Mercurial > code > home > repos > gcalendarwatch

annotate ingest.py @ 36:cb990883e52f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
deployment; dep updates
author drewp@bigasterisk.com
date Sun, 12 Nov 2023 23:19:23 -0800
parents e2209226b001
children d686e4a5b892
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
28
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
1 import logging
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
2 import re
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
3 from typing import Any, Dict, Iterable, List, Sequence, cast
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
4
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
5 import pymongo.collection
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
6 from dateutil.tz import tzlocal
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
7 from googleapiclient.discovery import Resource
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
8 from patchablegraph import PatchableGraph
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
9 from rdflib import Namespace
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
10
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
11 from calendar_connection import getCalendarService
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
12 from datetimemath import dayRange, limitDays, parse
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
13 from localtypes import Conf, Record
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
14 from graphconvert import asGraph
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
15
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
16 log = logging.getLogger()
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
17 EV = Namespace("http://bigasterisk.com/event#")
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
18
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
19
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
20 def feedFromCalId(conf: Conf, calId: str) -> str:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
21 return conf['event_uri_ns'] + 'feed/' + calId
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
22
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
23
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
24 def getFirstPageOfCalendars(service: Resource):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
25 for row in service.calendarList().list().execute()['items']:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
26 yield row['id']
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
27
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
28
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
29 def recordFromEv(conf: Conf, calId: str, ev: Dict) -> Record:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
30
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
31 def dateOrTime(d):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
32 if 'date' in d:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
33 return d['date']
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
34 return d['dateTime']
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
35
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
36 rec = {
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
37 'uri': conf['event_uri_ns'] + ev['id'],
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
38 'feed': feedFromCalId(conf, calId),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
39 'title': ev.get('summary', '?'),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
40 'start': dateOrTime(ev['start']),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
41 'end': dateOrTime(ev['end']),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
42 'endTimeUnspecified': ev.get('endTimeUnspecified', False),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
43 'htmlLink': ev.get('htmlLink', ''),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
44 'creatorEmail': ev.get('creator', {}).get('email', ''),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
45 }
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
46
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
47 for field, val in [('start', ev['start']), ('end', ev['end'])]:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
48 if 'date' in val:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
49 rec['%sTime' % field] = parse(val['date']).replace(tzinfo=tzlocal())
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
50 rec['%sDate' % field] = val['date']
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
51 else:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
52 rec['%sTime' % field] = parse(val['dateTime'])
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
53 rec['%sDate' % field] = parse(val['dateTime']).date().isoformat()
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
54 return rec
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
55
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
56
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
57 def filterStarred(recs: Sequence[Record], maxCount=15) -> List[Record]:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
58 recs = sorted(recs, key=lambda r: r['start'])
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
59 out = []
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
60 for rec in recs:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
61 if re.search(r'(.*)\*\s*$', rec['title']):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
62 out.append(rec)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
63 if len(out) >= maxCount:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
64 break
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
65 return out
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
66
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
67
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
68 class SyncToMongo(object):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
69 """reads gcal, writes to mongodb"""
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
70 collection: pymongo.collection.Collection
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
71
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
72 def __init__(self, conf: Conf, collection: pymongo.collection.Collection, agendaGraph: PatchableGraph,
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
73 countdownGraph: PatchableGraph):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
74 self.conf = conf
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
75 self.service = getCalendarService()
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
76 self.collection = collection
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
77 self.agendaGraph = agendaGraph
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
78 self.countdownGraph = countdownGraph
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
79
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
80 def update(self, days=10, cal=None) -> int:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
81 start, end = dayRange(days)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
82 spec: Dict[str, Any] = {"startTime": {"$gte": start, "$lte": end}}
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
83 if cal is not None:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
84 spec['feed'] = feedFromCalId(self.conf, cal)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
85 idsFormerlyInRange = [doc['_id'] for doc in self.collection.find(spec)]
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
86 n = self.collection.delete_many(spec)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
87 log.info(f'cleared {n} records before reread')
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
88
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
89 totalNew = 0
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
90 currentRecords = []
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
91 for calId in getFirstPageOfCalendars(self.service):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
92 if cal and calId != cal:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
93 continue
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
94 print('read %s' % calId)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
95 events = self.service.events().list(
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
96 calendarId=calId,
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
97 singleEvents=True,
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
98 timeMin=start.isoformat(),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
99 timeMax=end.isoformat(),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
100 showDeleted=False,
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
101 maxResults=1000,
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
102 ).execute()
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
103
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
104 for ev in events['items']:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
105 rec = recordFromEv(self.conf, calId, ev)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
106 self.upsertMongo(rec)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
107 if rec['uri'] not in idsFormerlyInRange:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
108 totalNew += 1
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
109 currentRecords.append(rec)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
110
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
111 self.updateGraphs(currentRecords)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
112 return totalNew
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
113
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
114 def upsertMongo(self, rec: Record) -> List[Record]:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
115 if self.collection.find_one({"_id": rec['uri']}) is not None:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
116 log.debug("existing record %s", rec['uri'])
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
117 # this is not yet noticing updates
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
118 return []
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
119 else:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
120 log.debug("add record %s", rec)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
121 d = cast(Dict[str, Any], rec.copy())
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
122 d['_id'] = d.pop('uri')
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
123 self.collection.insert_one(d)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
124 return [rec]
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
125
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
126 def updateGraphs(self, currentRecords: Iterable[Record]):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
127 currentRecords = list(currentRecords)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
128 self.agendaGraph.setToGraph(asGraph(limitDays(currentRecords, days=2)))
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
129 self.countdownGraph.setToGraph(asGraph(filterStarred(currentRecords, maxCount=15), extraClasses=[EV['CountdownEvent']]))