Mercurial > code > home > repos > gcalendarwatch

annotate ingest.py @ 38:d686e4a5b892

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
refactor; attempt clearer errors
author drewp@bigasterisk.com
date Sun, 12 Nov 2023 23:21:10 -0800
parents e2209226b001
children 7d9609edcf9c
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
28
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
1 import logging
38
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
2 import os
28
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
3 import re
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
4 from typing import Any, Dict, Iterable, List, Sequence, cast
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
5
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
6 import pymongo.collection
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
7 from dateutil.tz import tzlocal
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
8 from googleapiclient.discovery import Resource
38
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
9 from googleapiclient.errors import HttpError
28
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
10 from patchablegraph import PatchableGraph
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
11 from rdflib import Namespace
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
12
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
13 from calendar_connection import getCalendarService
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
14 from datetimemath import dayRange, limitDays, parse
38
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
15 from graphconvert import asGraph
28
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
16 from localtypes import Conf, Record
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
17
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
18 log = logging.getLogger()
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
19 EV = Namespace("http://bigasterisk.com/event#")
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
20
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
21
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
22 def feedFromCalId(conf: Conf, calId: str) -> str:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
23 return conf['event_uri_ns'] + 'feed/' + calId
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
24
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
25
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
26 def getFirstPageOfCalendars(service: Resource):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
27 for row in service.calendarList().list().execute()['items']:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
28 yield row['id']
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
29
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
30
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
31 def recordFromEv(conf: Conf, calId: str, ev: Dict) -> Record:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
32
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
33 def dateOrTime(d):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
34 if 'date' in d:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
35 return d['date']
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
36 return d['dateTime']
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
37
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
38 rec = {
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
39 'uri': conf['event_uri_ns'] + ev['id'],
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
40 'feed': feedFromCalId(conf, calId),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
41 'title': ev.get('summary', '?'),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
42 'start': dateOrTime(ev['start']),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
43 'end': dateOrTime(ev['end']),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
44 'endTimeUnspecified': ev.get('endTimeUnspecified', False),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
45 'htmlLink': ev.get('htmlLink', ''),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
46 'creatorEmail': ev.get('creator', {}).get('email', ''),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
47 }
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
48
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
49 for field, val in [('start', ev['start']), ('end', ev['end'])]:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
50 if 'date' in val:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
51 rec['%sTime' % field] = parse(val['date']).replace(tzinfo=tzlocal())
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
52 rec['%sDate' % field] = val['date']
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
53 else:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
54 rec['%sTime' % field] = parse(val['dateTime'])
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
55 rec['%sDate' % field] = parse(val['dateTime']).date().isoformat()
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
56 return rec
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
57
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
58
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
59 def filterStarred(recs: Sequence[Record], maxCount=15) -> List[Record]:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
60 recs = sorted(recs, key=lambda r: r['start'])
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
61 out = []
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
62 for rec in recs:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
63 if re.search(r'(.*)\*\s*$', rec['title']):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
64 out.append(rec)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
65 if len(out) >= maxCount:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
66 break
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
67 return out
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
68
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
69
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
70 class SyncToMongo(object):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
71 """reads gcal, writes to mongodb"""
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
72 collection: pymongo.collection.Collection
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
73
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
74 def __init__(self, conf: Conf, collection: pymongo.collection.Collection, agendaGraph: PatchableGraph,
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
75 countdownGraph: PatchableGraph):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
76 self.conf = conf
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
77 self.service = getCalendarService()
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
78 self.collection = collection
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
79 self.agendaGraph = agendaGraph
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
80 self.countdownGraph = countdownGraph
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
81
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
82 def update(self, days=10, cal=None) -> int:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
83 start, end = dayRange(days)
38
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
84 idsFormerlyInRange = self.clearByStartTime(cal, start, end)
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
85
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
86 totalNew, currentRecords = self.gatherNewEventsInRange(cal, start, end, idsFormerlyInRange)
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
87
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
88
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
89 self.updateGraphs(currentRecords)
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
90 return totalNew
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
91
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
92 def gatherNewEventsInRange(self, cal, start, end, idsFormerlyInRange):
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
93 totalNew = 0
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
94 currentRecords = []
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
95 try:
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
96 calIds = getFirstPageOfCalendars(self.service)
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
97 except HttpError:
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
98 log.error('on getFirstPageOfCalendars')
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
99 os.abort()
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
100 for calId in calIds:
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
101 if cal and calId != cal:
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
102 continue
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
103 try:
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
104 self.updateOneCal(start, end, idsFormerlyInRange, totalNew, currentRecords, calId)
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
105 except HttpError:
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
106 log.error(f"on cal {calId}")
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
107 return totalNew,currentRecords
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
108
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
109 def clearByStartTime(self, cal, start, end):
28
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
110 spec: Dict[str, Any] = {"startTime": {"$gte": start, "$lte": end}}
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
111 if cal is not None:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
112 spec['feed'] = feedFromCalId(self.conf, cal)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
113 idsFormerlyInRange = [doc['_id'] for doc in self.collection.find(spec)]
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
114 n = self.collection.delete_many(spec)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
115 log.info(f'cleared {n} records before reread')
38
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
116 return idsFormerlyInRange
28
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
117
38
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
118 def updateOneCal(self, start, end, idsFormerlyInRange, totalNew, currentRecords, calId):
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
119 print('read %s' % calId)
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
120 events = self.service.events().list(
28
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
121 calendarId=calId,
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
122 singleEvents=True,
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
123 timeMin=start.isoformat(),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
124 timeMax=end.isoformat(),
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
125 showDeleted=False,
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
126 maxResults=1000,
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
127 ).execute()
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
128
38
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
129 for ev in events['items']:
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
130 rec = recordFromEv(self.conf, calId, ev)
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
131 self.upsertMongo(rec)
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
132 if rec['uri'] not in idsFormerlyInRange:
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
133 totalNew += 1
d686e4a5b892 refactor; attempt clearer errors
drewp@bigasterisk.com
parents: 28
diff changeset
134 currentRecords.append(rec)
28
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
135
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
136 def upsertMongo(self, rec: Record) -> List[Record]:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
137 if self.collection.find_one({"_id": rec['uri']}) is not None:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
138 log.debug("existing record %s", rec['uri'])
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
139 # this is not yet noticing updates
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
140 return []
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
141 else:
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
142 log.debug("add record %s", rec)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
143 d = cast(Dict[str, Any], rec.copy())
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
144 d['_id'] = d.pop('uri')
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
145 self.collection.insert_one(d)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
146 return [rec]
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
147
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
148 def updateGraphs(self, currentRecords: Iterable[Record]):
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
149 currentRecords = list(currentRecords)
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
150 self.agendaGraph.setToGraph(asGraph(limitDays(currentRecords, days=2)))
e2209226b001 rewrite with starlette and background_loop
drewp@bigasterisk.com
parents:
diff changeset
151 self.countdownGraph.setToGraph(asGraph(filterStarred(currentRecords, maxCount=15), extraClasses=[EV['CountdownEvent']]))