diff ingest.py @ 45:e53a1bc87f99

cleanup and some fixes to starred event graph
author drewp@bigasterisk.com
date Thu, 06 Jun 2024 15:57:26 -0700
parents 7d9609edcf9c
children
line wrap: on
line diff
--- a/ingest.py	Mon Feb 19 13:53:51 2024 -0800
+++ b/ingest.py	Thu Jun 06 15:57:26 2024 -0700
@@ -19,21 +19,19 @@
 EV = Namespace("http://bigasterisk.com/event#")
 
 
-
-
-def getFirstPageOfCalendars(service: Resource) -> Iterable[tuple[str, str | None, str | None]]:
+def _getFirstPageOfCalendars(service: Resource) -> Iterable[tuple[str, str | None, str | None]]:
     for row in service.calendarList().list().execute()['items']:
         yield row['id'], row.get('summary'), row.get('description')
 
 
-def recordFromEv(conf: Conf, calId: str, ev: Dict) -> Record:
+def _recordFromEv(conf: Conf, calId: str, ev: Dict) -> Record:
 
     def dateOrTime(d):
         if 'date' in d:
             return d['date']
         return d['dateTime']
 
-    rec= {
+    rec = {
         'uri': conf['event_uri_ns'] + ev['id'],
         'feed': feedFromCalId(conf, calId),
         'title': ev.get('summary', '?'),
@@ -51,14 +49,16 @@
         else:
             rec['%sTime' % field] = parse(val['dateTime'])
             rec['%sDate' % field] = parse(val['dateTime']).date().isoformat()
-    return rec
+    return cast(Record, rec)
 
 
 def filterStarred(recs: Sequence[Record], maxCount=15) -> List[Record]:
     recs = sorted(recs, key=lambda r: r['start'])
     out = []
     for rec in recs:
-        if re.search(r'(.*)\*\s*$', rec['title']):
+        if m:=re.search(r'(.*)\*\s*$', rec['title']):
+            rec = rec.copy()
+            rec['title'] = m.group(1)
             out.append(rec)
             if len(out) >= maxCount:
                 break
@@ -79,18 +79,18 @@
 
     def update(self, days=10, cal=None) -> int:
         start, end = dayRange(days)
-        idsFormerlyInRange = self.clearByStartTime(cal, start, end)
+        idsFormerlyInRange = self._clearByStartTime(cal, start, end)
 
-        totalNew, currentRecords = self.gatherNewEventsInRange(cal, start, end, idsFormerlyInRange)
+        totalNew, currentRecords = self._gatherNewEventsInRange(cal, start, end, idsFormerlyInRange)
 
         self.updateGraphs(currentRecords)
         return totalNew
 
-    def gatherNewEventsInRange(self, cal, start, end, idsFormerlyInRange):
+    def _gatherNewEventsInRange(self, cal, start, end, idsFormerlyInRange):
         totalNew = 0
         currentRecords = []
         try:
-            cals = getFirstPageOfCalendars(self.service)
+            cals = _getFirstPageOfCalendars(self.service)
         except HttpError:
             log.error('on getFirstPageOfCalendars')
             os.abort()
@@ -102,12 +102,12 @@
             if cal and calId != cal:
                 continue
             try:
-                self.updateOneCal(start, end, idsFormerlyInRange, totalNew, currentRecords, calId)
+                self._updateOneCal(start, end, idsFormerlyInRange, totalNew, currentRecords, calId)
             except HttpError:
                 log.error(f"on cal {calId}")
         return totalNew, currentRecords
 
-    def clearByStartTime(self, cal, start, end):
+    def _clearByStartTime(self, cal, start, end):
         spec: Dict[str, Any] = {"startTime": {"$gte": start, "$lte": end}}
         if cal is not None:
             spec['feed'] = feedFromCalId(self.conf, cal)
@@ -116,7 +116,7 @@
         log.info(f'cleared {n} records before reread')
         return idsFormerlyInRange
 
-    def updateOneCal(self, start, end, idsFormerlyInRange, totalNew, currentRecords, calId):
+    def _updateOneCal(self, start, end, idsFormerlyInRange, totalNew, currentRecords, calId):
         print('read %s' % calId)
         events = self.service.events().list(
             calendarId=calId,
@@ -128,13 +128,13 @@
         ).execute()
 
         for ev in events['items']:
-            rec = recordFromEv(self.conf, calId, ev)
-            self.upsertMongo(rec)
+            rec = _recordFromEv(self.conf, calId, ev)
+            self._upsertMongo(rec)
             if rec['uri'] not in idsFormerlyInRange:
                 totalNew += 1
             currentRecords.append(rec)
 
-    def upsertMongo(self, rec: Record) -> List[Record]:
+    def _upsertMongo(self, rec: Record) -> List[Record]:
         if self.collection.find_one({"_id": rec['uri']}) is not None:
             log.debug("existing record %s", rec['uri'])
             # this is not yet noticing updates