diff ingest.py @ 42:7d9609edcf9c

track calendar feed summary/description text and emit them in graphs
author drewp@bigasterisk.com
date Sun, 18 Feb 2024 12:34:53 -0800
parents d686e4a5b892
children e53a1bc87f99
line wrap: on
line diff
--- a/ingest.py	Sun Feb 18 12:33:52 2024 -0800
+++ b/ingest.py	Sun Feb 18 12:34:53 2024 -0800
@@ -13,19 +13,17 @@
 from calendar_connection import getCalendarService
 from datetimemath import dayRange, limitDays, parse
 from graphconvert import asGraph
-from localtypes import Conf, Record
+from localtypes import Conf, Record, feedFromCalId
 
 log = logging.getLogger()
 EV = Namespace("http://bigasterisk.com/event#")
 
 
-def feedFromCalId(conf: Conf, calId: str) -> str:
-    return conf['event_uri_ns'] + 'feed/' + calId
 
 
-def getFirstPageOfCalendars(service: Resource):
+def getFirstPageOfCalendars(service: Resource) -> Iterable[tuple[str, str | None, str | None]]:
     for row in service.calendarList().list().execute()['items']:
-        yield row['id']
+        yield row['id'], row.get('summary'), row.get('description')
 
 
 def recordFromEv(conf: Conf, calId: str, ev: Dict) -> Record:
@@ -35,7 +33,7 @@
             return d['date']
         return d['dateTime']
 
-    rec = {
+    rec= {
         'uri': conf['event_uri_ns'] + ev['id'],
         'feed': feedFromCalId(conf, calId),
         'title': ev.get('summary', '?'),
@@ -68,14 +66,14 @@
 
 
 class SyncToMongo(object):
-    """reads gcal, writes to mongodb"""
+    """reads gcal, writes to mongodb and graphs"""
     collection: pymongo.collection.Collection
 
-    def __init__(self, conf: Conf, collection: pymongo.collection.Collection, agendaGraph: PatchableGraph,
-                 countdownGraph: PatchableGraph):
+    def __init__(self, conf: Conf, collection: pymongo.collection.Collection, agendaGraph: PatchableGraph, countdownGraph: PatchableGraph):
         self.conf = conf
         self.service = getCalendarService()
         self.collection = collection
+        self.calendarsCollection = collection.database.get_collection('gcalendar_cals')
         self.agendaGraph = agendaGraph
         self.countdownGraph = countdownGraph
 
@@ -85,7 +83,6 @@
 
         totalNew, currentRecords = self.gatherNewEventsInRange(cal, start, end, idsFormerlyInRange)
 
-
         self.updateGraphs(currentRecords)
         return totalNew
 
@@ -93,18 +90,22 @@
         totalNew = 0
         currentRecords = []
         try:
-            calIds = getFirstPageOfCalendars(self.service)
+            cals = getFirstPageOfCalendars(self.service)
         except HttpError:
             log.error('on getFirstPageOfCalendars')
             os.abort()
-        for calId in calIds:
+        for calId, summary, description in cals:
+            self.calendarsCollection.update_one({'_id': calId}, {'$set': {
+                'summary': summary,
+                'description': description,
+            }}, upsert=True)
             if cal and calId != cal:
                 continue
             try:
                 self.updateOneCal(start, end, idsFormerlyInRange, totalNew, currentRecords, calId)
             except HttpError:
                 log.error(f"on cal {calId}")
-        return totalNew,currentRecords
+        return totalNew, currentRecords
 
     def clearByStartTime(self, cal, start, end):
         spec: Dict[str, Any] = {"startTime": {"$gte": start, "$lte": end}}
@@ -118,13 +119,13 @@
     def updateOneCal(self, start, end, idsFormerlyInRange, totalNew, currentRecords, calId):
         print('read %s' % calId)
         events = self.service.events().list(
-                calendarId=calId,
-                singleEvents=True,
-                timeMin=start.isoformat(),
-                timeMax=end.isoformat(),
-                showDeleted=False,
-                maxResults=1000,
-            ).execute()
+            calendarId=calId,
+            singleEvents=True,
+            timeMin=start.isoformat(),
+            timeMax=end.isoformat(),
+            showDeleted=False,
+            maxResults=1000,
+        ).execute()
 
         for ev in events['items']:
             rec = recordFromEv(self.conf, calId, ev)
@@ -147,5 +148,6 @@
 
     def updateGraphs(self, currentRecords: Iterable[Record]):
         currentRecords = list(currentRecords)
-        self.agendaGraph.setToGraph(asGraph(limitDays(currentRecords, days=2)))
-        self.countdownGraph.setToGraph(asGraph(filterStarred(currentRecords, maxCount=15), extraClasses=[EV['CountdownEvent']]))
+        cals = list(self.calendarsCollection.find())
+        self.agendaGraph.setToGraph(asGraph(self.conf, cals, limitDays(currentRecords, days=2)))
+        self.countdownGraph.setToGraph(asGraph(self.conf, cals, filterStarred(currentRecords, maxCount=15), extraClasses=[EV['CountdownEvent']]))