Mercurial > code > home > repos > gcalendarwatch

changeset 45:e53a1bc87f99

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
cleanup and some fixes to starred event graph
author drewp@bigasterisk.com
date Thu, 06 Jun 2024 15:57:26 -0700
parents 1d9c4487f21a
children a53d79faac16
files gcalendarwatch.py graphconvert.py ingest.py
diffstat 3 files changed, 69 insertions(+), 114 deletions(-) [+]
line wrap: on
line diff
--- a/gcalendarwatch.py	Mon Feb 19 13:53:51 2024 -0800
+++ b/gcalendarwatch.py	Thu Jun 06 15:57:26 2024 -0700
@@ -1,6 +1,5 @@
 """
-sync google calendar into mongodb, return queries from that as
-JSON-LD.
+sync google calendar into mongodb, return a few preset queries as RDF graphs
 
 gcalendarwatch.conf looks like this:
 {
@@ -16,10 +15,9 @@
 import functools
 import json
 import logging
-import re
 import time
 import traceback
-from typing import Any, Dict, Iterable, Optional, cast
+from typing import Iterable, cast
 
 import background_loop
 import pymongo.collection
@@ -27,14 +25,14 @@
 from patchablegraph import PatchableGraph
 from patchablegraph.handler import GraphEvents, StaticGraph
 from pymongo import MongoClient
-from rdflib import RDF, ConjunctiveGraph, Graph, Namespace, URIRef
+from rdflib import Namespace, URIRef
 from starlette.applications import Starlette
 from starlette.requests import Request
-from starlette.responses import HTMLResponse, JSONResponse, PlainTextResponse, Response
+from starlette.responses import HTMLResponse, PlainTextResponse
 from starlette.routing import Route
 from starlette_exporter import PrometheusMiddleware, handle_metrics
 
-from datetimemath import dayRange, parse
+from datetimemath import dayRange
 from graphconvert import asGraph
 from ingest import SyncToMongo
 from localtypes import Conf, Record
@@ -65,55 +63,6 @@
 }"""
 
 
-def asJsonLd(events):
-    ret = {'@graph': []}  # type: Dict[Any, Any]
-    for ev in events:
-        ev['startTime'] = ev['startTime'].astimezone(tzlocal()).isoformat()
-        ev['endTime'] = ev['endTime'].astimezone(tzlocal()).isoformat()
-        ev['@id'] = ev.pop('uri')
-        ret['@graph'].append(ev)
-
-    ret['@context'] = {
-        "xsd": "http://www.w3.org/2001/XMLSchema#",
-        "ev": "http://bigasterisk.com/event#",
-        "startTime": {
-            "@id": "ev:startTime",
-            "@type": "xsd:dateTime"
-        },
-        "endTime": {
-            "@id": "ev:endTime",
-            "@type": "xsd:dateTime"
-        },
-        "startDate": {
-            "@id": "ev:startDate",
-            "@type": "xsd:date"
-        },
-        "endDate": {
-            "@id": "ev:endDate",
-            "@type": "xsd:date"
-        },
-        "title": "ev:title",
-        "feed": {
-            "@id": "ev:feed",
-            "@type": "@id"
-        },
-        "htmlLink": {
-            "@id": "ev:htmlLink",
-            "@type": "@id"
-        },
-    }
-    return ret
-
-
-def starred(graph: Graph, ev: URIRef) -> Optional[str]:
-    title = str(graph.value(ev, EV['title']))
-    m = re.search(r'(.*)\*\s*$', title)
-    if m:
-        return m.group(1)
-    else:
-        return None
-
-
 class ReadMongoEvents(object):
     """read events from mongodb"""
 
@@ -138,7 +87,7 @@
 ) -> int:
     log.info(f"updating {cal or 'all'}")
     try:
-        n = sync.update(cal=cal)
+        n = sync.update(cal=cal, days=120)
     except Exception:
         traceback.print_exc()
         log.error("update failed")
@@ -146,40 +95,6 @@
     return n
 
 
-# who uses this? pimscreen, frontdoor? they should use the GraphEvents version
-def EventsPage(conf: Conf, read: ReadMongoEvents, req: Request) -> Response:
-    t1, t2 = dayRange(int(req.query_params.get('days', default='2')))
-    if req.query_params.get('t1', default=None):
-        t1 = parse(req.query_params.get('t1', ''), tzinfo=tzlocal())
-    if req.query_params.get('t2', default=None):
-        t2 = parse(req.query_params.get('t2', ''), tzinfo=tzlocal())
-    log.info(f'get /events local t1={t1} t2={t2}')
-    return PlainTextResponse(media_type="text/n3", content=asGraph(conf, cals=[], events=read.getEvents(t1, t2)).serialize(format='n3'))
-
-
-def Countdowns(countdownGraph, req: Request) -> Response:
-    rows = []
-    graph = countdownGraph._graph
-    for ev in graph.subjects(RDF.type, EV['Event']):
-        starLabel = starred(graph, ev)
-        if starLabel is not None:
-            rows.append({'@type': 'countdown', 'time': graph.value(ev, EV['start']), 'label': starLabel})
-    return JSONResponse(media_type="application/ld+json",
-                        content={
-                            "@context": {
-                                "countdown": "http://bigasterisk.com/countdown#CountdownEvent",
-                                "label": "http://www.w3.org/2000/01/rdf-schema#label",
-                                "time": {
-                                    "@id": "http://bigasterisk.com/event#time",
-                                    "@type": "xsd:dateTime"
-                                },
-                                "xsd": "http://www.w3.org/2001/XMLSchema#",
-                                "rdfs": "http://www.w3.org/2000/01/rdf-schema#"
-                            },
-                            "@graph": rows,
-                        })
-
-
 def statusMsg(conf, loop: background_loop.Loop):
     period = conf['minutes_between_polls'] * 60
     ago = time.time() - loop.lastSuccessRun
@@ -221,7 +136,7 @@
     sync = SyncToMongo(conf, mongoOut, agendaGraph, countdownGraph)
     read = ReadMongoEvents(mongoOut)
 
-    s, e = dayRange(60)
+    s, e = dayRange(120)
     sync.updateGraphs(read.getEvents(s, e))
 
     loop = background_loop.loop_forever(functools.partial(update, sync=sync), conf['minutes_between_polls'] * 60)
@@ -239,8 +154,6 @@
                         Route('/graph/calendar/countdown/events', GraphEvents(countdownGraph)),
                         Route('/graph/currentEvents', StaticGraph(currentEventsGraph)),
                         Route('/graph/currentEvents/events', GraphEvents(currentEventsGraph)),
-                        Route('/countdowns.json', functools.partial(Countdowns, countdownGraph)),
-                        Route('/events', functools.partial(EventsPage, conf, read)),
                         Route('/pollNow', functools.partial(PollNow, loop), methods=['POST'])
                     ])
 
--- a/graphconvert.py	Mon Feb 19 13:53:51 2024 -0800
+++ b/graphconvert.py	Thu Jun 06 15:57:26 2024 -0700
@@ -1,10 +1,12 @@
+from dateutil.tz import tzlocal
+from rdflib import RDF, RDFS, ConjunctiveGraph, Literal, Namespace, URIRef
+
 from localtypes import Conf, feedFromCalId
-from rdflib import RDF, RDFS, ConjunctiveGraph, Literal, Namespace, URIRef
 
 EV = Namespace("http://bigasterisk.com/event#")
 
 
-def asGraph(conf: Conf, cals, events, extraClasses=[], ctxUri:URIRef=EV['gcalendar']):
+def asGraph(conf: Conf, cals, events, extraClasses=[], ctxUri: URIRef = EV['gcalendar']):
     ctx = ConjunctiveGraph(identifier=ctxUri)
     graph = ConjunctiveGraph()
     graph.namespace_manager.bind('ev', EV)
@@ -22,7 +24,7 @@
         if uri is None:
             raise ValueError(f"{ev=} had no event uri")
 
-        def add(p: URIRef, o: URIRef|Literal):
+        def add(p: URIRef, o: URIRef | Literal):
             return graph.add((uri, p, o, ctx))
 
         add(RDF.type, EV['Event'])
@@ -37,3 +39,43 @@
         if 'htmlLink' in ev:
             add(EV['htmlLink'], URIRef(ev['htmlLink']))
     return graph
+
+
+def asJsonLd(events) -> dict:
+    ret: dict = {'@graph': []}
+    for ev in events:
+        ev['startTime'] = ev['startTime'].astimezone(tzlocal()).isoformat()
+        ev['endTime'] = ev['endTime'].astimezone(tzlocal()).isoformat()
+        ev['@id'] = ev.pop('uri')
+        ret['@graph'].append(ev)
+
+    ret['@context'] = {
+        "xsd": "http://www.w3.org/2001/XMLSchema#",
+        "ev": "http://bigasterisk.com/event#",
+        "startTime": {
+            "@id": "ev:startTime",
+            "@type": "xsd:dateTime"
+        },
+        "endTime": {
+            "@id": "ev:endTime",
+            "@type": "xsd:dateTime"
+        },
+        "startDate": {
+            "@id": "ev:startDate",
+            "@type": "xsd:date"
+        },
+        "endDate": {
+            "@id": "ev:endDate",
+            "@type": "xsd:date"
+        },
+        "title": "ev:title",
+        "feed": {
+            "@id": "ev:feed",
+            "@type": "@id"
+        },
+        "htmlLink": {
+            "@id": "ev:htmlLink",
+            "@type": "@id"
+        },
+    }
+    return ret
--- a/ingest.py	Mon Feb 19 13:53:51 2024 -0800
+++ b/ingest.py	Thu Jun 06 15:57:26 2024 -0700
@@ -19,21 +19,19 @@
 EV = Namespace("http://bigasterisk.com/event#")
 
 
-
-
-def getFirstPageOfCalendars(service: Resource) -> Iterable[tuple[str, str | None, str | None]]:
+def _getFirstPageOfCalendars(service: Resource) -> Iterable[tuple[str, str | None, str | None]]:
     for row in service.calendarList().list().execute()['items']:
         yield row['id'], row.get('summary'), row.get('description')
 
 
-def recordFromEv(conf: Conf, calId: str, ev: Dict) -> Record:
+def _recordFromEv(conf: Conf, calId: str, ev: Dict) -> Record:
 
     def dateOrTime(d):
         if 'date' in d:
             return d['date']
         return d['dateTime']
 
-    rec= {
+    rec = {
         'uri': conf['event_uri_ns'] + ev['id'],
         'feed': feedFromCalId(conf, calId),
         'title': ev.get('summary', '?'),
@@ -51,14 +49,16 @@
         else:
             rec['%sTime' % field] = parse(val['dateTime'])
             rec['%sDate' % field] = parse(val['dateTime']).date().isoformat()
-    return rec
+    return cast(Record, rec)
 
 
 def filterStarred(recs: Sequence[Record], maxCount=15) -> List[Record]:
     recs = sorted(recs, key=lambda r: r['start'])
     out = []
     for rec in recs:
-        if re.search(r'(.*)\*\s*$', rec['title']):
+        if m:=re.search(r'(.*)\*\s*$', rec['title']):
+            rec = rec.copy()
+            rec['title'] = m.group(1)
             out.append(rec)
             if len(out) >= maxCount:
                 break
@@ -79,18 +79,18 @@
 
     def update(self, days=10, cal=None) -> int:
         start, end = dayRange(days)
-        idsFormerlyInRange = self.clearByStartTime(cal, start, end)
+        idsFormerlyInRange = self._clearByStartTime(cal, start, end)
 
-        totalNew, currentRecords = self.gatherNewEventsInRange(cal, start, end, idsFormerlyInRange)
+        totalNew, currentRecords = self._gatherNewEventsInRange(cal, start, end, idsFormerlyInRange)
 
         self.updateGraphs(currentRecords)
         return totalNew
 
-    def gatherNewEventsInRange(self, cal, start, end, idsFormerlyInRange):
+    def _gatherNewEventsInRange(self, cal, start, end, idsFormerlyInRange):
         totalNew = 0
         currentRecords = []
         try:
-            cals = getFirstPageOfCalendars(self.service)
+            cals = _getFirstPageOfCalendars(self.service)
         except HttpError:
             log.error('on getFirstPageOfCalendars')
             os.abort()
@@ -102,12 +102,12 @@
             if cal and calId != cal:
                 continue
             try:
-                self.updateOneCal(start, end, idsFormerlyInRange, totalNew, currentRecords, calId)
+                self._updateOneCal(start, end, idsFormerlyInRange, totalNew, currentRecords, calId)
             except HttpError:
                 log.error(f"on cal {calId}")
         return totalNew, currentRecords
 
-    def clearByStartTime(self, cal, start, end):
+    def _clearByStartTime(self, cal, start, end):
         spec: Dict[str, Any] = {"startTime": {"$gte": start, "$lte": end}}
         if cal is not None:
             spec['feed'] = feedFromCalId(self.conf, cal)
@@ -116,7 +116,7 @@
         log.info(f'cleared {n} records before reread')
         return idsFormerlyInRange
 
-    def updateOneCal(self, start, end, idsFormerlyInRange, totalNew, currentRecords, calId):
+    def _updateOneCal(self, start, end, idsFormerlyInRange, totalNew, currentRecords, calId):
         print('read %s' % calId)
         events = self.service.events().list(
             calendarId=calId,
@@ -128,13 +128,13 @@
         ).execute()
 
         for ev in events['items']:
-            rec = recordFromEv(self.conf, calId, ev)
-            self.upsertMongo(rec)
+            rec = _recordFromEv(self.conf, calId, ev)
+            self._upsertMongo(rec)
             if rec['uri'] not in idsFormerlyInRange:
                 totalNew += 1
             currentRecords.append(rec)
 
-    def upsertMongo(self, rec: Record) -> List[Record]:
+    def _upsertMongo(self, rec: Record) -> List[Record]:
         if self.collection.find_one({"_id": rec['uri']}) is not None:
             log.debug("existing record %s", rec['uri'])
             # this is not yet noticing updates