Mercurial > code > home > repos > gcalendarwatch
changeset 42:7d9609edcf9c
track calendar feed summary/description text and emit them in graphs
| author | drewp@bigasterisk.com |
|---|---|
| date | Sun, 18 Feb 2024 12:34:53 -0800 |
| parents | ab54c9f65f76 |
| children | b5d3d9a8c83d |
| files | gcalendarwatch.py graphconvert.py ingest.py localtypes.py |
| diffstat | 4 files changed, 49 insertions(+), 31 deletions(-) [+] |
line wrap: on
line diff
--- a/gcalendarwatch.py Sun Feb 18 12:33:52 2024 -0800 +++ b/gcalendarwatch.py Sun Feb 18 12:34:53 2024 -0800 @@ -22,6 +22,7 @@ from typing import Any, Dict, Iterable, Optional, cast import background_loop +import pymongo.collection from dateutil.tz import tzlocal from patchablegraph import PatchableGraph from patchablegraph.handler import GraphEvents, StaticGraph @@ -29,7 +30,7 @@ from rdflib import RDF, Graph, Namespace, URIRef from starlette.applications import Starlette from starlette.requests import Request -from starlette.responses import (HTMLResponse, JSONResponse, PlainTextResponse, Response) +from starlette.responses import HTMLResponse, JSONResponse, PlainTextResponse, Response from starlette.routing import Route from starlette_exporter import PrometheusMiddleware, handle_metrics @@ -105,7 +106,7 @@ def starred(graph: Graph, ev: URIRef) -> Optional[str]: - title = graph.value(ev, EV['title']) + title = str(graph.value(ev, EV['title'])) m = re.search(r'(.*)\*\s*$', title) if m: return m.group(1) @@ -116,7 +117,7 @@ class ReadMongoEvents(object): """read events from mongodb""" - def __init__(self, collection): + def __init__(self, collection: pymongo.collection.Collection): self.collection = collection def getEvents(self, t1: datetime.datetime, t2: datetime.datetime) -> Iterable[Record]: @@ -146,14 +147,14 @@ # who uses this? pimscreen, frontdoor? they should use the GraphEvents version -def EventsPage(read: ReadMongoEvents, req: Request) -> Response: +def EventsPage(conf: Conf, read: ReadMongoEvents, req: Request) -> Response: t1, t2 = dayRange(int(req.query_params.get('days', default='2'))) if req.query_params.get('t1', default=None): t1 = parse(req.query_params.get('t1', ''), tzinfo=tzlocal()) if req.query_params.get('t2', default=None): t2 = parse(req.query_params.get('t2', ''), tzinfo=tzlocal()) log.info(f'get /events local t1={t1} t2={t2}') - return PlainTextResponse(media_type="text/n3", content=asGraph(read.getEvents(t1, t2)).serialize(format='n3')) + return PlainTextResponse(media_type="text/n3", content=asGraph(conf, cals=[], events=read.getEvents(t1, t2)).serialize(format='n3')) def Countdowns(countdownGraph, req: Request) -> Response: @@ -229,7 +230,7 @@ Route('/graph/calendar/countdown', StaticGraph(countdownGraph)), Route('/graph/calendar/countdown/events', GraphEvents(countdownGraph)), Route('/countdowns.json', functools.partial(Countdowns, countdownGraph)), - Route('/events', functools.partial(EventsPage, read)), + Route('/events', functools.partial(EventsPage, conf, read)), Route('/pollNow', functools.partial(PollNow, loop), methods=['POST']) ])
--- a/graphconvert.py Sun Feb 18 12:33:52 2024 -0800 +++ b/graphconvert.py Sun Feb 18 12:34:53 2024 -0800 @@ -1,15 +1,26 @@ -from rdflib import RDF, ConjunctiveGraph, Literal, Namespace, URIRef +from localtypes import Conf, feedFromCalId +from rdflib import RDF, RDFS, ConjunctiveGraph, Literal, Namespace, URIRef EV = Namespace("http://bigasterisk.com/event#") -def asGraph(events, extraClasses=[], ctx=EV['gcalendar']): +def asGraph(conf: Conf, cals, events, extraClasses=[], ctxUri:URIRef=EV['gcalendar']): + ctx = ConjunctiveGraph(identifier=ctxUri) graph = ConjunctiveGraph() graph.namespace_manager.bind('ev', EV) + + for doc in cals: + feed = URIRef(feedFromCalId(conf, doc['_id'])) + graph.add((feed, RDF.type, EV['CalendarFeed'], ctx)) + if doc['summary']: + graph.add((feed, RDFS.label, Literal(doc['summary'].strip()), ctx)) + if doc['description']: + graph.add((feed, EV['description'], Literal(doc['description'].strip()), ctx)) + for ev in events: uri = URIRef(ev['uri']) - def add(p, o): + def add(p: URIRef, o: URIRef|Literal): return graph.add((uri, p, o, ctx)) add(RDF.type, EV['Event'])
--- a/ingest.py Sun Feb 18 12:33:52 2024 -0800 +++ b/ingest.py Sun Feb 18 12:34:53 2024 -0800 @@ -13,19 +13,17 @@ from calendar_connection import getCalendarService from datetimemath import dayRange, limitDays, parse from graphconvert import asGraph -from localtypes import Conf, Record +from localtypes import Conf, Record, feedFromCalId log = logging.getLogger() EV = Namespace("http://bigasterisk.com/event#") -def feedFromCalId(conf: Conf, calId: str) -> str: - return conf['event_uri_ns'] + 'feed/' + calId -def getFirstPageOfCalendars(service: Resource): +def getFirstPageOfCalendars(service: Resource) -> Iterable[tuple[str, str | None, str | None]]: for row in service.calendarList().list().execute()['items']: - yield row['id'] + yield row['id'], row.get('summary'), row.get('description') def recordFromEv(conf: Conf, calId: str, ev: Dict) -> Record: @@ -35,7 +33,7 @@ return d['date'] return d['dateTime'] - rec = { + rec= { 'uri': conf['event_uri_ns'] + ev['id'], 'feed': feedFromCalId(conf, calId), 'title': ev.get('summary', '?'), @@ -68,14 +66,14 @@ class SyncToMongo(object): - """reads gcal, writes to mongodb""" + """reads gcal, writes to mongodb and graphs""" collection: pymongo.collection.Collection - def __init__(self, conf: Conf, collection: pymongo.collection.Collection, agendaGraph: PatchableGraph, - countdownGraph: PatchableGraph): + def __init__(self, conf: Conf, collection: pymongo.collection.Collection, agendaGraph: PatchableGraph, countdownGraph: PatchableGraph): self.conf = conf self.service = getCalendarService() self.collection = collection + self.calendarsCollection = collection.database.get_collection('gcalendar_cals') self.agendaGraph = agendaGraph self.countdownGraph = countdownGraph @@ -85,7 +83,6 @@ totalNew, currentRecords = self.gatherNewEventsInRange(cal, start, end, idsFormerlyInRange) - self.updateGraphs(currentRecords) return totalNew @@ -93,18 +90,22 @@ totalNew = 0 currentRecords = [] try: - calIds = getFirstPageOfCalendars(self.service) + cals = getFirstPageOfCalendars(self.service) except HttpError: log.error('on getFirstPageOfCalendars') os.abort() - for calId in calIds: + for calId, summary, description in cals: + self.calendarsCollection.update_one({'_id': calId}, {'$set': { + 'summary': summary, + 'description': description, + }}, upsert=True) if cal and calId != cal: continue try: self.updateOneCal(start, end, idsFormerlyInRange, totalNew, currentRecords, calId) except HttpError: log.error(f"on cal {calId}") - return totalNew,currentRecords + return totalNew, currentRecords def clearByStartTime(self, cal, start, end): spec: Dict[str, Any] = {"startTime": {"$gte": start, "$lte": end}} @@ -118,13 +119,13 @@ def updateOneCal(self, start, end, idsFormerlyInRange, totalNew, currentRecords, calId): print('read %s' % calId) events = self.service.events().list( - calendarId=calId, - singleEvents=True, - timeMin=start.isoformat(), - timeMax=end.isoformat(), - showDeleted=False, - maxResults=1000, - ).execute() + calendarId=calId, + singleEvents=True, + timeMin=start.isoformat(), + timeMax=end.isoformat(), + showDeleted=False, + maxResults=1000, + ).execute() for ev in events['items']: rec = recordFromEv(self.conf, calId, ev) @@ -147,5 +148,6 @@ def updateGraphs(self, currentRecords: Iterable[Record]): currentRecords = list(currentRecords) - self.agendaGraph.setToGraph(asGraph(limitDays(currentRecords, days=2))) - self.countdownGraph.setToGraph(asGraph(filterStarred(currentRecords, maxCount=15), extraClasses=[EV['CountdownEvent']])) + cals = list(self.calendarsCollection.find()) + self.agendaGraph.setToGraph(asGraph(self.conf, cals, limitDays(currentRecords, days=2))) + self.countdownGraph.setToGraph(asGraph(self.conf, cals, filterStarred(currentRecords, maxCount=15), extraClasses=[EV['CountdownEvent']]))