Mercurial > code > home > repos > gcalendarwatch
diff ingest.py @ 45:e53a1bc87f99
cleanup and some fixes to starred event graph
author | drewp@bigasterisk.com |
---|---|
date | Thu, 06 Jun 2024 15:57:26 -0700 |
parents | 7d9609edcf9c |
children |
line wrap: on
line diff
--- a/ingest.py Mon Feb 19 13:53:51 2024 -0800 +++ b/ingest.py Thu Jun 06 15:57:26 2024 -0700 @@ -19,21 +19,19 @@ EV = Namespace("http://bigasterisk.com/event#") - - -def getFirstPageOfCalendars(service: Resource) -> Iterable[tuple[str, str | None, str | None]]: +def _getFirstPageOfCalendars(service: Resource) -> Iterable[tuple[str, str | None, str | None]]: for row in service.calendarList().list().execute()['items']: yield row['id'], row.get('summary'), row.get('description') -def recordFromEv(conf: Conf, calId: str, ev: Dict) -> Record: +def _recordFromEv(conf: Conf, calId: str, ev: Dict) -> Record: def dateOrTime(d): if 'date' in d: return d['date'] return d['dateTime'] - rec= { + rec = { 'uri': conf['event_uri_ns'] + ev['id'], 'feed': feedFromCalId(conf, calId), 'title': ev.get('summary', '?'), @@ -51,14 +49,16 @@ else: rec['%sTime' % field] = parse(val['dateTime']) rec['%sDate' % field] = parse(val['dateTime']).date().isoformat() - return rec + return cast(Record, rec) def filterStarred(recs: Sequence[Record], maxCount=15) -> List[Record]: recs = sorted(recs, key=lambda r: r['start']) out = [] for rec in recs: - if re.search(r'(.*)\*\s*$', rec['title']): + if m:=re.search(r'(.*)\*\s*$', rec['title']): + rec = rec.copy() + rec['title'] = m.group(1) out.append(rec) if len(out) >= maxCount: break @@ -79,18 +79,18 @@ def update(self, days=10, cal=None) -> int: start, end = dayRange(days) - idsFormerlyInRange = self.clearByStartTime(cal, start, end) + idsFormerlyInRange = self._clearByStartTime(cal, start, end) - totalNew, currentRecords = self.gatherNewEventsInRange(cal, start, end, idsFormerlyInRange) + totalNew, currentRecords = self._gatherNewEventsInRange(cal, start, end, idsFormerlyInRange) self.updateGraphs(currentRecords) return totalNew - def gatherNewEventsInRange(self, cal, start, end, idsFormerlyInRange): + def _gatherNewEventsInRange(self, cal, start, end, idsFormerlyInRange): totalNew = 0 currentRecords = [] try: - cals = getFirstPageOfCalendars(self.service) + cals = _getFirstPageOfCalendars(self.service) except HttpError: log.error('on getFirstPageOfCalendars') os.abort() @@ -102,12 +102,12 @@ if cal and calId != cal: continue try: - self.updateOneCal(start, end, idsFormerlyInRange, totalNew, currentRecords, calId) + self._updateOneCal(start, end, idsFormerlyInRange, totalNew, currentRecords, calId) except HttpError: log.error(f"on cal {calId}") return totalNew, currentRecords - def clearByStartTime(self, cal, start, end): + def _clearByStartTime(self, cal, start, end): spec: Dict[str, Any] = {"startTime": {"$gte": start, "$lte": end}} if cal is not None: spec['feed'] = feedFromCalId(self.conf, cal) @@ -116,7 +116,7 @@ log.info(f'cleared {n} records before reread') return idsFormerlyInRange - def updateOneCal(self, start, end, idsFormerlyInRange, totalNew, currentRecords, calId): + def _updateOneCal(self, start, end, idsFormerlyInRange, totalNew, currentRecords, calId): print('read %s' % calId) events = self.service.events().list( calendarId=calId, @@ -128,13 +128,13 @@ ).execute() for ev in events['items']: - rec = recordFromEv(self.conf, calId, ev) - self.upsertMongo(rec) + rec = _recordFromEv(self.conf, calId, ev) + self._upsertMongo(rec) if rec['uri'] not in idsFormerlyInRange: totalNew += 1 currentRecords.append(rec) - def upsertMongo(self, rec: Record) -> List[Record]: + def _upsertMongo(self, rec: Record) -> List[Record]: if self.collection.find_one({"_id": rec['uri']}) is not None: log.debug("existing record %s", rec['uri']) # this is not yet noticing updates