--- a/ingest.py	Sun Nov 12 23:20:37 2023 -0800
+++ b/ingest.py	Sun Nov 12 23:21:10 2023 -0800
@@ -1,17 +1,19 @@
 import logging
+import os
 import re
 from typing import Any, Dict, Iterable, List, Sequence, cast
 
 import pymongo.collection
 from dateutil.tz import tzlocal
 from googleapiclient.discovery import Resource
+from googleapiclient.errors import HttpError
 from patchablegraph import PatchableGraph
 from rdflib import Namespace
 
 from calendar_connection import getCalendarService
 from datetimemath import dayRange, limitDays, parse
+from graphconvert import asGraph
 from localtypes import Conf, Record
-from graphconvert import asGraph
 
 log = logging.getLogger()
 EV = Namespace("http://bigasterisk.com/event#")
@@ -79,20 +81,43 @@
 
     def update(self, days=10, cal=None) -> int:
         start, end = dayRange(days)
+        idsFormerlyInRange = self.clearByStartTime(cal, start, end)
+
+        totalNew, currentRecords = self.gatherNewEventsInRange(cal, start, end, idsFormerlyInRange)
+
+
+        self.updateGraphs(currentRecords)
+        return totalNew
+
+    def gatherNewEventsInRange(self, cal, start, end, idsFormerlyInRange):
+        totalNew = 0
+        currentRecords = []
+        try:
+            calIds = getFirstPageOfCalendars(self.service)
+        except HttpError:
+            log.error('on getFirstPageOfCalendars')
+            os.abort()
+        for calId in calIds:
+            if cal and calId != cal:
+                continue
+            try:
+                self.updateOneCal(start, end, idsFormerlyInRange, totalNew, currentRecords, calId)
+            except HttpError:
+                log.error(f"on cal {calId}")
+        return totalNew,currentRecords
+
+    def clearByStartTime(self, cal, start, end):
         spec: Dict[str, Any] = {"startTime": {"$gte": start, "$lte": end}}
         if cal is not None:
             spec['feed'] = feedFromCalId(self.conf, cal)
         idsFormerlyInRange = [doc['_id'] for doc in self.collection.find(spec)]
         n = self.collection.delete_many(spec)
         log.info(f'cleared {n} records before reread')
+        return idsFormerlyInRange
 
-        totalNew = 0
-        currentRecords = []
-        for calId in getFirstPageOfCalendars(self.service):
-            if cal and calId != cal:
-                continue
-            print('read %s' % calId)
-            events = self.service.events().list(
+    def updateOneCal(self, start, end, idsFormerlyInRange, totalNew, currentRecords, calId):
+        print('read %s' % calId)
+        events = self.service.events().list(
                 calendarId=calId,
                 singleEvents=True,
                 timeMin=start.isoformat(),
@@ -101,15 +126,12 @@
                 maxResults=1000,
             ).execute()
 
-            for ev in events['items']:
-                rec = recordFromEv(self.conf, calId, ev)
-                self.upsertMongo(rec)
-                if rec['uri'] not in idsFormerlyInRange:
-                    totalNew += 1
-                currentRecords.append(rec)
-
-        self.updateGraphs(currentRecords)
-        return totalNew
+        for ev in events['items']:
+            rec = recordFromEv(self.conf, calId, ev)
+            self.upsertMongo(rec)
+            if rec['uri'] not in idsFormerlyInRange:
+                totalNew += 1
+            currentRecords.append(rec)
 
     def upsertMongo(self, rec: Record) -> List[Record]:
         if self.collection.find_one({"_id": rec['uri']}) is not None: