diff --git a/bin/rdfdb b/bin/rdfdb --- a/bin/rdfdb +++ b/bin/rdfdb @@ -109,6 +109,8 @@ Our web ui: """ from twisted.internet import reactor import twisted.internet.error +from twisted.python.filepath import FilePath +from twisted.internet.inotify import humanReadableMask, IN_CREATE import sys, optparse, logging, json, os import cyclone.web, cyclone.httpclient, cyclone.websocket sys.path.append(".") @@ -155,54 +157,82 @@ class Client(object): def sendPatch(self, p): return syncedgraph.sendPatch(self.updateUri, p) -class Db(object): +class WatchedFiles(object): """ - the master graph, all the connected clients, all the files we're watching + find files, notice new files. + + This object watches directories. Each GraphFile watches its own file. """ - def __init__(self): + def __init__(self, topDirsToWatch, patch, getSubgraph): + self.topDirsToWatch = topDirsToWatch + self.patch, self.getSubgraph = patch, getSubgraph + # files from cwd become uris starting with this. *should* be # building uris from the show uri in $LIGHT9_SHOW/URI # instead. Who wants to keep their data in the same dir tree # as the source code?! self.topUri = URIRef("http://light9.bigasterisk.com/") - self.clients = [] - self.graph = ConjunctiveGraph() - + self.graphFiles = {} # context uri : GraphFile + self.notifier = INotify() self.notifier.startReading() - self.graphFiles = {} # context uri : GraphFile - + self.findAndLoadFiles() def findAndLoadFiles(self): self.initialLoad = True try: - dirs = [ - "show/dance2012/sessions", - "show/dance2012/subs", - "show/dance2012/subterms", - ] - - for topdir in dirs: + for topdir in self.topDirsToWatch: for dirpath, dirnames, filenames in os.walk(topdir): for base in filenames: self.watchFile(os.path.join(dirpath, base)) - # todo: also notice new files in this dir - - self.watchFile("show/dance2012/config.n3") - self.watchFile("show/dance2012/patch.n3") + self.notifier.watch(FilePath(dirpath), autoAdd=True, + callbacks=[self.dirChange]) finally: self.initialLoad = False - self.summarizeToLog() + def dirChange(self, watch, path, mask): + if mask & IN_CREATE: + self.watchFile(path.path) + + def watchFile(self, inFile): + if not isinstance(inFile, FilePath): + inFile = FilePath(inFile) + if not inFile.isfile(): + return + if inFile.splitext()[1] not in ['.n3']: + return + ctx = self.uriFromFile(inFile) + gf = GraphFile(self.notifier, inFile.path, ctx, + self.patch, self.getSubgraph) + self.graphFiles[ctx] = gf + gf.reread() + + def dirtyFiles(self, ctxs): + """mark dirty the files that we watch in these contexts. + + the ctx might not be a file that we already read; it might be + for a new file we have to create, or it might be for a + transient context that we're not going to save + + if it's a ctx with no file, error + """ + for ctx in ctxs: + g = self.getSubgraph(ctx) + + if ctx not in self.graphFiles: + outFile = self.fileForUri(ctx) + self.graphFiles[ctx] = GraphFile(self.notifier, outFile, ctx, + self.patch, self.getSubgraph) + + self.graphFiles[ctx].dirty(g) def uriFromFile(self, filename): - if filename.endswith('.n3'): - # some legacy files don't end with n3. when we write them - # back this might not go so well - filename = filename[:-len('.n3')] - return URIRef(self.topUri + filename) + if isinstance(filename, FilePath): + filename = filename.path + assert filename.endswith('.n3'), filename + return URIRef(self.topUri + filename[:-len('.n3')]) def fileForUri(self, ctx): assert isinstance(ctx, URIRef), ctx @@ -210,11 +240,20 @@ class Db(object): raise ValueError("don't know what filename to use for %s" % ctx) return ctx[len(self.topUri):] + ".n3" - def watchFile(self, inFile): - ctx = self.uriFromFile(inFile) - gf = GraphFile(self.notifier, inFile, ctx, self.patch, self.getSubgraph) - self.graphFiles[ctx] = gf - gf.reread() + +class Db(object): + """ + the master graph, all the connected clients, all the files we're watching + """ + def __init__(self, topDirsToWatch): + + self.clients = [] + self.graph = ConjunctiveGraph() + + self.watchedFiles = WatchedFiles(topDirsToWatch, + self.patch, self.getSubgraph) + + self.summarizeToLog() def patch(self, p, dueToFileChange=False): """ @@ -232,8 +271,7 @@ class Db(object): patchQuads(self.graph, p.delQuads, p.addQuads, perfect=True) senderUpdateUri = getattr(p, 'senderUpdateUri', None) - #if not self.initialLoad: - # self.summarizeToLog() + for c in self.clients: if c.updateUri == senderUpdateUri: # this client has self-applied the patch already @@ -241,28 +279,9 @@ class Db(object): d = c.sendPatch(p) d.addErrback(self.clientErrored, c) if not dueToFileChange: - self.dirtyFiles([ctx]) + self.watchedFiles.dirtyFiles([ctx]) sendToLiveClients(asJson=p.jsonRepr) - def dirtyFiles(self, ctxs): - """mark dirty the files that we watch in these contexts. - - the ctx might not be a file that we already read; it might be - for a new file we have to create, or it might be for a - transient context that we're not going to save - - if it's a ctx with no file, error - """ - for ctx in ctxs: - g = self.getSubgraph(ctx) - - if ctx not in self.graphFiles: - outFile = self.fileForUri(ctx) - self.graphFiles[ctx] = GraphFile(self.notifier, outFile, ctx, - self.patch, self.getSubgraph) - - self.graphFiles[ctx].dirty(g) - def clientErrored(self, err, c): err.trap(twisted.internet.error.ConnectError) log.info("connection error- dropping client %r" % c) @@ -375,7 +394,7 @@ if __name__ == "__main__": if not options.show: raise ValueError("missing --show http://...") - db = Db() + db = Db(topDirsToWatch=['show/dance2013']) from twisted.python import log as twlog twlog.startLogging(sys.stdout)