light9 Changeset - e53e78db7b17

Changeset - e53e78db7b17

Parent rev.

Child rev.

[Not reviewed]

default

0 2 0

Drew Perttula - 12 years ago 2013-06-04 22:46:34
drewp@bigasterisk.com

refactor file watching. notice new files and dirs.
Ignore-this: 7f571fd9b309b54cff22b2fee5e68322

2 files changed with 82 insertions and 56 deletions:

bin/rdfdb

light9/rdfdb/graphfile.py

0 comments (0 inline, 0 general)

bin/rdfdb

➞

Show inline comments

@@ @@ -109,6 +109,8 @@ Our web ui: @@
 """
 from twisted.internet import reactor
 import twisted.internet.error
 from twisted.python.filepath import FilePath
 from twisted.internet.inotify import humanReadableMask, IN_CREATE
 import sys, optparse, logging, json, os
 import cyclone.web, cyclone.httpclient, cyclone.websocket
 sys.path.append(".")
@@ @@ -155,54 +157,82 @@ class Client(object): @@
     def sendPatch(self, p):
         return syncedgraph.sendPatch(self.updateUri, p)
-class Db(object):
+class WatchedFiles(object):
     """
     the master graph, all the connected clients, all the files we're watching
     find files, notice new files.
     This object watches directories. Each GraphFile watches its own file.
     """
     def __init__(self):
     def __init__(self, topDirsToWatch, patch, getSubgraph):
         self.topDirsToWatch = topDirsToWatch
         self.patch, self.getSubgraph = patch, getSubgraph
         # files from cwd become uris starting with this. *should* be
         # building uris from the show uri in $LIGHT9_SHOW/URI
         # instead. Who wants to keep their data in the same dir tree
         # as the source code?!
         self.topUri = URIRef("http://light9.bigasterisk.com/")
         self.clients = []
         self.graph = ConjunctiveGraph()
         self.graphFiles = {} # context uri : GraphFile
         self.notifier = INotify()
         self.notifier.startReading()
         self.graphFiles = {} # context uri : GraphFile
         self.findAndLoadFiles()
     def findAndLoadFiles(self):
         self.initialLoad = True
         try:
             dirs = [
                 "show/dance2012/sessions",
                 "show/dance2012/subs",
                 "show/dance2012/subterms",
+                ]
             for topdir in dirs:
             for topdir in self.topDirsToWatch:
                 for dirpath, dirnames, filenames in os.walk(topdir):
                     for base in filenames:
                         self.watchFile(os.path.join(dirpath, base))
                 # todo: also notice new files in this dir
             self.watchFile("show/dance2012/config.n3")
             self.watchFile("show/dance2012/patch.n3")
                     self.notifier.watch(FilePath(dirpath), autoAdd=True,
                                         callbacks=[self.dirChange])
         finally:
             self.initialLoad = False
         self.summarizeToLog()
     def dirChange(self, watch, path, mask):
         if mask & IN_CREATE:
             self.watchFile(path.path)
     def watchFile(self, inFile):
         if not isinstance(inFile, FilePath):
             inFile = FilePath(inFile)
         if not inFile.isfile():
             return
         if inFile.splitext()[1] not in ['.n3']:
             return
         ctx = self.uriFromFile(inFile)
         gf = GraphFile(self.notifier, inFile.path, ctx,
                        self.patch, self.getSubgraph)
         self.graphFiles[ctx] = gf
         gf.reread()
     def dirtyFiles(self, ctxs):
         """mark dirty the files that we watch in these contexts.
         the ctx might not be a file that we already read; it might be
         for a new file we have to create, or it might be for a
         transient context that we're not going to save
         if it's a ctx with no file, error
         """
         for ctx in ctxs:
             g = self.getSubgraph(ctx)
             if ctx not in self.graphFiles:
                 outFile = self.fileForUri(ctx)
                 self.graphFiles[ctx] = GraphFile(self.notifier, outFile, ctx,
                                                  self.patch, self.getSubgraph)
             self.graphFiles[ctx].dirty(g)
     def uriFromFile(self, filename):
         if filename.endswith('.n3'):
             # some legacy files don't end with n3. when we write them
             # back this might not go so well
             filename = filename[:-len('.n3')]
         return URIRef(self.topUri + filename)
         if isinstance(filename, FilePath):
             filename = filename.path
         assert filename.endswith('.n3'), filename
         return URIRef(self.topUri + filename[:-len('.n3')])
     def fileForUri(self, ctx):
         assert isinstance(ctx, URIRef), ctx
@@ @@ -210,11 +240,20 @@ class Db(object): @@
             raise ValueError("don't know what filename to use for %s" % ctx)
         return ctx[len(self.topUri):] + ".n3"
     def watchFile(self, inFile):
         ctx = self.uriFromFile(inFile)
         gf = GraphFile(self.notifier, inFile, ctx, self.patch, self.getSubgraph)
         self.graphFiles[ctx] = gf
         gf.reread()
 class Db(object):
     """
     the master graph, all the connected clients, all the files we're watching
     """
     def __init__(self, topDirsToWatch):
         self.clients = []
         self.graph = ConjunctiveGraph()
         self.watchedFiles = WatchedFiles(topDirsToWatch,
                                          self.patch, self.getSubgraph)
         self.summarizeToLog()
     def patch(self, p, dueToFileChange=False):
         """
@@ @@ -232,8 +271,7 @@ class Db(object): @@
         patchQuads(self.graph, p.delQuads, p.addQuads, perfect=True)
         senderUpdateUri = getattr(p, 'senderUpdateUri', None)
         #if not self.initialLoad:
         #    self.summarizeToLog()
         for c in self.clients:
             if c.updateUri == senderUpdateUri:
                 # this client has self-applied the patch already
@@ @@ -241,28 +279,9 @@ class Db(object): @@
             d = c.sendPatch(p)
             d.addErrback(self.clientErrored, c)
         if not dueToFileChange:
             self.dirtyFiles([ctx])
+            self.watchedFiles.dirtyFiles([ctx])
         sendToLiveClients(asJson=p.jsonRepr)
     def dirtyFiles(self, ctxs):
         """mark dirty the files that we watch in these contexts.
         the ctx might not be a file that we already read; it might be
         for a new file we have to create, or it might be for a
         transient context that we're not going to save
         if it's a ctx with no file, error
         """
         for ctx in ctxs:
             g = self.getSubgraph(ctx)
             if ctx not in self.graphFiles:
                 outFile = self.fileForUri(ctx)
                 self.graphFiles[ctx] = GraphFile(self.notifier, outFile, ctx,
                                                  self.patch, self.getSubgraph)
             self.graphFiles[ctx].dirty(g)
     def clientErrored(self, err, c):
         err.trap(twisted.internet.error.ConnectError)
         log.info("connection error- dropping client %r" % c)
@@ @@ -375,7 +394,7 @@ if __name__ == "__main__": @@
     if not options.show:
         raise ValueError("missing --show http://...")
     db = Db()
+    db = Db(topDirsToWatch=['show/dance2013'])
     from twisted.python import log as twlog
     twlog.startLogging(sys.stdout)

light9/rdfdb/graphfile.py

➞

Show inline comments

@@ @@ -38,9 +38,14 @@ class GraphFile(object): @@
                        callbacks=[self.notify])
     def notify(self, notifier, filepath, mask):
         if filepath.getModificationTime() == self.lastWriteTimestamp:
             log.debug("file %s changed, but we did this write", filepath)
         try:
             if filepath.getModificationTime() == self.lastWriteTimestamp:
                 log.debug("file %s changed, but we did this write", filepath)
                 return
         except OSError as e:
             log.error("watched file %s: %r" % (filepath, e))
             return
         log.info("file %s changed", filepath)
         try:
             self.reread()
@@ @@ -57,10 +62,12 @@ class GraphFile(object): @@
             print e
             log.error("syntax error in %s" % self.path)
             return
         except IOError as e:
             log.error("rereading %s: %r" % (self.uri, e))
             return
         old = inContext(old, self.uri)
         new = inContext(new, self.uri)
         print "old %s new %s" % (old, new)
         p = Patch.fromDiff(old, new)
         if p:

0 comments (0 inline, 0 general)