changeset 42:541010f99d61

type fixes Ignore-this: fbd389c09e0909acc6f9da26999f313d
author Drew Perttula <drewp@bigasterisk.com>
date Mon, 27 May 2019 00:39:16 +0000
parents 3ee7386327c2
children 8d4822ae58bc
files rdfdb/file_vs_uri.py rdfdb/patchreceiver.py rdfdb/patchsender.py rdfdb/service.py run_mypy.sh
diffstat 5 files changed, 62 insertions(+), 50 deletions(-) [+]
line wrap: on
line diff
--- a/rdfdb/file_vs_uri.py	Sun May 26 19:33:00 2019 +0000
+++ b/rdfdb/file_vs_uri.py	Mon May 27 00:39:16 2019 +0000
@@ -4,23 +4,26 @@
 """
 import os
 from rdflib import URIRef
+from typing import Dict
 
-def uriFromFile(dirUriMap, filename):
-    assert filename.endswith('.n3'), filename
+DirUriMap = Dict[bytes, URIRef]
+
+def uriFromFile(dirUriMap: DirUriMap, filename: bytes) -> URIRef:
+    assert filename.endswith(b'.n3'), filename
     for d, prefix in list(dirUriMap.items()):
         if filename.startswith(d):
-            return URIRef(prefix + filename[len(d):-len('.n3')])
+            return URIRef(prefix + filename[len(d):-len(b'.n3')].decode('ascii'))
     raise ValueError("filename %s doesn't start with any of %s" %
                      (filename, list(dirUriMap.keys())))
 
-def fileForUri(dirUriMap, ctx):
+def fileForUri(dirUriMap: DirUriMap, ctx: URIRef) -> bytes:
     assert isinstance(ctx, URIRef), ctx
-    for d, prefix in list(dirUriMap.items()):
+    for d, prefix in dirUriMap.items():
         if ctx.startswith(prefix):
-            return d + ctx[len(prefix):] + '.n3'
+            return d + ctx[len(prefix):].encode('utf8') + b'.n3'
     raise ValueError("don't know what filename to use for %s" % ctx)
 
-def correctToTopdirPrefix(dirUriMap, inFile):
+def correctToTopdirPrefix(dirUriMap: DirUriMap, inFile: bytes) -> bytes:
     if not any(inFile.startswith(prefix) for prefix in dirUriMap):
         for prefix in dirUriMap:
             prefixAbs = os.path.abspath(prefix)
--- a/rdfdb/patchreceiver.py	Sun May 26 19:33:00 2019 +0000
+++ b/rdfdb/patchreceiver.py	Mon May 27 00:39:16 2019 +0000
@@ -1,4 +1,5 @@
-import logging, cyclone.httpclient, traceback, urllib.request, urllib.parse, urllib.error
+import logging, traceback, urllib.request, urllib.parse, urllib.error
+import cyclone.web, cyclone.httpclient
 from twisted.internet import reactor
 from rdfdb.patch import Patch
 log = logging.getLogger('syncedgraph')
--- a/rdfdb/patchsender.py	Sun May 26 19:33:00 2019 +0000
+++ b/rdfdb/patchsender.py	Mon May 27 00:39:16 2019 +0000
@@ -1,6 +1,7 @@
 import logging, time
 from typing import List
 import cyclone.httpclient
+from rdflib import URIRef
 from twisted.internet import defer
 from rdfdb.patch import Patch
 
@@ -93,7 +94,7 @@
         log.error(e)
         self._continueSending()
 
-def sendPatch(putUri, patch, **kw) -> defer.Deferred:
+def sendPatch(putUri: URIRef, patch: Patch, **kw) -> defer.Deferred:
     """
     PUT a patch as json to an http server. Returns deferred.
     
@@ -102,7 +103,7 @@
     t1 = time.time()
     body = patch.makeJsonRepr(kw)
     jsonTime = time.time() - t1
-    intro = body[:200].decode('utf8')
+    intro = body[:200]
     if len(body) > 200:
         intro = intro + "..."
     log.debug("send body (rendered %.1fkB in %.1fms): %s", len(body) / 1024, jsonTime * 1000, intro)
@@ -116,8 +117,8 @@
         return done
 
     return cyclone.httpclient.fetch(
-        url=putUri,
+        url=putUri.toPython().encode('ascii'),
         method=b'PUT',
         headers={b'Content-Type': [b'application/json']},
-        postdata=body,
+        postdata=body.encode('utf8'),
         ).addCallback(putDone)
--- a/rdfdb/service.py	Sun May 26 19:33:00 2019 +0000
+++ b/rdfdb/service.py	Mon May 27 00:39:16 2019 +0000
@@ -5,13 +5,13 @@
 from twisted.internet.inotify import IN_CREATE
 import sys, optparse, logging, json, os
 import cyclone.web, cyclone.httpclient, cyclone.websocket
-from typing import Dict, List, Set, Optional
+from typing import Dict, List, Set, Optional, Union
 
 from rdflib import ConjunctiveGraph, URIRef, Graph
 from rdfdb.graphfile import GraphFile
 from rdfdb.patch import Patch, ALLSTMTS
 from rdfdb.rdflibpatch import patchQuads
-from rdfdb.file_vs_uri import correctToTopdirPrefix, fileForUri, uriFromFile
+from rdfdb.file_vs_uri import correctToTopdirPrefix, fileForUri, uriFromFile, DirUriMap
 from rdfdb.patchsender import sendPatch
 from rdfdb.patchreceiver import makePatchEndpointPutMethod
 
@@ -22,20 +22,13 @@
 
 class WebsocketDisconnect(ValueError):
     pass
-
-def sendGraphToClient(graph, client):
-    """send the client the whole graph contents"""
-    log.info("sending all graphs to %r" % client)
-    client.sendPatch(Patch(
-        addQuads=graph.quads(ALLSTMTS),
-        delQuads=[]))
     
 
 class Client(object):
     """
     one of our syncedgraph clients
     """
-    def __init__(self, updateUri: bytes, label: str):
+    def __init__(self, updateUri: URIRef, label: str):
         self.label = label
         # todo: updateUri is used publicly to compare clients. Replace
         # it with Client.__eq__ so WsClient doesn't have to fake an
@@ -45,7 +38,7 @@
     def __repr__(self):
         return "<%s client at %s>" % (self.label, self.updateUri)
 
-    def sendPatch(self, p):
+    def sendPatch(self, p: Patch) -> defer.Deferred:
         """
         returns deferred. error will be interpreted as the client being
         broken.
@@ -53,16 +46,23 @@
         return sendPatch(self.updateUri, p)
         
 class WsClient(object):
-    def __init__(self, connectionId: bytes, sendMessage):
-        self.updateUri = connectionId
+    def __init__(self, connectionId: str, sendMessage):
+        self.updateUri = URIRef(connectionId)
         self.sendMessage = sendMessage
 
     def __repr__(self):
         return "<WsClient %s>" % self.updateUri
 
-    def sendPatch(self, p):
+    def sendPatch(self, p: Patch) -> defer.Deferred:
         self.sendMessage(p.makeJsonRepr())
         return defer.succeed(None)
+
+def sendGraphToClient(graph, client: Union[Client, WsClient]) -> None:
+    """send the client the whole graph contents"""
+    log.info("sending all graphs to %r" % client)
+    client.sendPatch(Patch(
+        addQuads=graph.quads(ALLSTMTS),
+        delQuads=[]))
         
 class WatchedFiles(object):
     """
@@ -70,7 +70,7 @@
 
     This object watches directories. Each GraphFile watches its own file.
     """
-    def __init__(self, dirUriMap, patch, getSubgraph, addlPrefixes):
+    def __init__(self, dirUriMap: DirUriMap, patch, getSubgraph, addlPrefixes):
         self.dirUriMap = dirUriMap # {abspath : uri prefix}
         self.patch, self.getSubgraph = patch, getSubgraph
         self.addlPrefixes = addlPrefixes
@@ -82,19 +82,22 @@
         
         self.findAndLoadFiles()
 
-    def findAndLoadFiles(self):
+    def findAndLoadFiles(self) -> None:
         self.initialLoad = True
         try:
             for topdir in self.dirUriMap:
                 for dirpath, dirnames, filenames in os.walk(topdir):
                     for base in filenames:
-                        self.watchFile(os.path.join(dirpath, base))
+                        p = os.path.join(dirpath, base)
+                        # why wasn't mypy catching this?
+                        assert isinstance(p, bytes)
+                        self.watchFile(p)
                     self.notifier.watch(FilePath(dirpath), autoAdd=True,
                                         callbacks=[self.dirChange])
         finally:
             self.initialLoad = False
 
-    def dirChange(self, watch, path, mask):
+    def dirChange(self, watch, path: FilePath, mask):
         if mask & IN_CREATE:
             if path.path.endswith((b'~', b'.swp', b'swx', b'.rdfdb-temp')):
                 return
@@ -114,10 +117,10 @@
             return
 
         inFile = correctToTopdirPrefix(self.dirUriMap, inFile)
-        if os.path.splitext(inFile)[1] not in ['.n3']:
+        if os.path.splitext(inFile)[1] not in [b'.n3']:
             return
 
-        if '/capture/' in inFile:
+        if b'/capture/' in inFile:
             # smaller graph for now
             return
             
@@ -126,13 +129,13 @@
         # SyncedGraph calls graphFromNQuad on the incoming data and
         # has a parse error. I'm not sure where this should be fixed
         # yet.
-        if '-rules' in inFile:
+        if b'-rules' in inFile:
             return
 
         # for legacy versions, compile all the config stuff you want
         # read into one file called config.n3. New versions won't read
         # it.
-        if inFile.endswith("config.n3"):
+        if inFile.endswith(b"config.n3"):
             return
             
         ctx = uriFromFile(self.dirUriMap, inFile)
@@ -186,9 +189,8 @@
     """
     the master graph, all the connected clients, all the files we're watching
     """
-    def __init__(self, dirUriMap, addlPrefixes):
-      
-        self.clients: List[Client] = []
+    def __init__(self, dirUriMap: DirUriMap, addlPrefixes):
+        self.clients: List[Union[Client, WsClient]] = []
         self.graph = ConjunctiveGraph()
 
         self.watchedFiles = WatchedFiles(dirUriMap,
@@ -259,9 +261,10 @@
             g.add(s)
         return g
 
-    def addClient(self, newClient):
-        [self.clients.remove(c)
-         for c in self.clients if c.updateUri == newClient.updateUri]
+    def addClient(self, newClient: Union[Client, WsClient]) -> None:
+        for c in self.clients:
+            if c.updateUri == newClient.updateUri:
+                self.clients.remove(c)
 
         log.info("new client %r" % newClient)
         sendGraphToClient(self.graph, newClient)
@@ -270,7 +273,7 @@
 
     def sendClientsToAllLivePages(self) -> None:
         sendToLiveClients({"clients": [
-            dict(updateUri=c.updateUri.decode('utf8'), label=repr(c))
+            dict(updateUri=c.updateUri.toPython(), label=repr(c))
             for c in self.clients]})
 
 class GraphResource(cyclone.web.RequestHandler):
@@ -304,8 +307,8 @@
     def get(self):
         pass
 
-    def post(self):
-        upd = self.get_argument("clientUpdate").encode('utf8')
+    def post(self) -> None:
+        upd = URIRef(self.get_argument("clientUpdate"))
         try:
             self.settings.db.addClient(Client(upd, self.get_argument("label")))
         except:
@@ -323,9 +326,9 @@
 class WebsocketClient(cyclone.websocket.WebSocketHandler):
 
     wsClient: Optional[WsClient] = None
-    def connectionMade(self, *args, **kwargs):
+    def connectionMade(self, *args, **kwargs) -> None:
         global _wsClientSerial
-        connectionId = ('connection-%s' % _wsClientSerial).encode('utf8')
+        connectionId = f'connection-{_wsClientSerial}'
         _wsClientSerial += 1
 
         self.wsClient = WsClient(connectionId, self.sendMessage)
@@ -343,6 +346,7 @@
             return
         log.info("got message from %r: %s", self.wsClient, message)
         p = Patch(jsonRepr=message.decode('utf8'))
+        assert self.wsClient is not None
         p.senderUpdateUri = self.wsClient.updateUri
         self.settings.db.patch(p)
 
@@ -375,15 +379,17 @@
         cyclone.web.StaticFileHandler.get(self, path, *args, **kw)
 
 
-def main(dirUriMap=None, prefixes=None, port=9999):
+def main(dirUriMap: Optional[DirUriMap]=None,
+         prefixes: Optional[Dict[str, URIRef]]=None,
+         port=9999):
 
     if dirUriMap is None:
-        dirUriMap = {'data/': URIRef('http://example.com/data/')}
+        dirUriMap = {b'data/': URIRef('http://example.com/data/')}
     if prefixes is None:
         prefixes = {
-            'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
-            'rdfs': 'http://www.w3.org/2000/01/rdf-schema#',
-            'xsd': 'http://www.w3.org/2001/XMLSchema#',
+            'rdf': URIRef('http://www.w3.org/1999/02/22-rdf-syntax-ns#'),
+            'rdfs': URIRef('http://www.w3.org/2000/01/rdf-schema#'),
+            'xsd': URIRef('http://www.w3.org/2001/XMLSchema#'),
         }
     
     logging.basicConfig()
--- a/run_mypy.sh	Sun May 26 19:33:00 2019 +0000
+++ b/run_mypy.sh	Mon May 27 00:39:16 2019 +0000
@@ -1,4 +1,5 @@
 #!/bin/sh
 
+# see light9 for a flake8 alternative
 pyflakes rdfdb/*.py
 MYPYPATH=stubs mypy --check-untyped-defs rdfdb/*.py