Mercurial > code > home > repos > rdfdb
changeset 42:541010f99d61
type fixes
Ignore-this: fbd389c09e0909acc6f9da26999f313d
author | Drew Perttula <drewp@bigasterisk.com> |
---|---|
date | Mon, 27 May 2019 00:39:16 +0000 |
parents | 3ee7386327c2 |
children | 8d4822ae58bc |
files | rdfdb/file_vs_uri.py rdfdb/patchreceiver.py rdfdb/patchsender.py rdfdb/service.py run_mypy.sh |
diffstat | 5 files changed, 62 insertions(+), 50 deletions(-) [+] |
line wrap: on
line diff
--- a/rdfdb/file_vs_uri.py Sun May 26 19:33:00 2019 +0000 +++ b/rdfdb/file_vs_uri.py Mon May 27 00:39:16 2019 +0000 @@ -4,23 +4,26 @@ """ import os from rdflib import URIRef +from typing import Dict -def uriFromFile(dirUriMap, filename): - assert filename.endswith('.n3'), filename +DirUriMap = Dict[bytes, URIRef] + +def uriFromFile(dirUriMap: DirUriMap, filename: bytes) -> URIRef: + assert filename.endswith(b'.n3'), filename for d, prefix in list(dirUriMap.items()): if filename.startswith(d): - return URIRef(prefix + filename[len(d):-len('.n3')]) + return URIRef(prefix + filename[len(d):-len(b'.n3')].decode('ascii')) raise ValueError("filename %s doesn't start with any of %s" % (filename, list(dirUriMap.keys()))) -def fileForUri(dirUriMap, ctx): +def fileForUri(dirUriMap: DirUriMap, ctx: URIRef) -> bytes: assert isinstance(ctx, URIRef), ctx - for d, prefix in list(dirUriMap.items()): + for d, prefix in dirUriMap.items(): if ctx.startswith(prefix): - return d + ctx[len(prefix):] + '.n3' + return d + ctx[len(prefix):].encode('utf8') + b'.n3' raise ValueError("don't know what filename to use for %s" % ctx) -def correctToTopdirPrefix(dirUriMap, inFile): +def correctToTopdirPrefix(dirUriMap: DirUriMap, inFile: bytes) -> bytes: if not any(inFile.startswith(prefix) for prefix in dirUriMap): for prefix in dirUriMap: prefixAbs = os.path.abspath(prefix)
--- a/rdfdb/patchreceiver.py Sun May 26 19:33:00 2019 +0000 +++ b/rdfdb/patchreceiver.py Mon May 27 00:39:16 2019 +0000 @@ -1,4 +1,5 @@ -import logging, cyclone.httpclient, traceback, urllib.request, urllib.parse, urllib.error +import logging, traceback, urllib.request, urllib.parse, urllib.error +import cyclone.web, cyclone.httpclient from twisted.internet import reactor from rdfdb.patch import Patch log = logging.getLogger('syncedgraph')
--- a/rdfdb/patchsender.py Sun May 26 19:33:00 2019 +0000 +++ b/rdfdb/patchsender.py Mon May 27 00:39:16 2019 +0000 @@ -1,6 +1,7 @@ import logging, time from typing import List import cyclone.httpclient +from rdflib import URIRef from twisted.internet import defer from rdfdb.patch import Patch @@ -93,7 +94,7 @@ log.error(e) self._continueSending() -def sendPatch(putUri, patch, **kw) -> defer.Deferred: +def sendPatch(putUri: URIRef, patch: Patch, **kw) -> defer.Deferred: """ PUT a patch as json to an http server. Returns deferred. @@ -102,7 +103,7 @@ t1 = time.time() body = patch.makeJsonRepr(kw) jsonTime = time.time() - t1 - intro = body[:200].decode('utf8') + intro = body[:200] if len(body) > 200: intro = intro + "..." log.debug("send body (rendered %.1fkB in %.1fms): %s", len(body) / 1024, jsonTime * 1000, intro) @@ -116,8 +117,8 @@ return done return cyclone.httpclient.fetch( - url=putUri, + url=putUri.toPython().encode('ascii'), method=b'PUT', headers={b'Content-Type': [b'application/json']}, - postdata=body, + postdata=body.encode('utf8'), ).addCallback(putDone)
--- a/rdfdb/service.py Sun May 26 19:33:00 2019 +0000 +++ b/rdfdb/service.py Mon May 27 00:39:16 2019 +0000 @@ -5,13 +5,13 @@ from twisted.internet.inotify import IN_CREATE import sys, optparse, logging, json, os import cyclone.web, cyclone.httpclient, cyclone.websocket -from typing import Dict, List, Set, Optional +from typing import Dict, List, Set, Optional, Union from rdflib import ConjunctiveGraph, URIRef, Graph from rdfdb.graphfile import GraphFile from rdfdb.patch import Patch, ALLSTMTS from rdfdb.rdflibpatch import patchQuads -from rdfdb.file_vs_uri import correctToTopdirPrefix, fileForUri, uriFromFile +from rdfdb.file_vs_uri import correctToTopdirPrefix, fileForUri, uriFromFile, DirUriMap from rdfdb.patchsender import sendPatch from rdfdb.patchreceiver import makePatchEndpointPutMethod @@ -22,20 +22,13 @@ class WebsocketDisconnect(ValueError): pass - -def sendGraphToClient(graph, client): - """send the client the whole graph contents""" - log.info("sending all graphs to %r" % client) - client.sendPatch(Patch( - addQuads=graph.quads(ALLSTMTS), - delQuads=[])) class Client(object): """ one of our syncedgraph clients """ - def __init__(self, updateUri: bytes, label: str): + def __init__(self, updateUri: URIRef, label: str): self.label = label # todo: updateUri is used publicly to compare clients. Replace # it with Client.__eq__ so WsClient doesn't have to fake an @@ -45,7 +38,7 @@ def __repr__(self): return "<%s client at %s>" % (self.label, self.updateUri) - def sendPatch(self, p): + def sendPatch(self, p: Patch) -> defer.Deferred: """ returns deferred. error will be interpreted as the client being broken. @@ -53,16 +46,23 @@ return sendPatch(self.updateUri, p) class WsClient(object): - def __init__(self, connectionId: bytes, sendMessage): - self.updateUri = connectionId + def __init__(self, connectionId: str, sendMessage): + self.updateUri = URIRef(connectionId) self.sendMessage = sendMessage def __repr__(self): return "<WsClient %s>" % self.updateUri - def sendPatch(self, p): + def sendPatch(self, p: Patch) -> defer.Deferred: self.sendMessage(p.makeJsonRepr()) return defer.succeed(None) + +def sendGraphToClient(graph, client: Union[Client, WsClient]) -> None: + """send the client the whole graph contents""" + log.info("sending all graphs to %r" % client) + client.sendPatch(Patch( + addQuads=graph.quads(ALLSTMTS), + delQuads=[])) class WatchedFiles(object): """ @@ -70,7 +70,7 @@ This object watches directories. Each GraphFile watches its own file. """ - def __init__(self, dirUriMap, patch, getSubgraph, addlPrefixes): + def __init__(self, dirUriMap: DirUriMap, patch, getSubgraph, addlPrefixes): self.dirUriMap = dirUriMap # {abspath : uri prefix} self.patch, self.getSubgraph = patch, getSubgraph self.addlPrefixes = addlPrefixes @@ -82,19 +82,22 @@ self.findAndLoadFiles() - def findAndLoadFiles(self): + def findAndLoadFiles(self) -> None: self.initialLoad = True try: for topdir in self.dirUriMap: for dirpath, dirnames, filenames in os.walk(topdir): for base in filenames: - self.watchFile(os.path.join(dirpath, base)) + p = os.path.join(dirpath, base) + # why wasn't mypy catching this? + assert isinstance(p, bytes) + self.watchFile(p) self.notifier.watch(FilePath(dirpath), autoAdd=True, callbacks=[self.dirChange]) finally: self.initialLoad = False - def dirChange(self, watch, path, mask): + def dirChange(self, watch, path: FilePath, mask): if mask & IN_CREATE: if path.path.endswith((b'~', b'.swp', b'swx', b'.rdfdb-temp')): return @@ -114,10 +117,10 @@ return inFile = correctToTopdirPrefix(self.dirUriMap, inFile) - if os.path.splitext(inFile)[1] not in ['.n3']: + if os.path.splitext(inFile)[1] not in [b'.n3']: return - if '/capture/' in inFile: + if b'/capture/' in inFile: # smaller graph for now return @@ -126,13 +129,13 @@ # SyncedGraph calls graphFromNQuad on the incoming data and # has a parse error. I'm not sure where this should be fixed # yet. - if '-rules' in inFile: + if b'-rules' in inFile: return # for legacy versions, compile all the config stuff you want # read into one file called config.n3. New versions won't read # it. - if inFile.endswith("config.n3"): + if inFile.endswith(b"config.n3"): return ctx = uriFromFile(self.dirUriMap, inFile) @@ -186,9 +189,8 @@ """ the master graph, all the connected clients, all the files we're watching """ - def __init__(self, dirUriMap, addlPrefixes): - - self.clients: List[Client] = [] + def __init__(self, dirUriMap: DirUriMap, addlPrefixes): + self.clients: List[Union[Client, WsClient]] = [] self.graph = ConjunctiveGraph() self.watchedFiles = WatchedFiles(dirUriMap, @@ -259,9 +261,10 @@ g.add(s) return g - def addClient(self, newClient): - [self.clients.remove(c) - for c in self.clients if c.updateUri == newClient.updateUri] + def addClient(self, newClient: Union[Client, WsClient]) -> None: + for c in self.clients: + if c.updateUri == newClient.updateUri: + self.clients.remove(c) log.info("new client %r" % newClient) sendGraphToClient(self.graph, newClient) @@ -270,7 +273,7 @@ def sendClientsToAllLivePages(self) -> None: sendToLiveClients({"clients": [ - dict(updateUri=c.updateUri.decode('utf8'), label=repr(c)) + dict(updateUri=c.updateUri.toPython(), label=repr(c)) for c in self.clients]}) class GraphResource(cyclone.web.RequestHandler): @@ -304,8 +307,8 @@ def get(self): pass - def post(self): - upd = self.get_argument("clientUpdate").encode('utf8') + def post(self) -> None: + upd = URIRef(self.get_argument("clientUpdate")) try: self.settings.db.addClient(Client(upd, self.get_argument("label"))) except: @@ -323,9 +326,9 @@ class WebsocketClient(cyclone.websocket.WebSocketHandler): wsClient: Optional[WsClient] = None - def connectionMade(self, *args, **kwargs): + def connectionMade(self, *args, **kwargs) -> None: global _wsClientSerial - connectionId = ('connection-%s' % _wsClientSerial).encode('utf8') + connectionId = f'connection-{_wsClientSerial}' _wsClientSerial += 1 self.wsClient = WsClient(connectionId, self.sendMessage) @@ -343,6 +346,7 @@ return log.info("got message from %r: %s", self.wsClient, message) p = Patch(jsonRepr=message.decode('utf8')) + assert self.wsClient is not None p.senderUpdateUri = self.wsClient.updateUri self.settings.db.patch(p) @@ -375,15 +379,17 @@ cyclone.web.StaticFileHandler.get(self, path, *args, **kw) -def main(dirUriMap=None, prefixes=None, port=9999): +def main(dirUriMap: Optional[DirUriMap]=None, + prefixes: Optional[Dict[str, URIRef]]=None, + port=9999): if dirUriMap is None: - dirUriMap = {'data/': URIRef('http://example.com/data/')} + dirUriMap = {b'data/': URIRef('http://example.com/data/')} if prefixes is None: prefixes = { - 'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#', - 'rdfs': 'http://www.w3.org/2000/01/rdf-schema#', - 'xsd': 'http://www.w3.org/2001/XMLSchema#', + 'rdf': URIRef('http://www.w3.org/1999/02/22-rdf-syntax-ns#'), + 'rdfs': URIRef('http://www.w3.org/2000/01/rdf-schema#'), + 'xsd': URIRef('http://www.w3.org/2001/XMLSchema#'), } logging.basicConfig()