# HG changeset patch # User drewp@bigasterisk.com # Date 1649095319 25200 # Node ID 22c9679dbf6744167d631e76a2e0d6086a9b9a4a # Parent e47dd82a7ddd77e499c418e44a7f764b0c3acfab reformat diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/__init__.py --- a/rdfdb/__init__.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/__init__.py Mon Apr 04 11:01:59 2022 -0700 @@ -1,1 +0,0 @@ - diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/autodepgraphapi.py --- a/rdfdb/autodepgraphapi.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/autodepgraphapi.py Mon Apr 04 11:01:59 2022 -0700 @@ -1,7 +1,10 @@ import logging -from typing import Callable, Dict, Set, Tuple, List +from typing import Callable, Dict, List, Set, Tuple + from rdflib import RDF, RDFS, URIRef + from rdfdb.currentstategraphapi import contextsForStatementNoWildcards + log = logging.getLogger('autodepgraphapi') @@ -22,8 +25,7 @@ def __init__(self): self._watchers = _GraphWatchers() - self.currentFuncs: List[Callable[[], None]] = [ - ] # stack of addHandler callers + self.currentFuncs: List[Callable[[], None]] = [] # stack of addHandler callers def addHandler(self, func: Callable[[], None]) -> None: """ @@ -56,8 +58,7 @@ raise finally: self.currentFuncs.pop() - log.debug('graph.currentFuncs pop %s. stack now has %s', func, - len(self.currentFuncs)) + log.debug('graph.currentFuncs pop %s. stack now has %s', func, len(self.currentFuncs)) def runDepsOnNewPatch(self, p): """ @@ -89,21 +90,12 @@ # between calls, but probably most of them don't do that (they # work from a starting uri) - def value(self, - subject=None, - predicate=RDF.value, - object=None, - default=None, - any=True): + def value(self, subject=None, predicate=RDF.value, object=None, default=None, any=True): if object is not None: raise NotImplementedError() func = self._getCurrentFunc() self._watchers.addSubjPredWatcher(func, subject, predicate) - return self._graph.value(subject, - predicate, - object=object, - default=default, - any=any) + return self._graph.value(subject, predicate, object=object, default=default, any=any) def objects(self, subject=None, predicate=None): func = self._getCurrentFunc() @@ -163,12 +155,9 @@ """ def __init__(self): - self._handlersSp: Dict[Tuple[URIRef, URIRef], HandlerSet] = { - } # (s,p): set(handlers) - self._handlersPo: Dict[Tuple[URIRef, URIRef], HandlerSet] = { - } # (p,o): set(handlers) - self._handlersSpo: Dict[Tuple[URIRef, URIRef, URIRef], HandlerSet] = { - } # (s,p,o): set(handlers) + self._handlersSp: Dict[Tuple[URIRef, URIRef], HandlerSet] = {} # (s,p): set(handlers) + self._handlersPo: Dict[Tuple[URIRef, URIRef], HandlerSet] = {} # (p,o): set(handlers) + self._handlersSpo: Dict[Tuple[URIRef, URIRef, URIRef], HandlerSet] = {} # (s,p,o): set(handlers) self._handlersS: Dict[URIRef, HandlerSet] = {} # s: set(handlers) def addSubjPredWatcher(self, func, s, p): @@ -197,29 +186,25 @@ """ #self.dependencies() ret: Set[Callable[[], None]] = set() - affectedSubjPreds = set([(s, p) for s, p, o, c in patch.addQuads] + - [(s, p) for s, p, o, c in patch.delQuads]) + affectedSubjPreds = set([(s, p) for s, p, o, c in patch.addQuads] + [(s, p) for s, p, o, c in patch.delQuads]) for (s, p), funcs in self._handlersSp.items(): if (s, p) in affectedSubjPreds: ret.update(funcs) funcs.clear() - affectedPredObjs = set([(p, o) for s, p, o, c in patch.addQuads] + - [(p, o) for s, p, o, c in patch.delQuads]) + affectedPredObjs = set([(p, o) for s, p, o, c in patch.addQuads] + [(p, o) for s, p, o, c in patch.delQuads]) for (p, o), funcs in self._handlersPo.items(): if (p, o) in affectedPredObjs: ret.update(funcs) funcs.clear() - affectedTriples = set([(s, p, o) for s, p, o, c in patch.addQuads] + - [(s, p, o) for s, p, o, c in patch.delQuads]) + affectedTriples = set([(s, p, o) for s, p, o, c in patch.addQuads] + [(s, p, o) for s, p, o, c in patch.delQuads]) for triple, funcs in self._handlersSpo.items(): if triple in affectedTriples: ret.update(funcs) funcs.clear() - affectedSubjs = set([s for s, p, o, c in patch.addQuads] + - [s for s, p, o, c in patch.delQuads]) + affectedSubjs = set([s for s, p, o, c in patch.addQuads] + [s for s, p, o, c in patch.delQuads]) for subj, funcs in self._handlersS.items(): if subj in affectedSubjs: ret.update(funcs) diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/currentstategraphapi.py --- a/rdfdb/currentstategraphapi.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/currentstategraphapi.py Mon Apr 04 11:01:59 2022 -0700 @@ -1,6 +1,12 @@ -import logging, traceback, time, itertools +import itertools +import logging +import time +import traceback + from rdflib import ConjunctiveGraph, URIRef + from rdfdb.rdflibpatch import contextsForStatement as rp_contextsForStatement + log = logging.getLogger("currentstate") @@ -12,8 +18,7 @@ self.graph = graph def __getattr__(self, attr): - if attr in ['subjects', 'value', 'objects', 'triples', - 'label']: # not complete + if attr in ['subjects', 'value', 'objects', 'triples', 'label']: # not complete return getattr(self.graph, attr) raise TypeError("can't access %r of read-only graph" % attr) @@ -58,13 +63,11 @@ if tripleFilter == (None, None, None): self2.logThisCopy(g, time.time() - t1) - setattr(g, 'contextsForStatement', lambda t: - contextsForStatementNoWildcards(g, t)) + setattr(g, 'contextsForStatement', lambda t: contextsForStatementNoWildcards(g, t)) return g def logThisCopy(self, g, sec): - log.info("copied graph %s statements (%.1f ms) " - "because of this:" % (len(g), sec * 1000)) + log.info("copied graph %s statements (%.1f ms) " "because of this:" % (len(g), sec * 1000)) for frame in traceback.format_stack(limit=4)[:-2]: for line in frame.splitlines(): log.info(" " + line) @@ -86,8 +89,7 @@ self._reservedSequentials = set() for i in itertools.count(1): newUri = URIRef(prefix + str(i)) - if newUri not in self._reservedSequentials and not list( - self._graph.triples((newUri, None, None))): + if newUri not in self._reservedSequentials and not list(self._graph.triples((newUri, None, None))): self._reservedSequentials.add(newUri) return newUri diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/currentstategraphapi_test.py --- a/rdfdb/currentstategraphapi_test.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/currentstategraphapi_test.py Mon Apr 04 11:01:59 2022 -0700 @@ -1,5 +1,7 @@ import unittest + from rdflib import URIRef + from rdfdb.syncedgraph import SyncedGraph @@ -7,7 +9,5 @@ def test_returnsSequentialUris(self): g = SyncedGraph(URIRef('http://example.com/db/'), label='test') - self.assertEqual(g.sequentialUri(URIRef('http://example.com/foo')), - URIRef('http://example.com/foo1')) - self.assertEqual(g.sequentialUri(URIRef('http://example.com/foo')), - URIRef('http://example.com/foo2')) + self.assertEqual(g.sequentialUri(URIRef('http://example.com/foo')), URIRef('http://example.com/foo1')) + self.assertEqual(g.sequentialUri(URIRef('http://example.com/foo')), URIRef('http://example.com/foo2')) diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/file_vs_uri.py --- a/rdfdb/file_vs_uri.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/file_vs_uri.py Mon Apr 04 11:01:59 2022 -0700 @@ -3,8 +3,9 @@ undefined results that depend on python's items() order. """ import os +from typing import Dict + from rdflib import URIRef -from typing import Dict DirUriMap = Dict[bytes, URIRef] @@ -13,10 +14,8 @@ assert filename.endswith(b'.n3'), filename for d, prefix in list(dirUriMap.items()): if filename.startswith(d): - return URIRef(prefix + - filename[len(d):-len(b'.n3')].decode('ascii')) - raise ValueError("filename %s doesn't start with any of %s" % - (filename, list(dirUriMap.keys()))) + return URIRef(prefix + filename[len(d):-len(b'.n3')].decode('ascii')) + raise ValueError("filename %s doesn't start with any of %s" % (filename, list(dirUriMap.keys()))) def fileForUri(dirUriMap: DirUriMap, ctx: URIRef) -> bytes: @@ -35,6 +34,5 @@ inFile = prefix + inFile[len(prefixAbs):] break else: - raise ValueError("can't correct %s to start with one of %s" % - (inFile, list(dirUriMap.keys()))) + raise ValueError("can't correct %s to start with one of %s" % (inFile, list(dirUriMap.keys()))) return inFile diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/grapheditapi.py --- a/rdfdb/grapheditapi.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/grapheditapi.py Mon Apr 04 11:01:59 2022 -0700 @@ -1,8 +1,12 @@ -import random, logging +import logging +import random from itertools import chain -from rdflib import URIRef, RDF + +from rdflib import RDF, URIRef from rdflib.term import Node + from rdfdb.patch import Patch, quadsWithContextUris + log = logging.getLogger('graphedit') @@ -21,15 +25,12 @@ """ existing = [] - for spoc in quadsWithContextUris( - self._graph.quads((subject, predicate, None, context))): + for spoc in quadsWithContextUris(self._graph.quads((subject, predicate, None, context))): existing.append(spoc) - toAdd = ([(subject, predicate, newObject, - context)] if newObject is not None else []) + toAdd = ([(subject, predicate, newObject, context)] if newObject is not None else []) return Patch(delQuads=existing, addQuads=toAdd).simplify() - def patchObject(self, context: URIRef, subject: Node, predicate: URIRef, - newObject: Node): + def patchObject(self, context: URIRef, subject: Node, predicate: URIRef, newObject: Node): p = self.getObjectPatch(context, subject, predicate, newObject) if not p.isNoop(): log.debug("patchObject %r" % p.jsonRepr) @@ -40,16 +41,13 @@ replace all statements in 'context' with the quads in newGraph. This is not cooperating with currentState. """ - old = set( - quadsWithContextUris(self._graph.quads( - (None, None, None, context)))) + old = set(quadsWithContextUris(self._graph.quads((None, None, None, context)))) new = set(quadsWithContextUris(newGraph)) p = Patch(delQuads=old - new, addQuads=new - old) self.patch(p) return p # for debugging - def patchMapping(self, context, subject, predicate, nodeClass, keyPred, - valuePred, newKey, newValue): + def patchMapping(self, context, subject, predicate, nodeClass, keyPred, valuePred, newKey, newValue): """ creates/updates a structure like this: @@ -66,27 +64,23 @@ # tripleFilter optimization, this looks like a mess. If # currentState became cheap, a lot of code here could go away. - with self.currentState(tripleFilter=(subject, predicate, - None)) as current: + with self.currentState(tripleFilter=(subject, predicate, None)) as current: adds = set([]) for setting in current.objects(subject, predicate): - with self.currentState(tripleFilter=(setting, keyPred, - None)) as current2: + with self.currentState(tripleFilter=(setting, keyPred, None)) as current2: match = current2.value(setting, keyPred) == newKey if match: break else: - setting = URIRef(subject + - "/map/%s" % random.randrange(999999999)) + setting = URIRef(subject + "/map/%s" % random.randrange(999999999)) adds.update([ (subject, predicate, setting, context), (setting, RDF.type, nodeClass, context), (setting, keyPred, newKey, context), ]) - with self.currentState(tripleFilter=(setting, valuePred, - None)) as current: + with self.currentState(tripleFilter=(setting, valuePred, None)) as current: dels = set([]) for prev in current.objects(setting, valuePred): dels.add((setting, valuePred, prev, context)) @@ -102,14 +96,14 @@ patchMapping made. """ p = Patch(delQuads=[ - spo + (context,) for spo in chain( - self._graph.triples((None, None, node), context=context), - self._graph.triples((node, None, None), context=context)) + spo + (context,) + for spo in chain(self._graph.triples((None, None, node), context=context), self._graph.triples((node, None, None), context=context)) ]) self.patch(p) import unittest + from rdflib import ConjunctiveGraph diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/graphfile.py --- a/rdfdb/graphfile.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/graphfile.py Mon Apr 04 11:01:59 2022 -0700 @@ -1,14 +1,17 @@ -import logging, traceback, os, time -from twisted.python.filepath import FilePath +import logging +import os +import time +import traceback +from typing import Dict, Optional, Protocol + +from rdflib import RDF, XSD, Graph, Literal, URIRef from twisted.internet import reactor +from twisted.internet.inotify import INotify, humanReadableMask from twisted.internet.interfaces import IDelayedCall -from twisted.internet.inotify import INotify, humanReadableMask -from rdflib import Graph, RDF, URIRef, Literal, XSD +from twisted.python.filepath import FilePath + from rdfdb.patch import Patch from rdfdb.rdflibpatch import inContext -from typing import Dict, Optional -from typing_extensions import Protocol - log = logging.getLogger('graphfile') iolog = logging.getLogger('io') @@ -16,7 +19,7 @@ def patchN3SerializerToUseLessWhitespace(cutColumn=75): # todo: make a n3serializer subclass with whitespace settings - from rdflib.plugins.serializers.turtle import TurtleSerializer, OBJECT, VERB, _GEN_QNAME_FOR_DT + from rdflib.plugins.serializers.turtle import (_GEN_QNAME_FOR_DT, OBJECT, VERB, TurtleSerializer) originalWrite = TurtleSerializer.write def write(self, s): @@ -73,22 +76,24 @@ num = node.toPython() return '%g' % num return node._literal_n3(use_plain=True, qname_callback=qname_callback) - + def label(self, node, position): if node == RDF.nil: return '()' if position is VERB and node in self.keywords: return self.keywords[node] if isinstance(node, Literal): - return custom_literal(node, # <- switch to this - qname_callback=lambda dt: self.getQName( - dt, _GEN_QNAME_FOR_DT)) + return custom_literal( + node, # <- switch to this + qname_callback=lambda dt: self.getQName(dt, _GEN_QNAME_FOR_DT)) else: node = self.relativize(node) return self.getQName(node, position == VERB) or node.n3() + TurtleSerializer.label = label # type: ignore + patchN3SerializerToUseLessWhitespace() @@ -109,9 +114,7 @@ one rdf file that we read from, write to, and notice external changes to """ - def __init__(self, notifier: INotify, path: bytes, uri: URIRef, - patch: PatchCb, getSubgraph: GetSubgraph, - globalPrefixes: Dict[str, URIRef], + def __init__(self, notifier: INotify, path: bytes, uri: URIRef, patch: PatchCb, getSubgraph: GetSubgraph, globalPrefixes: Dict[str, URIRef], ctxPrefixes: Dict[str, URIRef]): """ uri is the context for the triples in this file. We assume @@ -176,9 +179,7 @@ self.fileGone() return else: - log.warn( - "%s delete_self event but file is here. " - "probably a new version moved in", filepath) + log.warn("%s delete_self event but file is here. " "probably a new version moved in", filepath) # we could filter these out in the watch() call, but I want # the debugging @@ -207,8 +208,7 @@ """ our file is gone; remove the statements from that context """ - myQuads = [(s, p, o, self.uri) for s, p, o in self.getSubgraph(self.uri) - ] + myQuads = [(s, p, o, self.uri) for s, p, o in self.getSubgraph(self.uri)] log.debug("dropping all statements from context %s", self.uri) if myQuads: self.patch(Patch(delQuads=myQuads), dueToFileChange=True) @@ -223,8 +223,7 @@ try: contents = open(self.path).read() if contents.startswith("#new"): - log.debug("%s ignoring empty contents of my new file", - self.path) + log.debug("%s ignoring empty contents of my new file", self.path) # this is a new file we're starting, and we should not # patch our graph as if it had just been cleared. We # shouldn't even be here reading this, but @@ -277,8 +276,7 @@ self.writeCall.reset(self.flushDelay) else: # This awkward assignment is just to hide from mypy. - setattr(self, 'writeCall', - reactor.callLater(self.flushDelay, self.flush)) + setattr(self, 'writeCall', reactor.callLater(self.flushDelay, self.flush)) def flush(self) -> None: self.writeCall = None @@ -286,9 +284,7 @@ tmpOut = self.path + b".rdfdb-temp" f = open(tmpOut, 'wb') t1 = time.time() - for p, n in (list(self.globalPrefixes.items()) + - list(self.readPrefixes.items()) + - list(self.ctxPrefixes.items())): + for p, n in (list(self.globalPrefixes.items()) + list(self.readPrefixes.items()) + list(self.ctxPrefixes.items())): self.graphToWrite.bind(p, n) self.graphToWrite.serialize(destination=f, format='n3', encoding='utf8') serializeTime = time.time() - t1 @@ -298,5 +294,4 @@ iolog.info("%s rewrote in %.1f ms", self.path, serializeTime * 1000) def __repr__(self) -> str: - return "%s(path=%r, uri=%r, ...)" % (self.__class__.__name__, self.path, - self.uri) + return "%s(path=%r, uri=%r, ...)" % (self.__class__.__name__, self.path, self.uri) diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/graphfile_test.py --- a/rdfdb/graphfile_test.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/graphfile_test.py Mon Apr 04 11:01:59 2022 -0700 @@ -1,7 +1,9 @@ +import tempfile import unittest + import mock -import tempfile -from rdflib import URIRef, Graph +from rdflib import Graph, URIRef + from rdfdb.graphfile import GraphFile @@ -19,14 +21,11 @@ def getSubgraph(uri): return Graph() - gf = GraphFile(mock.Mock(), tf.name.encode('ascii'), URIRef('uri'), - mock.Mock(), getSubgraph, {}, {}) + gf = GraphFile(mock.Mock(), tf.name.encode('ascii'), URIRef('uri'), mock.Mock(), getSubgraph, {}, {}) gf.reread() newGraph = Graph() - newGraph.add((URIRef('http://example.com/boo'), - URIRef('http://example.com/n/two'), - URIRef('http://example.com/other/ns'))) + newGraph.add((URIRef('http://example.com/boo'), URIRef('http://example.com/n/two'), URIRef('http://example.com/other/ns'))) gf.dirty(newGraph) gf.flush() wroteContent = open(tf.name, 'rb').read() diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/localsyncedgraph.py --- a/rdfdb/localsyncedgraph.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/localsyncedgraph.py Mon Apr 04 11:01:59 2022 -0700 @@ -1,7 +1,7 @@ from rdflib import ConjunctiveGraph +from rdfdb.autodepgraphapi import AutoDepGraphApi from rdfdb.currentstategraphapi import CurrentStateGraphApi -from rdfdb.autodepgraphapi import AutoDepGraphApi from rdfdb.grapheditapi import GraphEditApi from rdfdb.rdflibpatch import patchQuads @@ -15,8 +15,5 @@ self._graph.parse(f, format='n3') def patch(self, p): - patchQuads(self._graph, - deleteQuads=p.delQuads, - addQuads=p.addQuads, - perfect=True) + patchQuads(self._graph, deleteQuads=p.delQuads, addQuads=p.addQuads, perfect=True) # no deps diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/mock_syncedgraph.py --- a/rdfdb/mock_syncedgraph.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/mock_syncedgraph.py Mon Apr 04 11:01:59 2022 -0700 @@ -1,4 +1,4 @@ -from rdflib import Graph, RDF, RDFS +from rdflib import RDF, RDFS, Graph from rdflib.parser import StringInputSource @@ -15,19 +15,10 @@ def addHandler(self, func): func() - def value(self, - subject=None, - predicate=RDF.value, - object=None, - default=None, - any=True): + def value(self, subject=None, predicate=RDF.value, object=None, default=None, any=True): if object is not None: raise NotImplementedError() - return self._graph.value(subject, - predicate, - object=object, - default=default, - any=any) + return self._graph.value(subject, predicate, object=object, default=default, any=any) def objects(self, subject=None, predicate=None): return self._graph.objects(subject, predicate) diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/patch.py --- a/rdfdb/patch.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/patch.py Mon Apr 04 11:01:59 2022 -0700 @@ -1,6 +1,11 @@ -import json, unittest -from rdflib import ConjunctiveGraph, Graph, URIRef, URIRef as U, Literal, Namespace +import json +import unittest from typing import Optional + +from rdflib import ConjunctiveGraph, Graph, Literal, Namespace +from rdflib import URIRef +from rdflib import URIRef as U + XSD = Namespace("http://www.w3.org/2001/XMLSchema#") from rdfdb.rdflibpatch import graphFromNQuad, graphFromQuads, serializeQuad @@ -30,12 +35,7 @@ the json representation includes the {"patch":...} wrapper """ - def __init__(self, - jsonRepr: Optional[str] = None, - addQuads=None, - delQuads=None, - addGraph=None, - delGraph=None): + def __init__(self, jsonRepr: Optional[str] = None, addQuads=None, delQuads=None, addGraph=None, delGraph=None): """ addQuads/delQuads can be lists or sets, but if we make them internally, they'll be lists @@ -87,16 +87,14 @@ """ if self._jsonRepr and self._jsonRepr.strip(): raise NotImplementedError() - return bool(self._addQuads or self._delQuads or self._addGraph or - self._delGraph) + return bool(self._addQuads or self._delQuads or self._addGraph or self._delGraph) @property def addQuads(self): if self._addQuads is None: if self._addGraph is None: return [] - self._addQuads = list( - quadsWithContextUris(self._addGraph.quads(ALLSTMTS))) + self._addQuads = list(quadsWithContextUris(self._addGraph.quads(ALLSTMTS))) return self._addQuads @property @@ -104,8 +102,7 @@ if self._delQuads is None: if self._delGraph is None: return [] - self._delQuads = list( - quadsWithContextUris(self._delGraph.quads(ALLSTMTS))) + self._delQuads = list(quadsWithContextUris(self._delGraph.quads(ALLSTMTS))) return self._delQuads @property @@ -180,9 +177,7 @@ ctx = q[3] if ctx != q[3]: - raise ValueError( - "patch applies to multiple contexts, at least %r and %r" % - (ctx, q[3])) + raise ValueError("patch applies to multiple contexts, at least %r and %r" % (ctx, q[3])) if ctx is None: raise ValueError("patch affects no contexts") assert isinstance(ctx, URIRef), ctx @@ -240,7 +235,5 @@ self.assertEqual(p.getContext(), U('http://ctx1')) def testMultiContextPatchFailsToReturnContext(self): - p = Patch(addQuads=[ - stmt1[:3] + (U('http://ctx1'),), stmt1[:3] + (U('http://ctx2'),) - ]) + p = Patch(addQuads=[stmt1[:3] + (U('http://ctx1'),), stmt1[:3] + (U('http://ctx2'),)]) self.assertRaises(ValueError, p.getContext) diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/rdflibpatch.py --- a/rdfdb/rdflibpatch.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/rdflibpatch.py Mon Apr 04 11:01:59 2022 -0700 @@ -2,12 +2,15 @@ this is a proposal for a ConjunctiveGraph method in rdflib """ import sys + if sys.path[0] == '/usr/lib/python2.7/dist-packages': # nosetests puts this in sys.path = sys.path[1:] import unittest -from rdflib import ConjunctiveGraph, Graph, URIRef as U, Literal + +from rdflib import ConjunctiveGraph, Graph, Literal +from rdflib import URIRef as U def patchQuads(graph, deleteQuads, addQuads, perfect=False): @@ -147,8 +150,7 @@ def testTwoContexts(self): g = graphFromQuads([(A, A, A, A), (A, A, A, B)]) - self.assertEqual(sorted(contextsForStatement(g, (A, A, A))), - sorted([A, B])) + self.assertEqual(sorted(contextsForStatement(g, (A, A, A))), sorted([A, B])) # There's a case where contextsForStatement was returning a Graph # with identifier, which I've fixed without a test diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/rdflibpatch_literal.py --- a/rdfdb/rdflibpatch_literal.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/rdflibpatch_literal.py Mon Apr 04 11:01:59 2022 -0700 @@ -1,11 +1,13 @@ import sys + if sys.path[0] == '/usr/lib/python2.7/dist-packages': # nosetests puts this in sys.path = sys.path[1:] -from rdflib.term import _PLAIN_LITERAL_TYPES, _XSD_DOUBLE, _XSD_DECIMAL, Literal from re import sub +from rdflib.term import (_PLAIN_LITERAL_TYPES, _XSD_DECIMAL, _XSD_DOUBLE, Literal) + def _literal_n3(self, use_plain=False, qname_callback=None): if use_plain and self.datatype in _PLAIN_LITERAL_TYPES: diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/service.py --- a/rdfdb/service.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/service.py Mon Apr 04 11:01:59 2022 -0700 @@ -1,43 +1,53 @@ -import sys, optparse, logging, json, os, time, itertools +import itertools +import json +import logging +import optparse +import os +import sys +import time from typing import Dict, List, Optional +import cyclone.web +import cyclone.websocket +import twisted.internet.error +from greplin import scales from greplin.scales.cyclonehandler import StatsHandler -from greplin import scales +from rdflib import ConjunctiveGraph, Graph, URIRef +from standardservice.scalessetup import gatherProcessStats from twisted.internet import reactor, task from twisted.internet.inotify import IN_CREATE, INotify from twisted.python.failure import Failure from twisted.python.filepath import FilePath -import cyclone.web, cyclone.websocket -from rdflib import ConjunctiveGraph, URIRef, Graph -import twisted.internet.error -from rdfdb.file_vs_uri import correctToTopdirPrefix, fileForUri, uriFromFile, DirUriMap -from rdfdb.graphfile import GraphFile, PatchCb, GetSubgraph -from rdfdb.patch import Patch, ALLSTMTS +from rdfdb.file_vs_uri import (DirUriMap, correctToTopdirPrefix, fileForUri, uriFromFile) +from rdfdb.graphfile import GetSubgraph, GraphFile, PatchCb +from rdfdb.patch import ALLSTMTS, Patch from rdfdb.rdflibpatch import patchQuads -from standardservice.scalessetup import gatherProcessStats gatherProcessStats() -stats = scales.collection('/webServer', - scales.IntStat('clients'), - scales.IntStat('liveClients'), - scales.PmfStat('setAttr'), +stats = scales.collection( + '/webServer', + scales.IntStat('clients'), + scales.IntStat('liveClients'), + scales.PmfStat('setAttr'), ) -graphStats = scales.collection('/graph', - scales.IntStat('statements'), - scales.RecentFpsStat('patchFps'), +graphStats = scales.collection( + '/graph', + scales.IntStat('statements'), + scales.RecentFpsStat('patchFps'), ) -fileStats = scales.collection('/file', - scales.IntStat('mappedGraphFiles'), - ) +fileStats = scales.collection( + '/file', + scales.IntStat('mappedGraphFiles'), +) log = logging.getLogger('rdfdb') + class WebsocketDisconnect(ValueError): pass - class WatchedFiles(object): """ find files, notice new files. @@ -45,8 +55,7 @@ This object watches directories. Each GraphFile watches its own file. """ - def __init__(self, dirUriMap: DirUriMap, patch: PatchCb, - getSubgraph: GetSubgraph, addlPrefixes: Dict[str, URIRef]): + def __init__(self, dirUriMap: DirUriMap, patch: PatchCb, getSubgraph: GetSubgraph, addlPrefixes: Dict[str, URIRef]): self.dirUriMap = dirUriMap # {abspath : uri prefix} self.patch, self.getSubgraph = patch, getSubgraph self.addlPrefixes = addlPrefixes @@ -68,9 +77,7 @@ # why wasn't mypy catching this? assert isinstance(p, bytes) self.watchFile(p) - self.notifier.watch(FilePath(dirpath), - autoAdd=True, - callbacks=[self.dirChange]) + self.notifier.watch(FilePath(dirpath), autoAdd=True, callbacks=[self.dirChange]) finally: self.initialLoad = False @@ -143,13 +150,7 @@ def _addGraphFile(self, ctx, path): self.addlPrefixes.setdefault(ctx, {}) self.addlPrefixes.setdefault(None, {}) - gf = GraphFile(self.notifier, - path, - ctx, - self.patch, - self.getSubgraph, - globalPrefixes=self.addlPrefixes[None], - ctxPrefixes=self.addlPrefixes[ctx]) + gf = GraphFile(self.notifier, path, ctx, self.patch, self.getSubgraph, globalPrefixes=self.addlPrefixes[None], ctxPrefixes=self.addlPrefixes[ctx]) self.graphFiles[ctx] = gf fileStats.mappedGraphFiles = len(self.graphFiles) return gf @@ -197,8 +198,7 @@ def connectionLost(self, reason): log.info("bye ws client %r: %s", self, reason) - self.settings.db.clientErrored(Failure(WebsocketDisconnect(reason)), - self) + self.settings.db.clientErrored(Failure(WebsocketDisconnect(reason)), self) def messageReceived(self, message: bytes): if message == b'PING': @@ -214,6 +214,7 @@ def __repr__(self): return f"" + class Db(object): """ the master graph, all the connected clients, all the files we're watching @@ -225,13 +226,12 @@ stats.graphLen = len(self.graph) stats.clients = len(self.clients) - self.watchedFiles = WatchedFiles(dirUriMap, self.patch, - self.getSubgraph, addlPrefixes) + self.watchedFiles = WatchedFiles(dirUriMap, self.patch, self.getSubgraph, addlPrefixes) self.summarizeToLog() @graphStats.patchFps.rate() - def patch(self, patch: Patch, sender: Optional[str]=None, dueToFileChange: bool = False) -> None: + def patch(self, patch: Patch, sender: Optional[str] = None, dueToFileChange: bool = False) -> None: """ apply this patch to the master graph then notify everyone about it @@ -239,23 +239,22 @@ *from* the file (such that we shouldn't write it back to the file) """ ctx = patch.getContext() - log.info("patching graph %s -%d +%d" % - (ctx, len(patch.delQuads), len(patch.addQuads))) + log.info("patching graph %s -%d +%d" % (ctx, len(patch.delQuads), len(patch.addQuads))) - if hasattr(self, 'watchedFiles'): # todo: eliminate this + if hasattr(self, 'watchedFiles'): # todo: eliminate this self.watchedFiles.aboutToPatch(ctx) # an error here needs to drop the sender, and reset everyone # else if we can't rollback the failing patch. patchQuads(self.graph, patch.delQuads, patch.addQuads, perfect=True) stats.graphLen = len(self.graph) - + self._syncPatchToOtherClients(patch, sender) if not dueToFileChange: self.watchedFiles.dirtyFiles([ctx]) graphStats.statements = len(self.graph) - def _syncPatchToOtherClients(self, p: Patch, sender: Optional[str]=None): + def _syncPatchToOtherClients(self, p: Patch, sender: Optional[str] = None): for c in self.clients: if sender is not None and c.connectionId == sender: # this client has self-applied the patch already @@ -263,7 +262,7 @@ continue log.debug('_syncPatchToOtherClients: send to %r', c) c.sendPatch(p) - + def clientErrored(self, err, c) -> None: err.trap(twisted.internet.error.ConnectError, WebsocketDisconnect) log.info("%r %r - dropping client", c, err.getErrorMessage()) @@ -274,8 +273,7 @@ def summarizeToLog(self): log.info("contexts in graph (%s total stmts):" % len(self.graph)) for c in self.graph.contexts(): - log.info(" %s: %s statements" % - (c.identifier, len(self.getSubgraph(c.identifier)))) + log.info(" %s: %s statements" % (c.identifier, len(self.getSubgraph(c.identifier)))) def getSubgraph(self, uri: URIRef) -> Graph: """ @@ -325,8 +323,7 @@ def post(self): suggestion = json.loads(self.request.body) addlPrefixes = self.settings.db.watchedFiles.addlPrefixes - addlPrefixes.setdefault(URIRef(suggestion['ctx']), - {}).update(suggestion['prefixes']) + addlPrefixes.setdefault(URIRef(suggestion['ctx']), {}).update(suggestion['prefixes']) class NoExts(cyclone.web.StaticFileHandler): @@ -337,9 +334,7 @@ cyclone.web.StaticFileHandler.get(self, path, *args, **kw) -def main(dirUriMap: Optional[DirUriMap] = None, - prefixes: Optional[Dict[str, URIRef]] = None, - port=9999): +def main(dirUriMap: Optional[DirUriMap] = None, prefixes: Optional[Dict[str, URIRef]] = None, port=9999): if dirUriMap is None: dirUriMap = {b'data/': URIRef('http://example.com/data/')} @@ -354,10 +349,7 @@ log = logging.getLogger() parser = optparse.OptionParser() - parser.add_option("-v", - "--verbose", - action="store_true", - help="logging.DEBUG") + parser.add_option("-v", "--verbose", action="store_true", help="logging.DEBUG") (options, args) = parser.parse_args() log.setLevel(logging.DEBUG if options.verbose else logging.INFO) @@ -373,7 +365,9 @@ (r'/graph', GraphResource), (r'/syncedGraph', WebsocketClient), (r'/prefixes', Prefixes), - (r'/stats/(.*)', StatsHandler, {'serverName': 'rdfdb'}), + (r'/stats/(.*)', StatsHandler, { + 'serverName': 'rdfdb' + }), (r'/(.*)', NoExts, { "path": FilePath(__file__).sibling("web").path, "default_filename": "index.html" diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/syncedgraph.py --- a/rdfdb/syncedgraph.py Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/syncedgraph.py Mon Apr 04 11:01:59 2022 -0700 @@ -14,31 +14,33 @@ WsClientProtocol one connection with the rdfdb server. """ -import json, logging, traceback +import json +import logging +import traceback +import urllib.parse from typing import Optional -import urllib.parse + +import autobahn.twisted.websocket +import treq +from rdflib import ConjunctiveGraph, URIRef +from twisted.internet import defer, reactor from rdfdb.autodepgraphapi import AutoDepGraphApi from rdfdb.currentstategraphapi import CurrentStateGraphApi from rdfdb.grapheditapi import GraphEditApi from rdfdb.patch import Patch from rdfdb.rdflibpatch import patchQuads -from rdflib import ConjunctiveGraph, URIRef -from twisted.internet import defer -from twisted.internet import reactor -import autobahn.twisted.websocket -import treq - # everybody who writes literals needs to get this from rdfdb.rdflibpatch_literal import patch + patch() log = logging.getLogger('syncedgraph') - class WsClientProtocol(autobahn.twisted.websocket.WebSocketClientProtocol): """The server for this is service.WebsocketClient""" + def __init__(self, sg): super().__init__() self.sg = sg @@ -51,7 +53,7 @@ def onOpen(self): log.info('ws open') self.sg.isConnected = True - + def onMessage(self, payload, isBinary): msg = json.loads(payload) if 'connectedAs' in msg: @@ -81,6 +83,7 @@ log.info("WebSocket connection closed: {0}".format(reason)) self.sg.lostRdfdbConnection() + class SyncedGraph(CurrentStateGraphApi, AutoDepGraphApi, GraphEditApi): """ graph for clients to use. Changes are synced with the master graph @@ -105,10 +108,7 @@ pending local changes) and get the data again from the server. """ - def __init__(self, - rdfdbRoot: URIRef, - label: str, - receiverHost: Optional[str] = None): + def __init__(self, rdfdbRoot: URIRef, label: str, receiverHost: Optional[str] = None): """ label is a string that the server will display in association with your connection @@ -131,8 +131,8 @@ self.patch(Patch(delQuads=self._graph.quads())) log.info(f'cleared graph to {len(self._graph)}') log.error('graph is not updating- you need to restart') - self.connectSocket() - + self.connectSocket() + def connectSocket(self) -> None: factory = autobahn.twisted.websocket.WebSocketClientFactory( self.rdfdbRoot.replace('http://', 'ws://') + 'syncedGraph', @@ -204,11 +204,7 @@ prefixes. async, not guaranteed to finish before any particular file flush """ - treq.post(self.rdfdbRoot + 'prefixes', - json.dumps({ - 'ctx': ctx, - 'prefixes': prefixes - }).encode('utf8')) + treq.post(self.rdfdbRoot + 'prefixes', json.dumps({'ctx': ctx, 'prefixes': prefixes}).encode('utf8')) def _applyPatchLocally(self, p: Patch): # .. and disconnect on failure @@ -224,7 +220,7 @@ log.debug('server has sent us %s', p.shortSummary()) else: log.debug('server has sent us %s', p) - + self._applyPatchLocally(p) try: self.runDepsOnNewPatch(p) diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/web/graphView.html --- a/rdfdb/web/graphView.html Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/web/graphView.html Mon Apr 04 11:01:59 2022 -0700 @@ -2,8 +2,7 @@ graphview - - +
starting...
- +
Messages
-

URI substring:

+

URI substring:

- + + + + - - +
subject predicate object contextsubjectpredicateobjectcontext
diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/web/style.css --- a/rdfdb/web/style.css Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/web/style.css Mon Apr 04 11:01:59 2022 -0700 @@ -1,54 +1,54 @@ body { - display: flex; - flex-direction: column; - position: absolute; - width: 100%; - height: 100%; - padding: 0px; - margin: 0; + display: flex; + flex-direction: column; + position: absolute; + width: 100%; + height: 100%; + padding: 0px; + margin: 0; } #edits { - flex-grow: 3; - display: flex; - flex-direction: column; + flex-grow: 3; + display: flex; + flex-direction: column; } #messages { - flex-grow: 1; - display: flex; - flex-direction: column; - margin-bottom: 3px; + flex-grow: 1; + display: flex; + flex-direction: column; + margin-bottom: 3px; } /* rdfdb */ #patches { - overflow-y: scroll; - flex: 1 1 0; + overflow-y: scroll; + flex: 1 1 0; } .patch { - border: 1px solid gray; - padding: 2px; - margin: 4px; + border: 1px solid gray; + padding: 2px; + margin: 4px; } .patch > div { - font-family: monospace; - font-size: 90%; - white-space: pre-wrap; -} + font-family: monospace; + font-size: 90%; + white-space: pre-wrap; +} .patch .adds { - color: #3AEA38; + color: #3aea38; } .patch .deletes { - color: #FF2828; + color: #ff2828; } #out { - white-space: pre-wrap; - overflow-y: auto; - /* height: 50px; */ - flex: 1 0 0; + white-space: pre-wrap; + overflow-y: auto; + /* height: 50px; */ + flex: 1 0 0; } .patch fieldset { - color: gray; - font-family: arial; - font-size: 75%; + color: gray; + font-family: arial; + font-size: 75%; } diff -r e47dd82a7ddd -r 22c9679dbf67 rdfdb/web/syncedgraph.js --- a/rdfdb/web/syncedgraph.js Mon Apr 04 10:57:33 2022 -0700 +++ b/rdfdb/web/syncedgraph.js Mon Apr 04 11:01:59 2022 -0700 @@ -8,7 +8,7 @@ */ var self = this; - + self.patch = function (p) { throw; @@ -24,6 +24,6 @@ function onMessage(d) { $('#out').append($('
').text(JSON.stringify(d))); } - + reconnectingWebSocket("liveSyncedGraph", onMessage); } diff -r e47dd82a7ddd -r 22c9679dbf67 tasks.py --- a/tasks.py Mon Apr 04 10:57:33 2022 -0700 +++ b/tasks.py Mon Apr 04 11:01:59 2022 -0700 @@ -4,10 +4,12 @@ sys.path.append('/my/proj/release') from release import local_release + @task def release(ctx): local_release(ctx) + @task def test(ctx): ctx.run('pdm run nose2 -v rdfdb.currentstategraphapi_test rdfdb.graphfile_test', pty=True)