Mercurial > code > home > repos > patchablegraph
view patchablegraph.py @ 20:8ec07d997cd5
declare labelnames on metrics
author | drewp@bigasterisk.com |
---|---|
date | Wed, 24 Nov 2021 20:29:32 -0800 |
parents | 388a5e15d249 |
children | 8d6ba6d372c8 |
line wrap: on
line source
""" Design: 1. Services each have (named) graphs, which they patch as things change. PatchableGraph is an object for holding this graph. 2. You can http GET that graph, or ... 3. You can http GET/SSE that graph and hear about modifications to it 4. The client that got the graph holds and maintains a copy. The client may merge together multiple graphs. 5. Client queries its graph with low-level APIs or client-side sparql. 6. When the graph changes, the client knows and can update itself at low or high granularity. See also: * http://iswc2007.semanticweb.org/papers/533.pdf RDFSync: efficient remote synchronization of RDF models * https://www.w3.org/2009/12/rdf-ws/papers/ws07 Supporting Change Propagation in RDF * https://www.w3.org/DesignIssues/lncs04/Diff.pdf Delta: an ontology for the distribution of differences between RDF graphs """ import html import itertools import json import logging from typing import Callable, List, Optional, cast import cyclone.sse import cyclone.web from cycloneerr import PrettyErrorHandler from prometheus_client import Counter, Gauge, Summary from rdfdb.grapheditapi import GraphEditApi from rdfdb.patch import Patch from rdfdb.rdflibpatch import inGraph, patchQuads from rdflib import ConjunctiveGraph from rdflib.namespace import NamespaceManager from rdflib.parser import StringInputSource from rdflib.plugins.serializers.jsonld import from_rdf from . import patch_cyclone patch_cyclone.patch_sse() log = logging.getLogger('patchablegraph') SERIALIZE_CALLS = Summary('serialize_calls', 'PatchableGraph.serialize calls', labelnames=['graph']) PATCH_CALLS = Summary('patch_calls', 'PatchableGraph.patch calls', labelnames=['graph']) STATEMENT_COUNT = Gauge('statement_count', 'current PatchableGraph graph size', labelnames=['graph']) OBSERVERS_CURRENT = Gauge('observers_current', 'current observer count', labelnames=['graph']) OBSERVERS_ADDED = Counter('observers_added', 'observers added', labelnames=['graph']) # forked from /my/proj/light9/light9/rdfdb/rdflibpatch.py def _graphFromQuads2(q): g = ConjunctiveGraph() #g.addN(q) # no effect on nquad output for s, p, o, c in q: g.get_context(c).add((s, p, o)) # kind of works with broken rdflib nquad serializer code #g.store.add((s,p,o), c) # no effect on nquad output return g def jsonFromPatch(p: Patch) -> str: return json.dumps( {'patch': { 'adds': from_rdf(_graphFromQuads2(p.addQuads)), 'deletes': from_rdf(_graphFromQuads2(p.delQuads)), }}) patchAsJson = jsonFromPatch # deprecated name def patchFromJson(j: str) -> Patch: body = json.loads(j)['patch'] a = ConjunctiveGraph() a.parse(StringInputSource(json.dumps(body['adds']).encode('utf8')), format='json-ld') d = ConjunctiveGraph() d.parse(StringInputSource(json.dumps(body['deletes']).encode('utf8')), format='json-ld') return Patch(addGraph=a, delGraph=d) def graphAsJson(g: ConjunctiveGraph) -> str: # This is not the same as g.serialize(format='json-ld')! That # version omits literal datatypes. return json.dumps(from_rdf(g)) _graphsInProcess = itertools.count() class PatchableGraph(GraphEditApi): """ Master graph that you modify with self.patch, and we get the updates to all current listeners. """ def __init__(self, label: Optional[str] = None): self._graph = ConjunctiveGraph() self._observers: List[Callable[[str], None]] = [] if label is None: label = f'patchableGraph{next(_graphsInProcess)}' self.label = label def serialize(self, *arg, **kw) -> bytes: with SERIALIZE_CALLS.labels(graph=self.label).time(): return cast(bytes, self._graph.serialize(*arg, **kw)) def patch(self, p: Patch): with PATCH_CALLS.labels(graph=self.label).time(): # assuming no stmt is both in p.addQuads and p.delQuads. dels = set([q for q in p.delQuads if inGraph(q, self._graph)]) adds = set([q for q in p.addQuads if not inGraph(q, self._graph)]) minimizedP = Patch(addQuads=adds, delQuads=dels) if minimizedP.isNoop(): return patchQuads(self._graph, deleteQuads=dels, addQuads=adds, perfect=False) # true? for ob in self._observers: ob(patchAsJson(p)) STATEMENT_COUNT.labels(graph=self.label).set(len(self._graph)) def asJsonLd(self) -> str: return graphAsJson(self._graph) def addObserver(self, onPatch: Callable[[str], None]): self._observers.append(onPatch) OBSERVERS_CURRENT.labels(graph=self.label).set(len(self._observers)) OBSERVERS_ADDED.labels(graph=self.label).inc() def removeObserver(self, onPatch: Callable[[str], None]): try: self._observers.remove(onPatch) except ValueError: pass self._currentObservers = len(self._observers) def setToGraph(self, newGraph: ConjunctiveGraph): self.patch(Patch.fromDiff(self._graph, newGraph)) SEND_SIMPLE_GRAPH = Summary('send_simple_graph', 'calls to _writeGraphResponse') class CycloneGraphHandler(PrettyErrorHandler, cyclone.web.RequestHandler): def initialize(self, masterGraph: PatchableGraph): self.masterGraph = masterGraph def get(self): with SEND_SIMPLE_GRAPH.time(): self._writeGraphResponse() def _writeGraphResponse(self): acceptHeader = self.request.headers.get( 'Accept', # see https://github.com/fiorix/cyclone/issues/20 self.request.headers.get('accept', '')) if acceptHeader == 'application/nquads': self.set_header('Content-type', 'application/nquads') self.masterGraph.serialize(self, format='nquads') elif acceptHeader == 'application/ld+json': self.set_header('Content-type', 'application/ld+json') self.masterGraph.serialize(self, format='json-ld', indent=2) else: if acceptHeader.startswith('text/html'): self._writeGraphForBrowser() return self.set_header('Content-type', 'application/x-trig') self.masterGraph.serialize(self, format='trig') def _writeGraphForBrowser(self): # We think this is a browser, so respond with a live graph view # (todo) self.set_header('Content-type', 'text/html') self.write(b''' <html><body><pre>''') ns = NamespaceManager(self.masterGraph._graph) # maybe these could be on the PatchableGraph instance ns.bind('ex', 'http://example.com/') ns.bind('', 'http://projects.bigasterisk.com/room/') ns.bind("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#") ns.bind("xsd", "http://www.w3.org/2001/XMLSchema#") for s, p, o, g in sorted(self.masterGraph._graph.quads()): g = g.identifier nquadLine = f'{s.n3(ns)} {p.n3(ns)} {o.n3(ns)} {g.n3(ns)} .\n' self.write(html.escape(nquadLine).encode('utf8')) self.write(b''' </pre> <p> <a href="#">[refresh]</a> <label><input type="checkbox"> Auto-refresh</label> </p> <script> if (new URL(window.location).searchParams.get('autorefresh') == 'on') { document.querySelector("input").checked = true; setTimeout(() => { requestAnimationFrame(() => { window.location.replace(window.location.href); }); }, 2000); } document.querySelector("a").addEventListener("click", (ev) => { ev.preventDefault(); window.location.replace(window.location.href); }); document.querySelector("input").addEventListener("change", (ev) => { if (document.querySelector("input").checked) { const u = new URL(window.location); u.searchParams.set('autorefresh', 'on'); window.location.replace(u.href); } else { const u = new URL(window.location); u.searchParams.delete('autorefresh'); window.location.replace(u.href); } }); </script> </body></html> ''') SEND_FULL_GRAPH = Summary('send_full_graph', 'fullGraph SSE events') SEND_PATCH = Summary('send_patch', 'patch SSE events') class CycloneGraphEventsHandler(cyclone.sse.SSEHandler): """ One session with one client. returns current graph plus future patches to keep remote version in sync with ours. intsead of turning off buffering all over, it may work for this response to send 'x-accel-buffering: no', per http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering """ def __init__(self, application, request, masterGraph): cyclone.sse.SSEHandler.__init__(self, application, request) self.masterGraph = masterGraph def bind(self): with SEND_FULL_GRAPH.time(): graphJson = self.masterGraph.asJsonLd() log.debug("send fullGraph event: %s", graphJson) self.sendEvent(message=graphJson, event=b'fullGraph') self.masterGraph.addObserver(self.onPatch) def onPatch(self, patchJson): with SEND_PATCH.time(): # throttle and combine patches here- ideally we could see how # long the latency to the client is to make a better rate choice self.sendEvent(message=patchJson, event=b'patch') def unbind(self): self.masterGraph.removeObserver(self.onPatch)