Mercurial > code > home > repos > homeauto
annotate lib/patchablegraph/patchablegraph.py @ 1505:3cdf76755812
reformat
Ignore-this: e1fe14c4b973c3c1fc2714b988ecc7c8
darcs-hash:055eedef2e8db685f2c524ce4d2b608040d60b33
| author | drewp <drewp@bigasterisk.com> |
|---|---|
| date | Mon, 03 Feb 2020 00:51:18 -0800 |
| parents | d8eba5a51c1f |
| children | e5c0d387c248 |
| rev | line source |
|---|---|
| 1317 | 1 """ |
| 2 Design: | |
| 3 | |
| 4 1. Services each have (named) graphs, which they patch as things | |
| 5 change. PatchableGraph is an object for holding this graph. | |
| 6 2. You can http GET that graph, or ... | |
| 7 3. You can http GET/SSE that graph and hear about modifications to it | |
| 8 4. The client that got the graph holds and maintains a copy. The | |
| 9 client may merge together multiple graphs. | |
| 10 5. Client queries its graph with low-level APIs or client-side sparql. | |
| 11 6. When the graph changes, the client knows and can update itself at | |
| 12 low or high granularity. | |
| 13 | |
| 14 | |
| 15 See also: | |
| 16 * http://iswc2007.semanticweb.org/papers/533.pdf RDFSync: efficient remote synchronization of RDF | |
| 17 models | |
| 18 * https://www.w3.org/2009/12/rdf-ws/papers/ws07 Supporting Change Propagation in RDF | |
| 19 * https://www.w3.org/DesignIssues/lncs04/Diff.pdf Delta: an ontology for the distribution of | |
| 20 differences between RDF graphs | |
| 21 | |
| 22 """ | |
| 23 import json, logging, itertools | |
| 24 | |
| 25 from greplin import scales | |
| 26 from rdfdb.grapheditapi import GraphEditApi | |
| 27 from rdflib import ConjunctiveGraph | |
| 28 from rdflib.parser import StringInputSource | |
| 29 from rdflib_jsonld.serializer import from_rdf | |
| 30 import cyclone.sse | |
| 31 | |
| 32 from cycloneerr import PrettyErrorHandler | |
| 33 from rdfdb.patch import Patch | |
| 34 from rdfdb.rdflibpatch import patchQuads, inGraph | |
| 35 | |
| 36 log = logging.getLogger('patchablegraph') | |
| 37 | |
| 38 def writeGraphResponse(req, graph, acceptHeader): | |
| 39 if acceptHeader == 'application/nquads': | |
| 40 req.set_header('Content-type', 'application/nquads') | |
| 41 graph.serialize(req, format='nquads') | |
| 42 elif acceptHeader == 'application/ld+json': | |
| 43 req.set_header('Content-type', 'application/ld+json') | |
| 44 graph.serialize(req, format='json-ld', indent=2) | |
| 45 else: | |
|
1504
d8eba5a51c1f
try a text/plain response if we think it's a browser asking for the graph
drewp <drewp@bigasterisk.com>
parents:
1317
diff
changeset
|
46 print(f'acceptHeader {acceptHeader}') |
|
d8eba5a51c1f
try a text/plain response if we think it's a browser asking for the graph
drewp <drewp@bigasterisk.com>
parents:
1317
diff
changeset
|
47 if acceptHeader.startswith('text/html'): |
|
d8eba5a51c1f
try a text/plain response if we think it's a browser asking for the graph
drewp <drewp@bigasterisk.com>
parents:
1317
diff
changeset
|
48 # browser; should arrange to pick live view |
|
d8eba5a51c1f
try a text/plain response if we think it's a browser asking for the graph
drewp <drewp@bigasterisk.com>
parents:
1317
diff
changeset
|
49 req.set_header('Content-type', 'text/plain') |
|
d8eba5a51c1f
try a text/plain response if we think it's a browser asking for the graph
drewp <drewp@bigasterisk.com>
parents:
1317
diff
changeset
|
50 lines = graph.serialize(format='nquads') |
|
d8eba5a51c1f
try a text/plain response if we think it's a browser asking for the graph
drewp <drewp@bigasterisk.com>
parents:
1317
diff
changeset
|
51 lines.sort() |
|
d8eba5a51c1f
try a text/plain response if we think it's a browser asking for the graph
drewp <drewp@bigasterisk.com>
parents:
1317
diff
changeset
|
52 req.write(''.join(lines)) |
|
d8eba5a51c1f
try a text/plain response if we think it's a browser asking for the graph
drewp <drewp@bigasterisk.com>
parents:
1317
diff
changeset
|
53 return |
|
d8eba5a51c1f
try a text/plain response if we think it's a browser asking for the graph
drewp <drewp@bigasterisk.com>
parents:
1317
diff
changeset
|
54 |
| 1317 | 55 req.set_header('Content-type', 'application/x-trig') |
| 56 graph.serialize(req, format='trig') | |
| 57 | |
| 58 # forked from /my/proj/light9/light9/rdfdb/rdflibpatch.py | |
| 59 def _graphFromQuads2(q): | |
| 60 g = ConjunctiveGraph() | |
| 61 #g.addN(q) # no effect on nquad output | |
| 62 for s,p,o,c in q: | |
| 63 g.get_context(c).add((s,p,o)) # kind of works with broken rdflib nquad serializer code | |
| 64 #g.store.add((s,p,o), c) # no effect on nquad output | |
| 65 return g | |
| 66 | |
| 67 def jsonFromPatch(p): | |
| 68 return json.dumps({'patch': { | |
| 69 'adds': from_rdf(_graphFromQuads2(p.addQuads)), | |
| 70 'deletes': from_rdf(_graphFromQuads2(p.delQuads)), | |
| 71 }}) | |
| 72 patchAsJson = jsonFromPatch # deprecated name | |
| 73 | |
| 1505 | 74 |
| 1317 | 75 def patchFromJson(j): |
| 76 body = json.loads(j)['patch'] | |
| 77 a = ConjunctiveGraph() | |
| 78 a.parse(StringInputSource(json.dumps(body['adds']).encode('utf8')), format='json-ld') | |
| 79 d = ConjunctiveGraph() | |
| 80 d.parse(StringInputSource(json.dumps(body['deletes']).encode('utf8')), format='json-ld') | |
| 81 return Patch(addGraph=a, delGraph=d) | |
| 82 | |
| 83 def graphAsJson(g): | |
| 84 # This is not the same as g.serialize(format='json-ld')! That | |
| 85 # version omits literal datatypes. | |
| 86 return json.dumps(from_rdf(g)) | |
| 87 | |
| 88 _graphsInProcess = itertools.count() | |
| 89 class PatchableGraph(GraphEditApi): | |
| 90 """ | |
| 91 Master graph that you modify with self.patch, and we get the | |
| 92 updates to all current listeners. | |
| 93 """ | |
| 94 def __init__(self): | |
| 95 self._graph = ConjunctiveGraph() | |
| 96 self._observers = [] | |
| 97 scales.init(self, '/patchableGraph%s' % next(_graphsInProcess)) | |
| 98 | |
| 99 _serialize = scales.PmfStat('serialize') | |
| 100 def serialize(self, to, **kw): | |
| 101 with self._serialize.time(): | |
| 102 return self._graph.serialize(to, **kw) | |
| 103 | |
| 104 _patch = scales.PmfStat('patch') | |
| 105 _len = scales.IntStat('statementCount') | |
| 106 def patch(self, p): | |
| 107 with self._patch.time(): | |
| 108 # assuming no stmt is both in p.addQuads and p.delQuads. | |
| 109 dels = set([q for q in p.delQuads if inGraph(q, self._graph)]) | |
| 110 adds = set([q for q in p.addQuads if not inGraph(q, self._graph)]) | |
| 111 minimizedP = Patch(addQuads=adds, delQuads=dels) | |
| 112 if minimizedP.isNoop(): | |
| 113 return | |
| 114 patchQuads(self._graph, | |
| 115 deleteQuads=dels, | |
| 116 addQuads=adds, | |
| 117 perfect=False) # true? | |
| 118 for ob in self._observers: | |
| 119 ob(patchAsJson(p)) | |
| 120 self._len = len(self._graph) | |
| 121 | |
| 122 def asJsonLd(self): | |
| 123 return graphAsJson(self._graph) | |
| 124 | |
| 125 _currentObservers = scales.IntStat('observers/current') | |
| 126 _observersAdded = scales.IntStat('observers/added') | |
| 127 def addObserver(self, onPatch): | |
| 128 self._observers.append(onPatch) | |
| 129 self._currentObservers = len(self._observers) | |
| 130 self._observersAdded += 1 | |
| 1505 | 131 |
| 1317 | 132 def removeObserver(self, onPatch): |
| 133 try: | |
| 134 self._observers.remove(onPatch) | |
| 135 except ValueError: | |
| 136 pass | |
| 137 self._currentObservers = len(self._observers) | |
| 138 | |
| 139 def setToGraph(self, newGraph): | |
| 140 self.patch(Patch.fromDiff(self._graph, newGraph)) | |
| 1505 | 141 |
| 1317 | 142 _sendSimpleGraph = scales.PmfStat('serve/simpleGraph') |
| 143 _sendFullGraph = scales.PmfStat('serve/events/sendFull') | |
| 144 _sendPatch = scales.PmfStat('serve/events/sendPatch') | |
| 145 | |
| 146 class CycloneGraphHandler(PrettyErrorHandler, cyclone.web.RequestHandler): | |
| 147 def initialize(self, masterGraph): | |
| 148 self.masterGraph = masterGraph | |
| 1505 | 149 |
| 1317 | 150 def get(self): |
| 151 with self.masterGraph._sendSimpleGraph.time(): | |
| 152 writeGraphResponse(self, self.masterGraph, | |
| 153 self.request.headers.get('accept')) | |
| 154 | |
| 155 | |
| 156 class CycloneGraphEventsHandler(cyclone.sse.SSEHandler): | |
| 157 """ | |
| 158 One session with one client. | |
| 1505 | 159 |
| 1317 | 160 returns current graph plus future patches to keep remote version |
| 161 in sync with ours. | |
| 162 | |
| 163 intsead of turning off buffering all over, it may work for this | |
| 164 response to send 'x-accel-buffering: no', per | |
| 165 http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering | |
| 166 """ | |
| 167 def __init__(self, application, request, masterGraph): | |
| 168 cyclone.sse.SSEHandler.__init__(self, application, request) | |
| 169 self.masterGraph = masterGraph | |
| 170 | |
| 171 def bind(self): | |
| 172 with self.masterGraph._sendFullGraph.time(): | |
| 173 graphJson = self.masterGraph.asJsonLd() | |
| 174 log.debug("send fullGraph event: %s", graphJson) | |
| 175 self.sendEvent(message=graphJson, event=b'fullGraph') | |
| 176 self.masterGraph.addObserver(self.onPatch) | |
| 177 | |
| 178 def onPatch(self, patchJson): | |
| 179 with self.masterGraph._sendPatch.time(): | |
| 180 # throttle and combine patches here- ideally we could see how | |
| 181 # long the latency to the client is to make a better rate choice | |
| 182 self.sendEvent(message=patchJson, event=b'patch') | |
| 1505 | 183 |
| 1317 | 184 def unbind(self): |
| 185 self.masterGraph.removeObserver(self.onPatch) |