Mercurial > code > home > repos > rdfdb
annotate rdfdb/service.py @ 87:a96f4eb95ef0
more typing and formatting
| author | drewp@bigasterisk.com |
|---|---|
| date | Mon, 04 Apr 2022 23:01:08 -0700 |
| parents | 5b6e90a708ce |
| children | 1120c6489888 |
| rev | line source |
|---|---|
| 80 | 1 import itertools |
| 2 import json | |
| 3 import logging | |
| 4 import optparse | |
| 5 import os | |
| 6 import sys | |
| 87 | 7 from typing import Dict, List, Optional, cast |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
8 |
| 80 | 9 import cyclone.web |
| 10 import cyclone.websocket | |
| 11 import twisted.internet.error | |
| 12 from rdflib import ConjunctiveGraph, Graph, URIRef | |
| 87 | 13 import twisted.internet.reactor |
|
46
3b36b2c8ae65
fix bug that echoed a patch back to KC. https://bigasterisk.com/light9/work/2019/kc-patch-echo-bug.png
Drew Perttula <drewp@bigasterisk.com>
parents:
45
diff
changeset
|
14 from twisted.internet.inotify import IN_CREATE, INotify |
|
3b36b2c8ae65
fix bug that echoed a patch back to KC. https://bigasterisk.com/light9/work/2019/kc-patch-echo-bug.png
Drew Perttula <drewp@bigasterisk.com>
parents:
45
diff
changeset
|
15 from twisted.python.failure import Failure |
|
3b36b2c8ae65
fix bug that echoed a patch back to KC. https://bigasterisk.com/light9/work/2019/kc-patch-echo-bug.png
Drew Perttula <drewp@bigasterisk.com>
parents:
45
diff
changeset
|
16 from twisted.python.filepath import FilePath |
| 87 | 17 from twisted.internet.interfaces import IReactorCore |
|
46
3b36b2c8ae65
fix bug that echoed a patch back to KC. https://bigasterisk.com/light9/work/2019/kc-patch-echo-bug.png
Drew Perttula <drewp@bigasterisk.com>
parents:
45
diff
changeset
|
18 |
| 80 | 19 from rdfdb.file_vs_uri import (DirUriMap, correctToTopdirPrefix, fileForUri, uriFromFile) |
| 20 from rdfdb.graphfile import GetSubgraph, GraphFile, PatchCb | |
| 21 from rdfdb.patch import ALLSTMTS, Patch | |
|
46
3b36b2c8ae65
fix bug that echoed a patch back to KC. https://bigasterisk.com/light9/work/2019/kc-patch-echo-bug.png
Drew Perttula <drewp@bigasterisk.com>
parents:
45
diff
changeset
|
22 from rdfdb.rdflibpatch import patchQuads |
| 87 | 23 reactor = cast(IReactorCore, twisted.internet.reactor) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
24 |
|
81
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
25 # gatherProcessStats() |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
26 # stats = scales.collection( |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
27 # '/webServer', |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
28 # scales.IntStat('clients'), |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
29 # scales.IntStat('liveClients'), |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
30 # scales.PmfStat('setAttr'), |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
31 # ) |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
32 # graphStats = scales.collection( |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
33 # '/graph', |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
34 # scales.IntStat('statements'), |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
35 # scales.RecentFpsStat('patchFps'), |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
36 # ) |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
37 # fileStats = scales.collection( |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
38 # '/file', |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
39 # scales.IntStat('mappedGraphFiles'), |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
40 # ) |
| 52 | 41 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
42 log = logging.getLogger('rdfdb') |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
43 |
| 80 | 44 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
45 class WebsocketDisconnect(ValueError): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
46 pass |
| 45 | 47 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
48 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
49 class WatchedFiles(object): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
50 """ |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
51 find files, notice new files. |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
52 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
53 This object watches directories. Each GraphFile watches its own file. |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
54 """ |
| 45 | 55 |
| 80 | 56 def __init__(self, dirUriMap: DirUriMap, patch: PatchCb, getSubgraph: GetSubgraph, addlPrefixes: Dict[str, URIRef]): |
| 45 | 57 self.dirUriMap = dirUriMap # {abspath : uri prefix} |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
58 self.patch, self.getSubgraph = patch, getSubgraph |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
59 self.addlPrefixes = addlPrefixes |
| 45 | 60 |
| 61 self.graphFiles: Dict[URIRef, GraphFile] = {} # context uri : GraphFile | |
| 62 | |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
63 self.notifier = INotify() |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
64 self.notifier.startReading() |
| 45 | 65 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
66 self.findAndLoadFiles() |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
67 |
| 42 | 68 def findAndLoadFiles(self) -> None: |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
69 self.initialLoad = True |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
70 try: |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
71 for topdir in self.dirUriMap: |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
72 for dirpath, dirnames, filenames in os.walk(topdir): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
73 for base in filenames: |
| 42 | 74 p = os.path.join(dirpath, base) |
| 75 # why wasn't mypy catching this? | |
| 76 assert isinstance(p, bytes) | |
| 77 self.watchFile(p) | |
| 80 | 78 self.notifier.watch(FilePath(dirpath), autoAdd=True, callbacks=[self.dirChange]) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
79 finally: |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
80 self.initialLoad = False |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
81 |
| 42 | 82 def dirChange(self, watch, path: FilePath, mask): |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
83 if mask & IN_CREATE: |
| 86 | 84 if cast(str, path.path).endswith((b'~', b'.swp', b'swx', b'.rdfdb-temp')): |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
85 return |
| 45 | 86 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
87 log.debug("%s created; consider adding a watch", path) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
88 self.watchFile(path.path) |
| 45 | 89 |
| 39 | 90 def watchFile(self, inFile: bytes): |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
91 """ |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
92 consider adding a GraphFile to self.graphFiles |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
93 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
94 inFile needs to be a relative path, not an absolute (e.g. in a |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
95 FilePath) because we use its exact relative form in the |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
96 context URI |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
97 """ |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
98 if not os.path.isfile(inFile): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
99 return |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
100 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
101 inFile = correctToTopdirPrefix(self.dirUriMap, inFile) |
| 42 | 102 if os.path.splitext(inFile)[1] not in [b'.n3']: |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
103 return |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
104 |
| 42 | 105 if b'/capture/' in inFile: |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
106 # smaller graph for now |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
107 return |
| 45 | 108 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
109 # an n3 file with rules makes it all the way past this reading |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
110 # and the serialization. Then, on the receiving side, a |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
111 # SyncedGraph calls graphFromNQuad on the incoming data and |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
112 # has a parse error. I'm not sure where this should be fixed |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
113 # yet. |
| 42 | 114 if b'-rules' in inFile: |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
115 return |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
116 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
117 # for legacy versions, compile all the config stuff you want |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
118 # read into one file called config.n3. New versions won't read |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
119 # it. |
| 42 | 120 if inFile.endswith(b"config.n3"): |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
121 return |
| 45 | 122 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
123 ctx = uriFromFile(self.dirUriMap, inFile) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
124 gf = self._addGraphFile(ctx, inFile) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
125 log.info("%s do initial read", inFile) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
126 gf.reread() |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
127 |
|
43
8d4822ae58bc
more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents:
42
diff
changeset
|
128 def aboutToPatch(self, ctx: URIRef): |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
129 """ |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
130 warn us that a patch is about to come to this context. it's more |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
131 straightforward to create the new file now |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
132 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
133 this is meant to make the file before we add triples, so we |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
134 wouldn't see the blank file and lose those triples. But it |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
135 didn't work, so there are other measures that make us not lose |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
136 the triples from a new file. Calling this before patching the |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
137 graph is still a reasonable thing to do, though. |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
138 """ |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
139 if ctx not in self.graphFiles: |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
140 outFile = fileForUri(self.dirUriMap, ctx) |
|
43
8d4822ae58bc
more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents:
42
diff
changeset
|
141 # mypy missed the next line because of |
|
8d4822ae58bc
more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents:
42
diff
changeset
|
142 # https://github.com/python/typeshed/issues/2937 ('str in |
|
8d4822ae58bc
more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents:
42
diff
changeset
|
143 # bytes' isn't an error) |
|
8d4822ae58bc
more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents:
42
diff
changeset
|
144 assert b'//' not in outFile, (outFile, self.dirUriMap, ctx) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
145 log.info("starting new file %r", outFile) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
146 self._addGraphFile(ctx, outFile) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
147 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
148 def _addGraphFile(self, ctx, path): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
149 self.addlPrefixes.setdefault(ctx, {}) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
150 self.addlPrefixes.setdefault(None, {}) |
| 80 | 151 gf = GraphFile(self.notifier, path, ctx, self.patch, self.getSubgraph, globalPrefixes=self.addlPrefixes[None], ctxPrefixes=self.addlPrefixes[ctx]) |
| 45 | 152 self.graphFiles[ctx] = gf |
| 54 | 153 fileStats.mappedGraphFiles = len(self.graphFiles) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
154 return gf |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
155 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
156 def dirtyFiles(self, ctxs): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
157 """mark dirty the files that we watch in these contexts. |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
158 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
159 the ctx might not be a file that we already read; it might be |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
160 for a new file we have to create, or it might be for a |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
161 transient context that we're not going to save |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
162 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
163 if it's a ctx with no file, error |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
164 """ |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
165 for ctx in ctxs: |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
166 g = self.getSubgraph(ctx) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
167 self.graphFiles[ctx].dirty(g) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
168 |
| 45 | 169 |
|
65
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
170 _wsClientSerial = itertools.count(0) |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
171 |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
172 |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
173 class WebsocketClient(cyclone.websocket.WebSocketHandler): |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
174 """ |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
175 Send patches to the client (starting with a client who has 0 |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
176 statements) to keep it in sync with the graph. |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
177 |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
178 Accept patches from the client, and assume that the client has |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
179 already applied them to its local graph. |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
180 |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
181 Treat a disconnect as 'out of sync'. Either the client thinks it |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
182 is out of sync and wants to start over, or we can't apply a patch |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
183 correctly therefore we disconnect to make the client start over. |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
184 |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
185 This socket may also carry some special messages meant for the |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
186 rdfdb web UI, e.g. about who is connected, etc. |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
187 """ |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
188 connectionId: str |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
189 |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
190 def connectionMade(self, *args, **kwargs) -> None: |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
191 self.connectionId = f'WS{next(_wsClientSerial)}' |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
192 |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
193 self.sendMessage(json.dumps({'connectedAs': self.connectionId})) |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
194 log.info("new ws client %r", self) |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
195 self.settings.db.addClient(self) |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
196 |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
197 def connectionLost(self, reason): |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
198 log.info("bye ws client %r: %s", self, reason) |
| 80 | 199 self.settings.db.clientErrored(Failure(WebsocketDisconnect(reason)), self) |
|
65
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
200 |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
201 def messageReceived(self, message: bytes): |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
202 if message == b'PING': |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
203 self.sendMessage('PONG') |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
204 return |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
205 log.debug("got message from %r: %s", self, message[:32]) |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
206 p = Patch(jsonRepr=message.decode('utf8')) |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
207 self.settings.db.patch(p, sender=self.connectionId) |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
208 |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
209 def sendPatch(self, p: Patch): |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
210 self.sendMessage(p.makeJsonRepr()) |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
211 |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
212 def __repr__(self): |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
213 return f"<SyncedGraph client {self.connectionId}>" |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
214 |
| 80 | 215 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
216 class Db(object): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
217 """ |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
218 the master graph, all the connected clients, all the files we're watching |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
219 """ |
| 45 | 220 |
| 42 | 221 def __init__(self, dirUriMap: DirUriMap, addlPrefixes): |
|
65
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
222 self.clients: List[WebsocketClient] = [] |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
223 self.graph = ConjunctiveGraph() |
|
81
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
224 # stats.graphLen = len(self.graph) |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
225 # stats.clients = len(self.clients) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
226 |
| 80 | 227 self.watchedFiles = WatchedFiles(dirUriMap, self.patch, self.getSubgraph, addlPrefixes) |
| 45 | 228 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
229 self.summarizeToLog() |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
230 |
|
81
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
231 # @graphStats.patchFps.rate() |
| 80 | 232 def patch(self, patch: Patch, sender: Optional[str] = None, dueToFileChange: bool = False) -> None: |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
233 """ |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
234 apply this patch to the master graph then notify everyone about it |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
235 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
236 dueToFileChange if this is a patch describing an edit we read |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
237 *from* the file (such that we shouldn't write it back to the file) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
238 """ |
|
43
8d4822ae58bc
more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents:
42
diff
changeset
|
239 ctx = patch.getContext() |
| 80 | 240 log.info("patching graph %s -%d +%d" % (ctx, len(patch.delQuads), len(patch.addQuads))) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
241 |
| 80 | 242 if hasattr(self, 'watchedFiles'): # todo: eliminate this |
|
66
109fefea80a7
fix on watchedFiles startup
Drew Perttula <drewp@bigasterisk.com>
parents:
65
diff
changeset
|
243 self.watchedFiles.aboutToPatch(ctx) |
| 45 | 244 |
|
65
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
245 # an error here needs to drop the sender, and reset everyone |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
246 # else if we can't rollback the failing patch. |
|
43
8d4822ae58bc
more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents:
42
diff
changeset
|
247 patchQuads(self.graph, patch.delQuads, patch.addQuads, perfect=True) |
| 52 | 248 stats.graphLen = len(self.graph) |
| 80 | 249 |
|
65
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
250 self._syncPatchToOtherClients(patch, sender) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
251 if not dueToFileChange: |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
252 self.watchedFiles.dirtyFiles([ctx]) |
| 52 | 253 graphStats.statements = len(self.graph) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
254 |
| 80 | 255 def _syncPatchToOtherClients(self, p: Patch, sender: Optional[str] = None): |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
256 for c in self.clients: |
|
69
6b01e1e5859b
drop patches and log, but don't otherwise error, if we're not connected to rdfdb
Drew Perttula <drewp@bigasterisk.com>
parents:
66
diff
changeset
|
257 if sender is not None and c.connectionId == sender: |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
258 # this client has self-applied the patch already |
|
65
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
259 log.debug("_syncPatchToOtherClients: don't resend to %r", c) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
260 continue |
|
65
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
261 log.debug('_syncPatchToOtherClients: send to %r', c) |
|
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
262 c.sendPatch(p) |
| 80 | 263 |
|
46
3b36b2c8ae65
fix bug that echoed a patch back to KC. https://bigasterisk.com/light9/work/2019/kc-patch-echo-bug.png
Drew Perttula <drewp@bigasterisk.com>
parents:
45
diff
changeset
|
264 def clientErrored(self, err, c) -> None: |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
265 err.trap(twisted.internet.error.ConnectError, WebsocketDisconnect) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
266 log.info("%r %r - dropping client", c, err.getErrorMessage()) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
267 if c in self.clients: |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
268 self.clients.remove(c) |
| 54 | 269 stats.clients = len(self.clients) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
270 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
271 def summarizeToLog(self): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
272 log.info("contexts in graph (%s total stmts):" % len(self.graph)) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
273 for c in self.graph.contexts(): |
| 80 | 274 log.info(" %s: %s statements" % (c.identifier, len(self.getSubgraph(c.identifier)))) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
275 |
|
43
8d4822ae58bc
more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents:
42
diff
changeset
|
276 def getSubgraph(self, uri: URIRef) -> Graph: |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
277 """ |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
278 this is meant to return a live view of the given subgraph, but |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
279 if i'm still working around an rdflib bug, it might return a |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
280 copy |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
281 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
282 and it's returning triples, but I think quads would be better |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
283 """ |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
284 # this is returning an empty Graph :( |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
285 #return self.graph.get_context(uri) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
286 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
287 g = Graph() |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
288 for s in self.graph.triples(ALLSTMTS, uri): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
289 g.add(s) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
290 return g |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
291 |
|
65
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
292 def addClient(self, newClient: WebsocketClient) -> None: |
|
72
8bf24d5a0d21
shorter debug log on initial graph sync
drewp@bigasterisk.com
parents:
71
diff
changeset
|
293 log.info("new connection: sending all graphs to %r" % newClient) |
|
65
9bc9de580033
big rewrite of rdfdb and syncedgraph to use a single websocket connecton
Drew Perttula <drewp@bigasterisk.com>
parents:
59
diff
changeset
|
294 newClient.sendPatch(Patch(addQuads=self.graph.quads(ALLSTMTS), delQuads=[])) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
295 self.clients.append(newClient) |
| 54 | 296 stats.clients = len(self.clients) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
297 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
298 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
299 class GraphResource(cyclone.web.RequestHandler): |
| 45 | 300 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
301 def get(self): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
302 accept = self.request.headers.get('accept', '') |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
303 format = 'n3' |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
304 if accept == 'text/plain': |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
305 format = 'nt' |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
306 elif accept == 'application/n-quads': |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
307 format = 'nquads' |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
308 elif accept == 'pickle': |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
309 # don't use this; it's just for speed comparison |
| 26 | 310 import pickle as pickle |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
311 pickle.dump(self.settings.db.graph, self, protocol=2) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
312 return |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
313 elif accept == 'msgpack': |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
314 self.write(repr(self.settings.db.graph.__getstate__)) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
315 return |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
316 self.write(self.settings.db.graph.serialize(format=format)) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
317 |
| 45 | 318 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
319 class Prefixes(cyclone.web.RequestHandler): |
| 45 | 320 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
321 def post(self): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
322 suggestion = json.loads(self.request.body) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
323 addlPrefixes = self.settings.db.watchedFiles.addlPrefixes |
| 80 | 324 addlPrefixes.setdefault(URIRef(suggestion['ctx']), {}).update(suggestion['prefixes']) |
| 45 | 325 |
| 326 | |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
327 class NoExts(cyclone.web.StaticFileHandler): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
328 # .html pages can be get() without .html on them |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
329 def get(self, path, *args, **kw): |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
330 if path and '.' not in path: |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
331 path = path + ".html" |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
332 cyclone.web.StaticFileHandler.get(self, path, *args, **kw) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
333 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
334 |
| 80 | 335 def main(dirUriMap: Optional[DirUriMap] = None, prefixes: Optional[Dict[str, URIRef]] = None, port=9999): |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
336 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
337 if dirUriMap is None: |
| 42 | 338 dirUriMap = {b'data/': URIRef('http://example.com/data/')} |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
339 if prefixes is None: |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
340 prefixes = { |
| 42 | 341 'rdf': URIRef('http://www.w3.org/1999/02/22-rdf-syntax-ns#'), |
| 342 'rdfs': URIRef('http://www.w3.org/2000/01/rdf-schema#'), | |
| 343 'xsd': URIRef('http://www.w3.org/2001/XMLSchema#'), | |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
344 } |
| 45 | 345 |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
346 logging.basicConfig() |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
347 log = logging.getLogger() |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
348 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
349 parser = optparse.OptionParser() |
| 80 | 350 parser.add_option("-v", "--verbose", action="store_true", help="logging.DEBUG") |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
351 (options, args) = parser.parse_args() |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
352 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
353 log.setLevel(logging.DEBUG if options.verbose else logging.INFO) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
354 |
| 45 | 355 db = Db(dirUriMap=dirUriMap, addlPrefixes={None: prefixes}) |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
356 |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
357 from twisted.python import log as twlog |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
358 twlog.startLogging(sys.stdout) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
359 |
| 45 | 360 reactor.listenTCP( |
| 361 port, | |
| 87 | 362 cyclone.web.Application( |
| 363 handlers=[ | |
| 364 (r'/graph', GraphResource), | |
| 365 (r'/syncedGraph', WebsocketClient), | |
| 366 (r'/prefixes', Prefixes), | |
| 367 # (r'/stats/(.*)', StatsHandler, { | |
| 368 # 'serverName': 'rdfdb' | |
| 369 # }), | |
| 370 (r'/(.*)', NoExts, { | |
| 371 "path": FilePath(__file__).sibling("web").path, | |
| 372 "default_filename": "index.html" | |
| 373 }), | |
| 374 ], | |
| 375 debug=True, | |
| 376 db=db)) | |
|
13
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
377 log.info("serving on %s" % port) |
|
c9d1764d64ad
add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
378 reactor.run() |
| 86 | 379 |
| 380 | |
|
81
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
381 if __name__ == '__main__': |
|
8d3c3e7cfb54
(barely) runnable demo service instance
drewp@bigasterisk.com
parents:
80
diff
changeset
|
382 main() |