Mercurial > code > home > repos > rdfdb

annotate rdfdb/service.py @ 43:8d4822ae58bc

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
more types, as needed on light9 keyboardcomposer Ignore-this: 9bc5d28702e6684fcce62104fc50966d
author Drew Perttula <drewp@bigasterisk.com>
date Mon, 27 May 2019 06:58:51 +0000
parents 541010f99d61
children dc61012eeace
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
1 from twisted.internet import reactor, defer
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
2 import twisted.internet.error
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
3 from twisted.python.filepath import FilePath
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
4 from twisted.python.failure import Failure
35
deaabf47d82b fix pyflakes warnings, including an error at import
Drew Perttula <drewp@bigasterisk.com>
parents: 31
diff changeset
5 from twisted.internet.inotify import IN_CREATE
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
6 import sys, optparse, logging, json, os
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
7 import cyclone.web, cyclone.httpclient, cyclone.websocket
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
8 from typing import Dict, List, Set, Optional, Union
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
9
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
10 from rdflib import ConjunctiveGraph, URIRef, Graph
43
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
11 from rdfdb.graphfile import GraphFile, PatchCb, GetSubgraph
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
12 from rdfdb.patch import Patch, ALLSTMTS
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
13 from rdfdb.rdflibpatch import patchQuads
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
14 from rdfdb.file_vs_uri import correctToTopdirPrefix, fileForUri, uriFromFile, DirUriMap
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
15 from rdfdb.patchsender import sendPatch
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
16 from rdfdb.patchreceiver import makePatchEndpointPutMethod
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
17
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
18 from twisted.internet.inotify import INotify
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
19
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
20 log = logging.getLogger('rdfdb')
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
21 log.setLevel(logging.DEBUG)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
22
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
23 class WebsocketDisconnect(ValueError):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
24 pass
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
25
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
26
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
27 class Client(object):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
28 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
29 one of our syncedgraph clients
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
30 """
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
31 def __init__(self, updateUri: URIRef, label: str):
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
32 self.label = label
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
33 # todo: updateUri is used publicly to compare clients. Replace
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
34 # it with Client.__eq__ so WsClient doesn't have to fake an
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
35 # updateUri.
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
36 self.updateUri = updateUri
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
37
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
38 def __repr__(self):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
39 return "<%s client at %s>" % (self.label, self.updateUri)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
40
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
41 def sendPatch(self, p: Patch) -> defer.Deferred:
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
42 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
43 returns deferred. error will be interpreted as the client being
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
44 broken.
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
45 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
46 return sendPatch(self.updateUri, p)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
47
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
48 class WsClient(object):
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
49 def __init__(self, connectionId: str, sendMessage):
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
50 self.updateUri = URIRef(connectionId)
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
51 self.sendMessage = sendMessage
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
52
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
53 def __repr__(self):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
54 return "<WsClient %s>" % self.updateUri
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
55
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
56 def sendPatch(self, p: Patch) -> defer.Deferred:
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
57 self.sendMessage(p.makeJsonRepr())
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
58 return defer.succeed(None)
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
59
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
60 def sendGraphToClient(graph, client: Union[Client, WsClient]) -> None:
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
61 """send the client the whole graph contents"""
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
62 log.info("sending all graphs to %r" % client)
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
63 client.sendPatch(Patch(
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
64 addQuads=graph.quads(ALLSTMTS),
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
65 delQuads=[]))
43
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
66
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
67 class WatchedFiles(object):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
68 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
69 find files, notice new files.
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
70
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
71 This object watches directories. Each GraphFile watches its own file.
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
72 """
43
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
73 def __init__(self, dirUriMap: DirUriMap, patch: PatchCb, getSubgraph: GetSubgraph, addlPrefixes: Dict[str, URIRef]):
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
74 self.dirUriMap = dirUriMap # {abspath : uri prefix}
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
75 self.patch, self.getSubgraph = patch, getSubgraph
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
76 self.addlPrefixes = addlPrefixes
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
77
31
3ca1a8774513 typing fixes. rewrite sequentialUri a bit
Drew Perttula <drewp@bigasterisk.com>
parents: 30
diff changeset
78 self.graphFiles: Dict[URIRef, GraphFile] = {} # context uri : GraphFile
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
79
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
80 self.notifier = INotify()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
81 self.notifier.startReading()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
82
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
83 self.findAndLoadFiles()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
84
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
85 def findAndLoadFiles(self) -> None:
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
86 self.initialLoad = True
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
87 try:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
88 for topdir in self.dirUriMap:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
89 for dirpath, dirnames, filenames in os.walk(topdir):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
90 for base in filenames:
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
91 p = os.path.join(dirpath, base)
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
92 # why wasn't mypy catching this?
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
93 assert isinstance(p, bytes)
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
94 self.watchFile(p)
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
95 self.notifier.watch(FilePath(dirpath), autoAdd=True,
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
96 callbacks=[self.dirChange])
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
97 finally:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
98 self.initialLoad = False
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
99
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
100 def dirChange(self, watch, path: FilePath, mask):
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
101 if mask & IN_CREATE:
39
83fc83e919e5 more type errors.
Drew Perttula <drewp@bigasterisk.com>
parents: 37
diff changeset
102 if path.path.endswith((b'~', b'.swp', b'swx', b'.rdfdb-temp')):
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
103 return
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
104
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
105 log.debug("%s created; consider adding a watch", path)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
106 self.watchFile(path.path)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
107
39
83fc83e919e5 more type errors.
Drew Perttula <drewp@bigasterisk.com>
parents: 37
diff changeset
108 def watchFile(self, inFile: bytes):
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
109 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
110 consider adding a GraphFile to self.graphFiles
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
111
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
112 inFile needs to be a relative path, not an absolute (e.g. in a
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
113 FilePath) because we use its exact relative form in the
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
114 context URI
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
115 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
116 if not os.path.isfile(inFile):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
117 return
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
118
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
119 inFile = correctToTopdirPrefix(self.dirUriMap, inFile)
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
120 if os.path.splitext(inFile)[1] not in [b'.n3']:
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
121 return
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
122
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
123 if b'/capture/' in inFile:
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
124 # smaller graph for now
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
125 return
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
126
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
127 # an n3 file with rules makes it all the way past this reading
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
128 # and the serialization. Then, on the receiving side, a
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
129 # SyncedGraph calls graphFromNQuad on the incoming data and
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
130 # has a parse error. I'm not sure where this should be fixed
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
131 # yet.
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
132 if b'-rules' in inFile:
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
133 return
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
134
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
135 # for legacy versions, compile all the config stuff you want
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
136 # read into one file called config.n3. New versions won't read
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
137 # it.
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
138 if inFile.endswith(b"config.n3"):
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
139 return
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
140
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
141 ctx = uriFromFile(self.dirUriMap, inFile)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
142 gf = self._addGraphFile(ctx, inFile)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
143 log.info("%s do initial read", inFile)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
144 gf.reread()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
145
43
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
146 def aboutToPatch(self, ctx: URIRef):
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
147 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
148 warn us that a patch is about to come to this context. it's more
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
149 straightforward to create the new file now
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
150
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
151 this is meant to make the file before we add triples, so we
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
152 wouldn't see the blank file and lose those triples. But it
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
153 didn't work, so there are other measures that make us not lose
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
154 the triples from a new file. Calling this before patching the
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
155 graph is still a reasonable thing to do, though.
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
156 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
157 if ctx not in self.graphFiles:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
158 outFile = fileForUri(self.dirUriMap, ctx)
43
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
159 # mypy missed the next line because of
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
160 # https://github.com/python/typeshed/issues/2937 ('str in
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
161 # bytes' isn't an error)
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
162 assert b'//' not in outFile, (outFile, self.dirUriMap, ctx)
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
163 log.info("starting new file %r", outFile)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
164 self._addGraphFile(ctx, outFile)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
165
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
166 def _addGraphFile(self, ctx, path):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
167 self.addlPrefixes.setdefault(ctx, {})
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
168 self.addlPrefixes.setdefault(None, {})
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
169 gf = GraphFile(self.notifier, path, ctx,
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
170 self.patch, self.getSubgraph,
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
171 globalPrefixes=self.addlPrefixes[None],
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
172 ctxPrefixes=self.addlPrefixes[ctx])
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
173 self.graphFiles[ctx] = gf
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
174 return gf
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
175
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
176
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
177 def dirtyFiles(self, ctxs):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
178 """mark dirty the files that we watch in these contexts.
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
179
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
180 the ctx might not be a file that we already read; it might be
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
181 for a new file we have to create, or it might be for a
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
182 transient context that we're not going to save
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
183
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
184 if it's a ctx with no file, error
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
185 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
186 for ctx in ctxs:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
187 g = self.getSubgraph(ctx)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
188 self.graphFiles[ctx].dirty(g)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
189
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
190
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
191 class Db(object):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
192 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
193 the master graph, all the connected clients, all the files we're watching
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
194 """
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
195 def __init__(self, dirUriMap: DirUriMap, addlPrefixes):
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
196 self.clients: List[Union[Client, WsClient]] = []
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
197 self.graph = ConjunctiveGraph()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
198
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
199 self.watchedFiles = WatchedFiles(dirUriMap,
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
200 self.patch, self.getSubgraph,
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
201 addlPrefixes)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
202
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
203 self.summarizeToLog()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
204
43
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
205 def patch(self, patch: Patch, dueToFileChange: bool=False) -> None:
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
206 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
207 apply this patch to the master graph then notify everyone about it
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
208
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
209 dueToFileChange if this is a patch describing an edit we read
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
210 *from* the file (such that we shouldn't write it back to the file)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
211
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
212 if p has a senderUpdateUri attribute, we won't send this patch
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
213 back to the sender with that updateUri
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
214 """
43
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
215 ctx = patch.getContext()
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
216 log.info("patching graph %s -%d +%d" % (
43
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
217 ctx, len(patch.delQuads), len(patch.addQuads)))
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
218
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
219 if hasattr(self, 'watchedFiles'): # not available during startup
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
220 self.watchedFiles.aboutToPatch(ctx)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
221
43
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
222 patchQuads(self.graph, patch.delQuads, patch.addQuads, perfect=True)
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
223 self._sendPatch(patch)
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
224 if not dueToFileChange:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
225 self.watchedFiles.dirtyFiles([ctx])
43
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
226 sendToLiveClients(asJson=patch.jsonRepr)
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
227
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
228 def _sendPatch(self, p):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
229 senderUpdateUri = getattr(p, 'senderUpdateUri', None)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
230
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
231 for c in self.clients:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
232 if c.updateUri == senderUpdateUri:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
233 # this client has self-applied the patch already
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
234 continue
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
235 d = c.sendPatch(p)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
236 d.addErrback(self.clientErrored, c)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
237
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
238 def clientErrored(self, err, c):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
239 err.trap(twisted.internet.error.ConnectError, WebsocketDisconnect)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
240 log.info("%r %r - dropping client", c, err.getErrorMessage())
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
241 if c in self.clients:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
242 self.clients.remove(c)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
243 self.sendClientsToAllLivePages()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
244
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
245 def summarizeToLog(self):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
246 log.info("contexts in graph (%s total stmts):" % len(self.graph))
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
247 for c in self.graph.contexts():
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
248 log.info(" %s: %s statements" %
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
249 (c.identifier, len(self.getSubgraph(c.identifier))))
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
250
43
8d4822ae58bc more types, as needed on light9 keyboardcomposer
Drew Perttula <drewp@bigasterisk.com>
parents: 42
diff changeset
251 def getSubgraph(self, uri: URIRef) -> Graph:
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
252 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
253 this is meant to return a live view of the given subgraph, but
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
254 if i'm still working around an rdflib bug, it might return a
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
255 copy
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
256
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
257 and it's returning triples, but I think quads would be better
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
258 """
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
259 # this is returning an empty Graph :(
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
260 #return self.graph.get_context(uri)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
261
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
262 g = Graph()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
263 for s in self.graph.triples(ALLSTMTS, uri):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
264 g.add(s)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
265 return g
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
266
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
267 def addClient(self, newClient: Union[Client, WsClient]) -> None:
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
268 for c in self.clients:
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
269 if c.updateUri == newClient.updateUri:
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
270 self.clients.remove(c)
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
271
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
272 log.info("new client %r" % newClient)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
273 sendGraphToClient(self.graph, newClient)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
274 self.clients.append(newClient)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
275 self.sendClientsToAllLivePages()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
276
37
9b4c551dddfb fix registration bug (tried to send bytes to json encoder)
Drew Perttula <drewp@bigasterisk.com>
parents: 35
diff changeset
277 def sendClientsToAllLivePages(self) -> None:
9b4c551dddfb fix registration bug (tried to send bytes to json encoder)
Drew Perttula <drewp@bigasterisk.com>
parents: 35
diff changeset
278 sendToLiveClients({"clients": [
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
279 dict(updateUri=c.updateUri.toPython(), label=repr(c))
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
280 for c in self.clients]})
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
281
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
282 class GraphResource(cyclone.web.RequestHandler):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
283 def get(self):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
284 accept = self.request.headers.get('accept', '')
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
285 format = 'n3'
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
286 if accept == 'text/plain':
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
287 format = 'nt'
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
288 elif accept == 'application/n-quads':
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
289 format = 'nquads'
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
290 elif accept == 'pickle':
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
291 # don't use this; it's just for speed comparison
26
95c57a5cb18e run 2to3
drewp@bigasterisk.com
parents: 13
diff changeset
292 import pickle as pickle
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
293 pickle.dump(self.settings.db.graph, self, protocol=2)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
294 return
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
295 elif accept == 'msgpack':
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
296 self.write(repr(self.settings.db.graph.__getstate__))
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
297 return
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
298 self.write(self.settings.db.graph.serialize(format=format))
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
299
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
300 class Patches(cyclone.web.RequestHandler):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
301 def __init__(self, *args, **kw):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
302 cyclone.web.RequestHandler.__init__(self, *args, **kw)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
303 p = makePatchEndpointPutMethod(self.settings.db.patch)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
304 self.put = lambda: p(self)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
305
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
306 def get(self):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
307 pass
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
308
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
309 class GraphClients(cyclone.web.RequestHandler):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
310 def get(self):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
311 pass
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
312
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
313 def post(self) -> None:
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
314 upd = URIRef(self.get_argument("clientUpdate"))
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
315 try:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
316 self.settings.db.addClient(Client(upd, self.get_argument("label")))
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
317 except:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
318 import traceback
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
319 traceback.print_exc()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
320 raise
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
321
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
322 class Prefixes(cyclone.web.RequestHandler):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
323 def post(self):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
324 suggestion = json.loads(self.request.body)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
325 addlPrefixes = self.settings.db.watchedFiles.addlPrefixes
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
326 addlPrefixes.setdefault(URIRef(suggestion['ctx']), {}).update(suggestion['prefixes'])
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
327
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
328 _wsClientSerial = 0
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
329 class WebsocketClient(cyclone.websocket.WebSocketHandler):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
330
39
83fc83e919e5 more type errors.
Drew Perttula <drewp@bigasterisk.com>
parents: 37
diff changeset
331 wsClient: Optional[WsClient] = None
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
332 def connectionMade(self, *args, **kwargs) -> None:
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
333 global _wsClientSerial
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
334 connectionId = f'connection-{_wsClientSerial}'
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
335 _wsClientSerial += 1
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
336
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
337 self.wsClient = WsClient(connectionId, self.sendMessage)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
338 log.info("new ws client %r", self.wsClient)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
339 self.settings.db.addClient(self.wsClient)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
340
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
341 def connectionLost(self, reason):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
342 log.info("bye ws client %r", self.wsClient)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
343 self.settings.db.clientErrored(
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
344 Failure(WebsocketDisconnect(reason)), self.wsClient)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
345
40
195779114472 str/bytes fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 39
diff changeset
346 def messageReceived(self, message: bytes):
195779114472 str/bytes fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 39
diff changeset
347 if message == b'PING':
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
348 self.sendMessage('PONG')
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
349 return
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
350 log.info("got message from %r: %s", self.wsClient, message)
40
195779114472 str/bytes fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 39
diff changeset
351 p = Patch(jsonRepr=message.decode('utf8'))
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
352 assert self.wsClient is not None
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
353 p.senderUpdateUri = self.wsClient.updateUri
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
354 self.settings.db.patch(p)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
355
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
356 class Live(cyclone.websocket.WebSocketHandler):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
357
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
358 def connectionMade(self, *args, **kwargs):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
359 log.info("websocket opened")
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
360 liveClients.add(self)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
361 self.settings.db.sendClientsToAllLivePages()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
362
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
363 def connectionLost(self, reason):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
364 log.info("websocket closed")
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
365 liveClients.remove(self)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
366
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
367 def messageReceived(self, message):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
368 log.info("got message %s" % message)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
369 self.sendMessage(message)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
370
35
deaabf47d82b fix pyflakes warnings, including an error at import
Drew Perttula <drewp@bigasterisk.com>
parents: 31
diff changeset
371 liveClients: Set[Live] = set()
deaabf47d82b fix pyflakes warnings, including an error at import
Drew Perttula <drewp@bigasterisk.com>
parents: 31
diff changeset
372 def sendToLiveClients(d=None, asJson=None):
deaabf47d82b fix pyflakes warnings, including an error at import
Drew Perttula <drewp@bigasterisk.com>
parents: 31
diff changeset
373 j = asJson or json.dumps(d)
deaabf47d82b fix pyflakes warnings, including an error at import
Drew Perttula <drewp@bigasterisk.com>
parents: 31
diff changeset
374 for c in liveClients:
deaabf47d82b fix pyflakes warnings, including an error at import
Drew Perttula <drewp@bigasterisk.com>
parents: 31
diff changeset
375 c.sendMessage(j)
deaabf47d82b fix pyflakes warnings, including an error at import
Drew Perttula <drewp@bigasterisk.com>
parents: 31
diff changeset
376
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
377 class NoExts(cyclone.web.StaticFileHandler):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
378 # .html pages can be get() without .html on them
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
379 def get(self, path, *args, **kw):
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
380 if path and '.' not in path:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
381 path = path + ".html"
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
382 cyclone.web.StaticFileHandler.get(self, path, *args, **kw)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
383
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
384
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
385 def main(dirUriMap: Optional[DirUriMap]=None,
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
386 prefixes: Optional[Dict[str, URIRef]]=None,
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
387 port=9999):
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
388
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
389 if dirUriMap is None:
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
390 dirUriMap = {b'data/': URIRef('http://example.com/data/')}
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
391 if prefixes is None:
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
392 prefixes = {
42
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
393 'rdf': URIRef('http://www.w3.org/1999/02/22-rdf-syntax-ns#'),
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
394 'rdfs': URIRef('http://www.w3.org/2000/01/rdf-schema#'),
541010f99d61 type fixes
Drew Perttula <drewp@bigasterisk.com>
parents: 40
diff changeset
395 'xsd': URIRef('http://www.w3.org/2001/XMLSchema#'),
13
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
396 }
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
397
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
398 logging.basicConfig()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
399 log = logging.getLogger()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
400
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
401 parser = optparse.OptionParser()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
402 parser.add_option("-v", "--verbose", action="store_true",
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
403 help="logging.DEBUG")
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
404 (options, args) = parser.parse_args()
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
405
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
406 log.setLevel(logging.DEBUG if options.verbose else logging.INFO)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
407
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
408 db = Db(dirUriMap=dirUriMap,
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
409 addlPrefixes={None: prefixes})
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
410
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
411 from twisted.python import log as twlog
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
412 twlog.startLogging(sys.stdout)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
413
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
414 reactor.listenTCP(port, cyclone.web.Application(handlers=[
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
415 (r'/live', Live),
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
416 (r'/graph', GraphResource),
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
417 (r'/patches', Patches),
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
418 (r'/graphClients', GraphClients),
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
419 (r'/syncedGraph', WebsocketClient),
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
420 (r'/prefixes', Prefixes),
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
421
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
422 (r'/(.*)', NoExts,
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
423 {"path" : FilePath(__file__).sibling("web").path,
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
424 "default_filename" : "index.html"}),
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
425
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
426 ], debug=True, db=db))
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
427 log.info("serving on %s" % port)
c9d1764d64ad add web server. remove more traces of light9
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
428 reactor.run()