annotate service/reasoning/sse_collector.py @ 302:46c5fae89823

factor out patchsource Ignore-this: a9757cc53b914cb8be1f880a6504336f
author drewp@bigasterisk.com
date Sun, 28 Aug 2016 23:43:03 -0700
parents 29f593aee67b
children 66fe7a93753d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
1 """
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
2 requesting /graph/foo returns an SSE patch stream that's the
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
3 result of fetching multiple other SSE patch streams. The result stream
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
4 may include new statements injected by this service.
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
5
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
6 Future:
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
7 - filter out unneeded stmts from the sources
298
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
8 - give a time resolution and concatenate any patches that come faster than that res
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
9 """
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
10
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
11 config = {
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
12 'streams': [
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
13 {'id': 'home',
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
14 'sources': [
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
15 #'http://bang:9059/graph/events',
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
16 'http://plus:9075/graph/events',
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
17 ]
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
18 },
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
19 ]
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
20 }
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
21
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
22 from crochet import no_setup
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
23 no_setup()
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
24
302
46c5fae89823 factor out patchsource
drewp@bigasterisk.com
parents: 301
diff changeset
25 import sys, logging, collections
46c5fae89823 factor out patchsource
drewp@bigasterisk.com
parents: 301
diff changeset
26 from twisted.internet import reactor
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
27 import cyclone.web, cyclone.sse
302
46c5fae89823 factor out patchsource
drewp@bigasterisk.com
parents: 301
diff changeset
28 from rdflib import URIRef, Namespace
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
29 from docopt import docopt
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
30
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
31
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
32 sys.path.append("../../lib")
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
33 from logsetup import log
302
46c5fae89823 factor out patchsource
drewp@bigasterisk.com
parents: 301
diff changeset
34 from patchablegraph import jsonFromPatch
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
35
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
36 sys.path.append("/my/proj/light9")
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
37 from light9.rdfdb.patch import Patch
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
38
302
46c5fae89823 factor out patchsource
drewp@bigasterisk.com
parents: 301
diff changeset
39 from patchsource import ReconnectingPatchSource
46c5fae89823 factor out patchsource
drewp@bigasterisk.com
parents: 301
diff changeset
40
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
41 ROOM = Namespace("http://projects.bigasterisk.com/room/")
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
42 COLLECTOR = URIRef('http://bigasterisk.com/sse_collector/')
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
43
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
44 class LocalStatements(object):
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
45 """
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
46 functions that make statements originating from sse_collector itself
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
47 """
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
48 def __init__(self, applyPatch):
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
49 self.applyPatch = applyPatch
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
50 self._sourceState = {} # source: state URIRef
298
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
51
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
52 def setSourceState(self, source, state):
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
53 """
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
54 add a patch to the COLLECTOR graph about the state of this
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
55 source. state=None to remove the source.
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
56 """
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
57 oldState = self._sourceState.get(source, None)
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
58 if state == oldState:
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
59 return
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
60 log.info('source state %s -> %s', source, state)
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
61 if oldState is None:
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
62 self._sourceState[source] = state
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
63 self.applyPatch(COLLECTOR, Patch(addQuads=[
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
64 (COLLECTOR, ROOM['source'], source, COLLECTOR),
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
65 (source, ROOM['state'], state, COLLECTOR),
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
66 ]))
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
67 elif state is None:
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
68 del self._sourceState[source]
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
69 self.applyPatch(COLLECTOR, Patch(delQuads=[
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
70 (COLLECTOR, ROOM['source'], source, COLLECTOR),
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
71 (source, ROOM['state'], oldState, COLLECTOR),
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
72 ]))
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
73 else:
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
74 self._sourceState[source] = state
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
75 self.applyPatch(COLLECTOR, Patch(
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
76 addQuads=[
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
77 (source, ROOM['state'], state, COLLECTOR),
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
78 ],
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
79 delQuads=[
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
80 (source, ROOM['state'], oldState, COLLECTOR),
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
81 ]))
298
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
82
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
83 def abbrevTerm(t):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
84 if isinstance(t, URIRef):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
85 return (t.replace('http://projects.bigasterisk.com/room/', 'room:')
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
86 .replace('http://bigasterisk.com/sse_collector/', 'sc:'))
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
87 return t
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
88
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
89 def abbrevStmt(stmt):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
90 return '(%s %s %s %s)' % tuple(map(abbrevTerm, stmt))
298
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
91
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
92 class ActiveStatements(object):
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
93 def __init__(self):
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
94
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
95 # This table holds statements asserted by any of our sources
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
96 # plus local statements that we introduce (source is
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
97 # http://bigasterisk.com/sse_collector/).
298
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
98 self.statements = collections.defaultdict(lambda: (set(), set())) # (s,p,o,c): (sourceUrls, handlers)`
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
99
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
100 def _postDeleteStatements(self):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
101 statements = self.statements
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
102 class PostDeleter(object):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
103 def __enter__(self):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
104 self._garbage = []
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
105 return self
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
106 def add(self, stmt):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
107 self._garbage.append(stmt)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
108 def __exit__(self, type, value, traceback):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
109 if type is not None:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
110 raise
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
111 for stmt in self._garbage:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
112 del statements[stmt]
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
113 return PostDeleter()
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
114
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
115 def pprintTable(self):
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
116 for i, (stmt, (sources, handlers)) in enumerate(sorted(self.statements.items())):
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
117 print "%03d. %-80s from %s to %s" % (
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
118 i, abbrevStmt(stmt), [abbrevTerm(s) for s in sources], handlers)
298
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
119
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
120 def makeSyncPatch(self, handler, sources):
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
121 # todo: this could run all handlers at once, which is how we use it anyway
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
122 adds = []
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
123 dels = []
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
124
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
125 with self._postDeleteStatements() as garbage:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
126 for stmt, (stmtSources, handlers) in self.statements.iteritems():
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
127 belongsInHandler = not set(sources).isdisjoint(stmtSources)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
128 handlerHasIt = handler in handlers
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
129 #log.debug("%s %s %s", abbrevStmt(stmt), belongsInHandler, handlerHasIt)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
130 if belongsInHandler and not handlerHasIt:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
131 adds.append(stmt)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
132 handlers.add(handler)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
133 elif not belongsInHandler and handlerHasIt:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
134 dels.append(stmt)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
135 handlers.remove(handler)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
136 if not handlers and not stmtSources:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
137 garbage.add(stmt)
298
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
138
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
139 return Patch(addQuads=adds, delQuads=dels)
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
140
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
141 def applySourcePatch(self, source, p):
298
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
142 for stmt in p.addQuads:
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
143 sourceUrls, handlers = self.statements[stmt]
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
144 if source in sourceUrls:
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
145 raise ValueError("%s added stmt that it already had: %s" %
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
146 (source, abbrevStmt(stmt)))
298
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
147 sourceUrls.add(source)
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
148
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
149 with self._postDeleteStatements() as garbage:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
150 for stmt in p.delQuads:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
151 sourceUrls, handlers = self.statements[stmt]
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
152 if source not in sourceUrls:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
153 raise ValueError("%s deleting stmt that it didn't have: %s" %
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
154 (source, abbrevStmt(stmt)))
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
155 sourceUrls.remove(source)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
156 # this is rare, since some handler probably still has
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
157 # the stmt we're deleting, but it can happen e.g. when
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
158 # a handler was just deleted
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
159 if not sourceUrls and not handlers:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
160 garbage.add(stmt)
298
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
161
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
162 def replaceSourceStatements(self, source, stmts):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
163 log.debug('replaceSourceStatements with %s stmts', len(stmts))
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
164 newStmts = set(stmts)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
165
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
166 with self._postDeleteStatements() as garbage:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
167 for stmt, (sources, handlers) in self.statements.iteritems():
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
168 if source in sources:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
169 if stmt not in stmts:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
170 sources.remove(source)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
171 if not sources and not handlers:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
172 garbage.add(stmt)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
173 else:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
174 if stmt in stmts:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
175 sources.add(source)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
176 newStmts.discard(stmt)
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
177
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
178 self.applySourcePatch(source, Patch(addQuads=newStmts, delQuads=[]))
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
179
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
180 def discardHandler(self, handler):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
181 with self._postDeleteStatements() as garbage:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
182 for stmt, (sources, handlers) in self.statements.iteritems():
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
183 handlers.discard(handler)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
184 if not sources and not handlers:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
185 garbage.add(stmt)
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
186
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
187 def discardSource(self, source):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
188 with self._postDeleteStatements() as garbage:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
189 for stmt, (sources, handlers) in self.statements.iteritems():
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
190 sources.discard(source)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
191 if not sources and not handlers:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
192 garbage.add(stmt)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
193
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
194 class GraphClients(object):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
195 """
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
196 All the active PatchSources and SSEHandlers
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
197
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
198 To handle all the overlapping-statement cases, we store a set of
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
199 true statements along with the sources that are currently
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
200 asserting them and the requesters who currently know them. As
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
201 statements come and go, we make patches to send to requesters.
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
202 """
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
203 def __init__(self):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
204 self.clients = {} # url: PatchSource (COLLECTOR is not listed)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
205 self.handlers = set() # handler
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
206 self.statements = ActiveStatements()
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
207
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
208 self._localStatements = LocalStatements(self._onPatch)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
209
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
210 def _sourcesForHandler(self, handler):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
211 streamId = handler.streamId
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
212 matches = [s for s in config['streams'] if s['id'] == streamId]
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
213 if len(matches) != 1:
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
214 raise ValueError("%s matches for %r" % (len(matches), streamId))
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
215 return map(URIRef, matches[0]['sources']) + [COLLECTOR]
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
216
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
217 def _onPatch(self, source, p, fullGraph=False):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
218 if fullGraph:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
219 # a reconnect may need to resend the full graph even
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
220 # though we've already sent some statements
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
221 self.statements.replaceSourceStatements(source, p.addQuads)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
222 else:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
223 self.statements.applySourcePatch(source, p)
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
224
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
225 self._sendUpdatePatch()
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
226
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
227 if log.isEnabledFor(logging.DEBUG):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
228 self.statements.pprintTable()
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
229
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
230 if source != COLLECTOR:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
231 self._localStatements.setSourceState(
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
232 source,
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
233 ROOM['fullGraphReceived'] if fullGraph else
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
234 ROOM['patchesReceived'])
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
235
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
236 def _sendUpdatePatch(self, handler=None):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
237 """
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
238 send a patch event out this handler to bring it up to date with
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
239 self.statements
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
240 """
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
241 # reduce loops here- prepare all patches at once
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
242 for h in (self.handlers if handler is None else [handler]):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
243 p = self.statements.makeSyncPatch(h, self._sourcesForHandler(h))
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
244 if not p.isNoop():
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
245 log.debug("send patch %s to %s", p.shortSummary(), h)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
246 h.sendEvent(message=jsonFromPatch(p), event='patch')
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
247
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
248 def addSseHandler(self, handler):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
249 log.info('addSseHandler %r %r', handler, handler.streamId)
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
250 self.handlers.add(handler)
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
251
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
252 for source in self._sourcesForHandler(handler):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
253 if source not in self.clients and source != COLLECTOR:
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
254 self._localStatements.setSourceState(source, ROOM['connect'])
302
46c5fae89823 factor out patchsource
drewp@bigasterisk.com
parents: 301
diff changeset
255 self.clients[source] = ReconnectingPatchSource(
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
256 source, listener=lambda p, fullGraph, source=source: self._onPatch(
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
257 source, p, fullGraph))
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
258 self._sendUpdatePatch(handler)
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
259
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
260 def removeSseHandler(self, handler):
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
261 log.info('removeSseHandler %r', handler)
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
262
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
263 self.statements.discardHandler(handler)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
264
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
265 for source in self._sourcesForHandler(handler):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
266 for otherHandler in self.handlers:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
267 if (otherHandler != handler and
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
268 source in self._sourcesForHandler(otherHandler)):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
269 break
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
270 else:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
271 self._stopClient(source)
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
272
298
8d89da1915df sse_collector now kind of gets concurrent requests right
drewp@bigasterisk.com
parents: 296
diff changeset
273 self.handlers.remove(handler)
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
274
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
275 def _stopClient(self, url):
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
276 if url == COLLECTOR:
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
277 return
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
278
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
279 self.clients[url].stop()
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
280
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
281 self.statements.discardSource(url)
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
282
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
283 self._localStatements.setSourceState(url, None)
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
284 del self.clients[url]
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
285
300
371af6e92b5e local state statements and self.statements rewrite
drewp@bigasterisk.com
parents: 299
diff changeset
286
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
287 class SomeGraph(cyclone.sse.SSEHandler):
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
288 _handlerSerial = 0
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
289 def __init__(self, application, request):
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
290 cyclone.sse.SSEHandler.__init__(self, application, request)
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
291 self.streamId = request.uri[len('/graph/'):]
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
292 self.graphClients = self.settings.graphClients
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
293
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
294 self._serial = SomeGraph._handlerSerial
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
295 SomeGraph._handlerSerial += 1
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
296
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
297 def __repr__(self):
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
298 return '<Handler #%s>' % self._serial
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
299
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
300 def bind(self):
301
29f593aee67b big rewrites in sse_collector
drewp@bigasterisk.com
parents: 300
diff changeset
301 self.graphClients.addSseHandler(self)
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
302
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
303 def unbind(self):
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
304 self.graphClients.removeSseHandler(self)
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
305
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
306 if __name__ == '__main__':
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
307
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
308 arg = docopt("""
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
309 Usage: sse_collector.py [options]
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
310
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
311 -v Verbose
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
312 """)
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
313
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
314 if arg['-v']:
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
315 import twisted.python.log
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
316 twisted.python.log.startLogging(sys.stdout)
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
317 log.setLevel(logging.DEBUG)
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
318
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
319
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
320 graphClients = GraphClients()
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
321
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
322 reactor.listenTCP(
299
5084a1f719c9 port 9072
drewp@bigasterisk.com
parents: 298
diff changeset
323 9072,
296
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
324 cyclone.web.Application(
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
325 handlers=[
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
326 (r'/graph/(.*)', SomeGraph),
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
327 ],
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
328 graphClients=graphClients),
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
329 interface='::')
233b81cf2712 start sse_collector
drewp@bigasterisk.com
parents:
diff changeset
330 reactor.run()