Mercurial > code > home > repos > light9
annotate bin/collector @ 1884:5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Ignore-this: ce9f2586b03f5a773accab7ca3bf6c5d
author | Drew Perttula <drewp@bigasterisk.com> |
---|---|
date | Tue, 28 May 2019 06:48:37 +0000 |
parents | baae0bdfde74 |
children | 070c6d185276 |
rev | line source |
---|---|
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
1 #!bin/python |
1590 | 2 """ |
3 Collector receives device attrs from multiple senders, combines | |
4 them, and sends output attrs to hardware. The combining part has | |
5 custom code for some attributes. | |
6 | |
7 Input can be over http or zmq. | |
8 """ | |
9 | |
1697
5c04a54df635
fix startup (this might be breaking mypy)
Drew Perttula <drewp@bigasterisk.com>
parents:
1692
diff
changeset
|
10 from run_local import log |
5c04a54df635
fix startup (this might be breaking mypy)
Drew Perttula <drewp@bigasterisk.com>
parents:
1692
diff
changeset
|
11 |
1493
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
12 from twisted.internet import reactor, utils |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
13 import json |
1289
5a4e74f1e36a
Fixed client session clearing bugs.
Drew Perttula <drewp@bigasterisk.com>
parents:
1288
diff
changeset
|
14 import logging |
1372
f427801da9f6
collector properly merges repeated attr settings in the same message
Drew Perttula <drewp@bigasterisk.com>
parents:
1307
diff
changeset
|
15 import optparse |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
16 import traceback |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
17 import cyclone.web, cyclone.websocket |
1289
5a4e74f1e36a
Fixed client session clearing bugs.
Drew Perttula <drewp@bigasterisk.com>
parents:
1288
diff
changeset
|
18 from greplin import scales |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
19 |
1861 | 20 from cycloneerr import PrettyErrorHandler |
1866
3c523c71da29
pyflakes cleanups and some refactors
Drew Perttula <drewp@bigasterisk.com>
parents:
1861
diff
changeset
|
21 from light9 import networking |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
22 from light9.collector.collector import Collector |
1866
3c523c71da29
pyflakes cleanups and some refactors
Drew Perttula <drewp@bigasterisk.com>
parents:
1861
diff
changeset
|
23 from light9.collector.weblisteners import WebListeners |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
24 from light9.greplin_cyclone import StatsForCyclone |
1866
3c523c71da29
pyflakes cleanups and some refactors
Drew Perttula <drewp@bigasterisk.com>
parents:
1861
diff
changeset
|
25 from light9.namespaces import L9 |
3c523c71da29
pyflakes cleanups and some refactors
Drew Perttula <drewp@bigasterisk.com>
parents:
1861
diff
changeset
|
26 from light9.zmqtransport import parseJsonMessage, startZmq |
3c523c71da29
pyflakes cleanups and some refactors
Drew Perttula <drewp@bigasterisk.com>
parents:
1861
diff
changeset
|
27 from rdfdb.syncedgraph import SyncedGraph |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
28 |
1884
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
29 from light9.collector.output import Udmx, DummyOutput # noqa |
1858 | 30 |
31 | |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
32 class Updates(cyclone.websocket.WebSocketHandler): |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
33 |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
34 def connectionMade(self, *args, **kwargs): |
1543
c8cffe82b537
collector gui updates
Drew Perttula <drewp@bigasterisk.com>
parents:
1541
diff
changeset
|
35 log.info('socket connect %s', self) |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
36 self.settings.listeners.addClient(self) |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
37 |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
38 def connectionLost(self, reason): |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
39 self.settings.listeners.delClient(self) |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
40 |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
41 def messageReceived(self, message): |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
42 json.loads(message) |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
43 |
1858 | 44 |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
45 stats = scales.collection('/webServer', scales.PmfStat('setAttr')) |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
46 |
1858 | 47 |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
48 class Attrs(PrettyErrorHandler, cyclone.web.RequestHandler): |
1858 | 49 |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
50 def put(self): |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
51 with stats.setAttr.time(): |
1858 | 52 client, clientSession, settings, sendTime = parseJsonMessage( |
53 self.request.body) | |
54 self.settings.collector.setAttrs(client, clientSession, settings, | |
55 sendTime) | |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
56 self.set_status(202) |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
57 |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
58 |
1490
649d482737e0
start of a collector --loadtest mode
Drew Perttula <drewp@bigasterisk.com>
parents:
1489
diff
changeset
|
59 def launch(graph, doLoadTest=False): |
1530 | 60 try: |
61 # todo: drive outputs with config files | |
62 outputs = [ | |
1884
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
63 Udmx(L9['output/dmxA/'], bus=None, address=None), |
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
64 DummyOutput(L9['output/dmxB/']), |
1530 | 65 ] |
1799
0bb7b9df12e5
collector warnings and errors. the reactor.crash isn't working.
drewp@bigasterisk.com
parents:
1798
diff
changeset
|
66 except Exception: |
0bb7b9df12e5
collector warnings and errors. the reactor.crash isn't working.
drewp@bigasterisk.com
parents:
1798
diff
changeset
|
67 log.error("setting up outputs:") |
1530 | 68 traceback.print_exc() |
69 raise | |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
70 listeners = WebListeners() |
1884
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
71 c: Collector = Collector(graph, outputs, listeners) |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
72 |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
73 startZmq(networking.collectorZmq.port, c) |
1858 | 74 |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
75 reactor.listenTCP(networking.collector.port, |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
76 cyclone.web.Application(handlers=[ |
1858 | 77 (r'/()', cyclone.web.StaticFileHandler, { |
78 "path": "light9/collector/web", | |
79 "default_filename": "index.html" | |
80 }), | |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
81 (r'/updates', Updates), |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
82 (r'/attrs', Attrs), |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
83 (r'/stats', StatsForCyclone), |
1858 | 84 ], |
85 collector=c, | |
86 listeners=listeners), | |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
87 interface='::') |
1289
5a4e74f1e36a
Fixed client session clearing bugs.
Drew Perttula <drewp@bigasterisk.com>
parents:
1288
diff
changeset
|
88 log.info('serving http on %s, zmq on %s', networking.collector.port, |
5a4e74f1e36a
Fixed client session clearing bugs.
Drew Perttula <drewp@bigasterisk.com>
parents:
1288
diff
changeset
|
89 networking.collectorZmq.port) |
1490
649d482737e0
start of a collector --loadtest mode
Drew Perttula <drewp@bigasterisk.com>
parents:
1489
diff
changeset
|
90 if doLoadTest: |
1493
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
91 # in a subprocess since we don't want this client to be |
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
92 # cooperating with the main event loop and only sending |
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
93 # requests when there's free time |
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
94 def afterWarmup(): |
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
95 log.info('running collector_loadtest') |
1858 | 96 d = utils.getProcessValue('bin/python', |
97 ['bin/collector_loadtest.py']) | |
98 | |
1493
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
99 def done(*a): |
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
100 log.info('loadtest done') |
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
101 reactor.stop() |
1858 | 102 |
1493
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
103 d.addCallback(done) |
1858 | 104 |
1493
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
105 reactor.callLater(2, afterWarmup) |
1858 | 106 |
107 | |
1307
8863b4485fd4
collector uses rdfdb
Drew Perttula <drewp@bigasterisk.com>
parents:
1289
diff
changeset
|
108 def main(): |
1372
f427801da9f6
collector properly merges repeated attr settings in the same message
Drew Perttula <drewp@bigasterisk.com>
parents:
1307
diff
changeset
|
109 parser = optparse.OptionParser() |
1858 | 110 parser.add_option("-v", |
111 "--verbose", | |
112 action="store_true", | |
1372
f427801da9f6
collector properly merges repeated attr settings in the same message
Drew Perttula <drewp@bigasterisk.com>
parents:
1307
diff
changeset
|
113 help="logging.DEBUG") |
1884
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
114 parser.add_option("--logdmx", action="store_true", help="log all dmx sends") |
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
115 |
1858 | 116 parser.add_option("--loadtest", |
117 action="store_true", | |
1490
649d482737e0
start of a collector --loadtest mode
Drew Perttula <drewp@bigasterisk.com>
parents:
1489
diff
changeset
|
118 help="call myself with some synthetic load then exit") |
1372
f427801da9f6
collector properly merges repeated attr settings in the same message
Drew Perttula <drewp@bigasterisk.com>
parents:
1307
diff
changeset
|
119 (options, args) = parser.parse_args() |
f427801da9f6
collector properly merges repeated attr settings in the same message
Drew Perttula <drewp@bigasterisk.com>
parents:
1307
diff
changeset
|
120 log.setLevel(logging.DEBUG if options.verbose else logging.INFO) |
1884
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
121 logging.getLogger('output').setLevel(logging.DEBUG) |
1307
8863b4485fd4
collector uses rdfdb
Drew Perttula <drewp@bigasterisk.com>
parents:
1289
diff
changeset
|
122 |
1884
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
123 logging.getLogger('output.allDmx').setLevel( |
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
124 logging.DEBUG if options.logdmx else logging.INFO) |
1596
7d5d6e7bc526
collector web view speedups- don't json encode all devs all the time
Drew Perttula <drewp@bigasterisk.com>
parents:
1595
diff
changeset
|
125 logging.getLogger('colormath').setLevel(logging.INFO) |
1858 | 126 |
1307
8863b4485fd4
collector uses rdfdb
Drew Perttula <drewp@bigasterisk.com>
parents:
1289
diff
changeset
|
127 graph = SyncedGraph(networking.rdfdb.url, "collector") |
8863b4485fd4
collector uses rdfdb
Drew Perttula <drewp@bigasterisk.com>
parents:
1289
diff
changeset
|
128 |
1858 | 129 graph.initiallySynced.addCallback(lambda _: launch(graph, options.loadtest) |
130 ).addErrback(lambda e: reactor.crash()) | |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
131 reactor.run() |
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
132 |
1858 | 133 |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
134 if __name__ == '__main__': |
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
135 main() |