Mercurial > code > home > repos > light9
annotate bin/collector @ 1998:3d28b1a54a29
reformat
Ignore-this: 28e00687983432a552e465291ef9b4b4
author | drewp@bigasterisk.com |
---|---|
date | Sun, 09 Jun 2019 21:56:03 +0000 |
parents | 5c8b6cd330da |
children | f9faa2ec824f |
rev | line source |
---|---|
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
1 #!bin/python |
1590 | 2 """ |
3 Collector receives device attrs from multiple senders, combines | |
4 them, and sends output attrs to hardware. The combining part has | |
5 custom code for some attributes. | |
6 | |
7 Input can be over http or zmq. | |
8 """ | |
9 | |
1697
5c04a54df635
fix startup (this might be breaking mypy)
Drew Perttula <drewp@bigasterisk.com>
parents:
1692
diff
changeset
|
10 from run_local import log |
5c04a54df635
fix startup (this might be breaking mypy)
Drew Perttula <drewp@bigasterisk.com>
parents:
1692
diff
changeset
|
11 |
1493
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
12 from twisted.internet import reactor, utils |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
13 import json |
1289
5a4e74f1e36a
Fixed client session clearing bugs.
Drew Perttula <drewp@bigasterisk.com>
parents:
1288
diff
changeset
|
14 import logging |
1372
f427801da9f6
collector properly merges repeated attr settings in the same message
Drew Perttula <drewp@bigasterisk.com>
parents:
1307
diff
changeset
|
15 import optparse |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
16 import traceback |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
17 import cyclone.web, cyclone.websocket |
1289
5a4e74f1e36a
Fixed client session clearing bugs.
Drew Perttula <drewp@bigasterisk.com>
parents:
1288
diff
changeset
|
18 from greplin import scales |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
19 |
1861 | 20 from cycloneerr import PrettyErrorHandler |
1866
3c523c71da29
pyflakes cleanups and some refactors
Drew Perttula <drewp@bigasterisk.com>
parents:
1861
diff
changeset
|
21 from light9 import networking |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
22 from light9.collector.collector import Collector |
1866
3c523c71da29
pyflakes cleanups and some refactors
Drew Perttula <drewp@bigasterisk.com>
parents:
1861
diff
changeset
|
23 from light9.collector.weblisteners import WebListeners |
1926
1a7e5b07bf17
use my greplin fork's stats/ code instead of an old local one
Drew Perttula <drewp@bigasterisk.com>
parents:
1922
diff
changeset
|
24 from greplin.scales.cyclonehandler import StatsHandler |
1866
3c523c71da29
pyflakes cleanups and some refactors
Drew Perttula <drewp@bigasterisk.com>
parents:
1861
diff
changeset
|
25 from light9.namespaces import L9 |
3c523c71da29
pyflakes cleanups and some refactors
Drew Perttula <drewp@bigasterisk.com>
parents:
1861
diff
changeset
|
26 from light9.zmqtransport import parseJsonMessage, startZmq |
3c523c71da29
pyflakes cleanups and some refactors
Drew Perttula <drewp@bigasterisk.com>
parents:
1861
diff
changeset
|
27 from rdfdb.syncedgraph import SyncedGraph |
1957
a745bee5c419
new process stats visualizers
Drew Perttula <drewp@bigasterisk.com>
parents:
1937
diff
changeset
|
28 from standardservice.scalessetup import gatherProcessStats |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
29 |
1980 | 30 from light9.collector.output import ArtnetDmx, DummyOutput # noqa |
1858 | 31 |
32 | |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
33 class Updates(cyclone.websocket.WebSocketHandler): |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
34 |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
35 def connectionMade(self, *args, **kwargs): |
1543
c8cffe82b537
collector gui updates
Drew Perttula <drewp@bigasterisk.com>
parents:
1541
diff
changeset
|
36 log.info('socket connect %s', self) |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
37 self.settings.listeners.addClient(self) |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
38 |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
39 def connectionLost(self, reason): |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
40 self.settings.listeners.delClient(self) |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
41 |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
42 def messageReceived(self, message): |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
43 json.loads(message) |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
44 |
1858 | 45 |
1957
a745bee5c419
new process stats visualizers
Drew Perttula <drewp@bigasterisk.com>
parents:
1937
diff
changeset
|
46 gatherProcessStats() |
1927 | 47 stats = scales.collection( |
48 '/webServer', | |
1937
f29e26811206
pmfstats now update at 1hz. some ui improvements to stats lines.
Drew Perttula <drewp@bigasterisk.com>
parents:
1927
diff
changeset
|
49 scales.PmfStat('setAttr', recalcPeriod=1), |
1927 | 50 scales.RecentFpsStat('setAttrFps'), |
1922
11e2f63bb2f2
more stats to measure sequencer framerate better
Drew Perttula <drewp@bigasterisk.com>
parents:
1916
diff
changeset
|
51 ) |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
52 |
1858 | 53 |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
54 class Attrs(PrettyErrorHandler, cyclone.web.RequestHandler): |
1858 | 55 |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
56 def put(self): |
1922
11e2f63bb2f2
more stats to measure sequencer framerate better
Drew Perttula <drewp@bigasterisk.com>
parents:
1916
diff
changeset
|
57 stats.setAttrFps.mark() |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
58 with stats.setAttr.time(): |
1858 | 59 client, clientSession, settings, sendTime = parseJsonMessage( |
60 self.request.body) | |
61 self.settings.collector.setAttrs(client, clientSession, settings, | |
62 sendTime) | |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
63 self.set_status(202) |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
64 |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
65 |
1490
649d482737e0
start of a collector --loadtest mode
Drew Perttula <drewp@bigasterisk.com>
parents:
1489
diff
changeset
|
66 def launch(graph, doLoadTest=False): |
1530 | 67 try: |
68 # todo: drive outputs with config files | |
1980 | 69 rate = 30 |
1530 | 70 outputs = [ |
1998 | 71 ArtnetDmx(L9['output/dmxA/'], |
72 host='127.0.0.1', | |
73 port=6445, | |
74 rate=rate), | |
1971
b26a1e7fcfbe
dmx out: lots of stats and more reconnection attempts after usb errors
drewp@bigasterisk.com
parents:
1957
diff
changeset
|
75 #DummyOutput(L9['output/dmxA/']), |
1530 | 76 ] |
1799
0bb7b9df12e5
collector warnings and errors. the reactor.crash isn't working.
drewp@bigasterisk.com
parents:
1798
diff
changeset
|
77 except Exception: |
0bb7b9df12e5
collector warnings and errors. the reactor.crash isn't working.
drewp@bigasterisk.com
parents:
1798
diff
changeset
|
78 log.error("setting up outputs:") |
1530 | 79 traceback.print_exc() |
80 raise | |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
81 listeners = WebListeners() |
1884
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
82 c: Collector = Collector(graph, outputs, listeners) |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
83 |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
84 startZmq(networking.collectorZmq.port, c) |
1858 | 85 |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
86 reactor.listenTCP(networking.collector.port, |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
87 cyclone.web.Application(handlers=[ |
1858 | 88 (r'/()', cyclone.web.StaticFileHandler, { |
89 "path": "light9/collector/web", | |
90 "default_filename": "index.html" | |
91 }), | |
1541
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
92 (r'/updates', Updates), |
c1bf296b0a74
collector uses cyclone and gets a web ui showing output attrs
Drew Perttula <drewp@bigasterisk.com>
parents:
1530
diff
changeset
|
93 (r'/attrs', Attrs), |
1926
1a7e5b07bf17
use my greplin fork's stats/ code instead of an old local one
Drew Perttula <drewp@bigasterisk.com>
parents:
1922
diff
changeset
|
94 (r'/stats/(.*)', StatsHandler, { |
1a7e5b07bf17
use my greplin fork's stats/ code instead of an old local one
Drew Perttula <drewp@bigasterisk.com>
parents:
1922
diff
changeset
|
95 'serverName': 'collector' |
1a7e5b07bf17
use my greplin fork's stats/ code instead of an old local one
Drew Perttula <drewp@bigasterisk.com>
parents:
1922
diff
changeset
|
96 }), |
1858 | 97 ], |
98 collector=c, | |
99 listeners=listeners), | |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
100 interface='::') |
1289
5a4e74f1e36a
Fixed client session clearing bugs.
Drew Perttula <drewp@bigasterisk.com>
parents:
1288
diff
changeset
|
101 log.info('serving http on %s, zmq on %s', networking.collector.port, |
5a4e74f1e36a
Fixed client session clearing bugs.
Drew Perttula <drewp@bigasterisk.com>
parents:
1288
diff
changeset
|
102 networking.collectorZmq.port) |
1490
649d482737e0
start of a collector --loadtest mode
Drew Perttula <drewp@bigasterisk.com>
parents:
1489
diff
changeset
|
103 if doLoadTest: |
1493
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
104 # in a subprocess since we don't want this client to be |
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
105 # cooperating with the main event loop and only sending |
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
106 # requests when there's free time |
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
107 def afterWarmup(): |
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
108 log.info('running collector_loadtest') |
1858 | 109 d = utils.getProcessValue('bin/python', |
110 ['bin/collector_loadtest.py']) | |
111 | |
1493
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
112 def done(*a): |
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
113 log.info('loadtest done') |
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
114 reactor.stop() |
1858 | 115 |
1493
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
116 d.addCallback(done) |
1858 | 117 |
1493
4294ed82ee16
move collector_loadtest and arrange for collector to be able to run the test itself, but that last part isn't working. you can run collector and collector_loadtest in two shells, though
drewp@bigasterisk.com
parents:
1492
diff
changeset
|
118 reactor.callLater(2, afterWarmup) |
1858 | 119 |
120 | |
1307
8863b4485fd4
collector uses rdfdb
Drew Perttula <drewp@bigasterisk.com>
parents:
1289
diff
changeset
|
121 def main(): |
1372
f427801da9f6
collector properly merges repeated attr settings in the same message
Drew Perttula <drewp@bigasterisk.com>
parents:
1307
diff
changeset
|
122 parser = optparse.OptionParser() |
1858 | 123 parser.add_option("-v", |
124 "--verbose", | |
125 action="store_true", | |
1372
f427801da9f6
collector properly merges repeated attr settings in the same message
Drew Perttula <drewp@bigasterisk.com>
parents:
1307
diff
changeset
|
126 help="logging.DEBUG") |
1884
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
127 parser.add_option("--logdmx", action="store_true", help="log all dmx sends") |
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
128 |
1858 | 129 parser.add_option("--loadtest", |
130 action="store_true", | |
1490
649d482737e0
start of a collector --loadtest mode
Drew Perttula <drewp@bigasterisk.com>
parents:
1489
diff
changeset
|
131 help="call myself with some synthetic load then exit") |
1372
f427801da9f6
collector properly merges repeated attr settings in the same message
Drew Perttula <drewp@bigasterisk.com>
parents:
1307
diff
changeset
|
132 (options, args) = parser.parse_args() |
f427801da9f6
collector properly merges repeated attr settings in the same message
Drew Perttula <drewp@bigasterisk.com>
parents:
1307
diff
changeset
|
133 log.setLevel(logging.DEBUG if options.verbose else logging.INFO) |
1884
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
134 logging.getLogger('output').setLevel(logging.DEBUG) |
1307
8863b4485fd4
collector uses rdfdb
Drew Perttula <drewp@bigasterisk.com>
parents:
1289
diff
changeset
|
135 |
1884
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
136 logging.getLogger('output.allDmx').setLevel( |
5cde72dfdc22
change collector output code to use very specific types. Might fix bugs too.
Drew Perttula <drewp@bigasterisk.com>
parents:
1881
diff
changeset
|
137 logging.DEBUG if options.logdmx else logging.INFO) |
1596
7d5d6e7bc526
collector web view speedups- don't json encode all devs all the time
Drew Perttula <drewp@bigasterisk.com>
parents:
1595
diff
changeset
|
138 logging.getLogger('colormath').setLevel(logging.INFO) |
1858 | 139 |
1307
8863b4485fd4
collector uses rdfdb
Drew Perttula <drewp@bigasterisk.com>
parents:
1289
diff
changeset
|
140 graph = SyncedGraph(networking.rdfdb.url, "collector") |
8863b4485fd4
collector uses rdfdb
Drew Perttula <drewp@bigasterisk.com>
parents:
1289
diff
changeset
|
141 |
1858 | 142 graph.initiallySynced.addCallback(lambda _: launch(graph, options.loadtest) |
143 ).addErrback(lambda e: reactor.crash()) | |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
144 reactor.run() |
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
145 |
1858 | 146 |
1288
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
147 if __name__ == '__main__': |
5e76c8fd8a03
rewrite dmx outputter to a new service
Drew Perttula <drewp@bigasterisk.com>
parents:
diff
changeset
|
148 main() |