annotate service/piNode/piNode.py @ 1745:d90cb7c06f15

try to crash if mqtt doesn't connect
author drewp@bigasterisk.com
date Thu, 09 Nov 2023 17:21:59 -0800
parents 9d074317e16a
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
636
7c04b4f675ec piNode to py3
drewp@bigasterisk.com
parents: 628
diff changeset
1 import logging, socket, json, time, pkg_resources
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
2 import cyclone.web
226
1e8ef0daf0f8 piNode uses new graph SSE code
drewp@bigasterisk.com
parents: 225
diff changeset
3 from rdflib import Namespace, URIRef, Literal, Graph, RDF, ConjunctiveGraph
184
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
4 from rdflib.parser import StringInputSource
636
7c04b4f675ec piNode to py3
drewp@bigasterisk.com
parents: 628
diff changeset
5 from twisted.internet import reactor
7c04b4f675ec piNode to py3
drewp@bigasterisk.com
parents: 628
diff changeset
6 from twisted.internet.defer import inlineCallbacks, maybeDeferred, returnValue
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
7 from twisted.internet.threads import deferToThread
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
8 from docopt import docopt
636
7c04b4f675ec piNode to py3
drewp@bigasterisk.com
parents: 628
diff changeset
9 import etcd3
466
1122016d16eb stats handler, events output fix, build updaets
drewp@bigasterisk.com
parents: 408
diff changeset
10 from greplin import scales
1122016d16eb stats handler, events output fix, build updaets
drewp@bigasterisk.com
parents: 408
diff changeset
11 from greplin.scales.cyclonehandler import StatsHandler
1681
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
12 import os
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
13 #os.environ['PIGPIO_ADDR'] = 'pigpio' # (aka the docker host)
636
7c04b4f675ec piNode to py3
drewp@bigasterisk.com
parents: 628
diff changeset
14 import pigpio
7c04b4f675ec piNode to py3
drewp@bigasterisk.com
parents: 628
diff changeset
15 import treq
331
a94f2a522d41 build and import updates for rdfdb, etc
drewp@bigasterisk.com
parents: 325
diff changeset
16
226
1e8ef0daf0f8 piNode uses new graph SSE code
drewp@bigasterisk.com
parents: 225
diff changeset
17 from patchablegraph import PatchableGraph, CycloneGraphHandler, CycloneGraphEventsHandler
589
f397ec8bd13d pinode devs can now poll in parallel (within one poll step). doesn't help much.
drewp@bigasterisk.com
parents: 553
diff changeset
18 from cycloneerr import PrettyErrorHandler
640
23ab244aa649 move bnode id optimization to its own file. more logging cleanups
drewp@bigasterisk.com
parents: 639
diff changeset
19 from standardservice.logsetup import log, verboseLogging
338
f64e20d3407e RgbPixelsAnimation and docker build updates
drewp@bigasterisk.com
parents: 331
diff changeset
20 from rdfdb.rdflibpatch import inContext
f64e20d3407e RgbPixelsAnimation and docker build updates
drewp@bigasterisk.com
parents: 331
diff changeset
21 from rdfdb.patch import Patch
640
23ab244aa649 move bnode id optimization to its own file. more logging cleanups
drewp@bigasterisk.com
parents: 639
diff changeset
22 from rdflib_pi_opt import patchRandid
23ab244aa649 move bnode id optimization to its own file. more logging cleanups
drewp@bigasterisk.com
parents: 639
diff changeset
23 from export_to_influxdb import InfluxExporter
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
24
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
25 import devices
233
4ebb5cc30002 server/browser graph sync. cut dependency on the WS version. merge some changes between arduino/pi code.
drewp@bigasterisk.com
parents: 230
diff changeset
26
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
27 ROOM = Namespace('http://projects.bigasterisk.com/room/')
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
28 HOST = Namespace('http://bigasterisk.com/ruler/host/')
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
29
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
30 hostname = socket.gethostname()
233
4ebb5cc30002 server/browser graph sync. cut dependency on the WS version. merge some changes between arduino/pi code.
drewp@bigasterisk.com
parents: 230
diff changeset
31 CTX = ROOM['pi/%s' % hostname]
4ebb5cc30002 server/browser graph sync. cut dependency on the WS version. merge some changes between arduino/pi code.
drewp@bigasterisk.com
parents: 230
diff changeset
32
466
1122016d16eb stats handler, events output fix, build updaets
drewp@bigasterisk.com
parents: 408
diff changeset
33 STATS = scales.collection('/root',
552
98384c3ccd33 update build and deps to use invoke and to use new lib layout, plus more stats collection
drewp@bigasterisk.com
parents: 466
diff changeset
34 scales.PmfStat('configReread'),
98384c3ccd33 update build and deps to use invoke and to use new lib layout, plus more stats collection
drewp@bigasterisk.com
parents: 466
diff changeset
35 scales.IntStat('pollException'),
625
69a84b3d1dfa frontdoor configs; move device class timing to greplin stats
drewp@bigasterisk.com
parents: 624
diff changeset
36 scales.PmfStat('pollAll'),
552
98384c3ccd33 update build and deps to use invoke and to use new lib layout, plus more stats collection
drewp@bigasterisk.com
parents: 466
diff changeset
37 scales.PmfStat('sendOneshot'),
98384c3ccd33 update build and deps to use invoke and to use new lib layout, plus more stats collection
drewp@bigasterisk.com
parents: 466
diff changeset
38 scales.PmfStat('outputStatements'),
1681
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
39 scales.IntStat('oneshotSuccess'),
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
40 scales.IntStat('oneshotFail'),
466
1122016d16eb stats handler, events output fix, build updaets
drewp@bigasterisk.com
parents: 408
diff changeset
41 )
258
0c4ec87d4498 try a speedup for raspi
drewp@bigasterisk.com
parents: 251
diff changeset
42
183
634d6e477953 get piNode working, for motionsensor at least
drewp@bigasterisk.com
parents: 182
diff changeset
43 class Config(object):
1681
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
44 def __init__(self, masterGraph):
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
45 log.info('connect to etcd-homeauto')
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
46 self.etcd = etcd3.client(host='etcd-homeauto', port=9022)
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
47 log.info('version %r', self.etcd.status().version)
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
48
408
0787cd64ecf8 cmdline flag on piNode to pick hub host
drewp@bigasterisk.com
parents: 382
diff changeset
49
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
50 self.masterGraph = masterGraph
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
51 self.configGraph = ConjunctiveGraph()
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
52 self.boards = []
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
53 self.etcPrefix = 'pi/'
628
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
54 self.rereadLater = None
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
55
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
56 self.reread()
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
57
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
58 deferToThread(self.watchEtcd)
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
59
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
60 def watchEtcd(self):
408
0787cd64ecf8 cmdline flag on piNode to pick hub host
drewp@bigasterisk.com
parents: 382
diff changeset
61 events, cancel = self.etcd.watch_prefix(self.etcPrefix)
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
62 reactor.addSystemEventTrigger('before', 'shutdown', cancel)
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
63 for ev in events:
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
64 log.info('%s changed', ev.key)
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
65 reactor.callFromThread(self.configChanged)
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
66
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
67 def configChanged(self):
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
68 self.cancelRead()
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
69 self.rereadLater = reactor.callLater(.1, self.reread)
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
70
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
71 def cancelRead(self):
628
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
72 if self.rereadLater:
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
73 self.rereadLater.cancel()
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
74 self.rereadLater = None
552
98384c3ccd33 update build and deps to use invoke and to use new lib layout, plus more stats collection
drewp@bigasterisk.com
parents: 466
diff changeset
75
98384c3ccd33 update build and deps to use invoke and to use new lib layout, plus more stats collection
drewp@bigasterisk.com
parents: 466
diff changeset
76 @STATS.configReread.time()
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
77 def reread(self):
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
78 self.rereadLater = None
183
634d6e477953 get piNode working, for motionsensor at least
drewp@bigasterisk.com
parents: 182
diff changeset
79 log.info('read config')
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
80 self.configGraph = ConjunctiveGraph()
408
0787cd64ecf8 cmdline flag on piNode to pick hub host
drewp@bigasterisk.com
parents: 382
diff changeset
81 for v, md in self.etcd.get_prefix(self.etcPrefix):
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
82 log.info(' read file %r', md.key)
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
83 self.configGraph.parse(StringInputSource(v), format='n3')
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
84 self.configGraph.bind('', ROOM)
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
85 self.configGraph.bind('rdf', RDF)
233
4ebb5cc30002 server/browser graph sync. cut dependency on the WS version. merge some changes between arduino/pi code.
drewp@bigasterisk.com
parents: 230
diff changeset
86 # config graph is too noisy; maybe make it a separate resource
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
87 #masterGraph.patch(Patch(addGraph=self.configGraph))
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
88 self.setupBoards()
624
1249dcc657b8 whitespace
drewp@bigasterisk.com
parents: 591
diff changeset
89
1249dcc657b8 whitespace
drewp@bigasterisk.com
parents: 591
diff changeset
90 def setupBoards(self):
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
91 thisHost = Literal(hostname)
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
92 for row in self.configGraph.query(
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
93 'SELECT ?board WHERE { ?board a :PiBoard; :hostname ?h }',
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
94 initBindings=dict(h=thisHost)):
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
95 thisBoard = row.board
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
96 break
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
97 else:
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
98 log.warn("config had no board for :hostname %s. Waiting for config update." %
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
99 thisHost)
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
100 self.boards = []
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
101 return
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
102
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
103 log.info("found config for board %r" % thisBoard)
1681
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
104 self.boards = [Board(self.configGraph, self.masterGraph, thisBoard)]
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
105
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
106
628
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
107 class DeviceRunner(object):
642
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
108 def __init__(self, dev, masterGraph, sendOneshot, influx):
628
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
109 self.dev = dev
642
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
110 self.masterGraph = masterGraph
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
111 self.sendOneshot = sendOneshot
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
112 self.influx = influx
628
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
113 self.period = getattr(self.dev, 'pollPeriod', .05)
642
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
114 self.latestStatementsFromInputs = set()
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
115 self.lastPollTime = None
628
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
116
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
117 reactor.callLater(0, self.poll)
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
118
642
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
119 def syncMasterGraphToHostStatements(self):
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
120 hostStmtCtx = URIRef(self.dev.uri + '/host')
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
121 newQuads = inContext(self.dev.hostStatements(), hostStmtCtx)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
122 p = self.masterGraph.patchSubgraph(hostStmtCtx, newQuads)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
123 if p:
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
124 log.debug("patch master with these host stmts %s", p)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
125
628
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
126 @inlineCallbacks
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
127 def poll(self):
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
128 now = time.time()
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
129 try:
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
130 with self.dev.stats.poll.time():
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
131 new = yield maybeDeferred(self.dev.poll)
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
132 finally:
642
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
133 reactor.callLater(max(0, self.period - (time.time() - now)), self.poll)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
134
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
135 if isinstance(new, dict): # new style
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
136 oneshot = set(new['oneshot'])
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
137 new = set(new['latest'])
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
138 else:
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
139 oneshot = set()
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
140 new = set(new)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
141
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
142 prev = self.latestStatementsFromInputs
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
143 # it's important that quads from different devices
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
144 # don't clash, since that can lead to inconsistent
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
145 # patches (e.g.
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
146 # dev1 changes value from 1 to 2;
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
147 # dev2 changes value from 2 to 3;
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
148 # dev1 changes from 2 to 4 but this patch will
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
149 # fail since the '2' statement is gone)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
150 self.masterGraph.patch(Patch.fromDiff(inContext(prev, self.dev.uri),
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
151 inContext(new, self.dev.uri)))
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
152 self.latestStatementsFromInputs = new
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
153
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
154 self.syncMasterGraphToHostStatements() # needed?
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
155
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
156 if oneshot:
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
157 self.sendOneshot(oneshot)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
158 self.lastPollTime = now
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
159
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
160 if self.latestStatementsFromInputs:
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
161 self.influx.exportToInflux(set.union(set(self.latestStatementsFromInputs)))
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
162
628
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
163 returnValue(new)
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
164
642
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
165 def filterIncomingStatements(self, stmts):
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
166 wanted = set()
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
167 unwanted = set(stmts)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
168 for pat in self.dev.outputPatterns():
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
169 if [term is None for term in pat] != [False, False, True]:
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
170 raise NotImplementedError
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
171 for stmt in stmts:
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
172 if stmt[:2] == pat[:2]:
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
173 wanted.add(stmt)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
174 unwanted.discard(stmt)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
175 return wanted, unwanted
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
176
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
177 def onPutStatements(self, stmts):
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
178 log.info("output goes to action handler for %s" % self.dev.uri)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
179 with self.dev.stats.output.time():
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
180 self.dev.sendOutput(stmts)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
181 self.syncMasterGraphToHostStatements()
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
182
1681
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
183 def sendOneshot(oneshot):
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
184 body = (' '.join('%s %s %s .' % (s.n3(), p.n3(), o.n3())
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
185 for s,p,o in oneshot)).encode('utf8')
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
186 url = 'http://reasoning:9071/oneShot'
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
187 log.debug('post to %r', url)
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
188 d = treq.post(
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
189 url=url.encode('ascii'),
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
190 headers={b'Content-Type': [b'text/n3']},
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
191 data=body,
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
192 timeout=5)
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
193
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
194 def ok(k):
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
195 log.debug('sendOneshot to %r success', url)
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
196 STATS.oneshotSuccess += 1
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
197 def err(e):
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
198 log.info('oneshot post to %r failed: %s',
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
199 url, e.getErrorMessage())
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
200 STATS.oneshotFail += 1
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
201 d.addCallbacks(ok, err)
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
202
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
203 class Board(object):
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
204 """similar to arduinoNode.Board but without the communications stuff"""
1681
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
205 def __init__(self, graph, masterGraph, uri):
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
206 self.graph, self.uri = graph, uri
226
1e8ef0daf0f8 piNode uses new graph SSE code
drewp@bigasterisk.com
parents: 225
diff changeset
207 self.masterGraph = masterGraph
636
7c04b4f675ec piNode to py3
drewp@bigasterisk.com
parents: 628
diff changeset
208
349
88bd46f4e28c more robust about not accumulating old stmts
drewp@bigasterisk.com
parents: 347
diff changeset
209 self.masterGraph.setToGraph(self.staticStmts())
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
210 self.pi = pigpio.pi()
642
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
211
293
fc0e42933baa save data to influxdb, not graphite
drewp@bigasterisk.com
parents: 271
diff changeset
212 self._influx = InfluxExporter(self.graph)
642
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
213 self._runners = [DeviceRunner(d, self.masterGraph, self.sendOneshot, self._influx)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
214 for d in devices.makeDevices(graph, self.uri, self.pi)]
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
215 log.debug('found %s devices', len(self._runners))
552
98384c3ccd33 update build and deps to use invoke and to use new lib layout, plus more stats collection
drewp@bigasterisk.com
parents: 466
diff changeset
216
98384c3ccd33 update build and deps to use invoke and to use new lib layout, plus more stats collection
drewp@bigasterisk.com
parents: 466
diff changeset
217 @STATS.sendOneshot.time()
642
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
218 def sendOneshot(self, oneshot):
1681
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
219 sendOneshot(oneshot)
251
254df9f881a6 start sending oneshot events from some devices
drewp@bigasterisk.com
parents: 243
diff changeset
220
552
98384c3ccd33 update build and deps to use invoke and to use new lib layout, plus more stats collection
drewp@bigasterisk.com
parents: 466
diff changeset
221 @STATS.outputStatements.time()
642
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
222 def outputStatements(self, stmts: set):
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
223 if not stmts:
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
224 return
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
225 for devRunner in self._runners:
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
226 wanted, unwanted = devRunner.filterIncomingStatements(stmts)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
227 log.info(f'\ndev {devRunner.dev.uri}:n wanted {wanted}. unwanted {unwanted}')
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
228 if len(wanted) == len(stmts):
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
229 devRunner.onPutStatements(stmts)
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
230 break
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
231 elif len(unwanted) == len(stmts):
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
232 continue
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
233 else:
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
234 raise NotImplementedError(f'dev {devRunner.dev.uri} wanted only {wanted}')
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
235 else:
233
4ebb5cc30002 server/browser graph sync. cut dependency on the WS version. merge some changes between arduino/pi code.
drewp@bigasterisk.com
parents: 230
diff changeset
236 log.info("Board %s doesn't care about these statements:", self.uri)
1681
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
237 for s in unwanted:
233
4ebb5cc30002 server/browser graph sync. cut dependency on the WS version. merge some changes between arduino/pi code.
drewp@bigasterisk.com
parents: 230
diff changeset
238 log.warn("%r", s)
4ebb5cc30002 server/browser graph sync. cut dependency on the WS version. merge some changes between arduino/pi code.
drewp@bigasterisk.com
parents: 230
diff changeset
239
226
1e8ef0daf0f8 piNode uses new graph SSE code
drewp@bigasterisk.com
parents: 225
diff changeset
240 def staticStmts(self):
233
4ebb5cc30002 server/browser graph sync. cut dependency on the WS version. merge some changes between arduino/pi code.
drewp@bigasterisk.com
parents: 230
diff changeset
241 return [(HOST[hostname], ROOM['connectedTo'], self.uri, CTX)]
4ebb5cc30002 server/browser graph sync. cut dependency on the WS version. merge some changes between arduino/pi code.
drewp@bigasterisk.com
parents: 230
diff changeset
242
183
634d6e477953 get piNode working, for motionsensor at least
drewp@bigasterisk.com
parents: 182
diff changeset
243 def description(self):
634d6e477953 get piNode working, for motionsensor at least
drewp@bigasterisk.com
parents: 182
diff changeset
244 """for web page"""
634d6e477953 get piNode working, for motionsensor at least
drewp@bigasterisk.com
parents: 182
diff changeset
245 return {
634d6e477953 get piNode working, for motionsensor at least
drewp@bigasterisk.com
parents: 182
diff changeset
246 'uri': self.uri,
642
0a39cb133ce5 remove pytz and time table
drewp@bigasterisk.com
parents: 640
diff changeset
247 'devices': [d.dev.description() for d in self._runners],
183
634d6e477953 get piNode working, for motionsensor at least
drewp@bigasterisk.com
parents: 182
diff changeset
248 'graph': 'http://sticker:9059/graph', #todo
628
accbbd521ecd wip for pytype support and separate device run loops on piNode
drewp@bigasterisk.com
parents: 627
diff changeset
249 }
624
1249dcc657b8 whitespace
drewp@bigasterisk.com
parents: 591
diff changeset
250
184
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
251 def rdfGraphBody(body, headers):
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
252 g = Graph()
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
253 g.parse(StringInputSource(body), format='nt')
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
254 return g
183
634d6e477953 get piNode working, for motionsensor at least
drewp@bigasterisk.com
parents: 182
diff changeset
255
591
1398f4ec01a5 pinode use prettyerrorhandler
drewp@bigasterisk.com
parents: 589
diff changeset
256 class OutputPage(PrettyErrorHandler, cyclone.web.RequestHandler):
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
257 def put(self):
184
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
258 arg = self.request.arguments
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
259 if arg.get('s') and arg.get('p'):
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
260 subj = URIRef(arg['s'][-1])
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
261 pred = URIRef(arg['p'][-1])
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
262 turtleLiteral = self.request.body
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
263 try:
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
264 obj = Literal(float(turtleLiteral))
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
265 except ValueError:
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
266 obj = Literal(turtleLiteral)
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
267 stmt = (subj, pred, obj)
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
268 else:
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
269 g = rdfGraphBody(self.request.body, self.request.headers)
e052416a2290 piNode allow nt graphs as the body of a PUT /output
drewp@bigasterisk.com
parents: 183
diff changeset
270 assert len(g) == 1, len(g)
636
7c04b4f675ec piNode to py3
drewp@bigasterisk.com
parents: 628
diff changeset
271 stmt = next(g.triples((None, None, None)))
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
272
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
273 for b in self.settings.config.boards:
636
7c04b4f675ec piNode to py3
drewp@bigasterisk.com
parents: 628
diff changeset
274 b.outputStatements({stmt})
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
275
591
1398f4ec01a5 pinode use prettyerrorhandler
drewp@bigasterisk.com
parents: 589
diff changeset
276 class Boards(PrettyErrorHandler, cyclone.web.RequestHandler):
183
634d6e477953 get piNode working, for motionsensor at least
drewp@bigasterisk.com
parents: 182
diff changeset
277 def get(self):
634d6e477953 get piNode working, for motionsensor at least
drewp@bigasterisk.com
parents: 182
diff changeset
278 self.set_header('Content-type', 'application/json')
634d6e477953 get piNode working, for motionsensor at least
drewp@bigasterisk.com
parents: 182
diff changeset
279 self.write(json.dumps({
233
4ebb5cc30002 server/browser graph sync. cut dependency on the WS version. merge some changes between arduino/pi code.
drewp@bigasterisk.com
parents: 230
diff changeset
280 'host': hostname,
347
97f5cbcac00b pi read config over etcd
drewp@bigasterisk.com
parents: 338
diff changeset
281 'boards': [b.description() for b in self.settings.config.boards]
183
634d6e477953 get piNode working, for motionsensor at least
drewp@bigasterisk.com
parents: 182
diff changeset
282 }, indent=2))
624
1249dcc657b8 whitespace
drewp@bigasterisk.com
parents: 591
diff changeset
283
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
284 def main():
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
285 arg = docopt("""
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
286 Usage: piNode.py [options]
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
287
408
0787cd64ecf8 cmdline flag on piNode to pick hub host
drewp@bigasterisk.com
parents: 382
diff changeset
288 -v Verbose
0787cd64ecf8 cmdline flag on piNode to pick hub host
drewp@bigasterisk.com
parents: 382
diff changeset
289 --ow Just report onewire device URIs and readings, then exit.
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
290 """)
640
23ab244aa649 move bnode id optimization to its own file. more logging cleanups
drewp@bigasterisk.com
parents: 639
diff changeset
291 verboseLogging(arg['-v'])
233
4ebb5cc30002 server/browser graph sync. cut dependency on the WS version. merge some changes between arduino/pi code.
drewp@bigasterisk.com
parents: 230
diff changeset
292
408
0787cd64ecf8 cmdline flag on piNode to pick hub host
drewp@bigasterisk.com
parents: 382
diff changeset
293 if arg['--ow']:
0787cd64ecf8 cmdline flag on piNode to pick hub host
drewp@bigasterisk.com
parents: 382
diff changeset
294 log.setLevel(logging.INFO)
0787cd64ecf8 cmdline flag on piNode to pick hub host
drewp@bigasterisk.com
parents: 382
diff changeset
295 for stmt in devices.OneWire().poll():
636
7c04b4f675ec piNode to py3
drewp@bigasterisk.com
parents: 628
diff changeset
296 print(stmt)
408
0787cd64ecf8 cmdline flag on piNode to pick hub host
drewp@bigasterisk.com
parents: 382
diff changeset
297 return
624
1249dcc657b8 whitespace
drewp@bigasterisk.com
parents: 591
diff changeset
298
640
23ab244aa649 move bnode id optimization to its own file. more logging cleanups
drewp@bigasterisk.com
parents: 639
diff changeset
299 patchRandid()
23ab244aa649 move bnode id optimization to its own file. more logging cleanups
drewp@bigasterisk.com
parents: 639
diff changeset
300
226
1e8ef0daf0f8 piNode uses new graph SSE code
drewp@bigasterisk.com
parents: 225
diff changeset
301 masterGraph = PatchableGraph()
1681
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
302 config = Config(masterGraph)
624
1249dcc657b8 whitespace
drewp@bigasterisk.com
parents: 591
diff changeset
303
552
98384c3ccd33 update build and deps to use invoke and to use new lib layout, plus more stats collection
drewp@bigasterisk.com
parents: 466
diff changeset
304 static = pkg_resources.resource_filename('homeauto_anynode', 'static/')
98384c3ccd33 update build and deps to use invoke and to use new lib layout, plus more stats collection
drewp@bigasterisk.com
parents: 466
diff changeset
305
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
306 reactor.listenTCP(9059, cyclone.web.Application([
627
c3f0a98fa65d lib upgrades; fix a static http server path
drewp@bigasterisk.com
parents: 625
diff changeset
307 (r"/(|output-widgets.html)", cyclone.web.StaticFileHandler, {
552
98384c3ccd33 update build and deps to use invoke and to use new lib layout, plus more stats collection
drewp@bigasterisk.com
parents: 466
diff changeset
308 "path": static, "default_filename": "index.html"}),
553
36e4304762ae fix /static path dir
drewp@bigasterisk.com
parents: 552
diff changeset
309 (r'/static/(.*)', cyclone.web.StaticFileHandler, {"path": static}),
466
1122016d16eb stats handler, events output fix, build updaets
drewp@bigasterisk.com
parents: 408
diff changeset
310 (r'/stats/(.*)', StatsHandler, {'serverName': 'piNode'}),
233
4ebb5cc30002 server/browser graph sync. cut dependency on the WS version. merge some changes between arduino/pi code.
drewp@bigasterisk.com
parents: 230
diff changeset
311 (r'/boards', Boards),
226
1e8ef0daf0f8 piNode uses new graph SSE code
drewp@bigasterisk.com
parents: 225
diff changeset
312 (r"/graph", CycloneGraphHandler, {'masterGraph': masterGraph}),
1e8ef0daf0f8 piNode uses new graph SSE code
drewp@bigasterisk.com
parents: 225
diff changeset
313 (r"/graph/events", CycloneGraphEventsHandler, {'masterGraph': masterGraph}),
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
314 (r'/output', OutputPage),
466
1122016d16eb stats handler, events output fix, build updaets
drewp@bigasterisk.com
parents: 408
diff changeset
315 ], config=config, debug=arg['-v']), interface='::')
258
0c4ec87d4498 try a speedup for raspi
drewp@bigasterisk.com
parents: 251
diff changeset
316 log.warn('serving on 9059')
182
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
317 reactor.run()
9fff29ebca71 start pinode
drewp@bigasterisk.com
parents:
diff changeset
318
1681
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
319 if __name__ == '__main__':
9d074317e16a checkpoint service/piNode
drewp@bigasterisk.com
parents: 642
diff changeset
320 main()