Mercurial > code > home > repos > homeauto
annotate service/wifi/wifi.py @ 1728:81aa0873b48d
port to skaffold, starlette, etc
| author | drewp@bigasterisk.com |
|---|---|
| date | Fri, 30 Jun 2023 22:03:55 -0700 |
| parents | f88ff1021ee0 |
| children |
| rev | line source |
|---|---|
| 0 | 1 """ |
| 2 scrape the tomato router status pages to see who's connected to the | |
| 3 wifi access points. Includes leases that aren't currently connected. | |
| 4 | |
| 5 Returns: | |
| 6 json listing (for magma page) | |
| 7 rdf graph (for reasoning) | |
| 8 activity stream, when we start saving history | |
| 9 | |
| 10 Todo: this should be the one polling and writing to mongo, not entrancemusic | |
| 162 | 11 |
| 0 | 12 """ |
| 1679 | 13 import datetime |
| 14 import logging | |
| 15 import time | |
| 16 import traceback | |
| 1728 | 17 from dataclasses import dataclass |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
18 from typing import List |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
19 |
| 1728 | 20 import background_loop |
| 0 | 21 from dateutil import tz |
| 1728 | 22 from patchablegraph import PatchableGraph |
| 23 from patchablegraph.handler import GraphEvents, StaticGraph | |
| 1679 | 24 from prometheus_client import Counter, Gauge, Summary |
| 1728 | 25 from pymongo import DESCENDING |
| 26 from pymongo import MongoClient as Connection | |
| 1679 | 27 from pymongo.collection import Collection |
| 1728 | 28 from rdflib import RDF, ConjunctiveGraph, Literal, Namespace |
| 29 from starlette.applications import Starlette | |
| 30 from starlette.routing import Route | |
| 31 from starlette_exporter import PrometheusMiddleware, handle_metrics | |
|
383
8f5a16a55f64
various docker setups and build fixes
drewp@bigasterisk.com
parents:
340
diff
changeset
|
32 |
| 1679 | 33 from scrape import SeenNode, Wifi |
| 36 | 34 |
| 1728 | 35 logging.basicConfig() |
| 36 log = logging.getLogger() | |
| 37 | |
| 422 | 38 AST = Namespace("http://bigasterisk.com/") |
| 0 | 39 DEV = Namespace("http://projects.bigasterisk.com/device/") |
| 40 ROOM = Namespace("http://projects.bigasterisk.com/room/") | |
| 1 | 41 |
| 1728 | 42 # class Index(PrettyErrorHandler, cyclone.web.RequestHandler): |
| 1679 | 43 |
| 1728 | 44 # def get(self): |
| 45 # age = time.time() - self.settings.poller.lastPollTime | |
| 46 # if age > 10: | |
| 47 # raise ValueError("poll data is stale. age=%s" % age) | |
| 659 | 48 |
| 1728 | 49 # self.set_header("Content-Type", "text/html") |
| 50 # self.write(open("index.html").read()) | |
|
51
d2842eedd56d
rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents:
50
diff
changeset
|
51 |
| 1679 | 52 |
|
175
c81a451f9b26
rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents:
162
diff
changeset
|
53 def whenConnected(mongo, macThatIsNowConnected): |
|
c81a451f9b26
rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents:
162
diff
changeset
|
54 lastArrive = None |
| 1679 | 55 for ev in mongo.find({'address': macThatIsNowConnected.upper()}, sort=[('created', -1)], max_time_ms=5000): |
|
175
c81a451f9b26
rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents:
162
diff
changeset
|
56 if ev['action'] == 'arrive': |
|
c81a451f9b26
rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents:
162
diff
changeset
|
57 lastArrive = ev |
|
c81a451f9b26
rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents:
162
diff
changeset
|
58 if ev['action'] == 'leave': |
|
c81a451f9b26
rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents:
162
diff
changeset
|
59 break |
|
c81a451f9b26
rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents:
162
diff
changeset
|
60 if lastArrive is None: |
|
c81a451f9b26
rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents:
162
diff
changeset
|
61 raise ValueError("no past arrivals") |
| 0 | 62 |
|
175
c81a451f9b26
rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents:
162
diff
changeset
|
63 return lastArrive['created'] |
|
c81a451f9b26
rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents:
162
diff
changeset
|
64 |
| 1679 | 65 |
| 1728 | 66 # class Table(PrettyErrorHandler, cyclone.web.RequestHandler): |
| 659 | 67 |
| 1728 | 68 # def get(self): |
| 1679 | 69 |
| 1728 | 70 # def rowDict(row): |
| 71 # row['cls'] = "signal" if row.get('connected') else "nosignal" | |
| 72 # if 'name' not in row: | |
| 73 # row['name'] = row.get('clientHostname', '-') | |
| 74 # if 'signal' not in row: | |
| 75 # row['signal'] = 'yes' if row.get('connected') else 'no' | |
|
62
f8cc3d1baa85
redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents:
52
diff
changeset
|
76 |
| 1728 | 77 # try: |
| 78 # conn = whenConnected(self.settings.mongo, row.get('mac', '??')) | |
| 79 # row['connectedAgo'] = connectedAgoString(conn) | |
| 80 # except ValueError: | |
| 81 # row['connectedAgo'] = 'yes' if row.get('connected') else '' | |
| 82 # row['router'] = row.get('ssid', '') | |
| 83 # return row | |
| 0 | 84 |
| 1728 | 85 # self.set_header("Content-Type", "application/xhtml+xml") |
| 86 # self.write( | |
| 87 # pystache.render( | |
| 88 # open("table.mustache").read(), | |
| 89 # dict(rows=sorted(map(rowDict, self.settings.poller.lastAddrs), | |
| 90 # key=lambda a: (not a.get('connected'), a.get('name')))))) | |
|
51
d2842eedd56d
rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents:
50
diff
changeset
|
91 |
| 1728 | 92 # class Json(PrettyErrorHandler, cyclone.web.RequestHandler): |
| 1679 | 93 |
| 1728 | 94 # def get(self): |
| 95 # self.set_header("Content-Type", "application/json") | |
| 96 # age = time.time() - self.settings.poller.lastPollTime | |
| 97 # if age > 10: | |
| 98 # raise ValueError("poll data is stale. age=%s" % age) | |
| 99 # self.write(json.dumps({"wifi": self.settings.poller.lastAddrs, "dataAge": age})) | |
| 1679 | 100 |
| 101 POLL = Summary('poll', 'Time in HTTP poll requests') | |
| 102 POLL_SUCCESSES = Counter('poll_successes', 'poll success count') | |
| 103 POLL_ERRORS = Counter('poll_errors', 'poll error count') | |
| 104 CURRENTLY_ON_WIFI = Gauge('currently_on_wifi', 'current nodes known to wifi router (some may be wired)') | |
| 105 MAC_ON_WIFI = Gauge('connected', 'mac addr is currently connected', ['mac']) | |
| 106 | |
| 0 | 107 |
| 1728 | 108 @dataclass |
| 109 class Poller: | |
| 110 wifi: Wifi | |
| 111 mongo: Collection | |
| 112 masterGraph: PatchableGraph | |
| 1679 | 113 |
| 1728 | 114 def __post_init__(self): |
| 1679 | 115 self.lastAddrs = [] # List[SeenNode] |
| 0 | 116 self.lastWithSignal = [] |
| 117 self.lastPollTime = 0 | |
| 118 | |
| 1728 | 119 async def poll(self, first_run): |
| 120 with POLL.time(): | |
| 121 try: | |
| 122 newAddrs = await self.wifi.getPresentMacAddrs() | |
| 123 self.onNodes(newAddrs) | |
| 124 POLL_SUCCESSES.inc() | |
| 125 except Exception as e: | |
| 126 log.error("poll error: %r\n%s", e, traceback.format_exc()) | |
| 127 POLL_ERRORS.inc() | |
| 0 | 128 |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
129 def onNodes(self, newAddrs: List[SeenNode]): |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
130 now = int(time.time()) |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
131 newWithSignal = [a for a in newAddrs if a.connected] |
| 1679 | 132 CURRENTLY_ON_WIFI.set(len(newWithSignal)) |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
133 |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
134 actions = self.computeActions(newWithSignal) |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
135 for action in actions: |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
136 log.info("action: %s", action) |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
137 action['created'] = datetime.datetime.now(tz.gettz('UTC')) |
| 1728 | 138 self.mongo.insert_one(action) |
| 1679 | 139 MAC_ON_WIFI.labels(mac=action['address'].lower()).set(1 if action['action'] == 'arrive' else 0) |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
140 if now // 3600 > self.lastPollTime // 3600: |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
141 log.info('hourly writes') |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
142 for addr in newWithSignal: |
| 1679 | 143 MAC_ON_WIFI.labels(mac=addr.mac.lower()).set(1) |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
144 |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
145 self.lastWithSignal = newWithSignal |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
146 self.lastAddrs = newAddrs |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
147 self.lastPollTime = now |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
148 |
| 1728 | 149 self.updateGraph(self.masterGraph) |
|
566
c3d06c350e24
no more immediateUpdate since we push patch events now. and the code was broken for py3 anyway
drewp@bigasterisk.com
parents:
564
diff
changeset
|
150 |
| 0 | 151 def computeActions(self, newWithSignal): |
| 152 actions = [] | |
| 153 | |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
154 def makeAction(addr: SeenNode, act: str): |
| 1679 | 155 d = dict( |
| 156 sensor="wifi", | |
| 157 address=addr.mac.upper(), # mongo data is legacy uppercase | |
| 158 action=act) | |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
159 if act == 'arrive': |
|
62
f8cc3d1baa85
redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents:
52
diff
changeset
|
160 # this won't cover the possible case that you get on |
|
f8cc3d1baa85
redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents:
52
diff
changeset
|
161 # wifi but don't have an ip yet. We'll record an |
|
f8cc3d1baa85
redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents:
52
diff
changeset
|
162 # action with no ip and then never record your ip. |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
163 d['ip'] = addr.ip |
| 659 | 164 return d |
| 0 | 165 |
| 166 for addr in newWithSignal: | |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
167 if addr.mac not in [r.mac for r in self.lastWithSignal]: |
| 0 | 168 actions.append(makeAction(addr, 'arrive')) |
| 169 | |
| 170 for addr in self.lastWithSignal: | |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
171 if addr.mac not in [r.mac for r in newWithSignal]: |
| 0 | 172 actions.append(makeAction(addr, 'leave')) |
| 173 | |
| 174 return actions | |
| 175 | |
| 176 def deltaSinceLastArrive(self, name): | |
| 1679 | 177 results = list(self.mongo.find({'name': name}).sort('created', DESCENDING).limit(1)) |
| 0 | 178 if not results: |
| 179 return datetime.timedelta.max | |
| 180 now = datetime.datetime.now(tz.gettz('UTC')) | |
| 181 last = results[0]['created'].replace(tzinfo=tz.gettz('UTC')) | |
| 182 return now - last | |
|
51
d2842eedd56d
rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents:
50
diff
changeset
|
183 |
| 340 | 184 def updateGraph(self, masterGraph): |
| 185 g = ConjunctiveGraph() | |
| 186 ctx = DEV['wifi'] | |
| 187 | |
| 188 # someday i may also record specific AP and their strength, | |
| 189 # for positioning. But many users just want to know that the | |
| 190 # device is connected to some bigasterisk AP. | |
| 191 age = time.time() - self.lastPollTime | |
| 192 if age > 10: | |
| 193 raise ValueError("poll data is stale. age=%s" % age) | |
| 194 | |
| 195 for dev in self.lastAddrs: | |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
196 if not dev.connected: |
| 340 | 197 continue |
|
427
db031d9ec28e
don't use 'connected' for time and for network. add rdf:type.
drewp@bigasterisk.com
parents:
423
diff
changeset
|
198 g.add((dev.uri, RDF.type, ROOM['NetworkedDevice'], ctx)) |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
199 g.add((dev.uri, ROOM['macAddress'], Literal(dev.mac), ctx)) |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
200 g.add((dev.uri, ROOM['ipAddress'], Literal(dev.ip), ctx)) |
| 340 | 201 |
| 1679 | 202 for s, p, o in dev.stmts: |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
203 g.add((s, p, o, ctx)) |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
204 |
| 340 | 205 try: |
| 1728 | 206 conn = whenConnected(self.mongo, dev.mac) |
| 340 | 207 except ValueError: |
| 208 traceback.print_exc() | |
| 209 pass | |
| 210 else: | |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
211 g.add((dev.uri, ROOM['connected'], Literal(conn), ctx)) |
| 340 | 212 masterGraph.setToGraph(g) |
| 213 | |
| 1679 | 214 |
| 1728 | 215 # class RemoteSuspend(PrettyErrorHandler, cyclone.web.RequestHandler): |
| 659 | 216 |
| 1728 | 217 # def post(self): |
| 218 # # windows is running shutter (https://www.den4b.com/products/shutter) | |
| 219 # fetch('http://DESKTOP-GOU4AC4:8011/action', postdata={'id': 'Sleep'}) | |
| 1679 | 220 |
| 221 | |
| 1728 | 222 def main(): |
| 223 log.setLevel(logging.INFO) | |
| 224 masterGraph = PatchableGraph() | |
| 1679 | 225 mongo = Connection('mongodb.default.svc.cluster.local', 27017, tz_aware=True)['visitor']['visitor'] |
| 0 | 226 |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
227 config = ConjunctiveGraph() |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
228 config.parse(open('private_config.n3'), format='n3') |
| 659 | 229 |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
230 wifi = Wifi(config) |
| 1728 | 231 poller = Poller(wifi, mongo, masterGraph) |
| 232 loop = background_loop.loop_forever(poller.poll, 10) | |
| 0 | 233 |
| 1728 | 234 app = Starlette(routes=[ |
| 235 Route('/graph/wifi', StaticGraph(masterGraph)), | |
| 236 Route('/graph/wifi/events', GraphEvents(masterGraph)), | |
| 237 ],) | |
| 238 | |
| 239 app.add_middleware(PrometheusMiddleware, app_name='environment') | |
| 240 app.add_route("/metrics", handle_metrics) | |
| 241 return app | |
| 242 | |
| 243 | |
| 244 app = main() |