Mercurial > code > home > repos > homeauto
changeset 1226:7de8f0cd3392
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
Ignore-this: bfecea6f4990d34b36cc6d97cc6c6fa2
darcs-hash:79a3d55bfae5776b2374faa2020b6e27f17eb390
| author | drewp <drewp@bigasterisk.com> |
|---|---|
| date | Sat, 30 Mar 2019 23:38:47 -0700 |
| parents | b8c0daabe5a5 |
| children | 21bc3b07a538 |
| files | service/wifi/Dockerfile service/wifi/makefile service/wifi/requirements.txt service/wifi/scrape.py service/wifi/scrape_unmaintained.py service/wifi/wifi.py |
| diffstat | 6 files changed, 181 insertions(+), 264 deletions(-) [+] |
line wrap: on
line diff
--- a/service/wifi/Dockerfile Sat Mar 30 18:59:19 2019 -0700 +++ b/service/wifi/Dockerfile Sat Mar 30 23:38:47 2019 -0700 @@ -9,7 +9,7 @@ # not sure why this doesn't work from inside requirements.txt RUN pip3 install -U 'https://github.com/drewp/cyclone/archive/python3.zip?v2' -COPY *.py *.n3 *.json *.html ./ +COPY *.py *.n3 *.html ./ EXPOSE 9070
--- a/service/wifi/makefile Sat Mar 30 18:59:19 2019 -0700 +++ b/service/wifi/makefile Sat Mar 30 23:38:47 2019 -0700 @@ -6,7 +6,7 @@ build_image: rm -rf tmp_ctx mkdir -p tmp_ctx - cp -a Dockerfile ../../lib/*.py *.py *.n3 *.json *.html req* tmp_ctx + cp -a Dockerfile ../../lib/*.py *.py *.n3 *.html req* tmp_ctx docker build --network=host -t ${TAG} tmp_ctx docker push ${TAG} rm -r tmp_ctx
--- a/service/wifi/requirements.txt Sat Mar 30 18:59:19 2019 -0700 +++ b/service/wifi/requirements.txt Sat Mar 30 23:38:47 2019 -0700 @@ -8,3 +8,4 @@ lxml==4.3.3 pystache==0.5.2 rdflib==4.2.2 +ago==0.0.93
--- a/service/wifi/scrape.py Sat Mar 30 18:59:19 2019 -0700 +++ b/service/wifi/scrape.py Sat Mar 30 23:38:47 2019 -0700 @@ -1,176 +1,78 @@ -import re, ast, logging, socket, json, base64 +import logging, json, base64 +from typing import List + +from cyclone.httpclient import fetch +from rdflib import Literal, Graph, RDF, URIRef, Namespace from twisted.internet.defer import inlineCallbacks, returnValue -from cyclone.httpclient import fetch -from rdflib import Literal, Graph, RDFS, URIRef log = logging.getLogger() +ROOM = Namespace("http://projects.bigasterisk.com/room/") +AST = Namespace("http://bigasterisk.com/") def macUri(macAddress: str) -> URIRef: - return URIRef("http://bigasterisk.com/mac/%s" % dev['mac'].lower()) + return URIRef("http://bigasterisk.com/mac/%s" % macAddress.lower()) +class SeenNode(object): + def __init__(self, uri: URIRef, mac: str, ip: str, pred_objs: List): + self.connected = True + self.uri = uri + self.mac = mac + self.ip = ip + self.stmts = [(uri, p, o) for p, o in pred_objs] + class Wifi(object): """ gather the users of wifi from the tomato routers """ - def __init__(self, accessN3="/my/proj/openid_proxy/access.n3"): - self.rereadConfig() - #self._loadRouters(accessN3, tomatoUrl) - - def rereadConfig(self): - self.graph = Graph() - self.graph.parse('config.n3', format='n3') - + def __init__(self, config: Graph): + self.config = config - def _loadRouters(self, accessN3, tomatoUrl): - g = Graph() - g.parse(accessN3, format="n3") - repl = { - '/wifiRouter1/' : None, - #'/tomato2/' : None - } - for k in repl: - rows = list(g.query(''' - PREFIX p: <http://bigasterisk.com/openid_proxy#> - SELECT ?prefix WHERE { - ?site - p:requestPrefix ?public; - p:proxyUrlPrefix ?prefix - . - }''', initBindings={"public" : Literal(k)})) - repl[k] = str(rows[0][0]) - log.debug('repl %r', repl) - - self.routers = [] - for url in tomatoUrl: - name = url - for k, v in repl.items(): - url = url.replace(k, v) - - r = Router() - http, tail = url.split('//', 1) - userPass, tail = tail.split("@", 1) - r.url = http + '//' + tail - r.headers = {'Authorization': ['Basic %s' % userPass.encode('base64').strip()]} - r.name = {'wifiRouter1' : 'bigasterisk5', - 'tomato2' : 'bigasterisk4'}[name.split('/')[1]] - self.routers.append(r) - @inlineCallbacks - def getPresentMacAddrs(self): - self.rereadConfig() - rows = yield loadOrbiData() - for row in rows: - if 'clientHostname' in row: - row['name'] = row['clientHostname'] - mac = macUri(row['mac'].lower()) - label = self.graph.value(mac, RDFS.label) - if label: - row['name'] = label + def getPresentMacAddrs(self): # returnValue List[SeenNode] + rows = yield self._loader()(self.config) returnValue(rows) - - @inlineCallbacks - def getPresentMacAddrs_multirouter(self): - rows = [] - - for router in self.routers: - log.debug("GET %s", router) - try: - resp = yield fetch(router.url, headers=router.headers, - timeout=2) - except socket.error: - log.warn("get on %s failed" % router) - continue - data = resp.body - if 'Wireless -- Authenticated Stations' in data: - # zyxel 'Station Info' page - rows.extend(self._parseZyxel(data, router.name)) - else: - # tomato page - rows.extend(self._parseTomato(data, router.name)) - for r in rows: - try: - r['name'] = self.knownMacAddr[r['mac']] - except KeyError: - pass - - returnValue(rows) - - def _parseZyxel(self, data, routerName): - import lxml.html.soupparser - - root = lxml.html.soupparser.fromstring(data) - for tr in root.cssselect('tr'): - mac, assoc, uth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] - if mac == "MAC": - continue - assoc = assoc.lower() == 'yes' - yield dict(router=routerName, mac=mac, assoc=assoc, connected=assoc) - - def _parseTomato(self, data, routerName): - for iface, mac, signal in jsValue(data, 'wldev'): - yield dict(router=routerName, mac=mac, signal=signal, connected=bool(signal)) + def _loader(self): + cls = self.config.value(ROOM['wifiScraper'], RDF.type) + if cls == ROOM['OrbiScraper']: + return loadOrbiData + raise NotImplementedError(cls) @inlineCallbacks -def loadUvaData(): - import lxml.html.soupparser +def loadOrbiData(config): + user = config.value(ROOM['wifiScraper'], ROOM['user']) + passwd = config.value(ROOM['wifiScraper'], ROOM['password']) + basicAuth = '%s:%s' % (user, passwd) + headers = { + b'Authorization': [ + b'Basic %s' % base64.encodebytes(basicAuth.encode('utf8')).strip()], + } + uri = config.value(ROOM['wifiScraper'], ROOM['deviceInfoPage']) + resp = yield fetch(uri.encode('utf8'), method=b'GET', headers=headers) - config = json.load(open("priv-uva.json")) - headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} - resp = yield fetch('http://10.2.0.2/wlstationlist.cmd', headers=headers) - root = lxml.html.soupparser.fromstring(resp.body) - byMac = {} - for tr in root.cssselect('tr'): - mac, connected, auth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] - if mac == "MAC": - continue - connected = connected.lower() == 'yes' - byMac[mac] = dict(mac=mac, connected=connected, auth=auth == 'Yes', ssid=ssid, iface=iface) - - resp = yield fetch('http://10.2.0.2/DHCPTable.asp', headers=headers) - for row in re.findall(r'new AAA\((.*)\)', resp.body): - clientHostname, ipaddr, mac, expires, iface = [s.strip("'") for s in row.rsplit(',', 4)] - if clientHostname == 'wlanadv.none': - continue - byMac.setdefault(mac, {}).update(dict( - clientHostname=clientHostname, connection=iface, ipaddr=ipaddr, dhcpExpires=expires)) - - returnValue(sorted(byMac.values())) - -@inlineCallbacks -def loadCiscoData(): - config = json.load(open("priv-uva.json")) - headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} - print(headers) - resp = yield fetch('http://10.2.0.2/', headers=headers) - print(resp.body) - returnValue([]) - -@inlineCallbacks -def loadOrbiData(): - config = json.load(open("priv-uva.json")) - headers = {b'Authorization': [ - b'Basic %s' % base64.encodebytes(config['userPass'].encode('utf8')).strip()]} - resp = yield fetch(b'http://orbi.bigasterisk.com/DEV_device_info.htm', method=b'GET', headers=headers) - print('back from fetch') - - if not resp.body.startswith((b'device=', b'device_changed=0\ndevice=', b'device_changed=1\ndevice=')): + if not resp.body.startswith((b'device=', + b'device_changed=0\ndevice=', + b'device_changed=1\ndevice=')): raise ValueError(resp.body) - ret = [] + log.debug(resp.body) + rows = [] for row in json.loads(resp.body.split(b'device=', 1)[-1]): - ret.append(dict( - connected=True, - ipaddr=row['ip'], + extra = [] + extra.append((ROOM['connected'], { + 'wireless': AST['wifiAccessPoints'], + '2.4G': AST['wifiAccessPoints'], + '5G': AST['wifiAccessPoints'], + '-': AST['wifiUnknownConnectionType'], + 'Unknown': AST['wifiUnknownConnectionType'], + 'wired': AST['houseOpenNet']}[row['contype']])) + if row['model'] != 'Unknown': + extra.append((ROOM['networkModel'], Literal(row['model']))) + + rows.append(SeenNode( + uri=macUri(row['mac'].lower()), mac=row['mac'].lower(), - contype=row['contype'], - model=row['model'], - clientHostname=row['name'] if row['name'] != 'Unknown' else None)) - returnValue(ret) - - -def jsValue(js, variableName): - # using literal_eval instead of json parser to handle the trailing commas - val = re.search(variableName + r'\s*=\s*(.*?);', js, re.DOTALL).group(1) - return ast.literal_eval(val) + ip=row['ip'], + pred_objs=extra)) + returnValue(rows)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/service/wifi/scrape_unmaintained.py Sat Mar 30 23:38:47 2019 -0700 @@ -0,0 +1,57 @@ + +@inlineCallbacks +def loadUvaData(): + import lxml.html.soupparser + + config = json.load(open("priv-uva.json")) + headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} + resp = yield fetch('http://10.2.0.2/wlstationlist.cmd', headers=headers) + root = lxml.html.soupparser.fromstring(resp.body) + byMac = {} + for tr in root.cssselect('tr'): + mac, connected, auth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] + if mac == "MAC": + continue + connected = connected.lower() == 'yes' + byMac[mac] = dict(mac=mac, connected=connected, auth=auth == 'Yes', ssid=ssid, iface=iface) + + resp = yield fetch('http://10.2.0.2/DHCPTable.asp', headers=headers) + for row in re.findall(r'new AAA\((.*)\)', resp.body): + clientHostname, ipaddr, mac, expires, iface = [s.strip("'") for s in row.rsplit(',', 4)] + if clientHostname == 'wlanadv.none': + continue + byMac.setdefault(mac, {}).update(dict( + clientHostname=clientHostname, connection=iface, ipaddr=ipaddr, dhcpExpires=expires)) + + returnValue(sorted(byMac.values())) + +@inlineCallbacks +def loadCiscoData(): + config = json.load(open("priv-uva.json")) + headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} + print(headers) + resp = yield fetch('http://10.2.0.2/', headers=headers) + print(resp.body) + returnValue([]) + + +def jsValue(js, variableName): + # using literal_eval instead of json parser to handle the trailing commas + val = re.search(variableName + r'\s*=\s*(.*?);', js, re.DOTALL).group(1) + return ast.literal_eval(val) + + +def _parseZyxel(self, data, routerName): + import lxml.html.soupparser + + root = lxml.html.soupparser.fromstring(data) + for tr in root.cssselect('tr'): + mac, assoc, uth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] + if mac == "MAC": + continue + assoc = assoc.lower() == 'yes' + yield dict(router=routerName, mac=mac, assoc=assoc, connected=assoc) + +def _parseTomato(self, data, routerName): + for iface, mac, signal in jsValue(data, 'wldev'): + yield dict(router=routerName, mac=mac, signal=signal, connected=bool(signal))
--- a/service/wifi/wifi.py Sat Mar 30 18:59:19 2019 -0700 +++ b/service/wifi/wifi.py Sat Mar 30 23:38:47 2019 -0700 @@ -1,4 +1,3 @@ -#!/usr/bin/python """ scrape the tomato router status pages to see who's connected to the wifi access points. Includes leases that aren't currently connected. @@ -11,25 +10,25 @@ Todo: this should be the one polling and writing to mongo, not entrancemusic """ -from __future__ import division -import sys, cyclone.web, json, traceback, time, pystache, datetime, logging +import sys, json, traceback, time, datetime, logging +from typing import List + from cyclone.httpclient import fetch - from dateutil import tz +from influxdb import InfluxDBClient +from pymongo import MongoClient as Connection, DESCENDING +from rdflib import Namespace, Literal, ConjunctiveGraph from twisted.internet import reactor, task from twisted.internet.defer import inlineCallbacks +import ago +import cyclone.web import docopt -from influxdb import InfluxDBClient -from pymongo import MongoClient as Connection, DESCENDING -from rdflib import Namespace, Literal, URIRef, ConjunctiveGraph - -from scrape import Wifi, macUri - -from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler +import pystache from cycloneerr import PrettyErrorHandler from logsetup import log - +from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler +from scrape import Wifi, SeenNode AST = Namespace("http://bigasterisk.com/") DEV = Namespace("http://projects.bigasterisk.com/device/") @@ -38,7 +37,6 @@ class Index(PrettyErrorHandler, cyclone.web.RequestHandler): def get(self): - age = time.time() - self.settings.poller.lastPollTime if age > 10: raise ValueError("poll data is stale. age=%s" % age) @@ -61,8 +59,7 @@ return lastArrive['created'] def connectedAgoString(conn): - return web.utils.datestr( - conn.astimezone(tz.tzutc()).replace(tzinfo=None)) + return ago.human(conn.astimezone(tz.tzutc()).replace(tzinfo=None)) class Table(PrettyErrorHandler, cyclone.web.RequestHandler): def get(self): @@ -88,7 +85,6 @@ rows=sorted(map(rowDict, self.settings.poller.lastAddrs), key=lambda a: (not a.get('connected'), a.get('name')))))) - class Json(PrettyErrorHandler, cyclone.web.RequestHandler): def get(self): @@ -97,13 +93,13 @@ if age > 10: raise ValueError("poll data is stale. age=%s" % age) self.write(json.dumps({"wifi" : self.settings.poller.lastAddrs, - "dataAge" : age})) + "dataAge" : age})) class Poller(object): def __init__(self, wifi, mongo): self.wifi = wifi self.mongo = mongo - self.lastAddrs = [] + self.lastAddrs = [] # List[SeenNode] self.lastWithSignal = [] self.lastPollTime = 0 @@ -113,50 +109,42 @@ @inlineCallbacks def poll(self): - now = int(time.time()) - - # UVA mode: - addDhcpData = lambda *args: None - try: newAddrs = yield self.wifi.getPresentMacAddrs() - addDhcpData(newAddrs) - - newWithSignal = [a for a in newAddrs if a.get('connected')] - - actions = self.computeActions(newWithSignal) - points = [] - for action in actions: - log.info("action: %s", action) - action['created'] = datetime.datetime.now(tz.gettz('UTC')) - mongo.save(action) - points.append( - self.influxPoint(now, action['address'].lower(), - 1 if action['action'] == 'arrive' else 0)) - try: - self.doEntranceMusic(action) - except Exception as e: - log.error("entrancemusic error: %r", e) - - if now // 3600 > self.lastPollTime // 3600: - log.info('hourly writes') - for addr in newWithSignal: - points.append(self.influxPoint(now, addr['mac'].lower(), 1)) - - influx.write_points(points, time_precision='s') - self.lastWithSignal = newWithSignal - if actions: # this doesn't currently include signal strength changes - fetch(reasoning + "immediateUpdate", - method='PUT', - timeout=2, - headers={'user-agent': ['tomatoWifi']}).addErrback(log.warn) - self.lastAddrs = newAddrs - self.lastPollTime = now - - self.updateGraph(masterGraph) + self.onNodes(newAddrs) except Exception as e: log.error("poll error: %r\n%s", e, traceback.format_exc()) + def onNodes(self, newAddrs: List[SeenNode]): + now = int(time.time()) + newWithSignal = [a for a in newAddrs if a.connected] + + actions = self.computeActions(newWithSignal) + points = [] + for action in actions: + log.info("action: %s", action) + action['created'] = datetime.datetime.now(tz.gettz('UTC')) + mongo.save(action) + points.append( + self.influxPoint(now, action['address'].lower(), + 1 if action['action'] == 'arrive' else 0)) + if now // 3600 > self.lastPollTime // 3600: + log.info('hourly writes') + for addr in newWithSignal: + points.append(self.influxPoint(now, addr.mac.lower(), 1)) + + influx.write_points(points, time_precision='s') + self.lastWithSignal = newWithSignal + if actions: # this doesn't currently include signal strength changes + fetch(reasoning + "immediateUpdate", + method='PUT', + timeout=2, + headers={'user-agent': ['wifi']}).addErrback(log.warn) + self.lastAddrs = newAddrs + self.lastPollTime = now + + self.updateGraph(masterGraph) + def influxPoint(self, now, address, value): return { 'measurement': 'presence', @@ -168,46 +156,27 @@ def computeActions(self, newWithSignal): actions = [] - def makeAction(addr, act): + def makeAction(addr: SeenNode, act: str): d = dict(sensor="wifi", - address=addr.get('mac').upper(), # mongo data is legacy uppercase - name=addr.get('name'), - networkName=addr.get('clientHostname'), + address=addr.mac.upper(), # mongo data is legacy uppercase action=act) - if act == 'arrive' and 'ip' in addr: + if act == 'arrive': # this won't cover the possible case that you get on # wifi but don't have an ip yet. We'll record an # action with no ip and then never record your ip. - d['ip'] = addr['ip'] + d['ip'] = addr.ip return d for addr in newWithSignal: - if addr['mac'] not in [r['mac'] for r in self.lastWithSignal]: + if addr.mac not in [r.mac for r in self.lastWithSignal]: actions.append(makeAction(addr, 'arrive')) for addr in self.lastWithSignal: - if addr['mac'] not in [r['mac'] for r in newWithSignal]: + if addr.mac not in [r.mac for r in newWithSignal]: actions.append(makeAction(addr, 'leave')) return actions - - # these need to move out to their own service - def doEntranceMusic(self, action): - import restkit, json - dt = self.deltaSinceLastArrive(action['name']) - log.debug("dt=%s", dt) - if dt > datetime.timedelta(hours=1): - hub = restkit.Resource( - # PSHB not working yet; "http://bang:9030/" - "http://slash:9049/" - ) - action = action.copy() - del action['created'] - del action['_id'] - log.info("post to %s", hub) - hub.post("visitorNet", payload=json.dumps(action)) - def deltaSinceLastArrive(self, name): results = list(self.mongo.find({'name' : name}).sort('created', DESCENDING).limit(1)) @@ -218,7 +187,6 @@ return now - last def updateGraph(self, masterGraph): - g = ConjunctiveGraph() ctx = DEV['wifi'] @@ -230,43 +198,30 @@ raise ValueError("poll data is stale. age=%s" % age) for dev in self.lastAddrs: - if not dev.get('connected'): + if not dev.connected: continue - mac = dev['mac'].lower() - uri = macUri(mac) - g.add((uri, ROOM['macAddress'], Literal(mac), ctx)) + g.add((dev.uri, ROOM['macAddress'], Literal(dev.mac), ctx)) + g.add((dev.uri, ROOM['ipAddress'], Literal(dev.ip), ctx)) - g.add((uri, ROOM['connected'], { - 'wireless': AST['wifiAccessPoints'], - '2.4G': AST['wifiAccessPoints'], - '5G': AST['wifiAccessPoints'], - '-': AST['wifiUnknownConnectionType'], - 'Unknown': AST['wifiUnknownConnectionType'], - 'wired': AST['houseOpenNet']}[dev['contype']], ctx)) - if 'clientHostname' in dev and dev['clientHostname']: - g.add((uri, ROOM['wifiNetworkName'], Literal(dev['clientHostname']), ctx)) - if 'name' in dev and dev['name']: - g.add((uri, ROOM['deviceName'], Literal(dev['name']), ctx)) - if 'signal' in dev: - g.add((uri, ROOM['signalStrength'], Literal(dev['signal']), ctx)) - if 'model' in dev: - g.add((uri, ROOM['networkModel'], Literal(dev['model']), ctx)) + for s,p,o in dev.stmts: + g.add((s, p, o, ctx)) + try: - conn = whenConnected(mongo, dev['mac']) + conn = whenConnected(mongo, dev.mac) except ValueError: traceback.print_exc() pass else: - g.add((uri, ROOM['connectedAgo'], + g.add((dev.uri, ROOM['connectedAgo'], Literal(connectedAgoString(conn)), ctx)) - g.add((uri, ROOM['connected'], Literal(conn), ctx)) + g.add((dev.uri, ROOM['connected'], Literal(conn), ctx)) masterGraph.setToGraph(g) if __name__ == '__main__': args = docopt.docopt(''' Usage: - tomatoWifi [options] + wifi.py [options] Options: -v, --verbose more logging @@ -282,8 +237,11 @@ mongo = Connection('bang', 27017, tz_aware=True)['visitor']['visitor'] influx = InfluxDBClient('bang', 9060, 'root', 'root', 'main') + config = ConjunctiveGraph() + config.parse(open('private_config.n3'), format='n3') + masterGraph = PatchableGraph() - wifi = Wifi() + wifi = Wifi(config) poller = Poller(wifi, mongo) task.LoopingCall(poller.poll).start(1/float(args['--poll'])) @@ -301,5 +259,4 @@ wifi=wifi, poller=poller, mongo=mongo)) - import twisted; print('twisted', twisted.__version__) reactor.run()