Mercurial > code > home > repos > homeauto
changeset 421:47d7dd31bb2c
port to py3
Ignore-this: e1a2e6bb730111e76f5a5dd2366d498a
| author | drewp@bigasterisk.com |
|---|---|
| date | Sat, 30 Mar 2019 18:27:17 -0700 |
| parents | a530d9c5b280 |
| children | 19460b3f7baf |
| files | service/wifi/Dockerfile service/wifi/makefile service/wifi/requirements.txt service/wifi/scrape.py service/wifi/tomatoWifi.py service/wifi/wifi.py |
| diffstat | 6 files changed, 474 insertions(+), 474 deletions(-) [+] |
line wrap: on
line diff
--- a/service/wifi/Dockerfile Sat Mar 30 16:58:08 2019 -0700 +++ b/service/wifi/Dockerfile Sat Mar 30 18:27:17 2019 -0700 @@ -5,10 +5,12 @@ RUN apt-get install -y libxml2-dev libxslt1-dev COPY requirements.txt ./ -RUN pip install -r requirements.txt +RUN pip3 install -r requirements.txt +# not sure why this doesn't work from inside requirements.txt +RUN pip3 install -U 'https://github.com/drewp/cyclone/archive/python3.zip?v2' COPY *.py *.n3 *.json *.html ./ EXPOSE 9070 -CMD [ "python", "tomatoWifi.py" ] +CMD [ "python3", "wifi.py" ]
--- a/service/wifi/makefile Sat Mar 30 16:58:08 2019 -0700 +++ b/service/wifi/makefile Sat Mar 30 18:27:17 2019 -0700 @@ -11,11 +11,11 @@ docker push ${TAG} rm -r tmp_ctx -shell: +shell: build_image docker run --rm -it --cap-add SYS_PTRACE --net=host ${TAG} /bin/bash -local_run: - docker run --rm -it -p ${PORT}:${PORT} \ - --net=host \ - ${TAG} \ - python tomatoWifi.py -v +local_run: build_image + docker run --rm -it -p ${PORT}:${PORT} --net=host ${TAG} python3 wifi.py -v + +local_run_strace: build_image + docker run --rm -it -p ${PORT}:${PORT} --cap-add=SYS_PTRACE --net=host ${TAG} strace -f -s 200 python3 wifi.py -v
--- a/service/wifi/requirements.txt Sat Mar 30 16:58:08 2019 -0700 +++ b/service/wifi/requirements.txt Sat Mar 30 18:27:17 2019 -0700 @@ -1,13 +1,10 @@ docopt pymongo -BeautifulSoup==3.2.1 -cyclone +git+http://github.com/drewp/scales.git@448d59fb491b7631877528e7695a93553bfaaa93#egg=scales +https://github.com/drewp/cyclone/archive/python3.zip +https://projects.bigasterisk.com/rdfdb/rdfdb-0.7.0.tar.gz influxdb==3.0.0 -lxml==4.2.5 +lxml==4.3.3 pystache==0.5.2 rdflib==4.2.2 -restkit==4.2.0 -web.py==0.37 - -https://projects.bigasterisk.com/rdfdb/rdfdb-0.6.0.tar.gz
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/service/wifi/scrape.py Sat Mar 30 18:27:17 2019 -0700 @@ -0,0 +1,177 @@ +import re, ast, logging, socket, json, base64 +from twisted.internet.defer import inlineCallbacks, returnValue +from cyclone.httpclient import fetch +from rdflib import Literal, Graph, RDFS, URIRef + +log = logging.getLogger() + +class Router(object): + def __repr__(self): + return repr(self.__dict__) + +class Wifi(object): + """ + gather the users of wifi from the tomato routers + """ + def __init__(self, accessN3="/my/proj/openid_proxy/access.n3"): + self.rereadConfig() + #self._loadRouters(accessN3, tomatoUrl) + + def rereadConfig(self): + self.graph = Graph() + self.graph.parse('config.n3', format='n3') + + + def _loadRouters(self, accessN3, tomatoUrl): + g = Graph() + g.parse(accessN3, format="n3") + repl = { + '/wifiRouter1/' : None, + #'/tomato2/' : None + } + for k in repl: + rows = list(g.query(''' + PREFIX p: <http://bigasterisk.com/openid_proxy#> + SELECT ?prefix WHERE { + ?site + p:requestPrefix ?public; + p:proxyUrlPrefix ?prefix + . + }''', initBindings={"public" : Literal(k)})) + repl[k] = str(rows[0][0]) + log.debug('repl %r', repl) + + self.routers = [] + for url in tomatoUrl: + name = url + for k, v in repl.items(): + url = url.replace(k, v) + + r = Router() + http, tail = url.split('//', 1) + userPass, tail = tail.split("@", 1) + r.url = http + '//' + tail + r.headers = {'Authorization': ['Basic %s' % userPass.encode('base64').strip()]} + r.name = {'wifiRouter1' : 'bigasterisk5', + 'tomato2' : 'bigasterisk4'}[name.split('/')[1]] + self.routers.append(r) + + @inlineCallbacks + def getPresentMacAddrs(self): + self.rereadConfig() + rows = yield loadOrbiData() + for row in rows: + if 'clientHostname' in row: + row['name'] = row['clientHostname'] + mac = URIRef('http://bigasterisk.com/mac/%s' % row['mac'].lower()) + label = self.graph.value(mac, RDFS.label) + if label: + row['name'] = label + returnValue(rows) + + @inlineCallbacks + def getPresentMacAddrs_multirouter(self): + rows = [] + + for router in self.routers: + log.debug("GET %s", router) + try: + resp = yield fetch(router.url, headers=router.headers, + timeout=2) + except socket.error: + log.warn("get on %s failed" % router) + continue + data = resp.body + if 'Wireless -- Authenticated Stations' in data: + # zyxel 'Station Info' page + rows.extend(self._parseZyxel(data, router.name)) + else: + # tomato page + rows.extend(self._parseTomato(data, router.name)) + + for r in rows: + try: + r['name'] = self.knownMacAddr[r['mac']] + except KeyError: + pass + + returnValue(rows) + + def _parseZyxel(self, data, routerName): + import lxml.html.soupparser + + root = lxml.html.soupparser.fromstring(data) + for tr in root.cssselect('tr'): + mac, assoc, uth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] + if mac == "MAC": + continue + assoc = assoc.lower() == 'yes' + yield dict(router=routerName, mac=mac, assoc=assoc, connected=assoc) + + def _parseTomato(self, data, routerName): + for iface, mac, signal in jsValue(data, 'wldev'): + yield dict(router=routerName, mac=mac, signal=signal, connected=bool(signal)) + + +@inlineCallbacks +def loadUvaData(): + import lxml.html.soupparser + + config = json.load(open("priv-uva.json")) + headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} + resp = yield fetch('http://10.2.0.2/wlstationlist.cmd', headers=headers) + root = lxml.html.soupparser.fromstring(resp.body) + byMac = {} + for tr in root.cssselect('tr'): + mac, connected, auth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] + if mac == "MAC": + continue + connected = connected.lower() == 'yes' + byMac[mac] = dict(mac=mac, connected=connected, auth=auth == 'Yes', ssid=ssid, iface=iface) + + resp = yield fetch('http://10.2.0.2/DHCPTable.asp', headers=headers) + for row in re.findall(r'new AAA\((.*)\)', resp.body): + clientHostname, ipaddr, mac, expires, iface = [s.strip("'") for s in row.rsplit(',', 4)] + if clientHostname == 'wlanadv.none': + continue + byMac.setdefault(mac, {}).update(dict( + clientHostname=clientHostname, connection=iface, ipaddr=ipaddr, dhcpExpires=expires)) + + returnValue(sorted(byMac.values())) + +@inlineCallbacks +def loadCiscoData(): + config = json.load(open("priv-uva.json")) + headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} + print(headers) + resp = yield fetch('http://10.2.0.2/', headers=headers) + print(resp.body) + returnValue([]) + +@inlineCallbacks +def loadOrbiData(): + config = json.load(open("priv-uva.json")) + headers = {b'Authorization': [ + b'Basic %s' % base64.encodebytes(config['userPass'].encode('utf8')).strip()]} + resp = yield fetch(b'http://orbi.bigasterisk.com/DEV_device_info.htm', method=b'GET', headers=headers) + print('back from fetch') + + if not resp.body.startswith((b'device=', b'device_changed=0\ndevice=', b'device_changed=1\ndevice=')): + raise ValueError(resp.body) + + ret = [] + for row in json.loads(resp.body.split(b'device=', 1)[-1]): + ret.append(dict( + connected=True, + ipaddr=row['ip'], + mac=row['mac'].lower(), + contype=row['contype'], + model=row['model'], + clientHostname=row['name'] if row['name'] != 'Unknown' else None)) + returnValue(ret) + + +def jsValue(js, variableName): + # using literal_eval instead of json parser to handle the trailing commas + val = re.search(variableName + r'\s*=\s*(.*?);', js, re.DOTALL).group(1) + return ast.literal_eval(val)
--- a/service/wifi/tomatoWifi.py Sat Mar 30 16:58:08 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,306 +0,0 @@ -#!/usr/bin/python -""" -scrape the tomato router status pages to see who's connected to the -wifi access points. Includes leases that aren't currently connected. - -Returns: - json listing (for magma page) - rdf graph (for reasoning) - activity stream, when we start saving history - -Todo: this should be the one polling and writing to mongo, not entrancemusic - -""" -from __future__ import division -import sys, cyclone.web, json, traceback, time, pystache, datetime, logging -import web.utils -from cyclone.httpclient import fetch - -from dateutil import tz -from twisted.internet import reactor, task -from twisted.internet.defer import inlineCallbacks -import docopt -from influxdb import InfluxDBClient -from pymongo import MongoClient as Connection, DESCENDING -from rdflib import Namespace, Literal, URIRef, ConjunctiveGraph - -from stategraph import StateGraph -from wifi import Wifi - -from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler - -from rdfdb.patch import Patch - -from cycloneerr import PrettyErrorHandler -from logsetup import log - - -DEV = Namespace("http://projects.bigasterisk.com/device/") -ROOM = Namespace("http://projects.bigasterisk.com/room/") -reasoning = "http://bang:9071/" - -class Index(PrettyErrorHandler, cyclone.web.RequestHandler): - def get(self): - - age = time.time() - self.settings.poller.lastPollTime - if age > 10: - raise ValueError("poll data is stale. age=%s" % age) - - self.set_header("Content-Type", "text/html") - self.write(open("index.html").read()) - -def whenConnected(mongo, macThatIsNowConnected): - lastArrive = None - for ev in mongo.find({'address': macThatIsNowConnected.upper()}, - sort=[('created', -1)], - max_scan=100000): - if ev['action'] == 'arrive': - lastArrive = ev - if ev['action'] == 'leave': - break - if lastArrive is None: - raise ValueError("no past arrivals") - - return lastArrive['created'] - -def connectedAgoString(conn): - return web.utils.datestr( - conn.astimezone(tz.tzutc()).replace(tzinfo=None)) - -class Table(PrettyErrorHandler, cyclone.web.RequestHandler): - def get(self): - def rowDict(row): - row['cls'] = "signal" if row.get('connected') else "nosignal" - if 'name' not in row: - row['name'] = row.get('clientHostname', '-') - if 'signal' not in row: - row['signal'] = 'yes' if row.get('connected') else 'no' - - try: - conn = whenConnected(self.settings.mongo, row.get('mac', '??')) - row['connectedAgo'] = connectedAgoString(conn) - except ValueError: - row['connectedAgo'] = 'yes' if row.get('connected') else '' - row['router'] = row.get('ssid', '') - return row - - self.set_header("Content-Type", "application/xhtml+xml") - self.write(pystache.render( - open("table.mustache").read(), - dict( - rows=sorted(map(rowDict, self.settings.poller.lastAddrs), - key=lambda a: (not a.get('connected'), - a.get('name')))))) - - -class Json(PrettyErrorHandler, cyclone.web.RequestHandler): - def get(self): - self.set_header("Content-Type", "application/json") - age = time.time() - self.settings.poller.lastPollTime - if age > 10: - raise ValueError("poll data is stale. age=%s" % age) - self.write(json.dumps({"wifi" : self.settings.poller.lastAddrs, - "dataAge" : age})) - -class Poller(object): - def __init__(self, wifi, mongo): - self.wifi = wifi - self.mongo = mongo - self.lastAddrs = [] - self.lastWithSignal = [] - self.lastPollTime = 0 - - def assertCurrent(self): - dt = time.time() - self.lastPollTime - assert dt < 10, "last poll was %s sec ago" % dt - - @inlineCallbacks - def poll(self): - connectedField = 'connected' - now = int(time.time()) - - # UVA mode: - addDhcpData = lambda *args: None - - try: - newAddrs = yield self.wifi.getPresentMacAddrs() - addDhcpData(newAddrs) - - newWithSignal = [a for a in newAddrs if a.get('connected')] - - actions = self.computeActions(newWithSignal) - points = [] - for action in actions: - log.info("action: %s", action) - action['created'] = datetime.datetime.now(tz.gettz('UTC')) - mongo.save(action) - points.append( - self.influxPoint(now, action['address'].lower(), - 1 if action['action'] == 'arrive' else 0)) - try: - self.doEntranceMusic(action) - except Exception, e: - log.error("entrancemusic error: %r", e) - - if now // 3600 > self.lastPollTime // 3600: - log.info('hourly writes') - for addr in newWithSignal: - points.append(self.influxPoint(now, addr['mac'].lower(), 1)) - - influx.write_points(points, time_precision='s') - self.lastWithSignal = newWithSignal - if actions: # this doesn't currently include signal strength changes - fetch(reasoning + "immediateUpdate", - method='PUT', - timeout=2, - headers={'user-agent': ['tomatoWifi']}).addErrback(log.warn) - self.lastAddrs = newAddrs - self.lastPollTime = now - - self.updateGraph(masterGraph) - except Exception, e: - log.error("poll error: %r\n%s", e, traceback.format_exc()) - - def influxPoint(self, now, address, value): - return { - 'measurement': 'presence', - 'tags': {'sensor': 'wifi', 'address': address,}, - 'fields': {'value': value}, - 'time': now, - } - - def computeActions(self, newWithSignal): - actions = [] - - def makeAction(addr, act): - d = dict(sensor="wifi", - address=addr.get('mac').upper(), # mongo data is legacy uppercase - name=addr.get('name'), - networkName=addr.get('clientHostname'), - action=act) - if act == 'arrive' and 'ip' in addr: - # this won't cover the possible case that you get on - # wifi but don't have an ip yet. We'll record an - # action with no ip and then never record your ip. - d['ip'] = addr['ip'] - return d - - for addr in newWithSignal: - if addr['mac'] not in [r['mac'] for r in self.lastWithSignal]: - actions.append(makeAction(addr, 'arrive')) - - for addr in self.lastWithSignal: - if addr['mac'] not in [r['mac'] for r in newWithSignal]: - actions.append(makeAction(addr, 'leave')) - - return actions - - - # these need to move out to their own service - def doEntranceMusic(self, action): - import restkit, json - dt = self.deltaSinceLastArrive(action['name']) - log.debug("dt=%s", dt) - if dt > datetime.timedelta(hours=1): - hub = restkit.Resource( - # PSHB not working yet; "http://bang:9030/" - "http://slash:9049/" - ) - action = action.copy() - del action['created'] - del action['_id'] - log.info("post to %s", hub) - hub.post("visitorNet", payload=json.dumps(action)) - - def deltaSinceLastArrive(self, name): - results = list(self.mongo.find({'name' : name}).sort('created', - DESCENDING).limit(1)) - if not results: - return datetime.timedelta.max - now = datetime.datetime.now(tz.gettz('UTC')) - last = results[0]['created'].replace(tzinfo=tz.gettz('UTC')) - return now - last - - def updateGraph(self, masterGraph): - - g = ConjunctiveGraph() - ctx = DEV['wifi'] - - # someday i may also record specific AP and their strength, - # for positioning. But many users just want to know that the - # device is connected to some bigasterisk AP. - age = time.time() - self.lastPollTime - if age > 10: - raise ValueError("poll data is stale. age=%s" % age) - - for dev in self.lastAddrs: - if not dev.get('connected'): - continue - uri = URIRef("http://bigasterisk.com/mac/%s" % dev['mac'].lower()) - g.add((uri, ROOM['macAddress'], Literal(dev['mac'].lower()), ctx)) - - g.add((uri, ROOM['connected'], { - 'wireless': URIRef("http://bigasterisk.com/wifiAccessPoints"), - '2.4G': URIRef("http://bigasterisk.com/wifiAccessPoints"), - '5G': URIRef("http://bigasterisk.com/wifiAccessPoints"), - '-': URIRef("http://bigasterisk.com/wifiUnknownConnectionType"), - 'Unknown': URIRef("http://bigasterisk.com/wifiUnknownConnectionType"), - 'wired': URIRef("http://bigasterisk.com/houseOpenNet")}[dev['contype']], ctx)) - if 'clientHostname' in dev and dev['clientHostname']: - g.add((uri, ROOM['wifiNetworkName'], Literal(dev['clientHostname']), ctx)) - if 'name' in dev and dev['name']: - g.add((uri, ROOM['deviceName'], Literal(dev['name']), ctx)) - if 'signal' in dev: - g.add((uri, ROOM['signalStrength'], Literal(dev['signal']), ctx)) - if 'model' in dev: - g.add((uri, ROOM['networkModel'], Literal(dev['model']), ctx)) - try: - conn = whenConnected(mongo, dev['mac']) - except ValueError: - traceback.print_exc() - pass - else: - g.add((uri, ROOM['connectedAgo'], - Literal(connectedAgoString(conn)), ctx)) - g.add((uri, ROOM['connected'], Literal(conn), ctx)) - masterGraph.setToGraph(g) - - -if __name__ == '__main__': - args = docopt.docopt(''' -Usage: - tomatoWifi [options] - -Options: - -v, --verbose more logging - --port=<n> serve on port [default: 9070] - --poll=<freq> poll frequency [default: .2] -''') - if args['--verbose']: - from twisted.python import log as twlog - twlog.startLogging(sys.stdout) - log.setLevel(10) - log.setLevel(logging.DEBUG) - - mongo = Connection('bang', 27017, tz_aware=True)['visitor']['visitor'] - influx = InfluxDBClient('bang', 9060, 'root', 'root', 'main') - - masterGraph = PatchableGraph() - wifi = Wifi() - poller = Poller(wifi, mongo) - task.LoopingCall(poller.poll).start(1/float(args['--poll'])) - - reactor.listenTCP(int(args['--port']), - cyclone.web.Application( - [ - (r"/", Index), - (r'/json', Json), - (r'/graph', CycloneGraphHandler, {'masterGraph': masterGraph}), - (r'/graph/events', CycloneGraphEventsHandler, {'masterGraph': masterGraph}), - (r'/table', Table), - #(r'/activity', Activity), - ], - wifi=wifi, - poller=poller, - mongo=mongo)) - reactor.run()
--- a/service/wifi/wifi.py Sat Mar 30 16:58:08 2019 -0700 +++ b/service/wifi/wifi.py Sat Mar 30 18:27:17 2019 -0700 @@ -1,172 +1,302 @@ -import re, ast, logging, socket, json -import lxml.html.soupparser -from twisted.internet.defer import inlineCallbacks, returnValue +#!/usr/bin/python +""" +scrape the tomato router status pages to see who's connected to the +wifi access points. Includes leases that aren't currently connected. + +Returns: + json listing (for magma page) + rdf graph (for reasoning) + activity stream, when we start saving history + +Todo: this should be the one polling and writing to mongo, not entrancemusic + +""" +from __future__ import division +import sys, cyclone.web, json, traceback, time, pystache, datetime, logging from cyclone.httpclient import fetch -from rdflib import Literal, Graph, RDFS, URIRef - -log = logging.getLogger() -class Router(object): - def __repr__(self): - return repr(self.__dict__) +from dateutil import tz +from twisted.internet import reactor, task +from twisted.internet.defer import inlineCallbacks +import docopt +from influxdb import InfluxDBClient +from pymongo import MongoClient as Connection, DESCENDING +from rdflib import Namespace, Literal, URIRef, ConjunctiveGraph + +from scrape import Wifi + +from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler + +from cycloneerr import PrettyErrorHandler +from logsetup import log -class Wifi(object): - """ - gather the users of wifi from the tomato routers - """ - def __init__(self, accessN3="/my/proj/openid_proxy/access.n3"): - self.rereadConfig() - #self._loadRouters(accessN3, tomatoUrl) + +DEV = Namespace("http://projects.bigasterisk.com/device/") +ROOM = Namespace("http://projects.bigasterisk.com/room/") +reasoning = "http://bang:9071/" + +class Index(PrettyErrorHandler, cyclone.web.RequestHandler): + def get(self): + + age = time.time() - self.settings.poller.lastPollTime + if age > 10: + raise ValueError("poll data is stale. age=%s" % age) + + self.set_header("Content-Type", "text/html") + self.write(open("index.html").read()) - def rereadConfig(self): - self.graph = Graph() - self.graph.parse('config.n3', format='n3') - - - def _loadRouters(self, accessN3, tomatoUrl): - g = Graph() - g.parse(accessN3, format="n3") - repl = { - '/wifiRouter1/' : None, - #'/tomato2/' : None - } - for k in repl: - rows = list(g.query(''' - PREFIX p: <http://bigasterisk.com/openid_proxy#> - SELECT ?prefix WHERE { - ?site - p:requestPrefix ?public; - p:proxyUrlPrefix ?prefix - . - }''', initBindings={"public" : Literal(k)})) - repl[k] = str(rows[0][0]) - log.debug('repl %r', repl) +def whenConnected(mongo, macThatIsNowConnected): + lastArrive = None + for ev in mongo.find({'address': macThatIsNowConnected.upper()}, + sort=[('created', -1)], + max_scan=100000): + if ev['action'] == 'arrive': + lastArrive = ev + if ev['action'] == 'leave': + break + if lastArrive is None: + raise ValueError("no past arrivals") + + return lastArrive['created'] + +def connectedAgoString(conn): + return web.utils.datestr( + conn.astimezone(tz.tzutc()).replace(tzinfo=None)) + +class Table(PrettyErrorHandler, cyclone.web.RequestHandler): + def get(self): + def rowDict(row): + row['cls'] = "signal" if row.get('connected') else "nosignal" + if 'name' not in row: + row['name'] = row.get('clientHostname', '-') + if 'signal' not in row: + row['signal'] = 'yes' if row.get('connected') else 'no' - self.routers = [] - for url in tomatoUrl: - name = url - for k, v in repl.items(): - url = url.replace(k, v) + try: + conn = whenConnected(self.settings.mongo, row.get('mac', '??')) + row['connectedAgo'] = connectedAgoString(conn) + except ValueError: + row['connectedAgo'] = 'yes' if row.get('connected') else '' + row['router'] = row.get('ssid', '') + return row + + self.set_header("Content-Type", "application/xhtml+xml") + self.write(pystache.render( + open("table.mustache").read(), + dict( + rows=sorted(map(rowDict, self.settings.poller.lastAddrs), + key=lambda a: (not a.get('connected'), + a.get('name')))))) + - r = Router() - http, tail = url.split('//', 1) - userPass, tail = tail.split("@", 1) - r.url = http + '//' + tail - r.headers = {'Authorization': ['Basic %s' % userPass.encode('base64').strip()]} - r.name = {'wifiRouter1' : 'bigasterisk5', - 'tomato2' : 'bigasterisk4'}[name.split('/')[1]] - self.routers.append(r) +class Json(PrettyErrorHandler, cyclone.web.RequestHandler): + def get(self): + self.set_header("Content-Type", "application/json") + age = time.time() - self.settings.poller.lastPollTime + if age > 10: + raise ValueError("poll data is stale. age=%s" % age) + self.write(json.dumps({"wifi" : self.settings.poller.lastAddrs, + "dataAge" : age})) + +class Poller(object): + def __init__(self, wifi, mongo): + self.wifi = wifi + self.mongo = mongo + self.lastAddrs = [] + self.lastWithSignal = [] + self.lastPollTime = 0 + + def assertCurrent(self): + dt = time.time() - self.lastPollTime + assert dt < 10, "last poll was %s sec ago" % dt @inlineCallbacks - def getPresentMacAddrs(self): - self.rereadConfig() - rows = yield loadOrbiData() - for row in rows: - if 'clientHostname' in row: - row['name'] = row['clientHostname'] - mac = URIRef('http://bigasterisk.com/mac/%s' % row['mac'].lower()) - label = self.graph.value(mac, RDFS.label) - if label: - row['name'] = label - returnValue(rows) - - @inlineCallbacks - def getPresentMacAddrs_multirouter(self): - rows = [] + def poll(self): + now = int(time.time()) + + # UVA mode: + addDhcpData = lambda *args: None - for router in self.routers: - log.debug("GET %s", router) - try: - resp = yield fetch(router.url, headers=router.headers, - timeout=2) - except socket.error: - log.warn("get on %s failed" % router) - continue - data = resp.body - if 'Wireless -- Authenticated Stations' in data: - # zyxel 'Station Info' page - rows.extend(self._parseZyxel(data, router.name)) - else: - # tomato page - rows.extend(self._parseTomato(data, router.name)) + try: + newAddrs = yield self.wifi.getPresentMacAddrs() + addDhcpData(newAddrs) + + newWithSignal = [a for a in newAddrs if a.get('connected')] + + actions = self.computeActions(newWithSignal) + points = [] + for action in actions: + log.info("action: %s", action) + action['created'] = datetime.datetime.now(tz.gettz('UTC')) + mongo.save(action) + points.append( + self.influxPoint(now, action['address'].lower(), + 1 if action['action'] == 'arrive' else 0)) + try: + self.doEntranceMusic(action) + except Exception as e: + log.error("entrancemusic error: %r", e) + + if now // 3600 > self.lastPollTime // 3600: + log.info('hourly writes') + for addr in newWithSignal: + points.append(self.influxPoint(now, addr['mac'].lower(), 1)) + + influx.write_points(points, time_precision='s') + self.lastWithSignal = newWithSignal + if actions: # this doesn't currently include signal strength changes + fetch(reasoning + "immediateUpdate", + method='PUT', + timeout=2, + headers={'user-agent': ['tomatoWifi']}).addErrback(log.warn) + self.lastAddrs = newAddrs + self.lastPollTime = now - for r in rows: - try: - r['name'] = self.knownMacAddr[r['mac']] - except KeyError: - pass - - returnValue(rows) + self.updateGraph(masterGraph) + except Exception as e: + log.error("poll error: %r\n%s", e, traceback.format_exc()) + + def influxPoint(self, now, address, value): + return { + 'measurement': 'presence', + 'tags': {'sensor': 'wifi', 'address': address,}, + 'fields': {'value': value}, + 'time': now, + } - def _parseZyxel(self, data, routerName): - root = lxml.html.soupparser.fromstring(data) - for tr in root.cssselect('tr'): - mac, assoc, uth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] - if mac == "MAC": - continue - assoc = assoc.lower() == 'yes' - yield dict(router=routerName, mac=mac, assoc=assoc, connected=assoc) + def computeActions(self, newWithSignal): + actions = [] - def _parseTomato(self, data, routerName): - for iface, mac, signal in jsValue(data, 'wldev'): - yield dict(router=routerName, mac=mac, signal=signal, connected=bool(signal)) + def makeAction(addr, act): + d = dict(sensor="wifi", + address=addr.get('mac').upper(), # mongo data is legacy uppercase + name=addr.get('name'), + networkName=addr.get('clientHostname'), + action=act) + if act == 'arrive' and 'ip' in addr: + # this won't cover the possible case that you get on + # wifi but don't have an ip yet. We'll record an + # action with no ip and then never record your ip. + d['ip'] = addr['ip'] + return d + + for addr in newWithSignal: + if addr['mac'] not in [r['mac'] for r in self.lastWithSignal]: + actions.append(makeAction(addr, 'arrive')) + + for addr in self.lastWithSignal: + if addr['mac'] not in [r['mac'] for r in newWithSignal]: + actions.append(makeAction(addr, 'leave')) + + return actions -@inlineCallbacks -def loadUvaData(): - config = json.load(open("priv-uva.json")) - headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} - resp = yield fetch('http://10.2.0.2/wlstationlist.cmd', headers=headers) - root = lxml.html.soupparser.fromstring(resp.body) - byMac = {} - for tr in root.cssselect('tr'): - mac, connected, auth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] - if mac == "MAC": - continue - connected = connected.lower() == 'yes' - byMac[mac] = dict(mac=mac, connected=connected, auth=auth == 'Yes', ssid=ssid, iface=iface) - - resp = yield fetch('http://10.2.0.2/DHCPTable.asp', headers=headers) - for row in re.findall(r'new AAA\((.*)\)', resp.body): - clientHostname, ipaddr, mac, expires, iface = [s.strip("'") for s in row.rsplit(',', 4)] - if clientHostname == 'wlanadv.none': - continue - byMac.setdefault(mac, {}).update(dict( - clientHostname=clientHostname, connection=iface, ipaddr=ipaddr, dhcpExpires=expires)) - - returnValue(sorted(byMac.values())) + # these need to move out to their own service + def doEntranceMusic(self, action): + import restkit, json + dt = self.deltaSinceLastArrive(action['name']) + log.debug("dt=%s", dt) + if dt > datetime.timedelta(hours=1): + hub = restkit.Resource( + # PSHB not working yet; "http://bang:9030/" + "http://slash:9049/" + ) + action = action.copy() + del action['created'] + del action['_id'] + log.info("post to %s", hub) + hub.post("visitorNet", payload=json.dumps(action)) + + def deltaSinceLastArrive(self, name): + results = list(self.mongo.find({'name' : name}).sort('created', + DESCENDING).limit(1)) + if not results: + return datetime.timedelta.max + now = datetime.datetime.now(tz.gettz('UTC')) + last = results[0]['created'].replace(tzinfo=tz.gettz('UTC')) + return now - last + + def updateGraph(self, masterGraph): + + g = ConjunctiveGraph() + ctx = DEV['wifi'] + + # someday i may also record specific AP and their strength, + # for positioning. But many users just want to know that the + # device is connected to some bigasterisk AP. + age = time.time() - self.lastPollTime + if age > 10: + raise ValueError("poll data is stale. age=%s" % age) + + for dev in self.lastAddrs: + if not dev.get('connected'): + continue + uri = URIRef("http://bigasterisk.com/mac/%s" % dev['mac'].lower()) + g.add((uri, ROOM['macAddress'], Literal(dev['mac'].lower()), ctx)) -@inlineCallbacks -def loadCiscoData(): - config = json.load(open("priv-uva.json")) - headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} - print headers - resp = yield fetch('http://10.2.0.2/', headers=headers) - print resp.body - returnValue([]) + g.add((uri, ROOM['connected'], { + 'wireless': URIRef("http://bigasterisk.com/wifiAccessPoints"), + '2.4G': URIRef("http://bigasterisk.com/wifiAccessPoints"), + '5G': URIRef("http://bigasterisk.com/wifiAccessPoints"), + '-': URIRef("http://bigasterisk.com/wifiUnknownConnectionType"), + 'Unknown': URIRef("http://bigasterisk.com/wifiUnknownConnectionType"), + 'wired': URIRef("http://bigasterisk.com/houseOpenNet")}[dev['contype']], ctx)) + if 'clientHostname' in dev and dev['clientHostname']: + g.add((uri, ROOM['wifiNetworkName'], Literal(dev['clientHostname']), ctx)) + if 'name' in dev and dev['name']: + g.add((uri, ROOM['deviceName'], Literal(dev['name']), ctx)) + if 'signal' in dev: + g.add((uri, ROOM['signalStrength'], Literal(dev['signal']), ctx)) + if 'model' in dev: + g.add((uri, ROOM['networkModel'], Literal(dev['model']), ctx)) + try: + conn = whenConnected(mongo, dev['mac']) + except ValueError: + traceback.print_exc() + pass + else: + g.add((uri, ROOM['connectedAgo'], + Literal(connectedAgoString(conn)), ctx)) + g.add((uri, ROOM['connected'], Literal(conn), ctx)) + masterGraph.setToGraph(g) -@inlineCallbacks -def loadOrbiData(): - config = json.load(open("priv-uva.json")) - headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} - resp = yield fetch('http://orbi.bigasterisk.com/DEV_device_info.htm', headers=headers) + +if __name__ == '__main__': + args = docopt.docopt(''' +Usage: + tomatoWifi [options] - if not resp.body.startswith(('device=', 'device_changed=0\ndevice=', 'device_changed=1\ndevice=')): - raise ValueError(resp.body) +Options: + -v, --verbose more logging + --port=<n> serve on port [default: 9070] + --poll=<freq> poll frequency [default: .2] +''') + if args['--verbose']: + from twisted.python import log as twlog + twlog.startLogging(sys.stdout) + log.setLevel(10) + log.setLevel(logging.DEBUG) + + mongo = Connection('bang', 27017, tz_aware=True)['visitor']['visitor'] + influx = InfluxDBClient('bang', 9060, 'root', 'root', 'main') - ret = [] - for row in json.loads(resp.body.split('device=', 1)[-1]): - ret.append(dict( - connected=True, - ipaddr=row['ip'], - mac=row['mac'].lower(), - contype=row['contype'], - model=row['model'], - clientHostname=row['name'] if row['name'] != 'Unknown' else None)) - returnValue(ret) + masterGraph = PatchableGraph() + wifi = Wifi() + poller = Poller(wifi, mongo) + task.LoopingCall(poller.poll).start(1/float(args['--poll'])) - -def jsValue(js, variableName): - # using literal_eval instead of json parser to handle the trailing commas - val = re.search(variableName + r'\s*=\s*(.*?);', js, re.DOTALL).group(1) - return ast.literal_eval(val) + reactor.listenTCP(int(args['--port']), + cyclone.web.Application( + [ + (r"/", Index), + (r'/json', Json), + (r'/graph', CycloneGraphHandler, {'masterGraph': masterGraph}), + (r'/graph/events', CycloneGraphEventsHandler, {'masterGraph': masterGraph}), + (r'/table', Table), + #(r'/activity', Activity), + ], + wifi=wifi, + poller=poller, + mongo=mongo)) + import twisted; print('twisted', twisted.__version__) + reactor.run()