Mercurial > code > home > repos > homeauto
changeset 420:a530d9c5b280
rename historical 'tomatoWifi'
Ignore-this: 8a3f1f261df50e8029cf9de5b11a6896
| author | drewp@bigasterisk.com |
|---|---|
| date | Sat, 30 Mar 2019 16:58:08 -0700 |
| parents | fdbdecdecd0b |
| children | 47d7dd31bb2c |
| files | service/tomatoWifi/Dockerfile service/tomatoWifi/index.html service/tomatoWifi/makefile service/tomatoWifi/requirements.txt service/tomatoWifi/table.mustache service/tomatoWifi/tomatoWifi.py service/tomatoWifi/wifi.py service/wifi/Dockerfile service/wifi/index.html service/wifi/makefile service/wifi/requirements.txt service/wifi/table.mustache service/wifi/tomatoWifi.py service/wifi/wifi.py |
| diffstat | 14 files changed, 636 insertions(+), 636 deletions(-) [+] |
line wrap: on
line diff
--- a/service/tomatoWifi/Dockerfile Sat Mar 30 16:57:08 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,14 +0,0 @@ -FROM bang6:5000/base_x86 - -WORKDIR /opt - -RUN apt-get install -y libxml2-dev libxslt1-dev - -COPY requirements.txt ./ -RUN pip install -r requirements.txt - -COPY *.py *.n3 *.json *.html ./ - -EXPOSE 9070 - -CMD [ "python", "tomatoWifi.py" ]
--- a/service/tomatoWifi/index.html Sat Mar 30 16:57:08 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,82 +0,0 @@ -<!doctype html> -<html> - <head> - <title>wifi</title> - <meta charset="utf-8" /> - <script src="/lib/polymer/1.0.9/webcomponentsjs/webcomponents-lite.min.js"></script> - <link rel="import" href="/lib/polymer/1.0.9/polymer/polymer.html"> - <script src="/lib/underscore-1.5.2.min.js"></script> - <link rel="import" href="/lib/polymer/1.0.9/iron-ajax/iron-ajax.html"> - <link rel="import" href="/rdf/n3+polymer/trig-store.html"> - </head> - <body> - <h1>Devices on wifi</h1> - <p><a href="../dhcpleases">go to dhcpleases</a></p> - <dom-module id="wifi-table"> - <template> - <iron-ajax auto url="graph" - handle-as="text" - last-response="{{ajaxResponse}}"></iron-ajax> - <trig-store id="ts" trig-input="{{ajaxResponse}}"></trig-store> - - <table> - <tr> - <th>name</th> - <th>MAC</th> - <th>Connected</th> - </tr> - <template is="dom-repeat" items="{{devices}}"> - <tr> - <td>{{item.deviceName}}</td> - <td>{{item.mac}}</td> - <td>{{item.connectedAgo}}</td> - </tr> - </template> - </table> - </template> - - <script> - HTMLImports.whenReady(function () { - Polymer({ - is: "wifi-table", - ready: function() { - this.$.ts.addEventListener('store-changed', this.storeChanged.bind(this)); - this.devices = []; - }, - storeChanged: function(ev) { - var store = ev.detail.value; - var find = function(s, p, o) { return store.findAllGraphs(s, p, o); }; - var findOne = function(s, p, o) { - var rows = find(s, p, o); - return rows[0]; - }; - - this.devices = []; - - find(null, "room:connected", "http://bigasterisk.com/wifiAccessPoints" - ).forEach(function(row) { - var out = { - mac: N3.Util.getLiteralValue( - findOne(row.subject, "room:macAddress", null).object), - connectedAgo: N3.Util.getLiteralValue( - findOne(row.subject, "room:connectedAgo", null).object) - - }; - try { - var dev = findOne(row.subject, "room:deviceName", null).object; - out.deviceName = N3.Util.getLiteralValue(dev); - } catch(e) { - - } - - this.devices.push(out); - }.bind(this)); - this.devices = _.sortBy(this.devices, 'deviceName'); - } - }); - }); - </script> - </dom-module> - <wifi-table></wifi-table> - </body> -</html>
--- a/service/tomatoWifi/makefile Sat Mar 30 16:57:08 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,21 +0,0 @@ -JOB=wifi -PORT=9070 - -TAG=bang6:5000/${JOB}_x86:latest - -build_image: - rm -rf tmp_ctx - mkdir -p tmp_ctx - cp -a Dockerfile ../../lib/*.py *.py *.n3 *.json *.html req* tmp_ctx - docker build --network=host -t ${TAG} tmp_ctx - docker push ${TAG} - rm -r tmp_ctx - -shell: - docker run --rm -it --cap-add SYS_PTRACE --net=host ${TAG} /bin/bash - -local_run: - docker run --rm -it -p ${PORT}:${PORT} \ - --net=host \ - ${TAG} \ - python tomatoWifi.py -v
--- a/service/tomatoWifi/requirements.txt Sat Mar 30 16:57:08 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,13 +0,0 @@ -docopt -pymongo - -BeautifulSoup==3.2.1 -cyclone -influxdb==3.0.0 -lxml==4.2.5 -pystache==0.5.2 -rdflib==4.2.2 -restkit==4.2.0 -web.py==0.37 - -https://projects.bigasterisk.com/rdfdb/rdfdb-0.6.0.tar.gz
--- a/service/tomatoWifi/table.mustache Sat Mar 30 16:57:08 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,28 +0,0 @@ -<?xml version="1.0" encoding="iso-8859-1"?> -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" -"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> -<div xmlns="http://www.w3.org/1999/xhtml" class="wifiNow"> - - <table><tr> - - <th class="name">Name</th> - <th class="connected">Connected</th> - <th class="ip">IP address</th> - <th class="rssi">dBm</th> - <th class="mac">MAC address</th> - <th class="router">Router</th> - </tr> - {{#rows}} - <tr class="{{cls}}"> - <td>{{name}}</td> - <td class="connected">{{connectedAgo}}</td> - <td class="ip">{{ip}}</td> - <td class="rssi">{{signal}}</td> - <td class="mac">{{mac}}</td> - <td>{{router}}</td> - </tr> - {{/rows}} - </table> - - -</div>
--- a/service/tomatoWifi/tomatoWifi.py Sat Mar 30 16:57:08 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,306 +0,0 @@ -#!/usr/bin/python -""" -scrape the tomato router status pages to see who's connected to the -wifi access points. Includes leases that aren't currently connected. - -Returns: - json listing (for magma page) - rdf graph (for reasoning) - activity stream, when we start saving history - -Todo: this should be the one polling and writing to mongo, not entrancemusic - -""" -from __future__ import division -import sys, cyclone.web, json, traceback, time, pystache, datetime, logging -import web.utils -from cyclone.httpclient import fetch - -from dateutil import tz -from twisted.internet import reactor, task -from twisted.internet.defer import inlineCallbacks -import docopt -from influxdb import InfluxDBClient -from pymongo import MongoClient as Connection, DESCENDING -from rdflib import Namespace, Literal, URIRef, ConjunctiveGraph - -from stategraph import StateGraph -from wifi import Wifi - -from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler - -from rdfdb.patch import Patch - -from cycloneerr import PrettyErrorHandler -from logsetup import log - - -DEV = Namespace("http://projects.bigasterisk.com/device/") -ROOM = Namespace("http://projects.bigasterisk.com/room/") -reasoning = "http://bang:9071/" - -class Index(PrettyErrorHandler, cyclone.web.RequestHandler): - def get(self): - - age = time.time() - self.settings.poller.lastPollTime - if age > 10: - raise ValueError("poll data is stale. age=%s" % age) - - self.set_header("Content-Type", "text/html") - self.write(open("index.html").read()) - -def whenConnected(mongo, macThatIsNowConnected): - lastArrive = None - for ev in mongo.find({'address': macThatIsNowConnected.upper()}, - sort=[('created', -1)], - max_scan=100000): - if ev['action'] == 'arrive': - lastArrive = ev - if ev['action'] == 'leave': - break - if lastArrive is None: - raise ValueError("no past arrivals") - - return lastArrive['created'] - -def connectedAgoString(conn): - return web.utils.datestr( - conn.astimezone(tz.tzutc()).replace(tzinfo=None)) - -class Table(PrettyErrorHandler, cyclone.web.RequestHandler): - def get(self): - def rowDict(row): - row['cls'] = "signal" if row.get('connected') else "nosignal" - if 'name' not in row: - row['name'] = row.get('clientHostname', '-') - if 'signal' not in row: - row['signal'] = 'yes' if row.get('connected') else 'no' - - try: - conn = whenConnected(self.settings.mongo, row.get('mac', '??')) - row['connectedAgo'] = connectedAgoString(conn) - except ValueError: - row['connectedAgo'] = 'yes' if row.get('connected') else '' - row['router'] = row.get('ssid', '') - return row - - self.set_header("Content-Type", "application/xhtml+xml") - self.write(pystache.render( - open("table.mustache").read(), - dict( - rows=sorted(map(rowDict, self.settings.poller.lastAddrs), - key=lambda a: (not a.get('connected'), - a.get('name')))))) - - -class Json(PrettyErrorHandler, cyclone.web.RequestHandler): - def get(self): - self.set_header("Content-Type", "application/json") - age = time.time() - self.settings.poller.lastPollTime - if age > 10: - raise ValueError("poll data is stale. age=%s" % age) - self.write(json.dumps({"wifi" : self.settings.poller.lastAddrs, - "dataAge" : age})) - -class Poller(object): - def __init__(self, wifi, mongo): - self.wifi = wifi - self.mongo = mongo - self.lastAddrs = [] - self.lastWithSignal = [] - self.lastPollTime = 0 - - def assertCurrent(self): - dt = time.time() - self.lastPollTime - assert dt < 10, "last poll was %s sec ago" % dt - - @inlineCallbacks - def poll(self): - connectedField = 'connected' - now = int(time.time()) - - # UVA mode: - addDhcpData = lambda *args: None - - try: - newAddrs = yield self.wifi.getPresentMacAddrs() - addDhcpData(newAddrs) - - newWithSignal = [a for a in newAddrs if a.get('connected')] - - actions = self.computeActions(newWithSignal) - points = [] - for action in actions: - log.info("action: %s", action) - action['created'] = datetime.datetime.now(tz.gettz('UTC')) - mongo.save(action) - points.append( - self.influxPoint(now, action['address'].lower(), - 1 if action['action'] == 'arrive' else 0)) - try: - self.doEntranceMusic(action) - except Exception, e: - log.error("entrancemusic error: %r", e) - - if now // 3600 > self.lastPollTime // 3600: - log.info('hourly writes') - for addr in newWithSignal: - points.append(self.influxPoint(now, addr['mac'].lower(), 1)) - - influx.write_points(points, time_precision='s') - self.lastWithSignal = newWithSignal - if actions: # this doesn't currently include signal strength changes - fetch(reasoning + "immediateUpdate", - method='PUT', - timeout=2, - headers={'user-agent': ['tomatoWifi']}).addErrback(log.warn) - self.lastAddrs = newAddrs - self.lastPollTime = now - - self.updateGraph(masterGraph) - except Exception, e: - log.error("poll error: %r\n%s", e, traceback.format_exc()) - - def influxPoint(self, now, address, value): - return { - 'measurement': 'presence', - 'tags': {'sensor': 'wifi', 'address': address,}, - 'fields': {'value': value}, - 'time': now, - } - - def computeActions(self, newWithSignal): - actions = [] - - def makeAction(addr, act): - d = dict(sensor="wifi", - address=addr.get('mac').upper(), # mongo data is legacy uppercase - name=addr.get('name'), - networkName=addr.get('clientHostname'), - action=act) - if act == 'arrive' and 'ip' in addr: - # this won't cover the possible case that you get on - # wifi but don't have an ip yet. We'll record an - # action with no ip and then never record your ip. - d['ip'] = addr['ip'] - return d - - for addr in newWithSignal: - if addr['mac'] not in [r['mac'] for r in self.lastWithSignal]: - actions.append(makeAction(addr, 'arrive')) - - for addr in self.lastWithSignal: - if addr['mac'] not in [r['mac'] for r in newWithSignal]: - actions.append(makeAction(addr, 'leave')) - - return actions - - - # these need to move out to their own service - def doEntranceMusic(self, action): - import restkit, json - dt = self.deltaSinceLastArrive(action['name']) - log.debug("dt=%s", dt) - if dt > datetime.timedelta(hours=1): - hub = restkit.Resource( - # PSHB not working yet; "http://bang:9030/" - "http://slash:9049/" - ) - action = action.copy() - del action['created'] - del action['_id'] - log.info("post to %s", hub) - hub.post("visitorNet", payload=json.dumps(action)) - - def deltaSinceLastArrive(self, name): - results = list(self.mongo.find({'name' : name}).sort('created', - DESCENDING).limit(1)) - if not results: - return datetime.timedelta.max - now = datetime.datetime.now(tz.gettz('UTC')) - last = results[0]['created'].replace(tzinfo=tz.gettz('UTC')) - return now - last - - def updateGraph(self, masterGraph): - - g = ConjunctiveGraph() - ctx = DEV['wifi'] - - # someday i may also record specific AP and their strength, - # for positioning. But many users just want to know that the - # device is connected to some bigasterisk AP. - age = time.time() - self.lastPollTime - if age > 10: - raise ValueError("poll data is stale. age=%s" % age) - - for dev in self.lastAddrs: - if not dev.get('connected'): - continue - uri = URIRef("http://bigasterisk.com/mac/%s" % dev['mac'].lower()) - g.add((uri, ROOM['macAddress'], Literal(dev['mac'].lower()), ctx)) - - g.add((uri, ROOM['connected'], { - 'wireless': URIRef("http://bigasterisk.com/wifiAccessPoints"), - '2.4G': URIRef("http://bigasterisk.com/wifiAccessPoints"), - '5G': URIRef("http://bigasterisk.com/wifiAccessPoints"), - '-': URIRef("http://bigasterisk.com/wifiUnknownConnectionType"), - 'Unknown': URIRef("http://bigasterisk.com/wifiUnknownConnectionType"), - 'wired': URIRef("http://bigasterisk.com/houseOpenNet")}[dev['contype']], ctx)) - if 'clientHostname' in dev and dev['clientHostname']: - g.add((uri, ROOM['wifiNetworkName'], Literal(dev['clientHostname']), ctx)) - if 'name' in dev and dev['name']: - g.add((uri, ROOM['deviceName'], Literal(dev['name']), ctx)) - if 'signal' in dev: - g.add((uri, ROOM['signalStrength'], Literal(dev['signal']), ctx)) - if 'model' in dev: - g.add((uri, ROOM['networkModel'], Literal(dev['model']), ctx)) - try: - conn = whenConnected(mongo, dev['mac']) - except ValueError: - traceback.print_exc() - pass - else: - g.add((uri, ROOM['connectedAgo'], - Literal(connectedAgoString(conn)), ctx)) - g.add((uri, ROOM['connected'], Literal(conn), ctx)) - masterGraph.setToGraph(g) - - -if __name__ == '__main__': - args = docopt.docopt(''' -Usage: - tomatoWifi [options] - -Options: - -v, --verbose more logging - --port=<n> serve on port [default: 9070] - --poll=<freq> poll frequency [default: .2] -''') - if args['--verbose']: - from twisted.python import log as twlog - twlog.startLogging(sys.stdout) - log.setLevel(10) - log.setLevel(logging.DEBUG) - - mongo = Connection('bang', 27017, tz_aware=True)['visitor']['visitor'] - influx = InfluxDBClient('bang', 9060, 'root', 'root', 'main') - - masterGraph = PatchableGraph() - wifi = Wifi() - poller = Poller(wifi, mongo) - task.LoopingCall(poller.poll).start(1/float(args['--poll'])) - - reactor.listenTCP(int(args['--port']), - cyclone.web.Application( - [ - (r"/", Index), - (r'/json', Json), - (r'/graph', CycloneGraphHandler, {'masterGraph': masterGraph}), - (r'/graph/events', CycloneGraphEventsHandler, {'masterGraph': masterGraph}), - (r'/table', Table), - #(r'/activity', Activity), - ], - wifi=wifi, - poller=poller, - mongo=mongo)) - reactor.run()
--- a/service/tomatoWifi/wifi.py Sat Mar 30 16:57:08 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,172 +0,0 @@ -import re, ast, logging, socket, json -import lxml.html.soupparser -from twisted.internet.defer import inlineCallbacks, returnValue -from cyclone.httpclient import fetch -from rdflib import Literal, Graph, RDFS, URIRef - -log = logging.getLogger() - -class Router(object): - def __repr__(self): - return repr(self.__dict__) - -class Wifi(object): - """ - gather the users of wifi from the tomato routers - """ - def __init__(self, accessN3="/my/proj/openid_proxy/access.n3"): - self.rereadConfig() - #self._loadRouters(accessN3, tomatoUrl) - - def rereadConfig(self): - self.graph = Graph() - self.graph.parse('config.n3', format='n3') - - - def _loadRouters(self, accessN3, tomatoUrl): - g = Graph() - g.parse(accessN3, format="n3") - repl = { - '/wifiRouter1/' : None, - #'/tomato2/' : None - } - for k in repl: - rows = list(g.query(''' - PREFIX p: <http://bigasterisk.com/openid_proxy#> - SELECT ?prefix WHERE { - ?site - p:requestPrefix ?public; - p:proxyUrlPrefix ?prefix - . - }''', initBindings={"public" : Literal(k)})) - repl[k] = str(rows[0][0]) - log.debug('repl %r', repl) - - self.routers = [] - for url in tomatoUrl: - name = url - for k, v in repl.items(): - url = url.replace(k, v) - - r = Router() - http, tail = url.split('//', 1) - userPass, tail = tail.split("@", 1) - r.url = http + '//' + tail - r.headers = {'Authorization': ['Basic %s' % userPass.encode('base64').strip()]} - r.name = {'wifiRouter1' : 'bigasterisk5', - 'tomato2' : 'bigasterisk4'}[name.split('/')[1]] - self.routers.append(r) - - @inlineCallbacks - def getPresentMacAddrs(self): - self.rereadConfig() - rows = yield loadOrbiData() - for row in rows: - if 'clientHostname' in row: - row['name'] = row['clientHostname'] - mac = URIRef('http://bigasterisk.com/mac/%s' % row['mac'].lower()) - label = self.graph.value(mac, RDFS.label) - if label: - row['name'] = label - returnValue(rows) - - @inlineCallbacks - def getPresentMacAddrs_multirouter(self): - rows = [] - - for router in self.routers: - log.debug("GET %s", router) - try: - resp = yield fetch(router.url, headers=router.headers, - timeout=2) - except socket.error: - log.warn("get on %s failed" % router) - continue - data = resp.body - if 'Wireless -- Authenticated Stations' in data: - # zyxel 'Station Info' page - rows.extend(self._parseZyxel(data, router.name)) - else: - # tomato page - rows.extend(self._parseTomato(data, router.name)) - - for r in rows: - try: - r['name'] = self.knownMacAddr[r['mac']] - except KeyError: - pass - - returnValue(rows) - - def _parseZyxel(self, data, routerName): - root = lxml.html.soupparser.fromstring(data) - for tr in root.cssselect('tr'): - mac, assoc, uth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] - if mac == "MAC": - continue - assoc = assoc.lower() == 'yes' - yield dict(router=routerName, mac=mac, assoc=assoc, connected=assoc) - - def _parseTomato(self, data, routerName): - for iface, mac, signal in jsValue(data, 'wldev'): - yield dict(router=routerName, mac=mac, signal=signal, connected=bool(signal)) - - -@inlineCallbacks -def loadUvaData(): - config = json.load(open("priv-uva.json")) - headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} - resp = yield fetch('http://10.2.0.2/wlstationlist.cmd', headers=headers) - root = lxml.html.soupparser.fromstring(resp.body) - byMac = {} - for tr in root.cssselect('tr'): - mac, connected, auth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] - if mac == "MAC": - continue - connected = connected.lower() == 'yes' - byMac[mac] = dict(mac=mac, connected=connected, auth=auth == 'Yes', ssid=ssid, iface=iface) - - resp = yield fetch('http://10.2.0.2/DHCPTable.asp', headers=headers) - for row in re.findall(r'new AAA\((.*)\)', resp.body): - clientHostname, ipaddr, mac, expires, iface = [s.strip("'") for s in row.rsplit(',', 4)] - if clientHostname == 'wlanadv.none': - continue - byMac.setdefault(mac, {}).update(dict( - clientHostname=clientHostname, connection=iface, ipaddr=ipaddr, dhcpExpires=expires)) - - returnValue(sorted(byMac.values())) - -@inlineCallbacks -def loadCiscoData(): - config = json.load(open("priv-uva.json")) - headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} - print headers - resp = yield fetch('http://10.2.0.2/', headers=headers) - print resp.body - returnValue([]) - -@inlineCallbacks -def loadOrbiData(): - config = json.load(open("priv-uva.json")) - headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} - resp = yield fetch('http://orbi.bigasterisk.com/DEV_device_info.htm', headers=headers) - - if not resp.body.startswith(('device=', 'device_changed=0\ndevice=', 'device_changed=1\ndevice=')): - raise ValueError(resp.body) - - ret = [] - for row in json.loads(resp.body.split('device=', 1)[-1]): - ret.append(dict( - connected=True, - ipaddr=row['ip'], - mac=row['mac'].lower(), - contype=row['contype'], - model=row['model'], - clientHostname=row['name'] if row['name'] != 'Unknown' else None)) - returnValue(ret) - - -def jsValue(js, variableName): - # using literal_eval instead of json parser to handle the trailing commas - val = re.search(variableName + r'\s*=\s*(.*?);', js, re.DOTALL).group(1) - return ast.literal_eval(val)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/service/wifi/Dockerfile Sat Mar 30 16:58:08 2019 -0700 @@ -0,0 +1,14 @@ +FROM bang6:5000/base_x86 + +WORKDIR /opt + +RUN apt-get install -y libxml2-dev libxslt1-dev + +COPY requirements.txt ./ +RUN pip install -r requirements.txt + +COPY *.py *.n3 *.json *.html ./ + +EXPOSE 9070 + +CMD [ "python", "tomatoWifi.py" ]
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/service/wifi/index.html Sat Mar 30 16:58:08 2019 -0700 @@ -0,0 +1,82 @@ +<!doctype html> +<html> + <head> + <title>wifi</title> + <meta charset="utf-8" /> + <script src="/lib/polymer/1.0.9/webcomponentsjs/webcomponents-lite.min.js"></script> + <link rel="import" href="/lib/polymer/1.0.9/polymer/polymer.html"> + <script src="/lib/underscore-1.5.2.min.js"></script> + <link rel="import" href="/lib/polymer/1.0.9/iron-ajax/iron-ajax.html"> + <link rel="import" href="/rdf/n3+polymer/trig-store.html"> + </head> + <body> + <h1>Devices on wifi</h1> + <p><a href="../dhcpleases">go to dhcpleases</a></p> + <dom-module id="wifi-table"> + <template> + <iron-ajax auto url="graph" + handle-as="text" + last-response="{{ajaxResponse}}"></iron-ajax> + <trig-store id="ts" trig-input="{{ajaxResponse}}"></trig-store> + + <table> + <tr> + <th>name</th> + <th>MAC</th> + <th>Connected</th> + </tr> + <template is="dom-repeat" items="{{devices}}"> + <tr> + <td>{{item.deviceName}}</td> + <td>{{item.mac}}</td> + <td>{{item.connectedAgo}}</td> + </tr> + </template> + </table> + </template> + + <script> + HTMLImports.whenReady(function () { + Polymer({ + is: "wifi-table", + ready: function() { + this.$.ts.addEventListener('store-changed', this.storeChanged.bind(this)); + this.devices = []; + }, + storeChanged: function(ev) { + var store = ev.detail.value; + var find = function(s, p, o) { return store.findAllGraphs(s, p, o); }; + var findOne = function(s, p, o) { + var rows = find(s, p, o); + return rows[0]; + }; + + this.devices = []; + + find(null, "room:connected", "http://bigasterisk.com/wifiAccessPoints" + ).forEach(function(row) { + var out = { + mac: N3.Util.getLiteralValue( + findOne(row.subject, "room:macAddress", null).object), + connectedAgo: N3.Util.getLiteralValue( + findOne(row.subject, "room:connectedAgo", null).object) + + }; + try { + var dev = findOne(row.subject, "room:deviceName", null).object; + out.deviceName = N3.Util.getLiteralValue(dev); + } catch(e) { + + } + + this.devices.push(out); + }.bind(this)); + this.devices = _.sortBy(this.devices, 'deviceName'); + } + }); + }); + </script> + </dom-module> + <wifi-table></wifi-table> + </body> +</html>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/service/wifi/makefile Sat Mar 30 16:58:08 2019 -0700 @@ -0,0 +1,21 @@ +JOB=wifi +PORT=9070 + +TAG=bang6:5000/${JOB}_x86:latest + +build_image: + rm -rf tmp_ctx + mkdir -p tmp_ctx + cp -a Dockerfile ../../lib/*.py *.py *.n3 *.json *.html req* tmp_ctx + docker build --network=host -t ${TAG} tmp_ctx + docker push ${TAG} + rm -r tmp_ctx + +shell: + docker run --rm -it --cap-add SYS_PTRACE --net=host ${TAG} /bin/bash + +local_run: + docker run --rm -it -p ${PORT}:${PORT} \ + --net=host \ + ${TAG} \ + python tomatoWifi.py -v
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/service/wifi/requirements.txt Sat Mar 30 16:58:08 2019 -0700 @@ -0,0 +1,13 @@ +docopt +pymongo + +BeautifulSoup==3.2.1 +cyclone +influxdb==3.0.0 +lxml==4.2.5 +pystache==0.5.2 +rdflib==4.2.2 +restkit==4.2.0 +web.py==0.37 + +https://projects.bigasterisk.com/rdfdb/rdfdb-0.6.0.tar.gz
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/service/wifi/table.mustache Sat Mar 30 16:58:08 2019 -0700 @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="iso-8859-1"?> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" +"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> +<div xmlns="http://www.w3.org/1999/xhtml" class="wifiNow"> + + <table><tr> + + <th class="name">Name</th> + <th class="connected">Connected</th> + <th class="ip">IP address</th> + <th class="rssi">dBm</th> + <th class="mac">MAC address</th> + <th class="router">Router</th> + </tr> + {{#rows}} + <tr class="{{cls}}"> + <td>{{name}}</td> + <td class="connected">{{connectedAgo}}</td> + <td class="ip">{{ip}}</td> + <td class="rssi">{{signal}}</td> + <td class="mac">{{mac}}</td> + <td>{{router}}</td> + </tr> + {{/rows}} + </table> + + +</div>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/service/wifi/tomatoWifi.py Sat Mar 30 16:58:08 2019 -0700 @@ -0,0 +1,306 @@ +#!/usr/bin/python +""" +scrape the tomato router status pages to see who's connected to the +wifi access points. Includes leases that aren't currently connected. + +Returns: + json listing (for magma page) + rdf graph (for reasoning) + activity stream, when we start saving history + +Todo: this should be the one polling and writing to mongo, not entrancemusic + +""" +from __future__ import division +import sys, cyclone.web, json, traceback, time, pystache, datetime, logging +import web.utils +from cyclone.httpclient import fetch + +from dateutil import tz +from twisted.internet import reactor, task +from twisted.internet.defer import inlineCallbacks +import docopt +from influxdb import InfluxDBClient +from pymongo import MongoClient as Connection, DESCENDING +from rdflib import Namespace, Literal, URIRef, ConjunctiveGraph + +from stategraph import StateGraph +from wifi import Wifi + +from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler + +from rdfdb.patch import Patch + +from cycloneerr import PrettyErrorHandler +from logsetup import log + + +DEV = Namespace("http://projects.bigasterisk.com/device/") +ROOM = Namespace("http://projects.bigasterisk.com/room/") +reasoning = "http://bang:9071/" + +class Index(PrettyErrorHandler, cyclone.web.RequestHandler): + def get(self): + + age = time.time() - self.settings.poller.lastPollTime + if age > 10: + raise ValueError("poll data is stale. age=%s" % age) + + self.set_header("Content-Type", "text/html") + self.write(open("index.html").read()) + +def whenConnected(mongo, macThatIsNowConnected): + lastArrive = None + for ev in mongo.find({'address': macThatIsNowConnected.upper()}, + sort=[('created', -1)], + max_scan=100000): + if ev['action'] == 'arrive': + lastArrive = ev + if ev['action'] == 'leave': + break + if lastArrive is None: + raise ValueError("no past arrivals") + + return lastArrive['created'] + +def connectedAgoString(conn): + return web.utils.datestr( + conn.astimezone(tz.tzutc()).replace(tzinfo=None)) + +class Table(PrettyErrorHandler, cyclone.web.RequestHandler): + def get(self): + def rowDict(row): + row['cls'] = "signal" if row.get('connected') else "nosignal" + if 'name' not in row: + row['name'] = row.get('clientHostname', '-') + if 'signal' not in row: + row['signal'] = 'yes' if row.get('connected') else 'no' + + try: + conn = whenConnected(self.settings.mongo, row.get('mac', '??')) + row['connectedAgo'] = connectedAgoString(conn) + except ValueError: + row['connectedAgo'] = 'yes' if row.get('connected') else '' + row['router'] = row.get('ssid', '') + return row + + self.set_header("Content-Type", "application/xhtml+xml") + self.write(pystache.render( + open("table.mustache").read(), + dict( + rows=sorted(map(rowDict, self.settings.poller.lastAddrs), + key=lambda a: (not a.get('connected'), + a.get('name')))))) + + +class Json(PrettyErrorHandler, cyclone.web.RequestHandler): + def get(self): + self.set_header("Content-Type", "application/json") + age = time.time() - self.settings.poller.lastPollTime + if age > 10: + raise ValueError("poll data is stale. age=%s" % age) + self.write(json.dumps({"wifi" : self.settings.poller.lastAddrs, + "dataAge" : age})) + +class Poller(object): + def __init__(self, wifi, mongo): + self.wifi = wifi + self.mongo = mongo + self.lastAddrs = [] + self.lastWithSignal = [] + self.lastPollTime = 0 + + def assertCurrent(self): + dt = time.time() - self.lastPollTime + assert dt < 10, "last poll was %s sec ago" % dt + + @inlineCallbacks + def poll(self): + connectedField = 'connected' + now = int(time.time()) + + # UVA mode: + addDhcpData = lambda *args: None + + try: + newAddrs = yield self.wifi.getPresentMacAddrs() + addDhcpData(newAddrs) + + newWithSignal = [a for a in newAddrs if a.get('connected')] + + actions = self.computeActions(newWithSignal) + points = [] + for action in actions: + log.info("action: %s", action) + action['created'] = datetime.datetime.now(tz.gettz('UTC')) + mongo.save(action) + points.append( + self.influxPoint(now, action['address'].lower(), + 1 if action['action'] == 'arrive' else 0)) + try: + self.doEntranceMusic(action) + except Exception, e: + log.error("entrancemusic error: %r", e) + + if now // 3600 > self.lastPollTime // 3600: + log.info('hourly writes') + for addr in newWithSignal: + points.append(self.influxPoint(now, addr['mac'].lower(), 1)) + + influx.write_points(points, time_precision='s') + self.lastWithSignal = newWithSignal + if actions: # this doesn't currently include signal strength changes + fetch(reasoning + "immediateUpdate", + method='PUT', + timeout=2, + headers={'user-agent': ['tomatoWifi']}).addErrback(log.warn) + self.lastAddrs = newAddrs + self.lastPollTime = now + + self.updateGraph(masterGraph) + except Exception, e: + log.error("poll error: %r\n%s", e, traceback.format_exc()) + + def influxPoint(self, now, address, value): + return { + 'measurement': 'presence', + 'tags': {'sensor': 'wifi', 'address': address,}, + 'fields': {'value': value}, + 'time': now, + } + + def computeActions(self, newWithSignal): + actions = [] + + def makeAction(addr, act): + d = dict(sensor="wifi", + address=addr.get('mac').upper(), # mongo data is legacy uppercase + name=addr.get('name'), + networkName=addr.get('clientHostname'), + action=act) + if act == 'arrive' and 'ip' in addr: + # this won't cover the possible case that you get on + # wifi but don't have an ip yet. We'll record an + # action with no ip and then never record your ip. + d['ip'] = addr['ip'] + return d + + for addr in newWithSignal: + if addr['mac'] not in [r['mac'] for r in self.lastWithSignal]: + actions.append(makeAction(addr, 'arrive')) + + for addr in self.lastWithSignal: + if addr['mac'] not in [r['mac'] for r in newWithSignal]: + actions.append(makeAction(addr, 'leave')) + + return actions + + + # these need to move out to their own service + def doEntranceMusic(self, action): + import restkit, json + dt = self.deltaSinceLastArrive(action['name']) + log.debug("dt=%s", dt) + if dt > datetime.timedelta(hours=1): + hub = restkit.Resource( + # PSHB not working yet; "http://bang:9030/" + "http://slash:9049/" + ) + action = action.copy() + del action['created'] + del action['_id'] + log.info("post to %s", hub) + hub.post("visitorNet", payload=json.dumps(action)) + + def deltaSinceLastArrive(self, name): + results = list(self.mongo.find({'name' : name}).sort('created', + DESCENDING).limit(1)) + if not results: + return datetime.timedelta.max + now = datetime.datetime.now(tz.gettz('UTC')) + last = results[0]['created'].replace(tzinfo=tz.gettz('UTC')) + return now - last + + def updateGraph(self, masterGraph): + + g = ConjunctiveGraph() + ctx = DEV['wifi'] + + # someday i may also record specific AP and their strength, + # for positioning. But many users just want to know that the + # device is connected to some bigasterisk AP. + age = time.time() - self.lastPollTime + if age > 10: + raise ValueError("poll data is stale. age=%s" % age) + + for dev in self.lastAddrs: + if not dev.get('connected'): + continue + uri = URIRef("http://bigasterisk.com/mac/%s" % dev['mac'].lower()) + g.add((uri, ROOM['macAddress'], Literal(dev['mac'].lower()), ctx)) + + g.add((uri, ROOM['connected'], { + 'wireless': URIRef("http://bigasterisk.com/wifiAccessPoints"), + '2.4G': URIRef("http://bigasterisk.com/wifiAccessPoints"), + '5G': URIRef("http://bigasterisk.com/wifiAccessPoints"), + '-': URIRef("http://bigasterisk.com/wifiUnknownConnectionType"), + 'Unknown': URIRef("http://bigasterisk.com/wifiUnknownConnectionType"), + 'wired': URIRef("http://bigasterisk.com/houseOpenNet")}[dev['contype']], ctx)) + if 'clientHostname' in dev and dev['clientHostname']: + g.add((uri, ROOM['wifiNetworkName'], Literal(dev['clientHostname']), ctx)) + if 'name' in dev and dev['name']: + g.add((uri, ROOM['deviceName'], Literal(dev['name']), ctx)) + if 'signal' in dev: + g.add((uri, ROOM['signalStrength'], Literal(dev['signal']), ctx)) + if 'model' in dev: + g.add((uri, ROOM['networkModel'], Literal(dev['model']), ctx)) + try: + conn = whenConnected(mongo, dev['mac']) + except ValueError: + traceback.print_exc() + pass + else: + g.add((uri, ROOM['connectedAgo'], + Literal(connectedAgoString(conn)), ctx)) + g.add((uri, ROOM['connected'], Literal(conn), ctx)) + masterGraph.setToGraph(g) + + +if __name__ == '__main__': + args = docopt.docopt(''' +Usage: + tomatoWifi [options] + +Options: + -v, --verbose more logging + --port=<n> serve on port [default: 9070] + --poll=<freq> poll frequency [default: .2] +''') + if args['--verbose']: + from twisted.python import log as twlog + twlog.startLogging(sys.stdout) + log.setLevel(10) + log.setLevel(logging.DEBUG) + + mongo = Connection('bang', 27017, tz_aware=True)['visitor']['visitor'] + influx = InfluxDBClient('bang', 9060, 'root', 'root', 'main') + + masterGraph = PatchableGraph() + wifi = Wifi() + poller = Poller(wifi, mongo) + task.LoopingCall(poller.poll).start(1/float(args['--poll'])) + + reactor.listenTCP(int(args['--port']), + cyclone.web.Application( + [ + (r"/", Index), + (r'/json', Json), + (r'/graph', CycloneGraphHandler, {'masterGraph': masterGraph}), + (r'/graph/events', CycloneGraphEventsHandler, {'masterGraph': masterGraph}), + (r'/table', Table), + #(r'/activity', Activity), + ], + wifi=wifi, + poller=poller, + mongo=mongo)) + reactor.run()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/service/wifi/wifi.py Sat Mar 30 16:58:08 2019 -0700 @@ -0,0 +1,172 @@ +import re, ast, logging, socket, json +import lxml.html.soupparser +from twisted.internet.defer import inlineCallbacks, returnValue +from cyclone.httpclient import fetch +from rdflib import Literal, Graph, RDFS, URIRef + +log = logging.getLogger() + +class Router(object): + def __repr__(self): + return repr(self.__dict__) + +class Wifi(object): + """ + gather the users of wifi from the tomato routers + """ + def __init__(self, accessN3="/my/proj/openid_proxy/access.n3"): + self.rereadConfig() + #self._loadRouters(accessN3, tomatoUrl) + + def rereadConfig(self): + self.graph = Graph() + self.graph.parse('config.n3', format='n3') + + + def _loadRouters(self, accessN3, tomatoUrl): + g = Graph() + g.parse(accessN3, format="n3") + repl = { + '/wifiRouter1/' : None, + #'/tomato2/' : None + } + for k in repl: + rows = list(g.query(''' + PREFIX p: <http://bigasterisk.com/openid_proxy#> + SELECT ?prefix WHERE { + ?site + p:requestPrefix ?public; + p:proxyUrlPrefix ?prefix + . + }''', initBindings={"public" : Literal(k)})) + repl[k] = str(rows[0][0]) + log.debug('repl %r', repl) + + self.routers = [] + for url in tomatoUrl: + name = url + for k, v in repl.items(): + url = url.replace(k, v) + + r = Router() + http, tail = url.split('//', 1) + userPass, tail = tail.split("@", 1) + r.url = http + '//' + tail + r.headers = {'Authorization': ['Basic %s' % userPass.encode('base64').strip()]} + r.name = {'wifiRouter1' : 'bigasterisk5', + 'tomato2' : 'bigasterisk4'}[name.split('/')[1]] + self.routers.append(r) + + @inlineCallbacks + def getPresentMacAddrs(self): + self.rereadConfig() + rows = yield loadOrbiData() + for row in rows: + if 'clientHostname' in row: + row['name'] = row['clientHostname'] + mac = URIRef('http://bigasterisk.com/mac/%s' % row['mac'].lower()) + label = self.graph.value(mac, RDFS.label) + if label: + row['name'] = label + returnValue(rows) + + @inlineCallbacks + def getPresentMacAddrs_multirouter(self): + rows = [] + + for router in self.routers: + log.debug("GET %s", router) + try: + resp = yield fetch(router.url, headers=router.headers, + timeout=2) + except socket.error: + log.warn("get on %s failed" % router) + continue + data = resp.body + if 'Wireless -- Authenticated Stations' in data: + # zyxel 'Station Info' page + rows.extend(self._parseZyxel(data, router.name)) + else: + # tomato page + rows.extend(self._parseTomato(data, router.name)) + + for r in rows: + try: + r['name'] = self.knownMacAddr[r['mac']] + except KeyError: + pass + + returnValue(rows) + + def _parseZyxel(self, data, routerName): + root = lxml.html.soupparser.fromstring(data) + for tr in root.cssselect('tr'): + mac, assoc, uth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] + if mac == "MAC": + continue + assoc = assoc.lower() == 'yes' + yield dict(router=routerName, mac=mac, assoc=assoc, connected=assoc) + + def _parseTomato(self, data, routerName): + for iface, mac, signal in jsValue(data, 'wldev'): + yield dict(router=routerName, mac=mac, signal=signal, connected=bool(signal)) + + +@inlineCallbacks +def loadUvaData(): + config = json.load(open("priv-uva.json")) + headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} + resp = yield fetch('http://10.2.0.2/wlstationlist.cmd', headers=headers) + root = lxml.html.soupparser.fromstring(resp.body) + byMac = {} + for tr in root.cssselect('tr'): + mac, connected, auth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] + if mac == "MAC": + continue + connected = connected.lower() == 'yes' + byMac[mac] = dict(mac=mac, connected=connected, auth=auth == 'Yes', ssid=ssid, iface=iface) + + resp = yield fetch('http://10.2.0.2/DHCPTable.asp', headers=headers) + for row in re.findall(r'new AAA\((.*)\)', resp.body): + clientHostname, ipaddr, mac, expires, iface = [s.strip("'") for s in row.rsplit(',', 4)] + if clientHostname == 'wlanadv.none': + continue + byMac.setdefault(mac, {}).update(dict( + clientHostname=clientHostname, connection=iface, ipaddr=ipaddr, dhcpExpires=expires)) + + returnValue(sorted(byMac.values())) + +@inlineCallbacks +def loadCiscoData(): + config = json.load(open("priv-uva.json")) + headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} + print headers + resp = yield fetch('http://10.2.0.2/', headers=headers) + print resp.body + returnValue([]) + +@inlineCallbacks +def loadOrbiData(): + config = json.load(open("priv-uva.json")) + headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} + resp = yield fetch('http://orbi.bigasterisk.com/DEV_device_info.htm', headers=headers) + + if not resp.body.startswith(('device=', 'device_changed=0\ndevice=', 'device_changed=1\ndevice=')): + raise ValueError(resp.body) + + ret = [] + for row in json.loads(resp.body.split('device=', 1)[-1]): + ret.append(dict( + connected=True, + ipaddr=row['ip'], + mac=row['mac'].lower(), + contype=row['contype'], + model=row['model'], + clientHostname=row['name'] if row['name'] != 'Unknown' else None)) + returnValue(ret) + + +def jsValue(js, variableName): + # using literal_eval instead of json parser to handle the trailing commas + val = re.search(variableName + r'\s*=\s*(.*?);', js, re.DOTALL).group(1) + return ast.literal_eval(val)