Mercurial > code > home > repos > homeauto

comparison service/wifi/wifi.py @ 1224:e1202af42d4d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
port to py3 Ignore-this: e1a2e6bb730111e76f5a5dd2366d498a darcs-hash:491edbe1604e4024b3c61145a8022caaec25fbd5
author drewp <drewp@bigasterisk.com>
date Sat, 30 Mar 2019 18:27:17 -0700
parents 34de6cfa0b6b
children b8c0daabe5a5
comparison
equal deleted inserted replaced
1223:34de6cfa0b6b 1224:e1202af42d4d
1 import re, ast, logging, socket, json 1 #!/usr/bin/python
2 import lxml.html.soupparser 2 """
3 from twisted.internet.defer import inlineCallbacks, returnValue 3 scrape the tomato router status pages to see who's connected to the
4 wifi access points. Includes leases that aren't currently connected.
5
6 Returns:
7 json listing (for magma page)
8 rdf graph (for reasoning)
9 activity stream, when we start saving history
10
11 Todo: this should be the one polling and writing to mongo, not entrancemusic
12
13 """
14 from __future__ import division
15 import sys, cyclone.web, json, traceback, time, pystache, datetime, logging
4 from cyclone.httpclient import fetch 16 from cyclone.httpclient import fetch
5 from rdflib import Literal, Graph, RDFS, URIRef 17
6 18 from dateutil import tz
7 log = logging.getLogger() 19 from twisted.internet import reactor, task
8 20 from twisted.internet.defer import inlineCallbacks
9 class Router(object): 21 import docopt
10 def __repr__(self): 22 from influxdb import InfluxDBClient
11 return repr(self.__dict__) 23 from pymongo import MongoClient as Connection, DESCENDING
12 24 from rdflib import Namespace, Literal, URIRef, ConjunctiveGraph
13 class Wifi(object): 25
14 """ 26 from scrape import Wifi
15 gather the users of wifi from the tomato routers 27
16 """ 28 from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler
17 def __init__(self, accessN3="/my/proj/openid_proxy/access.n3"): 29
18 self.rereadConfig() 30 from cycloneerr import PrettyErrorHandler
19 #self._loadRouters(accessN3, tomatoUrl) 31 from logsetup import log
20 32
21 def rereadConfig(self): 33
22 self.graph = Graph() 34 DEV = Namespace("http://projects.bigasterisk.com/device/")
23 self.graph.parse('config.n3', format='n3') 35 ROOM = Namespace("http://projects.bigasterisk.com/room/")
36 reasoning = "http://bang:9071/"
37
38 class Index(PrettyErrorHandler, cyclone.web.RequestHandler):
39 def get(self):
40
41 age = time.time() - self.settings.poller.lastPollTime
42 if age > 10:
43 raise ValueError("poll data is stale. age=%s" % age)
44
45 self.set_header("Content-Type", "text/html")
46 self.write(open("index.html").read())
47
48 def whenConnected(mongo, macThatIsNowConnected):
49 lastArrive = None
50 for ev in mongo.find({'address': macThatIsNowConnected.upper()},
51 sort=[('created', -1)],
52 max_scan=100000):
53 if ev['action'] == 'arrive':
54 lastArrive = ev
55 if ev['action'] == 'leave':
56 break
57 if lastArrive is None:
58 raise ValueError("no past arrivals")
59
60 return lastArrive['created']
61
62 def connectedAgoString(conn):
63 return web.utils.datestr(
64 conn.astimezone(tz.tzutc()).replace(tzinfo=None))
65
66 class Table(PrettyErrorHandler, cyclone.web.RequestHandler):
67 def get(self):
68 def rowDict(row):
69 row['cls'] = "signal" if row.get('connected') else "nosignal"
70 if 'name' not in row:
71 row['name'] = row.get('clientHostname', '-')
72 if 'signal' not in row:
73 row['signal'] = 'yes' if row.get('connected') else 'no'
74
75 try:
76 conn = whenConnected(self.settings.mongo, row.get('mac', '??'))
77 row['connectedAgo'] = connectedAgoString(conn)
78 except ValueError:
79 row['connectedAgo'] = 'yes' if row.get('connected') else ''
80 row['router'] = row.get('ssid', '')
81 return row
82
83 self.set_header("Content-Type", "application/xhtml+xml")
84 self.write(pystache.render(
85 open("table.mustache").read(),
86 dict(
87 rows=sorted(map(rowDict, self.settings.poller.lastAddrs),
88 key=lambda a: (not a.get('connected'),
89 a.get('name'))))))
90
91
92 class Json(PrettyErrorHandler, cyclone.web.RequestHandler):
93 def get(self):
94 self.set_header("Content-Type", "application/json")
95 age = time.time() - self.settings.poller.lastPollTime
96 if age > 10:
97 raise ValueError("poll data is stale. age=%s" % age)
98 self.write(json.dumps({"wifi" : self.settings.poller.lastAddrs,
99 "dataAge" : age}))
100
101 class Poller(object):
102 def __init__(self, wifi, mongo):
103 self.wifi = wifi
104 self.mongo = mongo
105 self.lastAddrs = []
106 self.lastWithSignal = []
107 self.lastPollTime = 0
108
109 def assertCurrent(self):
110 dt = time.time() - self.lastPollTime
111 assert dt < 10, "last poll was %s sec ago" % dt
112
113 @inlineCallbacks
114 def poll(self):
115 now = int(time.time())
24 116
117 # UVA mode:
118 addDhcpData = lambda *args: None
25 119
26 def _loadRouters(self, accessN3, tomatoUrl): 120 try:
27 g = Graph() 121 newAddrs = yield self.wifi.getPresentMacAddrs()
28 g.parse(accessN3, format="n3") 122 addDhcpData(newAddrs)
29 repl = { 123
30 '/wifiRouter1/' : None, 124 newWithSignal = [a for a in newAddrs if a.get('connected')]
31 #'/tomato2/' : None 125
126 actions = self.computeActions(newWithSignal)
127 points = []
128 for action in actions:
129 log.info("action: %s", action)
130 action['created'] = datetime.datetime.now(tz.gettz('UTC'))
131 mongo.save(action)
132 points.append(
133 self.influxPoint(now, action['address'].lower(),
134 1 if action['action'] == 'arrive' else 0))
135 try:
136 self.doEntranceMusic(action)
137 except Exception as e:
138 log.error("entrancemusic error: %r", e)
139
140 if now // 3600 > self.lastPollTime // 3600:
141 log.info('hourly writes')
142 for addr in newWithSignal:
143 points.append(self.influxPoint(now, addr['mac'].lower(), 1))
144
145 influx.write_points(points, time_precision='s')
146 self.lastWithSignal = newWithSignal
147 if actions: # this doesn't currently include signal strength changes
148 fetch(reasoning + "immediateUpdate",
149 method='PUT',
150 timeout=2,
151 headers={'user-agent': ['tomatoWifi']}).addErrback(log.warn)
152 self.lastAddrs = newAddrs
153 self.lastPollTime = now
154
155 self.updateGraph(masterGraph)
156 except Exception as e:
157 log.error("poll error: %r\n%s", e, traceback.format_exc())
158
159 def influxPoint(self, now, address, value):
160 return {
161 'measurement': 'presence',
162 'tags': {'sensor': 'wifi', 'address': address,},
163 'fields': {'value': value},
164 'time': now,
32 } 165 }
33 for k in repl:
34 rows = list(g.query('''
35 PREFIX p: <http://bigasterisk.com/openid_proxy#>
36 SELECT ?prefix WHERE {
37 ?site
38 p:requestPrefix ?public;
39 p:proxyUrlPrefix ?prefix
40 .
41 }''', initBindings={"public" : Literal(k)}))
42 repl[k] = str(rows[0][0])
43 log.debug('repl %r', repl)
44
45 self.routers = []
46 for url in tomatoUrl:
47 name = url
48 for k, v in repl.items():
49 url = url.replace(k, v)
50
51 r = Router()
52 http, tail = url.split('//', 1)
53 userPass, tail = tail.split("@", 1)
54 r.url = http + '//' + tail
55 r.headers = {'Authorization': ['Basic %s' % userPass.encode('base64').strip()]}
56 r.name = {'wifiRouter1' : 'bigasterisk5',
57 'tomato2' : 'bigasterisk4'}[name.split('/')[1]]
58 self.routers.append(r)
59
60 @inlineCallbacks
61 def getPresentMacAddrs(self):
62 self.rereadConfig()
63 rows = yield loadOrbiData()
64 for row in rows:
65 if 'clientHostname' in row:
66 row['name'] = row['clientHostname']
67 mac = URIRef('http://bigasterisk.com/mac/%s' % row['mac'].lower())
68 label = self.graph.value(mac, RDFS.label)
69 if label:
70 row['name'] = label
71 returnValue(rows)
72
73 @inlineCallbacks
74 def getPresentMacAddrs_multirouter(self):
75 rows = []
76 166
77 for router in self.routers: 167 def computeActions(self, newWithSignal):
78 log.debug("GET %s", router) 168 actions = []
169
170 def makeAction(addr, act):
171 d = dict(sensor="wifi",
172 address=addr.get('mac').upper(), # mongo data is legacy uppercase
173 name=addr.get('name'),
174 networkName=addr.get('clientHostname'),
175 action=act)
176 if act == 'arrive' and 'ip' in addr:
177 # this won't cover the possible case that you get on
178 # wifi but don't have an ip yet. We'll record an
179 # action with no ip and then never record your ip.
180 d['ip'] = addr['ip']
181 return d
182
183 for addr in newWithSignal:
184 if addr['mac'] not in [r['mac'] for r in self.lastWithSignal]:
185 actions.append(makeAction(addr, 'arrive'))
186
187 for addr in self.lastWithSignal:
188 if addr['mac'] not in [r['mac'] for r in newWithSignal]:
189 actions.append(makeAction(addr, 'leave'))
190
191 return actions
192
193
194 # these need to move out to their own service
195 def doEntranceMusic(self, action):
196 import restkit, json
197 dt = self.deltaSinceLastArrive(action['name'])
198 log.debug("dt=%s", dt)
199 if dt > datetime.timedelta(hours=1):
200 hub = restkit.Resource(
201 # PSHB not working yet; "http://bang:9030/"
202 "http://slash:9049/"
203 )
204 action = action.copy()
205 del action['created']
206 del action['_id']
207 log.info("post to %s", hub)
208 hub.post("visitorNet", payload=json.dumps(action))
209
210 def deltaSinceLastArrive(self, name):
211 results = list(self.mongo.find({'name' : name}).sort('created',
212 DESCENDING).limit(1))
213 if not results:
214 return datetime.timedelta.max
215 now = datetime.datetime.now(tz.gettz('UTC'))
216 last = results[0]['created'].replace(tzinfo=tz.gettz('UTC'))
217 return now - last
218
219 def updateGraph(self, masterGraph):
220
221 g = ConjunctiveGraph()
222 ctx = DEV['wifi']
223
224 # someday i may also record specific AP and their strength,
225 # for positioning. But many users just want to know that the
226 # device is connected to some bigasterisk AP.
227 age = time.time() - self.lastPollTime
228 if age > 10:
229 raise ValueError("poll data is stale. age=%s" % age)
230
231 for dev in self.lastAddrs:
232 if not dev.get('connected'):
233 continue
234 uri = URIRef("http://bigasterisk.com/mac/%s" % dev['mac'].lower())
235 g.add((uri, ROOM['macAddress'], Literal(dev['mac'].lower()), ctx))
236
237 g.add((uri, ROOM['connected'], {
238 'wireless': URIRef("http://bigasterisk.com/wifiAccessPoints"),
239 '2.4G': URIRef("http://bigasterisk.com/wifiAccessPoints"),
240 '5G': URIRef("http://bigasterisk.com/wifiAccessPoints"),
241 '-': URIRef("http://bigasterisk.com/wifiUnknownConnectionType"),
242 'Unknown': URIRef("http://bigasterisk.com/wifiUnknownConnectionType"),
243 'wired': URIRef("http://bigasterisk.com/houseOpenNet")}[dev['contype']], ctx))
244 if 'clientHostname' in dev and dev['clientHostname']:
245 g.add((uri, ROOM['wifiNetworkName'], Literal(dev['clientHostname']), ctx))
246 if 'name' in dev and dev['name']:
247 g.add((uri, ROOM['deviceName'], Literal(dev['name']), ctx))
248 if 'signal' in dev:
249 g.add((uri, ROOM['signalStrength'], Literal(dev['signal']), ctx))
250 if 'model' in dev:
251 g.add((uri, ROOM['networkModel'], Literal(dev['model']), ctx))
79 try: 252 try:
80 resp = yield fetch(router.url, headers=router.headers, 253 conn = whenConnected(mongo, dev['mac'])
81 timeout=2) 254 except ValueError:
82 except socket.error: 255 traceback.print_exc()
83 log.warn("get on %s failed" % router) 256 pass
84 continue
85 data = resp.body
86 if 'Wireless -- Authenticated Stations' in data:
87 # zyxel 'Station Info' page
88 rows.extend(self._parseZyxel(data, router.name))
89 else: 257 else:
90 # tomato page 258 g.add((uri, ROOM['connectedAgo'],
91 rows.extend(self._parseTomato(data, router.name)) 259 Literal(connectedAgoString(conn)), ctx))
92 260 g.add((uri, ROOM['connected'], Literal(conn), ctx))
93 for r in rows: 261 masterGraph.setToGraph(g)
94 try: 262
95 r['name'] = self.knownMacAddr[r['mac']] 263
96 except KeyError: 264 if __name__ == '__main__':
97 pass 265 args = docopt.docopt('''
98 266 Usage:
99 returnValue(rows) 267 tomatoWifi [options]
100 268
101 def _parseZyxel(self, data, routerName): 269 Options:
102 root = lxml.html.soupparser.fromstring(data) 270 -v, --verbose more logging
103 for tr in root.cssselect('tr'): 271 --port=<n> serve on port [default: 9070]
104 mac, assoc, uth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] 272 --poll=<freq> poll frequency [default: .2]
105 if mac == "MAC": 273 ''')
106 continue 274 if args['--verbose']:
107 assoc = assoc.lower() == 'yes' 275 from twisted.python import log as twlog
108 yield dict(router=routerName, mac=mac, assoc=assoc, connected=assoc) 276 twlog.startLogging(sys.stdout)
109 277 log.setLevel(10)
110 def _parseTomato(self, data, routerName): 278 log.setLevel(logging.DEBUG)
111 for iface, mac, signal in jsValue(data, 'wldev'): 279
112 yield dict(router=routerName, mac=mac, signal=signal, connected=bool(signal)) 280 mongo = Connection('bang', 27017, tz_aware=True)['visitor']['visitor']
113 281 influx = InfluxDBClient('bang', 9060, 'root', 'root', 'main')
114 282
115 @inlineCallbacks 283 masterGraph = PatchableGraph()
116 def loadUvaData(): 284 wifi = Wifi()
117 config = json.load(open("priv-uva.json")) 285 poller = Poller(wifi, mongo)
118 headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} 286 task.LoopingCall(poller.poll).start(1/float(args['--poll']))
119 resp = yield fetch('http://10.2.0.2/wlstationlist.cmd', headers=headers) 287
120 root = lxml.html.soupparser.fromstring(resp.body) 288 reactor.listenTCP(int(args['--port']),
121 byMac = {} 289 cyclone.web.Application(
122 for tr in root.cssselect('tr'): 290 [
123 mac, connected, auth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] 291 (r"/", Index),
124 if mac == "MAC": 292 (r'/json', Json),
125 continue 293 (r'/graph', CycloneGraphHandler, {'masterGraph': masterGraph}),
126 connected = connected.lower() == 'yes' 294 (r'/graph/events', CycloneGraphEventsHandler, {'masterGraph': masterGraph}),
127 byMac[mac] = dict(mac=mac, connected=connected, auth=auth == 'Yes', ssid=ssid, iface=iface) 295 (r'/table', Table),
128 296 #(r'/activity', Activity),
129 resp = yield fetch('http://10.2.0.2/DHCPTable.asp', headers=headers) 297 ],
130 for row in re.findall(r'new AAA\((.*)\)', resp.body): 298 wifi=wifi,
131 clientHostname, ipaddr, mac, expires, iface = [s.strip("'") for s in row.rsplit(',', 4)] 299 poller=poller,
132 if clientHostname == 'wlanadv.none': 300 mongo=mongo))
133 continue 301 import twisted; print('twisted', twisted.__version__)
134 byMac.setdefault(mac, {}).update(dict( 302 reactor.run()
135 clientHostname=clientHostname, connection=iface, ipaddr=ipaddr, dhcpExpires=expires))
136
137 returnValue(sorted(byMac.values()))
138
139 @inlineCallbacks
140 def loadCiscoData():
141 config = json.load(open("priv-uva.json"))
142 headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]}
143 print headers
144 resp = yield fetch('http://10.2.0.2/', headers=headers)
145 print resp.body
146 returnValue([])
147
148 @inlineCallbacks
149 def loadOrbiData():
150 config = json.load(open("priv-uva.json"))
151 headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]}
152 resp = yield fetch('http://orbi.bigasterisk.com/DEV_device_info.htm', headers=headers)
153
154 if not resp.body.startswith(('device=', 'device_changed=0\ndevice=', 'device_changed=1\ndevice=')):
155 raise ValueError(resp.body)
156
157 ret = []
158 for row in json.loads(resp.body.split('device=', 1)[-1]):
159 ret.append(dict(
160 connected=True,
161 ipaddr=row['ip'],
162 mac=row['mac'].lower(),
163 contype=row['contype'],
164 model=row['model'],
165 clientHostname=row['name'] if row['name'] != 'Unknown' else None))
166 returnValue(ret)
167
168
169 def jsValue(js, variableName):
170 # using literal_eval instead of json parser to handle the trailing commas
171 val = re.search(variableName + r'\s*=\s*(.*?);', js, re.DOTALL).group(1)
172 return ast.literal_eval(val)