Mercurial > code > home > repos > homeauto
annotate service/wifi/scrape.py @ 1225:b8c0daabe5a5
factor out some URI generation
Ignore-this: 3c982a1fdbadcc3154278fbae3d2ce0
darcs-hash:22be14eeb34ca3e87842f4619af6277851033561
| author | drewp <drewp@bigasterisk.com> |
|---|---|
| date | Sat, 30 Mar 2019 18:59:19 -0700 |
| parents | e1202af42d4d |
| children | 7de8f0cd3392 |
| rev | line source |
|---|---|
| 1224 | 1 import re, ast, logging, socket, json, base64 |
| 2 from twisted.internet.defer import inlineCallbacks, returnValue | |
| 3 from cyclone.httpclient import fetch | |
| 4 from rdflib import Literal, Graph, RDFS, URIRef | |
| 5 | |
| 6 log = logging.getLogger() | |
| 7 | |
|
1225
b8c0daabe5a5
factor out some URI generation
drewp <drewp@bigasterisk.com>
parents:
1224
diff
changeset
|
8 def macUri(macAddress: str) -> URIRef: |
|
b8c0daabe5a5
factor out some URI generation
drewp <drewp@bigasterisk.com>
parents:
1224
diff
changeset
|
9 return URIRef("http://bigasterisk.com/mac/%s" % dev['mac'].lower()) |
| 1224 | 10 |
| 11 class Wifi(object): | |
| 12 """ | |
| 13 gather the users of wifi from the tomato routers | |
| 14 """ | |
| 15 def __init__(self, accessN3="/my/proj/openid_proxy/access.n3"): | |
| 16 self.rereadConfig() | |
| 17 #self._loadRouters(accessN3, tomatoUrl) | |
| 18 | |
| 19 def rereadConfig(self): | |
| 20 self.graph = Graph() | |
| 21 self.graph.parse('config.n3', format='n3') | |
| 22 | |
| 23 | |
| 24 def _loadRouters(self, accessN3, tomatoUrl): | |
| 25 g = Graph() | |
| 26 g.parse(accessN3, format="n3") | |
| 27 repl = { | |
| 28 '/wifiRouter1/' : None, | |
| 29 #'/tomato2/' : None | |
| 30 } | |
| 31 for k in repl: | |
| 32 rows = list(g.query(''' | |
| 33 PREFIX p: <http://bigasterisk.com/openid_proxy#> | |
| 34 SELECT ?prefix WHERE { | |
| 35 ?site | |
| 36 p:requestPrefix ?public; | |
| 37 p:proxyUrlPrefix ?prefix | |
| 38 . | |
| 39 }''', initBindings={"public" : Literal(k)})) | |
| 40 repl[k] = str(rows[0][0]) | |
| 41 log.debug('repl %r', repl) | |
| 42 | |
| 43 self.routers = [] | |
| 44 for url in tomatoUrl: | |
| 45 name = url | |
| 46 for k, v in repl.items(): | |
| 47 url = url.replace(k, v) | |
| 48 | |
| 49 r = Router() | |
| 50 http, tail = url.split('//', 1) | |
| 51 userPass, tail = tail.split("@", 1) | |
| 52 r.url = http + '//' + tail | |
| 53 r.headers = {'Authorization': ['Basic %s' % userPass.encode('base64').strip()]} | |
| 54 r.name = {'wifiRouter1' : 'bigasterisk5', | |
| 55 'tomato2' : 'bigasterisk4'}[name.split('/')[1]] | |
| 56 self.routers.append(r) | |
| 57 | |
| 58 @inlineCallbacks | |
| 59 def getPresentMacAddrs(self): | |
| 60 self.rereadConfig() | |
| 61 rows = yield loadOrbiData() | |
| 62 for row in rows: | |
| 63 if 'clientHostname' in row: | |
| 64 row['name'] = row['clientHostname'] | |
|
1225
b8c0daabe5a5
factor out some URI generation
drewp <drewp@bigasterisk.com>
parents:
1224
diff
changeset
|
65 mac = macUri(row['mac'].lower()) |
| 1224 | 66 label = self.graph.value(mac, RDFS.label) |
| 67 if label: | |
| 68 row['name'] = label | |
| 69 returnValue(rows) | |
| 70 | |
| 71 @inlineCallbacks | |
| 72 def getPresentMacAddrs_multirouter(self): | |
| 73 rows = [] | |
| 74 | |
| 75 for router in self.routers: | |
| 76 log.debug("GET %s", router) | |
| 77 try: | |
| 78 resp = yield fetch(router.url, headers=router.headers, | |
| 79 timeout=2) | |
| 80 except socket.error: | |
| 81 log.warn("get on %s failed" % router) | |
| 82 continue | |
| 83 data = resp.body | |
| 84 if 'Wireless -- Authenticated Stations' in data: | |
| 85 # zyxel 'Station Info' page | |
| 86 rows.extend(self._parseZyxel(data, router.name)) | |
| 87 else: | |
| 88 # tomato page | |
| 89 rows.extend(self._parseTomato(data, router.name)) | |
| 90 | |
| 91 for r in rows: | |
| 92 try: | |
| 93 r['name'] = self.knownMacAddr[r['mac']] | |
| 94 except KeyError: | |
| 95 pass | |
| 96 | |
| 97 returnValue(rows) | |
| 98 | |
| 99 def _parseZyxel(self, data, routerName): | |
| 100 import lxml.html.soupparser | |
| 101 | |
| 102 root = lxml.html.soupparser.fromstring(data) | |
| 103 for tr in root.cssselect('tr'): | |
| 104 mac, assoc, uth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] | |
| 105 if mac == "MAC": | |
| 106 continue | |
| 107 assoc = assoc.lower() == 'yes' | |
| 108 yield dict(router=routerName, mac=mac, assoc=assoc, connected=assoc) | |
| 109 | |
| 110 def _parseTomato(self, data, routerName): | |
| 111 for iface, mac, signal in jsValue(data, 'wldev'): | |
| 112 yield dict(router=routerName, mac=mac, signal=signal, connected=bool(signal)) | |
| 113 | |
| 114 | |
| 115 @inlineCallbacks | |
| 116 def loadUvaData(): | |
| 117 import lxml.html.soupparser | |
| 118 | |
| 119 config = json.load(open("priv-uva.json")) | |
| 120 headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} | |
| 121 resp = yield fetch('http://10.2.0.2/wlstationlist.cmd', headers=headers) | |
| 122 root = lxml.html.soupparser.fromstring(resp.body) | |
| 123 byMac = {} | |
| 124 for tr in root.cssselect('tr'): | |
| 125 mac, connected, auth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()] | |
| 126 if mac == "MAC": | |
| 127 continue | |
| 128 connected = connected.lower() == 'yes' | |
| 129 byMac[mac] = dict(mac=mac, connected=connected, auth=auth == 'Yes', ssid=ssid, iface=iface) | |
| 130 | |
| 131 resp = yield fetch('http://10.2.0.2/DHCPTable.asp', headers=headers) | |
| 132 for row in re.findall(r'new AAA\((.*)\)', resp.body): | |
| 133 clientHostname, ipaddr, mac, expires, iface = [s.strip("'") for s in row.rsplit(',', 4)] | |
| 134 if clientHostname == 'wlanadv.none': | |
| 135 continue | |
| 136 byMac.setdefault(mac, {}).update(dict( | |
| 137 clientHostname=clientHostname, connection=iface, ipaddr=ipaddr, dhcpExpires=expires)) | |
| 138 | |
| 139 returnValue(sorted(byMac.values())) | |
| 140 | |
| 141 @inlineCallbacks | |
| 142 def loadCiscoData(): | |
| 143 config = json.load(open("priv-uva.json")) | |
| 144 headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]} | |
| 145 print(headers) | |
| 146 resp = yield fetch('http://10.2.0.2/', headers=headers) | |
| 147 print(resp.body) | |
| 148 returnValue([]) | |
| 149 | |
| 150 @inlineCallbacks | |
| 151 def loadOrbiData(): | |
| 152 config = json.load(open("priv-uva.json")) | |
| 153 headers = {b'Authorization': [ | |
| 154 b'Basic %s' % base64.encodebytes(config['userPass'].encode('utf8')).strip()]} | |
| 155 resp = yield fetch(b'http://orbi.bigasterisk.com/DEV_device_info.htm', method=b'GET', headers=headers) | |
| 156 print('back from fetch') | |
| 157 | |
| 158 if not resp.body.startswith((b'device=', b'device_changed=0\ndevice=', b'device_changed=1\ndevice=')): | |
| 159 raise ValueError(resp.body) | |
| 160 | |
| 161 ret = [] | |
| 162 for row in json.loads(resp.body.split(b'device=', 1)[-1]): | |
| 163 ret.append(dict( | |
| 164 connected=True, | |
| 165 ipaddr=row['ip'], | |
| 166 mac=row['mac'].lower(), | |
| 167 contype=row['contype'], | |
| 168 model=row['model'], | |
| 169 clientHostname=row['name'] if row['name'] != 'Unknown' else None)) | |
| 170 returnValue(ret) | |
| 171 | |
| 172 | |
| 173 def jsValue(js, variableName): | |
| 174 # using literal_eval instead of json parser to handle the trailing commas | |
| 175 val = re.search(variableName + r'\s*=\s*(.*?);', js, re.DOTALL).group(1) | |
| 176 return ast.literal_eval(val) |