Mercurial > code > home > repos > homeauto

annotate service/wifi/scrape.py @ 421:47d7dd31bb2c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
port to py3 Ignore-this: e1a2e6bb730111e76f5a5dd2366d498a
author drewp@bigasterisk.com
date Sat, 30 Mar 2019 18:27:17 -0700
parents service/wifi/wifi.py@a530d9c5b280
children 19460b3f7baf
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
1 import re, ast, logging, socket, json, base64
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
2 from twisted.internet.defer import inlineCallbacks, returnValue
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
3 from cyclone.httpclient import fetch
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
4 from rdflib import Literal, Graph, RDFS, URIRef
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
5
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
6 log = logging.getLogger()
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
7
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
8 class Router(object):
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
9 def __repr__(self):
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
10 return repr(self.__dict__)
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
11
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
12 class Wifi(object):
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
13 """
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
14 gather the users of wifi from the tomato routers
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
15 """
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
16 def __init__(self, accessN3="/my/proj/openid_proxy/access.n3"):
272
660d21bcfcde reload config wifi more often
drewp@bigasterisk.com
parents: 202
diff changeset
17 self.rereadConfig()
660d21bcfcde reload config wifi more often
drewp@bigasterisk.com
parents: 202
diff changeset
18 #self._loadRouters(accessN3, tomatoUrl)
660d21bcfcde reload config wifi more often
drewp@bigasterisk.com
parents: 202
diff changeset
19
660d21bcfcde reload config wifi more often
drewp@bigasterisk.com
parents: 202
diff changeset
20 def rereadConfig(self):
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
21 self.graph = Graph()
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
22 self.graph.parse('config.n3', format='n3')
272
660d21bcfcde reload config wifi more often
drewp@bigasterisk.com
parents: 202
diff changeset
23
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
24
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
25 def _loadRouters(self, accessN3, tomatoUrl):
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
26 g = Graph()
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
27 g.parse(accessN3, format="n3")
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
28 repl = {
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
29 '/wifiRouter1/' : None,
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
30 #'/tomato2/' : None
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
31 }
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
32 for k in repl:
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
33 rows = list(g.query('''
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
34 PREFIX p: <http://bigasterisk.com/openid_proxy#>
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
35 SELECT ?prefix WHERE {
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
36 ?site
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
37 p:requestPrefix ?public;
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
38 p:proxyUrlPrefix ?prefix
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
39 .
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
40 }''', initBindings={"public" : Literal(k)}))
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
41 repl[k] = str(rows[0][0])
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
42 log.debug('repl %r', repl)
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
43
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
44 self.routers = []
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
45 for url in tomatoUrl:
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
46 name = url
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
47 for k, v in repl.items():
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
48 url = url.replace(k, v)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
49
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
50 r = Router()
52
875a37be1228 fix fetch calls in tomatowifi
drewp@bigasterisk.com
parents: 51
diff changeset
51 http, tail = url.split('//', 1)
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
52 userPass, tail = tail.split("@", 1)
52
875a37be1228 fix fetch calls in tomatowifi
drewp@bigasterisk.com
parents: 51
diff changeset
53 r.url = http + '//' + tail
875a37be1228 fix fetch calls in tomatowifi
drewp@bigasterisk.com
parents: 51
diff changeset
54 r.headers = {'Authorization': ['Basic %s' % userPass.encode('base64').strip()]}
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
55 r.name = {'wifiRouter1' : 'bigasterisk5',
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
56 'tomato2' : 'bigasterisk4'}[name.split('/')[1]]
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
57 self.routers.append(r)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
58
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
59 @inlineCallbacks
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
60 def getPresentMacAddrs(self):
272
660d21bcfcde reload config wifi more often
drewp@bigasterisk.com
parents: 202
diff changeset
61 self.rereadConfig()
341
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
62 rows = yield loadOrbiData()
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
63 for row in rows:
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
64 if 'clientHostname' in row:
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
65 row['name'] = row['clientHostname']
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
66 mac = URIRef('http://bigasterisk.com/mac/%s' % row['mac'].lower())
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
67 label = self.graph.value(mac, RDFS.label)
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
68 if label:
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
69 row['name'] = label
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
70 returnValue(rows)
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
71
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
72 @inlineCallbacks
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
73 def getPresentMacAddrs_multirouter(self):
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
74 rows = []
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
75
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
76 for router in self.routers:
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
77 log.debug("GET %s", router)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
78 try:
52
875a37be1228 fix fetch calls in tomatowifi
drewp@bigasterisk.com
parents: 51
diff changeset
79 resp = yield fetch(router.url, headers=router.headers,
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
80 timeout=2)
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
81 except socket.error:
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
82 log.warn("get on %s failed" % router)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
83 continue
52
875a37be1228 fix fetch calls in tomatowifi
drewp@bigasterisk.com
parents: 51
diff changeset
84 data = resp.body
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
85 if 'Wireless -- Authenticated Stations' in data:
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
86 # zyxel 'Station Info' page
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
87 rows.extend(self._parseZyxel(data, router.name))
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
88 else:
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
89 # tomato page
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
90 rows.extend(self._parseTomato(data, router.name))
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
91
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
92 for r in rows:
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
93 try:
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
94 r['name'] = self.knownMacAddr[r['mac']]
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
95 except KeyError:
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
96 pass
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
97
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
98 returnValue(rows)
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
99
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
100 def _parseZyxel(self, data, routerName):
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
101 import lxml.html.soupparser
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
102
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
103 root = lxml.html.soupparser.fromstring(data)
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
104 for tr in root.cssselect('tr'):
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
105 mac, assoc, uth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()]
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
106 if mac == "MAC":
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
107 continue
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
108 assoc = assoc.lower() == 'yes'
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
109 yield dict(router=routerName, mac=mac, assoc=assoc, connected=assoc)
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
110
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
111 def _parseTomato(self, data, routerName):
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
112 for iface, mac, signal in jsValue(data, 'wldev'):
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
113 yield dict(router=routerName, mac=mac, signal=signal, connected=bool(signal))
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
114
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
115
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
116 @inlineCallbacks
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
117 def loadUvaData():
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
118 import lxml.html.soupparser
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
119
419
fdbdecdecd0b use rel path for config
drewp@bigasterisk.com
parents: 341
diff changeset
120 config = json.load(open("priv-uva.json"))
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
121 headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]}
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
122 resp = yield fetch('http://10.2.0.2/wlstationlist.cmd', headers=headers)
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
123 root = lxml.html.soupparser.fromstring(resp.body)
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
124 byMac = {}
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
125 for tr in root.cssselect('tr'):
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
126 mac, connected, auth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()]
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
127 if mac == "MAC":
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
128 continue
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
129 connected = connected.lower() == 'yes'
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
130 byMac[mac] = dict(mac=mac, connected=connected, auth=auth == 'Yes', ssid=ssid, iface=iface)
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
131
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
132 resp = yield fetch('http://10.2.0.2/DHCPTable.asp', headers=headers)
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
133 for row in re.findall(r'new AAA\((.*)\)', resp.body):
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
134 clientHostname, ipaddr, mac, expires, iface = [s.strip("'") for s in row.rsplit(',', 4)]
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
135 if clientHostname == 'wlanadv.none':
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
136 continue
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
137 byMac.setdefault(mac, {}).update(dict(
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
138 clientHostname=clientHostname, connection=iface, ipaddr=ipaddr, dhcpExpires=expires))
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
139
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
140 returnValue(sorted(byMac.values()))
202
7faf642438bc start of code for reading cisco router data
drewp@bigasterisk.com
parents: 175
diff changeset
141
7faf642438bc start of code for reading cisco router data
drewp@bigasterisk.com
parents: 175
diff changeset
142 @inlineCallbacks
7faf642438bc start of code for reading cisco router data
drewp@bigasterisk.com
parents: 175
diff changeset
143 def loadCiscoData():
419
fdbdecdecd0b use rel path for config
drewp@bigasterisk.com
parents: 341
diff changeset
144 config = json.load(open("priv-uva.json"))
202
7faf642438bc start of code for reading cisco router data
drewp@bigasterisk.com
parents: 175
diff changeset
145 headers = {'Authorization': ['Basic %s' % config['userPass'].encode('base64').strip()]}
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
146 print(headers)
202
7faf642438bc start of code for reading cisco router data
drewp@bigasterisk.com
parents: 175
diff changeset
147 resp = yield fetch('http://10.2.0.2/', headers=headers)
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
148 print(resp.body)
202
7faf642438bc start of code for reading cisco router data
drewp@bigasterisk.com
parents: 175
diff changeset
149 returnValue([])
341
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
150
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
151 @inlineCallbacks
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
152 def loadOrbiData():
419
fdbdecdecd0b use rel path for config
drewp@bigasterisk.com
parents: 341
diff changeset
153 config = json.load(open("priv-uva.json"))
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
154 headers = {b'Authorization': [
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
155 b'Basic %s' % base64.encodebytes(config['userPass'].encode('utf8')).strip()]}
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
156 resp = yield fetch(b'http://orbi.bigasterisk.com/DEV_device_info.htm', method=b'GET', headers=headers)
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
157 print('back from fetch')
341
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
158
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
159 if not resp.body.startswith((b'device=', b'device_changed=0\ndevice=', b'device_changed=1\ndevice=')):
341
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
160 raise ValueError(resp.body)
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
161
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
162 ret = []
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
163 for row in json.loads(resp.body.split(b'device=', 1)[-1]):
341
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
164 ret.append(dict(
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
165 connected=True,
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
166 ipaddr=row['ip'],
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
167 mac=row['mac'].lower(),
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
168 contype=row['contype'],
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
169 model=row['model'],
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
170 clientHostname=row['name'] if row['name'] != 'Unknown' else None))
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
171 returnValue(ret)
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
172
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
173
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
174 def jsValue(js, variableName):
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
175 # using literal_eval instead of json parser to handle the trailing commas
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
176 val = re.search(variableName + r'\s*=\s*(.*?);', js, re.DOTALL).group(1)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
177 return ast.literal_eval(val)