Mercurial > code > home > repos > homeauto

annotate service/tomatoWifi/wifi.py @ 62:f8cc3d1baa85

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table Ignore-this: 18bade72e14d40532bd019791d03fa7d
author drewp@bigasterisk.com
date Sat, 06 Apr 2013 17:08:19 -0700
parents 875a37be1228
children c81a451f9b26
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
1 import re, ast, logging, socket
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
2 import lxml.html.soupparser
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
3 from twisted.internet.defer import inlineCallbacks, returnValue
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
4 from cyclone.httpclient import fetch
36
0ab069867c64 rdflib and jsonlib api updates
drewp@bigasterisk.com
parents: 0
diff changeset
5 from rdflib import Literal, Graph
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
6
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
7 log = logging.getLogger()
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
8
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
9 class Router(object):
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
10 def __repr__(self):
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
11 return repr(self.__dict__)
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
12
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
13 class Wifi(object):
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
14 """
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
15 gather the users of wifi from the tomato routers
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
16
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
17 with host names from /var/lib/dhcp/dhcpd.leases
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
18 """
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
19 def __init__(self, tomatoConfig="/my/site/magma/tomato_config.js",
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
20 accessN3="/my/proj/openid_proxy/access.n3"):
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
21
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
22 # ideally this would all be in the same rdf store, with int and
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
23 # ext versions of urls
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
24
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
25 txt = open(tomatoConfig).read().replace('\n', '')
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
26 self.knownMacAddr = jsValue(txt, 'knownMacAddr')
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
27 tomatoUrl = jsValue(txt, 'tomatoUrl')
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
28
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
29 g = Graph()
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
30 g.parse(accessN3, format="n3")
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
31 repl = {'/tomato1/' : None, '/tomato2/' : None}
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
32 for k in repl:
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
33 rows = list(g.query('''
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
34 PREFIX p: <http://bigasterisk.com/openid_proxy#>
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
35 SELECT ?prefix WHERE {
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
36 [
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
37 p:requestPrefix ?public;
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
38 p:proxyUrlPrefix ?prefix
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
39 ]
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
40 }''', initBindings={"public" : Literal(k)}))
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
41 repl[k] = str(rows[0][0])
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
42
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
43 self.routers = []
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
44 for url in tomatoUrl:
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
45 name = url
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
46 for k, v in repl.items():
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
47 url = url.replace(k, v)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
48
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
49 r = Router()
52
875a37be1228 fix fetch calls in tomatowifi
drewp@bigasterisk.com
parents: 51
diff changeset
50 http, tail = url.split('//', 1)
875a37be1228 fix fetch calls in tomatowifi
drewp@bigasterisk.com
parents: 51
diff changeset
51 userPass, tail = tail.split("@", 1)
875a37be1228 fix fetch calls in tomatowifi
drewp@bigasterisk.com
parents: 51
diff changeset
52 r.url = http + '//' + tail
875a37be1228 fix fetch calls in tomatowifi
drewp@bigasterisk.com
parents: 51
diff changeset
53 r.headers = {'Authorization': ['Basic %s' % userPass.encode('base64').strip()]}
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
54 r.name = {'tomato1' : 'bigasterisk5',
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
55 'tomato2' : 'bigasterisk4'}[name.split('/')[1]]
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
56 self.routers.append(r)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
57
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
58 @inlineCallbacks
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
59 def getPresentMacAddrs(self):
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
60 rows = []
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
61
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
62 for router in self.routers:
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
63 log.debug("GET %s", router)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
64 try:
52
875a37be1228 fix fetch calls in tomatowifi
drewp@bigasterisk.com
parents: 51
diff changeset
65 resp = yield fetch(router.url, headers=router.headers,
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
66 timeout=2)
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
67 except socket.error:
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
68 log.warn("get on %s failed" % router)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
69 continue
52
875a37be1228 fix fetch calls in tomatowifi
drewp@bigasterisk.com
parents: 51
diff changeset
70 data = resp.body
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
71 if 'Wireless -- Authenticated Stations' in data:
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
72 # zyxel 'Station Info' page
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
73 rows.extend(self.parseZyxel(data, router.name))
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
74 else:
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
75 # tomato page
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
76 rows.extend(self.parseTomato(data, router.name))
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
77
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
78 for r in rows:
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
79 try:
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
80 r['name'] = self.knownMacAddr[r['mac']]
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
81 except KeyError:
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
82 pass
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
83
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
84 returnValue(rows)
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
85
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
86 def parseZyxel(self, data, routerName):
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
87 root = lxml.html.soupparser.fromstring(data)
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
88 for tr in root.cssselect('tr'):
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
89 mac, assoc, uth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()]
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
90 if mac == "MAC":
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
91 continue
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
92 assoc = assoc.lower() == 'yes'
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
93 yield dict(router=routerName, mac=mac, assoc=assoc, connected=assoc)
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
94
62
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
95 def parseTomato(self, data, routerName):
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
96 for iface, mac, signal in jsValue(data, 'wldev'):
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
97 yield dict(router=routerName, mac=mac, signal=signal, connected=bool(signal))
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
98
f8cc3d1baa85 redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp@bigasterisk.com
parents: 52
diff changeset
99
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
100 def jsValue(js, variableName):
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
101 # using literal_eval instead of json parser to handle the trailing commas
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
102 val = re.search(variableName + r'\s*=\s*(.*?);', js, re.DOTALL).group(1)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
103 return ast.literal_eval(val)