Mercurial > code > home > repos > homeauto

annotate service/tomatoWifi/wifi.py @ 971:fbe72d44f15a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
only recompile if the C code is new. redo Device class api. single temperature sensor is working Ignore-this: e78106d25dbb2ac8c5e5d8a81d576358 darcs-hash:20150411084359-312f9-7c50d2f4f21dd9d5a0fa8913873a1e6d7d325118
author drewp <drewp@bigasterisk.com>
date Sat, 11 Apr 2015 01:43:59 -0700
parents d9bc9a82dcca
children c81a451f9b26
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
856
4b386fc51325 rewrite tomatowifi from restkit to cyclone httpclient
drewp <drewp@bigasterisk.com>
parents: 841
diff changeset
1 import re, ast, logging, socket
867
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
2 import lxml.html.soupparser
856
4b386fc51325 rewrite tomatowifi from restkit to cyclone httpclient
drewp <drewp@bigasterisk.com>
parents: 841
diff changeset
3 from twisted.internet.defer import inlineCallbacks, returnValue
4b386fc51325 rewrite tomatowifi from restkit to cyclone httpclient
drewp <drewp@bigasterisk.com>
parents: 841
diff changeset
4 from cyclone.httpclient import fetch
841
ddc2cdbfde50 rdflib and jsonlib api updates
drewp <drewp@bigasterisk.com>
parents: 805
diff changeset
5 from rdflib import Literal, Graph
805
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
6
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
7 log = logging.getLogger()
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
8
856
4b386fc51325 rewrite tomatowifi from restkit to cyclone httpclient
drewp <drewp@bigasterisk.com>
parents: 841
diff changeset
9 class Router(object):
4b386fc51325 rewrite tomatowifi from restkit to cyclone httpclient
drewp <drewp@bigasterisk.com>
parents: 841
diff changeset
10 def __repr__(self):
4b386fc51325 rewrite tomatowifi from restkit to cyclone httpclient
drewp <drewp@bigasterisk.com>
parents: 841
diff changeset
11 return repr(self.__dict__)
4b386fc51325 rewrite tomatowifi from restkit to cyclone httpclient
drewp <drewp@bigasterisk.com>
parents: 841
diff changeset
12
805
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
13 class Wifi(object):
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
14 """
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
15 gather the users of wifi from the tomato routers
867
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
16
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
17 with host names from /var/lib/dhcp/dhcpd.leases
805
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
18 """
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
19 def __init__(self, tomatoConfig="/my/site/magma/tomato_config.js",
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
20 accessN3="/my/proj/openid_proxy/access.n3"):
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
21
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
22 # ideally this would all be in the same rdf store, with int and
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
23 # ext versions of urls
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
24
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
25 txt = open(tomatoConfig).read().replace('\n', '')
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
26 self.knownMacAddr = jsValue(txt, 'knownMacAddr')
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
27 tomatoUrl = jsValue(txt, 'tomatoUrl')
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
28
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
29 g = Graph()
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
30 g.parse(accessN3, format="n3")
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
31 repl = {'/tomato1/' : None, '/tomato2/' : None}
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
32 for k in repl:
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
33 rows = list(g.query('''
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
34 PREFIX p: <http://bigasterisk.com/openid_proxy#>
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
35 SELECT ?prefix WHERE {
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
36 [
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
37 p:requestPrefix ?public;
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
38 p:proxyUrlPrefix ?prefix
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
39 ]
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
40 }''', initBindings={"public" : Literal(k)}))
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
41 repl[k] = str(rows[0][0])
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
42
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
43 self.routers = []
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
44 for url in tomatoUrl:
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
45 name = url
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
46 for k, v in repl.items():
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
47 url = url.replace(k, v)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
48
856
4b386fc51325 rewrite tomatowifi from restkit to cyclone httpclient
drewp <drewp@bigasterisk.com>
parents: 841
diff changeset
49 r = Router()
857
1168b0cd90af fix fetch calls in tomatowifi
drewp <drewp@bigasterisk.com>
parents: 856
diff changeset
50 http, tail = url.split('//', 1)
1168b0cd90af fix fetch calls in tomatowifi
drewp <drewp@bigasterisk.com>
parents: 856
diff changeset
51 userPass, tail = tail.split("@", 1)
1168b0cd90af fix fetch calls in tomatowifi
drewp <drewp@bigasterisk.com>
parents: 856
diff changeset
52 r.url = http + '//' + tail
1168b0cd90af fix fetch calls in tomatowifi
drewp <drewp@bigasterisk.com>
parents: 856
diff changeset
53 r.headers = {'Authorization': ['Basic %s' % userPass.encode('base64').strip()]}
867
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
54 r.name = {'tomato1' : 'bigasterisk5',
805
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
55 'tomato2' : 'bigasterisk4'}[name.split('/')[1]]
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
56 self.routers.append(r)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
57
856
4b386fc51325 rewrite tomatowifi from restkit to cyclone httpclient
drewp <drewp@bigasterisk.com>
parents: 841
diff changeset
58 @inlineCallbacks
805
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
59 def getPresentMacAddrs(self):
867
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
60 rows = []
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
61
805
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
62 for router in self.routers:
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
63 log.debug("GET %s", router)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
64 try:
857
1168b0cd90af fix fetch calls in tomatowifi
drewp <drewp@bigasterisk.com>
parents: 856
diff changeset
65 resp = yield fetch(router.url, headers=router.headers,
856
4b386fc51325 rewrite tomatowifi from restkit to cyclone httpclient
drewp <drewp@bigasterisk.com>
parents: 841
diff changeset
66 timeout=2)
805
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
67 except socket.error:
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
68 log.warn("get on %s failed" % router)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
69 continue
857
1168b0cd90af fix fetch calls in tomatowifi
drewp <drewp@bigasterisk.com>
parents: 856
diff changeset
70 data = resp.body
867
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
71 if 'Wireless -- Authenticated Stations' in data:
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
72 # zyxel 'Station Info' page
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
73 rows.extend(self.parseZyxel(data, router.name))
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
74 else:
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
75 # tomato page
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
76 rows.extend(self.parseTomato(data, router.name))
856
4b386fc51325 rewrite tomatowifi from restkit to cyclone httpclient
drewp <drewp@bigasterisk.com>
parents: 841
diff changeset
77
867
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
78 for r in rows:
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
79 try:
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
80 r['name'] = self.knownMacAddr[r['mac']]
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
81 except KeyError:
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
82 pass
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
83
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
84 returnValue(rows)
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
85
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
86 def parseZyxel(self, data, routerName):
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
87 root = lxml.html.soupparser.fromstring(data)
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
88 for tr in root.cssselect('tr'):
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
89 mac, assoc, uth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()]
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
90 if mac == "MAC":
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
91 continue
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
92 assoc = assoc.lower() == 'yes'
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
93 yield dict(router=routerName, mac=mac, assoc=assoc, connected=assoc)
805
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
94
867
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
95 def parseTomato(self, data, routerName):
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
96 for iface, mac, signal in jsValue(data, 'wldev'):
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
97 yield dict(router=routerName, mac=mac, signal=signal, connected=bool(signal))
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
98
d9bc9a82dcca redo wifi scraper to work with zyxel router report page too. add last connected time (from mongo) to web table
drewp <drewp@bigasterisk.com>
parents: 857
diff changeset
99
805
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
100 def jsValue(js, variableName):
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
101 # using literal_eval instead of json parser to handle the trailing commas
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
102 val = re.search(variableName + r'\s*=\s*(.*?);', js, re.DOTALL).group(1)
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
103 return ast.literal_eval(val)