Mercurial > code > home > repos > homeauto

annotate service/wifi/scrape.py @ 641:5fd7aef3b2b2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
index page updates. include wifi badg data. Ignore-this: e4cecff1c9886d17f64c281dd4f52c7b
author drewp@bigasterisk.com
date Wed, 14 Aug 2019 16:40:57 -0700
parents cbb4b3ccdb53
children f88ff1021ee0
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
1 import logging, json, base64
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
2 from typing import List, Iterable
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
3
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
4 from cyclone.httpclient import fetch
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
5 from rdflib import Literal, Graph, RDF, URIRef, Namespace, RDFS
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
6 from twisted.internet.defer import inlineCallbacks, returnValue
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
7
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
8 log = logging.getLogger()
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
9 ROOM = Namespace("http://projects.bigasterisk.com/room/")
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
10 AST = Namespace("http://bigasterisk.com/")
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
11
422
19460b3f7baf factor out some URI generation
drewp@bigasterisk.com
parents: 421
diff changeset
12 def macUri(macAddress: str) -> URIRef:
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
13 return URIRef("http://bigasterisk.com/mac/%s" % macAddress.lower())
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
14
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
15 class SeenNode(object):
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
16 def __init__(self, uri: URIRef, mac: str, ip: str, stmts: Iterable):
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
17 self.connected = True
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
18 self.uri = uri
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
19 self.mac = mac
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
20 self.ip = ip
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
21 self.stmts = stmts
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
22
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
23 class Wifi(object):
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
24 """
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
25 gather the users of wifi from the tomato routers
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
26 """
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
27 def __init__(self, config: Graph):
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
28 self.config = config
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
29
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
30 @inlineCallbacks
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
31 def getPresentMacAddrs(self): # returnValue List[SeenNode]
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
32 rows = yield self._loader()(self.config)
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
33 returnValue(rows)
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
34
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
35 def _loader(self):
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
36 cls = self.config.value(ROOM['wifiScraper'], RDF.type)
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
37 if cls == ROOM['OrbiScraper']:
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
38 return loadOrbiData
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
39 raise NotImplementedError(cls)
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
40
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
41
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
42 @inlineCallbacks
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
43 def loadOrbiData(config):
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
44 user = config.value(ROOM['wifiScraper'], ROOM['user'])
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
45 passwd = config.value(ROOM['wifiScraper'], ROOM['password'])
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
46 basicAuth = '%s:%s' % (user, passwd)
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
47 headers = {
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
48 b'Authorization': [
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
49 b'Basic %s' % base64.encodebytes(basicAuth.encode('utf8')).strip()],
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
50 }
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
51 uri = config.value(ROOM['wifiScraper'], ROOM['deviceInfoPage'])
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
52 resp = yield fetch(uri.encode('utf8'), method=b'GET', headers=headers)
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
53
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
54 if not resp.body.startswith((b'device=',
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
55 b'device_changed=0\ndevice=',
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
56 b'device_changed=1\ndevice=')):
341
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
57 raise ValueError(resp.body)
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
58
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
59 log.debug(resp.body)
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
60 rows = []
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
61 for row in json.loads(resp.body.split(b'device=', 1)[-1]):
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
62 triples = set()
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
63 uri = macUri(row['mac'].lower())
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
64
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
65 if row['contype'] in ['2.4G', '5G']:
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
66 orbi = macUri(row['conn_orbi_mac'])
641
5fd7aef3b2b2 index page updates. include wifi badg data.
drewp@bigasterisk.com
parents: 565
diff changeset
67 ct = ROOM['wifiBand/%s' % row['contype']]
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
68 triples.add((uri, ROOM['connectedToAp'], orbi))
641
5fd7aef3b2b2 index page updates. include wifi badg data.
drewp@bigasterisk.com
parents: 565
diff changeset
69 triples.add((uri, ROOM['wifiBand'], ct))
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
70 triples.add((orbi, RDF.type, ROOM['AccessPoint']))
641
5fd7aef3b2b2 index page updates. include wifi badg data.
drewp@bigasterisk.com
parents: 565
diff changeset
71 triples.add((orbi, ROOM['wifiBand'], ct))
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
72 triples.add((orbi, ROOM['macAddress'],
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
73 Literal(row['conn_orbi_mac'].lower())))
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
74 triples.add((orbi, RDFS.label, Literal(row['conn_orbi_name'])))
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
75 elif row['contype'] == 'wireless':
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
76 pass
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
77 elif row['contype'] == 'wired':
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
78 pass
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
79 elif row['contype'] == '-':
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
80 pass
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
81 else:
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
82 pass
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
83 triples.add((uri, ROOM['connectedToNet'], ROOM['HouseOpenNet']))
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
84
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
85 if row['model'] != 'Unknown':
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
86 triples.add((uri, ROOM['networkModel'], Literal(row['model'])))
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
87
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
88 rows.append(SeenNode(
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
89 uri=uri,
341
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
90 mac=row['mac'].lower(),
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
91 ip=row['ip'],
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
92 stmts=triples))
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
93 returnValue(rows)