Mercurial > code > home > repos > homeauto

annotate service/wifi/scrape.py @ 605:ad4c4d7c1fb9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
reasoning output using treq, and keep writing to PUT calls forever (but not as fast as the reasoning loop runs) Ignore-this: 1633b16dc315082f759041d42e848ced
author drewp@bigasterisk.com
date Tue, 23 Jul 2019 17:30:46 -0700
parents cbb4b3ccdb53
children 5fd7aef3b2b2
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
1 import logging, json, base64
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
2 from typing import List, Iterable
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
3
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
4 from cyclone.httpclient import fetch
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
5 from rdflib import Literal, Graph, RDF, URIRef, Namespace, RDFS
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
6 from twisted.internet.defer import inlineCallbacks, returnValue
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
7
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
8 log = logging.getLogger()
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
9 ROOM = Namespace("http://projects.bigasterisk.com/room/")
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
10 AST = Namespace("http://bigasterisk.com/")
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
11
422
19460b3f7baf factor out some URI generation
drewp@bigasterisk.com
parents: 421
diff changeset
12 def macUri(macAddress: str) -> URIRef:
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
13 return URIRef("http://bigasterisk.com/mac/%s" % macAddress.lower())
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
14
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
15 class SeenNode(object):
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
16 def __init__(self, uri: URIRef, mac: str, ip: str, stmts: Iterable):
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
17 self.connected = True
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
18 self.uri = uri
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
19 self.mac = mac
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
20 self.ip = ip
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
21 self.stmts = stmts
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
22
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
23 class Wifi(object):
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
24 """
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
25 gather the users of wifi from the tomato routers
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
26 """
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
27 def __init__(self, config: Graph):
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
28 self.config = config
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
29
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
30 @inlineCallbacks
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
31 def getPresentMacAddrs(self): # returnValue List[SeenNode]
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
32 rows = yield self._loader()(self.config)
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
33 returnValue(rows)
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
34
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
35 def _loader(self):
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
36 cls = self.config.value(ROOM['wifiScraper'], RDF.type)
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
37 if cls == ROOM['OrbiScraper']:
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
38 return loadOrbiData
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
39 raise NotImplementedError(cls)
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
40
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
41
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
42 @inlineCallbacks
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
43 def loadOrbiData(config):
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
44 user = config.value(ROOM['wifiScraper'], ROOM['user'])
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
45 passwd = config.value(ROOM['wifiScraper'], ROOM['password'])
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
46 basicAuth = '%s:%s' % (user, passwd)
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
47 headers = {
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
48 b'Authorization': [
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
49 b'Basic %s' % base64.encodebytes(basicAuth.encode('utf8')).strip()],
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
50 }
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
51 uri = config.value(ROOM['wifiScraper'], ROOM['deviceInfoPage'])
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
52 resp = yield fetch(uri.encode('utf8'), method=b'GET', headers=headers)
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
53
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
54 if not resp.body.startswith((b'device=',
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
55 b'device_changed=0\ndevice=',
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
56 b'device_changed=1\ndevice=')):
341
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
57 raise ValueError(resp.body)
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
58
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
59 log.debug(resp.body)
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
60 rows = []
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
61 for row in json.loads(resp.body.split(b'device=', 1)[-1]):
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
62 triples = set()
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
63 uri = macUri(row['mac'].lower())
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
64
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
65 if row['contype'] in ['2.4G', '5G']:
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
66 orbi = macUri(row['conn_orbi_mac'])
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
67 triples.add((orbi, ROOM['wifiBand'],
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
68 ROOM['wifiBand/%s' % row['contype']]))
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
69 triples.add((uri, ROOM['connectedToAp'], orbi))
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
70 triples.add((orbi, RDF.type, ROOM['AccessPoint']))
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
71 triples.add((orbi, ROOM['macAddress'],
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
72 Literal(row['conn_orbi_mac'].lower())))
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
73 triples.add((orbi, RDFS.label, Literal(row['conn_orbi_name'])))
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
74 elif row['contype'] == 'wireless':
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
75 pass
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
76 elif row['contype'] == 'wired':
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
77 pass
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
78 elif row['contype'] == '-':
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
79 pass
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
80 else:
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
81 pass
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
82 triples.add((uri, ROOM['connectedToNet'], ROOM['HouseOpenNet']))
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
83
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
84 if row['model'] != 'Unknown':
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
85 triples.add((uri, ROOM['networkModel'], Literal(row['model'])))
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
86
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
87 rows.append(SeenNode(
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
88 uri=uri,
341
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
89 mac=row['mac'].lower(),
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
90 ip=row['ip'],
565
cbb4b3ccdb53 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents: 427
diff changeset
91 stmts=triples))
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
92 returnValue(rows)