Mercurial > code > home > repos > homeauto

annotate service/wifi/scrape.py @ 1444:4afb1830bb5e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
use rx version 3.x Ignore-this: 4232f8e780d35a8d0642e86521eb2801 darcs-hash:747608892b607f78260f4772a4ff2b24c7392f73
author drewp <drewp@bigasterisk.com>
date Tue, 24 Sep 2019 14:04:02 -0700
parents 140d8d419cf0
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
1 import logging, json, base64
1368
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
2 from typing import List, Iterable
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
3
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
4 from cyclone.httpclient import fetch
1368
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
5 from rdflib import Literal, Graph, RDF, URIRef, Namespace, RDFS
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
6 from twisted.internet.defer import inlineCallbacks, returnValue
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
7
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
8 log = logging.getLogger()
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
9 ROOM = Namespace("http://projects.bigasterisk.com/room/")
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
10 AST = Namespace("http://bigasterisk.com/")
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
11
1225
b8c0daabe5a5 factor out some URI generation
drewp <drewp@bigasterisk.com>
parents: 1224
diff changeset
12 def macUri(macAddress: str) -> URIRef:
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
13 return URIRef("http://bigasterisk.com/mac/%s" % macAddress.lower())
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
14
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
15 class SeenNode(object):
1368
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
16 def __init__(self, uri: URIRef, mac: str, ip: str, stmts: Iterable):
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
17 self.connected = True
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
18 self.uri = uri
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
19 self.mac = mac
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
20 self.ip = ip
1368
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
21 self.stmts = stmts
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
22
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
23 class Wifi(object):
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
24 """
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
25 gather the users of wifi from the tomato routers
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
26 """
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
27 def __init__(self, config: Graph):
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
28 self.config = config
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
29
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
30 @inlineCallbacks
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
31 def getPresentMacAddrs(self): # returnValue List[SeenNode]
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
32 rows = yield self._loader()(self.config)
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
33 returnValue(rows)
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
34
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
35 def _loader(self):
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
36 cls = self.config.value(ROOM['wifiScraper'], RDF.type)
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
37 if cls == ROOM['OrbiScraper']:
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
38 return loadOrbiData
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
39 raise NotImplementedError(cls)
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
40
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
41
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
42 @inlineCallbacks
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
43 def loadOrbiData(config):
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
44 user = config.value(ROOM['wifiScraper'], ROOM['user'])
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
45 passwd = config.value(ROOM['wifiScraper'], ROOM['password'])
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
46 basicAuth = '%s:%s' % (user, passwd)
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
47 headers = {
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
48 b'Authorization': [
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
49 b'Basic %s' % base64.encodebytes(basicAuth.encode('utf8')).strip()],
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
50 }
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
51 uri = config.value(ROOM['wifiScraper'], ROOM['deviceInfoPage'])
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
52 resp = yield fetch(uri.encode('utf8'), method=b'GET', headers=headers)
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
53
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
54 if not resp.body.startswith((b'device=',
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
55 b'device_changed=0\ndevice=',
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
56 b'device_changed=1\ndevice=')):
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
57 raise ValueError(resp.body)
1368
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
58
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
59 log.debug(resp.body)
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
60 rows = []
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
61 for row in json.loads(resp.body.split(b'device=', 1)[-1]):
1368
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
62 triples = set()
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
63 uri = macUri(row['mac'].lower())
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
64
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
65 if row['contype'] in ['2.4G', '5G']:
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
66 orbi = macUri(row['conn_orbi_mac'])
1444
4afb1830bb5e use rx version 3.x
drewp <drewp@bigasterisk.com>
parents: 1368
diff changeset
67 ct = ROOM['wifiBand/%s' % row['contype']]
1368
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
68 triples.add((uri, ROOM['connectedToAp'], orbi))
1444
4afb1830bb5e use rx version 3.x
drewp <drewp@bigasterisk.com>
parents: 1368
diff changeset
69 triples.add((uri, ROOM['wifiBand'], ct))
1368
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
70 triples.add((orbi, RDF.type, ROOM['AccessPoint']))
1444
4afb1830bb5e use rx version 3.x
drewp <drewp@bigasterisk.com>
parents: 1368
diff changeset
71 triples.add((orbi, ROOM['wifiBand'], ct))
1368
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
72 triples.add((orbi, ROOM['macAddress'],
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
73 Literal(row['conn_orbi_mac'].lower())))
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
74 triples.add((orbi, RDFS.label, Literal(row['conn_orbi_name'])))
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
75 elif row['contype'] == 'wireless':
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
76 pass
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
77 elif row['contype'] == 'wired':
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
78 pass
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
79 elif row['contype'] == '-':
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
80 pass
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
81 else:
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
82 pass
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
83 triples.add((uri, ROOM['connectedToNet'], ROOM['HouseOpenNet']))
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
84
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
85 if row['model'] != 'Unknown':
1368
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
86 triples.add((uri, ROOM['networkModel'], Literal(row['model'])))
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
87
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
88 rows.append(SeenNode(
1368
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
89 uri=uri,
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
90 mac=row['mac'].lower(),
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
91 ip=row['ip'],
1368
140d8d419cf0 redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp <drewp@bigasterisk.com>
parents: 1230
diff changeset
92 stmts=triples))
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
93 returnValue(rows)