Mercurial > code > home > repos > homeauto
annotate service/wifi/scrape.py @ 1679:f88ff1021ee0
checkpoint service/wifi
| author | drewp@bigasterisk.com |
|---|---|
| date | Mon, 27 Sep 2021 23:02:33 -0700 |
| parents | 5fd7aef3b2b2 |
| children | 41394bc1d1b0 |
| rev | line source |
|---|---|
| 1679 | 1 import base64 |
| 2 import json | |
| 3 import logging | |
| 4 import re | |
| 5 import time | |
| 6 from typing import Awaitable, Callable, Iterable, List | |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
7 |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
8 from cyclone.httpclient import fetch |
| 1679 | 9 from rdflib import Graph, Literal, Namespace, RDF, RDFS, URIRef |
| 0 | 10 |
| 11 log = logging.getLogger() | |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
12 ROOM = Namespace("http://projects.bigasterisk.com/room/") |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
13 AST = Namespace("http://bigasterisk.com/") |
| 0 | 14 |
| 1679 | 15 |
| 422 | 16 def macUri(macAddress: str) -> URIRef: |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
17 return URIRef("http://bigasterisk.com/mac/%s" % macAddress.lower()) |
|
51
d2842eedd56d
rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents:
36
diff
changeset
|
18 |
| 1679 | 19 |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
20 class SeenNode(object): |
| 1679 | 21 |
|
565
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
22 def __init__(self, uri: URIRef, mac: str, ip: str, stmts: Iterable): |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
23 self.connected = True |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
24 self.uri = uri |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
25 self.mac = mac |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
26 self.ip = ip |
|
565
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
27 self.stmts = stmts |
| 1679 | 28 |
| 29 | |
| 0 | 30 class Wifi(object): |
| 31 """ | |
| 32 gather the users of wifi from the tomato routers | |
| 33 """ | |
| 1679 | 34 |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
35 def __init__(self, config: Graph): |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
36 self.config = config |
|
51
d2842eedd56d
rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents:
36
diff
changeset
|
37 |
| 1679 | 38 async def getPresentMacAddrs(self) -> List[SeenNode]: |
| 39 rows = await self._loader()(self.config) | |
| 40 return rows | |
| 41 | |
| 42 def _loader(self) -> Callable[[Graph], Awaitable[List[SeenNode]]]: | |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
43 cls = self.config.value(ROOM['wifiScraper'], RDF.type) |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
44 if cls == ROOM['OrbiScraper']: |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
45 return loadOrbiData |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
46 raise NotImplementedError(cls) |
|
175
c81a451f9b26
rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents:
62
diff
changeset
|
47 |
|
c81a451f9b26
rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents:
62
diff
changeset
|
48 |
| 1679 | 49 async def loadOrbiData(config: Graph) -> List[SeenNode]: |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
50 user = config.value(ROOM['wifiScraper'], ROOM['user']) |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
51 passwd = config.value(ROOM['wifiScraper'], ROOM['password']) |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
52 basicAuth = '%s:%s' % (user, passwd) |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
53 headers = { |
| 1679 | 54 b'Authorization': [b'Basic %s' % base64.encodebytes(basicAuth.encode('utf8')).strip()], |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
55 } |
|
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
56 uri = config.value(ROOM['wifiScraper'], ROOM['deviceInfoPage']) |
| 1679 | 57 resp = await fetch(f"{uri}?ts={time.time()}".encode('utf8'), method=b'GET', headers=headers) |
| 58 | |
| 59 if not resp.body.startswith((b'device=', b'device_changed=0\ndevice=', b'device_changed=1\ndevice=')): | |
| 60 raise ValueError(resp.body) | |
| 421 | 61 |
| 1679 | 62 |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
63 rows = [] |
| 1679 | 64 for rowNum, row in enumerate(json.loads(resp.body.split(b'device=', 1)[-1])): |
| 65 log.debug('response row [%d] %r', rowNum, row) | |
| 66 if not re.match(r'\w\w:\w\w:\w\w:\w\w:\w\w:\w\w', row['mac']): | |
| 67 raise ValueError(f"corrupt response: mac was {row['mac']!r}") | |
|
565
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
68 triples = set() |
|
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
69 uri = macUri(row['mac'].lower()) |
| 1679 | 70 |
|
565
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
71 if row['contype'] in ['2.4G', '5G']: |
|
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
72 orbi = macUri(row['conn_orbi_mac']) |
|
641
5fd7aef3b2b2
index page updates. include wifi badg data.
drewp@bigasterisk.com
parents:
565
diff
changeset
|
73 ct = ROOM['wifiBand/%s' % row['contype']] |
|
565
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
74 triples.add((uri, ROOM['connectedToAp'], orbi)) |
|
641
5fd7aef3b2b2
index page updates. include wifi badg data.
drewp@bigasterisk.com
parents:
565
diff
changeset
|
75 triples.add((uri, ROOM['wifiBand'], ct)) |
|
565
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
76 triples.add((orbi, RDF.type, ROOM['AccessPoint'])) |
|
641
5fd7aef3b2b2
index page updates. include wifi badg data.
drewp@bigasterisk.com
parents:
565
diff
changeset
|
77 triples.add((orbi, ROOM['wifiBand'], ct)) |
| 1679 | 78 triples.add((orbi, ROOM['macAddress'], Literal(row['conn_orbi_mac'].lower()))) |
|
565
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
79 triples.add((orbi, RDFS.label, Literal(row['conn_orbi_name']))) |
|
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
80 elif row['contype'] == 'wireless': |
|
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
81 pass |
|
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
82 elif row['contype'] == 'wired': |
|
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
83 pass |
|
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
84 elif row['contype'] == '-': |
|
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
85 pass |
|
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
86 else: |
|
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
87 pass |
|
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
88 triples.add((uri, ROOM['connectedToNet'], ROOM['HouseOpenNet'])) |
|
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
89 |
|
423
e0703c7824e9
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents:
422
diff
changeset
|
90 if row['model'] != 'Unknown': |
|
565
cbb4b3ccdb53
redo wifi's statements about access points so we can distinguish who is connecting to where, and on which wifi band
drewp@bigasterisk.com
parents:
427
diff
changeset
|
91 triples.add((uri, ROOM['networkModel'], Literal(row['model']))) |
| 1679 | 92 |
| 93 rows.append(SeenNode(uri=uri, mac=row['mac'].lower(), ip=row['ip'], stmts=triples)) | |
| 94 return rows |