Mercurial > code > home > repos > homeauto

comparison service/wifi/scrape.py @ 1679:f88ff1021ee0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
checkpoint service/wifi
author drewp@bigasterisk.com
date Mon, 27 Sep 2021 23:02:33 -0700
parents 5fd7aef3b2b2
children 41394bc1d1b0
comparison
equal deleted inserted replaced
1678:7831b5de3572 1679:f88ff1021ee0
1 import logging, json, base64 1 import base64
2 from typing import List, Iterable 2 import json
3 import logging
4 import re
5 import time
6 from typing import Awaitable, Callable, Iterable, List
3 7
4 from cyclone.httpclient import fetch 8 from cyclone.httpclient import fetch
5 from rdflib import Literal, Graph, RDF, URIRef, Namespace, RDFS 9 from rdflib import Graph, Literal, Namespace, RDF, RDFS, URIRef
6 from twisted.internet.defer import inlineCallbacks, returnValue
7 10
8 log = logging.getLogger() 11 log = logging.getLogger()
9 ROOM = Namespace("http://projects.bigasterisk.com/room/") 12 ROOM = Namespace("http://projects.bigasterisk.com/room/")
10 AST = Namespace("http://bigasterisk.com/") 13 AST = Namespace("http://bigasterisk.com/")
11 14
15
12 def macUri(macAddress: str) -> URIRef: 16 def macUri(macAddress: str) -> URIRef:
13 return URIRef("http://bigasterisk.com/mac/%s" % macAddress.lower()) 17 return URIRef("http://bigasterisk.com/mac/%s" % macAddress.lower())
14 18
19
15 class SeenNode(object): 20 class SeenNode(object):
21
16 def __init__(self, uri: URIRef, mac: str, ip: str, stmts: Iterable): 22 def __init__(self, uri: URIRef, mac: str, ip: str, stmts: Iterable):
17 self.connected = True 23 self.connected = True
18 self.uri = uri 24 self.uri = uri
19 self.mac = mac 25 self.mac = mac
20 self.ip = ip 26 self.ip = ip
21 self.stmts = stmts 27 self.stmts = stmts
22 28
29
23 class Wifi(object): 30 class Wifi(object):
24 """ 31 """
25 gather the users of wifi from the tomato routers 32 gather the users of wifi from the tomato routers
26 """ 33 """
34
27 def __init__(self, config: Graph): 35 def __init__(self, config: Graph):
28 self.config = config 36 self.config = config
29
30 @inlineCallbacks
31 def getPresentMacAddrs(self): # returnValue List[SeenNode]
32 rows = yield self._loader()(self.config)
33 returnValue(rows)
34 37
35 def _loader(self): 38 async def getPresentMacAddrs(self) -> List[SeenNode]:
39 rows = await self._loader()(self.config)
40 return rows
41
42 def _loader(self) -> Callable[[Graph], Awaitable[List[SeenNode]]]:
36 cls = self.config.value(ROOM['wifiScraper'], RDF.type) 43 cls = self.config.value(ROOM['wifiScraper'], RDF.type)
37 if cls == ROOM['OrbiScraper']: 44 if cls == ROOM['OrbiScraper']:
38 return loadOrbiData 45 return loadOrbiData
39 raise NotImplementedError(cls) 46 raise NotImplementedError(cls)
40 47
41 48
42 @inlineCallbacks 49 async def loadOrbiData(config: Graph) -> List[SeenNode]:
43 def loadOrbiData(config):
44 user = config.value(ROOM['wifiScraper'], ROOM['user']) 50 user = config.value(ROOM['wifiScraper'], ROOM['user'])
45 passwd = config.value(ROOM['wifiScraper'], ROOM['password']) 51 passwd = config.value(ROOM['wifiScraper'], ROOM['password'])
46 basicAuth = '%s:%s' % (user, passwd) 52 basicAuth = '%s:%s' % (user, passwd)
47 headers = { 53 headers = {
48 b'Authorization': [ 54 b'Authorization': [b'Basic %s' % base64.encodebytes(basicAuth.encode('utf8')).strip()],
49 b'Basic %s' % base64.encodebytes(basicAuth.encode('utf8')).strip()],
50 } 55 }
51 uri = config.value(ROOM['wifiScraper'], ROOM['deviceInfoPage']) 56 uri = config.value(ROOM['wifiScraper'], ROOM['deviceInfoPage'])
52 resp = yield fetch(uri.encode('utf8'), method=b'GET', headers=headers) 57 resp = await fetch(f"{uri}?ts={time.time()}".encode('utf8'), method=b'GET', headers=headers)
53 58
54 if not resp.body.startswith((b'device=', 59 if not resp.body.startswith((b'device=', b'device_changed=0\ndevice=', b'device_changed=1\ndevice=')):
55 b'device_changed=0\ndevice=',
56 b'device_changed=1\ndevice=')):
57 raise ValueError(resp.body) 60 raise ValueError(resp.body)
58 61
59 log.debug(resp.body) 62
60 rows = [] 63 rows = []
61 for row in json.loads(resp.body.split(b'device=', 1)[-1]): 64 for rowNum, row in enumerate(json.loads(resp.body.split(b'device=', 1)[-1])):
65 log.debug('response row [%d] %r', rowNum, row)
66 if not re.match(r'\w\w:\w\w:\w\w:\w\w:\w\w:\w\w', row['mac']):
67 raise ValueError(f"corrupt response: mac was {row['mac']!r}")
62 triples = set() 68 triples = set()
63 uri = macUri(row['mac'].lower()) 69 uri = macUri(row['mac'].lower())
64 70
65 if row['contype'] in ['2.4G', '5G']: 71 if row['contype'] in ['2.4G', '5G']:
66 orbi = macUri(row['conn_orbi_mac']) 72 orbi = macUri(row['conn_orbi_mac'])
67 ct = ROOM['wifiBand/%s' % row['contype']] 73 ct = ROOM['wifiBand/%s' % row['contype']]
68 triples.add((uri, ROOM['connectedToAp'], orbi)) 74 triples.add((uri, ROOM['connectedToAp'], orbi))
69 triples.add((uri, ROOM['wifiBand'], ct)) 75 triples.add((uri, ROOM['wifiBand'], ct))
70 triples.add((orbi, RDF.type, ROOM['AccessPoint'])) 76 triples.add((orbi, RDF.type, ROOM['AccessPoint']))
71 triples.add((orbi, ROOM['wifiBand'], ct)) 77 triples.add((orbi, ROOM['wifiBand'], ct))
72 triples.add((orbi, ROOM['macAddress'], 78 triples.add((orbi, ROOM['macAddress'], Literal(row['conn_orbi_mac'].lower())))
73 Literal(row['conn_orbi_mac'].lower())))
74 triples.add((orbi, RDFS.label, Literal(row['conn_orbi_name']))) 79 triples.add((orbi, RDFS.label, Literal(row['conn_orbi_name'])))
75 elif row['contype'] == 'wireless': 80 elif row['contype'] == 'wireless':
76 pass 81 pass
77 elif row['contype'] == 'wired': 82 elif row['contype'] == 'wired':
78 pass 83 pass
82 pass 87 pass
83 triples.add((uri, ROOM['connectedToNet'], ROOM['HouseOpenNet'])) 88 triples.add((uri, ROOM['connectedToNet'], ROOM['HouseOpenNet']))
84 89
85 if row['model'] != 'Unknown': 90 if row['model'] != 'Unknown':
86 triples.add((uri, ROOM['networkModel'], Literal(row['model']))) 91 triples.add((uri, ROOM['networkModel'], Literal(row['model'])))
87 92
88 rows.append(SeenNode( 93 rows.append(SeenNode(uri=uri, mac=row['mac'].lower(), ip=row['ip'], stmts=triples))
89 uri=uri, 94 return rows
90 mac=row['mac'].lower(),
91 ip=row['ip'],
92 stmts=triples))
93 returnValue(rows)