Mercurial > code > home > repos > homeauto

annotate service/wifi/scrape.py @ 1226:7de8f0cd3392

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code Ignore-this: bfecea6f4990d34b36cc6d97cc6c6fa2 darcs-hash:79a3d55bfae5776b2374faa2020b6e27f17eb390
author drewp <drewp@bigasterisk.com>
date Sat, 30 Mar 2019 23:38:47 -0700
parents b8c0daabe5a5
children ec0db53aa833
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
1 import logging, json, base64
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
2 from typing import List
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
3
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
4 from cyclone.httpclient import fetch
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
5 from rdflib import Literal, Graph, RDF, URIRef, Namespace
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
6 from twisted.internet.defer import inlineCallbacks, returnValue
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
7
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
8 log = logging.getLogger()
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
9 ROOM = Namespace("http://projects.bigasterisk.com/room/")
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
10 AST = Namespace("http://bigasterisk.com/")
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
11
1225
b8c0daabe5a5 factor out some URI generation
drewp <drewp@bigasterisk.com>
parents: 1224
diff changeset
12 def macUri(macAddress: str) -> URIRef:
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
13 return URIRef("http://bigasterisk.com/mac/%s" % macAddress.lower())
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
14
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
15 class SeenNode(object):
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
16 def __init__(self, uri: URIRef, mac: str, ip: str, pred_objs: List):
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
17 self.connected = True
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
18 self.uri = uri
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
19 self.mac = mac
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
20 self.ip = ip
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
21 self.stmts = [(uri, p, o) for p, o in pred_objs]
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
22
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
23 class Wifi(object):
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
24 """
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
25 gather the users of wifi from the tomato routers
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
26 """
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
27 def __init__(self, config: Graph):
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
28 self.config = config
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
29
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
30 @inlineCallbacks
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
31 def getPresentMacAddrs(self): # returnValue List[SeenNode]
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
32 rows = yield self._loader()(self.config)
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
33 returnValue(rows)
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
34
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
35 def _loader(self):
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
36 cls = self.config.value(ROOM['wifiScraper'], RDF.type)
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
37 if cls == ROOM['OrbiScraper']:
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
38 return loadOrbiData
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
39 raise NotImplementedError(cls)
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
40
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
41
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
42 @inlineCallbacks
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
43 def loadOrbiData(config):
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
44 user = config.value(ROOM['wifiScraper'], ROOM['user'])
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
45 passwd = config.value(ROOM['wifiScraper'], ROOM['password'])
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
46 basicAuth = '%s:%s' % (user, passwd)
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
47 headers = {
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
48 b'Authorization': [
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
49 b'Basic %s' % base64.encodebytes(basicAuth.encode('utf8')).strip()],
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
50 }
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
51 uri = config.value(ROOM['wifiScraper'], ROOM['deviceInfoPage'])
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
52 resp = yield fetch(uri.encode('utf8'), method=b'GET', headers=headers)
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
53
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
54 if not resp.body.startswith((b'device=',
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
55 b'device_changed=0\ndevice=',
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
56 b'device_changed=1\ndevice=')):
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
57 raise ValueError(resp.body)
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
58
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
59 log.debug(resp.body)
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
60 rows = []
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
61 for row in json.loads(resp.body.split(b'device=', 1)[-1]):
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
62 extra = []
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
63 extra.append((ROOM['connected'], {
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
64 'wireless': AST['wifiAccessPoints'],
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
65 '2.4G': AST['wifiAccessPoints'],
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
66 '5G': AST['wifiAccessPoints'],
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
67 '-': AST['wifiUnknownConnectionType'],
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
68 'Unknown': AST['wifiUnknownConnectionType'],
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
69 'wired': AST['houseOpenNet']}[row['contype']]))
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
70 if row['model'] != 'Unknown':
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
71 extra.append((ROOM['networkModel'], Literal(row['model'])))
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
72
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
73 rows.append(SeenNode(
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
74 uri=macUri(row['mac'].lower()),
1224
e1202af42d4d port to py3
drewp <drewp@bigasterisk.com>
parents:
diff changeset
75 mac=row['mac'].lower(),
1226
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
76 ip=row['ip'],
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
77 pred_objs=extra))
7de8f0cd3392 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp <drewp@bigasterisk.com>
parents: 1225
diff changeset
78 returnValue(rows)