Mercurial > code > home > repos > homeauto

annotate service/wifi/scrape.py @ 427:db031d9ec28e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
don't use 'connected' for time and for network. add rdf:type. Ignore-this: 59899796d4785300f40137de2c2c44d5
author drewp@bigasterisk.com
date Thu, 04 Apr 2019 02:19:10 -0700
parents e0703c7824e9
children cbb4b3ccdb53
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
1 import logging, json, base64
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
2 from typing import List
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
3
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
4 from cyclone.httpclient import fetch
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
5 from rdflib import Literal, Graph, RDF, URIRef, Namespace
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
6 from twisted.internet.defer import inlineCallbacks, returnValue
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
7
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
8 log = logging.getLogger()
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
9 ROOM = Namespace("http://projects.bigasterisk.com/room/")
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
10 AST = Namespace("http://bigasterisk.com/")
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
11
422
19460b3f7baf factor out some URI generation
drewp@bigasterisk.com
parents: 421
diff changeset
12 def macUri(macAddress: str) -> URIRef:
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
13 return URIRef("http://bigasterisk.com/mac/%s" % macAddress.lower())
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
14
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
15 class SeenNode(object):
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
16 def __init__(self, uri: URIRef, mac: str, ip: str, pred_objs: List):
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
17 self.connected = True
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
18 self.uri = uri
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
19 self.mac = mac
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
20 self.ip = ip
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
21 self.stmts = [(uri, p, o) for p, o in pred_objs]
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
22
0
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
23 class Wifi(object):
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
24 """
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
25 gather the users of wifi from the tomato routers
Drew Perttula <drewp@bigasterisk.com>
parents:
diff changeset
26 """
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
27 def __init__(self, config: Graph):
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
28 self.config = config
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
29
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
30 @inlineCallbacks
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
31 def getPresentMacAddrs(self): # returnValue List[SeenNode]
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
32 rows = yield self._loader()(self.config)
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
33 returnValue(rows)
51
d2842eedd56d rewrite tomatowifi from restkit to cyclone httpclient
drewp@bigasterisk.com
parents: 36
diff changeset
34
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
35 def _loader(self):
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
36 cls = self.config.value(ROOM['wifiScraper'], RDF.type)
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
37 if cls == ROOM['OrbiScraper']:
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
38 return loadOrbiData
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
39 raise NotImplementedError(cls)
175
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
40
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
41
c81a451f9b26 rewrites for better graph export, removal of dhcp reader
drewp@bigasterisk.com
parents: 62
diff changeset
42 @inlineCallbacks
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
43 def loadOrbiData(config):
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
44 user = config.value(ROOM['wifiScraper'], ROOM['user'])
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
45 passwd = config.value(ROOM['wifiScraper'], ROOM['password'])
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
46 basicAuth = '%s:%s' % (user, passwd)
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
47 headers = {
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
48 b'Authorization': [
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
49 b'Basic %s' % base64.encodebytes(basicAuth.encode('utf8')).strip()],
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
50 }
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
51 uri = config.value(ROOM['wifiScraper'], ROOM['deviceInfoPage'])
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
52 resp = yield fetch(uri.encode('utf8'), method=b'GET', headers=headers)
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
53
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
54 if not resp.body.startswith((b'device=',
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
55 b'device_changed=0\ndevice=',
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
56 b'device_changed=1\ndevice=')):
341
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
57 raise ValueError(resp.body)
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
58
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
59 log.debug(resp.body)
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
60 rows = []
421
47d7dd31bb2c port to py3
drewp@bigasterisk.com
parents: 420
diff changeset
61 for row in json.loads(resp.body.split(b'device=', 1)[-1]):
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
62 extra = []
427
db031d9ec28e don't use 'connected' for time and for network. add rdf:type.
drewp@bigasterisk.com
parents: 423
diff changeset
63 extra.append((ROOM['connectedToNetwork'], {
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
64 'wireless': AST['wifiAccessPoints'],
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
65 '2.4G': AST['wifiAccessPoints'],
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
66 '5G': AST['wifiAccessPoints'],
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
67 '-': AST['wifiUnknownConnectionType'],
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
68 'Unknown': AST['wifiUnknownConnectionType'],
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
69 'wired': AST['houseOpenNet']}[row['contype']]))
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
70 if row['model'] != 'Unknown':
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
71 extra.append((ROOM['networkModel'], Literal(row['model'])))
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
72
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
73 rows.append(SeenNode(
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
74 uri=macUri(row['mac'].lower()),
341
f20e66ace980 wifi support for scraping Orbi admin page
drewp@bigasterisk.com
parents: 272
diff changeset
75 mac=row['mac'].lower(),
423
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
76 ip=row['ip'],
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
77 pred_objs=extra))
e0703c7824e9 very big rewrite. py3; orbi-only for now; n3 config file; delete or move out dead code
drewp@bigasterisk.com
parents: 422
diff changeset
78 returnValue(rows)