--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/service/tomatoWifi/dhcpparse.py	Sat Apr 06 17:08:19 2013 -0700
@@ -0,0 +1,23 @@
+def addDhcpData(rows):
+    """given dicts with 'mac', add other data from the dhcp cache"""
+    currentLease = None
+    for line in open('/var/lib/dhcp/dhcpd.leases'):
+        if line.startswith('lease '):
+            currentLease = {'ip': line.split()[1]}
+        elif line.startswith('  hardware ethernet '):
+            currentLease['mac'] = line.split()[2].strip(';').upper()
+        elif line.startswith('  client-hostname'):
+            currentLease['clientHostname'] = line.split(None, 2)[1].strip('";')
+        elif line.startswith('  binding state'):
+            currentLease['bindingState'] = line.split()[2].strip(';')
+        elif line.startswith('}'):
+            if currentLease.get('bindingState') == 'active':
+                # there seem to be a lot of 'active' blocks
+                # for the same ip addr and mac. I haven't
+                # looked into whether they hold different
+                # lease terms or what
+                for r in rows:
+                    if r['mac'] == currentLease['mac']:
+                        r.update(currentLease)
+                        
+            

--- a/service/tomatoWifi/table.mustache	Sun Feb 10 13:41:35 2013 -0800
+++ b/service/tomatoWifi/table.mustache	Sat Apr 06 17:08:19 2013 -0700
@@ -5,8 +5,8 @@
 
   <table><tr> 
 
-    <th class="name">Name</th> 
-    <th class="lease">Lease</th> 
+    <th class="name">Name</th>
+    <th class="connected">Connected</th>
     <th class="ip">IP address</th> 
     <th class="rssi">dBm</th> 
     <th class="mac">MAC address</th> 
@@ -15,7 +15,7 @@
   {{#rows}}
   <tr class="{{cls}}">
     <td>{{name}}</td>
-    <td class="lease">{{lease}}</td>
+    <td class="connected">{{connectedAgo}}</td>
     <td class="ip">{{ip}}</td>
     <td class="rssi">{{signal}}</td>
     <td class="mac">{{mac}}</td>
@@ -25,4 +25,4 @@
   </table>
 
 
-</div>
\ No newline at end of file
+</div>

--- a/service/tomatoWifi/tomatoWifi.py	Sun Feb 10 13:41:35 2013 -0800
+++ b/service/tomatoWifi/tomatoWifi.py	Sat Apr 06 17:08:19 2013 -0700
@@ -12,18 +12,19 @@
 """
 from __future__ import division
 import sys, cyclone.web, json, traceback, time, pystache, datetime, logging
+import web.utils
 from cyclone.httpclient import fetch
 sys.path.append("/home/drewp/projects/photo/lib/python2.7/site-packages")
 from dateutil import tz
 from twisted.internet import reactor, task
-from twisted.internet.defer import inlineCallbacks, returnValue
-
+from twisted.internet.defer import inlineCallbacks
 
 from pymongo import Connection, DESCENDING
 from rdflib import Namespace, Literal, URIRef
 sys.path.append("/my/site/magma")
 from stategraph import StateGraph
 from wifi import Wifi
+from dhcpparse import addDhcpData
 
 sys.path.append("/my/proj/homeauto/lib")
 from cycloneerr import PrettyErrorHandler
@@ -53,21 +54,44 @@
 
 class Table(PrettyErrorHandler, cyclone.web.RequestHandler):
     def get(self):
-        def rowDict(addr):
-            addr['cls'] = "signal" if addr.get('signal') else "nosignal"
-            if 'lease' in addr:
-                addr['lease'] = addr['lease'].replace("0 days, ", "")
-            return addr
+        def rowDict(row):
+            row['cls'] = "signal" if row.get('connected') else "nosignal"
+            if 'name' not in row:
+                row['name'] = row['clientHostname']
+            if 'signal' not in row:
+                row['signal'] = 'yes' if row['connected'] else 'no'
+
+            try:
+                conn = self.whenConnected(row['mac'])
+                row['connectedAgo'] = web.utils.datestr(
+                    conn.astimezone(tz.tzutc()).replace(tzinfo=None))
+            except ValueError:
+                pass
+            
+            return row
 
         self.set_header("Content-Type", "application/xhtml+xml")
         self.write(pystache.render(
             open("table.mustache").read(),
             dict(
                 rows=sorted(map(rowDict, self.settings.poller.lastAddrs),
-                            key=lambda a: (a.get('router'),
-                                           a.get('name'),
-                                           a.get('mac'))))))
+                            key=lambda a: (not a.get('connected'),
+                                           a.get('name'))))))
 
+    def whenConnected(self, macThatIsNowConnected):
+        lastArrive = None
+        for ev in self.settings.mongo.find({'address': macThatIsNowConnected},
+                                           sort=[('created', -1)],
+                                           max_scan=100000):
+            if ev['action'] == 'arrive':
+                lastArrive = ev
+            if ev['action'] == 'leave':
+                break
+        if lastArrive is None:
+            raise ValueError("no past arrivals")
+        
+        return lastArrive['created']
+        
 
 class Json(PrettyErrorHandler, cyclone.web.RequestHandler):
     def get(self):
@@ -91,36 +115,22 @@
             raise ValueError("poll data is stale. age=%s" % age)
 
         for dev in self.settings.poller.lastAddrs:
-            if not dev.get('signal'):
+            if not dev.get('connected'):
                 continue
             uri = URIRef("http://bigasterisk.com/wifiDevice/%s" % dev['mac'])
             g.add((uri, ROOM['macAddress'], Literal(dev['mac'])))
+            
             g.add((uri, ROOM['connected'], aps))
-            if 'rawName' in dev:
-                g.add((uri, ROOM['wifiNetworkName'], Literal(dev['rawName'])))
-            g.add((uri, ROOM['deviceName'], Literal(dev['name'])))
-            g.add((uri, ROOM['signalStrength'], Literal(dev['signal'])))
+            if 'clientHostname' in dev:
+                g.add((uri, ROOM['wifiNetworkName'], Literal(dev['clientHostname'])))
+            if 'name' in dev:
+                g.add((uri, ROOM['deviceName'], Literal(dev['name'])))
+            if 'signal' in dev:
+                g.add((uri, ROOM['signalStrength'], Literal(dev['signal'])))
 
         self.set_header('Content-type', 'application/x-trig')
         self.write(g.asTrig())
 
-class Application(cyclone.web.Application):
-    def __init__(self, wifi, poller):
-        handlers = [
-            (r"/", Index),
-            (r'/json', Json),
-            (r'/graph', GraphHandler),
-            (r'/table', Table),
-            #(r'/activity', Activity),
-        ]
-        settings = {
-            'wifi' : wifi,
-            'poller' : poller,
-            'mongo' : Connection('bang', 27017,
-                                 tz_aware=True)['house']['sensor']
-            }
-        cyclone.web.Application.__init__(self, handlers, **settings)
-
 class Poller(object):
     def __init__(self, wifi, mongo):
         self.wifi = wifi
@@ -137,8 +147,9 @@
     def poll(self):
         try:
             newAddrs = yield self.wifi.getPresentMacAddrs()
-
-            newWithSignal = [a for a in newAddrs if a.get('signal')]
+            addDhcpData(newAddrs)
+            
+            newWithSignal = [a for a in newAddrs if a.get('connected')]
 
             actions = self.computeActions(newWithSignal)
             for action in actions:
@@ -161,33 +172,27 @@
             log.error("poll error: %s\n%s", e, traceback.format_exc())
 
     def computeActions(self, newWithSignal):
-        def removeVolatile(a):
-            ret = dict((k,v) for k,v in a.items() if k in ['name', 'mac'])
-            ret['signal'] = bool(a.get('signal'))
-            return ret
-
-        def find(a, others):
-            a = removeVolatile(a)
-            return any(a == removeVolatile(o) for o in others)
-
         actions = []
 
         def makeAction(addr, act):
-            return dict(sensor="wifi",
+            d = dict(sensor="wifi",
                         address=addr.get('mac'),
                         name=addr.get('name'),
-                        networkName=addr.get('rawName'),
+                        networkName=addr.get('clientHostname'),
                         action=act)
+            if act == 'arrive' and 'ip' in addr:
+                # this won't cover the possible case that you get on
+                # wifi but don't have an ip yet. We'll record an
+                # action with no ip and then never record your ip.
+                d['ip'] = addr['ip']
+            return d                             
 
         for addr in newWithSignal:
-            if not find(addr, self.lastWithSignal):
-                # the point of all the removeVolatile stuff is so
-                # I have the complete addr object here, although
-                # it is currently mostly thrown out by makeAction
+            if addr['mac'] not in [r['mac'] for r in self.lastWithSignal]:
                 actions.append(makeAction(addr, 'arrive'))
 
         for addr in self.lastWithSignal:
-            if not find(addr, newWithSignal):
+            if addr['mac'] not in [r['mac'] for r in newWithSignal]:
                 actions.append(makeAction(addr, 'leave'))
 
         return actions
@@ -227,13 +232,24 @@
     from twisted.python import log as twlog
     #log.startLogging(sys.stdout)
     #log.setLevel(10)
-    log.setLevel(logging.DEBUG)
+    #log.setLevel(logging.DEBUG)
 
-    mongo = Connection('bang', 27017)['visitor']['visitor']
+    mongo = Connection('bang', 27017, tz_aware=True)['visitor']['visitor']
 
     wifi = Wifi()
     poller = Poller(wifi, mongo)
     task.LoopingCall(poller.poll).start(1/config['pollFrequency'])
 
-    reactor.listenTCP(config['servePort'], Application(wifi, poller))
+    reactor.listenTCP(config['servePort'],
+                      cyclone.web.Application(
+                          [
+                              (r"/", Index),
+                              (r'/json', Json),
+                              (r'/graph', GraphHandler),
+                              (r'/table', Table),
+                              #(r'/activity', Activity),
+                          ],
+                          wifi=wifi,
+                          poller=poller,
+                          mongo=mongo))
     reactor.run()

--- a/service/tomatoWifi/wifi.py	Sun Feb 10 13:41:35 2013 -0800
+++ b/service/tomatoWifi/wifi.py	Sat Apr 06 17:08:19 2013 -0700
@@ -1,4 +1,5 @@
 import re, ast, logging, socket
+import lxml.html.soupparser
 from twisted.internet.defer import inlineCallbacks, returnValue
 from cyclone.httpclient import fetch
 from rdflib import Literal, Graph
@@ -12,6 +13,8 @@
 class Wifi(object):
     """
     gather the users of wifi from the tomato routers
+
+    with host names from /var/lib/dhcp/dhcpd.leases
     """
     def __init__(self, tomatoConfig="/my/site/magma/tomato_config.js",
                  accessN3="/my/proj/openid_proxy/access.n3"):
@@ -48,15 +51,14 @@
 	    userPass, tail = tail.split("@", 1)
             r.url = http + '//' + tail
             r.headers = {'Authorization': ['Basic %s' % userPass.encode('base64').strip()]}
-            r.name = {'tomato1' : 'bigasterisk3',
+            r.name = {'tomato1' : 'bigasterisk5',
                       'tomato2' : 'bigasterisk4'}[name.split('/')[1]]
             self.routers.append(r)
 
     @inlineCallbacks
     def getPresentMacAddrs(self):
-        aboutIp = {}
-        byMac = {} # mac : [ip]
-
+        rows = []
+        
         for router in self.routers:
             log.debug("GET %s", router)
             try:
@@ -66,57 +68,35 @@
                 log.warn("get on %s failed" % router)
                 continue
             data = resp.body
-
-            for (ip, mac, iface) in jsValue(data, 'arplist'):
-                aboutIp.setdefault(ip, {}).update(dict(
-                    ip=ip,
-                    router=router.name,
-                    mac=mac,
-                    iface=iface,
-                    ))
-
-                byMac.setdefault(mac, set()).add(ip)
-
-            for (name, ip, mac, lease) in jsValue(data, 'dhcpd_lease'):
-                if lease.startswith('0 days, '):
-                    lease = lease[len('0 days, '):]
-                aboutIp.setdefault(ip, {}).update(dict(
-                    router=router.name,
-                    rawName=name,
-                    mac=mac,
-                    lease=lease
-                    ))
-
-                byMac.setdefault(mac, set()).add(ip)
+            if 'Wireless -- Authenticated Stations' in data:
+                # zyxel 'Station Info' page
+                rows.extend(self.parseZyxel(data, router.name))
+            else:
+                # tomato page
+                rows.extend(self.parseTomato(data, router.name))
 
-            for iface, mac, signal in jsValue(data, 'wldev'):
-                matched = False
-                for addr in aboutIp.values():
-                    if (addr['router'], addr['mac']) == (router.name, mac):
-                        addr.update(dict(signal=signal, iface=iface))
-                        matched = True
-                if not matched:
-                    aboutIp["mac-%s-%s" % (router, mac)] = dict(
-                        router=router.name,
-                        mac=mac,
-                        signal=signal,
-                        )
+        for r in rows:
+            try:
+                r['name'] = self.knownMacAddr[r['mac']]
+            except KeyError:
+                pass
+                
+        returnValue(rows)
+        
+    def parseZyxel(self, data, routerName):
+        root = lxml.html.soupparser.fromstring(data)
+        for tr in root.cssselect('tr'):
+            mac, assoc, uth, ssid, iface = [td.text_content().strip() for td in tr.getchildren()]
+            if mac == "MAC":
+                continue
+            assoc = assoc.lower() == 'yes'
+            yield dict(router=routerName, mac=mac, assoc=assoc, connected=assoc)
 
-        ret = []
-        for addr in aboutIp.values():
-            if addr.get('ip') in ['192.168.1.1', '192.168.1.2', '192.168.0.2']:
-                continue
-            try:
-                addr['name'] = self.knownMacAddr[addr['mac']]
-            except KeyError:
-                addr['name'] = addr.get('rawName')
-                if addr['name'] in [None, '*']:
-                    addr['name'] = 'unknown'
-            ret.append(addr)
-
-        returnValue(ret)
-
-
+    def parseTomato(self, data, routerName):
+        for iface, mac, signal in jsValue(data, 'wldev'):
+            yield dict(router=routerName, mac=mac, signal=signal, connected=bool(signal))
+        
+            
 def jsValue(js, variableName):
     # using literal_eval instead of json parser to handle the trailing commas
     val = re.search(variableName + r'\s*=\s*(.*?);', js, re.DOTALL).group(1)