Changeset - f29e26811206
[Not reviewed]
default
0 16 0
Drew Perttula - 6 years ago 2019-06-02 21:19:57
drewp@bigasterisk.com
pmfstats now update at 1hz. some ui improvements to stats lines.
Ignore-this: be2b416f56b6f6604a0af86f5cd69fef
16 files changed with 44 insertions and 40 deletions:
0 comments (0 inline, 0 general)
bin/captureDevice
Show inline comments
 
@@ -16,25 +16,26 @@ from run_local import log
 
from cycloneerr import PrettyErrorHandler
 

	
 
from light9.namespaces import L9, RDF
 
from light9 import networking, showconfig
 
from rdfdb.syncedgraph import SyncedGraph
 
from light9.paint.capture import writeCaptureDescription
 
from greplin.scales.cyclonehandler import StatsHandler
 
from light9.effect.settings import DeviceSettings
 
from light9.collector.collector_client import sendToCollector
 
from rdfdb.patch import Patch
 
from light9.zmqtransport import parseJsonMessage
 

	
 
stats = scales.collection('/webServer', scales.PmfStat('setAttr'))
 
stats = scales.collection('/webServer', scales.PmfStat('setAttr',
 
                                                       recalcPeriod=1))
 

	
 

	
 
class Camera(object):
 

	
 
    def __init__(self, imageUrl):
 
        self.imageUrl = imageUrl
 

	
 
    def takePic(self, uri, writePath):
 
        log.info('takePic %s', uri)
 
        return treq.get(
 
            self.imageUrl).addCallbacks(lambda r: self._done(writePath, r),
 
                                        log.error)
bin/collector
Show inline comments
 
@@ -35,25 +35,25 @@ class Updates(cyclone.websocket.WebSocke
 
        log.info('socket connect %s', self)
 
        self.settings.listeners.addClient(self)
 

	
 
    def connectionLost(self, reason):
 
        self.settings.listeners.delClient(self)
 

	
 
    def messageReceived(self, message):
 
        json.loads(message)
 

	
 

	
 
stats = scales.collection(
 
    '/webServer',
 
    scales.PmfStat('setAttr'),
 
    scales.PmfStat('setAttr', recalcPeriod=1),
 
    scales.RecentFpsStat('setAttrFps'),
 
)
 

	
 

	
 
class Attrs(PrettyErrorHandler, cyclone.web.RequestHandler):
 

	
 
    def put(self):
 
        stats.setAttrFps.mark()
 
        with stats.setAttr.time():
 
            client, clientSession, settings, sendTime = parseJsonMessage(
 
                self.request.body)
 
            self.settings.collector.setAttrs(client, clientSession, settings,
bin/effecteval
Show inline comments
 
@@ -105,25 +105,24 @@ class SongEffectsUpdates(cyclone.websock
 
            seen = set()
 
            for n in self.graph.objects(s, L9['note']):
 
                for uri in self.graph.objects(n, L9['effectClass']):
 
                    if uri in seen:
 
                        continue
 
                    seen.add(uri)
 
                    out[-1]['effects'].append({
 
                        'uri': uri,
 
                        'label': self.graph.label(uri)
 
                    })
 
            out[-1]['effects'].sort(key=lambda e: e['uri'])
 
                    
 
                    
 
        self.sendMessage({'songs': out})
 

	
 

	
 
class EffectUpdates(cyclone.websocket.WebSocketHandler):
 
    """
 
    stays alive for the life of the effect page
 
    """
 

	
 
    def connectionMade(self, *args, **kwargs):
 
        log.info("websocket opened")
 
        self.uri = URIRef(self.get_argument('uri'))
 
        self.sendMessage({'hello': repr(self)})
 
@@ -217,28 +216,28 @@ class SongEffectsEval(PrettyErrorHandler
 

	
 
class App(object):
 

	
 
    def __init__(self, show, outputWhere):
 
        self.show = show
 
        self.outputWhere = outputWhere
 
        self.graph = SyncedGraph(networking.rdfdb.url, "effectEval")
 
        self.graph.initiallySynced.addCallback(self.launch).addErrback(
 
            log.error)
 

	
 
        self.stats = scales.collection(
 
            '/',
 
            scales.PmfStat('sendLevels'),
 
            scales.PmfStat('getMusic'),
 
            scales.PmfStat('evals'),
 
            scales.PmfStat('sendOutput'),
 
            scales.PmfStat('sendLevels', recalcPeriod=1),
 
            scales.PmfStat('getMusic', recalcPeriod=1),
 
            scales.PmfStat('evals', recalcPeriod=1),
 
            scales.PmfStat('sendOutput', recalcPeriod=1),
 
            scales.IntStat('errors'),
 
        )
 

	
 
    def launch(self, *args):
 
        log.info('launch')
 
        if self.outputWhere:
 
            self.loop = makeEffectLoop(self.graph, self.stats, self.outputWhere)
 
            self.loop.startLoop()
 

	
 
        SFH = cyclone.web.StaticFileHandler
 
        self.cycloneApp = cyclone.web.Application(handlers=[
 
            (r'/()', SFH, {
bin/effectsequencer
Show inline comments
 
@@ -20,28 +20,28 @@ from light9 import clientsession
 

	
 
class App(object):
 

	
 
    def __init__(self, show, session):
 
        self.show = show
 
        self.session = session
 

	
 
        self.graph = SyncedGraph(networking.rdfdb.url, "effectSequencer")
 
        self.graph.initiallySynced.addCallback(self.launch)
 

	
 
        self.stats = scales.collection(
 
            '/',
 
            scales.PmfStat('sendLevels'),
 
            scales.PmfStat('getMusic'),
 
            scales.PmfStat('evals'),
 
            scales.PmfStat('sendOutput'),
 
            scales.PmfStat('sendLevels', recalcPeriod=1),
 
            scales.PmfStat('getMusic', recalcPeriod=1),
 
            scales.PmfStat('evals', recalcPeriod=1),
 
            scales.PmfStat('sendOutput', recalcPeriod=1),
 
            scales.IntStat('errors'),
 
        )
 

	
 
    def launch(self, *args):
 
        self.seq = Sequencer(
 
            self.graph,
 
            lambda settings: sendToCollector(
 
                'effectSequencer',
 
                self.session,
 
                settings,
 
                # This seems to be safe here (and lets us get from
 
                # 20fpx to 40fpx), even though it leads to big stalls
bin/paintserver
Show inline comments
 
@@ -62,25 +62,25 @@ class BestMatches(PrettyErrorHandler, cy
 
class App(object):
 

	
 
    def __init__(self, show, session):
 
        self.show = show
 
        self.session = session
 

	
 
        self.graph = SyncedGraph(networking.rdfdb.url, "paintServer")
 
        self.graph.initiallySynced.addCallback(self.launch).addErrback(
 
            log.error)
 

	
 
        self.stats = scales.collection(
 
            '/',
 
            scales.PmfStat('solve'),
 
            scales.PmfStat('solve', recalcPeriod=1),
 
        )
 

	
 
    def launch(self, *args):
 

	
 
        self.solver = light9.paint.solve.Solver(
 
            self.graph,
 
            sessions=[
 
                L9['show/dance2017/capture/aura1/cap1876596'],
 
                L9['show/dance2017/capture/aura2/cap1876792'],
 
                L9['show/dance2017/capture/aura3/cap1877057'],
 
                L9['show/dance2017/capture/aura4/cap1877241'],
 
                L9['show/dance2017/capture/aura5/cap1877406'],
light9/ascoltami/player.py
Show inline comments
 
#!/usr/bin/python
 
"""
 
alternate to the mpd music player, for ascoltami
 
"""
 

	
 
import time, logging, traceback
 
from gi.repository import Gst
 
from twisted.internet import task
 
from greplin import scales
 

	
 
log = logging.getLogger()
 

	
 
stats = scales.collection('/player',
 
stats = scales.collection(
 
    '/player',
 
                          scales.RecentFpsStat('currentTimeFps'),
 
)
 

	
 

	
 
class Player(object):
 

	
 
    def __init__(self, autoStopOffset=4, onEOS=None):
 
        """autoStopOffset is the number of seconds before the end of
 
        song before automatically stopping (which is really pausing).
 
        onEOS is an optional function to be called when we reach the
 
        end of a stream (for example, can be used to advance the song).
 
        It is called with one argument which is the URI of the song that
 
        just finished."""
 
        self.autoStopOffset = autoStopOffset
 
        self.playbin = self.pipeline = Gst.ElementFactory.make('playbin', None)
 

	
light9/ascoltami/webapp.py
Show inline comments
 
@@ -158,15 +158,17 @@ class goButton(PrettyErrorHandler, cyclo
 
        self.write("ok")
 

	
 

	
 
def makeWebApp(app):
 
    return cyclone.web.Application(handlers=[
 
        (r"/", root),
 
        (r"/time", timeResource),
 
        (r"/song", songResource),
 
        (r"/songs", songs),
 
        (r"/seekPlayOrPause", seekPlayOrPause),
 
        (r"/output", output),
 
        (r"/go", goButton),
 
        (r'/stats/(.*)', StatsHandler, {'serverName': 'ascoltami'}),
 
        (r'/stats/(.*)', StatsHandler, {
 
            'serverName': 'ascoltami'
 
        }),
 
    ],
 
                                   app=app)
light9/collector/collector_client.py
Show inline comments
 
@@ -3,25 +3,25 @@ from light9.effect.settings import Devic
 
from twisted.internet import defer
 
from txzmq import ZmqEndpoint, ZmqFactory, ZmqPushConnection
 
import json, time, logging
 
import treq
 
from greplin import scales
 

	
 
log = logging.getLogger('coll_client')
 

	
 
_zmqClient = None
 

	
 
stats = scales.collection(
 
    '/collectorClient/',
 
    scales.PmfStat('send'),
 
    scales.PmfStat('send', recalcPeriod=1),
 
)
 

	
 

	
 
class TwistedZmqClient(object):
 

	
 
    def __init__(self, service):
 
        zf = ZmqFactory()
 
        e = ZmqEndpoint('connect', 'tcp://%s:%s' % (service.host, service.port))
 
        self.conn = ZmqPushConnection(zf, e)
 

	
 
    def send(self, msg):
 
        self.conn.push(msg)
light9/collector/output.py
Show inline comments
 
@@ -36,25 +36,25 @@ class Output(object):
 
        """caller asks for the output to be this buffer"""
 
        self._currentBuffer = buf
 

	
 
    def _periodicLog(self):
 
        msg = '%s: %s' % (self.shortId(), ' '.join(map(str,
 
                                                       self._currentBuffer)))
 
        if msg != self._lastLoggedMsg:
 
            log.debug(msg)
 
            self._lastLoggedMsg = msg
 

	
 
    _writeSucceed = scales.IntStat('write/succeed')
 
    _writeFail = scales.IntStat('write/fail')
 
    _writeCall = scales.PmfStat('write/call')
 
    _writeCall = scales.PmfStat('write/call', recalcPeriod=1)
 
    _writeFps = scales.RecentFpsStat('write/fps')
 

	
 
    def _write(self, buf: bytes) -> None:
 
        """
 
        write buffer to output hardware (may be throttled if updates are
 
        too fast, or repeated if they are too slow)
 
        """
 
        pass
 

	
 

	
 
class DummyOutput(Output):
 

	
 
@@ -160,26 +160,26 @@ class UdmxOld(BackgroundLoopOutput):
 

	
 
        except usb.core.USBError as e:
 
            # not in main thread
 
            if e.errno != 75:
 
                msg = 'usb: sending %s bytes to %r; error %r' % (
 
                    len(buf), self.uri, e)
 
                log.warn(msg)
 
            raise
 
          
 
                                
 
# out of date
 
class EnttecDmx(BackgroundLoopOutput):
 
    stats = scales.collection('/output/enttecDmx', scales.PmfStat('write'),
 
                              scales.PmfStat('update'))
 
    stats = scales.collection('/output/enttecDmx', scales.PmfStat('write', recalcPeriod=1),
 
                              scales.PmfStat('update', recalcPeriod=1))
 

	
 
    def __init__(self, uri, devicePath='/dev/dmx0', numChannels=80):
 
        sys.path.append("dmx_usb_module")
 
        from dmx import Dmx
 
        self.dev = Dmx(devicePath)
 
        super().__init__(uri)
 

	
 

	
 
    @stats.update.time()
 
    def update(self, values):
 

	
 
        # I was outputting on 76 and it was turning on the light at
light9/effect/sequencer.py
Show inline comments
 
@@ -16,41 +16,41 @@ from typing import Any, Callable, Dict, 
 
from light9.namespaces import L9, RDF
 
from light9.newtypes import DeviceUri, DeviceAttr, NoteUri, Curve, Song
 
from light9.vidref.musictime import MusicTime
 
from light9.effect import effecteval
 
from light9.effect.settings import DeviceSettings
 
from light9.effect.simple_outputs import SimpleOutputs
 
from rdfdb.syncedgraph import SyncedGraph
 

	
 
from greplin import scales
 
import imp
 

	
 
log = logging.getLogger('sequencer')
 
stats = scales.collection('/sequencer/',)
 

	
 
updateStats = scales.collection(
 
    '/update/',
 
    scales.PmfStat('s0_getMusic'),
 
    scales.PmfStat('s1_eval'),
 
    scales.PmfStat('s2_sendToWeb'),
 
    scales.PmfStat('s3_send'),
 
    scales.PmfStat('sendPhase'),
 
    scales.PmfStat('updateLoopLatency'),
 
    scales.PmfStat('s0_getMusic', recalcPeriod=1),
 
    scales.PmfStat('s1_eval', recalcPeriod=1),
 
    scales.PmfStat('s2_sendToWeb', recalcPeriod=1),
 
    scales.PmfStat('s3_send', recalcPeriod=1),
 
    scales.PmfStat('sendPhase', recalcPeriod=1),
 
    scales.PmfStat('updateLoopLatency', recalcPeriod=1),
 
    scales.DoubleStat('updateLoopLatencyGoal'),
 
    scales.RecentFpsStat('updateFps'),
 
    scales.DoubleStat('goalFps'),
 
)
 
compileStats = scales.collection(
 
    '/compile/',
 
    scales.PmfStat('graph'),
 
    scales.PmfStat('song'),
 
    scales.PmfStat('graph', recalcPeriod=1),
 
    scales.PmfStat('song', recalcPeriod=1),
 
)
 

	
 

	
 
class Note(object):
 

	
 
    def __init__(self, graph: SyncedGraph, uri: NoteUri, effectevalModule,
 
                 simpleOutputs):
 
        g = self.graph = graph
 
        self.uri = uri
 
        self.effectEval = effectevalModule.EffectEval(
 
            graph, g.value(uri, L9['effectClass']), simpleOutputs)
 
        self.baseEffectSettings: Dict[URIRef, Any] = {}  # {effectAttr: value}
 
@@ -147,26 +147,25 @@ class CodeWatcher(object):
 

	
 
        def go():
 
            log.info("reload effecteval")
 
            imp.reload(effecteval)
 
            self.onChange()
 

	
 
        # in case we got an event at the start of the write
 
        reactor.callLater(.1, go)
 

	
 

	
 
class Sequencer(object):
 

	
 
    def __init__(
 
            self,
 
    def __init__(self,
 
            graph: SyncedGraph,
 
            sendToCollector: Callable[[DeviceSettings], defer.Deferred],
 
            fps=40):
 
        self.graph = graph
 
        self.fps = fps
 
        updateStats.goalFps = self.fps
 
        updateStats.updateLoopLatencyGoal = 1 / self.fps
 
        self.sendToCollector = sendToCollector
 
        self.music = MusicTime(period=.2, pollCurvecalc=False)
 

	
 
        self.recentUpdateTimes: List[float] = []
 
        self.lastStatLog = 0.0
light9/web/index.html
Show inline comments
 
@@ -15,27 +15,24 @@
 
    <dom-module id="service-button-row">
 
      <template>
 
        <style>
 
         :host { padding-bottom: 10px;  }
 
         a {
 
             color: #7d7dec;
 
         }
 
         div {
 
             display: flex;
 
             justify-content: space-between;
 
             padding: 2px 3px;
 
         }
 
         div:hover {
 
             background: gray;
 
         }
 
         .left {
 
             display: inline-block;
 
             margin-right: 3px;
 
             flex-grow: 1;
 
         }
 
         .window {
 
         }
 
         .serviceGrid > td {
 
             border: 5px solid red;
 
             display: inline-block;
 
         }
 
         :host > div { display: inline-block; vertical-align: top; }
light9/web/stats-line.js
Show inline comments
 
@@ -28,24 +28,25 @@ class StatsLine extends LitElement {
 
                        }
 
                });
 
                }
 
                reload();
 
            }
 
        });
 
    }
 
    
 
    static get styles() {
 
        return css`
 
        :host {
 
            border: 2px solid #46a79f;
 
            display: inline-block;
 
        }
 
        table { 
 
            border-collapse: collapse;
 
            background: #000;
 
            color: #ccc;
 
            font-family: sans-serif;
 
        }
 
        th, td { 
 
            outline: 1px solid #000; 
 
        }
 
        th {
 
            padding: 2px 4px;
 
@@ -98,35 +99,34 @@ class StatsLine extends LitElement {
 
                 </tr>
 
             </table>`;
 
        };
 
        const tdWrap = (content) => {
 
            return html`<td>${content}</td>`;
 
        }
 
        const recents = (d, path) => {
 
            const hi = Math.max.apply(null, d.recents);
 
            const scl = 30 / hi;
 
            
 
            const bar = (y) => {
 
                let color;
 
                if (y < hi * .85) {
 
                if (y < d.average) {
 
                    color="#6a6aff";
 
                } else {
 
                    color="#d09e4c";
 
                }
 
                return html`<div class="bar" style="height: ${y * scl}px; background: ${color};"></div>`;
 
            };
 
            return tdWrap(table({
 
                average: rounding(d.average, 3),
 
                recents: html`<div class="recents">${d.recents.map(bar)}</div>`
 
            }, path));
 
            return html`<td>
 
               <div class="recents">${d.recents.map(bar)}</div>
 
               <div>avg=${rounding(d.average ,3)}</div>`;
 

	
 
        };
 
        const pmf = (d, path) => {
 
            return tdWrap(table({
 
                count: d.count,
 
                'values [ms]': html`
 
                   <div>mean=${rounding(d.mean*1000, 3)}</div>
 
                   <div>sd=${rounding(d.stddev*1000, 3)}</div>
 
                   <div>99=${rounding(d['99percentile']*1000, 3)}</div>
 
                 `
 
            }, path));
 
        };
light9/web/style.css
Show inline comments
 
@@ -188,12 +188,16 @@ a.big {
 
    display: inline-block;
 
    border-radius: 5px;
 
}
 

	
 
table {
 
    border-collapse: collapse;
 
}
 

	
 
table.borders td, table.borders th {
 
    border: 1px solid #4a4a4a;
 
    padding: 2px 8px;
 
}
 
hr {
 
    width: 100%;
 
    border-color: #1d3e1d;
 
}
light9/zmqtransport.py
Show inline comments
 
@@ -13,25 +13,25 @@ def parseJsonMessage(msg):
 
    for device, attr, value in body['settings']:
 
        if isinstance(value, str) and value.startswith('http'):
 
            value = URIRef(value)
 
        else:
 
            value = Literal(value)
 
        settings.append((URIRef(device), URIRef(attr), value))
 
    return body['client'], body['clientSession'], settings, body['sendTime']
 

	
 

	
 
def startZmq(port, collector):
 
    stats = scales.collection(
 
        '/zmqServer',
 
        scales.PmfStat('setAttr'),
 
        scales.PmfStat('setAttr', recalcPeriod=1),
 
        scales.RecentFpsStat('setAttrFps'),
 
    )
 

	
 
    zf = ZmqFactory()
 
    addr = 'tcp://*:%s' % port
 
    log.info('creating zmq endpoint at %r', addr)
 
    e = ZmqEndpoint('bind', addr)
 

	
 
    class Pull(ZmqPullConnection):
 
        #highWaterMark = 3
 
        def onPull(self, message):
 
            stats.setAttrFps.mark()
requirements.txt
Show inline comments
 
@@ -27,19 +27,19 @@ watchdog==0.8.3
 
webcolors==1.7
 
udmx-pyusb==2.0.0
 
yapf==0.27.0
 

	
 
coverage==4.3.4
 
flake8
 
hunter
 
ipdb==0.10.2
 
ipython==5.3.0
 
mypy==0.701
 
typing_extensions
 

	
 
git+http://github.com/drewp/scales.git@3f2609c#egg=scales
 
git+http://github.com/drewp/scales.git@4b011434f7469a442c3fc1d7e81685c0bfa56eeb#egg=scales
 
git+http://github.com/11craft/louie.git@f18bb71010c114eca9c6b88c96453340e3b39454#egg=louie
 
git+http://github.com/webpy/webpy@ace0f8954c28311004b33c7a513c6e40a3c02470#egg=web
 
https://github.com/drewp/cyclone/archive/python3.zip#egg=cyclone
 

	
 
cycloneerr==0.3.0
 
rdfdb==0.19.0
stubs/greplin/scales/__init__.pyi
Show inline comments
 
@@ -107,25 +107,25 @@ class PmfStatDict(UserDict):
 
        def __enter__(self): ...
 
        def __exit__(self, *_: Any) -> None: ...
 
        def warn99(self, logger: Any, msg: Any, *args: Any) -> None: ...
 
        def discard(self) -> None: ...
 
        def __call__(self, func: Any): ...
 
    percentile99: Any = ...
 
    def __init__(self, sample: Optional[Any] = ...) -> None: ...
 
    def __getitem__(self, item: Any): ...
 
    def addValue(self, value: Any) -> None: ...
 
    def time(self): ...
 

	
 
class PmfStat(Stat):
 
    def __init__(self, name: Any, _: Optional[Any] = ...) -> None: ...
 
    def __init__(self, name: Any, _: Optional[Any] = ..., recalcPeriod: Optional[float]=20) -> None: ...
 
    def __set__(self, instance: Any, value: Any) -> None: ...
 

	
 
class NamedPmfDict(UserDict):
 
    def __init__(self) -> None: ...
 
    def __getitem__(self, item: Any): ...
 
    def __setitem__(self, key: Any, value: Any) -> None: ...
 

	
 
class NamedPmfDictStat(Stat): ...
 

	
 
class RecentFpsStat(Stat): ...
 

	
 
class StateTimeStatDict(UserDict):
0 comments (0 inline, 0 general)