changeset 1951:9f0f2b39ad95

vidref web is working Ignore-this: 686b512c0368f8cc419000e784f13935
author Drew Perttula <drewp@bigasterisk.com>
date Thu, 06 Jun 2019 00:09:39 +0000
parents 19c2e6216cf8
children 7088d60fde7e
files .boring bin/vidref light9/ascoltami/webapp.py light9/vidref/index.html light9/vidref/replay.py light9/vidref/videorecorder.py light9/web/light9-vidref-live.js light9/web/light9-vidref-replay-stack.js light9/web/light9-vidref-replay.js makefile
diffstat 10 files changed, 410 insertions(+), 345 deletions(-) [+]
line wrap: on
line diff
--- a/.boring	Thu Jun 06 00:07:07 2019 +0000
+++ b/.boring	Thu Jun 06 00:09:39 2019 +0000
@@ -158,6 +158,7 @@
 ^show/dance..../capture
 ^show/dance2017/networking.n3
 ^show/dance2019/model/
+^show/dance2019/video/
 ^timelapse/
 __pycache__
 rgbled/build-nano328/
--- a/bin/vidref	Thu Jun 06 00:07:07 2019 +0000
+++ b/bin/vidref	Thu Jun 06 00:09:39 2019 +0000
@@ -18,13 +18,15 @@
 
 from twisted.internet import reactor, defer
 
-import logging, optparse, json, base64
+import logging, optparse, json, base64, os, glob
 import cyclone.web, cyclone.httpclient, cyclone.websocket
-from light9 import networking
-from light9.vidref.replay import snapshotDir
+from light9 import networking, showconfig
 from light9.vidref import videorecorder
+from rdflib import URIRef
+from light9.newtypes import Song
+from light9.namespaces import L9
 from rdfdb.syncedgraph import SyncedGraph
-from io import BytesIO
+from cycloneerr import PrettyErrorHandler
 
 parser = optparse.OptionParser()
 parser.add_option("-v", "--verbose", action="store_true", help="logging.DEBUG")
@@ -40,11 +42,12 @@
         # save next pic
         # return /snapshot/path
         try:
+            snapshotDir = 'todo'
             outputFilename = yield self.settings.gui.snapshot()
 
-            assert outputFilename.startswith(snapshotDir())
+            assert outputFilename.startswith(snapshotDir)
             out = networking.vidref.path(
-                "snapshot/%s" % outputFilename[len(snapshotDir()):].lstrip('/'))
+                "snapshot/%s" % outputFilename[len(snapshotDir):].lstrip('/'))
 
             self.write(json.dumps({'snapshot': out}))
             self.set_header("Location", out)
@@ -71,15 +74,11 @@
 
     def onFrame(self, cf: videorecorder.CaptureFrame):
         if cf is None: return
-        output = BytesIO()
-        cf.img.save(output, 'jpeg', quality=80)
 
         self.sendMessage(
             json.dumps({
-                'jpeg':
-                base64.b64encode(output.getvalue()).decode('ascii'),
-                'description':
-                f't={cf.t}',
+                'jpeg': base64.b64encode(cf.asJpeg()).decode('ascii'),
+                'description': f't={cf.t}',
             }))
 
 
@@ -97,9 +96,48 @@
         self.set_status(202)
 
 
-#graph = SyncedGraph(networking.rdfdb.url, "vidref")
-outVideos = videorecorder.FramesToVideoFiles(pipeline.liveImages)
-#outVideos.save('/tmp/mov1')
+def takeUri(songPath: bytes):
+    p = songPath.decode('ascii').split('/')
+    take = p[-1].replace('.mp4', '')
+    song = p[-2].split('_')
+    return URIRef('/'.join(
+        ['http://light9.bigasterisk.com/show', song[-2], song[-1], take]))
+
+
+class ReplayMap(PrettyErrorHandler, cyclone.web.RequestHandler):
+
+    def get(self):
+        song = Song(self.get_argument('song'))
+        clips = []
+        for vid in glob.glob(os.path.join(videorecorder.songDir(song),
+                                          b'*.mp4')):
+            pts = []
+            for line in open(vid.replace(b'.mp4', b'.timing'), 'rb'):
+                _v, vt, _eq, _song, st = line.split()
+                pts.append([float(st), float(vt)])
+
+            url = vid[len(os.path.dirname(os.path.dirname(showconfig.root()))
+                         ):].decode('ascii')
+
+            clips.append({
+                'uri': takeUri(vid),
+                'videoUrl': url,
+                'songToVideo': pts
+            })
+
+        clips.sort(key=lambda c: len(c['songToVideo']))
+        clips = clips[-3:]
+        clips.sort(key=lambda c: c['uri'], reverse=True)
+
+        ret = json.dumps(clips)
+        log.info('replayMap had %s videos; json is %s bytes', len(clips),
+                 len(ret))
+        self.write(ret)
+
+
+graph = SyncedGraph(networking.rdfdb.url, "vidref")
+outVideos = videorecorder.FramesToVideoFiles(
+    pipeline.liveImages, os.path.join(showconfig.root(), b'video'))
 
 port = networking.vidref.port
 reactor.listenTCP(
@@ -115,9 +153,10 @@
                 'default_filename': 'setup.html'
             }),
             (r'/live', Live),
+            (r'/replayMap', ReplayMap),
             (r'/snapshot', Snapshot),
             (r'/snapshot/(.*)', SnapshotPic, {
-                "path": snapshotDir()
+                "path": 'todo',
             }),
             (r'/time', Time),
         ],
--- a/light9/ascoltami/webapp.py	Thu Jun 06 00:07:07 2019 +0000
+++ b/light9/ascoltami/webapp.py	Thu Jun 06 00:09:39 2019 +0000
@@ -1,16 +1,16 @@
-import json, socket, subprocess, os
+import json, socket, subprocess, os, logging, time
 
 from cyclone import template
 from rdflib import URIRef
-import cyclone.web
+import cyclone.web, cyclone.websocket
 from greplin.scales.cyclonehandler import StatsHandler
 
 from cycloneerr import PrettyErrorHandler
 from light9.namespaces import L9
 from light9.showconfig import getSongsFromShow, songOnDisk
-
+from twisted.internet import reactor
 _songUris = {}  # locationUri : song
-
+log = logging.getLogger()
 loader = template.Loader(os.path.dirname(__file__))
 
 
@@ -46,30 +46,32 @@
         return None
 
 
+def currentState(graph, player):
+    if player.isAutostopped():
+        nextAction = 'finish'
+    elif player.isPlaying():
+        nextAction = 'disabled'
+    else:
+        nextAction = 'play'
+
+    return {
+        "song": playerSongUri(graph, player),
+        "started": player.playStartTime,
+        "duration": player.duration(),
+        "playing": player.isPlaying(),
+        "t": player.currentTime(),
+        "state": player.states(),
+        "next": nextAction,
+    }
+
+
 class timeResource(PrettyErrorHandler, cyclone.web.RequestHandler):
 
     def get(self):
         player = self.settings.app.player
         graph = self.settings.app.graph
         self.set_header("Content-Type", "application/json")
-
-        if player.isAutostopped():
-            nextAction = 'finish'
-        elif player.isPlaying():
-            nextAction = 'disabled'
-        else:
-            nextAction = 'play'
-
-        self.write(
-            json.dumps({
-                "song": playerSongUri(graph, player),
-                "started": player.playStartTime,
-                "duration": player.duration(),
-                "playing": player.isPlaying(),
-                "t": player.currentTime(),
-                "state": player.states(),
-                "next": nextAction,
-            }))
+        self.write(json.dumps(currentState(graph, player)))
 
     def post(self):
         """
@@ -89,6 +91,28 @@
         self.write("ok")
 
 
+class timeStreamResource(cyclone.websocket.WebSocketHandler):
+
+    def connectionMade(self, *args, **kwargs) -> None:
+        self.lastSent = None
+        self.lastSentTime = 0.
+        self.loop()
+
+    def loop(self):
+        now = time.time()
+        msg = currentState(self.settings.app.graph, self.settings.app.player)
+        if msg != self.lastSent or now > self.lastSentTime + 2:
+            self.sendMessage(json.dumps(msg))
+            self.lastSent = msg
+            self.lastSentTime = now
+
+        if self.transport.connected:
+            reactor.callLater(.2, self.loop)
+
+    def connectionLost(self, reason):
+        log.info("bye ws client %r: %s", self, reason)
+
+
 class songs(PrettyErrorHandler, cyclone.web.RequestHandler):
 
     def get(self):
@@ -120,11 +144,16 @@
 
 
 class seekPlayOrPause(PrettyErrorHandler, cyclone.web.RequestHandler):
+    """curveCalc's ctrl-p or a vidref scrub"""
 
     def post(self):
         player = self.settings.app.player
 
         data = json.loads(self.request.body)
+        if 'scrub' in data:
+            player.pause()
+            player.seek(data['scrub'])
+            return
         if player.isPlaying():
             player.pause()
         else:
@@ -162,6 +191,7 @@
     return cyclone.web.Application(handlers=[
         (r"/", root),
         (r"/time", timeResource),
+        (r"/time/stream", timeStreamResource),
         (r"/song", songResource),
         (r"/songs", songs),
         (r"/seekPlayOrPause", seekPlayOrPause),
--- a/light9/vidref/index.html	Thu Jun 06 00:07:07 2019 +0000
+++ b/light9/vidref/index.html	Thu Jun 06 00:09:39 2019 +0000
@@ -4,20 +4,24 @@
     <title>vidref</title>
     <meta charset="utf-8" />
     <style>
-
      
     </style>
+    <script src="/lib/debug/debug-build.js"></script>
+    <script>
+     debug.enable('*');
+    </script>
     <script src="/lib/jquery/dist/jquery.slim.min.js"></script>
-
     <script src="/websocket.js"></script>
-    <script type="module"  src="/light9-vidref-live.js"></script>
-
-
+    <script type="module" src="/light9-vidref-live.js"></script>
+    <script type="module" src="/light9-vidref-replay-stack.js"></script>
   </head>
   <body>
     Live:
-    <light9-vidref-live></light9-vidref-live>
+    <div>
+      <light9-vidref-live></light9-vidref-live>
+    </div>
+    <light9-vidref-replay-stack></light9-vidref-replay-stack>
 
-
+stats of all disk usage
   </body>
 </html>
--- a/light9/vidref/replay.py	Thu Jun 06 00:07:07 2019 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,256 +0,0 @@
-import os, shutil, logging, time
-from bisect import bisect_left
-from decimal import Decimal
-log = logging.getLogger()
-
-framerate = 15
-
-
-def songDir(song):
-    safeUri = song.split('://')[-1].replace('/', '_')
-    return os.path.expanduser("~/light9-vidref/play-%s" % safeUri)
-
-
-def takeDir(songDir, startTime):
-    """
-    startTime: unix seconds (str ok)
-    """
-    return os.path.join(songDir, str(int(startTime)))
-
-
-def snapshotDir():
-    return os.path.expanduser("~/light9-vidref/snapshot")
-
-
-class ReplayViews(object):
-    """
-    the whole list of replay windows. parent is the scrolling area for
-    these windows to be added
-    """
-
-    def __init__(self, parent):
-        # today, parent is the vbox the replay windows should appear in
-        self.parent = parent
-        self.lastStart = None
-
-        self.views = []
-
-    def update(self, position):
-        """
-        freshen all replay windows. We get called this about every
-        time there's a new live video frame.
-
-        Calls loadViewsForSong if we change songs, or even if we just
-        restart the playback of the current song (since there could be
-        a new replay view)
-        """
-        t1 = time.time()
-        if position.get('started') != self.lastStart and position['song']:
-            self.loadViewsForSong(position['song'])
-            self.lastStart = position['started']
-        for v in self.views:
-            v.updatePic(position)
-        log.debug("update %s views in %.2fms", len(self.views),
-                  (time.time() - t1) * 1000)
-
-    def loadViewsForSong(self, song):
-        """
-        replace previous views, and cleanup short ones
-        """
-        for v in self.views:
-            v.destroy()
-        self.views[:] = []
-
-        d = songDir(song)
-        try:
-            takes = sorted(t for t in os.listdir(d) if t.isdigit())
-        except OSError:
-            return
-
-        for take in takes:
-            td = takeDir(songDir(song), take)
-            r = Replay(td)
-            if r.tooShort():
-                # this is happening even on full-song recordings, even
-                # after the Replay.__init__ attempt to catch it
-                log.warn("prob too short, but that's currently broken")
-                #log.warn("cleaning up %s; too short" % r.takeDir)
-                #r.deleteDir()
-                continue
-            rv = ReplayView(self.parent, r)
-            self.views.append(rv)
-
-
-class ReplayView(object):
-    """
-    one of the replay widgets
-    """
-
-    def __init__(self, parent, replay):
-        self.replay = replay
-        self.enabled = True
-        self.showingPic = None
-
-        # this *should* be a composite widget from glade
-
-        delImage = gtk.Image()
-        delImage.set_visible(True)
-        delImage.set_from_stock("gtk-delete", gtk.ICON_SIZE_BUTTON)
-
-        def withLabel(cls, label):
-            x = cls()
-            x.set_visible(True)
-            x.set_label(label)
-            return x
-
-        def labeledProperty(key, value, width=12):
-            lab = withLabel(gtk.Label, key)
-
-            ent = gtk.Entry()
-            ent.set_visible(True)
-            ent.props.editable = False
-            ent.props.width_chars = width
-            ent.props.text = value
-
-            cols = gtk.HBox()
-            cols.set_visible(True)
-            cols.add(lab)
-            cols.add(ent)
-            return cols
-
-        replayPanel = gtk.HBox()
-        replayPanel.set_visible(True)
-        if True:
-            af = gtk.AspectFrame()
-            af.set_visible(True)
-            af.set_shadow_type(gtk.SHADOW_OUT)
-            af.props.obey_child = True
-
-            img = gtk.Image()
-            img.set_visible(True)
-            self.picWidget = img
-
-            af.add(img)
-            replayPanel.pack_start(af, False, False, 0)
-
-        if True:
-            rows = []
-            rows.append(labeledProperty("Started:", self.replay.getTitle()))
-            rows.append(labeledProperty("Seconds:", self.replay.getDuration()))
-            if True:
-                en = withLabel(gtk.ToggleButton, "Enabled")
-                en.set_active(True)
-
-                def tog(w):
-                    self.enabled = w.get_active()
-
-                en.connect("toggled", tog)
-                rows.append(en)
-            if True:
-                d = withLabel(gtk.Button, "Delete")
-                d.props.image = delImage
-
-                def onClicked(w):
-                    self.replay.deleteDir()
-                    self.destroy()
-
-                d.connect("clicked", onClicked)
-                rows.append(d)
-            if True:
-                pin = withLabel(gtk.CheckButton, "Pin to top")
-                pin.props.draw_indicator = True
-                rows.append(pin)
-
-            stack = gtk.VBox()
-            stack.set_visible(True)
-            for r in rows:
-                stack.add(r)
-                stack.set_child_packing(r, False, False, 0, gtk.PACK_START)
-
-            replayPanel.pack_start(stack, False, False, 0)
-
-        parent.pack_start(replayPanel, False, False)
-        log.debug("packed ReplayView %s" % replayPanel)
-        self.replayPanel = replayPanel
-
-    def destroy(self):
-        self.replayPanel.destroy()
-        self.enabled = False
-
-    def updatePic(self, position, lag=.2):
-
-        # this should skip updating off-screen widgets! maybe that is
-        # done by declaring the widget dirty and then reacting to a
-        # paint message if one comes
-
-        if not self.enabled:
-            return
-
-        t = position.get('hoverTime', position['t'])
-        inPic = self.replay.findClosestFrame(t + lag)
-
-        if inPic == self.showingPic:
-            return
-        with gtk.gdk.lock:
-            self.picWidget.set_from_file(inPic)
-            if 0:
-                # force redraw of that widget
-                self.picWidget.queue_draw_area(0, 0, 320, 240)
-                self.picWidget.get_window().process_updates(True)
-        self.showingPic = inPic
-
-
-_existingFrames = {}  # takeDir : frames
-
-
-class Replay(object):
-    """
-    model for one of the replay widgets
-    """
-
-    def __init__(self, takeDir):
-        self.takeDir = takeDir
-        try:
-            self.existingFrames = _existingFrames[self.takeDir]
-        except KeyError:
-            log.info("scanning %s", self.takeDir)
-            self.existingFrames = sorted(
-                [Decimal(f.split('.jpg')[0]) for f in os.listdir(self.takeDir)])
-            if not self.existingFrames:
-                raise NotImplementedError(
-                    "suspiciously found no frames in dir %s" % self.takeDir)
-            _existingFrames[self.takeDir] = self.existingFrames
-
-    def tooShort(self, minSeconds=5):
-        return len(self.existingFrames) < (minSeconds * framerate)
-
-    def deleteDir(self):
-        try:
-            shutil.rmtree(self.takeDir)
-        except OSError:
-            # probably was writing frames into this dir at the same time!
-            log.warn("partial delete- frames were probably still writing "
-                     "into that dir")
-
-    def getTitle(self):
-        tm = time.localtime(int(os.path.basename(self.takeDir)))
-        return time.strftime("%a %H:%M:%S", tm)
-
-    def getDuration(self):
-        """total number of seconds represented, which is most probably
-        a continuous section, but we aren't saying where in the song
-        that is"""
-        return "%.1f" % (len(self.existingFrames) / framerate)
-
-    def findClosestFrame(self, t):
-        # this is weird to be snapping our playback time to the frames
-        # on disk. More efficient and accurate would be to schedule
-        # the disk frames to playback exactly as fast as they want
-        # to. This might spread cpu load since the recorded streams
-        # might be a little more out of phase. It would also
-        # accomodate changes in framerate between playback streams.
-        i = bisect_left(self.existingFrames, Decimal(str(t)))
-        if i >= len(self.existingFrames):
-            i = len(self.existingFrames) - 1
-        return os.path.join(self.takeDir,
-                            "%08.03f.jpg" % self.existingFrames[i])
--- a/light9/vidref/videorecorder.py	Thu Jun 06 00:07:07 2019 +0000
+++ b/light9/vidref/videorecorder.py	Thu Jun 06 00:09:39 2019 +0000
@@ -1,13 +1,13 @@
 from dataclasses import dataclass
-from queue import Queue
 from typing import Optional
-import time, logging, os, traceback, sys
+import time, logging, os, traceback
+from io import BytesIO
 
 import gi
 gi.require_version('Gst', '1.0')
 gi.require_version('GstBase', '1.0')
 
-from PIL import Image
+import PIL.Image
 from gi.repository import Gst
 from rx.subject import BehaviorSubject
 from twisted.internet import threads
@@ -16,21 +16,31 @@
 
 from light9.ascoltami.musictime_client import MusicTime
 from light9.newtypes import Song
-
-from IPython.core import ultratb
-sys.excepthook = ultratb.FormattedTB(mode='Verbose',
-                                     color_scheme='Linux',
-                                     call_pdb=1)
+from light9 import showconfig
 
 log = logging.getLogger()
 
 
-@dataclass(frozen=True)
+@dataclass
 class CaptureFrame:
-    img: Image
+    img: PIL.Image
     song: Song
     t: float
     isPlaying: bool
+    imgJpeg: Optional[bytes] = None
+
+    def asJpeg(self):
+        if not self.imgJpeg:
+            output = BytesIO()
+            self.img.save(output, 'jpeg', quality=80)
+            self.imgJpeg = output.getvalue()
+        return self.imgJpeg
+
+
+def songDir(song: Song) -> bytes:
+    return os.path.join(
+        showconfig.root(), b'video',
+        song.replace('http://', '').replace('/', '_').encode('ascii'))
 
 
 class FramesToVideoFiles:
@@ -51,8 +61,9 @@
     
     """
 
-    def __init__(self, frames: BehaviorSubject):
+    def __init__(self, frames: BehaviorSubject, root: bytes):
         self.frames = frames
+        self.root = root
         self.nextImg: Optional[CaptureFrame] = None
 
         self.currentOutputClip = None
@@ -69,7 +80,8 @@
             # start up
             self.nextWriteAction = 'saveFrames'
             self.currentOutputSong = cf.song
-            self.save('/tmp/out%s' % time.time())
+            self.save(
+                os.path.join(songDir(cf.song), b'take_%d' % int(time.time())))
         elif self.currentOutputClip and cf.isPlaying:
             self.nextWriteAction = 'saveFrames'
             # continue recording this
@@ -91,7 +103,11 @@
 
     def _bg_save(self, outBase):
         os.makedirs(os.path.dirname(outBase), exist_ok=True)
-        self.frameMap = open(outBase + '.timing', 'wt')
+        self.frameMap = open(outBase + b'.timing', 'wt')
+
+        # todo: see moviestore.py for a better-looking version where
+        # we get to call write_frame on a FFMPEG_VideoWriter instead
+        # of it calling us back.
 
         # (immediately calls make_frame)
         self.currentOutputClip = moviepy.editor.VideoClip(self._bg_make_frame,
@@ -101,7 +117,9 @@
         self.currentOutputClip.fps = 10
         log.info(f'write_videofile {outBase} start')
         try:
-            self.currentOutputClip.write_videofile(outBase + '.mp4',
+            self.currentOutputClip.write_videofile(outBase.decode('ascii') +
+                                                   '.mp4',
+                                                   codec='libx264',
                                                    audio=False,
                                                    preset='ultrafast',
                                                    logger=None,
@@ -124,7 +142,7 @@
 
         # should be a little queue to miss fewer frames
         while self.nextImg is None:
-            time.sleep(.03)
+            time.sleep(.015)
         cf, self.nextImg = self.nextImg, None
 
         self.frameMap.write(f'video {video_time_secs:g} = song {cf.t:g}\n')
@@ -139,8 +157,8 @@
         """
         Gst.init(None)
         self.musicTime = MusicTime(pollCurvecalc=False)
-        self.liveImages: BehaviorSubject[
-            Optional[CaptureFrame]] = BehaviorSubject(None)
+        self.liveImages: BehaviorSubject = BehaviorSubject(
+            None)  # stream of Optional[CaptureFrame]
 
         size = [640, 480]
 
@@ -173,7 +191,7 @@
             buf = sample.get_buffer()
             (result, mapinfo) = buf.map(Gst.MapFlags.READ)
             try:
-                img = Image.frombytes(
+                img = PIL.Image.frombytes(
                     'RGB', (caps.get_structure(0).get_value('width'),
                             caps.get_structure(0).get_value('height')),
                     mapinfo.data)
@@ -184,8 +202,10 @@
             latest = self.musicTime.getLatest()
             if 'song' in latest:
                 self.liveImages.on_next(
-                    CaptureFrame(img, Song(latest['song']), latest['t'],
-                                 latest['playing']))
+                    CaptureFrame(img=img,
+                                 song=Song(latest['song']),
+                                 t=latest['t'],
+                                 isPlaying=latest['playing']))
         except Exception:
             traceback.print_exc()
         return Gst.FlowReturn.OK
@@ -216,16 +236,12 @@
             log.error("ignoring error: %r" % messageText)
 
 
+'''
 class oldPipeline(object):
 
     def __init__(self):
         self.snapshotRequests = Queue()
 
-        try:
-            os.makedirs(snapshotDir())
-        except OSError:
-            pass
-
     def snapshot(self):
         """
         returns deferred to the path (which is under snapshotDir()) where
@@ -235,7 +251,7 @@
         d = defer.Deferred()
 
         def req(frame):
-            filename = "%s/%s.jpg" % (snapshotDir(), time.time())
+            filename = "%s/%s.jpg" % ('todo', time.time())
             log.debug("received snapshot; saving in %s", filename)
             frame.save(filename)
             d.callback(filename)
@@ -245,7 +261,6 @@
         return d
 
 
-'''
         self.imagesToSave = Queue()
         self.startBackgroundImageSaver(self.imagesToSave)
 
--- a/light9/web/light9-vidref-live.js	Thu Jun 06 00:07:07 2019 +0000
+++ b/light9/web/light9-vidref-live.js	Thu Jun 06 00:09:39 2019 +0000
@@ -1,14 +1,16 @@
 import { LitElement, TemplateResult, html, css } from '/node_modules/lit-element/lit-element.js';
+import { rounding }  from '/node_modules/significant-rounding/index.js';
+import './light9-vidref-replay.js';
+
 import debug from '/lib/debug/debug-build-es6.js';
-debug.enable('*');
 const log = debug('live');
-log('hi it is live')
 
 class Light9VidrefLive extends LitElement {
     
     static get properties() {
         return {
-            description: { type: String }
+            description: { type: String },
+            enabled: { type: Boolean }
         };
     }
 
@@ -20,25 +22,41 @@
         }
         `;
     }
-
-    firstUpdated() {
-        const ws = reconnectingWebSocket('live', (msg) => {
-            this.shadowRoot.querySelector('#live').src = 'data:image/jpeg;base64,' + msg.jpeg;
-            this.description = msg.description;
-        });
-        
+    
+    constructor() {
+        super();
+        this.live = null;
+    }
+    
+    onEnabled() {
+        if (this.shadowRoot.querySelector('#enabled').checked) {
+            
+            this.live = reconnectingWebSocket(
+                'live', (msg) => {
+                    this.shadowRoot.querySelector('#live').src = 'data:image/jpeg;base64,' + msg.jpeg;
+                    this.description = msg.description;
+                });
+            this.shadowRoot.querySelector('#liveWidget').style.display = 'block';
+        } else {
+            if (this.live) {
+                this.live.disconnect();
+                this.live = null;
+                this.shadowRoot.querySelector('#liveWidget').style.display = 'none';
+            }
+        }
     }
 
     disconnectedCallback() {
         log('bye');
+        //close socket
         
     }
     
     render() {
         return html`
 <div>
-<div><img id="live" ></div>
-<div>${this.description}</div>
+  <label><input type="checkbox" id="enabled" ?checked="${this.enabled}" @change="${this.onEnabled}">Show live</label>
+  <div id="liveWidget"><img id="live" ></div>
 </div>
 `;
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/light9/web/light9-vidref-replay-stack.js	Thu Jun 06 00:09:39 2019 +0000
@@ -0,0 +1,150 @@
+import { LitElement, TemplateResult, html, css } from '/node_modules/lit-element/lit-element.js';
+import debug from '/lib/debug/debug-build-es6.js';
+import _ from '/lib/underscore/underscore-min-es6.js';
+import { rounding }  from '/node_modules/significant-rounding/index.js';
+
+const log = debug('stack');
+
+class Light9VidrefReplayStack extends LitElement {
+    
+    static get properties() {
+        return {
+            songTime: { type: Number, attribute: false }, // from musicState.t but higher res
+            musicState: { type: Object, attribute: false },
+            players: { type: Array, attribute: false }
+        };
+    }
+
+    constructor() {
+        super();
+        this.musicState = {};
+    }
+
+    setVideoTimesFromSongTime() {
+        this.shadowRoot.querySelectorAll('light9-vidref-replay').forEach(
+            (r) => {
+                r.setVideoTimeFromSongTime(this.songTime);
+            });
+    }
+
+    fineTime() {       
+        if (this.musicState.playing) {
+            const sinceLastUpdate = (Date.now() - this.musicState.reportTime) / 1000;
+            this.songTime = sinceLastUpdate + this.musicState.tStart;
+            this.songTimeRangeInput.value = this.songTime;
+        } else  {
+            //this.songTime = this.musicState.t;
+        }
+        requestAnimationFrame(this.fineTime.bind(this));
+    }
+
+    updated(changedProperties) {
+        if (changedProperties.has('songTime')) {
+            this.setVideoTimesFromSongTime();
+        }
+    }
+
+    firstUpdated() {
+        this.songTimeRangeInput = this.shadowRoot.querySelector('#songTime');
+
+        const ws = reconnectingWebSocket('../ascoltami/time/stream',
+                                         this.receivedSongAndTime.bind(this));
+        // bug: upon connecting, clear this.song
+        this.fineTime();
+    }
+
+    receivedSongAndTime(msg) {
+        this.musicState = msg;
+        this.musicState.reportTime = Date.now();
+        this.musicState.tStart = this.musicState.t;            
+
+        this.songTimeRangeInput.max = this.musicState.duration;
+
+        if (this.musicState.song != this.song) {
+            this.song = this.musicState.song;
+            this.getReplayMapForSong(this.song);
+
+        }
+    }
+        
+    getReplayMapForSong(song) {
+        const u = new URL(window.location.href);
+        u.pathname = '/vidref/replayMap'
+        u.searchParams.set('song', song);
+        fetch(u.toString()).then((resp) => {
+            if (resp.ok) {
+                resp.json().then((msg) => {
+                    this.players = msg.map(this.makeClipRow.bind(this));
+                    this.updateComplete.then(this.setupClipRows.bind(this, msg));
+                });
+            }
+        });          
+    }
+    
+    setupClipRows(msg) {
+        const nodes = this.shadowRoot.querySelectorAll('light9-vidref-replay');
+        nodes.forEach((node, i) => {
+            node.uri = msg[i].uri;
+            node.videoUrl = msg[i].videoUrl;
+            node.songToVideo = msg[i].songToVideo;
+
+
+        });
+        this.setVideoTimesFromSongTime();
+    }
+    
+    makeClipRow(clip) {
+        return html`<light9-vidref-replay @clips-changed="${this.onClipsChanged}"></light9-vidref-replay>`;
+    }
+    
+    onClipsChanged(ev) {
+        this.getReplayMapForSong(this.song);
+    }
+    
+    disconnectedCallback() {
+        log('bye');
+        //close socket
+    }
+
+    userMovedSongTime(ev) {
+        const st = this.songTimeRangeInput.valueAsNumber;
+        this.songTime = st;
+
+        fetch('/ascoltami/seekPlayOrPause', {
+            method: 'POST',
+            body: JSON.stringify({scrub: st}),
+        });
+
+    }
+
+    static get styles() {
+        return css`
+        :host {
+         
+        }
+        #songTime {
+            width: 100%;
+        }
+#clips {
+display: flex;
+flex-direction: column;
+}
+        `;
+    }
+    
+    render() {
+        return html`
+<div>
+  <div><input id="songTime" type="range" @input="${this.userMovedSongTime}" min="0" max="0" step=".001"></div>
+  <div>${this.musicState.song}</div>
+  <div>showing song time ${rounding(this.songTime, 3)} (${rounding(this.musicState.t, 3)})</div>
+<div>clips:</div>
+<div id="clips">
+   ${this.players}
+</div>
+</div>
+`;
+
+    }
+}
+customElements.define('light9-vidref-replay-stack', Light9VidrefReplayStack);
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/light9/web/light9-vidref-replay.js	Thu Jun 06 00:09:39 2019 +0000
@@ -0,0 +1,64 @@
+import { LitElement, TemplateResult, html, css } from '/node_modules/lit-element/lit-element.js';
+import debug from '/lib/debug/debug-build-es6.js';
+import _ from '/lib/underscore/underscore-min-es6.js';
+
+const log = debug('replay');
+
+class Light9VidrefReplay extends LitElement {
+    
+    static get properties() {
+        return {
+            uri: { type: String },
+            videoUrl: { type: String },
+            songToVideo: { type: Object },
+            videoTime: { type: Number },
+        };
+    }
+
+    setVideoTimeFromSongTime(songTime) {
+        if (!this.songToVideo || !this.outVideo) {
+            return;
+        }
+        const i = _.sortedIndex(this.songToVideo, [songTime],
+                                (row) => { return row[0]; });
+        this.videoTime = this.songToVideo[Math.max(0, i - 1)][1];
+        this.outVideo.currentTime = this.videoTime;
+    }
+
+    firstUpdated() {
+        this.outVideo = this.shadowRoot.querySelector('#replay');
+    }
+
+    onDelete() {
+        const u = new URL(window.location.href);
+        u.pathname = '/vidref/clips'
+        u.searchParams.set('uri', this.uri);
+        fetch(u.toString(), {method: 'DELETE'}).then((resp) => {
+            let event = new CustomEvent('clips-changed', {detail: {}});
+            this.dispatchEvent(event);
+        });
+    }
+
+    static get styles() {
+        return css`
+        :host {
+            border: 2px solid #46a79f;
+            display: inline-block;
+        }
+        `;
+    }
+    
+    render() {
+        return html`
+<div>
+  <div><video id="replay" src="${this.videoUrl}"></video></div>
+  <div>take is ${this.uri} (${Object.keys(this.songToVideo).length} frames)</div>
+  <!-- a little canvas showing what coverage we have -->
+  <div>video time is ${this.videoTime}</div>
+  <button @click="${this.onDelete}">Delete</button>
+</div>
+`;
+
+    }
+}
+customElements.define('light9-vidref-replay', Light9VidrefReplay);
--- a/makefile	Thu Jun 06 00:07:07 2019 +0000
+++ b/makefile	Thu Jun 06 00:09:39 2019 +0000
@@ -41,7 +41,7 @@
 
 light9/web/lib/debug/debug-build-es6.js:
 	node_modules/browserify/bin/cmd.js light9/web/lib/debug/src/browser.js -o light9/web/lib/debug/debug-build-es6.js --standalone debug
-	echo "export default window.debug;" >> light9/web/lib/debug/debug-build-es6.js
+	echo "\nexport default window.debug;" >> light9/web/lib/debug/debug-build-es6.js
 
 lit_fix:
 	perl -pi -e "s,'lit-html,'/node_modules/lit-html,; s,lit-html',lit-html/lit-html.js'," node_modules/lit-element/lit-element.js
@@ -49,11 +49,11 @@
 round_fix:
 	perl -pi -e 's/module.exports = rounding/export { rounding }/' node_modules/significant-rounding/index.js
 
-debug_es6: 
-	node_modules/browserify/bin/cmd.js light9/web/lib/debug/src/browser.js -o light9/web/lib/debug/debug-build-es6.js
-	node_modules/cjs-to-es6/index.js light9/web/lib/debug/debug-build-es6.js
+light9/web/lib/underscore/underscore-min-es6.js:
+	cp light9/web/lib/underscore/underscore-min.js light9/web/lib/underscore/underscore-min-es6.js
+	perl -pi -e 's/call\(this\);/call(window); export default window._;/' light9/web/lib/underscore/underscore-min-es6.js
 
-npm: npm_install node_modules/n3/n3-browser.js light9/web/lib/debug/debug-build.js light9/web/lib/debug/debug-build-es6.js lit_fix round_fix debug_es6
+npm: npm_install node_modules/n3/n3-browser.js light9/web/lib/debug/debug-build.js light9/web/lib/debug/debug-build-es6.js lit_fix round_fix light9/web/lib/underscore/underscore-min-es6.js
 
 
 bin/ascoltami2: gst_packages link_to_sys_packages