light9 Changeset - 3ae1e7f8db23

Changeset - 3ae1e7f8db23

Parent rev.

Child rev.

[Not reviewed]

default

0 8 0

Drew Perttula - 6 years ago 2019-06-06 02:28:28
drewp@bigasterisk.com

vidref playback smoothness, autodelete short clips, manual-delete clips, vidref keyboard shortcuts
Ignore-this: 6daccf686fd66561029f3252ed4dbafd

8 files changed with 198 insertions and 50 deletions:

bin/vidref

light9/ascoltami/webapp.py

light9/vidref/index.html

light9/vidref/videorecorder.py

light9/web/light9-vidref-live.js

light9/web/light9-vidref-replay-stack.js

light9/web/light9-vidref-replay.js

light9/web/style.css

0 comments (0 inline, 0 general)

bin/vidref

➞

Show inline comments

@@ @@ -15,27 +15,27 @@ light9/web/light9-vidref-playback.js Lit @@
 """
 from run_local import log
 from twisted.internet import reactor, defer
 import logging, optparse, json, base64, os, glob
 import cyclone.web, cyclone.httpclient, cyclone.websocket
 from light9 import networking, showconfig
 from light9.vidref import videorecorder
 from rdflib import URIRef
 from light9.newtypes import Song
 from light9.namespaces import L9
 from rdfdb.syncedgraph import SyncedGraph
 from cycloneerr import PrettyErrorHandler
 from typing import cast
 parser = optparse.OptionParser()
 parser.add_option("-v", "--verbose", action="store_true", help="logging.DEBUG")
 (options, args) = parser.parse_args()
 log.setLevel(logging.DEBUG if options.verbose else logging.INFO)
 class Snapshot(cyclone.web.RequestHandler):
     @defer.inlineCallbacks
     def post(self):
@@ @@ -87,54 +87,53 @@ class SnapshotPic(cyclone.web.StaticFile @@
 class Time(cyclone.web.RequestHandler):
     def put(self):
         body = json.loads(self.request.body)
         t = body['t']
         source = body['source']
         self.settings.gui.incomingTime(t, source)
         self.set_status(202)
 def takeUri(songPath: bytes):
     p = songPath.decode('ascii').split('/')
     take = p[-1].replace('.mp4', '')
     song = p[-2].split('_')
     return URIRef('/'.join(
         ['http://light9.bigasterisk.com/show', song[-2], song[-1], take]))
 class Clips(PrettyErrorHandler, cyclone.web.RequestHandler):
     def delete(self):
         clip = URIRef(self.get_argument('uri'))
         videorecorder.deleteClip(clip)
 class ReplayMap(PrettyErrorHandler, cyclone.web.RequestHandler):
     def get(self):
         song = Song(self.get_argument('song'))
         clips = []
         for vid in glob.glob(os.path.join(videorecorder.songDir(song),
                                           b'*.mp4')):
             pts = []
             for line in open(vid.replace(b'.mp4', b'.timing'), 'rb'):
                 _v, vt, _eq, _song, st = line.split()
                 pts.append([float(st), float(vt)])
             url = vid[len(os.path.dirname(os.path.dirname(showconfig.root()))
                          ):].decode('ascii')
             clips.append({
                 'uri': takeUri(vid),
+                'uri': videorecorder.takeUri(vid),
                 'videoUrl': url,
                 'songToVideo': pts
             })
         clips.sort(key=lambda c: len(c['songToVideo']))
+        clips.sort(key=lambda c: len(cast(list, c['songToVideo'])))
         clips = clips[-3:]
         clips.sort(key=lambda c: c['uri'], reverse=True)
         ret = json.dumps(clips)
         log.info('replayMap had %s videos; json is %s bytes', len(clips),
                  len(ret))
         self.write(ret)
 graph = SyncedGraph(networking.rdfdb.url, "vidref")
 outVideos = videorecorder.FramesToVideoFiles(
     pipeline.liveImages, os.path.join(showconfig.root(), b'video'))
@@ @@ -144,24 +143,25 @@ reactor.listenTCP( @@
     port,
     cyclone.web.Application(
         handlers=[
             (r'/()', cyclone.web.StaticFileHandler, {
                 'path': 'light9/vidref',
                 'default_filename': 'index.html'
             }),
             (r'/setup/()', cyclone.web.StaticFileHandler, {
                 'path': 'light9/vidref',
                 'default_filename': 'setup.html'
             }),
             (r'/live', Live),
             (r'/clips', Clips),
             (r'/replayMap', ReplayMap),
             (r'/snapshot', Snapshot),
             (r'/snapshot/(.*)', SnapshotPic, {
                 "path": 'todo',
             }),
             (r'/time', Time),
         ],
         debug=True,
     ))
 log.info("serving on %s" % port)
 reactor.run()

light9/ascoltami/webapp.py

➞

Show inline comments

@@ @@ -145,24 +145,32 @@ class songResource(PrettyErrorHandler, c @@
 class seekPlayOrPause(PrettyErrorHandler, cyclone.web.RequestHandler):
     """curveCalc's ctrl-p or a vidref scrub"""
     def post(self):
         player = self.settings.app.player
         data = json.loads(self.request.body)
         if 'scrub' in data:
             player.pause()
             player.seek(data['scrub'])
             return
         if 'action' in data:
             if data['action'] == 'play':
                 player.resume()
             elif data['action'] == 'pause':
                 player.pause()
             else:
                 raise NotImplementedError
             return
         if player.isPlaying():
             player.pause()
         else:
             player.seek(data['t'])
             player.resume()
 class output(PrettyErrorHandler, cyclone.web.RequestHandler):
     def post(self):
         d = json.loads(self.request.body)
         subprocess.check_call(["bin/movesinks", str(d['sink'])])

light9/vidref/index.html

➞

Show inline comments

 <!doctype html>
 <html>
   <head>
     <title>vidref</title>
     <meta charset="utf-8" />
     <style>
+     <link rel="stylesheet" href="/style.css">
     </style>
     <script src="/lib/debug/debug-build.js"></script>
     <script>
      debug.enable('*');
     </script>
     <script src="/lib/jquery/dist/jquery.slim.min.js"></script>
     <script src="/websocket.js"></script>
     <script type="module" src="/light9-vidref-live.js"></script>
     <script type="module" src="/light9-vidref-replay-stack.js"></script>
   </head>
   <body>
-    Live:
+    <h1>vidref</h1>
     <div>
       <light9-vidref-live></light9-vidref-live>
     </div>
     <light9-vidref-replay-stack></light9-vidref-replay-stack>
     <light9-vidref-replay-stack id="rs"></light9-vidref-replay-stack>
     <div class="keys">Keys:
       <span class="keyCap">s</span> stop,
       <span class="keyCap">p</span> play,
       <span class="keyCap">,/.</span> step
     </div>
       <script>
        const log = debug('index');
        document.addEventListener('keypress', (ev) => {
          const nudge = (dt) => {
            const newTime = document.querySelector('#rs').songTime + dt;
            fetch('/ascoltami/seekPlayOrPause', {
              method: 'POST',
              body: JSON.stringify({scrub: newTime}),
            });
          };
 stats of all disk usage
          if (ev.code == 'KeyP') {
            fetch('/ascoltami/seekPlayOrPause',
                  {method: 'POST', body: JSON.stringify({action: 'play'})});
          } else if (ev.code == 'KeyS') {
            fetch('/ascoltami/seekPlayOrPause',
                  {method: 'POST', body: JSON.stringify({action: 'pause'})});
          } else if (ev.code == 'Comma') {
            nudge(-.1);
          } else if (ev.code == 'Period') {
            nudge(.1);
+         }
        });
       </script>
   </body>
 </html>

light9/vidref/videorecorder.py

➞

Show inline comments

@@ @@ -2,24 +2,25 @@ from dataclasses import dataclass @@
 from typing import Optional
 import time, logging, os, traceback
 from io import BytesIO
 import gi
 gi.require_version('Gst', '1.0')
 gi.require_version('GstBase', '1.0')
 import PIL.Image
 from gi.repository import Gst
 from rx.subject import BehaviorSubject
 from twisted.internet import threads
 from rdflib import URIRef
 import moviepy.editor
 import numpy
 from light9.ascoltami.musictime_client import MusicTime
 from light9.newtypes import Song
 from light9 import showconfig
 log = logging.getLogger()
 @dataclass
 class CaptureFrame:
@@ @@ -33,49 +34,65 @@ class CaptureFrame: @@
         if not self.imgJpeg:
             output = BytesIO()
             self.img.save(output, 'jpeg', quality=80)
             self.imgJpeg = output.getvalue()
         return self.imgJpeg
 def songDir(song: Song) -> bytes:
     return os.path.join(
         showconfig.root(), b'video',
         song.replace('http://', '').replace('/', '_').encode('ascii'))
 def takeUri(songPath: bytes) -> URIRef:
     p = songPath.decode('ascii').split('/')
     take = p[-1].replace('.mp4', '')
     song = p[-2].split('_')
     return URIRef('/'.join(
         ['http://light9.bigasterisk.com/show', song[-2], song[-1], take]))
 def deleteClip(uri: URIRef):
     # uri http://light9.bigasterisk.com/show/dance2019/song6/take_155
     # path show/dance2019/video/light9.bigasterisk.com_show_dance2019_song6/take_155.*
     w = uri.split('/')[-4:]
     path = '/'.join([w[0], w[1], 'video',
                      f'light9.bigasterisk.com_{w[0]}_{w[1]}_{w[2]}', w[3]])
     log.info(f'deleting {uri} {path}')
     for fn in [path + '.mp4', path + '.timing']:
         os.remove(fn)
 class FramesToVideoFiles:
     """
     nextWriteAction: 'ignore'
     currentOutputClip: None
     (frames come in for new video)
     nextWriteAction: 'saveFrame'
     currentOutputClip: new VideoClip
     (many frames)
     (music stops or song changes)
     nextWriteAction: 'close'
     currentOutputClip: None
     nextWriteAction: 'ignore'
     """
     def __init__(self, frames: BehaviorSubject, root: bytes):
         self.frames = frames
         self.root = root
         self.nextImg: Optional[CaptureFrame] = None
         self.currentOutputClip = None
+        self.currentOutputClip: Optional[moviepy.editor.VideoClip] = None
         self.currentOutputSong: Optional[Song] = None
         self.nextWriteAction = 'ignore'
         self.frames.subscribe(on_next=self.onFrame)
     def onFrame(self, cf: Optional[CaptureFrame]):
         if cf is None:
             return
         self.nextImg = cf
         if self.currentOutputClip is None and cf.isPlaying:
             # start up
             self.nextWriteAction = 'saveFrames'
@@ @@ -92,69 +109,78 @@ class FramesToVideoFiles: @@
             # stop
             self.nextWriteAction = 'close'
         else:
             raise NotImplementedError(str(vars()))
     def save(self, outBase):
         """
         receive frames (infinite) and wall-to-song times (stream ends with
         the song), and write a video file and a frame map
         """
         return threads.deferToThread(self._bg_save, outBase)
     def _bg_save(self, outBase):
+    def _bg_save(self, outBase: bytes):
         os.makedirs(os.path.dirname(outBase), exist_ok=True)
         self.frameMap = open(outBase + b'.timing', 'wt')
         # todo: see moviestore.py for a better-looking version where
         # we get to call write_frame on a FFMPEG_VideoWriter instead
         # of it calling us back.
         self.currentClipFrameCount = 0
         # (immediately calls make_frame)
         self.currentOutputClip = moviepy.editor.VideoClip(self._bg_make_frame,
                                                           duration=999.)
         # The fps recorded in the file doesn't matter much; we'll play
         # it back in sync with the music regardless.
         self.currentOutputClip.fps = 10
         log.info(f'write_videofile {outBase} start')
         try:
             self.currentOutputClip.write_videofile(outBase.decode('ascii') +
                                                    '.mp4',
             self.outMp4 = outBase.decode('ascii') + '.mp4'
             self.currentOutputClip.write_videofile(self.outMp4,
                                                    codec='libx264',
                                                    audio=False,
                                                    preset='ultrafast',
                                                    logger=None,
                                                    ffmpeg_params=['-g', '10'],
                                                    bitrate='150000')
         except (StopIteration, RuntimeError):
             self.frameMap.close()
         log.info('write_videofile done')
         self.currentOutputClip = None
         if self.currentClipFrameCount < 400:
             log.info('too small- deleting')
             deleteClip(takeUri(self.outMp4.encode('ascii')))
     def _bg_make_frame(self, video_time_secs):
         if self.nextWriteAction == 'close':
             raise StopIteration  # the one in write_videofile
         elif self.nextWriteAction == 'notWritingClip':
             raise NotImplementedError
         elif self.nextWriteAction == 'saveFrames':
             pass
         else:
             raise NotImplementedError(self.nextWriteAction)
         # should be a little queue to miss fewer frames
         while self.nextImg is None:
             time.sleep(.015)
         cf, self.nextImg = self.nextImg, None
         self.frameMap.write(f'video {video_time_secs:g} = song {cf.t:g}\n')
         self.currentClipFrameCount += 1
         return numpy.asarray(cf.img)
 class GstSource:
     def __init__(self, dev):
         """
         make new gst pipeline
         """
         Gst.init(None)
         self.musicTime = MusicTime(pollCurvecalc=False)
         self.liveImages: BehaviorSubject = BehaviorSubject(

light9/web/light9-vidref-live.js

➞

Show inline comments

@@ @@ -5,33 +5,24 @@ import './light9-vidref-replay.js'; @@
 import debug from '/lib/debug/debug-build-es6.js';
 const log = debug('live');
 class Light9VidrefLive extends LitElement {
     static get properties() {
         return {
             description: { type: String },
             enabled: { type: Boolean }
         };
+    }
     static get styles() {
         return css`
         :host {
             border: 2px solid #46a79f;
             display: inline-block;
+        }
         `;
+    }
     constructor() {
         super();
         this.live = null;
+    }
     onEnabled() {
         if (this.shadowRoot.querySelector('#enabled').checked) {
             this.live = reconnectingWebSocket(
                 'live', (msg) => {
                     this.shadowRoot.querySelector('#live').src = 'data:image/jpeg;base64,' + msg.jpeg;
                     this.description = msg.description;
@@ @@ -43,23 +34,32 @@ class Light9VidrefLive extends LitElemen @@
                 this.live = null;
                 this.shadowRoot.querySelector('#liveWidget').style.display = 'none';
+            }
+        }
+    }
     disconnectedCallback() {
         log('bye');
         //close socket
+    }
     static get styles() {
         return css`
         :host {
             display: inline-block;
+        }
 #live {
 border: 4px solid orange;
+}
         `;
+    }
     render() {
         return html`
 <div>
   <label><input type="checkbox" id="enabled" ?checked="${this.enabled}" @change="${this.onEnabled}">Show live</label>
   <div id="liveWidget"><img id="live" ></div>
 </div>
   <div id="liveWidget" style="display: none"><img id="live" ></div>
 `;
+    }
+}
 customElements.define('light9-vidref-live', Light9VidrefLive);

light9/web/light9-vidref-replay-stack.js

➞

Show inline comments

@@ @@ -14,35 +14,37 @@ class Light9VidrefReplayStack extends Li @@
             players: { type: Array, attribute: false }
         };
+    }
     constructor() {
         super();
         this.musicState = {};
+    }
     setVideoTimesFromSongTime() {
         this.shadowRoot.querySelectorAll('light9-vidref-replay').forEach(
             (r) => {
                 r.setVideoTimeFromSongTime(this.songTime);
+                r.setVideoTimeFromSongTime(this.songTime, this.musicState.playing);
             });
+    }
     nudgeTime(dt) {
         this.songTime += dt;
         log('song now', this.songTime);
+    }
     fineTime() {
         if (this.musicState.playing) {
             const sinceLastUpdate = (Date.now() - this.musicState.reportTime) / 1000;
             this.songTime = sinceLastUpdate + this.musicState.tStart;
             this.songTimeRangeInput.value = this.songTime;
         } else  {
-            //this.songTime = this.musicState.t;
             this.songTime = this.musicState.t;
+        }
         requestAnimationFrame(this.fineTime.bind(this));
+    }
     updated(changedProperties) {
         if (changedProperties.has('songTime')) {
             this.setVideoTimesFromSongTime();
+        }
+    }
     firstUpdated() {
         this.songTimeRangeInput = this.shadowRoot.querySelector('#songTime');
@@ @@ -54,50 +56,47 @@ class Light9VidrefReplayStack extends Li @@
+    }
     receivedSongAndTime(msg) {
         this.musicState = msg;
         this.musicState.reportTime = Date.now();
         this.musicState.tStart = this.musicState.t;
         this.songTimeRangeInput.max = this.musicState.duration;
         if (this.musicState.song != this.song) {
             this.song = this.musicState.song;
             this.getReplayMapForSong(this.song);
+        }
+    }
     getReplayMapForSong(song) {
         const u = new URL(window.location.href);
         u.pathname = '/vidref/replayMap'
         u.searchParams.set('song', song);
         fetch(u.toString()).then((resp) => {
             if (resp.ok) {
                 resp.json().then((msg) => {
                     this.players = msg.map(this.makeClipRow.bind(this));
                     this.updateComplete.then(this.setupClipRows.bind(this, msg));
                 });
+            }
         });
+    }
     setupClipRows(msg) {
         const nodes = this.shadowRoot.querySelectorAll('light9-vidref-replay');
         nodes.forEach((node, i) => {
             node.uri = msg[i].uri;
             node.videoUrl = msg[i].videoUrl;
             node.songToVideo = msg[i].songToVideo;
         });
         this.setVideoTimesFromSongTime();
+    }
     makeClipRow(clip) {
         return html`<light9-vidref-replay @clips-changed="${this.onClipsChanged}"></light9-vidref-replay>`;
+    }
     onClipsChanged(ev) {
         this.getReplayMapForSong(this.song);
+    }
@@ @@ -105,46 +104,56 @@ class Light9VidrefReplayStack extends Li @@
         log('bye');
         //close socket
+    }
     userMovedSongTime(ev) {
         const st = this.songTimeRangeInput.valueAsNumber;
         this.songTime = st;
         fetch('/ascoltami/seekPlayOrPause', {
             method: 'POST',
             body: JSON.stringify({scrub: st}),
         });
+    }
     static get styles() {
         return css`
         :host {
+           display: inline-block;
+        }
         #songTime {
             width: 100%;
+        }
 #clips {
 display: flex;
 flex-direction: column;
+}
         a {
             color: rgb(97, 97, 255);
+        }
         #songTime {
             font-size: 27px;
+        }
         `;
+    }
     render() {
         return html`
 <div>
   <div><input id="songTime" type="range" @input="${this.userMovedSongTime}" min="0" max="0" step=".001"></div>
   <div>${this.musicState.song}</div>
   <div>showing song time ${rounding(this.songTime, 3)} (${rounding(this.musicState.t, 3)})</div>
     <input id="songTime" type="range"
            .value="${this.songTime}"
            @input="${this.userMovedSongTime}"
            min="0" max="0" step=".001"></div>
   <div><a href="${this.musicState.song}">${this.musicState.song}</a></div>
   <div id="songTime">showing song time ${rounding(this.songTime, 3)}</div>
 <div>clips:</div>
 <div id="clips">
    ${this.players}
 </div>
   <div>
     <button @click="${this.onClipsChanged}">Refresh clips for song</button>
 </div>
 `;
+    }
+}
 customElements.define('light9-vidref-replay-stack', Light9VidrefReplayStack);

light9/web/light9-vidref-replay.js

➞

Show inline comments

 import { LitElement, TemplateResult, html, css } from '/node_modules/lit-element/lit-element.js';
 import debug from '/lib/debug/debug-build-es6.js';
 import _ from '/lib/underscore/underscore-min-es6.js';
 import { rounding }  from '/node_modules/significant-rounding/index.js';
 const log = debug('replay');
 class Light9VidrefReplay extends LitElement {
     static get properties() {
         return {
             uri: { type: String },
             videoUrl: { type: String },
             songToVideo: { type: Object },
             videoTime: { type: Number },
             outVideoCurrentTime: { type: Number },
             timeErr: { type: Number },
             playRate: { type: Number }
         };
+    }
     setVideoTimeFromSongTime(songTime) {
         if (!this.songToVideo || !this.outVideo) {
     estimateRate() {
         const n = this.songToVideo.length;
         const x0 = Math.round(n * .3);
         const x1 = Math.round(n * .6);
         const pt0 = this.songToVideo[x0];
         const pt1 = this.songToVideo[x1];
         return (pt1[1] - pt0[1]) / (pt1[0] - pt0[0]);
+    }
     setVideoTimeFromSongTime(songTime, isPlaying) {
         if (!this.songToVideo || !this.outVideo || this.outVideo.readyState < 1) {
             return;
+        }
         const i = _.sortedIndex(this.songToVideo, [songTime],
                                 (row) => { return row[0]; });
         this.videoTime = this.songToVideo[Math.max(0, i - 1)][1];
         this.outVideoCurrentTime = this.outVideo.currentTime;
         if (isPlaying) {
             if (this.outVideo.paused) {
                 this.outVideo.play();
                 this.setRate(this.estimateRate());
+            }
             const err = this.outVideo.currentTime - this.videoTime;
             this.timeErr = err;
             if (Math.abs(err) > window.thresh) {
         this.outVideo.currentTime = this.videoTime;
                 const p = window.p;
                 if (err > 0) {
                     this.setRate(this.playRate - err * p);
                 } else {
                     this.setRate(this.playRate - err * p);
+                }
+            }
         } else {
             this.outVideo.pause();
             this.outVideoCurrentTime = this.outVideo.currentTime = this.videoTime;
             this.timeErr = 0;
+        }
+    }
     setRate(r) {
         this.playRate = Math.max(.1, Math.min(4, r));
         this.outVideo.playbackRate = this.playRate;
+    }
     firstUpdated() {
         this.outVideo = this.shadowRoot.querySelector('#replay');
         this.playRate = this.outVideo.playbackRate = 1.0;
+    }
     onDelete() {
         const u = new URL(window.location.href);
         u.pathname = '/vidref/clips'
         u.searchParams.set('uri', this.uri);
         fetch(u.toString(), {method: 'DELETE'}).then((resp) => {
             let event = new CustomEvent('clips-changed', {detail: {}});
             this.dispatchEvent(event);
         });
+    }
     static get styles() {
         return css`
         :host {
             margin: 5px;
             border: 2px solid #46a79f;
             display: flex;
             flex-direction: column;
+        }
         div {
             padding: 5px;
+        }
         .num {
             display: inline-block;
             width: 4em;
             color: #29ffa0;
+        }
         a {
             color: rgb(97, 97, 255);
+        }
         `;
+    }
     render() {
         return html`
   <video id="replay" src="${this.videoUrl}"></video>
 <div>
   <div><video id="replay" src="${this.videoUrl}"></video></div>
   <div>take is ${this.uri} (${Object.keys(this.songToVideo).length} frames)</div>
   <!-- a little canvas showing what coverage we have -->
   <div>video time is ${this.videoTime}</div>
     take is <a href="${this.uri}">${this.uri}</a>
     (${Object.keys(this.songToVideo).length} frames)
   <button @click="${this.onDelete}">Delete</button>
 </div>
   <!-- here, put a little canvas showing what coverage we have with the
        actual/goal time cursors -->
   <div>
     video time should be <span class="num">${this.videoTime} </span>
     actual = <span class="num">${rounding(this.outVideoCurrentTime, 3)}</span>,
     err = <span class="num">${rounding(this.timeErr, 3)} </span>
     rate = <span class="num">${rounding(this.playRate, 3)}</span>
   </div>
 `;
+    }
+}
 customElements.define('light9-vidref-replay', Light9VidrefReplay);
 window.thresh=.3
 window.p=.3

light9/web/style.css

➞

Show inline comments

@@ @@ -64,24 +64,42 @@ button a { @@
 .commands button {
     background: black;
     color: white;
     padding: 20px
+}
 .commands button.active {
     background: #a90707;
+}
 .key {
     color: #888;
+}
 div.keys {
     margin-top: 10px;
     padding: 5px;
+}
 .keyCap {
     color: #ccc;
     background: #525252;
     display: inline-block;
     border: 1px outset #b3b3b3;
     padding: 2px 3px;
     margin: 3px 0;
     font-size: 16px;
     box-shadow: 0.9px 0.9px 0px 2px #565656;
     border-radius: 2px;
+}
 .currentSong button {
     background: #a90707;
+}
 .timeRow {
     margin: 14px;
+}
 .stalled {
     opacity: .5;
+}
 .num {
     font-size: 27px;
     color: rgb(233, 122, 122);

0 comments (0 inline, 0 general)