light9 Changeset - 3ae1e7f8db23

Changeset - 3ae1e7f8db23

Parent rev.

Child rev.

[Not reviewed]

default

0 8 0

Drew Perttula - 6 years ago 2019-06-06 02:28:28
drewp@bigasterisk.com

vidref playback smoothness, autodelete short clips, manual-delete clips, vidref keyboard shortcuts
Ignore-this: 6daccf686fd66561029f3252ed4dbafd

8 files changed with 198 insertions and 50 deletions:

bin/vidref

light9/ascoltami/webapp.py

light9/vidref/index.html

light9/vidref/videorecorder.py

light9/web/light9-vidref-live.js

light9/web/light9-vidref-replay-stack.js

light9/web/light9-vidref-replay.js

light9/web/style.css

0 comments (0 inline, 0 general)

bin/vidref

➞

Show inline comments

@@ @@ -21,15 +21,15 @@ from twisted.internet import reactor, de @@
 import logging, optparse, json, base64, os, glob
 import cyclone.web, cyclone.httpclient, cyclone.websocket
 from light9 import networking, showconfig
 from light9.vidref import videorecorder
 from rdflib import URIRef
 from light9.newtypes import Song
 from light9.namespaces import L9
 from rdfdb.syncedgraph import SyncedGraph
 from cycloneerr import PrettyErrorHandler
 from typing import cast
 parser = optparse.OptionParser()
 parser.add_option("-v", "--verbose", action="store_true", help="logging.DEBUG")
 (options, args) = parser.parse_args()
 log.setLevel(logging.DEBUG if options.verbose else logging.INFO)
@@ @@ -93,19 +93,18 @@ class Time(cyclone.web.RequestHandler): @@
         t = body['t']
         source = body['source']
         self.settings.gui.incomingTime(t, source)
         self.set_status(202)
 def takeUri(songPath: bytes):
     p = songPath.decode('ascii').split('/')
     take = p[-1].replace('.mp4', '')
     song = p[-2].split('_')
     return URIRef('/'.join(
         ['http://light9.bigasterisk.com/show', song[-2], song[-1], take]))
 class Clips(PrettyErrorHandler, cyclone.web.RequestHandler):
     def delete(self):
         clip = URIRef(self.get_argument('uri'))
         videorecorder.deleteClip(clip)
 class ReplayMap(PrettyErrorHandler, cyclone.web.RequestHandler):
     def get(self):
         song = Song(self.get_argument('song'))
         clips = []
@@ @@ -117,18 +116,18 @@ class ReplayMap(PrettyErrorHandler, cycl @@
                 pts.append([float(st), float(vt)])
             url = vid[len(os.path.dirname(os.path.dirname(showconfig.root()))
                          ):].decode('ascii')
             clips.append({
                 'uri': takeUri(vid),
+                'uri': videorecorder.takeUri(vid),
                 'videoUrl': url,
                 'songToVideo': pts
             })
         clips.sort(key=lambda c: len(c['songToVideo']))
+        clips.sort(key=lambda c: len(cast(list, c['songToVideo'])))
         clips = clips[-3:]
         clips.sort(key=lambda c: c['uri'], reverse=True)
         ret = json.dumps(clips)
         log.info('replayMap had %s videos; json is %s bytes', len(clips),
                  len(ret))
@@ @@ -150,12 +149,13 @@ reactor.listenTCP( @@
             }),
             (r'/setup/()', cyclone.web.StaticFileHandler, {
                 'path': 'light9/vidref',
                 'default_filename': 'setup.html'
             }),
             (r'/live', Live),
             (r'/clips', Clips),
             (r'/replayMap', ReplayMap),
             (r'/snapshot', Snapshot),
             (r'/snapshot/(.*)', SnapshotPic, {
                 "path": 'todo',
             }),
             (r'/time', Time),

light9/ascoltami/webapp.py

➞

Show inline comments

@@ @@ -151,12 +151,20 @@ class seekPlayOrPause(PrettyErrorHandler @@
         data = json.loads(self.request.body)
         if 'scrub' in data:
             player.pause()
             player.seek(data['scrub'])
             return
         if 'action' in data:
             if data['action'] == 'play':
                 player.resume()
             elif data['action'] == 'pause':
                 player.pause()
             else:
                 raise NotImplementedError
             return
         if player.isPlaying():
             player.pause()
         else:
             player.seek(data['t'])
             player.resume()

light9/vidref/index.html

➞

Show inline comments

 <!doctype html>
 <html>
   <head>
     <title>vidref</title>
     <meta charset="utf-8" />
     <style>
+     <link rel="stylesheet" href="/style.css">
     </style>
     <script src="/lib/debug/debug-build.js"></script>
     <script>
      debug.enable('*');
     </script>
     <script src="/lib/jquery/dist/jquery.slim.min.js"></script>
     <script src="/websocket.js"></script>
     <script type="module" src="/light9-vidref-live.js"></script>
     <script type="module" src="/light9-vidref-replay-stack.js"></script>
   </head>
   <body>
-    Live:
+    <h1>vidref</h1>
     <div>
       <light9-vidref-live></light9-vidref-live>
     </div>
     <light9-vidref-replay-stack></light9-vidref-replay-stack>
     <light9-vidref-replay-stack id="rs"></light9-vidref-replay-stack>
     <div class="keys">Keys:
       <span class="keyCap">s</span> stop,
       <span class="keyCap">p</span> play,
       <span class="keyCap">,/.</span> step
     </div>
       <script>
        const log = debug('index');
        document.addEventListener('keypress', (ev) => {
          const nudge = (dt) => {
            const newTime = document.querySelector('#rs').songTime + dt;
            fetch('/ascoltami/seekPlayOrPause', {
              method: 'POST',
              body: JSON.stringify({scrub: newTime}),
            });
          };
 stats of all disk usage
          if (ev.code == 'KeyP') {
            fetch('/ascoltami/seekPlayOrPause',
                  {method: 'POST', body: JSON.stringify({action: 'play'})});
          } else if (ev.code == 'KeyS') {
            fetch('/ascoltami/seekPlayOrPause',
                  {method: 'POST', body: JSON.stringify({action: 'pause'})});
          } else if (ev.code == 'Comma') {
            nudge(-.1);
          } else if (ev.code == 'Period') {
            nudge(.1);
+         }
        });
       </script>
   </body>
 </html>

light9/vidref/videorecorder.py

➞

Show inline comments

@@ @@ -8,12 +8,13 @@ gi.require_version('Gst', '1.0') @@
 gi.require_version('GstBase', '1.0')
 import PIL.Image
 from gi.repository import Gst
 from rx.subject import BehaviorSubject
 from twisted.internet import threads
 from rdflib import URIRef
 import moviepy.editor
 import numpy
 from light9.ascoltami.musictime_client import MusicTime
 from light9.newtypes import Song
 from light9 import showconfig
@@ @@ -39,12 +40,28 @@ class CaptureFrame: @@
 def songDir(song: Song) -> bytes:
     return os.path.join(
         showconfig.root(), b'video',
         song.replace('http://', '').replace('/', '_').encode('ascii'))
 def takeUri(songPath: bytes) -> URIRef:
     p = songPath.decode('ascii').split('/')
     take = p[-1].replace('.mp4', '')
     song = p[-2].split('_')
     return URIRef('/'.join(
         ['http://light9.bigasterisk.com/show', song[-2], song[-1], take]))
 def deleteClip(uri: URIRef):
     # uri http://light9.bigasterisk.com/show/dance2019/song6/take_155
     # path show/dance2019/video/light9.bigasterisk.com_show_dance2019_song6/take_155.*
     w = uri.split('/')[-4:]
     path = '/'.join([w[0], w[1], 'video',
                      f'light9.bigasterisk.com_{w[0]}_{w[1]}_{w[2]}', w[3]])
     log.info(f'deleting {uri} {path}')
     for fn in [path + '.mp4', path + '.timing']:
         os.remove(fn)
 class FramesToVideoFiles:
     """
     nextWriteAction: 'ignore'
     currentOutputClip: None
@@ @@ -63,13 +80,13 @@ class FramesToVideoFiles: @@
     def __init__(self, frames: BehaviorSubject, root: bytes):
         self.frames = frames
         self.root = root
         self.nextImg: Optional[CaptureFrame] = None
         self.currentOutputClip = None
+        self.currentOutputClip: Optional[moviepy.editor.VideoClip] = None
         self.currentOutputSong: Optional[Song] = None
         self.nextWriteAction = 'ignore'
         self.frames.subscribe(on_next=self.onFrame)
     def onFrame(self, cf: Optional[CaptureFrame]):
         if cf is None:
@@ @@ -98,41 +115,49 @@ class FramesToVideoFiles: @@
         """
         receive frames (infinite) and wall-to-song times (stream ends with
         the song), and write a video file and a frame map
         """
         return threads.deferToThread(self._bg_save, outBase)
     def _bg_save(self, outBase):
+    def _bg_save(self, outBase: bytes):
         os.makedirs(os.path.dirname(outBase), exist_ok=True)
         self.frameMap = open(outBase + b'.timing', 'wt')
         # todo: see moviestore.py for a better-looking version where
         # we get to call write_frame on a FFMPEG_VideoWriter instead
         # of it calling us back.
         self.currentClipFrameCount = 0
         # (immediately calls make_frame)
         self.currentOutputClip = moviepy.editor.VideoClip(self._bg_make_frame,
                                                           duration=999.)
         # The fps recorded in the file doesn't matter much; we'll play
         # it back in sync with the music regardless.
         self.currentOutputClip.fps = 10
         log.info(f'write_videofile {outBase} start')
         try:
             self.currentOutputClip.write_videofile(outBase.decode('ascii') +
                                                    '.mp4',
             self.outMp4 = outBase.decode('ascii') + '.mp4'
             self.currentOutputClip.write_videofile(self.outMp4,
                                                    codec='libx264',
                                                    audio=False,
                                                    preset='ultrafast',
                                                    logger=None,
                                                    ffmpeg_params=['-g', '10'],
                                                    bitrate='150000')
         except (StopIteration, RuntimeError):
             self.frameMap.close()
         log.info('write_videofile done')
         self.currentOutputClip = None
         if self.currentClipFrameCount < 400:
             log.info('too small- deleting')
             deleteClip(takeUri(self.outMp4.encode('ascii')))
     def _bg_make_frame(self, video_time_secs):
         if self.nextWriteAction == 'close':
             raise StopIteration  # the one in write_videofile
         elif self.nextWriteAction == 'notWritingClip':
             raise NotImplementedError
         elif self.nextWriteAction == 'saveFrames':
@@ @@ -143,12 +168,13 @@ class FramesToVideoFiles: @@
         # should be a little queue to miss fewer frames
         while self.nextImg is None:
             time.sleep(.015)
         cf, self.nextImg = self.nextImg, None
         self.frameMap.write(f'video {video_time_secs:g} = song {cf.t:g}\n')
         self.currentClipFrameCount += 1
         return numpy.asarray(cf.img)
 class GstSource:
     def __init__(self, dev):

light9/web/light9-vidref-live.js

➞

Show inline comments

@@ @@ -11,21 +11,12 @@ class Light9VidrefLive extends LitElemen @@
         return {
             description: { type: String },
             enabled: { type: Boolean }
         };
+    }
     static get styles() {
         return css`
         :host {
             border: 2px solid #46a79f;
             display: inline-block;
+        }
         `;
+    }
     constructor() {
         super();
         this.live = null;
+    }
     onEnabled() {
@@ @@ -49,17 +40,26 @@ class Light9VidrefLive extends LitElemen @@
     disconnectedCallback() {
         log('bye');
         //close socket
+    }
     static get styles() {
         return css`
         :host {
             display: inline-block;
+        }
 #live {
 border: 4px solid orange;
+}
         `;
+    }
     render() {
         return html`
 <div>
   <label><input type="checkbox" id="enabled" ?checked="${this.enabled}" @change="${this.onEnabled}">Show live</label>
   <div id="liveWidget"><img id="live" ></div>
 </div>
   <div id="liveWidget" style="display: none"><img id="live" ></div>
 `;
+    }
+}
 customElements.define('light9-vidref-live', Light9VidrefLive);

light9/web/light9-vidref-replay-stack.js

➞

Show inline comments

@@ @@ -20,23 +20,25 @@ class Light9VidrefReplayStack extends Li @@
         this.musicState = {};
+    }
     setVideoTimesFromSongTime() {
         this.shadowRoot.querySelectorAll('light9-vidref-replay').forEach(
             (r) => {
                 r.setVideoTimeFromSongTime(this.songTime);
+                r.setVideoTimeFromSongTime(this.songTime, this.musicState.playing);
             });
+    }
     nudgeTime(dt) {
         this.songTime += dt;
         log('song now', this.songTime);
+    }
     fineTime() {
         if (this.musicState.playing) {
             const sinceLastUpdate = (Date.now() - this.musicState.reportTime) / 1000;
             this.songTime = sinceLastUpdate + this.musicState.tStart;
             this.songTimeRangeInput.value = this.songTime;
         } else  {
-            //this.songTime = this.musicState.t;
             this.songTime = this.musicState.t;
+        }
         requestAnimationFrame(this.fineTime.bind(this));
+    }
     updated(changedProperties) {
         if (changedProperties.has('songTime')) {
@@ @@ -60,13 +62,12 @@ class Light9VidrefReplayStack extends Li @@
         this.songTimeRangeInput.max = this.musicState.duration;
         if (this.musicState.song != this.song) {
             this.song = this.musicState.song;
             this.getReplayMapForSong(this.song);
+        }
+    }
     getReplayMapForSong(song) {
         const u = new URL(window.location.href);
         u.pathname = '/vidref/replayMap'
@@ @@ -84,14 +85,12 @@ class Light9VidrefReplayStack extends Li @@
     setupClipRows(msg) {
         const nodes = this.shadowRoot.querySelectorAll('light9-vidref-replay');
         nodes.forEach((node, i) => {
             node.uri = msg[i].uri;
             node.videoUrl = msg[i].videoUrl;
             node.songToVideo = msg[i].songToVideo;
         });
         this.setVideoTimesFromSongTime();
+    }
     makeClipRow(clip) {
         return html`<light9-vidref-replay @clips-changed="${this.onClipsChanged}"></light9-vidref-replay>`;
@@ @@ -111,40 +110,50 @@ class Light9VidrefReplayStack extends Li @@
         this.songTime = st;
         fetch('/ascoltami/seekPlayOrPause', {
             method: 'POST',
             body: JSON.stringify({scrub: st}),
         });
+    }
     static get styles() {
         return css`
         :host {
+           display: inline-block;
+        }
         #songTime {
             width: 100%;
+        }
 #clips {
 display: flex;
 flex-direction: column;
+}
         a {
             color: rgb(97, 97, 255);
+        }
         #songTime {
             font-size: 27px;
+        }
         `;
+    }
     render() {
         return html`
 <div>
   <div><input id="songTime" type="range" @input="${this.userMovedSongTime}" min="0" max="0" step=".001"></div>
   <div>${this.musicState.song}</div>
   <div>showing song time ${rounding(this.songTime, 3)} (${rounding(this.musicState.t, 3)})</div>
     <input id="songTime" type="range"
            .value="${this.songTime}"
            @input="${this.userMovedSongTime}"
            min="0" max="0" step=".001"></div>
   <div><a href="${this.musicState.song}">${this.musicState.song}</a></div>
   <div id="songTime">showing song time ${rounding(this.songTime, 3)}</div>
 <div>clips:</div>
 <div id="clips">
    ${this.players}
 </div>
   <div>
     <button @click="${this.onClipsChanged}">Refresh clips for song</button>
 </div>
 `;
+    }
+}
 customElements.define('light9-vidref-replay-stack', Light9VidrefReplayStack);

light9/web/light9-vidref-replay.js

➞

Show inline comments

 import { LitElement, TemplateResult, html, css } from '/node_modules/lit-element/lit-element.js';
 import debug from '/lib/debug/debug-build-es6.js';
 import _ from '/lib/underscore/underscore-min-es6.js';
 import { rounding }  from '/node_modules/significant-rounding/index.js';
 const log = debug('replay');
 class Light9VidrefReplay extends LitElement {
     static get properties() {
         return {
             uri: { type: String },
             videoUrl: { type: String },
             songToVideo: { type: Object },
             videoTime: { type: Number },
             outVideoCurrentTime: { type: Number },
             timeErr: { type: Number },
             playRate: { type: Number }
         };
+    }
     setVideoTimeFromSongTime(songTime) {
         if (!this.songToVideo || !this.outVideo) {
     estimateRate() {
         const n = this.songToVideo.length;
         const x0 = Math.round(n * .3);
         const x1 = Math.round(n * .6);
         const pt0 = this.songToVideo[x0];
         const pt1 = this.songToVideo[x1];
         return (pt1[1] - pt0[1]) / (pt1[0] - pt0[0]);
+    }
     setVideoTimeFromSongTime(songTime, isPlaying) {
         if (!this.songToVideo || !this.outVideo || this.outVideo.readyState < 1) {
             return;
+        }
         const i = _.sortedIndex(this.songToVideo, [songTime],
                                 (row) => { return row[0]; });
         this.videoTime = this.songToVideo[Math.max(0, i - 1)][1];
         this.outVideoCurrentTime = this.outVideo.currentTime;
         if (isPlaying) {
             if (this.outVideo.paused) {
                 this.outVideo.play();
                 this.setRate(this.estimateRate());
+            }
             const err = this.outVideo.currentTime - this.videoTime;
             this.timeErr = err;
             if (Math.abs(err) > window.thresh) {
         this.outVideo.currentTime = this.videoTime;
                 const p = window.p;
                 if (err > 0) {
                     this.setRate(this.playRate - err * p);
                 } else {
                     this.setRate(this.playRate - err * p);
+                }
+            }
         } else {
             this.outVideo.pause();
             this.outVideoCurrentTime = this.outVideo.currentTime = this.videoTime;
             this.timeErr = 0;
+        }
+    }
     setRate(r) {
         this.playRate = Math.max(.1, Math.min(4, r));
         this.outVideo.playbackRate = this.playRate;
+    }
     firstUpdated() {
         this.outVideo = this.shadowRoot.querySelector('#replay');
         this.playRate = this.outVideo.playbackRate = 1.0;
+    }
     onDelete() {
         const u = new URL(window.location.href);
         u.pathname = '/vidref/clips'
         u.searchParams.set('uri', this.uri);
@@ @@ -39,26 +79,48 @@ class Light9VidrefReplay extends LitElem @@
         });
+    }
     static get styles() {
         return css`
         :host {
             margin: 5px;
             border: 2px solid #46a79f;
             display: flex;
             flex-direction: column;
+        }
         div {
             padding: 5px;
+        }
         .num {
             display: inline-block;
             width: 4em;
             color: #29ffa0;
+        }
         a {
             color: rgb(97, 97, 255);
+        }
         `;
+    }
     render() {
         return html`
   <video id="replay" src="${this.videoUrl}"></video>
 <div>
   <div><video id="replay" src="${this.videoUrl}"></video></div>
   <div>take is ${this.uri} (${Object.keys(this.songToVideo).length} frames)</div>
   <!-- a little canvas showing what coverage we have -->
   <div>video time is ${this.videoTime}</div>
     take is <a href="${this.uri}">${this.uri}</a>
     (${Object.keys(this.songToVideo).length} frames)
   <button @click="${this.onDelete}">Delete</button>
 </div>
   <!-- here, put a little canvas showing what coverage we have with the
        actual/goal time cursors -->
   <div>
     video time should be <span class="num">${this.videoTime} </span>
     actual = <span class="num">${rounding(this.outVideoCurrentTime, 3)}</span>,
     err = <span class="num">${rounding(this.timeErr, 3)} </span>
     rate = <span class="num">${rounding(this.playRate, 3)}</span>
   </div>
 `;
+    }
+}
 customElements.define('light9-vidref-replay', Light9VidrefReplay);
 window.thresh=.3
 window.p=.3

light9/web/style.css

➞

Show inline comments

@@ @@ -70,12 +70,30 @@ button a { @@
 .commands button.active {
     background: #a90707;
+}
 .key {
     color: #888;
+}
 div.keys {
     margin-top: 10px;
     padding: 5px;
+}
 .keyCap {
     color: #ccc;
     background: #525252;
     display: inline-block;
     border: 1px outset #b3b3b3;
     padding: 2px 3px;
     margin: 3px 0;
     font-size: 16px;
     box-shadow: 0.9px 0.9px 0px 2px #565656;
     border-radius: 2px;
+}
 .currentSong button {
     background: #a90707;
+}
 .timeRow {
     margin: 14px;
+}

0 comments (0 inline, 0 general)