changeset 1939:6f49dc917aa3

start vidref web version. v4l camera frames to web page is working Ignore-this: 34bcc3b6149a1a3bed31aa5f32a4ddc6
author Drew Perttula <drewp@bigasterisk.com>
date Mon, 03 Jun 2019 09:50:29 +0000
parents 60c5acfe4f5a
children cce016abe31e
files bin/vidref light9/vidref/replay.py light9/vidref/setup.html light9/vidref/videorecorder.py light9/web/light9-vidref-live.js requirements.txt
diffstat 6 files changed, 219 insertions(+), 152 deletions(-) [+]
line wrap: on
line diff
--- a/bin/vidref	Sun Jun 02 21:36:57 2019 +0000
+++ b/bin/vidref	Mon Jun 03 09:50:29 2019 +0000
@@ -1,22 +1,29 @@
 #!bin/python
+"""
+Camera images of the stage. View live on a web page and also save
+them to disk. Retrieve images based on the song and time that was
+playing when they were taken. Also, save snapshot images to a place
+they can be used again as thumbnails of effects.
+
+bin/vidref main
+light9/vidref/videorecorder.py capture frames and save them
+light9/vidref/replay.py backend for vidref.js playback element- figures out which frames go with the current song and time
+light9/vidref/index.html web ui for watching current stage and song playback
+light9/vidref/setup.html web ui for setup of camera params and frame crop
+light9/web/vidref.js LitElement for video playback
+
+"""
 from run_local import log
-import sys
-sys.path.append('/usr/lib/python2.7/dist-packages')  # For gtk
-from twisted.internet import gtk2reactor
-gtk2reactor.install()
+
 from twisted.internet import reactor, defer
-import gobject
-gobject.threads_init()
-import sys, logging, optparse, json
+
+import logging, optparse, json, base64
 import cyclone.web, cyclone.httpclient, cyclone.websocket
 from light9 import networking
-from light9.vidref.main import Gui
 from light9.vidref.replay import snapshotDir
+from light9.vidref import videorecorder
 from rdfdb.syncedgraph import SyncedGraph
-
-# find replay dirs correctly. show multiple
-# replays. trash. reorder/pin. dump takes that are too short; they're
-# just from seeking
+from io import BytesIO
 
 parser = optparse.OptionParser()
 parser.add_option("-v", "--verbose", action="store_true", help="logging.DEBUG")
@@ -47,6 +54,33 @@
             raise
 
 
+pipeline = videorecorder.GstSource(
+    '/dev/v4l/by-id/usb-Generic_FULL_HD_1080P_Webcam_200901010001-video-index0')
+
+
+class Live(cyclone.websocket.WebSocketHandler):
+
+    def connectionMade(self, *args, **kwargs):
+        pipeline.liveImages.subscribe(on_next=self.onFrame)
+
+    def connectionLost(self, reason):
+        0  #self.subj.dispose()
+
+    def onFrame(self, t_img):
+        t, img = t_img
+        if img is None: return
+        output = BytesIO()
+        img.save(output, 'jpeg', quality=80)
+
+        self.sendMessage(
+            json.dumps({
+                'jpeg':
+                base64.b64encode(output.getvalue()).decode('ascii'),
+                'description':
+                f't={t}',
+            }))
+
+
 class SnapshotPic(cyclone.web.StaticFileHandler):
     pass
 
@@ -63,24 +97,28 @@
 
 graph = SyncedGraph(networking.rdfdb.url, "vidref")
 
-gui = Gui(graph)
-
 port = networking.vidref.port
 reactor.listenTCP(
     port,
-    cyclone.web.Application(handlers=[
-        (r'/()', cyclone.web.StaticFileHandler, {
-            'path': 'light9/vidref',
-            'default_filename': 'vidref.html'
-        }),
-        (r'/snapshot', Snapshot),
-        (r'/snapshot/(.*)', SnapshotPic, {
-            "path": snapshotDir()
-        }),
-        (r'/time', Time),
-    ],
-                            debug=True,
-                            gui=gui))
+    cyclone.web.Application(
+        handlers=[
+            (r'/()', cyclone.web.StaticFileHandler, {
+                'path': 'light9/vidref',
+                'default_filename': 'vidref.html'
+            }),
+            (r'/setup/()', cyclone.web.StaticFileHandler, {
+                'path': 'light9/vidref',
+                'default_filename': 'setup.html'
+            }),
+            (r'/setup/live', Live),
+            (r'/snapshot', Snapshot),
+            (r'/snapshot/(.*)', SnapshotPic, {
+                "path": snapshotDir()
+            }),
+            (r'/time', Time),
+        ],
+        debug=True,
+    ))
 log.info("serving on %s" % port)
 
 reactor.run()
--- a/light9/vidref/replay.py	Sun Jun 02 21:36:57 2019 +0000
+++ b/light9/vidref/replay.py	Mon Jun 03 09:50:29 2019 +0000
@@ -1,4 +1,4 @@
-import os, gtk, shutil, logging, time
+import os, shutil, logging, time
 from bisect import bisect_left
 from decimal import Decimal
 log = logging.getLogger()
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/light9/vidref/setup.html	Mon Jun 03 09:50:29 2019 +0000
@@ -0,0 +1,20 @@
+<!doctype html>
+<html>
+  <head>
+    <title>vidref setup</title>
+    <meta charset="utf-8" />
+    <script src="/node_modules/@webcomponents/webcomponentsjs/webcomponents-lite.js"></script>
+    <link rel="stylesheet" href="/style.css">
+    <script src="/lib/jquery/dist/jquery.slim.min.js"></script>
+
+    <script src="/websocket.js"></script>
+    <script type="module"  src="/light9-vidref-live.js"></script>
+
+  </head>
+  <body>
+    Live:
+    <light9-vidref-live></light9-vidref-live>
+
+    
+  </body>
+</html>
--- a/light9/vidref/videorecorder.py	Sun Jun 02 21:36:57 2019 +0000
+++ b/light9/vidref/videorecorder.py	Mon Jun 03 09:50:29 2019 +0000
@@ -1,21 +1,101 @@
-import pygst
-pygst.require("0.10")
-import gst, gobject, time, logging, os, traceback
-import gtk
+import sys
+import gi
+gi.require_version('Gst', '1.0')
+gi.require_version('GstBase', '1.0')
+from gi.repository import Gst
+from rx.subjects import BehaviorSubject
+
+import time, logging, os, traceback
 from PIL import Image
-from threading import Thread
 from twisted.internet import defer
-from queue import Queue, Empty
+from queue import Queue
 from light9.vidref.replay import framerate, songDir, takeDir, snapshotDir
+from typing import Set
+
+from IPython.core import ultratb
+sys.excepthook = ultratb.FormattedTB(mode='Verbose',
+                                     color_scheme='Linux',
+                                     call_pdb=1)
+
 log = logging.getLogger()
 
 
-class Pipeline(object):
+class GstSource:
+
+    def __init__(self, dev):
+        """
+        make new gst pipeline
+        """
+        Gst.init(None)
+        self.liveImages = BehaviorSubject((0, None))
+
+        size = [800, 600]
+
+        log.info("new pipeline using device=%s" % dev)
+        
+        # using videocrop breaks the pipeline, may be this issue
+        # https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/issues/732
+        pipeStr = f"v4l2src device=\"{dev}\" ! videoconvert ! appsink emit-signals=true max-buffers=1 drop=true name=end0 caps=video/x-raw,format=RGB,width={size[0]},height={size[1]}"
+        log.info("pipeline: %s" % pipeStr)
+
+        self.pipe = Gst.parse_launch(pipeStr)
+
+        self.setupPipelineError(self.pipe, self.onError)
+
+        self.appsink = self.pipe.get_by_name('end0')
+        self.appsink.connect('new-sample', self.new_sample)
+
+        self.pipe.set_state(Gst.State.PLAYING)
+        log.info('recording video')
 
-    def __init__(self, liveVideoXid, musicTime, recordingTo):
-        self.musicTime = musicTime
-        self.liveVideoXid = liveVideoXid
-        self.recordingTo = recordingTo
+    def new_sample(self, appsink):
+        try:
+            sample = appsink.emit('pull-sample')
+            caps = sample.get_caps()
+            buf = sample.get_buffer()
+            (result, mapinfo) = buf.map(Gst.MapFlags.READ)
+            try:
+                img = Image.frombytes(
+                    'RGB', (caps.get_structure(0).get_value('width'),
+                            caps.get_structure(0).get_value('height')),
+                    mapinfo.data)
+                img = img.crop((0, 100, 800,  500))
+            finally:
+                buf.unmap(mapinfo)
+            self.liveImages.on_next((time.time(), img))
+        except Exception:
+            traceback.print_exc()
+        return Gst.FlowReturn.OK
+
+    def setupPipelineError(self, pipe, cb):
+        bus = pipe.get_bus()
+
+        def onBusMessage(bus, msg):
+
+            print('nusmsg', msg)
+            if msg.type == Gst.MessageType.ERROR:
+                _, txt = msg.parse_error()
+                cb(txt)
+            return True
+
+        # not working; use GST_DEBUG=4 to see errors
+        bus.add_watch(0, onBusMessage)
+        bus.connect('message', onBusMessage)
+
+    def onError(self, messageText):
+        if ('v4l2src' in messageText and
+            ('No such file or directory' in messageText or
+             'Resource temporarily unavailable' in messageText or
+             'No such device' in messageText)):
+            log.error(messageText)
+            os.abort()
+        else:
+            log.error("ignoring error: %r" % messageText)
+
+
+class oldPipeline(object):
+
+    def __init__(self):
         self.snapshotRequests = Queue()
 
         try:
@@ -41,80 +121,8 @@
         self.snapshotRequests.put(req)
         return d
 
-    def setInput(self, name):
-        sourcePipe = {
-            "auto": "autovideosrc name=src1",
-            "testpattern": "videotestsrc name=src1",
-            "dv": "dv1394src name=src1 ! dvdemux ! dvdec",
-            "v4l": "v4l2src device=/dev/video0 name=src1",
-        }[name]
 
-        cam = (
-            sourcePipe + " ! "
-            "videorate ! video/x-raw-yuv,framerate=%s/1 ! "
-            "videoscale ! video/x-raw-yuv,width=640,height=480;video/x-raw-rgb,width=320,height=240 ! "
-            "videocrop left=160 top=180 right=120 bottom=80 ! "
-            "queue name=vid" % framerate)
-
-        print(cam)
-        self.pipeline = gst.parse_launch(cam)
-
-        def makeElem(t, n=None):
-            e = gst.element_factory_make(t, n)
-            self.pipeline.add(e)
-            return e
-
-        sink = makeElem("xvimagesink")
-
-        def setRec(t):
-            # if you're selecting the text while gtk is updating it,
-            # you can get a crash in xcb_io
-            if getattr(self, '_lastRecText', None) == t:
-                return
-            with gtk.gdk.lock:
-                self.recordingTo.set_text(t)
-            self._lastRecText = t
-
-        recSink = VideoRecordSink(self.musicTime, setRec, self.snapshotRequests)
-        self.pipeline.add(recSink)
-
-        tee = makeElem("tee")
-
-        caps = makeElem("capsfilter")
-        caps.set_property('caps', gst.caps_from_string('video/x-raw-rgb'))
-
-        gst.element_link_many(self.pipeline.get_by_name("vid"), tee, sink)
-        gst.element_link_many(tee, makeElem("ffmpegcolorspace"), caps, recSink)
-        sink.set_xwindow_id(self.liveVideoXid)
-        self.pipeline.set_state(gst.STATE_PLAYING)
-
-    def setLiveVideo(self, on):
-
-        if on:
-            self.pipeline.set_state(gst.STATE_PLAYING)
-            # this is an attempt to bring the dv1394 source back, but
-            # it doesn't work right.
-            self.pipeline.get_by_name("src1").seek_simple(
-                gst.FORMAT_TIME, gst.SEEK_FLAG_FLUSH, 0 * gst.SECOND)
-        else:
-            self.pipeline.set_state(gst.STATE_READY)
-
-
-class VideoRecordSink(gst.Element):
-    _sinkpadtemplate = gst.PadTemplate("sinkpadtemplate", gst.PAD_SINK,
-                                       gst.PAD_ALWAYS, gst.caps_new_any())
-
-    def __init__(self, musicTime, updateRecordingTo, snapshotRequests):
-        gst.Element.__init__(self)
-        self.updateRecordingTo = updateRecordingTo
-        self.snapshotRequests = snapshotRequests
-        self.sinkpad = gst.Pad(self._sinkpadtemplate, "sink")
-        self.add_pad(self.sinkpad)
-        self.sinkpad.set_chain_function(self.chainfunc)
-        self.lastTime = 0
-
-        self.musicTime = musicTime
-
+'''
         self.imagesToSave = Queue()
         self.startBackgroundImageSaver(self.imagesToSave)
 
@@ -148,43 +156,4 @@
 
     def chainfunc(self, pad, buffer):
         position = self.musicTime.getLatest()
-
-        # if music is not playing and there's no pending snapshot
-        # request, we could skip the image conversions here.
-
-        try:
-            cap = buffer.caps[0]
-            #print "cap", (cap['width'], cap['height'])
-            img = Image.fromstring('RGB', (cap['width'], cap['height']),
-                                   buffer.data)
-            self.imagesToSave.put((position, img, buffer.timestamp))
-        except Exception:
-            traceback.print_exc()
-
-        return gst.FLOW_OK
-
-    def saveImg(self, position, img, bufferTimestamp):
-        if not position['song']:
-            return
-
-        t1 = time.time()
-        outDir = takeDir(songDir(position['song']), position['started'])
-        outFilename = "%s/%08.03f.jpg" % (outDir, position['t'])
-        if os.path.exists(outFilename):  # we're paused on one time
-            return
-
-        try:
-            os.makedirs(outDir)
-        except OSError:
-            pass
-
-        img.save(outFilename)
-
-        now = time.time()
-        log.info("wrote %s delay of %.2fms, took %.2fms", outFilename,
-                 (now - self.lastTime) * 1000, (now - t1) * 1000)
-        self.updateRecordingTo(outDir)
-        self.lastTime = now
-
-
-gobject.type_register(VideoRecordSink)
+'''
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/light9/web/light9-vidref-live.js	Mon Jun 03 09:50:29 2019 +0000
@@ -0,0 +1,39 @@
+import { LitElement, TemplateResult, html, css } from '/node_modules/lit-element/lit-element.js';
+
+
+class Light9VidrefLive extends LitElement {
+    
+    static get properties() {
+        return {
+            description: { type: String }
+        };
+    }
+
+    static get styles() {
+        return css`
+        :host {
+            border: 2px solid #46a79f;
+            display: inline-block;
+        }
+        `;
+    }
+
+    firstUpdated() {
+        const ws = reconnectingWebSocket('live', (msg) => {
+            this.shadowRoot.querySelector('#live').src = 'data:image/jpeg;base64,' + msg.jpeg;
+            this.description = msg.description;
+        });
+        
+    }
+
+    render() {
+        return html`
+<div>
+<div><img id="live" ></div>
+<div>${this.description}</div>
+</div>
+`;
+
+    }
+}
+customElements.define('light9-vidref-live', Light9VidrefLive);
--- a/requirements.txt	Sun Jun 02 21:36:57 2019 +0000
+++ b/requirements.txt	Mon Jun 03 09:50:29 2019 +0000
@@ -16,6 +16,7 @@
 pyusb==1.0.0
 rdflib==4.2.2
 requests==2.22.0
+rx==1.6.1
 scipy==1.3.0
 service_identity==18.1.0
 statprof==0.1.2