light9/web/light9-vidref-live.js LitElement for live video frames

light9/web/light9-vidref-playback.js LitElement for video playback

    '/dev/v4l/by-id/usb-Bison_HD_Webcam_200901010001-video-index0'

#    '/dev/v4l/by-id/usb-Generic_FULL_HD_1080P_Webcam_200901010001-video-index0'

)

    def onFrame(self, cf: videorecorder.CaptureFrame):

        if cf is None: return

        cf.img.save(output, 'jpeg', quality=80)

                f't={cf.t}',

#graph = SyncedGraph(networking.rdfdb.url, "vidref")

outVideos = videorecorder.FramesToVideoFiles(pipeline.liveImages)

#outVideos.save('/tmp/mov1')

import os

from bisect import bisect_left

from rdflib import URIRef

import sys

sys.path.append(

    '/home/drewp/Downloads/moviepy/lib/python2.7/site-packages')  # for moviepy

from moviepy.video.io.ffmpeg_writer import FFMPEG_VideoWriter

from moviepy.video.io.ffmpeg_reader import FFMPEG_VideoReader

class _ResourceDir(object):

    """the disk files for a resource"""

    def __init__(self, root, uri):

        self.root, self.uri = root, uri

        u = self.uri.replace('http://', '').replace('/', '_')

        self.topDir = os.path.join(self.root, u)

        try:

            os.makedirs(self.topDir)

        except OSError:

            pass

    def videoPath(self):

        return os.path.join(self.topDir, 'video.avi')

    def indexPath(self):

        return os.path.join(self.topDir, 'frame_times')

class Writer(object):

    """saves a video of a resource, receiving a frame at a time. Frame timing does not have to be regular."""

    def __init__(self, root, uri):

        self.rd = _ResourceDir(root, uri)

        self.ffmpegWriter = None  # lazy since we don't know the size yet

        self.index = open(self.rd.indexPath(), 'w')

        self.framesWritten = 0

    def save(self, t, img):

        if self.ffmpegWriter is None:

            self.ffmpegWriter = FFMPEG_VideoWriter(

                filename=self.rd.videoPath(),

                size=img.size,

                fps=10,  # doesn't matter, just for debugging playbacks

                codec='libx264')

        self.ffmpegWriter.write_frame(img)

        self.framesWritten = self.framesWritten + 1

        self.index.write('%d %g\n' % (self.framesWritten, t))

    def close(self):

        if self.ffmpegWriter is not None:

            self.ffmpegWriter.close()

        self.index.close()

class Reader(object):

    def __init__(self, resourceDir):

        self.timeFrame = []

        for line in open(resourceDir.indexPath()):

            f, t = line.strip().split()

            self.timeFrame.append((float(t), int(f)))

        self._reader = FFMPEG_VideoReader(resourceDir.videoPath())

    def getFrame(self, t):

        i = bisect_left(self.timeFrame, (t, None))

        i = min(i, len(self.timeFrame) - 1)

        f = self.timeFrame[i][1]

        return self._reader.get_frame(f)

class MultiReader(object):

    """loads the nearest existing frame of a resource's video. Supports random access of multiple resources."""

    def __init__(self, root):

        self.root = root

        # these should cleanup when they haven't been used in a while

        self.readers = {}  # uri: Reader

    def getFrame(self, uri, t):

        if uri not in self.readers:

            #self.readers.close all and pop them

            self.readers[uri] = Reader(_ResourceDir(self.root, uri))

        return self.readers[uri].getFrame(t)

if __name__ == '__main__':

    from PIL import Image

    take = URIRef(

        'http://light9.bigasterisk.com/show/dance2015/song10/1434249076/')

    if 0:

        w = Writer('/tmp/ms', take)

        for fn in sorted(

                os.listdir(

                    '/home/drewp/light9-vidref/play-light9.bigasterisk.com_show_dance2015_song10/1434249076'

                )):

            t = float(fn.replace('.jpg', ''))

            jpg = Image.open(

                '/home/drewp/light9-vidref/play-light9.bigasterisk.com_show_dance2015_song10/1434249076/%08.03f.jpg'

                % t)

            jpg = jpg.resize((450, 176))

            w.save(t, jpg)

        w.close()

    else:

        r = MultiReader('/tmp/ms')

        print((r.getFrame(take, 5.6)))

import time, logging, os, traceback, sys

from dataclasses import dataclass

from twisted.internet import defer, threads

from typing import Set, Optional

import moviepy.editor

import numpy

from light9.ascoltami.musictime_client import MusicTime

from light9.newtypes import Song

@dataclass(frozen=True)

class CaptureFrame:

    img: Image

    song: Song

    t: float

    isPlaying: bool

class FramesToVideoFiles:

    """

    nextWriteAction: 'ignore'

    currentOutputClip: None

    (frames come in for new video)

    nextWriteAction: 'saveFrame'

    currentOutputClip: new VideoClip

    (many frames)

    (music stops or song changes)

    nextWriteAction: 'close'

    currentOutputClip: None

    nextWriteAction: 'ignore'

    """

    def __init__(self, frames: BehaviorSubject):

        self.frames = frames

        self.nextImg = None

        self.currentOutputClip = None

        self.currentOutputSong = None

        self.nextWriteAction = 'ignore'

        self.frames.subscribe(on_next=self.onFrame)

    def onFrame(self, cf: Optional[CaptureFrame]):

        if cf is None:

            return

        self.nextImg = cf

        if self.currentOutputClip is None and cf.isPlaying:

            # start up

            self.nextWriteAction = 'saveFrames'

            self.currentOutputSong = cf.song

            self.save('/tmp/out%s' % time.time())

        elif self.currentOutputClip and cf.isPlaying:

            self.nextWriteAction = 'saveFrames'

            # continue recording this

        elif self.currentOutputClip is None and not cf.isPlaying:

            self.nextWriteAction  = 'notWritingClip'

            pass # continue waiting

        elif self.currentOutputClip and not cf.isPlaying or self.currentOutputSong != cf.song:

            # stop

            self.nextWriteAction = 'close'

        else:

            raise NotImplementedError        

    def save(self, outBase):

        """

        receive frames (infinite) and wall-to-song times (stream ends with

        the song), and write a video file and a frame map

        """

        return threads.deferToThread(self._bg_save, outBase)

    def _bg_save(self, outBase):

        self.frameMap = open(outBase + '.timing', 'wt')

        # (immediately calls make_frame)

        self.currentOutputClip = moviepy.editor.VideoClip(

            self._bg_make_frame, duration=999.)

        self.currentOutputClip.fps = 5

        log.info(f'write_videofile {outBase} start')

        try:

            self.currentOutputClip.write_videofile(

                outBase + '.mp4',

                audio=False, preset='ultrafast', verbose=True, bitrate='150000')

        except (StopIteration, RuntimeError):

            pass

        log.info('write_videofile done')

        self.currentOutputClip = None

    def _bg_make_frame(self, video_time_secs):

        if self.nextWriteAction == 'close':

            raise StopIteration # the one in write_videofile

        # should be a queue to miss fewer frames

        while self.nextImg is None:

            time.sleep(.03)

        cf, self.nextImg = self.nextImg, None

        self.frameMap.write(

            f'video {video_time_secs:g} = song {cf.t:g}\n')

        self.frameMap.flush()

        return numpy.asarray(cf.img)

        self.musicTime = MusicTime(pollCurvecalc=False)

        self.liveImages: BehaviorSubject[Optional[CaptureFrame]] = BehaviorSubject(None)

        size = [640, 480]

        pipeStr = (

            #f"v4l2src device=\"{dev}\""

            f'autovideosrc'

            f" ! videoconvert"

            f" ! appsink emit-signals=true max-buffers=1 drop=true name=end0 caps=video/x-raw,format=RGB,width={size[0]},height={size[1]}"

            )

        log.info('gst pipeline is recording video')

                img = img.crop((0, 100, 640, 380))

            # could get gst's frame time and pass it to getLatest

            latest = self.musicTime.getLatest()

            if 'song' in latest:

                self.liveImages.on_next(

                    CaptureFrame(img, Song(latest['song']),

                                 latest['t'], latest['playing']))

moviepy==1.0.0