Mercurial > code > home > repos > video

annotate video.py @ 44:239a83d46d48

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
make the server return urls with the (new) correct slashes
author drewp@bigasterisk.com
date Fri, 06 Dec 2024 01:01:05 -0800
parents b5b29f6ef5cb
children 882d0bb0f801
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
17
071943adf000 dnd a file or a url which we'll queue and fetch
drewp@bigasterisk.com
parents: 15
diff changeset
1 import asyncio
37
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
2 from functools import partial
17
071943adf000 dnd a file or a url which we'll queue and fetch
drewp@bigasterisk.com
parents: 15
diff changeset
3 import json
15
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
4 import logging
5
75b54be050bc show one subdir of archive at once, with folders and parents-breadcrumbs
drewp@bigasterisk.com
parents: 4
diff changeset
5
28
1e058bea3ac2 crash better when mongo is unreachable
drewp@bigasterisk.com
parents: 27
diff changeset
6 import uvicorn
3
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
7 from prometheus_client import Gauge
28
1e058bea3ac2 crash better when mongo is unreachable
drewp@bigasterisk.com
parents: 27
diff changeset
8 from sse_starlette.sse import EventSourceResponse
3
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
9 from starlette.applications import Starlette
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
10 from starlette.requests import Request
15
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
11 from starlette.responses import HTMLResponse, JSONResponse, Response
3
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
12 from starlette.routing import Route
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
13 from starlette_exporter import PrometheusMiddleware, handle_metrics
5
75b54be050bc show one subdir of archive at once, with folders and parents-breadcrumbs
drewp@bigasterisk.com
parents: 4
diff changeset
14
28
1e058bea3ac2 crash better when mongo is unreachable
drewp@bigasterisk.com
parents: 27
diff changeset
15 import dl_queue
15
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
16 from video_file_store import VideoFileStore
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
17 from video_ingest import VideoIngest
36
ed16fdbb3996 rewrite WIP. scan fs separately; store in db. thumbs are broken for now
drewp@bigasterisk.com
parents: 32
diff changeset
18 from mongo_required import open_mongo_or_die
37
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
19 import pymongo.database
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
20 import thumbnail
44
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
21 from urllib.parse import unquote
18
1b388ee5dd09 reformat
drewp@bigasterisk.com
parents: 17
diff changeset
22
3
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
23 logging.basicConfig(level=logging.DEBUG)
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
24 log = logging.getLogger()
17
071943adf000 dnd a file or a url which we'll queue and fetch
drewp@bigasterisk.com
parents: 15
diff changeset
25 logging.getLogger('sse_starlette').setLevel(logging.WARNING)
32
d83dbda8bccf quieter log
drewp@bigasterisk.com
parents: 28
diff changeset
26 logging.getLogger('pymongo').setLevel(logging.INFO)
0
drewp@bigasterisk.com
parents:
diff changeset
27
drewp@bigasterisk.com
parents:
diff changeset
28
3
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
29 def root(req):
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
30 return HTMLResponse("api")
0
drewp@bigasterisk.com
parents:
diff changeset
31
drewp@bigasterisk.com
parents:
diff changeset
32
6
ccfea3625cf6 render thumbs and display them (no video player at all atm)
drewp@bigasterisk.com
parents: 5
diff changeset
33 async def videos(req: Request) -> JSONResponse:
44
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
34 # either like /dir1/dir2/ or /dir1/dir2/vid1
38
0aea9e55899b hack in path router - dirs kind of work; putting a video in the path doesn't
drewp@bigasterisk.com
parents: 37
diff changeset
35 subdir = req.query_params.get('subdir', '/')
44
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
36
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
37 subdir = unquote(subdir)
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
38 webDirRelPath = subdir.rsplit('/', 1)[0] + '/'
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
39 autoplayRelPath = subdir if not subdir.endswith('/') else None
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
40
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
41 log.info(f'loading {webDirRelPath=} {autoplayRelPath=}')
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
42
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
43 assert webDirRelPath.startswith('/')
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
44 assert webDirRelPath.endswith('/')
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
45
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
46 resp: dict[str, object] = {
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
47 'webDirRelPath': webDirRelPath,
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
48 'dirLabel': webDirRelPath.strip('/').split('/')[-1], #todo
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
49 }
39
b5b29f6ef5cb cleanup and refactor
drewp@bigasterisk.com
parents: 38
diff changeset
50
44
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
51 d = webDirRelPath[1:-1] or '.'
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
52 log.info(f'loading {d=}')
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
53 vfInDir = list(store.findInDir(d))
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
54 resp["videos"] = [{
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
55 'webRelPath': '/' + vf.webRelPath,
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
56 'webDataPath': '/' + vf.webDataPath,
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
57 'label': vf.label,
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
58 } for vf in vfInDir]
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
59
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
60 if autoplayRelPath:
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
61 for vf in vfInDir:
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
62 if '/' + vf.webRelPath == autoplayRelPath:
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
63 resp['autoplay'] = {
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
64 'webRelPath': '/' + vf.webRelPath,
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
65 'webDataPath': '/' + vf.webDataPath,
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
66 'label': vf.label
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
67 }
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
68 break
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
69 else:
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
70 raise ValueError(f'{autoplayRelPath=} not in dir')
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
71
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
72 log.info(f'{subdir=}')
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
73 resp['subdirs'] = []
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
74 for s in store.findSubdirs(subdir.strip('/') or '/'):
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
75 resp['subdirs'].append({
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
76 'label': s['label'],
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
77 'path': '/' + s['path'] + '/',
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
78 })
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
79
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
80 return JSONResponse(resp)
4
c8a41359505c server provides video listing from disk
drewp@bigasterisk.com
parents: 3
diff changeset
81
15
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
82
20
cf6842952e13 serve api/folderTree
drewp@bigasterisk.com
parents: 18
diff changeset
83 def folderTree(req: Request) -> JSONResponse:
cf6842952e13 serve api/folderTree
drewp@bigasterisk.com
parents: 18
diff changeset
84 return JSONResponse(store.folderTree())
cf6842952e13 serve api/folderTree
drewp@bigasterisk.com
parents: 18
diff changeset
85
cf6842952e13 serve api/folderTree
drewp@bigasterisk.com
parents: 18
diff changeset
86
15
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
87 async def ingestVideoUrl(req: Request) -> Response:
23
9e56e86a6814 server supports downloading into a given folder
drewp@bigasterisk.com
parents: 20
diff changeset
88 folder = req.query_params['folder']
15
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
89 url = await req.body()
23
9e56e86a6814 server supports downloading into a given folder
drewp@bigasterisk.com
parents: 20
diff changeset
90 await svc.ingestUrl(url.decode('utf8'), folder)
15
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
91 return Response(status_code=202)
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
92
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
93
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
94 async def ingestVideoUpload(req: Request) -> Response:
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
95 name = req.query_params['name']
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
96 await svc.addContent(name, req.body())
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
97 return Response(status_code=200)
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
98
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
99
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
100 async def ingestQueue(req: Request) -> EventSourceResponse:
18
1b388ee5dd09 reformat
drewp@bigasterisk.com
parents: 17
diff changeset
101
17
071943adf000 dnd a file or a url which we'll queue and fetch
drewp@bigasterisk.com
parents: 15
diff changeset
102 async def g():
071943adf000 dnd a file or a url which we'll queue and fetch
drewp@bigasterisk.com
parents: 15
diff changeset
103 async for ev in svc.events():
071943adf000 dnd a file or a url which we'll queue and fetch
drewp@bigasterisk.com
parents: 15
diff changeset
104 yield json.dumps(ev)
18
1b388ee5dd09 reformat
drewp@bigasterisk.com
parents: 17
diff changeset
105
17
071943adf000 dnd a file or a url which we'll queue and fetch
drewp@bigasterisk.com
parents: 15
diff changeset
106 return EventSourceResponse(g())
15
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
107
39
b5b29f6ef5cb cleanup and refactor
drewp@bigasterisk.com
parents: 38
diff changeset
108
37
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
109 def getDiskPath(fs, webRelPath):
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
110 doc = fs.find_one({'webRelPath': webRelPath})
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
111 if doc is None:
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
112 raise ValueError
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
113 return doc['diskPath']
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
114
39
b5b29f6ef5cb cleanup and refactor
drewp@bigasterisk.com
parents: 38
diff changeset
115
b5b29f6ef5cb cleanup and refactor
drewp@bigasterisk.com
parents: 38
diff changeset
116 async def getThumbnail(db: pymongo.database.Database,
b5b29f6ef5cb cleanup and refactor
drewp@bigasterisk.com
parents: 38
diff changeset
117 req: Request) -> Response:
44
239a83d46d48 make the server return urls with the (new) correct slashes
drewp@bigasterisk.com
parents: 39
diff changeset
118 webRelPath = req.query_params['webRelPath'].lstrip('/')
37
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
119 fs = db.get_collection('fs')
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
120 diskPath = getDiskPath(fs, webRelPath)
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
121 th = db.get_collection('thumb')
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
122 async with asyncio.timeout(10):
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
123 data = await thumbnail.getThumbnailData(th, diskPath)
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
124 return Response(content=data, media_type='image/jpeg')
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
125
15
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
126
36
ed16fdbb3996 rewrite WIP. scan fs separately; store in db. thumbs are broken for now
drewp@bigasterisk.com
parents: 32
diff changeset
127 db = open_mongo_or_die().get_database('video')
ed16fdbb3996 rewrite WIP. scan fs separately; store in db. thumbs are broken for now
drewp@bigasterisk.com
parents: 32
diff changeset
128 store = VideoFileStore(db.get_collection('fs'))
15
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
129 svc = VideoIngest(store)
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
130
18
1b388ee5dd09 reformat
drewp@bigasterisk.com
parents: 17
diff changeset
131
4
c8a41359505c server provides video listing from disk
drewp@bigasterisk.com
parents: 3
diff changeset
132 def main():
3
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
133
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
134 app = Starlette(
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
135 debug=True,
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
136 routes=[
4
c8a41359505c server provides video listing from disk
drewp@bigasterisk.com
parents: 3
diff changeset
137 Route('/video/api/', root),
c8a41359505c server provides video listing from disk
drewp@bigasterisk.com
parents: 3
diff changeset
138 Route('/video/api/videos', videos),
20
cf6842952e13 serve api/folderTree
drewp@bigasterisk.com
parents: 18
diff changeset
139 Route('/video/api/folderTree', folderTree),
15
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
140 Route('/video/api/ingest/videoUpload',
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
141 ingestVideoUpload,
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
142 methods=['POST']),
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
143 Route('/video/api/ingest/videoUrl',
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
144 ingestVideoUrl,
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
145 methods=['POST']),
53d99454f394 support dropping url or file
drewp@bigasterisk.com
parents: 6
diff changeset
146 Route('/video/api/ingest/queue', ingestQueue),
37
7cacfae58430 thumbnails rewrite - store in db; don't use YT-provided pics for now
drewp@bigasterisk.com
parents: 36
diff changeset
147 Route('/video/api/thumbnail', partial(getThumbnail, db)),
3
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
148 ],
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
149 )
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
150
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
151 app.add_middleware(PrometheusMiddleware, app_name='video_api')
25
bf860a23d0c1 deployment and metrics
drewp@bigasterisk.com
parents: 23
diff changeset
152 app.add_route("/video/api/metrics", handle_metrics)
27
27a754f8e8f8 deployment fixes
drewp@bigasterisk.com
parents: 25
diff changeset
153 app.add_route("/metrics", handle_metrics)
18
1b388ee5dd09 reformat
drewp@bigasterisk.com
parents: 17
diff changeset
154
17
071943adf000 dnd a file or a url which we'll queue and fetch
drewp@bigasterisk.com
parents: 15
diff changeset
155 app.state.processTask = asyncio.create_task(dl_queue.process())
4
c8a41359505c server provides video listing from disk
drewp@bigasterisk.com
parents: 3
diff changeset
156 return app
3
ee55ed10faec start py service deployment
drewp@bigasterisk.com
parents: 0
diff changeset
157
39
b5b29f6ef5cb cleanup and refactor
drewp@bigasterisk.com
parents: 38
diff changeset
158
28
1e058bea3ac2 crash better when mongo is unreachable
drewp@bigasterisk.com
parents: 27
diff changeset
159 uvicorn.run(main,
1e058bea3ac2 crash better when mongo is unreachable
drewp@bigasterisk.com
parents: 27
diff changeset
160 host="0.0.0.0",
1e058bea3ac2 crash better when mongo is unreachable
drewp@bigasterisk.com
parents: 27
diff changeset
161 port=8004,
1e058bea3ac2 crash better when mongo is unreachable
drewp@bigasterisk.com
parents: 27
diff changeset
162 log_level=logging.INFO,
1e058bea3ac2 crash better when mongo is unreachable
drewp@bigasterisk.com
parents: 27
diff changeset
163 factory=True)