Mercurial > code > home > repos > sco-bot
view scobot/index/download_tasks.py @ 15:6ed25bcaaf1f
add prefect and rebuild flow to k8s
author | drewp@bigasterisk.com |
---|---|
date | Fri, 19 Jul 2024 00:30:47 -0700 |
parents | 6622bacb0b84 |
children | 7a87ba2f00d9 |
line wrap: on
line source
import datetime import time import httpx from prefect import task from prefect.artifacts import create_link_artifact from scobot.local_types import Url @task( task_run_name=lambda: f'getHttp-{int(time.time())}', cache_key_fn=lambda _, args: f'getHttp-{args["url"]}', cache_expiration=datetime.timedelta(seconds=86400), tags=['city'], # todo ratelimit based on tag ) def getCityMutableJson(url: Url): create_link_artifact("get", url) req = httpx.get(url) # todo async req.raise_for_status() return req.json() @task(task_run_name=lambda: f'getHttp-{int(time.time())}', cache_key_fn=lambda _, args: f'getHttp-{args["url"]}', tags=['city']) def getCityPermanent(url: Url) -> str: create_link_artifact("get", url) req = httpx.get(url) req.raise_for_status() return req.text @task def getYoutubePermanent(url: str): time.sleep(5) return 'video' * 10000