Mercurial > code > home > repos > sco-bot
view scobot/index/download_tasks.py @ 11:6622bacb0b84
first pass at reorg
author | drewp@bigasterisk.com |
---|---|
date | Thu, 11 Jul 2024 18:15:44 -0700 |
parents | flow/download.py@13438795d896 |
children | 7a87ba2f00d9 |
line wrap: on
line source
import datetime import time import httpx from prefect import task from prefect.artifacts import create_link_artifact from scobot.local_types import Url @task( task_run_name=lambda: f'getHttp-{int(time.time())}', cache_key_fn=lambda _, args: f'getHttp-{args["url"]}', cache_expiration=datetime.timedelta(seconds=86400), tags=['city'], # todo ratelimit based on tag ) def getCityMutableJson(url: Url): create_link_artifact("get", url) req = httpx.get(url) # todo async req.raise_for_status() return req.json() @task(task_run_name=lambda: f'getHttp-{int(time.time())}', cache_key_fn=lambda _, args: f'getHttp-{args["url"]}', tags=['city']) def getCityPermanent(url: Url) -> str: create_link_artifact("get", url) req = httpx.get(url) req.raise_for_status() return req.text @task def getYoutubePermanent(url: str): time.sleep(5) return 'video' * 10000