Mercurial > code > home > repos > sco-bot
view scobot/index/download_tasks.py @ 16:7a87ba2f00d9
reformat, fix some types, make more async
author | drewp@bigasterisk.com |
---|---|
date | Fri, 19 Jul 2024 00:49:38 -0700 |
parents | 6622bacb0b84 |
children |
line wrap: on
line source
import datetime import time import httpx from prefect import task from prefect.artifacts import create_link_artifact from scobot.local_types import Url @task( task_run_name=lambda: f'getHttp-{int(time.time())}', cache_key_fn=lambda _, args: f'getHttp-{args["url"]}', cache_expiration=datetime.timedelta(seconds=86400), tags=['city'], # todo ratelimit based on tag ) async def getCityMutableJson(url: Url): await create_link_artifact("get", url) req = httpx.get(url) # todo async req.raise_for_status() return req.json() @task(task_run_name=lambda: f'getHttp-{int(time.time())}', cache_key_fn=lambda _, args: f'getHttp-{args["url"]}', tags=['city']) async def getCityPermanent(url: Url) -> str: await create_link_artifact("get", url) req = httpx.get(url) req.raise_for_status() return req.text @task def getYoutubePermanent(url: str): time.sleep(5) return 'video' * 10000