view scobot/index/download_tasks.py @ 16:7a87ba2f00d9

reformat, fix some types, make more async
author drewp@bigasterisk.com
date Fri, 19 Jul 2024 00:49:38 -0700
parents 6622bacb0b84
children
line wrap: on
line source

import datetime
import time

import httpx
from prefect import task
from prefect.artifacts import create_link_artifact

from scobot.local_types import Url


@task(
    task_run_name=lambda: f'getHttp-{int(time.time())}',
    cache_key_fn=lambda _, args: f'getHttp-{args["url"]}',
    cache_expiration=datetime.timedelta(seconds=86400),
    tags=['city'],  # todo ratelimit based on tag
)
async def getCityMutableJson(url: Url):
    await create_link_artifact("get", url)
    req = httpx.get(url)  # todo async
    req.raise_for_status()
    return req.json()


@task(task_run_name=lambda: f'getHttp-{int(time.time())}',
      cache_key_fn=lambda _, args: f'getHttp-{args["url"]}',
      tags=['city'])
async def getCityPermanent(url: Url) -> str:
    await create_link_artifact("get", url)
    req = httpx.get(url)
    req.raise_for_status()
    return req.text


@task
def getYoutubePermanent(url: str):
    time.sleep(5)
    return 'video' * 10000