view scobot/index/download_tasks.py @ 11:6622bacb0b84

first pass at reorg
author drewp@bigasterisk.com
date Thu, 11 Jul 2024 18:15:44 -0700
parents flow/download.py@13438795d896
children 7a87ba2f00d9
line wrap: on
line source

import datetime
import time

import httpx
from prefect import task
from prefect.artifacts import create_link_artifact

from scobot.local_types import Url


@task(
    task_run_name=lambda: f'getHttp-{int(time.time())}',
    cache_key_fn=lambda _, args: f'getHttp-{args["url"]}',
    cache_expiration=datetime.timedelta(seconds=86400),
    tags=['city'],  # todo ratelimit based on tag
)
def getCityMutableJson(url: Url):
    create_link_artifact("get", url)
    req = httpx.get(url)  # todo async
    req.raise_for_status()
    return req.json()


@task(task_run_name=lambda: f'getHttp-{int(time.time())}',
      cache_key_fn=lambda _, args: f'getHttp-{args["url"]}',
      tags=['city'])
def getCityPermanent(url: Url) -> str:
    create_link_artifact("get", url)
    req = httpx.get(url)
    req.raise_for_status()
    return req.text


@task
def getYoutubePermanent(url: str):
    time.sleep(5)
    return 'video' * 10000