Mercurial > code > home > repos > sco-bot

comparison scobot/index/download_tasks.py @ 11:6622bacb0b84

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first pass at reorg
author drewp@bigasterisk.com
date Thu, 11 Jul 2024 18:15:44 -0700
parents flow/download.py@13438795d896
children 7a87ba2f00d9
comparison
equal deleted inserted replaced
10:13438795d896 11:6622bacb0b84
1 import datetime
2 import time
3
4 import httpx
5 from prefect import task
6 from prefect.artifacts import create_link_artifact
7
8 from scobot.local_types import Url
9
10
11 @task(
12 task_run_name=lambda: f'getHttp-{int(time.time())}',
13 cache_key_fn=lambda _, args: f'getHttp-{args["url"]}',
14 cache_expiration=datetime.timedelta(seconds=86400),
15 tags=['city'], # todo ratelimit based on tag
16 )
17 def getCityMutableJson(url: Url):
18 create_link_artifact("get", url)
19 req = httpx.get(url) # todo async
20 req.raise_for_status()
21 return req.json()
22
23
24 @task(task_run_name=lambda: f'getHttp-{int(time.time())}',
25 cache_key_fn=lambda _, args: f'getHttp-{args["url"]}',
26 tags=['city'])
27 def getCityPermanent(url: Url) -> str:
28 create_link_artifact("get", url)
29 req = httpx.get(url)
30 req.raise_for_status()
31 return req.text
32
33
34 @task
35 def getYoutubePermanent(url: str):
36 time.sleep(5)
37 return 'video' * 10000