Mercurial > code > home > repos > background_loop

annotate background_loop.py @ 25:a97a1b4f5950 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
release 1.7.0
author drewp@bigasterisk.com
date Thu, 24 Nov 2022 20:39:38 -0800
parents a1ed58edfccc
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
9
263ccb9e00df port to asyncio and pdm
drewp@bigasterisk.com
parents: 0
diff changeset
1 # dev copy
263ccb9e00df port to asyncio and pdm
drewp@bigasterisk.com
parents: 0
diff changeset
2 import asyncio
263ccb9e00df port to asyncio and pdm
drewp@bigasterisk.com
parents: 0
diff changeset
3 import logging
0
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
4 import time
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
5 import traceback
16
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
6 from typing import Any, Awaitable, Callable, Union
9
263ccb9e00df port to asyncio and pdm
drewp@bigasterisk.com
parents: 0
diff changeset
7
263ccb9e00df port to asyncio and pdm
drewp@bigasterisk.com
parents: 0
diff changeset
8 from prometheus_client import Gauge, Summary
0
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
9
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
10 log = logging.getLogger()
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
11
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
12
9
263ccb9e00df port to asyncio and pdm
drewp@bigasterisk.com
parents: 0
diff changeset
13 # throw this away (when net_routes is rewritten)
0
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
14 def loop_forever_sync(func, sleep_period, up_metric):
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
15 first_run = True
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
16 while True:
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
17 try:
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
18 func(first_run)
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
19 up_metric.set(1)
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
20 first_run = False
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
21 except Exception as ex:
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
22 log.error(ex)
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
23 traceback.print_exc()
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
24 up_metric.set(0)
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
25 time.sleep(sleep_period)
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
26
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
27
16
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
28 _created = []
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
29
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
30 # todo: some tricky typing
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
31 F_RET = Any
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
32 F_KWARGS = Any
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
33
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
34 UserAsyncFunc = Callable[..., # always called with at least kwarg first_run=bool
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
35 Awaitable[F_RET]]
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
36 UserSyncFunc = Callable[..., # see above
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
37 F_RET]
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
38 UserFunc = Union[UserAsyncFunc, UserSyncFunc]
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
39
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
40
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
41 class Loop:
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
42
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
43 def __init__(
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
44 self,
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
45 func: UserFunc,
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
46 sleep_period: float,
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
47 metric_prefix: str,
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
48 ):
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
49 self.func = func
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
50 self.sleep_period = sleep_period
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
51
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
52 self.up_metric = Gauge(f'{metric_prefix}_up', 'not erroring')
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
53 self.call_metric = Summary(f'{metric_prefix}_calls', 'calls')
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
54 self.lastSuccessRun = 0
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
55 self.everSucceeded = False
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
56
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
57 async def call_func(self, first_run: bool, call_kwargs={}) -> F_RET:
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
58 with self.call_metric.time():
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
59 if asyncio.iscoroutinefunction(self.func):
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
60 ret = await self.func(first_run=first_run, **call_kwargs)
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
61 else:
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
62 ret = self.func(first_run=first_run, **call_kwargs)
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
63 return ret
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
64
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
65 async def _run(self):
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
66 self.first_run = True
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
67 while True:
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
68 await self._runOne()
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
69 await asyncio.sleep(self.sleep_period)
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
70
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
71 async def runNow(self, **kwargs: F_KWARGS) -> F_RET:
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
72 """unlike background runs, you get to pass more args and get your func's
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
73 return value.
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
74 """
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
75 return await self._runOne(call_kwargs=kwargs)
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
76
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
77 async def _runOne(self, call_kwargs={}):
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
78 try:
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
79 result = await self.call_func(self.first_run, call_kwargs)
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
80 self.lastSuccessRun = time.time()
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
81 self.up_metric.set(1)
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
82 self.everSucceeded = True
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
83 self.first_run = False
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
84 except Exception as ex:
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
85 log.error(ex)
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
86 traceback.print_exc()
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
87 self.up_metric.set(0)
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
88 result = None
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
89 # todo: something that reveals error ratio
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
90 return result
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
91
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
92
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
93 def loop_forever(
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
94 func: UserFunc,
9
263ccb9e00df port to asyncio and pdm
drewp@bigasterisk.com
parents: 0
diff changeset
95 sleep_period: float,
16
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
96 metric_prefix='background_loop',
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
97 up_metric=None,
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
98 call_metric=None,
9
263ccb9e00df port to asyncio and pdm
drewp@bigasterisk.com
parents: 0
diff changeset
99 ):
0
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
100 """
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
101 sleep_period is the sleep time after however long func takes to run
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
102 """
16
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
103 if up_metric is not None or call_metric is not None:
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
104 raise NotImplementedError('remove old-style metrics')
0
a340a9bdaedb factor this out from wherever it was
drewp@bigasterisk.com
parents:
diff changeset
105
16
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
106 loop = Loop(func, sleep_period, metric_prefix)
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
107 _created.append(asyncio.create_task(loop._run()))
a1ed58edfccc does its own task; returns Loop that you can query for status and call runNow on; rewrote metrics naming
drewp@bigasterisk.com
parents: 9
diff changeset
108 return loop