Mercurial > code > home > repos > light9
annotate lib/background_loop.py @ 2216:acf1b68d031a
fancier background_loop reporting for faders
| author | drewp@bigasterisk.com |
|---|---|
| date | Tue, 23 May 2023 14:43:12 -0700 |
| parents | d8853f173568 |
| children | 436a1fdbfe4a |
| rev | line source |
|---|---|
| 2215 | 1 # dev copy |
| 2 import asyncio | |
| 3 import logging | |
| 4 import time | |
| 5 import traceback | |
| 6 from typing import Any, Awaitable, Callable, Union | |
| 7 | |
| 8 from prometheus_client import Gauge, Summary | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
9 from light9.recentfps import RecentFps |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
10 from braillegraph import horizontal_graph |
| 2215 | 11 |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
12 log = logging.getLogger('loop') |
| 2215 | 13 |
| 14 | |
| 15 # throw this away (when net_routes is rewritten) | |
| 16 def loop_forever_sync(func, sleep_period, up_metric): | |
| 17 first_run = True | |
| 18 while True: | |
| 19 try: | |
| 20 func(first_run) | |
| 21 up_metric.set(1) | |
| 22 first_run = False | |
| 23 except Exception as ex: | |
| 24 log.error(ex) | |
| 25 traceback.print_exc() | |
| 26 up_metric.set(0) | |
| 27 time.sleep(sleep_period) | |
| 28 | |
| 29 | |
| 30 _created = [] | |
| 31 | |
| 32 # todo: some tricky typing | |
| 33 F_RET = Any | |
| 34 F_KWARGS = Any | |
| 35 | |
| 36 UserAsyncFunc = Callable[..., # always called with at least kwarg first_run=bool | |
| 37 Awaitable[F_RET]] | |
| 38 UserSyncFunc = Callable[..., # see above | |
| 39 F_RET] | |
| 40 UserFunc = Union[UserAsyncFunc, UserSyncFunc] | |
| 41 | |
| 42 | |
| 43 class Loop: | |
| 44 | |
| 45 def __init__( | |
| 46 self, | |
| 47 func: UserFunc, | |
| 48 sleep_period: float, | |
| 49 metric_prefix: str, | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
50 extra_sleep_on_error: float = 2, |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
51 log_fps=False, |
| 2215 | 52 ): |
| 53 self.func = func | |
| 54 self.sleep_period = sleep_period | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
55 self.extra_sleep_on_error = extra_sleep_on_error |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
56 self.metric_prefix = metric_prefix |
| 2215 | 57 |
| 58 self.up_metric = Gauge(f'{metric_prefix}_up', 'not erroring') | |
| 59 self.call_metric = Summary(f'{metric_prefix}_calls', 'calls') | |
| 60 self.lastSuccessRun = 0 | |
| 61 self.everSucceeded = False | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
62 self.succeeding = False |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
63 self.fps = RecentFps() if log_fps else None |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
64 self.lastFpsLog = 0 |
| 2215 | 65 |
| 66 async def call_func(self, first_run: bool, call_kwargs={}) -> F_RET: | |
| 67 with self.call_metric.time(): | |
| 68 if asyncio.iscoroutinefunction(self.func): | |
| 69 ret = await self.func(first_run=first_run, **call_kwargs) | |
| 70 else: | |
| 71 ret = self.func(first_run=first_run, **call_kwargs) | |
| 72 return ret | |
| 73 | |
| 74 async def _run(self): | |
| 75 self.first_run = True | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
76 self.newlyFailing = True |
| 2215 | 77 while True: |
| 78 await self._runOne() | |
| 79 await asyncio.sleep(self.sleep_period) | |
| 80 | |
| 81 async def runNow(self, **kwargs: F_KWARGS) -> F_RET: | |
| 82 """unlike background runs, you get to pass more args and get your func's | |
| 83 return value. | |
| 84 """ | |
| 85 return await self._runOne(call_kwargs=kwargs) | |
| 86 | |
| 87 async def _runOne(self, call_kwargs={}): | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
88 now = time.time() |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
89 self._updateFps(now) |
| 2215 | 90 try: |
| 91 result = await self.call_func(self.first_run, call_kwargs) | |
| 92 self.lastSuccessRun = time.time() | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
93 if self.fps: |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
94 self.fps.mark() |
| 2215 | 95 self.up_metric.set(1) |
| 96 self.everSucceeded = True | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
97 self.newlyFailing = True |
| 2215 | 98 self.first_run = False |
| 99 except Exception as ex: | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
100 if self.newlyFailing: |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
101 traceback.print_exc() |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
102 self.newlyFailing = False |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
103 else: |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
104 log.error(ex) |
| 2215 | 105 self.up_metric.set(0) |
| 106 result = None | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
107 self.succeeding = False |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
108 |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
109 await asyncio.sleep(self.extra_sleep_on_error) |
| 2215 | 110 # todo: something that reveals error ratio |
| 111 return result | |
| 112 | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
113 def _updateFps(self, now: float): |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
114 if self.fps is None: |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
115 return |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
116 if now < self.lastFpsLog + 5: |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
117 return |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
118 d = self.fps() |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
119 y_hi = 1 / self.sleep_period |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
120 if not d: |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
121 return |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
122 pts = [int(min(4, y / y_hi * 4)) for y in d['recents']] |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
123 log.info(f'{self.metric_prefix} fps={d["average"]:3.1f} (req={y_hi:3.1f}) {horizontal_graph(pts)}') |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
124 self.lastFpsLog = now |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
125 |
| 2215 | 126 |
| 127 def loop_forever( | |
| 128 func: UserFunc, | |
| 129 sleep_period: float, | |
| 130 metric_prefix='background_loop', | |
| 131 up_metric=None, | |
| 132 call_metric=None, | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
133 extra_sleep_on_error=2, |
|
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
134 log_fps=False, |
| 2215 | 135 ): |
| 136 """ | |
| 137 sleep_period is the sleep time after however long func takes to run | |
| 138 """ | |
| 139 if up_metric is not None or call_metric is not None: | |
| 140 raise NotImplementedError('remove old-style metrics') | |
| 141 | |
|
2216
acf1b68d031a
fancier background_loop reporting for faders
drewp@bigasterisk.com
parents:
2215
diff
changeset
|
142 loop = Loop(func, sleep_period, metric_prefix, extra_sleep_on_error, log_fps) |
| 2215 | 143 _created.append(asyncio.create_task(loop._run())) |
| 144 return loop |