Mercurial > code > home > repos > reposync
annotate repo_local_status.py @ 22:b9fe6d26b3fa
minor changes, fixes, upgrades
| author | drewp@bigasterisk.com |
|---|---|
| date | Tue, 29 Mar 2022 21:15:51 -0700 |
| parents | cb71722bb75c |
| children |
| rev | line source |
|---|---|
|
18
6f38aa08408d
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
drewp@bigasterisk.com
parents:
17
diff
changeset
|
1 """ |
|
6f38aa08408d
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
drewp@bigasterisk.com
parents:
17
diff
changeset
|
2 configured hg dirs and settings -> rdf graph |
|
6f38aa08408d
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
drewp@bigasterisk.com
parents:
17
diff
changeset
|
3 """ |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
4 import datetime |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
5 import json |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
6 import time |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
7 import traceback |
| 17 | 8 from dataclasses import dataclass, field |
| 9 from pathlib import Path | |
| 20 | 10 from typing import Dict, Optional, Set, Tuple |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
11 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
12 import cyclone.httpserver |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
13 import cyclone.sse |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
14 import cyclone.web |
| 17 | 15 import docopt |
| 16 import treq | |
| 20 | 17 from background_loop import loop_forever_async |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
18 from dateutil.parser import parse |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
19 from dateutil.tz import tzlocal |
| 20 | 20 from patchablegraph import (CycloneGraphEventsHandler, CycloneGraphHandler, PatchableGraph) |
| 21 from prometheus_client import Counter, Gauge | |
| 17 | 22 from prometheus_client.exposition import generate_latest |
| 23 from prometheus_client.registry import REGISTRY | |
| 20 | 24 from rdfdb.currentstategraphapi import CurrentStateGraphApi |
| 25 from rdfdb.patch import Patch | |
| 26 from rdflib import RDF, Literal, Namespace, URIRef | |
| 27 from rdflib.term import Identifier | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
28 from ruamel.yaml import YAML |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
29 from standardservice.logsetup import log, verboseLogging |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
30 from twisted.internet import reactor |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
31 from twisted.internet.defer import inlineCallbacks, returnValue |
| 17 | 32 from twisted.internet.utils import _UnexpectedErrorOutput, getProcessOutput |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
33 |
|
18
6f38aa08408d
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
drewp@bigasterisk.com
parents:
17
diff
changeset
|
34 from patch_cyclone_sse import patchCycloneSse |
| 20 | 35 |
|
18
6f38aa08408d
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
drewp@bigasterisk.com
parents:
17
diff
changeset
|
36 patchCycloneSse() |
|
6f38aa08408d
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
drewp@bigasterisk.com
parents:
17
diff
changeset
|
37 |
| 20 | 38 Quad = Tuple[Identifier, Identifier, Identifier, Identifier] |
| 39 Triple = Tuple[Identifier, Identifier, Identifier] | |
| 40 EX = Namespace('http://example.com/') # todo | |
| 41 | |
| 42 HG_SYNC = Gauge('hg_sync', 'sync passes to hg') | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
43 |
| 21 | 44 _venv = None |
| 45 | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
46 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
47 @inlineCallbacks |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
48 def runHg(cwd, args): |
| 21 | 49 global _venv |
| 50 if _venv is None: | |
| 51 _venv = yield getProcessOutput('pipenv', ['--venv']) | |
| 52 _venv = _venv.decode('ascii').strip() | |
| 53 | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
54 if args[0] not in ['push']: |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
55 args.extend(['-T', 'json']) |
| 21 | 56 j = yield getProcessOutput(_venv + '/bin/hg', ['--cwd', str(cwd)] + args, env={'LANG': 'C.UTF-8'}) |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
57 returnValue(json.loads(j) if j else None) |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
58 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
59 |
| 20 | 60 # merge this into setToGraph |
| 61 def replaceContext(pg: PatchableGraph, ctx: URIRef, newTriples: Set[Triple]): | |
| 62 prevCtxStmts = set((s, p, o, g.identifier) for s, p, o, g in pg._graph.quads((None, None, None, ctx))) | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
63 |
| 20 | 64 currentStmts: Set[Quad] = set() |
| 65 for tri in newTriples: | |
| 66 currentStmts.add(tri + (ctx,)) | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
67 |
| 20 | 68 p = Patch(delQuads=prevCtxStmts.difference(currentStmts), addQuads=currentStmts.difference(prevCtxStmts)) |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
69 |
| 20 | 70 pg.patch(p) |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
71 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
72 |
| 17 | 73 class Metrics(cyclone.web.RequestHandler): |
| 74 | |
| 75 def get(self): | |
| 76 self.add_header('content-type', 'text/plain') | |
| 77 self.write(generate_latest(REGISTRY)) | |
| 78 | |
| 79 | |
| 20 | 80 class Index(cyclone.web.RequestHandler): |
| 81 | |
| 82 def get(self, *args): | |
| 83 self.add_header('content-type', 'text/html') | |
| 84 self.write('''<!DOCTYPE html> | |
| 85 <html> | |
| 86 <head> | |
| 87 <title>repo_local_status</title> | |
| 88 </head> | |
| 89 <body> | |
| 90 <a href="graph/localRepos">graph</a> | |
| 91 </body> | |
| 92 </html>''') | |
| 93 | |
| 94 | |
| 95 # share with other files | |
| 96 @dataclass | |
| 97 class Repo: | |
| 98 path: Path | |
| 99 github: bool | |
| 100 | |
| 101 def uri(self): | |
| 102 return URIRef(f'http://bigasterisk.com/repo/{self.path.name}') | |
| 103 | |
| 104 def localInstance(self): | |
| 105 return URIRef(f'http://bigasterisk.com/repo/{self.path.name}/local') | |
| 106 | |
| 107 | |
| 108 @inlineCallbacks | |
| 109 def updateOne(graph, repo: Repo): | |
| 110 try: | |
| 111 writeLocalRepo(graph, repo) | |
| 112 yield writeStatus(graph, repo) | |
| 113 writeLatestHgCommit(graph, repo) | |
| 114 except Exception: | |
| 115 log.warning(f'While updating {repo}:') | |
| 116 log.warning(traceback.format_exc()) | |
| 117 | |
| 118 | |
| 119 def uriForHgNode(n: str) -> URIRef: | |
| 120 return URIRef(f'http://bigasterisk.com/hg/node/{n}') | |
| 121 | |
| 21 | 122 |
| 20 | 123 def uriForHgPhase(phase: str) -> URIRef: |
| 124 return URIRef(f'http://bigasterisk.com/hg/phase/{phase}') | |
| 125 | |
| 21 | 126 |
| 20 | 127 @inlineCallbacks |
| 128 def writeLatestHgCommit(graph, repo): | |
| 129 rows = yield runHg(repo.path, ['log', '--limit', '1']) | |
| 130 commit = rows[0] | |
| 131 sec = commit['date'][0] | |
| 132 t = datetime.datetime.fromtimestamp(sec, tzlocal()) | |
| 133 latest = uriForHgNode(commit['node']) | |
| 134 | |
| 135 new: Set[Triple] = set() | |
| 136 new.add((repo.localInstance(), EX['latestCommit'], latest)) | |
| 137 new.add((latest, RDF.type, EX['HgCommit'])) | |
| 138 new.add((latest, EX['email'], Literal(commit['user']))) | |
| 139 new.add((latest, EX['commitMessage'], Literal(commit['desc']))) | |
| 140 new.add((latest, EX['created'], Literal(t))) | |
| 141 new.add((latest, EX['phase'], uriForHgNode(commit['phase']))) | |
| 142 | |
| 143 for t in commit['tags']: | |
| 144 new.add((latest, EX['tag'], Literal(t))) | |
| 145 for p in commit['parents']: | |
| 146 new.add((latest, EX['parent'], uriForHgNode(p))) | |
| 147 | |
| 148 replaceContext(graph, URIRef(repo.localInstance() + '/log'), new) | |
| 149 | |
| 150 | |
| 151 @inlineCallbacks | |
| 152 def writeStatus(graph, repo): | |
| 153 statusResp = yield runHg(repo.path, ['status']) | |
| 154 unknowns = len([row for row in statusResp if row['status'] == '?']) | |
| 22 | 155 replaceContext(graph, URIRef(repo.localInstance() + '/status'), { |
| 156 (repo.localInstance(), EX['unknownCount'], Literal(unknowns)), | |
| 157 (repo.localInstance(), EX['changed'], Literal(len(statusResp) - unknowns)), | |
| 158 }) | |
| 20 | 159 |
| 160 | |
| 161 def writeLocalRepo(graph, repo: Repo): | |
| 162 replaceContext(graph, URIRef(repo.uri() + '/config'), { | |
| 22 | 163 (repo.uri(), RDF.type, EX['Repo']), |
| 20 | 164 (repo.uri(), EX['localRepo'], repo.localInstance()), |
| 165 (repo.localInstance(), RDF.type, EX['HgRepo']), | |
| 166 }) | |
| 167 | |
| 168 | |
| 169 @inlineCallbacks | |
| 170 def update(graph, repos): | |
| 171 for r in repos: | |
| 172 yield updateOne(graph, r) | |
| 173 | |
| 174 | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
175 def main(): |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
176 args = docopt.docopt(''' |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
177 Usage: |
| 20 | 178 repo_local_status.py [options] |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
179 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
180 Options: |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
181 -v, --verbose more logging |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
182 ''') |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
183 verboseLogging(args['--verbose']) |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
184 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
185 yaml = YAML(typ='safe') |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
186 config = yaml.load(open('config.yaml')) |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
187 repos = [Repo(Path(row['dir']), row['github']) for row in config['hg_repos']] |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
188 |
| 20 | 189 class PG2(PatchableGraph, CurrentStateGraphApi): |
| 190 pass | |
| 191 | |
| 192 graph = PG2() | |
| 193 | |
| 194 @inlineCallbacks | |
| 195 def f(first): | |
| 196 yield update(graph, repos) | |
| 197 | |
| 198 loop_forever_async(f, 3600, HG_SYNC) | |
| 199 | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
200 class Application(cyclone.web.Application): |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
201 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
202 def __init__(self): |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
203 handlers = [ |
| 20 | 204 (r"/()", Index), |
| 205 (r'/graph/localRepos', CycloneGraphHandler, { | |
| 206 'masterGraph': graph | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
207 }), |
| 20 | 208 (r'/graph/localRepos/events', CycloneGraphEventsHandler, { |
| 209 'masterGraph': graph, | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
210 }), |
| 17 | 211 (r'/metrics', Metrics), |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
212 ] |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
213 cyclone.web.Application.__init__( |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
214 self, |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
215 handlers, |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
216 debug=args['--verbose'], |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
217 ) |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
218 |
| 20 | 219 reactor.listenTCP(8001, Application(), interface='::') |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
220 reactor.run() |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
221 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
222 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
223 if __name__ == '__main__': |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
224 main() |