Mercurial > code > home > repos > reposync
annotate repo_local_status.py @ 20:b59912649fc4
rewrite local hg scanner
| author | drewp@bigasterisk.com |
|---|---|
| date | Sun, 09 Jan 2022 20:47:57 -0800 |
| parents | 6f38aa08408d |
| children | cb71722bb75c |
| rev | line source |
|---|---|
|
18
6f38aa08408d
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
drewp@bigasterisk.com
parents:
17
diff
changeset
|
1 """ |
|
6f38aa08408d
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
drewp@bigasterisk.com
parents:
17
diff
changeset
|
2 configured hg dirs and settings -> rdf graph |
|
6f38aa08408d
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
drewp@bigasterisk.com
parents:
17
diff
changeset
|
3 """ |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
4 import datetime |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
5 import json |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
6 import time |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
7 import traceback |
| 17 | 8 from dataclasses import dataclass, field |
| 9 from pathlib import Path | |
| 20 | 10 from typing import Dict, Optional, Set, Tuple |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
11 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
12 import cyclone.httpserver |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
13 import cyclone.sse |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
14 import cyclone.web |
| 17 | 15 import docopt |
| 16 import treq | |
| 20 | 17 from background_loop import loop_forever_async |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
18 from dateutil.parser import parse |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
19 from dateutil.tz import tzlocal |
| 20 | 20 from patchablegraph import (CycloneGraphEventsHandler, CycloneGraphHandler, PatchableGraph) |
| 21 from prometheus_client import Counter, Gauge | |
| 17 | 22 from prometheus_client.exposition import generate_latest |
| 23 from prometheus_client.registry import REGISTRY | |
| 20 | 24 from rdfdb.currentstategraphapi import CurrentStateGraphApi |
| 25 from rdfdb.patch import Patch | |
| 26 from rdflib import RDF, Literal, Namespace, URIRef | |
| 27 from rdflib.term import Identifier | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
28 from ruamel.yaml import YAML |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
29 from standardservice.logsetup import log, verboseLogging |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
30 from twisted.internet import reactor |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
31 from twisted.internet.defer import inlineCallbacks, returnValue |
| 17 | 32 from twisted.internet.utils import _UnexpectedErrorOutput, getProcessOutput |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
33 |
|
18
6f38aa08408d
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
drewp@bigasterisk.com
parents:
17
diff
changeset
|
34 from patch_cyclone_sse import patchCycloneSse |
| 20 | 35 |
|
18
6f38aa08408d
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
drewp@bigasterisk.com
parents:
17
diff
changeset
|
36 patchCycloneSse() |
|
6f38aa08408d
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
drewp@bigasterisk.com
parents:
17
diff
changeset
|
37 |
| 20 | 38 Quad = Tuple[Identifier, Identifier, Identifier, Identifier] |
| 39 Triple = Tuple[Identifier, Identifier, Identifier] | |
| 40 EX = Namespace('http://example.com/') # todo | |
| 41 | |
| 42 HG_SYNC = Gauge('hg_sync', 'sync passes to hg') | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
43 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
44 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
45 @inlineCallbacks |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
46 def runHg(cwd, args): |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
47 if args[0] not in ['push']: |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
48 args.extend(['-T', 'json']) |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
49 j = yield getProcessOutput('/usr/local/bin/hg', args, path=cwd) |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
50 returnValue(json.loads(j) if j else None) |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
51 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
52 |
| 20 | 53 # merge this into setToGraph |
| 54 def replaceContext(pg: PatchableGraph, ctx: URIRef, newTriples: Set[Triple]): | |
| 55 prevCtxStmts = set((s, p, o, g.identifier) for s, p, o, g in pg._graph.quads((None, None, None, ctx))) | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
56 |
| 20 | 57 currentStmts: Set[Quad] = set() |
| 58 for tri in newTriples: | |
| 59 currentStmts.add(tri + (ctx,)) | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
60 |
| 20 | 61 p = Patch(delQuads=prevCtxStmts.difference(currentStmts), addQuads=currentStmts.difference(prevCtxStmts)) |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
62 |
| 20 | 63 pg.patch(p) |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
64 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
65 |
| 17 | 66 class Metrics(cyclone.web.RequestHandler): |
| 67 | |
| 68 def get(self): | |
| 69 self.add_header('content-type', 'text/plain') | |
| 70 self.write(generate_latest(REGISTRY)) | |
| 71 | |
| 72 | |
| 20 | 73 class Index(cyclone.web.RequestHandler): |
| 74 | |
| 75 def get(self, *args): | |
| 76 self.add_header('content-type', 'text/html') | |
| 77 self.write('''<!DOCTYPE html> | |
| 78 <html> | |
| 79 <head> | |
| 80 <title>repo_local_status</title> | |
| 81 </head> | |
| 82 <body> | |
| 83 <a href="graph/localRepos">graph</a> | |
| 84 </body> | |
| 85 </html>''') | |
| 86 | |
| 87 | |
| 88 # share with other files | |
| 89 @dataclass | |
| 90 class Repo: | |
| 91 path: Path | |
| 92 github: bool | |
| 93 | |
| 94 def uri(self): | |
| 95 return URIRef(f'http://bigasterisk.com/repo/{self.path.name}') | |
| 96 | |
| 97 def localInstance(self): | |
| 98 return URIRef(f'http://bigasterisk.com/repo/{self.path.name}/local') | |
| 99 | |
| 100 | |
| 101 @inlineCallbacks | |
| 102 def updateOne(graph, repo: Repo): | |
| 103 try: | |
| 104 writeLocalRepo(graph, repo) | |
| 105 yield writeStatus(graph, repo) | |
| 106 writeLatestHgCommit(graph, repo) | |
| 107 except Exception: | |
| 108 log.warning(f'While updating {repo}:') | |
| 109 log.warning(traceback.format_exc()) | |
| 110 | |
| 111 | |
| 112 def uriForHgNode(n: str) -> URIRef: | |
| 113 return URIRef(f'http://bigasterisk.com/hg/node/{n}') | |
| 114 | |
| 115 def uriForHgPhase(phase: str) -> URIRef: | |
| 116 return URIRef(f'http://bigasterisk.com/hg/phase/{phase}') | |
| 117 | |
| 118 @inlineCallbacks | |
| 119 def writeLatestHgCommit(graph, repo): | |
| 120 rows = yield runHg(repo.path, ['log', '--limit', '1']) | |
| 121 commit = rows[0] | |
| 122 sec = commit['date'][0] | |
| 123 t = datetime.datetime.fromtimestamp(sec, tzlocal()) | |
| 124 latest = uriForHgNode(commit['node']) | |
| 125 | |
| 126 new: Set[Triple] = set() | |
| 127 new.add((repo.localInstance(), EX['latestCommit'], latest)) | |
| 128 new.add((latest, RDF.type, EX['HgCommit'])) | |
| 129 new.add((latest, EX['email'], Literal(commit['user']))) | |
| 130 new.add((latest, EX['commitMessage'], Literal(commit['desc']))) | |
| 131 new.add((latest, EX['created'], Literal(t))) | |
| 132 new.add((latest, EX['phase'], uriForHgNode(commit['phase']))) | |
| 133 | |
| 134 for t in commit['tags']: | |
| 135 new.add((latest, EX['tag'], Literal(t))) | |
| 136 for p in commit['parents']: | |
| 137 new.add((latest, EX['parent'], uriForHgNode(p))) | |
| 138 | |
| 139 replaceContext(graph, URIRef(repo.localInstance() + '/log'), new) | |
| 140 | |
| 141 | |
| 142 @inlineCallbacks | |
| 143 def writeStatus(graph, repo): | |
| 144 statusResp = yield runHg(repo.path, ['status']) | |
| 145 unknowns = len([row for row in statusResp if row['status'] == '?']) | |
| 146 replaceContext( | |
| 147 graph, URIRef(repo.localInstance() + '/status'), { | |
| 148 (repo.localInstance(), EX['unknownCount'], Literal(unknowns)), | |
| 149 (repo.localInstance(), EX['changed'], Literal(len(statusResp) - unknowns)), | |
| 150 }) | |
| 151 | |
| 152 | |
| 153 def writeLocalRepo(graph, repo: Repo): | |
| 154 replaceContext(graph, URIRef(repo.uri() + '/config'), { | |
| 155 (repo.uri(), EX['localRepo'], repo.localInstance()), | |
| 156 (repo.localInstance(), RDF.type, EX['HgRepo']), | |
| 157 }) | |
| 158 | |
| 159 | |
| 160 @inlineCallbacks | |
| 161 def update(graph, repos): | |
| 162 for r in repos: | |
| 163 yield updateOne(graph, r) | |
| 164 | |
| 165 | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
166 def main(): |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
167 args = docopt.docopt(''' |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
168 Usage: |
| 20 | 169 repo_local_status.py [options] |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
170 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
171 Options: |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
172 -v, --verbose more logging |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
173 ''') |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
174 verboseLogging(args['--verbose']) |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
175 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
176 yaml = YAML(typ='safe') |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
177 config = yaml.load(open('config.yaml')) |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
178 repos = [Repo(Path(row['dir']), row['github']) for row in config['hg_repos']] |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
179 |
| 20 | 180 class PG2(PatchableGraph, CurrentStateGraphApi): |
| 181 pass | |
| 182 | |
| 183 graph = PG2() | |
| 184 | |
| 185 @inlineCallbacks | |
| 186 def f(first): | |
| 187 yield update(graph, repos) | |
| 188 | |
| 189 loop_forever_async(f, 3600, HG_SYNC) | |
| 190 | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
191 class Application(cyclone.web.Application): |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
192 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
193 def __init__(self): |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
194 handlers = [ |
| 20 | 195 (r"/()", Index), |
| 196 (r'/graph/localRepos', CycloneGraphHandler, { | |
| 197 'masterGraph': graph | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
198 }), |
| 20 | 199 (r'/graph/localRepos/events', CycloneGraphEventsHandler, { |
| 200 'masterGraph': graph, | |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
201 }), |
| 17 | 202 (r'/metrics', Metrics), |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
203 ] |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
204 cyclone.web.Application.__init__( |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
205 self, |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
206 handlers, |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
207 debug=args['--verbose'], |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
208 ) |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
209 |
| 20 | 210 reactor.listenTCP(8001, Application(), interface='::') |
|
4
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
211 reactor.run() |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
212 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
213 |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
214 if __name__ == '__main__': |
|
f714a6a7842c
start new web view for hgand github syncing
drewp@bigasterisk.com
parents:
diff
changeset
|
215 main() |