Mercurial > code > home > repos > reposync

comparison repo_local_status.py @ 18:6f38aa08408d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
starting over: make a web page that draws a streamed graph from collector, with plans for services to scrape the data that collector will subscribe to
author drewp@bigasterisk.com
date Sun, 09 Jan 2022 00:21:41 -0800
parents hg_status.py@a4778c56cc03
children b59912649fc4
comparison
equal deleted inserted replaced
17:a4778c56cc03 18:6f38aa08408d
1 """
2 configured hg dirs and settings -> rdf graph
3 """
4 import datetime
5 import json
6 import time
7 import traceback
8 from dataclasses import dataclass, field
9 from pathlib import Path
10 from typing import Dict, Optional, Tuple
11
12 import cyclone.httpserver
13 import cyclone.sse
14 import cyclone.web
15 import docopt
16 import treq
17 import tzlocal
18 from cycloneerr import PrettyErrorHandler
19 from dateutil.parser import parse
20 from dateutil.tz import tzlocal
21 from prometheus_client.exposition import generate_latest
22 from prometheus_client.registry import REGISTRY
23 from ruamel.yaml import YAML
24 from standardservice.logsetup import log, verboseLogging
25 from twisted.internet import reactor
26 from twisted.internet.defer import inlineCallbacks, returnValue
27 from twisted.internet.utils import _UnexpectedErrorOutput, getProcessOutput
28
29 from patch_cyclone_sse import patchCycloneSse
30 patchCycloneSse()
31
32 githubOwner = 'drewp'
33
34
35 @inlineCallbacks
36 def runHg(cwd, args):
37 if args[0] not in ['push']:
38 args.extend(['-T', 'json'])
39 j = yield getProcessOutput('/usr/local/bin/hg', args, path=cwd)
40 returnValue(json.loads(j) if j else None)
41
42
43 @dataclass
44 class Repo:
45 path: Path
46 github: bool
47 _cache: Dict[str, Tuple[float, object]] = field(default_factory=dict)
48
49 def _isStale(self, group) -> Optional[object]:
50 now = time.time()
51 if group not in self._cache:
52 return True
53 if now > self._cache[group][0] + 86400:
54 return True
55 print('fresh')
56 return False
57
58 def _save(self, group, obj):
59 now = time.time()
60 self._cache[group] = (now, obj)
61
62 def _get(self, group):
63 print('get')
64 return self._cache[group][1]
65
66 @inlineCallbacks
67 def getStatus(self):
68 if self._isStale('status'):
69 try:
70 statusResp = yield runHg(self.path, ['status'])
71 except Exception as e:
72 status = {'error': repr(e)}
73 else:
74 unknowns = len([row for row in statusResp if row['status'] == '?'])
75 status = {'unknown': unknowns, 'changed': len(statusResp) - unknowns}
76 self._save('status', status)
77 returnValue(self._get('status'))
78
79 @inlineCallbacks
80 def getLatestHgCommit(self):
81 if self._isStale('log'):
82 rows = yield runHg(self.path, ['log', '--limit', '1'])
83 commit = rows[0]
84 sec = commit['date'][0]
85 t = datetime.datetime.fromtimestamp(sec, tzlocal())
86 self._save('log', {'email': commit['user'], 't': t.isoformat(), 'message': commit['desc']})
87 returnValue(self._get('log'))
88
89 @inlineCallbacks
90 def getLatestGithubCommit(self):
91 if self._isStale('github'):
92 resp = yield treq.get(f'https://api.github.com/repos/{githubOwner}/{self.path.name}/commits?per_page=1',
93 timeout=5,
94 headers={
95 'User-agent': 'reposync by github.com/drewp',
96 'Accept': 'application/vnd.github.v3+json'
97 })
98 ret = yield treq.json_content(resp)
99 commit = ret[0]['commit']
100 t = parse(commit['committer']['date']).astimezone(tzlocal()).isoformat()
101 self._save('github', {'email': commit['committer']['email'], 't': t, 'message': commit['message']})
102 returnValue(self._get('github'))
103
104 @inlineCallbacks
105 def clearGithubMaster(self):
106 '''bang(pts/13):/tmp/reset% git init
107 Initialized empty Git repository in /tmp/reset/.git/
108 then github set current to a new branch called 'clearing' with https://developer.github.com/v3/repos/#update-a-repository
109 bang(pts/13):/tmp/reset% git remote add origin git@github.com:drewp/href.git
110 bang(pts/13):/tmp/reset% git push origin :master
111 To github.com:drewp/href.git
112 - [deleted] master
113 maybe --set-upstream origin
114 bang(pts/13):/tmp/reset% git remote set-branches origin master
115 ?
116 then push
117 then github setdefault to master
118 then github delete clearing
119 '''
120
121 @inlineCallbacks
122 def pushToGithub(self):
123 if not self.github:
124 raise ValueError
125 yield runHg(self.path, ['bookmark', '--rev', 'default', 'master'])
126 out = yield runHg(self.path, ['push', f'git+ssh://git@github.com/{githubOwner}/{self.path.name}.git'])
127 print(f'out fompushh {out}')
128
129
130 class GithubSync(PrettyErrorHandler, cyclone.web.RequestHandler):
131
132 @inlineCallbacks
133 def post(self):
134 try:
135 path = self.get_argument('repo')
136 repo = [r for r in self.settings.repos if str(r.path) == path][0]
137 yield repo.pushToGithub()
138 except Exception:
139 traceback.print_exc()
140 raise
141
142
143 class Statuses(cyclone.sse.SSEHandler):
144
145 def update(self, key, data):
146 self.sendEvent(json.dumps({'key': key, 'update': data}).encode('utf8'))
147
148 def bind(self):
149 self.toProcess = self.settings.repos[:]
150 reactor.callLater(0, self.runOne)
151
152 @inlineCallbacks
153 def runOne(self):
154 if not self.toProcess:
155 print('done')
156 return
157 repo = self.toProcess.pop(0)
158
159 try:
160 update = {'path': str(repo.path), 'github': repo.github, 'status': (yield repo.getStatus()), 'hgLatest': (yield repo.getLatestHgCommit())}
161 if repo.github:
162 update['githubLatest'] = (yield repo.getLatestGithubCommit())
163 self.update(str(repo.path), update)
164 except Exception:
165 log.warn(f'not reporting on {repo}')
166 traceback.print_exc()
167 reactor.callLater(0, self.runOne)
168
169
170 class Metrics(cyclone.web.RequestHandler):
171
172 def get(self):
173 self.add_header('content-type', 'text/plain')
174 self.write(generate_latest(REGISTRY))
175
176
177 def main():
178 args = docopt.docopt('''
179 Usage:
180 hg_status.py [options]
181
182 Options:
183 -v, --verbose more logging
184 ''')
185 verboseLogging(args['--verbose'])
186
187 # import sys
188 # sys.path.append('/usr/lib/python3/dist-packages')
189 # import OpenSSL
190
191 yaml = YAML(typ='safe')
192 config = yaml.load(open('config.yaml'))
193 repos = [Repo(Path(row['dir']), row['github']) for row in config['hg_repos']]
194
195 class Application(cyclone.web.Application):
196
197 def __init__(self):
198 handlers = [
199 (r"/()", cyclone.web.StaticFileHandler, {
200 'path': '.',
201 'default_filename': 'index.html'
202 }),
203 (r'/build/(bundle\.js)', cyclone.web.StaticFileHandler, {
204 'path': './build/'
205 }),
206 (r'/status/events', Statuses),
207 (r'/githubSync', GithubSync),
208 (r'/metrics', Metrics),
209 ]
210 cyclone.web.Application.__init__(
211 self,
212 handlers,
213 repos=repos,
214 debug=args['--verbose'],
215 template_path='.',
216 )
217
218 reactor.listenTCP(10001, Application())
219 reactor.run()
220
221
222 if __name__ == '__main__':
223 main()