Mercurial > code > home > repos > reposync
view repo_local_status.py @ 19:5751ef191454
read from github into local graph
author | drewp@bigasterisk.com |
---|---|
date | Sun, 09 Jan 2022 16:02:08 -0800 |
parents | 6f38aa08408d |
children | b59912649fc4 |
line wrap: on
line source
""" configured hg dirs and settings -> rdf graph """ import datetime import json import time import traceback from dataclasses import dataclass, field from pathlib import Path from typing import Dict, Optional, Tuple import cyclone.httpserver import cyclone.sse import cyclone.web import docopt import treq import tzlocal from cycloneerr import PrettyErrorHandler from dateutil.parser import parse from dateutil.tz import tzlocal from prometheus_client.exposition import generate_latest from prometheus_client.registry import REGISTRY from ruamel.yaml import YAML from standardservice.logsetup import log, verboseLogging from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.utils import _UnexpectedErrorOutput, getProcessOutput from patch_cyclone_sse import patchCycloneSse patchCycloneSse() githubOwner = 'drewp' @inlineCallbacks def runHg(cwd, args): if args[0] not in ['push']: args.extend(['-T', 'json']) j = yield getProcessOutput('/usr/local/bin/hg', args, path=cwd) returnValue(json.loads(j) if j else None) @dataclass class Repo: path: Path github: bool _cache: Dict[str, Tuple[float, object]] = field(default_factory=dict) def _isStale(self, group) -> Optional[object]: now = time.time() if group not in self._cache: return True if now > self._cache[group][0] + 86400: return True print('fresh') return False def _save(self, group, obj): now = time.time() self._cache[group] = (now, obj) def _get(self, group): print('get') return self._cache[group][1] @inlineCallbacks def getStatus(self): if self._isStale('status'): try: statusResp = yield runHg(self.path, ['status']) except Exception as e: status = {'error': repr(e)} else: unknowns = len([row for row in statusResp if row['status'] == '?']) status = {'unknown': unknowns, 'changed': len(statusResp) - unknowns} self._save('status', status) returnValue(self._get('status')) @inlineCallbacks def getLatestHgCommit(self): if self._isStale('log'): rows = yield runHg(self.path, ['log', '--limit', '1']) commit = rows[0] sec = commit['date'][0] t = datetime.datetime.fromtimestamp(sec, tzlocal()) self._save('log', {'email': commit['user'], 't': t.isoformat(), 'message': commit['desc']}) returnValue(self._get('log')) @inlineCallbacks def getLatestGithubCommit(self): if self._isStale('github'): resp = yield treq.get(f'https://api.github.com/repos/{githubOwner}/{self.path.name}/commits?per_page=1', timeout=5, headers={ 'User-agent': 'reposync by github.com/drewp', 'Accept': 'application/vnd.github.v3+json' }) ret = yield treq.json_content(resp) commit = ret[0]['commit'] t = parse(commit['committer']['date']).astimezone(tzlocal()).isoformat() self._save('github', {'email': commit['committer']['email'], 't': t, 'message': commit['message']}) returnValue(self._get('github')) @inlineCallbacks def clearGithubMaster(self): '''bang(pts/13):/tmp/reset% git init Initialized empty Git repository in /tmp/reset/.git/ then github set current to a new branch called 'clearing' with https://developer.github.com/v3/repos/#update-a-repository bang(pts/13):/tmp/reset% git remote add origin git@github.com:drewp/href.git bang(pts/13):/tmp/reset% git push origin :master To github.com:drewp/href.git - [deleted] master maybe --set-upstream origin bang(pts/13):/tmp/reset% git remote set-branches origin master ? then push then github setdefault to master then github delete clearing ''' @inlineCallbacks def pushToGithub(self): if not self.github: raise ValueError yield runHg(self.path, ['bookmark', '--rev', 'default', 'master']) out = yield runHg(self.path, ['push', f'git+ssh://git@github.com/{githubOwner}/{self.path.name}.git']) print(f'out fompushh {out}') class GithubSync(PrettyErrorHandler, cyclone.web.RequestHandler): @inlineCallbacks def post(self): try: path = self.get_argument('repo') repo = [r for r in self.settings.repos if str(r.path) == path][0] yield repo.pushToGithub() except Exception: traceback.print_exc() raise class Statuses(cyclone.sse.SSEHandler): def update(self, key, data): self.sendEvent(json.dumps({'key': key, 'update': data}).encode('utf8')) def bind(self): self.toProcess = self.settings.repos[:] reactor.callLater(0, self.runOne) @inlineCallbacks def runOne(self): if not self.toProcess: print('done') return repo = self.toProcess.pop(0) try: update = {'path': str(repo.path), 'github': repo.github, 'status': (yield repo.getStatus()), 'hgLatest': (yield repo.getLatestHgCommit())} if repo.github: update['githubLatest'] = (yield repo.getLatestGithubCommit()) self.update(str(repo.path), update) except Exception: log.warn(f'not reporting on {repo}') traceback.print_exc() reactor.callLater(0, self.runOne) class Metrics(cyclone.web.RequestHandler): def get(self): self.add_header('content-type', 'text/plain') self.write(generate_latest(REGISTRY)) def main(): args = docopt.docopt(''' Usage: hg_status.py [options] Options: -v, --verbose more logging ''') verboseLogging(args['--verbose']) # import sys # sys.path.append('/usr/lib/python3/dist-packages') # import OpenSSL yaml = YAML(typ='safe') config = yaml.load(open('config.yaml')) repos = [Repo(Path(row['dir']), row['github']) for row in config['hg_repos']] class Application(cyclone.web.Application): def __init__(self): handlers = [ (r"/()", cyclone.web.StaticFileHandler, { 'path': '.', 'default_filename': 'index.html' }), (r'/build/(bundle\.js)', cyclone.web.StaticFileHandler, { 'path': './build/' }), (r'/status/events', Statuses), (r'/githubSync', GithubSync), (r'/metrics', Metrics), ] cyclone.web.Application.__init__( self, handlers, repos=repos, debug=args['--verbose'], template_path='.', ) reactor.listenTCP(10001, Application()) reactor.run() if __name__ == '__main__': main()