Mercurial > code > home > repos > reposync
view hg_status.py @ 17:a4778c56cc03
update deps and k8s setup
author | drewp@bigasterisk.com |
---|---|
date | Sun, 12 Dec 2021 22:32:25 -0800 |
parents | db4037285592 |
children |
line wrap: on
line source
import datetime import json import time import traceback from dataclasses import dataclass, field from pathlib import Path from typing import Dict, Optional, Tuple import cyclone.httpserver import cyclone.sse import cyclone.web import docopt import treq import tzlocal from cycloneerr import PrettyErrorHandler from dateutil.parser import parse from dateutil.tz import tzlocal from prometheus_client.exposition import generate_latest from prometheus_client.registry import REGISTRY from ruamel.yaml import YAML from standardservice.logsetup import log, verboseLogging from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.utils import _UnexpectedErrorOutput, getProcessOutput githubOwner = 'drewp' @inlineCallbacks def runHg(cwd, args): if args[0] not in ['push']: args.extend(['-T', 'json']) j = yield getProcessOutput('/usr/local/bin/hg', args, path=cwd) returnValue(json.loads(j) if j else None) @dataclass class Repo: path: Path github: bool _cache: Dict[str, Tuple[float, object]] = field(default_factory=dict) def _isStale(self, group) -> Optional[object]: now = time.time() if group not in self._cache: return True if now > self._cache[group][0] + 86400: return True print('fresh') return False def _save(self, group, obj): now = time.time() self._cache[group] = (now, obj) def _get(self, group): print('get') return self._cache[group][1] @inlineCallbacks def getStatus(self): if self._isStale('status'): try: statusResp = yield runHg(self.path, ['status']) except Exception as e: status = {'error': repr(e)} else: unknowns = len([row for row in statusResp if row['status'] == '?']) status = {'unknown': unknowns, 'changed': len(statusResp) - unknowns} self._save('status', status) returnValue(self._get('status')) @inlineCallbacks def getLatestHgCommit(self): if self._isStale('log'): rows = yield runHg(self.path, ['log', '--limit', '1']) commit = rows[0] sec = commit['date'][0] t = datetime.datetime.fromtimestamp(sec, tzlocal()) self._save('log', {'email': commit['user'], 't': t.isoformat(), 'message': commit['desc']}) returnValue(self._get('log')) @inlineCallbacks def getLatestGithubCommit(self): if self._isStale('github'): resp = yield treq.get(f'https://api.github.com/repos/{githubOwner}/{self.path.name}/commits?per_page=1', timeout=5, headers={ 'User-agent': 'reposync by github.com/drewp', 'Accept': 'application/vnd.github.v3+json' }) ret = yield treq.json_content(resp) commit = ret[0]['commit'] t = parse(commit['committer']['date']).astimezone(tzlocal()).isoformat() self._save('github', {'email': commit['committer']['email'], 't': t, 'message': commit['message']}) returnValue(self._get('github')) @inlineCallbacks def clearGithubMaster(self): '''bang(pts/13):/tmp/reset% git init Initialized empty Git repository in /tmp/reset/.git/ then github set current to a new branch called 'clearing' with https://developer.github.com/v3/repos/#update-a-repository bang(pts/13):/tmp/reset% git remote add origin git@github.com:drewp/href.git bang(pts/13):/tmp/reset% git push origin :master To github.com:drewp/href.git - [deleted] master maybe --set-upstream origin bang(pts/13):/tmp/reset% git remote set-branches origin master ? then push then github setdefault to master then github delete clearing ''' @inlineCallbacks def pushToGithub(self): if not self.github: raise ValueError yield runHg(self.path, ['bookmark', '--rev', 'default', 'master']) out = yield runHg(self.path, ['push', f'git+ssh://git@github.com/{githubOwner}/{self.path.name}.git']) print(f'out fompushh {out}') class GithubSync(PrettyErrorHandler, cyclone.web.RequestHandler): @inlineCallbacks def post(self): try: path = self.get_argument('repo') repo = [r for r in self.settings.repos if str(r.path) == path][0] yield repo.pushToGithub() except Exception: traceback.print_exc() raise class Statuses(cyclone.sse.SSEHandler): def update(self, key, data): self.sendEvent(json.dumps({'key': key, 'update': data}).encode('utf8')) def bind(self): self.toProcess = self.settings.repos[:] reactor.callLater(0, self.runOne) @inlineCallbacks def runOne(self): if not self.toProcess: print('done') return repo = self.toProcess.pop(0) try: update = {'path': str(repo.path), 'github': repo.github, 'status': (yield repo.getStatus()), 'hgLatest': (yield repo.getLatestHgCommit())} if repo.github: update['githubLatest'] = (yield repo.getLatestGithubCommit()) self.update(str(repo.path), update) except Exception: log.warn(f'not reporting on {repo}') traceback.print_exc() reactor.callLater(0, self.runOne) class Metrics(cyclone.web.RequestHandler): def get(self): self.add_header('content-type', 'text/plain') self.write(generate_latest(REGISTRY)) def main(): args = docopt.docopt(''' Usage: hg_status.py [options] Options: -v, --verbose more logging ''') verboseLogging(args['--verbose']) # import sys # sys.path.append('/usr/lib/python3/dist-packages') # import OpenSSL yaml = YAML(typ='safe') config = yaml.load(open('config.yaml')) repos = [Repo(Path(row['dir']), row['github']) for row in config['hg_repos']] class Application(cyclone.web.Application): def __init__(self): handlers = [ (r"/()", cyclone.web.StaticFileHandler, { 'path': '.', 'default_filename': 'index.html' }), (r'/build/(bundle\.js)', cyclone.web.StaticFileHandler, { 'path': './build/' }), (r'/status/events', Statuses), (r'/githubSync', GithubSync), (r'/metrics', Metrics), ] cyclone.web.Application.__init__( self, handlers, repos=repos, debug=args['--verbose'], template_path='.', ) reactor.listenTCP(10001, Application()) reactor.run() if __name__ == '__main__': main()