Mercurial > code > home > repos > reposync
annotate sync.py @ 9:2c4d383d464c
move config stuff into Project
| author | drewp@bigasterisk.com |
|---|---|
| date | Fri, 16 Jul 2021 00:19:01 -0700 |
| parents | cc3321b8adc1 |
| children | 460a2cf8b22b |
| rev | line source |
|---|---|
| 0 | 1 #!bin/python |
| 2 | |
| 7 | 3 import os, subprocess, json, logging, traceback, time, re |
| 4 from pathlib import Path | |
|
1
1da34eecdfd8
port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents:
0
diff
changeset
|
5 from github import Github, GithubException |
| 0 | 6 logging.basicConfig(level=logging.INFO) |
| 7 log = logging.getLogger() | |
| 8 | |
| 9 | 9 |
| 10 class Project: | |
| 11 def __init__(self, projRoot: Path): | |
| 12 self.config = json.load(open(Path(__file__).parent / "config.json")) | |
| 13 self.config['SSH_AUTH_SOCK'] = getSshAuthSock() | |
| 14 | |
| 15 self.gh = Github(self.config['githubToken']).get_user() | |
| 7 | 16 self.projRoot = projRoot |
| 17 self.name = projRoot.name | |
|
2
22ccc05756de
don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents:
1
diff
changeset
|
18 |
|
22ccc05756de
don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents:
1
diff
changeset
|
19 def darcsTime(self): |
|
22ccc05756de
don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents:
1
diff
changeset
|
20 j = os.path.join |
|
22ccc05756de
don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents:
1
diff
changeset
|
21 darcsPriv = j(self.config['darcsDir'], self.name, '_darcs') |
|
22ccc05756de
don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents:
1
diff
changeset
|
22 for n in ['inventory', 'hashed_inventory']: |
|
22ccc05756de
don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents:
1
diff
changeset
|
23 if os.path.exists(j(darcsPriv, n)): |
|
22ccc05756de
don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents:
1
diff
changeset
|
24 return os.path.getmtime(j(darcsPriv, n)) |
|
22ccc05756de
don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents:
1
diff
changeset
|
25 raise ValueError("can't find a darcs time") |
| 7 | 26 |
| 0 | 27 def gitDir(self): |
| 28 gitDir = os.path.join(self.config['gitSyncDir'], self.name) | |
| 29 try: | |
| 30 os.mkdir(gitDir) | |
| 31 except OSError: pass | |
| 32 return gitDir | |
| 7 | 33 |
| 0 | 34 def syncToLocalGit(self): |
| 35 darcsDir = os.path.join(self.config['darcsDir'], self.name) | |
| 36 try: | |
| 37 os.rmdir(os.path.join(darcsDir, 'darcs_testing_for_nfs')) | |
| 38 except OSError: pass | |
|
3
4077903a9520
upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents:
2
diff
changeset
|
39 self.runGitCommand([self.config['darcsToGitCmd'], '--verbose', darcsDir]) |
| 0 | 40 |
| 7 | 41 def runGitCommand(self, args, callKw={}): |
|
1
1da34eecdfd8
port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents:
0
diff
changeset
|
42 try: |
|
1da34eecdfd8
port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents:
0
diff
changeset
|
43 subprocess.check_call(args, cwd=self.gitDir(), |
|
1da34eecdfd8
port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents:
0
diff
changeset
|
44 env={'SSH_AUTH_SOCK': self.config['SSH_AUTH_SOCK'], |
|
1da34eecdfd8
port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents:
0
diff
changeset
|
45 'HOME': os.environ['HOME'], # darcs-to-git uses this |
| 7 | 46 }, **callKw) |
|
1
1da34eecdfd8
port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents:
0
diff
changeset
|
47 except: |
|
1da34eecdfd8
port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents:
0
diff
changeset
|
48 log.error("in %s" % self.gitDir()) |
|
1da34eecdfd8
port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents:
0
diff
changeset
|
49 raise |
| 0 | 50 |
| 8 | 51 def makeGithubRepo(self): |
| 0 | 52 try: |
|
1
1da34eecdfd8
port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents:
0
diff
changeset
|
53 self.gh.create_repo(self.name) |
| 7 | 54 except GithubException as e: |
|
3
4077903a9520
upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents:
2
diff
changeset
|
55 assert e.data['errors'][0]['message'].startswith('name already exists'), (e, self.name) |
| 0 | 56 return |
| 57 self.runGitCommand(['git', 'remote', 'add', 'origin', | |
|
1
1da34eecdfd8
port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents:
0
diff
changeset
|
58 'git@github.com:%s/%s.git' % (self.gh.login, |
| 0 | 59 self.name)]) |
| 60 | |
| 8 | 61 def pushToGithub(self): |
| 0 | 62 self.runGitCommand(['git', 'push', 'origin', 'master']) |
| 63 | |
| 8 | 64 def hgToGithub(self): |
| 7 | 65 subprocess.check_call(['hg', 'bookmark', '-r', 'default', 'main'], |
| 66 cwd=self.projRoot) | |
| 67 subprocess.check_call(['hg', 'push', | |
| 68 f'git+ssh://git@github.com/{self.gh.login}/{self.name}' | |
| 69 ], | |
| 70 cwd=self.projRoot, | |
| 71 env={'SSH_AUTH_SOCK': self.config['SSH_AUTH_SOCK']}) | |
| 72 | |
|
3
4077903a9520
upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents:
2
diff
changeset
|
73 def getSshAuthSock(): |
|
4077903a9520
upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents:
2
diff
changeset
|
74 keychain = subprocess.check_output([ |
| 7 | 75 "keychain", "--noask", "--quiet", "--eval", "id_rsa"]).decode('ascii') |
|
3
4077903a9520
upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents:
2
diff
changeset
|
76 m = re.search(r'SSH_AUTH_SOCK=([^; \n]+)', keychain) |
| 7 | 77 if m is None: |
|
3
4077903a9520
upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents:
2
diff
changeset
|
78 raise ValueError("couldn't find SSH_AUTH_SOCK in output " |
|
4077903a9520
upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents:
2
diff
changeset
|
79 "from keychain: %r" % keychain) |
| 7 | 80 return m.group(1) |
| 81 | |
| 82 if __name__ == '__main__': | |
|
1
1da34eecdfd8
port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents:
0
diff
changeset
|
83 |
| 7 | 84 # to get this token: |
| 85 # curl -u drewp https://api.github.com/authorizations -d '{"scopes":["repo"]}' | |
| 86 # --new: | |
| 87 # from https://docs.github.com/en/developers/apps/building-oauth-apps/authorizing-oauth-apps#non-web-application-flow -> https://github.com/settings/tokens to make one | |
| 88 | |
| 89 for proj in os.listdir(config['darcsDir']): | |
| 90 if not os.path.isdir(os.path.join(config['darcsDir'], proj)): | |
|
2
22ccc05756de
don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents:
1
diff
changeset
|
91 continue |
| 7 | 92 try: |
| 9 | 93 p = Project(proj) |
| 7 | 94 |
| 95 if p.darcsTime() < time.time() - 86400*config['tooOldDays']: | |
| 96 continue | |
| 97 | |
| 98 log.info("syncing %s", proj) | |
| 99 p.syncToLocalGit() | |
| 8 | 100 p.makeGithubRepo() |
| 101 p.pushToGithub() | |
| 7 | 102 except Exception as e: |
| 103 traceback.print_exc() |