Mercurial > code > home > repos > reposync

annotate sync.py @ 6:a0d9679c4f4a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
loginbar element moves out
author drewp@bigasterisk.com
date Fri, 24 Jul 2020 15:22:06 -0700
parents 4077903a9520
children 7f479502a8ab
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
drewp@bigasterisk.com
parents:
diff changeset
1 #!bin/python
drewp@bigasterisk.com
parents:
diff changeset
2
3
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
3 import os, subprocess, urllib2, json, logging, traceback, time, re
1
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
4 from github import Github, GithubException
0
drewp@bigasterisk.com
parents:
diff changeset
5 logging.basicConfig(level=logging.INFO)
drewp@bigasterisk.com
parents:
diff changeset
6 log = logging.getLogger()
drewp@bigasterisk.com
parents:
diff changeset
7
drewp@bigasterisk.com
parents:
diff changeset
8 class Project(object):
drewp@bigasterisk.com
parents:
diff changeset
9 def __init__(self, config, gh, name):
drewp@bigasterisk.com
parents:
diff changeset
10 self.config = config
drewp@bigasterisk.com
parents:
diff changeset
11 self.gh = gh
drewp@bigasterisk.com
parents:
diff changeset
12 self.name = name
2
22ccc05756de don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents: 1
diff changeset
13
22ccc05756de don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents: 1
diff changeset
14 def darcsTime(self):
22ccc05756de don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents: 1
diff changeset
15 j = os.path.join
22ccc05756de don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents: 1
diff changeset
16 darcsPriv = j(self.config['darcsDir'], self.name, '_darcs')
22ccc05756de don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents: 1
diff changeset
17 for n in ['inventory', 'hashed_inventory']:
22ccc05756de don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents: 1
diff changeset
18 if os.path.exists(j(darcsPriv, n)):
22ccc05756de don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents: 1
diff changeset
19 return os.path.getmtime(j(darcsPriv, n))
22ccc05756de don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents: 1
diff changeset
20 raise ValueError("can't find a darcs time")
0
drewp@bigasterisk.com
parents:
diff changeset
21
drewp@bigasterisk.com
parents:
diff changeset
22 def gitDir(self):
drewp@bigasterisk.com
parents:
diff changeset
23 gitDir = os.path.join(self.config['gitSyncDir'], self.name)
drewp@bigasterisk.com
parents:
diff changeset
24 try:
drewp@bigasterisk.com
parents:
diff changeset
25 os.mkdir(gitDir)
drewp@bigasterisk.com
parents:
diff changeset
26 except OSError: pass
drewp@bigasterisk.com
parents:
diff changeset
27 return gitDir
drewp@bigasterisk.com
parents:
diff changeset
28
drewp@bigasterisk.com
parents:
diff changeset
29 def syncToLocalGit(self):
drewp@bigasterisk.com
parents:
diff changeset
30 darcsDir = os.path.join(self.config['darcsDir'], self.name)
drewp@bigasterisk.com
parents:
diff changeset
31 try:
drewp@bigasterisk.com
parents:
diff changeset
32 os.rmdir(os.path.join(darcsDir, 'darcs_testing_for_nfs'))
drewp@bigasterisk.com
parents:
diff changeset
33 except OSError: pass
3
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
34 self.runGitCommand([self.config['darcsToGitCmd'], '--verbose', darcsDir])
0
drewp@bigasterisk.com
parents:
diff changeset
35
drewp@bigasterisk.com
parents:
diff changeset
36 def runGitCommand(self, args):
1
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
37 try:
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
38 subprocess.check_call(args, cwd=self.gitDir(),
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
39 env={'SSH_AUTH_SOCK': self.config['SSH_AUTH_SOCK'],
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
40 'HOME': os.environ['HOME'], # darcs-to-git uses this
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
41 })
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
42 except:
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
43 log.error("in %s" % self.gitDir())
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
44 raise
0
drewp@bigasterisk.com
parents:
diff changeset
45
drewp@bigasterisk.com
parents:
diff changeset
46 def makeGitHubRepo(self):
drewp@bigasterisk.com
parents:
diff changeset
47 try:
1
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
48 self.gh.create_repo(self.name)
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
49 except GithubException, e:
3
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
50 assert e.data['errors'][0]['message'].startswith('name already exists'), (e, self.name)
0
drewp@bigasterisk.com
parents:
diff changeset
51 return
drewp@bigasterisk.com
parents:
diff changeset
52 self.runGitCommand(['git', 'remote', 'add', 'origin',
1
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
53 'git@github.com:%s/%s.git' % (self.gh.login,
0
drewp@bigasterisk.com
parents:
diff changeset
54 self.name)])
drewp@bigasterisk.com
parents:
diff changeset
55
drewp@bigasterisk.com
parents:
diff changeset
56 def pushToGitHub(self):
drewp@bigasterisk.com
parents:
diff changeset
57 self.runGitCommand(['git', 'push', 'origin', 'master'])
drewp@bigasterisk.com
parents:
diff changeset
58
3
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
59 def getSshAuthSock():
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
60 keychain = subprocess.check_output([
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
61 "keychain", "--noask", "--quiet", "--eval", "id_rsa"])
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
62 m = re.search(r'SSH_AUTH_SOCK=([^; \n]+)', keychain)
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
63 if m is not None:
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
64 return m.group(1)
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
65 else:
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
66 raise ValueError("couldn't find SSH_AUTH_SOCK in output "
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
67 "from keychain: %r" % keychain)
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
68
1
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
69 config = json.loads(open("config.json").read())
3
4077903a9520 upgrade PyGithub. use keychain for ssh sock path
drewp@bigasterisk.com
parents: 2
diff changeset
70 config['SSH_AUTH_SOCK'] = getSshAuthSock()
1
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
71
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
72 # to get this token:
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
73 # curl -u drewp https://api.github.com/authorizations -d '{"scopes":["repo"]}'
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
74 # from http://developer.github.com/v3/oauth/#oauth-authorizations-api
1da34eecdfd8 port to PyGithub so repo creates work again
drewp@bigasterisk.com
parents: 0
diff changeset
75 gh = Github(config['gitHubToken']).get_user()
0
drewp@bigasterisk.com
parents:
diff changeset
76
drewp@bigasterisk.com
parents:
diff changeset
77 for proj in os.listdir(config['darcsDir']):
drewp@bigasterisk.com
parents:
diff changeset
78 if not os.path.isdir(os.path.join(config['darcsDir'], proj)):
drewp@bigasterisk.com
parents:
diff changeset
79 continue
drewp@bigasterisk.com
parents:
diff changeset
80 try:
drewp@bigasterisk.com
parents:
diff changeset
81 p = Project(config, gh, proj)
2
22ccc05756de don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents: 1
diff changeset
82
22ccc05756de don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents: 1
diff changeset
83 if p.darcsTime() < time.time() - 86400*config['tooOldDays']:
22ccc05756de don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents: 1
diff changeset
84 continue
22ccc05756de don't waste time scanning repos that appear to have no recent activity
drewp@bigasterisk.com
parents: 1
diff changeset
85
0
drewp@bigasterisk.com
parents:
diff changeset
86 log.info("syncing %s" % proj)
drewp@bigasterisk.com
parents:
diff changeset
87 p.syncToLocalGit()
drewp@bigasterisk.com
parents:
diff changeset
88 p.makeGitHubRepo()
drewp@bigasterisk.com
parents:
diff changeset
89 p.pushToGitHub()
drewp@bigasterisk.com
parents:
diff changeset
90 except Exception, e:
drewp@bigasterisk.com
parents:
diff changeset
91 traceback.print_exc()
drewp@bigasterisk.com
parents:
diff changeset
92