From 54f3c7394be3f4ccc0f29ed0f7e2a840aa341360 Mon Sep 17 00:00:00 2001 From: Lin Yang Date: Mon, 30 Jul 2012 12:21:11 +0800 Subject: [PATCH] use git clone --reference to improve network performance --- aiaiaicheck.py | 14 +++++++++++--- buildcheck.py | 14 +++++++++++--- git.py | 31 +++++-------------------------- policycheck.py | 16 +++++++++++----- runner.py | 5 +++-- submitobs.py | 17 ++++++++++++----- 6 files changed, 53 insertions(+), 44 deletions(-) diff --git a/aiaiaicheck.py b/aiaiaicheck.py index c0b05ca..32b8ea0 100755 --- a/aiaiaicheck.py +++ b/aiaiaicheck.py @@ -29,8 +29,16 @@ if __name__ == '__main__': aiaiaicmd = '%s/aiaiai/aiaiai-test-patchset' % JENKINS_HOME - if utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), GERRIT_PROJECT, giturl)): - shutil.copytree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), prjdir, True) + # update local git tree from remote + try: + if os.path.exists(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)): + print '\nuse local repo as reference to clone' + if runner.show('git clone %s/%s --reference %s %s' % (giturl, GERRIT_PROJECT, os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), '%s/%s' % (tmpdir,GERRIT_PROJECT)))[0]: + print 'use local repo as reference to clone: Failed.' + shutil.rmtree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)) + except Exception, ex: + print '\nExcept occur when use reference repo to clone code' + print ex if not utils.retry(git.update_git_project, (tmpdir, GERRIT_PROJECT, giturl)): end(1) @@ -51,5 +59,5 @@ if __name__ == '__main__': if line: msg.append(line) if msg: - runner.quiet('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, 'aiaiai check result:\n- '+'\n'.join(msg))) + runner.show('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, 'aiaiai check result:\n- '+'\n'.join(msg))) end() diff --git a/buildcheck.py b/buildcheck.py index df12234..be494f7 100755 --- a/buildcheck.py +++ b/buildcheck.py @@ -39,8 +39,16 @@ if __name__ == '__main__': if not utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), mappingprj, giturl)): end(1) - if utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), GERRIT_PROJECT, giturl)): - shutil.copytree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), prjdir, True) + # update local git tree from remote + try: + if os.path.exists(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)): + print '\nuse local repo as reference to clone' + if runner.show('git clone %s/%s --reference %s %s' % (giturl, GERRIT_PROJECT, os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), '%s/%s' % (tmpdir,GERRIT_PROJECT)))[0]: + print 'use local repo as reference to clone: Failed.' + shutil.rmtree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)) + except Exception, ex: + print '\nExcept occur when use reference repo to clone code' + print ex if not utils.retry(git.update_git_project, (tmpdir, GERRIT_PROJECT, giturl)): end(1) @@ -100,5 +108,5 @@ if __name__ == '__main__': for arch in status[repo]: msg = msg + '\n\n %-15s%-15s%-15s' %(repo, arch, status[repo][arch]) print msg - runner.quiet('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, msg)) + runner.show('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, msg)) end() diff --git a/git.py b/git.py index 19e83bd..cfc96f2 100644 --- a/git.py +++ b/git.py @@ -51,7 +51,7 @@ class Git: msger.debug('run command: %s' % cmdln) with Workdir(self.path): - ret, outs = runner.runtool(cmdln) + ret, outs = runner.show(cmdln) #if ret: # raise errors.GitError("command error for: %s" % cmdln) @@ -149,10 +149,8 @@ class Git: def checkout(self, *args): """checkout repository branch 'br' """ - print('git checkout %s' % ' '.join(args)) with Workdir(self.path): ret, outs = self._exec_git('checkout', list(args)) - print(outs) def clean_branch(self, br): """Clean up repository branch 'br' @@ -320,50 +318,40 @@ class Git: def show(self, *args): """show commit details """ - print('git show %s' % ' '.join(args)) with Workdir(self.path): ret, outs = self._exec_git('show', list(args)) - print(outs) if not ret: return outs.splitlines() def fetch(self, *args): """Download objects and refs from another repository """ - print('git fetch %s' % ' '.join(args)) with Workdir(self.path): ret, outs = self._exec_git('fetch', list(args)) - print(outs) if ret: raise errors.GitError("git fetch error") def describe(self, *args): """Show the most recent tag that is reachable from a commit """ - print('git describe %s' % ' '.join(args)) with Workdir(self.path): ret, outs = self._exec_git('describe', list(args)) - print(outs) if not ret: return outs.strip() def pull(self, *args): """Fetch from and merge with another repository or a local branch """ - print('git pull %s' % ' '.join(args)) with Workdir(self.path): ret, outs = self._exec_git('pull', list(args)) - print(outs) if ret: return False else: return True def clean(self, *args): - print('git clean %s' % ' '.join(args)) with Workdir(self.path): ret, outs = self._exec_git('clean', list(args)) - print(outs) def get_base_commit(self, commit): out = self._exec_git('log', @@ -372,28 +360,22 @@ class Git: return out[1].strip() def format_patch(self, *args): - print('git format-patch %s' % ' '.join(args)) with Workdir(self.path): ret, outs = self._exec_git('format-patch', list(args)) - print(outs) if not ret: return outs.splitlines() def branch_contains(self, cmitid): - print('git branch --contains %s' % cmitid) with Workdir(self.path): ret, outs = self._exec_git('branch --contains', [cmitid]) - print(outs) if not ret: branchs = [] for br in outs.splitlines(): - print 'br', type(br),br if br.startswith('* '): branchs.append(br[len('* '):]) else: branchs.append(br) - print 'branchs', branchs - print(branchs) + print 'git branch contains %s' % branchs return branchs else: return None @@ -410,20 +392,17 @@ def update_git_project(workdir, prj, giturl): except errors.GitError, err: print('git execption: ', err) shutil.rmtree(prjdir) - ret, outs = runner.runtool('git clone %s/%s %s' % (giturl, prj, prj)) - print('git clone %s/%s %s\n%s' % (giturl, prj, prj, outs)) + ret, outs = runner.show('git clone %s/%s %s' % (giturl, prj, prj)) if ret !=0: result = False else: if not gitprj.pull(): shutil.rmtree(prjdir) - ret, outs = runner.runtool('git clone %s/%s %s' % (giturl, prj, prj)) - print('git clone %s/%s %s\n%s' % (giturl, prj, prj, outs)) + ret, outs = runner.show('git clone %s/%s %s' % (giturl, prj, prj)) if ret !=0: result = False else: - ret, outs = runner.runtool('git clone %s/%s %s' % (giturl, prj, prj)) - print('git clone %s/%s %s\n%s' % (giturl, prj, prj, outs)) + ret, outs = runner.show('git clone %s/%s %s' % (giturl, prj, prj)) if ret !=0: result = False diff --git a/policycheck.py b/policycheck.py index b1147d2..1246672 100755 --- a/policycheck.py +++ b/policycheck.py @@ -53,9 +53,15 @@ if __name__ == '__main__': end(1) # update local git tree from remote - if utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), GERRIT_PROJECT, giturl)): - print('Copy %s to %s' % (os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), prjdir)) - shutil.copytree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), prjdir, True) + try: + if os.path.exists(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)): + print '\nuse local repo as reference to clone' + if runner.show('git clone %s/%s --reference %s %s' % (giturl, GERRIT_PROJECT, os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), '%s/%s' % (tmpdir,GERRIT_PROJECT)))[0]: + print 'use local repo as reference to clone: Failed.' + shutil.rmtree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)) + except Exception, ex: + print '\nExcept occur when use reference repo to clone code' + print ex if not utils.retry(git.update_git_project, (tmpdir, GERRIT_PROJECT, giturl)): end(1) @@ -137,6 +143,6 @@ if __name__ == '__main__': print('\n-------------------------------\ncheck obs target result:\n%s' % checkobsmsg) print('\n-------------------------------\ncheck specfile result:\n%s' % checkspecmsg) - runner.quiet('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, checkobsmsg)) - runner.quiet('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, checkspecmsg)) + runner.show('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, checkobsmsg)) + runner.show('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, checkspecmsg)) end() diff --git a/runner.py b/runner.py index fd8082e..6797352 100644 --- a/runner.py +++ b/runner.py @@ -78,6 +78,7 @@ def runtool(cmdln_or_args, catch=1): if catch != 3: os.close(dev_null) + #print 'execute cmd: %s\nreturncode: %s\n%s' %(cmdln_or_args, process.returncode, out) return (process.returncode, out) def show(cmdln_or_args): @@ -99,8 +100,8 @@ def show(cmdln_or_args): msg += '\n | %s' % line msg += '\n +----------------' - msger.verbose(msg) - return rcode + print msg + return rcode, out def outs(cmdln_or_args, catch=1): # get the outputs of tools diff --git a/submitobs.py b/submitobs.py index b924228..40d220f 100755 --- a/submitobs.py +++ b/submitobs.py @@ -67,9 +67,16 @@ if __name__ == '__main__': end('success') # update local git tree from remote - if utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), GERRIT_PROJECT, giturl)): - print('Copy %s to %s' % (os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), prjdir)) - shutil.copytree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), prjdir, True) + try: + if utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), GERRIT_PROJECT, giturl)): + if os.path.exists(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)): + print '\nuse local repo as reference to clone' + if runner.show('git clone %s/%s --reference %s %s' % (giturl, GERRIT_PROJECT, os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), '%s/%s' % (tmpdir,GERRIT_PROJECT)))[0]: + print 'use local repo as reference to clone: Failed.' + shutil.rmtree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)) + except Exception, ex: + print '\nExcept occur when use reference repo to clone code' + print ex if not utils.retry(git.update_git_project, (tmpdir, GERRIT_PROJECT, giturl)): end('retry') @@ -150,7 +157,7 @@ if __name__ == '__main__': # use gbs export to generate tarball outdir = tempfile.mkdtemp(prefix=tmpdir+'/') with utils.Workdir(prjdir): - runner.quiet('gbs export --spec=%s -o %s' % (specfile, outdir)) + runner.show('gbs export --spec=%s -o %s' % (specfile, outdir)) print '\ngbs export --spec=%s -o %s' % (specfile, outdir) tarballdir = os.path.join(outdir, os.listdir(outdir)[0]) @@ -203,7 +210,7 @@ if __name__ == '__main__': if GERRIT_CHANGE_NUMBER and GERRIT_CHANGE_NUMBER: comment = 'A request has been created to submit this commit to OBS %s project.\n- Submitter: %s <%s>\nTag: %s\nRequest ID: %s' % (' '.join(dstprjlist), tagger['author'], tagger['email'], tag, ' '.join(reqlist)) print '%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, comment) - runner.quiet('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, comment)) + runner.show('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, comment)) else: print 'GERRIT_CHANGE_NUMBER %s and GERRIT_CHANGE_NUMBER %s is empty.' % (GERRIT_CHANGE_NUMBER, GERRIT_CHANGE_NUMBER) end('success') -- 2.7.4