use git clone --reference to improve network performance

diff --git a/aiaiaicheck.py b/aiaiaicheck.py
index c0b05ca..32b8ea0 100755 (executable)
--- a/aiaiaicheck.py
+++ b/aiaiaicheck.py
@@ -29,8 +29,16 @@ if __name__ == '__main__':
 
     aiaiaicmd = '%s/aiaiai/aiaiai-test-patchset' % JENKINS_HOME
 
-    if utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), GERRIT_PROJECT, giturl)):
-        shutil.copytree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), prjdir, True)
+    # update local git tree from remote
+    try:
+        if os.path.exists(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)):
+            print '\nuse local repo as reference to clone'
+            if runner.show('git clone %s/%s --reference %s %s' % (giturl, GERRIT_PROJECT, os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), '%s/%s' % (tmpdir,GERRIT_PROJECT)))[0]:
+                print 'use local repo as reference to clone: Failed.'
+                shutil.rmtree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT))
+    except Exception, ex:
+        print '\nExcept occur when use reference repo to clone code'
+        print ex
     if not utils.retry(git.update_git_project, (tmpdir, GERRIT_PROJECT, giturl)):
         end(1)
 
@@ -51,5 +59,5 @@ if __name__ == '__main__':
         if line:
             msg.append(line)
     if msg:
-        runner.quiet('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, 'aiaiai check result:\n- '+'\n'.join(msg)))
+        runner.show('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, 'aiaiai check result:\n- '+'\n'.join(msg)))
     end()

diff --git a/buildcheck.py b/buildcheck.py
index df12234..be494f7 100755 (executable)
--- a/buildcheck.py
+++ b/buildcheck.py
@@ -39,8 +39,16 @@ if __name__ == '__main__':
         if not utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), mappingprj, giturl)):
             end(1)
 
-    if utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), GERRIT_PROJECT, giturl)):
-        shutil.copytree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), prjdir, True)
+    # update local git tree from remote
+    try:
+        if os.path.exists(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)):
+            print '\nuse local repo as reference to clone'
+            if runner.show('git clone %s/%s --reference %s %s' % (giturl, GERRIT_PROJECT, os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), '%s/%s' % (tmpdir,GERRIT_PROJECT)))[0]:
+                print 'use local repo as reference to clone: Failed.'
+                shutil.rmtree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT))
+    except Exception, ex:
+        print '\nExcept occur when use reference repo to clone code'
+        print ex
     if not utils.retry(git.update_git_project, (tmpdir, GERRIT_PROJECT, giturl)):
         end(1)
 
@@ -100,5 +108,5 @@ if __name__ == '__main__':
             for arch in status[repo]:
                 msg = msg + '\n\n %-15s%-15s%-15s' %(repo, arch, status[repo][arch])
         print msg
-        runner.quiet('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, msg))
+        runner.show('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, msg))
     end()

diff --git a/git.py b/git.py
index 19e83bd..cfc96f2 100644 (file)
--- a/git.py
+++ b/git.py
@@ -51,7 +51,7 @@ class Git:
         msger.debug('run command: %s' % cmdln)
 
         with Workdir(self.path):
-            ret, outs = runner.runtool(cmdln)
+            ret, outs = runner.show(cmdln)
 
         #if ret:
         #    raise errors.GitError("command error for: %s" % cmdln)
@@ -149,10 +149,8 @@ class Git:
     def checkout(self, *args):
         """checkout repository branch 'br'
         """
-        print('git checkout %s' % ' '.join(args))
         with Workdir(self.path):
             ret, outs = self._exec_git('checkout', list(args))
-            print(outs)
 
     def clean_branch(self, br):
         """Clean up repository branch 'br'
@@ -320,50 +318,40 @@ class Git:
     def show(self, *args):
         """show commit details
         """
-        print('git show %s' % ' '.join(args))
         with Workdir(self.path):
             ret, outs = self._exec_git('show', list(args))
-            print(outs)
             if not ret:
                 return outs.splitlines()
 
     def fetch(self, *args):
         """Download objects and refs from another repository
         """
-        print('git fetch %s' % ' '.join(args))
         with Workdir(self.path):
             ret, outs = self._exec_git('fetch', list(args))
-            print(outs)
             if ret:
                 raise errors.GitError("git fetch error")             
 
     def describe(self, *args):
         """Show the most recent tag that is reachable from a commit
         """
-        print('git describe %s' % ' '.join(args))
         with Workdir(self.path):
             ret, outs = self._exec_git('describe', list(args))
-            print(outs)
             if not ret:
                 return outs.strip()
 
     def pull(self, *args):
         """Fetch from and merge with another repository or a local branch
         """
-        print('git pull %s' % ' '.join(args))
         with Workdir(self.path):
             ret, outs = self._exec_git('pull', list(args))
-            print(outs)
             if ret:
                 return False
             else:
                 return True
 
     def clean(self, *args):
-        print('git clean %s' % ' '.join(args))
         with Workdir(self.path):
             ret, outs = self._exec_git('clean', list(args))
-            print(outs)
 
     def get_base_commit(self, commit):
         out = self._exec_git('log',
@@ -372,28 +360,22 @@ class Git:
         return out[1].strip()
 
     def format_patch(self, *args):
-        print('git format-patch %s' % ' '.join(args))
         with Workdir(self.path):
             ret, outs = self._exec_git('format-patch', list(args))
-            print(outs)
         if not ret:
             return outs.splitlines()
 
     def branch_contains(self, cmitid):
-        print('git branch --contains %s' % cmitid)
         with Workdir(self.path):
             ret, outs = self._exec_git('branch --contains', [cmitid])
-            print(outs)
         if not ret:
             branchs = []
             for br in outs.splitlines():
-                print 'br', type(br),br
                 if br.startswith('* '):
                     branchs.append(br[len('* '):])
                 else:
                     branchs.append(br)
-                print 'branchs', branchs
-            print(branchs)
+            print 'git branch contains %s' % branchs
             return branchs 
         else:
             return None
@@ -410,20 +392,17 @@ def update_git_project(workdir, prj, giturl):
             except errors.GitError, err:
                 print('git execption: ', err)
                 shutil.rmtree(prjdir)
-                ret, outs = runner.runtool('git clone %s/%s %s' % (giturl, prj, prj))
-                print('git clone %s/%s %s\n%s' % (giturl, prj, prj, outs))
+                ret, outs = runner.show('git clone %s/%s %s' % (giturl, prj, prj))
                 if ret !=0:
                     result = False
             else:
                 if not gitprj.pull():
                     shutil.rmtree(prjdir)
-                    ret, outs = runner.runtool('git clone %s/%s %s' % (giturl, prj, prj))
-                    print('git clone %s/%s %s\n%s' % (giturl, prj, prj, outs))
+                    ret, outs = runner.show('git clone %s/%s %s' % (giturl, prj, prj))
                     if ret !=0:
                         result = False
         else:
-            ret, outs = runner.runtool('git clone %s/%s %s' % (giturl, prj, prj))
-            print('git clone %s/%s %s\n%s' % (giturl, prj, prj, outs))
+            ret, outs = runner.show('git clone %s/%s %s' % (giturl, prj, prj))
             if ret !=0:
                 result = False
 

diff --git a/policycheck.py b/policycheck.py
index b1147d2..1246672 100755 (executable)
--- a/policycheck.py
+++ b/policycheck.py
@@ -53,9 +53,15 @@ if __name__ == '__main__':
             end(1)
 
     # update local git tree from remote
-    if utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), GERRIT_PROJECT, giturl)):
-        print('Copy %s to %s' % (os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), prjdir))
-        shutil.copytree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), prjdir, True)
+    try:
+        if os.path.exists(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)):
+            print '\nuse local repo as reference to clone'
+            if runner.show('git clone %s/%s --reference %s %s' % (giturl, GERRIT_PROJECT, os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), '%s/%s' % (tmpdir,GERRIT_PROJECT)))[0]:
+                print 'use local repo as reference to clone: Failed.'
+                shutil.rmtree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT))
+    except Exception, ex:
+        print '\nExcept occur when use reference repo to clone code'
+        print ex
     if not utils.retry(git.update_git_project, (tmpdir, GERRIT_PROJECT, giturl)):
         end(1)
 
@@ -137,6 +143,6 @@ if __name__ == '__main__':
 
     print('\n-------------------------------\ncheck obs target result:\n%s' % checkobsmsg)
     print('\n-------------------------------\ncheck specfile result:\n%s' % checkspecmsg)
-    runner.quiet('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, checkobsmsg))
-    runner.quiet('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, checkspecmsg))
+    runner.show('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, checkobsmsg))
+    runner.show('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, checkspecmsg))
     end()

diff --git a/runner.py b/runner.py
index fd8082e..6797352 100644 (file)
--- a/runner.py
+++ b/runner.py
@@ -78,6 +78,7 @@ def runtool(cmdln_or_args, catch=1):
         if catch != 3:
             os.close(dev_null)
 
+    #print 'execute cmd: %s\nreturncode: %s\n%s' %(cmdln_or_args, process.returncode, out) 
     return (process.returncode, out)
 
 def show(cmdln_or_args):
@@ -99,8 +100,8 @@ def show(cmdln_or_args):
             msg += '\n  | %s' % line
         msg += '\n  +----------------'
 
-    msger.verbose(msg)
-    return rcode
+    print msg
+    return rcode, out
 
 def outs(cmdln_or_args, catch=1):
     # get the outputs of tools

diff --git a/submitobs.py b/submitobs.py
index b924228..40d220f 100755 (executable)
--- a/submitobs.py
+++ b/submitobs.py
@@ -67,9 +67,16 @@ if __name__ == '__main__':
             end('success')
 
     # update local git tree from remote
-    if utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), GERRIT_PROJECT, giturl)):
-        print('Copy %s to %s' % (os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), prjdir))
-        shutil.copytree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), prjdir, True)
+    try:
+        if utils.retry(git.update_git_project, (os.path.join(JENKINS_HOME, 'git'), GERRIT_PROJECT, giturl)):
+            if os.path.exists(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT)):
+                print '\nuse local repo as reference to clone'
+                if runner.show('git clone %s/%s --reference %s %s' % (giturl, GERRIT_PROJECT, os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT), '%s/%s' % (tmpdir,GERRIT_PROJECT)))[0]:
+                    print 'use local repo as reference to clone: Failed.'
+                    shutil.rmtree(os.path.join(JENKINS_HOME, 'git', GERRIT_PROJECT))
+    except Exception, ex:
+        print '\nExcept occur when use reference repo to clone code'
+        print ex
     if not utils.retry(git.update_git_project, (tmpdir, GERRIT_PROJECT, giturl)):
         end('retry')
 
@@ -150,7 +157,7 @@ if __name__ == '__main__':
         # use gbs export to generate tarball
         outdir = tempfile.mkdtemp(prefix=tmpdir+'/')
         with utils.Workdir(prjdir):
-            runner.quiet('gbs export --spec=%s -o %s' % (specfile, outdir))
+            runner.show('gbs export --spec=%s -o %s' % (specfile, outdir))
             print '\ngbs export --spec=%s -o %s' % (specfile, outdir)
         tarballdir = os.path.join(outdir, os.listdir(outdir)[0])
 
@@ -203,7 +210,7 @@ if __name__ == '__main__':
     if GERRIT_CHANGE_NUMBER and GERRIT_CHANGE_NUMBER:
         comment = 'A request has been created to submit this commit to OBS %s project.\n- Submitter: %s <%s>\nTag: %s\nRequest ID: %s' % (' '.join(dstprjlist), tagger['author'], tagger['email'], tag, ' '.join(reqlist))
         print '%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, comment)
-        runner.quiet('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, comment))
+        runner.show('%s %s %s,%s --message \'"%s"\'' % (gerritcmd, 'review', GERRIT_CHANGE_NUMBER, GERRIT_PATCHSET_NUMBER, comment))
     else:
         print 'GERRIT_CHANGE_NUMBER %s and GERRIT_CHANGE_NUMBER %s is empty.' % (GERRIT_CHANGE_NUMBER, GERRIT_CHANGE_NUMBER)
     end('success')