improve function of cloning git project to local

Change-Id: I939dfaa8d055c740a92683e0cdbe2015802e2383
Signed-off-by: Lin Yang <lin.a.yang@intel.com>

diff --git a/common/git.py b/common/git.py
index 7a7dd3f..c636744 100644 (file)
--- a/common/git.py
+++ b/common/git.py
@@ -23,7 +23,8 @@ import shutil
 # internal modules
 import runner
 import errors
-from utils import Workdir
+from utils import Workdir, retry
+
 class Git:
     def __init__(self, path):
         if not os.path.isdir(os.path.join(path, '.git')):
@@ -422,30 +423,59 @@ class Git:
         else:
             return None
 
-def update_git_project(workdir, prj, giturl):
-    print('Update git tree %s under %s' % (prj, workdir))
+def _update_gitproject(localdir):
+    """Fetch latest code to local dir"""
+
+    print '\nUpdating local git: %s' % localdir
+    try:
+        localgit = Git(localdir)
+    except errors.GitError, err:
+        print('git execption: ', err)
+        shutil.rmtree(localdir)
+        return False
+
+    if not (localgit.pull('--all') and localgit.fetch('--tags')):
+        shutil.rmtree(prjdir)
+        return False
+
+    return True
+
+def _clone_gitproject(gerritprj, localdir):
+    """Clone gerrit project from remote to local dir"""
+
     result = True
-    prjdir = os.path.join(workdir, prj)
-
-    with Workdir(workdir):
-        if os.path.isdir(prjdir):
-            try:
-                gitprj = Git(prjdir)
-            except errors.GitError, err:
-                print('git execption: ', err)
-                shutil.rmtree(prjdir)
-                result = False
-            else:
-                if not (gitprj.pull('--all') and gitprj.fetch('--tags')):
-                    shutil.rmtree(prjdir)
-                    result = False
+
+    # fetch latest code if local dir already exists
+    if os.path.isdir(localdir) and _update_gitproject(localdir):
+        return True
+
+    # clone gerrit project from remote to local dir
+    giturl = 'ssh://%s@%s:%s' % (os.getenv('GERRIT_USERNAME'),
+                                 os.getenv('GERRIT_HOSTNAME'),
+                                 os.getenv('GERRIT_SSHPORT'))
+    try:
+        if os.path.isdir(os.path.join(os.getenv('GIT_CACHE_DIR'), gerritprj)):
+            # use local cache repo as reference to clone
+            gitcmd = 'git clone %s/%s --reference %s %s' % (giturl,
+                                                            gerritprj,
+                                                            os.path.join(os.getenv('GIT_CACHE_DIR'), gerritprj),
+                                                            localdir)
         else:
+            gitcmd = 'git clone %s/%s %s' % (giturl, gerritprj, localdir)
+
+        if runner.show(gitcmd)[0] != 0:
             result = False
+    except Exception, ex:
+        result = False
+        print '\nExcept occur when clone gerrit project'
+        print ex
+
+    if not result:
+        print '\nClone gerrit project Failed.'
+        shutil.rmtree(localdir)
 
-        if not result:
-            ret, outs = runner.show('git clone %s/%s %s' % (giturl, prj, prj))
-            if ret !=0:
-                shutil.rmtree(prjdir)
-            else:
-                result = True
     return result
+
+def clone_gitproject(gerritprj, localdir):
+    """Clone gerrit project from remote to local dir"""
+    return retry(_clone_gitproject, (gerritprj, localdir))

diff --git a/job_deletetag.py b/job_deletetag.py
index 2f2bcc2..d3b20f8 100755 (executable)
--- a/job_deletetag.py
+++ b/job_deletetag.py
@@ -10,7 +10,7 @@ import shutil
 
 from common import runner
 from common import utils
-from common import git
+from common.git import Git, clone_gitproject
 from common.envparas import export
 from common.gerrit import Gerrit
 
@@ -31,19 +31,11 @@ GIT_URL = 'ssh://%s@%s:%s' % (GERRIT_USERNAME, GERRIT_HOSTNAME, GERRIT_SSHPORT)
 def cloneTmpGit(tmpdir):
     gitdir = os.path.join(GIT_CACHE_DIR, GERRIT_PROJECT)
 
-    try:
-        if os.path.exists(gitdir):
-           print '\nuse local repo as reference to clone'
-           if runner.show('git clone %s/%s --reference %s %s' % (GIT_URL, GERRIT_PROJECT, gitdir, '%s/%s' % (tmpdir, GERRIT_PROJECT)))[0]:
-               print 'use locl repo as reference to clone: Failed.'
-    except Exception, ex:
-        print '\nExcept occur when use reference repo to clone code'
-        print ex
-    if not utils.retry(git.update_git_project, (tmpdir, GERRIT_PROJECT, GIT_URL)):
-        shutil.rmtree(tmpdir)
+    # clone gerrit project to local dir
+    if not clone_gitproject(GERRIT_PROJECT, os.path.join(tmpdir, GERRIT_PROJECT)):
         exit(-1)
 
-    return git.Git(os.path.join(tmpdir, GERRIT_PROJECT))
+    return Git(os.path.join(tmpdir, GERRIT_PROJECT))
 
 def deleteTags(mygit, revision):
 

diff --git a/job_policycheck.py b/job_policycheck.py
index 9eabca6..792e5c8 100755 (executable)
--- a/job_policycheck.py
+++ b/job_policycheck.py
@@ -13,7 +13,7 @@ import shutil
 # internal module
 from common import runner
 from common import utils
-from common import git
+from common.git import Git, clone_gitproject
 from common import obspkg
 from common import gerrit
 from common import mapping
@@ -87,20 +87,11 @@ if __name__ == '__main__':
         mygerrit.review(commit = GERRIT_PATCHSET_REVISION, message = checkmappingmsg)
         end()
 
-    # update local git tree from remote
-    try:
-        if os.path.exists(os.path.join(GIT_CACHE_DIR, GERRIT_PROJECT)):
-            print '\nuse local repo as reference to clone'
-            if runner.show('git clone %s/%s --reference %s %s' % (GIT_URL, GERRIT_PROJECT, os.path.join(GIT_CACHE_DIR, GERRIT_PROJECT), '%s/%s' % (tmpdir,GERRIT_PROJECT)))[0]:
-                print 'use local repo as reference to clone: Failed.'
-                shutil.rmtree(os.path.join(GIT_CACHE_DIR, GERRIT_PROJECT))
-    except Exception, ex:
-        print '\nExcept occur when use reference repo to clone code'
-        print ex
-    if not utils.retry(git.update_git_project, (tmpdir, GERRIT_PROJECT, GIT_URL)):
+    # clone gerrit project to local dir
+    if not clone_gitproject(GERRIT_PROJECT, prjdir):
         end(1)
 
-    mygit = git.Git(prjdir)
+    mygit = Git(prjdir)
     mygit.fetch('origin', GERRIT_REFSPEC, '-t')
     mygit.checkout('FETCH_HEAD')
 

diff --git a/job_pre_release.py b/job_pre_release.py
index 9c6d6c7..5cb1848 100755 (executable)
--- a/job_pre_release.py
+++ b/job_pre_release.py
@@ -6,7 +6,7 @@
 
 from common.envparas import export
 from common import utils
-from common import git
+from common.git import Git, clone_gitproject
 from common.tempbuildpkg import BuildService2, TempBuildService
 from common.buildtrigger import trigger_info
 from testprojects.prerelease import PreRelease
@@ -33,22 +33,13 @@ export(envparas, locals())
 GIT_URL = 'ssh://%s@%s:%s' % (GERRIT_USERNAME, GERRIT_HOSTNAME, GERRIT_SSHPORT)
 
 def create(prerelease):
-    try:
-        if os.path.exists(os.path.join(GIT_CACHE_DIR, GIT_PROJECT)):
-            print '\nuse local repo as reference to clone'
-            if runner.show('git clone %s/%s --reference %s %s' % (GIT_URL, GIT_PROJECT, os.path.join('GIT_CACHE_DIR', GIT_PROJECT), '%s/%s' % (tmpdir,GIT_PROJECT)))[0]:
-                print 'use local repo as reference to clone: Failed.'
-                shutil.rmtree(os.path.join(GIT_CACHE_DIR, GERRIT_PROJECT))
-    except Exception, ex:
-        print '\nExcept occur when use reference repo to clone code'
-        print ex
-    if not utils.retry(git.update_git_project, (WORKSPACE, GIT_PROJECT, GIT_URL)):
-        end(1)
-    sys.stdout.flush()
-
     prjdir = os.path.join(WORKSPACE, GIT_PROJECT)
 
-    mygit = git.Git(prjdir)
+    # clone gerrit project to local dir
+    if not clone_gitproject(GERRIT_PROJECT, prjdir):
+        exit(1)
+
+    mygit = Git(prjdir)
 
     bs = TempBuildService(prerelease.obs_project_name(), OBS_API_URL, OBS_API_USERNAME, OBS_API_PASSWD)
 

diff --git a/job_request.py b/job_request.py
index 66fd201..4cdf460 100755 (executable)
--- a/job_request.py
+++ b/job_request.py
@@ -2,7 +2,7 @@
 from common import runner
 from common import utils
 from common.envparas import export
-from common import git
+from common.git import Git, clone_gitproject
 from common import buildservice
 from common.gerrit import Gerrit
 from common.buildtrigger import trigger_info, trigger_next
@@ -44,24 +44,19 @@ def request_url(request_id):
     else:
         return '%s' %(request_id)
 
-def tag_info(prjdir, tag):
+def tag_info(prj, tag):
+    try:
+        mygit = Git(os.path.join(GIT_CACHE_DIR, prj))
+        if mygit.find_tag(tag):
+            return mygit.get_tag(tag)
+    except GitError, err:
+        print err
 
-    gitprj = git.Git(os.path.join(GIT_CACHE_DIR, prjdir))
-    if gitprj.find_tag(tag):
-        return gitprj.get_tag(tag)
-    else:
-        # if GIT_CACHE_DIR don't have this tag, update local git tree from remote
-        GIT_URL = 'ssh://%s@%s:%s' % (GERRIT_USERNAME, GERRIT_HOSTNAME, GERRIT_SSHPORT)
-        if os.path.exists(os.path.join(GIT_CACHE_DIR, prjdir)):
-            runner.show('git clone %s/%s --reference %s %s' % (GIT_URL, prjdir, os.path.join(GIT_CACHE_DIR, prjdir), '%s/%s' % (WORKSPACE, prjdir)))
-        if utils.retry(git.update_git_project, (WORKSPACE, prjdir, GIT_URL)):
-            gitprj = git.Git(os.path.join(WORKSPACE, prjdir))
-            return gitprj.get_tag(tag)
-        else:
-            print '-------------------------------------'
-            print "FATAL ERROR: Can not pull git tree from remote"
-            print '-------------------------------------'
-            exit(0)
+    # clone gerrit project to local dir
+    if not clone_gitproject(prj, os.path.join(WORKSPACE, prj)):                               
+        exit(1)
+    mygit = Git(os.path.join(WORKSPACE, prj))
+    return mygit.get_tag(tag)
 
 def OBS_Git_data(event_fields):
 
@@ -259,7 +254,7 @@ notify_submiter(event_fields, data)
 
 gerrit = Gerrit(GERRIT_HOSTNAME, GERRIT_USERNAME, GERRIT_SSHPORT)
 
-gitprj = git.Git('%s/%s' %(GIT_CACHE_DIR, data['GIT_PROJECT']))
+gitprj = Git('%s/%s' %(GIT_CACHE_DIR, data['GIT_PROJECT']))
 
 if event_fields['type'] == 'OBS_SRCSRV_REQUEST_STATECHANGE':
     if event_fields['state'] == 'declined':

diff --git a/job_submitobs.py b/job_submitobs.py
index 473a1a3..b9b4981 100755 (executable)
--- a/job_submitobs.py
+++ b/job_submitobs.py
@@ -16,7 +16,7 @@ import random
 # internal module
 from common import runner
 from common import utils
-from common import git
+from common.git import Git, clone_gitproject
 from common import obspkg
 from common import mapping
 from common.envparas import export
@@ -210,14 +210,11 @@ def main():
             print '\nREFNAME "%s" is deleted, exit now' % GERRIT_REFNAME
             end('success')
 
-    # check whether exist git-obs-mapping.xml in local
-    if GERRIT_PROJECT == MAPPING_PRJ or not os.path.isfile('%s/%s/git-obs-mapping.xml' % (GIT_CACHE_DIR, MAPPING_PRJ)):
+    # check whether git-obs-mapping.xml exist in local
+    if not os.path.isfile('%s/%s/git-obs-mapping.xml' % (GIT_CACHE_DIR, MAPPING_PRJ)):
         print('Update %s/git-obs-mapping.xml to local.' % MAPPING_PRJ)
-        if not utils.retry(git.update_git_project, (GIT_CACHE_DIR, MAPPING_PRJ, GIT_URL)):
+        if not clone_gitproject(MAPPING_PRJ, os.path.join(GIT_CACHE_DIR, MAPPING_PRJ)):
             end('retry')
-        if GERRIT_PROJECT == MAPPING_PRJ:
-            print '\nPulled scm/git-obs-mapping change to local, exit now'
-            end('success')
 
     # quit directly if project do not map to any OBS project
     mymapping = mapping.Mapping('%s/%s/git-obs-mapping.xml' % (GIT_CACHE_DIR, MAPPING_PRJ))
@@ -225,17 +222,8 @@ def main():
         print '\nThis project do not map to any OBS project, exit now'
         end('success')
 
-    # update local git tree from remote
-    try:
-        if os.path.exists(os.path.join(GIT_CACHE_DIR, GERRIT_PROJECT)):
-            print '\nuse local repo as reference to clone'
-            if runner.show('git clone %s/%s --reference %s %s' % (GIT_URL, GERRIT_PROJECT, os.path.join(GIT_CACHE_DIR, GERRIT_PROJECT), '%s/%s' % (tmpdir,GERRIT_PROJECT)))[0]:
-                print 'use local repo as reference to clone: Failed.'
-                shutil.rmtree(os.path.join(GIT_CACHE_DIR, GERRIT_PROJECT))
-    except Exception, ex:
-        print '\nExcept occur when use reference repo to clone code'
-        print ex
-    if not utils.retry(git.update_git_project, (tmpdir, GERRIT_PROJECT, GIT_URL)):
+    # clone gerrit project to local dir
+    if not clone_gitproject(GERRIT_PROJECT, prjdir):
         end('retry')
 
     # if scm/rpmlint-config changed, update HOME/.config/rpmlint
@@ -247,7 +235,7 @@ def main():
         print '\nPulled scm/rpmlint-config change to local, exit now'
         end('success')
 
-    mygit = git.Git(prjdir)
+    mygit = Git(prjdir)
     mygerrit = gerrit.Gerrit(GERRIT_HOSTNAME, GERRIT_USERNAME, GERRIT_SSHPORT)
 
     if GERRIT_EVENT_TYPE == "REF_UPDATED":

diff --git a/job_update_local_git.py b/job_update_local_git.py
index bdb4b80..c2f5632 100755 (executable)
--- a/job_update_local_git.py
+++ b/job_update_local_git.py
@@ -3,29 +3,18 @@
 
 """This script will pull latest change to local when remote ref updated..
 """
+import os
 
 # internal module
-from common import utils
-from common import git
+from common.git import clone_gitproject
 from common.envparas import export
 
 envparas = ['GERRIT_PROJECT',
-            'GERRIT_SSHPORT',
-            'GERRIT_USERNAME',
-            'GERRIT_HOSTNAME',
             'GIT_CACHE_DIR']
 export(envparas, locals())
-GIT_URL = 'ssh://%s@%s:%s' % (GERRIT_USERNAME, GERRIT_HOSTNAME, GERRIT_SSHPORT)
 
 if __name__ == '__main__':
     print '---[JOB STARTED]----------------------------------------'
 
-    # update local git tree from remote
-    try:
-        if not utils.retry(git.update_git_project, (GIT_CACHE_DIR, GERRIT_PROJECT, GIT_URL)):
-            print '\nUpdate local git failed!'
-            exit(1)
-    except Exception, ex:
-        print '\nExcept occur when updating local git tree'
-        print ex
-        exit(1)
+    # clone gerrit project to local dir
+    clone_gitproject(GERRIT_PROJECT, os.path.join(GIT_CACHE_DIR, GERRIT_PROJECT))