From: Ed Bartosh Date: Fri, 26 Oct 2012 21:18:00 +0000 (+0300) Subject: Implemented uploading only changed files to OBS X-Git-Tag: 0.11~3 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f484f0a8485d5c6e87b5c21cdd545a430f60e955;p=tools%2Fgbs.git Implemented uploading only changed files to OBS Before uploading local files to OBS remotebuild code now checks if it's really needed. Only new and changed files are uploaded. Old files are deleted from OBS project. This change also reorganizes remotebuild code a bit to avoid repeated queries to OBS. Fixes: #474 Change-Id: Ib67e84765b6ffc39de7185c437342d7a261018de Signed-off-by: Ed Bartosh --- diff --git a/gitbuildsys/cmd_remotebuild.py b/gitbuildsys/cmd_remotebuild.py index a58652d..98585c7 100644 --- a/gitbuildsys/cmd_remotebuild.py +++ b/gitbuildsys/cmd_remotebuild.py @@ -160,17 +160,6 @@ def main(args): % '\n'.join(results)) return 0 - msger.info('checking status of obs project: %s ...' % target_prj) - if not api.exists(target_prj): - msger.info('copying settings of %s to %s' % (base_prj, target_prj)) - api.copy_project(base_prj, target_prj) - - if api.exists(target_prj, package): - msger.info('cleaning existing package') - api.remove_files(target_prj, package) - else: - msger.info('creating new package %s/%s' % (target_prj, package)) - api.create_package(target_prj, package) except OSCError, err: msger.error(str(err)) @@ -189,16 +178,40 @@ def main(args): except GitRepositoryError, exc: msger.error('failed to get commit info: %s' % exc) - msger.info('commit packaging files to build server ...') + files = glob.glob("%s/*" % exportdir) try: - api.commit_files(target_prj, package, - glob.glob("%s/*" % exportdir), commit_msg) - except errors.ObsError, exc: - msger.error('commit packages fail: %s, please check the permission '\ - 'of target project:%s' % (exc, target_prj)) + msger.info('checking status of obs project: %s ...' % target_prj) + if api.exists(target_prj, package): + old, _not_changed, changed, new = api.diff_files(target_prj, + package, files) + if old: + msger.info("removing old files from OBS project") + api.remove_files(target_prj, package, old) + commit_files = changed + new + else: + msger.info('copying settings of %s to %s' % (base_prj, target_prj)) + api.copy_project(base_prj, target_prj) + msger.info('creating new package %s/%s' % (target_prj, package)) + api.create_package(target_prj, package) + # new project - submitting all local files + commit_files = files + except OSCError, err: + msger.error(str(err)) + + if not commit_files: + msger.warning("No local changes found. Triggering rebuild") + api.rebuild(target_prj, package, obs_arch) + else: + msger.info('commit packaging files to build server ...') + commit_files = [(fpath, fpath in commit_files) for fpath in files] + try: + api.commit_files(target_prj, package, commit_files, commit_msg) + except errors.ObsError, exc: + msger.error('commit packages fail: %s, please check the permission '\ + 'of target project:%s' % (exc, target_prj)) + msger.info('local changes submitted to build server successfully') - msger.info('local changes submitted to build server successfully') msger.info('follow the link to monitor the build progress:\n' ' %s/package/show?package=%s&project=%s' \ % (apiurl.replace('api', 'build'), package, target_prj)) diff --git a/gitbuildsys/oscapi.py b/gitbuildsys/oscapi.py index 89e9b69..c5fcf54 100644 --- a/gitbuildsys/oscapi.py +++ b/gitbuildsys/oscapi.py @@ -167,6 +167,59 @@ class OSC(object): return True + def rebuild(self, prj, pkg, arch): + """Rebuild package.""" + try: + return core.rebuild(self.apiurl, prj, pkg, repo=None, arch=arch) + except (urllib2.URLError, M2Crypto.m2urllib2.URLError, \ + M2Crypto.SSL.SSLError), err: + raise ObsError("Can't trigger rebuild for %s/%s: %s" % \ + (prj, pkg, str(err))) + except SSLVerificationError: + raise ObsError("SSL verification error.") + + def diff_files(self, prj, pkg, paths): + """ + Find difference between local and remote filelists + Return 4 lists: (old, not changed, changed, new) + where: + old - present only remotely + changed - present remotely and locally and differ + not changed - present remotely and locally and does not not differ + new - present only locally + old is a list of remote filenames + changed, not changed and new are lists of local filepaths + """ + + # Get list of files from the OBS + rfiles = core.meta_get_filelist(self.apiurl, prj, pkg, verbose=True) + + old, not_changed, changed, new = [], [], [], [] + + if not rfiles: + # no remote files - all local files are new + return old, not_changed, changed, paths[:] + + # Helper dictionary helps to avoid looping over remote files + rdict = dict((fobj.name, (fobj.size, fobj.md5)) for fobj in rfiles) + + for lpath in paths: + lname = os.path.basename(lpath) + if lname in rdict: + lsize = os.path.getsize(lpath) + rsize, rmd5 = rdict[lname] + if rsize == lsize and rmd5 == core.dgst(lpath): + not_changed.append(lpath) + else: + changed.append(lpath) + # remove processed files from the remote dict + # after processing only old files will be letf there + rdict.pop(lname) + else: + new.append(lpath) + + return rdict.keys(), not_changed, changed, new + def commit_files(self, prj, pkg, files, message): """Commits files to OBS.""" @@ -176,7 +229,7 @@ class OSC(object): url = core.makeurl(self.apiurl, ['source', prj, pkg], query=query) xml = "" - for fpath in files: + for fpath, _ in files: with open(fpath) as fhandle: xml += '' % \ (os.path.basename(fpath), hexdigest(fhandle)) @@ -184,12 +237,13 @@ class OSC(object): try: self.core_http(core.http_POST, url, data=xml) - for fpath in files: - put_url = core.makeurl( - self.apiurl, ['source', prj, pkg, - pathname2url(os.path.basename(fpath))], - query="rev=repository") - self.core_http(core.http_PUT, put_url, filep=fpath) + for fpath, commit_flag in files: + if commit_flag: + put_url = core.makeurl( + self.apiurl, ['source', prj, pkg, + pathname2url(os.path.basename(fpath))], + query="rev=repository") + self.core_http(core.http_PUT, put_url, filep=fpath) self.core_http(core.http_POST, url, data=xml) except OSCError, err: raise ObsError("can't commit files to %s/%s: %s" % (prj, pkg, err))