From d51e41a9bb69b19a4bebaf4bd6ca844e94c5d760 Mon Sep 17 00:00:00 2001 From: Hasan Wan Date: Sun, 6 Jan 2013 14:03:47 +0800 Subject: [PATCH] resolve local .oscrc issue, add project filter for repomaker Change-Id: I91fda621e39b55a0bf960f0ca0dd60b929f3c909 Signed-off-by: Hasan Wan --- common/buildservice.py | 102 -------------------------- common/tempbuildpkg.py | 190 ++++++++++++++++++++++++++++++++++++++++++++++++- job_pre_release.py | 13 ++-- job_repomaker.py | 42 ++++++++--- 4 files changed, 226 insertions(+), 121 deletions(-) diff --git a/common/buildservice.py b/common/buildservice.py index 0684cb8..3be77de 100644 --- a/common/buildservice.py +++ b/common/buildservice.py @@ -1270,105 +1270,3 @@ class BuildService(object): except (M2Crypto.m2urllib2.URLError, M2Crypto.SSL.SSLError), err: raise errors.ObsError(str(err)) -PRJ_TEMPLATE = """ - - %(desp)s - - - - - <%(build)s/> - - - - i586 - -""" - -class TempBuildService(): - - def __init__(self, project, apiurl = None, oscrc = None): - self.project = project - self.bs = BuildService(apiurl = apiurl, oscrc = oscrc) - - def get_repostatus(self): - return self.bs.getRepoState(self.project) - - def cleanup(self): - os.system("osc -H -d -v rdelete --recursive %s -m 'deleted'" %(self.project)) - return self.bs.deleteProject(self.project) - - def is_published(self): - try: - return set(self.get_repostatus().values()) == set(['published']) - except: - return False - - def get_abs_repopath(self): - # FIXME: hard code here - return "/srv/obs/repos/%s" %(self.project.replace(":", ":/")) - - def _su_osbrun(self): - os.setuid(getpwnam('obsrun').pw_uid) - os.setgid(getpwnam('obsrun').pw_gid) - - def extract_image_ks(self): - pg = re.compile("image-configurations.*.rpm") - repopath = self.get_abs_repopath() - found = False - for root, dirs, files in os.walk(repopath): - for f in files: - print root,f - if re.match(pg, f): - print "image configurations found: %s/%s" %(root,f) - tmpdir = tempfile.mkdtemp() - pwd = os.getcwd() - os.chdir(tmpdir) - os.system("rpm2cpio %s/%s | cpio -idmv" %(root,f)) - os.system("mkdir -p %s/%s" %(repopath, 'pre-release-images/image-configs')) - - if os.path.exists("%s/usr/share/image-configurations/image-configs.xml" %tmpdir): - shutil.copyfile("%s/usr/share/image-configurations/image-configs.xml" %tmpdir, '%s/pre-release-images/image-configs.xml' %repopath) - - os.system("cp %s/usr/share/image-configurations/*.ks %s/builddata/image-configs" %(tmpdir, repopath)) - os.chdir(pwd) - shutil.rmtree(tmpdir) - found = True - break - - if found: break - - if not found: - print "did not find any image configurations" - - def disable_build(self, user): - - targetmeta = """ - - - - - - <%(build)s/> - -"""% {'target': self.project, 'user': user, 'build': 'disable'} - print targetmeta - fileh, filename = tempfile.mkstemp(prefix="osc_metafile.", - suffix=".xml", text=True) - os.write(fileh, targetmeta) - os.close(fileh) - os.system("osc -d -v meta prj %s -F %s" %(self.project, filename)) - sys.stdout.flush() - os.unlink(filename) - - def link_project(self, src, user, desp = ''): - targetmeta = PRJ_TEMPLATE % {'target': self.project, 'src': src, - 'user': user, 'build': 'enable', - 'desp': desp} - - fileh, filename = tempfile.mkstemp(prefix="osc_metafile.", - suffix=".xml", text=True) - os.write(fileh, targetmeta) - os.close(fileh) - os.system("osc meta prj %s -F %s" %(self.project, filename)) - os.unlink(filename) diff --git a/common/tempbuildpkg.py b/common/tempbuildpkg.py index 795a2ca..5259814 100644 --- a/common/tempbuildpkg.py +++ b/common/tempbuildpkg.py @@ -1,9 +1,92 @@ import os import time - +import tempfile +import base64 import runner import buildservice +def encode_passwd(passwd): + '''encode passwd by bz2 and base64''' + return base64.b64encode(passwd.encode('bz2')) + +OSCRC_TEMPLATE = """[general] +apiurl = %(apiurl)s +plaintext_passwd=0 +use_keyring=0 +http_debug = %(http_debug)s +debug = %(debug)s +gnome_keyring=0 +[%(apiurl)s] +user=%(user)s +passx=%(passwdx)s +""" + + +# Copy from gitbuildsys.utils +class Temp(object): + """ + Create temporary file or directory. + Delete it automatically when object is destroyed. + + """ + + def __init__(self, suffix='', prefix='tmp', dirn=None, + directory=False, content=None): + """ + Create file or directory using tempfile.mk[sd]temp. + If content is provided write it to the file. + + """ + self.directory = directory + self.path = None + + try: + if dirn: + target_dir = os.path.abspath(os.path.join(dirn, prefix)) + else: + target_dir = os.path.abspath(prefix) + target_dir = os.path.dirname(target_dir) + + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + if directory: + path = tempfile.mkdtemp(suffix, prefix, dirn) + else: + (fds, path) = tempfile.mkstemp(suffix, prefix, dirn) + os.close(fds) + if content: + with file(path, 'w+') as fobj: + fobj.write(content) + except OSError, err: + raise errors.GbsError("Failed to create dir or file on %s: %s" % \ + (target_dir, str(err))) + self.path = path + + def __del__(self): + """Remove it when object is destroyed.""" + if self.path and os.path.exists(self.path): + if self.directory: + shutil.rmtree(self.path, True) + else: + os.unlink(self.path) + + +class BuildService2(buildservice.BuildService): + def __init__(self, apiurl, apiuser, apipasswd): + oscrc = OSCRC_TEMPLATE % { + "http_debug": 0, + "debug": 0, + "apiurl": apiurl, + "user": apiuser, + "passwdx": encode_passwd(apipasswd) + } + + self.apiurl = apiurl + tmpf = Temp(prefix='.oscrc', content=oscrc) + oscrcpath = tmpf.path + buildservice.BuildService(apiurl, oscrc=oscrcpath) + class TempPackageBuild: def __init__(self, apiurl, oscrc, project, package = None): @@ -94,3 +177,108 @@ class TempPackageBuild: def del_itself(self): self.bs.deleteProject(self.project) + +PRJ_TEMPLATE = """ + + %(desp)s + + + + + <%(build)s/> + + + + i586 + +""" + +class TempBuildService(): + + def __init__(self, project, apiurl, apiuser, apipasswd): + self.project = project + self.bs = BuildService2(apiurl = apiurl, + apiuser = apiuser, + apipasswd = apipasswd) + + def get_repostatus(self): + return self.bs.getRepoState(self.project) + + def cleanup(self): + os.system("osc -H -d -v rdelete --recursive %s -m 'deleted'" %(self.project)) + return self.bs.deleteProject(self.project) + + def is_published(self): + try: + return set(self.get_repostatus().values()) == set(['published']) + except: + return False + + def get_abs_repopath(self): + # FIXME: hard code here + return "/srv/obs/repos/%s" %(self.project.replace(":", ":/")) + + def _su_osbrun(self): + os.setuid(getpwnam('obsrun').pw_uid) + os.setgid(getpwnam('obsrun').pw_gid) + + def extract_image_ks(self): + pg = re.compile("image-configurations.*.rpm") + repopath = self.get_abs_repopath() + found = False + for root, dirs, files in os.walk(repopath): + for f in files: + print root,f + if re.match(pg, f): + print "image configurations found: %s/%s" %(root,f) + tmpdir = tempfile.mkdtemp() + pwd = os.getcwd() + os.chdir(tmpdir) + os.system("rpm2cpio %s/%s | cpio -idmv" %(root,f)) + os.system("mkdir -p %s/%s" %(repopath, 'pre-release-images/image-configs')) + + if os.path.exists("%s/usr/share/image-configurations/image-configs.xml" %tmpdir): + shutil.copyfile("%s/usr/share/image-configurations/image-configs.xml" %tmpdir, '%s/pre-release-images/image-configs.xml' %repopath) + + os.system("cp %s/usr/share/image-configurations/*.ks %s/builddata/image-configs" %(tmpdir, repopath)) + os.chdir(pwd) + shutil.rmtree(tmpdir) + found = True + break + + if found: break + + if not found: + print "did not find any image configurations" + + def disable_build(self, user): + + targetmeta = """ + + + + + + <%(build)s/> + +"""% {'target': self.project, 'user': user, 'build': 'disable'} + print targetmeta + fileh, filename = tempfile.mkstemp(prefix="osc_metafile.", + suffix=".xml", text=True) + os.write(fileh, targetmeta) + os.close(fileh) + os.system("osc -d -v meta prj %s -F %s" %(self.project, filename)) + sys.stdout.flush() + os.unlink(filename) + + def link_project(self, src, user, desp = ''): + targetmeta = PRJ_TEMPLATE % {'target': self.project, 'src': src, + 'user': user, 'build': 'enable', + 'desp': desp} + + fileh, filename = tempfile.mkstemp(prefix="osc_metafile.", + suffix=".xml", text=True) + os.write(fileh, targetmeta) + os.close(fileh) + os.system("osc meta prj %s -F %s" %(self.project, filename)) + os.unlink(filename) diff --git a/job_pre_release.py b/job_pre_release.py index ade34d6..9c6d6c7 100755 --- a/job_pre_release.py +++ b/job_pre_release.py @@ -7,9 +7,8 @@ from common.envparas import export from common import utils from common import git -from common import buildservice +from common.tempbuildpkg import BuildService2, TempBuildService from common.buildtrigger import trigger_info -from common.buildservice import TempBuildService from testprojects.prerelease import PreRelease import os import tempfile @@ -20,12 +19,12 @@ envparas = ['JENKINS_HOME', 'WORKSPACE', 'TRIGGER_INFO', 'OBS_API_URL', - 'OBS_OSCRC_PATH', + 'OBS_API_USERNAME', + 'OBS_API_PASSWD', 'GERRIT_HOSTNAME', 'GERRIT_USERNAME', 'GERRIT_SSHPORT', 'PATH_BUILDS', - 'PATH_OSCRC', 'GIT_PROJECT', 'GIT_CACHE_DIR'] @@ -51,13 +50,13 @@ def create(prerelease): mygit = git.Git(prjdir) - bs = TempBuildService(prerelease.obs_project_name(), OBS_API_URL, OBS_OSCRC_PATH) + bs = TempBuildService(prerelease.obs_project_name(), OBS_API_URL, OBS_API_USERNAME, OBS_API_PASSWD) prerelease.create(bs, mygit) def cleanup(prerelease): - bs = TempBuildService(prerelease.obs_project, OBS_API_URL, PATH_OSCRC) + bs = TempBuildService(prerelease.obs_project, OBS_API_URL, OBS_API_USERNAME, OBS_API_PASSWD) bs.cleanup() print "OBS Project %s deleted" %prerelease.obs_project @@ -76,7 +75,7 @@ if __name__ == '__main__': locals().update(trigger_info(TRIGGER_INFO)) - prerelease = PreRelease(OBS_REQ_PRJ, GERRIT_USERNAME, GIT_TAG, GIT_PROJECT) + prerelease = PreRelease(OBS_REQ_PRJ, OBS_API_USERNAME, GIT_TAG, GIT_PROJECT) if action == 'create': create(prerelease) diff --git a/job_repomaker.py b/job_repomaker.py index 9eb3191..85ee4cd 100755 --- a/job_repomaker.py +++ b/job_repomaker.py @@ -14,7 +14,7 @@ except ImportError: import simplejson as json from common.envparas import export -from common.buildservice import BuildService, TempBuildService +from common.tempbuildpkg import BuildService2 from common.buildtrigger import trigger_info, trigger_next import repomaker import base64 @@ -36,12 +36,13 @@ CONFS = {} PARAM_LIST = ['PATH_BUILDS', 'PATH_REPO_CONF', 'OBS_API_URL', - 'OBS_USERNAME', - 'PATH_OSCRC', + 'OBS_API_USERNAME', + 'OBS_API_PASSWD', 'WORKSPACE', 'PATH_RAW_REPOS', 'OBS_TRIGGERS_PATH', 'OBS_BUILDING_PATH', + 'REPOMAKER_PROJECTS', 'NO_ARMV8', 'MAILTO', 'SANDBOX_REPO_BASEURL', @@ -56,6 +57,16 @@ PARAM_LIST = ['PATH_BUILDS', export(PARAM_LIST, locals()) +def is_intrested_project(project): + for keyword in REPOMAKER_PROJECTS.split(' '): + if keyword.endswith('*'): + if project.startswith(keyword[:-1]): + return True + else: + if project == keyword: + return True + return False + def image_dispatch(repoinfo): ready = repoinfo['SNAPSHOT'] @@ -89,7 +100,9 @@ def image_dispatch(repoinfo): def create_repo(serverconf, event_fields): try: - bs = BuildService(apiurl=serverconf.apiurl, oscrc=serverconf.oscrc) + bs = BuildService2(apiurl = serverconf.apiurl, + apiuser = serverconf.apiuser, + apipasswd = serverconf.apipasswd) except Exception, e: print 'OBS access errors: ', str(e) sys.exit(-1) @@ -111,13 +124,13 @@ def temp_repo_define(event_files, repos_conf): AddRepos: %(addrepos)s Target: %(target)s SandboxOf: %(sandboxof)s - SnapshotDir: /srv/snapshots/pre-release/ + SnapshotDir: %(path_builds)s/pre-release/ Release: "%(release)s" Architectures: %(arch)s """ project = event_files["project"] repo = event_files["repo"] - match = re.match("(.*):prerelease:([\.\w-]*)", project) + match = re.match(".*:prerelease:(.*):([\.\w-]*)", project) if not match: raise Exception, "Ivalid project name found" @@ -131,6 +144,7 @@ def temp_repo_define(event_files, repos_conf): "sandbox_repo": os.path.basename(repos_conf.get_repo(target_prj)['Location']), "project": project, "target": repo, + "path_builds": PATH_BUILDS, "release": timestamp, "arch":["ia32"], "addrepos": ["Tizen:Base"] #FIXME @@ -150,7 +164,8 @@ if __name__ == "__main__": server_conf.raw_repos = PATH_RAW_REPOS server_conf.repos_conf = PATH_REPO_CONF server_conf.apiurl = OBS_API_URL - server_conf.oscrc = PATH_OSCRC + server_conf.apiuser = OBS_API_USERNAME + server_conf.apipasswd = OBS_API_PASSWD server_conf.builds = PATH_BUILDS server_conf.obs_triggers_path = OBS_TRIGGERS_PATH or '/srv/obs/repos_sync' server_conf.obs_building_path = OBS_BUILDING_PATH or '/srv/obs/build' @@ -162,11 +177,16 @@ if __name__ == "__main__": repo_conf_fn = None event_fields = trigger_info(TRIGGER_INFO) - + if not event_fields: print "Invalid OBS event: %s" %(OBS_EVENT_STRING) sys.exit(-1) + project = event_fields["project"] + if not is_intrested_project(project): + print "Pass, not intrested project %s" %project + sys.exit(0) + # Pre-release image creation temp project tempbuild = None if event_fields["project"].find(":prerelease:") > 0: @@ -180,9 +200,9 @@ if __name__ == "__main__": server_conf.repos_conf = temp_conf_fn project = event_fields["project"] - match = re.match("(.*):prerelease:([\.\w-]*)", project) - - bs = BuildService(apiurl=server_conf.apiurl, oscrc=server_conf.oscrc) + bs = BuildService2(apiurl = server_conf.apiurl, + apiuser = server_conf.apiuser, + apipasswd = server_conf.apipasswd) review_repo = repomaker.ReviewRepo(project, server_conf, bs) review_repo.create() -- 2.7.4