From 7c80bbf134bb599aa25913fbd353dcb87705ccc1 Mon Sep 17 00:00:00 2001 From: gaoxuesx Date: Thu, 31 Jul 2014 15:37:36 +0800 Subject: [PATCH] [draft]Job for syncing IRIS data from scm/meta/git project client.py is IRIS Client which is a wrap of python requests, it against CSRF protection. This job will clone scm/meta/git project then check domains and git-trees files syntax and update them to IRIS by call IRIS Client. Change-Id: I9722faf15f0bf7b7bb18142c0073062d96a46b34 Signed-off-by: gaoxuesx --- common/client.py | 61 ++++++++++++++++++++++ job_scm.py | 112 +++++++++++++++++++++++++++++++++++++++++ packaging/jenkins-scripts.spec | 2 + 3 files changed, 175 insertions(+) create mode 100644 common/client.py create mode 100755 job_scm.py diff --git a/common/client.py b/common/client.py new file mode 100644 index 0000000..aa407ca --- /dev/null +++ b/common/client.py @@ -0,0 +1,61 @@ +"""IRIS requests client""" + + +import os +import sys +import urlparse +from distutils.sysconfig import get_python_lib + +# Sets python lib as the first item in python search path, +# directs Python to search modules in it first. +sys.path.insert(0, get_python_lib()) + +import requests + + +class IrisRestClient(object): + def __init__(self, server): + self.server = server + self.session = None + self.csrftoken = None + + def login(self, user, pwd): + """ + Get csrftoken from server address, use csrftoken to login server, + then get new csrftoken for accessing the server. + """ + self.session = requests.Session() + self.session.get(self.server) + csrftoken = self.session.cookies.get('csrftoken') + + assert csrftoken + + data = dict(username=user, password=pwd) + header = {'X-CSRFToken': csrftoken} + + r = self.session.post(urlparse.urljoin(self.server, '/login/'), data=data, + headers=header) + + self.csrftoken = self.session.cookies['csrftoken'] + self.header = {'X-CSRFToken': self.csrftoken} + return r + + def _post(self, url, **kwargs): + """Commom post method with header""" + r = self.session.post(urlparse.urljoin(self.server, url), + headers=self.header, **kwargs) + detail = r.json() + detail['status'] = r.status_code + return detail + + def scm_update(self, domain_file, gittree_file): + """Update scm data to IRIS""" + with open(domain_file, 'rb') as domains, open(gittree_file, 'rb') as gittrees: + files = (('domains', domains), ('gittrees', gittrees)) + return self._post('app/packagedb/scm/update', files=files) + + def scm_check(self, domain_file, gittree_file): + """Check scm project""" + with open(domain_file, 'rb') as domains, open(gittree_file, 'rb') as gittrees: + files = (('domains', domains), ('gittrees', gittrees)) + return self._post('app/packagedb/scm/check', files=files) diff --git a/job_scm.py b/job_scm.py new file mode 100755 index 0000000..c3cd0ef --- /dev/null +++ b/job_scm.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python + +"""The job will clone scm/meta/git project, and update it to IRIS""" + + +import sys +import os +import base64 +import argparse +from urllib2 import HTTPError +from distutils.sysconfig import get_python_lib + +sys.path.insert(0, get_python_lib()) + +import requests + +from common.tempbuildpkg import TempPackageBuild +from common.gerrit import Gerrit, GerritError, get_gerrit_event +from common.git import Git, clone_gitproject +from common.client import IrisRestClient +from job_submitobs import find_submit_tag + +# set default char-set endcoding to utf-8 +reload(sys) +sys.setdefaultencoding('utf-8') # pylint: disable-msg=E1101 + +USERNAME = os.getenv('IRIS_USERNAME') +PASSWORD = base64.b64decode(os.getenv('IRIS_PASSWORDX','')) +SERVER = os.getenv('IRIS_SERVER') +GERRIT_PROJECT = os.getenv('GERRIT_PROJECT') +OBS_API_USERNAME = os.getenv('OBS_API_USERNAME') +WORKSPACE = os.getenv('WORKSPACE') +GERRIT_HOSTNAME = os.getenv('GERRIT_HOSTNAME') +GERRIT_USERNAME = os.getenv('GERRIT_USERNAME') +GERRIT_SSHPORT = os.getenv('GERRIT_SSHPORT') + + +def scm_check(client, gerrit, events, domains, gittrees): + """ + Print scm check information and return errors, check successfully + only when errors's value is zero. + """ + print '====scm_check====================================' + result = client.scm_check(domains, gittrees) + print result + print '########################################' + + if result['status'] == 200: + return 0 + if result['status'] == 406: + try: + gerrit.review(commit=events['patchset_revision'], + message=result['detail'], verified=-1) + except GerritError, err: + print >> sys.stderr, 'Error posting review comment '\ + 'back to Gerrit: %s' % str(err) + # return 1 if this exception is not caused by invalid commit + if 'no such patch set' not in str(err): + return 1 + return 1 + + +def scm_update(client, domains, gittrees): + """Print scm update information""" + print '====scm update====================================' + result = client.scm_update(domains, gittrees) + print result + print '########################################' + + if result['status'] == 200: + return 0 + return 1 + + +def main(): + """The main body""" + + desc = ("This job can check syntax of domains and git-trees files in " + "scm/meta/git project and update them to IRIS by call IRIS client " + "api.") + parser = argparse.ArgumentParser(description=desc) + parser.parse_args() + + events = get_gerrit_event() + + if not GERRIT_PROJECT: + print 'Invalid gerrit project' + return -1 + + proj_path = os.path.join(WORKSPACE, GERRIT_PROJECT) + if not clone_gitproject(GERRIT_PROJECT, proj_path): + return -1 + + git = Git(proj_path) + gerrit = Gerrit(GERRIT_HOSTNAME, GERRIT_USERNAME, GERRIT_SSHPORT, + GERRIT_SILENT_MOD) + + domains = os.path.join(proj_path, 'domains') + gittrees = os.path.join(proj_path, 'git-trees') + + client = IrisRestClient(SERVER) + client.login(USERNAME, PASSWORD) + + if events['event_type'] == 'patchset-created': + return scm_check(client, gerrit, events, domains, gittrees) + elif events['event_type'] == 'change-merged': + return scm_update(client, domains, gittrees) + else: + return 1 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index 244e365..3be24b8 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -115,6 +115,7 @@ fi %{destdir}/common/utils.py %{destdir}/common/manifest.py %{destdir}/common/upload_service.py +%{destdir}/common/client.py %{destdir}/job_create_snapshot.py %{destdir}/job_release_snapshot.py %{destdir}/job_buildlogs.py @@ -126,6 +127,7 @@ fi %dir %{destdir}/templates %{destdir}/templates/index.html %{destdir}/job_update_local_git.py +%{destdir}/job_scm.py %files tzs %defattr(-,jenkins,jenkins) -- 2.7.4