From 18eb567cc577ca0f478ff95f9d0f4941b0b951cb Mon Sep 17 00:00:00 2001 From: Lingchaox Xin Date: Tue, 24 Dec 2013 18:02:40 +0800 Subject: [PATCH] Integrate snapdiff into jenkins-scripts Change-Id: Idaf2791be10ab1d0e72536200c11c065692024f2 --- job_create_snapshot.py | 7 +- job_sync_snapdiff.py | 186 +++++++++++++++++++++++++++++++++++++++++ packaging/jenkins-scripts.spec | 4 + templates/index.html | 75 +++++++++++++++++ 4 files changed, 270 insertions(+), 2 deletions(-) create mode 100755 job_sync_snapdiff.py create mode 100644 templates/index.html diff --git a/job_create_snapshot.py b/job_create_snapshot.py index cb6175a..1e4325d 100755 --- a/job_create_snapshot.py +++ b/job_create_snapshot.py @@ -46,6 +46,7 @@ def trigger_image_creation(images_ks, build_id, path_repo, project, 'repo_path': path_repo, 'url_pub_base': url_pub_base } + yield data trigger_next('%s/image_trigger_%s' % (os.getenv('WORKSPACE'), index), data) @@ -158,8 +159,10 @@ def main(): base_path, live_repo_base, buildconf = buildconf_str) # trigger image creation jobs - trigger_image_creation(repo_data['imagedata'].ksi, repo_data['build_id'], - repo_data['repo_path'], project, base_url) + datable = trigger_image_creation(repo_data['imagedata'].ksi, \ + repo_data['build_id'], repo_data['repo_path'], project, base_url) + + trigger_next('snapdiff-trigger', {repo: [data for data in datable]}) # trigger post snapshot creation job with repo data data = repo_data.copy() diff --git a/job_sync_snapdiff.py b/job_sync_snapdiff.py new file mode 100755 index 0000000..9cb9a72 --- /dev/null +++ b/job_sync_snapdiff.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python +""" + job_sync_snapdiff.py + ~~~~~~~~~~~~~~~~~~~~ + + Generate repo and image diffs, sync them to tizen.org. + +""" + +import os +import re +import sys +from distutils.sysconfig import get_python_lib + +sys.path.insert(0, get_python_lib()) + +import bs4 +from jinja2 import Environment, FileSystemLoader +import requests +import snapdiff + +from common.buildtrigger import trigger_info +from common.utils import sync + + +def search_last_build(build_url, released=False): + """Search the lastest build in build_url. + + :param build_url: build url contains builds. + :param released: is a released url or not. + :return: path. the last build url. + """ + res = requests.get(build_url) + if res.status_code != 200: + return + soup = bs4.BeautifulSoup(res.text) + pattern = re.compile('^tizen_[0-9]{8}.[0-9]+/$') + builds = [i.text for i in soup.find_all(href=pattern)] + + if released: + last_build = builds[-1] + elif len(builds) > 1: + last_build = builds[-2] + else: + last_build = None + + return os.path.join(build_url, last_build) if last_build else None + +def get_last_build(url_pub_base, repo_path): + """Return the second lastest build. + + :param url_pub_base: the public base url. + :param repo_path: the relative repo path. + :return: path. the second lastest build url. + """ + last_build_url = os.path.join(url_pub_base, repo_path[:repo_path.rfind('/')]) + return search_last_build(last_build_url) + +def get_released_build(release_base, repo_path): + """Return the lastest released build. + + :param release_base: release base url. + :param repo_path: the relative repo path. + :return: path tuple. the lastest daily and weekly builds. + """ + base_contains = '/'.join(repo_path.split('/')[1:-1]) + daily_url = os.path.join(release_base, 'daily', base_contains) + weekly_url = os.path.join(release_base, 'weekly', base_contains) + return search_last_build(daily_url, True), search_last_build(weekly_url, True) + +def _get_buildid(url): + """Get build id from url""" + result = re.search(r'\w*_\d{8}\.\d*', url) + return result.group(0) if result else url + +def _get_name(url): + """Get image name from url""" + name = url[:-1].split('/')[-1] + return name or url + +def generate_diff(old_url, new_url, name, style='repo'): + """Generate repo or image diff. + + :param old_url: old repo url. + :param new_url: new repo url. + :param repo_path: the repo base path. + :param name: diff prefix name. + """ + sync_out = os.path.join(os.getenv('WORKSPACE'), 'outdir') + reports = os.path.join(sync_out, 'builddata', 'reports') + diff_name = '-'.join([name, _get_buildid(old_url), _get_buildid(new_url)]) + snapdiff.diff_to_dist(old_url, new_url, reports, style, diff_name) + return '%s.html' % diff_name + +def sync_world(template_name, repo_path, **kwargs): + """Sync repo and image diff. + + :param template_name: template we will use generate sumary index. + :param kwargs: all context we use render template. + """ + template_path = os.path.join(os.path.dirname(__file__), 'templates') + env = Environment(loader = FileSystemLoader(template_path)) + template = env.get_template(template_name) + content = template.render(kwargs) + sync_out = os.path.join(os.getenv('WORKSPACE'), 'outdir') + with open(os.path.join(sync_out, 'builddata', 'reports', 'index.html'), 'w') as fp: + fp.write(content) + sync(sync_out, os.path.join(os.getenv('IMG_SYNC_DEST_BASE'), repo_path)) + +def main(): + """The main body""" + info = trigger_info(os.getenv('TRIGGER_INFO')) + repo_name = list(info)[0] + content = info[repo_name] + + if content is None: + return + + url_pub_base = content[0]['url_pub_base'] + repo_path = content[0]['repo_path'] + + diff_root = os.path.join(url_pub_base, repo_path) + current_repo_url = os.path.join(diff_root, 'repos', repo_name, 'packages') + + context = {'id': content[0]['buildid']} + context['repo'] = [] + + # generate current and last repo diff + last_repo_base = get_last_build(url_pub_base, repo_path) + + if last_repo_base: + last_repo_url = os.path.join(last_repo_base, 'repos', repo_name, 'packages') + last_current = generate_diff(last_repo_url, current_repo_url, 'last_current') + last_id = _get_buildid(last_repo_url) + context['repo'].append(('last_build', last_id, last_current)) + + # releases' url + releases_url = os.path.join(url_pub_base, 'releases') + + # generate current and releases repo diff + daily_repo_base, weekly_repo_base = get_released_build(releases_url, repo_path) + + if daily_repo_base: + daily_repo_url = os.path.join(daily_repo_base, 'repos', repo_name, 'packages') + daily_current = generate_diff(daily_repo_url, current_repo_url, 'daily_current') + daily_id = _get_buildid(daily_repo_url) + context['repo'].append(('daily_build', daily_id, daily_current)) + + if weekly_repo_base: + weekly_repo_url = os.path.join(weekly_repo_base, 'repos', repo_name, 'packages') + weekly_current = generate_diff(weekly_repo_url, current_repo_url, 'weekly_current') + weekly_id = _get_buildid(weekly_repo_url) + context['repo'].append(('weekly_build', weekly_id, weekly_current)) + + # generate image diff + context['images'] = {} + + for item in content: + current_image_url = os.path.join(diff_root, item['images_path']) + last_image_url = os.path.join(last_repo_base, item['images_path']) if last_repo_base else None + daily_image_url = os.path.join(daily_repo_base, item['images_path']) if daily_repo_base else None + weekly_image_url = os.path.join(weekly_repo_base, item['images_path']) if weekly_repo_base else None + + context['images'][item['name']] = [] + + # collect each image information. + if requests.get(current_image_url).status_code == 200: + if last_image_url and requests.get(last_image_url).status_code == 200: + name = generate_diff(last_image_url, current_image_url, item['name']+'_last_current', style='image') + context['images'][item['name']].append(('last_build', last_id, name)) + if daily_image_url and requests.get(daily_image_url).status_code == 200: + name = generate_diff(daily_image_url, current_image_url, item['name']+'_daily_current', style='image') + context['images'][item['name']].append(('daily_build', daily_id, name)) + if weekly_image_url and requests.get(weekly_image_url).status_code == 200: + name = generate_diff(weekly_image_url, current_image_url, item['name']+'_weekly_current', style='image') + context['images'][item['name']].append(('weekly_build', weekly_id, name)) + + # if image is empty, pop it. + if not context['images'][item['name']]: + context['images'].pop(item['name']) + + # sync all + sync_world('index.html', repo_path, **context) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/packaging/jenkins-scripts.spec b/packaging/jenkins-scripts.spec index 40b9597..231c818 100644 --- a/packaging/jenkins-scripts.spec +++ b/packaging/jenkins-scripts.spec @@ -32,8 +32,11 @@ Group: Development/Tools/Building Requires: createrepo_c Requires: git-core Requires: tizen-gbp-rpm >= 20131017 +Requires: python-beautifulsoup4 Requires: python-cheetah Requires: python-redis +Requires: python-requests +Requires: python-snapdiff Requires: python-yaml Requires: gbs-api @@ -119,6 +122,7 @@ fi %{destdir}/job_imager.py %{destdir}/job_mail_sender.py %{destdir}/job_load_repos.yaml.py +%{destdir}/job_sync_snapdiff.py %{destdir}/job_update_local_git.py %files tzs diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..fdbfc41 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,75 @@ + + + + + + + + + +
+
+ +

+ Report for Build {{ id }} +

+

+ Repository +

+ +

+ Images +

+ + {% for image in images %} +

+ + + + {{ image.capitalize() }} ({{ image }}) + +

+

+ Difference to last, daily and weekly builds: +

+ + {% endfor %} +
+
+ + + -- 2.7.4