From: Lin Yang Date: Thu, 21 Nov 2013 05:34:21 +0000 (+0800) Subject: Initial version X-Git-Tag: submit/devel/20190730.075508~43 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=dc6dddce3199d30f40ee271b0450591acdf9825f;p=services%2Fpython-snapdiff.git Initial version Change-Id: I67703258907f356768442cc75b7e11b97797f52f Signed-off-by: Lin Yang --- diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..ef63250 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include builddiff/templates/*.html + diff --git a/builddiff/__init__.py b/builddiff/__init__.py new file mode 100644 index 0000000..1e5a5e7 --- /dev/null +++ b/builddiff/__init__.py @@ -0,0 +1,2 @@ + +__all__ = ['render', 'repo'] diff --git a/builddiff/render.py b/builddiff/render.py new file mode 100644 index 0000000..707a806 --- /dev/null +++ b/builddiff/render.py @@ -0,0 +1,16 @@ +from jinja2 import Environment, FileSystemLoader +import os + +_SETTINGS = { + 'template_path': os.path.join(os.path.dirname(__file__), 'templates'), + } + +def output_html(template_name, **kwargs): + """Render a template file with given kwargs""" + env = Environment( + loader = FileSystemLoader(_SETTINGS['template_path']), + # auto_reload = True, + ) + template = env.get_template(template_name) + return template.render(kwargs) + diff --git a/builddiff/repo.py b/builddiff/repo.py new file mode 100644 index 0000000..4d20365 --- /dev/null +++ b/builddiff/repo.py @@ -0,0 +1,148 @@ +from . import utils +from .render import output_html + +import gzip +import json +import os +import tempfile +import urllib2 +import xml.etree.cElementTree as ET + + +class RepoError(Exception): + """Local custom Exception class, handle Repo Errors""" + pass + + +def _get_primary_md(url, workspace, name): + """Get primary.xml.gz file from remote repodata directory""" + tree = ET.ElementTree(file=_download(url + '/repodata/repomd.xml', \ + workspace, name)) + for elem in tree.iter(tag='{http://linux.duke.edu/metadata/repo}data'): + if elem.attrib['type'] == 'primary': + for c in elem: + if c.tag == '{http://linux.duke.edu/metadata/repo}location': + href = c.attrib['href'] + if href: + return _download(url + href, workspace, \ + href.split('/')[-1]) + else: + raise RepoError('Repo primary metadata can\'t be found !') + +def _download(url, workspace, name): + """Download needed xml file to local by given url""" + xs_file = os.path.join(workspace, name) + + try: + rf = urllib2.urlopen(url) + except urllib2.HTTPError: + xs_file = None + else: + with open(xs_file, 'wb') as xs: + xs.write(rf.read()) + + return xs_file + + +class Repo(object): + """Stuff packages' info""" + + def __init__(self, url): + workspace = tempfile.mkdtemp(dir='/var/tmp') + primary_md = _get_primary_md(url, workspace, 'repomd.xml') + self._et = utils.xml2obj(gzip.open(primary_md)) + + @property + def packages(self): + packages_info = {} + for _package in self._et.package: + packages_info[_package.name] = (packages_info.get(_package.name) \ + or []) + [_package] + return packages_info + +def diff_to_JSON(old_url, new_url): + """Output diffs' json format""" + + if not old_url or not new_url: + return + + old, new = Repo(old_url).packages, Repo(new_url).packages + + added, removed, modified, rebuilded = [], [], [], [] + package_names = set(old.keys() + new.keys()) + + def _pair_old_new(): + for name in package_names: + if old.get(name) is None: + for pkg in new[name]: + yield (None, pkg) + elif new.get(name) is None: + for pkg in old[name]: + yield (pkg, None) + else: + for old_pkg in old[name]: + for new_pkg in new[name]: + if old_pkg.version.ver == new_pkg.version.ver: + yield (old_pkg, new_pkg) + old[name].remove(old_pkg) + new[name].remove(new_pkg) + for pair in map(None, old[name], new[name]): + yield pair + for old_pkg, new_pkg in _pair_old_new(): + if old_pkg is None: + added.append(new_pkg) + elif new_pkg is None: + removed.append(old_pkg) + elif old_pkg.version.ver == new_pkg.version.ver and \ + old_pkg.version.vcs == new_pkg.version.vcs: + rebuilded.append((old_pkg, new_pkg)) + else: + modified.append((old_pkg, new_pkg)) + + obj = {'repo': {'old': old_url, 'new': new_url}, + 'diff': { + 'added': [{'oldpkg': None, 'newpkg': {'name': _new.name, \ + 'version': {'epoch': _new.version.epoch, \ + 'rel': _new.version.rel, 'ver': _new.version.rel}, \ + 'vcs': _new.version.vcs}, 'codediff': None} \ + for _new in added], + 'removed': [{'oldpkg': {'name': _old.name, \ + 'version': {'epoch': _old.version.epoch, \ + 'rel': _old.version.rel, 'ver': _old.version.ver}, \ + 'vcs': _old.version.vcs}, 'newpkg': None, \ + 'codediff': None} for _old in added], + 'modified': [{'oldpkg': {'name': _old.name, \ + 'version': {'epoch': _old.version.epoch, \ + 'rel': _old.version.rel, 'ver': _old.version.ver}, \ + 'vcs': _old.version.vcs}, \ + 'newpkg': {'name': _new.name, \ + 'version': {'epoch': _new.version.epoch, \ + 'rel': _new.version.rel, 'ver': _new.version.ver}, + 'vcs': _new.version.vcs}, 'codediff': None} \ + for _old, _new in modified], + 'rebuilded': [{'oldpkg': {'name': _old.name, \ + 'version': {'epoch': _old.version.epoch, \ + 'rel': _old.version.rel, 'ver': _old.version.ver}, \ + 'vcs': _old.version.vcs}, \ + 'newpkg': {'name': _new.name, \ + 'version': {'epoch': _new.version.epoch, \ + 'rel': _new.version.rel, 'ver': _new.version.ver}, + 'vcs': _new.version.vcs}, 'codediff': None} \ + for _old, _new in rebuilded], + } + } + + return json.dumps(obj, indent=4) + +def diff_to_HTML(old_url, new_url): + """Output diffs' html format""" + + data = json.loads(diff_to_JSON(old_url, new_url)) + + context = {'old_url': old_url, + 'new_url': new_url, + 'diff': data['diff'], + } + + return output_html('diff.html', **context) + diff --git a/builddiff/templates/diff.html b/builddiff/templates/diff.html new file mode 100644 index 0000000..664bc0e --- /dev/null +++ b/builddiff/templates/diff.html @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + +
+
+ +
+
+

+ Repository Difference and Changelogs +

+ Go back...
+

Difference between + {{ old_url }} and {{ new_url }} +

+

Highlights

+ + + + diff --git a/builddiff/utils.py b/builddiff/utils.py new file mode 100644 index 0000000..256c371 --- /dev/null +++ b/builddiff/utils.py @@ -0,0 +1,86 @@ +import re +import xml.sax.handler + +def xml2obj(src): + """ + A simple function to converts XML data into native Python object, + from http://goo.gl/Ymc6Nl, Licensed under the PSF License. + """ + + non_id_char = re.compile('[^_0-9a-zA-Z]') + def _name_mangle(name): + return non_id_char.sub('_', name) + + class DataNode(object): + def __init__(self): + self._attrs = {} # XML attributes and child elements + self.data = None # child text data + def __len__(self): + # treat single element as a list of 1 + return 1 + def __getitem__(self, key): + if isinstance(key, basestring): + return self._attrs.get(key, None) + else: + return [self][key] + def __contains__(self, name): + return self._attrs.has_key(name) + def __nonzero__(self): + return bool(self._attrs or self.data) + def __getattr__(self, name): + if name.startswith('__'): + # need to do this for Python special methods??? + raise AttributeError(name) + return self._attrs.get(name, None) + def _add_xml_attr(self, name, value): + if name in self._attrs: + # multiple attribute of the same name are represented by a list + children = self._attrs[name] + if not isinstance(children, list): + children = [children] + self._attrs[name] = children + children.append(value) + else: + self._attrs[name] = value + def __str__(self): + return self.data or '' + def __repr__(self): + items = sorted(self._attrs.items()) + if self.data: + items.append(('data', self.data)) + return u'{%s}' % ', '.join([u'%s:%s' % (k, repr(v)) for k, v in items]) + + class TreeBuilder(xml.sax.handler.ContentHandler): + def __init__(self): + self.stack = [] + self.root = DataNode() + self.current = self.root + self.text_parts = [] + def startElement(self, name, attrs): + self.stack.append((self.current, self.text_parts)) + self.current = DataNode() + self.text_parts = [] + # xml attributes --> python attributes + for k, v in attrs.items(): + self.current._add_xml_attr(_name_mangle(k), v) + def endElement(self, name): + text = ''.join(self.text_parts).strip() + if text: + self.current.data = text + if self.current._attrs: + obj = self.current + else: + # a text only node is simply represented by the string + obj = text or '' + self.current, self.text_parts = self.stack.pop() + self.current._add_xml_attr(_name_mangle(name), obj) + def characters(self, content): + self.text_parts.append(content) + + builder = TreeBuilder() + if isinstance(src, basestring): + xml.sax.parseString(src, builder) + else: + xml.sax.parse(src, builder) + return builder.root._attrs.values()[0] + diff --git a/packaging/builddiff.changes b/packaging/builddiff.changes new file mode 100644 index 0000000..a90f011 --- /dev/null +++ b/packaging/builddiff.changes @@ -0,0 +1,2 @@ +* Wed Nov 20 2013 Lingchao Xin - 0.1 + - Initial packaging diff --git a/packaging/builddiff.spec b/packaging/builddiff.spec new file mode 100644 index 0000000..1e0dcda --- /dev/null +++ b/packaging/builddiff.spec @@ -0,0 +1,41 @@ +# +# spec file for package builddiff +# +# Copyright (c) 2013 Lingchao Xin . +# + +Name: builddiff +Version: 0.1 +Release: 0 +License: GPL-2.0+ +Summary: Generate image and repo diffs +Url: http://www.tizen.org +Group: Development/Tools/Other +Source: %{name}-%{version}.tar.gz +BuildRequires: python-devel +Requires: python >= 2.7 +Requires: python-jinja2 +BuildRoot: %{_tmppath}/%{name}-%{version}-build +BuildArch: noarch +%{!?python_sitelib: %define python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} + +%description +Builddiff is used to generate image and repo diffs. + +%prep +%setup -q + +%build +CFLAGS="%{optflags}" python setup.py build + +%install +python setup.py install --prefix=%{_prefix} --root=%{buildroot} + +%clean +rm -rf %{buildroot} + +%files +%defattr(-,root,root) +%{_bindir}/* +%{python_sitelib}/* + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d1e23ef --- /dev/null +++ b/setup.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +setup(name = 'builddiff', + version = '0.1', + description = 'Generate image and repo diffs', + author = 'Lingchao Xin', + author_email = 'lingchaox.xin@intel.com', + url = 'http://www.tizen.org/', + scripts = ['tools/repodiff',], + packages=['builddiff',], + include_package_data=True, + install_requires=['Jinja2>=2.6'], + zip_safe=False, +) + diff --git a/tools/repodiff b/tools/repodiff new file mode 100755 index 0000000..f553cbb --- /dev/null +++ b/tools/repodiff @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +from builddiff.repo import diff_to_JSON, diff_to_HTML + +import argparse +import sys + +def main(argv): + description = 'Diff two repos with different urls' + parser = argparse.ArgumentParser(description=description) + parser.add_argument(dest='old', help='old repo') + parser.add_argument(dest='new', help='new repo') + parser.add_argument('--json', help='output json diffs', action='store_true') + args = parser.parse_args(argv) + + if args.json: + print diff_to_JSON(args.old, args.new) + else: + print diff_to_HTML(args.old, args.new) + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) +