initial import imager and reports job script

reuse boss-imager-participant and boss-reports-participant

diff --git a/boss-imager.py b/boss-imager.py
new file mode 100755 (executable)
index 0000000..641ed4e
--- /dev/null
+++ b/boss-imager.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+
+from __future__ import with_statement
+
+import os, sys, io, stat, re
+import ConfigParser
+import optparse
+import shutil
+import tempfile
+import subprocess
+from string import Template
+from datetime import datetime
+#import boss_info
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+from common.envparas import export
+
+P_NAME = 'imager' # component name in boss
+
+DEFAULTCONF = """
+[$pname]
+# dir to store temp ks files
+task_dir = /srv/boss-imager-tasks
+
+# dir to store final images, mostly will be overrided by WI params
+dest_dir = /srv/images
+
+# used by bossinfo module
+redis_server = 127.0.0.1
+"""
+
+CONFS = {}
+
+# global BossInfo instance for updating status
+BI = None
+
+HOSTNAME = os.uname()[1]
+
+PARAM_LIST = {
+        'OBS_EVENT_STRING',
+        'TASK_DIRECTORY',
+        'DESTINATION_DIRECTORY',
+        'REDIS_SERVER'
+    }
+
+export(PARAM_LIST, locals())
+
+def _save_ks(ks_str, ks_dir, fname):
+    if not os.path.exists(ks_dir):
+        os.makedirs(ks_dir)
+
+    ksfp = "%s/%s.ks" % (ks_dir, fname)
+    with open(ksfp, 'w') as wf:
+        wf.write(ks_str)
+
+def _change_url(ksc, conf_baseurl, relative_uri):
+    if '@BUILD_ID@' not in ksc:
+        return ksc
+
+    lines = []
+    baseurl_re = re.compile(r"baseurl=.*(?=@BUILD_ID@)")
+    conf_baseurl = conf_baseurl.rstrip('/') + '/'
+    replace_url = conf_baseurl
+
+    if relative_uri:
+        if relative_uri not in conf_baseurl:
+            replace_url = replace_url + relative_uri
+
+    replace_url = replace_url.rstrip('/') + '/'
+
+    for line in ksc.splitlines():
+        if line.startswith('repo') and \
+           '@BUILD_ID@' in line and \
+           'baseurl=' in line:
+            line = baseurl_re.sub(r"baseurl=" + replace_url , line)
+        lines.append(line)
+
+    return '\n'.join(lines)
+
+
+def main():
+    fields = json.loads(OBS_EVENT_STRING)
+
+    # make sure the needed dirs exists
+    try:
+        os.makedirs(CONFS['task_dir'])
+        os.makedirs(CONFS['dest_dir'])
+    except:
+        pass
+
+    try:
+        kickstart = fields["kickstart"]
+        buildid = fields["buildid"]
+        name = fields["name"]
+        timeout = '90m'
+        conf_baseurl = fields["conf_baseurl"]
+        starttime = fields["starttime"]
+        relative_uri = fields["relative_uri"] or ''
+    except KeyError, e:
+        print 'Lost field: %s' % str(e)
+        sys.exit(-1)
+
+    # replace repo's baseurl inside kickstart
+    ks_str = _change_url(kickstart,
+                         conf_baseurl,
+                         relative_uri).replace('@BUILD_ID@', buildid)
+
+    # save internal workable ks
+    ks_fp = "%s/%s-%s" %(CONFS['task_dir'], name, buildid)
+    _save_ks(ks_str, ks_fp, name)
+
+    # update image creation status to bossinfo server
+    #taskid = BI.get_imgtask_by_buildid(buildid)
+    #buildtime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    #BI.set_img_status(taskid, name, 'running')
+    #BI.set_img_starttime(taskid, name, buildtime)
+    #BI.set_img_buildhost(taskid, name, HOSTNAME)
+    #BI.set_server_status(HOSTNAME, self.worker, taskid, name)
+
+    print 'starting kvmic instance to create image...'
+    try:
+        ret = subprocess.call("/usr/bin/kvmic -b %s -k %s -d %s -T %s" %
+                                                (buildid,
+                                                 "%s/%s.ks" % (ks_fp, name),
+                                                 CONFS['dest_dir'],
+                                                 timeout),
+                               stdout=sys.stdout,
+                               stderr=sys.stderr,
+                               shell=True)
+    except OSError, e:
+        print 'Launch kvmic error:', str(e)
+        return
+
+    # save original ks to images dir to public
+    ks_pub = "%s/%s/%s/%s" %(CONFS['dest_dir'], buildid, 'images', name)
+    _save_ks(kickstart, ks_pub, name)
+
+    try:
+        status = {0: 'ok', 1: 'failed', 2: 'timeout'}[ret]
+        #BI.set_img_status(taskid, name, status)
+    except KeyError:
+        status = 'kvmic runtime error'
+        #BI.set_img_status(taskid, name, 'failed')
+
+    #buildtime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    #BI.set_img_endtime(taskid, name, buildtime)
+
+    #BI.set_server_status(HOSTNAME, self.worker, '0', None)
+    print 'kvmic quit, with status: %s' % status
+    sys.exit(status)
+
+if __name__ == "__main__":
+    parser = optparse.OptionParser()
+    parser.add_option("-c", "--config", dest="config",
+                      help="specify configuration file")
+    parser.add_option("", "--get-defconf", dest="get_defconf", action="store_true",
+                      help="Print out the default  configuration file")
+
+    (opts, args) = parser.parse_args()
+
+    temp = Template(DEFAULTCONF)
+    str_conf = temp.substitute(pname=P_NAME)
+
+    if opts.get_defconf:
+        print str_conf
+        sys.exit(0)
+
+    if opts.config:
+        with open(opts.config) as f:
+            str_conf = f.read()
+
+    config = ConfigParser.ConfigParser()
+    config.readfp(io.BytesIO(str_conf))
+
+    try:
+        CONFS['task_dir'] = TASK_DIRECTORY or config.get(P_NAME, 'task_dir')
+        CONFS['dest_dir'] = DESTINATION_DIRECTORY or config.get(P_NAME, 'dest_dir')
+        CONFS['redis_server'] = REDIS_SERVER or config.get(P_NAME, 'redis_server')
+    except ConfigParser.NoOptionError, e:
+        print 'In config, %s' % str(e)
+        sys.exit(1)
+
+    #starttime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    #BI = boss_info.BossInfo('redis://%s' % CONFS['redis_server'])
+    #BI.init_server(HOSTNAME, starttime, nworkers)
+
+    main()

diff --git a/boss-reports.py b/boss-reports.py
new file mode 100755 (executable)
index 0000000..3964c1e
--- /dev/null
+++ b/boss-reports.py
@@ -0,0 +1,282 @@
+#!/usr/bin/python
+
+from __future__ import with_statement
+import os, sys, io
+import ConfigParser
+import optparse
+from string import Template
+from builddiff.imagediff import ImageDiff
+from builddiff.repodiff import builddiff
+
+try:
+  from lxml import etree
+except ImportError:
+  try:
+    # Python 2.5
+    import xml.etree.cElementTree as etree
+  except ImportError:
+    try:
+      # Python 2.5
+      import xml.etree.ElementTree as etree
+    except ImportError:
+      try:
+        # normal cElementTree install
+        import cElementTree as etree
+      except ImportError:
+        try:
+          # normal ElementTree install
+          import elementtree.ElementTree as etree
+        except ImportError:
+          print("Failed to import ElementTree from any known place")
+
+from common.envparas import export
+
+PARAM_LIST = ['RELATIVE_URI',
+              'release_build_id']
+
+export(PARAM_LIST, locals())
+
+P_NAME = 'reports' # component name in boss
+
+DEFAULTCONF = """
+[$pname]
+daemon = Yes
+logfile = /var/log/boss/${pname}.log
+pidfile = /var/run/boss/${pname}.pid
+runas_user = obsrun
+runas_group = obsrun
+
+# ${pname} specific configrations
+
+project_name = tizen.org
+
+release_dir = /srv/www/download/releases
+snapshots_dir = /srv/www/download/snapshots
+"""
+
+def find_desc(name, configs):
+  for ks in configs.findall('config'):
+    n = ks.find('name').text
+    n = n[0:n.rindex('.ks')]
+    if name == n or name == "meego-%s" %n:
+      #print "%s=%s" %(n,name)
+      return ks.find('description').text
+
+def compare(buildid, relative, project_set, oldid = None, output_dir = None):
+    """Support tizen
+    """
+
+    if buildid.startswith('tizen'):
+        _tizen_compare(buildid, relative, project_set, oldid, output_dir)
+    else:
+        pass
+
+def _tizen_compare(buildid, relative, project_set, oldid, output_dir):
+    distro = 'tizen'
+    release_dir = project_set['release_dir']
+    parent = project_set['snapshots_dir'] + '/' + relative
+
+    style_dir = "/usr/share/python-builddiff/styles/" + distro
+
+    # sample buildid:
+    #   tizen_20111228.2
+    #   tizen-1.0_20111228.1
+    try:
+        sid = buildid.split("_")
+        distro_ver = sid[0]
+        builds = sid[-1]
+        major, build_no  = builds.split('.')
+    except:
+        print 'Invalid build id: %s' % buildid
+        return None
+
+    # daily
+    daily = []
+    weekly = []
+    compare_to = []
+    if oldid:
+        compare_to = [(oldid, parent)]
+    else:
+        daily_dir = release_dir + '/daily/' + relative.lstrip('/')
+        if os.path.isdir(daily_dir):
+            _d = os.listdir(daily_dir)
+            daily = sorted([(x, daily_dir) for x in _d if x.startswith(distro_ver)],
+                           key = lambda bid: map(int, bid[0].split('_')[-1].split('.')))[-2:]
+        else:
+            daily = []
+
+        print "daily: %s" %daily
+
+        weekly_dir = release_dir + '/weekly/' + relative.lstrip('/')
+        if os.path.isdir(weekly_dir):
+            _d = os.listdir(weekly_dir)
+            weekly = sorted([(x, weekly_dir) for x in _d if x.startswith(distro_ver)],
+                           key = lambda bid: map(int, bid[0].split('_')[-1].split('.')))[-1:]
+        else:
+            weekly = []
+
+        print "weekly: %s" %weekly
+
+        sys.stdout.flush()
+        compare_to = weekly+daily
+
+    print "going to compare with ", compare_to
+    sys.stdout.flush()
+
+    if buildid and len(compare_to) > 0:
+        if output_dir is not None:
+            outputdir = "%s/%s" %(output_dir, buildid)
+        else:
+            outputdir = "%s/%s/builddata/reports" %(parent, buildid)
+
+        print "All reports will be stored in %s" %outputdir
+        if not os.path.exists(outputdir):
+            os.mkdir(outputdir, 0755)
+
+        for old, olddir in set(compare_to):
+            oldurl = "%s/%s" %(olddir, old)
+            newurl = "%s/%s" %(parent, buildid)
+
+            repodiff = builddiff(oldurl, newurl, outputdir = outputdir, project_name = project_set['project_name'], relative_uri=relative)
+
+            imgdiff = ImageDiff(oldurl, newurl, project_name = project_set['project_name'])
+            imgdiff.repodiff = repodiff
+            imgdiff.outputdir = outputdir
+
+            if not os.path.exists(imgdiff.outputdir):
+                os.mkdir(imgdiff.outputdir, 0755)
+
+            for image in os.listdir("%s/%s/images" %(parent, buildid)):
+                if os.path.exists("%s/%s/images" %(olddir, old)):
+                  print "working on %s" %image
+                  imgdiff.image_diff(image, relative)
+
+        create_report(parent, buildid, style_dir, outputdir)
+    else:
+        print "nothing to report"
+
+    sys.stdout.flush()
+
+def create_report(parent, id, styledir, output):
+    files = {}
+    repos = []
+    for f in os.listdir(output):
+        if 'repodiff' in f:
+            nohtml =  f[9:f.rindex('.html')]
+            old, new = nohtml.split("--")
+            repos.append( { 'html': f, 'new': new, 'old': old } )
+            continue
+
+        if '.html' in f and f != 'index.html':
+            nohtml =  f[0:f.rindex('.html')]
+            ks, old, new = nohtml.split("--")
+
+            configs = etree.parse('%s/%s/builddata/image-configs.xml' %(parent, id))
+            desc = find_desc(ks, configs)
+            manfile = '%s/%s/images/%s/MANIFEST' %(parent, id, ks)
+            if os.path.exists(manfile):
+                manifest = open(manfile, "r")
+                manifest_lines = [m.split() for m in manifest.readlines()]
+                manifest.close()
+            else:
+                manifest_lines = []
+
+            if not files.has_key(ks):
+                files[ks] = [{'new': new, 'old': old, 'description': desc, 'html': f, 'manifest': manifest_lines}]
+            else:
+                files[ks].append({'new': new, 'old': old, 'description': desc, 'html': f, 'manifest': manifest_lines})
+
+    xmlroot = etree.Element("report")
+    etree.SubElement(xmlroot, "build_id").text = id
+    for k in files.keys():
+        images = etree.SubElement(xmlroot, "images")
+        image = etree.SubElement(images, "image")
+        etree.SubElement(image, "name").text = k
+        etree.SubElement(image, "description").text = files[k][0]['description']
+        imgf = etree.SubElement(image, "files")
+        for imf in files[k][0]['manifest']:
+            ff = etree.SubElement(imgf, "file")
+            ff.set("md5sum", imf[0])
+            ff.set("name", imf[1].lstrip('*'))
+
+        for d in files[k]:
+            diff = etree.SubElement(image, "diff")
+            for key in d.keys():
+                if key != 'description' and key != 'manifest':
+                    diff.set(key, d[key])
+
+    reposel = etree.SubElement(xmlroot, "repos")
+    for r in repos:
+        repo = etree.SubElement(reposel, "repo")
+        for key in r.keys():
+            repo.set(key, r[key])
+
+    tree = etree.ElementTree(xmlroot)
+    #tree.write('%s/%s/builddata/reports/index.xml' %(parent, id))
+
+    # transform
+    xslt_root = etree.parse('%s/index.xslt' %styledir)
+    transform = etree.XSLT(xslt_root)
+    html = transform(tree)
+    hfp = open("%s/index.html" %(output), 'w')
+    hfp.write(str(html))
+    hfp.close()
+
+def main(project_set):
+
+    compare(release_build_id, RELATIVE_URI, project_set)
+
+if __name__ == "__main__":
+    parser = optparse.OptionParser()
+    parser.add_option("-c", "--config", dest="config", 
+                      help="specify configuration file")
+    parser.add_option("", "--get-defconf", dest="get_defconf", action="store_true",
+                      help="Print out the default  configuration file")
+
+    parser.add_option("", "--build-id", dest="buildid", 
+                      help="Create report for build id")
+    parser.add_option("", "--compare-with", dest="comparewith", 
+                      help="Compare with this id")
+    parser.add_option("", "--output-dir", dest="output_dir", 
+                      help="Output directory")
+    (opts, args) = parser.parse_args()
+
+    if opts.buildid:
+        print 'will support command line later'
+        """
+        if opts.output_dir:
+            compare(opts.buildid, opts.comparewith,  opts.output_dir)
+        else:
+            compare(opts.buildid)
+        """
+    else:
+        temp = Template(DEFAULTCONF)
+        str_conf = temp.substitute(pname=P_NAME)
+
+        if opts.get_defconf:
+            print str_conf
+            sys.exit(0)
+
+        if opts.config:
+            with open(opts.config) as f:
+                str_conf = f.read()
+
+        config = ConfigParser.ConfigParser()
+        config.readfp(io.BytesIO(str_conf))
+
+        project_set = {}
+        try:
+            # reports participant specific conf
+            if config.has_option(P_NAME, 'project_name'):
+                project_set['project_name'] = config.get(P_NAME, 'project_name')
+            else:
+                project_set['project_name'] = None
+
+            project_set['release_dir'] = config.get(P_NAME, 'release_dir')
+            project_set['snapshots_dir'] = config.get(P_NAME, 'snapshots_dir')
+
+        except ConfigParser.NoOptionError, e:
+            print 'In config, %s' % str(e)
+            sys.exit(1)
+
+        main(project_set)