Feature : Generates .files which contains package name and its file list

make the images_files file.ex)tizen-3.0-tv_20160324.9_tv-emulator.files

Change-Id: I6477485b5ff7aeafb649e06c80ef8d483f4d81c9

diff --git a/job_imager.py b/job_imager.py
index 468153d..a69528b 100755 (executable)
--- a/job_imager.py
+++ b/job_imager.py
@@ -18,16 +18,115 @@
 #
 """This script is used to create image"""
 
+import re
 import os
 import sys
 import shutil
 import subprocess
+import gzip
+import xml.dom.minidom
+import urllib2
+import codecs
+
+from xml.dom.minidom import parse,parseString
+from collections import defaultdict
+from StringIO import StringIO
+
+
 from random import randint
 
 from common.buildtrigger import trigger_info, trigger_next
 from common.utils import sync, set_permissions, Workdir
 
 
+# Returns the list of baseurls from kickstart configuration.
+def get_base_url(kickstart,buildid):
+    baseurl = []
+    for line in kickstart.split('\n'):
+        if line.startswith('repo') and line.find('--baseurl') > 0:
+           burl = re.search("(?P<url>https?://[^\s]+)", line).group('url')
+           if burl.find('@BUILD_ID@') >0 :
+               burl =  burl.replace('@BUILD_ID@',buildid)
+           baseurl.append(burl)
+    return baseurl
+
+
+# Returns the url location of filelist.xml.gz
+def get_gz_file_url(baseurl):
+    repomdurl = os.path.join(baseurl, 'repodata/repomd.xml')
+    response = urllib2.urlopen(repomdurl)
+    xml_res = response.read()
+    DOMTree = xml.dom.minidom.parseString(xml_res)
+    collection = DOMTree.documentElement
+    data = collection.getElementsByTagName("data")
+    gz_url = None
+    for d in data:
+        if d.hasAttribute('type') and d.getAttribute('type') == 'filelists':
+            gz_file_name = d.getElementsByTagName('location')[0].getAttribute('href')
+            gz_url = repomdurl.replace('repodata/repomd.xml',gz_file_name)
+            break
+    return gz_url
+
+#  Returns the decompressed xml data from given url location
+def decompress_url_data(gzurl):
+    if gzurl is None:
+        return ''
+    request = urllib2.Request(gzurl)
+    request.add_header('Accept-encoding', 'gzip')
+    response = urllib2.urlopen(request)
+    buf = StringIO( response.read())
+    f = gzip.GzipFile(fileobj=buf)
+    xml_data = f.read()
+    f.close()
+    return xml_data
+
+# Generates .files with filelist of packages
+def gen_pkg_filelist(fields,base_path):
+
+    fpkg= os.path.join(base_path,fields['name'],fields['buildid']+'_'+fields['name']+'.packages')
+    fout= os.path.join(base_path,fields['name'],fields['buildid']+'_'+fields['name']+'.files')
+
+    if not os.path.isfile(fpkg):
+        print 'The file %s doesn\'t exists. Can\'t generate .files'%(fpkg)
+        return -1
+
+    baseurl = get_base_url(fields['kickstart'],fields['buildid'])
+
+    if not baseurl:
+        print 'No baseurl found in kick start file...'
+        return -1
+    pkgdict = defaultdict()
+
+    for turl in baseurl:
+        print 'baseurl %s'%(turl)
+        gzurl =  get_gz_file_url(turl)
+        print 'filelists url %s'%(gzurl)
+        xml_content =  decompress_url_data(gzurl)
+
+        DOMTree = xml.dom.minidom.parseString(xml_content)
+        collection = DOMTree.documentElement
+        packages = collection.getElementsByTagName("package")
+
+        # The contents of xml are converted to dictionary in which key format is <package_name>.<arch> <version>-<release> and value is list of files for the package
+        for pkg in packages:
+            name_arch=  pkg.getAttribute('name')+'.'+pkg.getAttribute('arch')
+            f = pkg.getElementsByTagName('version') [0]
+            ver_rel= f.getAttribute('ver') + '-'+f.getAttribute('rel')
+            pkgdict[name_arch+' '+ver_rel] = [ f.childNodes[0].data  for f in   pkg.getElementsByTagName('file') ]
+
+
+
+    print 'Writing the list of files in packages to '+fout
+    # For each package in the .packages file, get the list of pkg files from dictionary and writes them to .files
+    with open (fpkg,'r') as pkg_f, codecs.open (fout,'w','utf-8') as out_f:
+        for line in pkg_f:
+            key =  line.split()[0]+' '+line.split()[1]
+            out_f.write('\n\n'+key)
+            if key in pkgdict.keys():
+                for t in pkgdict[key]:
+                    out_f.write('\n\t'+t)
+
+
 def get_xml(image_dir, ks_name):
     """ Get the xml the image, return the body as string """
 
@@ -179,6 +278,9 @@ def main():
 
     sync_src = os.path.join(outdir, build_id)
 
+    # Generates .files which contains package name and its file list
+    gen_pkg_filelist(fields,os.path.join(sync_src,'images'))
+
     convert_image_dir(os.path.join(sync_src, 'images'),
                       fields['repo'], fields['name'])
     if sync(sync_src, sync_dest):