tools/getpacs/diff-packages.py

   1 #!/usr/bin/python
   2
   3 from sets import Set
   4 import csv
   5 import urllib2
   6 from optparse import OptionParser
   7 import os
   8 import re, base64
   9 import ConfigParser
  10
  11 username = ""
  12 password = ""
  13
  14 def read_config(config_file):
  15     config_file = os.path.expanduser(config_file)
  16     parser = ConfigParser.SafeConfigParser()
  17     parser.read(config_file)
  18     return parser
  19
  20 def http_get(url):
  21     print "Downloading %s" %url
  22     request = urllib2.Request(url)
  23     if username and password:
  24         base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
  25         request.add_header("Authorization", "Basic %s" % base64string)
  26     html_page = urllib2.urlopen(request)
  27     return html_page
  28
  29 def download(url, out):
  30     if not os.path.exists(out):
  31         ret = http_get(url)
  32         cache = open(out, "w")
  33         cache.write(ret.read())
  34         cache.close()
  35     else:
  36         print "Already exists: %s" % out
  37
  38 def get_package_list(image_name, base_url, build_id):
  39     cache_dir = "cache"
  40     if not os.path.exists(cache_dir):
  41         os.makedirs(cache_dir)
  42     cache_file = "%s/%s-%s.packages" %(cache_dir, image_name, build_id )
  43     package_file = None
  44     if not os.path.exists(cache_file):
  45         image_packages = "%s/%s/images/%s/%s-%s.packages" %(base_url, build_id, image_name, image_name, build_id )
  46         package_file = http_get(image_packages)
  47         cache = open(cache_file, "w")
  48         cache.write(package_file.read())
  49         cache.close()
  50     package_file = open(cache_file, "rb")
  51
  52     packages = []
  53
  54     pkgreader = csv.reader(package_file, delimiter=' ', quotechar='|')
  55     for row in pkgreader:
  56         pkg = row[0].split(".")
  57         name = "%s-%s.%s" %(pkg[0], row[1], pkg[1])
  58         packages.append(name)
  59
  60     package_file.close()
  61     return packages
  62
  63 def get_package_list2(image_name, base_url, build_id):
  64     cache_dir = "cache"
  65     if not os.path.exists(cache_dir):
  66         os.makedirs(cache_dir)
  67     cache_file = "%s/%s-%s.packages" %(cache_dir, image_name, build_id )
  68     package_file = None
  69     if not os.path.exists(cache_file):
  70         image_packages = "%s/%s/images/%s/%s-%s.packages" %(base_url, build_id, image_name, image_name, build_id )
  71         #print image_packages
  72         package_file = http_get(image_packages)
  73         cache = open(cache_file, "w")
  74         cache.write(package_file.read())
  75         cache.close()
  76     package_file = open(cache_file, "rb")
  77
  78     packages = {}
  79
  80     pkgreader = csv.reader(package_file, delimiter=' ', quotechar='|')
  81     for row in pkgreader:
  82         pkg = row[0].split(".")
  83         if len(row)>2:
  84             packages[pkg[0]] = {'scm': row[2], 'version': row[1], 'arch': pkg[1]}
  85         else:
  86             packages[pkg[0]] = {'scm': None, 'version': row[1], 'arch': pkg[1]}
  87
  88     package_file.close()
  89     return packages
  90
  91 parser = OptionParser()
  92 parser.add_option("-o", "--old",  dest="old", metavar="OLD", help="Old snapshot")
  93 parser.add_option("-n", "--new",  dest="new", metavar="NEW", help="New snapshot")
  94 parser.add_option("-t", "--type",  dest="type", metavar="TYPE", help="Release type")
  95 parser.add_option("-i", "--image",  dest="image", metavar="IMAGE", help="Image Name")
  96 parser.add_option("-u", "--username",  dest="username", metavar="USERNAME", help="Username for https")
  97 parser.add_option("-p", "--password",  dest="password", metavar="PASSWD", help="Password for https")
  98
  99 (options, args) = parser.parse_args()
 100
 101 config = read_config('~/.swuprc')
 102
 103 DAILY="/pc/releases/daily/trunk"
 104 WEEKLY="/pc/releases/weekly/trunk"
 105 SNAPSHOTS="/snapshots/trunk/pc/"
 106 BASE="https://download.tz.otcshare.org/"
 107
 108 if options.type == "daily":
 109     release_url = "%s/%s" %(BASE, DAILY)
 110 if options.type == "weekly":
 111     release_url = "%s/%s" %(BASE, WEEKLY)
 112 else:
 113     release_url = "%s/%s" %(BASE, SNAPSHOTS)
 114
 115 if options.username:
 116     username = options.username
 117 elif config.has_option('DEFAULT', 'username'):
 118     username = config.get('DEFAULT', 'username')
 119 if options.password:
 120     password = options.password
 121 elif config.has_option('DEFAULT', 'password'):
 122     password = config.get('DEFAULT', 'password')
 123
 124 p1 = get_package_list2(options.image, release_url, options.old)
 125 p2 = get_package_list2(options.image, release_url, options.new)
 126
 127 pkgs1 = {'%s|%s' % (pkg, attr['arch']) for pkg, attr in p1.iteritems()}
 128 pkgs2 = {'%s|%s' % (pkg, attr['arch']) for pkg, attr in p2.iteritems()}
 129 newpkgs = [pkg.split('|')[0] for pkg in pkgs2.difference(pkgs1)]
 130
 131 pkgs1 = {'%s|%s' % (pkg, attr['version']) for pkg, attr in p1.iteritems()}
 132 pkgs2 = {'%s|%s' % (pkg, attr['version']) for pkg, attr in p2.iteritems()}
 133 changedpkgs = [pkg.split('|')[0] for pkg in pkgs2.difference(pkgs1) if pkg.split('|')[0] in p1]
 134
 135 if not os.path.exists('old'):
 136     os.makedirs('old')
 137 if not os.path.exists('new'):
 138     os.makedirs('new')
 139 if not os.path.exists('rpms'):
 140     os.makedirs('rpms')
 141
 142 for p in newpkgs:
 143     rpm = "%s-%s.%s.rpm" % (p, p2[p]['version'], p2[p]['arch'])
 144     arch = p2[p]['arch']
 145     download("%s/%s/repos/pc/x86_64/packages/%s/%s" % (release_url, options.new, arch, rpm), "new/%s" % rpm)
 146
 147 for p in changedpkgs:
 148     rpm = "%s-%s.%s.rpm" % (p, p1[p]['version'], p1[p]['arch'])
 149     arch = p1[p]['arch']
 150     download("%s/%s/repos/pc/x86_64/packages/%s/%s" % (release_url, options.old, arch, rpm), "old/%s" % rpm)
 151     rpm = "%s-%s.%s.rpm" % (p, p2[p]['version'], p2[p]['arch'])
 152     download("%s/%s/repos/pc/x86_64/packages/%s/%s" %(release_url, options.new, arch, rpm), "rpms/%s" % rpm)
 153