external/fetch_sources.py

   1 # -*- coding: utf-8 -*-
   2
   3 import os
   4 import sys
   5 import shutil
   6 import tarfile
   7 import urllib2
   8 import hashlib
   9 import argparse
  10
  11 EXTERNAL_DIR    = os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
  12
  13 class SourcePackage:
  14         def __init__(self, url, filename, checksum, dstDir, postExtract=None):
  15                 self.url                        = url
  16                 self.filename           = filename
  17                 self.checksum           = checksum
  18                 self.dstDir                     = dstDir
  19                 self.postExtract        = postExtract
  20
  21 def computeChecksum (data):
  22         return hashlib.sha256(data).hexdigest()
  23
  24 def clean (pkg):
  25         srcPath = os.path.join(EXTERNAL_DIR, pkg.dstDir)
  26
  27         for entry in os.listdir(srcPath):
  28                 if entry == "CMakeLists.txt":
  29                         continue
  30
  31                 fullPath = os.path.join(srcPath, entry)
  32
  33                 if os.path.isfile(fullPath):
  34                         os.unlink(fullPath)
  35                 elif os.path.isdir(fullPath):
  36                         shutil.rmtree(fullPath, ignore_errors=False)
  37
  38 def fetch (pkg):
  39         print "Fetching %s" % pkg.url
  40
  41         req                     = urllib2.urlopen(pkg.url)
  42         data            = req.read()
  43         checksum        = computeChecksum(data)
  44         dstPath         = os.path.join(EXTERNAL_DIR, pkg.filename)
  45
  46         if checksum != pkg.checksum:
  47                 raise Exception("Checksum mismatch for %s, exepected %s, got %s" % (pkg.filename, pkg.checksum, checksum))
  48
  49         out = open(dstPath, 'wb')
  50         out.write(data)
  51         out.close()
  52
  53 def extract (pkg):
  54         print "Extracting %s to %s" % (pkg.filename, pkg.dstDir)
  55
  56         srcPath = os.path.join(EXTERNAL_DIR, pkg.filename)
  57         tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % pkg.dstDir)
  58         dstPath = os.path.join(EXTERNAL_DIR, pkg.dstDir)
  59         archive = tarfile.open(srcPath)
  60
  61         if os.path.exists(tmpPath):
  62                 shutil.rmtree(tmpPath, ignore_errors=False)
  63
  64         os.mkdir(tmpPath)
  65
  66         archive.extractall(tmpPath)
  67         archive.close()
  68
  69         extractedEntries = os.listdir(tmpPath)
  70         if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
  71                 raise Exception("%s doesn't contain single top-level directory") % pkg.filename
  72
  73         topLevelPath = os.path.join(tmpPath, extractedEntries[0])
  74
  75         for entry in os.listdir(topLevelPath):
  76                 if os.path.exists(os.path.join(dstPath, entry)):
  77                         print "  skipping %s" % entry
  78                         continue
  79
  80                 shutil.move(os.path.join(topLevelPath, entry), dstPath)
  81
  82         shutil.rmtree(tmpPath, ignore_errors=True)
  83
  84         if pkg.postExtract != None:
  85                 pkg.postExtract(dstPath)
  86
  87 def postExtractLibpng (path):
  88         shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
  89                                 os.path.join(path, "pnglibconf.h"))
  90
  91 PACKAGES = [
  92         SourcePackage("http://zlib.net/zlib-1.2.8.tar.gz",
  93                                   "zlib-1.2.8.tar.gz",
  94                                   "36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d",
  95                                   "zlib"),
  96         SourcePackage("ftp://ftp.simplesystems.org/pub/libpng/png/src/libpng16/libpng-1.6.16.tar.gz",
  97                                   "libpng-1.6.14.tar.gz",
  98                                   "02f96b6bad5a381d36d7ba7a5d9be3b06f7fe6c274da00707509c23592a073ad",
  99                                   "libpng",
 100                                   postExtract = postExtractLibpng),
 101 ]
 102
 103 def parseArgs ():
 104         parser = argparse.ArgumentParser(description = "Fetch external sources")
 105         parser.add_argument('--clean-only', dest='cleanOnly', action='store_true', default=False,
 106                                                 help='Clean only, do not fetch/extract')
 107         parser.add_argument('--keep-archive', dest='keepArchive', action='store_true', default=False,
 108                                                 help='Keep archive after extracting')
 109         return parser.parse_args()
 110
 111 if __name__ == "__main__":
 112         args = parseArgs()
 113
 114         for pkg in PACKAGES:
 115                 clean(pkg)
 116
 117                 if args.cleanOnly:
 118                         continue
 119
 120                 fetch(pkg)
 121                 extract(pkg)
 122
 123                 if not args.keepArchive:
 124                         os.unlink(os.path.join(EXTERNAL_DIR, pkg.filename))