external/fetch_sources.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2015 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 import os
  24 import sys
  25 import shutil
  26 import tarfile
  27 import urllib2
  28 import hashlib
  29 import argparse
  30 import subprocess
  31
  32 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
  33
  34 from build.common import *
  35
  36 EXTERNAL_DIR    = os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
  37
  38 def computeChecksum (data):
  39         return hashlib.sha256(data).hexdigest()
  40
  41 class Source:
  42         def __init__(self, baseDir, extractDir):
  43                 self.baseDir            = baseDir
  44                 self.extractDir         = extractDir
  45
  46         def clean (self):
  47                 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
  48                 if os.path.exists(fullDstPath):
  49                         shutil.rmtree(fullDstPath, ignore_errors=False)
  50
  51 class SourcePackage (Source):
  52         def __init__(self, url, filename, checksum, baseDir, extractDir = "src", postExtract=None):
  53                 Source.__init__(self, baseDir, extractDir)
  54                 self.url                        = url
  55                 self.filename           = filename
  56                 self.checksum           = checksum
  57                 self.archiveDir         = "packages"
  58                 self.postExtract        = postExtract
  59
  60         def clean (self):
  61                 Source.clean(self)
  62                 self.removeArchives()
  63
  64         def update (self):
  65                 if not self.isArchiveUpToDate():
  66                         self.fetchAndVerifyArchive()
  67
  68                 if self.getExtractedChecksum() != self.checksum:
  69                         Source.clean(self)
  70                         self.extract()
  71                         self.storeExtractedChecksum(self.checksum)
  72
  73         def removeArchives (self):
  74                 archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
  75                 if os.path.exists(archiveDir):
  76                         shutil.rmtree(archiveDir, ignore_errors=False)
  77
  78         def isArchiveUpToDate (self):
  79                 archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
  80                 if os.path.exists(archiveFile):
  81                         return computeChecksum(readFile(archiveFile)) == self.checksum
  82                 else:
  83                         return False
  84
  85         def getExtractedChecksumFilePath (self):
  86                 return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
  87
  88         def getExtractedChecksum (self):
  89                 extractedChecksumFile = self.getExtractedChecksumFilePath()
  90
  91                 if os.path.exists(extractedChecksumFile):
  92                         return readFile(extractedChecksumFile)
  93                 else:
  94                         return None
  95
  96         def storeExtractedChecksum (self, checksum):
  97                 writeFile(self.getExtractedChecksumFilePath(), checksum)
  98
  99         def fetchAndVerifyArchive (self):
 100                 print "Fetching %s" % self.url
 101
 102                 req                     = urllib2.urlopen(self.url)
 103                 data            = req.read()
 104                 checksum        = computeChecksum(data)
 105                 dstPath         = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
 106
 107                 if checksum != self.checksum:
 108                         raise Exception("Checksum mismatch for %s, exepected %s, got %s" % (self.filename, self.checksum, checksum))
 109
 110                 if not os.path.exists(os.path.dirname(dstPath)):
 111                         os.mkdir(os.path.dirname(dstPath))
 112
 113                 writeFile(dstPath, data)
 114
 115         def extract (self):
 116                 print "Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir)
 117
 118                 srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
 119                 tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
 120                 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
 121                 archive = tarfile.open(srcPath)
 122
 123                 if os.path.exists(tmpPath):
 124                         shutil.rmtree(tmpPath, ignore_errors=False)
 125
 126                 os.mkdir(tmpPath)
 127
 128                 archive.extractall(tmpPath)
 129                 archive.close()
 130
 131                 extractedEntries = os.listdir(tmpPath)
 132                 if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
 133                         raise Exception("%s doesn't contain single top-level directory" % self.filename)
 134
 135                 topLevelPath = os.path.join(tmpPath, extractedEntries[0])
 136
 137                 if not os.path.exists(dstPath):
 138                         os.mkdir(dstPath)
 139
 140                 for entry in os.listdir(topLevelPath):
 141                         if os.path.exists(os.path.join(dstPath, entry)):
 142                                 raise Exception("%s exists already" % entry)
 143
 144                         shutil.move(os.path.join(topLevelPath, entry), dstPath)
 145
 146                 shutil.rmtree(tmpPath, ignore_errors=True)
 147
 148                 if self.postExtract != None:
 149                         self.postExtract(dstPath)
 150
 151 class GitRepo (Source):
 152         def __init__(self, url, revision, baseDir, extractDir = "src"):
 153                 Source.__init__(self, baseDir, extractDir)
 154                 self.url                = url
 155                 self.revision   = revision
 156
 157         def update (self):
 158                 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
 159
 160                 if not os.path.exists(fullDstPath):
 161                         execute(["git", "clone", "--no-checkout", self.url, fullDstPath])
 162
 163                 pushWorkingDir(fullDstPath)
 164                 try:
 165                         execute(["git", "fetch", self.url, "+refs/heads/*:refs/remotes/origin/*"])
 166                         execute(["git", "checkout", self.revision])
 167                 finally:
 168                         popWorkingDir()
 169
 170 def postExtractLibpng (path):
 171         shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
 172                                 os.path.join(path, "pnglibconf.h"))
 173
 174 PACKAGES = [
 175         SourcePackage(
 176                 "http://zlib.net/zlib-1.2.8.tar.gz",
 177                 "zlib-1.2.8.tar.gz",
 178                 "36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d",
 179                 "zlib"),
 180         SourcePackage(
 181                 "http://prdownloads.sourceforge.net/libpng/libpng-1.6.17.tar.gz",
 182                 "libpng-1.6.17.tar.gz",
 183                 "a18233c99e1dc59a256180e6871d9305a42e91b3f98799b3ceb98e87e9ec5e31",
 184                 "libpng",
 185                 postExtract = postExtractLibpng),
 186         GitRepo(
 187                 "https://github.com/KhronosGroup/SPIRV-Tools.git",
 188                 "f18e1f293b7cf8f24a42587b48f35bc49912a5dd",
 189                 "spirv-tools"),
 190         GitRepo(
 191                 "https://github.com/KhronosGroup/glslang.git",
 192                 "e3aa654c4b0c761b28d7864192ca8ceea6faf70a",
 193                 "glslang"),
 194         GitRepo(
 195                 "https://github.com/KhronosGroup/SPIRV-Headers.git",
 196                 "e4e22fd832810c45f46d542c67910b0c6bf4ca2d",
 197                 "spirv-headers"),
 198 ]
 199
 200 def parseArgs ():
 201         parser = argparse.ArgumentParser(description = "Fetch external sources")
 202         parser.add_argument('--clean', dest='clean', action='store_true', default=False,
 203                                                 help='Remove sources instead of fetching')
 204         return parser.parse_args()
 205
 206 if __name__ == "__main__":
 207         args = parseArgs()
 208
 209         for pkg in PACKAGES:
 210                 if args.clean:
 211                         pkg.clean()
 212                 else:
 213                         pkg.update()