external/fetch_sources.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2015 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 import os
  24 import sys
  25 import shutil
  26 import tarfile
  27 import hashlib
  28 import argparse
  29 import subprocess
  30 import ssl
  31 import stat
  32
  33 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
  34
  35 from build.common import *
  36
  37 EXTERNAL_DIR    = os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
  38
  39 def computeChecksum (data):
  40         return hashlib.sha256(data).hexdigest()
  41
  42 def onReadonlyRemoveError (func, path, exc_info):
  43         os.chmod(path, stat.S_IWRITE)
  44         os.unlink(path)
  45
  46 class Source:
  47         def __init__(self, baseDir, extractDir):
  48                 self.baseDir            = baseDir
  49                 self.extractDir         = extractDir
  50
  51         def clean (self):
  52                 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
  53                 # Remove read-only first
  54                 readonlydir = os.path.join(fullDstPath, ".git", "objects", "pack")
  55                 if os.path.exists(readonlydir):
  56                         shutil.rmtree(readonlydir, onerror = onReadonlyRemoveError )
  57                 if os.path.exists(fullDstPath):
  58                         shutil.rmtree(fullDstPath, ignore_errors=False)
  59
  60 class SourcePackage (Source):
  61         def __init__(self, url, filename, checksum, baseDir, extractDir = "src", postExtract=None):
  62                 Source.__init__(self, baseDir, extractDir)
  63                 self.url                        = url
  64                 self.filename           = filename
  65                 self.checksum           = checksum
  66                 self.archiveDir         = "packages"
  67                 self.postExtract        = postExtract
  68
  69         def clean (self):
  70                 Source.clean(self)
  71                 self.removeArchives()
  72
  73         def update (self, cmdProtocol = None, force = False):
  74                 if not self.isArchiveUpToDate():
  75                         self.fetchAndVerifyArchive()
  76
  77                 if self.getExtractedChecksum() != self.checksum:
  78                         Source.clean(self)
  79                         self.extract()
  80                         self.storeExtractedChecksum(self.checksum)
  81
  82         def removeArchives (self):
  83                 archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
  84                 if os.path.exists(archiveDir):
  85                         shutil.rmtree(archiveDir, ignore_errors=False)
  86
  87         def isArchiveUpToDate (self):
  88                 archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
  89                 if os.path.exists(archiveFile):
  90                         return computeChecksum(readBinaryFile(archiveFile)) == self.checksum
  91                 else:
  92                         return False
  93
  94         def getExtractedChecksumFilePath (self):
  95                 return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
  96
  97         def getExtractedChecksum (self):
  98                 extractedChecksumFile = self.getExtractedChecksumFilePath()
  99
 100                 if os.path.exists(extractedChecksumFile):
 101                         return readFile(extractedChecksumFile)
 102                 else:
 103                         return None
 104
 105         def storeExtractedChecksum (self, checksum):
 106                 checksum_bytes = checksum.encode("utf-8")
 107                 writeBinaryFile(self.getExtractedChecksumFilePath(), checksum_bytes)
 108
 109         def connectToUrl (self, url):
 110                 result = None
 111
 112                 if sys.version_info < (3, 0):
 113                         from urllib2 import urlopen
 114                 else:
 115                         from urllib.request import urlopen
 116
 117                 if args.insecure:
 118                         print("Ignoring certificate checks")
 119                         ssl_context = ssl._create_unverified_context()
 120                         result = urlopen(url, context=ssl_context)
 121                 else:
 122                         result = urlopen(url)
 123
 124                 return result
 125
 126         def fetchAndVerifyArchive (self):
 127                 print("Fetching %s" % self.url)
 128
 129                 req                     = self.connectToUrl(self.url)
 130                 data            = req.read()
 131                 checksum        = computeChecksum(data)
 132                 dstPath         = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
 133
 134                 if checksum != self.checksum:
 135                         raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
 136
 137                 if not os.path.exists(os.path.dirname(dstPath)):
 138                         os.mkdir(os.path.dirname(dstPath))
 139
 140                 writeBinaryFile(dstPath, data)
 141
 142         def extract (self):
 143                 print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
 144
 145                 srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
 146                 tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
 147                 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
 148                 archive = tarfile.open(srcPath)
 149
 150                 if os.path.exists(tmpPath):
 151                         shutil.rmtree(tmpPath, ignore_errors=False)
 152
 153                 os.mkdir(tmpPath)
 154
 155                 archive.extractall(tmpPath)
 156                 archive.close()
 157
 158                 extractedEntries = os.listdir(tmpPath)
 159                 if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
 160                         raise Exception("%s doesn't contain single top-level directory" % self.filename)
 161
 162                 topLevelPath = os.path.join(tmpPath, extractedEntries[0])
 163
 164                 if not os.path.exists(dstPath):
 165                         os.mkdir(dstPath)
 166
 167                 for entry in os.listdir(topLevelPath):
 168                         if os.path.exists(os.path.join(dstPath, entry)):
 169                                 raise Exception("%s exists already" % entry)
 170
 171                         shutil.move(os.path.join(topLevelPath, entry), dstPath)
 172
 173                 shutil.rmtree(tmpPath, ignore_errors=True)
 174
 175                 if self.postExtract != None:
 176                         self.postExtract(dstPath)
 177
 178 class SourceFile (Source):
 179         def __init__(self, url, filename, checksum, baseDir, extractDir = "src"):
 180                 Source.__init__(self, baseDir, extractDir)
 181                 self.url                        = url
 182                 self.filename           = filename
 183                 self.checksum           = checksum
 184
 185         def update (self, cmdProtocol = None, force = False):
 186                 if not self.isFileUpToDate():
 187                         Source.clean(self)
 188                         self.fetchAndVerifyFile()
 189
 190         def isFileUpToDate (self):
 191                 file = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.extractDir, pkg.filename)
 192                 if os.path.exists(file):
 193                         data = readFile(file)
 194                         return computeChecksum(data.encode('utf-8')) == self.checksum
 195                 else:
 196                         return False
 197
 198         def connectToUrl (self, url):
 199                 result = None
 200
 201                 if sys.version_info < (3, 0):
 202                         from urllib2 import urlopen
 203                 else:
 204                         from urllib.request import urlopen
 205
 206                 if args.insecure:
 207                         print("Ignoring certificate checks")
 208                         ssl_context = ssl._create_unverified_context()
 209                         result = urlopen(url, context=ssl_context)
 210                 else:
 211                         result = urlopen(url)
 212
 213                 return result
 214
 215         def fetchAndVerifyFile (self):
 216                 print("Fetching %s" % self.url)
 217
 218                 req                     = self.connectToUrl(self.url)
 219                 data            = req.read()
 220                 checksum        = computeChecksum(data)
 221                 dstPath         = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir, self.filename)
 222
 223                 if checksum != self.checksum:
 224                         raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
 225
 226                 if not os.path.exists(os.path.dirname(dstPath)):
 227                         os.mkdir(os.path.dirname(dstPath))
 228
 229                 writeBinaryFile(dstPath, data)
 230
 231 class GitRepo (Source):
 232         def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src", removeTags = []):
 233                 Source.__init__(self, baseDir, extractDir)
 234                 self.httpsUrl   = httpsUrl
 235                 self.sshUrl             = sshUrl
 236                 self.revision   = revision
 237                 self.removeTags = removeTags
 238
 239         def checkout(self, url, fullDstPath, force):
 240                 if not os.path.exists(os.path.join(fullDstPath, '.git')):
 241                         execute(["git", "clone", "--no-checkout", url, fullDstPath])
 242
 243                 pushWorkingDir(fullDstPath)
 244                 try:
 245                         for tag in self.removeTags:
 246                                 proc = subprocess.Popen(['git', 'tag', '-l', tag], stdout=subprocess.PIPE)
 247                                 (stdout, stderr) = proc.communicate()
 248                                 if len(stdout) > 0:
 249                                         execute(["git", "tag", "-d",tag])
 250                         force_arg = ['--force'] if force else []
 251                         execute(["git", "fetch"] + force_arg + ["--tags", url, "+refs/heads/*:refs/remotes/origin/*"])
 252                         execute(["git", "checkout"] + force_arg + [self.revision])
 253                 finally:
 254                         popWorkingDir()
 255
 256         def update (self, cmdProtocol, force = False):
 257                 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
 258                 url         = self.httpsUrl
 259                 backupUrl   = self.sshUrl
 260
 261                 # If url is none then start with ssh
 262                 if cmdProtocol == 'ssh' or url == None:
 263                         url       = self.sshUrl
 264                         backupUrl = self.httpsUrl
 265
 266                 try:
 267                         self.checkout(url, fullDstPath, force)
 268                 except:
 269                         if backupUrl != None:
 270                                 self.checkout(backupUrl, fullDstPath, force)
 271
 272 def postExtractLibpng (path):
 273         shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
 274                                 os.path.join(path, "pnglibconf.h"))
 275
 276 PACKAGES = [
 277         SourcePackage(
 278                 "http://zlib.net/zlib-1.2.12.tar.gz",
 279                 "zlib-1.2.12.tar.gz",
 280                 "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9",
 281                 "zlib"),
 282         SourcePackage(
 283                 "http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
 284                 "libpng-1.6.27.tar.gz",
 285                 "c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
 286                 "libpng",
 287                 postExtract = postExtractLibpng),
 288         SourceFile(
 289                 "https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h",
 290                 "renderdoc_app.h",
 291                 "e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e",
 292                 "renderdoc"),
 293         GitRepo(
 294                 "https://github.com/KhronosGroup/SPIRV-Tools.git",
 295                 "git@github.com:KhronosGroup/SPIRV-Tools.git",
 296                 "b0ce31fd2d8fdf0bdf87832a63d3da3289202fdf",
 297                 "spirv-tools"),
 298         GitRepo(
 299                 "https://github.com/KhronosGroup/glslang.git",
 300                 "git@github.com:KhronosGroup/glslang.git",
 301                 "abbe466451ca975fecfdba453ef9073df52aefc5",
 302                 "glslang",
 303                 removeTags = ["master-tot"]),
 304         GitRepo(
 305                 "https://github.com/KhronosGroup/SPIRV-Headers.git",
 306                 "git@github.com:KhronosGroup/SPIRV-Headers.git",
 307                 "4995a2f2723c401eb0ea3e10c81298906bf1422b",
 308                 "spirv-headers"),
 309         GitRepo(
 310                 "https://gitlab.khronos.org/vulkan/vulkan.git",
 311                 "git@gitlab.khronos.org:vulkan/vulkan.git",
 312                 "7c7d5305655a171ca495e7a6e5b8cff88e67d40a",
 313                 "vulkan-docs"),
 314         GitRepo(
 315                 "https://github.com/google/amber.git",
 316                 "git@github.com:google/amber.git",
 317                 "8b145a6c89dcdb4ec28173339dd176fb7b6f43ed",
 318                 "amber"),
 319 ]
 320
 321 def parseArgs ():
 322         versionsForInsecure = ((2,7,9), (3,4,3))
 323         versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
 324
 325         parser = argparse.ArgumentParser(description = "Fetch external sources")
 326         parser.add_argument('--clean', dest='clean', action='store_true', default=False,
 327                                                 help='Remove sources instead of fetching')
 328         parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
 329                                                 help="Disable certificate check for external sources."
 330                                                 " Minimum python version required " + versionsForInsecureStr)
 331         parser.add_argument('--protocol', dest='protocol', default='https', choices=['ssh', 'https'],
 332                                                 help="Select protocol to checkout git repositories.")
 333         parser.add_argument('--force', dest='force', action='store_true', default=False,
 334                                                 help="Pass --force to git fetch and checkout commands")
 335
 336         args = parser.parse_args()
 337
 338         if args.insecure:
 339                 for versionItem in versionsForInsecure:
 340                         if (sys.version_info.major == versionItem[0]):
 341                                 if sys.version_info < versionItem:
 342                                         parser.error("For --insecure minimum required python version is " +
 343                                                                 versionsForInsecureStr)
 344                                 break;
 345
 346         return args
 347
 348 if __name__ == "__main__":
 349         args = parseArgs()
 350
 351         for pkg in PACKAGES:
 352                 if args.clean:
 353                         pkg.clean()
 354                 else:
 355                         pkg.update(args.protocol, args.force)