external/fetch_sources.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2015 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 import os
  24 import sys
  25 import shutil
  26 import tarfile
  27 import zipfile
  28 import hashlib
  29 import argparse
  30 import subprocess
  31 import ssl
  32 import stat
  33
  34 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
  35
  36 from ctsbuild.common import *
  37
  38 EXTERNAL_DIR    = os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
  39
  40 def computeChecksum (data):
  41         return hashlib.sha256(data).hexdigest()
  42
  43 def onReadonlyRemoveError (func, path, exc_info):
  44         os.chmod(path, stat.S_IWRITE)
  45         os.unlink(path)
  46
  47 class Source:
  48         def __init__(self, baseDir, extractDir):
  49                 self.baseDir            = baseDir
  50                 self.extractDir         = extractDir
  51
  52         def clean (self):
  53                 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
  54                 # Remove read-only first
  55                 readonlydir = os.path.join(fullDstPath, ".git")
  56                 if os.path.exists(readonlydir):
  57                         shutil.rmtree(readonlydir, onerror = onReadonlyRemoveError)
  58                 if os.path.exists(fullDstPath):
  59                         shutil.rmtree(fullDstPath, ignore_errors=False)
  60
  61 class SourcePackage (Source):
  62         def __init__(self, url, filename, checksum, baseDir, extractDir = "src", postExtract=None):
  63                 Source.__init__(self, baseDir, extractDir)
  64                 self.url                        = url
  65                 self.filename           = filename
  66                 self.checksum           = checksum
  67                 self.archiveDir         = "packages"
  68                 self.postExtract        = postExtract
  69
  70                 if ("FFmpeg" in url):
  71                         ndx = {"Windows":0, "Linux":1}[platform.system()]
  72                         self.url = self.url.split()[ndx]
  73                         self.checksum = self.checksum.split()[ndx]
  74                         self.filename = self.filename.split()[ndx]
  75
  76         def clean (self):
  77                 Source.clean(self)
  78                 self.removeArchives()
  79
  80         def update (self, cmdProtocol = None, force = False):
  81                 if not self.isArchiveUpToDate():
  82                         self.fetchAndVerifyArchive()
  83
  84                 if self.getExtractedChecksum() != self.checksum:
  85                         Source.clean(self)
  86                         self.extract()
  87                         self.storeExtractedChecksum(self.checksum)
  88
  89         def removeArchives (self):
  90                 archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
  91                 if os.path.exists(archiveDir):
  92                         shutil.rmtree(archiveDir, ignore_errors=False)
  93
  94         def isArchiveUpToDate (self):
  95                 archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
  96                 if os.path.exists(archiveFile):
  97                         return computeChecksum(readBinaryFile(archiveFile)) == self.checksum
  98                 else:
  99                         return False
 100
 101         def getExtractedChecksumFilePath (self):
 102                 return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
 103
 104         def getExtractedChecksum (self):
 105                 extractedChecksumFile = self.getExtractedChecksumFilePath()
 106
 107                 if os.path.exists(extractedChecksumFile):
 108                         return readFile(extractedChecksumFile)
 109                 else:
 110                         return None
 111
 112         def storeExtractedChecksum (self, checksum):
 113                 checksum_bytes = checksum.encode("utf-8")
 114                 writeBinaryFile(self.getExtractedChecksumFilePath(), checksum_bytes)
 115
 116         def connectToUrl (self, url):
 117                 result = None
 118
 119                 if sys.version_info < (3, 0):
 120                         from urllib2 import urlopen
 121                 else:
 122                         from urllib.request import urlopen
 123
 124                 if args.insecure:
 125                         print("Ignoring certificate checks")
 126                         ssl_context = ssl._create_unverified_context()
 127                         result = urlopen(url, context=ssl_context)
 128                 else:
 129                         result = urlopen(url)
 130
 131                 return result
 132
 133         def fetchAndVerifyArchive (self):
 134                 print("Fetching %s" % self.url)
 135
 136                 req                     = self.connectToUrl(self.url)
 137                 data            = req.read()
 138                 checksum        = computeChecksum(data)
 139                 dstPath         = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
 140
 141                 if checksum != self.checksum:
 142                         raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
 143
 144                 if not os.path.exists(os.path.dirname(dstPath)):
 145                         os.mkdir(os.path.dirname(dstPath))
 146
 147                 writeBinaryFile(dstPath, data)
 148
 149         def extract (self):
 150                 print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
 151
 152                 srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
 153                 tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
 154                 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
 155
 156                 if self.filename.endswith(".zip"):
 157                         archive = zipfile.ZipFile(srcPath)
 158                 else:
 159                         archive = tarfile.open(srcPath)
 160
 161                 if os.path.exists(tmpPath):
 162                         shutil.rmtree(tmpPath, ignore_errors=False)
 163
 164                 os.mkdir(tmpPath)
 165
 166                 archive.extractall(tmpPath)
 167                 archive.close()
 168
 169                 extractedEntries = os.listdir(tmpPath)
 170                 if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
 171                         raise Exception("%s doesn't contain single top-level directory" % self.filename)
 172
 173                 topLevelPath = os.path.join(tmpPath, extractedEntries[0])
 174
 175                 if not os.path.exists(dstPath):
 176                         os.mkdir(dstPath)
 177
 178                 for entry in os.listdir(topLevelPath):
 179                         if os.path.exists(os.path.join(dstPath, entry)):
 180                                 raise Exception("%s exists already" % entry)
 181
 182                         shutil.move(os.path.join(topLevelPath, entry), dstPath)
 183
 184                 shutil.rmtree(tmpPath, ignore_errors=True)
 185
 186                 if self.postExtract != None:
 187                         self.postExtract(dstPath)
 188
 189 class SourceFile (Source):
 190         def __init__(self, url, filename, checksum, baseDir, extractDir = "src"):
 191                 Source.__init__(self, baseDir, extractDir)
 192                 self.url                        = url
 193                 self.filename           = filename
 194                 self.checksum           = checksum
 195
 196         def update (self, cmdProtocol = None, force = False):
 197                 if not self.isFileUpToDate():
 198                         Source.clean(self)
 199                         self.fetchAndVerifyFile()
 200
 201         def isFileUpToDate (self):
 202                 file = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.extractDir, pkg.filename)
 203                 if os.path.exists(file):
 204                         data = readFile(file)
 205                         return computeChecksum(data.encode('utf-8')) == self.checksum
 206                 else:
 207                         return False
 208
 209         def connectToUrl (self, url):
 210                 result = None
 211
 212                 if sys.version_info < (3, 0):
 213                         from urllib2 import urlopen
 214                 else:
 215                         from urllib.request import urlopen
 216
 217                 if args.insecure:
 218                         print("Ignoring certificate checks")
 219                         ssl_context = ssl._create_unverified_context()
 220                         result = urlopen(url, context=ssl_context)
 221                 else:
 222                         result = urlopen(url)
 223
 224                 return result
 225
 226         def fetchAndVerifyFile (self):
 227                 print("Fetching %s" % self.url)
 228
 229                 req                     = self.connectToUrl(self.url)
 230                 data            = req.read()
 231                 checksum        = computeChecksum(data)
 232                 dstPath         = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir, self.filename)
 233
 234                 if checksum != self.checksum:
 235                         raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
 236
 237                 if not os.path.exists(os.path.dirname(dstPath)):
 238                         os.mkdir(os.path.dirname(dstPath))
 239
 240                 writeBinaryFile(dstPath, data)
 241
 242 class GitRepo (Source):
 243         def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src", removeTags = [], patch = ""):
 244                 Source.__init__(self, baseDir, extractDir)
 245                 self.httpsUrl   = httpsUrl
 246                 self.sshUrl             = sshUrl
 247                 self.revision   = revision
 248                 self.removeTags = removeTags
 249                 self.patch              = patch
 250
 251         def checkout(self, url, fullDstPath, force):
 252                 if not os.path.exists(os.path.join(fullDstPath, '.git')):
 253                         execute(["git", "clone", "--no-checkout", url, fullDstPath])
 254
 255                 pushWorkingDir(fullDstPath)
 256                 try:
 257                         for tag in self.removeTags:
 258                                 proc = subprocess.Popen(['git', 'tag', '-l', tag], stdout=subprocess.PIPE)
 259                                 (stdout, stderr) = proc.communicate()
 260                                 if len(stdout) > 0:
 261                                         execute(["git", "tag", "-d",tag])
 262                         force_arg = ['--force'] if force else []
 263                         execute(["git", "fetch"] + force_arg + ["--tags", url, "+refs/heads/*:refs/remotes/origin/*"])
 264                         execute(["git", "checkout"] + force_arg + [self.revision])
 265
 266                         if(self.patch != ""):
 267                                 patchFile = os.path.join(EXTERNAL_DIR, self.patch)
 268                                 execute(["git", "reset", "--hard", "HEAD"])
 269                                 execute(["git", "apply", patchFile])
 270                 finally:
 271                         popWorkingDir()
 272
 273         def update (self, cmdProtocol, force = False):
 274                 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
 275                 url         = self.httpsUrl
 276                 backupUrl   = self.sshUrl
 277
 278                 # If url is none then start with ssh
 279                 if cmdProtocol == 'ssh' or url == None:
 280                         url       = self.sshUrl
 281                         backupUrl = self.httpsUrl
 282
 283                 try:
 284                         self.checkout(url, fullDstPath, force)
 285                 except:
 286                         if backupUrl != None:
 287                                 self.checkout(backupUrl, fullDstPath, force)
 288
 289 def postExtractLibpng (path):
 290         shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
 291                                 os.path.join(path, "pnglibconf.h"))
 292
 293 PACKAGES = [
 294         SourcePackage(
 295                 "http://zlib.net/fossils/zlib-1.2.13.tar.gz",
 296                 "zlib-1.2.13.tar.gz",
 297                 "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30",
 298                 "zlib"),
 299         SourcePackage(
 300                 "http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
 301                 "libpng-1.6.27.tar.gz",
 302                 "c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
 303                 "libpng",
 304                 postExtract = postExtractLibpng),
 305         SourcePackage(
 306                 "http://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2021-05-31-13-09/ffmpeg-N-102631-gbaf5cc5b7a-win64-lgpl-shared.zip https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2021-05-31-13-09/ffmpeg-N-102631-gbaf5cc5b7a-linux64-lgpl-shared.tar.xz",
 307                 "ffmpeg-N-102631-gbaf5cc5b7a-win64-lgpl-shared.zip ffmpeg-N-102631-gbaf5cc5b7a-linux64-lgpl-shared.tar.xz",
 308                 "9c1855b066d75de6ea6b2c3bb2c1cb87d3bd51dee056adfdcb00e4eaa1e437ad 22454a9a9639898171351b07b6f6c17288249596f96a28b7da67823988a316f9",
 309                 "ffmpeg"),
 310         SourceFile(
 311                 "https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h",
 312                 "renderdoc_app.h",
 313                 "e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e",
 314                 "renderdoc"),
 315         GitRepo(
 316                 "https://github.com/KhronosGroup/SPIRV-Tools.git",
 317                 "git@github.com:KhronosGroup/SPIRV-Tools.git",
 318                 "f98473ceeb1d33700d01e20910433583e5256030",
 319                 "spirv-tools"),
 320         GitRepo(
 321                 "https://github.com/KhronosGroup/glslang.git",
 322                 "git@github.com:KhronosGroup/glslang.git",
 323                 "a0ad0d7067521fff880e36acfb8ce453421c3f25",
 324                 "glslang",
 325                 removeTags = ["master-tot"]),
 326         GitRepo(
 327                 "https://github.com/KhronosGroup/SPIRV-Headers.git",
 328                 "git@github.com:KhronosGroup/SPIRV-Headers.git",
 329                 "87d5b782bec60822aa878941e6b13c0a9a954c9b",
 330                 "spirv-headers"),
 331         GitRepo(
 332                 "https://github.com/KhronosGroup/Vulkan-Docs.git",
 333                 "git@github.com:KhronosGroup/Vulkan-Docs.git",
 334                 "9a2e576a052a1e65a5d41b593e693ff02745604b",
 335                 "vulkan-docs"),
 336         GitRepo(
 337                 "https://github.com/google/amber.git",
 338                 "git@github.com:google/amber.git",
 339                 "8b145a6c89dcdb4ec28173339dd176fb7b6f43ed",
 340                 "amber"),
 341         GitRepo(
 342                 "https://github.com/open-source-parsers/jsoncpp.git",
 343                 "git@github.com:open-source-parsers/jsoncpp.git",
 344                 "9059f5cad030ba11d37818847443a53918c327b1",
 345                 "jsoncpp"),
 346         GitRepo(
 347                 "https://github.com/nvpro-samples/vk_video_samples.git",
 348                 None,
 349                 "68398ce672fb3b331ee6c998392951bba37e7e4d",
 350                 "video-parser"),
 351 ]
 352
 353 def parseArgs ():
 354         versionsForInsecure = ((2,7,9), (3,4,3))
 355         versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
 356
 357         parser = argparse.ArgumentParser(description = "Fetch external sources")
 358         parser.add_argument('--clean', dest='clean', action='store_true', default=False,
 359                                                 help='Remove sources instead of fetching')
 360         parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
 361                                                 help="Disable certificate check for external sources."
 362                                                 " Minimum python version required " + versionsForInsecureStr)
 363         parser.add_argument('--protocol', dest='protocol', default='https', choices=['ssh', 'https'],
 364                                                 help="Select protocol to checkout git repositories.")
 365         parser.add_argument('--force', dest='force', action='store_true', default=False,
 366                                                 help="Pass --force to git fetch and checkout commands")
 367
 368         args = parser.parse_args()
 369
 370         if args.insecure:
 371                 for versionItem in versionsForInsecure:
 372                         if (sys.version_info.major == versionItem[0]):
 373                                 if sys.version_info < versionItem:
 374                                         parser.error("For --insecure minimum required python version is " +
 375                                                                 versionsForInsecureStr)
 376                                 break;
 377
 378         return args
 379
 380 def run(*popenargs, **kwargs):
 381         process = subprocess.Popen(*popenargs, **kwargs)
 382
 383         try:
 384                 stdout, stderr = process.communicate(None)
 385         except:
 386                 process.kill()
 387                 process.wait()
 388                 raise
 389
 390         retcode = process.poll()
 391
 392         if retcode:
 393                 raise subprocess.CalledProcessError(retcode, process.args, output=stdout, stderr=stderr)
 394
 395         return retcode, stdout, stderr
 396
 397 if __name__ == "__main__":
 398         # Rerun script with python3 as python2 does not have lzma (xz) decompression support
 399         if sys.version_info < (3, 0):
 400                 cmd = {"Windows": ['py', '-3'], "Linux": ['python3']}[platform.system()]
 401                 cmd = cmd + sys.argv
 402                 run(cmd)
 403         else:
 404                 args = parseArgs()
 405
 406                 for pkg in PACKAGES:
 407                         if args.clean:
 408                                 pkg.clean()
 409                         else:
 410                                 pkg.update(args.protocol, args.force)