external/fetch_sources.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2015 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 import os
  24 import sys
  25 import shutil
  26 import tarfile
  27 import zipfile
  28 import hashlib
  29 import argparse
  30 import subprocess
  31 import ssl
  32 import stat
  33
  34 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts"))
  35
  36 from ctsbuild.common import *
  37
  38 EXTERNAL_DIR    = os.path.realpath(os.path.normpath(os.path.dirname(__file__)))
  39
  40 def computeChecksum (data):
  41         return hashlib.sha256(data).hexdigest()
  42
  43 def onReadonlyRemoveError (func, path, exc_info):
  44         os.chmod(path, stat.S_IWRITE)
  45         os.unlink(path)
  46
  47 class Source:
  48         def __init__(self, baseDir, extractDir):
  49                 self.baseDir            = baseDir
  50                 self.extractDir         = extractDir
  51
  52         def clean (self):
  53                 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
  54                 # Remove read-only first
  55                 readonlydir = os.path.join(fullDstPath, ".git")
  56                 if os.path.exists(readonlydir):
  57                         shutil.rmtree(readonlydir, onerror = onReadonlyRemoveError)
  58                 if os.path.exists(fullDstPath):
  59                         shutil.rmtree(fullDstPath, ignore_errors=False)
  60
  61 class SourcePackage (Source):
  62         def __init__(self, url, checksum, baseDir, extractDir = "src", postExtract=None):
  63                 Source.__init__(self, baseDir, extractDir)
  64                 self.url                        = url
  65                 self.filename           = os.path.basename(self.url)
  66                 self.checksum           = checksum
  67                 self.archiveDir         = "packages"
  68                 self.postExtract        = postExtract
  69                 self.sysNdx                     = {"Windows":0, "Linux":1, "Darwin":2}[platform.system()]
  70                 self.FFmpeg                     = "FFmpeg" in url
  71
  72         def clean (self):
  73                 Source.clean(self)
  74                 self.removeArchives()
  75
  76         def update (self, cmdProtocol = None, force = False):
  77                 if self.sysNdx != 2:
  78                         if not self.isArchiveUpToDate():
  79                                 self.fetchAndVerifyArchive()
  80
  81                         if self.getExtractedChecksum() != self.checksum:
  82                                 Source.clean(self)
  83                                 self.extract()
  84                                 self.storeExtractedChecksum(self.checksum)
  85
  86         def removeArchives (self):
  87                 archiveDir = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir)
  88                 if os.path.exists(archiveDir):
  89                         shutil.rmtree(archiveDir, ignore_errors=False)
  90
  91         def isArchiveUpToDate (self):
  92                 archiveFile = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, pkg.filename)
  93                 if os.path.exists(archiveFile):
  94                         return computeChecksum(readBinaryFile(archiveFile)) == self.checksum
  95                 else:
  96                         return False
  97
  98         def getExtractedChecksumFilePath (self):
  99                 return os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.archiveDir, "extracted")
 100
 101         def getExtractedChecksum (self):
 102                 extractedChecksumFile = self.getExtractedChecksumFilePath()
 103
 104                 if os.path.exists(extractedChecksumFile):
 105                         return readFile(extractedChecksumFile)
 106                 else:
 107                         return None
 108
 109         def storeExtractedChecksum (self, checksum):
 110                 checksum_bytes = checksum.encode("utf-8")
 111                 writeBinaryFile(self.getExtractedChecksumFilePath(), checksum_bytes)
 112
 113         def connectToUrl (self, url):
 114                 result = None
 115
 116                 if sys.version_info < (3, 0):
 117                         from urllib2 import urlopen
 118                 else:
 119                         from urllib.request import urlopen
 120
 121                 if args.insecure:
 122                         print("Ignoring certificate checks")
 123                         ssl_context = ssl._create_unverified_context()
 124                         result = urlopen(url, context=ssl_context)
 125                 else:
 126                         result = urlopen(url)
 127
 128                 return result
 129
 130         def fetchAndVerifyArchive (self):
 131                 print("Fetching %s" % self.url)
 132
 133                 req                     = self.connectToUrl(self.url)
 134                 data            = req.read()
 135                 checksum        = computeChecksum(data)
 136                 dstPath         = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
 137
 138                 if checksum != self.checksum:
 139                         raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
 140
 141                 if not os.path.exists(os.path.dirname(dstPath)):
 142                         os.mkdir(os.path.dirname(dstPath))
 143
 144                 writeBinaryFile(dstPath, data)
 145
 146         def extract (self):
 147                 print("Extracting %s to %s/%s" % (self.filename, self.baseDir, self.extractDir))
 148
 149                 srcPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.archiveDir, self.filename)
 150                 tmpPath = os.path.join(EXTERNAL_DIR, ".extract-tmp-%s" % self.baseDir)
 151                 dstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
 152
 153                 if self.filename.endswith(".zip"):
 154                         archive = zipfile.ZipFile(srcPath)
 155                 else:
 156                         archive = tarfile.open(srcPath)
 157
 158                 if os.path.exists(tmpPath):
 159                         shutil.rmtree(tmpPath, ignore_errors=False)
 160
 161                 os.mkdir(tmpPath)
 162
 163                 archive.extractall(tmpPath)
 164                 archive.close()
 165
 166                 extractedEntries = os.listdir(tmpPath)
 167                 if len(extractedEntries) != 1 or not os.path.isdir(os.path.join(tmpPath, extractedEntries[0])):
 168                         raise Exception("%s doesn't contain single top-level directory" % self.filename)
 169
 170                 topLevelPath = os.path.join(tmpPath, extractedEntries[0])
 171
 172                 if not os.path.exists(dstPath):
 173                         os.mkdir(dstPath)
 174
 175                 for entry in os.listdir(topLevelPath):
 176                         if os.path.exists(os.path.join(dstPath, entry)):
 177                                 raise Exception("%s exists already" % entry)
 178
 179                         shutil.move(os.path.join(topLevelPath, entry), dstPath)
 180
 181                 shutil.rmtree(tmpPath, ignore_errors=True)
 182
 183                 if self.postExtract != None:
 184                         self.postExtract(dstPath)
 185
 186 class SourceFile (Source):
 187         def __init__(self, url, filename, checksum, baseDir, extractDir = "src"):
 188                 Source.__init__(self, baseDir, extractDir)
 189                 self.url                        = url
 190                 self.filename           = filename
 191                 self.checksum           = checksum
 192
 193         def update (self, cmdProtocol = None, force = False):
 194                 if not self.isFileUpToDate():
 195                         Source.clean(self)
 196                         self.fetchAndVerifyFile()
 197
 198         def isFileUpToDate (self):
 199                 file = os.path.join(EXTERNAL_DIR, pkg.baseDir, pkg.extractDir, pkg.filename)
 200                 if os.path.exists(file):
 201                         data = readFile(file)
 202                         return computeChecksum(data.encode('utf-8')) == self.checksum
 203                 else:
 204                         return False
 205
 206         def connectToUrl (self, url):
 207                 result = None
 208
 209                 if sys.version_info < (3, 0):
 210                         from urllib2 import urlopen
 211                 else:
 212                         from urllib.request import urlopen
 213
 214                 if args.insecure:
 215                         print("Ignoring certificate checks")
 216                         ssl_context = ssl._create_unverified_context()
 217                         result = urlopen(url, context=ssl_context)
 218                 else:
 219                         result = urlopen(url)
 220
 221                 return result
 222
 223         def fetchAndVerifyFile (self):
 224                 print("Fetching %s" % self.url)
 225
 226                 req                     = self.connectToUrl(self.url)
 227                 data            = req.read()
 228                 checksum        = computeChecksum(data)
 229                 dstPath         = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir, self.filename)
 230
 231                 if checksum != self.checksum:
 232                         raise Exception("Checksum mismatch for %s, expected %s, got %s" % (self.filename, self.checksum, checksum))
 233
 234                 if not os.path.exists(os.path.dirname(dstPath)):
 235                         os.mkdir(os.path.dirname(dstPath))
 236
 237                 writeBinaryFile(dstPath, data)
 238
 239 class GitRepo (Source):
 240         def __init__(self, httpsUrl, sshUrl, revision, baseDir, extractDir = "src", removeTags = [], patch = ""):
 241                 Source.__init__(self, baseDir, extractDir)
 242                 self.httpsUrl   = httpsUrl
 243                 self.sshUrl             = sshUrl
 244                 self.revision   = revision
 245                 self.removeTags = removeTags
 246                 self.patch              = patch
 247
 248         def checkout(self, url, fullDstPath, force):
 249                 if not os.path.exists(os.path.join(fullDstPath, '.git')):
 250                         execute(["git", "clone", "--no-checkout", url, fullDstPath])
 251
 252                 pushWorkingDir(fullDstPath)
 253                 try:
 254                         for tag in self.removeTags:
 255                                 proc = subprocess.Popen(['git', 'tag', '-l', tag], stdout=subprocess.PIPE)
 256                                 (stdout, stderr) = proc.communicate()
 257                                 if len(stdout) > 0:
 258                                         execute(["git", "tag", "-d",tag])
 259                         force_arg = ['--force'] if force else []
 260                         execute(["git", "fetch"] + force_arg + ["--tags", url, "+refs/heads/*:refs/remotes/origin/*"])
 261                         execute(["git", "checkout"] + force_arg + [self.revision])
 262
 263                         if(self.patch != ""):
 264                                 patchFile = os.path.join(EXTERNAL_DIR, self.patch)
 265                                 execute(["git", "reset", "--hard", "HEAD"])
 266                                 execute(["git", "apply", patchFile])
 267                 finally:
 268                         popWorkingDir()
 269
 270         def update (self, cmdProtocol, force = False):
 271                 fullDstPath = os.path.join(EXTERNAL_DIR, self.baseDir, self.extractDir)
 272                 url         = self.httpsUrl
 273                 backupUrl   = self.sshUrl
 274
 275                 # If url is none then start with ssh
 276                 if cmdProtocol == 'ssh' or url == None:
 277                         url       = self.sshUrl
 278                         backupUrl = self.httpsUrl
 279
 280                 try:
 281                         self.checkout(url, fullDstPath, force)
 282                 except:
 283                         if backupUrl != None:
 284                                 self.checkout(backupUrl, fullDstPath, force)
 285
 286 def postExtractLibpng (path):
 287         shutil.copy(os.path.join(path, "scripts", "pnglibconf.h.prebuilt"),
 288                                 os.path.join(path, "pnglibconf.h"))
 289
 290 PACKAGES = [
 291         SourcePackage(
 292                 "http://zlib.net/fossils/zlib-1.2.13.tar.gz",
 293                 "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30",
 294                 "zlib"),
 295         SourcePackage(
 296                 "http://prdownloads.sourceforge.net/libpng/libpng-1.6.27.tar.gz",
 297                 "c9d164ec247f426a525a7b89936694aefbc91fb7a50182b198898b8fc91174b4",
 298                 "libpng",
 299                 postExtract = postExtractLibpng),
 300         SourcePackage(
 301         {"Windows":"https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2022-05-31-12-34/ffmpeg-n4.4.2-1-g8e98dfc57f-win64-lgpl-shared-4.4.zip", "Linux": "https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2022-05-31-12-34/ffmpeg-n4.4.2-1-g8e98dfc57f-linux64-gpl-shared-4.4.tar.xz", "Darwin":""}[platform.system()],
 302         {"Windows":"670df8e9d2ddd5e761459b3538f64b8826566270ef1ed13bcbfc63e73aab3fd9","Linux":"817f8c93ff1ef7ede3dad15b20415d5e366bcd6848844d55046111fd3de827d0", "Darwin":""}[platform.system()],
 303                 "ffmpeg"),
 304         SourceFile(
 305                 "https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h",
 306                 "renderdoc_app.h",
 307                 "e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e",
 308                 "renderdoc"),
 309         GitRepo(
 310                 "https://github.com/KhronosGroup/SPIRV-Tools.git",
 311                 "git@github.com:KhronosGroup/SPIRV-Tools.git",
 312                 "f98473ceeb1d33700d01e20910433583e5256030",
 313                 "spirv-tools"),
 314         GitRepo(
 315                 "https://github.com/KhronosGroup/glslang.git",
 316                 "git@github.com:KhronosGroup/glslang.git",
 317                 "a0ad0d7067521fff880e36acfb8ce453421c3f25",
 318                 "glslang",
 319                 removeTags = ["master-tot"]),
 320         GitRepo(
 321                 "https://github.com/KhronosGroup/SPIRV-Headers.git",
 322                 "git@github.com:KhronosGroup/SPIRV-Headers.git",
 323                 "87d5b782bec60822aa878941e6b13c0a9a954c9b",
 324                 "spirv-headers"),
 325         GitRepo(
 326                 "https://github.com/KhronosGroup/Vulkan-Docs.git",
 327                 "git@github.com:KhronosGroup/Vulkan-Docs.git",
 328                 "9a2e576a052a1e65a5d41b593e693ff02745604b",
 329                 "vulkan-docs"),
 330         GitRepo(
 331                 "https://github.com/google/amber.git",
 332                 "git@github.com:google/amber.git",
 333                 "8b145a6c89dcdb4ec28173339dd176fb7b6f43ed",
 334                 "amber"),
 335         GitRepo(
 336                 "https://github.com/open-source-parsers/jsoncpp.git",
 337                 "git@github.com:open-source-parsers/jsoncpp.git",
 338                 "9059f5cad030ba11d37818847443a53918c327b1",
 339                 "jsoncpp"),
 340         GitRepo(
 341                 "https://github.com/nvpro-samples/vk_video_samples.git",
 342                 None,
 343                 "7d68747d3524842afaf050c5e00a10f5b8c07904",
 344                 "video-parser"),
 345 ]
 346
 347 def parseArgs ():
 348         versionsForInsecure = ((2,7,9), (3,4,3))
 349         versionsForInsecureStr = ' or '.join(('.'.join(str(x) for x in v)) for v in versionsForInsecure)
 350
 351         parser = argparse.ArgumentParser(description = "Fetch external sources")
 352         parser.add_argument('--clean', dest='clean', action='store_true', default=False,
 353                                                 help='Remove sources instead of fetching')
 354         parser.add_argument('--insecure', dest='insecure', action='store_true', default=False,
 355                                                 help="Disable certificate check for external sources."
 356                                                 " Minimum python version required " + versionsForInsecureStr)
 357         parser.add_argument('--protocol', dest='protocol', default='https', choices=['ssh', 'https'],
 358                                                 help="Select protocol to checkout git repositories.")
 359         parser.add_argument('--force', dest='force', action='store_true', default=False,
 360                                                 help="Pass --force to git fetch and checkout commands")
 361
 362         args = parser.parse_args()
 363
 364         if args.insecure:
 365                 for versionItem in versionsForInsecure:
 366                         if (sys.version_info.major == versionItem[0]):
 367                                 if sys.version_info < versionItem:
 368                                         parser.error("For --insecure minimum required python version is " +
 369                                                                 versionsForInsecureStr)
 370                                 break;
 371
 372         return args
 373
 374 def run(*popenargs, **kwargs):
 375         process = subprocess.Popen(*popenargs, **kwargs)
 376
 377         try:
 378                 stdout, stderr = process.communicate(None)
 379         except:
 380                 process.kill()
 381                 process.wait()
 382                 raise
 383
 384         retcode = process.poll()
 385
 386         if retcode:
 387                 raise subprocess.CalledProcessError(retcode, process.args, output=stdout, stderr=stderr)
 388
 389         return retcode, stdout, stderr
 390
 391 if __name__ == "__main__":
 392         # Rerun script with python3 as python2 does not have lzma (xz) decompression support
 393         if sys.version_info < (3, 0):
 394                 cmd = {"Windows": ['py', '-3'], "Linux": ['python3'], "Darwin": ['python3']}[platform.system()]
 395                 cmd = cmd + sys.argv
 396                 run(cmd)
 397         else:
 398                 args = parseArgs()
 399
 400                 for pkg in PACKAGES:
 401                         if args.clean:
 402                                 pkg.clean()
 403                         else:
 404                                 pkg.update(args.protocol, args.force)