scripts/android/build_apk.py

   1 # -*- coding: utf-8 -*-
   2
   3 #-------------------------------------------------------------------------
   4 # drawElements Quality Program utilities
   5 # --------------------------------------
   6 #
   7 # Copyright 2017 The Android Open Source Project
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License");
  10 # you may not use this file except in compliance with the License.
  11 # You may obtain a copy of the License at
  12 #
  13 #      http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS,
  17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18 # See the License for the specific language governing permissions and
  19 # limitations under the License.
  20 #
  21 #-------------------------------------------------------------------------
  22
  23 # \todo [2017-04-10 pyry]
  24 # * Use smarter asset copy in main build
  25 #   * cmake -E copy_directory doesn't copy timestamps which will cause
  26 #     assets to be always re-packaged
  27 # * Consider adding an option for downloading SDK & NDK
  28
  29 import os
  30 import re
  31 import sys
  32 import glob
  33 import string
  34 import shutil
  35 import argparse
  36 import tempfile
  37 import xml.etree.ElementTree
  38
  39 # Import from <root>/scripts
  40 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
  41
  42 from build.common import *
  43 from build.config import *
  44 from build.build import *
  45
  46 class SDKEnv:
  47         def __init__(self, path):
  48                 self.path                               = path
  49                 self.buildToolsVersion  = SDKEnv.selectBuildToolsVersion(self.path)
  50
  51         @staticmethod
  52         def getBuildToolsVersions (path):
  53                 buildToolsPath  = os.path.join(path, "build-tools")
  54                 versions                = []
  55
  56                 if os.path.exists(buildToolsPath):
  57                         for item in os.listdir(buildToolsPath):
  58                                 m = re.match(r'^([0-9]+)\.([0-9]+)\.([0-9]+)$', item)
  59                                 if m != None:
  60                                         versions.append((int(m.group(1)), int(m.group(2)), int(m.group(3))))
  61
  62                 return versions
  63
  64         @staticmethod
  65         def selectBuildToolsVersion (path):
  66                 preferred       = [(25, 0, 2)]
  67                 versions        = SDKEnv.getBuildToolsVersions(path)
  68
  69                 if len(versions) == 0:
  70                         return (0,0,0)
  71
  72                 for candidate in preferred:
  73                         if candidate in versions:
  74                                 return candidate
  75
  76                 # Pick newest
  77                 versions.sort()
  78                 return versions[-1]
  79
  80         def getPlatformLibrary (self, apiVersion):
  81                 return os.path.join(self.path, "platforms", "android-%d" % apiVersion, "android.jar")
  82
  83         def getBuildToolsPath (self):
  84                 return os.path.join(self.path, "build-tools", "%d.%d.%d" % self.buildToolsVersion)
  85
  86 class NDKEnv:
  87         def __init__(self, path):
  88                 self.path               = path
  89                 self.version    = NDKEnv.detectVersion(self.path)
  90                 self.hostOsName = NDKEnv.detectHostOsName(self.path)
  91
  92         @staticmethod
  93         def getKnownAbis ():
  94                 return ["armeabi-v7a", "arm64-v8a", "x86", "x86_64"]
  95
  96         @staticmethod
  97         def getAbiPrebuiltsName (abiName):
  98                 prebuilts = {
  99                         "armeabi-v7a":  'android-arm',
 100                         "arm64-v8a":    'android-arm64',
 101                         "x86":                  'android-x86',
 102                         "x86_64":               'android-x86_64',
 103                 }
 104
 105                 if not abiName in prebuilts:
 106                         raise Exception("Unknown ABI: " + abiName)
 107
 108                 return prebuilts[abiName]
 109
 110         @staticmethod
 111         def detectVersion (path):
 112                 propFilePath = os.path.join(path, "source.properties")
 113                 try:
 114                         with open(propFilePath) as propFile:
 115                                 for line in propFile:
 116                                         keyValue = list(map(lambda x: x.strip(), line.split("=")))
 117                                         if keyValue[0] == "Pkg.Revision":
 118                                                 versionParts = keyValue[1].split(".")
 119                                                 return tuple(map(int, versionParts[0:2]))
 120                 except Exception as e:
 121                         raise Exception("Failed to read source prop file '%s': %s" % (propFilePath, str(e)))
 122                 except:
 123                         raise Exception("Failed to read source prop file '%s': unkown error")
 124
 125                 raise Exception("Failed to detect NDK version (does %s/source.properties have Pkg.Revision?)" % path)
 126
 127         @staticmethod
 128         def isHostOsSupported (hostOsName):
 129                 os                      = HostInfo.getOs()
 130                 bits            = HostInfo.getArchBits()
 131                 hostOsParts     = hostOsName.split('-')
 132
 133                 if len(hostOsParts) > 1:
 134                         assert(len(hostOsParts) == 2)
 135                         assert(hostOsParts[1] == "x86_64")
 136
 137                         if bits != 64:
 138                                 return False
 139
 140                 if os == HostInfo.OS_WINDOWS:
 141                         return hostOsParts[0] == 'windows'
 142                 elif os == HostInfo.OS_LINUX:
 143                         return hostOsParts[0] == 'linux'
 144                 elif os == HostInfo.OS_OSX:
 145                         return hostOsParts[0] == 'darwin'
 146                 else:
 147                         raise Exception("Unhandled HostInfo.getOs() '%d'" % os)
 148
 149         @staticmethod
 150         def detectHostOsName (path):
 151                 hostOsNames = [
 152                         "windows",
 153                         "windows-x86_64",
 154                         "darwin-x86",
 155                         "darwin-x86_64",
 156                         "linux-x86",
 157                         "linux-x86_64"
 158                 ]
 159
 160                 for name in hostOsNames:
 161                         if os.path.exists(os.path.join(path, "prebuilt", name)):
 162                                 return name
 163
 164                 raise Exception("Failed to determine NDK host OS")
 165
 166 class Environment:
 167         def __init__(self, sdk, ndk):
 168                 self.sdk                = sdk
 169                 self.ndk                = ndk
 170
 171 class Configuration:
 172         def __init__(self, env, buildPath, abis, nativeApi, minApi, nativeBuildType, gtfTarget, verbose, layers, angle):
 173                 self.env                                = env
 174                 self.sourcePath                 = DEQP_DIR
 175                 self.buildPath                  = buildPath
 176                 self.abis                               = abis
 177                 self.nativeApi                  = nativeApi
 178                 self.javaApi                    = 28
 179                 self.minApi                             = minApi
 180                 self.nativeBuildType    = nativeBuildType
 181                 self.gtfTarget                  = gtfTarget
 182                 self.verbose                    = verbose
 183                 self.layers                             = layers
 184                 self.angle                              = angle
 185                 self.cmakeGenerator             = selectFirstAvailableGenerator([NINJA_GENERATOR, MAKEFILE_GENERATOR, NMAKE_GENERATOR])
 186
 187         def check (self):
 188                 if self.cmakeGenerator == None:
 189                         raise Exception("Failed to find build tools for CMake")
 190
 191                 if not os.path.exists(self.env.ndk.path):
 192                         raise Exception("Android NDK not found at %s" % self.env.ndk.path)
 193
 194                 if not NDKEnv.isHostOsSupported(self.env.ndk.hostOsName):
 195                         raise Exception("NDK '%s' is not supported on this machine" % self.env.ndk.hostOsName)
 196
 197                 if self.env.ndk.version[0] < 15:
 198                         raise Exception("Android NDK version %d is not supported; build requires NDK version >= 15" % (self.env.ndk.version[0]))
 199
 200                 if not (self.minApi <= self.javaApi <= self.nativeApi):
 201                         raise Exception("Requires: min-api (%d) <= java-api (%d) <= native-api (%d)" % (self.minApi, self.javaApi, self.nativeApi))
 202
 203                 if self.env.sdk.buildToolsVersion == (0,0,0):
 204                         raise Exception("No build tools directory found at %s" % os.path.join(self.env.sdk.path, "build-tools"))
 205
 206                 androidBuildTools = ["aapt", "zipalign", "dx"]
 207                 for tool in androidBuildTools:
 208                         if which(tool, [self.env.sdk.getBuildToolsPath()]) == None:
 209                                 raise Exception("Missing Android build tool: %s" % toolPath)
 210
 211                 requiredToolsInPath = ["javac", "jar", "jarsigner", "keytool"]
 212                 for tool in requiredToolsInPath:
 213                         if which(tool) == None:
 214                                 raise Exception("%s not in PATH" % tool)
 215
 216 def log (config, msg):
 217         if config.verbose:
 218                 print(msg)
 219
 220 def executeAndLog (config, args):
 221         if config.verbose:
 222                 print(" ".join(args))
 223         execute(args)
 224
 225 # Path components
 226
 227 class ResolvablePathComponent:
 228         def __init__ (self):
 229                 pass
 230
 231 class SourceRoot (ResolvablePathComponent):
 232         def resolve (self, config):
 233                 return config.sourcePath
 234
 235 class BuildRoot (ResolvablePathComponent):
 236         def resolve (self, config):
 237                 return config.buildPath
 238
 239 class NativeBuildPath (ResolvablePathComponent):
 240         def __init__ (self, abiName):
 241                 self.abiName = abiName
 242
 243         def resolve (self, config):
 244                 return getNativeBuildPath(config, self.abiName)
 245
 246 class GeneratedResSourcePath (ResolvablePathComponent):
 247         def __init__ (self, package):
 248                 self.package = package
 249
 250         def resolve (self, config):
 251                 packageComps    = self.package.getPackageName(config).split('.')
 252                 packageDir              = os.path.join(*packageComps)
 253
 254                 return os.path.join(config.buildPath, self.package.getAppDirName(), "src", packageDir, "R.java")
 255
 256 def resolvePath (config, path):
 257         resolvedComps = []
 258
 259         for component in path:
 260                 if isinstance(component, ResolvablePathComponent):
 261                         resolvedComps.append(component.resolve(config))
 262                 else:
 263                         resolvedComps.append(str(component))
 264
 265         return os.path.join(*resolvedComps)
 266
 267 def resolvePaths (config, paths):
 268         return list(map(lambda p: resolvePath(config, p), paths))
 269
 270 class BuildStep:
 271         def __init__ (self):
 272                 pass
 273
 274         def getInputs (self):
 275                 return []
 276
 277         def getOutputs (self):
 278                 return []
 279
 280         @staticmethod
 281         def expandPathsToFiles (paths):
 282                 """
 283                 Expand mixed list of file and directory paths into a flattened list
 284                 of files. Any non-existent input paths are preserved as is.
 285                 """
 286
 287                 def getFiles (dirPath):
 288                         for root, dirs, files in os.walk(dirPath):
 289                                 for file in files:
 290                                         yield os.path.join(root, file)
 291
 292                 files = []
 293                 for path in paths:
 294                         if os.path.isdir(path):
 295                                 files += list(getFiles(path))
 296                         else:
 297                                 files.append(path)
 298
 299                 return files
 300
 301         def isUpToDate (self, config):
 302                 inputs                          = resolvePaths(config, self.getInputs())
 303                 outputs                         = resolvePaths(config, self.getOutputs())
 304
 305                 assert len(inputs) > 0 and len(outputs) > 0
 306
 307                 expandedInputs          = BuildStep.expandPathsToFiles(inputs)
 308                 expandedOutputs         = BuildStep.expandPathsToFiles(outputs)
 309
 310                 existingInputs          = list(filter(os.path.exists, expandedInputs))
 311                 existingOutputs         = list(filter(os.path.exists, expandedOutputs))
 312
 313                 if len(existingInputs) != len(expandedInputs):
 314                         for file in expandedInputs:
 315                                 if file not in existingInputs:
 316                                         print("ERROR: Missing input file: %s" % file)
 317                         die("Missing input files")
 318
 319                 if len(existingOutputs) != len(expandedOutputs):
 320                         return False # One or more output files are missing
 321
 322                 lastInputChange         = max(map(os.path.getmtime, existingInputs))
 323                 firstOutputChange       = min(map(os.path.getmtime, existingOutputs))
 324
 325                 return lastInputChange <= firstOutputChange
 326
 327         def update (config):
 328                 die("BuildStep.update() not implemented")
 329
 330 def getNativeBuildPath (config, abiName):
 331         return os.path.join(config.buildPath, "%s-%s-%d" % (abiName, config.nativeBuildType, config.nativeApi))
 332
 333 def clearCMakeCacheVariables(args):
 334         # New value, so clear the necessary cmake variables
 335         args.append('-UANGLE_LIBS')
 336         args.append('-UGLES1_LIBRARY')
 337         args.append('-UGLES2_LIBRARY')
 338         args.append('-UEGL_LIBRARY')
 339
 340 def buildNativeLibrary (config, abiName):
 341         def makeNDKVersionString (version):
 342                 minorVersionString = (chr(ord('a') + version[1]) if version[1] > 0 else "")
 343                 return "r%d%s" % (version[0], minorVersionString)
 344
 345         def getBuildArgs (config, abiName):
 346                 args = ['-DDEQP_TARGET=android',
 347                                 '-DDEQP_TARGET_TOOLCHAIN=ndk-modern',
 348                                 '-DCMAKE_C_FLAGS=-Werror',
 349                                 '-DCMAKE_CXX_FLAGS=-Werror',
 350                                 '-DANDROID_NDK_PATH=%s' % config.env.ndk.path,
 351                                 '-DANDROID_ABI=%s' % abiName,
 352                                 '-DDE_ANDROID_API=%s' % config.nativeApi,
 353                                 '-DGLCTS_GTF_TARGET=%s' % config.gtfTarget]
 354
 355                 if config.angle is None:
 356                         # Find any previous builds that may have embedded ANGLE libs and clear the CMake cache
 357                         for abi in NDKEnv.getKnownAbis():
 358                                 cMakeCachePath = os.path.join(getNativeBuildPath(config, abi), "CMakeCache.txt")
 359                                 try:
 360                                         if 'ANGLE_LIBS' in open(cMakeCachePath).read():
 361                                                 clearCMakeCacheVariables(args)
 362                                 except IOError:
 363                                         pass
 364                 else:
 365                         cMakeCachePath = os.path.join(getNativeBuildPath(config, abiName), "CMakeCache.txt")
 366                         angleLibsDir = os.path.join(config.angle, abiName)
 367                         # Check if the user changed where the ANGLE libs are being loaded from
 368                         try:
 369                                 if angleLibsDir not in open(cMakeCachePath).read():
 370                                         clearCMakeCacheVariables(args)
 371                         except IOError:
 372                                 pass
 373                         args.append('-DANGLE_LIBS=%s' % angleLibsDir)
 374
 375                 return args
 376
 377         nativeBuildPath = getNativeBuildPath(config, abiName)
 378         buildConfig             = BuildConfig(nativeBuildPath, config.nativeBuildType, getBuildArgs(config, abiName))
 379
 380         build(buildConfig, config.cmakeGenerator, ["deqp"])
 381
 382 def executeSteps (config, steps):
 383         for step in steps:
 384                 if not step.isUpToDate(config):
 385                         step.update(config)
 386
 387 def parsePackageName (manifestPath):
 388         tree = xml.etree.ElementTree.parse(manifestPath)
 389
 390         if not 'package' in tree.getroot().attrib:
 391                 raise Exception("'package' attribute missing from root element in %s" % manifestPath)
 392
 393         return tree.getroot().attrib['package']
 394
 395 class PackageDescription:
 396         def __init__ (self, appDirName, appName, hasResources = True):
 397                 self.appDirName         = appDirName
 398                 self.appName            = appName
 399                 self.hasResources       = hasResources
 400
 401         def getAppName (self):
 402                 return self.appName
 403
 404         def getAppDirName (self):
 405                 return self.appDirName
 406
 407         def getPackageName (self, config):
 408                 manifestPath    = resolvePath(config, self.getManifestPath())
 409
 410                 return parsePackageName(manifestPath)
 411
 412         def getManifestPath (self):
 413                 return [SourceRoot(), "android", self.appDirName, "AndroidManifest.xml"]
 414
 415         def getResPath (self):
 416                 return [SourceRoot(), "android", self.appDirName, "res"]
 417
 418         def getSourcePaths (self):
 419                 return [
 420                                 [SourceRoot(), "android", self.appDirName, "src"]
 421                         ]
 422
 423         def getAssetsPath (self):
 424                 return [BuildRoot(), self.appDirName, "assets"]
 425
 426         def getClassesJarPath (self):
 427                 return [BuildRoot(), self.appDirName, "bin", "classes.jar"]
 428
 429         def getClassesDexPath (self):
 430                 return [BuildRoot(), self.appDirName, "bin", "classes.dex"]
 431
 432         def getAPKPath (self):
 433                 return [BuildRoot(), self.appDirName, "bin", self.appName + ".apk"]
 434
 435 # Build step implementations
 436
 437 class BuildNativeLibrary (BuildStep):
 438         def __init__ (self, abi):
 439                 self.abi = abi
 440
 441         def isUpToDate (self, config):
 442                 return False
 443
 444         def update (self, config):
 445                 log(config, "BuildNativeLibrary: %s" % self.abi)
 446                 buildNativeLibrary(config, self.abi)
 447
 448 class GenResourcesSrc (BuildStep):
 449         def __init__ (self, package):
 450                 self.package = package
 451
 452         def getInputs (self):
 453                 return [self.package.getResPath(), self.package.getManifestPath()]
 454
 455         def getOutputs (self):
 456                 return [[GeneratedResSourcePath(self.package)]]
 457
 458         def update (self, config):
 459                 aaptPath        = which("aapt", [config.env.sdk.getBuildToolsPath()])
 460                 dstDir          = os.path.dirname(resolvePath(config, [GeneratedResSourcePath(self.package)]))
 461
 462                 if not os.path.exists(dstDir):
 463                         os.makedirs(dstDir)
 464
 465                 executeAndLog(config, [
 466                                 aaptPath,
 467                                 "package",
 468                                 "-f",
 469                                 "-m",
 470                                 "-S", resolvePath(config, self.package.getResPath()),
 471                                 "-M", resolvePath(config, self.package.getManifestPath()),
 472                                 "-J", resolvePath(config, [BuildRoot(), self.package.getAppDirName(), "src"]),
 473                                 "-I", config.env.sdk.getPlatformLibrary(config.javaApi)
 474                         ])
 475
 476 # Builds classes.jar from *.java files
 477 class BuildJavaSource (BuildStep):
 478         def __init__ (self, package, libraries = []):
 479                 self.package    = package
 480                 self.libraries  = libraries
 481
 482         def getSourcePaths (self):
 483                 srcPaths = self.package.getSourcePaths()
 484
 485                 if self.package.hasResources:
 486                         srcPaths.append([BuildRoot(), self.package.getAppDirName(), "src"]) # Generated sources
 487
 488                 return srcPaths
 489
 490         def getInputs (self):
 491                 inputs = self.getSourcePaths()
 492
 493                 for lib in self.libraries:
 494                         inputs.append(lib.getClassesJarPath())
 495
 496                 return inputs
 497
 498         def getOutputs (self):
 499                 return [self.package.getClassesJarPath()]
 500
 501         def update (self, config):
 502                 srcPaths        = resolvePaths(config, self.getSourcePaths())
 503                 srcFiles        = BuildStep.expandPathsToFiles(srcPaths)
 504                 jarPath         = resolvePath(config, self.package.getClassesJarPath())
 505                 objPath         = resolvePath(config, [BuildRoot(), self.package.getAppDirName(), "obj"])
 506                 classPaths      = [objPath] + [resolvePath(config, lib.getClassesJarPath()) for lib in self.libraries]
 507                 pathSep         = ";" if HostInfo.getOs() == HostInfo.OS_WINDOWS else ":"
 508
 509                 if os.path.exists(objPath):
 510                         shutil.rmtree(objPath)
 511
 512                 os.makedirs(objPath)
 513
 514                 for srcFile in srcFiles:
 515                         executeAndLog(config, [
 516                                         "javac",
 517                                         "-source", "1.7",
 518                                         "-target", "1.7",
 519                                         "-d", objPath,
 520                                         "-bootclasspath", config.env.sdk.getPlatformLibrary(config.javaApi),
 521                                         "-classpath", pathSep.join(classPaths),
 522                                         "-sourcepath", pathSep.join(srcPaths),
 523                                         srcFile
 524                                 ])
 525
 526                 if not os.path.exists(os.path.dirname(jarPath)):
 527                         os.makedirs(os.path.dirname(jarPath))
 528
 529                 try:
 530                         pushWorkingDir(objPath)
 531                         executeAndLog(config, [
 532                                         "jar",
 533                                         "cf",
 534                                         jarPath,
 535                                         "."
 536                                 ])
 537                 finally:
 538                         popWorkingDir()
 539
 540 class BuildDex (BuildStep):
 541         def __init__ (self, package, libraries):
 542                 self.package    = package
 543                 self.libraries  = libraries
 544
 545         def getInputs (self):
 546                 return [self.package.getClassesJarPath()] + [lib.getClassesJarPath() for lib in self.libraries]
 547
 548         def getOutputs (self):
 549                 return [self.package.getClassesDexPath()]
 550
 551         def update (self, config):
 552                 dxPath          = which("dx", [config.env.sdk.getBuildToolsPath()])
 553                 srcPaths        = resolvePaths(config, self.getInputs())
 554                 dexPath         = resolvePath(config, self.package.getClassesDexPath())
 555                 jarPaths        = [resolvePath(config, self.package.getClassesJarPath())]
 556
 557                 for lib in self.libraries:
 558                         jarPaths.append(resolvePath(config, lib.getClassesJarPath()))
 559
 560                 executeAndLog(config, [
 561                                 dxPath,
 562                                 "--dex",
 563                                 "--output", dexPath
 564                         ] + jarPaths)
 565
 566 class CreateKeystore (BuildStep):
 567         def __init__ (self):
 568                 self.keystorePath       = [BuildRoot(), "debug.keystore"]
 569
 570         def getOutputs (self):
 571                 return [self.keystorePath]
 572
 573         def isUpToDate (self, config):
 574                 return os.path.exists(resolvePath(config, self.keystorePath))
 575
 576         def update (self, config):
 577                 executeAndLog(config, [
 578                                 "keytool",
 579                                 "-genkey",
 580                                 "-keystore", resolvePath(config, self.keystorePath),
 581                                 "-storepass", "android",
 582                                 "-alias", "androiddebugkey",
 583                                 "-keypass", "android",
 584                                 "-keyalg", "RSA",
 585                                 "-keysize", "2048",
 586                                 "-validity", "10000",
 587                                 "-dname", "CN=, OU=, O=, L=, S=, C=",
 588                         ])
 589
 590 # Builds APK without code
 591 class BuildBaseAPK (BuildStep):
 592         def __init__ (self, package, libraries = []):
 593                 self.package    = package
 594                 self.libraries  = libraries
 595                 self.dstPath    = [BuildRoot(), self.package.getAppDirName(), "tmp", "base.apk"]
 596
 597         def getResPaths (self):
 598                 paths = []
 599                 for pkg in [self.package] + self.libraries:
 600                         if pkg.hasResources:
 601                                 paths.append(pkg.getResPath())
 602                 return paths
 603
 604         def getInputs (self):
 605                 return [self.package.getManifestPath()] + self.getResPaths()
 606
 607         def getOutputs (self):
 608                 return [self.dstPath]
 609
 610         def update (self, config):
 611                 aaptPath        = which("aapt", [config.env.sdk.getBuildToolsPath()])
 612                 dstPath         = resolvePath(config, self.dstPath)
 613
 614                 if not os.path.exists(os.path.dirname(dstPath)):
 615                         os.makedirs(os.path.dirname(dstPath))
 616
 617                 args = [
 618                         aaptPath,
 619                         "package",
 620                         "-f",
 621                         "--min-sdk-version", str(config.minApi),
 622                         "--target-sdk-version", str(config.javaApi),
 623                         "-M", resolvePath(config, self.package.getManifestPath()),
 624                         "-I", config.env.sdk.getPlatformLibrary(config.javaApi),
 625                         "-F", dstPath,
 626                 ]
 627
 628                 for resPath in self.getResPaths():
 629                         args += ["-S", resolvePath(config, resPath)]
 630
 631                 if config.verbose:
 632                         args.append("-v")
 633
 634                 executeAndLog(config, args)
 635
 636 def addFilesToAPK (config, apkPath, baseDir, relFilePaths):
 637         aaptPath                = which("aapt", [config.env.sdk.getBuildToolsPath()])
 638         maxBatchSize    = 25
 639
 640         pushWorkingDir(baseDir)
 641         try:
 642                 workQueue = list(relFilePaths)
 643                 # Workaround for Windows.
 644                 if os.path.sep == "\\":
 645                         workQueue = [i.replace("\\", "/") for i in workQueue]
 646
 647                 while len(workQueue) > 0:
 648                         batchSize       = min(len(workQueue), maxBatchSize)
 649                         items           = workQueue[0:batchSize]
 650
 651                         executeAndLog(config, [
 652                                         aaptPath,
 653                                         "add",
 654                                         "-f", apkPath,
 655                                 ] + items)
 656
 657                         del workQueue[0:batchSize]
 658         finally:
 659                 popWorkingDir()
 660
 661 def addFileToAPK (config, apkPath, baseDir, relFilePath):
 662         addFilesToAPK(config, apkPath, baseDir, [relFilePath])
 663
 664 class AddJavaToAPK (BuildStep):
 665         def __init__ (self, package):
 666                 self.package    = package
 667                 self.srcPath    = BuildBaseAPK(self.package).getOutputs()[0]
 668                 self.dstPath    = [BuildRoot(), self.package.getAppDirName(), "tmp", "with-java.apk"]
 669
 670         def getInputs (self):
 671                 return [
 672                                 self.srcPath,
 673                                 self.package.getClassesDexPath(),
 674                         ]
 675
 676         def getOutputs (self):
 677                 return [self.dstPath]
 678
 679         def update (self, config):
 680                 srcPath         = resolvePath(config, self.srcPath)
 681                 dstPath         = resolvePath(config, self.getOutputs()[0])
 682                 dexPath         = resolvePath(config, self.package.getClassesDexPath())
 683
 684                 shutil.copyfile(srcPath, dstPath)
 685                 addFileToAPK(config, dstPath, os.path.dirname(dexPath), os.path.basename(dexPath))
 686
 687 class AddAssetsToAPK (BuildStep):
 688         def __init__ (self, package, abi):
 689                 self.package    = package
 690                 self.buildPath  = [NativeBuildPath(abi)]
 691                 self.srcPath    = AddJavaToAPK(self.package).getOutputs()[0]
 692                 self.dstPath    = [BuildRoot(), self.package.getAppDirName(), "tmp", "with-assets.apk"]
 693
 694         def getInputs (self):
 695                 return [
 696                                 self.srcPath,
 697                                 self.buildPath + ["assets"]
 698                         ]
 699
 700         def getOutputs (self):
 701                 return [self.dstPath]
 702
 703         @staticmethod
 704         def getAssetFiles (buildPath):
 705                 allFiles = BuildStep.expandPathsToFiles([os.path.join(buildPath, "assets")])
 706                 return [os.path.relpath(p, buildPath) for p in allFiles]
 707
 708         def update (self, config):
 709                 srcPath         = resolvePath(config, self.srcPath)
 710                 dstPath         = resolvePath(config, self.getOutputs()[0])
 711                 buildPath       = resolvePath(config, self.buildPath)
 712                 assetFiles      = AddAssetsToAPK.getAssetFiles(buildPath)
 713
 714                 shutil.copyfile(srcPath, dstPath)
 715
 716                 addFilesToAPK(config, dstPath, buildPath, assetFiles)
 717
 718 class AddNativeLibsToAPK (BuildStep):
 719         def __init__ (self, package, abis):
 720                 self.package    = package
 721                 self.abis               = abis
 722                 self.srcPath    = AddAssetsToAPK(self.package, "").getOutputs()[0]
 723                 self.dstPath    = [BuildRoot(), self.package.getAppDirName(), "tmp", "with-native-libs.apk"]
 724
 725         def getInputs (self):
 726                 paths = [self.srcPath]
 727                 for abi in self.abis:
 728                         paths.append([NativeBuildPath(abi), "libdeqp.so"])
 729                 return paths
 730
 731         def getOutputs (self):
 732                 return [self.dstPath]
 733
 734         def update (self, config):
 735                 srcPath         = resolvePath(config, self.srcPath)
 736                 dstPath         = resolvePath(config, self.getOutputs()[0])
 737                 pkgPath         = resolvePath(config, [BuildRoot(), self.package.getAppDirName()])
 738                 libFiles        = []
 739
 740                 # Create right directory structure first
 741                 for abi in self.abis:
 742                         libSrcPath      = resolvePath(config, [NativeBuildPath(abi), "libdeqp.so"])
 743                         libRelPath      = os.path.join("lib", abi, "libdeqp.so")
 744                         libAbsPath      = os.path.join(pkgPath, libRelPath)
 745
 746                         if not os.path.exists(os.path.dirname(libAbsPath)):
 747                                 os.makedirs(os.path.dirname(libAbsPath))
 748
 749                         shutil.copyfile(libSrcPath, libAbsPath)
 750                         libFiles.append(libRelPath)
 751
 752                         if config.layers:
 753                                 layersGlob = os.path.join(config.layers, abi, "libVkLayer_*.so")
 754                                 libVkLayers = glob.glob(layersGlob)
 755                                 for layer in libVkLayers:
 756                                         layerFilename = os.path.basename(layer)
 757                                         layerRelPath = os.path.join("lib", abi, layerFilename)
 758                                         layerAbsPath = os.path.join(pkgPath, layerRelPath)
 759                                         shutil.copyfile(layer, layerAbsPath)
 760                                         libFiles.append(layerRelPath)
 761                                         print("Adding layer binary: %s" % (layer,))
 762
 763                         if config.angle:
 764                                 angleGlob = os.path.join(config.angle, abi, "lib*_angle.so")
 765                                 libAngle = glob.glob(angleGlob)
 766                                 for lib in libAngle:
 767                                         libFilename = os.path.basename(lib)
 768                                         libRelPath = os.path.join("lib", abi, libFilename)
 769                                         libAbsPath = os.path.join(pkgPath, libRelPath)
 770                                         shutil.copyfile(lib, libAbsPath)
 771                                         libFiles.append(libRelPath)
 772                                         print("Adding ANGLE binary: %s" % (lib,))
 773
 774                 shutil.copyfile(srcPath, dstPath)
 775                 addFilesToAPK(config, dstPath, pkgPath, libFiles)
 776
 777 class SignAPK (BuildStep):
 778         def __init__ (self, package):
 779                 self.package            = package
 780                 self.srcPath            = AddNativeLibsToAPK(self.package, []).getOutputs()[0]
 781                 self.dstPath            = [BuildRoot(), self.package.getAppDirName(), "tmp", "signed.apk"]
 782                 self.keystorePath       = CreateKeystore().getOutputs()[0]
 783
 784         def getInputs (self):
 785                 return [self.srcPath, self.keystorePath]
 786
 787         def getOutputs (self):
 788                 return [self.dstPath]
 789
 790         def update (self, config):
 791                 srcPath         = resolvePath(config, self.srcPath)
 792                 dstPath         = resolvePath(config, self.dstPath)
 793
 794                 executeAndLog(config, [
 795                                 "jarsigner",
 796                                 "-keystore", resolvePath(config, self.keystorePath),
 797                                 "-storepass", "android",
 798                                 "-keypass", "android",
 799                                 "-signedjar", dstPath,
 800                                 srcPath,
 801                                 "androiddebugkey"
 802                         ])
 803
 804 def getBuildRootRelativeAPKPath (package):
 805         return os.path.join(package.getAppDirName(), package.getAppName() + ".apk")
 806
 807 class FinalizeAPK (BuildStep):
 808         def __init__ (self, package):
 809                 self.package            = package
 810                 self.srcPath            = SignAPK(self.package).getOutputs()[0]
 811                 self.dstPath            = [BuildRoot(), getBuildRootRelativeAPKPath(self.package)]
 812                 self.keystorePath       = CreateKeystore().getOutputs()[0]
 813
 814         def getInputs (self):
 815                 return [self.srcPath]
 816
 817         def getOutputs (self):
 818                 return [self.dstPath]
 819
 820         def update (self, config):
 821                 srcPath                 = resolvePath(config, self.srcPath)
 822                 dstPath                 = resolvePath(config, self.dstPath)
 823                 zipalignPath    = os.path.join(config.env.sdk.getBuildToolsPath(), "zipalign")
 824
 825                 executeAndLog(config, [
 826                                 zipalignPath,
 827                                 "-f", "4",
 828                                 srcPath,
 829                                 dstPath
 830                         ])
 831
 832 def getBuildStepsForPackage (abis, package, libraries = []):
 833         steps = []
 834
 835         assert len(abis) > 0
 836
 837         # Build native code first
 838         for abi in abis:
 839                 steps += [BuildNativeLibrary(abi)]
 840
 841         # Build library packages
 842         for library in libraries:
 843                 if library.hasResources:
 844                         steps.append(GenResourcesSrc(library))
 845                 steps.append(BuildJavaSource(library))
 846
 847         # Build main package .java sources
 848         if package.hasResources:
 849                 steps.append(GenResourcesSrc(package))
 850         steps.append(BuildJavaSource(package, libraries))
 851         steps.append(BuildDex(package, libraries))
 852
 853         # Build base APK
 854         steps.append(BuildBaseAPK(package, libraries))
 855         steps.append(AddJavaToAPK(package))
 856
 857         # Add assets from first ABI
 858         steps.append(AddAssetsToAPK(package, abis[0]))
 859
 860         # Add native libs to APK
 861         steps.append(AddNativeLibsToAPK(package, abis))
 862
 863         # Finalize APK
 864         steps.append(CreateKeystore())
 865         steps.append(SignAPK(package))
 866         steps.append(FinalizeAPK(package))
 867
 868         return steps
 869
 870 def getPackageAndLibrariesForTarget (target):
 871         deqpPackage     = PackageDescription("package", "dEQP")
 872         ctsPackage      = PackageDescription("openglcts", "Khronos-CTS", hasResources = False)
 873
 874         if target == 'deqp':
 875                 return (deqpPackage, [])
 876         elif target == 'openglcts':
 877                 return (ctsPackage, [deqpPackage])
 878         else:
 879                 raise Exception("Uknown target '%s'" % target)
 880
 881 def findNDK ():
 882         ndkBuildPath = which('ndk-build')
 883         if ndkBuildPath != None:
 884                 return os.path.dirname(ndkBuildPath)
 885         else:
 886                 return None
 887
 888 def findSDK ():
 889         sdkBuildPath = which('android')
 890         if sdkBuildPath != None:
 891                 return os.path.dirname(os.path.dirname(sdkBuildPath))
 892         else:
 893                 return None
 894
 895 def getDefaultBuildRoot ():
 896         return os.path.join(tempfile.gettempdir(), "deqp-android-build")
 897
 898 def parseArgs ():
 899         nativeBuildTypes        = ['Release', 'Debug', 'MinSizeRel', 'RelWithAsserts', 'RelWithDebInfo']
 900         defaultNDKPath          = findNDK()
 901         defaultSDKPath          = findSDK()
 902         defaultBuildRoot        = getDefaultBuildRoot()
 903
 904         parser = argparse.ArgumentParser(os.path.basename(__file__),
 905                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 906         parser.add_argument('--native-build-type',
 907                 dest='nativeBuildType',
 908                 default="RelWithAsserts",
 909                 choices=nativeBuildTypes,
 910                 help="Native code build type")
 911         parser.add_argument('--build-root',
 912                 dest='buildRoot',
 913                 default=defaultBuildRoot,
 914                 help="Root build directory")
 915         parser.add_argument('--abis',
 916                 dest='abis',
 917                 default=",".join(NDKEnv.getKnownAbis()),
 918                 help="ABIs to build")
 919         parser.add_argument('--native-api',
 920                 type=int,
 921                 dest='nativeApi',
 922                 default=28,
 923                 help="Android API level to target in native code")
 924         parser.add_argument('--min-api',
 925                 type=int,
 926                 dest='minApi',
 927                 default=22,
 928                 help="Minimum Android API level for which the APK can be installed")
 929         parser.add_argument('--sdk',
 930                 dest='sdkPath',
 931                 default=defaultSDKPath,
 932                 help="Android SDK path",
 933                 required=(True if defaultSDKPath == None else False))
 934         parser.add_argument('--ndk',
 935                 dest='ndkPath',
 936                 default=defaultNDKPath,
 937                 help="Android NDK path",
 938                 required=(True if defaultNDKPath == None else False))
 939         parser.add_argument('-v', '--verbose',
 940                 dest='verbose',
 941                 help="Verbose output",
 942                 default=False,
 943                 action='store_true')
 944         parser.add_argument('--target',
 945                 dest='target',
 946                 help='Build target',
 947                 choices=['deqp', 'openglcts'],
 948                 default='deqp')
 949         parser.add_argument('--kc-cts-target',
 950                 dest='gtfTarget',
 951                 default='gles32',
 952                 choices=['gles32', 'gles31', 'gles3', 'gles2', 'gl'],
 953                 help="KC-CTS (GTF) target API (only used in openglcts target)")
 954         parser.add_argument('--layers-path',
 955                 dest='layers',
 956                 default=None,
 957                 required=False)
 958         parser.add_argument('--angle-path',
 959                 dest='angle',
 960                 default=None,
 961                 required=False)
 962
 963         args = parser.parse_args()
 964
 965         def parseAbis (abisStr):
 966                 knownAbis       = set(NDKEnv.getKnownAbis())
 967                 abis            = []
 968
 969                 for abi in abisStr.split(','):
 970                         abi = abi.strip()
 971                         if not abi in knownAbis:
 972                                 raise Exception("Unknown ABI: %s" % abi)
 973                         abis.append(abi)
 974
 975                 return abis
 976
 977         # Custom parsing & checks
 978         try:
 979                 args.abis = parseAbis(args.abis)
 980                 if len(args.abis) == 0:
 981                         raise Exception("--abis can't be empty")
 982         except Exception as e:
 983                 print("ERROR: %s" % str(e))
 984                 parser.print_help()
 985                 sys.exit(-1)
 986
 987         return args
 988
 989 if __name__ == "__main__":
 990         args            = parseArgs()
 991
 992         ndk                     = NDKEnv(os.path.realpath(args.ndkPath))
 993         sdk                     = SDKEnv(os.path.realpath(args.sdkPath))
 994         buildPath       = os.path.realpath(args.buildRoot)
 995         env                     = Environment(sdk, ndk)
 996         config          = Configuration(env, buildPath, abis=args.abis, nativeApi=args.nativeApi, minApi=args.minApi, nativeBuildType=args.nativeBuildType, gtfTarget=args.gtfTarget,
 997                                                  verbose=args.verbose, layers=args.layers, angle=args.angle)
 998
 999         try:
1000                 config.check()
1001         except Exception as e:
1002                 print("ERROR: %s" % str(e))
1003                 print("")
1004                 print("Please check your configuration:")
1005                 print("  --sdk=%s" % args.sdkPath)
1006                 print("  --ndk=%s" % args.ndkPath)
1007                 sys.exit(-1)
1008
1009         pkg, libs       = getPackageAndLibrariesForTarget(args.target)
1010         steps           = getBuildStepsForPackage(config.abis, pkg, libs)
1011
1012         executeSteps(config, steps)
1013
1014         print("")
1015         print("Built %s" % os.path.join(buildPath, getBuildRootRelativeAPKPath(pkg)))