From 7e01a414adc237914552ab970ab3c8c81a8ff799 Mon Sep 17 00:00:00 2001 From: Pyry Haulos Date: Mon, 20 Oct 2014 11:09:56 -0700 Subject: [PATCH] Improvements to standalone Android build scripts * If multiple connected devices are detected, install.py prompts to select one by default if no additional arguments are given. * Add -a option to install.py that installs package to all connected devices. * Use ninja on Linux / OS X, if installed, for faster builds (apt-get install ninja-build to get the awesomeness). * If make is used, pass in -j{CPUs} based on number of cores detected. * Prefer adb in path to avoid adb version mismatch in certain environments, for example when doing Android OS builds. * Clean up libs/ dir to avoid stale versions for archs that are not being built. * Do not unnecessarily force re-linking of libtestercore.so. * Optimize asset cleanup by performing that only for the build that is used for the assets. * Auto-detect ANDROID_NDK_HOST_OS. Bug: 18329517 Change-Id: I44e1b0acb5e6bcafeff7df30147002f216a76deb (cherry picked from commit 89e0405021b883114cca9e748166c17bd7ebcc98) --- android/scripts/build.py | 88 +++++++++++++--------- android/scripts/common.py | 184 ++++++++++++++++++++++++++++++--------------- android/scripts/install.py | 46 +++++++++--- 3 files changed, 209 insertions(+), 109 deletions(-) diff --git a/android/scripts/build.py b/android/scripts/build.py index 12419cc..37a3f52 100644 --- a/android/scripts/build.py +++ b/android/scripts/build.py @@ -8,6 +8,8 @@ import argparse import common +BASE_LIBS_DIR = os.path.join(common.ANDROID_DIR, "package", "libs") + def getStoreKeyPasswords (filename): f = open(filename) storepass = None @@ -28,25 +30,16 @@ def getNativeBuildDir (nativeLib, buildType): deqpDir = os.path.normpath(os.path.join(common.ANDROID_DIR, "..")) return os.path.normpath(os.path.join(deqpDir, "android", "build", "%s-%d-%s" % (buildType.lower(), nativeLib.apiVersion, nativeLib.abiVersion))) +def getAssetsDir (nativeLib, buildType): + return os.path.join(getNativeBuildDir(nativeLib, buildType), "assets") + def buildNative (nativeLib, buildType): deqpDir = os.path.normpath(os.path.join(common.ANDROID_DIR, "..")) buildDir = getNativeBuildDir(nativeLib, buildType) - assetsDir = os.path.join(buildDir, "assets") - libsDir = os.path.join(common.ANDROID_DIR, "package", "libs", nativeLib.abiVersion) + libsDir = os.path.join(BASE_LIBS_DIR, nativeLib.abiVersion) srcLibFile = os.path.join(buildDir, "libtestercore.so") dstLibFile = os.path.join(libsDir, "lib%s.so" % nativeLib.libName) - # Remove old lib files if such exist - if os.path.exists(srcLibFile): - os.unlink(srcLibFile) - - if os.path.exists(dstLibFile): - os.unlink(dstLibFile) - - # Remove assets directory so that we don't collect unnecessary cruft to the APK - if os.path.exists(assetsDir): - shutil.rmtree(assetsDir) - # Make build directory if necessary if not os.path.exists(buildDir): os.makedirs(buildDir) @@ -65,7 +58,7 @@ def buildNative (nativeLib, buildType): ]) os.chdir(buildDir) - common.execute(common.BUILD_CMD) + common.execArgs(['cmake', '--build', '.'] + common.EXTRA_BUILD_ARGS) if not os.path.exists(libsDir): os.makedirs(libsDir) @@ -85,48 +78,69 @@ def buildNative (nativeLib, buildType): # Make sure there is no gdbserver if build is not debug build os.unlink(os.path.join(libsDir, "gdbserver")) -def copyAssets (nativeLib, buildType): - srcDir = os.path.join(getNativeBuildDir(nativeLib, buildType), "assets") - dstDir = os.path.join(common.ANDROID_DIR, "package", "assets") - - if os.path.exists(dstDir): - shutil.rmtree(dstDir) - - if os.path.exists(srcDir): - shutil.copytree(srcDir, dstDir) - -def fileContains (filename, str): - f = open(filename, 'rb') - data = f.read() - f.close() - - return data.find(str) >= 0 - def buildApp (isRelease): appDir = os.path.join(common.ANDROID_DIR, "package") # Set up app os.chdir(appDir) - common.execute("%s update project --name dEQP --path . --target %s" % (common.shellquote(common.ANDROID_BIN), common.ANDROID_JAVA_API)) + common.execArgs([ + common.ANDROID_BIN, + 'update', 'project', + '--name', 'dEQP', + '--path', '.', + '--target', str(common.ANDROID_JAVA_API), + ]) # Build - common.execute("%s %s" % (common.shellquote(common.ANT_BIN), "release" if isRelease else "debug")) + common.execArgs([common.ANT_BIN, "release" if isRelease else "debug"]) def signApp (keystore, keyname, storepass, keypass): os.chdir(os.path.join(common.ANDROID_DIR, "package")) - common.execute("%s -keystore %s -storepass %s -keypass %s -sigfile CERT -digestalg SHA1 -sigalg MD5withRSA -signedjar bin/dEQP-unaligned.apk bin/dEQP-release-unsigned.apk %s" % (common.shellquote(common.JARSIGNER_BIN), common.shellquote(keystore), storepass, keypass, keyname)) - common.execute("%s -f 4 bin/dEQP-unaligned.apk bin/dEQP-release.apk" % (common.shellquote(common.ZIPALIGN_BIN))) + common.execArgs([ + common.JARSIGNER_BIN, + '-keystore', keystore, + '-storepass', storepass, + '-keypass', keypass, + '-sigfile', 'CERT', + '-digestalg', 'SHA1', + '-sigalg', 'MD5withRSA', + '-signedjar', 'bin/dEQP-unaligned.apk', + 'bin/dEQP-release-unsigned.apk', + keyname + ]) + common.execArgs([ + common.ZIPALIGN_BIN, + '-f', '4', + 'bin/dEQP-unaligned.apk', + 'bin/dEQP-release.apk' + ]) def build (isRelease=False, nativeBuildType="Release"): curDir = os.getcwd() try: + assetsSrcDir = getAssetsDir(common.NATIVE_LIBS[0], nativeBuildType) + assetsDstDir = os.path.join(common.ANDROID_DIR, "package", "assets") + + # Remove assets from the first build dir where we copy assets from + # to avoid collecting cruft there. + if os.path.exists(assetsSrcDir): + shutil.rmtree(assetsSrcDir) + if os.path.exists(assetsDstDir): + shutil.rmtree(assetsDstDir) + + # Remove old libs dir to avoid collecting out-of-date versions + # of libs for ABIs not built this time. + if os.path.exists(BASE_LIBS_DIR): + shutil.rmtree(BASE_LIBS_DIR) + # Build native code for lib in common.NATIVE_LIBS: buildNative(lib, nativeBuildType) - # Copy assets from first build dir - copyAssets(common.NATIVE_LIBS[0], nativeBuildType) + # Copy assets + if os.path.exists(assetsSrcDir): + shutil.copytree(assetsSrcDir, assetsDstDir) # Build java code and .apk buildApp(isRelease) diff --git a/android/scripts/common.py b/android/scripts/common.py index 9cd12b5..3d44c39 100644 --- a/android/scripts/common.py +++ b/android/scripts/common.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- import os +import re import sys import shlex import subprocess +import multiprocessing class NativeLib: def __init__ (self, libName, apiVersion, abiVersion): @@ -17,7 +19,7 @@ def getPlatform (): else: return sys.platform -def getCfg (variants): +def selectByOS (variants): platform = getPlatform() if platform in variants: return variants[platform] @@ -41,12 +43,19 @@ def which (binName): def isBinaryInPath (binName): return which(binName) != None -def selectBin (basePaths, relBinPath): - for basePath in basePaths: - fullPath = os.path.normpath(os.path.join(basePath, relBinPath)) - if isExecutable(fullPath): - return fullPath - return which(os.path.basename(relBinPath)) +def selectFirstExistingBinary (filenames): + for filename in filenames: + if filename != None and isExecutable(filename): + return filename + + return None + +def selectFirstExistingDir (paths): + for path in paths: + if path != None and os.path.isdir(path): + return path + + return None def die (msg): print msg @@ -66,78 +75,129 @@ def execArgs (args): if retcode != 0: raise Exception("Failed to execute '%s', got %d" % (str(args), retcode)) +class Device: + def __init__(self, serial, product, model, device): + self.serial = serial + self.product = product + self.model = model + self.device = device + + def __str__ (self): + return "%s: {product: %s, model: %s, device: %s}" % (self.serial, self.product, self.model, self.device) + +def getDevices (adb): + proc = subprocess.Popen([adb, 'devices', '-l'], stdout=subprocess.PIPE) + (stdout, stderr) = proc.communicate() + + if proc.returncode != 0: + raise Exception("adb devices -l failed, got %d" % retcode) + + ptrn = re.compile(r'^([a-zA-Z0-9]+)\s+.*product:([^\s]+)\s+model:([^\s]+)\s+device:([^\s]+)') + devices = [] + for line in stdout.splitlines()[1:]: + if len(line.strip()) == 0: + continue + + m = ptrn.match(line) + if m == None: + raise Exception("Failed to parse device info '%s'" % line) + + devices.append(Device(m.group(1), m.group(2), m.group(3), m.group(4))) + + return devices + +def getWin32Generator (): + if which("jom.exe") != None: + return "NMake Makefiles JOM" + else: + return "NMake Makefiles" + +def isNinjaSupported (): + return which("ninja") != None + +def getUnixGenerator (): + if isNinjaSupported(): + return "Ninja" + else: + return "Unix Makefiles" + +def getExtraBuildArgs (generator): + if generator == "Unix Makefiles": + return ["--", "-j%d" % multiprocessing.cpu_count()] + else: + return [] + +NDK_HOST_OS_NAMES = [ + "windows", + "windows_x86-64", + "darwin-x86", + "darwin-x86-64", + "linux-x86", + "linux-x86_64" +] + +def getNDKHostOsName (ndkPath): + for name in NDK_HOST_OS_NAMES: + if os.path.exists(os.path.join(ndkPath, "prebuilt", name)): + return name + + raise Exception("Couldn't determine NDK host OS") + # deqp/android path ANDROID_DIR = os.path.realpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) # Build configuration NATIVE_LIBS = [ # library name API ABI -# NativeLib("testercore", 13, "armeabi"), # ARM v5 ABI NativeLib("testercore", 13, "armeabi-v7a"), # ARM v7a ABI NativeLib("testercore", 13, "x86"), # x86 -# NativeLib("testercore", 21, "arm64-v8a"), # ARM64 v8a ABI + NativeLib("testercore", 21, "arm64-v8a"), # ARM64 v8a ABI ] ANDROID_JAVA_API = "android-13" # NDK paths -ANDROID_NDK_HOST_OS = getCfg({ - 'win32': "windows", - 'darwin': "darwin-x86", - 'linux': "linux-x86" - }) -ANDROID_NDK_PATH = getCfg({ - 'win32': "C:/android/android-ndk-r9d", - 'darwin': os.path.expanduser("~/android-ndk-r9d"), - 'linux': os.path.expanduser("~/android-ndk-r9d") - }) -ANDROID_NDK_TOOLCHAIN_VERSION = "clang-r9d" # Toolchain file is selected based on this - -def getWin32Generator (): - if which("jom.exe") != None: - return "NMake Makefiles JOM" - else: - return "NMake Makefiles" +ANDROID_NDK_PATH = selectFirstExistingDir([ + os.path.expanduser("~/android-ndk-r10c"), + "C:/android/android-ndk-r10c", + ]) +ANDROID_NDK_HOST_OS = getNDKHostOsName(ANDROID_NDK_PATH) +ANDROID_NDK_TOOLCHAIN_VERSION = "r10c" # Toolchain file is selected based on this # Native code build settings -CMAKE_GENERATOR = getCfg({ +CMAKE_GENERATOR = selectByOS({ 'win32': getWin32Generator(), - 'darwin': "Unix Makefiles", - 'linux': "Unix Makefiles" - }) -BUILD_CMD = getCfg({ - 'win32': "cmake --build .", - 'darwin': "cmake --build . -- -j 4", - 'linux': "cmake --build . -- -j 4" + 'other': getUnixGenerator() }) +EXTRA_BUILD_ARGS = getExtraBuildArgs(CMAKE_GENERATOR) # SDK paths -ANDROID_SDK_PATHS = [ - "C:/android/android-sdk-windows", - os.path.expanduser("~/android-sdk-mac_x86"), - os.path.expanduser("~/android-sdk-linux") - ] -ANDROID_BIN = getCfg({ - 'win32': selectBin(ANDROID_SDK_PATHS, "tools/android.bat"), - 'other': selectBin(ANDROID_SDK_PATHS, "tools/android"), - }) -ADB_BIN = getCfg({ - 'win32': selectBin(ANDROID_SDK_PATHS, "platform-tools/adb.exe"), - 'other': selectBin(ANDROID_SDK_PATHS, "platform-tools/adb"), - }) -ZIPALIGN_BIN = getCfg({ - 'win32': selectBin(ANDROID_SDK_PATHS, "tools/zipalign.exe"), - 'other': selectBin(ANDROID_SDK_PATHS, "tools/zipalign"), - }) -JARSIGNER_BIN = "jarsigner" +ANDROID_SDK_PATH = selectFirstExistingDir([ + os.path.expanduser("~/android-sdk-linux"), + os.path.expanduser("~/android-sdk-mac_x86"), + "C:/android/android-sdk-windows", + ]) +ANDROID_BIN = selectFirstExistingBinary([ + os.path.join(ANDROID_SDK_PATH, "tools", "android"), + os.path.join(ANDROID_SDK_PATH, "tools", "android.bat"), + which('android'), + ]) +ADB_BIN = selectFirstExistingBinary([ + which('adb'), # \note Prefer adb in path to avoid version issues on dev machines + os.path.join(ANDROID_SDK_PATH, "platform-tools", "adb"), + os.path.join(ANDROID_SDK_PATH, "platform-tools", "adb.exe"), + ]) +ZIPALIGN_BIN = selectFirstExistingBinary([ + os.path.join(ANDROID_SDK_PATH, "tools", "zipalign"), + os.path.join(ANDROID_SDK_PATH, "tools", "zipalign.exe"), + which('zipalign'), + ]) +JARSIGNER_BIN = which('jarsigner') # Apache ant -ANT_PATHS = [ - "C:/android/apache-ant-1.8.4", - "C:/android/apache-ant-1.9.2", - "C:/android/apache-ant-1.9.3", - "C:/android/apache-ant-1.9.4", - ] -ANT_BIN = getCfg({ - 'win32': selectBin(ANT_PATHS, "bin/ant.bat"), - 'other': selectBin(ANT_PATHS, "bin/ant") - }) +ANT_BIN = selectFirstExistingBinary([ + which('ant'), + "C:/android/apache-ant-1.8.4/bin/ant.bat", + "C:/android/apache-ant-1.9.2/bin/ant.bat", + "C:/android/apache-ant-1.9.3/bin/ant.bat", + "C:/android/apache-ant-1.9.4/bin/ant.bat", + ]) diff --git a/android/scripts/install.py b/android/scripts/install.py index 8d2205f..2ef02cd 100644 --- a/android/scripts/install.py +++ b/android/scripts/install.py @@ -7,29 +7,55 @@ import string import common -def install (extraArgs = ""): +def install (extraArgs = []): curDir = os.getcwd() try: os.chdir(common.ANDROID_DIR) - adbCmd = common.shellquote(common.ADB_BIN) - if len(extraArgs) > 0: - adbCmd += " %s" % extraArgs - print "Removing old dEQP Package..." - common.execute("%s uninstall com.drawelements.deqp" % adbCmd) + common.execArgs([common.ADB_BIN] + extraArgs + [ + 'uninstall', + 'com.drawelements.deqp' + ]) print "" print "Installing dEQP Package..." - common.execute("%s install -r package/bin/dEQP-debug.apk" % adbCmd) + common.execArgs([common.ADB_BIN] + extraArgs + [ + 'install', + '-r', + 'package/bin/dEQP-debug.apk' + ]) print "" finally: # Restore working dir os.chdir(curDir) - + +def installToDevice (device): + print "Installing to %s (%s)..." % (device.serial, device.model) + install(['-s', device.serial]) + +def installToAllDevices (): + devices = common.getDevices(common.ADB_BIN) + for device in devices: + installToDevice(device) + if __name__ == "__main__": if len(sys.argv) > 1: - install(string.join(sys.argv[1:], " ")) + if sys.argv[1] == '-a': + installToAllDevices() + else: + install(sys.argv[1:]) else: - install() + devices = common.getDevices(common.ADB_BIN) + if len(devices) == 0: + common.die('No devices connected') + elif len(devices) == 1: + installToDevice(devices[0]) + else: + print "More than one device connected:" + for i in range(0, len(devices)): + print "%3d: %16s %s" % ((i+1), devices[i].serial, devices[i].model) + + deviceNdx = int(raw_input("Choose device (1-%d): " % len(devices))) + installToDevice(devices[deviceNdx-1]) -- 2.7.4