Improvements to standalone Android build scripts
authorPyry Haulos <phaulos@google.com>
Mon, 20 Oct 2014 18:09:56 +0000 (11:09 -0700)
committerPyry Haulos <phaulos@google.com>
Tue, 11 Nov 2014 16:41:06 +0000 (16:41 +0000)
 * If multiple connected devices are detected, install.py prompts to
select one by default if no additional arguments are given.

 * Add -a option to install.py that installs package to all connected
devices.

 * Use ninja on Linux / OS X, if installed, for faster builds (apt-get
install ninja-build to get the awesomeness).

 * If make is used, pass in -j{CPUs} based on number of cores detected.

 * Prefer adb in path to avoid adb version mismatch in certain
environments, for example when doing Android OS builds.

 * Clean up libs/ dir to avoid stale versions for archs that are not
being built.

 * Do not unnecessarily force re-linking of libtestercore.so.

 * Optimize asset cleanup by performing that only for the build that is
used for the assets.

 * Auto-detect ANDROID_NDK_HOST_OS.

Bug: 18329517
Change-Id: I44e1b0acb5e6bcafeff7df30147002f216a76deb
(cherry picked from commit 89e0405021b883114cca9e748166c17bd7ebcc98)

android/scripts/build.py
android/scripts/common.py
android/scripts/install.py

index 12419cc..37a3f52 100644 (file)
@@ -8,6 +8,8 @@ import argparse
 
 import common
 
+BASE_LIBS_DIR = os.path.join(common.ANDROID_DIR, "package", "libs")
+
 def getStoreKeyPasswords (filename):
        f                       = open(filename)
        storepass       = None
@@ -28,25 +30,16 @@ def getNativeBuildDir (nativeLib, buildType):
        deqpDir = os.path.normpath(os.path.join(common.ANDROID_DIR, ".."))
        return os.path.normpath(os.path.join(deqpDir, "android", "build", "%s-%d-%s" % (buildType.lower(), nativeLib.apiVersion, nativeLib.abiVersion)))
 
+def getAssetsDir (nativeLib, buildType):
+       return os.path.join(getNativeBuildDir(nativeLib, buildType), "assets")
+
 def buildNative (nativeLib, buildType):
        deqpDir         = os.path.normpath(os.path.join(common.ANDROID_DIR, ".."))
        buildDir        = getNativeBuildDir(nativeLib, buildType)
-       assetsDir       = os.path.join(buildDir, "assets")
-       libsDir         = os.path.join(common.ANDROID_DIR, "package", "libs", nativeLib.abiVersion)
+       libsDir         = os.path.join(BASE_LIBS_DIR, nativeLib.abiVersion)
        srcLibFile      = os.path.join(buildDir, "libtestercore.so")
        dstLibFile      = os.path.join(libsDir, "lib%s.so" % nativeLib.libName)
 
-       # Remove old lib files if such exist
-       if os.path.exists(srcLibFile):
-               os.unlink(srcLibFile)
-
-       if os.path.exists(dstLibFile):
-               os.unlink(dstLibFile)
-
-       # Remove assets directory so that we don't collect unnecessary cruft to the APK
-       if os.path.exists(assetsDir):
-               shutil.rmtree(assetsDir)
-
        # Make build directory if necessary
        if not os.path.exists(buildDir):
                os.makedirs(buildDir)
@@ -65,7 +58,7 @@ def buildNative (nativeLib, buildType):
                        ])
 
        os.chdir(buildDir)
-       common.execute(common.BUILD_CMD)
+       common.execArgs(['cmake', '--build', '.'] + common.EXTRA_BUILD_ARGS)
 
        if not os.path.exists(libsDir):
                os.makedirs(libsDir)
@@ -85,48 +78,69 @@ def buildNative (nativeLib, buildType):
                # Make sure there is no gdbserver if build is not debug build
                os.unlink(os.path.join(libsDir, "gdbserver"))
 
-def copyAssets (nativeLib, buildType):
-       srcDir = os.path.join(getNativeBuildDir(nativeLib, buildType), "assets")
-       dstDir = os.path.join(common.ANDROID_DIR, "package", "assets")
-
-       if os.path.exists(dstDir):
-               shutil.rmtree(dstDir)
-
-       if os.path.exists(srcDir):
-               shutil.copytree(srcDir, dstDir)
-
-def fileContains (filename, str):
-       f = open(filename, 'rb')
-       data = f.read()
-       f.close()
-
-       return data.find(str) >= 0
-
 def buildApp (isRelease):
        appDir  = os.path.join(common.ANDROID_DIR, "package")
 
        # Set up app
        os.chdir(appDir)
-       common.execute("%s update project --name dEQP --path . --target %s" % (common.shellquote(common.ANDROID_BIN), common.ANDROID_JAVA_API))
+       common.execArgs([
+                       common.ANDROID_BIN,
+                       'update', 'project',
+                       '--name', 'dEQP',
+                       '--path', '.',
+                       '--target', str(common.ANDROID_JAVA_API),
+               ])
 
        # Build
-       common.execute("%s %s" % (common.shellquote(common.ANT_BIN), "release" if isRelease else "debug"))
+       common.execArgs([common.ANT_BIN, "release" if isRelease else "debug"])
 
 def signApp (keystore, keyname, storepass, keypass):
        os.chdir(os.path.join(common.ANDROID_DIR, "package"))
-       common.execute("%s -keystore %s -storepass %s -keypass %s -sigfile CERT -digestalg SHA1 -sigalg MD5withRSA -signedjar bin/dEQP-unaligned.apk bin/dEQP-release-unsigned.apk %s" % (common.shellquote(common.JARSIGNER_BIN), common.shellquote(keystore), storepass, keypass, keyname))
-       common.execute("%s -f 4 bin/dEQP-unaligned.apk bin/dEQP-release.apk" % (common.shellquote(common.ZIPALIGN_BIN)))
+       common.execArgs([
+                       common.JARSIGNER_BIN,
+                       '-keystore', keystore,
+                       '-storepass', storepass,
+                       '-keypass', keypass,
+                       '-sigfile', 'CERT',
+                       '-digestalg', 'SHA1',
+                       '-sigalg', 'MD5withRSA',
+                       '-signedjar', 'bin/dEQP-unaligned.apk',
+                       'bin/dEQP-release-unsigned.apk',
+                       keyname
+               ])
+       common.execArgs([
+                       common.ZIPALIGN_BIN,
+                       '-f', '4',
+                       'bin/dEQP-unaligned.apk',
+                       'bin/dEQP-release.apk'
+               ])
 
 def build (isRelease=False, nativeBuildType="Release"):
        curDir = os.getcwd()
 
        try:
+               assetsSrcDir    = getAssetsDir(common.NATIVE_LIBS[0], nativeBuildType)
+               assetsDstDir    = os.path.join(common.ANDROID_DIR, "package", "assets")
+
+               # Remove assets from the first build dir where we copy assets from
+               # to avoid collecting cruft there.
+               if os.path.exists(assetsSrcDir):
+                       shutil.rmtree(assetsSrcDir)
+               if os.path.exists(assetsDstDir):
+                       shutil.rmtree(assetsDstDir)
+
+               # Remove old libs dir to avoid collecting out-of-date versions
+               # of libs for ABIs not built this time.
+               if os.path.exists(BASE_LIBS_DIR):
+                       shutil.rmtree(BASE_LIBS_DIR)
+
                # Build native code
                for lib in common.NATIVE_LIBS:
                        buildNative(lib, nativeBuildType)
 
-               # Copy assets from first build dir
-               copyAssets(common.NATIVE_LIBS[0], nativeBuildType)
+               # Copy assets
+               if os.path.exists(assetsSrcDir):
+                       shutil.copytree(assetsSrcDir, assetsDstDir)
 
                # Build java code and .apk
                buildApp(isRelease)
index 9cd12b5..3d44c39 100644 (file)
@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 
 import os
+import re
 import sys
 import shlex
 import subprocess
+import multiprocessing
 
 class NativeLib:
        def __init__ (self, libName, apiVersion, abiVersion):
@@ -17,7 +19,7 @@ def getPlatform ():
        else:
                return sys.platform
 
-def getCfg (variants):
+def selectByOS (variants):
        platform = getPlatform()
        if platform in variants:
                return variants[platform]
@@ -41,12 +43,19 @@ def which (binName):
 def isBinaryInPath (binName):
        return which(binName) != None
 
-def selectBin (basePaths, relBinPath):
-       for basePath in basePaths:
-               fullPath = os.path.normpath(os.path.join(basePath, relBinPath))
-               if isExecutable(fullPath):
-                       return fullPath
-       return which(os.path.basename(relBinPath))
+def selectFirstExistingBinary (filenames):
+       for filename in filenames:
+               if filename != None and isExecutable(filename):
+                       return filename
+
+       return None
+
+def selectFirstExistingDir (paths):
+       for path in paths:
+               if path != None and os.path.isdir(path):
+                       return path
+
+       return None
 
 def die (msg):
        print msg
@@ -66,78 +75,129 @@ def execArgs (args):
        if retcode != 0:
                raise Exception("Failed to execute '%s', got %d" % (str(args), retcode))
 
+class Device:
+       def __init__(self, serial, product, model, device):
+               self.serial             = serial
+               self.product    = product
+               self.model              = model
+               self.device             = device
+
+       def __str__ (self):
+               return "%s: {product: %s, model: %s, device: %s}" % (self.serial, self.product, self.model, self.device)
+
+def getDevices (adb):
+       proc = subprocess.Popen([adb, 'devices', '-l'], stdout=subprocess.PIPE)
+       (stdout, stderr) = proc.communicate()
+
+       if proc.returncode != 0:
+               raise Exception("adb devices -l failed, got %d" % retcode)
+
+       ptrn = re.compile(r'^([a-zA-Z0-9]+)\s+.*product:([^\s]+)\s+model:([^\s]+)\s+device:([^\s]+)')
+       devices = []
+       for line in stdout.splitlines()[1:]:
+               if len(line.strip()) == 0:
+                       continue
+
+               m = ptrn.match(line)
+               if m == None:
+                       raise Exception("Failed to parse device info '%s'" % line)
+
+               devices.append(Device(m.group(1), m.group(2), m.group(3), m.group(4)))
+
+       return devices
+
+def getWin32Generator ():
+       if which("jom.exe") != None:
+               return "NMake Makefiles JOM"
+       else:
+               return "NMake Makefiles"
+
+def isNinjaSupported ():
+       return which("ninja") != None
+
+def getUnixGenerator ():
+       if isNinjaSupported():
+               return "Ninja"
+       else:
+               return "Unix Makefiles"
+
+def getExtraBuildArgs (generator):
+       if generator == "Unix Makefiles":
+               return ["--", "-j%d" % multiprocessing.cpu_count()]
+       else:
+               return []
+
+NDK_HOST_OS_NAMES = [
+       "windows",
+       "windows_x86-64",
+       "darwin-x86",
+       "darwin-x86-64",
+       "linux-x86",
+       "linux-x86_64"
+]
+
+def getNDKHostOsName (ndkPath):
+       for name in NDK_HOST_OS_NAMES:
+               if os.path.exists(os.path.join(ndkPath, "prebuilt", name)):
+                       return name
+
+       raise Exception("Couldn't determine NDK host OS")
+
 # deqp/android path
 ANDROID_DIR                            = os.path.realpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
 
 # Build configuration
 NATIVE_LIBS                            = [
                #                 library name          API             ABI
-#              NativeLib("testercore",         13,             "armeabi"),                     # ARM v5 ABI
                NativeLib("testercore",         13,             "armeabi-v7a"),         # ARM v7a ABI
                NativeLib("testercore",         13,             "x86"),                         # x86
-#              NativeLib("testercore",         21,             "arm64-v8a"),           # ARM64 v8a ABI
+               NativeLib("testercore",         21,             "arm64-v8a"),           # ARM64 v8a ABI
        ]
 ANDROID_JAVA_API               = "android-13"
 
 # NDK paths
-ANDROID_NDK_HOST_OS            = getCfg({
-               'win32':        "windows",
-               'darwin':       "darwin-x86",
-               'linux':        "linux-x86"
-       })
-ANDROID_NDK_PATH               = getCfg({
-               'win32':        "C:/android/android-ndk-r9d",
-               'darwin':       os.path.expanduser("~/android-ndk-r9d"),
-               'linux':        os.path.expanduser("~/android-ndk-r9d")
-       })
-ANDROID_NDK_TOOLCHAIN_VERSION = "clang-r9d" # Toolchain file is selected based on this
-
-def getWin32Generator ():
-       if which("jom.exe") != None:
-               return "NMake Makefiles JOM"
-       else:
-               return "NMake Makefiles"
+ANDROID_NDK_PATH               = selectFirstExistingDir([
+               os.path.expanduser("~/android-ndk-r10c"),
+               "C:/android/android-ndk-r10c",
+       ])
+ANDROID_NDK_HOST_OS                            = getNDKHostOsName(ANDROID_NDK_PATH)
+ANDROID_NDK_TOOLCHAIN_VERSION  = "r10c" # Toolchain file is selected based on this
 
 # Native code build settings
-CMAKE_GENERATOR                        = getCfg({
+CMAKE_GENERATOR                        = selectByOS({
                'win32':        getWin32Generator(),
-               'darwin':       "Unix Makefiles",
-               'linux':        "Unix Makefiles"
-       })
-BUILD_CMD                              = getCfg({
-               'win32':        "cmake --build .",
-               'darwin':       "cmake --build . -- -j 4",
-               'linux':        "cmake --build . -- -j 4"
+               'other':        getUnixGenerator()
        })
+EXTRA_BUILD_ARGS               = getExtraBuildArgs(CMAKE_GENERATOR)
 
 # SDK paths
-ANDROID_SDK_PATHS              = [
-       "C:/android/android-sdk-windows",
-       os.path.expanduser("~/android-sdk-mac_x86"),
-       os.path.expanduser("~/android-sdk-linux")
-       ]
-ANDROID_BIN                            = getCfg({
-               'win32':        selectBin(ANDROID_SDK_PATHS, "tools/android.bat"),
-               'other':        selectBin(ANDROID_SDK_PATHS, "tools/android"),
-       })
-ADB_BIN                                        = getCfg({
-               'win32':        selectBin(ANDROID_SDK_PATHS, "platform-tools/adb.exe"),
-               'other':        selectBin(ANDROID_SDK_PATHS, "platform-tools/adb"),
-       })
-ZIPALIGN_BIN                   = getCfg({
-               'win32':        selectBin(ANDROID_SDK_PATHS, "tools/zipalign.exe"),
-               'other':        selectBin(ANDROID_SDK_PATHS, "tools/zipalign"),
-       })
-JARSIGNER_BIN                  = "jarsigner"
+ANDROID_SDK_PATH               = selectFirstExistingDir([
+               os.path.expanduser("~/android-sdk-linux"),
+               os.path.expanduser("~/android-sdk-mac_x86"),
+               "C:/android/android-sdk-windows",
+       ])
+ANDROID_BIN                            = selectFirstExistingBinary([
+               os.path.join(ANDROID_SDK_PATH, "tools", "android"),
+               os.path.join(ANDROID_SDK_PATH, "tools", "android.bat"),
+               which('android'),
+       ])
+ADB_BIN                                        = selectFirstExistingBinary([
+               which('adb'), # \note Prefer adb in path to avoid version issues on dev machines
+               os.path.join(ANDROID_SDK_PATH, "platform-tools", "adb"),
+               os.path.join(ANDROID_SDK_PATH, "platform-tools", "adb.exe"),
+       ])
+ZIPALIGN_BIN                   = selectFirstExistingBinary([
+               os.path.join(ANDROID_SDK_PATH, "tools", "zipalign"),
+               os.path.join(ANDROID_SDK_PATH, "tools", "zipalign.exe"),
+               which('zipalign'),
+       ])
+JARSIGNER_BIN                  = which('jarsigner')
 
 # Apache ant
-ANT_PATHS                              = [
-       "C:/android/apache-ant-1.8.4",
-       "C:/android/apache-ant-1.9.2",
-       "C:/android/apache-ant-1.9.3",
-       "C:/android/apache-ant-1.9.4",
-       ]
-ANT_BIN                                        = getCfg({
-               'win32':        selectBin(ANT_PATHS, "bin/ant.bat"),
-               'other':        selectBin(ANT_PATHS, "bin/ant")
-       })
+ANT_BIN                                        = selectFirstExistingBinary([
+               which('ant'),
+               "C:/android/apache-ant-1.8.4/bin/ant.bat",
+               "C:/android/apache-ant-1.9.2/bin/ant.bat",
+               "C:/android/apache-ant-1.9.3/bin/ant.bat",
+               "C:/android/apache-ant-1.9.4/bin/ant.bat",
+       ])
index 8d2205f..2ef02cd 100644 (file)
@@ -7,29 +7,55 @@ import string
 
 import common
 
-def install (extraArgs = ""):
+def install (extraArgs = []):
        curDir = os.getcwd()
        try:
                os.chdir(common.ANDROID_DIR)
 
-               adbCmd = common.shellquote(common.ADB_BIN)
-               if len(extraArgs) > 0:
-                       adbCmd += " %s" % extraArgs
-
                print "Removing old dEQP Package..."
-               common.execute("%s uninstall com.drawelements.deqp" % adbCmd)
+               common.execArgs([common.ADB_BIN] + extraArgs + [
+                               'uninstall',
+                               'com.drawelements.deqp'
+                       ])
                print ""
 
                print "Installing dEQP Package..."
-               common.execute("%s install -r package/bin/dEQP-debug.apk" % adbCmd)
+               common.execArgs([common.ADB_BIN] + extraArgs + [
+                               'install',
+                               '-r',
+                               'package/bin/dEQP-debug.apk'
+                       ])
                print ""
 
        finally:
                # Restore working dir
                os.chdir(curDir)
-               
+
+def installToDevice (device):
+       print "Installing to %s (%s)..." % (device.serial, device.model)
+       install(['-s', device.serial])
+
+def installToAllDevices ():
+       devices = common.getDevices(common.ADB_BIN)
+       for device in devices:
+               installToDevice(device)
+
 if __name__ == "__main__":
        if len(sys.argv) > 1:
-               install(string.join(sys.argv[1:], " "))
+               if sys.argv[1] == '-a':
+                       installToAllDevices()
+               else:
+                       install(sys.argv[1:])
        else:
-               install()
+               devices = common.getDevices(common.ADB_BIN)
+               if len(devices) == 0:
+                       common.die('No devices connected')
+               elif len(devices) == 1:
+                       installToDevice(devices[0])
+               else:
+                       print "More than one device connected:"
+                       for i in range(0, len(devices)):
+                               print "%3d: %16s %s" % ((i+1), devices[i].serial, devices[i].model)
+
+                       deviceNdx = int(raw_input("Choose device (1-%d): " % len(devices)))
+                       installToDevice(devices[deviceNdx-1])