src/native_client/pnacl/build.sh

   1 #!/bin/bash
   2 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5 #
   6 #@                 PNaCl toolchain build script
   7 #@-------------------------------------------------------------------
   8 #@ This script builds the ARM and PNaCl untrusted toolchains.
   9 #@ It MUST be run from the native_client/ directory.
  10 ######################################################################
  11 # Directory Layout Description
  12 ######################################################################
  13 #
  14 # All directories are relative to BASE which is
  15 # On Linux: native_client/toolchain/linux_x86/pnacl_newlib/
  16 # On Mac: native_client/toolchain/mac_x86/pnacl_newlib/
  17 # On Windows: native_client/toolchain/win_x86/pnacl_newlib/
  18 #
  19 ######################################################################
  20
  21 set -o nounset
  22 set -o errexit
  23
  24 PWD_ON_ENTRY="$(pwd)"
  25 # The script is located in "native_client/pnacl/".
  26 # Set pwd to pnacl/
  27 cd "$(dirname "$0")"
  28 if [[ $(basename "$(pwd)") != "pnacl" ]] ; then
  29   echo "ERROR: cannot find pnacl/ directory"
  30   exit -1
  31 fi
  32
  33 source scripts/common-tools.sh
  34
  35 readonly PNACL_ROOT="$(pwd)"
  36 readonly NACL_ROOT="$(GetAbsolutePath ..)"
  37 readonly SCONS_OUT="${NACL_ROOT}/scons-out"
  38
  39 SetScriptPath "${PNACL_ROOT}/build.sh"
  40 SetLogDirectory "${PNACL_ROOT}/build/log"
  41
  42 readonly TOOLCHAIN_BUILD="${NACL_ROOT}/toolchain_build/toolchain_build_pnacl.py"
  43
  44 # For different levels of make parallelism change this in your env
  45 readonly PNACL_CONCURRENCY=${PNACL_CONCURRENCY:-8}
  46 # Concurrency for builds using the host's system compiler (which might be goma)
  47 readonly PNACL_CONCURRENCY_HOST=${PNACL_CONCURRENCY_HOST:-${PNACL_CONCURRENCY}}
  48 PNACL_PRUNE=${PNACL_PRUNE:-false}
  49 PNACL_BUILD_ARM=true
  50 PNACL_BUILD_MIPS=${PNACL_BUILD_MIPS:-false}
  51
  52 if ${BUILD_PLATFORM_MAC} || ${BUILD_PLATFORM_WIN}; then
  53   # We don't yet support building ARM tools for mac or windows.
  54   PNACL_BUILD_ARM=false
  55   PNACL_BUILD_MIPS=false
  56 fi
  57
  58 # PNaCl builds libc++/libc++abi as well as libstdc++, allowing users to
  59 # choose which to use through the -stdlib=XXX command-line argument.
  60 #
  61 # The following strings are used for banner names as well as file and
  62 # folder names. These names are created by the libraries themselves, and
  63 # expected by their dependents. Changing them would be ill-advised.
  64 readonly LIB_CXX_NAME="libc++"
  65 readonly LIB_STDCPP_NAME="libstdc++"
  66
  67 # TODO(pdox): Decide what the target should really permanently be
  68 readonly CROSS_TARGET_ARM=arm-none-linux-gnueabi
  69 readonly BINUTILS_TARGET=arm-pc-nacl
  70 readonly REAL_CROSS_TARGET=le32-nacl
  71 readonly NACL64_TARGET=x86_64-nacl
  72
  73 readonly DRIVER_DIR="${PNACL_ROOT}/driver"
  74 readonly ARM_ARCH=armv7-a
  75 readonly ARM_FPU=vfp
  76
  77 readonly TOOLCHAIN_ROOT="${NACL_ROOT}/toolchain"
  78 readonly TOOLCHAIN_BASE="${TOOLCHAIN_ROOT}/${SCONS_BUILD_PLATFORM}_x86"
  79
  80 readonly NNACL_NEWLIB_ROOT="${TOOLCHAIN_BASE}/nacl_x86_newlib"
  81 readonly NNACL_ARM_NEWLIB_ROOT="${TOOLCHAIN_BASE}/nacl_arm_newlib"
  82
  83 readonly PNACL_MAKE_OPTS="${PNACL_MAKE_OPTS:-}"
  84 readonly MAKE_OPTS="-j${PNACL_CONCURRENCY} VERBOSE=1 ${PNACL_MAKE_OPTS}"
  85 readonly MAKE_OPTS_HOST="-j${PNACL_CONCURRENCY_HOST} VERBOSE=1 ${PNACL_MAKE_OPTS}"
  86
  87 readonly NONEXISTENT_PATH="/going/down/the/longest/road/to/nowhere"
  88
  89 # For speculative build status output. ( see status function )
  90 # Leave this blank, it will be filled during processing.
  91 SPECULATIVE_REBUILD_SET=""
  92
  93 readonly PNACL_SUPPORT="${PNACL_ROOT}/support"
  94
  95 readonly THIRD_PARTY="${NACL_ROOT}"/../third_party
  96 readonly NACL_SRC_THIRD_PARTY="${NACL_ROOT}/src/third_party"
  97
  98 # Git sources
  99 readonly PNACL_GIT_ROOT="${NACL_ROOT}/toolchain_build/src"
 100 readonly TC_SRC_BINUTILS="${PNACL_GIT_ROOT}/binutils"
 101 readonly TC_SRC_LLVM="${PNACL_GIT_ROOT}/llvm"
 102 readonly TC_SRC_GCC="${PNACL_GIT_ROOT}/pnacl-gcc"
 103 readonly TC_SRC_NEWLIB="${PNACL_GIT_ROOT}/pnacl-newlib"
 104 readonly TC_SRC_LIBSTDCPP="${TC_SRC_GCC}/${LIB_STDCPP_NAME}-v3"
 105 readonly TC_SRC_COMPILER_RT="${PNACL_GIT_ROOT}/compiler-rt"
 106 readonly TC_SRC_CLANG="${PNACL_GIT_ROOT}/clang"
 107 readonly TC_SRC_LIBCXX="${PNACL_GIT_ROOT}/libcxx"
 108
 109 readonly SERVICE_RUNTIME_SRC="${NACL_ROOT}/src/trusted/service_runtime"
 110 readonly EXPORT_HEADER_SCRIPT="${SERVICE_RUNTIME_SRC}/export_header.py"
 111 readonly NACL_SYS_HEADERS="${SERVICE_RUNTIME_SRC}/include"
 112 readonly NEWLIB_INCLUDE_DIR="${TC_SRC_NEWLIB}/newlib/libc/sys/nacl"
 113
 114 # The location of each project. These should be absolute paths.
 115 readonly TC_BUILD="${PNACL_ROOT}/build"
 116 readonly TC_BUILD_LLVM="${TC_BUILD}/llvm_${HOST_ARCH}"
 117 readonly TC_BUILD_BINUTILS="${TC_BUILD}/binutils_${HOST_ARCH}"
 118 readonly TC_BUILD_BINUTILS_LIBERTY="${TC_BUILD}/binutils-liberty"
 119 TC_BUILD_NEWLIB="${TC_BUILD}/newlib"
 120 readonly TC_BUILD_COMPILER_RT="${TC_BUILD}/compiler_rt"
 121 readonly TC_BUILD_GCC="${TC_BUILD}/gcc"
 122 readonly NACL_HEADERS_TS="${TC_BUILD}/nacl.sys.timestamp"
 123
 124 readonly TIMESTAMP_FILENAME="make-timestamp"
 125
 126 # PNaCl toolchain installation directories (absolute paths)
 127 readonly INSTALL_ROOT="${TOOLCHAIN_BASE}/pnacl_newlib"
 128 readonly INSTALL_BIN="${INSTALL_ROOT}/bin"
 129
 130 # Bitcode lib directories (including static bitcode libs)
 131 INSTALL_LIB="${INSTALL_ROOT}/lib"
 132
 133 # Native nacl lib directories
 134 # The pattern `${INSTALL_LIB_NATIVE}${arch}' is used in many places.
 135 readonly INSTALL_LIB_NATIVE="${INSTALL_ROOT}/translator/"
 136 readonly INSTALL_LIB_ARM="${INSTALL_LIB_NATIVE}arm/lib"
 137 readonly INSTALL_LIB_X8632="${INSTALL_LIB_NATIVE}x86-32/lib"
 138 readonly INSTALL_LIB_X8664="${INSTALL_LIB_NATIVE}x86-64/lib"
 139 readonly INSTALL_LIB_MIPS32="${INSTALL_LIB_NATIVE}mips32/lib"
 140
 141 # PNaCl client-translators (sandboxed) binary locations
 142 readonly INSTALL_TRANSLATOR="${TOOLCHAIN_BASE}/pnacl_translator"
 143
 144
 145 # The INSTALL_HOST directory has host binaries and libs which
 146 # are part of the toolchain (e.g. llvm and binutils).
 147 # There are also tools-x86 and tools-arm which have host binaries which
 148 # are not part of the toolchain but might be useful in the SDK, e.g.
 149 # arm sel_ldr and x86-hosted arm/mips validators.
 150 readonly INSTALL_HOST="${INSTALL_ROOT}"
 151
 152 # Component installation directories
 153 readonly LLVM_INSTALL_DIR="${INSTALL_HOST}"
 154 readonly BINUTILS_INSTALL_DIR="${INSTALL_HOST}"
 155 readonly BFD_PLUGIN_DIR="${BINUTILS_INSTALL_DIR}/lib/bfd-plugins"
 156 readonly FAKE_INSTALL_DIR="${INSTALL_HOST}/fake"
 157 NEWLIB_INSTALL_DIR="${INSTALL_ROOT}/usr"
 158
 159 # Location of the PNaCl tools defined for configure invocations.
 160 readonly PNACL_CC="${INSTALL_BIN}/pnacl-clang"
 161 readonly PNACL_CXX="${INSTALL_BIN}/pnacl-clang++"
 162 readonly PNACL_LD="${INSTALL_BIN}/pnacl-ld"
 163 readonly PNACL_PP="${INSTALL_BIN}/pnacl-clang -E"
 164 readonly PNACL_AR="${INSTALL_BIN}/pnacl-ar"
 165 readonly PNACL_RANLIB="${INSTALL_BIN}/pnacl-ranlib"
 166 readonly PNACL_AS="${INSTALL_BIN}/pnacl-as"
 167 readonly PNACL_DIS="${INSTALL_BIN}/pnacl-dis"
 168 readonly PNACL_FINALIZE="${INSTALL_BIN}/pnacl-finalize"
 169 readonly PNACL_NM="${INSTALL_BIN}/pnacl-nm"
 170 readonly PNACL_TRANSLATE="${INSTALL_BIN}/pnacl-translate"
 171 readonly PNACL_READELF="${INSTALL_BIN}/pnacl-readelf"
 172 readonly PNACL_SIZE="${BINUTILS_INSTALL_DIR}/bin/${REAL_CROSS_TARGET}-size"
 173 readonly PNACL_STRIP="${INSTALL_BIN}/pnacl-strip"
 174 readonly ILLEGAL_TOOL="${INSTALL_BIN}"/pnacl-illegal
 175
 176 GetNNaClTool() {
 177   local arch=$1
 178   case ${arch} in
 179     x86-32) echo ${NNACL_NEWLIB_ROOT}/bin/i686-nacl-gcc ;;
 180     x86-64) echo ${NNACL_NEWLIB_ROOT}/bin/x86_64-nacl-gcc ;;
 181     arm) echo ${NNACL_ARM_NEWLIB_ROOT}/bin/arm-nacl-gcc ;;
 182     mips32)
 183       # No NNaCl for mips
 184       echo -n "${PNACL_CC} -arch mips32 --pnacl-bias=mips32 "
 185       echo "--pnacl-allow-translate --pnacl-allow-native" ;;
 186     *) Fatal "Unexpected argument to GetNNaClTool: $*" ;;
 187   esac
 188 }
 189
 190 # For a production (release) build, we want the sandboxed
 191 # translator to only contain the code needed to handle
 192 # its own architecture. For example, the translator shipped with
 193 # an X86-32 browser would only be able to translate to X86-32 code.
 194 # This is so that the translator binary is as small as possible.
 195 #
 196 # If SBTC_PRODUCTION is true, then the translators are built
 197 # separately, one for each architecture, so that each translator
 198 # can only target its own architecture.
 199 #
 200 # If SBTC_PRODUCTION is false, then we instead use PNaCl to
 201 # build a `fat` translator which can target all supported
 202 # architectures. This translator is built as a .pexe
 203 # which can then be translated to each individual architecture.
 204 SBTC_PRODUCTION=${SBTC_PRODUCTION:-true}
 205
 206 # Which arches to build for our sandboxed toolchain.
 207 SBTC_ARCHES_ALL=${SBTC_ARCHES_ALL:-"armv7 i686 x86_64"}
 208
 209 get-sbtc-llvm-arches() {
 210 # For LLVM i686 brings in both i686 and x86_64.  De-dupe that.
 211   echo ${SBTC_ARCHES_ALL} \
 212     | sed 's/x86_64/i686/' | sed 's/i686\(.*\)i686/i686\1/'
 213 }
 214 SBTC_ARCHES_LLVM=$(get-sbtc-llvm-arches)
 215
 216
 217 CC=${CC:-gcc}
 218 CXX=${CXX:-g++}
 219 AR=${AR:-ar}
 220 RANLIB=${RANLIB:-ranlib}
 221
 222 if ${HOST_ARCH_X8632}; then
 223   # These are simple compiler wrappers to force 32bit builds
 224   CC="${PNACL_ROOT}/scripts/mygcc32"
 225   CXX="${PNACL_ROOT}/scripts/myg++32"
 226 fi
 227
 228 # Set up some environment variables to build flavored bitcode libs
 229 setup-biased-bitcode-env() {
 230   local arch=$1
 231   # Do this to avoid .S files in newlib that clang doesn't like
 232   NEWLIB_TARGET="${REAL_CROSS_TARGET}"
 233   TC_BUILD_NEWLIB="${TC_BUILD}/newlib-${arch}"
 234   LIB_CPP_BUILD="${TC_BUILD}/c++-stdlib-newlib-${arch}"
 235   case ${arch} in
 236     portable)
 237       BIASED_BC_CFLAGS=""
 238       NEWLIB_INSTALL_DIR="${INSTALL_ROOT}/usr"
 239       INSTALL_LIB="${INSTALL_ROOT}/lib"
 240       LIB_CPP_INSTALL_DIR="${INSTALL_ROOT}/usr"
 241       ;;
 242     arm)
 243       BIASED_BC_CFLAGS="--target=armv7-unknown-nacl-gnueabihf -mfloat-abi=hard"
 244       NEWLIB_INSTALL_DIR="${INSTALL_ROOT}/usr-bc-${arch}"
 245       INSTALL_LIB="${INSTALL_ROOT}/lib-bc-${arch}"
 246       LIB_CPP_INSTALL_DIR="${INSTALL_ROOT}/usr-bc-${arch}"
 247       ;;
 248     x86-32)
 249       BIASED_BC_CFLAGS="--target=i686-unknown-nacl"
 250       NEWLIB_INSTALL_DIR="${INSTALL_ROOT}/usr-bc-${arch}"
 251       INSTALL_LIB="${INSTALL_ROOT}/lib-bc-${arch}"
 252       LIB_CPP_INSTALL_DIR="${INSTALL_ROOT}/usr-bc-${arch}"
 253       ;;
 254     x86-64)
 255       BIASED_BC_CFLAGS="--target=x86_64-unknown-nacl"
 256       NEWLIB_INSTALL_DIR="${INSTALL_ROOT}/usr-bc-${arch}"
 257       INSTALL_LIB="${INSTALL_ROOT}/lib-bc-${arch}"
 258       LIB_CPP_INSTALL_DIR="${INSTALL_ROOT}/usr-bc-${arch}"
 259       ;;
 260     mips32)
 261       # MIPS doesn't use biased bitcode.
 262       BIASED_BC_CFLAGS=""
 263       NEWLIB_INSTALL_DIR="${INSTALL_ROOT}/usr-bc-${arch}"
 264       INSTALL_LIB="${INSTALL_ROOT}/lib-bc-${arch}"
 265       LIB_CPP_INSTALL_DIR="${INSTALL_ROOT}/usr-bc-${arch}"
 266       ;;
 267     *)
 268       echo "Newlib architecture not implemented yet"
 269       exit 1
 270   esac
 271 }
 272
 273 setup-lib-cpp-env() {
 274   # NOTE: we do not expect the assembler or linker to be used for libs
 275   #       hence the use of ILLEGAL_TOOL.
 276
 277   STD_ENV_FOR_LIB_CPP=(
 278     CC_FOR_BUILD="${CC}"
 279     CC="${PNACL_CC}"
 280     CXX="${PNACL_CXX}"
 281     RAW_CXX_FOR_TARGET="${PNACL_CXX}"
 282     LD="${ILLEGAL_TOOL}"
 283     CFLAGS="-g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS}"
 284     CXXFLAGS="-g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS}"
 285     CFLAGS_FOR_TARGET="-g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS}"
 286     CXXFLAGS_FOR_TARGET="-g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS}"
 287     CC_FOR_TARGET="${PNACL_CC}"
 288     GCC_FOR_TARGET="${PNACL_CC}"
 289     CXX_FOR_TARGET="${PNACL_CXX}"
 290     AR="${PNACL_AR}"
 291     AR_FOR_TARGET="${PNACL_AR}"
 292     NM_FOR_TARGET="${PNACL_NM}"
 293     RANLIB="${PNACL_RANLIB}"
 294     RANLIB_FOR_TARGET="${PNACL_RANLIB}"
 295     AS_FOR_TARGET="${ILLEGAL_TOOL}"
 296     LD_FOR_TARGET="${ILLEGAL_TOOL}"
 297     OBJDUMP_FOR_TARGET="${ILLEGAL_TOOL}" )
 298 }
 299
 300 setup-newlib-env() {
 301   local arch=$1
 302   setup-biased-bitcode-env ${arch}
 303
 304   STD_ENV_FOR_NEWLIB=(
 305     # TODO(robertm): get rid of '-allow-asm' here once we have a way of
 306     # distinguishing "good" from "bad" asms.
 307     CFLAGS_FOR_TARGET="-allow-asm -g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS}"
 308     CXXFLAGS_FOR_TARGET="-allow-asm -g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS}"
 309     CC_FOR_TARGET="${PNACL_CC}"
 310     GCC_FOR_TARGET="${PNACL_CC}"
 311     CXX_FOR_TARGET="${PNACL_CXX}"
 312     AR_FOR_TARGET="${PNACL_AR}"
 313     NM_FOR_TARGET="${PNACL_NM}"
 314     RANLIB_FOR_TARGET="${PNACL_RANLIB}"
 315     READELF_FOR_TARGET="${PNACL_READELF}"
 316     OBJDUMP_FOR_TARGET="${ILLEGAL_TOOL}"
 317     AS_FOR_TARGET="${ILLEGAL_TOOL}"
 318     LD_FOR_TARGET="${ILLEGAL_TOOL}"
 319     STRIP_FOR_TARGET="${ILLEGAL_TOOL}" )
 320 }
 321
 322 # The gold plugin that we use is documented at
 323 # http://llvm.org/docs/GoldPlugin.html
 324 # Despite its name it is actually used by both gold and bfd. The changes to
 325 # this file to enable its use are:
 326 # * Build shared
 327 # * --enable-gold and --enable-plugin when building binutils
 328 # * --with-binutils-include when building binutils
 329 # * linking the plugin in bfd-plugins
 330
 331 ######################################################################
 332 ######################################################################
 333 #
 334 #                     < USER ACCESSIBLE FUNCTIONS >
 335 #
 336 ######################################################################
 337 ######################################################################
 338
 339 # Convert a path given on the command-line to an absolute path.
 340 # This takes into account the fact that we changed directories at the
 341 # beginning of this script. PWD_ON_ENTRY is used to remember the
 342 # initial working directory.
 343 ArgumentToAbsolutePath() {
 344   local relpath="$1"
 345   local savepwd="$(pwd)"
 346   cd "${PWD_ON_ENTRY}"
 347   GetAbsolutePath "${relpath}"
 348   cd "${savepwd}"
 349 }
 350
 351 #@ sync-sources          - check out repos needed to build toolchain
 352 sync-sources() {
 353   StepBanner "SYNC SOURCES"
 354   python ${TOOLCHAIN_BUILD} --legacy-repo-sync
 355   newlib-nacl-headers
 356 }
 357
 358 git-sync() {
 359   local gitbase="${PNACL_GIT_ROOT}"
 360   # Disable depot tools auto-update for these invocations. (It
 361   # will still get updated when it is invoked at the start of each build,
 362   # and we don't want it updating from cygwin here)
 363   export DEPOT_TOOLS_UPDATE=0
 364
 365   mkdir -p "${gitbase}"
 366   cp "${PNACL_ROOT}"/gclient_template "${gitbase}/.gclient"
 367
 368   if ! [ -d "${gitbase}/dummydir" ]; then
 369     spushd "${gitbase}"
 370     ${GCLIENT} update --verbose -j1
 371     spopd
 372   fi
 373
 374   newlib-nacl-headers-clean
 375   cp "${PNACL_ROOT}"/DEPS "${gitbase}"/dummydir
 376   spushd "${gitbase}"
 377   ${GCLIENT} update --verbose -j1
 378   spopd
 379
 380   # Copy nacl headers into newlib tree.
 381   newlib-nacl-headers
 382 }
 383
 384 gerrit-upload() {
 385   # Upload to gerrit code review
 386   # (Note: normally we use Rietveld code review. This is only useful if
 387   # e.g. we want external users to use gerrit instead of rietveld so we can
 388   # easily merge their commits without giving them direct access). But
 389   # this at least documents the specifics of uploading to our repo
 390   # TODO(dschuff): Find a good way to persist the user's login. maybe
 391   # do something like what depot tools does
 392   if ! [ -f ~/.gerrit_username ]; then
 393     echo "Enter your gerrit username: "
 394     read user
 395     echo "Saving gerrit username (${user}) to ~/.gerrit_username"
 396     echo ${user} > ~/.gerrit_username
 397   else
 398     local user=$(cat ~/.gerrit_username)
 399   fi
 400   cd "${TC_SRC_LLVM}"
 401   git push ssh://${user}@gerrit.chromium.org:29418/native_client/pnacl-llvm \
 402     HEAD:refs/for/master
 403 }
 404
 405 #@-------------------------------------------------------------------------
 406
 407 #@ download-trusted      - Download and Install trusted SDKs (arm,x86-32,x86-64)
 408 #@                         (your untrusted build will not be modified)
 409 download-trusted() {
 410   StepBanner "DOWNLOAD-TRUSTED" "Downloading trusted toolchains"
 411   local installdir="${INSTALL_ROOT}"
 412   local tmpdir="${installdir}-backup"
 413   local dldir="${installdir}-downloaded"
 414
 415   rm -rf "${dldir}"
 416
 417   if [ -d "${tmpdir}" ]; then
 418     echo "ERROR: It is not expected that directory '${tmpdir}' exists."
 419     echo "       Please delete it if you don't need it"
 420     exit -1
 421   fi
 422
 423   if [ -d "${installdir}" ]; then
 424     mv "${installdir}" "${tmpdir}"
 425   fi
 426
 427   download-toolchains
 428
 429   if [ -d "${installdir}" ]; then
 430     mv "${installdir}" "${dldir}"
 431   fi
 432
 433   if [ -d "${tmpdir}" ]; then
 434     mv "${tmpdir}" "${installdir}"
 435   fi
 436 }
 437
 438 #@-------------------------------------------------------------------------
 439
 440 #@ download-toolchains   - Download and Install all SDKs (arm,x86-32,x86-64)
 441
 442 download-toolchains() {
 443   ${GCLIENT} runhooks --force --verbose
 444 }
 445
 446 #@ libs                  - install native libs and build bitcode libs
 447 libs() {
 448   libs-clean
 449   # MIPS32 doesn't use biased bitcode.
 450   for arch in portable arm x86-32 x86-64; do
 451     newlib ${arch}
 452   done
 453   libs-support
 454   for arch in arm x86-32 x86-64 mips32 arm-nonsfi x86-32-nonsfi; do
 455     dummy-irt-shim ${arch}
 456   done
 457   compiler-rt-all
 458   libgcc_eh-all
 459   # MIPS32 doesn't use biased bitcode.
 460   for arch in portable arm x86-32 x86-64; do
 461     lib-cpp ${LIB_CXX_NAME} ${arch}
 462     lib-cpp ${LIB_STDCPP_NAME} ${arch}
 463   done
 464 }
 465
 466 #@ everything            - Build and install untrusted SDK. no translator
 467 everything() {
 468   sync-sources
 469   build-all
 470 }
 471
 472 #@ build-all does everything AFTER getting the sources
 473 build-all() {
 474   mkdir -p "${INSTALL_ROOT}"
 475   # This is needed to build misc-tools and run ARM tests.
 476   # We check this early so that there are no surprises later, and we can
 477   # handle all user interaction early.
 478   check-for-trusted
 479
 480   clean-install
 481
 482   clean-logs
 483
 484   build-host
 485
 486   driver
 487
 488   libs
 489
 490   # NOTE: we delay the tool building till after the sdk is essentially
 491   #      complete, so that sdk sanity checks don't fail
 492   misc-tools
 493   cp "${PNACL_ROOT}/README" "${INSTALL_ROOT}"
 494   verify
 495
 496   if ${PNACL_PRUNE}; then
 497     prune
 498   fi
 499 }
 500
 501 #@ Build just the host binaries
 502 build-host() {
 503   binutils
 504   llvm
 505   if ${PNACL_PRUNE}; then
 506     prune-host
 507   fi
 508 }
 509
 510 #@ everything-translator   - Build and install untrusted SDK AND translator
 511 everything-translator() {
 512   everything
 513   translator-all
 514   driver-install-translator
 515 }
 516
 517 #+ translator-clean-all  - Clean all translator install/build directories
 518 translator-clean-all() {
 519   StepBanner "TRANSLATOR" "Clean all"
 520   rm -rf "${TC_BUILD}"/translator*
 521   rm -rf "${INSTALL_TRANSLATOR}"*
 522 }
 523
 524 #@ translator-all   -  Build and install all of the translators.
 525 translator-all() {
 526   StepBanner \
 527     "SANDBOXED TC [prod=${SBTC_PRODUCTION}] [arches=${SBTC_ARCHES_ALL}]"
 528
 529   # Build the SDK if it not already present.
 530   sdk
 531   # Also build private libs to allow building nexes without the IRT
 532   # segment gap.  Specifically, only the sandboxed translator nexes
 533   # are built without IRT support to gain address space and reduce
 534   # swap file usage. Also libsrpc and its dependencies are now considered
 535   # private libs because they are not in the real SDK
 536   sdk-private-libs
 537
 538   binutils-liberty
 539   if ${SBTC_PRODUCTION}; then
 540     # Build each architecture separately.
 541     local arch
 542     for arch in ${SBTC_ARCHES_LLVM} ; do
 543       llvm-sb ${arch}
 544     done
 545     for arch in ${SBTC_ARCHES_ALL} ; do
 546       binutils-gold-sb ${arch}
 547     done
 548   else
 549     # Using arch `universal` builds the sandboxed tools from a single
 550     # .pexe which support all targets.
 551     llvm-sb universal
 552     binutils-gold-sb universal
 553     if ${PNACL_PRUNE}; then
 554       # The universal pexes have already been translated.
 555       # They don't need to stick around.
 556       rm -rf "$(GetTranslatorInstallDir universal)"
 557     fi
 558   fi
 559
 560   # Copy native libs to translator install dir.
 561   mkdir -p ${INSTALL_TRANSLATOR}/translator
 562   cp -a ${INSTALL_LIB_NATIVE}* ${INSTALL_TRANSLATOR}/translator
 563
 564   driver-install-translator
 565
 566   if ${PNACL_PRUNE}; then
 567     sdk-clean newlib
 568   fi
 569 }
 570
 571
 572 #+ translator-prune    Remove leftover files like pexes from pnacl_translator
 573 #+                     build and from translator-archive-pexes.
 574 translator-prune() {
 575   find "${INSTALL_TRANSLATOR}" -name "*.pexe" -exec "rm" {} +
 576 }
 577
 578
 579 #+ translator-clean <arch> -
 580 #+     Clean one translator install/build
 581 translator-clean() {
 582   local arch=$1
 583   StepBanner "TRANSLATOR" "Clean ${arch}"
 584   rm -rf "$(GetTranslatorInstallDir ${arch})"
 585   rm -rf "$(GetTranslatorBuildDir ${arch})"
 586 }
 587
 588 #@ all                   - Alias for 'everything'
 589 all() {
 590   everything
 591 }
 592
 593 #@ status                - Show status of build directories
 594 status() {
 595   # TODO(robertm): this is currently broken
 596   StepBanner "BUILD STATUS"
 597
 598   status-helper "BINUTILS"             binutils
 599   status-helper "LLVM"                 llvm
 600
 601   status-helper "NEWLIB"               newlib
 602   status-helper "C++ Standard Library" lib-cpp
 603 }
 604
 605 status-helper() {
 606   local title="$1"
 607   local mod="$2"
 608
 609   if ${mod}-needs-configure; then
 610     StepBanner "$title" "NEEDS FULL REBUILD"
 611     speculative-add "${mod}"
 612   elif ${mod}-needs-make; then
 613     StepBanner "$title" "NEEDS MAKE (INCREMENTAL)"
 614     speculative-add "${mod}"
 615   else
 616     StepBanner "$title" "OK (UP TO DATE)"
 617   fi
 618 }
 619
 620 speculative-add() {
 621   local mod="$1"
 622   SPECULATIVE_REBUILD_SET="${SPECULATIVE_REBUILD_SET} ${mod}"
 623 }
 624
 625 speculative-check() {
 626   local mod="$1"
 627   local search=$(echo "${SPECULATIVE_REBUILD_SET}" | grep -F "$mod")
 628   [ ${#search} -gt 0 ]
 629   return $?
 630 }
 631
 632
 633
 634 #@ clean                 - Clean the build and install directories.
 635 clean() {
 636   StepBanner "CLEAN" "Cleaning build, log, and install directories."
 637
 638   clean-logs
 639   clean-build
 640   clean-install
 641   clean-scons
 642 }
 643
 644 #@ fast-clean            - Clean everything except LLVM.
 645 fast-clean() {
 646   local did_backup=false
 647   local backup_dir="${PNACL_ROOT}/fast-clean-llvm-backup"
 648
 649   if [ -d "${TC_BUILD_LLVM}" ]; then
 650     rm -rf "${backup_dir}"
 651     mv "${TC_BUILD_LLVM}" "${backup_dir}"
 652     did_backup=true
 653   fi
 654
 655   clean
 656
 657   if ${did_backup} ; then
 658     mkdir -p "${TC_BUILD}"
 659     mv "${backup_dir}" "${TC_BUILD_LLVM}"
 660   fi
 661 }
 662
 663 #+ clean-scons           - Clean scons-out directory
 664 clean-scons() {
 665   rm -rf "${SCONS_OUT}"
 666 }
 667
 668 #+ clean-build           - Clean all build directories
 669 clean-build() {
 670   rm -rf "${TC_BUILD}"
 671 }
 672
 673 #+ clean-install         - Clean install directories
 674 clean-install() {
 675   rm -rf "${INSTALL_ROOT}"
 676 }
 677
 678 #+ libs-clean            - Removes the library directories
 679 libs-clean() {
 680   StepBanner "LIBS-CLEAN" "Cleaning ${INSTALL_ROOT}/libs-*"
 681   rm -rf "${INSTALL_LIB}"/*
 682   rm -rf "${INSTALL_LIB_NATIVE}"*
 683 }
 684
 685
 686 #@-------------------------------------------------------------------------
 687
 688 prune-host() {
 689   echo "stripping binaries (binutils)"
 690   strip "${BINUTILS_INSTALL_DIR}"/bin/*
 691
 692   echo "stripping binaries (llvm)"
 693   if ! strip "${LLVM_INSTALL_DIR}"/bin/* ; then
 694     echo "NOTE: some failures during stripping are expected"
 695   fi
 696
 697   echo "removing unused clang shared lib"
 698   rm -rf "${LLVM_INSTALL_DIR}"/${SO_DIR}/*clang${SO_EXT}
 699
 700   echo "removing unused binutils binaries"
 701   rm -rf "${LLVM_INSTALL_DIR}"/bin/le32-nacl-elfedit
 702   rm -rf "${LLVM_INSTALL_DIR}"/bin/le32-nacl-gprof
 703   rm -rf "${LLVM_INSTALL_DIR}"/bin/le32-nacl-objcopy
 704
 705   echo "removing unused LLVM/Clang binaries"
 706   rm -rf "${LLVM_INSTALL_DIR}"/bin/bc-wrap
 707   rm -rf "${LLVM_INSTALL_DIR}"/bin/bugpoint
 708   rm -rf "${LLVM_INSTALL_DIR}"/bin/c-index-test
 709   rm -rf "${LLVM_INSTALL_DIR}"/bin/clang-*
 710   rm -rf "${LLVM_INSTALL_DIR}"/bin/llc
 711   rm -rf "${LLVM_INSTALL_DIR}"/bin/lli
 712   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-ar
 713   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-bcanalyzer
 714   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-config
 715   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-cov
 716   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-diff
 717   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-dwarfdump
 718   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-extract
 719   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-mcmarkup
 720   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-prof
 721   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-ranlib
 722   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-readobj
 723   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-rtdyld
 724   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-size
 725   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-stress
 726   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-symbolizer
 727   rm -rf "${LLVM_INSTALL_DIR}"/bin/macho-dump
 728   rm -rf "${LLVM_INSTALL_DIR}"/bin/pso-stub
 729   rm -rf "${LLVM_INSTALL_DIR}"/bin/*-tblgen
 730
 731   echo "removing llvm & clang headers"
 732   rm -rf "${LLVM_INSTALL_DIR}"/include
 733
 734   echo "removing docs/ and share/"
 735   rm -rf "${LLVM_INSTALL_DIR}"/docs
 736   rm -rf "${LLVM_INSTALL_DIR}"/share
 737
 738   echo "removing unused libs"
 739   rm -rf "${LLVM_INSTALL_DIR}"/lib/*.a
 740   rm -rf "${LLVM_INSTALL_DIR}"/lib/bfd-plugins
 741   rm -rf "${LLVM_INSTALL_DIR}"/lib/BugpointPasses.so
 742   rm -rf "${LLVM_INSTALL_DIR}"/lib/LLVMHello.so
 743 }
 744
 745 #+ prune                 - Prune toolchain
 746 prune() {
 747   StepBanner "PRUNE" "Pruning toolchain"
 748   local dir_size_before=$(get_dir_size_in_mb ${INSTALL_ROOT})
 749
 750   SubBanner "Size before: ${INSTALL_ROOT} ${dir_size_before}MB"
 751
 752   prune-host
 753
 754   echo "removing .pyc files"
 755   rm -f "${INSTALL_BIN}"/pydir/*.pyc
 756
 757   local dir_size_after=$(get_dir_size_in_mb "${INSTALL_ROOT}")
 758   SubBanner "Size after: ${INSTALL_ROOT} ${dir_size_after}MB"
 759 }
 760
 761 #+ tarball <filename>    - Produce tarball file
 762 tarball() {
 763   if [ ! -n "${1:-}" ]; then
 764     echo "Error: tarball needs a tarball name." >&2
 765     exit 1
 766   fi
 767   local tarball="$(ArgumentToAbsolutePath "$1")"
 768   StepBanner "TARBALL" "Creating tar ball ${tarball}"
 769   tar zcf "${tarball}" -C "${INSTALL_ROOT}" .
 770   ls -l ${tarball}
 771 }
 772
 773 translator-tarball() {
 774   if [ ! -n "${1:-}" ]; then
 775     echo "Error: tarball needs a tarball name." >&2
 776     exit 1
 777   fi
 778   local tarball="$(ArgumentToAbsolutePath "$1")"
 779   StepBanner "TARBALL" "Creating translator tar ball ${tarball}"
 780   tar zcf "${tarball}" -C "${INSTALL_TRANSLATOR}" .
 781 }
 782
 783 #########################################################################
 784 #                              < LLVM >
 785 #########################################################################
 786
 787
 788 #+-------------------------------------------------------------------------
 789 #+ llvm                  - Configure, build and install LLVM.
 790
 791 llvm() {
 792   StepBanner "LLVM (${HOST_ARCH} HOST)"
 793
 794   local srcdir="${TC_SRC_LLVM}"
 795
 796   assert-dir "${srcdir}" "You need to checkout LLVM."
 797
 798   if llvm-needs-configure; then
 799     llvm-clean
 800     llvm-configure
 801   else
 802     SkipBanner "LLVM" "configure"
 803   fi
 804
 805   if llvm-needs-make; then
 806     llvm-make
 807   else
 808     SkipBanner "LLVM" "make"
 809   fi
 810
 811   llvm-install
 812 }
 813
 814 #+ llvm-clean            - Clean LLVM completely
 815 llvm-clean() {
 816   StepBanner "LLVM" "Clean"
 817   local objdir="${TC_BUILD_LLVM}"
 818   rm -rf ${objdir}
 819   mkdir -p ${objdir}
 820 }
 821
 822 #+ llvm-link-clang       - Add tools/clang symlink into llvm directory
 823 llvm-link-clang() {
 824   rm -f "${TC_SRC_LLVM}"/tools/clang
 825   # Symbolic link named : ${TC_SRC_LLVM}/tools/clang
 826   # Needs to point to   : ${TC_SRC_CLANG}
 827   ln -sf "${TC_SRC_CLANG}" "${TC_SRC_LLVM}"/tools/clang
 828 }
 829
 830 #+ llvm-unlink-clang     - Remove tools/clang symlink from llvm directory
 831 llvm-unlink-clang() {
 832   if [ -d "${TC_SRC_LLVM}" ]; then
 833     rm -f "${TC_SRC_LLVM}"/tools/clang
 834   fi
 835 }
 836
 837 # Default case - Optimized configure
 838 LLVM_EXTRA_OPTIONS="--enable-optimized"
 839
 840 #+ llvm-configure        - Run LLVM configure
 841 llvm-configure() {
 842   StepBanner "LLVM" "Configure"
 843
 844   local srcdir="${TC_SRC_LLVM}"
 845   local objdir="${TC_BUILD_LLVM}"
 846
 847   mkdir -p "${objdir}"
 848   spushd "${objdir}"
 849
 850   # Some components like ARMDisassembler may time out when built with -O3.
 851   # If we ever start using MSVC, we may also need to tone down the opt level
 852   # (see the settings in the CMake file for those components).
 853   local llvm_extra_opts=${LLVM_EXTRA_OPTIONS}
 854   if ${BUILD_PLATFORM_MAC}; then
 855     llvm_extra_opts="${LLVM_EXTRA_OPTIONS} --with-optimize-option=-O2"
 856   fi
 857
 858   llvm-link-clang
 859   # The --with-binutils-include is to allow llvm to build the gold plugin
 860   # re: --enable-targets  "x86" brings in both i686 and x86_64.
 861   #
 862   # Disabling zlib features for now to reduce set of shared lib deps.
 863   # This may disable things like compressed debug info:
 864   # https://code.google.com/p/nativeclient/issues/detail?id=3592
 865   #
 866   # Disabling terminfo removes a dependency on libtinfo on Linux which
 867   # is awkward mostly because it would require a multilib x86-32 copy
 868   # of libtinfo installed.  This just disables output colouring.
 869   local binutils_include="${TC_SRC_BINUTILS}/include"
 870   RunWithLog "llvm.configure" \
 871       env -i PATH="${PATH}" \
 872              MAKE_OPTS=${MAKE_OPTS_HOST} \
 873              CC="${CC}" \
 874              CXX="${CXX}" \
 875              ${srcdir}/configure \
 876              --enable-shared \
 877              --disable-zlib \
 878              --disable-terminfo \
 879              --disable-jit \
 880              --with-binutils-include=${binutils_include} \
 881              --enable-targets=x86,arm,mips \
 882              --prefix="${LLVM_INSTALL_DIR}" \
 883              --program-prefix= \
 884              ${llvm_extra_opts}
 885
 886
 887   spopd
 888 }
 889
 890 #+ llvm-configure-ninja - Configure with cmake for ninja build
 891 # Not used by default. Call manually. Pass build type (Release or Debug)
 892 # as an argument; The default is Release.
 893 llvm-configure-ninja() {
 894   StepBanner "LLVM" "Configure (Cmake-ninja)"
 895
 896   local srcdir="${TC_SRC_LLVM}"
 897   local objdir="${TC_BUILD_LLVM}"
 898   local buildtype="${1:-Release}"
 899
 900   mkdir -p "${objdir}"
 901   spushd "${objdir}"
 902
 903   llvm-link-clang
 904   local binutils_include="${TC_SRC_BINUTILS}/include"
 905   # Disabling zlib features for now to reduce set of shared lib deps.
 906   # This may disable things like compressed debug info:
 907   # https://code.google.com/p/nativeclient/issues/detail?id=3592
 908   RunWithLog "llvm.configure.cmake" \
 909     env \
 910       cmake -G Ninja \
 911       -DCMAKE_BUILD_TYPE=${buildtype} \
 912       -DCMAKE_INSTALL_PREFIX="${LLVM_INSTALL_DIR}" \
 913       -DCMAKE_INSTALL_RPATH='$ORIGIN/../lib' \
 914       -DBUILD_SHARED_LIBS=ON \
 915       -DLLVM_TARGETS_TO_BUILD="X86;ARM;Mips" \
 916       -DLLVM_ENABLE_ASSERTIONS=ON \
 917       -ULLVM_ENABLE_ZLIB \
 918       -DLLVM_ENABLE_TERMINFO=OFF \
 919       -DLLVM_BUILD_TESTS=ON \
 920       -DLLVM_APPEND_VC_REV=ON \
 921       -DLLVM_BINUTILS_INCDIR="${binutils_include}" \
 922       ${srcdir}
 923   spopd
 924 }
 925
 926 #+ llvm-configure-dbg        - Run LLVM configure
 927 #  Not used by default. Call manually.
 928 llvm-configure-dbg() {
 929   StepBanner "LLVM" "Configure With Debugging"
 930   LLVM_EXTRA_OPTIONS="--disable-optimized \
 931         --enable-debug-runtime \
 932         --enable-assertions "
 933   llvm-configure
 934 }
 935
 936
 937 llvm-needs-configure() {
 938   [ ! -f "${TC_BUILD_LLVM}/config.status" \
 939     -a ! -f "${TC_BUILD_LLVM}/build.ninja" ]
 940   return $?
 941 }
 942
 943 llvm-needs-make() {
 944   local objdir="${TC_BUILD_LLVM}"
 945   ts-modified "${TC_SRC_LLVM}" "${objdir}" ||
 946     ts-modified "${TC_SRC_CLANG}" "${objdir}"
 947   return $?
 948 }
 949
 950 #+ llvm-make             - Run LLVM 'make'
 951 llvm-make() {
 952   StepBanner "LLVM" "Make"
 953
 954   local srcdir="${TC_SRC_LLVM}"
 955   local objdir="${TC_BUILD_LLVM}"
 956   llvm-link-clang
 957
 958   spushd "${objdir}"
 959
 960   ts-touch-open "${objdir}"
 961
 962   if [ -f "${TC_BUILD_LLVM}/build.ninja" ]; then
 963     if [ -f "${TC_BUILD_LLVM}/config.status" ]; then
 964       echo "ERROR: Found multiple build system files in ${TC_BUILD_LLVM}"
 965       exit 1
 966     fi
 967     echo "Using ninja"
 968     ninja
 969   else
 970     RunWithLog llvm.make \
 971       env -i PATH="${PATH}" \
 972       MAKE_OPTS="${MAKE_OPTS_HOST}" \
 973       NACL_SANDBOX=0 \
 974       CC="${CC}" \
 975       CXX="${CXX}" \
 976       make ${MAKE_OPTS_HOST} all
 977   fi
 978
 979   ts-touch-commit  "${objdir}"
 980
 981   spopd
 982 }
 983
 984 #+ llvm-install          - Install LLVM
 985 llvm-install() {
 986   StepBanner "LLVM" "Install"
 987
 988   spushd "${TC_BUILD_LLVM}"
 989   llvm-link-clang
 990   if [ -f "${TC_BUILD_LLVM}/build.ninja" ]; then
 991     echo "Using ninja"
 992     RunWithLog llvm.install ninja install
 993   else
 994     RunWithLog llvm.install \
 995       env -i PATH=/usr/bin/:/bin \
 996       MAKE_OPTS="${MAKE_OPTS}" \
 997       NACL_SANDBOX=0 \
 998       CC="${CC}" \
 999       CXX="${CXX}" \
1000       make ${MAKE_OPTS} install
1001   fi
1002   spopd
1003
1004   # This is really part of libgcc_eh, but gets blown away by the ninja build.
1005   install-unwind-header
1006 }
1007
1008 #########################################################################
1009 #########################################################################
1010 #     < LIBGCC_EH >
1011 #########################################################################
1012 #########################################################################
1013
1014 libgcc_eh-all() {
1015   StepBanner "LIBGCC_EH (from GCC 4.6)"
1016
1017   libgcc_eh arm
1018   libgcc_eh mips32
1019   libgcc_eh x86-32
1020   libgcc_eh x86-64
1021 }
1022
1023 libgcc_eh-setup() {
1024   local arch=$1
1025   local flags=$2
1026   # For x86 we use nacl-gcc to build libgcc_eh because of some issues with
1027   # LLVM's handling of the gcc intrinsics used in the library. See
1028   # https://code.google.com/p/nativeclient/issues/detail?id=1933
1029   # and http://llvm.org/bugs/show_bug.cgi?id=8541
1030   # For ARM, LLVM does work and we use it to avoid dealing with the fact that
1031   # arm-nacl-gcc uses different libgcc support functions than PNaCl.
1032   if [ ${arch} == "arm" ]; then
1033     LIBGCC_EH_ENV=(
1034       CC="${PNACL_CC} ${flags} -arch ${arch} --pnacl-bias=${arch} \
1035          --pnacl-allow-translate --pnacl-allow-native" \
1036       AR="${PNACL_AR}" \
1037       NM="${PNACL_NM}" \
1038       RANLIB="${PNACL_RANLIB}")
1039   else
1040     LIBGCC_EH_ENV=(
1041       CC="$(GetNNaClTool ${arch}) ${flags}" \
1042       AR="${PNACL_AR}" \
1043       NM="${PNACL_NM}" \
1044       RANLIB="${PNACL_RANLIB}")
1045   fi
1046 }
1047
1048 libgcc_eh() {
1049   local arch=$1
1050
1051   local objdir="${TC_BUILD}/libgcc_eh-${arch}-newlib"
1052   local subdir="${objdir}/fake-target/libgcc"
1053   local installdir="${INSTALL_LIB_NATIVE}${arch}"
1054   local label="(${arch} libgcc_eh.a)"
1055
1056   mkdir -p "${installdir}"
1057   rm -rf "${installdir}"/libgcc_eh*
1058   rm -rf "${objdir}"
1059
1060   # Setup fake gcc build directory.
1061   mkdir -p "${objdir}"/gcc
1062   cp -a "${PNACL_ROOT}"/scripts/libgcc-newlib.mvars \
1063         "${objdir}"/gcc/libgcc.mvars
1064   cp -a "${PNACL_ROOT}"/scripts/libgcc-tconfig.h "${objdir}"/gcc/tconfig.h
1065   touch "${objdir}"/gcc/tm.h
1066
1067   install-unwind-header
1068
1069   mkdir -p "${subdir}"
1070   spushd "${subdir}"
1071   local flags=""
1072   flags+=" -DENABLE_RUNTIME_CHECKING"
1073
1074   libgcc_eh-setup ${arch} "${flags}"
1075
1076   StepBanner "LIBGCC_EH" "Configure ${label}"
1077   RunWithLog libgcc.${arch}.configure \
1078     env -i \
1079       PATH="/usr/bin:/bin" \
1080       /bin/sh \
1081       "${TC_SRC_GCC}"/libgcc/configure \
1082         "${LIBGCC_EH_ENV[@]}" \
1083         --prefix="${FAKE_INSTALL_DIR}" \
1084         --enable-shared \
1085         --host=i686-nacl
1086         # --enable-shared is needed because the EH functions end
1087         # up in libgcc.a if shared libs are disabled.
1088         #
1089         # The "--host" field is needed to enable cross-compiling.
1090         # It introduces extra configuration files in libgcc/config, but these
1091         # appear to affect libgcc, not libgcc_eh.
1092         # TODO(pdox): Create a fake target to get rid of i686 here.
1093
1094   StepBanner "LIBGCC_EH" "Make ${label}"
1095   RunWithLog libgcc.${arch}.make make libgcc_eh.a
1096   spopd
1097
1098   StepBanner "LIBGCC_EH" "Install ${label}"
1099   cp ${subdir}/libgcc_eh.a "${installdir}"
1100 }
1101
1102 install-unwind-header() {
1103   # TODO(pdox):
1104   # We need to establish an unwind ABI, since this is part of the ABI
1105   # exposed to the bitcode by the translator. This header should not vary
1106   # across compilers or C libraries.
1107   local install="/usr/bin/install -c -m 644"
1108   ${install} ${TC_SRC_GCC}/gcc/unwind-generic.h \
1109              ${LLVM_INSTALL_DIR}/lib/clang/3.4/include/unwind.h
1110 }
1111
1112 #########################################################################
1113 #########################################################################
1114 #     < COMPILER-RT >
1115 #########################################################################
1116 #########################################################################
1117
1118 compiler-rt-all() {
1119   StepBanner "COMPILER-RT (LIBGCC)"
1120   compiler-rt arm
1121   compiler-rt mips32
1122   compiler-rt x86-32
1123   compiler-rt x86-64
1124   compiler-rt arm-nonsfi
1125   compiler-rt x86-32-nonsfi
1126 }
1127
1128
1129 #+ compiler-rt           - build/install llvm's replacement for libgcc.a
1130 compiler-rt() {
1131   local arch=$1
1132   local src="${TC_SRC_COMPILER_RT}/lib"
1133   local objdir="${TC_BUILD_COMPILER_RT}-${arch}"
1134   local installdir="${INSTALL_LIB_NATIVE}${arch}"
1135   StepBanner "compiler rt" "build (${arch})"
1136
1137   rm -rf "${objdir}"
1138   mkdir -p "${objdir}"
1139
1140   spushd "${objdir}"
1141   RunWithLog libgcc.${arch}.make \
1142       make -j ${PNACL_CONCURRENCY} -f ${src}/Makefile-pnacl libgcc.a \
1143         CC="${PNACL_CC}" \
1144         AR="${PNACL_AR}" \
1145         "SRC_DIR=${src}" \
1146         "CFLAGS=-arch ${arch} -DPNACL_${arch} --pnacl-allow-translate -O3"
1147
1148   StepBanner "compiler rt" "install (${arch})"
1149   mkdir -p "${installdir}"
1150   cp libgcc.a "${installdir}"
1151   spopd
1152 }
1153
1154 #########################################################################
1155 #########################################################################
1156 #                          < LIBSTDC++/LIBC++ >
1157 #########################################################################
1158 #########################################################################
1159
1160 check-lib-cpp() {
1161   local lib=$1
1162   if [ ${lib} != ${LIB_CXX_NAME} ] && [ ${lib} != ${LIB_STDCPP_NAME} ]; then
1163     echo "ERROR: Unsupported C++ Standard Library '${lib}'. Choose one of: ${LIB_CXX_NAME}, ${LIB_STDCPP_NAME}"
1164     exit -1
1165   fi
1166 }
1167
1168 LIB_CPP_IS_SETUP=false
1169 lib-cpp-setup() {
1170   if ${LIB_CPP_IS_SETUP} && [ $# -eq 0 ]; then
1171     return 0
1172   fi
1173   local arch=$1
1174   setup-biased-bitcode-env ${arch}
1175   LIB_CPP_IS_SETUP=true
1176 }
1177
1178 lib-cpp() {
1179   local lib=$1
1180   local arch=$2
1181   check-lib-cpp "${lib}"
1182   lib-cpp-setup "${arch}"
1183   StepBanner "C++ Standard Library" "(BITCODE $*)"
1184
1185   if lib-cpp-needs-configure "${lib}" "${arch}"; then
1186     lib-cpp-clean "${lib}"
1187     lib-cpp-configure "${lib}" "${arch}"
1188   else
1189     StepBanner "${lib}" "configure"
1190   fi
1191
1192   if lib-cpp-needs-make "${lib}" "${arch}"; then
1193     lib-cpp-make "${lib}" "${arch}"
1194   else
1195     SkipBanner "${lib}" "make"
1196   fi
1197
1198   lib-cpp-install "${lib}" "${arch}"
1199   LIB_CPP_IS_SETUP=false
1200 }
1201
1202 #+ lib-cpp-clean - clean libc++/libstdc++ in bitcode
1203 lib-cpp-clean() {
1204   local lib=$1
1205   StepBanner "${lib}" "Clean"
1206   rm -rf ${TC_BUILD}/${lib}*
1207 }
1208
1209 lib-cpp-needs-configure() {
1210   local lib=$1
1211   local arch=$2
1212   lib-cpp-setup "${arch}"
1213   local objdir="${LIB_CPP_BUILD}-${lib}"
1214   ts-newer-than "${TC_BUILD_LLVM}" "${objdir}" && return 0
1215   [ ! -f "${objdir}/config.status" ]
1216   return $?
1217 }
1218
1219 lib-cpp-configure() {
1220   local lib=$1
1221   local arch=$2
1222   lib-cpp-setup "${arch}"
1223   StepBanner "${lib}" "Configure"
1224   local objdir="${LIB_CPP_BUILD}-${lib}"
1225   local subdir="${objdir}/pnacl-target"
1226   local flags=""
1227   mkdir -p "${subdir}"
1228   spushd "${subdir}"
1229
1230   flags+="--with-newlib --disable-shared --disable-rpath"
1231
1232   setup-lib-cpp-env
1233
1234   if [ ${lib} == ${LIB_CXX_NAME} ]; then
1235     local srcdir="${TC_SRC_LIBCXX}"
1236     # HAS_THREAD_LOCAL is used by libc++abi's exception storage, the
1237     # fallback is pthread otherwise.
1238     local cflags="-g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS} \
1239       -DHAS_THREAD_LOCAL=1"
1240     # LLVM's lit is used to test libc++. run.py serves as a shell that
1241     # translates pexe->nexe and executes in sel_ldr. The libc++ test
1242     # suite needs to be told to use pnacl-clang++'s "system library",
1243     # which happens to be libc++ when the -stdlib parameter is used.
1244     # The target architecture is set through the PNACL_RUN_ARCH
1245     # environment variable (``-arch env``).
1246     local litargs="--verbose"
1247     litargs+=" --param shell_prefix='${NACL_ROOT}/run.py -arch env --retries=1'"
1248     litargs+=" --param exe_suffix='.pexe'"
1249     litargs+=" --param use_system_lib=true"
1250     litargs+=" --param link_flags='-std=gnu++11 --pnacl-exceptions=sjlj'"
1251     # TODO(jfb) CMAKE_???_COMPILER_WORKS can be removed once the PNaCl
1252     #           driver scripts stop confusing cmake for libc++. See:
1253     #           https://code.google.com/p/nativeclient/issues/detail?id=3661
1254     RunWithLog libcxx.configure \
1255       env \
1256       cmake -G "Unix Makefiles" \
1257       -DCMAKE_CXX_COMPILER_WORKS=1 \
1258       -DCMAKE_C_COMPILER_WORKS=1 \
1259       -DCMAKE_SYSTEM_NAME=nacl \
1260       -DCMAKE_INSTALL_PREFIX="${LIB_CPP_INSTALL_DIR}" \
1261       -DCMAKE_BUILD_TYPE=Release \
1262       -DCMAKE_C_COMPILER="${PNACL_CC}" \
1263       -DCMAKE_CXX_COMPILER="${PNACL_CXX}" \
1264       -DCMAKE_AR="${PNACL_AR}" \
1265       -DCMAKE_NM="${PNACL_NM}" \
1266       -DCMAKE_RANLIB="${PNACL_RANLIB}" \
1267       -DCMAKE_LD="${ILLEGAL_TOOL}" \
1268       -DCMAKE_AS="${ILLEGAL_TOOL}" \
1269       -DCMAKE_OBJDUMP="${ILLEGAL_TOOL}" \
1270       -DCMAKE_C_FLAGS="-std=gnu11 ${cflags}" \
1271       -DCMAKE_CXX_FLAGS="-std=gnu++11 ${cflags}" \
1272       -DLIT_EXECUTABLE="${TC_SRC_LLVM}/utils/lit/lit.py" \
1273       -DLLVM_LIT_ARGS="${litargs}" \
1274       -DLIBCXX_ENABLE_CXX0X=0 \
1275       -DLIBCXX_ENABLE_SHARED=0 \
1276       -DLIBCXX_CXX_ABI=libcxxabi \
1277       -DLIBCXX_LIBCXXABI_INCLUDE_PATHS="${TC_SRC_LIBCXX}/../libcxxabi/include" \
1278       ${flags} \
1279       ${srcdir}
1280   else
1281     local srcdir="${TC_SRC_LIBSTDCPP}"
1282     RunWithLog libstdcpp.configure \
1283       env -i PATH=/usr/bin/:/bin \
1284       "${STD_ENV_FOR_LIB_CPP[@]}" \
1285       "${srcdir}"/configure \
1286       --host="${CROSS_TARGET_ARM}" \
1287       --prefix="${LIB_CPP_INSTALL_DIR}" \
1288       --enable-cxx-flags="-D__SIZE_MAX__=4294967295" \
1289       --disable-multilib \
1290       --disable-linux-futex \
1291       --disable-libstdcxx-time \
1292       --disable-sjlj-exceptions \
1293       --disable-libstdcxx-pch \
1294       ${flags}
1295   fi
1296   spopd
1297 }
1298
1299 lib-cpp-needs-make() {
1300   local lib=$1
1301   local arch=$2
1302   lib-cpp-setup "${arch}"
1303   local srcdir="${TC_SRC_LIBSTDCPP}" && \
1304     [ ${lib} == ${LIB_CXX_NAME} ] && srcdir="${TC_SRC_LIBCXX}"
1305   local objdir="${LIB_CPP_BUILD}-${lib}"
1306
1307   ts-modified "${srcdir}" "${objdir}"
1308   return $?
1309 }
1310
1311 lib-cpp-make() {
1312   local lib=$1
1313   local arch=$2
1314   lib-cpp-setup "${arch}"
1315   StepBanner "${lib}" "Make"
1316   local objdir="${LIB_CPP_BUILD}-${lib}"
1317
1318   ts-touch-open "${objdir}"
1319
1320   spushd "${objdir}/pnacl-target"
1321   setup-lib-cpp-env
1322   RunWithLog "${lib}.make" \
1323     env -i PATH=/usr/bin/:/bin \
1324     make \
1325     "${STD_ENV_FOR_LIB_CPP[@]}" \
1326     ${MAKE_OPTS}
1327   spopd
1328
1329   ts-touch-commit "${objdir}"
1330 }
1331
1332 lib-cpp-install() {
1333   local lib=$1
1334   local arch=$2
1335   lib-cpp-setup "${arch}"
1336   StepBanner "${lib}" "Install"
1337   local objdir="${LIB_CPP_BUILD}-${lib}"
1338
1339   spushd "${objdir}/pnacl-target"
1340
1341   # Clean the existing installation
1342   local include_dir=""
1343   if [ ${lib} == ${LIB_CXX_NAME} ]; then
1344     include_dir="v[0-9]"
1345   else
1346     include_dir="[0-9].[0-9].[0-9]"
1347   fi
1348   rm -rf "${LIB_CPP_INSTALL_DIR}"/include/c++/"${include_dir}"
1349   rm -rf "${LIB_CPP_INSTALL_DIR}"/lib/"${lib}"*
1350
1351   # install headers (=install-data)
1352   # for good measure make sure we do not keep any old headers
1353   setup-lib-cpp-env
1354   if [ ${lib} == ${LIB_CXX_NAME} ]; then
1355     RunWithLog "${lib}.install" \
1356       make \
1357       "${STD_ENV_FOR_LIB_CPP[@]}" \
1358       ${MAKE_OPTS} install
1359   else
1360     RunWithLog "${lib}.install" \
1361       make \
1362       "${STD_ENV_FOR_LIB_CPP[@]}" \
1363       ${MAKE_OPTS} install-data
1364   fi
1365
1366   # Install bitcode library
1367   mkdir -p "${LIB_CPP_INSTALL_DIR}/lib"
1368   if [ ${lib} == ${LIB_CXX_NAME} ]; then
1369     cp "${objdir}/pnacl-target/lib/${LIB_CXX_NAME}.a" \
1370       "${LIB_CPP_INSTALL_DIR}/lib"
1371   else
1372     cp "${objdir}/pnacl-target/src/.libs/${LIB_STDCPP_NAME}.a" \
1373       "${LIB_CPP_INSTALL_DIR}/lib"
1374   fi
1375   spopd
1376
1377   # libstdc++ installs a file with an abnormal name: "libstdc++*-gdb.py"
1378   # The asterisk may be due to a bug in libstdc++ Makefile/configure.
1379   # This causes problems on the Windows bot (during cleanup, toolchain
1380   # directory delete fails due to the bad character).
1381   # Rename it to get rid of the asterisk.
1382   if [ ${lib} == ${LIB_STDCPP_NAME} ]; then
1383     spushd "${LIB_CPP_INSTALL_DIR}/lib"
1384     mv -f ${LIB_STDCPP_NAME}'*'-gdb.py ${LIB_STDCPP_NAME}-gdb.py
1385     spopd
1386   fi
1387 }
1388
1389 #########################################################################
1390 #########################################################################
1391 #                          < Tools >
1392 #########################################################################
1393 #########################################################################
1394
1395 build-validator() {
1396   local arch=$1
1397   StepBanner "MISC-TOOLS" "Building validator ("${arch}")"
1398   spushd "${NACL_ROOT}"
1399   RunWithLog ${arch}_ncval_core \
1400     ./scons MODE=opt-host \
1401     sysinfo=0 \
1402     ${arch}-ncval-core
1403   cp ${SCONS_OUT}/opt-linux-x86-32/obj/src/trusted/\
1404 validator_${arch}/${arch}-ncval-core ${INSTALL_ROOT}/tools-x86
1405   spopd
1406 }
1407
1408 #+ misc-tools            - Build and install sel_ldr and validator for ARM.
1409 misc-tools() {
1410   if ${PNACL_BUILD_ARM} ; then
1411     StepBanner "MISC-TOOLS" "Building sel_ldr (arm)"
1412
1413     # TODO(robertm): revisit some of these options
1414     spushd "${NACL_ROOT}"
1415     RunWithLog arm_sel_ldr \
1416       ./scons MODE=opt-host \
1417       platform=arm \
1418       naclsdk_validate=0 \
1419       sysinfo=0 \
1420       -j${PNACL_CONCURRENCY} \
1421       sel_ldr
1422     rm -rf  "${INSTALL_ROOT}/tools-arm"
1423     mkdir "${INSTALL_ROOT}/tools-arm"
1424     local sconsdir="${SCONS_OUT}/opt-${SCONS_BUILD_PLATFORM}-arm"
1425     cp "${sconsdir}/obj/src/trusted/service_runtime/sel_ldr" \
1426        "${INSTALL_ROOT}/tools-arm"
1427     spopd
1428   else
1429     StepBanner "MISC-TOOLS" "Skipping arm sel_ldr (No trusted arm toolchain)"
1430   fi
1431
1432   # TODO(petarj): It would be nice to build MIPS sel_ldr on builbots later.
1433   if [ "${PNACL_BUILD_MIPS}" == "true" ] ; then
1434     StepBanner "MISC-TOOLS" "Building sel_ldr (mips32)"
1435     spushd "${NACL_ROOT}"
1436     RunWithLog mips32_sel_ldr \
1437       ./scons MODE=opt-host \
1438       platform=mips32 \
1439       naclsdk_validate=0 \
1440       sysinfo=0 \
1441       -j${PNACL_CONCURRENCY} \
1442       sel_ldr
1443     rm -rf  "${INSTALL_ROOT}/tools-mips32"
1444     mkdir "${INSTALL_ROOT}/tools-mips32"
1445     local sconsdir="scons-out/opt-${SCONS_BUILD_PLATFORM}-mips32"
1446     cp "${sconsdir}/obj/src/trusted/service_runtime/sel_ldr" \
1447        "${INSTALL_ROOT}/tools-mips32"
1448     spopd
1449   else
1450     StepBanner "MISC-TOOLS" "Skipping mips sel_ldr (No trusted mips toolchain)"
1451   fi
1452
1453   if ${BUILD_PLATFORM_LINUX} ; then
1454     rm -rf  "${INSTALL_ROOT}/tools-x86"
1455     mkdir "${INSTALL_ROOT}/tools-x86"
1456     for target in arm mips; do
1457       build-validator $target
1458     done
1459   else
1460     for target in arm mips; do
1461       StepBanner "MISC-"${target} "Skipping " ${target} " validator (Not yet supported on Mac)"
1462     done
1463   fi
1464 }
1465
1466 #########################################################################
1467 #     < BINUTILS >
1468 #########################################################################
1469
1470 #+-------------------------------------------------------------------------
1471 #+ binutils          - Build and install binutils (cross-tools)
1472 binutils() {
1473   StepBanner "BINUTILS (${HOST_ARCH} HOST)"
1474
1475   local srcdir="${TC_SRC_BINUTILS}"
1476
1477   assert-dir "${srcdir}" "You need to checkout binutils."
1478
1479   if binutils-needs-configure; then
1480     binutils-clean
1481     binutils-configure
1482   else
1483     SkipBanner "BINUTILS" "configure"
1484   fi
1485
1486   if binutils-needs-make; then
1487     binutils-make
1488   else
1489     SkipBanner "BINUTILS" "make"
1490   fi
1491   binutils-install
1492 }
1493
1494 #+ binutils-clean    - Clean binutils
1495 binutils-clean() {
1496   StepBanner "BINUTILS" "Clean"
1497   local objdir="${TC_BUILD_BINUTILS}"
1498   rm -rf ${objdir}
1499 }
1500
1501 #+ binutils-configure- Configure binutils for ARM
1502 binutils-configure() {
1503   StepBanner "BINUTILS" "Configure"
1504
1505   local srcdir="${TC_SRC_BINUTILS}"
1506   local objdir="${TC_BUILD_BINUTILS}"
1507
1508   # enable multiple targets so that we can use the same ar with all .o files
1509   local targ="arm-pc-nacl,i686-pc-nacl,x86_64-pc-nacl,mipsel-pc-nacl"
1510   mkdir -p "${objdir}"
1511   spushd "${objdir}"
1512
1513   # The linux escaping is horrible but apparently the only way of doing this:
1514   # c.f.:  http://sourceware.org/ml/binutils/2009-05/msg00252.html
1515   # all we try to do here is to add "$ORIGIN/../lib to "rpath".
1516   # If you ever touch this please make sure that rpath is correct via:
1517   # objdump -p ${TOOLCHAIN_BASE}/pnacl_newlib/host/bin/le32-nacl-ld.gold
1518   # objdump -p ${TOOLCHAIN_BASE}/pnacl_newlib/host/bin/le32-nacl-objdump
1519   if ${BUILD_PLATFORM_LINUX} ; then
1520       local flags='-Xlinker -rpath -Xlinker '"'"'$\\$$\$$\\$$\$$ORIGIN/../lib'"'"
1521       local shared='yes'
1522       local zlib='--without-zlib'
1523   else
1524       # The shared build for binutils on mac is currently disabled.
1525       # A mac-expert needs to look at this but
1526       # It seems that on mac the linker is storing "full" library paths into
1527       # the dynamic image, e.g, for the llc dynamic image we see  paths like:
1528       # @executable_path/../lib/libLLVM-3.3svn.dylib
1529       # This only works if at linktime the libraries are already at
1530       # @executable_path/../lib which is not the case for mac
1531       #local flags="-Xlinker -rpath -Xlinker '@executable_path/../lib'"
1532       local flags=''
1533       local shared='no'
1534       local zlib=''
1535   fi
1536   # The --enable-gold and --enable-plugins options are on so that we
1537   # can use gold's support for plugin to link PNaCl modules, and use
1538   # gold as the final linker. We do not use bfd ld, and it is disabled
1539   # in part because we do not have its MIPS support downstream.
1540
1541   # We llvm's mc for assembly so we no longer build gas
1542   RunWithLog binutils.configure \
1543       env -i \
1544       PATH="${PATH}" \
1545       CC="${CC}" \
1546       CXX="${CXX}" \
1547       LDFLAGS="${flags}" \
1548       ${srcdir}/configure \
1549           --prefix="${BINUTILS_INSTALL_DIR}" \
1550           --target=${BINUTILS_TARGET} \
1551           --program-prefix=${REAL_CROSS_TARGET}- \
1552           --enable-targets=${targ} \
1553           --enable-shared=${shared} \
1554           --enable-gold=default \
1555           --enable-ld=no \
1556           --disable-nls \
1557           --enable-plugins \
1558           --disable-werror \
1559           --without-gas \
1560           ${zlib} \
1561           --with-sysroot="${NONEXISTENT_PATH}"
1562   # There's no point in setting the correct path as sysroot, because we
1563   # want the toolchain to be relocatable. The driver will use ld command-line
1564   # option --sysroot= to override this value and set it to the correct path.
1565   # However, we need to include --with-sysroot during configure to get this
1566   # option. So fill in a non-sense, non-existent path.
1567
1568   spopd
1569 }
1570
1571 binutils-needs-configure() {
1572   [ ! -f "${TC_BUILD_BINUTILS}/config.status" ]
1573   return $?
1574 }
1575
1576 binutils-needs-make() {
1577   local srcdir="${TC_SRC_BINUTILS}"
1578   local objdir="${TC_BUILD_BINUTILS}"
1579   local ret=1
1580   ts-modified "$srcdir" "$objdir" && ret=0
1581   return ${ret}
1582 }
1583
1584 #+ binutils-make     - Make binutils for ARM
1585 binutils-make() {
1586   StepBanner "BINUTILS" "Make"
1587   local objdir="${TC_BUILD_BINUTILS}"
1588   spushd "${objdir}"
1589
1590   ts-touch-open "${objdir}"
1591
1592   if ${BUILD_PLATFORM_LINUX}; then
1593     RunWithLog binutils.make \
1594       env -i PATH="/usr/bin:/bin" \
1595       ac_cv_search_zlibVersion=no \
1596       make ${MAKE_OPTS_HOST}
1597   else
1598     local control_parallel=""
1599     if ${BUILD_PLATFORM_MAC}; then
1600       # Try to avoid parallel builds for Mac to see if it avoids flake.
1601       # http://code.google.com/p/nativeclient/issues/detail?id=2926
1602       control_parallel="-j1"
1603     fi
1604     RunWithLog binutils.make \
1605       env -i PATH="${PATH}" \
1606       make ${MAKE_OPTS_HOST} ${control_parallel}
1607   fi
1608
1609   ts-touch-commit "${objdir}"
1610
1611   spopd
1612 }
1613
1614 #+ binutils-install  - Install binutils for ARM
1615 binutils-install() {
1616   StepBanner "BINUTILS" "Install"
1617   local objdir="${TC_BUILD_BINUTILS}"
1618   spushd "${objdir}"
1619
1620   RunWithLog binutils.install \
1621     env -i PATH="/usr/bin:/bin" \
1622     make \
1623       install ${MAKE_OPTS}
1624
1625   # TODO(robertm): remove this once we manage to avoid building
1626   #                ld in the first place
1627   echo "pruning hack: ${BINUTILS_INSTALL_DIR}"
1628   rm -f "${BINUTILS_INSTALL_DIR}/bin/${REAL_CROSS_TARGET}-ld"
1629   rm -f "${BINUTILS_INSTALL_DIR}/bin/${REAL_CROSS_TARGET}-ld.bfd"
1630
1631   # Also remove "${BINUTILS_INSTALL_DIR}/${BINUTILS_TARGET}" which contains
1632   # duplicate binaries and unused linker scripts
1633   echo "remove unused ${BINUTILS_INSTALL_DIR}/${BINUTILS_TARGET}/"
1634   rm -rf "${BINUTILS_INSTALL_DIR}/${BINUTILS_TARGET}/"
1635
1636   # Move binutils shared libs to host/lib.
1637   # The first "*" expands to the host string, e.g.
1638   # x86_64-unknown-linux-gnu
1639   if ${BUILD_PLATFORM_LINUX} ; then
1640     echo "move shared libs to ${BINUTILS_INSTALL_DIR}/${SO_DIR}"
1641     mkdir -p ${BINUTILS_INSTALL_DIR}/${SO_DIR}
1642     for lib in ${BINUTILS_INSTALL_DIR}/*/${BINUTILS_TARGET}/lib/lib*${SO_EXT}
1643     do
1644       echo "moving ${lib}"
1645       mv ${lib} ${BINUTILS_INSTALL_DIR}/${SO_DIR}
1646     done
1647   fi
1648
1649   # NOTE: this needs to come AFTER the moving of the shared libs
1650   # TODO(robertm): this needs to be augmented for mac and windows
1651   echo "remove unused (remaining) binutils static libs and headers"
1652   rm -rf "${BINUTILS_INSTALL_DIR}/i686-pc-linux-gnu" \
1653          "${BINUTILS_INSTALL_DIR}/x86_64-unknown-linux-gnu"
1654
1655   spopd
1656 }
1657
1658 #+-------------------------------------------------------------------------
1659 #+ binutils-liberty      - Build native binutils libiberty
1660 binutils-liberty() {
1661   local srcdir="${TC_SRC_BINUTILS}"
1662
1663   assert-dir "${srcdir}" "You need to checkout binutils."
1664
1665   if binutils-liberty-needs-configure; then
1666     binutils-liberty-clean
1667     binutils-liberty-configure
1668   else
1669     SkipBanner "BINUTILS-LIBERTY" "configure"
1670   fi
1671
1672   if binutils-liberty-needs-make; then
1673     binutils-liberty-make
1674   else
1675     SkipBanner "BINUTILS-LIBERTY" "make"
1676   fi
1677 }
1678
1679 binutils-liberty-needs-configure() {
1680   [ ! -f "${TC_BUILD_BINUTILS_LIBERTY}/config.status" ]
1681   return $?
1682 }
1683
1684 #+ binutils-liberty-clean    - Clean binutils-liberty
1685 binutils-liberty-clean() {
1686   StepBanner "BINUTILS-LIBERTY" "Clean"
1687   local objdir="${TC_BUILD_BINUTILS_LIBERTY}"
1688   rm -rf ${objdir}
1689 }
1690
1691 #+ binutils-liberty-configure - Configure binutils-liberty
1692 binutils-liberty-configure() {
1693   StepBanner "BINUTILS-LIBERTY" "Configure"
1694
1695   local srcdir="${TC_SRC_BINUTILS}"
1696   local objdir="${TC_BUILD_BINUTILS_LIBERTY}"
1697
1698   mkdir -p "${objdir}"
1699   spushd "${objdir}"
1700   RunWithLog binutils.liberty.configure \
1701       env -i \
1702       PATH="${PATH}" \
1703       CC="${CC}" \
1704       CXX="${CXX}" \
1705       ${srcdir}/configure
1706   spopd
1707 }
1708
1709 binutils-liberty-needs-make() {
1710   local srcdir="${TC_SRC_BINUTILS}"
1711   local objdir="${TC_BUILD_BINUTILS_LIBERTY}"
1712
1713   ts-modified "$srcdir" "$objdir"
1714   return $?
1715 }
1716
1717 #+ binutils-liberty-make - Make binutils-liberty
1718 binutils-liberty-make() {
1719   StepBanner "BINUTILS-LIBERTY" "Make"
1720   local srcdir="${TC_SRC_BINUTILS}"
1721   local objdir="${TC_BUILD_BINUTILS_LIBERTY}"
1722   spushd "${objdir}"
1723
1724   ts-touch-open "${objdir}"
1725
1726   RunWithLog binutils.liberty.make \
1727       env -i \
1728       PATH="${PATH}" \
1729       CC="${CC}" \
1730       CXX="${CXX}" \
1731       make ${MAKE_OPTS_HOST} all-libiberty
1732
1733   ts-touch-commit "${objdir}"
1734
1735   spopd
1736 }
1737
1738 #########################################################################
1739 #     CLIENT BINARIES (SANDBOXED)
1740 #########################################################################
1741
1742 check-arch() {
1743   local arch=$1
1744   for valid_arch in i686 x86_64 armv7 universal ; do
1745     if [ "${arch}" == "${valid_arch}" ] ; then
1746       return
1747     fi
1748   done
1749
1750   Fatal "ERROR: Unsupported arch [$1]. Must be: i686, x86_64, armv7, universal"
1751 }
1752
1753 llvm-sb-setup() {
1754   local arch=$1
1755   LLVM_SB_LOG_PREFIX="llvm.sb.${arch}"
1756   LLVM_SB_OBJDIR="$(GetTranslatorBuildDir ${arch})/llvm-sb"
1757
1758   # The SRPC headers are included directly from the nacl tree, as they are
1759   # not in the SDK. libsrpc should have already been built by the
1760   # build.sh sdk-private-libs step.
1761   # This is always statically linked.
1762   # The LLVM sandboxed build uses the normally-disallowed external
1763   # function __nacl_get_arch().  Allow that for now.
1764   local flags="-static -I$(GetAbsolutePath ${NACL_ROOT}/..) \
1765     --pnacl-disable-abi-check "
1766
1767   LLVM_SB_CONFIGURE_ENV=(
1768     AR="${PNACL_AR}" \
1769     AS="${PNACL_AS}" \
1770     CC="${PNACL_CC} ${flags}" \
1771     CXX="${PNACL_CXX} ${flags}" \
1772     LD="${PNACL_LD} ${flags}" \
1773     NM="${PNACL_NM}" \
1774     RANLIB="${PNACL_RANLIB}")
1775 }
1776
1777 #+-------------------------------------------------------------------------
1778 #+ llvm-sb <arch>- Build and install llvm tools (sandboxed)
1779 llvm-sb() {
1780   local arch=$1
1781   check-arch ${arch}
1782   llvm-sb-setup ${arch}
1783   StepBanner "LLVM-SB" "Sandboxed pnacl-llc [${arch}]"
1784   local srcdir="${TC_SRC_LLVM}"
1785   assert-dir "${srcdir}" "You need to checkout llvm."
1786
1787   if llvm-sb-needs-configure ${arch} ; then
1788     llvm-sb-clean ${arch}
1789     llvm-sb-configure ${arch}
1790   else
1791     SkipBanner "LLVM-SB" "configure ${arch}"
1792   fi
1793
1794   llvm-sb-make ${arch}
1795   llvm-sb-install ${arch}
1796 }
1797
1798 llvm-sb-needs-configure() {
1799   [ ! -f "${LLVM_SB_OBJDIR}/config.status" ]
1800   return $?
1801 }
1802
1803 # llvm-sb-clean          - Clean llvm tools (sandboxed)
1804 llvm-sb-clean() {
1805   local arch=$1
1806   StepBanner "LLVM-SB" "Clean ${arch}"
1807   local objdir="${LLVM_SB_OBJDIR}"
1808
1809   rm -rf "${objdir}"
1810   mkdir -p "${objdir}"
1811 }
1812
1813 # llvm-sb-configure - Configure llvm tools (sandboxed)
1814 llvm-sb-configure() {
1815   local arch=$1
1816
1817   StepBanner "LLVM-SB" "Configure ${arch}"
1818   local srcdir="${TC_SRC_LLVM}"
1819   local objdir="${LLVM_SB_OBJDIR}"
1820   local installdir="$(GetTranslatorInstallDir ${arch})"
1821   local targets=""
1822   # For LLVM, "x86" brings in both i686 and x86_64.
1823   case ${arch} in
1824     i686) targets=x86 ;;
1825     x86_64) targets=x86 ;;
1826     armv7) targets=arm ;;
1827     universal) targets=x86,arm ;;
1828   esac
1829
1830   spushd "${objdir}"
1831   # TODO(jvoung): remove ac_cv_func_getrusage=no once newlib has getrusage
1832   # in its headers.  Otherwise, configure thinks that we can link in
1833   # getrusage (stub is in libnacl), but we can't actually compile code
1834   # that uses ::getrusage because it's not in headers:
1835   # https://code.google.com/p/nativeclient/issues/detail?id=3657
1836   RunWithLog \
1837       ${LLVM_SB_LOG_PREFIX}.configure \
1838       env -i \
1839       PATH="/usr/bin:/bin" \
1840       ${srcdir}/configure \
1841         "${LLVM_SB_CONFIGURE_ENV[@]}" \
1842         --prefix=${installdir} \
1843         --host=nacl \
1844         --enable-targets=${targets} \
1845         --disable-assertions \
1846         --enable-pic=no \
1847         --enable-static \
1848         --enable-shared=no \
1849         --disable-jit \
1850         --enable-optimized \
1851         --target=${CROSS_TARGET_ARM} \
1852         llvm_cv_link_use_export_dynamic=no \
1853         ac_cv_func_getrusage=no
1854   spopd
1855 }
1856
1857 # llvm-sb-make - Make llvm tools (sandboxed)
1858 llvm-sb-make() {
1859   local arch=$1
1860   StepBanner "LLVM-SB" "Make ${arch}"
1861   local objdir="${LLVM_SB_OBJDIR}"
1862
1863   spushd "${objdir}"
1864   ts-touch-open "${objdir}"
1865
1866   local tools_to_build="pnacl-llc"
1867   local export_dyn_env="llvm_cv_link_use_export_dynamic=no"
1868   local isjit=0
1869   RunWithLog ${LLVM_SB_LOG_PREFIX}.make \
1870       env -i PATH="/usr/bin:/bin" \
1871       ONLY_TOOLS="${tools_to_build}" \
1872       NACL_SANDBOX=1 \
1873       KEEP_SYMBOLS=1 \
1874       VERBOSE=1 \
1875       ${export_dyn_env} \
1876       make ${MAKE_OPTS} tools-only
1877
1878   ts-touch-commit "${objdir}"
1879
1880   spopd
1881 }
1882
1883 # llvm-sb-install - Install llvm tools (sandboxed)
1884 llvm-sb-install() {
1885   local arch=$1
1886   StepBanner "LLVM-SB" "Install ${arch}"
1887
1888   local toolname="pnacl-llc"
1889   local installdir="$(GetTranslatorInstallDir ${arch})"/bin
1890   mkdir -p "${installdir}"
1891   spushd "${installdir}"
1892   local objdir="${LLVM_SB_OBJDIR}"
1893   cp -f "${objdir}"/Release*/bin/${toolname} .
1894   mv -f ${toolname} ${toolname}.pexe
1895   local arches=${arch}
1896   if [[ "${arch}" == "universal" ]]; then
1897     arches="${SBTC_ARCHES_ALL}"
1898   elif [[ "${arch}" == "i686" ]]; then
1899     # LLVM does not separate the i686 and x86_64 backends.
1900     # Translate twice to get both nexes.
1901     arches="i686 x86_64"
1902   fi
1903   translate-sb-tool ${toolname} "${arches}"
1904   install-sb-tool ${toolname} "${arches}"
1905   spopd
1906 }
1907
1908 # translate-sb-tool <toolname> <arches>
1909 #
1910 # Translate <toolname>.pexe to <toolname>.<arch>.nexe in the current directory.
1911 translate-sb-tool() {
1912   local toolname=$1
1913   local arches=$2
1914   local pexe="${toolname}.pexe"
1915   if ${PNACL_PRUNE}; then
1916     # Only strip debug, to preserve symbol names for testing. This is okay
1917     # because we strip the native nexe later anyway.
1918     # So, why bother stripping here at all?
1919     # It does appear to affect the size of the nexe:
1920     # http://code.google.com/p/nativeclient/issues/detail?id=3305
1921     ${PNACL_STRIP} --strip-debug "${pexe}"
1922   fi
1923
1924   local tarch
1925   for tarch in ${arches}; do
1926     local nexe="${toolname}.${tarch}.nexe"
1927     StepBanner "TRANSLATE" \
1928                "Translating ${toolname}.pexe to ${tarch} (background)"
1929     # NOTE: we are using --noirt to build without a segment gap
1930     # since we aren't loading the IRT for the translator nexes.
1931     #
1932     # Compiling with -ffunction-sections, -fdata-sections, --gc-sections
1933     # helps reduce the size a bit. If you want to use --gc-sections to test out:
1934     # http://code.google.com/p/nativeclient/issues/detail?id=1591
1935     # you will need to do a build without these flags.
1936     "${PNACL_TRANSLATE}" -ffunction-sections -fdata-sections --gc-sections \
1937       --allow-llvm-bitcode-input --noirt -arch ${tarch} "${pexe}" -o "${nexe}" &
1938     QueueLastProcess
1939   done
1940   StepBanner "TRANSLATE" "Waiting for translation processes to finish"
1941   QueueWait
1942
1943   # Test that certain symbols have been pruned before stripping.
1944   if [ "${toolname}" == "pnacl-llc" ]; then
1945     for tarch in ${arches}; do
1946       local nexe="${toolname}.${tarch}.nexe"
1947       local llvm_host_glob="${LLVM_INSTALL_DIR}/lib/libLLVM*so"
1948       python "${PNACL_ROOT}/prune_test.py" "${PNACL_NM}" \
1949         "${llvm_host_glob}" "${nexe}"
1950     done
1951   fi
1952
1953   if ${PNACL_PRUNE}; then
1954     # Strip the nexes.
1955     for tarch in ${arches}; do
1956       local nexe="${toolname}.${tarch}.nexe"
1957       ${PNACL_STRIP} "${nexe}"
1958     done
1959   fi
1960   StepBanner "TRANSLATE" "Done."
1961 }
1962
1963 # install-sb-tool <toolname> <arches>
1964 #
1965 # Install <toolname>.<arch>.nexe from the current directory
1966 # into the right location (as <toolname>.nexe)
1967 install-sb-tool() {
1968   local toolname="$1"
1969   local arches="$2"
1970   local tarch
1971   for tarch in ${arches}; do
1972     local installbin="$(GetTranslatorInstallDir ${tarch})"/bin
1973     mkdir -p "${installbin}"
1974     mv -f ${toolname}.${tarch}.nexe "${installbin}"/${toolname}.nexe
1975     if ${PNACL_BUILDBOT}; then
1976       spushd "${installbin}"
1977       local tag="${toolname}_${tarch}_size"
1978       print-tagged-tool-sizes "${tag}" "$(pwd)/${toolname}.nexe"
1979       spopd
1980     fi
1981   done
1982 }
1983
1984 GetTranslatorBuildDir() {
1985   local arch="$1"
1986   echo "${TC_BUILD}/translator-${arch//_/-}"
1987 }
1988
1989 GetTranslatorInstallDir() {
1990   local arch
1991   case $1 in
1992     i686) arch=x86-32 ;;
1993     x86_64) arch=x86-64 ;;
1994     armv7) arch=arm ;;
1995     default) arch=$1 ;;
1996   esac
1997   echo "${INSTALL_TRANSLATOR}"/translator/${arch}
1998 }
1999
2000 ### Sandboxed version of gold.
2001
2002 #+-------------------------------------------------------------------------
2003 #+ binutils-gold-sb - Build and install gold (sandboxed)
2004 #+                    This is the replacement for the old
2005 #+                    final sandboxed linker which was bfd based.
2006 #+                    It has nothing to do with the bitcode linker
2007 #+                    which is also gold based.
2008 binutils-gold-sb() {
2009   local arch=$1
2010   check-arch ${arch}
2011   StepBanner "GOLD-NATIVE-SB" "(libiberty + gold) ${arch}"
2012
2013   local srcdir="${TC_SRC_BINUTILS}"
2014   assert-dir "${srcdir}" "You need to checkout gold."
2015
2016   binutils-gold-sb-clean ${arch}
2017   binutils-gold-sb-configure ${arch}
2018   binutils-gold-sb-make ${arch}
2019   binutils-gold-sb-install ${arch}
2020 }
2021
2022 # binutils-gold-sb-clean - Clean gold
2023 binutils-gold-sb-clean() {
2024   local arch=$1
2025   StepBanner "GOLD-NATIVE-SB" "Clean ${arch}"
2026   local objdir="$(GetTranslatorBuildDir ${arch})/binutils-gold-sb"
2027
2028   rm -rf "${objdir}"
2029   mkdir -p "${objdir}"
2030 }
2031
2032 # binutils-gold-sb-configure - Configure binutils for gold (unsandboxed)
2033 binutils-gold-sb-configure() {
2034   local arch=$1
2035   local srcdir="${TC_SRC_BINUTILS}"
2036   local objdir="$(GetTranslatorBuildDir ${arch})/binutils-gold-sb"
2037   local installbin="$(GetTranslatorInstallDir ${arch})/bin"
2038
2039   # The SRPC headers are included directly from the nacl tree, as they are
2040   # not in the SDK. libsrpc should have already been built by the
2041   # build.sh sdk-private-libs step
2042   # The Gold sandboxed build uses the normally-disallowed external
2043   # function __nacl_get_arch().  Allow that for now.
2044   #
2045   local flags="-static -I$(GetAbsolutePath ${NACL_ROOT}/..) \
2046     -fno-exceptions -O3 --pnacl-disable-abi-check "
2047   local configure_env=(
2048     AR="${PNACL_AR}" \
2049     AS="${PNACL_AS}" \
2050     CC="${PNACL_CC} ${flags}" \
2051     CXX="${PNACL_CXX} ${flags}" \
2052     CC_FOR_BUILD="${CC}" \
2053     CXX_FOR_BUILD="${CXX}" \
2054     LD="${PNACL_LD} ${flags}" \
2055     NM="${PNACL_NM}" \
2056     RANLIB="${PNACL_RANLIB}"
2057   )
2058   local gold_targets=""
2059   case ${arch} in
2060     i686)      gold_targets=i686-pc-nacl ;;
2061     x86_64)    gold_targets=x86_64-pc-nacl ;;
2062     armv7)     gold_targets=arm-pc-nacl ;;
2063     universal)
2064       gold_targets=i686-pc-nacl,x86_64-pc-nacl,arm-pc-nacl ;;
2065   esac
2066
2067   # gold always adds "target" to the enabled targets so we are
2068   # little careful to not build too much
2069   # Note: we are (ab)using target for both --host and --target
2070   #       which configure expects to be present
2071   local target
2072   if [ ${arch} == "universal" ] ; then
2073     target=i686-pc-nacl
2074   else
2075     target=${gold_targets}
2076   fi
2077
2078   StepBanner "GOLD-NATIVE-SB" "Configure (libiberty)"
2079   # Gold depends on liberty only for a few functions:
2080   # xrealloc, lbasename, etc.
2081   # we could remove these if necessary.
2082
2083   mkdir -p "${objdir}/libiberty"
2084   spushd "${objdir}/libiberty"
2085   StepBanner "GOLD-NATIVE-SB" "Dir [$(pwd)]"
2086   local log_prefix="binutils-gold.sb.${arch}"
2087   RunWithLog "${log_prefix}".configure \
2088     env -i \
2089     PATH="/usr/bin:/bin" \
2090     "${configure_env[@]}" \
2091     ${srcdir}/libiberty/configure --prefix="${installbin}" \
2092     --host=${target} \
2093     --target=${target}
2094   spopd
2095
2096   StepBanner "GOLD-NATIVE-SB" "Configure (gold) ${arch}"
2097   mkdir -p "${objdir}/gold"
2098   spushd "${objdir}/gold"
2099   StepBanner "GOLD-NATIVE-SB" "Dir [$(pwd)]"
2100   # Removed -Werror until upstream gold no longer has problems with new clang
2101   # warnings. http://code.google.com/p/nativeclient/issues/detail?id=2861
2102   # TODO(sehr,robertm): remove this when gold no longer has these.
2103   RunWithLog "${log_prefix}".configure \
2104     env -i \
2105     PATH="/usr/bin:/bin" \
2106     "${configure_env[@]}" \
2107     CXXFLAGS="" \
2108     CFLAGS="" \
2109     ac_cv_search_zlibVersion=no \
2110     ac_cv_header_sys_mman_h=no \
2111     ac_cv_func_mmap=no \
2112     ac_cv_func_mallinfo=no \
2113     ac_cv_prog_cc_g=no \
2114     ac_cv_prog_cxx_g=no \
2115     ${srcdir}/gold/configure --prefix="${installbin}" \
2116                                       --enable-targets=${gold_targets} \
2117                                       --host=${target} \
2118                                       --target=${target} \
2119                                       --disable-nls \
2120                                       --enable-plugins=no \
2121                                       --enable-naclsrpc=yes \
2122                                       --disable-werror \
2123                                       --with-sysroot="${NONEXISTENT_PATH}"
2124   # Note: the extra ac_cv settings:
2125   # * eliminate unnecessary use of zlib
2126   # * eliminate use of mmap
2127   # (those should not have much impact on the non-sandboxed
2128   # version but help in the sandboxed case)
2129   # We also disable debug (-g) because the bitcode files become too big
2130   # (with ac_cv_prog_cc_g).
2131
2132   # There's no point in setting the correct path as sysroot, because we
2133   # want the toolchain to be relocatable. The driver will use ld command-line
2134   # option --sysroot= to override this value and set it to the correct path.
2135   # However, we need to include --with-sysroot during configure to get this
2136   # option. So fill in a non-sense, non-existent path.
2137   spopd
2138 }
2139
2140 # binutils-gold-sb-make - Make binutils (sandboxed)
2141 binutils-gold-sb-make() {
2142   local arch=${1}
2143   local objdir="$(GetTranslatorBuildDir ${arch})/binutils-gold-sb"
2144   ts-touch-open "${objdir}/"
2145
2146   StepBanner "GOLD-NATIVE-SB" "Make (liberty) ${arch}"
2147   spushd "${objdir}/libiberty"
2148
2149   RunWithLog "binutils-gold.liberty.sb.${arch}".make \
2150       env -i PATH="/usr/bin:/bin" \
2151       make ${MAKE_OPTS}
2152   spopd
2153
2154   StepBanner "GOLD-NATIVE-SB" "Make (gold) ${arch}"
2155   spushd "${objdir}/gold"
2156   RunWithLog "binutils-gold.sb.${arch}".make \
2157       env -i PATH="/usr/bin:/bin" \
2158       make ${MAKE_OPTS} ld-new
2159   spopd
2160
2161   ts-touch-commit "${objdir}"
2162 }
2163
2164 # binutils-gold-sb-install - Install gold
2165 binutils-gold-sb-install() {
2166   local arch=$1
2167   local objdir="$(GetTranslatorBuildDir ${arch})/binutils-gold-sb"
2168   local installbin="$(GetTranslatorInstallDir ${arch})/bin"
2169
2170   StepBanner "GOLD-NATIVE-SB" "Install [${installbin}] ${arch}"
2171
2172   mkdir -p "${installbin}"
2173   spushd "${installbin}"
2174
2175   # Install just "ld"
2176   cp "${objdir}"/gold/ld-new ld.pexe
2177
2178   # Translate and install
2179   local arches=${arch}
2180   if [[ "${arch}" == "universal" ]]; then
2181     arches="${SBTC_ARCHES_ALL}"
2182   fi
2183   translate-sb-tool ld "${arches}"
2184   install-sb-tool ld "${arches}"
2185   spopd
2186 }
2187
2188
2189 #+-------------------------------------------------------------------------
2190 #########################################################################
2191 #     < NEWLIB-BITCODE >
2192 #########################################################################
2193
2194 #+ newlib - Build and install newlib in bitcode.
2195 newlib() {
2196   local arch=$1
2197   setup-newlib-env ${arch}
2198   StepBanner "NEWLIB (${arch})"
2199
2200   if newlib-needs-configure; then
2201     newlib-clean
2202     newlib-configure ${arch}
2203   else
2204     SkipBanner "NEWLIB" "configure"
2205   fi
2206
2207   if newlib-needs-make; then
2208     newlib-make ${arch}
2209   else
2210     SkipBanner "NEWLIB" "make"
2211   fi
2212
2213   newlib-install ${arch}
2214 }
2215
2216 #+ newlib-clean  - Clean bitcode newlib.
2217 newlib-clean() {
2218   StepBanner "NEWLIB" "Clean"
2219   rm -rf "${TC_BUILD_NEWLIB}"
2220 }
2221
2222 newlib-needs-configure() {
2223   ts-newer-than "${TC_BUILD_LLVM}" \
2224                 "${TC_BUILD_NEWLIB}" && return 0
2225
2226   [ ! -f "${TC_BUILD_NEWLIB}/config.status" ]
2227   return #?
2228 }
2229
2230 #+ newlib-configure - Configure bitcode Newlib
2231 newlib-configure() {
2232   local arch=$1
2233   setup-newlib-env ${arch}
2234   StepBanner "NEWLIB" "Configure (${NEWLIB_TARGET})"
2235
2236   local srcdir="${TC_SRC_NEWLIB}"
2237   local objdir="${TC_BUILD_NEWLIB}"
2238   mkdir -p "${objdir}"
2239   spushd "${objdir}"
2240
2241   RunWithLog "newlib.${arch}.configure" \
2242     env -i \
2243     PATH="/usr/bin:/bin" \
2244     "${STD_ENV_FOR_NEWLIB[@]}" \
2245     ${srcdir}/configure \
2246         --disable-multilib \
2247         --prefix="${NEWLIB_INSTALL_DIR}" \
2248         --disable-newlib-supplied-syscalls \
2249         --disable-texinfo \
2250         --disable-libgloss \
2251         --enable-newlib-iconv \
2252         --enable-newlib-iconv-from-encodings=UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4 \
2253         --enable-newlib-iconv-to-encodings=UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4 \
2254         --enable-newlib-io-long-long \
2255         --enable-newlib-io-long-double \
2256         --enable-newlib-io-c99-formats \
2257         --enable-newlib-mb \
2258         --target="${NEWLIB_TARGET}"
2259
2260   spopd
2261 }
2262
2263 newlib-needs-make() {
2264   local srcdir="${TC_SRC_NEWLIB}"
2265   local objdir="${TC_BUILD_NEWLIB}"
2266
2267   ts-modified "$srcdir" "$objdir"
2268   return $?
2269 }
2270
2271 #+ newlib-make           - Make bitcode Newlib
2272 newlib-make() {
2273   local arch=$1
2274   setup-newlib-env ${arch}
2275   StepBanner "NEWLIB" "Make"
2276   local srcdir="${TC_SRC_NEWLIB}"
2277   local objdir="${TC_BUILD_NEWLIB}"
2278
2279   ts-touch-open "${objdir}"
2280
2281   spushd "${objdir}"
2282   RunWithLog "newlib.${arch}.make" \
2283     env -i PATH="/usr/bin:/bin" \
2284     make \
2285       "${STD_ENV_FOR_NEWLIB[@]}" \
2286       ${MAKE_OPTS}
2287   spopd
2288
2289   ts-touch-commit "${objdir}"
2290
2291 }
2292
2293 #+ newlib-install        - Install Bitcode Newlib using build env.
2294 newlib-install() {
2295   local arch=$1
2296   setup-newlib-env ${arch}
2297   StepBanner "NEWLIB" "Install"
2298   local objdir="${TC_BUILD_NEWLIB}"
2299
2300   spushd "${objdir}"
2301
2302   # NOTE: we might be better off not using install, as we are already
2303   #       doing a bunch of copying of headers and libs further down
2304   RunWithLog newlib.install \
2305     env -i PATH="/usr/bin:/bin" \
2306       make \
2307       "${STD_ENV_FOR_NEWLIB[@]}" \
2308       install ${MAKE_OPTS}
2309   spopd
2310
2311   # Newlib installs files into usr/${NEWLIB_TARGET}/*
2312   # Get rid of the ${NEWLIB_TARGET}/ prefix.
2313   spushd "${NEWLIB_INSTALL_DIR}"
2314   mkdir -p lib include
2315   mv -f ${NEWLIB_TARGET}/lib/* lib
2316   rm -rf  include/sys include/machine
2317   mv -f ${NEWLIB_TARGET}/include/* include
2318   rmdir ${NEWLIB_TARGET}/lib
2319   rmdir ${NEWLIB_TARGET}/include
2320   rmdir ${NEWLIB_TARGET}
2321
2322   cp "${NACL_ROOT}/src/untrusted/pthread/pthread.h" \
2323     "${NACL_ROOT}/src/untrusted/pthread/semaphore.h" \
2324     include
2325   # Copy the nacl_random.h header, which is needed by the libc++ build. It
2326   # uses the IRT so should be safe to include in the toolchain tarball.
2327   mkdir include/nacl
2328   cp "${NACL_ROOT}/src/untrusted/nacl/nacl_random.h" \
2329     include/nacl
2330   spopd
2331 }
2332
2333 libs-support() {
2334   StepBanner "LIBS-SUPPORT"
2335   libs-support-newlib-crt1 portable
2336   libs-support-newlib-crt1 x86-64
2337   libs-support-bitcode portable
2338   libs-support-bitcode x86-64
2339
2340   local arch
2341   for arch in arm x86-32 x86-64 mips32 arm-nonsfi x86-32-nonsfi; do
2342     libs-support-native ${arch}
2343   done
2344
2345   libs-support-unsandboxed
2346 }
2347
2348 libs-support-newlib-crt1() {
2349   local arch=$1
2350   setup-biased-bitcode-env ${arch}
2351   mkdir -p "${INSTALL_LIB}"
2352   spushd "${PNACL_SUPPORT}/bitcode"
2353
2354   StepBanner "LIBS-SUPPORT-NEWLIB" \
2355              "Install crt1.x & crt1_for_eh.x (linker scripts)"
2356   # Two versions of crt1.x exist, for different scenarios (with and without EH).
2357   # See: https://code.google.com/p/nativeclient/issues/detail?id=3069
2358   cp crt1.x "${INSTALL_LIB}"/crt1.x
2359   cp crt1_for_eh.x "${INSTALL_LIB}"/crt1_for_eh.x
2360   spopd
2361 }
2362
2363 libs-support-bitcode() {
2364   local arch=$1
2365   setup-biased-bitcode-env ${arch}
2366   local build_dir="${TC_BUILD}/libs-support-bitcode-${arch}"
2367   local cc_cmd="${PNACL_CC} -Wall -Werror -O2 ${BIASED_BC_CFLAGS}"
2368
2369   mkdir -p "${build_dir}"
2370   spushd "${PNACL_SUPPORT}/bitcode"
2371   # Install crti.bc (empty _init/_fini)
2372   StepBanner "LIBS-SUPPORT" "Install ${arch} crti.bc"
2373   ${cc_cmd} -c crti.c -o "${build_dir}"/crti.bc
2374
2375   # Install crtbegin bitcode (__cxa_finalize for C++)
2376   StepBanner "LIBS-SUPPORT" "Install ${arch} crtbegin.bc"
2377   # NOTE: we do not have an "end" version of ththis
2378   ${cc_cmd} -c crtbegin.c -o "${build_dir}"/crtbegin.bc
2379
2380   # Install unwind_stubs.bc (stubs for _Unwind_* functions when libgcc_eh
2381   # is not included in the native link).
2382   StepBanner "LIBS-SUPPORT" "Install ${arch} unwind_stubs.bc"
2383   ${cc_cmd} -c unwind_stubs.c -o "${build_dir}"/unwind_stubs.bc
2384   StepBanner "LIBS-SUPPORT" "Install ${arch} sjlj_eh_redirect.bc"
2385   ${cc_cmd} -c sjlj_eh_redirect.c -o "${build_dir}"/sjlj_eh_redirect.bc
2386
2387   # Install libpnaclmm.a (__atomic_*).
2388   StepBanner "LIBS-SUPPORT" "Install ${arch} libpnaclmm.a"
2389   ${cc_cmd} -c pnaclmm.c -o "${build_dir}"/pnaclmm.bc
2390   ${PNACL_AR} rc "${build_dir}"/libpnaclmm.a "${build_dir}"/pnaclmm.bc
2391
2392   spopd
2393
2394   # Install to actual lib directories.
2395   spushd "${build_dir}"
2396   local files="crti.bc crtbegin.bc libpnaclmm.a unwind_stubs.bc sjlj_eh_redirect.bc"
2397   mkdir -p "${INSTALL_LIB}"
2398   cp -f ${files} "${INSTALL_LIB}"
2399   spopd
2400 }
2401
2402 libs-support-native() {
2403   local arch=$1
2404   local destdir="${INSTALL_LIB_NATIVE}"${arch}
2405   local label="LIBS-SUPPORT (${arch})"
2406   mkdir -p "${destdir}"
2407
2408   local flags="--pnacl-allow-native --pnacl-allow-translate -Wall -Werror"
2409   local cc_cmd="${PNACL_CC} -arch ${arch} --pnacl-bias=${arch} ${flags} \
2410       -I../../.. -O3"
2411
2412   spushd "${PNACL_SUPPORT}"
2413
2414   # NOTE: one of the reasons why we build these native is
2415   #       the shim library on x86-64
2416   #       c.f.: https://sites.google.com/a/google.com/nativeclient/project-pnacl/c-exception-handling
2417   # Compile crtbegin.o / crtend.o
2418   StepBanner "${label}" "Install crtbegin.o / crtend.o"
2419
2420   # Build two versions of crtbegin: one that relies on later linking with
2421   # libgcc_eh and one that doesn't.
2422   # See: https://code.google.com/p/nativeclient/issues/detail?id=3069
2423   ${cc_cmd} -c crtbegin.c -o "${destdir}"/crtbegin.o
2424   ${cc_cmd} -c crtbegin.c -DLINKING_WITH_LIBGCC_EH \
2425             -o "${destdir}"/crtbegin_for_eh.o
2426   ${cc_cmd} -c crtend.c -o "${destdir}"/crtend.o
2427
2428   # Make libcrt_platform.a
2429   StepBanner "${label}" "Install libcrt_platform.a"
2430   local tmpdir="${TC_BUILD}/libs-support-native"
2431   rm -rf "${tmpdir}"
2432   mkdir -p "${tmpdir}"
2433   ${cc_cmd} -c pnacl_irt.c -o "${tmpdir}"/pnacl_irt.o
2434   ${cc_cmd} -c relocate.c -o "${tmpdir}"/relocate.o
2435
2436   local setjmp_arch="$arch"
2437   if [ "$setjmp_arch" = "x86-32-nonsfi" ]; then
2438     setjmp_arch=x86-32
2439   elif [ "$setjmp_arch" = "arm-nonsfi" ]; then
2440     setjmp_arch=arm
2441   fi
2442   ${cc_cmd} -c setjmp_${setjmp_arch/-/_}.S -o "${tmpdir}"/setjmp.o
2443
2444   if [ "$arch" = "x86-32-nonsfi" ]; then
2445     ${cc_cmd} -c entry_linux.c -o "${tmpdir}"/entry_linux.o
2446     ${cc_cmd} -c entry_linux_x86_32.S -o "${tmpdir}"/entry_linux_asm.o
2447   elif [ "$arch" = "arm-nonsfi" ]; then
2448     ${cc_cmd} -c entry_linux.c -o "${tmpdir}"/entry_linux.o
2449     ${cc_cmd} -c entry_linux_arm.S -o "${tmpdir}"/entry_linux_asm.o
2450   fi
2451
2452   # Some of the support code lives in third_party/ because it's based on code
2453   # from other open-source projects.
2454   ${cc_cmd} \
2455     -c "${NACL_SRC_THIRD_PARTY}/pnacl_native_newlib_subset/string.c" \
2456     -std=c99 -o "${tmpdir}"/string.o
2457   # Pull in the no-errno __ieee754_fmod from newlib and rename it to fmod.
2458   # This is to support the LLVM frem instruction.
2459   ${cc_cmd} \
2460     -c "${TC_SRC_NEWLIB}/newlib/libm/math/e_fmod.c" \
2461     -I"${TC_SRC_NEWLIB}/newlib/libm/common/" \
2462     -D__ieee754_fmod=fmod \
2463     -std=c99 -o "${tmpdir}"/e_fmod.o
2464   ${cc_cmd} \
2465     -c "${TC_SRC_NEWLIB}/newlib/libm/math/ef_fmod.c" \
2466     -I"${TC_SRC_NEWLIB}/newlib/libm/common/" \
2467     -D__ieee754_fmodf=fmodf \
2468     -std=c99 -o "${tmpdir}"/ef_mod.o
2469
2470   # For ARM, also compile aeabi_read_tp.S
2471   if  [ ${arch} == arm ] ; then
2472     ${cc_cmd} -c aeabi_read_tp.S -o "${tmpdir}"/aeabi_read_tp.o
2473   fi
2474   spopd
2475
2476   ${PNACL_AR} rc "${destdir}"/libcrt_platform.a "${tmpdir}"/*.o
2477 }
2478
2479 libs-support-unsandboxed() {
2480   if ${BUILD_PLATFORM_LINUX} || ${BUILD_PLATFORM_MAC}; then
2481     local arch="x86-32-${BUILD_PLATFORM}"
2482     StepBanner "LIBS-SUPPORT (${arch})" "Install unsandboxed_irt.o"
2483     local destdir="${INSTALL_LIB_NATIVE}"${arch}
2484     mkdir -p ${destdir}
2485     # The NaCl headers insist on having a platform macro such as
2486     # NACL_LINUX defined, but nonsfi/irt/irt_interfaces.c does not
2487     # itself use any of these macros, so defining NACL_LINUX here even
2488     # on non-Linux systems is OK.
2489     gcc -m32 -O2 -Wall -Werror -I${NACL_ROOT}/.. -c \
2490         -DNACL_LINUX=1 -DDEFINE_MAIN \
2491         ${NACL_ROOT}/src/nonsfi/irt/irt_interfaces.c \
2492         -o ${destdir}/unsandboxed_irt.o
2493     gcc -m32 -O2 -Wall -Werror -I${NACL_ROOT}/.. -c \
2494         ${NACL_ROOT}/src/untrusted/irt/irt_query_list.c \
2495         -o ${destdir}/irt_query_list.o
2496   fi
2497 }
2498
2499
2500 # Build the dummy "libpnacl_irt_shim.a", which is useful for building
2501 # commandline programs.  It cannot be used to build PPAPI programs
2502 # because it does not actually shim the PPAPI interfaces.
2503 # The library is named the same as the real PPAPI shim to ensure that
2504 # the commandlines are the same.
2505 # This must be built after newlib(), since it uses headers like <stdint.h>.
2506 dummy-irt-shim() {
2507   local arch=$1
2508   local destdir="${INSTALL_LIB_NATIVE}"${arch}
2509   local label="DUMMY-IRT-SHIM (${arch})"
2510   mkdir -p "${destdir}"
2511
2512   local flags="--pnacl-allow-native --pnacl-allow-translate -Wall -Werror"
2513   local cc_cmd="${PNACL_CC} -arch ${arch} ${flags}"
2514
2515   spushd "${PNACL_SUPPORT}"
2516   StepBanner "${label}" "Install libpnacl_irt_shim_dummy.a"
2517   local tmpdir="${TC_BUILD}/dummy-irt-shim"
2518   rm -rf "${tmpdir}"
2519   mkdir -p "${tmpdir}"
2520   ${cc_cmd} -c dummy_shim_entry.c -o "${tmpdir}"/dummy_shim_entry.o
2521   spopd
2522
2523   ${PNACL_AR} rc "${destdir}"/libpnacl_irt_shim_dummy.a "${tmpdir}"/*.o
2524 }
2525
2526 #########################################################################
2527 #     < SDK >
2528 #########################################################################
2529 SCONS_ARGS=(MODE=nacl
2530             -j${PNACL_CONCURRENCY}
2531             bitcode=1
2532             disable_nosys_linker_warnings=1
2533             naclsdk_validate=0
2534             --verbose)
2535
2536 SDK_IS_SETUP=false
2537 sdk-setup() {
2538   if ${SDK_IS_SETUP} && [ $# -eq 0 ]; then
2539     return 0
2540   fi
2541   SDK_IS_SETUP=true
2542
2543   SDK_INSTALL_ROOT="${INSTALL_ROOT}/sdk"
2544   SDK_INSTALL_LIB="${SDK_INSTALL_ROOT}/lib"
2545   SDK_INSTALL_INCLUDE="${SDK_INSTALL_ROOT}/include"
2546 }
2547
2548 sdk() {
2549   sdk-setup "$@"
2550   StepBanner "SDK"
2551   sdk-clean
2552   sdk-headers
2553   sdk-libs
2554   sdk-verify
2555 }
2556
2557 #+ sdk-clean             - Clean sdk stuff
2558 sdk-clean() {
2559   sdk-setup "$@"
2560   StepBanner "SDK" "Clean"
2561   rm -rf "${SDK_INSTALL_ROOT}"
2562
2563   # clean scons obj dirs
2564   rm -rf "${SCONS_OUT}"/nacl-*-pnacl*
2565 }
2566
2567 sdk-headers() {
2568   sdk-setup "$@"
2569   mkdir -p "${SDK_INSTALL_INCLUDE}"
2570
2571   local extra_flags=""
2572   local neutral_platform="x86-32"
2573
2574   StepBanner "SDK" "Install headers"
2575   spushd "${NACL_ROOT}"
2576   RunWithLog "sdk.headers" \
2577       ./scons \
2578       "${SCONS_ARGS[@]}" \
2579       ${extra_flags} \
2580       platform=${neutral_platform} \
2581       pnacl_newlib_dir="${INSTALL_ROOT}" \
2582       install_headers \
2583       includedir="$(PosixToSysPath "${SDK_INSTALL_INCLUDE}")"
2584   spopd
2585 }
2586
2587 sdk-libs() {
2588   sdk-setup "$@"
2589   StepBanner "SDK" "Install libraries"
2590   mkdir -p "${SDK_INSTALL_LIB}"
2591
2592   local extra_flags=""
2593   local neutral_platform="x86-32"
2594
2595   spushd "${NACL_ROOT}"
2596   RunWithLog "sdk.libs.bitcode" \
2597       ./scons \
2598       "${SCONS_ARGS[@]}" \
2599       ${extra_flags} \
2600       platform=${neutral_platform} \
2601       pnacl_newlib_dir="${INSTALL_ROOT}" \
2602       install_lib \
2603       libdir="$(PosixToSysPath "${SDK_INSTALL_LIB}")"
2604   spopd
2605 }
2606
2607 # This builds lib*_private.a, to allow building the llc and ld nexes without
2608 # the IRT and without the segment gap.
2609 sdk-private-libs() {
2610   sdk-setup "$@"
2611   StepBanner "SDK" "Private (non-IRT) libs"
2612   spushd "${NACL_ROOT}"
2613
2614   local neutral_platform="x86-32"
2615   RunWithLog "sdk.libs_private.bitcode" \
2616     ./scons \
2617     -j${PNACL_CONCURRENCY} \
2618     bitcode=1 \
2619     platform=${neutral_platform} \
2620     pnacl_newlib_dir="${INSTALL_ROOT}" \
2621     --verbose \
2622     libnacl_sys_private \
2623     libpthread_private \
2624     libnacl_dyncode_private \
2625     libplatform \
2626     libimc \
2627     libimc_syscalls \
2628     libsrpc \
2629     libgio
2630
2631   local out_dir_prefix="${SCONS_OUT}"/nacl-x86-32-pnacl-pexe-clang
2632   local outdir="${out_dir_prefix}"/lib
2633   mkdir -p "${SDK_INSTALL_LIB}"
2634   cp "${outdir}"/lib*_private.a \
2635      "${outdir}"/lib{platform,imc,imc_syscalls,srpc,gio}.a "${SDK_INSTALL_LIB}"
2636   spopd
2637 }
2638
2639 sdk-verify() {
2640   sdk-setup "$@"
2641   StepBanner "SDK" "Verify"
2642
2643   # Verify bitcode libraries
2644   verify-bitcode-dir "${SDK_INSTALL_LIB}"
2645 }
2646
2647 newlib-nacl-headers-clean() {
2648   # Clean the include directory and revert it to its pure state
2649   if [ -d "${TC_SRC_NEWLIB}" ]; then
2650     rm -rf "${NEWLIB_INCLUDE_DIR}"
2651     # If the script is interrupted right here,
2652     # then NEWLIB_INCLUDE_DIR will not exist, and the repository
2653     # will be in a bad state. This will be fixed during the next
2654     # invocation by newlib-nacl-headers.
2655
2656     spushd "$(dirname "${NEWLIB_INCLUDE_DIR}")"
2657     RunWithLog "newlib-nacl-headers-clean" \
2658       ${GIT} checkout "$(basename "${NEWLIB_INCLUDE_DIR}")"
2659     spopd
2660   fi
2661 }
2662
2663 #+ newlib-nacl-headers   - Install NaCl headers to newlib
2664 newlib-nacl-headers() {
2665   StepBanner "newlib-nacl-headers" "Adding nacl headers to newlib"
2666
2667   assert-dir "${TC_SRC_NEWLIB}" "Newlib is not checked out"
2668
2669   # Make sure the headers directory has no local changes
2670   newlib-nacl-headers-check
2671   newlib-nacl-headers-clean
2672
2673   # Install the headers
2674   "${EXPORT_HEADER_SCRIPT}" \
2675       "${NACL_SYS_HEADERS}" \
2676       "${NEWLIB_INCLUDE_DIR}"
2677
2678   # Record the header install time
2679   ts-touch "${NACL_HEADERS_TS}"
2680 }
2681
2682 #+ newlib-nacl-headers-check - Make sure the newlib nacl headers haven't
2683 #+                             been modified since the last install.
2684 newlib-nacl-headers-check() {
2685   # The condition where NEWLIB_INCLUDE_DIR does not exist may have been
2686   # caused by an incomplete call to newlib-nacl-headers-clean().
2687   # Let it pass this check so that the clean will be able to finish.
2688   # See the comment in newlib-nacl-headers-clean()
2689   if ! [ -d "${TC_SRC_NEWLIB}" ] ||
2690      ! [ -d "${NEWLIB_INCLUDE_DIR}" ]; then
2691     return 0
2692   fi
2693
2694   # Already clean?
2695   if ! git-has-changes "${NEWLIB_INCLUDE_DIR}" ; then
2696     return 0
2697   fi
2698
2699   if ts-dir-changed "${NACL_HEADERS_TS}" "${NEWLIB_INCLUDE_DIR}"; then
2700     echo ""
2701     echo "*******************************************************************"
2702     echo "*                            ERROR                                *"
2703     echo "*      The NewLib include directory has local modifications       *"
2704     echo "*******************************************************************"
2705     echo "* The NewLib include directory should not be modified directly.   *"
2706     echo "* Instead, modifications should be done from:                     *"
2707     echo "*   src/trusted/service_runtime/include                           *"
2708     echo "*                                                                 *"
2709     echo "* To destroy the local changes to newlib, run:                    *"
2710     echo "*  pnacl/build.sh newlib-nacl-headers-clean                       *"
2711     echo "*******************************************************************"
2712     echo ""
2713     if ${PNACL_BUILDBOT} ; then
2714       newlib-nacl-headers-clean
2715     else
2716       exit -1
2717     fi
2718   fi
2719 }
2720
2721
2722 #+-------------------------------------------------------------------------
2723 #@ driver                - Install driver scripts.
2724 driver() {
2725   StepBanner "DRIVER"
2726   driver-install
2727 }
2728
2729 # install python scripts and redirector shell/batch scripts
2730 driver-install-python() {
2731   local destdir="$1"
2732   shift
2733   local pydir="${destdir}/pydir"
2734
2735   StepBanner "DRIVER" "Installing driver adaptors to ${destdir}"
2736   rm -rf "${destdir}"
2737   mkdir -p "${destdir}"
2738   mkdir -p "${pydir}"
2739
2740   spushd "${DRIVER_DIR}"
2741
2742   # Copy python scripts
2743   cp $@ driver_log.py driver_env.py driver_temps.py \
2744     *tools.py filetype.py loader.py nativeld.py "${pydir}"
2745
2746   # Install redirector shell/batch scripts
2747   for name in $@; do
2748     local dest="${destdir}/${name/.py}"
2749     # In some situations cygwin cp messes up the permissions of the redirector
2750     # shell/batch scripts. Using cp -a seems to make sure it preserves them.
2751     cp -a redirect.sh "${dest}"
2752     chmod +x "${dest}"
2753     if ${BUILD_PLATFORM_WIN}; then
2754       cp -a redirect.bat "${dest}".bat
2755     fi
2756   done
2757   spopd
2758 }
2759
2760 feature-version-file-install() {
2761   local install_root=$1
2762   # Scons tests can check this version number to decide whether to
2763   # enable tests for toolchain bug fixes or new features.  This allows
2764   # tests to be enabled on the toolchain buildbots/trybots before the
2765   # new toolchain version is rolled into the pinned version (i.e. before
2766   # the tests would pass on the main NaCl buildbots/trybots).
2767   #
2768   # If you are adding a test that depends on a toolchain change, you
2769   # can increment this version number manually.
2770   echo 6 > "${install_root}/FEATURE_VERSION"
2771 }
2772
2773 # The driver is a simple python script which changes its behavior
2774 # depending on the name it is invoked as.
2775 driver-install() {
2776   local bindir=bin
2777   # On Linux we ship a fat toolchain with 2 sets of binaries defaulting to
2778   # x86-32 (mostly because of the 32 bit chrome bots). So the default
2779   # bin dir is 32, and the bin64 driver runs the 64 bit binaries
2780   if ${HOST_ARCH_X8664} && ${BUILD_PLATFORM_LINUX}; then
2781     bindir="bin64"
2782     # We want to be able to locally test a toolchain on 64 bit hosts without
2783     # building it twice and without extra env vars. So if a 32 bit toolchain
2784     # has not already been built, just symlink the bin dirs together.
2785     if [[ ! -d "${INSTALL_BIN}" ]]; then
2786       mkdir -p "${INSTALL_ROOT}"
2787       ln -s ${bindir} "${INSTALL_BIN}"
2788     fi
2789   fi
2790
2791   # This directory (the ${INSTALL_ROOT}/${bindir} part)
2792   # should be kept in sync with INSTALL_BIN et al.
2793   local destdir="${INSTALL_ROOT}/${bindir}"
2794
2795   driver-install-python "${destdir}" "pnacl-*.py"
2796
2797   # Tell the driver the library mode and host arch
2798   echo """HAS_FRONTEND=1
2799 HOST_ARCH=${HOST_ARCH}""" > "${destdir}"/driver.conf
2800
2801   # On windows, copy the cygwin DLLs needed by the driver tools
2802   if ${BUILD_PLATFORM_WIN}; then
2803     StepBanner "DRIVER" "Copying cygwin libraries"
2804     local deps="gcc_s-1 iconv-2 win1 intl-8 stdc++-6 z"
2805     for name in ${deps}; do
2806       cp "/bin/cyg${name}.dll" "${destdir}"
2807     done
2808   fi
2809
2810   # Install a REV file so that "pnacl-clang --version" knows the version
2811   # of the drivers themselves.
2812   DumpAllRevisions > "${destdir}/REV"
2813
2814   feature-version-file-install ${INSTALL_ROOT}
2815 }
2816
2817 #@ driver-install-translator - Install driver scripts for translator component
2818 driver-install-translator() {
2819   local destdir="${INSTALL_TRANSLATOR}/bin"
2820
2821   driver-install-python "${destdir}" pnacl-translate.py
2822
2823   echo """HAS_FRONTEND=0""" > "${destdir}"/driver.conf
2824
2825   feature-version-file-install ${INSTALL_TRANSLATOR}
2826 }
2827
2828 ######################################################################
2829 ######################################################################
2830 #
2831 #                           HELPER FUNCTIONS
2832 #
2833 #             (These should not generally be used directly)
2834 #
2835 ######################################################################
2836 ######################################################################
2837
2838 DumpAllRevisions() {
2839   one-line-rev-info ${NACL_ROOT}
2840   for d in ${PNACL_GIT_ROOT}/*/ ; do
2841     one-line-rev-info $d
2842   done
2843 }
2844
2845 ######################################################################
2846 ######################################################################
2847 #     < VERIFY >
2848 ######################################################################
2849 ######################################################################
2850
2851 # Note: we could replace this with a modified version of tools/elf_checker.py
2852 #       if we do not want to depend on binutils
2853 readonly NACL_OBJDUMP=${BINUTILS_INSTALL_DIR}/bin/${REAL_CROSS_TARGET}-objdump
2854
2855 # Usage: VerifyArchive <checker> <pattern> <filename>
2856 ExtractAndCheck() {
2857   local checker="$1"
2858   local pattern="$2"
2859   local archive="$3"
2860   local tmp="/tmp/ar-verify-${RANDOM}"
2861   rm -rf ${tmp}
2862   mkdir -p ${tmp}
2863   cp "${archive}" "${tmp}"
2864   spushd ${tmp}
2865   ${PNACL_AR} x $(basename ${archive})
2866   # extract all the files
2867   local count=0
2868   for i in ${pattern} ; do
2869     if [ ! -e "$i" ]; then
2870       # we may also see the unexpanded pattern here if there is no match
2871       continue
2872     fi
2873     count=$((count+1))
2874     ${checker} $i
2875   done
2876   echo "PASS  (${count} files)"
2877   rm -rf "${tmp}"
2878   spopd
2879 }
2880
2881 IsLinkerScript() {
2882   local fname="$1"
2883   local type="$(file --brief --mime-type "${fname}")"
2884   case "$type" in
2885     text/x-c)
2886       # A linker script with C comments looks like C to "file".
2887       return 0
2888       ;;
2889     text/plain)
2890       return 0
2891       ;;
2892   esac
2893   return 1
2894 }
2895
2896 # Usage: VerifyLinkerScript <filename>
2897 VerifyLinkerScript() {
2898   local archive="$1"
2899   # Use preprocessor to strip the C-style comments.
2900   ${PNACL_PP} -xc "${archive}" | awk -v archive="$(basename ${archive})" '
2901     BEGIN { status = 0 }
2902     NF == 0 || $1 == "#" { next }
2903     $1 == "INPUT" && $2 == "(" && $NF == ")" { next }
2904     {
2905       print "FAIL - unexpected linker script(?) contents:", archive
2906       status = 1
2907       exit(status)
2908     }
2909     END { if (status == 0) print "PASS  (trivial linker script)" }
2910 ' || exit -1
2911 }
2912
2913 # Usage: VerifyArchive <checker> <pattern> <filename>
2914 VerifyArchive() {
2915   local checker="$1"
2916   local pattern="$2"
2917   local archive="$3"
2918   echo -n "verify $(basename "${archive}"): "
2919   if IsLinkerScript "${archive}"; then
2920     VerifyLinkerScript "${archive}"
2921   else
2922     ExtractAndCheck "$checker" "$pattern" "$archive"
2923   fi
2924 }
2925
2926 #
2927 # verify-object-llvm <obj>
2928 #
2929 #   Verifies that a given .o file is bitcode and free of ASMSs
2930 verify-object-llvm() {
2931   if ${PNACL_DIS} "$1" -o - | grep asm ; then
2932     echo
2933     echo "ERROR asm in $1"
2934     echo
2935     exit -1
2936   fi
2937   if [ ${PIPESTATUS[0]} -ne 0 ]; then
2938     exit -1
2939   fi
2940 }
2941
2942
2943 check-elf-abi() {
2944   # Temporarily disable ELF abi check until DEPS roll
2945   return 0
2946
2947   local arch_info="$(${NACL_OBJDUMP} -f $1)"
2948   if ! grep -q $2 <<< ${arch_info} ; then
2949     echo "ERROR $1 - bad file format: $2 vs ${arch_info}\n"
2950     echo ${arch_info}
2951     exit -1
2952   fi
2953 }
2954
2955
2956 # verify-object-arm <obj>
2957 #
2958 #   Ensure that the ARCH properties are what we expect, this is a little
2959 #   fragile and needs to be updated when tools change
2960 verify-object-arm() {
2961   check-elf-abi $1 "elf32-littlearm"
2962   # llvm-mc does not automatically insert these tags (unlike gnu-as).
2963   # So we exclude llvm-mc generated object files for now
2964   if [[ $1 == aeabi_read_tp.o || $1 == setjmp.o ]] ; then
2965     return
2966   fi
2967   arch_info="$("${PNACL_READELF}" -A "$1")"
2968   #TODO(robertm): some refactoring and cleanup needed
2969   if ! grep -q "Tag_FP_arch: VFPv3" <<< ${arch_info} ; then
2970     echo "ERROR $1 - bad Tag_FP_arch"
2971     #TODO(robertm): figure out what the right thing to do is here, c.f.
2972     # http://code.google.com/p/nativeclient/issues/detail?id=966
2973     "${PNACL_READELF}" -A $1 | grep  Tag_FP_arch
2974     exit -1
2975   fi
2976
2977   if ! grep -q "Tag_CPU_arch: v7" <<< ${arch_info} ; then
2978     echo "FAIL bad $1 Tag_CPU_arch"
2979     "${PNACL_READELF}" -A $1 | grep Tag_CPU_arch
2980     exit -1
2981   fi
2982
2983   if ! grep -q "Tag_Advanced_SIMD_arch: NEONv1" <<< ${arch_info} ; then
2984     echo "FAIL bad $1 Tag_Advanced_SIMD_arch"
2985     "${PNACL_READELF}" -A $1 | grep Tag_Advanced_SIMD_arch
2986   fi
2987
2988   # Check that the file uses the ARM hard-float ABI (where VFP
2989   # registers D0-D7 (s0-s15) are used to pass arguments and results).
2990   if ! grep -q "Tag_ABI_VFP_args: VFP registers" <<< ${arch_info} ; then
2991     echo "FAIL bad $1 Tag_ABI_VFP_args"
2992     "${PNACL_READELF}" -A $1 | grep Tag_ABI_VFP_args
2993   fi
2994 }
2995
2996
2997 # verify-object-x86-32 <obj>
2998 #
2999 verify-object-x86-32() {
3000   check-elf-abi $1 "elf32-i386"
3001 }
3002
3003 # verify-object-x86-64 <obj>
3004 #
3005 verify-object-x86-64() {
3006   check-elf-abi $1 "elf64-x86-64"
3007 }
3008
3009 #+ verify-bitcode-dir    - Verify that the files in a directory are bitcode.
3010 verify-bitcode-dir() {
3011   local dir="$1"
3012   # This avoids errors when * finds no matches.
3013   shopt -s nullglob
3014   SubBanner "VERIFY: ${dir}"
3015   for i in "${dir}"/*.a ; do
3016     verify-archive-llvm "$i"
3017   done
3018   for i in "${dir}"/*.bc ; do
3019     echo -n "verify $(basename "$i"): "
3020     verify-object-llvm "$i"
3021     echo "PASS (bitcode)"
3022   done
3023   for i in "${dir}"/*.o ; do
3024     Fatal "Native object file $i inside bitcode directory"
3025   done
3026   shopt -u nullglob
3027 }
3028
3029
3030 #+ verify-native-dir     - Verify that files in a directory are native for arch.
3031 verify-native-dir() {
3032   local arch="$1"
3033   local dir="$2"
3034
3035   SubBanner "VERIFY: ${dir}"
3036
3037   # This avoids errors when * finds no matches.
3038   shopt -s nullglob
3039   for i in "${dir}"/*.o ; do
3040     verify-object-${arch} "$i"
3041   done
3042
3043   for i in "${dir}"/*.a ; do
3044     verify-archive-${arch} "$i"
3045   done
3046
3047   for i in "${dir}"/*.bc "${dir}"/*.pso ; do
3048     Fatal "Bitcode file $i found inside native directory"
3049   done
3050   shopt -u nullglob
3051 }
3052
3053 #
3054 # verify-archive-llvm <archive>
3055 # Verifies that a given archive is bitcode and free of ASMSs
3056 #
3057 verify-archive-llvm() {
3058   # Currently all the files are .o in the llvm archives.
3059   # Eventually more and more should be .bc.
3060   VerifyArchive verify-object-llvm '*.bc *.o' "$@"
3061 }
3062
3063 #
3064 # verify-archive-arm <archive>
3065 # Verifies that a given archive is a proper arm achive
3066 #
3067 verify-archive-arm() {
3068   VerifyArchive verify-object-arm '*.o *.ons' "$@"
3069 }
3070
3071 #
3072 # verify-archive-x86-32 <archive>
3073 # Verifies that a given archive is a proper x86-32 achive
3074 #
3075 verify-archive-x86-32() {
3076   VerifyArchive verify-object-x86-32 '*.o *.ons' "$@"
3077 }
3078
3079 #
3080 # verify-archive-x86-64 <archive>
3081 # Verifies that a given archive is a proper x86-64 achive
3082 #
3083 verify-archive-x86-64() {
3084   VerifyArchive verify-object-x86-64 '*.o *.ons' "$@"
3085 }
3086
3087 #@-------------------------------------------------------------------------
3088 #+ verify                - Verifies that the pnacl-untrusted ELF files
3089 #+                         are of the correct architecture.
3090 verify() {
3091   StepBanner "VERIFY"
3092   verify-bitcode
3093   verify-native
3094 }
3095
3096 verify-bitcode() {
3097   verify-bitcode-dir "${INSTALL_LIB}"
3098 }
3099
3100 verify-native() {
3101   local arch
3102   for arch in arm x86-32 x86-64; do
3103     verify-native-dir ${arch} "${INSTALL_LIB_NATIVE}${arch}"
3104   done
3105 }
3106
3107 #+ verify-triple-build <arch>
3108 #+     Verify that the sandboxed translator produces an identical
3109 #+     translation of itself (pnacl-llc.pexe) as the unsandboxed translator.
3110 #+     (NOTE: This function is experimental/untested)
3111 verify-triple-build() {
3112   local arch=$1
3113   StepBanner "VERIFY" "Verifying triple build for ${arch}"
3114
3115   local bindir="$(GetTranslatorInstallDir ${arch})/bin"
3116   local llc_nexe="${bindir}/pnacl-llc.nexe"
3117   local llc_pexe="${bindir}/pnacl-llc.pexe"
3118   assert-file "${llc_nexe}" "sandboxed llc for ${arch} does not exist"
3119   assert-file "${llc_pexe}" "pnacl-llc.pexe does not exist"
3120
3121   local flags="--pnacl-sb --pnacl-driver-verbose"
3122
3123   if [ ${arch} == "arm" ] ; then
3124     # Use emulator if we are not on ARM
3125     local hostarch=$(uname -m)
3126     if ! [[ "${BUILD_ARCH}" =~ arm ]]; then
3127       flags+=" --pnacl-use-emulator"
3128     fi
3129   fi
3130
3131   local triple_install_dir="$(GetTranslatorInstallDir ${arch})/triple-build"
3132   mkdir -p ${triple_install_dir}
3133   local new_llc_nexe="${triple_install_dir}/pnacl-llc.rebuild.nexe"
3134   mkdir -p "${triple_install_dir}"
3135   StepBanner "VERIFY" "Translating ${llc_pexe} using sandboxed tools (${arch})"
3136   local sb_translator="${INSTALL_TRANSLATOR}/bin/pnacl-translate"
3137   RunWithLog "verify.triple.build" \
3138     "${sb_translator}" ${flags} -arch ${arch} "${llc_pexe}" -o "${new_llc_nexe}"
3139
3140   if ! cmp --silent "${llc_nexe}" "${new_llc_nexe}" ; then
3141     Banner "TRIPLE BUILD VERIFY FAILED"
3142     echo "Expected these files to be identical, but they are not:"
3143     echo "  ${archllc}"
3144     echo "  ${newllc}"
3145     exit -1
3146   fi
3147   StepBanner "VERIFY" "Verified ${arch} OK"
3148 }
3149
3150 ######################################################################
3151 ######################################################################
3152 #
3153 # UTILITIES
3154 #
3155 ######################################################################
3156 ######################################################################
3157
3158 #@-------------------------------------------------------------------------
3159 #@ show-config
3160 show-config() {
3161   Banner "Config Settings:"
3162   echo "PNACL_BUILDBOT:    ${PNACL_BUILDBOT}"
3163   echo "PNACL_CONCURRENCY: ${PNACL_CONCURRENCY}"
3164   echo "PNACL_DEBUG:       ${PNACL_DEBUG}"
3165   echo "PNACL_PRUNE:       ${PNACL_PRUNE}"
3166   echo "PNACL_VERBOSE:     ${PNACL_VERBOSE}"
3167   echo "INSTALL_ROOT:      ${INSTALL_ROOT}"
3168   Banner "Your Environment:"
3169   env | grep PNACL
3170   Banner "uname info for builder:"
3171   uname -a
3172 }
3173
3174 #@ help                  - Usage information.
3175 help() {
3176   Usage
3177 }
3178
3179 #@ help-full             - Usage information including internal functions.
3180 help-full() {
3181   Usage2
3182 }
3183
3184 has-trusted-toolchain() {
3185   if [ -f ${TOOLCHAIN_BASE}/arm_trusted/ld_script_arm_trusted ]; then
3186     return 0
3187   else
3188     return 1
3189   fi
3190 }
3191
3192 check-for-trusted() {
3193   if ! ${PNACL_BUILD_ARM} ; then
3194     return
3195   fi
3196
3197   if ! has-trusted-toolchain; then
3198     echo '*******************************************************************'
3199     echo '*   The ARM trusted toolchain does not appear to be installed yet *'
3200     echo '*   It is needed to run ARM tests.                                *'
3201     echo '*                                                                 *'
3202     echo '*   To download and install the trusted toolchain, run:           *'
3203     echo '*                                                                 *'
3204     echo '*       $ pnacl/build.sh download-trusted                         *'
3205     echo '*                                                                 *'
3206     echo '*   To compile the trusted toolchain, use:                        *'
3207     echo '*                                                                 *'
3208     echo '*       $ tools/llvm/trusted-toolchain-creator.sh trusted_sdk     *'
3209     echo '*               (warning: this takes a while)                     *'
3210     echo '*******************************************************************'
3211
3212     # If building on the bots, do not continue since it needs to run ARM tests.
3213     if ${PNACL_BUILDBOT} ; then
3214       echo "Building on bots --> need ARM trusted toolchain to run tests!"
3215       exit -1
3216     elif trusted-tc-confirm ; then
3217       echo "Continuing without ARM trusted toolchain"
3218       PNACL_BUILD_ARM=false
3219     else
3220       echo "Okay, stopping."
3221       exit -1
3222     fi
3223   fi
3224 }
3225
3226 trusted-tc-confirm() {
3227   echo
3228   echo "Do you wish to continue without the ARM trusted TC (skip ARM testing)?"
3229   echo ""
3230   confirm-yes "Continue"
3231   return $?
3232 }
3233
3234 DebugRun() {
3235   if ${PNACL_DEBUG} || ${PNACL_BUILDBOT}; then
3236     "$@"
3237   fi
3238 }
3239
3240 ######################################################################
3241 # Generate chromium perf bot logs for tracking the size of a binary.
3242 #
3243 print-tagged-tool-sizes() {
3244   local tag="$1"
3245   local binary="$2"
3246
3247   # size output look like:
3248   #    text   data     bss     dec    hex  filename
3249   #  354421  16132  168920  539473  83b51  .../tool
3250   local sizes=($(${PNACL_SIZE} -B "${binary}" | grep '[0-9]\+'))
3251   echo "RESULT ${tag}: text= ${sizes[0]} bytes"
3252   echo "RESULT ${tag}: data= ${sizes[1]} bytes"
3253   echo "RESULT ${tag}: bss= ${sizes[2]} bytes"
3254   echo "RESULT ${tag}: total= ${sizes[3]} bytes"
3255
3256   local file_size=($(du --bytes "${binary}"))
3257   echo "RESULT ${tag}: file_size= ${file_size[0]} bytes"
3258 }
3259
3260 ######################################################################
3261 ######################################################################
3262 #
3263 #                           < TIME STAMPING >
3264 #
3265 ######################################################################
3266 ######################################################################
3267
3268 ts-dir-changed() {
3269   local tsfile="$1"
3270   local dir="$2"
3271
3272   if [ -f "${tsfile}" ]; then
3273     local MODIFIED=$(find "${dir}" -type f -newer "${tsfile}")
3274     [ ${#MODIFIED} -gt 0 ]
3275     ret=$?
3276   else
3277     true
3278     ret=$?
3279   fi
3280   return $ret
3281 }
3282
3283 # Check if the source for a given build has been modified
3284 ts-modified() {
3285   local srcdir="$1"
3286   local objdir="$2"
3287   local tsfile="${objdir}/${TIMESTAMP_FILENAME}"
3288
3289   ts-dir-changed "${tsfile}" "${srcdir}"
3290   return $?
3291 }
3292
3293 ts-touch() {
3294   local tsfile="$1"
3295   touch "${tsfile}"
3296 }
3297
3298 # Record the time when make begins, but don't yet
3299 # write that to the timestamp file.
3300 # (Just in case make fails)
3301
3302 ts-touch-open() {
3303   local objdir="$1"
3304   local tsfile="${objdir}/${TIMESTAMP_FILENAME}"
3305   local tsfile_open="${objdir}/${TIMESTAMP_FILENAME}_OPEN"
3306
3307   rm -f "${tsfile}"
3308   touch "${tsfile_open}"
3309 }
3310
3311
3312 # Write the timestamp. (i.e. make has succeeded)
3313
3314 ts-touch-commit() {
3315   local objdir="$1"
3316   local tsfile="${objdir}/${TIMESTAMP_FILENAME}"
3317   local tsfile_open="${objdir}/${TIMESTAMP_FILENAME}_OPEN"
3318
3319   mv -f "${tsfile_open}" "${tsfile}"
3320 }
3321
3322
3323 # ts-newer-than dirA dirB
3324 # Compare the make timestamps in both object directories.
3325 # returns true (0) if dirA is newer than dirB
3326 # returns false (1) otherwise.
3327 #
3328 # This functions errs on the side of returning 0, since
3329 # that forces a rebuild anyway.
3330
3331 ts-newer-than() {
3332   local objdir1="$1"
3333   local objdir2="$2"
3334
3335   local tsfile1="${objdir1}/${TIMESTAMP_FILENAME}"
3336   local tsfile2="${objdir2}/${TIMESTAMP_FILENAME}"
3337
3338   if [ ! -d "${objdir1}" ]; then return 0; fi
3339   if [ ! -d "${objdir2}" ]; then return 0; fi
3340
3341   if [ ! -f "${tsfile1}" ]; then return 0; fi
3342   if [ ! -f "${tsfile2}" ]; then return 0; fi
3343
3344   local MODIFIED=$(find "${tsfile1}" -newer "${tsfile2}")
3345   if [ ${#MODIFIED} -gt 0 ]; then
3346     return 0
3347   fi
3348   return 1
3349 }
3350
3351
3352 # Don't define any functions after this or they won't show up in completions
3353 function-completions() {
3354   if [ $# = 0 ]; then set -- ""; fi
3355   compgen -A function -- $1
3356   exit 0
3357 }
3358
3359 ######################################################################
3360 ######################################################################
3361 #
3362 #                               < MAIN >
3363 #
3364 ######################################################################
3365 ######################################################################
3366
3367 mkdir -p "${INSTALL_ROOT}"
3368
3369 if [ $# = 0 ]; then set -- help; fi  # Avoid reference to undefined $1.
3370
3371 # Accept one -- argument for some compatibility with google3
3372 if [ $1 = "--tab_completion_word" ]; then
3373   set -- function-completions $2
3374 fi
3375
3376 if [ "$(type -t $1)" != "function" ]; then
3377   #Usage
3378   echo "ERROR: unknown function '$1'." >&2
3379   echo "For help, try:"
3380   echo "    $0 help"
3381   exit 1
3382 fi
3383
3384 "$@"