src/native_client/pnacl/build.sh

   1 #!/bin/bash
   2 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5 #
   6 #@                 PNaCl toolchain build script
   7 #@-------------------------------------------------------------------
   8 #@ This script builds the ARM and PNaCl untrusted toolchains.
   9 #@ It MUST be run from the native_client/ directory.
  10 ######################################################################
  11 # Directory Layout Description
  12 ######################################################################
  13 #
  14 # All directories are relative to BASE which is
  15 # On Linux: native_client/toolchain/pnacl_linux_x86/
  16 # On Mac: native_client/toolchain/pnacl_mac_x86/
  17 # On Windows: native_client/toolchain/pnacl_win_x86/
  18 #
  19 ######################################################################
  20
  21 set -o nounset
  22 set -o errexit
  23
  24 PWD_ON_ENTRY="$(pwd)"
  25 # The script is located in "native_client/pnacl/".
  26 # Set pwd to pnacl/
  27 cd "$(dirname "$0")"
  28 if [[ $(basename "$(pwd)") != "pnacl" ]] ; then
  29   echo "ERROR: cannot find pnacl/ directory"
  30   exit -1
  31 fi
  32
  33 source scripts/common-tools.sh
  34
  35 readonly PNACL_ROOT="$(pwd)"
  36 readonly NACL_ROOT="$(GetAbsolutePath ..)"
  37 readonly SCONS_OUT="${NACL_ROOT}/scons-out"
  38
  39 SetScriptPath "${PNACL_ROOT}/build.sh"
  40 SetLogDirectory "${PNACL_ROOT}/build/log"
  41
  42 readonly TOOLCHAIN_BUILD="${NACL_ROOT}/toolchain_build/toolchain_build_pnacl.py"
  43
  44 # For different levels of make parallelism change this in your env
  45 readonly PNACL_CONCURRENCY=${PNACL_CONCURRENCY:-8}
  46 # Concurrency for builds using the host's system compiler (which might be goma)
  47 readonly PNACL_CONCURRENCY_HOST=${PNACL_CONCURRENCY_HOST:-${PNACL_CONCURRENCY}}
  48 PNACL_PRUNE=${PNACL_PRUNE:-false}
  49 PNACL_BUILD_ARM=true
  50 PNACL_BUILD_MIPS=${PNACL_BUILD_MIPS:-false}
  51
  52 if ${BUILD_PLATFORM_MAC} || ${BUILD_PLATFORM_WIN}; then
  53   # We don't yet support building ARM tools for mac or windows.
  54   PNACL_BUILD_ARM=false
  55   PNACL_BUILD_MIPS=false
  56 fi
  57
  58 # PNaCl builds libc++/libc++abi as well as libstdc++, allowing users to
  59 # choose which to use through the -stdlib=XXX command-line argument.
  60 #
  61 # The following strings are used for banner names as well as file and
  62 # folder names. These names are created by the libraries themselves, and
  63 # expected by their dependents. Changing them would be ill-advised.
  64 readonly LIB_CXX_NAME="libc++"
  65 readonly LIB_STDCPP_NAME="libstdc++"
  66
  67 # TODO(pdox): Decide what the target should really permanently be
  68 readonly CROSS_TARGET_ARM=arm-none-linux-gnueabi
  69 readonly BINUTILS_TARGET=arm-pc-nacl
  70 readonly REAL_CROSS_TARGET=le32-nacl
  71 readonly NACL64_TARGET=x86_64-nacl
  72
  73 readonly DRIVER_DIR="${PNACL_ROOT}/driver"
  74 readonly ARM_ARCH=armv7-a
  75 readonly ARM_FPU=vfp
  76
  77 readonly TOOLCHAIN_ROOT="${NACL_ROOT}/toolchain"
  78
  79 readonly NNACL_BASE="${TOOLCHAIN_ROOT}/${SCONS_BUILD_PLATFORM}_x86"
  80 readonly NNACL_NEWLIB_ROOT="${NNACL_BASE}_newlib"
  81 readonly NNACL_ARM_NEWLIB_ROOT="${TOOLCHAIN_ROOT}/linux_arm_newlib"
  82
  83 readonly PNACL_MAKE_OPTS="${PNACL_MAKE_OPTS:-}"
  84 readonly MAKE_OPTS="-j${PNACL_CONCURRENCY} VERBOSE=1 ${PNACL_MAKE_OPTS}"
  85 readonly MAKE_OPTS_HOST="-j${PNACL_CONCURRENCY_HOST} VERBOSE=1 ${PNACL_MAKE_OPTS}"
  86
  87 readonly NONEXISTENT_PATH="/going/down/the/longest/road/to/nowhere"
  88
  89 # For speculative build status output. ( see status function )
  90 # Leave this blank, it will be filled during processing.
  91 SPECULATIVE_REBUILD_SET=""
  92
  93 readonly PNACL_SUPPORT="${PNACL_ROOT}/support"
  94
  95 readonly THIRD_PARTY="${NACL_ROOT}"/../third_party
  96 readonly NACL_SRC_THIRD_PARTY_MOD="${NACL_ROOT}/src/third_party_mod"
  97
  98 # Git sources
  99 readonly PNACL_GIT_ROOT="${PNACL_ROOT}/git"
 100 readonly TC_SRC_BINUTILS="${PNACL_GIT_ROOT}/binutils"
 101 readonly TC_SRC_LLVM="${PNACL_GIT_ROOT}/llvm"
 102 readonly TC_SRC_GCC="${PNACL_GIT_ROOT}/gcc"
 103 readonly TC_SRC_NEWLIB="${PNACL_GIT_ROOT}/nacl-newlib"
 104 readonly TC_SRC_LIBSTDCPP="${TC_SRC_GCC}/${LIB_STDCPP_NAME}-v3"
 105 readonly TC_SRC_COMPILER_RT="${PNACL_GIT_ROOT}/compiler-rt"
 106 readonly TC_SRC_CLANG="${PNACL_GIT_ROOT}/clang"
 107 readonly TC_SRC_LIBCXX="${PNACL_GIT_ROOT}/libcxx"
 108
 109 readonly SERVICE_RUNTIME_SRC="${NACL_ROOT}/src/trusted/service_runtime"
 110 readonly EXPORT_HEADER_SCRIPT="${SERVICE_RUNTIME_SRC}/export_header.py"
 111 readonly NACL_SYS_HEADERS="${SERVICE_RUNTIME_SRC}/include"
 112 readonly NEWLIB_INCLUDE_DIR="${TC_SRC_NEWLIB}/newlib/libc/sys/nacl"
 113
 114 # The location of each project. These should be absolute paths.
 115 readonly TC_BUILD="${PNACL_ROOT}/build"
 116 readonly TC_BUILD_LLVM="${TC_BUILD}/llvm_${HOST_ARCH}"
 117 readonly TC_BUILD_BINUTILS="${TC_BUILD}/binutils_${HOST_ARCH}"
 118 readonly TC_BUILD_BINUTILS_LIBERTY="${TC_BUILD}/binutils-liberty"
 119 TC_BUILD_NEWLIB="${TC_BUILD}/newlib"
 120 readonly TC_BUILD_COMPILER_RT="${TC_BUILD}/compiler_rt"
 121 readonly TC_BUILD_GCC="${TC_BUILD}/gcc"
 122 readonly NACL_HEADERS_TS="${TC_BUILD}/nacl.sys.timestamp"
 123
 124 readonly TIMESTAMP_FILENAME="make-timestamp"
 125
 126 # PNaCl toolchain installation directories (absolute paths)
 127 readonly TOOLCHAIN_LABEL="${TOOLCHAIN_LABEL:-pnacl_${BUILD_PLATFORM}_x86}"
 128 readonly INSTALL_ROOT="${TOOLCHAIN_ROOT}/${TOOLCHAIN_LABEL}"
 129 readonly INSTALL_BIN="${INSTALL_ROOT}/bin"
 130
 131 # Bitcode lib directories (including static bitcode libs)
 132 INSTALL_LIB="${INSTALL_ROOT}/lib"
 133
 134 # Native nacl lib directories
 135 # The pattern `${INSTALL_LIB_NATIVE}${arch}' is used in many places.
 136 readonly INSTALL_LIB_NATIVE="${INSTALL_ROOT}/lib-"
 137 readonly INSTALL_LIB_ARM="${INSTALL_LIB_NATIVE}arm"
 138 readonly INSTALL_LIB_X8632="${INSTALL_LIB_NATIVE}x86-32"
 139 readonly INSTALL_LIB_X8664="${INSTALL_LIB_NATIVE}x86-64"
 140 readonly INSTALL_LIB_MIPS32="${INSTALL_LIB_NATIVE}mips32"
 141
 142 # PNaCl client-translators (sandboxed) binary locations
 143 readonly INSTALL_TRANSLATOR="${TOOLCHAIN_ROOT}/pnacl_translator"
 144
 145
 146 # The INSTALL_HOST directory has host binaries and libs which
 147 # are part of the toolchain (e.g. llvm and binutils).
 148 # There are also tools-x86 and tools-arm which have host binaries which
 149 # are not part of the toolchain but might be useful in the SDK, e.g.
 150 # arm sel_ldr and x86-hosted arm/mips validators.
 151 readonly INSTALL_HOST="${INSTALL_ROOT}/host_${HOST_ARCH}"
 152
 153 # Component installation directories
 154 readonly LLVM_INSTALL_DIR="${INSTALL_HOST}"
 155 readonly BINUTILS_INSTALL_DIR="${INSTALL_HOST}"
 156 readonly BFD_PLUGIN_DIR="${BINUTILS_INSTALL_DIR}/lib/bfd-plugins"
 157 readonly FAKE_INSTALL_DIR="${INSTALL_HOST}/fake"
 158 NEWLIB_INSTALL_DIR="${INSTALL_ROOT}/usr"
 159
 160 # Location of the PNaCl tools defined for configure invocations.
 161 readonly PNACL_CC="${INSTALL_BIN}/pnacl-clang"
 162 readonly PNACL_CXX="${INSTALL_BIN}/pnacl-clang++"
 163 readonly PNACL_LD="${INSTALL_BIN}/pnacl-ld"
 164 readonly PNACL_PP="${INSTALL_BIN}/pnacl-clang -E"
 165 readonly PNACL_AR="${INSTALL_BIN}/pnacl-ar"
 166 readonly PNACL_RANLIB="${INSTALL_BIN}/pnacl-ranlib"
 167 readonly PNACL_AS="${INSTALL_BIN}/pnacl-as"
 168 readonly PNACL_DIS="${INSTALL_BIN}/pnacl-dis"
 169 readonly PNACL_FINALIZE="${INSTALL_BIN}/pnacl-finalize"
 170 readonly PNACL_NM="${INSTALL_BIN}/pnacl-nm"
 171 readonly PNACL_TRANSLATE="${INSTALL_BIN}/pnacl-translate"
 172 readonly PNACL_READELF="${INSTALL_BIN}/pnacl-readelf"
 173 readonly PNACL_SIZE="${BINUTILS_INSTALL_DIR}/bin/${REAL_CROSS_TARGET}-size"
 174 readonly PNACL_STRIP="${INSTALL_BIN}/pnacl-strip"
 175 readonly ILLEGAL_TOOL="${INSTALL_BIN}"/pnacl-illegal
 176
 177 GetNNaClTool() {
 178   local arch=$1
 179   case ${arch} in
 180     x86-32) echo ${NNACL_NEWLIB_ROOT}/bin/i686-nacl-gcc ;;
 181     x86-64) echo ${NNACL_NEWLIB_ROOT}/bin/x86_64-nacl-gcc ;;
 182     arm) echo ${NNACL_ARM_NEWLIB_ROOT}/bin/arm-nacl-gcc ;;
 183     mips32)
 184       # No NNaCl for mips
 185       echo -n "${PNACL_CC} -arch mips32 --pnacl-bias=mips32 "
 186       echo "--pnacl-allow-translate --pnacl-allow-native" ;;
 187     *) Fatal "Unexpected argument to GetNNaClTool: $*" ;;
 188   esac
 189 }
 190
 191 # For a production (release) build, we want the sandboxed
 192 # translator to only contain the code needed to handle
 193 # its own architecture. For example, the translator shipped with
 194 # an X86-32 browser would only be able to translate to X86-32 code.
 195 # This is so that the translator binary is as small as possible.
 196 #
 197 # If SBTC_PRODUCTION is true, then the translators are built
 198 # separately, one for each architecture, so that each translator
 199 # can only target its own architecture.
 200 #
 201 # If SBTC_PRODUCTION is false, then we instead use PNaCl to
 202 # build a `fat` translator which can target all supported
 203 # architectures. This translator is built as a .pexe
 204 # which can then be translated to each individual architecture.
 205 SBTC_PRODUCTION=${SBTC_PRODUCTION:-true}
 206
 207 # Which arches to build for our sandboxed toolchain.
 208 SBTC_ARCHES_ALL=${SBTC_ARCHES_ALL:-"armv7 i686 x86_64"}
 209
 210 get-sbtc-llvm-arches() {
 211 # For LLVM i686 brings in both i686 and x86_64.  De-dupe that.
 212   echo ${SBTC_ARCHES_ALL} \
 213     | sed 's/x86_64/i686/' | sed 's/i686\(.*\)i686/i686\1/'
 214 }
 215 SBTC_ARCHES_LLVM=$(get-sbtc-llvm-arches)
 216
 217
 218 CC=${CC:-gcc}
 219 CXX=${CXX:-g++}
 220 AR=${AR:-ar}
 221 RANLIB=${RANLIB:-ranlib}
 222
 223 if ${HOST_ARCH_X8632}; then
 224   # These are simple compiler wrappers to force 32bit builds
 225   CC="${PNACL_ROOT}/scripts/mygcc32"
 226   CXX="${PNACL_ROOT}/scripts/myg++32"
 227 fi
 228
 229 # Set up some environment variables to build flavored bitcode libs
 230 setup-biased-bitcode-env() {
 231  local arch=$1
 232   case ${arch} in
 233     portable)
 234       BIASED_BC_CFLAGS=""
 235       NEWLIB_INSTALL_DIR="${INSTALL_ROOT}/usr"
 236       NEWLIB_TARGET=${REAL_CROSS_TARGET}
 237       TC_BUILD_NEWLIB="${TC_BUILD}/newlib-portable"
 238       INSTALL_LIB="${INSTALL_ROOT}/lib"
 239       LIB_CPP_BUILD="${TC_BUILD}/c++-stdlib-newlib-portable"
 240       LIB_CPP_INSTALL_DIR="${INSTALL_ROOT}/usr"
 241       ;;
 242     x86-64)
 243       BIASED_BC_CFLAGS="--target=x86_64-nacl"
 244       NEWLIB_INSTALL_DIR="${INSTALL_ROOT}/usr-bc-${arch}"
 245       # Do this to avoid .S files in newlib that clang doesn't like
 246       NEWLIB_TARGET="${REAL_CROSS_TARGET}"
 247       TC_BUILD_NEWLIB="${TC_BUILD}/newlib-${arch}"
 248       INSTALL_LIB="${INSTALL_ROOT}/lib-bc-${arch}"
 249       LIB_CPP_BUILD="${TC_BUILD}/c++-stdlib-newlib-${arch}"
 250       LIB_CPP_INSTALL_DIR="${INSTALL_ROOT}/usr-bc-${arch}"
 251       ;;
 252     *)
 253       echo "Newlib architectures other than portable and x86-64 not implemented yet"
 254       exit 1
 255   esac
 256 }
 257
 258 setup-lib-cpp-env() {
 259   # NOTE: we do not expect the assembler or linker to be used for libs
 260   #       hence the use of ILLEGAL_TOOL.
 261
 262   STD_ENV_FOR_LIB_CPP=(
 263     CC_FOR_BUILD="${CC}"
 264     CC="${PNACL_CC}"
 265     CXX="${PNACL_CXX}"
 266     RAW_CXX_FOR_TARGET="${PNACL_CXX}"
 267     LD="${ILLEGAL_TOOL}"
 268     CFLAGS="-g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS}"
 269     CXXFLAGS="-g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS}"
 270     CFLAGS_FOR_TARGET="-g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS}"
 271     CXXFLAGS_FOR_TARGET="-g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS}"
 272     CC_FOR_TARGET="${PNACL_CC}"
 273     GCC_FOR_TARGET="${PNACL_CC}"
 274     CXX_FOR_TARGET="${PNACL_CXX}"
 275     AR="${PNACL_AR}"
 276     AR_FOR_TARGET="${PNACL_AR}"
 277     NM_FOR_TARGET="${PNACL_NM}"
 278     RANLIB="${PNACL_RANLIB}"
 279     RANLIB_FOR_TARGET="${PNACL_RANLIB}"
 280     AS_FOR_TARGET="${ILLEGAL_TOOL}"
 281     LD_FOR_TARGET="${ILLEGAL_TOOL}"
 282     OBJDUMP_FOR_TARGET="${ILLEGAL_TOOL}" )
 283 }
 284
 285 setup-newlib-env() {
 286   local arch=$1
 287   setup-biased-bitcode-env ${arch}
 288
 289   STD_ENV_FOR_NEWLIB=(
 290     # TODO(robertm): get rid of '-allow-asm' here once we have a way of
 291     # distinguishing "good" from "bad" asms.
 292     CFLAGS_FOR_TARGET="-allow-asm -g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS}"
 293     CXXFLAGS_FOR_TARGET="-allow-asm -g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS}"
 294     CC_FOR_TARGET="${PNACL_CC}"
 295     GCC_FOR_TARGET="${PNACL_CC}"
 296     CXX_FOR_TARGET="${PNACL_CXX}"
 297     AR_FOR_TARGET="${PNACL_AR}"
 298     NM_FOR_TARGET="${PNACL_NM}"
 299     RANLIB_FOR_TARGET="${PNACL_RANLIB}"
 300     READELF_FOR_TARGET="${PNACL_READELF}"
 301     OBJDUMP_FOR_TARGET="${ILLEGAL_TOOL}"
 302     AS_FOR_TARGET="${ILLEGAL_TOOL}"
 303     LD_FOR_TARGET="${ILLEGAL_TOOL}"
 304     STRIP_FOR_TARGET="${ILLEGAL_TOOL}" )
 305 }
 306
 307 # The gold plugin that we use is documented at
 308 # http://llvm.org/docs/GoldPlugin.html
 309 # Despite its name it is actually used by both gold and bfd. The changes to
 310 # this file to enable its use are:
 311 # * Build shared
 312 # * --enable-gold and --enable-plugin when building binutils
 313 # * --with-binutils-include when building binutils
 314 # * linking the plugin in bfd-plugins
 315
 316 ######################################################################
 317 ######################################################################
 318 #
 319 #                     < USER ACCESSIBLE FUNCTIONS >
 320 #
 321 ######################################################################
 322 ######################################################################
 323
 324 # Convert a path given on the command-line to an absolute path.
 325 # This takes into account the fact that we changed directories at the
 326 # beginning of this script. PWD_ON_ENTRY is used to remember the
 327 # initial working directory.
 328 ArgumentToAbsolutePath() {
 329   local relpath="$1"
 330   local savepwd="$(pwd)"
 331   cd "${PWD_ON_ENTRY}"
 332   GetAbsolutePath "${relpath}"
 333   cd "${savepwd}"
 334 }
 335
 336 #@ sync-sources          - check out repos needed to build toolchain
 337 sync-sources() {
 338   StepBanner "SYNC SOURCES"
 339   python ${TOOLCHAIN_BUILD} --legacy-repo-sync
 340   newlib-nacl-headers
 341 }
 342
 343 git-sync() {
 344   local gitbase="${PNACL_GIT_ROOT}"
 345   # Disable depot tools auto-update for these invocations. (It
 346   # will still get updated when it is invoked at the start of each build,
 347   # and we don't want it updating from cygwin here)
 348   export DEPOT_TOOLS_UPDATE=0
 349
 350   mkdir -p "${gitbase}"
 351   cp "${PNACL_ROOT}"/gclient_template "${gitbase}/.gclient"
 352
 353   if ! [ -d "${gitbase}/dummydir" ]; then
 354     spushd "${gitbase}"
 355     ${GCLIENT} update --verbose -j1
 356     spopd
 357   fi
 358
 359   newlib-nacl-headers-clean
 360   cp "${PNACL_ROOT}"/DEPS "${gitbase}"/dummydir
 361   spushd "${gitbase}"
 362   ${GCLIENT} update --verbose -j1
 363   spopd
 364
 365   # Copy nacl headers into newlib tree.
 366   newlib-nacl-headers
 367 }
 368
 369 gerrit-upload() {
 370   # Upload to gerrit code review
 371   # (Note: normally we use Rietveld code review. This is only useful if
 372   # e.g. we want external users to use gerrit instead of rietveld so we can
 373   # easily merge their commits without giving them direct access). But
 374   # this at least documents the specifics of uploading to our repo
 375   # TODO(dschuff): Find a good way to persist the user's login. maybe
 376   # do something like what depot tools does
 377   if ! [ -f ~/.gerrit_username ]; then
 378     echo "Enter your gerrit username: "
 379     read user
 380     echo "Saving gerrit username (${user}) to ~/.gerrit_username"
 381     echo ${user} > ~/.gerrit_username
 382   else
 383     local user=$(cat ~/.gerrit_username)
 384   fi
 385   cd "${TC_SRC_LLVM}"
 386   git push ssh://${user}@gerrit.chromium.org:29418/native_client/pnacl-llvm \
 387     HEAD:refs/for/master
 388 }
 389
 390 #@-------------------------------------------------------------------------
 391
 392 #@ download-trusted      - Download and Install trusted SDKs (arm,x86-32,x86-64)
 393 #@                         (your untrusted build will not be modified)
 394 download-trusted() {
 395   StepBanner "DOWNLOAD-TRUSTED" "Downloading trusted toolchains"
 396   local installdir="${INSTALL_ROOT}"
 397   local tmpdir="${installdir}-backup"
 398   local dldir="${installdir}-downloaded"
 399
 400   rm -rf "${dldir}"
 401
 402   if [ -d "${tmpdir}" ]; then
 403     echo "ERROR: It is not expected that directory '${tmpdir}' exists."
 404     echo "       Please delete it if you don't need it"
 405     exit -1
 406   fi
 407
 408   if [ -d "${installdir}" ]; then
 409     mv "${installdir}" "${tmpdir}"
 410   fi
 411
 412   download-toolchains
 413
 414   if [ -d "${installdir}" ]; then
 415     mv "${installdir}" "${dldir}"
 416   fi
 417
 418   if [ -d "${tmpdir}" ]; then
 419     mv "${tmpdir}" "${installdir}"
 420   fi
 421 }
 422
 423 #@-------------------------------------------------------------------------
 424
 425 #@ download-toolchains   - Download and Install all SDKs (arm,x86-32,x86-64)
 426
 427 download-toolchains() {
 428   ${GCLIENT} runhooks --force --verbose
 429 }
 430
 431 #@ libs                  - install native libs and build bitcode libs
 432 libs() {
 433   libs-clean
 434   newlib portable
 435   newlib x86-64
 436   libs-support
 437   for arch in arm x86-32 x86-64 mips32; do
 438     dummy-irt-shim ${arch}
 439   done
 440   compiler-rt-all
 441   libgcc_eh-all
 442   lib-cpp ${LIB_CXX_NAME} portable
 443   lib-cpp ${LIB_CXX_NAME} x86-64
 444   lib-cpp ${LIB_STDCPP_NAME} portable
 445   lib-cpp ${LIB_STDCPP_NAME} x86-64
 446 }
 447
 448 #@ everything            - Build and install untrusted SDK. no translator
 449 everything() {
 450   sync-sources
 451   build-all
 452 }
 453
 454 #@ build-all does everything AFTER getting the sources
 455 build-all() {
 456   mkdir -p "${INSTALL_ROOT}"
 457   # This is needed to build misc-tools and run ARM tests.
 458   # We check this early so that there are no surprises later, and we can
 459   # handle all user interaction early.
 460   check-for-trusted
 461
 462   clean-install
 463
 464   clean-logs
 465
 466   build-host
 467
 468   driver
 469
 470   libs
 471
 472   # NOTE: we delay the tool building till after the sdk is essentially
 473   #      complete, so that sdk sanity checks don't fail
 474   misc-tools
 475   cp "${PNACL_ROOT}/README" "${INSTALL_ROOT}"
 476   verify
 477
 478   if ${PNACL_PRUNE}; then
 479     prune
 480   fi
 481 }
 482
 483 #@ Build just the host binaries
 484 build-host() {
 485   binutils
 486   llvm
 487   if ${PNACL_PRUNE}; then
 488     prune-host
 489   fi
 490 }
 491
 492 #@ everything-translator   - Build and install untrusted SDK AND translator
 493 everything-translator() {
 494   everything
 495   translator-all
 496   driver-install-translator
 497 }
 498
 499 #@ translator-archive-pexes <tarball> -archive gold and llc pexes
 500 #@                      <tarball> should be an absolute pathname,
 501 #@                      and should have tar.bz2 as the suffix (as that is
 502 #@                      what will be produced).
 503 #@
 504 #@                      This must run before pexes are pruned.
 505 translator-archive-pexes() {
 506   local tarball=$1
 507
 508   if [[ "${tarball#*.}" != "tar.bz2" ]]; then
 509     echo "translator-archive-pexes: ${tarball} not named with .tar.bz2 suffix"
 510     exit 1
 511   fi
 512   local tarball_no_bz2=${tarball%.*}
 513
 514   # Clear tarball before appending files to it.
 515   rm -f ${tarball}
 516
 517   # Archive LD
 518   for arch in ${SBTC_ARCHES_ALL} ; do
 519     local pexe_dir="$(GetTranslatorInstallDir ${arch})/bin"
 520     echo "pexe_dir is ${pexe_dir}"
 521     # Label the pexes by architecture.
 522     local pexe_ld="${pexe_dir}/ld.${arch}"
 523     cp "${pexe_dir}/ld.pexe" "${pexe_ld}.nonfinal.pexe"
 524     ${PNACL_FINALIZE} "${pexe_ld}.nonfinal.pexe" -o "${pexe_ld}.final.pexe"
 525     local all="${pexe_ld}.final.pexe"
 526     file ${all}
 527     ls -l ${all}
 528     # strip all path components
 529     tar rf ${tarball_no_bz2}  --transform 's!^.*/!!' ${all}
 530   done
 531
 532   # Archive LLC (which combines x86 and x86_64).
 533   for arch in ${SBTC_ARCHES_LLVM} ; do
 534     local pexe_dir="$(GetTranslatorInstallDir ${arch})/bin"
 535     echo "pexe_dir is ${pexe_dir}"
 536     # Label the pexes by architecture.
 537     local pexe_llc="${pexe_dir}/pnacl-llc.${arch}"
 538     cp "${pexe_dir}/pnacl-llc.pexe" "${pexe_llc}.nonfinal.pexe"
 539     ${PNACL_FINALIZE} "${pexe_llc}.nonfinal.pexe" \
 540       -o "${pexe_llc}.final.pexe"
 541     local all="${pexe_llc}.final.pexe"
 542     file ${all}
 543     ls -l ${all}
 544     # strip all path components
 545     tar rf ${tarball_no_bz2}  --transform 's!^.*/!!' ${all}
 546   done
 547   bzip2 ${tarball_no_bz2}
 548 }
 549
 550
 551 #+ translator-clean-all  - Clean all translator install/build directories
 552 translator-clean-all() {
 553   StepBanner "TRANSLATOR" "Clean all"
 554   rm -rf "${TC_BUILD}"/translator*
 555   rm -rf "${INSTALL_TRANSLATOR}"*
 556 }
 557
 558 #@ translator-all   -  Build and install all of the translators.
 559 translator-all() {
 560   StepBanner \
 561     "SANDBOXED TC [prod=${SBTC_PRODUCTION}] [arches=${SBTC_ARCHES_ALL}]"
 562
 563   # Build the SDK if it not already present.
 564   sdk
 565   # Also build private libs to allow building nexes without the IRT
 566   # segment gap.  Specifically, only the sandboxed translator nexes
 567   # are built without IRT support to gain address space and reduce
 568   # swap file usage. Also libsrpc and its dependencies are now considered
 569   # private libs because they are not in the real SDK
 570   sdk-private-libs
 571
 572   binutils-liberty
 573   if ${SBTC_PRODUCTION}; then
 574     # Build each architecture separately.
 575     local arch
 576     for arch in ${SBTC_ARCHES_LLVM} ; do
 577       llvm-sb ${arch}
 578     done
 579     for arch in ${SBTC_ARCHES_ALL} ; do
 580       binutils-gold-sb ${arch}
 581     done
 582   else
 583     # Using arch `universal` builds the sandboxed tools from a single
 584     # .pexe which support all targets.
 585     llvm-sb universal
 586     binutils-gold-sb universal
 587     if ${PNACL_PRUNE}; then
 588       # The universal pexes have already been translated.
 589       # They don't need to stick around.
 590       rm -rf "$(GetTranslatorInstallDir universal)"
 591     fi
 592   fi
 593
 594   # Copy native libs to translator install dir.
 595   cp -a ${INSTALL_LIB_NATIVE}* ${INSTALL_TRANSLATOR}
 596
 597   driver-install-translator
 598
 599   if ${PNACL_PRUNE}; then
 600     sdk-clean newlib
 601   fi
 602 }
 603
 604
 605 #+ translator-prune    Remove leftover files like pexes from pnacl_translator
 606 #+                     build and from translator-archive-pexes.
 607 translator-prune() {
 608   find "${INSTALL_TRANSLATOR}" -name "*.pexe" -exec "rm" {} +
 609 }
 610
 611
 612 #+ translator-clean <arch> -
 613 #+     Clean one translator install/build
 614 translator-clean() {
 615   local arch=$1
 616   StepBanner "TRANSLATOR" "Clean ${arch}"
 617   rm -rf "$(GetTranslatorInstallDir ${arch})"
 618   rm -rf "$(GetTranslatorBuildDir ${arch})"
 619 }
 620
 621 #@ all                   - Alias for 'everything'
 622 all() {
 623   everything
 624 }
 625
 626 #@ status                - Show status of build directories
 627 status() {
 628   # TODO(robertm): this is currently broken
 629   StepBanner "BUILD STATUS"
 630
 631   status-helper "BINUTILS"             binutils
 632   status-helper "LLVM"                 llvm
 633
 634   status-helper "NEWLIB"               newlib
 635   status-helper "C++ Standard Library" lib-cpp
 636 }
 637
 638 status-helper() {
 639   local title="$1"
 640   local mod="$2"
 641
 642   if ${mod}-needs-configure; then
 643     StepBanner "$title" "NEEDS FULL REBUILD"
 644     speculative-add "${mod}"
 645   elif ${mod}-needs-make; then
 646     StepBanner "$title" "NEEDS MAKE (INCREMENTAL)"
 647     speculative-add "${mod}"
 648   else
 649     StepBanner "$title" "OK (UP TO DATE)"
 650   fi
 651 }
 652
 653 speculative-add() {
 654   local mod="$1"
 655   SPECULATIVE_REBUILD_SET="${SPECULATIVE_REBUILD_SET} ${mod}"
 656 }
 657
 658 speculative-check() {
 659   local mod="$1"
 660   local search=$(echo "${SPECULATIVE_REBUILD_SET}" | grep -F "$mod")
 661   [ ${#search} -gt 0 ]
 662   return $?
 663 }
 664
 665
 666
 667 #@ clean                 - Clean the build and install directories.
 668 clean() {
 669   StepBanner "CLEAN" "Cleaning build, log, and install directories."
 670
 671   clean-logs
 672   clean-build
 673   clean-install
 674   clean-scons
 675 }
 676
 677 #@ fast-clean            - Clean everything except LLVM.
 678 fast-clean() {
 679   local did_backup=false
 680   local backup_dir="${PNACL_ROOT}/fast-clean-llvm-backup"
 681
 682   if [ -d "${TC_BUILD_LLVM}" ]; then
 683     rm -rf "${backup_dir}"
 684     mv "${TC_BUILD_LLVM}" "${backup_dir}"
 685     did_backup=true
 686   fi
 687
 688   clean
 689
 690   if ${did_backup} ; then
 691     mkdir -p "${TC_BUILD}"
 692     mv "${backup_dir}" "${TC_BUILD_LLVM}"
 693   fi
 694 }
 695
 696 #+ clean-scons           - Clean scons-out directory
 697 clean-scons() {
 698   rm -rf "${SCONS_OUT}"
 699 }
 700
 701 #+ clean-build           - Clean all build directories
 702 clean-build() {
 703   rm -rf "${TC_BUILD}"
 704 }
 705
 706 #+ clean-install         - Clean install directories
 707 clean-install() {
 708   rm -rf "${INSTALL_ROOT}"
 709 }
 710
 711 #+ libs-clean            - Removes the library directories
 712 libs-clean() {
 713   StepBanner "LIBS-CLEAN" "Cleaning ${INSTALL_ROOT}/libs-*"
 714   rm -rf "${INSTALL_LIB}"/*
 715   rm -rf "${INSTALL_LIB_NATIVE}"*
 716 }
 717
 718
 719 #@-------------------------------------------------------------------------
 720
 721 prune-host() {
 722   echo "stripping binaries (binutils)"
 723   strip "${BINUTILS_INSTALL_DIR}"/bin/*
 724
 725   echo "stripping binaries (llvm)"
 726   if ! strip "${LLVM_INSTALL_DIR}"/bin/* ; then
 727     echo "NOTE: some failures during stripping are expected"
 728   fi
 729
 730   echo "removing unused clang shared lib"
 731   rm -rf "${LLVM_INSTALL_DIR}"/${SO_DIR}/*clang${SO_EXT}
 732
 733   echo "removing unused binutils binaries"
 734   rm -rf "${LLVM_INSTALL_DIR}"/bin/le32-nacl-elfedit
 735   rm -rf "${LLVM_INSTALL_DIR}"/bin/le32-nacl-gprof
 736   rm -rf "${LLVM_INSTALL_DIR}"/bin/le32-nacl-objcopy
 737
 738   echo "removing unused LLVM/Clang binaries"
 739   rm -rf "${LLVM_INSTALL_DIR}"/bin/bc-wrap
 740   rm -rf "${LLVM_INSTALL_DIR}"/bin/bugpoint
 741   rm -rf "${LLVM_INSTALL_DIR}"/bin/c-index-test
 742   rm -rf "${LLVM_INSTALL_DIR}"/bin/clang-*
 743   rm -rf "${LLVM_INSTALL_DIR}"/bin/llc
 744   rm -rf "${LLVM_INSTALL_DIR}"/bin/lli
 745   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-ar
 746   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-bcanalyzer
 747   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-config
 748   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-cov
 749   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-diff
 750   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-dwarfdump
 751   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-extract
 752   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-mcmarkup
 753   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-prof
 754   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-ranlib
 755   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-readobj
 756   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-rtdyld
 757   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-size
 758   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-stress
 759   rm -rf "${LLVM_INSTALL_DIR}"/bin/llvm-symbolizer
 760   rm -rf "${LLVM_INSTALL_DIR}"/bin/macho-dump
 761   rm -rf "${LLVM_INSTALL_DIR}"/bin/pso-stub
 762   rm -rf "${LLVM_INSTALL_DIR}"/bin/*-tblgen
 763
 764   echo "removing llvm & clang headers"
 765   rm -rf "${LLVM_INSTALL_DIR}"/include
 766
 767   echo "removing docs/ and share/"
 768   rm -rf "${LLVM_INSTALL_DIR}"/docs
 769   rm -rf "${LLVM_INSTALL_DIR}"/share
 770
 771   echo "removing unused libs"
 772   rm -rf "${LLVM_INSTALL_DIR}"/lib/*.a
 773   rm -rf "${LLVM_INSTALL_DIR}"/lib/bfd-plugins
 774   rm -rf "${LLVM_INSTALL_DIR}"/lib/BugpointPasses.so
 775   rm -rf "${LLVM_INSTALL_DIR}"/lib/LLVMHello.so
 776 }
 777
 778 #+ prune                 - Prune toolchain
 779 prune() {
 780   StepBanner "PRUNE" "Pruning toolchain"
 781   local dir_size_before=$(get_dir_size_in_mb ${INSTALL_ROOT})
 782
 783   SubBanner "Size before: ${INSTALL_ROOT} ${dir_size_before}MB"
 784
 785   prune-host
 786
 787   echo "removing .pyc files"
 788   rm -f "${INSTALL_BIN}"/pydir/*.pyc
 789
 790   local dir_size_after=$(get_dir_size_in_mb "${INSTALL_ROOT}")
 791   SubBanner "Size after: ${INSTALL_ROOT} ${dir_size_after}MB"
 792 }
 793
 794 #+ tarball <filename>    - Produce tarball file
 795 tarball() {
 796   if [ ! -n "${1:-}" ]; then
 797     echo "Error: tarball needs a tarball name." >&2
 798     exit 1
 799   fi
 800   local tarball="$(ArgumentToAbsolutePath "$1")"
 801   StepBanner "TARBALL" "Creating tar ball ${tarball}"
 802   tar zcf "${tarball}" -C "${INSTALL_ROOT}" .
 803   ls -l ${tarball}
 804 }
 805
 806 translator-tarball() {
 807   if [ ! -n "${1:-}" ]; then
 808     echo "Error: tarball needs a tarball name." >&2
 809     exit 1
 810   fi
 811   local tarball="$(ArgumentToAbsolutePath "$1")"
 812   StepBanner "TARBALL" "Creating translator tar ball ${tarball}"
 813   tar zcf "${tarball}" -C "${INSTALL_TRANSLATOR}" .
 814 }
 815
 816 #########################################################################
 817 #                              < LLVM >
 818 #########################################################################
 819
 820
 821 #+-------------------------------------------------------------------------
 822 #+ llvm                  - Configure, build and install LLVM.
 823
 824 llvm() {
 825   StepBanner "LLVM (${HOST_ARCH} HOST)"
 826
 827   local srcdir="${TC_SRC_LLVM}"
 828
 829   assert-dir "${srcdir}" "You need to checkout LLVM."
 830
 831   if llvm-needs-configure; then
 832     llvm-clean
 833     llvm-configure
 834   else
 835     SkipBanner "LLVM" "configure"
 836   fi
 837
 838   if llvm-needs-make; then
 839     llvm-make
 840   else
 841     SkipBanner "LLVM" "make"
 842   fi
 843
 844   llvm-install
 845 }
 846
 847 #+ llvm-clean            - Clean LLVM completely
 848 llvm-clean() {
 849   StepBanner "LLVM" "Clean"
 850   local objdir="${TC_BUILD_LLVM}"
 851   rm -rf ${objdir}
 852   mkdir -p ${objdir}
 853 }
 854
 855 #+ llvm-link-clang       - Add tools/clang symlink into llvm directory
 856 llvm-link-clang() {
 857   rm -f "${TC_SRC_LLVM}"/tools/clang
 858   # Symbolic link named : ${TC_SRC_LLVM}/tools/clang
 859   # Needs to point to   : ${TC_SRC_CLANG}
 860   ln -sf "${TC_SRC_CLANG}" "${TC_SRC_LLVM}"/tools/clang
 861 }
 862
 863 #+ llvm-unlink-clang     - Remove tools/clang symlink from llvm directory
 864 llvm-unlink-clang() {
 865   if [ -d "${TC_SRC_LLVM}" ]; then
 866     rm -f "${TC_SRC_LLVM}"/tools/clang
 867   fi
 868 }
 869
 870 # Default case - Optimized configure
 871 LLVM_EXTRA_OPTIONS="--enable-optimized"
 872
 873 #+ llvm-configure        - Run LLVM configure
 874 llvm-configure() {
 875   StepBanner "LLVM" "Configure"
 876
 877   local srcdir="${TC_SRC_LLVM}"
 878   local objdir="${TC_BUILD_LLVM}"
 879
 880   mkdir -p "${objdir}"
 881   spushd "${objdir}"
 882
 883   # Some components like ARMDisassembler may time out when built with -O3.
 884   # If we ever start using MSVC, we may also need to tone down the opt level
 885   # (see the settings in the CMake file for those components).
 886   local llvm_extra_opts=${LLVM_EXTRA_OPTIONS}
 887   if ${BUILD_PLATFORM_MAC}; then
 888     llvm_extra_opts="${LLVM_EXTRA_OPTIONS} --with-optimize-option=-O2"
 889   fi
 890
 891   llvm-link-clang
 892   # The --with-binutils-include is to allow llvm to build the gold plugin
 893   # re: --enable-targets  "x86" brings in both i686 and x86_64.
 894   # Disabling zlib features for now to reduce set of shared lib deps.
 895   # This may disable things like compressed debug info:
 896   # https://code.google.com/p/nativeclient/issues/detail?id=3592
 897   local binutils_include="${TC_SRC_BINUTILS}/include"
 898   RunWithLog "llvm.configure" \
 899       env -i PATH="${PATH}" \
 900              MAKE_OPTS=${MAKE_OPTS_HOST} \
 901              CC="${CC}" \
 902              CXX="${CXX}" \
 903              ${srcdir}/configure \
 904              --enable-shared \
 905              --enable-zlib=no \
 906              --disable-jit \
 907              --with-binutils-include=${binutils_include} \
 908              --enable-targets=x86,arm,mips \
 909              --prefix="${LLVM_INSTALL_DIR}" \
 910              --program-prefix= \
 911              ${llvm_extra_opts}
 912
 913
 914   spopd
 915 }
 916
 917 #+ llvm-configure-ninja - Configure with cmake for ninja build
 918 # Not used by default. Call manually. Pass build type (Release or Debug)
 919 # as an argument; The default is Release.
 920 llvm-configure-ninja() {
 921   StepBanner "LLVM" "Configure (Cmake-ninja)"
 922
 923   local srcdir="${TC_SRC_LLVM}"
 924   local objdir="${TC_BUILD_LLVM}"
 925   local buildtype="${1:-Release}"
 926
 927   mkdir -p "${objdir}"
 928   spushd "${objdir}"
 929
 930   llvm-link-clang
 931   local binutils_include="${TC_SRC_BINUTILS}/include"
 932   # Disabling zlib features for now to reduce set of shared lib deps.
 933   # This may disable things like compressed debug info:
 934   # https://code.google.com/p/nativeclient/issues/detail?id=3592
 935   RunWithLog "llvm.configure.cmake" \
 936     env \
 937       cmake -G Ninja \
 938       -DCMAKE_BUILD_TYPE=${buildtype} \
 939       -DCMAKE_INSTALL_PREFIX="${LLVM_INSTALL_DIR}" \
 940       -DCMAKE_INSTALL_RPATH='$ORIGIN/../lib' \
 941       -DBUILD_SHARED_LIBS=ON \
 942       -DLLVM_TARGETS_TO_BUILD="X86;ARM;Mips" \
 943       -DLLVM_ENABLE_ASSERTIONS=ON \
 944       -ULLVM_ENABLE_ZLIB \
 945       -DLLVM_BUILD_TESTS=ON \
 946       -DLLVM_APPEND_VC_REV=ON \
 947       -DLLVM_BINUTILS_INCDIR="${binutils_include}" \
 948       ${srcdir}
 949   spopd
 950 }
 951
 952 #+ llvm-configure-dbg        - Run LLVM configure
 953 #  Not used by default. Call manually.
 954 llvm-configure-dbg() {
 955   StepBanner "LLVM" "Configure With Debugging"
 956   LLVM_EXTRA_OPTIONS="--disable-optimized \
 957         --enable-debug-runtime \
 958         --enable-assertions "
 959   llvm-configure
 960 }
 961
 962
 963 llvm-needs-configure() {
 964   [ ! -f "${TC_BUILD_LLVM}/config.status" \
 965     -a ! -f "${TC_BUILD_LLVM}/build.ninja" ]
 966   return $?
 967 }
 968
 969 llvm-needs-make() {
 970   local objdir="${TC_BUILD_LLVM}"
 971   ts-modified "${TC_SRC_LLVM}" "${objdir}" ||
 972     ts-modified "${TC_SRC_CLANG}" "${objdir}"
 973   return $?
 974 }
 975
 976 #+ llvm-make             - Run LLVM 'make'
 977 llvm-make() {
 978   StepBanner "LLVM" "Make"
 979
 980   local srcdir="${TC_SRC_LLVM}"
 981   local objdir="${TC_BUILD_LLVM}"
 982   llvm-link-clang
 983
 984   spushd "${objdir}"
 985
 986   ts-touch-open "${objdir}"
 987
 988   if [ -f "${TC_BUILD_LLVM}/build.ninja" ]; then
 989     if [ -f "${TC_BUILD_LLVM}/config.status" ]; then
 990       echo "ERROR: Found multiple build system files in ${TC_BUILD_LLVM}"
 991       exit 1
 992     fi
 993     echo "Using ninja"
 994     ninja
 995   else
 996     RunWithLog llvm.make \
 997       env -i PATH="${PATH}" \
 998       MAKE_OPTS="${MAKE_OPTS_HOST}" \
 999       NACL_SANDBOX=0 \
1000       CC="${CC}" \
1001       CXX="${CXX}" \
1002       make ${MAKE_OPTS_HOST} all
1003   fi
1004
1005   ts-touch-commit  "${objdir}"
1006
1007   spopd
1008 }
1009
1010 #+ llvm-install          - Install LLVM
1011 llvm-install() {
1012   StepBanner "LLVM" "Install"
1013
1014   spushd "${TC_BUILD_LLVM}"
1015   llvm-link-clang
1016   if [ -f "${TC_BUILD_LLVM}/build.ninja" ]; then
1017     echo "Using ninja"
1018     RunWithLog llvm.install ninja install
1019   else
1020     RunWithLog llvm.install \
1021       env -i PATH=/usr/bin/:/bin \
1022       MAKE_OPTS="${MAKE_OPTS}" \
1023       NACL_SANDBOX=0 \
1024       CC="${CC}" \
1025       CXX="${CXX}" \
1026       make ${MAKE_OPTS} install
1027   fi
1028   spopd
1029
1030   # This is really part of libgcc_eh, but gets blown away by the ninja build.
1031   install-unwind-header
1032 }
1033
1034 #########################################################################
1035 #########################################################################
1036 #     < LIBGCC_EH >
1037 #########################################################################
1038 #########################################################################
1039
1040 libgcc_eh-all() {
1041   StepBanner "LIBGCC_EH (from GCC 4.6)"
1042
1043   libgcc_eh arm
1044   libgcc_eh mips32
1045   libgcc_eh x86-32
1046   libgcc_eh x86-64
1047 }
1048
1049 libgcc_eh-setup() {
1050   local arch=$1
1051   local flags=$2
1052   # For x86 we use nacl-gcc to build libgcc_eh because of some issues with
1053   # LLVM's handling of the gcc intrinsics used in the library. See
1054   # https://code.google.com/p/nativeclient/issues/detail?id=1933
1055   # and http://llvm.org/bugs/show_bug.cgi?id=8541
1056   # For ARM, LLVM does work and we use it to avoid dealing with the fact that
1057   # arm-nacl-gcc uses different libgcc support functions than PNaCl.
1058   if [ ${arch} == "arm" ]; then
1059     LIBGCC_EH_ENV=(
1060       CC="${PNACL_CC} ${flags} -arch ${arch} --pnacl-bias=${arch} \
1061          --pnacl-allow-translate --pnacl-allow-native" \
1062       AR="${PNACL_AR}" \
1063       NM="${PNACL_NM}" \
1064       RANLIB="${PNACL_RANLIB}")
1065   else
1066     LIBGCC_EH_ENV=(
1067       CC="$(GetNNaClTool ${arch}) ${flags}" \
1068       AR="${PNACL_AR}" \
1069       NM="${PNACL_NM}" \
1070       RANLIB="${PNACL_RANLIB}")
1071   fi
1072 }
1073
1074 libgcc_eh() {
1075   local arch=$1
1076
1077   local objdir="${TC_BUILD}/libgcc_eh-${arch}-newlib"
1078   local subdir="${objdir}/fake-target/libgcc"
1079   local installdir="${INSTALL_LIB_NATIVE}${arch}"
1080   local label="(${arch} libgcc_eh.a)"
1081
1082   mkdir -p "${installdir}"
1083   rm -rf "${installdir}"/libgcc_eh*
1084   rm -rf "${objdir}"
1085
1086   # Setup fake gcc build directory.
1087   mkdir -p "${objdir}"/gcc
1088   cp -a "${PNACL_ROOT}"/scripts/libgcc-newlib.mvars \
1089         "${objdir}"/gcc/libgcc.mvars
1090   cp -a "${PNACL_ROOT}"/scripts/libgcc-tconfig.h "${objdir}"/gcc/tconfig.h
1091   touch "${objdir}"/gcc/tm.h
1092
1093   install-unwind-header
1094
1095   mkdir -p "${subdir}"
1096   spushd "${subdir}"
1097   local flags=""
1098   flags+=" -DENABLE_RUNTIME_CHECKING"
1099
1100   libgcc_eh-setup ${arch} "${flags}"
1101
1102   StepBanner "LIBGCC_EH" "Configure ${label}"
1103   RunWithLog libgcc.${arch}.configure \
1104     env -i \
1105       PATH="/usr/bin:/bin" \
1106       /bin/sh \
1107       "${TC_SRC_GCC}"/libgcc/configure \
1108         "${LIBGCC_EH_ENV[@]}" \
1109         --prefix="${FAKE_INSTALL_DIR}" \
1110         --enable-shared \
1111         --host=i686-nacl
1112         # --enable-shared is needed because the EH functions end
1113         # up in libgcc.a if shared libs are disabled.
1114         #
1115         # The "--host" field is needed to enable cross-compiling.
1116         # It introduces extra configuration files in libgcc/config, but these
1117         # appear to affect libgcc, not libgcc_eh.
1118         # TODO(pdox): Create a fake target to get rid of i686 here.
1119
1120   StepBanner "LIBGCC_EH" "Make ${label}"
1121   RunWithLog libgcc.${arch}.make make libgcc_eh.a
1122   spopd
1123
1124   StepBanner "LIBGCC_EH" "Install ${label}"
1125   cp ${subdir}/libgcc_eh.a "${installdir}"
1126 }
1127
1128 install-unwind-header() {
1129   # TODO(pdox):
1130   # We need to establish an unwind ABI, since this is part of the ABI
1131   # exposed to the bitcode by the translator. This header should not vary
1132   # across compilers or C libraries.
1133   local install="/usr/bin/install -c -m 644"
1134   ${install} ${TC_SRC_GCC}/gcc/unwind-generic.h \
1135              ${LLVM_INSTALL_DIR}/lib/clang/3.3/include/unwind.h
1136 }
1137
1138 #########################################################################
1139 #########################################################################
1140 #     < COMPILER-RT >
1141 #########################################################################
1142 #########################################################################
1143
1144 compiler-rt-all() {
1145   StepBanner "COMPILER-RT (LIBGCC)"
1146   compiler-rt arm
1147   compiler-rt mips32
1148   compiler-rt x86-32
1149   compiler-rt x86-64
1150 }
1151
1152
1153 #+ compiler-rt           - build/install llvm's replacement for libgcc.a
1154 compiler-rt() {
1155   local arch=$1
1156   local src="${TC_SRC_COMPILER_RT}/lib"
1157   local objdir="${TC_BUILD_COMPILER_RT}-${arch}"
1158   local installdir="${INSTALL_LIB_NATIVE}${arch}"
1159   StepBanner "compiler rt" "build (${arch})"
1160
1161   rm -rf "${objdir}"
1162   mkdir -p "${objdir}"
1163
1164   spushd "${objdir}"
1165   RunWithLog libgcc.${arch}.make \
1166       make -j ${PNACL_CONCURRENCY} -f ${src}/Makefile-pnacl libgcc.a \
1167         CC="${PNACL_CC}" \
1168         AR="${PNACL_AR}" \
1169         "SRC_DIR=${src}" \
1170         "CFLAGS=-arch ${arch} -DPNACL_${arch} --pnacl-allow-translate -O3"
1171
1172   StepBanner "compiler rt" "install (${arch})"
1173   mkdir -p "${installdir}"
1174   cp libgcc.a "${installdir}"
1175   spopd
1176 }
1177
1178 #########################################################################
1179 #########################################################################
1180 #                          < LIBSTDC++/LIBC++ >
1181 #########################################################################
1182 #########################################################################
1183
1184 check-lib-cpp() {
1185   local lib=$1
1186   if [ ${lib} != ${LIB_CXX_NAME} ] && [ ${lib} != ${LIB_STDCPP_NAME} ]; then
1187     echo "ERROR: Unsupported C++ Standard Library '${lib}'. Choose one of: ${LIB_CXX_NAME}, ${LIB_STDCPP_NAME}"
1188     exit -1
1189   fi
1190 }
1191
1192 LIB_CPP_IS_SETUP=false
1193 lib-cpp-setup() {
1194   if ${LIB_CPP_IS_SETUP} && [ $# -eq 0 ]; then
1195     return 0
1196   fi
1197   local arch=$1
1198   setup-biased-bitcode-env ${arch}
1199   LIB_CPP_IS_SETUP=true
1200 }
1201
1202 lib-cpp() {
1203   local lib=$1
1204   local arch=$2
1205   check-lib-cpp "${lib}"
1206   lib-cpp-setup "${arch}"
1207   StepBanner "C++ Standard Library" "(BITCODE $*)"
1208
1209   if lib-cpp-needs-configure "${lib}" "${arch}"; then
1210     lib-cpp-clean "${lib}"
1211     lib-cpp-configure "${lib}" "${arch}"
1212   else
1213     StepBanner "${lib}" "configure"
1214   fi
1215
1216   if lib-cpp-needs-make "${lib}" "${arch}"; then
1217     lib-cpp-make "${lib}" "${arch}"
1218   else
1219     SkipBanner "${lib}" "make"
1220   fi
1221
1222   lib-cpp-install "${lib}" "${arch}"
1223   LIB_CPP_IS_SETUP=false
1224 }
1225
1226 #+ lib-cpp-clean - clean libc++/libstdc++ in bitcode
1227 lib-cpp-clean() {
1228   local lib=$1
1229   StepBanner "${lib}" "Clean"
1230   rm -rf ${TC_BUILD}/${lib}*
1231 }
1232
1233 lib-cpp-needs-configure() {
1234   local lib=$1
1235   local arch=$2
1236   lib-cpp-setup "${arch}"
1237   local objdir="${LIB_CPP_BUILD}-${lib}"
1238   ts-newer-than "${TC_BUILD_LLVM}" "${objdir}" && return 0
1239   [ ! -f "${objdir}/config.status" ]
1240   return $?
1241 }
1242
1243 lib-cpp-configure() {
1244   local lib=$1
1245   local arch=$2
1246   lib-cpp-setup "${arch}"
1247   StepBanner "${lib}" "Configure"
1248   local objdir="${LIB_CPP_BUILD}-${lib}"
1249   local subdir="${objdir}/pnacl-target"
1250   local flags=""
1251   mkdir -p "${subdir}"
1252   spushd "${subdir}"
1253
1254   flags+="--with-newlib --disable-shared --disable-rpath"
1255
1256   setup-lib-cpp-env
1257
1258   if [ ${lib} == ${LIB_CXX_NAME} ]; then
1259     local srcdir="${TC_SRC_LIBCXX}"
1260     # HAS_THREAD_LOCAL is used by libc++abi's exception storage, the
1261     # fallback is pthread otherwise.
1262     local cflags="-g -O2 -mllvm -inline-threshold=5 ${BIASED_BC_CFLAGS} \
1263       -DHAS_THREAD_LOCAL=1"
1264     # LLVM's lit is used to test libc++. run.py serves as a shell that
1265     # translates pexe->nexe and executes in sel_ldr. The libc++ test
1266     # suite needs to be told to use pnacl-clang++'s "system library",
1267     # which happens to be libc++ when the -stdlib parameter is used.
1268     # The target architecture is set through the PNACL_RUN_ARCH
1269     # environment variable (``-arch env``).
1270     local litargs="--verbose"
1271     litargs+=" --param shell_prefix='${NACL_ROOT}/run.py -arch env --retries=1'"
1272     litargs+=" --param exe_suffix='.pexe'"
1273     litargs+=" --param use_system_lib=true"
1274     litargs+=" --param link_flags='-std=gnu++11 --pnacl-exceptions=sjlj'"
1275     # TODO(jfb) CMAKE_???_COMPILER_WORKS can be removed once the PNaCl
1276     #           driver scripts stop confusing cmake for libc++. See:
1277     #           https://code.google.com/p/nativeclient/issues/detail?id=3661
1278     RunWithLog libcxx.configure \
1279       env \
1280       cmake -G "Unix Makefiles" \
1281       -DCMAKE_CXX_COMPILER_WORKS=1 \
1282       -DCMAKE_C_COMPILER_WORKS=1 \
1283       -DCMAKE_INSTALL_PREFIX="${LIB_CPP_INSTALL_DIR}" \
1284       -DCMAKE_BUILD_TYPE=Release \
1285       -DCMAKE_C_COMPILER="${PNACL_CC}" \
1286       -DCMAKE_CXX_COMPILER="${PNACL_CXX}" \
1287       -DCMAKE_AR="${PNACL_AR}" \
1288       -DCMAKE_NM="${PNACL_NM}" \
1289       -DCMAKE_RANLIB="${PNACL_RANLIB}" \
1290       -DCMAKE_LD="${ILLEGAL_TOOL}" \
1291       -DCMAKE_AS="${ILLEGAL_TOOL}" \
1292       -DCMAKE_OBJDUMP="${ILLEGAL_TOOL}" \
1293       -DCMAKE_C_FLAGS="-std=gnu11 ${cflags}" \
1294       -DCMAKE_CXX_FLAGS="-std=gnu++11 ${cflags}" \
1295       -DLIT_EXECUTABLE="${TC_SRC_LLVM}/utils/lit/lit.py" \
1296       -DLLVM_LIT_ARGS="${litargs}" \
1297       -DLIBCXX_ENABLE_CXX0X=0 \
1298       -DLIBCXX_ENABLE_SHARED=0 \
1299       -DLIBCXX_CXX_ABI=libcxxabi \
1300       -DLIBCXX_LIBCXXABI_INCLUDE_PATHS="${TC_SRC_LIBCXX}/../libcxxabi/include" \
1301       ${flags} \
1302       ${srcdir}
1303   else
1304     local srcdir="${TC_SRC_LIBSTDCPP}"
1305     RunWithLog libstdcpp.configure \
1306       env -i PATH=/usr/bin/:/bin \
1307       "${STD_ENV_FOR_LIB_CPP[@]}" \
1308       "${srcdir}"/configure \
1309       --host="${CROSS_TARGET_ARM}" \
1310       --prefix="${LIB_CPP_INSTALL_DIR}" \
1311       --enable-cxx-flags="-D__SIZE_MAX__=4294967295" \
1312       --disable-multilib \
1313       --disable-linux-futex \
1314       --disable-libstdcxx-time \
1315       --disable-sjlj-exceptions \
1316       --disable-libstdcxx-pch \
1317       ${flags}
1318   fi
1319   spopd
1320 }
1321
1322 lib-cpp-needs-make() {
1323   local lib=$1
1324   local arch=$2
1325   lib-cpp-setup "${arch}"
1326   local srcdir="${TC_SRC_LIBSTDCPP}" && \
1327     [ ${lib} == ${LIB_CXX_NAME} ] && srcdir="${TC_SRC_LIBCXX}"
1328   local objdir="${LIB_CPP_BUILD}-${lib}"
1329
1330   ts-modified "${srcdir}" "${objdir}"
1331   return $?
1332 }
1333
1334 lib-cpp-make() {
1335   local lib=$1
1336   local arch=$2
1337   lib-cpp-setup "${arch}"
1338   StepBanner "${lib}" "Make"
1339   local objdir="${LIB_CPP_BUILD}-${lib}"
1340
1341   ts-touch-open "${objdir}"
1342
1343   spushd "${objdir}/pnacl-target"
1344   setup-lib-cpp-env
1345   RunWithLog "${lib}.make" \
1346     env -i PATH=/usr/bin/:/bin \
1347     make \
1348     "${STD_ENV_FOR_LIB_CPP[@]}" \
1349     ${MAKE_OPTS}
1350   spopd
1351
1352   ts-touch-commit "${objdir}"
1353 }
1354
1355 lib-cpp-install() {
1356   local lib=$1
1357   local arch=$2
1358   lib-cpp-setup "${arch}"
1359   StepBanner "${lib}" "Install"
1360   local objdir="${LIB_CPP_BUILD}-${lib}"
1361
1362   spushd "${objdir}/pnacl-target"
1363
1364   # Clean the existing installation
1365   local include_dir=""
1366   if [ ${lib} == ${LIB_CXX_NAME} ]; then
1367     include_dir="v[0-9]"
1368   else
1369     include_dir="[0-9].[0-9].[0-9]"
1370   fi
1371   rm -rf "${LIB_CPP_INSTALL_DIR}"/include/c++/"${include_dir}"
1372   rm -rf "${LIB_CPP_INSTALL_DIR}"/lib/"${lib}"*
1373
1374   # install headers (=install-data)
1375   # for good measure make sure we do not keep any old headers
1376   setup-lib-cpp-env
1377   if [ ${lib} == ${LIB_CXX_NAME} ]; then
1378     RunWithLog "${lib}.install" \
1379       make \
1380       "${STD_ENV_FOR_LIB_CPP[@]}" \
1381       ${MAKE_OPTS} install
1382   else
1383     RunWithLog "${lib}.install" \
1384       make \
1385       "${STD_ENV_FOR_LIB_CPP[@]}" \
1386       ${MAKE_OPTS} install-data
1387   fi
1388
1389   # Install bitcode library
1390   mkdir -p "${LIB_CPP_INSTALL_DIR}/lib"
1391   if [ ${lib} == ${LIB_CXX_NAME} ]; then
1392     cp "${objdir}/pnacl-target/lib/${LIB_CXX_NAME}.a" \
1393       "${LIB_CPP_INSTALL_DIR}/lib"
1394   else
1395     cp "${objdir}/pnacl-target/src/.libs/${LIB_STDCPP_NAME}.a" \
1396       "${LIB_CPP_INSTALL_DIR}/lib"
1397   fi
1398   spopd
1399
1400   # libstdc++ installs a file with an abnormal name: "libstdc++*-gdb.py"
1401   # The asterisk may be due to a bug in libstdc++ Makefile/configure.
1402   # This causes problems on the Windows bot (during cleanup, toolchain
1403   # directory delete fails due to the bad character).
1404   # Rename it to get rid of the asterisk.
1405   if [ ${lib} == ${LIB_STDCPP_NAME} ]; then
1406     spushd "${LIB_CPP_INSTALL_DIR}/lib"
1407     mv -f ${LIB_STDCPP_NAME}'*'-gdb.py ${LIB_STDCPP_NAME}-gdb.py
1408     spopd
1409   fi
1410 }
1411
1412 #########################################################################
1413 #########################################################################
1414 #                          < Tools >
1415 #########################################################################
1416 #########################################################################
1417
1418 build-validator() {
1419   local arch=$1
1420   StepBanner "MISC-TOOLS" "Building validator ("${arch}")"
1421   spushd "${NACL_ROOT}"
1422   RunWithLog ${arch}_ncval_core \
1423     ./scons MODE=opt-host \
1424     sysinfo=0 \
1425     ${arch}-ncval-core
1426   cp ${SCONS_OUT}/opt-linux-x86-32/obj/src/trusted/\
1427 validator_${arch}/${arch}-ncval-core ${INSTALL_ROOT}/tools-x86
1428   spopd
1429 }
1430
1431 #+ misc-tools            - Build and install sel_ldr and validator for ARM.
1432 misc-tools() {
1433   if ${PNACL_BUILD_ARM} ; then
1434     StepBanner "MISC-TOOLS" "Building sel_ldr (arm)"
1435
1436     # TODO(robertm): revisit some of these options
1437     spushd "${NACL_ROOT}"
1438     RunWithLog arm_sel_ldr \
1439       ./scons MODE=opt-host \
1440       platform=arm \
1441       naclsdk_validate=0 \
1442       sysinfo=0 \
1443       -j${PNACL_CONCURRENCY} \
1444       sel_ldr
1445     rm -rf  "${INSTALL_ROOT}/tools-arm"
1446     mkdir "${INSTALL_ROOT}/tools-arm"
1447     local sconsdir="${SCONS_OUT}/opt-${SCONS_BUILD_PLATFORM}-arm"
1448     cp "${sconsdir}/obj/src/trusted/service_runtime/sel_ldr" \
1449        "${INSTALL_ROOT}/tools-arm"
1450     spopd
1451   else
1452     StepBanner "MISC-TOOLS" "Skipping arm sel_ldr (No trusted arm toolchain)"
1453   fi
1454
1455   # TODO(petarj): It would be nice to build MIPS sel_ldr on builbots later.
1456   if [ "${PNACL_BUILD_MIPS}" == "true" ] ; then
1457     StepBanner "MISC-TOOLS" "Building sel_ldr (mips32)"
1458     spushd "${NACL_ROOT}"
1459     RunWithLog mips32_sel_ldr \
1460       ./scons MODE=opt-host \
1461       platform=mips32 \
1462       naclsdk_validate=0 \
1463       sysinfo=0 \
1464       -j${PNACL_CONCURRENCY} \
1465       sel_ldr
1466     rm -rf  "${INSTALL_ROOT}/tools-mips32"
1467     mkdir "${INSTALL_ROOT}/tools-mips32"
1468     local sconsdir="scons-out/opt-${SCONS_BUILD_PLATFORM}-mips32"
1469     cp "${sconsdir}/obj/src/trusted/service_runtime/sel_ldr" \
1470        "${INSTALL_ROOT}/tools-mips32"
1471     spopd
1472   else
1473     StepBanner "MISC-TOOLS" "Skipping mips sel_ldr (No trusted mips toolchain)"
1474   fi
1475
1476   if ${BUILD_PLATFORM_LINUX} ; then
1477     rm -rf  "${INSTALL_ROOT}/tools-x86"
1478     mkdir "${INSTALL_ROOT}/tools-x86"
1479     for target in arm mips; do
1480       build-validator $target
1481     done
1482   else
1483     for target in arm mips; do
1484       StepBanner "MISC-"${target} "Skipping " ${target} " validator (Not yet supported on Mac)"
1485     done
1486   fi
1487 }
1488
1489 #########################################################################
1490 #     < BINUTILS >
1491 #########################################################################
1492
1493 #+-------------------------------------------------------------------------
1494 #+ binutils          - Build and install binutils (cross-tools)
1495 binutils() {
1496   StepBanner "BINUTILS (${HOST_ARCH} HOST)"
1497
1498   local srcdir="${TC_SRC_BINUTILS}"
1499
1500   assert-dir "${srcdir}" "You need to checkout binutils."
1501
1502   if binutils-needs-configure; then
1503     binutils-clean
1504     binutils-configure
1505   else
1506     SkipBanner "BINUTILS" "configure"
1507   fi
1508
1509   if binutils-needs-make; then
1510     binutils-make
1511   else
1512     SkipBanner "BINUTILS" "make"
1513   fi
1514   binutils-install
1515 }
1516
1517 #+ binutils-clean    - Clean binutils
1518 binutils-clean() {
1519   StepBanner "BINUTILS" "Clean"
1520   local objdir="${TC_BUILD_BINUTILS}"
1521   rm -rf ${objdir}
1522 }
1523
1524 #+ binutils-configure- Configure binutils for ARM
1525 binutils-configure() {
1526   StepBanner "BINUTILS" "Configure"
1527
1528   local srcdir="${TC_SRC_BINUTILS}"
1529   local objdir="${TC_BUILD_BINUTILS}"
1530
1531   # enable multiple targets so that we can use the same ar with all .o files
1532   local targ="arm-pc-nacl,i686-pc-nacl,x86_64-pc-nacl,mipsel-pc-nacl"
1533   mkdir -p "${objdir}"
1534   spushd "${objdir}"
1535
1536   # The linux escaping is horrible but apparently the only way of doing this:
1537   # c.f.:  http://sourceware.org/ml/binutils/2009-05/msg00252.html
1538   # all we try to do here is to add "$ORIGIN/../lib to "rpath".
1539   # If you ever touch this please make sure that rpath is correct via:
1540   # objdump -p toolchain/${TOOLCHAIN_LABEL}/host/bin/le32-nacl-ld.gold
1541   # objdump -p toolchain/${TOOLCHAIN_LABEL}/host/bin/le32-nacl-objdump
1542   if ${BUILD_PLATFORM_LINUX} ; then
1543       local flags='-Xlinker -rpath -Xlinker '"'"'$\\$$\$$\\$$\$$ORIGIN/../lib'"'"
1544       local shared='yes'
1545       local zlib='--without-zlib'
1546   else
1547       # The shared build for binutils on mac is currently disabled.
1548       # A mac-expert needs to look at this but
1549       # It seems that on mac the linker is storing "full" library paths into
1550       # the dynamic image, e.g, for the llc dynamic image we see  paths like:
1551       # @executable_path/../lib/libLLVM-3.3svn.dylib
1552       # This only works if at linktime the libraries are already at
1553       # @executable_path/../lib which is not the case for mac
1554       #local flags="-Xlinker -rpath -Xlinker '@executable_path/../lib'"
1555       local flags=''
1556       local shared='no'
1557       local zlib=''
1558   fi
1559   # The --enable-gold and --enable-plugins options are on so that we
1560   # can use gold's support for plugin to link PNaCl modules, and use
1561   # gold as the final linker. We do not use bfd ld, and it is disabled
1562   # in part because we do not have its MIPS support downstream.
1563
1564   # We llvm's mc for assembly so we no longer build gas
1565   RunWithLog binutils.configure \
1566       env -i \
1567       PATH="${PATH}" \
1568       CC="${CC}" \
1569       CXX="${CXX}" \
1570       LDFLAGS="${flags}" \
1571       ${srcdir}/configure \
1572           --prefix="${BINUTILS_INSTALL_DIR}" \
1573           --target=${BINUTILS_TARGET} \
1574           --program-prefix=${REAL_CROSS_TARGET}- \
1575           --enable-targets=${targ} \
1576           --enable-shared=${shared} \
1577           --enable-gold=default \
1578           --enable-ld=no \
1579           --disable-nls \
1580           --enable-plugins \
1581           --disable-werror \
1582           --without-gas \
1583           ${zlib} \
1584           --with-sysroot="${NONEXISTENT_PATH}"
1585   # There's no point in setting the correct path as sysroot, because we
1586   # want the toolchain to be relocatable. The driver will use ld command-line
1587   # option --sysroot= to override this value and set it to the correct path.
1588   # However, we need to include --with-sysroot during configure to get this
1589   # option. So fill in a non-sense, non-existent path.
1590
1591   spopd
1592 }
1593
1594 binutils-needs-configure() {
1595   [ ! -f "${TC_BUILD_BINUTILS}/config.status" ]
1596   return $?
1597 }
1598
1599 binutils-needs-make() {
1600   local srcdir="${TC_SRC_BINUTILS}"
1601   local objdir="${TC_BUILD_BINUTILS}"
1602   local ret=1
1603   ts-modified "$srcdir" "$objdir" && ret=0
1604   return ${ret}
1605 }
1606
1607 #+ binutils-make     - Make binutils for ARM
1608 binutils-make() {
1609   StepBanner "BINUTILS" "Make"
1610   local objdir="${TC_BUILD_BINUTILS}"
1611   spushd "${objdir}"
1612
1613   ts-touch-open "${objdir}"
1614
1615   if ${BUILD_PLATFORM_LINUX}; then
1616     RunWithLog binutils.make \
1617       env -i PATH="/usr/bin:/bin" \
1618       ac_cv_search_zlibVersion=no \
1619       make ${MAKE_OPTS_HOST}
1620   else
1621     local control_parallel=""
1622     if ${BUILD_PLATFORM_MAC}; then
1623       # Try to avoid parallel builds for Mac to see if it avoids flake.
1624       # http://code.google.com/p/nativeclient/issues/detail?id=2926
1625       control_parallel="-j1"
1626     fi
1627     RunWithLog binutils.make \
1628       env -i PATH="${PATH}" \
1629       make ${MAKE_OPTS_HOST} ${control_parallel}
1630   fi
1631
1632   ts-touch-commit "${objdir}"
1633
1634   spopd
1635 }
1636
1637 #+ binutils-install  - Install binutils for ARM
1638 binutils-install() {
1639   StepBanner "BINUTILS" "Install"
1640   local objdir="${TC_BUILD_BINUTILS}"
1641   spushd "${objdir}"
1642
1643   RunWithLog binutils.install \
1644     env -i PATH="/usr/bin:/bin" \
1645     make \
1646       install ${MAKE_OPTS}
1647
1648   # TODO(robertm): remove this once we manage to avoid building
1649   #                ld in the first place
1650   echo "pruning hack: ${BINUTILS_INSTALL_DIR}"
1651   rm -f "${BINUTILS_INSTALL_DIR}/bin/${REAL_CROSS_TARGET}-ld"
1652   rm -f "${BINUTILS_INSTALL_DIR}/bin/${REAL_CROSS_TARGET}-ld.bfd"
1653
1654   # Also remove "${BINUTILS_INSTALL_DIR}/${BINUTILS_TARGET}" which contains
1655   # duplicate binaries and unused linker scripts
1656   echo "remove unused ${BINUTILS_INSTALL_DIR}/${BINUTILS_TARGET}/"
1657   rm -rf "${BINUTILS_INSTALL_DIR}/${BINUTILS_TARGET}/"
1658
1659   # Move binutils shared libs to host/lib.
1660   # The first "*" expands to the host string, e.g.
1661   # x86_64-unknown-linux-gnu
1662   if ${BUILD_PLATFORM_LINUX} ; then
1663     echo "move shared libs to ${BINUTILS_INSTALL_DIR}/${SO_DIR}"
1664     for lib in ${BINUTILS_INSTALL_DIR}/*/${BINUTILS_TARGET}/lib/lib*${SO_EXT}
1665     do
1666       echo "moving ${lib}"
1667       mv ${lib} ${BINUTILS_INSTALL_DIR}/${SO_DIR}
1668     done
1669   fi
1670
1671   # NOTE: this needs to come AFTER the moving of the shared libs
1672   # TODO(robertm): this needs to be augmented for mac and windows
1673   echo "remove unused (remaining) binutils static libs and headers"
1674   rm -rf "${BINUTILS_INSTALL_DIR}/i686-pc-linux-gnu" \
1675          "${BINUTILS_INSTALL_DIR}/x86_64-unknown-linux-gnu"
1676
1677   spopd
1678 }
1679
1680 #+-------------------------------------------------------------------------
1681 #+ binutils-liberty      - Build native binutils libiberty
1682 binutils-liberty() {
1683   local srcdir="${TC_SRC_BINUTILS}"
1684
1685   assert-dir "${srcdir}" "You need to checkout binutils."
1686
1687   if binutils-liberty-needs-configure; then
1688     binutils-liberty-clean
1689     binutils-liberty-configure
1690   else
1691     SkipBanner "BINUTILS-LIBERTY" "configure"
1692   fi
1693
1694   if binutils-liberty-needs-make; then
1695     binutils-liberty-make
1696   else
1697     SkipBanner "BINUTILS-LIBERTY" "make"
1698   fi
1699 }
1700
1701 binutils-liberty-needs-configure() {
1702   [ ! -f "${TC_BUILD_BINUTILS_LIBERTY}/config.status" ]
1703   return $?
1704 }
1705
1706 #+ binutils-liberty-clean    - Clean binutils-liberty
1707 binutils-liberty-clean() {
1708   StepBanner "BINUTILS-LIBERTY" "Clean"
1709   local objdir="${TC_BUILD_BINUTILS_LIBERTY}"
1710   rm -rf ${objdir}
1711 }
1712
1713 #+ binutils-liberty-configure - Configure binutils-liberty
1714 binutils-liberty-configure() {
1715   StepBanner "BINUTILS-LIBERTY" "Configure"
1716
1717   local srcdir="${TC_SRC_BINUTILS}"
1718   local objdir="${TC_BUILD_BINUTILS_LIBERTY}"
1719
1720   mkdir -p "${objdir}"
1721   spushd "${objdir}"
1722   RunWithLog binutils.liberty.configure \
1723       env -i \
1724       PATH="${PATH}" \
1725       CC="${CC}" \
1726       CXX="${CXX}" \
1727       ${srcdir}/configure
1728   spopd
1729 }
1730
1731 binutils-liberty-needs-make() {
1732   local srcdir="${TC_SRC_BINUTILS}"
1733   local objdir="${TC_BUILD_BINUTILS_LIBERTY}"
1734
1735   ts-modified "$srcdir" "$objdir"
1736   return $?
1737 }
1738
1739 #+ binutils-liberty-make - Make binutils-liberty
1740 binutils-liberty-make() {
1741   StepBanner "BINUTILS-LIBERTY" "Make"
1742   local srcdir="${TC_SRC_BINUTILS}"
1743   local objdir="${TC_BUILD_BINUTILS_LIBERTY}"
1744   spushd "${objdir}"
1745
1746   ts-touch-open "${objdir}"
1747
1748   RunWithLog binutils.liberty.make \
1749       env -i \
1750       PATH="${PATH}" \
1751       CC="${CC}" \
1752       CXX="${CXX}" \
1753       make ${MAKE_OPTS_HOST} all-libiberty
1754
1755   ts-touch-commit "${objdir}"
1756
1757   spopd
1758 }
1759
1760 #########################################################################
1761 #     CLIENT BINARIES (SANDBOXED)
1762 #########################################################################
1763
1764 check-arch() {
1765   local arch=$1
1766   for valid_arch in i686 x86_64 armv7 universal ; do
1767     if [ "${arch}" == "${valid_arch}" ] ; then
1768       return
1769     fi
1770   done
1771
1772   Fatal "ERROR: Unsupported arch [$1]. Must be: i686, x86_64, armv7, universal"
1773 }
1774
1775 llvm-sb-setup() {
1776   local arch=$1
1777   LLVM_SB_LOG_PREFIX="llvm.sb.${arch}"
1778   LLVM_SB_OBJDIR="$(GetTranslatorBuildDir ${arch})/llvm-sb"
1779
1780   # The SRPC headers are included directly from the nacl tree, as they are
1781   # not in the SDK. libsrpc should have already been built by the
1782   # build.sh sdk-private-libs step.
1783   # This is always statically linked.
1784   # The LLVM sandboxed build uses the normally-disallowed external
1785   # function __nacl_get_arch().  Allow that for now.
1786   local flags="-static -I$(GetAbsolutePath ${NACL_ROOT}/..) \
1787     --pnacl-disable-abi-check "
1788
1789   LLVM_SB_CONFIGURE_ENV=(
1790     AR="${PNACL_AR}" \
1791     AS="${PNACL_AS}" \
1792     CC="${PNACL_CC} ${flags}" \
1793     CXX="${PNACL_CXX} ${flags}" \
1794     LD="${PNACL_LD} ${flags}" \
1795     NM="${PNACL_NM}" \
1796     RANLIB="${PNACL_RANLIB}")
1797 }
1798
1799 #+-------------------------------------------------------------------------
1800 #+ llvm-sb <arch>- Build and install llvm tools (sandboxed)
1801 llvm-sb() {
1802   local arch=$1
1803   check-arch ${arch}
1804   llvm-sb-setup ${arch}
1805   StepBanner "LLVM-SB" "Sandboxed pnacl-llc [${arch}]"
1806   local srcdir="${TC_SRC_LLVM}"
1807   assert-dir "${srcdir}" "You need to checkout llvm."
1808
1809   if llvm-sb-needs-configure ${arch} ; then
1810     llvm-sb-clean ${arch}
1811     llvm-sb-configure ${arch}
1812   else
1813     SkipBanner "LLVM-SB" "configure ${arch}"
1814   fi
1815
1816   llvm-sb-make ${arch}
1817   llvm-sb-install ${arch}
1818 }
1819
1820 llvm-sb-needs-configure() {
1821   [ ! -f "${LLVM_SB_OBJDIR}/config.status" ]
1822   return $?
1823 }
1824
1825 # llvm-sb-clean          - Clean llvm tools (sandboxed)
1826 llvm-sb-clean() {
1827   local arch=$1
1828   StepBanner "LLVM-SB" "Clean ${arch}"
1829   local objdir="${LLVM_SB_OBJDIR}"
1830
1831   rm -rf "${objdir}"
1832   mkdir -p "${objdir}"
1833 }
1834
1835 # llvm-sb-configure - Configure llvm tools (sandboxed)
1836 llvm-sb-configure() {
1837   local arch=$1
1838
1839   StepBanner "LLVM-SB" "Configure ${arch}"
1840   local srcdir="${TC_SRC_LLVM}"
1841   local objdir="${LLVM_SB_OBJDIR}"
1842   local installdir="$(GetTranslatorInstallDir ${arch})"
1843   local targets=""
1844   # For LLVM, "x86" brings in both i686 and x86_64.
1845   case ${arch} in
1846     i686) targets=x86 ;;
1847     x86_64) targets=x86 ;;
1848     armv7) targets=arm ;;
1849     universal) targets=x86,arm ;;
1850   esac
1851
1852   spushd "${objdir}"
1853   # TODO(jvoung): remove ac_cv_func_getrusage=no once newlib has getrusage
1854   # in its headers.  Otherwise, configure thinks that we can link in
1855   # getrusage (stub is in libnacl), but we can't actually compile code
1856   # that uses ::getrusage because it's not in headers:
1857   # https://code.google.com/p/nativeclient/issues/detail?id=3657
1858   RunWithLog \
1859       ${LLVM_SB_LOG_PREFIX}.configure \
1860       env -i \
1861       PATH="/usr/bin:/bin" \
1862       ${srcdir}/configure \
1863         "${LLVM_SB_CONFIGURE_ENV[@]}" \
1864         --prefix=${installdir} \
1865         --host=nacl \
1866         --enable-targets=${targets} \
1867         --disable-assertions \
1868         --enable-pic=no \
1869         --enable-static \
1870         --enable-shared=no \
1871         --disable-jit \
1872         --enable-optimized \
1873         --target=${CROSS_TARGET_ARM} \
1874         llvm_cv_link_use_export_dynamic=no \
1875         ac_cv_func_getrusage=no
1876   spopd
1877 }
1878
1879 # llvm-sb-make - Make llvm tools (sandboxed)
1880 llvm-sb-make() {
1881   local arch=$1
1882   StepBanner "LLVM-SB" "Make ${arch}"
1883   local objdir="${LLVM_SB_OBJDIR}"
1884
1885   spushd "${objdir}"
1886   ts-touch-open "${objdir}"
1887
1888   local tools_to_build="pnacl-llc"
1889   local export_dyn_env="llvm_cv_link_use_export_dynamic=no"
1890   local isjit=0
1891   RunWithLog ${LLVM_SB_LOG_PREFIX}.make \
1892       env -i PATH="/usr/bin:/bin" \
1893       ONLY_TOOLS="${tools_to_build}" \
1894       NACL_SANDBOX=1 \
1895       KEEP_SYMBOLS=1 \
1896       VERBOSE=1 \
1897       ${export_dyn_env} \
1898       make ${MAKE_OPTS} tools-only
1899
1900   ts-touch-commit "${objdir}"
1901
1902   spopd
1903 }
1904
1905 # llvm-sb-install - Install llvm tools (sandboxed)
1906 llvm-sb-install() {
1907   local arch=$1
1908   StepBanner "LLVM-SB" "Install ${arch}"
1909
1910   local toolname="pnacl-llc"
1911   local installdir="$(GetTranslatorInstallDir ${arch})"/bin
1912   mkdir -p "${installdir}"
1913   spushd "${installdir}"
1914   local objdir="${LLVM_SB_OBJDIR}"
1915   cp -f "${objdir}"/Release*/bin/${toolname} .
1916   mv -f ${toolname} ${toolname}.pexe
1917   local arches=${arch}
1918   if [[ "${arch}" == "universal" ]]; then
1919     arches="${SBTC_ARCHES_ALL}"
1920   elif [[ "${arch}" == "i686" ]]; then
1921     # LLVM does not separate the i686 and x86_64 backends.
1922     # Translate twice to get both nexes.
1923     arches="i686 x86_64"
1924   fi
1925   translate-sb-tool ${toolname} "${arches}"
1926   install-sb-tool ${toolname} "${arches}"
1927   spopd
1928 }
1929
1930 # translate-sb-tool <toolname> <arches>
1931 #
1932 # Translate <toolname>.pexe to <toolname>.<arch>.nexe in the current directory.
1933 translate-sb-tool() {
1934   local toolname=$1
1935   local arches=$2
1936   local pexe="${toolname}.pexe"
1937   if ${PNACL_PRUNE}; then
1938     # Only strip debug, to preserve symbol names for testing. This is okay
1939     # because we strip the native nexe later anyway.
1940     # So, why bother stripping here at all?
1941     # It does appear to affect the size of the nexe:
1942     # http://code.google.com/p/nativeclient/issues/detail?id=3305
1943     ${PNACL_STRIP} --strip-debug "${pexe}"
1944   fi
1945
1946   local tarch
1947   for tarch in ${arches}; do
1948     local nexe="${toolname}.${tarch}.nexe"
1949     StepBanner "TRANSLATE" \
1950                "Translating ${toolname}.pexe to ${tarch} (background)"
1951     # NOTE: we are using --noirt to build without a segment gap
1952     # since we aren't loading the IRT for the translator nexes.
1953     #
1954     # Compiling with -ffunction-sections, -fdata-sections, --gc-sections
1955     # helps reduce the size a bit. If you want to use --gc-sections to test out:
1956     # http://code.google.com/p/nativeclient/issues/detail?id=1591
1957     # you will need to do a build without these flags.
1958     "${PNACL_TRANSLATE}" -ffunction-sections -fdata-sections --gc-sections \
1959       --allow-llvm-bitcode-input --noirt -arch ${tarch} "${pexe}" -o "${nexe}" &
1960     QueueLastProcess
1961   done
1962   StepBanner "TRANSLATE" "Waiting for translation processes to finish"
1963   QueueWait
1964
1965   # Test that certain symbols have been pruned before stripping.
1966   if [ "${toolname}" == "pnacl-llc" ]; then
1967     for tarch in ${arches}; do
1968       local nexe="${toolname}.${tarch}.nexe"
1969       local llvm_host_glob="${LLVM_INSTALL_DIR}/lib/libLLVM*so"
1970       python "${PNACL_ROOT}/prune_test.py" "${PNACL_NM}" \
1971         "${llvm_host_glob}" "${nexe}"
1972     done
1973   fi
1974
1975   if ${PNACL_PRUNE}; then
1976     # Strip the nexes.
1977     for tarch in ${arches}; do
1978       local nexe="${toolname}.${tarch}.nexe"
1979       ${PNACL_STRIP} "${nexe}"
1980     done
1981   fi
1982   StepBanner "TRANSLATE" "Done."
1983 }
1984
1985 # install-sb-tool <toolname> <arches>
1986 #
1987 # Install <toolname>.<arch>.nexe from the current directory
1988 # into the right location (as <toolname>.nexe)
1989 install-sb-tool() {
1990   local toolname="$1"
1991   local arches="$2"
1992   local tarch
1993   for tarch in ${arches}; do
1994     local installbin="$(GetTranslatorInstallDir ${tarch})"/bin
1995     mkdir -p "${installbin}"
1996     mv -f ${toolname}.${tarch}.nexe "${installbin}"/${toolname}.nexe
1997     if ${PNACL_BUILDBOT}; then
1998       spushd "${installbin}"
1999       local tag="${toolname}_${tarch}_size"
2000       print-tagged-tool-sizes "${tag}" "$(pwd)/${toolname}.nexe"
2001       spopd
2002     fi
2003   done
2004 }
2005
2006 GetTranslatorBuildDir() {
2007   local arch="$1"
2008   echo "${TC_BUILD}/translator-${arch//_/-}"
2009 }
2010
2011 GetTranslatorInstallDir() {
2012   local arch="$1"
2013   echo "${INSTALL_TRANSLATOR}"/${arch}
2014 }
2015
2016 ### Sandboxed version of gold.
2017
2018 #+-------------------------------------------------------------------------
2019 #+ binutils-gold-sb - Build and install gold (sandboxed)
2020 #+                    This is the replacement for the old
2021 #+                    final sandboxed linker which was bfd based.
2022 #+                    It has nothing to do with the bitcode linker
2023 #+                    which is also gold based.
2024 binutils-gold-sb() {
2025   local arch=$1
2026   check-arch ${arch}
2027   StepBanner "GOLD-NATIVE-SB" "(libiberty + gold) ${arch}"
2028
2029   local srcdir="${TC_SRC_BINUTILS}"
2030   assert-dir "${srcdir}" "You need to checkout gold."
2031
2032   binutils-gold-sb-clean ${arch}
2033   binutils-gold-sb-configure ${arch}
2034   binutils-gold-sb-make ${arch}
2035   binutils-gold-sb-install ${arch}
2036 }
2037
2038 # binutils-gold-sb-clean - Clean gold
2039 binutils-gold-sb-clean() {
2040   local arch=$1
2041   StepBanner "GOLD-NATIVE-SB" "Clean ${arch}"
2042   local objdir="$(GetTranslatorBuildDir ${arch})/binutils-gold-sb"
2043
2044   rm -rf "${objdir}"
2045   mkdir -p "${objdir}"
2046 }
2047
2048 # binutils-gold-sb-configure - Configure binutils for gold (unsandboxed)
2049 binutils-gold-sb-configure() {
2050   local arch=$1
2051   local srcdir="${TC_SRC_BINUTILS}"
2052   local objdir="$(GetTranslatorBuildDir ${arch})/binutils-gold-sb"
2053   local installbin="$(GetTranslatorInstallDir ${arch})/bin"
2054
2055   # The SRPC headers are included directly from the nacl tree, as they are
2056   # not in the SDK. libsrpc should have already been built by the
2057   # build.sh sdk-private-libs step
2058   # The Gold sandboxed build uses the normally-disallowed external
2059   # function __nacl_get_arch().  Allow that for now.
2060   #
2061   # TODO(jfb) Gold currently only builds with libstdc++.
2062   local flags="-static -I$(GetAbsolutePath ${NACL_ROOT}/..) \
2063     -fno-exceptions -O3 --pnacl-disable-abi-check -stdlib=${LIB_STDCPP_NAME} "
2064   local configure_env=(
2065     AR="${PNACL_AR}" \
2066     AS="${PNACL_AS}" \
2067     CC="${PNACL_CC} ${flags}" \
2068     CXX="${PNACL_CXX} ${flags}" \
2069     CC_FOR_BUILD="${CC}" \
2070     CXX_FOR_BUILD="${CXX}" \
2071     LD="${PNACL_LD} ${flags}" \
2072     NM="${PNACL_NM}" \
2073     RANLIB="${PNACL_RANLIB}"
2074   )
2075   local gold_targets=""
2076   case ${arch} in
2077     i686)      gold_targets=i686-pc-nacl ;;
2078     x86_64)    gold_targets=x86_64-pc-nacl ;;
2079     armv7)     gold_targets=arm-pc-nacl ;;
2080     universal)
2081       gold_targets=i686-pc-nacl,x86_64-pc-nacl,arm-pc-nacl ;;
2082   esac
2083
2084   # gold always adds "target" to the enabled targets so we are
2085   # little careful to not build too much
2086   # Note: we are (ab)using target for both --host and --target
2087   #       which configure expects to be present
2088   local target
2089   if [ ${arch} == "universal" ] ; then
2090     target=i686-pc-nacl
2091   else
2092     target=${gold_targets}
2093   fi
2094
2095   StepBanner "GOLD-NATIVE-SB" "Configure (libiberty)"
2096   # Gold depends on liberty only for a few functions:
2097   # xrealloc, lbasename, etc.
2098   # we could remove these if necessary.
2099
2100   mkdir -p "${objdir}/libiberty"
2101   spushd "${objdir}/libiberty"
2102   StepBanner "GOLD-NATIVE-SB" "Dir [$(pwd)]"
2103   local log_prefix="binutils-gold.sb.${arch}"
2104   RunWithLog "${log_prefix}".configure \
2105     env -i \
2106     PATH="/usr/bin:/bin" \
2107     "${configure_env[@]}" \
2108     ${srcdir}/libiberty/configure --prefix="${installbin}" \
2109     --host=${target} \
2110     --target=${target}
2111   spopd
2112
2113   StepBanner "GOLD-NATIVE-SB" "Configure (gold) ${arch}"
2114   mkdir -p "${objdir}/gold"
2115   spushd "${objdir}/gold"
2116   StepBanner "GOLD-NATIVE-SB" "Dir [$(pwd)]"
2117   # Removed -Werror until upstream gold no longer has problems with new clang
2118   # warnings. http://code.google.com/p/nativeclient/issues/detail?id=2861
2119   # TODO(sehr,robertm): remove this when gold no longer has these.
2120   RunWithLog "${log_prefix}".configure \
2121     env -i \
2122     PATH="/usr/bin:/bin" \
2123     "${configure_env[@]}" \
2124     CXXFLAGS="" \
2125     CFLAGS="" \
2126     ac_cv_search_zlibVersion=no \
2127     ac_cv_header_sys_mman_h=no \
2128     ac_cv_func_mmap=no \
2129     ac_cv_func_mallinfo=no \
2130     ac_cv_prog_cc_g=no \
2131     ac_cv_prog_cxx_g=no \
2132     ${srcdir}/gold/configure --prefix="${installbin}" \
2133                                       --enable-targets=${gold_targets} \
2134                                       --host=${target} \
2135                                       --target=${target} \
2136                                       --disable-nls \
2137                                       --enable-plugins=no \
2138                                       --enable-naclsrpc=yes \
2139                                       --disable-werror \
2140                                       --with-sysroot="${NONEXISTENT_PATH}"
2141   # Note: the extra ac_cv settings:
2142   # * eliminate unnecessary use of zlib
2143   # * eliminate use of mmap
2144   # (those should not have much impact on the non-sandboxed
2145   # version but help in the sandboxed case)
2146   # We also disable debug (-g) because the bitcode files become too big
2147   # (with ac_cv_prog_cc_g).
2148
2149   # There's no point in setting the correct path as sysroot, because we
2150   # want the toolchain to be relocatable. The driver will use ld command-line
2151   # option --sysroot= to override this value and set it to the correct path.
2152   # However, we need to include --with-sysroot during configure to get this
2153   # option. So fill in a non-sense, non-existent path.
2154   spopd
2155 }
2156
2157 # binutils-gold-sb-make - Make binutils (sandboxed)
2158 binutils-gold-sb-make() {
2159   local arch=${1}
2160   local objdir="$(GetTranslatorBuildDir ${arch})/binutils-gold-sb"
2161   ts-touch-open "${objdir}/"
2162
2163   StepBanner "GOLD-NATIVE-SB" "Make (liberty) ${arch}"
2164   spushd "${objdir}/libiberty"
2165
2166   RunWithLog "binutils-gold.liberty.sb.${arch}".make \
2167       env -i PATH="/usr/bin:/bin" \
2168       make ${MAKE_OPTS}
2169   spopd
2170
2171   StepBanner "GOLD-NATIVE-SB" "Make (gold) ${arch}"
2172   spushd "${objdir}/gold"
2173   RunWithLog "binutils-gold.sb.${arch}".make \
2174       env -i PATH="/usr/bin:/bin" \
2175       make ${MAKE_OPTS} ld-new
2176   spopd
2177
2178   ts-touch-commit "${objdir}"
2179 }
2180
2181 # binutils-gold-sb-install - Install gold
2182 binutils-gold-sb-install() {
2183   local arch=$1
2184   local objdir="$(GetTranslatorBuildDir ${arch})/binutils-gold-sb"
2185   local installbin="$(GetTranslatorInstallDir ${arch})/bin"
2186
2187   StepBanner "GOLD-NATIVE-SB" "Install [${installbin}] ${arch}"
2188
2189   mkdir -p "${installbin}"
2190   spushd "${installbin}"
2191
2192   # Install just "ld"
2193   cp "${objdir}"/gold/ld-new ld.pexe
2194
2195   # Translate and install
2196   local arches=${arch}
2197   if [[ "${arch}" == "universal" ]]; then
2198     arches="${SBTC_ARCHES_ALL}"
2199   fi
2200   translate-sb-tool ld "${arches}"
2201   install-sb-tool ld "${arches}"
2202   spopd
2203 }
2204
2205
2206 #+-------------------------------------------------------------------------
2207 #########################################################################
2208 #     < NEWLIB-BITCODE >
2209 #########################################################################
2210
2211 #+ newlib - Build and install newlib in bitcode.
2212 newlib() {
2213   local arch=$1
2214   setup-newlib-env ${arch}
2215   StepBanner "NEWLIB (${arch})"
2216
2217   if newlib-needs-configure; then
2218     newlib-clean
2219     newlib-configure ${arch}
2220   else
2221     SkipBanner "NEWLIB" "configure"
2222   fi
2223
2224   if newlib-needs-make; then
2225     newlib-make ${arch}
2226   else
2227     SkipBanner "NEWLIB" "make"
2228   fi
2229
2230   newlib-install ${arch}
2231 }
2232
2233 #+ newlib-clean  - Clean bitcode newlib.
2234 newlib-clean() {
2235   StepBanner "NEWLIB" "Clean"
2236   rm -rf "${TC_BUILD_NEWLIB}"
2237 }
2238
2239 newlib-needs-configure() {
2240   ts-newer-than "${TC_BUILD_LLVM}" \
2241                 "${TC_BUILD_NEWLIB}" && return 0
2242
2243   [ ! -f "${TC_BUILD_NEWLIB}/config.status" ]
2244   return #?
2245 }
2246
2247 #+ newlib-configure - Configure bitcode Newlib
2248 newlib-configure() {
2249   local arch=$1
2250   setup-newlib-env ${arch}
2251   StepBanner "NEWLIB" "Configure (${NEWLIB_TARGET})"
2252
2253   local srcdir="${TC_SRC_NEWLIB}"
2254   local objdir="${TC_BUILD_NEWLIB}"
2255   mkdir -p "${objdir}"
2256   spushd "${objdir}"
2257
2258   RunWithLog "newlib.${arch}.configure" \
2259     env -i \
2260     PATH="/usr/bin:/bin" \
2261     "${STD_ENV_FOR_NEWLIB[@]}" \
2262     ${srcdir}/configure \
2263         --disable-multilib \
2264         --prefix="${NEWLIB_INSTALL_DIR}" \
2265         --disable-newlib-supplied-syscalls \
2266         --disable-texinfo \
2267         --disable-libgloss \
2268         --enable-newlib-iconv \
2269         --enable-newlib-iconv-from-encodings=UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4 \
2270         --enable-newlib-iconv-to-encodings=UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4 \
2271         --enable-newlib-io-long-long \
2272         --enable-newlib-io-long-double \
2273         --enable-newlib-io-c99-formats \
2274         --enable-newlib-mb \
2275         --target="${NEWLIB_TARGET}"
2276
2277   spopd
2278 }
2279
2280 newlib-needs-make() {
2281   local srcdir="${TC_SRC_NEWLIB}"
2282   local objdir="${TC_BUILD_NEWLIB}"
2283
2284   ts-modified "$srcdir" "$objdir"
2285   return $?
2286 }
2287
2288 #+ newlib-make           - Make bitcode Newlib
2289 newlib-make() {
2290   local arch=$1
2291   setup-newlib-env ${arch}
2292   StepBanner "NEWLIB" "Make"
2293   local srcdir="${TC_SRC_NEWLIB}"
2294   local objdir="${TC_BUILD_NEWLIB}"
2295
2296   ts-touch-open "${objdir}"
2297
2298   spushd "${objdir}"
2299   RunWithLog "newlib.${arch}.make" \
2300     env -i PATH="/usr/bin:/bin" \
2301     make \
2302       "${STD_ENV_FOR_NEWLIB[@]}" \
2303       ${MAKE_OPTS}
2304   spopd
2305
2306   ts-touch-commit "${objdir}"
2307
2308 }
2309
2310 #+ newlib-install        - Install Bitcode Newlib using build env.
2311 newlib-install() {
2312   local arch=$1
2313   setup-newlib-env ${arch}
2314   StepBanner "NEWLIB" "Install"
2315   local objdir="${TC_BUILD_NEWLIB}"
2316
2317   spushd "${objdir}"
2318
2319   # NOTE: we might be better off not using install, as we are already
2320   #       doing a bunch of copying of headers and libs further down
2321   RunWithLog newlib.install \
2322     env -i PATH="/usr/bin:/bin" \
2323       make \
2324       "${STD_ENV_FOR_NEWLIB[@]}" \
2325       install ${MAKE_OPTS}
2326   spopd
2327
2328   # Newlib installs files into usr/${NEWLIB_TARGET}/*
2329   # Get rid of the ${NEWLIB_TARGET}/ prefix.
2330   spushd "${NEWLIB_INSTALL_DIR}"
2331   mkdir -p lib include
2332   mv -f ${NEWLIB_TARGET}/lib/* lib
2333   rm -rf  include/sys include/machine
2334   mv -f ${NEWLIB_TARGET}/include/* include
2335   rmdir ${NEWLIB_TARGET}/lib
2336   rmdir ${NEWLIB_TARGET}/include
2337   rmdir ${NEWLIB_TARGET}
2338
2339   cp "${NACL_ROOT}/src/untrusted/pthread/pthread.h" \
2340     "${NACL_ROOT}/src/untrusted/pthread/semaphore.h" \
2341     include
2342   # Copy the nacl_random.h header, which is needed by the libc++ build. It
2343   # uses the IRT so should be safe to include in the toolchain tarball.
2344   mkdir include/nacl
2345   cp "${NACL_ROOT}/src/untrusted/nacl/nacl_random.h" \
2346     include/nacl
2347   spopd
2348 }
2349
2350 libs-support() {
2351   StepBanner "LIBS-SUPPORT"
2352   libs-support-newlib-crt1 portable
2353   libs-support-newlib-crt1 x86-64
2354   libs-support-bitcode portable
2355   libs-support-bitcode x86-64
2356
2357   local arch
2358   for arch in arm x86-32 x86-64 mips32; do
2359     libs-support-native ${arch}
2360   done
2361
2362   libs-support-unsandboxed
2363 }
2364
2365 libs-support-newlib-crt1() {
2366   local arch=$1
2367   setup-biased-bitcode-env ${arch}
2368   mkdir -p "${INSTALL_LIB}"
2369   spushd "${PNACL_SUPPORT}/bitcode"
2370
2371   StepBanner "LIBS-SUPPORT-NEWLIB" \
2372              "Install crt1.x & crt1_for_eh.x (linker scripts)"
2373   # Two versions of crt1.x exist, for different scenarios (with and without EH).
2374   # See: https://code.google.com/p/nativeclient/issues/detail?id=3069
2375   cp crt1.x "${INSTALL_LIB}"/crt1.x
2376   cp crt1_for_eh.x "${INSTALL_LIB}"/crt1_for_eh.x
2377   spopd
2378 }
2379
2380 libs-support-bitcode() {
2381   local arch=$1
2382   setup-biased-bitcode-env ${arch}
2383   local build_dir="${TC_BUILD}/libs-support-bitcode-${arch}"
2384   local cc_cmd="${PNACL_CC} -Wall -Werror -O2 ${BIASED_BC_CFLAGS}"
2385
2386   mkdir -p "${build_dir}"
2387   spushd "${PNACL_SUPPORT}/bitcode"
2388   # Install crti.bc (empty _init/_fini)
2389   StepBanner "LIBS-SUPPORT" "Install ${arch} crti.bc"
2390   ${cc_cmd} -c crti.c -o "${build_dir}"/crti.bc
2391
2392   # Install crtbegin bitcode (__cxa_finalize for C++)
2393   StepBanner "LIBS-SUPPORT" "Install ${arch} crtbegin.bc"
2394   # NOTE: we do not have an "end" version of ththis
2395   ${cc_cmd} -c crtbegin.c -o "${build_dir}"/crtbegin.bc
2396
2397   # Install unwind_stubs.bc (stubs for _Unwind_* functions when libgcc_eh
2398   # is not included in the native link).
2399   StepBanner "LIBS-SUPPORT" "Install ${arch} unwind_stubs.bc"
2400   ${cc_cmd} -c unwind_stubs.c -o "${build_dir}"/unwind_stubs.bc
2401   StepBanner "LIBS-SUPPORT" "Install ${arch} sjlj_eh_redirect.bc"
2402   ${cc_cmd} -c sjlj_eh_redirect.c -o "${build_dir}"/sjlj_eh_redirect.bc
2403
2404   # Install libpnaclmm.a (__atomic_*).
2405   StepBanner "LIBS-SUPPORT" "Install ${arch} libpnaclmm.a"
2406   ${cc_cmd} -c pnaclmm.c -o "${build_dir}"/pnaclmm.bc
2407   ${PNACL_AR} rc "${build_dir}"/libpnaclmm.a "${build_dir}"/pnaclmm.bc
2408
2409   spopd
2410
2411   # Install to actual lib directories.
2412   spushd "${build_dir}"
2413   local files="crti.bc crtbegin.bc libpnaclmm.a unwind_stubs.bc sjlj_eh_redirect.bc"
2414   mkdir -p "${INSTALL_LIB}"
2415   cp -f ${files} "${INSTALL_LIB}"
2416   spopd
2417 }
2418
2419 libs-support-native() {
2420   local arch=$1
2421   local destdir="${INSTALL_LIB_NATIVE}"${arch}
2422   local label="LIBS-SUPPORT (${arch})"
2423   mkdir -p "${destdir}"
2424
2425   local flags="--pnacl-allow-native --pnacl-allow-translate -Wall -Werror"
2426   local cc_cmd="${PNACL_CC} -arch ${arch} --pnacl-bias=${arch} ${flags} \
2427       -I../../.. -O3"
2428
2429   spushd "${PNACL_SUPPORT}"
2430
2431   # NOTE: one of the reasons why we build these native is
2432   #       the shim library on x86-64
2433   #       c.f.: https://sites.google.com/a/google.com/nativeclient/project-pnacl/c-exception-handling
2434   # Compile crtbegin.o / crtend.o
2435   StepBanner "${label}" "Install crtbegin.o / crtend.o"
2436
2437   # Build two versions of crtbegin: one that relies on later linking with
2438   # libgcc_eh and one that doesn't.
2439   # See: https://code.google.com/p/nativeclient/issues/detail?id=3069
2440   ${cc_cmd} -c crtbegin.c -o "${destdir}"/crtbegin.o
2441   ${cc_cmd} -c crtbegin.c -DLINKING_WITH_LIBGCC_EH \
2442             -o "${destdir}"/crtbegin_for_eh.o
2443   ${cc_cmd} -c crtend.c -o "${destdir}"/crtend.o
2444
2445   # Make libcrt_platform.a
2446   StepBanner "${label}" "Install libcrt_platform.a"
2447   local tmpdir="${TC_BUILD}/libs-support-native"
2448   rm -rf "${tmpdir}"
2449   mkdir -p "${tmpdir}"
2450   ${cc_cmd} -c pnacl_irt.c -o "${tmpdir}"/pnacl_irt.o
2451   ${cc_cmd} -c setjmp_${arch/-/_}.S -o "${tmpdir}"/setjmp.o
2452
2453   # Some of the support code lives in third_party/ because it's based on code
2454   # from other open-source projects.
2455   ${cc_cmd} \
2456     -c "${NACL_SRC_THIRD_PARTY_MOD}/pnacl_native_newlib_subset/string.c" \
2457     -std=c99 -o "${tmpdir}"/string.o
2458
2459   # For ARM, also compile aeabi_read_tp.S
2460   if  [ ${arch} == arm ] ; then
2461     ${cc_cmd} -c aeabi_read_tp.S -o "${tmpdir}"/aeabi_read_tp.o
2462   fi
2463   spopd
2464
2465   ${PNACL_AR} rc "${destdir}"/libcrt_platform.a "${tmpdir}"/*.o
2466 }
2467
2468 libs-support-unsandboxed() {
2469   if ${BUILD_PLATFORM_LINUX}; then
2470     local arch=linux-x86-32
2471     StepBanner "LIBS-SUPPORT (${arch})" "Install unsandboxed_irt.o"
2472     local destdir="${INSTALL_LIB_NATIVE}"${arch}
2473     mkdir -p ${destdir}
2474     gcc -m32 -O2 -Wall -Werror -I${NACL_ROOT}/.. -DNACL_LINUX=1 -c \
2475         ${PNACL_SUPPORT}/unsandboxed_irt.c -o ${destdir}/unsandboxed_irt.o
2476   fi
2477 }
2478
2479
2480 # Build the dummy "libpnacl_irt_shim.a", which is useful for building
2481 # commandline programs.  It cannot be used to build PPAPI programs
2482 # because it does not actually shim the PPAPI interfaces.
2483 # The library is named the same as the real PPAPI shim to ensure that
2484 # the commandlines are the same.
2485 # This must be built after newlib(), since it uses headers like <stdint.h>.
2486 dummy-irt-shim() {
2487   local arch=$1
2488   local destdir="${INSTALL_LIB_NATIVE}"${arch}
2489   local label="DUMMY-IRT-SHIM (${arch})"
2490   mkdir -p "${destdir}"
2491
2492   local flags="--pnacl-allow-native --pnacl-allow-translate -Wall -Werror"
2493   local cc_cmd="${PNACL_CC} -arch ${arch} ${flags}"
2494
2495   spushd "${PNACL_SUPPORT}"
2496   StepBanner "${label}" "Install libpnacl_irt_shim_dummy.a"
2497   local tmpdir="${TC_BUILD}/dummy-irt-shim"
2498   rm -rf "${tmpdir}"
2499   mkdir -p "${tmpdir}"
2500   ${cc_cmd} -c dummy_shim_entry.c -o "${tmpdir}"/dummy_shim_entry.o
2501   spopd
2502
2503   ${PNACL_AR} rc "${destdir}"/libpnacl_irt_shim_dummy.a "${tmpdir}"/*.o
2504 }
2505
2506 #########################################################################
2507 #     < SDK >
2508 #########################################################################
2509 SCONS_ARGS=(MODE=nacl
2510             -j${PNACL_CONCURRENCY}
2511             bitcode=1
2512             disable_nosys_linker_warnings=1
2513             naclsdk_validate=0
2514             --verbose)
2515
2516 SDK_IS_SETUP=false
2517 sdk-setup() {
2518   if ${SDK_IS_SETUP} && [ $# -eq 0 ]; then
2519     return 0
2520   fi
2521   SDK_IS_SETUP=true
2522
2523   SDK_INSTALL_ROOT="${INSTALL_ROOT}/sdk"
2524   SDK_INSTALL_LIB="${SDK_INSTALL_ROOT}/lib"
2525   SDK_INSTALL_INCLUDE="${SDK_INSTALL_ROOT}/include"
2526 }
2527
2528 sdk() {
2529   sdk-setup "$@"
2530   StepBanner "SDK"
2531   sdk-clean
2532   sdk-headers
2533   sdk-libs
2534   sdk-verify
2535 }
2536
2537 #+ sdk-clean             - Clean sdk stuff
2538 sdk-clean() {
2539   sdk-setup "$@"
2540   StepBanner "SDK" "Clean"
2541   rm -rf "${SDK_INSTALL_ROOT}"
2542
2543   # clean scons obj dirs
2544   rm -rf "${SCONS_OUT}"/nacl-*-pnacl*
2545 }
2546
2547 sdk-headers() {
2548   sdk-setup "$@"
2549   mkdir -p "${SDK_INSTALL_INCLUDE}"
2550
2551   local extra_flags=""
2552   local neutral_platform="x86-32"
2553
2554   StepBanner "SDK" "Install headers"
2555   spushd "${NACL_ROOT}"
2556   # TODO(pnacl-team): remove this pnaclsdk_mode once we have a better story
2557   # about host binary type (x86-32 vs x86-64).  SCons only knows how to use
2558   # x86-32 host binaries right now, so we need pnaclsdk_mode to override that.
2559   RunWithLog "sdk.headers" \
2560       ./scons \
2561       "${SCONS_ARGS[@]}" \
2562       ${extra_flags} \
2563       platform=${neutral_platform} \
2564       pnaclsdk_mode="custom:${INSTALL_ROOT}" \
2565       install_headers \
2566       includedir="$(PosixToSysPath "${SDK_INSTALL_INCLUDE}")"
2567   spopd
2568 }
2569
2570 sdk-libs() {
2571   sdk-setup "$@"
2572   StepBanner "SDK" "Install libraries"
2573   mkdir -p "${SDK_INSTALL_LIB}"
2574
2575   local extra_flags=""
2576   local neutral_platform="x86-32"
2577
2578   spushd "${NACL_ROOT}"
2579   # See above TODO about pnaclsdk_mode.
2580   RunWithLog "sdk.libs.bitcode" \
2581       ./scons \
2582       "${SCONS_ARGS[@]}" \
2583       ${extra_flags} \
2584       platform=${neutral_platform} \
2585       pnaclsdk_mode="custom:${INSTALL_ROOT}" \
2586       install_lib \
2587       libdir="$(PosixToSysPath "${SDK_INSTALL_LIB}")"
2588   spopd
2589 }
2590
2591 # This builds lib*_private.a, to allow building the llc and ld nexes without
2592 # the IRT and without the segment gap.
2593 sdk-private-libs() {
2594   sdk-setup "$@"
2595   StepBanner "SDK" "Private (non-IRT) libs"
2596   spushd "${NACL_ROOT}"
2597
2598   local neutral_platform="x86-32"
2599   # See above TODO about pnaclsdk_mode.
2600   RunWithLog "sdk.libs_private.bitcode" \
2601     ./scons \
2602     -j${PNACL_CONCURRENCY} \
2603     bitcode=1 \
2604     platform=${neutral_platform} \
2605     pnaclsdk_mode="custom:${INSTALL_ROOT}" \
2606     --verbose \
2607     libnacl_sys_private \
2608     libpthread_private \
2609     libnacl_dyncode_private \
2610     libplatform \
2611     libimc \
2612     libimc_syscalls \
2613     libsrpc \
2614     libgio
2615
2616   local out_dir_prefix="${SCONS_OUT}"/nacl-x86-32-pnacl-pexe-clang
2617   local outdir="${out_dir_prefix}"/lib
2618   mkdir -p "${SDK_INSTALL_LIB}"
2619   cp "${outdir}"/lib*_private.a \
2620      "${outdir}"/lib{platform,imc,imc_syscalls,srpc,gio}.a "${SDK_INSTALL_LIB}"
2621   spopd
2622 }
2623
2624 sdk-verify() {
2625   sdk-setup "$@"
2626   StepBanner "SDK" "Verify"
2627
2628   # Verify bitcode libraries
2629   verify-bitcode-dir "${SDK_INSTALL_LIB}"
2630 }
2631
2632 newlib-nacl-headers-clean() {
2633   # Clean the include directory and revert it to its pure state
2634   if [ -d "${TC_SRC_NEWLIB}" ]; then
2635     rm -rf "${NEWLIB_INCLUDE_DIR}"
2636     # If the script is interrupted right here,
2637     # then NEWLIB_INCLUDE_DIR will not exist, and the repository
2638     # will be in a bad state. This will be fixed during the next
2639     # invocation by newlib-nacl-headers.
2640
2641     spushd "$(dirname "${NEWLIB_INCLUDE_DIR}")"
2642     RunWithLog "newlib-nacl-headers-clean" \
2643       ${GIT} checkout "$(basename "${NEWLIB_INCLUDE_DIR}")"
2644     spopd
2645   fi
2646 }
2647
2648 #+ newlib-nacl-headers   - Install NaCl headers to newlib
2649 newlib-nacl-headers() {
2650   StepBanner "newlib-nacl-headers" "Adding nacl headers to newlib"
2651
2652   assert-dir "${TC_SRC_NEWLIB}" "Newlib is not checked out"
2653
2654   # Make sure the headers directory has no local changes
2655   newlib-nacl-headers-check
2656   newlib-nacl-headers-clean
2657
2658   # Install the headers
2659   "${EXPORT_HEADER_SCRIPT}" \
2660       "${NACL_SYS_HEADERS}" \
2661       "${NEWLIB_INCLUDE_DIR}"
2662
2663   # Record the header install time
2664   ts-touch "${NACL_HEADERS_TS}"
2665 }
2666
2667 #+ newlib-nacl-headers-check - Make sure the newlib nacl headers haven't
2668 #+                             been modified since the last install.
2669 newlib-nacl-headers-check() {
2670   # The condition where NEWLIB_INCLUDE_DIR does not exist may have been
2671   # caused by an incomplete call to newlib-nacl-headers-clean().
2672   # Let it pass this check so that the clean will be able to finish.
2673   # See the comment in newlib-nacl-headers-clean()
2674   if ! [ -d "${TC_SRC_NEWLIB}" ] ||
2675      ! [ -d "${NEWLIB_INCLUDE_DIR}" ]; then
2676     return 0
2677   fi
2678
2679   # Already clean?
2680   if ! git-has-changes "${NEWLIB_INCLUDE_DIR}" ; then
2681     return 0
2682   fi
2683
2684   if ts-dir-changed "${NACL_HEADERS_TS}" "${NEWLIB_INCLUDE_DIR}"; then
2685     echo ""
2686     echo "*******************************************************************"
2687     echo "*                            ERROR                                *"
2688     echo "*      The NewLib include directory has local modifications       *"
2689     echo "*******************************************************************"
2690     echo "* The NewLib include directory should not be modified directly.   *"
2691     echo "* Instead, modifications should be done from:                     *"
2692     echo "*   src/trusted/service_runtime/include                           *"
2693     echo "*                                                                 *"
2694     echo "* To destroy the local changes to newlib, run:                    *"
2695     echo "*  pnacl/build.sh newlib-nacl-headers-clean                       *"
2696     echo "*******************************************************************"
2697     echo ""
2698     if ${PNACL_BUILDBOT} ; then
2699       newlib-nacl-headers-clean
2700     else
2701       exit -1
2702     fi
2703   fi
2704 }
2705
2706
2707 #+-------------------------------------------------------------------------
2708 #@ driver                - Install driver scripts.
2709 driver() {
2710   StepBanner "DRIVER"
2711   driver-install
2712 }
2713
2714 # install python scripts and redirector shell/batch scripts
2715 driver-install-python() {
2716   local destdir="$1"
2717   shift
2718   local pydir="${destdir}/pydir"
2719
2720   StepBanner "DRIVER" "Installing driver adaptors to ${destdir}"
2721   rm -rf "${destdir}"
2722   mkdir -p "${destdir}"
2723   mkdir -p "${pydir}"
2724
2725   spushd "${DRIVER_DIR}"
2726
2727   # Copy python scripts
2728   cp $@ driver_log.py driver_env.py *tools.py filetype.py loader.py "${pydir}"
2729
2730   # Install redirector shell/batch scripts
2731   for name in $@; do
2732     local dest="${destdir}/${name/.py}"
2733     # In some situations cygwin cp messes up the permissions of the redirector
2734     # shell/batch scripts. Using cp -a seems to make sure it preserves them.
2735     cp -a redirect.sh "${dest}"
2736     chmod +x "${dest}"
2737     if ${BUILD_PLATFORM_WIN}; then
2738       cp -a redirect.bat "${dest}".bat
2739     fi
2740   done
2741   spopd
2742 }
2743
2744 feature-version-file-install() {
2745   # Scons tests can check this version number to decide whether to
2746   # enable tests for toolchain bug fixes or new features.  This allows
2747   # tests to be enabled on the toolchain buildbots/trybots before the
2748   # new toolchain version is rolled into TOOL_REVISIONS (i.e. before
2749   # the tests would pass on the main NaCl buildbots/trybots).
2750   #
2751   # If you are adding a test that depends on a toolchain change, you
2752   # can increment this version number manually.
2753   echo 4 > "${INSTALL_ROOT}/FEATURE_VERSION"
2754 }
2755
2756 # The driver is a simple python script which changes its behavior
2757 # depending on the name it is invoked as.
2758 driver-install() {
2759   local bindir=bin
2760   # On Linux we ship a fat toolchain with 2 sets of binaries defaulting to
2761   # x86-32 (mostly because of the 32 bit chrome bots). So the default
2762   # bin dir is 32, and the bin64 driver runs the 64 bit binaries
2763   if ${HOST_ARCH_X8664} && ${BUILD_PLATFORM_LINUX}; then
2764     bindir="bin64"
2765     # We want to be able to locally test a toolchain on 64 bit hosts without
2766     # building it twice and without extra env vars. So if a 32 bit toolchain
2767     # has not already been built, just symlink the bin dirs together.
2768     if [[ ! -d "${INSTALL_BIN}" ]]; then
2769       mkdir -p "${INSTALL_ROOT}"
2770       ln -s ${bindir} "${INSTALL_BIN}"
2771     fi
2772   fi
2773
2774   # This directory (the ${INSTALL_ROOT}/${bindir} part)
2775   # should be kept in sync with INSTALL_BIN et al.
2776   local destdir="${INSTALL_ROOT}/${bindir}"
2777
2778   driver-install-python "${destdir}" "pnacl-*.py"
2779
2780   # Tell the driver the library mode and host arch
2781   echo """HAS_FRONTEND=1
2782 HOST_ARCH=${HOST_ARCH}""" > "${destdir}"/driver.conf
2783
2784   # On windows, copy the cygwin DLLs needed by the driver tools
2785   if ${BUILD_PLATFORM_WIN}; then
2786     StepBanner "DRIVER" "Copying cygwin libraries"
2787     local deps="gcc_s-1 iconv-2 win1 intl-8 stdc++-6 z"
2788     for name in ${deps}; do
2789       cp "/bin/cyg${name}.dll" "${destdir}"
2790     done
2791   fi
2792
2793   # Install a REV file so that "pnacl-clang --version" knows the version
2794   # of the drivers themselves.
2795   DumpAllRevisions > "${destdir}/REV"
2796
2797   feature-version-file-install
2798 }
2799
2800 #@ driver-install-translator - Install driver scripts for translator component
2801 driver-install-translator() {
2802   local destdir="${INSTALL_TRANSLATOR}/bin"
2803
2804   driver-install-python "${destdir}" pnacl-translate.py pnacl-nativeld.py
2805
2806   echo """HAS_FRONTEND=0""" > "${destdir}"/driver.conf
2807 }
2808
2809 ######################################################################
2810 ######################################################################
2811 #
2812 #                           HELPER FUNCTIONS
2813 #
2814 #             (These should not generally be used directly)
2815 #
2816 ######################################################################
2817 ######################################################################
2818
2819 DumpAllRevisions() {
2820   one-line-rev-info ${NACL_ROOT}
2821   for d in ${PNACL_ROOT}/git/*/ ; do
2822     one-line-rev-info $d
2823   done
2824 }
2825
2826 ######################################################################
2827 ######################################################################
2828 #     < VERIFY >
2829 ######################################################################
2830 ######################################################################
2831
2832 # Note: we could replace this with a modified version of tools/elf_checker.py
2833 #       if we do not want to depend on binutils
2834 readonly NACL_OBJDUMP=${BINUTILS_INSTALL_DIR}/bin/${REAL_CROSS_TARGET}-objdump
2835
2836 # Usage: VerifyArchive <checker> <pattern> <filename>
2837 ExtractAndCheck() {
2838   local checker="$1"
2839   local pattern="$2"
2840   local archive="$3"
2841   local tmp="/tmp/ar-verify-${RANDOM}"
2842   rm -rf ${tmp}
2843   mkdir -p ${tmp}
2844   cp "${archive}" "${tmp}"
2845   spushd ${tmp}
2846   ${PNACL_AR} x $(basename ${archive})
2847   # extract all the files
2848   local count=0
2849   for i in ${pattern} ; do
2850     if [ ! -e "$i" ]; then
2851       # we may also see the unexpanded pattern here if there is no match
2852       continue
2853     fi
2854     count=$((count+1))
2855     ${checker} $i
2856   done
2857   echo "PASS  (${count} files)"
2858   rm -rf "${tmp}"
2859   spopd
2860 }
2861
2862 IsLinkerScript() {
2863   local fname="$1"
2864   local type="$(file --brief --mime-type "${fname}")"
2865   case "$type" in
2866     text/x-c)
2867       # A linker script with C comments looks like C to "file".
2868       return 0
2869       ;;
2870     text/plain)
2871       return 0
2872       ;;
2873   esac
2874   return 1
2875 }
2876
2877 # Usage: VerifyLinkerScript <filename>
2878 VerifyLinkerScript() {
2879   local archive="$1"
2880   # Use preprocessor to strip the C-style comments.
2881   ${PNACL_PP} -xc "${archive}" | awk -v archive="$(basename ${archive})" '
2882     BEGIN { status = 0 }
2883     NF == 0 || $1 == "#" { next }
2884     $1 == "INPUT" && $2 == "(" && $NF == ")" { next }
2885     {
2886       print "FAIL - unexpected linker script(?) contents:", archive
2887       status = 1
2888       exit(status)
2889     }
2890     END { if (status == 0) print "PASS  (trivial linker script)" }
2891 ' || exit -1
2892 }
2893
2894 # Usage: VerifyArchive <checker> <pattern> <filename>
2895 VerifyArchive() {
2896   local checker="$1"
2897   local pattern="$2"
2898   local archive="$3"
2899   echo -n "verify $(basename "${archive}"): "
2900   if IsLinkerScript "${archive}"; then
2901     VerifyLinkerScript "${archive}"
2902   else
2903     ExtractAndCheck "$checker" "$pattern" "$archive"
2904   fi
2905 }
2906
2907 #
2908 # verify-object-llvm <obj>
2909 #
2910 #   Verifies that a given .o file is bitcode and free of ASMSs
2911 verify-object-llvm() {
2912   if ${PNACL_DIS} "$1" -o - | grep asm ; then
2913     echo
2914     echo "ERROR asm in $1"
2915     echo
2916     exit -1
2917   fi
2918   if [ ${PIPESTATUS[0]} -ne 0 ]; then
2919     exit -1
2920   fi
2921 }
2922
2923
2924 check-elf-abi() {
2925   # Temporarily disable ELF abi check until DEPS roll
2926   return 0
2927
2928   local arch_info="$(${NACL_OBJDUMP} -f $1)"
2929   if ! grep -q $2 <<< ${arch_info} ; then
2930     echo "ERROR $1 - bad file format: $2 vs ${arch_info}\n"
2931     echo ${arch_info}
2932     exit -1
2933   fi
2934 }
2935
2936
2937 # verify-object-arm <obj>
2938 #
2939 #   Ensure that the ARCH properties are what we expect, this is a little
2940 #   fragile and needs to be updated when tools change
2941 verify-object-arm() {
2942   check-elf-abi $1 "elf32-littlearm"
2943   # llvm-mc does not automatically insert these tags (unlike gnu-as).
2944   # So we exclude llvm-mc generated object files for now
2945   if [[ $1 == aeabi_read_tp.o || $1 == setjmp.o ]] ; then
2946     return
2947   fi
2948   arch_info="$("${PNACL_READELF}" -A "$1")"
2949   #TODO(robertm): some refactoring and cleanup needed
2950   if ! grep -q "Tag_FP_arch: VFPv3" <<< ${arch_info} ; then
2951     echo "ERROR $1 - bad Tag_FP_arch"
2952     #TODO(robertm): figure out what the right thing to do is here, c.f.
2953     # http://code.google.com/p/nativeclient/issues/detail?id=966
2954     "${PNACL_READELF}" -A $1 | grep  Tag_FP_arch
2955     exit -1
2956   fi
2957
2958   if ! grep -q "Tag_CPU_arch: v7" <<< ${arch_info} ; then
2959     echo "FAIL bad $1 Tag_CPU_arch"
2960     "${PNACL_READELF}" -A $1 | grep Tag_CPU_arch
2961     exit -1
2962   fi
2963
2964   if ! grep -q "Tag_Advanced_SIMD_arch: NEONv1" <<< ${arch_info} ; then
2965     echo "FAIL bad $1 Tag_Advanced_SIMD_arch"
2966     "${PNACL_READELF}" -A $1 | grep Tag_Advanced_SIMD_arch
2967   fi
2968
2969   # Check that the file uses the ARM hard-float ABI (where VFP
2970   # registers D0-D7 (s0-s15) are used to pass arguments and results).
2971   if ! grep -q "Tag_ABI_VFP_args: VFP registers" <<< ${arch_info} ; then
2972     echo "FAIL bad $1 Tag_ABI_VFP_args"
2973     "${PNACL_READELF}" -A $1 | grep Tag_ABI_VFP_args
2974   fi
2975 }
2976
2977
2978 # verify-object-x86-32 <obj>
2979 #
2980 verify-object-x86-32() {
2981   check-elf-abi $1 "elf32-i386"
2982 }
2983
2984 # verify-object-x86-64 <obj>
2985 #
2986 verify-object-x86-64() {
2987   check-elf-abi $1 "elf64-x86-64"
2988 }
2989
2990 #+ verify-bitcode-dir    - Verify that the files in a directory are bitcode.
2991 verify-bitcode-dir() {
2992   local dir="$1"
2993   # This avoids errors when * finds no matches.
2994   shopt -s nullglob
2995   SubBanner "VERIFY: ${dir}"
2996   for i in "${dir}"/*.a ; do
2997     verify-archive-llvm "$i"
2998   done
2999   for i in "${dir}"/*.bc ; do
3000     echo -n "verify $(basename "$i"): "
3001     verify-object-llvm "$i"
3002     echo "PASS (bitcode)"
3003   done
3004   for i in "${dir}"/*.o ; do
3005     Fatal "Native object file $i inside bitcode directory"
3006   done
3007   shopt -u nullglob
3008 }
3009
3010
3011 #+ verify-native-dir     - Verify that files in a directory are native for arch.
3012 verify-native-dir() {
3013   local arch="$1"
3014   local dir="$2"
3015
3016   SubBanner "VERIFY: ${dir}"
3017
3018   # This avoids errors when * finds no matches.
3019   shopt -s nullglob
3020   for i in "${dir}"/*.o ; do
3021     verify-object-${arch} "$i"
3022   done
3023
3024   for i in "${dir}"/*.a ; do
3025     verify-archive-${arch} "$i"
3026   done
3027
3028   for i in "${dir}"/*.bc "${dir}"/*.pso ; do
3029     Fatal "Bitcode file $i found inside native directory"
3030   done
3031   shopt -u nullglob
3032 }
3033
3034 #
3035 # verify-archive-llvm <archive>
3036 # Verifies that a given archive is bitcode and free of ASMSs
3037 #
3038 verify-archive-llvm() {
3039   # Currently all the files are .o in the llvm archives.
3040   # Eventually more and more should be .bc.
3041   VerifyArchive verify-object-llvm '*.bc *.o' "$@"
3042 }
3043
3044 #
3045 # verify-archive-arm <archive>
3046 # Verifies that a given archive is a proper arm achive
3047 #
3048 verify-archive-arm() {
3049   VerifyArchive verify-object-arm '*.o *.ons' "$@"
3050 }
3051
3052 #
3053 # verify-archive-x86-32 <archive>
3054 # Verifies that a given archive is a proper x86-32 achive
3055 #
3056 verify-archive-x86-32() {
3057   VerifyArchive verify-object-x86-32 '*.o *.ons' "$@"
3058 }
3059
3060 #
3061 # verify-archive-x86-64 <archive>
3062 # Verifies that a given archive is a proper x86-64 achive
3063 #
3064 verify-archive-x86-64() {
3065   VerifyArchive verify-object-x86-64 '*.o *.ons' "$@"
3066 }
3067
3068 #@-------------------------------------------------------------------------
3069 #+ verify                - Verifies that toolchain/pnacl-untrusted ELF files
3070 #+                         are of the correct architecture.
3071 verify() {
3072   StepBanner "VERIFY"
3073   verify-bitcode
3074   verify-native
3075 }
3076
3077 verify-bitcode() {
3078   verify-bitcode-dir "${INSTALL_LIB}"
3079 }
3080
3081 verify-native() {
3082   local arch
3083   for arch in arm x86-32 x86-64; do
3084     verify-native-dir ${arch} "${INSTALL_LIB_NATIVE}${arch}"
3085   done
3086 }
3087
3088 #+ verify-triple-build <arch>
3089 #+     Verify that the sandboxed translator produces an identical
3090 #+     translation of itself (pnacl-llc.pexe) as the unsandboxed translator.
3091 #+     (NOTE: This function is experimental/untested)
3092 verify-triple-build() {
3093   local arch=$1
3094   StepBanner "VERIFY" "Verifying triple build for ${arch}"
3095
3096   local bindir="$(GetTranslatorInstallDir ${arch})/bin"
3097   local llc_nexe="${bindir}/pnacl-llc.nexe"
3098   local llc_pexe="${bindir}/pnacl-llc.pexe"
3099   assert-file "${llc_nexe}" "sandboxed llc for ${arch} does not exist"
3100   assert-file "${llc_pexe}" "pnacl-llc.pexe does not exist"
3101
3102   local flags="--pnacl-sb --pnacl-driver-verbose"
3103
3104   if [ ${arch} == "arm" ] ; then
3105     # Use emulator if we are not on ARM
3106     local hostarch=$(uname -m)
3107     if ! [[ "${BUILD_ARCH}" =~ arm ]]; then
3108       flags+=" --pnacl-use-emulator"
3109     fi
3110   fi
3111
3112   local triple_install_dir="$(GetTranslatorInstallDir ${arch})/triple-build"
3113   mkdir -p ${triple_install_dir}
3114   local new_llc_nexe="${triple_install_dir}/pnacl-llc.rebuild.nexe"
3115   mkdir -p "${triple_install_dir}"
3116   StepBanner "VERIFY" "Translating ${llc_pexe} using sandboxed tools (${arch})"
3117   local sb_translator="${INSTALL_TRANSLATOR}/bin/pnacl-translate"
3118   RunWithLog "verify.triple.build" \
3119     "${sb_translator}" ${flags} -arch ${arch} "${llc_pexe}" -o "${new_llc_nexe}"
3120
3121   if ! cmp --silent "${llc_nexe}" "${new_llc_nexe}" ; then
3122     Banner "TRIPLE BUILD VERIFY FAILED"
3123     echo "Expected these files to be identical, but they are not:"
3124     echo "  ${archllc}"
3125     echo "  ${newllc}"
3126     exit -1
3127   fi
3128   StepBanner "VERIFY" "Verified ${arch} OK"
3129 }
3130
3131 ######################################################################
3132 ######################################################################
3133 #
3134 # UTILITIES
3135 #
3136 ######################################################################
3137 ######################################################################
3138
3139 #@-------------------------------------------------------------------------
3140 #@ show-config
3141 show-config() {
3142   Banner "Config Settings:"
3143   echo "PNACL_BUILDBOT:    ${PNACL_BUILDBOT}"
3144   echo "PNACL_CONCURRENCY: ${PNACL_CONCURRENCY}"
3145   echo "PNACL_DEBUG:       ${PNACL_DEBUG}"
3146   echo "PNACL_PRUNE:       ${PNACL_PRUNE}"
3147   echo "PNACL_VERBOSE:     ${PNACL_VERBOSE}"
3148   echo "INSTALL_ROOT:      ${INSTALL_ROOT}"
3149   Banner "Your Environment:"
3150   env | grep PNACL
3151   Banner "uname info for builder:"
3152   uname -a
3153   Banner "Revisions:"
3154   DumpAllRevisions
3155 }
3156
3157 #@ help                  - Usage information.
3158 help() {
3159   Usage
3160 }
3161
3162 #@ help-full             - Usage information including internal functions.
3163 help-full() {
3164   Usage2
3165 }
3166
3167 has-trusted-toolchain() {
3168   if [ -f ${TOOLCHAIN_ROOT}/linux_arm-trusted/ld_script_arm_trusted ]; then
3169     return 0
3170   else
3171     return 1
3172   fi
3173 }
3174
3175 check-for-trusted() {
3176   if ! ${PNACL_BUILD_ARM} ; then
3177     return
3178   fi
3179
3180   if ! has-trusted-toolchain; then
3181     echo '*******************************************************************'
3182     echo '*   The ARM trusted toolchain does not appear to be installed yet *'
3183     echo '*   It is needed to run ARM tests.                                *'
3184     echo '*                                                                 *'
3185     echo '*   To download and install the trusted toolchain, run:           *'
3186     echo '*                                                                 *'
3187     echo '*       $ pnacl/build.sh download-trusted                         *'
3188     echo '*                                                                 *'
3189     echo '*   To compile the trusted toolchain, use:                        *'
3190     echo '*                                                                 *'
3191     echo '*       $ tools/llvm/trusted-toolchain-creator.sh trusted_sdk     *'
3192     echo '*               (warning: this takes a while)                     *'
3193     echo '*******************************************************************'
3194
3195     # If building on the bots, do not continue since it needs to run ARM tests.
3196     if ${PNACL_BUILDBOT} ; then
3197       echo "Building on bots --> need ARM trusted toolchain to run tests!"
3198       exit -1
3199     elif trusted-tc-confirm ; then
3200       echo "Continuing without ARM trusted toolchain"
3201       PNACL_BUILD_ARM=false
3202     else
3203       echo "Okay, stopping."
3204       exit -1
3205     fi
3206   fi
3207 }
3208
3209 trusted-tc-confirm() {
3210   echo
3211   echo "Do you wish to continue without the ARM trusted TC (skip ARM testing)?"
3212   echo ""
3213   confirm-yes "Continue"
3214   return $?
3215 }
3216
3217 DebugRun() {
3218   if ${PNACL_DEBUG} || ${PNACL_BUILDBOT}; then
3219     "$@"
3220   fi
3221 }
3222
3223 ######################################################################
3224 # Generate chromium perf bot logs for tracking the size of a binary.
3225 #
3226 print-tagged-tool-sizes() {
3227   local tag="$1"
3228   local binary="$2"
3229
3230   # size output look like:
3231   #    text   data     bss     dec    hex  filename
3232   #  354421  16132  168920  539473  83b51  .../tool
3233   local sizes=($(${PNACL_SIZE} -B "${binary}" | grep '[0-9]\+'))
3234   echo "RESULT ${tag}: text= ${sizes[0]} bytes"
3235   echo "RESULT ${tag}: data= ${sizes[1]} bytes"
3236   echo "RESULT ${tag}: bss= ${sizes[2]} bytes"
3237   echo "RESULT ${tag}: total= ${sizes[3]} bytes"
3238 }
3239
3240 ######################################################################
3241 ######################################################################
3242 #
3243 #                           < TIME STAMPING >
3244 #
3245 ######################################################################
3246 ######################################################################
3247
3248 ts-dir-changed() {
3249   local tsfile="$1"
3250   local dir="$2"
3251
3252   if [ -f "${tsfile}" ]; then
3253     local MODIFIED=$(find "${dir}" -type f -newer "${tsfile}")
3254     [ ${#MODIFIED} -gt 0 ]
3255     ret=$?
3256   else
3257     true
3258     ret=$?
3259   fi
3260   return $ret
3261 }
3262
3263 # Check if the source for a given build has been modified
3264 ts-modified() {
3265   local srcdir="$1"
3266   local objdir="$2"
3267   local tsfile="${objdir}/${TIMESTAMP_FILENAME}"
3268
3269   ts-dir-changed "${tsfile}" "${srcdir}"
3270   return $?
3271 }
3272
3273 ts-touch() {
3274   local tsfile="$1"
3275   touch "${tsfile}"
3276 }
3277
3278 # Record the time when make begins, but don't yet
3279 # write that to the timestamp file.
3280 # (Just in case make fails)
3281
3282 ts-touch-open() {
3283   local objdir="$1"
3284   local tsfile="${objdir}/${TIMESTAMP_FILENAME}"
3285   local tsfile_open="${objdir}/${TIMESTAMP_FILENAME}_OPEN"
3286
3287   rm -f "${tsfile}"
3288   touch "${tsfile_open}"
3289 }
3290
3291
3292 # Write the timestamp. (i.e. make has succeeded)
3293
3294 ts-touch-commit() {
3295   local objdir="$1"
3296   local tsfile="${objdir}/${TIMESTAMP_FILENAME}"
3297   local tsfile_open="${objdir}/${TIMESTAMP_FILENAME}_OPEN"
3298
3299   mv -f "${tsfile_open}" "${tsfile}"
3300 }
3301
3302
3303 # ts-newer-than dirA dirB
3304 # Compare the make timestamps in both object directories.
3305 # returns true (0) if dirA is newer than dirB
3306 # returns false (1) otherwise.
3307 #
3308 # This functions errs on the side of returning 0, since
3309 # that forces a rebuild anyway.
3310
3311 ts-newer-than() {
3312   local objdir1="$1"
3313   local objdir2="$2"
3314
3315   local tsfile1="${objdir1}/${TIMESTAMP_FILENAME}"
3316   local tsfile2="${objdir2}/${TIMESTAMP_FILENAME}"
3317
3318   if [ ! -d "${objdir1}" ]; then return 0; fi
3319   if [ ! -d "${objdir2}" ]; then return 0; fi
3320
3321   if [ ! -f "${tsfile1}" ]; then return 0; fi
3322   if [ ! -f "${tsfile2}" ]; then return 0; fi
3323
3324   local MODIFIED=$(find "${tsfile1}" -newer "${tsfile2}")
3325   if [ ${#MODIFIED} -gt 0 ]; then
3326     return 0
3327   fi
3328   return 1
3329 }
3330
3331
3332 # Don't define any functions after this or they won't show up in completions
3333 function-completions() {
3334   if [ $# = 0 ]; then set -- ""; fi
3335   compgen -A function -- $1
3336   exit 0
3337 }
3338
3339 ######################################################################
3340 ######################################################################
3341 #
3342 #                               < MAIN >
3343 #
3344 ######################################################################
3345 ######################################################################
3346
3347 mkdir -p "${INSTALL_ROOT}"
3348 PackageCheck
3349
3350 if [ $# = 0 ]; then set -- help; fi  # Avoid reference to undefined $1.
3351
3352 # Accept one -- argument for some compatibility with google3
3353 if [ $1 = "--tab_completion_word" ]; then
3354   set -- function-completions $2
3355 fi
3356
3357 if [ "$(type -t $1)" != "function" ]; then
3358   #Usage
3359   echo "ERROR: unknown function '$1'." >&2
3360   echo "For help, try:"
3361   echo "    $0 help"
3362   exit 1
3363 fi
3364
3365 "$@"