src/native_client/toolchain_build/toolchain_build.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Recipes for NativeClient toolchain packages.
   7
   8 The real entry plumbing is in toolchain_main.py.
   9 """
  10
  11 import fnmatch
  12 import platform
  13 import os
  14 import re
  15 import sys
  16
  17 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  18 import pynacl.gsd_storage
  19 import pynacl.platform
  20
  21 import command
  22 import toolchain_main
  23
  24
  25 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
  26 NACL_DIR = os.path.dirname(SCRIPT_DIR)
  27
  28
  29 # See command.GenerateGitPatches for the schema of entries in this dict.
  30 # Additionally, each may contain a 'repo' key whose value is the name
  31 # to use in place of the package name when calling GitUrl (below).
  32 GIT_REVISIONS = {
  33     'binutils': {
  34         'rev': '7deface59753c3b249ac08d854d471951796343f',
  35         'upstream-branch': 'upstream/binutils-2_24-branch',
  36         'upstream-name': 'binutils-2.24',
  37         # This is tag binutils-2_24, but Gerrit won't let us push
  38         # non-annotated tags, and the upstream tag is not annotated.
  39         'upstream-base': '237df3fa4a1d939e6fd1af0c3e5029a25a137310',
  40         },
  41     'gcc': {
  42         'rev': 'a2e4a3140c035233409598487c56b76108f8c74d',
  43         'upstream-branch': 'upstream/gcc-4_8-branch',
  44         'upstream-name': 'gcc-4.8.2',
  45          # Upstream tag gcc-4_8_2-release:
  46         'upstream-base': '9bcca88e24e64d4e23636aafa3404088b13bcb0e',
  47         },
  48     'newlib': {
  49         'rev': 'a9ae3c60b36dea3d8a10e18b1b6db952d21268c2',
  50         'upstream-branch': 'upstream/master',
  51         'upstream-name': 'newlib-2.0.0',
  52         # Upstream tag newlib_2_0_0:
  53         'upstream-base': 'c3fc84e062cacc2b3e13c1f6b9151d0cc85392ba',
  54         },
  55     'gdb': {
  56         'rev': '41afdf9d511d95073c4403c838d8ccba582d8eed',
  57         'repo': 'binutils',
  58         'upstream-branch': 'upstream/gdb-7.7-branch',
  59         'upstream-name': 'gdb-7.7',
  60         # Upstream tag gdb-7.7-release:
  61         'upstream-base': 'fe284cd86ba9761655a9281fef470d364e27eb85',
  62         },
  63     }
  64
  65 TAR_FILES = {
  66     'gmp': command.path.join('gmp', 'gmp-5.1.3.tar.bz2'),
  67     'mpfr': command.path.join('mpfr', 'mpfr-3.1.2.tar.bz2'),
  68     'mpc': command.path.join('mpc', 'mpc-1.0.2.tar.gz'),
  69     'isl': command.path.join('cloog', 'isl-0.12.2.tar.bz2'),
  70     'cloog': command.path.join('cloog', 'cloog-0.18.1.tar.gz'),
  71     'expat': command.path.join('expat', 'expat-2.0.1.tar.gz'),
  72     }
  73
  74 GIT_BASE_URL = 'https://chromium.googlesource.com/native_client'
  75
  76
  77 def GitUrl(package):
  78   repo = GIT_REVISIONS[package].get('repo', package)
  79   return '%s/nacl-%s.git' % (GIT_BASE_URL, repo)
  80
  81
  82 def CollectSources():
  83   sources = {}
  84
  85   for package in TAR_FILES:
  86     tar_file = TAR_FILES[package]
  87     if fnmatch.fnmatch(tar_file, '*.bz2'):
  88       extract = EXTRACT_STRIP_TBZ2
  89     elif fnmatch.fnmatch(tar_file, '*.gz'):
  90       extract = EXTRACT_STRIP_TGZ
  91     else:
  92       raise Exception('unexpected file name pattern in TAR_FILES[%r]' % package)
  93     sources[package] = {
  94         'type': 'source',
  95         'commands': [
  96             command.Command(extract + [command.path.join('%(abs_top_srcdir)s',
  97                                                          '..', 'third_party',
  98                                                          tar_file)],
  99                             cwd='%(output)s'),
 100             ],
 101         }
 102
 103   patch_packages = []
 104   patch_commands = []
 105   for package, info in GIT_REVISIONS.iteritems():
 106     sources[package] = {
 107         'type': 'source',
 108         'commands': [command.SyncGitRepo(GitUrl(package), '%(output)s',
 109                                          info['rev'])],
 110         }
 111     patch_packages.append(package)
 112     patch_commands.append(
 113         command.GenerateGitPatches('%(' + package + ')s/.git', info))
 114
 115   sources['patches'] = {
 116       'type': 'build',
 117       'dependencies': patch_packages,
 118       'commands': patch_commands,
 119       }
 120
 121   # The gcc_libs component gets the whole GCC source tree.
 122   sources['gcc_libs'] = sources['gcc']
 123
 124   # The gcc component omits all the source directories that are used solely
 125   # for building target libraries.  We don't want those included in the
 126   # input hash calculation so that we don't rebuild the compiler when the
 127   # the only things that have changed are target libraries.
 128   sources['gcc'] = {
 129         'type': 'source',
 130         'dependencies': ['gcc_libs'],
 131         'commands': [command.CopyTree('%(gcc_libs)s', '%(output)s', [
 132             'boehm-gc',
 133             'libada',
 134             'libatomic',
 135             'libffi',
 136             'libgcc',
 137             'libgfortran',
 138             'libgo',
 139             'libgomp',
 140             'libitm',
 141             'libjava',
 142             'libmudflap',
 143             'libobjc',
 144             'libquadmath',
 145             'libsanitizer',
 146             'libssp',
 147             'libstdc++-v3',
 148             ])]
 149       }
 150
 151   # We have to populate the newlib source tree with the "exported" form of
 152   # some headers from the native_client source tree.  The newlib build
 153   # needs these to be in the expected place.  By doing this in the source
 154   # target, these files will be part of the input hash and so we don't need
 155   # to do anything else to keep track of when they might have changed in
 156   # the native_client source tree.
 157   newlib_sys_nacl = command.path.join('%(output)s',
 158                                       'newlib', 'libc', 'sys', 'nacl')
 159   newlib_unpack = [command.RemoveDirectory(command.path.join(newlib_sys_nacl,
 160                                                              dirname))
 161                    for dirname in ['bits', 'sys', 'machine']]
 162   newlib_unpack.append(command.Command([
 163       'python',
 164       command.path.join('%(top_srcdir)s', 'src',
 165                         'trusted', 'service_runtime', 'export_header.py'),
 166       command.path.join('%(top_srcdir)s', 'src',
 167                         'trusted', 'service_runtime', 'include'),
 168       newlib_sys_nacl,
 169       ]))
 170   sources['newlib']['commands'] += newlib_unpack
 171
 172   return sources
 173
 174
 175 # Canonical tuples we use for hosts.
 176 WINDOWS_HOST_TUPLE = pynacl.platform.PlatformTriple('win', 'x86-64')
 177 MAC_HOST_TUPLE = pynacl.platform.PlatformTriple('darwin', 'x86-64')
 178 ARM_HOST_TUPLE = pynacl.platform.PlatformTriple('linux', 'arm')
 179 LINUX_X86_32_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-32')
 180 LINUX_X86_64_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-64')
 181
 182 # Map of native host tuple to extra tuples that it cross-builds for.
 183 EXTRA_HOSTS_MAP = {
 184     LINUX_X86_64_TUPLE: [
 185         LINUX_X86_32_TUPLE,
 186         ARM_HOST_TUPLE,
 187         WINDOWS_HOST_TUPLE,
 188         ],
 189     }
 190
 191 # Map of native host tuple to host tuples that are "native enough".
 192 # For these hosts, we will do a native-style build even though it's
 193 # not the native tuple, just passing some extra compiler flags.
 194 NATIVE_ENOUGH_MAP = {
 195     LINUX_X86_64_TUPLE: {
 196         LINUX_X86_32_TUPLE: ['-m32'],
 197         },
 198     }
 199
 200
 201 # The list of targets to build toolchains for.
 202 TARGET_LIST = ['arm', 'i686']
 203
 204 # These are extra arguments to pass gcc's configure that vary by target.
 205 TARGET_GCC_CONFIG = {
 206 # TODO(mcgrathr): Disabled tuning for now, tickling a constant-pool layout bug.
 207 #    'arm': ['--with-tune=cortex-a15'],
 208     }
 209
 210 PACKAGE_NAME = 'Native Client SDK [%(build_signature)s]'
 211 BUG_URL = 'http://gonacl.com/reportissue'
 212
 213 TAR_XV = ['tar', '-x', '-v']
 214 EXTRACT_STRIP_TGZ = TAR_XV + ['--gzip', '--strip-components=1', '-f']
 215 EXTRACT_STRIP_TBZ2 = TAR_XV + ['--bzip2', '--strip-components=1', '-f']
 216 CONFIGURE_CMD = ['sh', '%(src)s/configure']
 217 MAKE_PARALLEL_CMD = ['make', '-j%(cores)s']
 218 MAKE_CHECK_CMD = MAKE_PARALLEL_CMD + ['check']
 219 MAKE_DESTDIR_CMD = ['make', 'DESTDIR=%(abs_output)s']
 220
 221 # This file gets installed by multiple packages' install steps, but it is
 222 # never useful when installed in isolation.  So we remove it from the
 223 # installation directories before packaging up.
 224 REMOVE_INFO_DIR = command.Remove(command.path.join('%(output)s',
 225                                                    'share', 'info', 'dir'))
 226
 227 def ConfigureHostArch(host):
 228   configure_args = []
 229
 230   is_cross = CrossCompiling(host)
 231
 232   if is_cross:
 233     extra_cc_args = []
 234     configure_args.append('--host=' + host)
 235   else:
 236     extra_cc_args = NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}).get(host, [])
 237     if extra_cc_args:
 238       # The host we've chosen is "native enough", such as x86-32 on x86-64.
 239       # But it's not what config.guess will yield, so we need to supply
 240       # a --build switch to ensure things build correctly.
 241       configure_args.append('--build=' + host)
 242
 243   extra_cxx_args = list(extra_cc_args)
 244   if fnmatch.fnmatch(host, '*-linux*'):
 245     # Avoid shipping binaries with a runtime dependency on
 246     # a particular version of the libstdc++ shared library.
 247     # TODO(mcgrathr): Do we want this for MinGW and/or Mac too?
 248     extra_cxx_args.append('-static-libstdc++')
 249
 250   if extra_cc_args:
 251     # These are the defaults when there is no setting, but we will add
 252     # additional switches, so we must supply the command name too.
 253     if is_cross:
 254       cc = host + '-gcc'
 255     else:
 256       cc = 'gcc'
 257     configure_args.append('CC=' + ' '.join([cc] + extra_cc_args))
 258
 259   if extra_cxx_args:
 260     # These are the defaults when there is no setting, but we will add
 261     # additional switches, so we must supply the command name too.
 262     if is_cross:
 263       cxx = host + '-g++'
 264     else:
 265       cxx = 'g++'
 266     configure_args.append('CXX=' + ' '.join([cxx] + extra_cxx_args))
 267
 268   if HostIsWindows(host):
 269     # The i18n support brings in runtime dependencies on MinGW DLLs
 270     # that we don't want to have to distribute alongside our binaries.
 271     # So just disable it, and compiler messages will always be in US English.
 272     configure_args.append('--disable-nls')
 273
 274   return configure_args
 275
 276
 277 def ConfigureHostCommon(host):
 278   return ConfigureHostArch(host) + [
 279       '--prefix=',
 280       '--disable-silent-rules',
 281       '--without-gcc-arch',
 282       ]
 283
 284
 285 def ConfigureHostLib(host):
 286   return ConfigureHostCommon(host) + [
 287       '--disable-shared',
 288       ]
 289
 290
 291 def ConfigureHostTool(host):
 292   return ConfigureHostCommon(host) + [
 293       '--with-pkgversion=' + PACKAGE_NAME,
 294       '--with-bugurl=' + BUG_URL,
 295       '--without-zlib',
 296       ]
 297
 298
 299 def MakeCommand(host, extra_args=[]):
 300   if HostIsWindows(host):
 301     # There appears to be nothing we can pass at top-level configure time
 302     # that will prevent the configure scripts from finding MinGW's libiconv
 303     # and using it.  We have to force this variable into the environment
 304     # of the sub-configure runs, which are run via make.
 305     make_command = MAKE_PARALLEL_CMD + ['HAVE_LIBICONV=no']
 306   else:
 307     make_command = MAKE_PARALLEL_CMD
 308   return make_command + extra_args
 309
 310
 311 # Return the 'make check' command to run.
 312 # When cross-compiling, don't try to run test suites.
 313 def MakeCheckCommand(host):
 314   if CrossCompiling(host):
 315     return ['true']
 316   return MAKE_CHECK_CMD
 317
 318
 319 def InstallDocFiles(subdir, files):
 320   doc_dir = command.path.join('%(output)s', 'share', 'doc', subdir)
 321   dirs = sorted(set([command.path.dirname(command.path.join(doc_dir, file))
 322                      for file in files]))
 323   commands = ([command.Mkdir(dir, parents=True) for dir in dirs] +
 324               [command.Copy(command.path.join('%(' + subdir + ')s', file),
 325                             command.path.join(doc_dir, file))
 326                for file in files])
 327   return commands
 328
 329
 330 def NewlibLibcScript(arch):
 331   template = """/*
 332  * This is a linker script that gets installed as libc.a for the
 333  * newlib-based NaCl toolchain.  It brings in the constituent
 334  * libraries that make up what -lc means semantically.
 335  */
 336 OUTPUT_FORMAT(%s)
 337 GROUP ( libnacl.a libcrt_common.a )
 338 """
 339   if arch == 'arm':
 340     # Listing three formats instead of one makes -EL/-EB switches work
 341     # for the endian-switchable ARM backend.
 342     format_list = ['elf32-littlearm-nacl',
 343                    'elf32-bigarm-nacl',
 344                    'elf32-littlearm-nacl']
 345   elif arch == 'i686':
 346     format_list = 'elf32-i386-nacl'
 347   elif arch == 'x86_64':
 348     format_list = 'elf32-x86_64-nacl'
 349   else:
 350     raise Exception('TODO(mcgrathr): OUTPUT_FORMAT for %s' % arch)
 351   return template % ', '.join(['"' + fmt + '"' for fmt in format_list])
 352
 353
 354 # The default strip behavior removes debugging and symbol table
 355 # sections, but it leaves the .comment section.  This contains the
 356 # compiler version string, and so it changes when the compiler changes
 357 # even if the actual machine code it produces is completely identical.
 358 # Hence, the target library packages will always change when the
 359 # compiler changes unless these sections are removed.  Doing this
 360 # requires somehow teaching the makefile rules to pass the
 361 # --remove-section=.comment switch to TARGET-strip.  For the GCC
 362 # target libraries, setting STRIP_FOR_TARGET is sufficient.  But
 363 # quoting nightmares make it difficult to pass a command with a space
 364 # in it as the STRIP_FOR_TARGET value.  So the build writes a little
 365 # script that can be invoked with a simple name.
 366 #
 367 # Though the gcc target libraries' makefiles are smart enough to obey
 368 # STRIP_FOR_TARGET for library files, the newlib makefiles just
 369 # blindly use $(INSTALL_DATA) for both header (text) files and library
 370 # files.  Hence it's necessary to override its INSTALL_DATA setting to
 371 # one that will do stripping using this script, and thus the script
 372 # must silently do nothing to non-binary files.
 373 def ConfigureTargetPrep(arch):
 374   script_file = 'strip_for_target'
 375
 376   config_target = arch + '-nacl'
 377   script_contents = """\
 378 #!/bin/sh
 379 mode=--strip-all
 380 for arg; do
 381   case "$arg" in
 382   -*) ;;
 383   *)
 384     type=`file --brief --mime-type "$arg"`
 385     case "$type" in
 386       application/x-executable|application/x-sharedlib) ;;
 387       application/x-archive|application/x-object) mode=--strip-debug ;;
 388       *) exit 0 ;;
 389     esac
 390     ;;
 391   esac
 392 done
 393 exec %s-strip $mode --remove-section=.comment "$@"
 394 """ % config_target
 395
 396   return [
 397       command.WriteData(script_contents, script_file),
 398       command.Command(['chmod', '+x', script_file]),
 399       ]
 400
 401
 402 def ConfigureTargetArgs(arch):
 403   config_target = arch + '-nacl'
 404   return [
 405       '--target=' + config_target,
 406       '--with-sysroot=/' + config_target,
 407       'STRIP_FOR_TARGET=%(cwd)s/strip_for_target',
 408       ]
 409
 410
 411 def CommandsInBuild(command_lines):
 412   return [
 413       command.RemoveDirectory('build'),
 414       command.Mkdir('build'),
 415       ] + [command.Command(cmd, cwd='build')
 416            for cmd in command_lines]
 417
 418
 419 def PopulateDeps(dep_dirs):
 420   commands = [command.RemoveDirectory('all_deps'),
 421               command.Mkdir('all_deps')]
 422   commands += [command.Command('cp -r "%s/"* all_deps' % dirname, shell=True)
 423                for dirname in dep_dirs]
 424   return commands
 425
 426
 427 def WithDepsOptions(options, component=None):
 428   if component is None:
 429     directory = command.path.join('%(cwd)s', 'all_deps')
 430   else:
 431     directory = '%(abs_' + component + ')s'
 432   return ['--with-' + option + '=' + directory
 433           for option in options]
 434
 435
 436 # Return the component name we'll use for a base component name and
 437 # a host tuple.  The component names cannot contain dashes or other
 438 # non-identifier characters, because the names of the files uploaded
 439 # to Google Storage are constrained.  GNU configuration tuples contain
 440 # dashes, which we translate to underscores.
 441 def ForHost(component_name, host):
 442   return component_name + '_' + pynacl.gsd_storage.LegalizeName(host)
 443
 444
 445 # These are libraries that go into building the compiler itself.
 446 def HostGccLibs(host):
 447   def H(component_name):
 448     return ForHost(component_name, host)
 449   host_gcc_libs = {
 450       H('gmp'): {
 451           'type': 'build',
 452           'dependencies': ['gmp'],
 453           'commands': [
 454               command.Command(ConfigureCommand('gmp') +
 455                               ConfigureHostLib(host) + [
 456                                   '--with-sysroot=%(abs_output)s',
 457                                   '--enable-cxx',
 458                                   # Without this, the built library will
 459                                   # assume the instruction set details
 460                                   # available on the build machine.  With
 461                                   # this, it dynamically chooses what code
 462                                   # to use based on the details of the
 463                                   # actual host CPU at runtime.
 464                                   '--enable-fat',
 465                                   ]),
 466               command.Command(MakeCommand(host)),
 467               command.Command(MakeCheckCommand(host)),
 468               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 469               ],
 470           },
 471       H('mpfr'): {
 472           'type': 'build',
 473           'dependencies': ['mpfr', H('gmp')],
 474           'commands': [
 475               command.Command(ConfigureCommand('mpfr') +
 476                               ConfigureHostLib(host) +
 477                               WithDepsOptions(['sysroot', 'gmp'], H('gmp'))),
 478               command.Command(MakeCommand(host)),
 479               command.Command(MakeCheckCommand(host)),
 480               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 481               ],
 482           },
 483       H('mpc'): {
 484           'type': 'build',
 485           'dependencies': ['mpc', H('gmp'), H('mpfr')],
 486           'commands': PopulateDeps(['%(' + H('gmp') + ')s',
 487                                     '%(' + H('mpfr') + ')s']) + [
 488               command.Command(ConfigureCommand('mpc') +
 489                               ConfigureHostLib(host) +
 490                               WithDepsOptions(['sysroot', 'gmp', 'mpfr'])),
 491               command.Command(MakeCommand(host)),
 492               command.Command(MakeCheckCommand(host)),
 493               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 494               ],
 495           },
 496       H('isl'): {
 497           'type': 'build',
 498           'dependencies': ['isl', H('gmp')],
 499           'commands': [
 500               command.Command(ConfigureCommand('isl') +
 501                               ConfigureHostLib(host) +
 502                               WithDepsOptions(['sysroot', 'gmp-prefix'],
 503                                               H('gmp'))),
 504               command.Command(MakeCommand(host)),
 505               command.Command(MakeCheckCommand(host)),
 506               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 507               # The .pc files wind up containing some absolute paths
 508               # that make the output depend on the build directory name.
 509               # The dependents' configure scripts don't need them anyway.
 510               command.RemoveDirectory(command.path.join(
 511                   '%(output)s', 'lib', 'pkgconfig')),
 512               ],
 513           },
 514       H('cloog'): {
 515           'type': 'build',
 516           'dependencies': ['cloog', H('gmp'), H('isl')],
 517           'commands': PopulateDeps(['%(' + H('gmp') + ')s',
 518                                     '%(' + H('isl') + ')s']) + [
 519               command.Command(ConfigureCommand('cloog') +
 520                               ConfigureHostLib(host) + [
 521                                   '--with-bits=gmp',
 522                                   '--with-isl=system',
 523                                   ] + WithDepsOptions(['sysroot',
 524                                                        'gmp-prefix',
 525                                                        'isl-prefix'])),
 526               command.Command(MakeCommand(host)),
 527               command.Command(MakeCheckCommand(host)),
 528               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 529               # The .pc files wind up containing some absolute paths
 530               # that make the output depend on the build directory name.
 531               # The dependents' configure scripts don't need them anyway.
 532               command.RemoveDirectory(command.path.join(
 533                   '%(output)s', 'lib', 'pkgconfig')),
 534               ],
 535           },
 536       H('expat'): {
 537           'type': 'build',
 538           'dependencies': ['expat'],
 539           'commands': [
 540               command.Command(ConfigureCommand('expat') +
 541                               ConfigureHostLib(host)),
 542               command.Command(MakeCommand(host)),
 543               command.Command(MakeCheckCommand(host)),
 544               command.Command(MAKE_DESTDIR_CMD + [
 545                   # expat does not support the install-strip target.
 546                   'installlib',
 547                   'INSTALL=%(expat)s/conftools/install-sh -c -s',
 548                   'INSTALL_DATA=%(expat)s/conftools/install-sh -c -m 644',
 549                   ]),
 550               ],
 551           },
 552       }
 553   return host_gcc_libs
 554
 555
 556 HOST_GCC_LIBS_DEPS = ['gmp', 'mpfr', 'mpc', 'isl', 'cloog']
 557
 558 def HostGccLibsDeps(host):
 559   return [ForHost(package, host) for package in HOST_GCC_LIBS_DEPS]
 560
 561
 562 def ConfigureCommand(source_component):
 563   return [command % {'src': '%(' + source_component + ')s'}
 564           for command in CONFIGURE_CMD]
 565
 566
 567 # When doing a Canadian cross, we need native-hosted cross components
 568 # to do the GCC build.
 569 def GccDeps(host, target):
 570   components = ['binutils_' + target]
 571   if CrossCompiling(host):
 572     components.append('gcc_' + target)
 573     host = NATIVE_TUPLE
 574   return [ForHost(component, host) for component in components]
 575
 576
 577 def GccCommand(host, target, cmd):
 578   components_for_path = GccDeps(host, target)
 579   return command.Command(
 580       cmd, path_dirs=[command.path.join('%(abs_' + component + ')s', 'bin')
 581                       for component in components_for_path])
 582
 583
 584 def ConfigureGccCommand(source_component, host, target, extra_args=[]):
 585   return GccCommand(
 586       host,
 587       target,
 588       ConfigureCommand(source_component) +
 589       ConfigureHostTool(host) +
 590       ConfigureTargetArgs(target) +
 591       TARGET_GCC_CONFIG.get(target, []) + [
 592           '--with-gmp=%(abs_' + ForHost('gmp', host) + ')s',
 593           '--with-mpfr=%(abs_' + ForHost('mpfr', host) + ')s',
 594           '--with-mpc=%(abs_' + ForHost('mpc', host) + ')s',
 595           '--with-isl=%(abs_' + ForHost('isl', host) + ')s',
 596           '--with-cloog=%(abs_' + ForHost('cloog', host) + ')s',
 597           '--enable-cloog-backend=isl',
 598           '--disable-dlopen',
 599           '--disable-shared',
 600           '--with-newlib',
 601           '--with-linker-hash-style=gnu',
 602           '--enable-linker-build-id',
 603           '--enable-languages=c,c++,lto',
 604           ] + extra_args)
 605
 606
 607
 608 def HostTools(host, target):
 609   def H(component_name):
 610     return ForHost(component_name, host)
 611   tools = {
 612       H('binutils_' + target): {
 613           'type': 'build',
 614           'dependencies': ['binutils'],
 615           'commands': ConfigureTargetPrep(target) + [
 616               command.Command(
 617                   ConfigureCommand('binutils') +
 618                   ConfigureHostTool(host) +
 619                   ConfigureTargetArgs(target) + [
 620                       '--enable-deterministic-archives',
 621                       '--enable-gold',
 622                       ] + ([] if HostIsWindows(host) else [
 623                           '--enable-plugins',
 624                           ])),
 625               command.Command(MakeCommand(host)),
 626               command.Command(MakeCheckCommand(host)),
 627               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 628               REMOVE_INFO_DIR,
 629               ] + InstallDocFiles('binutils',
 630                                   ['COPYING3'] +
 631                                   [command.path.join(subdir, 'NEWS')
 632                                    for subdir in
 633                                    ['binutils', 'gas', 'ld', 'gold']]) +
 634               # The top-level lib* directories contain host libraries
 635               # that we don't want to include in the distribution.
 636               [command.RemoveDirectory(command.path.join('%(output)s', name))
 637                for name in ['lib', 'lib32', 'lib64']],
 638           },
 639
 640       H('gcc_' + target): {
 641           'type': 'build',
 642           'dependencies': (['gcc'] + HostGccLibsDeps(host) +
 643                            GccDeps(host, target)),
 644           'commands': ConfigureTargetPrep(target) + [
 645               ConfigureGccCommand('gcc', host, target),
 646               # GCC's configure step writes configargs.h with some strings
 647               # including the configure command line, which get embedded
 648               # into the gcc driver binary.  The build only works if we use
 649               # absolute paths in some of the configure switches, but
 650               # embedding those paths makes the output differ in repeated
 651               # builds done in different directories, which we do not want.
 652               # So force the generation of that file early and then edit it
 653               # in place to replace the absolute paths with something that
 654               # never varies.  Note that the 'configure-gcc' target will
 655               # actually build some components before running gcc/configure.
 656               GccCommand(host, target,
 657                          MakeCommand(host, ['configure-gcc'])),
 658               command.Command(['sed', '-i', '-e',
 659                                ';'.join(['s@%%(abs_%s)s@.../%s_install@g' %
 660                                          (component, component)
 661                                          for component in
 662                                          HostGccLibsDeps(host)] +
 663                                         ['s@%(cwd)s@...@g']),
 664                                command.path.join('gcc', 'configargs.h')]),
 665               # gcc/Makefile's install rules ordinarily look at the
 666               # installed include directory for a limits.h to decide
 667               # whether the lib/gcc/.../include-fixed/limits.h header
 668               # should be made to expect a libc-supplied limits.h or not.
 669               # Since we're doing this build in a clean environment without
 670               # any libc installed, we need to force its hand here.
 671               GccCommand(host, target,
 672                          MakeCommand(host, ['all-gcc', 'LIMITS_H_TEST=true'])),
 673               # gcc/Makefile's install targets populate this directory
 674               # only if it already exists.
 675               command.Mkdir(command.path.join('%(output)s',
 676                                               target + '-nacl', 'bin'),
 677                             True),
 678               GccCommand(host, target,
 679                          MAKE_DESTDIR_CMD + ['install-strip-gcc']),
 680               REMOVE_INFO_DIR,
 681               # Note we include COPYING.RUNTIME here and not with gcc_libs.
 682               ] + InstallDocFiles('gcc', ['COPYING3', 'COPYING.RUNTIME']),
 683           },
 684
 685       # GDB can support all the targets in one host tool.
 686       H('gdb'): {
 687           'type': 'build',
 688           'dependencies': ['gdb', H('expat')],
 689           'commands': [
 690               command.Command(
 691                   ConfigureCommand('gdb') +
 692                   ConfigureHostTool(host) + [
 693                       '--target=x86_64-nacl',
 694                       '--enable-targets=arm-none-eabi-nacl',
 695                       '--with-expat',
 696                       'CPPFLAGS=-I%(abs_' + H('expat') + ')s/include',
 697                       'LDFLAGS=-L%(abs_' + H('expat') + ')s/lib',
 698                       ] +
 699                   (['--without-python'] if HostIsWindows(host) else []) +
 700                   # TODO(mcgrathr): The default -Werror only breaks because
 701                   # the OSX default compiler is an old front-end that does
 702                   # not understand all the GCC options.  Maybe switch to
 703                   # using clang (system or Chromium-supplied) on Mac.
 704                   (['--disable-werror'] if HostIsMac(host) else [])),
 705               command.Command(MakeCommand(host) + ['all-gdb']),
 706               command.Command(MAKE_DESTDIR_CMD + [
 707                   '-C', 'gdb', 'install-strip',
 708                   ]),
 709               REMOVE_INFO_DIR,
 710               ] + InstallDocFiles('gdb', [
 711                   'COPYING3',
 712                   command.path.join('gdb', 'NEWS'),
 713                   ]),
 714           },
 715       }
 716
 717   # TODO(mcgrathr): The ARM cross environment does not supply a termcap
 718   # library, so it cannot build GDB.
 719   if host.startswith('arm') and CrossCompiling(host):
 720     del tools[H('gdb')]
 721
 722   return tools
 723
 724 def TargetCommands(host, target, command_list):
 725   # First we have to copy the host tools into a common directory.
 726   # We can't just have both directories in our PATH, because the
 727   # compiler looks for the assembler and linker relative to itself.
 728   commands = PopulateDeps(['%(' + ForHost('binutils_' + target, host) + ')s',
 729                            '%(' + ForHost('gcc_' + target, host) + ')s'])
 730   bindir = command.path.join('%(cwd)s', 'all_deps', 'bin')
 731   commands += [command.Command(cmd, path_dirs=[bindir])
 732                for cmd in command_list]
 733   return commands
 734
 735
 736 def TargetLibs(host, target):
 737   lib_deps = [ForHost(component + '_' + target, host)
 738               for component in ['binutils', 'gcc']]
 739
 740   def NewlibFile(subdir, name):
 741     return command.path.join('%(output)s', target + '-nacl', subdir, name)
 742
 743   newlib_sysroot = '%(abs_newlib_' + target + ')s'
 744   newlib_tooldir = '%s/%s-nacl' % (newlib_sysroot, target)
 745
 746   # See the comment at ConfigureTargetPrep, above.
 747   newlib_install_data = ' '.join(['STRIPPROG=%(cwd)s/strip_for_target',
 748                                   '%(abs_newlib)s/install-sh',
 749                                   '-c', '-s', '-m', '644'])
 750
 751   iconv_encodings = 'UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4'
 752   newlib_configure_args = [
 753       '--disable-libgloss',
 754       '--enable-newlib-iconv',
 755       '--enable-newlib-iconv-from-encodings=' + iconv_encodings,
 756       '--enable-newlib-iconv-to-encodings=' + iconv_encodings,
 757       '--enable-newlib-io-long-long',
 758       '--enable-newlib-io-long-double',
 759       '--enable-newlib-io-c99-formats',
 760       '--enable-newlib-mb',
 761       'CFLAGS=-O2',
 762       'INSTALL_DATA=' + newlib_install_data,
 763       ]
 764
 765   newlib_post_install = [
 766       command.Rename(NewlibFile('lib', 'libc.a'),
 767                      NewlibFile('lib', 'libcrt_common.a')),
 768       command.WriteData(NewlibLibcScript(target),
 769                         NewlibFile('lib', 'libc.a')),
 770       ] + [
 771       command.Copy(
 772           command.path.join('%(pthread_headers)s', header),
 773           NewlibFile('include', header))
 774       for header in ('pthread.h', 'semaphore.h')
 775       ]
 776
 777
 778   libs = {
 779       'newlib_' + target: {
 780           'type': 'build',
 781           'dependencies': ['newlib'] + lib_deps,
 782           'inputs': { 'pthread_headers':
 783                       os.path.join(NACL_DIR, 'src', 'untrusted',
 784                                    'pthread') },
 785           'commands': (ConfigureTargetPrep(target) +
 786                        TargetCommands(host, target, [
 787                            ConfigureCommand('newlib') +
 788                            ConfigureHostTool(host) +
 789                            ConfigureTargetArgs(target) +
 790                            newlib_configure_args,
 791                            MakeCommand(host),
 792                            MAKE_DESTDIR_CMD + ['install-strip'],
 793                            ]) +
 794                        newlib_post_install +
 795                        InstallDocFiles('newlib', ['COPYING.NEWLIB'])),
 796           },
 797
 798       'gcc_libs_' + target: {
 799           'type': 'build',
 800           'dependencies': (['gcc_libs'] + lib_deps + ['newlib_' + target] +
 801                            HostGccLibsDeps(host)),
 802           # This actually builds the compiler again and uses that compiler
 803           # to build the target libraries.  That's by far the easiest thing
 804           # to get going given the interdependencies of the target
 805           # libraries (especially libgcc) on the gcc subdirectory, and
 806           # building the compiler doesn't really take all that long in the
 807           # grand scheme of things.
 808           # TODO(mcgrathr): If upstream ever cleans up all their
 809           # interdependencies better, unpack the compiler, configure with
 810           # --disable-gcc, and just build all-target.
 811           'commands': ConfigureTargetPrep(target) + [
 812               ConfigureGccCommand('gcc_libs', host, target, [
 813                   '--with-build-sysroot=' + newlib_sysroot,
 814                   ]),
 815               GccCommand(host, target,
 816                          MakeCommand(host) + [
 817                              'build_tooldir=' + newlib_tooldir,
 818                              'all-target',
 819                              ]),
 820               GccCommand(host, target,
 821                          MAKE_DESTDIR_CMD + ['install-strip-target']),
 822               REMOVE_INFO_DIR,
 823               ],
 824           },
 825       }
 826   return libs
 827
 828 # Compute it once.
 829 NATIVE_TUPLE = pynacl.platform.PlatformTriple()
 830
 831
 832 # For our purposes, "cross-compiling" means not literally that we are
 833 # targetting a host that does not match NATIVE_TUPLE, but that we are
 834 # targetting a host whose binaries we cannot run locally.  So x86-32
 835 # on x86-64 does not count as cross-compiling.  See NATIVE_ENOUGH_MAP, above.
 836 def CrossCompiling(host):
 837   return (host != NATIVE_TUPLE and
 838           host not in NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}))
 839
 840
 841 def HostIsWindows(host):
 842   return host == WINDOWS_HOST_TUPLE
 843
 844
 845 def HostIsMac(host):
 846   return host == MAC_HOST_TUPLE
 847
 848
 849 # We build target libraries only on Linux for two reasons:
 850 # 1. We only need to build them once.
 851 # 2. Linux is the fastest to build.
 852 # TODO(mcgrathr): In future set up some scheme whereby non-Linux
 853 # bots can build target libraries but not archive them, only verifying
 854 # that the results came out the same as the ones archived by the
 855 # official builder bot.  That will serve as a test of the host tools
 856 # on the other host platforms.
 857 def BuildTargetLibsOn(host):
 858   return host == LINUX_X86_64_TUPLE
 859
 860
 861 def CollectPackagesForHost(host, targets):
 862   packages = HostGccLibs(host).copy()
 863   for target in targets:
 864     packages.update(HostTools(host, target))
 865     if BuildTargetLibsOn(host):
 866       packages.update(TargetLibs(host, target))
 867   return packages
 868
 869
 870 def CollectPackages(targets):
 871   packages = CollectSources()
 872
 873   packages.update(CollectPackagesForHost(NATIVE_TUPLE, targets))
 874
 875   for host in EXTRA_HOSTS_MAP.get(NATIVE_TUPLE, []):
 876     packages.update(CollectPackagesForHost(host, targets))
 877
 878   return packages
 879
 880
 881 PACKAGES = CollectPackages(TARGET_LIST)
 882
 883
 884 if __name__ == '__main__':
 885   tb = toolchain_main.PackageBuilder(PACKAGES, sys.argv[1:])
 886   # TODO(mcgrathr): The bot ought to run some native_client tests
 887   # using the new toolchain, like the old x86 toolchain bots do.
 888   tb.Main()