src/native_client/toolchain_build/toolchain_build.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Recipes for NativeClient toolchain packages.
   7
   8 The real entry plumbing is in toolchain_main.py.
   9 """
  10
  11 import collections
  12 import fnmatch
  13 import platform
  14 import os
  15 import re
  16 import sys
  17
  18 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  19 import pynacl.gsd_storage
  20 import pynacl.platform
  21
  22 import command
  23 import toolchain_main
  24
  25
  26 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
  27 NACL_DIR = os.path.dirname(SCRIPT_DIR)
  28
  29
  30 # See command.GenerateGitPatches for the schema of entries in this dict.
  31 # Additionally, each may contain a 'repo' key whose value is the name
  32 # to use in place of the package name when calling GitUrl (below).
  33 GIT_REVISIONS = {
  34     'binutils': {
  35         'rev': 'b08b9f0894e43f0bb966f3ad9094a4405ce6f570',
  36         'upstream-branch': 'upstream/binutils-2_24-branch',
  37         'upstream-name': 'binutils-2.24',
  38         # This is tag binutils-2_24, but Gerrit won't let us push
  39         # non-annotated tags, and the upstream tag is not annotated.
  40         'upstream-base': '237df3fa4a1d939e6fd1af0c3e5029a25a137310',
  41         },
  42     'gcc': {
  43         'rev': 'daf66f6da8d77ceb3cc75e63f3a7156abb09d564',
  44         'upstream-branch': 'upstream/gcc-4_9-branch',
  45         'upstream-name': 'gcc-4.9.2',
  46          # Upstream tag gcc-4_9_2-release:
  47         'upstream-base': 'c1283af40b65f1ad862cf5b27e2d9ed10b2076b6',
  48         },
  49     'newlib': {
  50         'rev': 'bf66148d14c7fca26b9198dd5dc81e743893bb66',
  51         'upstream-branch': 'upstream/master',
  52         'upstream-name': 'newlib-2.1.0',
  53         # Upstream tag newlib_2_1_0:
  54         'upstream-base': '99fc6c167467b41466ec90e8260e9c49cbe3d13c',
  55         },
  56     'gdb': {
  57         'rev': '5deb4793a5e3f2f48d7899f424bb4484686020f8',
  58         'repo': 'binutils',
  59         'upstream-branch': 'upstream/gdb-7.7-branch',
  60         'upstream-name': 'gdb-7.7.1',
  61         # Upstream tag gdb-7.7-release:
  62         'upstream-base': '4bd8fc3a1362970d9800a263987af8093798338b',
  63         },
  64     }
  65
  66 TAR_FILES = {
  67     'gmp': command.path.join('gmp', 'gmp-6.0.0a.tar.bz2'),
  68     'mpfr': command.path.join('mpfr', 'mpfr-3.1.2.tar.bz2'),
  69     'mpc': command.path.join('mpc', 'mpc-1.0.2.tar.gz'),
  70     'isl': command.path.join('cloog', 'isl-0.12.2.tar.bz2'),
  71     'cloog': command.path.join('cloog', 'cloog-0.18.1.tar.gz'),
  72     'expat': command.path.join('expat', 'expat-2.1.0.tar.gz'),
  73     }
  74
  75 GIT_BASE_URL = 'https://chromium.googlesource.com/native_client'
  76 GIT_PUSH_URL = 'ssh://gerrit.chromium.org/native_client'
  77
  78 ALT_GIT_BASE_URL = 'https://chromium.googlesource.com/a/native_client'
  79
  80 KNOWN_MIRRORS = [('http://git.chromium.org/native_client', GIT_BASE_URL)]
  81 PUSH_MIRRORS = [('http://git.chromium.org/native_client', GIT_PUSH_URL),
  82                 (ALT_GIT_BASE_URL, GIT_PUSH_URL),
  83                 (GIT_BASE_URL, GIT_PUSH_URL)]
  84
  85
  86 def GitUrl(package, push_url=False):
  87   repo = GIT_REVISIONS[package].get('repo', package)
  88   if push_url:
  89     base_url = GIT_PUSH_URL
  90   else:
  91     base_url = GIT_BASE_URL
  92
  93   return '%s/nacl-%s.git' % (base_url, repo)
  94
  95
  96 def CollectSources():
  97   sources = {}
  98
  99   for package in TAR_FILES:
 100     tar_file = TAR_FILES[package]
 101     if fnmatch.fnmatch(tar_file, '*.bz2'):
 102       extract = EXTRACT_STRIP_TBZ2
 103     elif fnmatch.fnmatch(tar_file, '*.gz'):
 104       extract = EXTRACT_STRIP_TGZ
 105     else:
 106       raise Exception('unexpected file name pattern in TAR_FILES[%r]' % package)
 107     sources[package] = {
 108         'type': 'source',
 109         'commands': [
 110             command.Command(extract + [command.path.join('%(abs_top_srcdir)s',
 111                                                          '..', 'third_party',
 112                                                          tar_file)],
 113                             cwd='%(output)s'),
 114             ],
 115         }
 116
 117   patch_packages = []
 118   patch_commands = []
 119   for package, info in GIT_REVISIONS.iteritems():
 120     sources[package] = {
 121         'type': 'source',
 122         'commands': command.SyncGitRepoCmds(GitUrl(package), '%(output)s',
 123                                             info['rev'],
 124                                             git_cache='%(git_cache_dir)s',
 125                                             push_url=GitUrl(package, True),
 126                                             known_mirrors=KNOWN_MIRRORS,
 127                                             push_mirrors=PUSH_MIRRORS),
 128         }
 129     patch_packages.append(package)
 130     patch_info = {'name': package}
 131     patch_info.update(info)
 132     patch_commands.append(
 133         command.GenerateGitPatches('%(' + package + ')s/.git', patch_info))
 134
 135   sources['patches'] = {
 136       'type': 'build',
 137       'dependencies': patch_packages,
 138       'commands': patch_commands,
 139       }
 140
 141   # The gcc_libs component gets the whole GCC source tree.
 142   sources['gcc_libs'] = sources['gcc']
 143
 144   # The gcc component omits all the source directories that are used solely
 145   # for building target libraries.  We don't want those included in the
 146   # input hash calculation so that we don't rebuild the compiler when the
 147   # the only things that have changed are target libraries.
 148   sources['gcc'] = {
 149         'type': 'source',
 150         'dependencies': ['gcc_libs'],
 151         'commands': [command.CopyTree('%(gcc_libs)s', '%(output)s', [
 152             'boehm-gc',
 153             'libada',
 154             'libatomic',
 155             'libffi',
 156             'libgcc',
 157             'libgfortran',
 158             'libgo',
 159             'libgomp',
 160             'libitm',
 161             'libjava',
 162             'libmudflap',
 163             'libobjc',
 164             'libquadmath',
 165             'libsanitizer',
 166             'libssp',
 167             'libstdc++-v3',
 168             ])]
 169       }
 170
 171   # We have to populate the newlib source tree with the "exported" form of
 172   # some headers from the native_client source tree.  The newlib build
 173   # needs these to be in the expected place.  By doing this in the source
 174   # target, these files will be part of the input hash and so we don't need
 175   # to do anything else to keep track of when they might have changed in
 176   # the native_client source tree.
 177   newlib_sys_nacl = command.path.join('%(output)s',
 178                                       'newlib', 'libc', 'sys', 'nacl')
 179   newlib_unpack = [command.RemoveDirectory(command.path.join(newlib_sys_nacl,
 180                                                              dirname))
 181                    for dirname in ['bits', 'sys', 'machine']]
 182   newlib_unpack.append(command.Command([
 183       'python',
 184       command.path.join('%(top_srcdir)s', 'src',
 185                         'trusted', 'service_runtime', 'export_header.py'),
 186       command.path.join('%(top_srcdir)s', 'src',
 187                         'trusted', 'service_runtime', 'include'),
 188       newlib_sys_nacl,
 189       ]))
 190   sources['newlib']['commands'] += newlib_unpack
 191
 192   return sources
 193
 194
 195 # Canonical tuples we use for hosts.
 196 WINDOWS_HOST_TUPLE = pynacl.platform.PlatformTriple('win', 'x86-32')
 197 MAC_HOST_TUPLE = pynacl.platform.PlatformTriple('darwin', 'x86-64')
 198 ARM_HOST_TUPLE = pynacl.platform.PlatformTriple('linux', 'arm')
 199 LINUX_X86_32_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-32')
 200 LINUX_X86_64_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-64')
 201
 202 # Map of native host tuple to extra tuples that it cross-builds for.
 203 EXTRA_HOSTS_MAP = {
 204     LINUX_X86_64_TUPLE: [
 205         LINUX_X86_32_TUPLE,
 206         ARM_HOST_TUPLE,
 207         WINDOWS_HOST_TUPLE,
 208         ],
 209     }
 210
 211 # Map of native host tuple to host tuples that are "native enough".
 212 # For these hosts, we will do a native-style build even though it's
 213 # not the native tuple, just passing some extra compiler flags.
 214 NATIVE_ENOUGH_MAP = {
 215     LINUX_X86_64_TUPLE: {
 216         LINUX_X86_32_TUPLE: ['-m32'],
 217         },
 218     }
 219
 220 # The list of targets to build toolchains for.
 221 TARGET_LIST = ['arm', 'i686']
 222
 223 # List upload targets for each host we want to upload packages for.
 224 TARGET = collections.namedtuple('TARGET', ['name', 'pkg_prefix'])
 225 HOST_TARGET = collections.namedtuple('HOST_TARGET',
 226                                      ['os', 'arch', 'differ3264', 'targets'])
 227
 228 STANDARD_TARGETS = [TARGET('arm', '')]
 229 LINUX_X86_64_TARGETS = [TARGET('arm', ''), TARGET('i686', 'ng_')]
 230
 231 UPLOAD_HOST_TARGETS = [
 232     HOST_TARGET('win', 'x86-32', False, STANDARD_TARGETS),
 233     HOST_TARGET('darwin', 'x86-64', False, STANDARD_TARGETS),
 234     HOST_TARGET('linux', 'arm', False, STANDARD_TARGETS),
 235     HOST_TARGET('linux', 'x86-32', False, STANDARD_TARGETS),
 236     HOST_TARGET('linux', 'x86-64', True, LINUX_X86_64_TARGETS),
 237     ]
 238
 239 # GDB is built by toolchain_build but injected into package targets built by
 240 # other means. List out what package targets, packages, and the tar file we are
 241 # injecting on top of here.
 242 GDB_INJECT_HOSTS = [
 243   ('win', 'x86-32'),
 244   ('darwin', 'x86-64'),
 245   ('linux', 'x86-32'),
 246   ]
 247
 248 GDB_INJECT_PACKAGES = [
 249   ('nacl_x86_newlib', ['naclsdk.tgz']),
 250   ('nacl_x86_glibc', ['toolchain.tar.bz2']),
 251   ]
 252
 253 # These are extra arguments to pass gcc's configure that vary by target.
 254 TARGET_GCC_CONFIG = {
 255     'arm': ['--with-tune=cortex-a15'],
 256     }
 257
 258 PACKAGE_NAME = 'Native Client SDK [%(build_signature)s]'
 259 BUG_URL = 'http://gonacl.com/reportissue'
 260
 261 TAR_XV = ['tar', '-x', '-v']
 262 EXTRACT_STRIP_TGZ = TAR_XV + ['--gzip', '--strip-components=1', '-f']
 263 EXTRACT_STRIP_TBZ2 = TAR_XV + ['--bzip2', '--strip-components=1', '-f']
 264 CONFIGURE_CMD = ['sh', '%(src)s/configure']
 265 MAKE_PARALLEL_CMD = ['make', '-j%(cores)s']
 266 MAKE_CHECK_CMD = MAKE_PARALLEL_CMD + ['check']
 267 MAKE_DESTDIR_CMD = ['make', 'DESTDIR=%(abs_output)s']
 268
 269 # This file gets installed by multiple packages' install steps, but it is
 270 # never useful when installed in isolation.  So we remove it from the
 271 # installation directories before packaging up.
 272 REMOVE_INFO_DIR = command.Remove(command.path.join('%(output)s',
 273                                                    'share', 'info', 'dir'))
 274
 275 def ConfigureHostArch(host):
 276   configure_args = []
 277
 278   is_cross = CrossCompiling(host)
 279
 280   if is_cross:
 281     extra_cc_args = []
 282     configure_args.append('--host=' + host)
 283   else:
 284     extra_cc_args = NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}).get(host, [])
 285     if extra_cc_args:
 286       # The host we've chosen is "native enough", such as x86-32 on x86-64.
 287       # But it's not what config.guess will yield, so we need to supply
 288       # a --build switch to ensure things build correctly.
 289       configure_args.append('--build=' + host)
 290
 291   extra_cxx_args = list(extra_cc_args)
 292   if fnmatch.fnmatch(host, '*-linux*'):
 293     # Avoid shipping binaries with a runtime dependency on
 294     # a particular version of the libstdc++ shared library.
 295     # TODO(mcgrathr): Do we want this for MinGW and/or Mac too?
 296     extra_cxx_args.append('-static-libstdc++')
 297
 298   if extra_cc_args:
 299     # These are the defaults when there is no setting, but we will add
 300     # additional switches, so we must supply the command name too.
 301     if is_cross:
 302       cc = host + '-gcc'
 303     else:
 304       cc = 'gcc'
 305     configure_args.append('CC=' + ' '.join([cc] + extra_cc_args))
 306
 307   if extra_cxx_args:
 308     # These are the defaults when there is no setting, but we will add
 309     # additional switches, so we must supply the command name too.
 310     if is_cross:
 311       cxx = host + '-g++'
 312     else:
 313       cxx = 'g++'
 314     configure_args.append('CXX=' + ' '.join([cxx] + extra_cxx_args))
 315
 316   if HostIsWindows(host):
 317     # The i18n support brings in runtime dependencies on MinGW DLLs
 318     # that we don't want to have to distribute alongside our binaries.
 319     # So just disable it, and compiler messages will always be in US English.
 320     configure_args.append('--disable-nls')
 321
 322   return configure_args
 323
 324
 325 def ConfigureHostCommon(host):
 326   return ConfigureHostArch(host) + [
 327       '--prefix=',
 328       '--disable-silent-rules',
 329       '--without-gcc-arch',
 330       ]
 331
 332
 333 def ConfigureHostLib(host):
 334   return ConfigureHostCommon(host) + [
 335       '--disable-shared',
 336       ]
 337
 338
 339 def ConfigureHostTool(host):
 340   return ConfigureHostCommon(host) + [
 341       '--with-pkgversion=' + PACKAGE_NAME,
 342       '--with-bugurl=' + BUG_URL,
 343       '--without-zlib',
 344       ]
 345
 346
 347 def MakeCommand(host, extra_args=[]):
 348   if HostIsWindows(host):
 349     # There appears to be nothing we can pass at top-level configure time
 350     # that will prevent the configure scripts from finding MinGW's libiconv
 351     # and using it.  We have to force this variable into the environment
 352     # of the sub-configure runs, which are run via make.
 353     make_command = MAKE_PARALLEL_CMD + ['HAVE_LIBICONV=no']
 354   else:
 355     make_command = MAKE_PARALLEL_CMD
 356   return make_command + extra_args
 357
 358
 359 # Return the 'make check' command to run.
 360 # When cross-compiling, don't try to run test suites.
 361 def MakeCheckCommand(host):
 362   if CrossCompiling(host):
 363     return ['true']
 364   return MAKE_CHECK_CMD
 365
 366
 367 def InstallDocFiles(subdir, files):
 368   doc_dir = command.path.join('%(output)s', 'share', 'doc', subdir)
 369   dirs = sorted(set([command.path.dirname(command.path.join(doc_dir, file))
 370                      for file in files]))
 371   commands = ([command.Mkdir(dir, parents=True) for dir in dirs] +
 372               [command.Copy(command.path.join('%(' + subdir + ')s', file),
 373                             command.path.join(doc_dir, file))
 374                for file in files])
 375   return commands
 376
 377
 378 def NewlibLibcScript(arch):
 379   template = """/*
 380  * This is a linker script that gets installed as libc.a for the
 381  * newlib-based NaCl toolchain.  It brings in the constituent
 382  * libraries that make up what -lc means semantically.
 383  */
 384 OUTPUT_FORMAT(%s)
 385 GROUP ( libnacl.a libcrt_common.a )
 386 """
 387   if arch == 'arm':
 388     # Listing three formats instead of one makes -EL/-EB switches work
 389     # for the endian-switchable ARM backend.
 390     format_list = ['elf32-littlearm-nacl',
 391                    'elf32-bigarm-nacl',
 392                    'elf32-littlearm-nacl']
 393   elif arch == 'i686':
 394     format_list = ['elf32-i386-nacl']
 395   elif arch == 'x86_64':
 396     format_list = ['elf32-x86_64-nacl']
 397   else:
 398     raise Exception('TODO(mcgrathr): OUTPUT_FORMAT for %s' % arch)
 399   return template % ', '.join(['"' + fmt + '"' for fmt in format_list])
 400
 401
 402 # The default strip behavior removes debugging and symbol table
 403 # sections, but it leaves the .comment section.  This contains the
 404 # compiler version string, and so it changes when the compiler changes
 405 # even if the actual machine code it produces is completely identical.
 406 # Hence, the target library packages will always change when the
 407 # compiler changes unless these sections are removed.  Doing this
 408 # requires somehow teaching the makefile rules to pass the
 409 # --remove-section=.comment switch to TARGET-strip.  For the GCC
 410 # target libraries, setting STRIP_FOR_TARGET is sufficient.  But
 411 # quoting nightmares make it difficult to pass a command with a space
 412 # in it as the STRIP_FOR_TARGET value.  So the build writes a little
 413 # script that can be invoked with a simple name.
 414 #
 415 # Though the gcc target libraries' makefiles are smart enough to obey
 416 # STRIP_FOR_TARGET for library files, the newlib makefiles just
 417 # blindly use $(INSTALL_DATA) for both header (text) files and library
 418 # files.  Hence it's necessary to override its INSTALL_DATA setting to
 419 # one that will do stripping using this script, and thus the script
 420 # must silently do nothing to non-binary files.
 421 def ConfigureTargetPrep(arch):
 422   script_file = 'strip_for_target'
 423
 424   config_target = arch + '-nacl'
 425   script_contents = """\
 426 #!/bin/sh
 427 mode=--strip-all
 428 for arg; do
 429   case "$arg" in
 430   -*) ;;
 431   *)
 432     type=`file --brief --mime-type "$arg"`
 433     case "$type" in
 434       application/x-executable|application/x-sharedlib) ;;
 435       application/x-archive|application/x-object) mode=--strip-debug ;;
 436       *) exit 0 ;;
 437     esac
 438     ;;
 439   esac
 440 done
 441 exec %s-strip $mode --remove-section=.comment "$@"
 442 """ % config_target
 443
 444   return [
 445       command.WriteData(script_contents, script_file),
 446       command.Command(['chmod', '+x', script_file]),
 447       ]
 448
 449
 450 def ConfigureTargetArgs(arch):
 451   config_target = arch + '-nacl'
 452   return [
 453       '--target=' + config_target,
 454       '--with-sysroot=/' + config_target,
 455       'STRIP_FOR_TARGET=%(cwd)s/strip_for_target',
 456       ]
 457
 458
 459 def CommandsInBuild(command_lines):
 460   return [
 461       command.RemoveDirectory('build'),
 462       command.Mkdir('build'),
 463       ] + [command.Command(cmd, cwd='build')
 464            for cmd in command_lines]
 465
 466
 467 def PopulateDeps(dep_dirs):
 468   commands = [command.RemoveDirectory('all_deps'),
 469               command.Mkdir('all_deps')]
 470   commands += [command.Command('cp -r "%s/"* all_deps' % dirname, shell=True)
 471                for dirname in dep_dirs]
 472   return commands
 473
 474
 475 def WithDepsOptions(options, component=None):
 476   if component is None:
 477     directory = command.path.join('%(cwd)s', 'all_deps')
 478   else:
 479     directory = '%(abs_' + component + ')s'
 480   return ['--with-' + option + '=' + directory
 481           for option in options]
 482
 483
 484 # Return the component name we'll use for a base component name and
 485 # a host tuple.  The component names cannot contain dashes or other
 486 # non-identifier characters, because the names of the files uploaded
 487 # to Google Storage are constrained.  GNU configuration tuples contain
 488 # dashes, which we translate to underscores.
 489 def ForHost(component_name, host):
 490   return component_name + '_' + pynacl.gsd_storage.LegalizeName(host)
 491
 492
 493 # These are libraries that go into building the compiler itself.
 494 def HostGccLibs(host):
 495   def H(component_name):
 496     return ForHost(component_name, host)
 497   host_gcc_libs = {
 498       H('gmp'): {
 499           'type': 'build',
 500           'dependencies': ['gmp'],
 501           'commands': [
 502               command.Command(ConfigureCommand('gmp') +
 503                               ConfigureHostLib(host) + [
 504                                   '--with-sysroot=%(abs_output)s',
 505                                   '--enable-cxx',
 506                                   # Without this, the built library will
 507                                   # assume the instruction set details
 508                                   # available on the build machine.  With
 509                                   # this, it dynamically chooses what code
 510                                   # to use based on the details of the
 511                                   # actual host CPU at runtime.
 512                                   '--enable-fat',
 513                                   ]),
 514               command.Command(MakeCommand(host)),
 515               command.Command(MakeCheckCommand(host)),
 516               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 517               ],
 518           },
 519       H('mpfr'): {
 520           'type': 'build',
 521           'dependencies': ['mpfr', H('gmp')],
 522           'commands': [
 523               command.Command(ConfigureCommand('mpfr') +
 524                               ConfigureHostLib(host) +
 525                               WithDepsOptions(['sysroot', 'gmp'], H('gmp'))),
 526               command.Command(MakeCommand(host)),
 527               command.Command(MakeCheckCommand(host)),
 528               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 529               ],
 530           },
 531       H('mpc'): {
 532           'type': 'build',
 533           'dependencies': ['mpc', H('gmp'), H('mpfr')],
 534           'commands': PopulateDeps(['%(' + H('gmp') + ')s',
 535                                     '%(' + H('mpfr') + ')s']) + [
 536               command.Command(ConfigureCommand('mpc') +
 537                               ConfigureHostLib(host) +
 538                               WithDepsOptions(['sysroot', 'gmp', 'mpfr'])),
 539               command.Command(MakeCommand(host)),
 540               command.Command(MakeCheckCommand(host)),
 541               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 542               ],
 543           },
 544       H('isl'): {
 545           'type': 'build',
 546           'dependencies': ['isl', H('gmp')],
 547           'commands': [
 548               command.Command(ConfigureCommand('isl') +
 549                               ConfigureHostLib(host) +
 550                               WithDepsOptions(['sysroot', 'gmp-prefix'],
 551                                               H('gmp'))),
 552               command.Command(MakeCommand(host)),
 553               command.Command(MakeCheckCommand(host)),
 554               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 555               # The .pc files wind up containing some absolute paths
 556               # that make the output depend on the build directory name.
 557               # The dependents' configure scripts don't need them anyway.
 558               command.RemoveDirectory(command.path.join(
 559                   '%(output)s', 'lib', 'pkgconfig')),
 560               ],
 561           },
 562       H('cloog'): {
 563           'type': 'build',
 564           'dependencies': ['cloog', H('gmp'), H('isl')],
 565           'commands': PopulateDeps(['%(' + H('gmp') + ')s',
 566                                     '%(' + H('isl') + ')s']) + [
 567               command.Command(ConfigureCommand('cloog') +
 568                               ConfigureHostLib(host) + [
 569                                   '--with-bits=gmp',
 570                                   '--with-isl=system',
 571                                   ] + WithDepsOptions(['sysroot',
 572                                                        'gmp-prefix',
 573                                                        'isl-prefix'])),
 574               command.Command(MakeCommand(host)),
 575               command.Command(MakeCheckCommand(host)),
 576               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 577               # The .pc files wind up containing some absolute paths
 578               # that make the output depend on the build directory name.
 579               # The dependents' configure scripts don't need them anyway.
 580               command.RemoveDirectory(command.path.join(
 581                   '%(output)s', 'lib', 'pkgconfig')),
 582               ],
 583           },
 584       H('expat'): {
 585           'type': 'build',
 586           'dependencies': ['expat'],
 587           'commands': [
 588               command.Command(ConfigureCommand('expat') +
 589                               ConfigureHostLib(host)),
 590               command.Command(MakeCommand(host)),
 591               command.Command(MakeCheckCommand(host)),
 592               command.Command(MAKE_DESTDIR_CMD + [
 593                   # expat does not support the install-strip target.
 594                   'installlib',
 595                   'INSTALL=%(expat)s/conftools/install-sh -c -s',
 596                   'INSTALL_DATA=%(expat)s/conftools/install-sh -c -m 644',
 597                   ]),
 598               ],
 599           },
 600       }
 601   return host_gcc_libs
 602
 603
 604 HOST_GCC_LIBS_DEPS = ['gmp', 'mpfr', 'mpc', 'isl', 'cloog']
 605
 606 def HostGccLibsDeps(host):
 607   return [ForHost(package, host) for package in HOST_GCC_LIBS_DEPS]
 608
 609
 610 def ConfigureCommand(source_component):
 611   return [command % {'src': '%(' + source_component + ')s'}
 612           for command in CONFIGURE_CMD]
 613
 614
 615 # When doing a Canadian cross, we need native-hosted cross components
 616 # to do the GCC build.
 617 def GccDeps(host, target):
 618   components = ['binutils_' + target]
 619   if CrossCompiling(host):
 620     components.append('gcc_' + target)
 621     host = NATIVE_TUPLE
 622   return [ForHost(component, host) for component in components]
 623
 624
 625 def GccCommand(host, target, cmd):
 626   components_for_path = GccDeps(host, target)
 627   return command.Command(
 628       cmd, path_dirs=[command.path.join('%(abs_' + component + ')s', 'bin')
 629                       for component in components_for_path])
 630
 631
 632 def ConfigureGccCommand(source_component, host, target, extra_args=[]):
 633   return GccCommand(
 634       host,
 635       target,
 636       ConfigureCommand(source_component) +
 637       ConfigureHostTool(host) +
 638       ConfigureTargetArgs(target) +
 639       TARGET_GCC_CONFIG.get(target, []) + [
 640           '--with-gmp=%(abs_' + ForHost('gmp', host) + ')s',
 641           '--with-mpfr=%(abs_' + ForHost('mpfr', host) + ')s',
 642           '--with-mpc=%(abs_' + ForHost('mpc', host) + ')s',
 643           '--with-isl=%(abs_' + ForHost('isl', host) + ')s',
 644           '--with-cloog=%(abs_' + ForHost('cloog', host) + ')s',
 645           '--enable-cloog-backend=isl',
 646           '--disable-dlopen',
 647           '--disable-shared',
 648           '--with-newlib',
 649           '--with-linker-hash-style=gnu',
 650           '--enable-linker-build-id',
 651           '--enable-languages=c,c++,lto',
 652           ] + extra_args)
 653
 654
 655
 656 def HostTools(host, target):
 657   def H(component_name):
 658     return ForHost(component_name, host)
 659
 660   def WindowsAlternate(if_windows, if_not_windows, if_mac=None):
 661     if if_mac is not None and HostIsMac(host):
 662       return if_mac
 663     elif HostIsWindows(host):
 664       return if_windows
 665     else:
 666       return if_not_windows
 667
 668   # Return the file name with the appropriate suffix for an executable file.
 669   def Exe(file):
 670     return file + WindowsAlternate('.exe', '')
 671
 672   tools = {
 673       H('binutils_' + target): {
 674           'type': 'build',
 675           'dependencies': ['binutils'],
 676           'commands': ConfigureTargetPrep(target) + [
 677               command.Command(
 678                   ConfigureCommand('binutils') +
 679                   ConfigureHostTool(host) +
 680                   ConfigureTargetArgs(target) + [
 681                       '--enable-deterministic-archives',
 682                       '--enable-gold',
 683                       ] + WindowsAlternate([], ['--enable-plugins'])),
 684               command.Command(MakeCommand(host)),
 685               command.Command(MakeCheckCommand(host)),
 686               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 687               REMOVE_INFO_DIR,
 688               ] + InstallDocFiles('binutils',
 689                                   ['COPYING3'] +
 690                                   [command.path.join(subdir, 'NEWS')
 691                                    for subdir in
 692                                    ['binutils', 'gas', 'ld', 'gold']]) +
 693               # The top-level lib* directories contain host libraries
 694               # that we don't want to include in the distribution.
 695               [command.RemoveDirectory(command.path.join('%(output)s', name))
 696                for name in ['lib', 'lib32', 'lib64']],
 697           },
 698
 699       H('gcc_' + target): {
 700           'type': 'build',
 701           'dependencies': (['gcc'] + HostGccLibsDeps(host) +
 702                            GccDeps(host, target)),
 703           'commands': ConfigureTargetPrep(target) + [
 704               ConfigureGccCommand('gcc', host, target),
 705               # GCC's configure step writes configargs.h with some strings
 706               # including the configure command line, which get embedded
 707               # into the gcc driver binary.  The build only works if we use
 708               # absolute paths in some of the configure switches, but
 709               # embedding those paths makes the output differ in repeated
 710               # builds done in different directories, which we do not want.
 711               # So force the generation of that file early and then edit it
 712               # in place to replace the absolute paths with something that
 713               # never varies.  Note that the 'configure-gcc' target will
 714               # actually build some components before running gcc/configure.
 715               GccCommand(host, target,
 716                          MakeCommand(host, ['configure-gcc'])),
 717               command.Command(['sed', '-i', '-e',
 718                                ';'.join(['s@%%(abs_%s)s@.../%s_install@g' %
 719                                          (component, component)
 720                                          for component in
 721                                          HostGccLibsDeps(host)] +
 722                                         ['s@%(cwd)s@...@g']),
 723                                command.path.join('gcc', 'configargs.h')]),
 724               # gcc/Makefile's install rules ordinarily look at the
 725               # installed include directory for a limits.h to decide
 726               # whether the lib/gcc/.../include-fixed/limits.h header
 727               # should be made to expect a libc-supplied limits.h or not.
 728               # Since we're doing this build in a clean environment without
 729               # any libc installed, we need to force its hand here.
 730               GccCommand(host, target,
 731                          MakeCommand(host, ['all-gcc', 'LIMITS_H_TEST=true'])),
 732               # gcc/Makefile's install targets populate this directory
 733               # only if it already exists.
 734               command.Mkdir(command.path.join('%(output)s',
 735                                               target + '-nacl', 'bin'),
 736                             True),
 737               GccCommand(host, target,
 738                          MAKE_DESTDIR_CMD + ['install-strip-gcc']),
 739               REMOVE_INFO_DIR,
 740               # Note we include COPYING.RUNTIME here and not with gcc_libs.
 741               ] + InstallDocFiles('gcc', ['COPYING3', 'COPYING.RUNTIME']),
 742           },
 743
 744       # GDB can support all the targets in one host tool.
 745       H('gdb'): {
 746           'type': 'build',
 747           'dependencies': ['gdb', H('expat')],
 748           'commands': [
 749               command.Command(
 750                   ConfigureCommand('gdb') +
 751                   ConfigureHostTool(host) + [
 752                       '--target=x86_64-nacl',
 753                       '--enable-targets=arm-none-eabi-nacl',
 754                       '--with-expat',
 755                       # Windows (MinGW) is missing ncurses; we need to
 756                       # build one here and link it in statically for
 757                       # --enable-tui.  See issue nativeclient:3911.
 758                       '--%s-tui' % WindowsAlternate('disable', 'enable'),
 759                       'CPPFLAGS=-I%(abs_' + H('expat') + ')s/include',
 760                       'LDFLAGS=-L%(abs_' + H('expat') + ')s/lib',
 761                       ] +
 762                   # TODO(mcgrathr): Should use --with-python to ensure
 763                   # we have it on Linux/Mac.
 764                   WindowsAlternate(['--without-python'], []) +
 765                   # TODO(mcgrathr): The default -Werror only breaks because
 766                   # the OSX default compiler is an old front-end that does
 767                   # not understand all the GCC options.  Maybe switch to
 768                   # using clang (system or Chromium-supplied) on Mac.
 769                   (['--disable-werror'] if HostIsMac(host) else [])),
 770               command.Command(MakeCommand(host) + ['all-gdb']),
 771               command.Command(MAKE_DESTDIR_CMD + [
 772                   '-C', 'gdb', 'install-strip',
 773                   ]),
 774               REMOVE_INFO_DIR,
 775               ] + [command.Command(['ln', '-f',
 776                                     command.path.join('%(abs_output)s',
 777                                                       'bin',
 778                                                       Exe('x86_64-nacl-gdb')),
 779                                     command.path.join('%(abs_output)s',
 780                                                       'bin',
 781                                                       Exe(arch + '-nacl-gdb'))])
 782                    for arch in ['i686', 'arm']] + InstallDocFiles('gdb', [
 783                        'COPYING3',
 784                        command.path.join('gdb', 'NEWS'),
 785                        ]),
 786           },
 787       }
 788
 789   # TODO(mcgrathr): The ARM cross environment does not supply a termcap
 790   # library, so it cannot build GDB.
 791   if host.startswith('arm') and CrossCompiling(host):
 792     del tools[H('gdb')]
 793
 794   return tools
 795
 796 def TargetCommands(host, target, command_list):
 797   # First we have to copy the host tools into a common directory.
 798   # We can't just have both directories in our PATH, because the
 799   # compiler looks for the assembler and linker relative to itself.
 800   commands = PopulateDeps(['%(' + ForHost('binutils_' + target, host) + ')s',
 801                            '%(' + ForHost('gcc_' + target, host) + ')s'])
 802   bindir = command.path.join('%(cwd)s', 'all_deps', 'bin')
 803   commands += [command.Command(cmd, path_dirs=[bindir])
 804                for cmd in command_list]
 805   return commands
 806
 807
 808 def TargetLibs(host, target):
 809   lib_deps = [ForHost(component + '_' + target, host)
 810               for component in ['binutils', 'gcc']]
 811
 812   def NewlibFile(subdir, name):
 813     return command.path.join('%(output)s', target + '-nacl', subdir, name)
 814
 815   newlib_sysroot = '%(abs_newlib_' + target + ')s'
 816   newlib_tooldir = '%s/%s-nacl' % (newlib_sysroot, target)
 817
 818   # See the comment at ConfigureTargetPrep, above.
 819   newlib_install_data = ' '.join(['STRIPPROG=%(cwd)s/strip_for_target',
 820                                   '%(abs_newlib)s/install-sh',
 821                                   '-c', '-s', '-m', '644'])
 822
 823   iconv_encodings = 'UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4'
 824   newlib_configure_args = [
 825       '--disable-libgloss',
 826       '--enable-newlib-iconv',
 827       '--enable-newlib-iconv-from-encodings=' + iconv_encodings,
 828       '--enable-newlib-iconv-to-encodings=' + iconv_encodings,
 829       '--enable-newlib-io-long-long',
 830       '--enable-newlib-io-long-double',
 831       '--enable-newlib-io-c99-formats',
 832       '--enable-newlib-mb',
 833       'CFLAGS=-O2',
 834       'INSTALL_DATA=' + newlib_install_data,
 835       ]
 836
 837   newlib_post_install = [
 838       command.Rename(NewlibFile('lib', 'libc.a'),
 839                      NewlibFile('lib', 'libcrt_common.a')),
 840       command.WriteData(NewlibLibcScript(target),
 841                         NewlibFile('lib', 'libc.a')),
 842       ] + [
 843       command.Copy(
 844           command.path.join('%(pthread_headers)s', header),
 845           NewlibFile('include', header))
 846       for header in ('pthread.h', 'semaphore.h')
 847       ]
 848
 849
 850   libs = {
 851       'newlib_' + target: {
 852           'type': 'build',
 853           'dependencies': ['newlib'] + lib_deps,
 854           'inputs': { 'pthread_headers':
 855                       os.path.join(NACL_DIR, 'src', 'untrusted',
 856                                    'pthread') },
 857           'commands': (ConfigureTargetPrep(target) +
 858                        TargetCommands(host, target, [
 859                            ConfigureCommand('newlib') +
 860                            ConfigureHostTool(host) +
 861                            ConfigureTargetArgs(target) +
 862                            newlib_configure_args,
 863                            MakeCommand(host),
 864                            MAKE_DESTDIR_CMD + ['install-strip'],
 865                            ]) +
 866                        newlib_post_install +
 867                        InstallDocFiles('newlib', ['COPYING.NEWLIB'])),
 868           },
 869
 870       'gcc_libs_' + target: {
 871           'type': 'build',
 872           'dependencies': (['gcc_libs'] + lib_deps + ['newlib_' + target] +
 873                            HostGccLibsDeps(host)),
 874           # This actually builds the compiler again and uses that compiler
 875           # to build the target libraries.  That's by far the easiest thing
 876           # to get going given the interdependencies of the target
 877           # libraries (especially libgcc) on the gcc subdirectory, and
 878           # building the compiler doesn't really take all that long in the
 879           # grand scheme of things.
 880           # TODO(mcgrathr): If upstream ever cleans up all their
 881           # interdependencies better, unpack the compiler, configure with
 882           # --disable-gcc, and just build all-target.
 883           'commands': ConfigureTargetPrep(target) + [
 884               ConfigureGccCommand('gcc_libs', host, target, [
 885                   '--with-build-sysroot=' + newlib_sysroot,
 886                   ]),
 887               GccCommand(host, target,
 888                          MakeCommand(host) + [
 889                              'build_tooldir=' + newlib_tooldir,
 890                              'all-target',
 891                              ]),
 892               GccCommand(host, target,
 893                          MAKE_DESTDIR_CMD + ['install-strip-target']),
 894               REMOVE_INFO_DIR,
 895               ],
 896           },
 897       }
 898   return libs
 899
 900 # Compute it once.
 901 NATIVE_TUPLE = pynacl.platform.PlatformTriple()
 902
 903
 904 # For our purposes, "cross-compiling" means not literally that we are
 905 # targetting a host that does not match NATIVE_TUPLE, but that we are
 906 # targetting a host whose binaries we cannot run locally.  So x86-32
 907 # on x86-64 does not count as cross-compiling.  See NATIVE_ENOUGH_MAP, above.
 908 def CrossCompiling(host):
 909   return (host != NATIVE_TUPLE and
 910           host not in NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}))
 911
 912
 913 def HostIsWindows(host):
 914   return host == WINDOWS_HOST_TUPLE
 915
 916
 917 def HostIsMac(host):
 918   return host == MAC_HOST_TUPLE
 919
 920
 921 # We build target libraries only on Linux for two reasons:
 922 # 1. We only need to build them once.
 923 # 2. Linux is the fastest to build.
 924 # TODO(mcgrathr): In future set up some scheme whereby non-Linux
 925 # bots can build target libraries but not archive them, only verifying
 926 # that the results came out the same as the ones archived by the
 927 # official builder bot.  That will serve as a test of the host tools
 928 # on the other host platforms.
 929 def BuildTargetLibsOn(host):
 930   return host == LINUX_X86_64_TUPLE
 931
 932
 933 def GetPackageTargets():
 934   """Package Targets describes all the final package targets.
 935
 936   This build can be built among many build bots, but eventually all things
 937   will be combined together. This package target dictionary describes the final
 938   output of the entire build.
 939   """
 940   package_targets = {}
 941
 942   # Add in standard upload targets.
 943   for host_target in UPLOAD_HOST_TARGETS:
 944     for target in host_target.targets:
 945       target_arch = target.name
 946       package_prefix = target.pkg_prefix
 947
 948       # Each package target contains non-platform specific newlib and gcc libs.
 949       # These packages are added inside of TargetLibs(host, target).
 950       newlib_package = 'newlib_%s' % target_arch
 951       gcc_lib_package = 'gcc_libs_%s' % target_arch
 952       shared_packages = [newlib_package, gcc_lib_package]
 953
 954       # Each package target contains arm binutils and gcc.
 955       # These packages are added inside of HostTools(host, target).
 956       platform_triple = pynacl.platform.PlatformTriple(host_target.os,
 957                                                        host_target.arch)
 958       binutils_package = ForHost('binutils_%s' % target_arch, platform_triple)
 959       gcc_package = ForHost('gcc_%s' % target_arch, platform_triple)
 960       gdb_package = ForHost('gdb', platform_triple)
 961
 962       # Create a list of packages for a target.
 963       platform_packages = [binutils_package, gcc_package, gdb_package]
 964       combined_packages = shared_packages + platform_packages
 965
 966       os_name = pynacl.platform.GetOS(host_target.os)
 967       if host_target.differ3264:
 968         arch_name = pynacl.platform.GetArch3264(host_target.arch)
 969       else:
 970         arch_name = pynacl.platform.GetArch(host_target.arch)
 971       package_target = '%s_%s' % (os_name, arch_name)
 972       package_name = '%snacl_%s_newlib' % (package_prefix,
 973                                            pynacl.platform.GetArch(target_arch))
 974
 975       package_target_dict = package_targets.setdefault(package_target, {})
 976       package_target_dict.setdefault(package_name, []).extend(combined_packages)
 977
 978   # GDB is a special and shared, we will inject it into various other packages.
 979   for platform, arch in GDB_INJECT_HOSTS:
 980     platform_triple = pynacl.platform.PlatformTriple(platform, arch)
 981     os_name = pynacl.platform.GetOS(platform)
 982     arch_name = pynacl.platform.GetArch(arch)
 983
 984     gdb_packages = [ForHost('gdb', platform_triple)]
 985     package_target = '%s_%s' % (os_name, arch_name)
 986
 987     for package_name, package_archives in GDB_INJECT_PACKAGES:
 988       combined_packages = package_archives + gdb_packages
 989       package_target_dict = package_targets.setdefault(package_target, {})
 990       package_target_dict.setdefault(package_name, []).extend(combined_packages)
 991
 992   return dict(package_targets)
 993
 994 def CollectPackagesForHost(host, targets):
 995   packages = HostGccLibs(host).copy()
 996   for target in targets:
 997     packages.update(HostTools(host, target))
 998     if BuildTargetLibsOn(host):
 999       packages.update(TargetLibs(host, target))
1000   return packages
1001
1002
1003 def CollectPackages(targets):
1004   packages = CollectSources()
1005
1006   packages.update(CollectPackagesForHost(NATIVE_TUPLE, targets))
1007
1008   for host in EXTRA_HOSTS_MAP.get(NATIVE_TUPLE, []):
1009     packages.update(CollectPackagesForHost(host, targets))
1010
1011   return packages
1012
1013
1014 PACKAGES = CollectPackages(TARGET_LIST)
1015 PACKAGE_TARGETS = GetPackageTargets()
1016
1017
1018 if __name__ == '__main__':
1019   tb = toolchain_main.PackageBuilder(PACKAGES, PACKAGE_TARGETS, sys.argv[1:])
1020   # TODO(mcgrathr): The bot ought to run some native_client tests
1021   # using the new toolchain, like the old x86 toolchain bots do.
1022   tb.Main()