src/native_client/toolchain_build/toolchain_build.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Recipes for NativeClient toolchain packages.
   7
   8 The real entry plumbing is in toolchain_main.py.
   9 """
  10
  11 import fnmatch
  12 import platform
  13 import os
  14 import re
  15 import sys
  16
  17 import command
  18 import gsd_storage
  19 import toolchain_main
  20 import repo_tools
  21
  22 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
  23 NACL_DIR = os.path.dirname(SCRIPT_DIR)
  24
  25 # See command.GenerateGitPatches for the schema of entries in this dict.
  26 # Additionally, each may contain a 'repo' key whose value is the name
  27 # to use in place of the package name when calling GitUrl (below).
  28 GIT_REVISIONS = {
  29     'binutils': {
  30         'rev': '38dbda270a4248ab5b7facc012b9c8d8527f6fb2',
  31         'upstream-branch': 'upstream/binutils-2_24-branch',
  32         'upstream-name': 'binutils-2.24',
  33         # This is tag binutils-2_24, but Gerrit won't let us push
  34         # non-annotated tags, and the upstream tag is not annotated.
  35         'upstream-base': '237df3fa4a1d939e6fd1af0c3e5029a25a137310',
  36         },
  37     'gcc': {
  38         'rev': '145a627d95b98f54915d037dddbcbb0f7b283494',
  39         'upstream-branch': 'upstream/gcc-4_8-branch',
  40         'upstream-name': 'gcc-4.8.2',
  41          # Upstream tag gcc-4_8_2-release:
  42         'upstream-base': '9bcca88e24e64d4e23636aafa3404088b13bcb0e',
  43         },
  44     'newlib': {
  45         'rev': 'a9ae3c60b36dea3d8a10e18b1b6db952d21268c2',
  46         'upstream-branch': 'upstream/master',
  47         'upstream-name': 'newlib-2.0.0',
  48         # Upstream tag newlib_2_0_0:
  49         'upstream-base': 'c3fc84e062cacc2b3e13c1f6b9151d0cc85392ba',
  50         },
  51     'gdb': {
  52         'rev': '41afdf9d511d95073c4403c838d8ccba582d8eed',
  53         'repo': 'binutils',
  54         'upstream-branch': 'upstream/gdb-7.7-branch',
  55         'upstream-name': 'gdb-7.7',
  56         # Upstream tag gdb-7.7-release:
  57         'upstream-base': 'fe284cd86ba9761655a9281fef470d364e27eb85',
  58         },
  59     }
  60
  61 TAR_FILES = {
  62     'gmp': command.path.join('gmp', 'gmp-5.1.3.tar.bz2'),
  63     'mpfr': command.path.join('mpfr', 'mpfr-3.1.2.tar.bz2'),
  64     'mpc': command.path.join('mpc', 'mpc-1.0.2.tar.gz'),
  65     'isl': command.path.join('cloog', 'isl-0.12.2.tar.bz2'),
  66     'cloog': command.path.join('cloog', 'cloog-0.18.1.tar.gz'),
  67     'expat': command.path.join('expat', 'expat-2.0.1.tar.gz'),
  68     }
  69
  70 GIT_BASE_URL = 'https://chromium.googlesource.com/native_client'
  71
  72
  73 def GitUrl(package):
  74   repo = GIT_REVISIONS[package].get('repo', package)
  75   return '%s/nacl-%s.git' % (GIT_BASE_URL, repo)
  76
  77
  78 def CollectSources():
  79   sources = {}
  80
  81   for package in TAR_FILES:
  82     tar_file = TAR_FILES[package]
  83     if fnmatch.fnmatch(tar_file, '*.bz2'):
  84       extract = EXTRACT_STRIP_TBZ2
  85     elif fnmatch.fnmatch(tar_file, '*.gz'):
  86       extract = EXTRACT_STRIP_TGZ
  87     else:
  88       raise Exception('unexpected file name pattern in TAR_FILES[%r]' % package)
  89     sources[package] = {
  90         'type': 'source',
  91         'commands': [
  92             command.Command(extract + [command.path.join('%(abs_top_srcdir)s',
  93                                                          '..', 'third_party',
  94                                                          tar_file)],
  95                             cwd='%(output)s'),
  96             ],
  97         }
  98
  99   patch_packages = []
 100   patch_commands = []
 101   for package, info in GIT_REVISIONS.iteritems():
 102     sources[package] = {
 103         'type': 'source',
 104         'commands': [command.SyncGitRepo(GitUrl(package), '%(output)s',
 105                                          info['rev'])],
 106         }
 107     patch_packages.append(package)
 108     patch_commands.append(
 109         command.GenerateGitPatches('%(' + package + ')s/.git', info))
 110
 111   sources['patches'] = {
 112       'type': 'build',
 113       'dependencies': patch_packages,
 114       'commands': patch_commands,
 115       }
 116
 117   # The gcc_libs component gets the whole GCC source tree.
 118   sources['gcc_libs'] = sources['gcc']
 119
 120   # The gcc component omits all the source directories that are used solely
 121   # for building target libraries.  We don't want those included in the
 122   # input hash calculation so that we don't rebuild the compiler when the
 123   # the only things that have changed are target libraries.
 124   sources['gcc'] = {
 125         'type': 'source',
 126         'dependencies': ['gcc_libs'],
 127         'commands': [command.CopyTree('%(gcc_libs)s', '%(output)s', [
 128             'boehm-gc',
 129             'libada',
 130             'libatomic',
 131             'libffi',
 132             'libgcc',
 133             'libgfortran',
 134             'libgo',
 135             'libgomp',
 136             'libitm',
 137             'libjava',
 138             'libmudflap',
 139             'libobjc',
 140             'libquadmath',
 141             'libsanitizer',
 142             'libssp',
 143             'libstdc++-v3',
 144             ])]
 145       }
 146
 147   # We have to populate the newlib source tree with the "exported" form of
 148   # some headers from the native_client source tree.  The newlib build
 149   # needs these to be in the expected place.  By doing this in the source
 150   # target, these files will be part of the input hash and so we don't need
 151   # to do anything else to keep track of when they might have changed in
 152   # the native_client source tree.
 153   newlib_sys_nacl = command.path.join('%(output)s',
 154                                       'newlib', 'libc', 'sys', 'nacl')
 155   newlib_unpack = [command.RemoveDirectory(command.path.join(newlib_sys_nacl,
 156                                                              dirname))
 157                    for dirname in ['bits', 'sys', 'machine']]
 158   newlib_unpack.append(command.Command([
 159       'python',
 160       command.path.join('%(top_srcdir)s', 'src',
 161                         'trusted', 'service_runtime', 'export_header.py'),
 162       command.path.join('%(top_srcdir)s', 'src',
 163                         'trusted', 'service_runtime', 'include'),
 164       newlib_sys_nacl,
 165       ]))
 166   sources['newlib']['commands'] += newlib_unpack
 167
 168   return sources
 169
 170
 171 # Canonical tuples we use for hosts.
 172 WINDOWS_HOST_TUPLE = 'i686-w64-mingw32'
 173 MAC_HOST_TUPLE = 'x86_64-apple-darwin'
 174 ARM_HOST_TUPLE = 'arm-linux-gnueabihf'
 175 LINUX_X86_32_TUPLE = 'i686-linux'
 176 LINUX_X86_64_TUPLE = 'x86_64-linux'
 177
 178 # Map of native host tuple to extra tuples that it cross-builds for.
 179 EXTRA_HOSTS_MAP = {
 180     LINUX_X86_64_TUPLE: [
 181         LINUX_X86_32_TUPLE,
 182         ARM_HOST_TUPLE,
 183         WINDOWS_HOST_TUPLE,
 184         ],
 185     }
 186
 187 # Map of native host tuple to host tuples that are "native enough".
 188 # For these hosts, we will do a native-style build even though it's
 189 # not the native tuple, just passing some extra compiler flags.
 190 NATIVE_ENOUGH_MAP = {
 191     LINUX_X86_64_TUPLE: {
 192         LINUX_X86_32_TUPLE: ['-m32'],
 193         },
 194     }
 195
 196
 197 # The list of targets to build toolchains for.
 198 TARGET_LIST = ['arm', 'i686']
 199
 200 # These are extra arguments to pass gcc's configure that vary by target.
 201 TARGET_GCC_CONFIG = {
 202 # TODO(mcgrathr): Disabled tuning for now, tickling a constant-pool layout bug.
 203 #    'arm': ['--with-tune=cortex-a15'],
 204     }
 205
 206 PACKAGE_NAME = 'Native Client SDK [%(build_signature)s]'
 207 BUG_URL = 'http://gonacl.com/reportissue'
 208
 209 TAR_XV = ['tar', '-x', '-v']
 210 EXTRACT_STRIP_TGZ = TAR_XV + ['--gzip', '--strip-components=1', '-f']
 211 EXTRACT_STRIP_TBZ2 = TAR_XV + ['--bzip2', '--strip-components=1', '-f']
 212 CONFIGURE_CMD = ['sh', '%(src)s/configure']
 213 MAKE_PARALLEL_CMD = ['make', '-j%(cores)s']
 214 MAKE_CHECK_CMD = MAKE_PARALLEL_CMD + ['check']
 215 MAKE_DESTDIR_CMD = ['make', 'DESTDIR=%(abs_output)s']
 216
 217 # This file gets installed by multiple packages' install steps, but it is
 218 # never useful when installed in isolation.  So we remove it from the
 219 # installation directories before packaging up.
 220 REMOVE_INFO_DIR = command.Remove(command.path.join('%(output)s',
 221                                                    'share', 'info', 'dir'))
 222
 223 def ConfigureHostArch(host):
 224   configure_args = []
 225
 226   is_cross = CrossCompiling(host)
 227
 228   if is_cross:
 229     extra_cc_args = []
 230     configure_args.append('--host=' + host)
 231   else:
 232     extra_cc_args = NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}).get(host, [])
 233     if extra_cc_args:
 234       # The host we've chosen is "native enough", such as x86-32 on x86-64.
 235       # But it's not what config.guess will yield, so we need to supply
 236       # a --build switch to ensure things build correctly.
 237       configure_args.append('--build=' + host)
 238
 239   extra_cxx_args = list(extra_cc_args)
 240   if fnmatch.fnmatch(host, '*-linux*'):
 241     # Avoid shipping binaries with a runtime dependency on
 242     # a particular version of the libstdc++ shared library.
 243     # TODO(mcgrathr): Do we want this for MinGW and/or Mac too?
 244     extra_cxx_args.append('-static-libstdc++')
 245
 246   if extra_cc_args:
 247     # These are the defaults when there is no setting, but we will add
 248     # additional switches, so we must supply the command name too.
 249     if is_cross:
 250       cc = host + '-gcc'
 251     else:
 252       cc = 'gcc'
 253     configure_args.append('CC=' + ' '.join([cc] + extra_cc_args))
 254
 255   if extra_cxx_args:
 256     # These are the defaults when there is no setting, but we will add
 257     # additional switches, so we must supply the command name too.
 258     if is_cross:
 259       cxx = host + '-g++'
 260     else:
 261       cxx = 'g++'
 262     configure_args.append('CXX=' + ' '.join([cxx] + extra_cxx_args))
 263
 264   if HostIsWindows(host):
 265     # The i18n support brings in runtime dependencies on MinGW DLLs
 266     # that we don't want to have to distribute alongside our binaries.
 267     # So just disable it, and compiler messages will always be in US English.
 268     configure_args.append('--disable-nls')
 269
 270   return configure_args
 271
 272
 273 def ConfigureHostCommon(host):
 274   return ConfigureHostArch(host) + [
 275       '--prefix=',
 276       '--disable-silent-rules',
 277       '--without-gcc-arch',
 278       ]
 279
 280
 281 def ConfigureHostLib(host):
 282   return ConfigureHostCommon(host) + [
 283       '--disable-shared',
 284       ]
 285
 286
 287 def ConfigureHostTool(host):
 288   return ConfigureHostCommon(host) + [
 289       '--with-pkgversion=' + PACKAGE_NAME,
 290       '--with-bugurl=' + BUG_URL,
 291       '--without-zlib',
 292       ]
 293
 294
 295 def MakeCommand(host, extra_args=[]):
 296   if HostIsWindows(host):
 297     # There appears to be nothing we can pass at top-level configure time
 298     # that will prevent the configure scripts from finding MinGW's libiconv
 299     # and using it.  We have to force this variable into the environment
 300     # of the sub-configure runs, which are run via make.
 301     make_command = MAKE_PARALLEL_CMD + ['HAVE_LIBICONV=no']
 302   else:
 303     make_command = MAKE_PARALLEL_CMD
 304   return make_command + extra_args
 305
 306
 307 # Return the 'make check' command to run.
 308 # When cross-compiling, don't try to run test suites.
 309 def MakeCheckCommand(host):
 310   if CrossCompiling(host):
 311     return ['true']
 312   return MAKE_CHECK_CMD
 313
 314
 315 def InstallDocFiles(subdir, files):
 316   doc_dir = command.path.join('%(output)s', 'share', 'doc', subdir)
 317   dirs = sorted(set([command.path.dirname(command.path.join(doc_dir, file))
 318                      for file in files]))
 319   commands = ([command.Mkdir(dir, parents=True) for dir in dirs] +
 320               [command.Copy(command.path.join('%(' + subdir + ')s', file),
 321                             command.path.join(doc_dir, file))
 322                for file in files])
 323   return commands
 324
 325
 326 def NewlibLibcScript(arch):
 327   template = """/*
 328  * This is a linker script that gets installed as libc.a for the
 329  * newlib-based NaCl toolchain.  It brings in the constituent
 330  * libraries that make up what -lc means semantically.
 331  */
 332 OUTPUT_FORMAT(%s)
 333 GROUP ( libcrt_common.a libnacl.a )
 334 """
 335   if arch == 'arm':
 336     # Listing three formats instead of one makes -EL/-EB switches work
 337     # for the endian-switchable ARM backend.
 338     format_list = ['elf32-littlearm-nacl',
 339                    'elf32-bigarm-nacl',
 340                    'elf32-littlearm-nacl']
 341   elif arch == 'i686':
 342     format_list = 'elf32-i386-nacl'
 343   elif arch == 'x86_64':
 344     format_list = 'elf32-x86_64-nacl'
 345   else:
 346     raise Exception('TODO(mcgrathr): OUTPUT_FORMAT for %s' % arch)
 347   return template % ', '.join(['"' + fmt + '"' for fmt in format_list])
 348
 349
 350 # The default strip behavior removes debugging and symbol table
 351 # sections, but it leaves the .comment section.  This contains the
 352 # compiler version string, and so it changes when the compiler changes
 353 # even if the actual machine code it produces is completely identical.
 354 # Hence, the target library packages will always change when the
 355 # compiler changes unless these sections are removed.  Doing this
 356 # requires somehow teaching the makefile rules to pass the
 357 # --remove-section=.comment switch to TARGET-strip.  For the GCC
 358 # target libraries, setting STRIP_FOR_TARGET is sufficient.  But
 359 # quoting nightmares make it difficult to pass a command with a space
 360 # in it as the STRIP_FOR_TARGET value.  So the build writes a little
 361 # script that can be invoked with a simple name.
 362 #
 363 # Though the gcc target libraries' makefiles are smart enough to obey
 364 # STRIP_FOR_TARGET for library files, the newlib makefiles just
 365 # blindly use $(INSTALL_DATA) for both header (text) files and library
 366 # files.  Hence it's necessary to override its INSTALL_DATA setting to
 367 # one that will do stripping using this script, and thus the script
 368 # must silently do nothing to non-binary files.
 369 def ConfigureTargetPrep(arch):
 370   script_file = 'strip_for_target'
 371
 372   config_target = arch + '-nacl'
 373   script_contents = """\
 374 #!/bin/sh
 375 mode=--strip-all
 376 for arg; do
 377   case "$arg" in
 378   -*) ;;
 379   *)
 380     type=`file --brief --mime-type "$arg"`
 381     case "$type" in
 382       application/x-executable|application/x-sharedlib) ;;
 383       application/x-archive|application/x-object) mode=--strip-debug ;;
 384       *) exit 0 ;;
 385     esac
 386     ;;
 387   esac
 388 done
 389 exec %s-strip $mode --remove-section=.comment "$@"
 390 """ % config_target
 391
 392   return [
 393       command.WriteData(script_contents, script_file),
 394       command.Command(['chmod', '+x', script_file]),
 395       ]
 396
 397
 398 def ConfigureTargetArgs(arch):
 399   config_target = arch + '-nacl'
 400   return [
 401       '--target=' + config_target,
 402       '--with-sysroot=/' + config_target,
 403       'STRIP_FOR_TARGET=%(cwd)s/strip_for_target',
 404       ]
 405
 406
 407 def CommandsInBuild(command_lines):
 408   return [
 409       command.RemoveDirectory('build'),
 410       command.Mkdir('build'),
 411       ] + [command.Command(cmd, cwd='build')
 412            for cmd in command_lines]
 413
 414
 415 def PopulateDeps(dep_dirs):
 416   commands = [command.RemoveDirectory('all_deps'),
 417               command.Mkdir('all_deps')]
 418   commands += [command.Command('cp -r "%s/"* all_deps' % dirname, shell=True)
 419                for dirname in dep_dirs]
 420   return commands
 421
 422
 423 def WithDepsOptions(options, component=None):
 424   if component is None:
 425     directory = command.path.join('%(cwd)s', 'all_deps')
 426   else:
 427     directory = '%(abs_' + component + ')s'
 428   return ['--with-' + option + '=' + directory
 429           for option in options]
 430
 431
 432 # Return the component name we'll use for a base component name and
 433 # a host tuple.  The component names cannot contain dashes or other
 434 # non-identifier characters, because the names of the files uploaded
 435 # to Google Storage are constrained.  GNU configuration tuples contain
 436 # dashes, which we translate to underscores.
 437 def ForHost(component_name, host):
 438   return component_name + '_' + gsd_storage.LegalizeName(host)
 439
 440
 441 # These are libraries that go into building the compiler itself.
 442 def HostGccLibs(host):
 443   def H(component_name):
 444     return ForHost(component_name, host)
 445   host_gcc_libs = {
 446       H('gmp'): {
 447           'type': 'build',
 448           'dependencies': ['gmp'],
 449           'commands': [
 450               command.Command(ConfigureCommand('gmp') +
 451                               ConfigureHostLib(host) + [
 452                                   '--with-sysroot=%(abs_output)s',
 453                                   '--enable-cxx',
 454                                   # Without this, the built library will
 455                                   # assume the instruction set details
 456                                   # available on the build machine.  With
 457                                   # this, it dynamically chooses what code
 458                                   # to use based on the details of the
 459                                   # actual host CPU at runtime.
 460                                   '--enable-fat',
 461                                   ]),
 462               command.Command(MakeCommand(host)),
 463               command.Command(MakeCheckCommand(host)),
 464               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 465               ],
 466           },
 467       H('mpfr'): {
 468           'type': 'build',
 469           'dependencies': ['mpfr', H('gmp')],
 470           'commands': [
 471               command.Command(ConfigureCommand('mpfr') +
 472                               ConfigureHostLib(host) +
 473                               WithDepsOptions(['sysroot', 'gmp'], H('gmp'))),
 474               command.Command(MakeCommand(host)),
 475               command.Command(MakeCheckCommand(host)),
 476               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 477               ],
 478           },
 479       H('mpc'): {
 480           'type': 'build',
 481           'dependencies': ['mpc', H('gmp'), H('mpfr')],
 482           'commands': PopulateDeps(['%(' + H('gmp') + ')s',
 483                                     '%(' + H('mpfr') + ')s']) + [
 484               command.Command(ConfigureCommand('mpc') +
 485                               ConfigureHostLib(host) +
 486                               WithDepsOptions(['sysroot', 'gmp', 'mpfr'])),
 487               command.Command(MakeCommand(host)),
 488               command.Command(MakeCheckCommand(host)),
 489               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 490               ],
 491           },
 492       H('isl'): {
 493           'type': 'build',
 494           'dependencies': ['isl', H('gmp')],
 495           'commands': [
 496               command.Command(ConfigureCommand('isl') +
 497                               ConfigureHostLib(host) +
 498                               WithDepsOptions(['sysroot', 'gmp-prefix'],
 499                                               H('gmp'))),
 500               command.Command(MakeCommand(host)),
 501               command.Command(MakeCheckCommand(host)),
 502               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 503               # The .pc files wind up containing some absolute paths
 504               # that make the output depend on the build directory name.
 505               # The dependents' configure scripts don't need them anyway.
 506               command.RemoveDirectory(command.path.join(
 507                   '%(output)s', 'lib', 'pkgconfig')),
 508               ],
 509           },
 510       H('cloog'): {
 511           'type': 'build',
 512           'dependencies': ['cloog', H('gmp'), H('isl')],
 513           'commands': PopulateDeps(['%(' + H('gmp') + ')s',
 514                                     '%(' + H('isl') + ')s']) + [
 515               command.Command(ConfigureCommand('cloog') +
 516                               ConfigureHostLib(host) + [
 517                                   '--with-bits=gmp',
 518                                   '--with-isl=system',
 519                                   ] + WithDepsOptions(['sysroot',
 520                                                        'gmp-prefix',
 521                                                        'isl-prefix'])),
 522               command.Command(MakeCommand(host)),
 523               command.Command(MakeCheckCommand(host)),
 524               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 525               # The .pc files wind up containing some absolute paths
 526               # that make the output depend on the build directory name.
 527               # The dependents' configure scripts don't need them anyway.
 528               command.RemoveDirectory(command.path.join(
 529                   '%(output)s', 'lib', 'pkgconfig')),
 530               ],
 531           },
 532       H('expat'): {
 533           'type': 'build',
 534           'dependencies': ['expat'],
 535           'commands': [
 536               command.Command(ConfigureCommand('expat') +
 537                               ConfigureHostLib(host)),
 538               command.Command(MakeCommand(host)),
 539               command.Command(MakeCheckCommand(host)),
 540               command.Command(MAKE_DESTDIR_CMD + [
 541                   # expat does not support the install-strip target.
 542                   'installlib',
 543                   'INSTALL=%(expat)s/conftools/install-sh -c -s',
 544                   'INSTALL_DATA=%(expat)s/conftools/install-sh -c -m 644',
 545                   ]),
 546               ],
 547           },
 548       }
 549   return host_gcc_libs
 550
 551
 552 HOST_GCC_LIBS_DEPS = ['gmp', 'mpfr', 'mpc', 'isl', 'cloog']
 553
 554 def HostGccLibsDeps(host):
 555   return [ForHost(package, host) for package in HOST_GCC_LIBS_DEPS]
 556
 557
 558 def ConfigureCommand(source_component):
 559   return [command % {'src': '%(' + source_component + ')s'}
 560           for command in CONFIGURE_CMD]
 561
 562
 563 # When doing a Canadian cross, we need native-hosted cross components
 564 # to do the GCC build.
 565 def GccDeps(host, target):
 566   components = ['binutils_' + target]
 567   if CrossCompiling(host):
 568     components.append('gcc_' + target)
 569     host = NATIVE_TUPLE
 570   return [ForHost(component, host) for component in components]
 571
 572
 573 def GccCommand(host, target, cmd):
 574   components_for_path = GccDeps(host, target)
 575   return command.Command(
 576       cmd, path_dirs=[command.path.join('%(abs_' + component + ')s', 'bin')
 577                       for component in components_for_path])
 578
 579
 580 def ConfigureGccCommand(source_component, host, target, extra_args=[]):
 581   return GccCommand(
 582       host,
 583       target,
 584       ConfigureCommand(source_component) +
 585       ConfigureHostTool(host) +
 586       ConfigureTargetArgs(target) +
 587       TARGET_GCC_CONFIG.get(target, []) + [
 588           '--with-gmp=%(abs_' + ForHost('gmp', host) + ')s',
 589           '--with-mpfr=%(abs_' + ForHost('mpfr', host) + ')s',
 590           '--with-mpc=%(abs_' + ForHost('mpc', host) + ')s',
 591           '--with-isl=%(abs_' + ForHost('isl', host) + ')s',
 592           '--with-cloog=%(abs_' + ForHost('cloog', host) + ')s',
 593           '--enable-cloog-backend=isl',
 594           '--disable-dlopen',
 595           '--disable-shared',
 596           '--with-newlib',
 597           '--with-linker-hash-style=gnu',
 598           '--enable-linker-build-id',
 599           '--enable-languages=c,c++,lto',
 600           ] + extra_args)
 601
 602
 603
 604 def HostTools(host, target):
 605   def H(component_name):
 606     return ForHost(component_name, host)
 607   tools = {
 608       H('binutils_' + target): {
 609           'type': 'build',
 610           'dependencies': ['binutils'],
 611           'commands': ConfigureTargetPrep(target) + [
 612               command.Command(
 613                   ConfigureCommand('binutils') +
 614                   ConfigureHostTool(host) +
 615                   ConfigureTargetArgs(target) + [
 616                       '--enable-deterministic-archives',
 617                       '--enable-gold',
 618                       ] + ([] if HostIsWindows(host) else [
 619                           '--enable-plugins',
 620                           ])),
 621               command.Command(MakeCommand(host)),
 622               command.Command(MakeCheckCommand(host)),
 623               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 624               REMOVE_INFO_DIR,
 625               ] + InstallDocFiles('binutils',
 626                                   ['COPYING3'] +
 627                                   [command.path.join(subdir, 'NEWS')
 628                                    for subdir in
 629                                    ['binutils', 'gas', 'ld', 'gold']]) +
 630               # The top-level lib* directories contain host libraries
 631               # that we don't want to include in the distribution.
 632               [command.RemoveDirectory(command.path.join('%(output)s', name))
 633                for name in ['lib', 'lib32', 'lib64']],
 634           },
 635
 636       H('gcc_' + target): {
 637           'type': 'build',
 638           'dependencies': (['gcc'] + HostGccLibsDeps(host) +
 639                            GccDeps(host, target)),
 640           'commands': ConfigureTargetPrep(target) + [
 641               ConfigureGccCommand('gcc', host, target),
 642               # GCC's configure step writes configargs.h with some strings
 643               # including the configure command line, which get embedded
 644               # into the gcc driver binary.  The build only works if we use
 645               # absolute paths in some of the configure switches, but
 646               # embedding those paths makes the output differ in repeated
 647               # builds done in different directories, which we do not want.
 648               # So force the generation of that file early and then edit it
 649               # in place to replace the absolute paths with something that
 650               # never varies.  Note that the 'configure-gcc' target will
 651               # actually build some components before running gcc/configure.
 652               GccCommand(host, target,
 653                          MakeCommand(host, ['configure-gcc'])),
 654               command.Command(['sed', '-i', '-e',
 655                                ';'.join(['s@%%(abs_%s)s@.../%s_install@g' %
 656                                          (component, component)
 657                                          for component in
 658                                          HostGccLibsDeps(host)] +
 659                                         ['s@%(cwd)s@...@g']),
 660                                command.path.join('gcc', 'configargs.h')]),
 661               # gcc/Makefile's install rules ordinarily look at the
 662               # installed include directory for a limits.h to decide
 663               # whether the lib/gcc/.../include-fixed/limits.h header
 664               # should be made to expect a libc-supplied limits.h or not.
 665               # Since we're doing this build in a clean environment without
 666               # any libc installed, we need to force its hand here.
 667               GccCommand(host, target,
 668                          MakeCommand(host, ['all-gcc', 'LIMITS_H_TEST=true'])),
 669               # gcc/Makefile's install targets populate this directory
 670               # only if it already exists.
 671               command.Mkdir(command.path.join('%(output)s',
 672                                               target + '-nacl', 'bin'),
 673                             True),
 674               GccCommand(host, target,
 675                          MAKE_DESTDIR_CMD + ['install-strip-gcc']),
 676               REMOVE_INFO_DIR,
 677               # Note we include COPYING.RUNTIME here and not with gcc_libs.
 678               ] + InstallDocFiles('gcc', ['COPYING3', 'COPYING.RUNTIME']),
 679           },
 680
 681       # GDB can support all the targets in one host tool.
 682       H('gdb'): {
 683           'type': 'build',
 684           'dependencies': ['gdb', H('expat')],
 685           'commands': [
 686               command.Command(
 687                   ConfigureCommand('gdb') +
 688                   ConfigureHostTool(host) + [
 689                       '--target=x86_64-nacl',
 690                       '--enable-targets=arm-none-eabi-nacl',
 691                       '--with-expat',
 692                       'CPPFLAGS=-I%(abs_' + H('expat') + ')s/include',
 693                       'LDFLAGS=-L%(abs_' + H('expat') + ')s/lib',
 694                       ] +
 695                   (['--without-python'] if HostIsWindows(host) else []) +
 696                   # TODO(mcgrathr): The default -Werror only breaks because
 697                   # the OSX default compiler is an old front-end that does
 698                   # not understand all the GCC options.  Maybe switch to
 699                   # using clang (system or Chromium-supplied) on Mac.
 700                   (['--disable-werror'] if HostIsMac(host) else [])),
 701               command.Command(MakeCommand(host) + ['all-gdb']),
 702               command.Command(MAKE_DESTDIR_CMD + [
 703                   '-C', 'gdb', 'install-strip',
 704                   ]),
 705               REMOVE_INFO_DIR,
 706               ] + InstallDocFiles('gdb', [
 707                   'COPYING3',
 708                   command.path.join('gdb', 'NEWS'),
 709                   ]),
 710           },
 711       }
 712
 713   # TODO(mcgrathr): The ARM cross environment does not supply a termcap
 714   # library, so it cannot build GDB.
 715   if host.startswith('arm') and CrossCompiling(host):
 716     del tools[H('gdb')]
 717
 718   return tools
 719
 720 def TargetCommands(host, target, command_list):
 721   # First we have to copy the host tools into a common directory.
 722   # We can't just have both directories in our PATH, because the
 723   # compiler looks for the assembler and linker relative to itself.
 724   commands = PopulateDeps(['%(' + ForHost('binutils_' + target, host) + ')s',
 725                            '%(' + ForHost('gcc_' + target, host) + ')s'])
 726   bindir = command.path.join('%(cwd)s', 'all_deps', 'bin')
 727   commands += [command.Command(cmd, path_dirs=[bindir])
 728                for cmd in command_list]
 729   return commands
 730
 731
 732 def TargetLibs(host, target):
 733   lib_deps = [ForHost(component + '_' + target, host)
 734               for component in ['binutils', 'gcc']]
 735
 736   def NewlibFile(subdir, name):
 737     return command.path.join('%(output)s', target + '-nacl', subdir, name)
 738
 739   newlib_sysroot = '%(abs_newlib_' + target + ')s'
 740   newlib_tooldir = '%s/%s-nacl' % (newlib_sysroot, target)
 741
 742   # See the comment at ConfigureTargetPrep, above.
 743   newlib_install_data = ' '.join(['STRIPPROG=%(cwd)s/strip_for_target',
 744                                   '%(abs_newlib)s/install-sh',
 745                                   '-c', '-s', '-m', '644'])
 746
 747   iconv_encodings = 'UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4'
 748   newlib_configure_args = [
 749       '--disable-libgloss',
 750       '--enable-newlib-iconv',
 751       '--enable-newlib-iconv-from-encodings=' + iconv_encodings,
 752       '--enable-newlib-iconv-to-encodings=' + iconv_encodings,
 753       '--enable-newlib-io-long-long',
 754       '--enable-newlib-io-long-double',
 755       '--enable-newlib-io-c99-formats',
 756       '--enable-newlib-mb',
 757       'CFLAGS=-O2',
 758       'INSTALL_DATA=' + newlib_install_data,
 759       ]
 760
 761   newlib_post_install = [
 762       command.Rename(NewlibFile('lib', 'libc.a'),
 763                      NewlibFile('lib', 'libcrt_common.a')),
 764       command.WriteData(NewlibLibcScript(target),
 765                         NewlibFile('lib', 'libc.a')),
 766       ] + [
 767       command.Copy(
 768           command.path.join('%(pthread_headers)s', header),
 769           NewlibFile('include', header))
 770       for header in ('pthread.h', 'semaphore.h')
 771       ]
 772
 773
 774   libs = {
 775       'newlib_' + target: {
 776           'type': 'build',
 777           'dependencies': ['newlib'] + lib_deps,
 778           'inputs': { 'pthread_headers':
 779                       os.path.join(NACL_DIR, 'src', 'untrusted',
 780                                    'pthread') },
 781           'commands': (ConfigureTargetPrep(target) +
 782                        TargetCommands(host, target, [
 783                            ConfigureCommand('newlib') +
 784                            ConfigureHostTool(host) +
 785                            ConfigureTargetArgs(target) +
 786                            newlib_configure_args,
 787                            MakeCommand(host),
 788                            MAKE_DESTDIR_CMD + ['install-strip'],
 789                            ]) +
 790                        newlib_post_install +
 791                        InstallDocFiles('newlib', ['COPYING.NEWLIB'])),
 792           },
 793
 794       'gcc_libs_' + target: {
 795           'type': 'build',
 796           'dependencies': (['gcc_libs'] + lib_deps + ['newlib_' + target] +
 797                            HostGccLibsDeps(host)),
 798           # This actually builds the compiler again and uses that compiler
 799           # to build the target libraries.  That's by far the easiest thing
 800           # to get going given the interdependencies of the target
 801           # libraries (especially libgcc) on the gcc subdirectory, and
 802           # building the compiler doesn't really take all that long in the
 803           # grand scheme of things.
 804           # TODO(mcgrathr): If upstream ever cleans up all their
 805           # interdependencies better, unpack the compiler, configure with
 806           # --disable-gcc, and just build all-target.
 807           'commands': ConfigureTargetPrep(target) + [
 808               ConfigureGccCommand('gcc_libs', host, target, [
 809                   '--with-build-sysroot=' + newlib_sysroot,
 810                   ]),
 811               GccCommand(host, target,
 812                          MakeCommand(host) + [
 813                              'build_tooldir=' + newlib_tooldir,
 814                              'all-target',
 815                              ]),
 816               GccCommand(host, target,
 817                          MAKE_DESTDIR_CMD + ['install-strip-target']),
 818               REMOVE_INFO_DIR,
 819               ],
 820           },
 821       }
 822   return libs
 823
 824
 825 def NativeTuple():
 826   if sys.platform.startswith('linux'):
 827     machine = platform.machine().lower()
 828     if machine.startswith('arm'):
 829       # TODO(mcgrathr): How to distinguish gnueabi vs gnueabihf?
 830       return ARM_HOST_TUPLE
 831     if fnmatch.fnmatch(machine, 'i?86*'):
 832       return LINUX_X86_32_TUPLE
 833     if any(fnmatch.fnmatch(machine, pattern) for pattern in
 834            ['x86_64*', 'amd64*', 'x64*']):
 835       return LINUX_X86_64_TUPLE
 836     raise Exception('Machine %s not recognized' % machine)
 837   elif sys.platform.startswith('win'):
 838     return WINDOWS_HOST_TUPLE
 839   elif sys.platform.startswith('darwin'):
 840     return MAC_HOST_TUPLE
 841   raise Exception('Platform %s not recognized' % sys.platform)
 842
 843 # Compute it once.
 844 NATIVE_TUPLE = NativeTuple()
 845
 846
 847 # For our purposes, "cross-compiling" means not literally that we are
 848 # targetting a host that does not match NATIVE_TUPLE, but that we are
 849 # targetting a host whose binaries we cannot run locally.  So x86-32
 850 # on x86-64 does not count as cross-compiling.  See NATIVE_ENOUGH_MAP, above.
 851 def CrossCompiling(host):
 852   return (host != NATIVE_TUPLE and
 853           host not in NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}))
 854
 855
 856 def HostIsWindows(host):
 857   return host == WINDOWS_HOST_TUPLE
 858
 859
 860 def HostIsMac(host):
 861   return host == MAC_HOST_TUPLE
 862
 863
 864 # We build target libraries only on Linux for two reasons:
 865 # 1. We only need to build them once.
 866 # 2. Linux is the fastest to build.
 867 # TODO(mcgrathr): In future set up some scheme whereby non-Linux
 868 # bots can build target libraries but not archive them, only verifying
 869 # that the results came out the same as the ones archived by the
 870 # official builder bot.  That will serve as a test of the host tools
 871 # on the other host platforms.
 872 def BuildTargetLibsOn(host):
 873   return host == LINUX_X86_64_TUPLE
 874
 875
 876 def CollectPackagesForHost(host, targets):
 877   packages = HostGccLibs(host).copy()
 878   for target in targets:
 879     packages.update(HostTools(host, target))
 880     if BuildTargetLibsOn(host):
 881       packages.update(TargetLibs(host, target))
 882   return packages
 883
 884
 885 def CollectPackages(targets):
 886   packages = CollectSources()
 887
 888   packages.update(CollectPackagesForHost(NATIVE_TUPLE, targets))
 889
 890   for host in EXTRA_HOSTS_MAP.get(NATIVE_TUPLE, []):
 891     packages.update(CollectPackagesForHost(host, targets))
 892
 893   return packages
 894
 895
 896 PACKAGES = CollectPackages(TARGET_LIST)
 897
 898
 899 if __name__ == '__main__':
 900   tb = toolchain_main.PackageBuilder(PACKAGES, sys.argv[1:])
 901   # TODO(mcgrathr): The bot ought to run some native_client tests
 902   # using the new toolchain, like the old x86 toolchain bots do.
 903   tb.Main()