src/native_client/toolchain_build/toolchain_build.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Recipes for NativeClient toolchain packages.
   7
   8 The real entry plumbing is in toolchain_main.py.
   9 """
  10
  11 import fnmatch
  12 import platform
  13 import os
  14 import re
  15 import sys
  16
  17 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  18 import pynacl.gsd_storage
  19 import pynacl.platform
  20
  21 import command
  22 import toolchain_main
  23
  24
  25 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
  26 NACL_DIR = os.path.dirname(SCRIPT_DIR)
  27
  28
  29 # See command.GenerateGitPatches for the schema of entries in this dict.
  30 # Additionally, each may contain a 'repo' key whose value is the name
  31 # to use in place of the package name when calling GitUrl (below).
  32 GIT_REVISIONS = {
  33     'binutils': {
  34         'rev': '7deface59753c3b249ac08d854d471951796343f',
  35         'upstream-branch': 'upstream/binutils-2_24-branch',
  36         'upstream-name': 'binutils-2.24',
  37         # This is tag binutils-2_24, but Gerrit won't let us push
  38         # non-annotated tags, and the upstream tag is not annotated.
  39         'upstream-base': '237df3fa4a1d939e6fd1af0c3e5029a25a137310',
  40         },
  41     'gcc': {
  42         'rev': 'a2e4a3140c035233409598487c56b76108f8c74d',
  43         'upstream-branch': 'upstream/gcc-4_8-branch',
  44         'upstream-name': 'gcc-4.8.2',
  45          # Upstream tag gcc-4_8_2-release:
  46         'upstream-base': '9bcca88e24e64d4e23636aafa3404088b13bcb0e',
  47         },
  48     'newlib': {
  49         'rev': 'a9ae3c60b36dea3d8a10e18b1b6db952d21268c2',
  50         'upstream-branch': 'upstream/master',
  51         'upstream-name': 'newlib-2.0.0',
  52         # Upstream tag newlib_2_0_0:
  53         'upstream-base': 'c3fc84e062cacc2b3e13c1f6b9151d0cc85392ba',
  54         },
  55     'gdb': {
  56         'rev': '5deb4793a5e3f2f48d7899f424bb4484686020f8',
  57         'repo': 'binutils',
  58         'upstream-branch': 'upstream/gdb-7.7-branch',
  59         'upstream-name': 'gdb-7.7.1',
  60         # Upstream tag gdb-7.7-release:
  61         'upstream-base': '4bd8fc3a1362970d9800a263987af8093798338b',
  62         },
  63     }
  64
  65 TAR_FILES = {
  66     'gmp': command.path.join('gmp', 'gmp-6.0.0a.tar.bz2'),
  67     'mpfr': command.path.join('mpfr', 'mpfr-3.1.2.tar.bz2'),
  68     'mpc': command.path.join('mpc', 'mpc-1.0.2.tar.gz'),
  69     'isl': command.path.join('cloog', 'isl-0.12.2.tar.bz2'),
  70     'cloog': command.path.join('cloog', 'cloog-0.18.1.tar.gz'),
  71     'expat': command.path.join('expat', 'expat-2.1.0.tar.gz'),
  72     }
  73
  74 GIT_BASE_URL = 'https://chromium.googlesource.com/native_client'
  75
  76
  77 def GitUrl(package):
  78   repo = GIT_REVISIONS[package].get('repo', package)
  79   return '%s/nacl-%s.git' % (GIT_BASE_URL, repo)
  80
  81
  82 def CollectSources():
  83   sources = {}
  84
  85   for package in TAR_FILES:
  86     tar_file = TAR_FILES[package]
  87     if fnmatch.fnmatch(tar_file, '*.bz2'):
  88       extract = EXTRACT_STRIP_TBZ2
  89     elif fnmatch.fnmatch(tar_file, '*.gz'):
  90       extract = EXTRACT_STRIP_TGZ
  91     else:
  92       raise Exception('unexpected file name pattern in TAR_FILES[%r]' % package)
  93     sources[package] = {
  94         'type': 'source',
  95         'commands': [
  96             command.Command(extract + [command.path.join('%(abs_top_srcdir)s',
  97                                                          '..', 'third_party',
  98                                                          tar_file)],
  99                             cwd='%(output)s'),
 100             ],
 101         }
 102
 103   patch_packages = []
 104   patch_commands = []
 105   for package, info in GIT_REVISIONS.iteritems():
 106     sources[package] = {
 107         'type': 'source',
 108         'commands': [command.SyncGitRepo(GitUrl(package), '%(output)s',
 109                                          info['rev'])],
 110         }
 111     patch_packages.append(package)
 112     patch_info = {'name': package}
 113     patch_info.update(info)
 114     patch_commands.append(
 115         command.GenerateGitPatches('%(' + package + ')s/.git', patch_info))
 116
 117   sources['patches'] = {
 118       'type': 'build',
 119       'dependencies': patch_packages,
 120       'commands': patch_commands,
 121       }
 122
 123   # The gcc_libs component gets the whole GCC source tree.
 124   sources['gcc_libs'] = sources['gcc']
 125
 126   # The gcc component omits all the source directories that are used solely
 127   # for building target libraries.  We don't want those included in the
 128   # input hash calculation so that we don't rebuild the compiler when the
 129   # the only things that have changed are target libraries.
 130   sources['gcc'] = {
 131         'type': 'source',
 132         'dependencies': ['gcc_libs'],
 133         'commands': [command.CopyTree('%(gcc_libs)s', '%(output)s', [
 134             'boehm-gc',
 135             'libada',
 136             'libatomic',
 137             'libffi',
 138             'libgcc',
 139             'libgfortran',
 140             'libgo',
 141             'libgomp',
 142             'libitm',
 143             'libjava',
 144             'libmudflap',
 145             'libobjc',
 146             'libquadmath',
 147             'libsanitizer',
 148             'libssp',
 149             'libstdc++-v3',
 150             ])]
 151       }
 152
 153   # We have to populate the newlib source tree with the "exported" form of
 154   # some headers from the native_client source tree.  The newlib build
 155   # needs these to be in the expected place.  By doing this in the source
 156   # target, these files will be part of the input hash and so we don't need
 157   # to do anything else to keep track of when they might have changed in
 158   # the native_client source tree.
 159   newlib_sys_nacl = command.path.join('%(output)s',
 160                                       'newlib', 'libc', 'sys', 'nacl')
 161   newlib_unpack = [command.RemoveDirectory(command.path.join(newlib_sys_nacl,
 162                                                              dirname))
 163                    for dirname in ['bits', 'sys', 'machine']]
 164   newlib_unpack.append(command.Command([
 165       'python',
 166       command.path.join('%(top_srcdir)s', 'src',
 167                         'trusted', 'service_runtime', 'export_header.py'),
 168       command.path.join('%(top_srcdir)s', 'src',
 169                         'trusted', 'service_runtime', 'include'),
 170       newlib_sys_nacl,
 171       ]))
 172   sources['newlib']['commands'] += newlib_unpack
 173
 174   return sources
 175
 176
 177 # List of all platform and architectures we target and will distribute for.
 178 HOSTS = [
 179     ('win', 'x86-64'),
 180     ('darwin', 'x86-64'),
 181     ('linux', 'arm'),
 182     ('linux', 'x86-32')
 183     ]
 184
 185 # Canonical tuples we use for hosts.
 186 WINDOWS_HOST_TUPLE = pynacl.platform.PlatformTriple('win', 'x86-64')
 187 MAC_HOST_TUPLE = pynacl.platform.PlatformTriple('darwin', 'x86-64')
 188 ARM_HOST_TUPLE = pynacl.platform.PlatformTriple('linux', 'arm')
 189 LINUX_X86_32_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-32')
 190 LINUX_X86_64_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-64')
 191
 192 # Map of native host tuple to extra tuples that it cross-builds for.
 193 EXTRA_HOSTS_MAP = {
 194     LINUX_X86_64_TUPLE: [
 195         LINUX_X86_32_TUPLE,
 196         ARM_HOST_TUPLE,
 197         WINDOWS_HOST_TUPLE,
 198         ],
 199     }
 200
 201 # Map of native host tuple to host tuples that are "native enough".
 202 # For these hosts, we will do a native-style build even though it's
 203 # not the native tuple, just passing some extra compiler flags.
 204 NATIVE_ENOUGH_MAP = {
 205     LINUX_X86_64_TUPLE: {
 206         LINUX_X86_32_TUPLE: ['-m32'],
 207         },
 208     }
 209
 210 # The list of targets to build toolchains for.
 211 TARGET_LIST = ['arm', 'i686']
 212 UPLOAD_TARGETS = ['arm']
 213
 214 # GDB is built by toolchain_build but injected into package targets built by
 215 # other means. List out what package targets, packages, and the tar file we are
 216 # injecting on top of here.
 217 GDB_INJECT_HOSTS = [
 218   ('win', 'x86-64'),
 219   ('darwin', 'x86-64'),
 220   ('linux', 'x86-32'),
 221   ]
 222
 223 GDB_INJECT_PACKAGES = [
 224   ('nacl_x86_newlib', ['naclsdk.tgz']),
 225   ('nacl_x86_glibc', ['toolchain.tar.bz2']),
 226   ]
 227
 228 # These are extra arguments to pass gcc's configure that vary by target.
 229 TARGET_GCC_CONFIG = {
 230 # TODO(mcgrathr): Disabled tuning for now, tickling a constant-pool layout bug.
 231 #    'arm': ['--with-tune=cortex-a15'],
 232     }
 233
 234 PACKAGE_NAME = 'Native Client SDK [%(build_signature)s]'
 235 BUG_URL = 'http://gonacl.com/reportissue'
 236
 237 TAR_XV = ['tar', '-x', '-v']
 238 EXTRACT_STRIP_TGZ = TAR_XV + ['--gzip', '--strip-components=1', '-f']
 239 EXTRACT_STRIP_TBZ2 = TAR_XV + ['--bzip2', '--strip-components=1', '-f']
 240 CONFIGURE_CMD = ['sh', '%(src)s/configure']
 241 MAKE_PARALLEL_CMD = ['make', '-j%(cores)s']
 242 MAKE_CHECK_CMD = MAKE_PARALLEL_CMD + ['check']
 243 MAKE_DESTDIR_CMD = ['make', 'DESTDIR=%(abs_output)s']
 244
 245 # This file gets installed by multiple packages' install steps, but it is
 246 # never useful when installed in isolation.  So we remove it from the
 247 # installation directories before packaging up.
 248 REMOVE_INFO_DIR = command.Remove(command.path.join('%(output)s',
 249                                                    'share', 'info', 'dir'))
 250
 251 def ConfigureHostArch(host):
 252   configure_args = []
 253
 254   is_cross = CrossCompiling(host)
 255
 256   if is_cross:
 257     extra_cc_args = []
 258     configure_args.append('--host=' + host)
 259   else:
 260     extra_cc_args = NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}).get(host, [])
 261     if extra_cc_args:
 262       # The host we've chosen is "native enough", such as x86-32 on x86-64.
 263       # But it's not what config.guess will yield, so we need to supply
 264       # a --build switch to ensure things build correctly.
 265       configure_args.append('--build=' + host)
 266
 267   extra_cxx_args = list(extra_cc_args)
 268   if fnmatch.fnmatch(host, '*-linux*'):
 269     # Avoid shipping binaries with a runtime dependency on
 270     # a particular version of the libstdc++ shared library.
 271     # TODO(mcgrathr): Do we want this for MinGW and/or Mac too?
 272     extra_cxx_args.append('-static-libstdc++')
 273
 274   if extra_cc_args:
 275     # These are the defaults when there is no setting, but we will add
 276     # additional switches, so we must supply the command name too.
 277     if is_cross:
 278       cc = host + '-gcc'
 279     else:
 280       cc = 'gcc'
 281     configure_args.append('CC=' + ' '.join([cc] + extra_cc_args))
 282
 283   if extra_cxx_args:
 284     # These are the defaults when there is no setting, but we will add
 285     # additional switches, so we must supply the command name too.
 286     if is_cross:
 287       cxx = host + '-g++'
 288     else:
 289       cxx = 'g++'
 290     configure_args.append('CXX=' + ' '.join([cxx] + extra_cxx_args))
 291
 292   if HostIsWindows(host):
 293     # The i18n support brings in runtime dependencies on MinGW DLLs
 294     # that we don't want to have to distribute alongside our binaries.
 295     # So just disable it, and compiler messages will always be in US English.
 296     configure_args.append('--disable-nls')
 297
 298   return configure_args
 299
 300
 301 def ConfigureHostCommon(host):
 302   return ConfigureHostArch(host) + [
 303       '--prefix=',
 304       '--disable-silent-rules',
 305       '--without-gcc-arch',
 306       ]
 307
 308
 309 def ConfigureHostLib(host):
 310   return ConfigureHostCommon(host) + [
 311       '--disable-shared',
 312       ]
 313
 314
 315 def ConfigureHostTool(host):
 316   return ConfigureHostCommon(host) + [
 317       '--with-pkgversion=' + PACKAGE_NAME,
 318       '--with-bugurl=' + BUG_URL,
 319       '--without-zlib',
 320       ]
 321
 322
 323 def MakeCommand(host, extra_args=[]):
 324   if HostIsWindows(host):
 325     # There appears to be nothing we can pass at top-level configure time
 326     # that will prevent the configure scripts from finding MinGW's libiconv
 327     # and using it.  We have to force this variable into the environment
 328     # of the sub-configure runs, which are run via make.
 329     make_command = MAKE_PARALLEL_CMD + ['HAVE_LIBICONV=no']
 330   else:
 331     make_command = MAKE_PARALLEL_CMD
 332   return make_command + extra_args
 333
 334
 335 # Return the 'make check' command to run.
 336 # When cross-compiling, don't try to run test suites.
 337 def MakeCheckCommand(host):
 338   if CrossCompiling(host):
 339     return ['true']
 340   return MAKE_CHECK_CMD
 341
 342
 343 def InstallDocFiles(subdir, files):
 344   doc_dir = command.path.join('%(output)s', 'share', 'doc', subdir)
 345   dirs = sorted(set([command.path.dirname(command.path.join(doc_dir, file))
 346                      for file in files]))
 347   commands = ([command.Mkdir(dir, parents=True) for dir in dirs] +
 348               [command.Copy(command.path.join('%(' + subdir + ')s', file),
 349                             command.path.join(doc_dir, file))
 350                for file in files])
 351   return commands
 352
 353
 354 def NewlibLibcScript(arch):
 355   template = """/*
 356  * This is a linker script that gets installed as libc.a for the
 357  * newlib-based NaCl toolchain.  It brings in the constituent
 358  * libraries that make up what -lc means semantically.
 359  */
 360 OUTPUT_FORMAT(%s)
 361 GROUP ( libnacl.a libcrt_common.a )
 362 """
 363   if arch == 'arm':
 364     # Listing three formats instead of one makes -EL/-EB switches work
 365     # for the endian-switchable ARM backend.
 366     format_list = ['elf32-littlearm-nacl',
 367                    'elf32-bigarm-nacl',
 368                    'elf32-littlearm-nacl']
 369   elif arch == 'i686':
 370     format_list = 'elf32-i386-nacl'
 371   elif arch == 'x86_64':
 372     format_list = 'elf32-x86_64-nacl'
 373   else:
 374     raise Exception('TODO(mcgrathr): OUTPUT_FORMAT for %s' % arch)
 375   return template % ', '.join(['"' + fmt + '"' for fmt in format_list])
 376
 377
 378 # The default strip behavior removes debugging and symbol table
 379 # sections, but it leaves the .comment section.  This contains the
 380 # compiler version string, and so it changes when the compiler changes
 381 # even if the actual machine code it produces is completely identical.
 382 # Hence, the target library packages will always change when the
 383 # compiler changes unless these sections are removed.  Doing this
 384 # requires somehow teaching the makefile rules to pass the
 385 # --remove-section=.comment switch to TARGET-strip.  For the GCC
 386 # target libraries, setting STRIP_FOR_TARGET is sufficient.  But
 387 # quoting nightmares make it difficult to pass a command with a space
 388 # in it as the STRIP_FOR_TARGET value.  So the build writes a little
 389 # script that can be invoked with a simple name.
 390 #
 391 # Though the gcc target libraries' makefiles are smart enough to obey
 392 # STRIP_FOR_TARGET for library files, the newlib makefiles just
 393 # blindly use $(INSTALL_DATA) for both header (text) files and library
 394 # files.  Hence it's necessary to override its INSTALL_DATA setting to
 395 # one that will do stripping using this script, and thus the script
 396 # must silently do nothing to non-binary files.
 397 def ConfigureTargetPrep(arch):
 398   script_file = 'strip_for_target'
 399
 400   config_target = arch + '-nacl'
 401   script_contents = """\
 402 #!/bin/sh
 403 mode=--strip-all
 404 for arg; do
 405   case "$arg" in
 406   -*) ;;
 407   *)
 408     type=`file --brief --mime-type "$arg"`
 409     case "$type" in
 410       application/x-executable|application/x-sharedlib) ;;
 411       application/x-archive|application/x-object) mode=--strip-debug ;;
 412       *) exit 0 ;;
 413     esac
 414     ;;
 415   esac
 416 done
 417 exec %s-strip $mode --remove-section=.comment "$@"
 418 """ % config_target
 419
 420   return [
 421       command.WriteData(script_contents, script_file),
 422       command.Command(['chmod', '+x', script_file]),
 423       ]
 424
 425
 426 def ConfigureTargetArgs(arch):
 427   config_target = arch + '-nacl'
 428   return [
 429       '--target=' + config_target,
 430       '--with-sysroot=/' + config_target,
 431       'STRIP_FOR_TARGET=%(cwd)s/strip_for_target',
 432       ]
 433
 434
 435 def CommandsInBuild(command_lines):
 436   return [
 437       command.RemoveDirectory('build'),
 438       command.Mkdir('build'),
 439       ] + [command.Command(cmd, cwd='build')
 440            for cmd in command_lines]
 441
 442
 443 def PopulateDeps(dep_dirs):
 444   commands = [command.RemoveDirectory('all_deps'),
 445               command.Mkdir('all_deps')]
 446   commands += [command.Command('cp -r "%s/"* all_deps' % dirname, shell=True)
 447                for dirname in dep_dirs]
 448   return commands
 449
 450
 451 def WithDepsOptions(options, component=None):
 452   if component is None:
 453     directory = command.path.join('%(cwd)s', 'all_deps')
 454   else:
 455     directory = '%(abs_' + component + ')s'
 456   return ['--with-' + option + '=' + directory
 457           for option in options]
 458
 459
 460 # Return the component name we'll use for a base component name and
 461 # a host tuple.  The component names cannot contain dashes or other
 462 # non-identifier characters, because the names of the files uploaded
 463 # to Google Storage are constrained.  GNU configuration tuples contain
 464 # dashes, which we translate to underscores.
 465 def ForHost(component_name, host):
 466   return component_name + '_' + pynacl.gsd_storage.LegalizeName(host)
 467
 468
 469 # These are libraries that go into building the compiler itself.
 470 def HostGccLibs(host):
 471   def H(component_name):
 472     return ForHost(component_name, host)
 473   host_gcc_libs = {
 474       H('gmp'): {
 475           'type': 'build',
 476           'dependencies': ['gmp'],
 477           'commands': [
 478               command.Command(ConfigureCommand('gmp') +
 479                               ConfigureHostLib(host) + [
 480                                   '--with-sysroot=%(abs_output)s',
 481                                   '--enable-cxx',
 482                                   # Without this, the built library will
 483                                   # assume the instruction set details
 484                                   # available on the build machine.  With
 485                                   # this, it dynamically chooses what code
 486                                   # to use based on the details of the
 487                                   # actual host CPU at runtime.
 488                                   '--enable-fat',
 489                                   ]),
 490               command.Command(MakeCommand(host)),
 491               command.Command(MakeCheckCommand(host)),
 492               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 493               ],
 494           },
 495       H('mpfr'): {
 496           'type': 'build',
 497           'dependencies': ['mpfr', H('gmp')],
 498           'commands': [
 499               command.Command(ConfigureCommand('mpfr') +
 500                               ConfigureHostLib(host) +
 501                               WithDepsOptions(['sysroot', 'gmp'], H('gmp'))),
 502               command.Command(MakeCommand(host)),
 503               command.Command(MakeCheckCommand(host)),
 504               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 505               ],
 506           },
 507       H('mpc'): {
 508           'type': 'build',
 509           'dependencies': ['mpc', H('gmp'), H('mpfr')],
 510           'commands': PopulateDeps(['%(' + H('gmp') + ')s',
 511                                     '%(' + H('mpfr') + ')s']) + [
 512               command.Command(ConfigureCommand('mpc') +
 513                               ConfigureHostLib(host) +
 514                               WithDepsOptions(['sysroot', 'gmp', 'mpfr'])),
 515               command.Command(MakeCommand(host)),
 516               command.Command(MakeCheckCommand(host)),
 517               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 518               ],
 519           },
 520       H('isl'): {
 521           'type': 'build',
 522           'dependencies': ['isl', H('gmp')],
 523           'commands': [
 524               command.Command(ConfigureCommand('isl') +
 525                               ConfigureHostLib(host) +
 526                               WithDepsOptions(['sysroot', 'gmp-prefix'],
 527                                               H('gmp'))),
 528               command.Command(MakeCommand(host)),
 529               command.Command(MakeCheckCommand(host)),
 530               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 531               # The .pc files wind up containing some absolute paths
 532               # that make the output depend on the build directory name.
 533               # The dependents' configure scripts don't need them anyway.
 534               command.RemoveDirectory(command.path.join(
 535                   '%(output)s', 'lib', 'pkgconfig')),
 536               ],
 537           },
 538       H('cloog'): {
 539           'type': 'build',
 540           'dependencies': ['cloog', H('gmp'), H('isl')],
 541           'commands': PopulateDeps(['%(' + H('gmp') + ')s',
 542                                     '%(' + H('isl') + ')s']) + [
 543               command.Command(ConfigureCommand('cloog') +
 544                               ConfigureHostLib(host) + [
 545                                   '--with-bits=gmp',
 546                                   '--with-isl=system',
 547                                   ] + WithDepsOptions(['sysroot',
 548                                                        'gmp-prefix',
 549                                                        'isl-prefix'])),
 550               command.Command(MakeCommand(host)),
 551               command.Command(MakeCheckCommand(host)),
 552               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 553               # The .pc files wind up containing some absolute paths
 554               # that make the output depend on the build directory name.
 555               # The dependents' configure scripts don't need them anyway.
 556               command.RemoveDirectory(command.path.join(
 557                   '%(output)s', 'lib', 'pkgconfig')),
 558               ],
 559           },
 560       H('expat'): {
 561           'type': 'build',
 562           'dependencies': ['expat'],
 563           'commands': [
 564               command.Command(ConfigureCommand('expat') +
 565                               ConfigureHostLib(host)),
 566               command.Command(MakeCommand(host)),
 567               command.Command(MakeCheckCommand(host)),
 568               command.Command(MAKE_DESTDIR_CMD + [
 569                   # expat does not support the install-strip target.
 570                   'installlib',
 571                   'INSTALL=%(expat)s/conftools/install-sh -c -s',
 572                   'INSTALL_DATA=%(expat)s/conftools/install-sh -c -m 644',
 573                   ]),
 574               ],
 575           },
 576       }
 577   return host_gcc_libs
 578
 579
 580 HOST_GCC_LIBS_DEPS = ['gmp', 'mpfr', 'mpc', 'isl', 'cloog']
 581
 582 def HostGccLibsDeps(host):
 583   return [ForHost(package, host) for package in HOST_GCC_LIBS_DEPS]
 584
 585
 586 def ConfigureCommand(source_component):
 587   return [command % {'src': '%(' + source_component + ')s'}
 588           for command in CONFIGURE_CMD]
 589
 590
 591 # When doing a Canadian cross, we need native-hosted cross components
 592 # to do the GCC build.
 593 def GccDeps(host, target):
 594   components = ['binutils_' + target]
 595   if CrossCompiling(host):
 596     components.append('gcc_' + target)
 597     host = NATIVE_TUPLE
 598   return [ForHost(component, host) for component in components]
 599
 600
 601 def GccCommand(host, target, cmd):
 602   components_for_path = GccDeps(host, target)
 603   return command.Command(
 604       cmd, path_dirs=[command.path.join('%(abs_' + component + ')s', 'bin')
 605                       for component in components_for_path])
 606
 607
 608 def ConfigureGccCommand(source_component, host, target, extra_args=[]):
 609   return GccCommand(
 610       host,
 611       target,
 612       ConfigureCommand(source_component) +
 613       ConfigureHostTool(host) +
 614       ConfigureTargetArgs(target) +
 615       TARGET_GCC_CONFIG.get(target, []) + [
 616           '--with-gmp=%(abs_' + ForHost('gmp', host) + ')s',
 617           '--with-mpfr=%(abs_' + ForHost('mpfr', host) + ')s',
 618           '--with-mpc=%(abs_' + ForHost('mpc', host) + ')s',
 619           '--with-isl=%(abs_' + ForHost('isl', host) + ')s',
 620           '--with-cloog=%(abs_' + ForHost('cloog', host) + ')s',
 621           '--enable-cloog-backend=isl',
 622           '--disable-dlopen',
 623           '--disable-shared',
 624           '--with-newlib',
 625           '--with-linker-hash-style=gnu',
 626           '--enable-linker-build-id',
 627           '--enable-languages=c,c++,lto',
 628           ] + extra_args)
 629
 630
 631
 632 def HostTools(host, target):
 633   def H(component_name):
 634     return ForHost(component_name, host)
 635
 636   # Return the file name with the appropriate suffix for an executable file.
 637   def Exe(file):
 638     if HostIsWindows(host):
 639       return file + '.exe'
 640     else:
 641       return file
 642
 643   tools = {
 644       H('binutils_' + target): {
 645           'type': 'build',
 646           'dependencies': ['binutils'],
 647           'commands': ConfigureTargetPrep(target) + [
 648               command.Command(
 649                   ConfigureCommand('binutils') +
 650                   ConfigureHostTool(host) +
 651                   ConfigureTargetArgs(target) + [
 652                       '--enable-deterministic-archives',
 653                       '--enable-gold',
 654                       ] + ([] if HostIsWindows(host) else [
 655                           '--enable-plugins',
 656                           ])),
 657               command.Command(MakeCommand(host)),
 658               command.Command(MakeCheckCommand(host)),
 659               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 660               REMOVE_INFO_DIR,
 661               ] + InstallDocFiles('binutils',
 662                                   ['COPYING3'] +
 663                                   [command.path.join(subdir, 'NEWS')
 664                                    for subdir in
 665                                    ['binutils', 'gas', 'ld', 'gold']]) +
 666               # The top-level lib* directories contain host libraries
 667               # that we don't want to include in the distribution.
 668               [command.RemoveDirectory(command.path.join('%(output)s', name))
 669                for name in ['lib', 'lib32', 'lib64']],
 670           },
 671
 672       H('gcc_' + target): {
 673           'type': 'build',
 674           'dependencies': (['gcc'] + HostGccLibsDeps(host) +
 675                            GccDeps(host, target)),
 676           'commands': ConfigureTargetPrep(target) + [
 677               ConfigureGccCommand('gcc', host, target),
 678               # GCC's configure step writes configargs.h with some strings
 679               # including the configure command line, which get embedded
 680               # into the gcc driver binary.  The build only works if we use
 681               # absolute paths in some of the configure switches, but
 682               # embedding those paths makes the output differ in repeated
 683               # builds done in different directories, which we do not want.
 684               # So force the generation of that file early and then edit it
 685               # in place to replace the absolute paths with something that
 686               # never varies.  Note that the 'configure-gcc' target will
 687               # actually build some components before running gcc/configure.
 688               GccCommand(host, target,
 689                          MakeCommand(host, ['configure-gcc'])),
 690               command.Command(['sed', '-i', '-e',
 691                                ';'.join(['s@%%(abs_%s)s@.../%s_install@g' %
 692                                          (component, component)
 693                                          for component in
 694                                          HostGccLibsDeps(host)] +
 695                                         ['s@%(cwd)s@...@g']),
 696                                command.path.join('gcc', 'configargs.h')]),
 697               # gcc/Makefile's install rules ordinarily look at the
 698               # installed include directory for a limits.h to decide
 699               # whether the lib/gcc/.../include-fixed/limits.h header
 700               # should be made to expect a libc-supplied limits.h or not.
 701               # Since we're doing this build in a clean environment without
 702               # any libc installed, we need to force its hand here.
 703               GccCommand(host, target,
 704                          MakeCommand(host, ['all-gcc', 'LIMITS_H_TEST=true'])),
 705               # gcc/Makefile's install targets populate this directory
 706               # only if it already exists.
 707               command.Mkdir(command.path.join('%(output)s',
 708                                               target + '-nacl', 'bin'),
 709                             True),
 710               GccCommand(host, target,
 711                          MAKE_DESTDIR_CMD + ['install-strip-gcc']),
 712               REMOVE_INFO_DIR,
 713               # Note we include COPYING.RUNTIME here and not with gcc_libs.
 714               ] + InstallDocFiles('gcc', ['COPYING3', 'COPYING.RUNTIME']),
 715           },
 716
 717       # GDB can support all the targets in one host tool.
 718       H('gdb'): {
 719           'type': 'build',
 720           'dependencies': ['gdb', H('expat')],
 721           'commands': [
 722               command.Command(
 723                   ConfigureCommand('gdb') +
 724                   ConfigureHostTool(host) + [
 725                       '--target=x86_64-nacl',
 726                       '--enable-targets=arm-none-eabi-nacl',
 727                       '--with-expat',
 728                       'CPPFLAGS=-I%(abs_' + H('expat') + ')s/include',
 729                       'LDFLAGS=-L%(abs_' + H('expat') + ')s/lib',
 730                       ] +
 731                   (['--without-python'] if HostIsWindows(host) else []) +
 732                   # TODO(mcgrathr): The default -Werror only breaks because
 733                   # the OSX default compiler is an old front-end that does
 734                   # not understand all the GCC options.  Maybe switch to
 735                   # using clang (system or Chromium-supplied) on Mac.
 736                   (['--disable-werror'] if HostIsMac(host) else [])),
 737               command.Command(MakeCommand(host) + ['all-gdb']),
 738               command.Command(MAKE_DESTDIR_CMD + [
 739                   '-C', 'gdb', 'install-strip',
 740                   ]),
 741               REMOVE_INFO_DIR,
 742               ] + [command.Command(['ln', '-f',
 743                                     command.path.join('%(abs_output)s',
 744                                                       'bin',
 745                                                       Exe('x86_64-nacl-gdb')),
 746                                     command.path.join('%(abs_output)s',
 747                                                       'bin',
 748                                                       Exe(arch + '-nacl-gdb'))])
 749                    for arch in ['i686', 'arm']] + InstallDocFiles('gdb', [
 750                        'COPYING3',
 751                        command.path.join('gdb', 'NEWS'),
 752                        ]),
 753           },
 754       }
 755
 756   # TODO(mcgrathr): The ARM cross environment does not supply a termcap
 757   # library, so it cannot build GDB.
 758   if host.startswith('arm') and CrossCompiling(host):
 759     del tools[H('gdb')]
 760
 761   return tools
 762
 763 def TargetCommands(host, target, command_list):
 764   # First we have to copy the host tools into a common directory.
 765   # We can't just have both directories in our PATH, because the
 766   # compiler looks for the assembler and linker relative to itself.
 767   commands = PopulateDeps(['%(' + ForHost('binutils_' + target, host) + ')s',
 768                            '%(' + ForHost('gcc_' + target, host) + ')s'])
 769   bindir = command.path.join('%(cwd)s', 'all_deps', 'bin')
 770   commands += [command.Command(cmd, path_dirs=[bindir])
 771                for cmd in command_list]
 772   return commands
 773
 774
 775 def TargetLibs(host, target):
 776   lib_deps = [ForHost(component + '_' + target, host)
 777               for component in ['binutils', 'gcc']]
 778
 779   def NewlibFile(subdir, name):
 780     return command.path.join('%(output)s', target + '-nacl', subdir, name)
 781
 782   newlib_sysroot = '%(abs_newlib_' + target + ')s'
 783   newlib_tooldir = '%s/%s-nacl' % (newlib_sysroot, target)
 784
 785   # See the comment at ConfigureTargetPrep, above.
 786   newlib_install_data = ' '.join(['STRIPPROG=%(cwd)s/strip_for_target',
 787                                   '%(abs_newlib)s/install-sh',
 788                                   '-c', '-s', '-m', '644'])
 789
 790   iconv_encodings = 'UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4'
 791   newlib_configure_args = [
 792       '--disable-libgloss',
 793       '--enable-newlib-iconv',
 794       '--enable-newlib-iconv-from-encodings=' + iconv_encodings,
 795       '--enable-newlib-iconv-to-encodings=' + iconv_encodings,
 796       '--enable-newlib-io-long-long',
 797       '--enable-newlib-io-long-double',
 798       '--enable-newlib-io-c99-formats',
 799       '--enable-newlib-mb',
 800       'CFLAGS=-O2',
 801       'INSTALL_DATA=' + newlib_install_data,
 802       ]
 803
 804   newlib_post_install = [
 805       command.Rename(NewlibFile('lib', 'libc.a'),
 806                      NewlibFile('lib', 'libcrt_common.a')),
 807       command.WriteData(NewlibLibcScript(target),
 808                         NewlibFile('lib', 'libc.a')),
 809       ] + [
 810       command.Copy(
 811           command.path.join('%(pthread_headers)s', header),
 812           NewlibFile('include', header))
 813       for header in ('pthread.h', 'semaphore.h')
 814       ]
 815
 816
 817   libs = {
 818       'newlib_' + target: {
 819           'type': 'build',
 820           'dependencies': ['newlib'] + lib_deps,
 821           'inputs': { 'pthread_headers':
 822                       os.path.join(NACL_DIR, 'src', 'untrusted',
 823                                    'pthread') },
 824           'commands': (ConfigureTargetPrep(target) +
 825                        TargetCommands(host, target, [
 826                            ConfigureCommand('newlib') +
 827                            ConfigureHostTool(host) +
 828                            ConfigureTargetArgs(target) +
 829                            newlib_configure_args,
 830                            MakeCommand(host),
 831                            MAKE_DESTDIR_CMD + ['install-strip'],
 832                            ]) +
 833                        newlib_post_install +
 834                        InstallDocFiles('newlib', ['COPYING.NEWLIB'])),
 835           },
 836
 837       'gcc_libs_' + target: {
 838           'type': 'build',
 839           'dependencies': (['gcc_libs'] + lib_deps + ['newlib_' + target] +
 840                            HostGccLibsDeps(host)),
 841           # This actually builds the compiler again and uses that compiler
 842           # to build the target libraries.  That's by far the easiest thing
 843           # to get going given the interdependencies of the target
 844           # libraries (especially libgcc) on the gcc subdirectory, and
 845           # building the compiler doesn't really take all that long in the
 846           # grand scheme of things.
 847           # TODO(mcgrathr): If upstream ever cleans up all their
 848           # interdependencies better, unpack the compiler, configure with
 849           # --disable-gcc, and just build all-target.
 850           'commands': ConfigureTargetPrep(target) + [
 851               ConfigureGccCommand('gcc_libs', host, target, [
 852                   '--with-build-sysroot=' + newlib_sysroot,
 853                   ]),
 854               GccCommand(host, target,
 855                          MakeCommand(host) + [
 856                              'build_tooldir=' + newlib_tooldir,
 857                              'all-target',
 858                              ]),
 859               GccCommand(host, target,
 860                          MAKE_DESTDIR_CMD + ['install-strip-target']),
 861               REMOVE_INFO_DIR,
 862               ],
 863           },
 864       }
 865   return libs
 866
 867 # Compute it once.
 868 NATIVE_TUPLE = pynacl.platform.PlatformTriple()
 869
 870
 871 # For our purposes, "cross-compiling" means not literally that we are
 872 # targetting a host that does not match NATIVE_TUPLE, but that we are
 873 # targetting a host whose binaries we cannot run locally.  So x86-32
 874 # on x86-64 does not count as cross-compiling.  See NATIVE_ENOUGH_MAP, above.
 875 def CrossCompiling(host):
 876   return (host != NATIVE_TUPLE and
 877           host not in NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}))
 878
 879
 880 def HostIsWindows(host):
 881   return host == WINDOWS_HOST_TUPLE
 882
 883
 884 def HostIsMac(host):
 885   return host == MAC_HOST_TUPLE
 886
 887
 888 # We build target libraries only on Linux for two reasons:
 889 # 1. We only need to build them once.
 890 # 2. Linux is the fastest to build.
 891 # TODO(mcgrathr): In future set up some scheme whereby non-Linux
 892 # bots can build target libraries but not archive them, only verifying
 893 # that the results came out the same as the ones archived by the
 894 # official builder bot.  That will serve as a test of the host tools
 895 # on the other host platforms.
 896 def BuildTargetLibsOn(host):
 897   return host == LINUX_X86_64_TUPLE
 898
 899 def GetPackageTargets():
 900   """Package Targets describes all the final package targets.
 901
 902   This build can be built among many build bots, but eventually all things
 903   will be combined together. This package target dictionary describes the final
 904   output of the entire build.
 905   """
 906   package_targets = {}
 907
 908   for target_arch in UPLOAD_TARGETS:
 909     # Each package target contains non-platform specific newlib and gcc libs.
 910     # These packages are added inside of TargetLibs(host, target).
 911     newlib_package = 'newlib_%s' % target_arch
 912     gcc_lib_package = 'gcc_libs_%s' % target_arch
 913     shared_packages = [newlib_package, gcc_lib_package]
 914
 915     for platform, arch in HOSTS:
 916       # Each package target contains arm binutils and gcc.
 917       # These packages are added inside of HostTools(host, target).
 918       platform_triple = pynacl.platform.PlatformTriple(platform, arch)
 919       binutils_package = ForHost('binutils_%s' % target_arch, platform_triple)
 920       gcc_package = ForHost('gcc_%s' % target_arch, platform_triple)
 921       gdb_package = ForHost('gdb', platform_triple)
 922
 923       # Create a list of packages for a target.
 924       platform_packages = [binutils_package, gcc_package, gdb_package]
 925       combined_packages = shared_packages + platform_packages
 926
 927       os_name = pynacl.platform.GetOS(platform)
 928       arch_name = pynacl.platform.GetArch(arch)
 929       package_target = '%s_%s' % (os_name, arch_name)
 930       package_name = 'nacl_%s_newlib' % (pynacl.platform.GetArch(target_arch))
 931
 932       package_target_dict = package_targets.setdefault(package_target, {})
 933       package_target_dict.setdefault(package_name, []).extend(combined_packages)
 934
 935   # GDB is a special and shared, we will inject it into various other packages.
 936   for platform, arch in GDB_INJECT_HOSTS:
 937     platform_triple = pynacl.platform.PlatformTriple(platform, arch)
 938     os_name = pynacl.platform.GetOS(platform)
 939     arch_name = pynacl.platform.GetArch(arch)
 940
 941     gdb_packages = [ForHost('gdb', platform_triple)]
 942     package_target = '%s_%s' % (os_name, arch_name)
 943
 944     for package_name, package_archives in GDB_INJECT_PACKAGES:
 945       combined_packages = package_archives + gdb_packages
 946       package_target_dict = package_targets.setdefault(package_target, {})
 947       package_target_dict.setdefault(package_name, []).extend(combined_packages)
 948
 949   return dict(package_targets)
 950
 951 def CollectPackagesForHost(host, targets):
 952   packages = HostGccLibs(host).copy()
 953   for target in targets:
 954     packages.update(HostTools(host, target))
 955     if BuildTargetLibsOn(host):
 956       packages.update(TargetLibs(host, target))
 957   return packages
 958
 959
 960 def CollectPackages(targets):
 961   packages = CollectSources()
 962
 963   packages.update(CollectPackagesForHost(NATIVE_TUPLE, targets))
 964
 965   for host in EXTRA_HOSTS_MAP.get(NATIVE_TUPLE, []):
 966     packages.update(CollectPackagesForHost(host, targets))
 967
 968   return packages
 969
 970
 971 PACKAGES = CollectPackages(TARGET_LIST)
 972 PACKAGE_TARGETS = GetPackageTargets()
 973
 974
 975 if __name__ == '__main__':
 976   tb = toolchain_main.PackageBuilder(PACKAGES, PACKAGE_TARGETS, sys.argv[1:])
 977   # TODO(mcgrathr): The bot ought to run some native_client tests
 978   # using the new toolchain, like the old x86 toolchain bots do.
 979   tb.Main()