src/native_client/toolchain_build/toolchain_build.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Recipes for NativeClient toolchain packages.
   7
   8 The real entry plumbing is in toolchain_main.py.
   9 """
  10
  11 import collections
  12 import fnmatch
  13 import platform
  14 import os
  15 import re
  16 import sys
  17
  18 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  19 import pynacl.gsd_storage
  20 import pynacl.platform
  21
  22 import command
  23 import toolchain_main
  24
  25
  26 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
  27 NACL_DIR = os.path.dirname(SCRIPT_DIR)
  28
  29
  30 # See command.GenerateGitPatches for the schema of entries in this dict.
  31 # Additionally, each may contain a 'repo' key whose value is the name
  32 # to use in place of the package name when calling GitUrl (below).
  33 GIT_REVISIONS = {
  34     'binutils': {
  35         'rev': '7deface59753c3b249ac08d854d471951796343f',
  36         'upstream-branch': 'upstream/binutils-2_24-branch',
  37         'upstream-name': 'binutils-2.24',
  38         # This is tag binutils-2_24, but Gerrit won't let us push
  39         # non-annotated tags, and the upstream tag is not annotated.
  40         'upstream-base': '237df3fa4a1d939e6fd1af0c3e5029a25a137310',
  41         },
  42     'gcc': {
  43         'rev': '95260f2088fc67f43c329c864f03293a5718269a',
  44         'upstream-branch': 'upstream/gcc-4_8-branch',
  45         'upstream-name': 'gcc-4.8.3',
  46          # Upstream tag gcc-4_8_3-release:
  47         'upstream-base': '6bbf0dec66c0e719b06cd2fe67559fda6df09000',
  48         },
  49     'newlib': {
  50         'rev': 'e7b1ccd4b5eec14e840f6bf875f4d6fa4cff045c',
  51         'upstream-branch': 'upstream/master',
  52         'upstream-name': 'newlib-2.1.0',
  53         # Upstream tag newlib_2_1_0:
  54         'upstream-base': '99fc6c167467b41466ec90e8260e9c49cbe3d13c',
  55         },
  56     'gdb': {
  57         'rev': '5deb4793a5e3f2f48d7899f424bb4484686020f8',
  58         'repo': 'binutils',
  59         'upstream-branch': 'upstream/gdb-7.7-branch',
  60         'upstream-name': 'gdb-7.7.1',
  61         # Upstream tag gdb-7.7-release:
  62         'upstream-base': '4bd8fc3a1362970d9800a263987af8093798338b',
  63         },
  64     }
  65
  66 TAR_FILES = {
  67     'gmp': command.path.join('gmp', 'gmp-6.0.0a.tar.bz2'),
  68     'mpfr': command.path.join('mpfr', 'mpfr-3.1.2.tar.bz2'),
  69     'mpc': command.path.join('mpc', 'mpc-1.0.2.tar.gz'),
  70     'isl': command.path.join('cloog', 'isl-0.12.2.tar.bz2'),
  71     'cloog': command.path.join('cloog', 'cloog-0.18.1.tar.gz'),
  72     'expat': command.path.join('expat', 'expat-2.1.0.tar.gz'),
  73     }
  74
  75 GIT_BASE_URL = 'https://chromium.googlesource.com/native_client'
  76 GIT_PUSH_URL = 'ssh://gerrit.chromium.org/native_client'
  77
  78 ALT_GIT_BASE_URL = 'https://chromium.googlesource.com/a/native_client'
  79
  80 KNOWN_MIRRORS = [('http://git.chromium.org/native_client', GIT_BASE_URL)]
  81 PUSH_MIRRORS = [('http://git.chromium.org/native_client', GIT_PUSH_URL),
  82                 (ALT_GIT_BASE_URL, GIT_PUSH_URL),
  83                 (GIT_BASE_URL, GIT_PUSH_URL)]
  84
  85
  86 def GitUrl(package, push_url=False):
  87   repo = GIT_REVISIONS[package].get('repo', package)
  88   if push_url:
  89     base_url = GIT_PUSH_URL
  90   else:
  91     base_url = GIT_BASE_URL
  92
  93   return '%s/nacl-%s.git' % (base_url, repo)
  94
  95
  96 def CollectSources():
  97   sources = {}
  98
  99   for package in TAR_FILES:
 100     tar_file = TAR_FILES[package]
 101     if fnmatch.fnmatch(tar_file, '*.bz2'):
 102       extract = EXTRACT_STRIP_TBZ2
 103     elif fnmatch.fnmatch(tar_file, '*.gz'):
 104       extract = EXTRACT_STRIP_TGZ
 105     else:
 106       raise Exception('unexpected file name pattern in TAR_FILES[%r]' % package)
 107     sources[package] = {
 108         'type': 'source',
 109         'commands': [
 110             command.Command(extract + [command.path.join('%(abs_top_srcdir)s',
 111                                                          '..', 'third_party',
 112                                                          tar_file)],
 113                             cwd='%(output)s'),
 114             ],
 115         }
 116
 117   patch_packages = []
 118   patch_commands = []
 119   for package, info in GIT_REVISIONS.iteritems():
 120     sources[package] = {
 121         'type': 'source',
 122         'commands': command.SyncGitRepoCmds(GitUrl(package), '%(output)s',
 123                                             info['rev'],
 124                                             git_cache='%(git_cache_dir)s',
 125                                             push_url=GitUrl(package, True),
 126                                             known_mirrors=KNOWN_MIRRORS,
 127                                             push_mirrors=PUSH_MIRRORS),
 128         }
 129     patch_packages.append(package)
 130     patch_info = {'name': package}
 131     patch_info.update(info)
 132     patch_commands.append(
 133         command.GenerateGitPatches('%(' + package + ')s/.git', patch_info))
 134
 135   sources['patches'] = {
 136       'type': 'build',
 137       'dependencies': patch_packages,
 138       'commands': patch_commands,
 139       }
 140
 141   # The gcc_libs component gets the whole GCC source tree.
 142   sources['gcc_libs'] = sources['gcc']
 143
 144   # The gcc component omits all the source directories that are used solely
 145   # for building target libraries.  We don't want those included in the
 146   # input hash calculation so that we don't rebuild the compiler when the
 147   # the only things that have changed are target libraries.
 148   sources['gcc'] = {
 149         'type': 'source',
 150         'dependencies': ['gcc_libs'],
 151         'commands': [command.CopyTree('%(gcc_libs)s', '%(output)s', [
 152             'boehm-gc',
 153             'libada',
 154             'libatomic',
 155             'libffi',
 156             'libgcc',
 157             'libgfortran',
 158             'libgo',
 159             'libgomp',
 160             'libitm',
 161             'libjava',
 162             'libmudflap',
 163             'libobjc',
 164             'libquadmath',
 165             'libsanitizer',
 166             'libssp',
 167             'libstdc++-v3',
 168             ])]
 169       }
 170
 171   # We have to populate the newlib source tree with the "exported" form of
 172   # some headers from the native_client source tree.  The newlib build
 173   # needs these to be in the expected place.  By doing this in the source
 174   # target, these files will be part of the input hash and so we don't need
 175   # to do anything else to keep track of when they might have changed in
 176   # the native_client source tree.
 177   newlib_sys_nacl = command.path.join('%(output)s',
 178                                       'newlib', 'libc', 'sys', 'nacl')
 179   newlib_unpack = [command.RemoveDirectory(command.path.join(newlib_sys_nacl,
 180                                                              dirname))
 181                    for dirname in ['bits', 'sys', 'machine']]
 182   newlib_unpack.append(command.Command([
 183       'python',
 184       command.path.join('%(top_srcdir)s', 'src',
 185                         'trusted', 'service_runtime', 'export_header.py'),
 186       command.path.join('%(top_srcdir)s', 'src',
 187                         'trusted', 'service_runtime', 'include'),
 188       newlib_sys_nacl,
 189       ]))
 190   sources['newlib']['commands'] += newlib_unpack
 191
 192   return sources
 193
 194
 195 # Canonical tuples we use for hosts.
 196 WINDOWS_HOST_TUPLE = pynacl.platform.PlatformTriple('win', 'x86-32')
 197 MAC_HOST_TUPLE = pynacl.platform.PlatformTriple('darwin', 'x86-64')
 198 ARM_HOST_TUPLE = pynacl.platform.PlatformTriple('linux', 'arm')
 199 LINUX_X86_32_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-32')
 200 LINUX_X86_64_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-64')
 201
 202 # Map of native host tuple to extra tuples that it cross-builds for.
 203 EXTRA_HOSTS_MAP = {
 204     LINUX_X86_64_TUPLE: [
 205         LINUX_X86_32_TUPLE,
 206         ARM_HOST_TUPLE,
 207         WINDOWS_HOST_TUPLE,
 208         ],
 209     }
 210
 211 # Map of native host tuple to host tuples that are "native enough".
 212 # For these hosts, we will do a native-style build even though it's
 213 # not the native tuple, just passing some extra compiler flags.
 214 NATIVE_ENOUGH_MAP = {
 215     LINUX_X86_64_TUPLE: {
 216         LINUX_X86_32_TUPLE: ['-m32'],
 217         },
 218     }
 219
 220 # The list of targets to build toolchains for.
 221 TARGET_LIST = ['arm', 'i686']
 222
 223 # List upload targets for each host we want to upload packages for.
 224 TARGET = collections.namedtuple('TARGET', ['name', 'pkg_prefix'])
 225 HOST_TARGET = collections.namedtuple('HOST_TARGET',
 226                                      ['os', 'arch', 'differ3264', 'targets'])
 227
 228 STANDARD_TARGETS = [TARGET('arm', '')]
 229 LINUX_X86_64_TARGETS = [TARGET('arm', ''), TARGET('i686', 'ng_')]
 230
 231 UPLOAD_HOST_TARGETS = [
 232     HOST_TARGET('win', 'x86-32', False, STANDARD_TARGETS),
 233     HOST_TARGET('darwin', 'x86-64', False, STANDARD_TARGETS),
 234     HOST_TARGET('linux', 'arm', False, STANDARD_TARGETS),
 235     HOST_TARGET('linux', 'x86-32', False, STANDARD_TARGETS),
 236     HOST_TARGET('linux', 'x86-64', True, LINUX_X86_64_TARGETS),
 237     ]
 238
 239 # GDB is built by toolchain_build but injected into package targets built by
 240 # other means. List out what package targets, packages, and the tar file we are
 241 # injecting on top of here.
 242 GDB_INJECT_HOSTS = [
 243   ('win', 'x86-32'),
 244   ('darwin', 'x86-64'),
 245   ('linux', 'x86-32'),
 246   ]
 247
 248 GDB_INJECT_PACKAGES = [
 249   ('nacl_x86_newlib', ['naclsdk.tgz']),
 250   ('nacl_x86_glibc', ['toolchain.tar.bz2']),
 251   ]
 252
 253 # These are extra arguments to pass gcc's configure that vary by target.
 254 TARGET_GCC_CONFIG = {
 255 # TODO(mcgrathr): Disabled tuning for now, tickling a constant-pool layout bug.
 256 #    'arm': ['--with-tune=cortex-a15'],
 257     }
 258
 259 PACKAGE_NAME = 'Native Client SDK [%(build_signature)s]'
 260 BUG_URL = 'http://gonacl.com/reportissue'
 261
 262 TAR_XV = ['tar', '-x', '-v']
 263 EXTRACT_STRIP_TGZ = TAR_XV + ['--gzip', '--strip-components=1', '-f']
 264 EXTRACT_STRIP_TBZ2 = TAR_XV + ['--bzip2', '--strip-components=1', '-f']
 265 CONFIGURE_CMD = ['sh', '%(src)s/configure']
 266 MAKE_PARALLEL_CMD = ['make', '-j%(cores)s']
 267 MAKE_CHECK_CMD = MAKE_PARALLEL_CMD + ['check']
 268 MAKE_DESTDIR_CMD = ['make', 'DESTDIR=%(abs_output)s']
 269
 270 # This file gets installed by multiple packages' install steps, but it is
 271 # never useful when installed in isolation.  So we remove it from the
 272 # installation directories before packaging up.
 273 REMOVE_INFO_DIR = command.Remove(command.path.join('%(output)s',
 274                                                    'share', 'info', 'dir'))
 275
 276 def ConfigureHostArch(host):
 277   configure_args = []
 278
 279   is_cross = CrossCompiling(host)
 280
 281   if is_cross:
 282     extra_cc_args = []
 283     configure_args.append('--host=' + host)
 284   else:
 285     extra_cc_args = NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}).get(host, [])
 286     if extra_cc_args:
 287       # The host we've chosen is "native enough", such as x86-32 on x86-64.
 288       # But it's not what config.guess will yield, so we need to supply
 289       # a --build switch to ensure things build correctly.
 290       configure_args.append('--build=' + host)
 291
 292   extra_cxx_args = list(extra_cc_args)
 293   if fnmatch.fnmatch(host, '*-linux*'):
 294     # Avoid shipping binaries with a runtime dependency on
 295     # a particular version of the libstdc++ shared library.
 296     # TODO(mcgrathr): Do we want this for MinGW and/or Mac too?
 297     extra_cxx_args.append('-static-libstdc++')
 298
 299   if extra_cc_args:
 300     # These are the defaults when there is no setting, but we will add
 301     # additional switches, so we must supply the command name too.
 302     if is_cross:
 303       cc = host + '-gcc'
 304     else:
 305       cc = 'gcc'
 306     configure_args.append('CC=' + ' '.join([cc] + extra_cc_args))
 307
 308   if extra_cxx_args:
 309     # These are the defaults when there is no setting, but we will add
 310     # additional switches, so we must supply the command name too.
 311     if is_cross:
 312       cxx = host + '-g++'
 313     else:
 314       cxx = 'g++'
 315     configure_args.append('CXX=' + ' '.join([cxx] + extra_cxx_args))
 316
 317   if HostIsWindows(host):
 318     # The i18n support brings in runtime dependencies on MinGW DLLs
 319     # that we don't want to have to distribute alongside our binaries.
 320     # So just disable it, and compiler messages will always be in US English.
 321     configure_args.append('--disable-nls')
 322
 323   return configure_args
 324
 325
 326 def ConfigureHostCommon(host):
 327   return ConfigureHostArch(host) + [
 328       '--prefix=',
 329       '--disable-silent-rules',
 330       '--without-gcc-arch',
 331       ]
 332
 333
 334 def ConfigureHostLib(host):
 335   return ConfigureHostCommon(host) + [
 336       '--disable-shared',
 337       ]
 338
 339
 340 def ConfigureHostTool(host):
 341   return ConfigureHostCommon(host) + [
 342       '--with-pkgversion=' + PACKAGE_NAME,
 343       '--with-bugurl=' + BUG_URL,
 344       '--without-zlib',
 345       ]
 346
 347
 348 def MakeCommand(host, extra_args=[]):
 349   if HostIsWindows(host):
 350     # There appears to be nothing we can pass at top-level configure time
 351     # that will prevent the configure scripts from finding MinGW's libiconv
 352     # and using it.  We have to force this variable into the environment
 353     # of the sub-configure runs, which are run via make.
 354     make_command = MAKE_PARALLEL_CMD + ['HAVE_LIBICONV=no']
 355   else:
 356     make_command = MAKE_PARALLEL_CMD
 357   return make_command + extra_args
 358
 359
 360 # Return the 'make check' command to run.
 361 # When cross-compiling, don't try to run test suites.
 362 def MakeCheckCommand(host):
 363   if CrossCompiling(host):
 364     return ['true']
 365   return MAKE_CHECK_CMD
 366
 367
 368 def InstallDocFiles(subdir, files):
 369   doc_dir = command.path.join('%(output)s', 'share', 'doc', subdir)
 370   dirs = sorted(set([command.path.dirname(command.path.join(doc_dir, file))
 371                      for file in files]))
 372   commands = ([command.Mkdir(dir, parents=True) for dir in dirs] +
 373               [command.Copy(command.path.join('%(' + subdir + ')s', file),
 374                             command.path.join(doc_dir, file))
 375                for file in files])
 376   return commands
 377
 378
 379 def NewlibLibcScript(arch):
 380   template = """/*
 381  * This is a linker script that gets installed as libc.a for the
 382  * newlib-based NaCl toolchain.  It brings in the constituent
 383  * libraries that make up what -lc means semantically.
 384  */
 385 OUTPUT_FORMAT(%s)
 386 GROUP ( libnacl.a libcrt_common.a )
 387 """
 388   if arch == 'arm':
 389     # Listing three formats instead of one makes -EL/-EB switches work
 390     # for the endian-switchable ARM backend.
 391     format_list = ['elf32-littlearm-nacl',
 392                    'elf32-bigarm-nacl',
 393                    'elf32-littlearm-nacl']
 394   elif arch == 'i686':
 395     format_list = ['elf32-i386-nacl']
 396   elif arch == 'x86_64':
 397     format_list = ['elf32-x86_64-nacl']
 398   else:
 399     raise Exception('TODO(mcgrathr): OUTPUT_FORMAT for %s' % arch)
 400   return template % ', '.join(['"' + fmt + '"' for fmt in format_list])
 401
 402
 403 # The default strip behavior removes debugging and symbol table
 404 # sections, but it leaves the .comment section.  This contains the
 405 # compiler version string, and so it changes when the compiler changes
 406 # even if the actual machine code it produces is completely identical.
 407 # Hence, the target library packages will always change when the
 408 # compiler changes unless these sections are removed.  Doing this
 409 # requires somehow teaching the makefile rules to pass the
 410 # --remove-section=.comment switch to TARGET-strip.  For the GCC
 411 # target libraries, setting STRIP_FOR_TARGET is sufficient.  But
 412 # quoting nightmares make it difficult to pass a command with a space
 413 # in it as the STRIP_FOR_TARGET value.  So the build writes a little
 414 # script that can be invoked with a simple name.
 415 #
 416 # Though the gcc target libraries' makefiles are smart enough to obey
 417 # STRIP_FOR_TARGET for library files, the newlib makefiles just
 418 # blindly use $(INSTALL_DATA) for both header (text) files and library
 419 # files.  Hence it's necessary to override its INSTALL_DATA setting to
 420 # one that will do stripping using this script, and thus the script
 421 # must silently do nothing to non-binary files.
 422 def ConfigureTargetPrep(arch):
 423   script_file = 'strip_for_target'
 424
 425   config_target = arch + '-nacl'
 426   script_contents = """\
 427 #!/bin/sh
 428 mode=--strip-all
 429 for arg; do
 430   case "$arg" in
 431   -*) ;;
 432   *)
 433     type=`file --brief --mime-type "$arg"`
 434     case "$type" in
 435       application/x-executable|application/x-sharedlib) ;;
 436       application/x-archive|application/x-object) mode=--strip-debug ;;
 437       *) exit 0 ;;
 438     esac
 439     ;;
 440   esac
 441 done
 442 exec %s-strip $mode --remove-section=.comment "$@"
 443 """ % config_target
 444
 445   return [
 446       command.WriteData(script_contents, script_file),
 447       command.Command(['chmod', '+x', script_file]),
 448       ]
 449
 450
 451 def ConfigureTargetArgs(arch):
 452   config_target = arch + '-nacl'
 453   return [
 454       '--target=' + config_target,
 455       '--with-sysroot=/' + config_target,
 456       'STRIP_FOR_TARGET=%(cwd)s/strip_for_target',
 457       ]
 458
 459
 460 def CommandsInBuild(command_lines):
 461   return [
 462       command.RemoveDirectory('build'),
 463       command.Mkdir('build'),
 464       ] + [command.Command(cmd, cwd='build')
 465            for cmd in command_lines]
 466
 467
 468 def PopulateDeps(dep_dirs):
 469   commands = [command.RemoveDirectory('all_deps'),
 470               command.Mkdir('all_deps')]
 471   commands += [command.Command('cp -r "%s/"* all_deps' % dirname, shell=True)
 472                for dirname in dep_dirs]
 473   return commands
 474
 475
 476 def WithDepsOptions(options, component=None):
 477   if component is None:
 478     directory = command.path.join('%(cwd)s', 'all_deps')
 479   else:
 480     directory = '%(abs_' + component + ')s'
 481   return ['--with-' + option + '=' + directory
 482           for option in options]
 483
 484
 485 # Return the component name we'll use for a base component name and
 486 # a host tuple.  The component names cannot contain dashes or other
 487 # non-identifier characters, because the names of the files uploaded
 488 # to Google Storage are constrained.  GNU configuration tuples contain
 489 # dashes, which we translate to underscores.
 490 def ForHost(component_name, host):
 491   return component_name + '_' + pynacl.gsd_storage.LegalizeName(host)
 492
 493
 494 # These are libraries that go into building the compiler itself.
 495 def HostGccLibs(host):
 496   def H(component_name):
 497     return ForHost(component_name, host)
 498   host_gcc_libs = {
 499       H('gmp'): {
 500           'type': 'build',
 501           'dependencies': ['gmp'],
 502           'commands': [
 503               command.Command(ConfigureCommand('gmp') +
 504                               ConfigureHostLib(host) + [
 505                                   '--with-sysroot=%(abs_output)s',
 506                                   '--enable-cxx',
 507                                   # Without this, the built library will
 508                                   # assume the instruction set details
 509                                   # available on the build machine.  With
 510                                   # this, it dynamically chooses what code
 511                                   # to use based on the details of the
 512                                   # actual host CPU at runtime.
 513                                   '--enable-fat',
 514                                   ]),
 515               command.Command(MakeCommand(host)),
 516               command.Command(MakeCheckCommand(host)),
 517               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 518               ],
 519           },
 520       H('mpfr'): {
 521           'type': 'build',
 522           'dependencies': ['mpfr', H('gmp')],
 523           'commands': [
 524               command.Command(ConfigureCommand('mpfr') +
 525                               ConfigureHostLib(host) +
 526                               WithDepsOptions(['sysroot', 'gmp'], H('gmp'))),
 527               command.Command(MakeCommand(host)),
 528               command.Command(MakeCheckCommand(host)),
 529               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 530               ],
 531           },
 532       H('mpc'): {
 533           'type': 'build',
 534           'dependencies': ['mpc', H('gmp'), H('mpfr')],
 535           'commands': PopulateDeps(['%(' + H('gmp') + ')s',
 536                                     '%(' + H('mpfr') + ')s']) + [
 537               command.Command(ConfigureCommand('mpc') +
 538                               ConfigureHostLib(host) +
 539                               WithDepsOptions(['sysroot', 'gmp', 'mpfr'])),
 540               command.Command(MakeCommand(host)),
 541               command.Command(MakeCheckCommand(host)),
 542               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 543               ],
 544           },
 545       H('isl'): {
 546           'type': 'build',
 547           'dependencies': ['isl', H('gmp')],
 548           'commands': [
 549               command.Command(ConfigureCommand('isl') +
 550                               ConfigureHostLib(host) +
 551                               WithDepsOptions(['sysroot', 'gmp-prefix'],
 552                                               H('gmp'))),
 553               command.Command(MakeCommand(host)),
 554               command.Command(MakeCheckCommand(host)),
 555               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 556               # The .pc files wind up containing some absolute paths
 557               # that make the output depend on the build directory name.
 558               # The dependents' configure scripts don't need them anyway.
 559               command.RemoveDirectory(command.path.join(
 560                   '%(output)s', 'lib', 'pkgconfig')),
 561               ],
 562           },
 563       H('cloog'): {
 564           'type': 'build',
 565           'dependencies': ['cloog', H('gmp'), H('isl')],
 566           'commands': PopulateDeps(['%(' + H('gmp') + ')s',
 567                                     '%(' + H('isl') + ')s']) + [
 568               command.Command(ConfigureCommand('cloog') +
 569                               ConfigureHostLib(host) + [
 570                                   '--with-bits=gmp',
 571                                   '--with-isl=system',
 572                                   ] + WithDepsOptions(['sysroot',
 573                                                        'gmp-prefix',
 574                                                        'isl-prefix'])),
 575               command.Command(MakeCommand(host)),
 576               command.Command(MakeCheckCommand(host)),
 577               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 578               # The .pc files wind up containing some absolute paths
 579               # that make the output depend on the build directory name.
 580               # The dependents' configure scripts don't need them anyway.
 581               command.RemoveDirectory(command.path.join(
 582                   '%(output)s', 'lib', 'pkgconfig')),
 583               ],
 584           },
 585       H('expat'): {
 586           'type': 'build',
 587           'dependencies': ['expat'],
 588           'commands': [
 589               command.Command(ConfigureCommand('expat') +
 590                               ConfigureHostLib(host)),
 591               command.Command(MakeCommand(host)),
 592               command.Command(MakeCheckCommand(host)),
 593               command.Command(MAKE_DESTDIR_CMD + [
 594                   # expat does not support the install-strip target.
 595                   'installlib',
 596                   'INSTALL=%(expat)s/conftools/install-sh -c -s',
 597                   'INSTALL_DATA=%(expat)s/conftools/install-sh -c -m 644',
 598                   ]),
 599               ],
 600           },
 601       }
 602   return host_gcc_libs
 603
 604
 605 HOST_GCC_LIBS_DEPS = ['gmp', 'mpfr', 'mpc', 'isl', 'cloog']
 606
 607 def HostGccLibsDeps(host):
 608   return [ForHost(package, host) for package in HOST_GCC_LIBS_DEPS]
 609
 610
 611 def ConfigureCommand(source_component):
 612   return [command % {'src': '%(' + source_component + ')s'}
 613           for command in CONFIGURE_CMD]
 614
 615
 616 # When doing a Canadian cross, we need native-hosted cross components
 617 # to do the GCC build.
 618 def GccDeps(host, target):
 619   components = ['binutils_' + target]
 620   if CrossCompiling(host):
 621     components.append('gcc_' + target)
 622     host = NATIVE_TUPLE
 623   return [ForHost(component, host) for component in components]
 624
 625
 626 def GccCommand(host, target, cmd):
 627   components_for_path = GccDeps(host, target)
 628   return command.Command(
 629       cmd, path_dirs=[command.path.join('%(abs_' + component + ')s', 'bin')
 630                       for component in components_for_path])
 631
 632
 633 def ConfigureGccCommand(source_component, host, target, extra_args=[]):
 634   return GccCommand(
 635       host,
 636       target,
 637       ConfigureCommand(source_component) +
 638       ConfigureHostTool(host) +
 639       ConfigureTargetArgs(target) +
 640       TARGET_GCC_CONFIG.get(target, []) + [
 641           '--with-gmp=%(abs_' + ForHost('gmp', host) + ')s',
 642           '--with-mpfr=%(abs_' + ForHost('mpfr', host) + ')s',
 643           '--with-mpc=%(abs_' + ForHost('mpc', host) + ')s',
 644           '--with-isl=%(abs_' + ForHost('isl', host) + ')s',
 645           '--with-cloog=%(abs_' + ForHost('cloog', host) + ')s',
 646           '--enable-cloog-backend=isl',
 647           '--disable-dlopen',
 648           '--disable-shared',
 649           '--with-newlib',
 650           '--with-linker-hash-style=gnu',
 651           '--enable-linker-build-id',
 652           '--enable-languages=c,c++,lto',
 653           ] + extra_args)
 654
 655
 656
 657 def HostTools(host, target):
 658   def H(component_name):
 659     return ForHost(component_name, host)
 660
 661   def WindowsAlternate(if_windows, if_not_windows, if_mac=None):
 662     if if_mac is not None and HostIsMac(host):
 663       return if_mac
 664     elif HostIsWindows(host):
 665       return if_windows
 666     else:
 667       return if_not_windows
 668
 669   # Return the file name with the appropriate suffix for an executable file.
 670   def Exe(file):
 671     return file + WindowsAlternate('.exe', '')
 672
 673   tools = {
 674       H('binutils_' + target): {
 675           'type': 'build',
 676           'dependencies': ['binutils'],
 677           'commands': ConfigureTargetPrep(target) + [
 678               command.Command(
 679                   ConfigureCommand('binutils') +
 680                   ConfigureHostTool(host) +
 681                   ConfigureTargetArgs(target) + [
 682                       '--enable-deterministic-archives',
 683                       '--enable-gold',
 684                       ] + WindowsAlternate([], ['--enable-plugins'])),
 685               command.Command(MakeCommand(host)),
 686               command.Command(MakeCheckCommand(host)),
 687               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
 688               REMOVE_INFO_DIR,
 689               ] + InstallDocFiles('binutils',
 690                                   ['COPYING3'] +
 691                                   [command.path.join(subdir, 'NEWS')
 692                                    for subdir in
 693                                    ['binutils', 'gas', 'ld', 'gold']]) +
 694               # The top-level lib* directories contain host libraries
 695               # that we don't want to include in the distribution.
 696               [command.RemoveDirectory(command.path.join('%(output)s', name))
 697                for name in ['lib', 'lib32', 'lib64']],
 698           },
 699
 700       H('gcc_' + target): {
 701           'type': 'build',
 702           'dependencies': (['gcc'] + HostGccLibsDeps(host) +
 703                            GccDeps(host, target)),
 704           'commands': ConfigureTargetPrep(target) + [
 705               ConfigureGccCommand('gcc', host, target),
 706               # GCC's configure step writes configargs.h with some strings
 707               # including the configure command line, which get embedded
 708               # into the gcc driver binary.  The build only works if we use
 709               # absolute paths in some of the configure switches, but
 710               # embedding those paths makes the output differ in repeated
 711               # builds done in different directories, which we do not want.
 712               # So force the generation of that file early and then edit it
 713               # in place to replace the absolute paths with something that
 714               # never varies.  Note that the 'configure-gcc' target will
 715               # actually build some components before running gcc/configure.
 716               GccCommand(host, target,
 717                          MakeCommand(host, ['configure-gcc'])),
 718               command.Command(['sed', '-i', '-e',
 719                                ';'.join(['s@%%(abs_%s)s@.../%s_install@g' %
 720                                          (component, component)
 721                                          for component in
 722                                          HostGccLibsDeps(host)] +
 723                                         ['s@%(cwd)s@...@g']),
 724                                command.path.join('gcc', 'configargs.h')]),
 725               # gcc/Makefile's install rules ordinarily look at the
 726               # installed include directory for a limits.h to decide
 727               # whether the lib/gcc/.../include-fixed/limits.h header
 728               # should be made to expect a libc-supplied limits.h or not.
 729               # Since we're doing this build in a clean environment without
 730               # any libc installed, we need to force its hand here.
 731               GccCommand(host, target,
 732                          MakeCommand(host, ['all-gcc', 'LIMITS_H_TEST=true'])),
 733               # gcc/Makefile's install targets populate this directory
 734               # only if it already exists.
 735               command.Mkdir(command.path.join('%(output)s',
 736                                               target + '-nacl', 'bin'),
 737                             True),
 738               GccCommand(host, target,
 739                          MAKE_DESTDIR_CMD + ['install-strip-gcc']),
 740               REMOVE_INFO_DIR,
 741               # Note we include COPYING.RUNTIME here and not with gcc_libs.
 742               ] + InstallDocFiles('gcc', ['COPYING3', 'COPYING.RUNTIME']),
 743           },
 744
 745       # GDB can support all the targets in one host tool.
 746       H('gdb'): {
 747           'type': 'build',
 748           'dependencies': ['gdb', H('expat')],
 749           'commands': [
 750               command.Command(
 751                   ConfigureCommand('gdb') +
 752                   ConfigureHostTool(host) + [
 753                       '--target=x86_64-nacl',
 754                       '--enable-targets=arm-none-eabi-nacl',
 755                       '--with-expat',
 756                       # Windows (MinGW) is missing ncurses; we need to
 757                       # build one here and link it in statically for
 758                       # --enable-tui.  See issue nativeclient:3911.
 759                       '--%s-tui' % WindowsAlternate('disable', 'enable'),
 760                       'CPPFLAGS=-I%(abs_' + H('expat') + ')s/include',
 761                       'LDFLAGS=-L%(abs_' + H('expat') + ')s/lib',
 762                       ] +
 763                   # TODO(mcgrathr): Should use --with-python to ensure
 764                   # we have it on Linux/Mac.
 765                   WindowsAlternate(['--without-python'], []) +
 766                   # TODO(mcgrathr): The default -Werror only breaks because
 767                   # the OSX default compiler is an old front-end that does
 768                   # not understand all the GCC options.  Maybe switch to
 769                   # using clang (system or Chromium-supplied) on Mac.
 770                   (['--disable-werror'] if HostIsMac(host) else [])),
 771               command.Command(MakeCommand(host) + ['all-gdb']),
 772               command.Command(MAKE_DESTDIR_CMD + [
 773                   '-C', 'gdb', 'install-strip',
 774                   ]),
 775               REMOVE_INFO_DIR,
 776               ] + [command.Command(['ln', '-f',
 777                                     command.path.join('%(abs_output)s',
 778                                                       'bin',
 779                                                       Exe('x86_64-nacl-gdb')),
 780                                     command.path.join('%(abs_output)s',
 781                                                       'bin',
 782                                                       Exe(arch + '-nacl-gdb'))])
 783                    for arch in ['i686', 'arm']] + InstallDocFiles('gdb', [
 784                        'COPYING3',
 785                        command.path.join('gdb', 'NEWS'),
 786                        ]),
 787           },
 788       }
 789
 790   # TODO(mcgrathr): The ARM cross environment does not supply a termcap
 791   # library, so it cannot build GDB.
 792   if host.startswith('arm') and CrossCompiling(host):
 793     del tools[H('gdb')]
 794
 795   return tools
 796
 797 def TargetCommands(host, target, command_list):
 798   # First we have to copy the host tools into a common directory.
 799   # We can't just have both directories in our PATH, because the
 800   # compiler looks for the assembler and linker relative to itself.
 801   commands = PopulateDeps(['%(' + ForHost('binutils_' + target, host) + ')s',
 802                            '%(' + ForHost('gcc_' + target, host) + ')s'])
 803   bindir = command.path.join('%(cwd)s', 'all_deps', 'bin')
 804   commands += [command.Command(cmd, path_dirs=[bindir])
 805                for cmd in command_list]
 806   return commands
 807
 808
 809 def TargetLibs(host, target):
 810   lib_deps = [ForHost(component + '_' + target, host)
 811               for component in ['binutils', 'gcc']]
 812
 813   def NewlibFile(subdir, name):
 814     return command.path.join('%(output)s', target + '-nacl', subdir, name)
 815
 816   newlib_sysroot = '%(abs_newlib_' + target + ')s'
 817   newlib_tooldir = '%s/%s-nacl' % (newlib_sysroot, target)
 818
 819   # See the comment at ConfigureTargetPrep, above.
 820   newlib_install_data = ' '.join(['STRIPPROG=%(cwd)s/strip_for_target',
 821                                   '%(abs_newlib)s/install-sh',
 822                                   '-c', '-s', '-m', '644'])
 823
 824   iconv_encodings = 'UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4'
 825   newlib_configure_args = [
 826       '--disable-libgloss',
 827       '--enable-newlib-iconv',
 828       '--enable-newlib-iconv-from-encodings=' + iconv_encodings,
 829       '--enable-newlib-iconv-to-encodings=' + iconv_encodings,
 830       '--enable-newlib-io-long-long',
 831       '--enable-newlib-io-long-double',
 832       '--enable-newlib-io-c99-formats',
 833       '--enable-newlib-mb',
 834       'CFLAGS=-O2',
 835       'INSTALL_DATA=' + newlib_install_data,
 836       ]
 837
 838   newlib_post_install = [
 839       command.Rename(NewlibFile('lib', 'libc.a'),
 840                      NewlibFile('lib', 'libcrt_common.a')),
 841       command.WriteData(NewlibLibcScript(target),
 842                         NewlibFile('lib', 'libc.a')),
 843       ] + [
 844       command.Copy(
 845           command.path.join('%(pthread_headers)s', header),
 846           NewlibFile('include', header))
 847       for header in ('pthread.h', 'semaphore.h')
 848       ]
 849
 850
 851   libs = {
 852       'newlib_' + target: {
 853           'type': 'build',
 854           'dependencies': ['newlib'] + lib_deps,
 855           'inputs': { 'pthread_headers':
 856                       os.path.join(NACL_DIR, 'src', 'untrusted',
 857                                    'pthread') },
 858           'commands': (ConfigureTargetPrep(target) +
 859                        TargetCommands(host, target, [
 860                            ConfigureCommand('newlib') +
 861                            ConfigureHostTool(host) +
 862                            ConfigureTargetArgs(target) +
 863                            newlib_configure_args,
 864                            MakeCommand(host),
 865                            MAKE_DESTDIR_CMD + ['install-strip'],
 866                            ]) +
 867                        newlib_post_install +
 868                        InstallDocFiles('newlib', ['COPYING.NEWLIB'])),
 869           },
 870
 871       'gcc_libs_' + target: {
 872           'type': 'build',
 873           'dependencies': (['gcc_libs'] + lib_deps + ['newlib_' + target] +
 874                            HostGccLibsDeps(host)),
 875           # This actually builds the compiler again and uses that compiler
 876           # to build the target libraries.  That's by far the easiest thing
 877           # to get going given the interdependencies of the target
 878           # libraries (especially libgcc) on the gcc subdirectory, and
 879           # building the compiler doesn't really take all that long in the
 880           # grand scheme of things.
 881           # TODO(mcgrathr): If upstream ever cleans up all their
 882           # interdependencies better, unpack the compiler, configure with
 883           # --disable-gcc, and just build all-target.
 884           'commands': ConfigureTargetPrep(target) + [
 885               ConfigureGccCommand('gcc_libs', host, target, [
 886                   '--with-build-sysroot=' + newlib_sysroot,
 887                   ]),
 888               GccCommand(host, target,
 889                          MakeCommand(host) + [
 890                              'build_tooldir=' + newlib_tooldir,
 891                              'all-target',
 892                              ]),
 893               GccCommand(host, target,
 894                          MAKE_DESTDIR_CMD + ['install-strip-target']),
 895               REMOVE_INFO_DIR,
 896               ],
 897           },
 898       }
 899   return libs
 900
 901 # Compute it once.
 902 NATIVE_TUPLE = pynacl.platform.PlatformTriple()
 903
 904
 905 # For our purposes, "cross-compiling" means not literally that we are
 906 # targetting a host that does not match NATIVE_TUPLE, but that we are
 907 # targetting a host whose binaries we cannot run locally.  So x86-32
 908 # on x86-64 does not count as cross-compiling.  See NATIVE_ENOUGH_MAP, above.
 909 def CrossCompiling(host):
 910   return (host != NATIVE_TUPLE and
 911           host not in NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}))
 912
 913
 914 def HostIsWindows(host):
 915   return host == WINDOWS_HOST_TUPLE
 916
 917
 918 def HostIsMac(host):
 919   return host == MAC_HOST_TUPLE
 920
 921
 922 # We build target libraries only on Linux for two reasons:
 923 # 1. We only need to build them once.
 924 # 2. Linux is the fastest to build.
 925 # TODO(mcgrathr): In future set up some scheme whereby non-Linux
 926 # bots can build target libraries but not archive them, only verifying
 927 # that the results came out the same as the ones archived by the
 928 # official builder bot.  That will serve as a test of the host tools
 929 # on the other host platforms.
 930 def BuildTargetLibsOn(host):
 931   return host == LINUX_X86_64_TUPLE
 932
 933
 934 def GetPackageTargets():
 935   """Package Targets describes all the final package targets.
 936
 937   This build can be built among many build bots, but eventually all things
 938   will be combined together. This package target dictionary describes the final
 939   output of the entire build.
 940   """
 941   package_targets = {}
 942
 943   # Add in standard upload targets.
 944   for host_target in UPLOAD_HOST_TARGETS:
 945     for target in host_target.targets:
 946       target_arch = target.name
 947       package_prefix = target.pkg_prefix
 948
 949       # Each package target contains non-platform specific newlib and gcc libs.
 950       # These packages are added inside of TargetLibs(host, target).
 951       newlib_package = 'newlib_%s' % target_arch
 952       gcc_lib_package = 'gcc_libs_%s' % target_arch
 953       shared_packages = [newlib_package, gcc_lib_package]
 954
 955       # Each package target contains arm binutils and gcc.
 956       # These packages are added inside of HostTools(host, target).
 957       platform_triple = pynacl.platform.PlatformTriple(host_target.os,
 958                                                        host_target.arch)
 959       binutils_package = ForHost('binutils_%s' % target_arch, platform_triple)
 960       gcc_package = ForHost('gcc_%s' % target_arch, platform_triple)
 961       gdb_package = ForHost('gdb', platform_triple)
 962
 963       # Create a list of packages for a target.
 964       platform_packages = [binutils_package, gcc_package, gdb_package]
 965       combined_packages = shared_packages + platform_packages
 966
 967       os_name = pynacl.platform.GetOS(host_target.os)
 968       if host_target.differ3264:
 969         arch_name = pynacl.platform.GetArch3264(host_target.arch)
 970       else:
 971         arch_name = pynacl.platform.GetArch(host_target.arch)
 972       package_target = '%s_%s' % (os_name, arch_name)
 973       package_name = '%snacl_%s_newlib' % (package_prefix,
 974                                            pynacl.platform.GetArch(target_arch))
 975
 976       package_target_dict = package_targets.setdefault(package_target, {})
 977       package_target_dict.setdefault(package_name, []).extend(combined_packages)
 978
 979   # GDB is a special and shared, we will inject it into various other packages.
 980   for platform, arch in GDB_INJECT_HOSTS:
 981     platform_triple = pynacl.platform.PlatformTriple(platform, arch)
 982     os_name = pynacl.platform.GetOS(platform)
 983     arch_name = pynacl.platform.GetArch(arch)
 984
 985     gdb_packages = [ForHost('gdb', platform_triple)]
 986     package_target = '%s_%s' % (os_name, arch_name)
 987
 988     for package_name, package_archives in GDB_INJECT_PACKAGES:
 989       combined_packages = package_archives + gdb_packages
 990       package_target_dict = package_targets.setdefault(package_target, {})
 991       package_target_dict.setdefault(package_name, []).extend(combined_packages)
 992
 993   return dict(package_targets)
 994
 995 def CollectPackagesForHost(host, targets):
 996   packages = HostGccLibs(host).copy()
 997   for target in targets:
 998     packages.update(HostTools(host, target))
 999     if BuildTargetLibsOn(host):
1000       packages.update(TargetLibs(host, target))
1001   return packages
1002
1003
1004 def CollectPackages(targets):
1005   packages = CollectSources()
1006
1007   packages.update(CollectPackagesForHost(NATIVE_TUPLE, targets))
1008
1009   for host in EXTRA_HOSTS_MAP.get(NATIVE_TUPLE, []):
1010     packages.update(CollectPackagesForHost(host, targets))
1011
1012   return packages
1013
1014
1015 PACKAGES = CollectPackages(TARGET_LIST)
1016 PACKAGE_TARGETS = GetPackageTargets()
1017
1018
1019 if __name__ == '__main__':
1020   tb = toolchain_main.PackageBuilder(PACKAGES, PACKAGE_TARGETS, sys.argv[1:])
1021   # TODO(mcgrathr): The bot ought to run some native_client tests
1022   # using the new toolchain, like the old x86 toolchain bots do.
1023   tb.Main()