Upstream version 11.40.277.0
[platform/framework/web/crosswalk.git] / src / native_client / toolchain_build / toolchain_build.py
1 #!/usr/bin/python
2 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 """Recipes for NativeClient toolchain packages.
7
8 The real entry plumbing is in toolchain_main.py.
9 """
10
11 import collections
12 import fnmatch
13 import platform
14 import os
15 import re
16 import sys
17
18 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
19 import pynacl.gsd_storage
20 import pynacl.platform
21
22 import command
23 import toolchain_main
24
25
26 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
27 NACL_DIR = os.path.dirname(SCRIPT_DIR)
28
29
30 # See command.GenerateGitPatches for the schema of entries in this dict.
31 # Additionally, each may contain a 'repo' key whose value is the name
32 # to use in place of the package name when calling GitUrl (below).
33 GIT_REVISIONS = {
34     'binutils': {
35         'rev': 'b08b9f0894e43f0bb966f3ad9094a4405ce6f570',
36         'upstream-branch': 'upstream/binutils-2_24-branch',
37         'upstream-name': 'binutils-2.24',
38         # This is tag binutils-2_24, but Gerrit won't let us push
39         # non-annotated tags, and the upstream tag is not annotated.
40         'upstream-base': '237df3fa4a1d939e6fd1af0c3e5029a25a137310',
41         },
42     'gcc': {
43         'rev': 'daf66f6da8d77ceb3cc75e63f3a7156abb09d564',
44         'upstream-branch': 'upstream/gcc-4_9-branch',
45         'upstream-name': 'gcc-4.9.2',
46          # Upstream tag gcc-4_9_2-release:
47         'upstream-base': 'c1283af40b65f1ad862cf5b27e2d9ed10b2076b6',
48         },
49     'newlib': {
50         'rev': 'bf66148d14c7fca26b9198dd5dc81e743893bb66',
51         'upstream-branch': 'upstream/master',
52         'upstream-name': 'newlib-2.1.0',
53         # Upstream tag newlib_2_1_0:
54         'upstream-base': '99fc6c167467b41466ec90e8260e9c49cbe3d13c',
55         },
56     'gdb': {
57         'rev': '5deb4793a5e3f2f48d7899f424bb4484686020f8',
58         'repo': 'binutils',
59         'upstream-branch': 'upstream/gdb-7.7-branch',
60         'upstream-name': 'gdb-7.7.1',
61         # Upstream tag gdb-7.7-release:
62         'upstream-base': '4bd8fc3a1362970d9800a263987af8093798338b',
63         },
64     }
65
66 TAR_FILES = {
67     'gmp': command.path.join('gmp', 'gmp-6.0.0a.tar.bz2'),
68     'mpfr': command.path.join('mpfr', 'mpfr-3.1.2.tar.bz2'),
69     'mpc': command.path.join('mpc', 'mpc-1.0.2.tar.gz'),
70     'isl': command.path.join('cloog', 'isl-0.12.2.tar.bz2'),
71     'cloog': command.path.join('cloog', 'cloog-0.18.1.tar.gz'),
72     'expat': command.path.join('expat', 'expat-2.1.0.tar.gz'),
73     }
74
75 GIT_BASE_URL = 'https://chromium.googlesource.com/native_client'
76 GIT_PUSH_URL = 'ssh://gerrit.chromium.org/native_client'
77
78 ALT_GIT_BASE_URL = 'https://chromium.googlesource.com/a/native_client'
79
80 KNOWN_MIRRORS = [('http://git.chromium.org/native_client', GIT_BASE_URL)]
81 PUSH_MIRRORS = [('http://git.chromium.org/native_client', GIT_PUSH_URL),
82                 (ALT_GIT_BASE_URL, GIT_PUSH_URL),
83                 (GIT_BASE_URL, GIT_PUSH_URL)]
84
85
86 def GitUrl(package, push_url=False):
87   repo = GIT_REVISIONS[package].get('repo', package)
88   if push_url:
89     base_url = GIT_PUSH_URL
90   else:
91     base_url = GIT_BASE_URL
92
93   return '%s/nacl-%s.git' % (base_url, repo)
94
95
96 def CollectSources():
97   sources = {}
98
99   for package in TAR_FILES:
100     tar_file = TAR_FILES[package]
101     if fnmatch.fnmatch(tar_file, '*.bz2'):
102       extract = EXTRACT_STRIP_TBZ2
103     elif fnmatch.fnmatch(tar_file, '*.gz'):
104       extract = EXTRACT_STRIP_TGZ
105     else:
106       raise Exception('unexpected file name pattern in TAR_FILES[%r]' % package)
107     sources[package] = {
108         'type': 'source',
109         'commands': [
110             command.Command(extract + [command.path.join('%(abs_top_srcdir)s',
111                                                          '..', 'third_party',
112                                                          tar_file)],
113                             cwd='%(output)s'),
114             ],
115         }
116
117   patch_packages = []
118   patch_commands = []
119   for package, info in GIT_REVISIONS.iteritems():
120     sources[package] = {
121         'type': 'source',
122         'commands': command.SyncGitRepoCmds(GitUrl(package), '%(output)s',
123                                             info['rev'],
124                                             git_cache='%(git_cache_dir)s',
125                                             push_url=GitUrl(package, True),
126                                             known_mirrors=KNOWN_MIRRORS,
127                                             push_mirrors=PUSH_MIRRORS),
128         }
129     patch_packages.append(package)
130     patch_info = {'name': package}
131     patch_info.update(info)
132     patch_commands.append(
133         command.GenerateGitPatches('%(' + package + ')s/.git', patch_info))
134
135   sources['patches'] = {
136       'type': 'build',
137       'dependencies': patch_packages,
138       'commands': patch_commands,
139       }
140
141   # The gcc_libs component gets the whole GCC source tree.
142   sources['gcc_libs'] = sources['gcc']
143
144   # The gcc component omits all the source directories that are used solely
145   # for building target libraries.  We don't want those included in the
146   # input hash calculation so that we don't rebuild the compiler when the
147   # the only things that have changed are target libraries.
148   sources['gcc'] = {
149         'type': 'source',
150         'dependencies': ['gcc_libs'],
151         'commands': [command.CopyTree('%(gcc_libs)s', '%(output)s', [
152             'boehm-gc',
153             'libada',
154             'libatomic',
155             'libffi',
156             'libgcc',
157             'libgfortran',
158             'libgo',
159             'libgomp',
160             'libitm',
161             'libjava',
162             'libmudflap',
163             'libobjc',
164             'libquadmath',
165             'libsanitizer',
166             'libssp',
167             'libstdc++-v3',
168             ])]
169       }
170
171   # We have to populate the newlib source tree with the "exported" form of
172   # some headers from the native_client source tree.  The newlib build
173   # needs these to be in the expected place.  By doing this in the source
174   # target, these files will be part of the input hash and so we don't need
175   # to do anything else to keep track of when they might have changed in
176   # the native_client source tree.
177   newlib_sys_nacl = command.path.join('%(output)s',
178                                       'newlib', 'libc', 'sys', 'nacl')
179   newlib_unpack = [command.RemoveDirectory(command.path.join(newlib_sys_nacl,
180                                                              dirname))
181                    for dirname in ['bits', 'sys', 'machine']]
182   newlib_unpack.append(command.Command([
183       'python',
184       command.path.join('%(top_srcdir)s', 'src',
185                         'trusted', 'service_runtime', 'export_header.py'),
186       command.path.join('%(top_srcdir)s', 'src',
187                         'trusted', 'service_runtime', 'include'),
188       newlib_sys_nacl,
189       ]))
190   sources['newlib']['commands'] += newlib_unpack
191
192   return sources
193
194
195 # Canonical tuples we use for hosts.
196 WINDOWS_HOST_TUPLE = pynacl.platform.PlatformTriple('win', 'x86-32')
197 MAC_HOST_TUPLE = pynacl.platform.PlatformTriple('darwin', 'x86-64')
198 ARM_HOST_TUPLE = pynacl.platform.PlatformTriple('linux', 'arm')
199 LINUX_X86_32_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-32')
200 LINUX_X86_64_TUPLE = pynacl.platform.PlatformTriple('linux', 'x86-64')
201
202 # Map of native host tuple to extra tuples that it cross-builds for.
203 EXTRA_HOSTS_MAP = {
204     LINUX_X86_64_TUPLE: [
205         LINUX_X86_32_TUPLE,
206         ARM_HOST_TUPLE,
207         WINDOWS_HOST_TUPLE,
208         ],
209     }
210
211 # Map of native host tuple to host tuples that are "native enough".
212 # For these hosts, we will do a native-style build even though it's
213 # not the native tuple, just passing some extra compiler flags.
214 NATIVE_ENOUGH_MAP = {
215     LINUX_X86_64_TUPLE: {
216         LINUX_X86_32_TUPLE: ['-m32'],
217         },
218     }
219
220 # The list of targets to build toolchains for.
221 TARGET_LIST = ['arm', 'i686']
222
223 # List upload targets for each host we want to upload packages for.
224 TARGET = collections.namedtuple('TARGET', ['name', 'pkg_prefix'])
225 HOST_TARGET = collections.namedtuple('HOST_TARGET',
226                                      ['os', 'arch', 'differ3264', 'targets'])
227
228 STANDARD_TARGETS = [TARGET('arm', '')]
229 LINUX_X86_64_TARGETS = [TARGET('arm', ''), TARGET('i686', 'ng_')]
230
231 UPLOAD_HOST_TARGETS = [
232     HOST_TARGET('win', 'x86-32', False, STANDARD_TARGETS),
233     HOST_TARGET('darwin', 'x86-64', False, STANDARD_TARGETS),
234     HOST_TARGET('linux', 'arm', False, STANDARD_TARGETS),
235     HOST_TARGET('linux', 'x86-32', False, STANDARD_TARGETS),
236     HOST_TARGET('linux', 'x86-64', True, LINUX_X86_64_TARGETS),
237     ]
238
239 # GDB is built by toolchain_build but injected into package targets built by
240 # other means. List out what package targets, packages, and the tar file we are
241 # injecting on top of here.
242 GDB_INJECT_HOSTS = [
243   ('win', 'x86-32'),
244   ('darwin', 'x86-64'),
245   ('linux', 'x86-32'),
246   ]
247
248 GDB_INJECT_PACKAGES = [
249   ('nacl_x86_newlib', ['naclsdk.tgz']),
250   ('nacl_x86_glibc', ['toolchain.tar.bz2']),
251   ]
252
253 # These are extra arguments to pass gcc's configure that vary by target.
254 TARGET_GCC_CONFIG = {
255     'arm': ['--with-tune=cortex-a15'],
256     }
257
258 PACKAGE_NAME = 'Native Client SDK [%(build_signature)s]'
259 BUG_URL = 'http://gonacl.com/reportissue'
260
261 TAR_XV = ['tar', '-x', '-v']
262 EXTRACT_STRIP_TGZ = TAR_XV + ['--gzip', '--strip-components=1', '-f']
263 EXTRACT_STRIP_TBZ2 = TAR_XV + ['--bzip2', '--strip-components=1', '-f']
264 CONFIGURE_CMD = ['sh', '%(src)s/configure']
265 MAKE_PARALLEL_CMD = ['make', '-j%(cores)s']
266 MAKE_CHECK_CMD = MAKE_PARALLEL_CMD + ['check']
267 MAKE_DESTDIR_CMD = ['make', 'DESTDIR=%(abs_output)s']
268
269 # This file gets installed by multiple packages' install steps, but it is
270 # never useful when installed in isolation.  So we remove it from the
271 # installation directories before packaging up.
272 REMOVE_INFO_DIR = command.Remove(command.path.join('%(output)s',
273                                                    'share', 'info', 'dir'))
274
275 def ConfigureHostArch(host):
276   configure_args = []
277
278   is_cross = CrossCompiling(host)
279
280   if is_cross:
281     extra_cc_args = []
282     configure_args.append('--host=' + host)
283   else:
284     extra_cc_args = NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}).get(host, [])
285     if extra_cc_args:
286       # The host we've chosen is "native enough", such as x86-32 on x86-64.
287       # But it's not what config.guess will yield, so we need to supply
288       # a --build switch to ensure things build correctly.
289       configure_args.append('--build=' + host)
290
291   extra_cxx_args = list(extra_cc_args)
292   if fnmatch.fnmatch(host, '*-linux*'):
293     # Avoid shipping binaries with a runtime dependency on
294     # a particular version of the libstdc++ shared library.
295     # TODO(mcgrathr): Do we want this for MinGW and/or Mac too?
296     extra_cxx_args.append('-static-libstdc++')
297
298   if extra_cc_args:
299     # These are the defaults when there is no setting, but we will add
300     # additional switches, so we must supply the command name too.
301     if is_cross:
302       cc = host + '-gcc'
303     else:
304       cc = 'gcc'
305     configure_args.append('CC=' + ' '.join([cc] + extra_cc_args))
306
307   if extra_cxx_args:
308     # These are the defaults when there is no setting, but we will add
309     # additional switches, so we must supply the command name too.
310     if is_cross:
311       cxx = host + '-g++'
312     else:
313       cxx = 'g++'
314     configure_args.append('CXX=' + ' '.join([cxx] + extra_cxx_args))
315
316   if HostIsWindows(host):
317     # The i18n support brings in runtime dependencies on MinGW DLLs
318     # that we don't want to have to distribute alongside our binaries.
319     # So just disable it, and compiler messages will always be in US English.
320     configure_args.append('--disable-nls')
321
322   return configure_args
323
324
325 def ConfigureHostCommon(host):
326   return ConfigureHostArch(host) + [
327       '--prefix=',
328       '--disable-silent-rules',
329       '--without-gcc-arch',
330       ]
331
332
333 def ConfigureHostLib(host):
334   return ConfigureHostCommon(host) + [
335       '--disable-shared',
336       ]
337
338
339 def ConfigureHostTool(host):
340   return ConfigureHostCommon(host) + [
341       '--with-pkgversion=' + PACKAGE_NAME,
342       '--with-bugurl=' + BUG_URL,
343       '--without-zlib',
344       ]
345
346
347 def MakeCommand(host, extra_args=[]):
348   if HostIsWindows(host):
349     # There appears to be nothing we can pass at top-level configure time
350     # that will prevent the configure scripts from finding MinGW's libiconv
351     # and using it.  We have to force this variable into the environment
352     # of the sub-configure runs, which are run via make.
353     make_command = MAKE_PARALLEL_CMD + ['HAVE_LIBICONV=no']
354   else:
355     make_command = MAKE_PARALLEL_CMD
356   return make_command + extra_args
357
358
359 # Return the 'make check' command to run.
360 # When cross-compiling, don't try to run test suites.
361 def MakeCheckCommand(host):
362   if CrossCompiling(host):
363     return ['true']
364   return MAKE_CHECK_CMD
365
366
367 def InstallDocFiles(subdir, files):
368   doc_dir = command.path.join('%(output)s', 'share', 'doc', subdir)
369   dirs = sorted(set([command.path.dirname(command.path.join(doc_dir, file))
370                      for file in files]))
371   commands = ([command.Mkdir(dir, parents=True) for dir in dirs] +
372               [command.Copy(command.path.join('%(' + subdir + ')s', file),
373                             command.path.join(doc_dir, file))
374                for file in files])
375   return commands
376
377
378 def NewlibLibcScript(arch):
379   template = """/*
380  * This is a linker script that gets installed as libc.a for the
381  * newlib-based NaCl toolchain.  It brings in the constituent
382  * libraries that make up what -lc means semantically.
383  */
384 OUTPUT_FORMAT(%s)
385 GROUP ( libnacl.a libcrt_common.a )
386 """
387   if arch == 'arm':
388     # Listing three formats instead of one makes -EL/-EB switches work
389     # for the endian-switchable ARM backend.
390     format_list = ['elf32-littlearm-nacl',
391                    'elf32-bigarm-nacl',
392                    'elf32-littlearm-nacl']
393   elif arch == 'i686':
394     format_list = ['elf32-i386-nacl']
395   elif arch == 'x86_64':
396     format_list = ['elf32-x86_64-nacl']
397   else:
398     raise Exception('TODO(mcgrathr): OUTPUT_FORMAT for %s' % arch)
399   return template % ', '.join(['"' + fmt + '"' for fmt in format_list])
400
401
402 # The default strip behavior removes debugging and symbol table
403 # sections, but it leaves the .comment section.  This contains the
404 # compiler version string, and so it changes when the compiler changes
405 # even if the actual machine code it produces is completely identical.
406 # Hence, the target library packages will always change when the
407 # compiler changes unless these sections are removed.  Doing this
408 # requires somehow teaching the makefile rules to pass the
409 # --remove-section=.comment switch to TARGET-strip.  For the GCC
410 # target libraries, setting STRIP_FOR_TARGET is sufficient.  But
411 # quoting nightmares make it difficult to pass a command with a space
412 # in it as the STRIP_FOR_TARGET value.  So the build writes a little
413 # script that can be invoked with a simple name.
414 #
415 # Though the gcc target libraries' makefiles are smart enough to obey
416 # STRIP_FOR_TARGET for library files, the newlib makefiles just
417 # blindly use $(INSTALL_DATA) for both header (text) files and library
418 # files.  Hence it's necessary to override its INSTALL_DATA setting to
419 # one that will do stripping using this script, and thus the script
420 # must silently do nothing to non-binary files.
421 def ConfigureTargetPrep(arch):
422   script_file = 'strip_for_target'
423
424   config_target = arch + '-nacl'
425   script_contents = """\
426 #!/bin/sh
427 mode=--strip-all
428 for arg; do
429   case "$arg" in
430   -*) ;;
431   *)
432     type=`file --brief --mime-type "$arg"`
433     case "$type" in
434       application/x-executable|application/x-sharedlib) ;;
435       application/x-archive|application/x-object) mode=--strip-debug ;;
436       *) exit 0 ;;
437     esac
438     ;;
439   esac
440 done
441 exec %s-strip $mode --remove-section=.comment "$@"
442 """ % config_target
443
444   return [
445       command.WriteData(script_contents, script_file),
446       command.Command(['chmod', '+x', script_file]),
447       ]
448
449
450 def ConfigureTargetArgs(arch):
451   config_target = arch + '-nacl'
452   return [
453       '--target=' + config_target,
454       '--with-sysroot=/' + config_target,
455       'STRIP_FOR_TARGET=%(cwd)s/strip_for_target',
456       ]
457
458
459 def CommandsInBuild(command_lines):
460   return [
461       command.RemoveDirectory('build'),
462       command.Mkdir('build'),
463       ] + [command.Command(cmd, cwd='build')
464            for cmd in command_lines]
465
466
467 def PopulateDeps(dep_dirs):
468   commands = [command.RemoveDirectory('all_deps'),
469               command.Mkdir('all_deps')]
470   commands += [command.Command('cp -r "%s/"* all_deps' % dirname, shell=True)
471                for dirname in dep_dirs]
472   return commands
473
474
475 def WithDepsOptions(options, component=None):
476   if component is None:
477     directory = command.path.join('%(cwd)s', 'all_deps')
478   else:
479     directory = '%(abs_' + component + ')s'
480   return ['--with-' + option + '=' + directory
481           for option in options]
482
483
484 # Return the component name we'll use for a base component name and
485 # a host tuple.  The component names cannot contain dashes or other
486 # non-identifier characters, because the names of the files uploaded
487 # to Google Storage are constrained.  GNU configuration tuples contain
488 # dashes, which we translate to underscores.
489 def ForHost(component_name, host):
490   return component_name + '_' + pynacl.gsd_storage.LegalizeName(host)
491
492
493 # These are libraries that go into building the compiler itself.
494 def HostGccLibs(host):
495   def H(component_name):
496     return ForHost(component_name, host)
497   host_gcc_libs = {
498       H('gmp'): {
499           'type': 'build',
500           'dependencies': ['gmp'],
501           'commands': [
502               command.Command(ConfigureCommand('gmp') +
503                               ConfigureHostLib(host) + [
504                                   '--with-sysroot=%(abs_output)s',
505                                   '--enable-cxx',
506                                   # Without this, the built library will
507                                   # assume the instruction set details
508                                   # available on the build machine.  With
509                                   # this, it dynamically chooses what code
510                                   # to use based on the details of the
511                                   # actual host CPU at runtime.
512                                   '--enable-fat',
513                                   ]),
514               command.Command(MakeCommand(host)),
515               command.Command(MakeCheckCommand(host)),
516               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
517               ],
518           },
519       H('mpfr'): {
520           'type': 'build',
521           'dependencies': ['mpfr', H('gmp')],
522           'commands': [
523               command.Command(ConfigureCommand('mpfr') +
524                               ConfigureHostLib(host) +
525                               WithDepsOptions(['sysroot', 'gmp'], H('gmp'))),
526               command.Command(MakeCommand(host)),
527               command.Command(MakeCheckCommand(host)),
528               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
529               ],
530           },
531       H('mpc'): {
532           'type': 'build',
533           'dependencies': ['mpc', H('gmp'), H('mpfr')],
534           'commands': PopulateDeps(['%(' + H('gmp') + ')s',
535                                     '%(' + H('mpfr') + ')s']) + [
536               command.Command(ConfigureCommand('mpc') +
537                               ConfigureHostLib(host) +
538                               WithDepsOptions(['sysroot', 'gmp', 'mpfr'])),
539               command.Command(MakeCommand(host)),
540               command.Command(MakeCheckCommand(host)),
541               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
542               ],
543           },
544       H('isl'): {
545           'type': 'build',
546           'dependencies': ['isl', H('gmp')],
547           'commands': [
548               command.Command(ConfigureCommand('isl') +
549                               ConfigureHostLib(host) +
550                               WithDepsOptions(['sysroot', 'gmp-prefix'],
551                                               H('gmp'))),
552               command.Command(MakeCommand(host)),
553               command.Command(MakeCheckCommand(host)),
554               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
555               # The .pc files wind up containing some absolute paths
556               # that make the output depend on the build directory name.
557               # The dependents' configure scripts don't need them anyway.
558               command.RemoveDirectory(command.path.join(
559                   '%(output)s', 'lib', 'pkgconfig')),
560               ],
561           },
562       H('cloog'): {
563           'type': 'build',
564           'dependencies': ['cloog', H('gmp'), H('isl')],
565           'commands': PopulateDeps(['%(' + H('gmp') + ')s',
566                                     '%(' + H('isl') + ')s']) + [
567               command.Command(ConfigureCommand('cloog') +
568                               ConfigureHostLib(host) + [
569                                   '--with-bits=gmp',
570                                   '--with-isl=system',
571                                   ] + WithDepsOptions(['sysroot',
572                                                        'gmp-prefix',
573                                                        'isl-prefix'])),
574               command.Command(MakeCommand(host)),
575               command.Command(MakeCheckCommand(host)),
576               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
577               # The .pc files wind up containing some absolute paths
578               # that make the output depend on the build directory name.
579               # The dependents' configure scripts don't need them anyway.
580               command.RemoveDirectory(command.path.join(
581                   '%(output)s', 'lib', 'pkgconfig')),
582               ],
583           },
584       H('expat'): {
585           'type': 'build',
586           'dependencies': ['expat'],
587           'commands': [
588               command.Command(ConfigureCommand('expat') +
589                               ConfigureHostLib(host)),
590               command.Command(MakeCommand(host)),
591               command.Command(MakeCheckCommand(host)),
592               command.Command(MAKE_DESTDIR_CMD + [
593                   # expat does not support the install-strip target.
594                   'installlib',
595                   'INSTALL=%(expat)s/conftools/install-sh -c -s',
596                   'INSTALL_DATA=%(expat)s/conftools/install-sh -c -m 644',
597                   ]),
598               ],
599           },
600       }
601   return host_gcc_libs
602
603
604 HOST_GCC_LIBS_DEPS = ['gmp', 'mpfr', 'mpc', 'isl', 'cloog']
605
606 def HostGccLibsDeps(host):
607   return [ForHost(package, host) for package in HOST_GCC_LIBS_DEPS]
608
609
610 def ConfigureCommand(source_component):
611   return [command % {'src': '%(' + source_component + ')s'}
612           for command in CONFIGURE_CMD]
613
614
615 # When doing a Canadian cross, we need native-hosted cross components
616 # to do the GCC build.
617 def GccDeps(host, target):
618   components = ['binutils_' + target]
619   if CrossCompiling(host):
620     components.append('gcc_' + target)
621     host = NATIVE_TUPLE
622   return [ForHost(component, host) for component in components]
623
624
625 def GccCommand(host, target, cmd):
626   components_for_path = GccDeps(host, target)
627   return command.Command(
628       cmd, path_dirs=[command.path.join('%(abs_' + component + ')s', 'bin')
629                       for component in components_for_path])
630
631
632 def ConfigureGccCommand(source_component, host, target, extra_args=[]):
633   return GccCommand(
634       host,
635       target,
636       ConfigureCommand(source_component) +
637       ConfigureHostTool(host) +
638       ConfigureTargetArgs(target) +
639       TARGET_GCC_CONFIG.get(target, []) + [
640           '--with-gmp=%(abs_' + ForHost('gmp', host) + ')s',
641           '--with-mpfr=%(abs_' + ForHost('mpfr', host) + ')s',
642           '--with-mpc=%(abs_' + ForHost('mpc', host) + ')s',
643           '--with-isl=%(abs_' + ForHost('isl', host) + ')s',
644           '--with-cloog=%(abs_' + ForHost('cloog', host) + ')s',
645           '--enable-cloog-backend=isl',
646           '--disable-dlopen',
647           '--disable-shared',
648           '--with-newlib',
649           '--with-linker-hash-style=gnu',
650           '--enable-linker-build-id',
651           '--enable-languages=c,c++,lto',
652           ] + extra_args)
653
654
655
656 def HostTools(host, target):
657   def H(component_name):
658     return ForHost(component_name, host)
659
660   def WindowsAlternate(if_windows, if_not_windows, if_mac=None):
661     if if_mac is not None and HostIsMac(host):
662       return if_mac
663     elif HostIsWindows(host):
664       return if_windows
665     else:
666       return if_not_windows
667
668   # Return the file name with the appropriate suffix for an executable file.
669   def Exe(file):
670     return file + WindowsAlternate('.exe', '')
671
672   tools = {
673       H('binutils_' + target): {
674           'type': 'build',
675           'dependencies': ['binutils'],
676           'commands': ConfigureTargetPrep(target) + [
677               command.Command(
678                   ConfigureCommand('binutils') +
679                   ConfigureHostTool(host) +
680                   ConfigureTargetArgs(target) + [
681                       '--enable-deterministic-archives',
682                       '--enable-gold',
683                       ] + WindowsAlternate([], ['--enable-plugins'])),
684               command.Command(MakeCommand(host)),
685               command.Command(MakeCheckCommand(host)),
686               command.Command(MAKE_DESTDIR_CMD + ['install-strip']),
687               REMOVE_INFO_DIR,
688               ] + InstallDocFiles('binutils',
689                                   ['COPYING3'] +
690                                   [command.path.join(subdir, 'NEWS')
691                                    for subdir in
692                                    ['binutils', 'gas', 'ld', 'gold']]) +
693               # The top-level lib* directories contain host libraries
694               # that we don't want to include in the distribution.
695               [command.RemoveDirectory(command.path.join('%(output)s', name))
696                for name in ['lib', 'lib32', 'lib64']],
697           },
698
699       H('gcc_' + target): {
700           'type': 'build',
701           'dependencies': (['gcc'] + HostGccLibsDeps(host) +
702                            GccDeps(host, target)),
703           'commands': ConfigureTargetPrep(target) + [
704               ConfigureGccCommand('gcc', host, target),
705               # GCC's configure step writes configargs.h with some strings
706               # including the configure command line, which get embedded
707               # into the gcc driver binary.  The build only works if we use
708               # absolute paths in some of the configure switches, but
709               # embedding those paths makes the output differ in repeated
710               # builds done in different directories, which we do not want.
711               # So force the generation of that file early and then edit it
712               # in place to replace the absolute paths with something that
713               # never varies.  Note that the 'configure-gcc' target will
714               # actually build some components before running gcc/configure.
715               GccCommand(host, target,
716                          MakeCommand(host, ['configure-gcc'])),
717               command.Command(['sed', '-i', '-e',
718                                ';'.join(['s@%%(abs_%s)s@.../%s_install@g' %
719                                          (component, component)
720                                          for component in
721                                          HostGccLibsDeps(host)] +
722                                         ['s@%(cwd)s@...@g']),
723                                command.path.join('gcc', 'configargs.h')]),
724               # gcc/Makefile's install rules ordinarily look at the
725               # installed include directory for a limits.h to decide
726               # whether the lib/gcc/.../include-fixed/limits.h header
727               # should be made to expect a libc-supplied limits.h or not.
728               # Since we're doing this build in a clean environment without
729               # any libc installed, we need to force its hand here.
730               GccCommand(host, target,
731                          MakeCommand(host, ['all-gcc', 'LIMITS_H_TEST=true'])),
732               # gcc/Makefile's install targets populate this directory
733               # only if it already exists.
734               command.Mkdir(command.path.join('%(output)s',
735                                               target + '-nacl', 'bin'),
736                             True),
737               GccCommand(host, target,
738                          MAKE_DESTDIR_CMD + ['install-strip-gcc']),
739               REMOVE_INFO_DIR,
740               # Note we include COPYING.RUNTIME here and not with gcc_libs.
741               ] + InstallDocFiles('gcc', ['COPYING3', 'COPYING.RUNTIME']),
742           },
743
744       # GDB can support all the targets in one host tool.
745       H('gdb'): {
746           'type': 'build',
747           'dependencies': ['gdb', H('expat')],
748           'commands': [
749               command.Command(
750                   ConfigureCommand('gdb') +
751                   ConfigureHostTool(host) + [
752                       '--target=x86_64-nacl',
753                       '--enable-targets=arm-none-eabi-nacl',
754                       '--with-expat',
755                       # Windows (MinGW) is missing ncurses; we need to
756                       # build one here and link it in statically for
757                       # --enable-tui.  See issue nativeclient:3911.
758                       '--%s-tui' % WindowsAlternate('disable', 'enable'),
759                       'CPPFLAGS=-I%(abs_' + H('expat') + ')s/include',
760                       'LDFLAGS=-L%(abs_' + H('expat') + ')s/lib',
761                       ] +
762                   # TODO(mcgrathr): Should use --with-python to ensure
763                   # we have it on Linux/Mac.
764                   WindowsAlternate(['--without-python'], []) +
765                   # TODO(mcgrathr): The default -Werror only breaks because
766                   # the OSX default compiler is an old front-end that does
767                   # not understand all the GCC options.  Maybe switch to
768                   # using clang (system or Chromium-supplied) on Mac.
769                   (['--disable-werror'] if HostIsMac(host) else [])),
770               command.Command(MakeCommand(host) + ['all-gdb']),
771               command.Command(MAKE_DESTDIR_CMD + [
772                   '-C', 'gdb', 'install-strip',
773                   ]),
774               REMOVE_INFO_DIR,
775               ] + [command.Command(['ln', '-f',
776                                     command.path.join('%(abs_output)s',
777                                                       'bin',
778                                                       Exe('x86_64-nacl-gdb')),
779                                     command.path.join('%(abs_output)s',
780                                                       'bin',
781                                                       Exe(arch + '-nacl-gdb'))])
782                    for arch in ['i686', 'arm']] + InstallDocFiles('gdb', [
783                        'COPYING3',
784                        command.path.join('gdb', 'NEWS'),
785                        ]),
786           },
787       }
788
789   # TODO(mcgrathr): The ARM cross environment does not supply a termcap
790   # library, so it cannot build GDB.
791   if host.startswith('arm') and CrossCompiling(host):
792     del tools[H('gdb')]
793
794   return tools
795
796 def TargetCommands(host, target, command_list):
797   # First we have to copy the host tools into a common directory.
798   # We can't just have both directories in our PATH, because the
799   # compiler looks for the assembler and linker relative to itself.
800   commands = PopulateDeps(['%(' + ForHost('binutils_' + target, host) + ')s',
801                            '%(' + ForHost('gcc_' + target, host) + ')s'])
802   bindir = command.path.join('%(cwd)s', 'all_deps', 'bin')
803   commands += [command.Command(cmd, path_dirs=[bindir])
804                for cmd in command_list]
805   return commands
806
807
808 def TargetLibs(host, target):
809   lib_deps = [ForHost(component + '_' + target, host)
810               for component in ['binutils', 'gcc']]
811
812   def NewlibFile(subdir, name):
813     return command.path.join('%(output)s', target + '-nacl', subdir, name)
814
815   newlib_sysroot = '%(abs_newlib_' + target + ')s'
816   newlib_tooldir = '%s/%s-nacl' % (newlib_sysroot, target)
817
818   # See the comment at ConfigureTargetPrep, above.
819   newlib_install_data = ' '.join(['STRIPPROG=%(cwd)s/strip_for_target',
820                                   '%(abs_newlib)s/install-sh',
821                                   '-c', '-s', '-m', '644'])
822
823   iconv_encodings = 'UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4'
824   newlib_configure_args = [
825       '--disable-libgloss',
826       '--enable-newlib-iconv',
827       '--enable-newlib-iconv-from-encodings=' + iconv_encodings,
828       '--enable-newlib-iconv-to-encodings=' + iconv_encodings,
829       '--enable-newlib-io-long-long',
830       '--enable-newlib-io-long-double',
831       '--enable-newlib-io-c99-formats',
832       '--enable-newlib-mb',
833       'CFLAGS=-O2',
834       'INSTALL_DATA=' + newlib_install_data,
835       ]
836
837   newlib_post_install = [
838       command.Rename(NewlibFile('lib', 'libc.a'),
839                      NewlibFile('lib', 'libcrt_common.a')),
840       command.WriteData(NewlibLibcScript(target),
841                         NewlibFile('lib', 'libc.a')),
842       ] + [
843       command.Copy(
844           command.path.join('%(pthread_headers)s', header),
845           NewlibFile('include', header))
846       for header in ('pthread.h', 'semaphore.h')
847       ]
848
849
850   libs = {
851       'newlib_' + target: {
852           'type': 'build',
853           'dependencies': ['newlib'] + lib_deps,
854           'inputs': { 'pthread_headers':
855                       os.path.join(NACL_DIR, 'src', 'untrusted',
856                                    'pthread') },
857           'commands': (ConfigureTargetPrep(target) +
858                        TargetCommands(host, target, [
859                            ConfigureCommand('newlib') +
860                            ConfigureHostTool(host) +
861                            ConfigureTargetArgs(target) +
862                            newlib_configure_args,
863                            MakeCommand(host),
864                            MAKE_DESTDIR_CMD + ['install-strip'],
865                            ]) +
866                        newlib_post_install +
867                        InstallDocFiles('newlib', ['COPYING.NEWLIB'])),
868           },
869
870       'gcc_libs_' + target: {
871           'type': 'build',
872           'dependencies': (['gcc_libs'] + lib_deps + ['newlib_' + target] +
873                            HostGccLibsDeps(host)),
874           # This actually builds the compiler again and uses that compiler
875           # to build the target libraries.  That's by far the easiest thing
876           # to get going given the interdependencies of the target
877           # libraries (especially libgcc) on the gcc subdirectory, and
878           # building the compiler doesn't really take all that long in the
879           # grand scheme of things.
880           # TODO(mcgrathr): If upstream ever cleans up all their
881           # interdependencies better, unpack the compiler, configure with
882           # --disable-gcc, and just build all-target.
883           'commands': ConfigureTargetPrep(target) + [
884               ConfigureGccCommand('gcc_libs', host, target, [
885                   '--with-build-sysroot=' + newlib_sysroot,
886                   ]),
887               GccCommand(host, target,
888                          MakeCommand(host) + [
889                              'build_tooldir=' + newlib_tooldir,
890                              'all-target',
891                              ]),
892               GccCommand(host, target,
893                          MAKE_DESTDIR_CMD + ['install-strip-target']),
894               REMOVE_INFO_DIR,
895               ],
896           },
897       }
898   return libs
899
900 # Compute it once.
901 NATIVE_TUPLE = pynacl.platform.PlatformTriple()
902
903
904 # For our purposes, "cross-compiling" means not literally that we are
905 # targetting a host that does not match NATIVE_TUPLE, but that we are
906 # targetting a host whose binaries we cannot run locally.  So x86-32
907 # on x86-64 does not count as cross-compiling.  See NATIVE_ENOUGH_MAP, above.
908 def CrossCompiling(host):
909   return (host != NATIVE_TUPLE and
910           host not in NATIVE_ENOUGH_MAP.get(NATIVE_TUPLE, {}))
911
912
913 def HostIsWindows(host):
914   return host == WINDOWS_HOST_TUPLE
915
916
917 def HostIsMac(host):
918   return host == MAC_HOST_TUPLE
919
920
921 # We build target libraries only on Linux for two reasons:
922 # 1. We only need to build them once.
923 # 2. Linux is the fastest to build.
924 # TODO(mcgrathr): In future set up some scheme whereby non-Linux
925 # bots can build target libraries but not archive them, only verifying
926 # that the results came out the same as the ones archived by the
927 # official builder bot.  That will serve as a test of the host tools
928 # on the other host platforms.
929 def BuildTargetLibsOn(host):
930   return host == LINUX_X86_64_TUPLE
931
932
933 def GetPackageTargets():
934   """Package Targets describes all the final package targets.
935
936   This build can be built among many build bots, but eventually all things
937   will be combined together. This package target dictionary describes the final
938   output of the entire build.
939   """
940   package_targets = {}
941
942   # Add in standard upload targets.
943   for host_target in UPLOAD_HOST_TARGETS:
944     for target in host_target.targets:
945       target_arch = target.name
946       package_prefix = target.pkg_prefix
947
948       # Each package target contains non-platform specific newlib and gcc libs.
949       # These packages are added inside of TargetLibs(host, target).
950       newlib_package = 'newlib_%s' % target_arch
951       gcc_lib_package = 'gcc_libs_%s' % target_arch
952       shared_packages = [newlib_package, gcc_lib_package]
953
954       # Each package target contains arm binutils and gcc.
955       # These packages are added inside of HostTools(host, target).
956       platform_triple = pynacl.platform.PlatformTriple(host_target.os,
957                                                        host_target.arch)
958       binutils_package = ForHost('binutils_%s' % target_arch, platform_triple)
959       gcc_package = ForHost('gcc_%s' % target_arch, platform_triple)
960       gdb_package = ForHost('gdb', platform_triple)
961
962       # Create a list of packages for a target.
963       platform_packages = [binutils_package, gcc_package, gdb_package]
964       combined_packages = shared_packages + platform_packages
965
966       os_name = pynacl.platform.GetOS(host_target.os)
967       if host_target.differ3264:
968         arch_name = pynacl.platform.GetArch3264(host_target.arch)
969       else:
970         arch_name = pynacl.platform.GetArch(host_target.arch)
971       package_target = '%s_%s' % (os_name, arch_name)
972       package_name = '%snacl_%s_newlib' % (package_prefix,
973                                            pynacl.platform.GetArch(target_arch))
974
975       package_target_dict = package_targets.setdefault(package_target, {})
976       package_target_dict.setdefault(package_name, []).extend(combined_packages)
977
978   # GDB is a special and shared, we will inject it into various other packages.
979   for platform, arch in GDB_INJECT_HOSTS:
980     platform_triple = pynacl.platform.PlatformTriple(platform, arch)
981     os_name = pynacl.platform.GetOS(platform)
982     arch_name = pynacl.platform.GetArch(arch)
983
984     gdb_packages = [ForHost('gdb', platform_triple)]
985     package_target = '%s_%s' % (os_name, arch_name)
986
987     for package_name, package_archives in GDB_INJECT_PACKAGES:
988       combined_packages = package_archives + gdb_packages
989       package_target_dict = package_targets.setdefault(package_target, {})
990       package_target_dict.setdefault(package_name, []).extend(combined_packages)
991
992   return dict(package_targets)
993
994 def CollectPackagesForHost(host, targets):
995   packages = HostGccLibs(host).copy()
996   for target in targets:
997     packages.update(HostTools(host, target))
998     if BuildTargetLibsOn(host):
999       packages.update(TargetLibs(host, target))
1000   return packages
1001
1002
1003 def CollectPackages(targets):
1004   packages = CollectSources()
1005
1006   packages.update(CollectPackagesForHost(NATIVE_TUPLE, targets))
1007
1008   for host in EXTRA_HOSTS_MAP.get(NATIVE_TUPLE, []):
1009     packages.update(CollectPackagesForHost(host, targets))
1010
1011   return packages
1012
1013
1014 PACKAGES = CollectPackages(TARGET_LIST)
1015 PACKAGE_TARGETS = GetPackageTargets()
1016
1017
1018 if __name__ == '__main__':
1019   tb = toolchain_main.PackageBuilder(PACKAGES, PACKAGE_TARGETS, sys.argv[1:])
1020   # TODO(mcgrathr): The bot ought to run some native_client tests
1021   # using the new toolchain, like the old x86 toolchain bots do.
1022   tb.Main()