2 # Copyright (c) 2013 The Native Client Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Recipes for PNaCl toolchain packages.
8 Recipes consist of specially-structured dictionaries, with keys for package
9 name, type, commands to execute, etc. The structure is documented in the
10 PackageBuilder docstring in toolchain_main.py.
12 The real entry plumbing and CLI flags are also in toolchain_main.py.
22 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
23 import pynacl.gsd_storage
24 import pynacl.platform
25 import pynacl.repo_tools
29 import pnacl_targetlibs
32 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
33 NACL_DIR = os.path.dirname(SCRIPT_DIR)
34 # Use the argparse from third_party to ensure it's the same on all platorms
35 python_lib_dir = os.path.join(os.path.dirname(NACL_DIR), 'third_party',
36 'python_libs', 'argparse')
37 sys.path.insert(0, python_lib_dir)
40 # Scons tests can check this version number to decide whether to enable tests
41 # for toolchain bug fixes or new features. This allows tests to be enabled on
42 # the toolchain buildbots/trybots before the new toolchain version is pinned
43 # (i.e. before the tests would pass on the main NaCl buildbots/trybots).
44 # If you are adding a test that depends on a toolchain change, you can
45 # increment this version number manually.
48 # For backward compatibility, these key names match the directory names
49 # previously used with gclient
51 'binutils': 'nacl-binutils.git',
52 'clang': 'pnacl-clang.git',
53 'llvm': 'pnacl-llvm.git',
54 'gcc': 'pnacl-gcc.git',
55 'libcxx': 'pnacl-libcxx.git',
56 'libcxxabi': 'pnacl-libcxxabi.git',
57 'nacl-newlib': 'nacl-newlib.git',
58 'llvm-test-suite': 'pnacl-llvm-testsuite.git',
59 'compiler-rt': 'pnacl-compiler-rt.git',
62 GIT_BASE_URL = 'https://chromium.googlesource.com/native_client/'
63 GIT_PUSH_URL = 'ssh://gerrit.chromium.org/native_client/'
64 GIT_DEPS_FILE = os.path.join(NACL_DIR, 'pnacl', 'COMPONENT_REVISIONS')
66 KNOWN_MIRRORS = [('http://git.chromium.org/native_client/', GIT_BASE_URL)]
67 PUSH_MIRRORS = [('http://git.chromium.org/native_client/', GIT_PUSH_URL),
68 (GIT_BASE_URL, GIT_PUSH_URL)]
70 # TODO(dschuff): Some of this mingw logic duplicates stuff in command.py
71 BUILD_CROSS_MINGW = False
72 # Path to the mingw cross-compiler libs on Ubuntu
73 CROSS_MINGW_LIBPATH = '/usr/lib/gcc/i686-w64-mingw32/4.6'
74 # Path and version of the native mingw compiler to be installed on Windows hosts
75 MINGW_PATH = os.path.join(NACL_DIR, 'mingw32')
76 MINGW_VERSION = 'i686-w64-mingw32-4.8.1'
78 ALL_ARCHES = ('x86-32', 'x86-64', 'arm', 'mips32',
79 'x86-32-nonsfi', 'arm-nonsfi')
80 # MIPS32 doesn't use biased bitcode, and nonsfi targets don't need it.
81 BITCODE_BIASES = tuple(bias for bias in ('portable', 'x86-32', 'x86-64', 'arm'))
83 MAKE_DESTDIR_CMD = ['make', 'DESTDIR=%(abs_output)s']
85 def TripleIsWindows(t):
86 return fnmatch.fnmatch(t, '*-mingw32*')
88 def TripleIsCygWin(t):
89 return fnmatch.fnmatch(t, '*-cygwin*')
92 def CompilersForHost(host):
94 # For now we only do native builds for linux and mac
95 'i686-linux': ('gcc', 'g++'), # treat 32-bit linux like a native build
96 'x86_64-linux': ('gcc', 'g++'),
97 # TODO(dschuff): switch to clang on mac
98 'x86_64-apple-darwin': ('gcc', 'g++'),
99 # Windows build should work for native and cross
100 'i686-w64-mingw32': ('i686-w64-mingw32-gcc', 'i686-w64-mingw32-g++'),
101 # TODO: add arm-hosted support
102 'i686-pc-cygwin': ('gcc', 'g++'),
104 return compiler[host]
107 def ConfigureHostArchFlags(host):
111 native = pynacl.platform.PlatformTriple()
112 is_cross = host != native
114 if (pynacl.platform.IsLinux64() and
115 fnmatch.fnmatch(host, '*-linux*')):
116 # 64 bit linux can build 32 bit linux binaries while still being a native
117 # build for our purposes. But it's not what config.guess will yield, so
118 # use --build to force it and make sure things build correctly.
119 configure_args.append('--build=' + host)
120 extra_cc_args = ['-m32']
122 configure_args.append('--host=' + host)
124 extra_cxx_args = list(extra_cc_args)
126 cc, cxx = CompilersForHost(host)
129 # LLVM's linux->mingw cross build needs this
130 configure_args.append('CC_FOR_BUILD=gcc')
132 configure_args.append('CC=' + ' '.join([cc] + extra_cc_args))
133 configure_args.append('CXX=' + ' '.join([cxx] + extra_cxx_args))
135 if TripleIsWindows(host):
136 # The i18n support brings in runtime dependencies on MinGW DLLs
137 # that we don't want to have to distribute alongside our binaries.
138 # So just disable it, and compiler messages will always be in US English.
139 configure_args.append('--disable-nls')
140 configure_args.extend(['LDFLAGS=-L%(abs_libdl)s',
141 'CFLAGS=-isystem %(abs_libdl)s',
142 'CXXFLAGS=-isystem %(abs_libdl)s'])
143 return configure_args
146 def CmakeHostArchFlags(host, options):
149 cc ='%(abs_top_srcdir)s/../third_party/llvm-build/Release+Asserts/bin/clang'
152 cc, cxx = CompilersForHost(host)
153 cmake_flags.extend(['-DCMAKE_C_COMPILER='+cc, '-DCMAKE_CXX_COMPILER='+cxx])
155 if pynacl.platform.IsLinux64() and pynacl.platform.PlatformTriple() != host:
156 # Currently the only supported "cross" build is 64-bit Linux to 32-bit
157 # Linux. Enable it. Also disable libxml and libtinfo because our Ubuntu
158 # doesn't have 32-bit libxml or libtinfo build, and users may not have them
160 cmake_flags.extend(['-DLLVM_BUILD_32_BITS=ON',
161 '-DLLVM_ENABLE_LIBXML=OFF',
162 '-DLLVM_ENABLE_TERMINFO=OFF'])
165 cmake_flags.extend(['-DCMAKE_%s_FLAGS=-fsanitize=%s' % (c, options.sanitize)
166 for c in ('C', 'CXX')])
167 cmake_flags.append('-DCMAKE_EXE_LINKER_FLAGS=-fsanitize=%s' %
172 def MakeCommand(host):
173 make_command = ['make']
174 if not pynacl.platform.IsWindows() or pynacl.platform.IsCygWin():
175 # The make that ships with msys sometimes hangs when run with -j.
176 # The ming32-make that comes with the compiler itself reportedly doesn't
177 # have this problem, but it has issues with pathnames with LLVM's build.
178 make_command.append('-j%(cores)s')
180 if TripleIsWindows(host):
181 # There appears to be nothing we can pass at top-level configure time
182 # that will prevent the configure scripts from finding MinGW's libiconv
183 # and using it. We have to force this variable into the environment
184 # of the sub-configure runs, which are run via make.
185 make_command.append('HAVE_LIBICONV=no')
189 def CopyWindowsHostLibs(host):
191 if not TripleIsWindows(host) and not TripleIsCygWin(host):
194 if TripleIsCygWin(host):
196 libs = ('cyggcc_s-1.dll', 'cygiconv-2.dll', 'cygwin1.dll',
197 'cygintl-8.dll', 'cygstdc++-6.dll', 'cygz.dll')
198 elif pynacl.platform.IsWindows():
199 lib_path = os.path.join(MINGW_PATH, 'bin')
200 # The native minGW compiler uses winpthread, but the Ubuntu cross compiler
202 libs = ('libgcc_s_sjlj-1.dll', 'libstdc++-6.dll', 'libwinpthread-1.dll')
204 lib_path = os.path.join(CROSS_MINGW_LIBPATH)
205 libs = ('libgcc_s_sjlj-1.dll', 'libstdc++-6.dll')
206 return [command.Copy(
207 os.path.join(lib_path, lib),
208 os.path.join('%(output)s', 'bin', lib))
211 def GetGitSyncCmdsCallback(revisions):
212 """Return a callback which returns the git sync commands for a component.
214 This allows all the revision information to be processed here while giving
215 other modules like pnacl_targetlibs.py the ability to define their own
216 source targets with minimal boilerplate.
218 def GetGitSyncCmds(component):
219 git_url = GIT_BASE_URL + GIT_REPOS[component]
220 git_push_url = GIT_PUSH_URL + GIT_REPOS[component]
222 # This replaces build.sh's newlib-nacl-headers-clean step by cleaning the
223 # the newlib repo on checkout (while silently blowing away any local
224 # changes). TODO(dschuff): find a better way to handle nacl newlib headers.
225 is_newlib = component == 'nacl-newlib'
226 return (command.SyncGitRepoCmds(git_url, '%(output)s', revisions[component],
228 git_cache='%(git_cache_dir)s',
229 push_url=git_push_url,
230 known_mirrors=KNOWN_MIRRORS,
231 push_mirrors=PUSH_MIRRORS) +
232 [command.Runnable(None,
233 pnacl_commands.CmdCheckoutGitBundleForTrybot,
234 component, '%(output)s')])
236 return GetGitSyncCmds
239 def HostToolsSources(GetGitSyncCmds):
241 'binutils_pnacl_src': {
243 'output_dirname': 'binutils',
244 'commands': GetGitSyncCmds('binutils'),
246 # For some reason, the llvm build using --with-clang-srcdir chokes if the
247 # clang source directory is named something other than 'clang', so don't
248 # change output_dirname for clang.
251 'output_dirname': 'clang',
252 'commands': GetGitSyncCmds('clang'),
256 'output_dirname': 'llvm',
257 'commands': GetGitSyncCmds('llvm'),
263 def TestsuiteSources(GetGitSyncCmds):
265 'llvm_testsuite_src': {
267 'output_dirname': 'llvm-test-suite',
268 'commands': GetGitSyncCmds('llvm-test-suite'),
276 if TripleIsWindows(host):
277 if pynacl.platform.IsWindows():
280 ar = 'i686-w64-mingw32-ar'
285 'inputs' : { 'src' : os.path.join(NACL_DIR, '..', 'third_party',
288 command.CopyTree('%(src)s', '.'),
289 command.Command(['i686-w64-mingw32-gcc',
290 '-o', 'dlfcn.o', '-c', 'dlfcn.c',
291 '-Wall', '-O3', '-fomit-frame-pointer']),
292 command.Command([ar, 'cru',
293 'libdl.a', 'dlfcn.o']),
294 command.Copy('libdl.a',
295 os.path.join('%(output)s', 'libdl.a')),
296 command.Copy('dlfcn.h',
297 os.path.join('%(output)s', 'dlfcn.h')),
304 def HostTools(host, options):
305 def H(component_name):
306 # Return a package name for a component name with a host triple.
307 return component_name + '_' + pynacl.gsd_storage.LegalizeName(host)
309 return fnmatch.fnmatch(host, 'x86_64*')
310 def HostSubdir(host):
311 return 'host_x86_64' if IsHost64(host) else 'host_x86_32'
313 return 'bin64' if host == 'x86_64-linux' else 'bin'
314 # Return the file name with the appropriate suffix for an executable file.
316 if TripleIsWindows(host):
321 H('binutils_pnacl'): {
322 'dependencies': ['binutils_pnacl_src'],
324 'output_subdir': HostSubdir(host),
326 command.SkipForIncrementalCommand([
328 '%(binutils_pnacl_src)s/configure'] +
329 ConfigureHostArchFlags(host) +
331 '--disable-silent-rules',
332 '--target=arm-pc-nacl',
333 '--program-prefix=le32-nacl-',
334 '--enable-targets=arm-pc-nacl,i686-pc-nacl,x86_64-pc-nacl,' +
336 '--enable-deterministic-archives',
337 '--enable-shared=no',
338 '--enable-gold=default',
343 '--with-sysroot=/arm-pc-nacl']),
344 command.Command(MakeCommand(host)),
345 command.Command(MAKE_DESTDIR_CMD + ['install-strip'])] +
346 [command.RemoveDirectory(os.path.join('%(output)s', dir))
347 for dir in ('arm-pc-nacl', 'lib', 'lib32')]
351 'output_subdir': BinSubdir(host),
352 'inputs': { 'src': os.path.join(NACL_DIR, 'pnacl', 'driver')},
356 pnacl_commands.InstallDriverScripts,
357 '%(src)s', '%(output)s',
358 host_windows=TripleIsWindows(host) or TripleIsCygWin(host),
359 host_64bit=IsHost64(host))
365 'dependencies': ['clang_src', 'llvm_src', 'binutils_pnacl_src'],
367 'output_subdir': HostSubdir(host),
369 command.SkipForIncrementalCommand([
370 'cmake', '-G', 'Ninja'] +
371 CmakeHostArchFlags(host, options) +
372 ['-DCMAKE_BUILD_TYPE=RelWithDebInfo',
373 '-DCMAKE_INSTALL_PREFIX=%(output)s',
374 '-DCMAKE_INSTALL_RPATH=$ORIGIN/../lib',
375 '-DBUILD_SHARED_LIBS=ON',
376 '-DLLVM_ENABLE_ASSERTIONS=ON',
377 '-DLLVM_ENABLE_ZLIB=OFF',
378 '-DLLVM_BUILD_TESTS=ON',
379 '-DLLVM_APPEND_VC_REV=ON',
380 '-DLLVM_BINUTILS_INCDIR=%(abs_binutils_pnacl_src)s/include',
381 '-DLLVM_EXTERNAL_CLANG_SOURCE_DIR=%(clang_src)s',
383 command.Command(['ninja', '-v']),
384 command.Command(['ninja', 'install']),
390 'dependencies': ['clang_src', 'llvm_src', 'binutils_pnacl_src'],
392 'output_subdir': HostSubdir(host),
394 command.SkipForIncrementalCommand([
396 '%(llvm_src)s/configure'] +
397 ConfigureHostArchFlags(host) +
401 '--disable-terminfo',
403 '--disable-bindings', # ocaml is currently the only binding.
404 '--with-binutils-include=%(abs_binutils_pnacl_src)s/include',
405 '--enable-targets=x86,arm,mips',
407 '--enable-optimized',
408 '--with-clang-srcdir=%(abs_clang_src)s']),
409 command.Command(MakeCommand(host) + [
413 command.Command(MAKE_DESTDIR_CMD + ['install']),
414 command.Remove(*[os.path.join('%(output)s', 'lib', f) for f in
415 '*.a', '*Hello.*', 'BugpointPasses.*']),
416 command.Remove(*[os.path.join('%(output)s', 'bin', f) for f in
417 Exe('clang-format'), Exe('clang-check'),
418 Exe('c-index-test'), Exe('clang-tblgen'),
419 Exe('llvm-tblgen')])] +
420 CopyWindowsHostLibs(host),
424 tools.update(llvm_cmake)
426 tools.update(llvm_autoconf)
427 if TripleIsWindows(host):
428 tools[H('binutils_pnacl')]['dependencies'].append('libdl')
429 tools[H('llvm')]['dependencies'].append('libdl')
432 # TODO(dschuff): The REV file should probably go here rather than in the driver
438 'inputs': { 'readme': os.path.join(NACL_DIR, 'pnacl', 'README') },
440 command.Copy('%(readme)s', os.path.join('%(output)s', 'README')),
441 command.WriteData(str(FEATURE_VERSION),
442 os.path.join('%(output)s', 'FEATURE_VERSION')),
448 def ParseComponentRevisionsFile(filename):
449 ''' Parse a simple-format deps file, with fields of the form:
451 Keys should match the keys in GIT_REPOS above, which match the previous
452 directory names used by gclient (with the exception that '_' in the file is
453 replaced by '-' in the returned key name).
454 Values are the git hashes for each repo.
455 Empty lines or lines beginning with '#' are ignored.
456 This function returns a dictionary mapping the keys found in the file to their
459 with open(filename) as f:
462 stripped = line.strip()
463 if stripped.startswith('#') or len(stripped) == 0:
465 tokens = stripped.split('=')
467 raise Exception('Malformed component revisions file: ' + filename)
468 deps[tokens[0].replace('_', '-')] = tokens[1]
472 def GetSyncPNaClReposSource(revisions, GetGitSyncCmds):
474 for repo, revision in revisions.iteritems():
475 sources['legacy_pnacl_%s_src' % repo] = {
477 'output_dirname': os.path.join(NACL_DIR, 'pnacl', 'git', repo),
478 'commands': GetGitSyncCmds(repo),
483 def InstallMinGWHostCompiler():
484 """Install the MinGW host compiler used to build the host tools on Windows.
486 We could use an ordinary source rule for this, but that would require hashing
487 hundreds of MB of toolchain files on every build. Instead, check for the
488 presence of the specially-named file <version>.installed in the install
489 directory. If it is absent, check for the presence of the zip file
490 <version>.zip. If it is absent, attempt to download it from Google Storage.
491 Then extract the zip file and create the install file.
493 if not os.path.isfile(os.path.join(MINGW_PATH, MINGW_VERSION + '.installed')):
494 downloader = pynacl.gsd_storage.GSDStorage([], ['nativeclient-mingw'])
495 zipfilename = MINGW_VERSION + '.zip'
496 zipfilepath = os.path.join(NACL_DIR, zipfilename)
497 # If the zip file is not present, try to download it from Google Storage.
498 # If that fails, bail out.
499 if (not os.path.isfile(zipfilepath) and
500 not downloader.GetSecureFile(zipfilename, zipfilepath)):
501 print >>sys.stderr, 'Failed to install MinGW tools:'
502 print >>sys.stderr, 'could not find or download', zipfilename
504 logging.info('Extracting %s' % zipfilename)
505 zf = zipfile.ZipFile(zipfilepath)
506 if os.path.exists(MINGW_PATH):
507 shutil.rmtree(MINGW_PATH)
508 zf.extractall(NACL_DIR)
509 with open(os.path.join(MINGW_PATH, MINGW_VERSION + '.installed'), 'w') as _:
511 os.environ['MINGW'] = MINGW_PATH
514 def GetUploadPackageTargets():
515 """Package Targets describes all the archived package targets.
517 This build can be built among many build bots, but eventually all things
518 will be combined together. This package target dictionary describes the final
519 output of the entire build.
523 common_packages = ['metadata']
525 # Target native libraries
526 for arch in ALL_ARCHES:
527 legal_arch = pynacl.gsd_storage.LegalizeName(arch)
528 common_packages.append('libs_support_native_%s' % legal_arch)
529 common_packages.append('compiler_rt_%s' % legal_arch)
530 if not 'nonsfi' in arch:
531 common_packages.append('libgcc_eh_%s' % legal_arch)
533 # Target bitcode libraries
534 for bias in BITCODE_BIASES:
535 legal_bias = pynacl.gsd_storage.LegalizeName(bias)
536 common_packages.append('newlib_%s' % legal_bias)
537 common_packages.append('libcxx_%s' % legal_bias)
538 common_packages.append('libstdcxx_%s' % legal_bias)
539 common_packages.append('libs_support_bitcode_%s' % legal_bias)
543 for os_name, arch in (('cygwin', 'x86-32'),
546 ('linux', 'x86-64')):
547 triple = pynacl.platform.PlatformTriple(os_name, arch)
548 legal_triple = pynacl.gsd_storage.LegalizeName(triple)
549 host_packages.setdefault(os_name, []).extend(
550 ['binutils_pnacl_%s' % legal_triple,
551 'llvm_%s' % legal_triple,
552 'driver_%s' % legal_triple])
554 # Unsandboxed target IRT libraries
555 for os_name in ('linux', 'mac'):
556 legal_triple = pynacl.gsd_storage.LegalizeName('x86-32-' + os_name)
557 host_packages[os_name].append('unsandboxed_irt_%s' % legal_triple)
559 for os_name, os_packages in host_packages.iteritems():
560 package_target = '%s_x86' % pynacl.platform.GetOS(os_name)
561 package_targets[package_target] = {}
562 package_name = 'pnacl_newlib'
563 combined_packages = os_packages + common_packages
564 package_targets[package_target][package_name] = combined_packages
566 return package_targets
568 if __name__ == '__main__':
569 # This sets the logging for gclient-alike repo sync. It will be overridden
570 # by the package builder based on the command-line flags.
571 logging.getLogger().setLevel(logging.DEBUG)
572 parser = argparse.ArgumentParser(add_help=False)
573 parser.add_argument('--legacy-repo-sync', action='store_true',
574 dest='legacy_repo_sync', default=False,
575 help='Sync the git repo directories used by build.sh')
576 parser.add_argument('--build-64bit-host', action='store_true',
577 dest='build_64bit_host', default=False,
578 help='Build 64-bit Linux host binaries in addition to 32')
579 parser.add_argument('--cmake', action='store_true', default=False,
580 help="Use LLVM's cmake ninja build instead of autoconf")
581 parser.add_argument('--clang', action='store_true', default=False,
582 help="Use clang instead of gcc with LLVM's cmake build")
583 parser.add_argument('--sanitize', choices=['address', 'thread', 'undefined'],
584 help="Use a sanitizer with LLVM's clang cmake build")
585 parser.add_argument('--testsuite-sync', action='store_true', default=False,
586 help=('Sync the sources for the LLVM testsuite. '
587 'Only useful if --sync/ is also enabled'))
588 args, leftover_args = parser.parse_known_args()
589 if '-h' in leftover_args or '--help' in leftover_args:
590 print 'The following arguments are specific to toolchain_build_pnacl.py:'
592 print 'The rest of the arguments are generic, in toolchain_main.py'
594 if args.sanitize and not args.cmake:
595 print 'Use of sanitizers requires a cmake build'
597 if args.clang and not args.cmake:
598 print 'Use of clang is currently only supported with cmake/ninja'
605 rev = ParseComponentRevisionsFile(GIT_DEPS_FILE)
606 if args.legacy_repo_sync:
607 packages = GetSyncPNaClReposSource(rev, GetGitSyncCmdsCallback(rev))
609 # Make sure sync is inside of the args to toolchain_main.
610 if not set(['-y', '--sync', '--sync-only']).intersection(leftover_args):
611 leftover_args.append('--sync-only')
613 upload_packages = GetUploadPackageTargets()
614 if pynacl.platform.IsWindows():
615 InstallMinGWHostCompiler()
617 packages.update(HostToolsSources(GetGitSyncCmdsCallback(rev)))
618 if args.testsuite_sync:
619 packages.update(TestsuiteSources(GetGitSyncCmdsCallback(rev)))
621 if pynacl.platform.IsLinux64():
622 hosts = ['i686-linux']
623 if args.build_64bit_host:
624 hosts.append(pynacl.platform.PlatformTriple())
626 hosts = [pynacl.platform.PlatformTriple()]
627 if pynacl.platform.IsLinux() and BUILD_CROSS_MINGW:
628 hosts.append(pynacl.platform.PlatformTriple('win', 'x86-32'))
630 packages.update(HostLibs(host))
631 packages.update(HostTools(host, args))
632 # Don't build the target libs on Windows because of pathname issues.
633 # Don't build the target libs on Mac because the gold plugin's rpaths
635 # On linux use the 32-bit compiler to build the target libs since that's
636 # what most developers will be using. (hosts[0] is i686-linux on linux64)
637 # TODO(dschuff): Figure out a better way to test things on toolchain bots.
638 if pynacl.platform.IsLinux():
639 packages.update(pnacl_targetlibs.TargetLibsSrc(
640 GetGitSyncCmdsCallback(rev)))
641 for bias in BITCODE_BIASES:
642 packages.update(pnacl_targetlibs.BitcodeLibs(hosts[0], bias))
643 for arch in ALL_ARCHES:
644 packages.update(pnacl_targetlibs.NativeLibs(hosts[0], arch))
645 packages.update(Metadata())
646 if pynacl.platform.IsLinux() or pynacl.platform.IsMac():
647 packages.update(pnacl_targetlibs.UnsandboxedIRT(
648 'x86-32-%s' % pynacl.platform.GetOS()))
651 tb = toolchain_main.PackageBuilder(packages,