2 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Recipes for NativeClient toolchain packages.
8 The real entry plumbing is in toolchain_main.py.
11 # Done first to setup python module path.
24 import toolchain_build
27 from file_update import Mkdir, Rmdir, Symlink
28 from file_update import NeedsUpdate, UpdateFromTo, UpdateText
30 BIONIC_VERSION = '171e152bc66aec14029e90edc69d94af4f289bab'
34 BUILD_SCRIPT = os.path.abspath(__file__)
35 TOOLCHAIN_BUILD = os.path.dirname(BUILD_SCRIPT)
36 TOOLCHAIN_BUILD_SRC = os.path.join(TOOLCHAIN_BUILD, 'src')
37 TOOLCHAIN_BUILD_OUT = os.path.join(TOOLCHAIN_BUILD, 'out')
39 BIONIC_SRC = os.path.join(TOOLCHAIN_BUILD_SRC, 'bionic')
41 NATIVE_CLIENT = os.path.dirname(TOOLCHAIN_BUILD)
42 TOOLCHAIN = os.path.join(NATIVE_CLIENT, 'toolchain')
44 ARM_NEWLIB = os.path.join(TOOLCHAIN, 'linux_arm_newlib')
45 ARM_BIONIC = os.path.join(TOOLCHAIN, 'linux_arm_bionic')
46 X86_NEWLIB = os.path.join(TOOLCHAIN, 'linux_x86_newlib')
47 X86_BIONIC = os.path.join(TOOLCHAIN, 'linux_x86_bionic')
57 def ReplaceText(text, maplist):
60 text = text.replace(key, m[key])
64 def ReplaceArch(text, arch, subarch=None):
82 '$NACL': NACL_ARCHES[arch],
83 '$GCC': GCC_ARCHES[arch],
84 '$CPU': CPU_ARCHES[arch],
85 '$SUB': subarch or '',
87 '$VER': VERSION_MAP[arch]
89 return ReplaceText(text, [REPLACE_MAP])
93 Rmdir(os.path.join(TOOLCHAIN_BUILD, 'cache'))
95 Rmdir(os.path.join(TOOLCHAIN, 'linux_%s_bionic' % arch))
96 for workdir in PROJECTS:
97 Rmdir(os.path.join(TOOLCHAIN_BUILD_OUT, workdir % arch))
100 def FetchAndBuildGCC():
102 tc_args = ['-y', '--no-use-remote-cache', 'gcc_libs_arm']
103 toolchain_main.PackageBuilder(toolchain_build.PACKAGES, tc_args).Main()
106 process.Run(['make', 'sync'], cwd=os.path.join(NATIVE_CLIENT, 'tools'))
109 def FetchBionicSources():
111 url = '%s/nacl-%s.git' % (toolchain_build.GIT_BASE_URL, project)
112 repo_tools.SyncGitRepo(url,
113 os.path.join(TOOLCHAIN_BUILD_SRC, project),
117 def CreateBasicToolchain():
118 # Create a toolchain directory containing only the toolchain binaries and
119 # basic files line nacl_arm_macros.s.
121 UpdateFromTo(ARM_NEWLIB, ARM_BIONIC,
122 filters=['*arm-nacl/include*', '*arm-nacl/lib*','*.a', '*.o'])
123 UpdateFromTo(ARM_NEWLIB, ARM_BIONIC, paterns=['*.s'])
125 UpdateFromTo(X86_NEWLIB, X86_BIONIC,
126 filters=['*x86_64-nacl/include*', '*x86_64-nacl/lib*','*.o'])
127 UpdateFromTo(X86_NEWLIB, X86_BIONIC, paterns=['*.s'])
130 # crt1.o crti.o 4.8.2/crtbeginT.o ... 4.8.2/crtend.o crtn.o
131 # -shared build uses:
132 # crti.o 4.8.2/crtbeginS.o ... 4.8.2/crtendS.o crtn.o crtn.o
133 # However we only provide a crtbegin(S) and crtend(S)
135 * This is a dummy linker script.
136 * libnacl.a, libcrt_common.a, crt0.o crt1.o crti.o and crtn.o are all
137 * empty. Instead all the work is done by crtbegin(S).o and crtend(S).o and
138 * the bionic libc. These are provided for compatability with the newlib
139 * toolchain binaries.
141 EMPTY_FILES = ['crt0.o', 'crt1.o', 'crti.o', 'crtn.o',
142 'libnacl.a', 'libcrt_common.a', 'libpthread.a']
144 # Bionic uses the following include paths
146 ('bionic/libc/include', '$NACL-nacl/include'),
147 ('bionic/libc/arch-nacl/syscalls/irt.h', '$NACL-nacl/include/irt.h'),
148 ('bionic/libc/arch-nacl/syscalls/irt_syscalls.h',
149 '$NACL-nacl/include/irt_syscalls.h'),
150 ('bionic/libc/arch-nacl/syscalls/nacl_stat.h',
151 '$NACL-nacl/include/nacl_stat.h'),
152 ('../../src/untrusted/irt/irt_dev.h', '$NACL-nacl/include/irt_dev.h'),
153 ('bionic/libc/arch-$ARCH/include/machine',
154 '$NACL-nacl/include/machine'),
155 ('bionic/libc/kernel/common', '$NACL-nacl/include'),
156 ('bionic/libc/kernel/arch-$ARCH/asm', '$NACL-nacl/include/asm'),
157 ('bionic/libm/include', '$NACL-nacl/include'),
158 ('bionic/libm/$CPU', '$NACL-nacl/include'),
159 ('bionic/safe-iop/include', '$NACL-nacl/include'),
160 ('bionic/libstdc++/nacl',
161 '$NACL-nacl/include/c++/4.8.2/$NACL-nacl'),
162 ('bionic/nacl/$ARCH', '.'),
166 inspath = os.path.join(TOOLCHAIN, 'linux_$ARCH_bionic')
167 inspath = ReplaceArch(inspath, arch)
169 # Create empty objects and libraries
170 libpath = ReplaceArch(os.path.join(inspath, '$NACL-nacl', 'lib'), arch)
171 for name in EMPTY_FILES:
172 UpdateText(os.path.join(libpath, name), EMPTY)
174 # Copy BIONIC files to toolchain
175 for src, dst in BIONIC_PAIRS:
176 srcpath = ReplaceArch(os.path.join(TOOLCHAIN_BUILD_SRC, src), arch)
177 dstpath = ReplaceArch(os.path.join(inspath, dst), arch)
178 UpdateFromTo(srcpath, dstpath)
180 workpath = os.path.join(TOOLCHAIN_BUILD_OUT, 'bionic_$ARCH_work')
181 workpath = ReplaceArch(workpath, arch)
182 ConfigureAndBuild(arch, 'bionic/libc', workpath, inspath,
186 gcc = ReplaceArch(os.path.join(inspath, 'bin', '$NACL-nacl-gcc'), arch)
187 lib = ReplaceArch(os.path.join(inspath, 'lib/gcc/$NACL-nacl/$VER'), arch)
188 specs = os.path.join(lib, 'specs')
189 with open(specs, 'w') as specfile:
190 process.Run([gcc, '-dumpspecs'], cwd=None, shell=False,
191 outfile=specfile, verbose=False)
192 text = open(specs, 'r').read()
194 # Replace items in the spec file
195 text = ReplaceText(text, [{
196 '-lgcc': '-lgcc %{!shared: -lgcc_eh}',
197 '--hash-style=gnu': '--hash-style=sysv',
200 open(specs, 'w').write(text)
203 def ConfigureAndBuildBionic():
205 inspath = os.path.join(TOOLCHAIN, 'linux_$ARCH_bionic')
206 inspath = ReplaceArch(inspath, arch)
208 workpath = os.path.join(TOOLCHAIN_BUILD_OUT, 'bionic_$ARCH_work')
209 workpath = ReplaceArch(workpath, arch)
210 ConfigureAndBuild(arch, 'bionic/libc', workpath, inspath)
211 ConfigureAndBuild(arch, 'bionic/libm', workpath, inspath)
212 ConfigureAndBuild(arch, 'bionic/linker', workpath, inspath)
215 def ConfigureAndInstallForGCC(arch, project, cfg, workpath, inspath):
216 # configure does not always have +x
217 filepath = os.path.abspath(os.path.join(workpath, cfg[0]))
218 st_info = os.stat(filepath)
219 os.chmod(filepath, st_info.st_mode | stat.S_IEXEC)
223 newpath = os.path.join(ARM_BIONIC, 'bin') + ':' + env['PATH']
225 newpath = os.path.join(X86_BIONIC, 'bin') + ':' + env['PATH']
226 proj = '%s %s' % (project, arch)
227 setpath = ['/usr/bin/env', 'PATH=' + newpath]
229 # Check if config changed or script is new
230 config_path = os.path.join(workpath, 'config.info')
231 updated = UpdateText(config_path, ' '.join(cfg))
232 updated |= NeedsUpdate(config_path, BUILD_SCRIPT)
235 print 'Configure ' + proj
236 if process.Run(setpath + cfg, cwd=workpath, env=env, outfile=sys.stdout):
237 raise RuntimeError('Failed to configure %s.\n' % proj)
239 print 'Reusing config for %s.' % proj
242 if process.Run(setpath + ['make', '-j16', 'V=1'],
243 cwd=workpath, outfile=sys.stdout):
244 raise RuntimeError('Failed to build %s.\n' % proj)
245 print 'Install ' + proj
246 if process.Run(setpath + ['make', '-j16', 'install', 'V=1'],
247 cwd=workpath, outfile=sys.stdout):
248 raise RuntimeError('Failed to install %s.\n' % proj)
252 def ConfigureAndBuildArmGCC(config=False):
255 tcpath = os.path.join(TOOLCHAIN, 'linux_$ARCH_bionic')
256 tcpath = ReplaceArch(tcpath, arch)
259 workpath = os.path.join(TOOLCHAIN_BUILD_OUT, 'gcc_$GCC_bionic_work')
260 workpath = ReplaceArch(workpath, arch)
262 Symlink('../gcc_libs_arm_work/gcc' , os.path.join(workpath, 'gcc'))
265 inspath = os.path.join(TOOLCHAIN_BUILD_OUT, 'gcc_$GCC_bionic_install')
266 inspath = ReplaceArch(inspath, arch)
268 dstpath = ReplaceArch(os.path.join(workpath, '$NACL-nacl/libgcc'), arch)
270 '../../../../src/gcc_libs/libgcc/configure',
272 '--build=i686-linux',
275 '--enable-shared-libgcc',
278 '--prefix=' + inspath,
279 'CFLAGS=-I../../../gcc_lib_arm_work'
281 ConfigureAndInstallForGCC(arch, project, cfg, dstpath, inspath)
282 UpdateFromTo(os.path.join(inspath, 'lib', 'gcc'),
283 os.path.join(tcpath, 'lib', 'gcc'),
285 UpdateFromTo(os.path.join(inspath, 'lib', 'libgcc_s.so.1'),
286 os.path.join(tcpath, 'arm-nacl', 'lib', 'libgcc_s.so.1'))
287 UpdateFromTo(os.path.join(inspath, 'lib', 'libgcc_s.so'),
288 os.path.join(tcpath, 'arm-nacl', 'lib', 'libgcc_s.so'))
291 def ConfigureAndBuildX86GCC(config=False):
295 tcpath = os.path.join(TOOLCHAIN, 'linux_$ARCH_bionic')
296 tcpath = ReplaceArch(tcpath, arch)
299 workpath = os.path.join(TOOLCHAIN_BUILD_OUT, 'gcc_$GCC_bionic_work')
300 workpath = ReplaceArch(workpath, arch)
302 Symlink('../gcc_libs_arm_work/gcc' , os.path.join(workpath, 'gcc'))
305 inspath = os.path.join(TOOLCHAIN_BUILD_OUT, 'gcc_$GCC_bionic_install')
306 inspath = ReplaceArch(inspath, arch)
308 dstpath = ReplaceArch(os.path.join(workpath, '$NACL-nacl/libgcc'), arch)
310 '../../../../SRC/gcc/libgcc/configure',
311 '--target=x86_64-nacl',
313 '--enable-shared-libgcc',
314 '--host=x86_64-nacl',
315 ',--build=i686-linux'
318 def ConfigureAndBuildArmCXX(config=False):
320 project = 'libstdc++'
321 tcpath = os.path.join(TOOLCHAIN, 'linux_$ARCH_bionic')
322 tcpath = ReplaceArch(tcpath, arch)
325 workpath = os.path.join(TOOLCHAIN_BUILD_OUT, 'cxx_$GCC_bionic_work')
326 workpath = ReplaceArch(workpath, arch)
328 Symlink('../gcc_libs_arm_work/gcc' , os.path.join(workpath, 'gcc'))
331 inspath = os.path.join(TOOLCHAIN_BUILD_OUT, 'cxx_$GCC_bionic_install')
332 inspath = ReplaceArch(inspath, arch)
334 dstpath = ReplaceArch(os.path.join(workpath, '$NACL-nacl/libstdc++-v3'), arch)
337 '../../../../src/gcc_libs/libstdc++-v3/configure',
339 '--build=i686-linux',
343 '--disable-libstdcxx-pch',
344 '--enable-shared-libgcc',
346 '--prefix=' + inspath,
347 'CFLAGS=-I../../../gcc_lib_arm_work'
349 ConfigureAndInstallForGCC(arch, project, cfg, dstpath, inspath)
350 UpdateFromTo(os.path.join(inspath, 'lib'),
351 os.path.join(tcpath, 'arm-nacl', 'lib'))
352 UpdateFromTo(os.path.join(inspath, 'include'),
353 os.path.join(tcpath, 'arm-nacl', 'include'))
357 def CreateProject(arch, src, dst, ins=None):
358 MAKEFILE_TEMPLATE = """
359 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
360 # Use of this source code is governed by a BSD-style license that can be
361 # found in the LICENSE file.
363 # GNU Makefile based on shared rules provided by the Native Client SDK.
364 # See README.Makefiles for more details.
366 TOOLCHAIN_PATH?=$(tc_path)/linux_$ARCH_bionic
367 TOOLCHAIN_PREFIX:=$(TOOLCHAIN_PATH)/bin/$GCC-nacl-
369 CC:=$(TOOLCHAIN_PREFIX)gcc
370 CXX:=$(TOOLCHAIN_PREFIX)g++
371 AR:=$(TOOLCHAIN_PREFIX)ar
380 include $(build_tc_path)/tc_bionic.mk
381 include $(src_path)/Makefile
386 '$(ins_path)': ins or dst,
387 '$(tc_path)': TOOLCHAIN,
388 '$(build_tc_path)': TOOLCHAIN_BUILD
390 text = ReplaceText(MAKEFILE_TEMPLATE, [remap])
391 text = ReplaceArch(text, arch)
393 print 'Create dst dir: ' + dst
396 print 'Create ins dir: ' + ins
399 UpdateText(os.path.join(dst, 'Makefile'), text)
402 def ConfigureAndBuild(arch, project, workpath, inspath,
403 tcpath=None, target=None):
405 # Create project for CRTx and LIBC files
406 print 'Configure %s for %s.\n' % (project, arch)
407 srcpath = os.path.join(TOOLCHAIN_BUILD_SRC, project)
408 dstpath = ReplaceArch(os.path.join(workpath, '$NACL-nacl', project), arch)
409 libpath = ReplaceArch(os.path.join(inspath, '$NACL-nacl', 'lib'), arch)
410 CreateProject(arch, srcpath, dstpath, libpath)
411 print 'Building %s for %s at %s.' % (project, arch, dstpath)
413 args = ['make', '-j12', 'V=1']
416 if process.Run(args, cwd=dstpath, outfile=sys.stdout):
417 raise RuntimeError('Failed to build %s for %s.\n' % (project, arch))
419 UpdateFromTo(inspath, tcpath)
420 print 'Done with %s for %s.\n' % (project, arch)
423 def ConfigureAndBuildGCC(clobber=False):
424 ConfigureAndBuildArmGCC(clobber)
427 def ConfigureAndBuildCXX(clobber=False):
428 ConfigureAndBuildArmCXX(clobber)
431 def ConfigureAndBuildTests(clobber=False):
433 workpath = os.path.join(TOOLCHAIN_BUILD_OUT,
434 ReplaceArch('bionic_$GCC_test_work', arch))
435 inspath = os.path.join(TOOLCHAIN_BUILD_OUT,
436 ReplaceArch('bionic_$GCC_test_install', arch))
443 ConfigureAndBuild(arch, 'bionic/tests', workpath, inspath)
446 def ArchiveAndUpload(version, zipname, zippath):
447 if 'BUILDBOT_BUILDERNAME' in os.environ:
448 GSUTIL = '../buildbot/gsutil.sh'
451 GSUTIL_ARGS = [GSUTIL, 'cp', '-a', 'public-read']
452 GSUTIL_PATH = 'gs://nativeclient-archive2/toolchain'
454 urldir = os.path.join(GSUTIL_PATH, version)
455 zipurl = os.path.join(urldir, zipname)
456 zipname = os.path.join(TOOLCHAIN_BUILD_OUT, zipname)
464 print >>sys.stderr, '@@@STEP_LINK@download@%s@@@' % urldir
466 if process.Run(['tar', '-czf', zipname, zippath],
469 raise RuntimeError('Failed to zip %s from %s.\n' % (zipname, zippath))
471 hashzipname = zipname + '.sha1hash'
472 hashzipurl = zipurl + '.sha1hash'
473 hashval = hashing_tools.HashFileContents(zipname)
475 with open(hashzipname, 'w') as f:
478 if process.Run(GSUTIL_ARGS + [zipname, zipurl],
481 err = 'Failed to upload zip %s to %s.\n' % (zipname, zipurl)
482 raise RuntimeError(err)
484 if process.Run(GSUTIL_ARGS + [hashzipname, hashzipurl],
487 err = 'Failed to upload hash %s to %s.\n' % (hashzipname, hashzipurl)
488 raise RuntimeError(err)
492 parser = optparse.OptionParser()
494 '-v', '--verbose', dest='verbose',
495 default=False, action='store_true',
496 help='Produce more output.')
498 '-c', '--clobber', dest='clobber',
499 default=False, action='store_true',
500 help='Clobber working directories before building.')
502 '-s', '--sync', dest='sync',
503 default=False, action='store_true',
504 help='Sync sources first.')
507 '-b', '--buildbot', dest='buildbot',
508 default=False, action='store_true',
509 help='Running on the buildbot.')
512 '-e', '--empty', dest='empty',
513 default=False, action='store_true',
514 help='Only create empty toolchain')
517 '-u', '--upload', dest='upload',
518 default=False, action='store_true',
519 help='Upload build artifacts.')
522 '--skip-gcc', dest='skip_gcc',
523 default=False, action='store_true',
524 help='Skip building GCC components.')
526 options, args = parser.parse_args(argv)
527 if options.buildbot or options.upload:
528 version = os.environ['BUILDBOT_REVISION']
530 if options.clobber or options.empty:
533 if options.sync or options.buildbot:
536 CreateBasicToolchain()
537 if not options.empty and not options.skip_gcc:
539 ConfigureAndBuildGCC(options.clobber)
540 ConfigureAndBuildCXX(options.clobber)
542 ConfigureAndBuildBionic()
544 # Can not test on buildot until sel_ldr, irt, etc.. are built
545 if not options.buildbot:
546 ConfigureAndBuildTests(clobber=False)
548 if options.buildbot or options.upload:
549 zipname = 'naclsdk_linux_arm_bionic.tgz'
550 ArchiveAndUpload(version, zipname, 'linux_x86_bionic')
553 if __name__ == '__main__':
554 sys.exit(main(sys.argv))