2 # Build many configurations of glibc.
3 # Copyright (C) 2016-2020 Free Software Foundation, Inc.
4 # This file is part of the GNU C Library.
6 # The GNU C Library is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU Lesser General Public
8 # License as published by the Free Software Foundation; either
9 # version 2.1 of the License, or (at your option) any later version.
11 # The GNU C Library is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # Lesser General Public License for more details.
16 # You should have received a copy of the GNU Lesser General Public
17 # License along with the GNU C Library; if not, see
18 # <https://www.gnu.org/licenses/>.
20 """Build many configurations of glibc.
22 This script takes as arguments a directory name (containing a src
23 subdirectory with sources of the relevant toolchain components) and a
24 description of what to do: 'checkout', to check out sources into that
25 directory, 'bot-cycle', to run a series of checkout and build steps,
26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
27 libraries required by the toolchain, 'compilers', to build
28 cross-compilers for various configurations, or 'glibcs', to build
29 glibc for various configurations and run the compilation parts of the
30 testsuite. Subsequent arguments name the versions of components to
31 check out (<component>-<version), for 'checkout', or, for actions
32 other than 'checkout' and 'bot-cycle', name configurations for which
33 compilers or glibc are to be built.
35 The 'list-compilers' command prints the name of each available
36 compiler configuration, without building anything. The 'list-glibcs'
37 command prints the name of each glibc compiler configuration, followed
38 by the space, followed by the name of the compiler configuration used
39 for building this glibc variant.
45 import email.mime.text
61 class _CompletedProcess:
62 def __init__(self, args, returncode, stdout=None, stderr=None):
64 self.returncode = returncode
68 def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
69 assert(timeout is None)
70 with subprocess.Popen(*popenargs, **kwargs) as process:
72 stdout, stderr = process.communicate(input)
77 returncode = process.poll()
78 if check and returncode:
79 raise subprocess.CalledProcessError(returncode, popenargs)
80 return _CompletedProcess(popenargs, returncode, stdout, stderr)
85 class Context(object):
86 """The global state associated with builds in a given directory."""
88 def __init__(self, topdir, parallelism, keep, replace_sources, strip,
89 full_gcc, action, shallow=False):
90 """Initialize the context."""
92 self.parallelism = parallelism
94 self.replace_sources = replace_sources
96 self.full_gcc = full_gcc
97 self.shallow = shallow
98 self.srcdir = os.path.join(topdir, 'src')
99 self.versions_json = os.path.join(self.srcdir, 'versions.json')
100 self.build_state_json = os.path.join(topdir, 'build-state.json')
101 self.bot_config_json = os.path.join(topdir, 'bot-config.json')
102 self.installdir = os.path.join(topdir, 'install')
103 self.host_libraries_installdir = os.path.join(self.installdir,
105 self.builddir = os.path.join(topdir, 'build')
106 self.logsdir = os.path.join(topdir, 'logs')
107 self.logsdir_old = os.path.join(topdir, 'logs-old')
108 self.makefile = os.path.join(self.builddir, 'Makefile')
109 self.wrapper = os.path.join(self.builddir, 'wrapper')
110 self.save_logs = os.path.join(self.builddir, 'save-logs')
111 self.script_text = self.get_script_text()
112 if action not in ('checkout', 'list-compilers', 'list-glibcs'):
113 self.build_triplet = self.get_build_triplet()
114 self.glibc_version = self.get_glibc_version()
116 self.glibc_configs = {}
117 self.makefile_pieces = ['.PHONY: all\n']
118 self.add_all_configs()
119 self.load_versions_json()
120 self.load_build_state_json()
121 self.status_log_list = []
122 self.email_warning = False
124 def get_script_text(self):
125 """Return the text of this script."""
126 with open(sys.argv[0], 'r') as f:
130 """Re-execute this script with the same arguments."""
132 os.execv(sys.executable, [sys.executable] + sys.argv)
134 def get_build_triplet(self):
135 """Determine the build triplet with config.guess."""
136 config_guess = os.path.join(self.component_srcdir('gcc'),
138 cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
139 check=True, universal_newlines=True).stdout
140 return cg_out.rstrip()
142 def get_glibc_version(self):
143 """Determine the glibc version number (major.minor)."""
144 version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
145 with open(version_h, 'r') as f:
146 lines = f.readlines()
147 starttext = '#define VERSION "'
149 if l.startswith(starttext):
150 l = l[len(starttext):]
152 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
153 return '%s.%s' % m.group(1, 2)
154 print('error: could not determine glibc version')
157 def add_all_configs(self):
158 """Add all known glibc build configurations."""
159 self.add_config(arch='aarch64',
161 extra_glibcs=[{'variant': 'disable-multi-arch',
162 'cfg': ['--disable-multi-arch']}])
163 self.add_config(arch='aarch64_be',
165 self.add_config(arch='alpha',
167 self.add_config(arch='arm',
168 os_name='linux-gnueabi',
169 extra_glibcs=[{'variant': 'v4t',
170 'ccopts': '-march=armv4t'}])
171 self.add_config(arch='armeb',
172 os_name='linux-gnueabi')
173 self.add_config(arch='armeb',
174 os_name='linux-gnueabi',
176 gcc_cfg=['--with-arch=armv7-a'])
177 self.add_config(arch='arm',
178 os_name='linux-gnueabihf',
179 gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
180 extra_glibcs=[{'variant': 'v7a',
181 'ccopts': '-march=armv7-a -mfpu=vfpv3'},
182 {'variant': 'v7a-disable-multi-arch',
183 'ccopts': '-march=armv7-a -mfpu=vfpv3',
184 'cfg': ['--disable-multi-arch']}])
185 self.add_config(arch='armeb',
186 os_name='linux-gnueabihf',
187 gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
188 self.add_config(arch='armeb',
189 os_name='linux-gnueabihf',
191 gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
193 self.add_config(arch='csky',
194 os_name='linux-gnuabiv2',
196 gcc_cfg=['--disable-multilib'])
197 self.add_config(arch='csky',
198 os_name='linux-gnuabiv2',
199 gcc_cfg=['--with-float=hard', '--disable-multilib'])
200 self.add_config(arch='hppa',
202 self.add_config(arch='i686',
204 self.add_config(arch='ia64',
206 first_gcc_cfg=['--with-system-libunwind'])
207 self.add_config(arch='m68k',
209 gcc_cfg=['--disable-multilib'])
210 self.add_config(arch='m68k',
213 gcc_cfg=['--with-arch=cf', '--disable-multilib'])
214 self.add_config(arch='m68k',
216 variant='coldfire-soft',
217 gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
218 '--disable-multilib'])
219 self.add_config(arch='microblaze',
221 gcc_cfg=['--disable-multilib'])
222 self.add_config(arch='microblazeel',
224 gcc_cfg=['--disable-multilib'])
225 self.add_config(arch='mips64',
227 gcc_cfg=['--with-mips-plt'],
228 glibcs=[{'variant': 'n32'},
230 'ccopts': '-mabi=32'},
232 'ccopts': '-mabi=64'}])
233 self.add_config(arch='mips64',
236 gcc_cfg=['--with-mips-plt', '--with-float=soft'],
237 glibcs=[{'variant': 'n32-soft'},
240 'ccopts': '-mabi=32'},
241 {'variant': 'n64-soft',
242 'ccopts': '-mabi=64'}])
243 self.add_config(arch='mips64',
246 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
247 '--with-arch-64=mips64r2',
248 '--with-arch-32=mips32r2'],
249 glibcs=[{'variant': 'n32-nan2008'},
250 {'variant': 'nan2008',
252 'ccopts': '-mabi=32'},
253 {'variant': 'n64-nan2008',
254 'ccopts': '-mabi=64'}])
255 self.add_config(arch='mips64',
257 variant='nan2008-soft',
258 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
259 '--with-arch-64=mips64r2',
260 '--with-arch-32=mips32r2',
261 '--with-float=soft'],
262 glibcs=[{'variant': 'n32-nan2008-soft'},
263 {'variant': 'nan2008-soft',
265 'ccopts': '-mabi=32'},
266 {'variant': 'n64-nan2008-soft',
267 'ccopts': '-mabi=64'}])
268 self.add_config(arch='mips64el',
270 gcc_cfg=['--with-mips-plt'],
271 glibcs=[{'variant': 'n32'},
273 'ccopts': '-mabi=32'},
275 'ccopts': '-mabi=64'}])
276 self.add_config(arch='mips64el',
279 gcc_cfg=['--with-mips-plt', '--with-float=soft'],
280 glibcs=[{'variant': 'n32-soft'},
283 'ccopts': '-mabi=32'},
284 {'variant': 'n64-soft',
285 'ccopts': '-mabi=64'}])
286 self.add_config(arch='mips64el',
289 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
290 '--with-arch-64=mips64r2',
291 '--with-arch-32=mips32r2'],
292 glibcs=[{'variant': 'n32-nan2008'},
293 {'variant': 'nan2008',
295 'ccopts': '-mabi=32'},
296 {'variant': 'n64-nan2008',
297 'ccopts': '-mabi=64'}])
298 self.add_config(arch='mips64el',
300 variant='nan2008-soft',
301 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
302 '--with-arch-64=mips64r2',
303 '--with-arch-32=mips32r2',
304 '--with-float=soft'],
305 glibcs=[{'variant': 'n32-nan2008-soft'},
306 {'variant': 'nan2008-soft',
308 'ccopts': '-mabi=32'},
309 {'variant': 'n64-nan2008-soft',
310 'ccopts': '-mabi=64'}])
311 self.add_config(arch='mipsisa64r6el',
313 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
314 '--with-arch-64=mips64r6',
315 '--with-arch-32=mips32r6',
316 '--with-float=hard'],
317 glibcs=[{'variant': 'n32'},
318 {'arch': 'mipsisa32r6el',
319 'ccopts': '-mabi=32'},
321 'ccopts': '-mabi=64'}])
322 self.add_config(arch='nios2',
324 self.add_config(arch='powerpc',
326 gcc_cfg=['--disable-multilib', '--enable-secureplt'],
327 extra_glibcs=[{'variant': 'power4',
328 'ccopts': '-mcpu=power4',
329 'cfg': ['--with-cpu=power4']}])
330 self.add_config(arch='powerpc',
333 gcc_cfg=['--disable-multilib', '--with-float=soft',
334 '--enable-secureplt'])
335 self.add_config(arch='powerpc64',
337 gcc_cfg=['--disable-multilib', '--enable-secureplt'])
338 self.add_config(arch='powerpc64le',
340 gcc_cfg=['--disable-multilib', '--enable-secureplt'])
341 self.add_config(arch='riscv64',
343 variant='rv64imac-lp64',
344 gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
345 '--disable-multilib'])
346 self.add_config(arch='riscv64',
348 variant='rv64imafdc-lp64',
349 gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
350 '--disable-multilib'])
351 self.add_config(arch='riscv64',
353 variant='rv64imafdc-lp64d',
354 gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
355 '--disable-multilib'])
356 self.add_config(arch='s390x',
359 {'arch': 's390', 'ccopts': '-m31'}])
360 self.add_config(arch='sh3',
362 self.add_config(arch='sh3eb',
364 self.add_config(arch='sh4',
366 self.add_config(arch='sh4eb',
368 self.add_config(arch='sh4',
371 gcc_cfg=['--without-fp'])
372 self.add_config(arch='sh4eb',
375 gcc_cfg=['--without-fp'])
376 self.add_config(arch='sparc64',
380 'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
381 extra_glibcs=[{'variant': 'leon3',
383 'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
384 {'variant': 'disable-multi-arch',
385 'cfg': ['--disable-multi-arch']},
386 {'variant': 'disable-multi-arch',
388 'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
389 'cfg': ['--disable-multi-arch']}])
390 self.add_config(arch='x86_64',
392 gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
394 {'variant': 'x32', 'ccopts': '-mx32'},
395 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
396 extra_glibcs=[{'variant': 'disable-multi-arch',
397 'cfg': ['--disable-multi-arch']},
398 {'variant': 'enable-obsolete',
399 'cfg': ['--enable-obsolete-rpc']},
400 {'variant': 'static-pie',
401 'cfg': ['--enable-static-pie']},
402 {'variant': 'x32-static-pie',
404 'cfg': ['--enable-static-pie']},
405 {'variant': 'static-pie',
407 'ccopts': '-m32 -march=i686',
408 'cfg': ['--enable-static-pie']},
409 {'variant': 'disable-multi-arch',
411 'ccopts': '-m32 -march=i686',
412 'cfg': ['--disable-multi-arch']},
413 {'variant': 'enable-obsolete',
415 'ccopts': '-m32 -march=i686',
416 'cfg': ['--enable-obsolete-rpc']},
418 'ccopts': '-m32 -march=i486'},
420 'ccopts': '-m32 -march=i586'}])
422 def add_config(self, **args):
423 """Add an individual build configuration."""
424 cfg = Config(self, **args)
425 if cfg.name in self.configs:
426 print('error: duplicate config %s' % cfg.name)
428 self.configs[cfg.name] = cfg
429 for c in cfg.all_glibcs:
430 if c.name in self.glibc_configs:
431 print('error: duplicate glibc config %s' % c.name)
433 self.glibc_configs[c.name] = c
435 def component_srcdir(self, component):
436 """Return the source directory for a given component, e.g. gcc."""
437 return os.path.join(self.srcdir, component)
439 def component_builddir(self, action, config, component, subconfig=None):
440 """Return the directory to use for a build."""
443 assert subconfig is None
444 return os.path.join(self.builddir, action, component)
445 if subconfig is None:
446 return os.path.join(self.builddir, action, config, component)
448 # glibc build as part of compiler build.
449 return os.path.join(self.builddir, action, config, component,
452 def compiler_installdir(self, config):
453 """Return the directory in which to install a compiler."""
454 return os.path.join(self.installdir, 'compilers', config)
456 def compiler_bindir(self, config):
457 """Return the directory in which to find compiler binaries."""
458 return os.path.join(self.compiler_installdir(config), 'bin')
460 def compiler_sysroot(self, config):
461 """Return the sysroot directory for a compiler."""
462 return os.path.join(self.compiler_installdir(config), 'sysroot')
464 def glibc_installdir(self, config):
465 """Return the directory in which to install glibc."""
466 return os.path.join(self.installdir, 'glibcs', config)
468 def run_builds(self, action, configs):
469 """Run the requested builds."""
470 if action == 'checkout':
471 self.checkout(configs)
473 if action == 'bot-cycle':
475 print('error: configurations specified for bot-cycle')
481 print('error: configurations specified for bot')
485 if action in ('host-libraries', 'list-compilers',
486 'list-glibcs') and configs:
487 print('error: configurations specified for ' + action)
489 if action == 'list-compilers':
490 for name in sorted(self.configs.keys()):
493 if action == 'list-glibcs':
494 for config in sorted(self.glibc_configs.values(),
495 key=lambda c: c.name):
496 print(config.name, config.compiler.name)
498 self.clear_last_build_state(action)
499 build_time = datetime.datetime.utcnow()
500 if action == 'host-libraries':
501 build_components = ('gmp', 'mpfr', 'mpc')
504 self.build_host_libraries()
505 elif action == 'compilers':
506 build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
508 old_components = ('gmp', 'mpfr', 'mpc')
509 old_versions = self.build_state['host-libraries']['build-versions']
510 self.build_compilers(configs)
512 build_components = ('glibc',)
513 old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
514 'mig', 'gnumach', 'hurd')
515 old_versions = self.build_state['compilers']['build-versions']
516 if action == 'update-syscalls':
517 self.update_syscalls(configs)
519 self.build_glibcs(configs)
523 # Partial build, do not update stored state.
526 for k in build_components:
527 if k in self.versions:
528 build_versions[k] = {'version': self.versions[k]['version'],
529 'revision': self.versions[k]['revision']}
530 for k in old_components:
531 if k in old_versions:
532 build_versions[k] = {'version': old_versions[k]['version'],
533 'revision': old_versions[k]['revision']}
534 self.update_build_state(action, build_time, build_versions)
537 def remove_dirs(*args):
538 """Remove directories and their contents if they exist."""
540 shutil.rmtree(dir, ignore_errors=True)
543 def remove_recreate_dirs(*args):
544 """Remove directories if they exist, and create them as empty."""
545 Context.remove_dirs(*args)
547 os.makedirs(dir, exist_ok=True)
549 def add_makefile_cmdlist(self, target, cmdlist, logsdir):
550 """Add makefile text for a list of commands."""
551 commands = cmdlist.makefile_commands(self.wrapper, logsdir)
552 self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
553 (target, target, target, commands))
554 self.status_log_list.extend(cmdlist.status_logs(logsdir))
556 def write_files(self):
557 """Write out the Makefile and wrapper script."""
558 mftext = ''.join(self.makefile_pieces)
559 with open(self.makefile, 'w') as f:
569 'prev_status=$prev_base-status.txt\n'
570 'this_status=$this_base-status.txt\n'
571 'this_log=$this_base-log.txt\n'
572 'date > "$this_log"\n'
573 'echo >> "$this_log"\n'
574 'echo "Description: $desc" >> "$this_log"\n'
575 'printf "%s" "Command:" >> "$this_log"\n'
576 'for word in "$@"; do\n'
577 ' if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
578 ' printf " %s" "$word"\n'
581 ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
584 'done >> "$this_log"\n'
585 'echo >> "$this_log"\n'
586 'echo "Directory: $dir" >> "$this_log"\n'
587 'echo "Path addition: $path" >> "$this_log"\n'
588 'echo >> "$this_log"\n'
591 ' echo >> "$this_log"\n'
592 ' echo "$1: $desc" > "$this_status"\n'
593 ' echo "$1: $desc" >> "$this_log"\n'
594 ' echo >> "$this_log"\n'
595 ' date >> "$this_log"\n'
596 ' echo "$1: $desc"\n'
601 ' if [ "$1" != "0" ]; then\n'
602 ' record_status FAIL\n'
605 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
606 ' record_status UNRESOLVED\n'
608 'if [ "$dir" ]; then\n'
610 ' check_error "$?"\n'
612 'if [ "$path" ]; then\n'
613 ' PATH=$path:$PATH\n'
615 '"$@" < /dev/null >> "$this_log" 2>&1\n'
617 'record_status PASS\n')
618 with open(self.wrapper, 'w') as f:
619 f.write(wrapper_text)
621 mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
622 stat.S_IROTH|stat.S_IXOTH)
623 os.chmod(self.wrapper, mode_exec)
626 'if ! [ -f tests.sum ]; then\n'
627 ' echo "No test summary available."\n'
632 ' echo "Contents of $1:"\n'
636 ' echo "End of contents of $1."\n'
639 'save_file tests.sum\n'
640 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
641 'for t in $non_pass_tests; do\n'
642 ' if [ -f "$t.out" ]; then\n'
643 ' save_file "$t.out"\n'
646 with open(self.save_logs, 'w') as f:
647 f.write(save_logs_text)
648 os.chmod(self.save_logs, mode_exec)
651 """Do the actual build."""
652 cmd = ['make', '-j%d' % self.parallelism]
653 subprocess.run(cmd, cwd=self.builddir, check=True)
655 def build_host_libraries(self):
656 """Build the host libraries."""
657 installdir = self.host_libraries_installdir
658 builddir = os.path.join(self.builddir, 'host-libraries')
659 logsdir = os.path.join(self.logsdir, 'host-libraries')
660 self.remove_recreate_dirs(installdir, builddir, logsdir)
661 cmdlist = CommandList('host-libraries', self.keep)
662 self.build_host_library(cmdlist, 'gmp')
663 self.build_host_library(cmdlist, 'mpfr',
664 ['--with-gmp=%s' % installdir])
665 self.build_host_library(cmdlist, 'mpc',
666 ['--with-gmp=%s' % installdir,
667 '--with-mpfr=%s' % installdir])
668 cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
669 self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
671 def build_host_library(self, cmdlist, lib, extra_opts=None):
672 """Build one host library."""
673 srcdir = self.component_srcdir(lib)
674 builddir = self.component_builddir('host-libraries', None, lib)
675 installdir = self.host_libraries_installdir
676 cmdlist.push_subdesc(lib)
677 cmdlist.create_use_dir(builddir)
678 cfg_cmd = [os.path.join(srcdir, 'configure'),
679 '--prefix=%s' % installdir,
682 cfg_cmd.extend (extra_opts)
683 cmdlist.add_command('configure', cfg_cmd)
684 cmdlist.add_command('build', ['make'])
685 cmdlist.add_command('check', ['make', 'check'])
686 cmdlist.add_command('install', ['make', 'install'])
687 cmdlist.cleanup_dir()
688 cmdlist.pop_subdesc()
690 def build_compilers(self, configs):
691 """Build the compilers."""
693 self.remove_dirs(os.path.join(self.builddir, 'compilers'))
694 self.remove_dirs(os.path.join(self.installdir, 'compilers'))
695 self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
696 configs = sorted(self.configs.keys())
698 self.configs[c].build()
700 def build_glibcs(self, configs):
701 """Build the glibcs."""
703 self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
704 self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
705 self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
706 configs = sorted(self.glibc_configs.keys())
708 self.glibc_configs[c].build()
710 def update_syscalls(self, configs):
711 """Update the glibc syscall lists."""
713 self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
714 self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
715 configs = sorted(self.glibc_configs.keys())
717 self.glibc_configs[c].update_syscalls()
719 def load_versions_json(self):
720 """Load information about source directory versions."""
721 if not os.access(self.versions_json, os.F_OK):
724 with open(self.versions_json, 'r') as f:
725 self.versions = json.load(f)
727 def store_json(self, data, filename):
728 """Store information in a JSON file."""
729 filename_tmp = filename + '.tmp'
730 with open(filename_tmp, 'w') as f:
731 json.dump(data, f, indent=2, sort_keys=True)
732 os.rename(filename_tmp, filename)
734 def store_versions_json(self):
735 """Store information about source directory versions."""
736 self.store_json(self.versions, self.versions_json)
738 def set_component_version(self, component, version, explicit, revision):
739 """Set the version information for a component."""
740 self.versions[component] = {'version': version,
741 'explicit': explicit,
742 'revision': revision}
743 self.store_versions_json()
745 def checkout(self, versions):
746 """Check out the desired component versions."""
747 default_versions = {'binutils': 'vcs-2.34',
749 'glibc': 'vcs-mainline',
754 'mig': 'vcs-mainline',
755 'gnumach': 'vcs-mainline',
756 'hurd': 'vcs-mainline'}
758 explicit_versions = {}
761 for k in default_versions.keys():
765 if k in use_versions:
766 print('error: multiple versions for %s' % k)
769 explicit_versions[k] = True
773 print('error: unknown component in %s' % v)
775 for k in default_versions.keys():
776 if k not in use_versions:
777 if k in self.versions and self.versions[k]['explicit']:
778 use_versions[k] = self.versions[k]['version']
779 explicit_versions[k] = True
781 use_versions[k] = default_versions[k]
782 explicit_versions[k] = False
783 os.makedirs(self.srcdir, exist_ok=True)
784 for k in sorted(default_versions.keys()):
785 update = os.access(self.component_srcdir(k), os.F_OK)
788 k in self.versions and
789 v != self.versions[k]['version']):
790 if not self.replace_sources:
791 print('error: version of %s has changed from %s to %s, '
792 'use --replace-sources to check out again' %
793 (k, self.versions[k]['version'], v))
795 shutil.rmtree(self.component_srcdir(k))
797 if v.startswith('vcs-'):
798 revision = self.checkout_vcs(k, v[4:], update)
800 self.checkout_tar(k, v, update)
802 self.set_component_version(k, v, explicit_versions[k], revision)
803 if self.get_script_text() != self.script_text:
804 # Rerun the checkout process in case the updated script
805 # uses different default versions or new components.
808 def checkout_vcs(self, component, version, update):
809 """Check out the given version of the given component from version
810 control. Return a revision identifier."""
811 if component == 'binutils':
812 git_url = 'git://sourceware.org/git/binutils-gdb.git'
813 if version == 'mainline':
814 git_branch = 'master'
816 trans = str.maketrans({'.': '_'})
817 git_branch = 'binutils-%s-branch' % version.translate(trans)
818 return self.git_checkout(component, git_url, git_branch, update)
819 elif component == 'gcc':
820 if version == 'mainline':
823 branch = 'releases/gcc-%s' % version
824 return self.gcc_checkout(branch, update)
825 elif component == 'glibc':
826 git_url = 'git://sourceware.org/git/glibc.git'
827 if version == 'mainline':
828 git_branch = 'master'
830 git_branch = 'release/%s/master' % version
831 r = self.git_checkout(component, git_url, git_branch, update)
832 self.fix_glibc_timestamps()
834 elif component == 'gnumach':
835 git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
836 git_branch = 'master'
837 r = self.git_checkout(component, git_url, git_branch, update)
838 subprocess.run(['autoreconf', '-i'],
839 cwd=self.component_srcdir(component), check=True)
841 elif component == 'mig':
842 git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
843 git_branch = 'master'
844 r = self.git_checkout(component, git_url, git_branch, update)
845 subprocess.run(['autoreconf', '-i'],
846 cwd=self.component_srcdir(component), check=True)
848 elif component == 'hurd':
849 git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
850 git_branch = 'master'
851 r = self.git_checkout(component, git_url, git_branch, update)
852 subprocess.run(['autoconf'],
853 cwd=self.component_srcdir(component), check=True)
856 print('error: component %s coming from VCS' % component)
859 def git_checkout(self, component, git_url, git_branch, update):
860 """Check out a component from git. Return a commit identifier."""
862 subprocess.run(['git', 'remote', 'prune', 'origin'],
863 cwd=self.component_srcdir(component), check=True)
864 if self.replace_sources:
865 subprocess.run(['git', 'clean', '-dxfq'],
866 cwd=self.component_srcdir(component), check=True)
867 subprocess.run(['git', 'pull', '-q'],
868 cwd=self.component_srcdir(component), check=True)
871 depth_arg = ('--depth', '1')
874 subprocess.run(['git', 'clone', '-q', '-b', git_branch,
876 self.component_srcdir(component)], check=True)
877 r = subprocess.run(['git', 'rev-parse', 'HEAD'],
878 cwd=self.component_srcdir(component),
879 stdout=subprocess.PIPE,
880 check=True, universal_newlines=True).stdout
883 def fix_glibc_timestamps(self):
884 """Fix timestamps in a glibc checkout."""
885 # Ensure that builds do not try to regenerate generated files
886 # in the source tree.
887 srcdir = self.component_srcdir('glibc')
888 # These files have Makefile dependencies to regenerate them in
889 # the source tree that may be active during a normal build.
890 # Some other files have such dependencies but do not need to
891 # be touched because nothing in a build depends on the files
893 for f in ('sysdeps/mach/hurd/bits/errno.h',):
894 to_touch = os.path.join(srcdir, f)
895 subprocess.run(['touch', '-c', to_touch], check=True)
896 for dirpath, dirnames, filenames in os.walk(srcdir):
898 if (f == 'configure' or
899 f == 'preconfigure' or
900 f.endswith('-kw.h')):
901 to_touch = os.path.join(dirpath, f)
902 subprocess.run(['touch', to_touch], check=True)
904 def gcc_checkout(self, branch, update):
905 """Check out GCC from git. Return the commit identifier."""
906 if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
908 if not self.replace_sources:
909 print('error: GCC has moved from SVN to git, use '
910 '--replace-sources to check out again')
912 shutil.rmtree(self.component_srcdir('gcc'))
915 self.git_checkout('gcc', 'git://gcc.gnu.org/git/gcc.git',
917 subprocess.run(['contrib/gcc_update', '--silent'],
918 cwd=self.component_srcdir('gcc'), check=True)
919 r = subprocess.run(['git', 'rev-parse', 'HEAD'],
920 cwd=self.component_srcdir('gcc'),
921 stdout=subprocess.PIPE,
922 check=True, universal_newlines=True).stdout
925 def checkout_tar(self, component, version, update):
926 """Check out the given version of the given component from a
930 url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
931 'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
932 'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
933 'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
934 'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
935 'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
936 'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
937 'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
938 'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
939 if component not in url_map:
940 print('error: component %s coming from tarball' % component)
942 version_major = version.split('.')[0]
943 url = url_map[component] % {'version': version, 'major': version_major}
944 filename = os.path.join(self.srcdir, url.split('/')[-1])
945 response = urllib.request.urlopen(url)
946 data = response.read()
947 with open(filename, 'wb') as f:
949 subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
951 os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
952 self.component_srcdir(component))
955 def load_build_state_json(self):
956 """Load information about the state of previous builds."""
957 if os.access(self.build_state_json, os.F_OK):
958 with open(self.build_state_json, 'r') as f:
959 self.build_state = json.load(f)
961 self.build_state = {}
962 for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
963 if k not in self.build_state:
964 self.build_state[k] = {}
965 if 'build-time' not in self.build_state[k]:
966 self.build_state[k]['build-time'] = ''
967 if 'build-versions' not in self.build_state[k]:
968 self.build_state[k]['build-versions'] = {}
969 if 'build-results' not in self.build_state[k]:
970 self.build_state[k]['build-results'] = {}
971 if 'result-changes' not in self.build_state[k]:
972 self.build_state[k]['result-changes'] = {}
973 if 'ever-passed' not in self.build_state[k]:
974 self.build_state[k]['ever-passed'] = []
976 def store_build_state_json(self):
977 """Store information about the state of previous builds."""
978 self.store_json(self.build_state, self.build_state_json)
980 def clear_last_build_state(self, action):
981 """Clear information about the state of part of the build."""
982 # We clear the last build time and versions when starting a
983 # new build. The results of the last build are kept around,
984 # as comparison is still meaningful if this build is aborted
985 # and a new one started.
986 self.build_state[action]['build-time'] = ''
987 self.build_state[action]['build-versions'] = {}
988 self.store_build_state_json()
990 def update_build_state(self, action, build_time, build_versions):
991 """Update the build state after a build."""
992 build_time = build_time.replace(microsecond=0)
993 self.build_state[action]['build-time'] = str(build_time)
994 self.build_state[action]['build-versions'] = build_versions
996 for log in self.status_log_list:
997 with open(log, 'r') as f:
999 log_text = log_text.rstrip()
1000 m = re.fullmatch('([A-Z]+): (.*)', log_text)
1002 test_name = m.group(2)
1003 assert test_name not in build_results
1004 build_results[test_name] = result
1005 old_build_results = self.build_state[action]['build-results']
1006 self.build_state[action]['build-results'] = build_results
1008 all_tests = set(old_build_results.keys()) | set(build_results.keys())
1010 if t in old_build_results:
1011 old_res = old_build_results[t]
1013 old_res = '(New test)'
1014 if t in build_results:
1015 new_res = build_results[t]
1017 new_res = '(Test removed)'
1018 if old_res != new_res:
1019 result_changes[t] = '%s -> %s' % (old_res, new_res)
1020 self.build_state[action]['result-changes'] = result_changes
1021 old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1022 if t in build_results}
1023 new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1024 self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1026 self.store_build_state_json()
1028 def load_bot_config_json(self):
1029 """Load bot configuration."""
1030 with open(self.bot_config_json, 'r') as f:
1031 self.bot_config = json.load(f)
1033 def part_build_old(self, action, delay):
1034 """Return whether the last build for a given action was at least a
1035 given number of seconds ago, or does not have a time recorded."""
1036 old_time_str = self.build_state[action]['build-time']
1037 if not old_time_str:
1039 old_time = datetime.datetime.strptime(old_time_str,
1040 '%Y-%m-%d %H:%M:%S')
1041 new_time = datetime.datetime.utcnow()
1042 delta = new_time - old_time
1043 return delta.total_seconds() >= delay
1045 def bot_cycle(self):
1046 """Run a single round of checkout and builds."""
1047 print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1048 self.load_bot_config_json()
1049 actions = ('host-libraries', 'compilers', 'glibcs')
1050 self.bot_run_self(['--replace-sources'], 'checkout')
1051 self.load_versions_json()
1052 if self.get_script_text() != self.script_text:
1053 print('Script changed, re-execing.')
1054 # On script change, all parts of the build should be rerun.
1056 self.clear_last_build_state(a)
1058 check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1059 'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1060 'mig', 'gnumach', 'hurd'),
1061 'glibcs': ('glibc',)}
1064 build_vers = self.build_state[a]['build-versions']
1065 must_build[a] = False
1066 if not self.build_state[a]['build-time']:
1067 must_build[a] = True
1070 for c in check_components[a]:
1072 old_vers[c] = build_vers[c]
1073 new_vers[c] = {'version': self.versions[c]['version'],
1074 'revision': self.versions[c]['revision']}
1075 if new_vers == old_vers:
1076 print('Versions for %s unchanged.' % a)
1078 print('Versions changed or rebuild forced for %s.' % a)
1079 if a == 'compilers' and not self.part_build_old(
1080 a, self.bot_config['compilers-rebuild-delay']):
1081 print('Not requiring rebuild of compilers this soon.')
1083 must_build[a] = True
1084 if must_build['host-libraries']:
1085 must_build['compilers'] = True
1086 if must_build['compilers']:
1087 must_build['glibcs'] = True
1090 print('Must rebuild %s.' % a)
1091 self.clear_last_build_state(a)
1093 print('No need to rebuild %s.' % a)
1094 if os.access(self.logsdir, os.F_OK):
1095 shutil.rmtree(self.logsdir_old, ignore_errors=True)
1096 shutil.copytree(self.logsdir, self.logsdir_old)
1099 build_time = datetime.datetime.utcnow()
1100 print('Rebuilding %s at %s.' % (a, str(build_time)))
1101 self.bot_run_self([], a)
1102 self.load_build_state_json()
1103 self.bot_build_mail(a, build_time)
1104 print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1106 def bot_build_mail(self, action, build_time):
1107 """Send email with the results of a build."""
1108 if not ('email-from' in self.bot_config and
1109 'email-server' in self.bot_config and
1110 'email-subject' in self.bot_config and
1111 'email-to' in self.bot_config):
1112 if not self.email_warning:
1113 print("Email not configured, not sending.")
1114 self.email_warning = True
1117 build_time = build_time.replace(microsecond=0)
1118 subject = (self.bot_config['email-subject'] %
1120 'build-time': str(build_time)})
1121 results = self.build_state[action]['build-results']
1122 changes = self.build_state[action]['result-changes']
1123 ever_passed = set(self.build_state[action]['ever-passed'])
1124 versions = self.build_state[action]['build-versions']
1125 new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1126 all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1127 all_fails = {k for k in results if results[k] == 'FAIL'}
1129 new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1130 new_reg_text = ('New regressions:\n\n%s\n\n' %
1131 '\n'.join(new_reg_list))
1135 all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1136 all_reg_text = ('All regressions:\n\n%s\n\n' %
1137 '\n'.join(all_reg_list))
1141 all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1142 all_fail_text = ('All failures:\n\n%s\n\n' %
1143 '\n'.join(all_fail_list))
1147 changes_list = sorted(changes.keys())
1148 changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1149 changes_text = ('All changed results:\n\n%s\n\n' %
1150 '\n'.join(changes_list))
1153 results_text = (new_reg_text + all_reg_text + all_fail_text +
1155 if not results_text:
1156 results_text = 'Clean build with unchanged results.\n\n'
1157 versions_list = sorted(versions.keys())
1158 versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1159 versions[k]['revision'])
1160 for k in versions_list]
1161 versions_text = ('Component versions for this build:\n\n%s\n' %
1162 '\n'.join(versions_list))
1163 body_text = results_text + versions_text
1164 msg = email.mime.text.MIMEText(body_text)
1165 msg['Subject'] = subject
1166 msg['From'] = self.bot_config['email-from']
1167 msg['To'] = self.bot_config['email-to']
1168 msg['Message-ID'] = email.utils.make_msgid()
1169 msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1170 with smtplib.SMTP(self.bot_config['email-server']) as s:
1173 def bot_run_self(self, opts, action, check=True):
1174 """Run a copy of this script with given options."""
1175 cmd = [sys.executable, sys.argv[0], '--keep=none',
1176 '-j%d' % self.parallelism]
1178 cmd.append('--full-gcc')
1180 cmd.extend([self.topdir, action])
1182 subprocess.run(cmd, check=check)
1185 """Run repeated rounds of checkout and builds."""
1187 self.load_bot_config_json()
1188 if not self.bot_config['run']:
1189 print('Bot exiting by request.')
1191 self.bot_run_self([], 'bot-cycle', check=False)
1192 self.load_bot_config_json()
1193 if not self.bot_config['run']:
1194 print('Bot exiting by request.')
1196 time.sleep(self.bot_config['delay'])
1197 if self.get_script_text() != self.script_text:
1198 print('Script changed, bot re-execing.')
1201 class LinuxHeadersPolicyForBuild(object):
1202 """Names and directories for installing Linux headers. Build variant."""
1204 def __init__(self, config):
1205 self.arch = config.arch
1206 self.srcdir = config.ctx.component_srcdir('linux')
1207 self.builddir = config.component_builddir('linux')
1208 self.headers_dir = os.path.join(config.sysroot, 'usr')
1210 class LinuxHeadersPolicyForUpdateSyscalls(object):
1211 """Names and directories for Linux headers. update-syscalls variant."""
1213 def __init__(self, glibc, headers_dir):
1214 self.arch = glibc.compiler.arch
1215 self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1216 self.builddir = glibc.ctx.component_builddir(
1217 'update-syscalls', glibc.name, 'build-linux')
1218 self.headers_dir = headers_dir
1220 def install_linux_headers(policy, cmdlist):
1221 """Install Linux kernel headers."""
1222 arch_map = {'aarch64': 'arm64',
1233 'microblaze': 'microblaze',
1236 'powerpc': 'powerpc',
1245 if policy.arch.startswith(k):
1246 linux_arch = arch_map[k]
1248 assert linux_arch is not None
1249 cmdlist.push_subdesc('linux')
1250 cmdlist.create_use_dir(policy.builddir)
1251 cmdlist.add_command('install-headers',
1252 ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1253 'ARCH=%s' % linux_arch,
1254 'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1256 cmdlist.cleanup_dir()
1257 cmdlist.pop_subdesc()
1259 class Config(object):
1260 """A configuration for building a compiler and associated libraries."""
1262 def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1263 first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1264 """Initialize a Config object."""
1268 self.variant = variant
1270 self.name = '%s-%s' % (arch, os_name)
1272 self.name = '%s-%s-%s' % (arch, os_name, variant)
1273 self.triplet = '%s-glibc-%s' % (arch, os_name)
1277 self.gcc_cfg = gcc_cfg
1278 if first_gcc_cfg is None:
1279 self.first_gcc_cfg = []
1281 self.first_gcc_cfg = first_gcc_cfg
1283 glibcs = [{'variant': variant}]
1284 if extra_glibcs is None:
1286 glibcs = [Glibc(self, **g) for g in glibcs]
1287 extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1288 self.all_glibcs = glibcs + extra_glibcs
1289 self.compiler_glibcs = glibcs
1290 self.installdir = ctx.compiler_installdir(self.name)
1291 self.bindir = ctx.compiler_bindir(self.name)
1292 self.sysroot = ctx.compiler_sysroot(self.name)
1293 self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1294 self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1296 def component_builddir(self, component):
1297 """Return the directory to use for a (non-glibc) build."""
1298 return self.ctx.component_builddir('compilers', self.name, component)
1301 """Generate commands to build this compiler."""
1302 self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1304 cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1305 cmdlist.add_command('check-host-libraries',
1307 os.path.join(self.ctx.host_libraries_installdir,
1309 cmdlist.use_path(self.bindir)
1310 self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1312 '--disable-gdbserver',
1313 '--disable-libdecnumber',
1314 '--disable-readline',
1316 if self.os.startswith('linux'):
1317 install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1318 self.build_gcc(cmdlist, True)
1319 if self.os == 'gnu':
1320 self.install_gnumach_headers(cmdlist)
1321 self.build_cross_tool(cmdlist, 'mig', 'mig')
1322 self.install_hurd_headers(cmdlist)
1323 for g in self.compiler_glibcs:
1324 cmdlist.push_subdesc('glibc')
1325 cmdlist.push_subdesc(g.name)
1326 g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1327 cmdlist.pop_subdesc()
1328 cmdlist.pop_subdesc()
1329 self.build_gcc(cmdlist, False)
1330 cmdlist.add_command('done', ['touch',
1331 os.path.join(self.installdir, 'ok')])
1332 self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1335 def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1336 """Build one cross tool."""
1337 srcdir = self.ctx.component_srcdir(tool_src)
1338 builddir = self.component_builddir(tool_build)
1339 cmdlist.push_subdesc(tool_build)
1340 cmdlist.create_use_dir(builddir)
1341 cfg_cmd = [os.path.join(srcdir, 'configure'),
1342 '--prefix=%s' % self.installdir,
1343 '--build=%s' % self.ctx.build_triplet,
1344 '--host=%s' % self.ctx.build_triplet,
1345 '--target=%s' % self.triplet,
1346 '--with-sysroot=%s' % self.sysroot]
1348 cfg_cmd.extend(extra_opts)
1349 cmdlist.add_command('configure', cfg_cmd)
1350 cmdlist.add_command('build', ['make'])
1351 # Parallel "make install" for GCC has race conditions that can
1352 # cause it to fail; see
1353 # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>. Such
1354 # problems are not known for binutils, but doing the
1355 # installation in parallel within a particular toolchain build
1356 # (as opposed to installation of one toolchain from
1357 # build-many-glibcs.py running in parallel to the installation
1358 # of other toolchains being built) is not known to be
1359 # significantly beneficial, so it is simplest just to disable
1360 # parallel install for cross tools here.
1361 cmdlist.add_command('install', ['make', '-j1', 'install'])
1362 cmdlist.cleanup_dir()
1363 cmdlist.pop_subdesc()
1365 def install_gnumach_headers(self, cmdlist):
1366 """Install GNU Mach headers."""
1367 srcdir = self.ctx.component_srcdir('gnumach')
1368 builddir = self.component_builddir('gnumach')
1369 cmdlist.push_subdesc('gnumach')
1370 cmdlist.create_use_dir(builddir)
1371 cmdlist.add_command('configure',
1372 [os.path.join(srcdir, 'configure'),
1373 '--build=%s' % self.ctx.build_triplet,
1374 '--host=%s' % self.triplet,
1376 'CC=%s-gcc -nostdlib' % self.triplet])
1377 cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1379 cmdlist.cleanup_dir()
1380 cmdlist.pop_subdesc()
1382 def install_hurd_headers(self, cmdlist):
1383 """Install Hurd headers."""
1384 srcdir = self.ctx.component_srcdir('hurd')
1385 builddir = self.component_builddir('hurd')
1386 cmdlist.push_subdesc('hurd')
1387 cmdlist.create_use_dir(builddir)
1388 cmdlist.add_command('configure',
1389 [os.path.join(srcdir, 'configure'),
1390 '--build=%s' % self.ctx.build_triplet,
1391 '--host=%s' % self.triplet,
1393 '--disable-profile', '--without-parted',
1394 'CC=%s-gcc -nostdlib' % self.triplet])
1395 cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1396 'no_deps=t', 'install-headers'])
1397 cmdlist.cleanup_dir()
1398 cmdlist.pop_subdesc()
1400 def build_gcc(self, cmdlist, bootstrap):
1402 # libssp is of little relevance with glibc's own stack
1403 # checking support. libcilkrts does not support GNU/Hurd (and
1404 # has been removed in GCC 8, so --disable-libcilkrts can be
1405 # removed once glibc no longer supports building with older
1407 cfg_opts = list(self.gcc_cfg)
1408 cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1409 host_libs = self.ctx.host_libraries_installdir
1410 cfg_opts += ['--with-gmp=%s' % host_libs,
1411 '--with-mpfr=%s' % host_libs,
1412 '--with-mpc=%s' % host_libs]
1414 tool_build = 'gcc-first'
1415 # Building a static-only, C-only compiler that is
1416 # sufficient to build glibc. Various libraries and
1417 # features that may require libc headers must be disabled.
1418 # When configuring with a sysroot, --with-newlib is
1419 # required to define inhibit_libc (to stop some parts of
1420 # libgcc including libc headers); --without-headers is not
1422 cfg_opts += ['--enable-languages=c', '--disable-shared',
1423 '--disable-threads',
1424 '--disable-libatomic',
1425 '--disable-decimal-float',
1427 '--disable-libgomp',
1430 '--disable-libquadmath',
1431 '--disable-libsanitizer',
1432 '--without-headers', '--with-newlib',
1433 '--with-glibc-version=%s' % self.ctx.glibc_version
1435 cfg_opts += self.first_gcc_cfg
1438 # libsanitizer commonly breaks because of glibc header
1439 # changes, or on unusual targets. C++ pre-compiled
1440 # headers are not used during the glibc build and are
1441 # expensive to create.
1442 if not self.ctx.full_gcc:
1443 cfg_opts += ['--disable-libsanitizer',
1444 '--disable-libstdcxx-pch']
1445 langs = 'all' if self.ctx.full_gcc else 'c,c++'
1446 cfg_opts += ['--enable-languages=%s' % langs,
1447 '--enable-shared', '--enable-threads']
1448 self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1450 class GlibcPolicyDefault(object):
1451 """Build policy for glibc: common defaults."""
1453 def __init__(self, glibc):
1454 self.srcdir = glibc.ctx.component_srcdir('glibc')
1455 self.use_usr = glibc.os != 'gnu'
1456 self.prefix = '/usr' if self.use_usr else ''
1457 self.configure_args = [
1458 '--prefix=%s' % self.prefix,
1460 '--build=%s' % glibc.ctx.build_triplet,
1461 '--host=%s' % glibc.triplet,
1462 'CC=%s' % glibc.tool_name('gcc'),
1463 'CXX=%s' % glibc.tool_name('g++'),
1464 'AR=%s' % glibc.tool_name('ar'),
1465 'AS=%s' % glibc.tool_name('as'),
1466 'LD=%s' % glibc.tool_name('ld'),
1467 'NM=%s' % glibc.tool_name('nm'),
1468 'OBJCOPY=%s' % glibc.tool_name('objcopy'),
1469 'OBJDUMP=%s' % glibc.tool_name('objdump'),
1470 'RANLIB=%s' % glibc.tool_name('ranlib'),
1471 'READELF=%s' % glibc.tool_name('readelf'),
1472 'STRIP=%s' % glibc.tool_name('strip'),
1474 if glibc.os == 'gnu':
1475 self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1476 self.configure_args += glibc.cfg
1478 def configure(self, cmdlist):
1479 """Invoked to add the configure command to the command list."""
1480 cmdlist.add_command('configure',
1481 [os.path.join(self.srcdir, 'configure'),
1482 *self.configure_args])
1484 def extra_commands(self, cmdlist):
1485 """Invoked to inject additional commands (make check) after build."""
1488 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1489 """Build policy for glibc during the compilers stage."""
1491 def __init__(self, glibc):
1492 super().__init__(glibc)
1493 self.builddir = glibc.ctx.component_builddir(
1494 'compilers', glibc.compiler.name, 'glibc', glibc.name)
1495 self.installdir = glibc.compiler.sysroot
1497 class GlibcPolicyForBuild(GlibcPolicyDefault):
1498 """Build policy for glibc during the glibcs stage."""
1500 def __init__(self, glibc):
1501 super().__init__(glibc)
1502 self.builddir = glibc.ctx.component_builddir(
1503 'glibcs', glibc.name, 'glibc')
1504 self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1506 self.strip = glibc.tool_name('strip')
1509 self.save_logs = glibc.ctx.save_logs
1511 def extra_commands(self, cmdlist):
1513 # Avoid picking up libc.so and libpthread.so, which are
1514 # linker scripts stored in /lib on Hurd. libc and
1515 # libpthread are still stripped via their libc-X.YY.so
1516 # implementation files.
1517 find_command = (('find %s/lib* -name "*.so"'
1518 + r' \! -name libc.so \! -name libpthread.so')
1520 cmdlist.add_command('strip', ['sh', '-c', ('%s $(%s)' %
1521 (self.strip, find_command))])
1522 cmdlist.add_command('check', ['make', 'check'])
1523 cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1525 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1526 """Build policy for glibc during update-syscalls."""
1528 def __init__(self, glibc):
1529 super().__init__(glibc)
1530 self.builddir = glibc.ctx.component_builddir(
1531 'update-syscalls', glibc.name, 'glibc')
1532 self.linuxdir = glibc.ctx.component_builddir(
1533 'update-syscalls', glibc.name, 'linux')
1534 self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1535 glibc, self.linuxdir)
1536 self.configure_args.insert(
1537 0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1538 # self.installdir not set because installation is not supported
1540 class Glibc(object):
1541 """A configuration for building glibc."""
1543 def __init__(self, compiler, arch=None, os_name=None, variant=None,
1544 cfg=None, ccopts=None):
1545 """Initialize a Glibc object."""
1546 self.ctx = compiler.ctx
1547 self.compiler = compiler
1549 self.arch = compiler.arch
1553 self.os = compiler.os
1556 self.variant = variant
1558 self.name = '%s-%s' % (self.arch, self.os)
1560 self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1561 self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1566 self.ccopts = ccopts
1568 def tool_name(self, tool):
1569 """Return the name of a cross-compilation tool."""
1570 ctool = '%s-%s' % (self.compiler.triplet, tool)
1571 if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1572 ctool = '%s %s' % (ctool, self.ccopts)
1576 """Generate commands to build this glibc."""
1577 builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1578 installdir = self.ctx.glibc_installdir(self.name)
1579 logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1580 self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1581 cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1582 cmdlist.add_command('check-compilers',
1584 os.path.join(self.compiler.installdir, 'ok')])
1585 cmdlist.use_path(self.compiler.bindir)
1586 self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1587 self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1590 def build_glibc(self, cmdlist, policy):
1591 """Generate commands to build this glibc, either as part of a compiler
1592 build or with the bootstrapped compiler (and in the latter case, run
1594 cmdlist.create_use_dir(policy.builddir)
1595 policy.configure(cmdlist)
1596 cmdlist.add_command('build', ['make'])
1597 cmdlist.add_command('install', ['make', 'install',
1598 'install_root=%s' % policy.installdir])
1599 # GCC uses paths such as lib/../lib64, so make sure lib
1600 # directories always exist.
1601 mkdir_cmd = ['mkdir', '-p',
1602 os.path.join(policy.installdir, 'lib')]
1604 mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1605 cmdlist.add_command('mkdir-lib', mkdir_cmd)
1606 policy.extra_commands(cmdlist)
1607 cmdlist.cleanup_dir()
1609 def update_syscalls(self):
1610 if self.os == 'gnu':
1611 # Hurd does not have system call tables that need updating.
1614 policy = GlibcPolicyForUpdateSyscalls(self)
1615 logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1616 self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1617 cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1618 cmdlist.add_command('check-compilers',
1620 os.path.join(self.compiler.installdir, 'ok')])
1621 cmdlist.use_path(self.compiler.bindir)
1623 install_linux_headers(policy.linux_policy, cmdlist)
1625 cmdlist.create_use_dir(policy.builddir)
1626 policy.configure(cmdlist)
1627 cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1628 cmdlist.cleanup_dir()
1629 self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1632 class Command(object):
1633 """A command run in the build process."""
1635 def __init__(self, desc, num, dir, path, command, always_run=False):
1636 """Initialize a Command object."""
1640 trans = str.maketrans({' ': '-'})
1641 self.logbase = '%03d-%s' % (num, desc.translate(trans))
1642 self.command = command
1643 self.always_run = always_run
1646 def shell_make_quote_string(s):
1647 """Given a string not containing a newline, quote it for use by the
1649 assert '\n' not in s
1650 if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1652 strans = str.maketrans({"'": "'\\''"})
1653 s = "'%s'" % s.translate(strans)
1654 mtrans = str.maketrans({'$': '$$'})
1655 return s.translate(mtrans)
1658 def shell_make_quote_list(l, translate_make):
1659 """Given a list of strings not containing newlines, quote them for use
1660 by the shell and make, returning a single string. If translate_make
1661 is true and the first string is 'make', change it to $(MAKE)."""
1662 l = [Command.shell_make_quote_string(s) for s in l]
1663 if translate_make and l[0] == 'make':
1667 def shell_make_quote(self):
1668 """Return this command quoted for the shell and make."""
1669 return self.shell_make_quote_list(self.command, True)
1672 class CommandList(object):
1673 """A list of commands run in the build process."""
1675 def __init__(self, desc, keep):
1676 """Initialize a CommandList object."""
1683 def desc_txt(self, desc):
1684 """Return the description to use for a command."""
1685 return '%s %s' % (' '.join(self.desc), desc)
1687 def use_dir(self, dir):
1688 """Set the default directory for subsequent commands."""
1691 def use_path(self, path):
1692 """Set a directory to be prepended to the PATH for subsequent
1696 def push_subdesc(self, subdesc):
1697 """Set the default subdescription for subsequent commands (e.g., the
1698 name of a component being built, within the series of commands
1700 self.desc.append(subdesc)
1702 def pop_subdesc(self):
1703 """Pop a subdescription from the list of descriptions."""
1706 def create_use_dir(self, dir):
1707 """Remove and recreate a directory and use it for subsequent
1709 self.add_command_dir('rm', None, ['rm', '-rf', dir])
1710 self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1713 def add_command_dir(self, desc, dir, command, always_run=False):
1714 """Add a command to run in a given directory."""
1715 cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1716 command, always_run)
1717 self.cmdlist.append(cmd)
1719 def add_command(self, desc, command, always_run=False):
1720 """Add a command to run in the default directory."""
1721 cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1722 self.path, command, always_run)
1723 self.cmdlist.append(cmd)
1725 def cleanup_dir(self, desc='cleanup', dir=None):
1726 """Clean up a build directory. If no directory is specified, the
1727 default directory is cleaned up and ceases to be the default
1732 if self.keep != 'all':
1733 self.add_command_dir(desc, None, ['rm', '-rf', dir],
1734 always_run=(self.keep == 'none'))
1736 def makefile_commands(self, wrapper, logsdir):
1737 """Return the sequence of commands in the form of text for a Makefile.
1738 The given wrapper script takes arguments: base of logs for
1739 previous command, or empty; base of logs for this command;
1740 description; directory; PATH addition; the command itself."""
1741 # prev_base is the base of the name for logs of the previous
1742 # command that is not always-run (that is, a build command,
1743 # whose failure should stop subsequent build commands from
1744 # being run, as opposed to a cleanup command, which is run
1745 # even if previous commands failed).
1748 for c in self.cmdlist:
1749 ctxt = c.shell_make_quote()
1750 if prev_base and not c.always_run:
1751 prev_log = os.path.join(logsdir, prev_base)
1754 this_log = os.path.join(logsdir, c.logbase)
1755 if not c.always_run:
1756 prev_base = c.logbase
1765 prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1766 prelim_txt = Command.shell_make_quote_list(prelims, False)
1767 cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1768 return '\n'.join(cmds)
1770 def status_logs(self, logsdir):
1771 """Return the list of log files with command status."""
1772 return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1773 for c in self.cmdlist]
1777 """Return an argument parser for this module."""
1778 parser = argparse.ArgumentParser(description=__doc__)
1779 parser.add_argument('-j', dest='parallelism',
1780 help='Run this number of jobs in parallel',
1781 type=int, default=os.cpu_count())
1782 parser.add_argument('--keep', dest='keep',
1783 help='Whether to keep all build directories, '
1784 'none or only those from failed builds',
1785 default='none', choices=('none', 'all', 'failed'))
1786 parser.add_argument('--replace-sources', action='store_true',
1787 help='Remove and replace source directories '
1788 'with the wrong version of a component')
1789 parser.add_argument('--strip', action='store_true',
1790 help='Strip installed glibc libraries')
1791 parser.add_argument('--full-gcc', action='store_true',
1792 help='Build GCC with all languages and libsanitizer')
1793 parser.add_argument('--shallow', action='store_true',
1794 help='Do not download Git history during checkout')
1795 parser.add_argument('topdir',
1796 help='Toplevel working directory')
1797 parser.add_argument('action',
1799 choices=('checkout', 'bot-cycle', 'bot',
1800 'host-libraries', 'compilers', 'glibcs',
1801 'update-syscalls', 'list-compilers',
1803 parser.add_argument('configs',
1804 help='Versions to check out or configurations to build',
1810 """The main entry point."""
1811 parser = get_parser()
1812 opts = parser.parse_args(argv)
1813 topdir = os.path.abspath(opts.topdir)
1814 ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1815 opts.strip, opts.full_gcc, opts.action,
1816 shallow=opts.shallow)
1817 ctx.run_builds(opts.action, opts.configs)
1820 if __name__ == '__main__':