scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2020 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <https://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 The 'list-compilers' command prints the name of each available
  36 compiler configuration, without building anything.  The 'list-glibcs'
  37 command prints the name of each glibc compiler configuration, followed
  38 by the space, followed by the name of the compiler configuration used
  39 for building this glibc variant.
  40
  41 """
  42
  43 import argparse
  44 import datetime
  45 import email.mime.text
  46 import email.utils
  47 import json
  48 import os
  49 import re
  50 import shutil
  51 import smtplib
  52 import stat
  53 import subprocess
  54 import sys
  55 import time
  56 import urllib.request
  57
  58 try:
  59     subprocess.run
  60 except:
  61     class _CompletedProcess:
  62         def __init__(self, args, returncode, stdout=None, stderr=None):
  63             self.args = args
  64             self.returncode = returncode
  65             self.stdout = stdout
  66             self.stderr = stderr
  67
  68     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  69         assert(timeout is None)
  70         with subprocess.Popen(*popenargs, **kwargs) as process:
  71             try:
  72                 stdout, stderr = process.communicate(input)
  73             except:
  74                 process.kill()
  75                 process.wait()
  76                 raise
  77             returncode = process.poll()
  78             if check and returncode:
  79                 raise subprocess.CalledProcessError(returncode, popenargs)
  80         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  81
  82     subprocess.run = _run
  83
  84
  85 class Context(object):
  86     """The global state associated with builds in a given directory."""
  87
  88     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  89                  full_gcc, action, shallow=False):
  90         """Initialize the context."""
  91         self.topdir = topdir
  92         self.parallelism = parallelism
  93         self.keep = keep
  94         self.replace_sources = replace_sources
  95         self.strip = strip
  96         self.full_gcc = full_gcc
  97         self.shallow = shallow
  98         self.srcdir = os.path.join(topdir, 'src')
  99         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 100         self.build_state_json = os.path.join(topdir, 'build-state.json')
 101         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 102         self.installdir = os.path.join(topdir, 'install')
 103         self.host_libraries_installdir = os.path.join(self.installdir,
 104                                                       'host-libraries')
 105         self.builddir = os.path.join(topdir, 'build')
 106         self.logsdir = os.path.join(topdir, 'logs')
 107         self.logsdir_old = os.path.join(topdir, 'logs-old')
 108         self.makefile = os.path.join(self.builddir, 'Makefile')
 109         self.wrapper = os.path.join(self.builddir, 'wrapper')
 110         self.save_logs = os.path.join(self.builddir, 'save-logs')
 111         self.script_text = self.get_script_text()
 112         if action not in ('checkout', 'list-compilers', 'list-glibcs'):
 113             self.build_triplet = self.get_build_triplet()
 114             self.glibc_version = self.get_glibc_version()
 115         self.configs = {}
 116         self.glibc_configs = {}
 117         self.makefile_pieces = ['.PHONY: all\n']
 118         self.add_all_configs()
 119         self.load_versions_json()
 120         self.load_build_state_json()
 121         self.status_log_list = []
 122         self.email_warning = False
 123
 124     def get_script_text(self):
 125         """Return the text of this script."""
 126         with open(sys.argv[0], 'r') as f:
 127             return f.read()
 128
 129     def exec_self(self):
 130         """Re-execute this script with the same arguments."""
 131         sys.stdout.flush()
 132         os.execv(sys.executable, [sys.executable] + sys.argv)
 133
 134     def get_build_triplet(self):
 135         """Determine the build triplet with config.guess."""
 136         config_guess = os.path.join(self.component_srcdir('gcc'),
 137                                     'config.guess')
 138         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 139                                 check=True, universal_newlines=True).stdout
 140         return cg_out.rstrip()
 141
 142     def get_glibc_version(self):
 143         """Determine the glibc version number (major.minor)."""
 144         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 145         with open(version_h, 'r') as f:
 146             lines = f.readlines()
 147         starttext = '#define VERSION "'
 148         for l in lines:
 149             if l.startswith(starttext):
 150                 l = l[len(starttext):]
 151                 l = l.rstrip('"\n')
 152                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 153                 return '%s.%s' % m.group(1, 2)
 154         print('error: could not determine glibc version')
 155         exit(1)
 156
 157     def add_all_configs(self):
 158         """Add all known glibc build configurations."""
 159         self.add_config(arch='aarch64',
 160                         os_name='linux-gnu',
 161                         extra_glibcs=[{'variant': 'disable-multi-arch',
 162                                        'cfg': ['--disable-multi-arch']}])
 163         self.add_config(arch='aarch64_be',
 164                         os_name='linux-gnu')
 165         self.add_config(arch='alpha',
 166                         os_name='linux-gnu')
 167         self.add_config(arch='arm',
 168                         os_name='linux-gnueabi',
 169                         extra_glibcs=[{'variant': 'v4t',
 170                                        'ccopts': '-march=armv4t'}])
 171         self.add_config(arch='armeb',
 172                         os_name='linux-gnueabi')
 173         self.add_config(arch='armeb',
 174                         os_name='linux-gnueabi',
 175                         variant='be8',
 176                         gcc_cfg=['--with-arch=armv7-a'])
 177         self.add_config(arch='arm',
 178                         os_name='linux-gnueabihf',
 179                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 180                         extra_glibcs=[{'variant': 'v7a',
 181                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 182                                       {'variant': 'v7a-disable-multi-arch',
 183                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 184                                        'cfg': ['--disable-multi-arch']}])
 185         self.add_config(arch='armeb',
 186                         os_name='linux-gnueabihf',
 187                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 188         self.add_config(arch='armeb',
 189                         os_name='linux-gnueabihf',
 190                         variant='be8',
 191                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 192                                  '--with-fpu=vfpv3'])
 193         self.add_config(arch='csky',
 194                         os_name='linux-gnuabiv2',
 195                         variant='soft',
 196                         gcc_cfg=['--disable-multilib'])
 197         self.add_config(arch='csky',
 198                         os_name='linux-gnuabiv2',
 199                         gcc_cfg=['--with-float=hard', '--disable-multilib'])
 200         self.add_config(arch='hppa',
 201                         os_name='linux-gnu')
 202         self.add_config(arch='i686',
 203                         os_name='gnu')
 204         self.add_config(arch='ia64',
 205                         os_name='linux-gnu',
 206                         first_gcc_cfg=['--with-system-libunwind'])
 207         self.add_config(arch='m68k',
 208                         os_name='linux-gnu',
 209                         gcc_cfg=['--disable-multilib'])
 210         self.add_config(arch='m68k',
 211                         os_name='linux-gnu',
 212                         variant='coldfire',
 213                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 214         self.add_config(arch='m68k',
 215                         os_name='linux-gnu',
 216                         variant='coldfire-soft',
 217                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 218                                  '--disable-multilib'])
 219         self.add_config(arch='microblaze',
 220                         os_name='linux-gnu',
 221                         gcc_cfg=['--disable-multilib'])
 222         self.add_config(arch='microblazeel',
 223                         os_name='linux-gnu',
 224                         gcc_cfg=['--disable-multilib'])
 225         self.add_config(arch='mips64',
 226                         os_name='linux-gnu',
 227                         gcc_cfg=['--with-mips-plt'],
 228                         glibcs=[{'variant': 'n32'},
 229                                 {'arch': 'mips',
 230                                  'ccopts': '-mabi=32'},
 231                                 {'variant': 'n64',
 232                                  'ccopts': '-mabi=64'}])
 233         self.add_config(arch='mips64',
 234                         os_name='linux-gnu',
 235                         variant='soft',
 236                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 237                         glibcs=[{'variant': 'n32-soft'},
 238                                 {'variant': 'soft',
 239                                  'arch': 'mips',
 240                                  'ccopts': '-mabi=32'},
 241                                 {'variant': 'n64-soft',
 242                                  'ccopts': '-mabi=64'}])
 243         self.add_config(arch='mips64',
 244                         os_name='linux-gnu',
 245                         variant='nan2008',
 246                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 247                                  '--with-arch-64=mips64r2',
 248                                  '--with-arch-32=mips32r2'],
 249                         glibcs=[{'variant': 'n32-nan2008'},
 250                                 {'variant': 'nan2008',
 251                                  'arch': 'mips',
 252                                  'ccopts': '-mabi=32'},
 253                                 {'variant': 'n64-nan2008',
 254                                  'ccopts': '-mabi=64'}])
 255         self.add_config(arch='mips64',
 256                         os_name='linux-gnu',
 257                         variant='nan2008-soft',
 258                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 259                                  '--with-arch-64=mips64r2',
 260                                  '--with-arch-32=mips32r2',
 261                                  '--with-float=soft'],
 262                         glibcs=[{'variant': 'n32-nan2008-soft'},
 263                                 {'variant': 'nan2008-soft',
 264                                  'arch': 'mips',
 265                                  'ccopts': '-mabi=32'},
 266                                 {'variant': 'n64-nan2008-soft',
 267                                  'ccopts': '-mabi=64'}])
 268         self.add_config(arch='mips64el',
 269                         os_name='linux-gnu',
 270                         gcc_cfg=['--with-mips-plt'],
 271                         glibcs=[{'variant': 'n32'},
 272                                 {'arch': 'mipsel',
 273                                  'ccopts': '-mabi=32'},
 274                                 {'variant': 'n64',
 275                                  'ccopts': '-mabi=64'}])
 276         self.add_config(arch='mips64el',
 277                         os_name='linux-gnu',
 278                         variant='soft',
 279                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 280                         glibcs=[{'variant': 'n32-soft'},
 281                                 {'variant': 'soft',
 282                                  'arch': 'mipsel',
 283                                  'ccopts': '-mabi=32'},
 284                                 {'variant': 'n64-soft',
 285                                  'ccopts': '-mabi=64'}])
 286         self.add_config(arch='mips64el',
 287                         os_name='linux-gnu',
 288                         variant='nan2008',
 289                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 290                                  '--with-arch-64=mips64r2',
 291                                  '--with-arch-32=mips32r2'],
 292                         glibcs=[{'variant': 'n32-nan2008'},
 293                                 {'variant': 'nan2008',
 294                                  'arch': 'mipsel',
 295                                  'ccopts': '-mabi=32'},
 296                                 {'variant': 'n64-nan2008',
 297                                  'ccopts': '-mabi=64'}])
 298         self.add_config(arch='mips64el',
 299                         os_name='linux-gnu',
 300                         variant='nan2008-soft',
 301                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 302                                  '--with-arch-64=mips64r2',
 303                                  '--with-arch-32=mips32r2',
 304                                  '--with-float=soft'],
 305                         glibcs=[{'variant': 'n32-nan2008-soft'},
 306                                 {'variant': 'nan2008-soft',
 307                                  'arch': 'mipsel',
 308                                  'ccopts': '-mabi=32'},
 309                                 {'variant': 'n64-nan2008-soft',
 310                                  'ccopts': '-mabi=64'}])
 311         self.add_config(arch='mipsisa64r6el',
 312                         os_name='linux-gnu',
 313                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 314                                  '--with-arch-64=mips64r6',
 315                                  '--with-arch-32=mips32r6',
 316                                  '--with-float=hard'],
 317                         glibcs=[{'variant': 'n32'},
 318                                 {'arch': 'mipsisa32r6el',
 319                                  'ccopts': '-mabi=32'},
 320                                 {'variant': 'n64',
 321                                  'ccopts': '-mabi=64'}])
 322         self.add_config(arch='nios2',
 323                         os_name='linux-gnu')
 324         self.add_config(arch='powerpc',
 325                         os_name='linux-gnu',
 326                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 327                         extra_glibcs=[{'variant': 'power4',
 328                                        'ccopts': '-mcpu=power4',
 329                                        'cfg': ['--with-cpu=power4']}])
 330         self.add_config(arch='powerpc',
 331                         os_name='linux-gnu',
 332                         variant='soft',
 333                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 334                                  '--enable-secureplt'])
 335         self.add_config(arch='powerpc64',
 336                         os_name='linux-gnu',
 337                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 338         self.add_config(arch='powerpc64le',
 339                         os_name='linux-gnu',
 340                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 341         self.add_config(arch='riscv64',
 342                         os_name='linux-gnu',
 343                         variant='rv64imac-lp64',
 344                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 345                                  '--disable-multilib'])
 346         self.add_config(arch='riscv64',
 347                         os_name='linux-gnu',
 348                         variant='rv64imafdc-lp64',
 349                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 350                                  '--disable-multilib'])
 351         self.add_config(arch='riscv64',
 352                         os_name='linux-gnu',
 353                         variant='rv64imafdc-lp64d',
 354                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 355                                  '--disable-multilib'])
 356         self.add_config(arch='s390x',
 357                         os_name='linux-gnu',
 358                         glibcs=[{},
 359                                 {'arch': 's390', 'ccopts': '-m31'}])
 360         self.add_config(arch='sh3',
 361                         os_name='linux-gnu')
 362         self.add_config(arch='sh3eb',
 363                         os_name='linux-gnu')
 364         self.add_config(arch='sh4',
 365                         os_name='linux-gnu')
 366         self.add_config(arch='sh4eb',
 367                         os_name='linux-gnu')
 368         self.add_config(arch='sh4',
 369                         os_name='linux-gnu',
 370                         variant='soft',
 371                         gcc_cfg=['--without-fp'])
 372         self.add_config(arch='sh4eb',
 373                         os_name='linux-gnu',
 374                         variant='soft',
 375                         gcc_cfg=['--without-fp'])
 376         self.add_config(arch='sparc64',
 377                         os_name='linux-gnu',
 378                         glibcs=[{},
 379                                 {'arch': 'sparcv9',
 380                                  'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
 381                         extra_glibcs=[{'variant': 'leon3',
 382                                        'arch' : 'sparcv8',
 383                                        'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
 384                                       {'variant': 'disable-multi-arch',
 385                                        'cfg': ['--disable-multi-arch']},
 386                                       {'variant': 'disable-multi-arch',
 387                                        'arch': 'sparcv9',
 388                                        'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
 389                                        'cfg': ['--disable-multi-arch']}])
 390         self.add_config(arch='x86_64',
 391                         os_name='linux-gnu',
 392                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 393                         glibcs=[{},
 394                                 {'variant': 'x32', 'ccopts': '-mx32'},
 395                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 396                         extra_glibcs=[{'variant': 'disable-multi-arch',
 397                                        'cfg': ['--disable-multi-arch']},
 398                                       {'variant': 'enable-obsolete',
 399                                        'cfg': ['--enable-obsolete-rpc',
 400                                                '--enable-obsolete-nsl']},
 401                                       {'variant': 'static-pie',
 402                                        'cfg': ['--enable-static-pie']},
 403                                       {'variant': 'x32-static-pie',
 404                                        'ccopts': '-mx32',
 405                                        'cfg': ['--enable-static-pie']},
 406                                       {'variant': 'static-pie',
 407                                        'arch': 'i686',
 408                                        'ccopts': '-m32 -march=i686',
 409                                        'cfg': ['--enable-static-pie']},
 410                                       {'variant': 'disable-multi-arch',
 411                                        'arch': 'i686',
 412                                        'ccopts': '-m32 -march=i686',
 413                                        'cfg': ['--disable-multi-arch']},
 414                                       {'variant': 'enable-obsolete',
 415                                        'arch': 'i686',
 416                                        'ccopts': '-m32 -march=i686',
 417                                        'cfg': ['--enable-obsolete-rpc',
 418                                                '--enable-obsolete-nsl']},
 419                                       {'arch': 'i486',
 420                                        'ccopts': '-m32 -march=i486'},
 421                                       {'arch': 'i586',
 422                                        'ccopts': '-m32 -march=i586'}])
 423
 424     def add_config(self, **args):
 425         """Add an individual build configuration."""
 426         cfg = Config(self, **args)
 427         if cfg.name in self.configs:
 428             print('error: duplicate config %s' % cfg.name)
 429             exit(1)
 430         self.configs[cfg.name] = cfg
 431         for c in cfg.all_glibcs:
 432             if c.name in self.glibc_configs:
 433                 print('error: duplicate glibc config %s' % c.name)
 434                 exit(1)
 435             self.glibc_configs[c.name] = c
 436
 437     def component_srcdir(self, component):
 438         """Return the source directory for a given component, e.g. gcc."""
 439         return os.path.join(self.srcdir, component)
 440
 441     def component_builddir(self, action, config, component, subconfig=None):
 442         """Return the directory to use for a build."""
 443         if config is None:
 444             # Host libraries.
 445             assert subconfig is None
 446             return os.path.join(self.builddir, action, component)
 447         if subconfig is None:
 448             return os.path.join(self.builddir, action, config, component)
 449         else:
 450             # glibc build as part of compiler build.
 451             return os.path.join(self.builddir, action, config, component,
 452                                 subconfig)
 453
 454     def compiler_installdir(self, config):
 455         """Return the directory in which to install a compiler."""
 456         return os.path.join(self.installdir, 'compilers', config)
 457
 458     def compiler_bindir(self, config):
 459         """Return the directory in which to find compiler binaries."""
 460         return os.path.join(self.compiler_installdir(config), 'bin')
 461
 462     def compiler_sysroot(self, config):
 463         """Return the sysroot directory for a compiler."""
 464         return os.path.join(self.compiler_installdir(config), 'sysroot')
 465
 466     def glibc_installdir(self, config):
 467         """Return the directory in which to install glibc."""
 468         return os.path.join(self.installdir, 'glibcs', config)
 469
 470     def run_builds(self, action, configs):
 471         """Run the requested builds."""
 472         if action == 'checkout':
 473             self.checkout(configs)
 474             return
 475         if action == 'bot-cycle':
 476             if configs:
 477                 print('error: configurations specified for bot-cycle')
 478                 exit(1)
 479             self.bot_cycle()
 480             return
 481         if action == 'bot':
 482             if configs:
 483                 print('error: configurations specified for bot')
 484                 exit(1)
 485             self.bot()
 486             return
 487         if action in ('host-libraries', 'list-compilers',
 488                       'list-glibcs') and configs:
 489             print('error: configurations specified for ' + action)
 490             exit(1)
 491         if action == 'list-compilers':
 492             for name in sorted(self.configs.keys()):
 493                 print(name)
 494             return
 495         if action == 'list-glibcs':
 496             for config in sorted(self.glibc_configs.values(),
 497                                  key=lambda c: c.name):
 498                 print(config.name, config.compiler.name)
 499             return
 500         self.clear_last_build_state(action)
 501         build_time = datetime.datetime.utcnow()
 502         if action == 'host-libraries':
 503             build_components = ('gmp', 'mpfr', 'mpc')
 504             old_components = ()
 505             old_versions = {}
 506             self.build_host_libraries()
 507         elif action == 'compilers':
 508             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 509                                 'gnumach', 'hurd')
 510             old_components = ('gmp', 'mpfr', 'mpc')
 511             old_versions = self.build_state['host-libraries']['build-versions']
 512             self.build_compilers(configs)
 513         else:
 514             build_components = ('glibc',)
 515             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 516                               'mig', 'gnumach', 'hurd')
 517             old_versions = self.build_state['compilers']['build-versions']
 518             if action == 'update-syscalls':
 519                 self.update_syscalls(configs)
 520             else:
 521                 self.build_glibcs(configs)
 522         self.write_files()
 523         self.do_build()
 524         if configs:
 525             # Partial build, do not update stored state.
 526             return
 527         build_versions = {}
 528         for k in build_components:
 529             if k in self.versions:
 530                 build_versions[k] = {'version': self.versions[k]['version'],
 531                                      'revision': self.versions[k]['revision']}
 532         for k in old_components:
 533             if k in old_versions:
 534                 build_versions[k] = {'version': old_versions[k]['version'],
 535                                      'revision': old_versions[k]['revision']}
 536         self.update_build_state(action, build_time, build_versions)
 537
 538     @staticmethod
 539     def remove_dirs(*args):
 540         """Remove directories and their contents if they exist."""
 541         for dir in args:
 542             shutil.rmtree(dir, ignore_errors=True)
 543
 544     @staticmethod
 545     def remove_recreate_dirs(*args):
 546         """Remove directories if they exist, and create them as empty."""
 547         Context.remove_dirs(*args)
 548         for dir in args:
 549             os.makedirs(dir, exist_ok=True)
 550
 551     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 552         """Add makefile text for a list of commands."""
 553         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 554         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 555                                     (target, target, target, commands))
 556         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 557
 558     def write_files(self):
 559         """Write out the Makefile and wrapper script."""
 560         mftext = ''.join(self.makefile_pieces)
 561         with open(self.makefile, 'w') as f:
 562             f.write(mftext)
 563         wrapper_text = (
 564             '#!/bin/sh\n'
 565             'prev_base=$1\n'
 566             'this_base=$2\n'
 567             'desc=$3\n'
 568             'dir=$4\n'
 569             'path=$5\n'
 570             'shift 5\n'
 571             'prev_status=$prev_base-status.txt\n'
 572             'this_status=$this_base-status.txt\n'
 573             'this_log=$this_base-log.txt\n'
 574             'date > "$this_log"\n'
 575             'echo >> "$this_log"\n'
 576             'echo "Description: $desc" >> "$this_log"\n'
 577             'printf "%s" "Command:" >> "$this_log"\n'
 578             'for word in "$@"; do\n'
 579             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 580             '    printf " %s" "$word"\n'
 581             '  else\n'
 582             '    printf " \'"\n'
 583             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 584             '    printf "\'"\n'
 585             '  fi\n'
 586             'done >> "$this_log"\n'
 587             'echo >> "$this_log"\n'
 588             'echo "Directory: $dir" >> "$this_log"\n'
 589             'echo "Path addition: $path" >> "$this_log"\n'
 590             'echo >> "$this_log"\n'
 591             'record_status ()\n'
 592             '{\n'
 593             '  echo >> "$this_log"\n'
 594             '  echo "$1: $desc" > "$this_status"\n'
 595             '  echo "$1: $desc" >> "$this_log"\n'
 596             '  echo >> "$this_log"\n'
 597             '  date >> "$this_log"\n'
 598             '  echo "$1: $desc"\n'
 599             '  exit 0\n'
 600             '}\n'
 601             'check_error ()\n'
 602             '{\n'
 603             '  if [ "$1" != "0" ]; then\n'
 604             '    record_status FAIL\n'
 605             '  fi\n'
 606             '}\n'
 607             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 608             '    record_status UNRESOLVED\n'
 609             'fi\n'
 610             'if [ "$dir" ]; then\n'
 611             '  cd "$dir"\n'
 612             '  check_error "$?"\n'
 613             'fi\n'
 614             'if [ "$path" ]; then\n'
 615             '  PATH=$path:$PATH\n'
 616             'fi\n'
 617             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 618             'check_error "$?"\n'
 619             'record_status PASS\n')
 620         with open(self.wrapper, 'w') as f:
 621             f.write(wrapper_text)
 622         # Mode 0o755.
 623         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 624                      stat.S_IROTH|stat.S_IXOTH)
 625         os.chmod(self.wrapper, mode_exec)
 626         save_logs_text = (
 627             '#!/bin/sh\n'
 628             'if ! [ -f tests.sum ]; then\n'
 629             '  echo "No test summary available."\n'
 630             '  exit 0\n'
 631             'fi\n'
 632             'save_file ()\n'
 633             '{\n'
 634             '  echo "Contents of $1:"\n'
 635             '  echo\n'
 636             '  cat "$1"\n'
 637             '  echo\n'
 638             '  echo "End of contents of $1."\n'
 639             '  echo\n'
 640             '}\n'
 641             'save_file tests.sum\n'
 642             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 643             'for t in $non_pass_tests; do\n'
 644             '  if [ -f "$t.out" ]; then\n'
 645             '    save_file "$t.out"\n'
 646             '  fi\n'
 647             'done\n')
 648         with open(self.save_logs, 'w') as f:
 649             f.write(save_logs_text)
 650         os.chmod(self.save_logs, mode_exec)
 651
 652     def do_build(self):
 653         """Do the actual build."""
 654         cmd = ['make', '-j%d' % self.parallelism]
 655         subprocess.run(cmd, cwd=self.builddir, check=True)
 656
 657     def build_host_libraries(self):
 658         """Build the host libraries."""
 659         installdir = self.host_libraries_installdir
 660         builddir = os.path.join(self.builddir, 'host-libraries')
 661         logsdir = os.path.join(self.logsdir, 'host-libraries')
 662         self.remove_recreate_dirs(installdir, builddir, logsdir)
 663         cmdlist = CommandList('host-libraries', self.keep)
 664         self.build_host_library(cmdlist, 'gmp')
 665         self.build_host_library(cmdlist, 'mpfr',
 666                                 ['--with-gmp=%s' % installdir])
 667         self.build_host_library(cmdlist, 'mpc',
 668                                 ['--with-gmp=%s' % installdir,
 669                                 '--with-mpfr=%s' % installdir])
 670         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 671         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 672
 673     def build_host_library(self, cmdlist, lib, extra_opts=None):
 674         """Build one host library."""
 675         srcdir = self.component_srcdir(lib)
 676         builddir = self.component_builddir('host-libraries', None, lib)
 677         installdir = self.host_libraries_installdir
 678         cmdlist.push_subdesc(lib)
 679         cmdlist.create_use_dir(builddir)
 680         cfg_cmd = [os.path.join(srcdir, 'configure'),
 681                    '--prefix=%s' % installdir,
 682                    '--disable-shared']
 683         if extra_opts:
 684             cfg_cmd.extend (extra_opts)
 685         cmdlist.add_command('configure', cfg_cmd)
 686         cmdlist.add_command('build', ['make'])
 687         cmdlist.add_command('check', ['make', 'check'])
 688         cmdlist.add_command('install', ['make', 'install'])
 689         cmdlist.cleanup_dir()
 690         cmdlist.pop_subdesc()
 691
 692     def build_compilers(self, configs):
 693         """Build the compilers."""
 694         if not configs:
 695             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 696             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 697             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 698             configs = sorted(self.configs.keys())
 699         for c in configs:
 700             self.configs[c].build()
 701
 702     def build_glibcs(self, configs):
 703         """Build the glibcs."""
 704         if not configs:
 705             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 706             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 707             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 708             configs = sorted(self.glibc_configs.keys())
 709         for c in configs:
 710             self.glibc_configs[c].build()
 711
 712     def update_syscalls(self, configs):
 713         """Update the glibc syscall lists."""
 714         if not configs:
 715             self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
 716             self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
 717             configs = sorted(self.glibc_configs.keys())
 718         for c in configs:
 719             self.glibc_configs[c].update_syscalls()
 720
 721     def load_versions_json(self):
 722         """Load information about source directory versions."""
 723         if not os.access(self.versions_json, os.F_OK):
 724             self.versions = {}
 725             return
 726         with open(self.versions_json, 'r') as f:
 727             self.versions = json.load(f)
 728
 729     def store_json(self, data, filename):
 730         """Store information in a JSON file."""
 731         filename_tmp = filename + '.tmp'
 732         with open(filename_tmp, 'w') as f:
 733             json.dump(data, f, indent=2, sort_keys=True)
 734         os.rename(filename_tmp, filename)
 735
 736     def store_versions_json(self):
 737         """Store information about source directory versions."""
 738         self.store_json(self.versions, self.versions_json)
 739
 740     def set_component_version(self, component, version, explicit, revision):
 741         """Set the version information for a component."""
 742         self.versions[component] = {'version': version,
 743                                     'explicit': explicit,
 744                                     'revision': revision}
 745         self.store_versions_json()
 746
 747     def checkout(self, versions):
 748         """Check out the desired component versions."""
 749         default_versions = {'binutils': 'vcs-2.34',
 750                             'gcc': 'vcs-10',
 751                             'glibc': 'vcs-mainline',
 752                             'gmp': '6.2.0',
 753                             'linux': '5.7',
 754                             'mpc': '1.1.0',
 755                             'mpfr': '4.0.2',
 756                             'mig': 'vcs-mainline',
 757                             'gnumach': 'vcs-mainline',
 758                             'hurd': 'vcs-mainline'}
 759         use_versions = {}
 760         explicit_versions = {}
 761         for v in versions:
 762             found_v = False
 763             for k in default_versions.keys():
 764                 kx = k + '-'
 765                 if v.startswith(kx):
 766                     vx = v[len(kx):]
 767                     if k in use_versions:
 768                         print('error: multiple versions for %s' % k)
 769                         exit(1)
 770                     use_versions[k] = vx
 771                     explicit_versions[k] = True
 772                     found_v = True
 773                     break
 774             if not found_v:
 775                 print('error: unknown component in %s' % v)
 776                 exit(1)
 777         for k in default_versions.keys():
 778             if k not in use_versions:
 779                 if k in self.versions and self.versions[k]['explicit']:
 780                     use_versions[k] = self.versions[k]['version']
 781                     explicit_versions[k] = True
 782                 else:
 783                     use_versions[k] = default_versions[k]
 784                     explicit_versions[k] = False
 785         os.makedirs(self.srcdir, exist_ok=True)
 786         for k in sorted(default_versions.keys()):
 787             update = os.access(self.component_srcdir(k), os.F_OK)
 788             v = use_versions[k]
 789             if (update and
 790                 k in self.versions and
 791                 v != self.versions[k]['version']):
 792                 if not self.replace_sources:
 793                     print('error: version of %s has changed from %s to %s, '
 794                           'use --replace-sources to check out again' %
 795                           (k, self.versions[k]['version'], v))
 796                     exit(1)
 797                 shutil.rmtree(self.component_srcdir(k))
 798                 update = False
 799             if v.startswith('vcs-'):
 800                 revision = self.checkout_vcs(k, v[4:], update)
 801             else:
 802                 self.checkout_tar(k, v, update)
 803                 revision = v
 804             self.set_component_version(k, v, explicit_versions[k], revision)
 805         if self.get_script_text() != self.script_text:
 806             # Rerun the checkout process in case the updated script
 807             # uses different default versions or new components.
 808             self.exec_self()
 809
 810     def checkout_vcs(self, component, version, update):
 811         """Check out the given version of the given component from version
 812         control.  Return a revision identifier."""
 813         if component == 'binutils':
 814             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 815             if version == 'mainline':
 816                 git_branch = 'master'
 817             else:
 818                 trans = str.maketrans({'.': '_'})
 819                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 820             return self.git_checkout(component, git_url, git_branch, update)
 821         elif component == 'gcc':
 822             if version == 'mainline':
 823                 branch = 'master'
 824             else:
 825                 branch = 'releases/gcc-%s' % version
 826             return self.gcc_checkout(branch, update)
 827         elif component == 'glibc':
 828             git_url = 'git://sourceware.org/git/glibc.git'
 829             if version == 'mainline':
 830                 git_branch = 'master'
 831             else:
 832                 git_branch = 'release/%s/master' % version
 833             r = self.git_checkout(component, git_url, git_branch, update)
 834             self.fix_glibc_timestamps()
 835             return r
 836         elif component == 'gnumach':
 837             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 838             git_branch = 'master'
 839             r = self.git_checkout(component, git_url, git_branch, update)
 840             subprocess.run(['autoreconf', '-i'],
 841                            cwd=self.component_srcdir(component), check=True)
 842             return r
 843         elif component == 'mig':
 844             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 845             git_branch = 'master'
 846             r = self.git_checkout(component, git_url, git_branch, update)
 847             subprocess.run(['autoreconf', '-i'],
 848                            cwd=self.component_srcdir(component), check=True)
 849             return r
 850         elif component == 'hurd':
 851             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 852             git_branch = 'master'
 853             r = self.git_checkout(component, git_url, git_branch, update)
 854             subprocess.run(['autoconf'],
 855                            cwd=self.component_srcdir(component), check=True)
 856             return r
 857         else:
 858             print('error: component %s coming from VCS' % component)
 859             exit(1)
 860
 861     def git_checkout(self, component, git_url, git_branch, update):
 862         """Check out a component from git.  Return a commit identifier."""
 863         if update:
 864             subprocess.run(['git', 'remote', 'prune', 'origin'],
 865                            cwd=self.component_srcdir(component), check=True)
 866             if self.replace_sources:
 867                 subprocess.run(['git', 'clean', '-dxfq'],
 868                                cwd=self.component_srcdir(component), check=True)
 869             subprocess.run(['git', 'pull', '-q'],
 870                            cwd=self.component_srcdir(component), check=True)
 871         else:
 872             if self.shallow:
 873                 depth_arg = ('--depth', '1')
 874             else:
 875                 depth_arg = ()
 876             subprocess.run(['git', 'clone', '-q', '-b', git_branch,
 877                             *depth_arg, git_url,
 878                             self.component_srcdir(component)], check=True)
 879         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 880                            cwd=self.component_srcdir(component),
 881                            stdout=subprocess.PIPE,
 882                            check=True, universal_newlines=True).stdout
 883         return r.rstrip()
 884
 885     def fix_glibc_timestamps(self):
 886         """Fix timestamps in a glibc checkout."""
 887         # Ensure that builds do not try to regenerate generated files
 888         # in the source tree.
 889         srcdir = self.component_srcdir('glibc')
 890         # These files have Makefile dependencies to regenerate them in
 891         # the source tree that may be active during a normal build.
 892         # Some other files have such dependencies but do not need to
 893         # be touched because nothing in a build depends on the files
 894         # in question.
 895         for f in ('sysdeps/mach/hurd/bits/errno.h',):
 896             to_touch = os.path.join(srcdir, f)
 897             subprocess.run(['touch', '-c', to_touch], check=True)
 898         for dirpath, dirnames, filenames in os.walk(srcdir):
 899             for f in filenames:
 900                 if (f == 'configure' or
 901                     f == 'preconfigure' or
 902                     f.endswith('-kw.h')):
 903                     to_touch = os.path.join(dirpath, f)
 904                     subprocess.run(['touch', to_touch], check=True)
 905
 906     def gcc_checkout(self, branch, update):
 907         """Check out GCC from git.  Return the commit identifier."""
 908         if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
 909                      os.F_OK):
 910             if not self.replace_sources:
 911                 print('error: GCC has moved from SVN to git, use '
 912                       '--replace-sources to check out again')
 913                 exit(1)
 914             shutil.rmtree(self.component_srcdir('gcc'))
 915             update = False
 916         if not update:
 917             self.git_checkout('gcc', 'git://gcc.gnu.org/git/gcc.git',
 918                               branch, update)
 919         subprocess.run(['contrib/gcc_update', '--silent'],
 920                        cwd=self.component_srcdir('gcc'), check=True)
 921         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 922                            cwd=self.component_srcdir('gcc'),
 923                            stdout=subprocess.PIPE,
 924                            check=True, universal_newlines=True).stdout
 925         return r.rstrip()
 926
 927     def checkout_tar(self, component, version, update):
 928         """Check out the given version of the given component from a
 929         tarball."""
 930         if update:
 931             return
 932         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 933                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
 934                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 935                    'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
 936                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 937                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 938                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 939                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 940                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 941         if component not in url_map:
 942             print('error: component %s coming from tarball' % component)
 943             exit(1)
 944         version_major = version.split('.')[0]
 945         url = url_map[component] % {'version': version, 'major': version_major}
 946         filename = os.path.join(self.srcdir, url.split('/')[-1])
 947         response = urllib.request.urlopen(url)
 948         data = response.read()
 949         with open(filename, 'wb') as f:
 950             f.write(data)
 951         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 952                        check=True)
 953         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 954                   self.component_srcdir(component))
 955         os.remove(filename)
 956
 957     def load_build_state_json(self):
 958         """Load information about the state of previous builds."""
 959         if os.access(self.build_state_json, os.F_OK):
 960             with open(self.build_state_json, 'r') as f:
 961                 self.build_state = json.load(f)
 962         else:
 963             self.build_state = {}
 964         for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
 965             if k not in self.build_state:
 966                 self.build_state[k] = {}
 967             if 'build-time' not in self.build_state[k]:
 968                 self.build_state[k]['build-time'] = ''
 969             if 'build-versions' not in self.build_state[k]:
 970                 self.build_state[k]['build-versions'] = {}
 971             if 'build-results' not in self.build_state[k]:
 972                 self.build_state[k]['build-results'] = {}
 973             if 'result-changes' not in self.build_state[k]:
 974                 self.build_state[k]['result-changes'] = {}
 975             if 'ever-passed' not in self.build_state[k]:
 976                 self.build_state[k]['ever-passed'] = []
 977
 978     def store_build_state_json(self):
 979         """Store information about the state of previous builds."""
 980         self.store_json(self.build_state, self.build_state_json)
 981
 982     def clear_last_build_state(self, action):
 983         """Clear information about the state of part of the build."""
 984         # We clear the last build time and versions when starting a
 985         # new build.  The results of the last build are kept around,
 986         # as comparison is still meaningful if this build is aborted
 987         # and a new one started.
 988         self.build_state[action]['build-time'] = ''
 989         self.build_state[action]['build-versions'] = {}
 990         self.store_build_state_json()
 991
 992     def update_build_state(self, action, build_time, build_versions):
 993         """Update the build state after a build."""
 994         build_time = build_time.replace(microsecond=0)
 995         self.build_state[action]['build-time'] = str(build_time)
 996         self.build_state[action]['build-versions'] = build_versions
 997         build_results = {}
 998         for log in self.status_log_list:
 999             with open(log, 'r') as f:
1000                 log_text = f.read()
1001             log_text = log_text.rstrip()
1002             m = re.fullmatch('([A-Z]+): (.*)', log_text)
1003             result = m.group(1)
1004             test_name = m.group(2)
1005             assert test_name not in build_results
1006             build_results[test_name] = result
1007         old_build_results = self.build_state[action]['build-results']
1008         self.build_state[action]['build-results'] = build_results
1009         result_changes = {}
1010         all_tests = set(old_build_results.keys()) | set(build_results.keys())
1011         for t in all_tests:
1012             if t in old_build_results:
1013                 old_res = old_build_results[t]
1014             else:
1015                 old_res = '(New test)'
1016             if t in build_results:
1017                 new_res = build_results[t]
1018             else:
1019                 new_res = '(Test removed)'
1020             if old_res != new_res:
1021                 result_changes[t] = '%s -> %s' % (old_res, new_res)
1022         self.build_state[action]['result-changes'] = result_changes
1023         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1024                            if t in build_results}
1025         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1026         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1027                                                          new_passes)
1028         self.store_build_state_json()
1029
1030     def load_bot_config_json(self):
1031         """Load bot configuration."""
1032         with open(self.bot_config_json, 'r') as f:
1033             self.bot_config = json.load(f)
1034
1035     def part_build_old(self, action, delay):
1036         """Return whether the last build for a given action was at least a
1037         given number of seconds ago, or does not have a time recorded."""
1038         old_time_str = self.build_state[action]['build-time']
1039         if not old_time_str:
1040             return True
1041         old_time = datetime.datetime.strptime(old_time_str,
1042                                               '%Y-%m-%d %H:%M:%S')
1043         new_time = datetime.datetime.utcnow()
1044         delta = new_time - old_time
1045         return delta.total_seconds() >= delay
1046
1047     def bot_cycle(self):
1048         """Run a single round of checkout and builds."""
1049         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1050         self.load_bot_config_json()
1051         actions = ('host-libraries', 'compilers', 'glibcs')
1052         self.bot_run_self(['--replace-sources'], 'checkout')
1053         self.load_versions_json()
1054         if self.get_script_text() != self.script_text:
1055             print('Script changed, re-execing.')
1056             # On script change, all parts of the build should be rerun.
1057             for a in actions:
1058                 self.clear_last_build_state(a)
1059             self.exec_self()
1060         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1061                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1062                                           'mig', 'gnumach', 'hurd'),
1063                             'glibcs': ('glibc',)}
1064         must_build = {}
1065         for a in actions:
1066             build_vers = self.build_state[a]['build-versions']
1067             must_build[a] = False
1068             if not self.build_state[a]['build-time']:
1069                 must_build[a] = True
1070             old_vers = {}
1071             new_vers = {}
1072             for c in check_components[a]:
1073                 if c in build_vers:
1074                     old_vers[c] = build_vers[c]
1075                 new_vers[c] = {'version': self.versions[c]['version'],
1076                                'revision': self.versions[c]['revision']}
1077             if new_vers == old_vers:
1078                 print('Versions for %s unchanged.' % a)
1079             else:
1080                 print('Versions changed or rebuild forced for %s.' % a)
1081                 if a == 'compilers' and not self.part_build_old(
1082                         a, self.bot_config['compilers-rebuild-delay']):
1083                     print('Not requiring rebuild of compilers this soon.')
1084                 else:
1085                     must_build[a] = True
1086         if must_build['host-libraries']:
1087             must_build['compilers'] = True
1088         if must_build['compilers']:
1089             must_build['glibcs'] = True
1090         for a in actions:
1091             if must_build[a]:
1092                 print('Must rebuild %s.' % a)
1093                 self.clear_last_build_state(a)
1094             else:
1095                 print('No need to rebuild %s.' % a)
1096         if os.access(self.logsdir, os.F_OK):
1097             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1098             shutil.copytree(self.logsdir, self.logsdir_old)
1099         for a in actions:
1100             if must_build[a]:
1101                 build_time = datetime.datetime.utcnow()
1102                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1103                 self.bot_run_self([], a)
1104                 self.load_build_state_json()
1105                 self.bot_build_mail(a, build_time)
1106         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1107
1108     def bot_build_mail(self, action, build_time):
1109         """Send email with the results of a build."""
1110         if not ('email-from' in self.bot_config and
1111                 'email-server' in self.bot_config and
1112                 'email-subject' in self.bot_config and
1113                 'email-to' in self.bot_config):
1114             if not self.email_warning:
1115                 print("Email not configured, not sending.")
1116                 self.email_warning = True
1117             return
1118
1119         build_time = build_time.replace(microsecond=0)
1120         subject = (self.bot_config['email-subject'] %
1121                    {'action': action,
1122                     'build-time': str(build_time)})
1123         results = self.build_state[action]['build-results']
1124         changes = self.build_state[action]['result-changes']
1125         ever_passed = set(self.build_state[action]['ever-passed'])
1126         versions = self.build_state[action]['build-versions']
1127         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1128         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1129         all_fails = {k for k in results if results[k] == 'FAIL'}
1130         if new_regressions:
1131             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1132             new_reg_text = ('New regressions:\n\n%s\n\n' %
1133                             '\n'.join(new_reg_list))
1134         else:
1135             new_reg_text = ''
1136         if all_regressions:
1137             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1138             all_reg_text = ('All regressions:\n\n%s\n\n' %
1139                             '\n'.join(all_reg_list))
1140         else:
1141             all_reg_text = ''
1142         if all_fails:
1143             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1144             all_fail_text = ('All failures:\n\n%s\n\n' %
1145                              '\n'.join(all_fail_list))
1146         else:
1147             all_fail_text = ''
1148         if changes:
1149             changes_list = sorted(changes.keys())
1150             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1151             changes_text = ('All changed results:\n\n%s\n\n' %
1152                             '\n'.join(changes_list))
1153         else:
1154             changes_text = ''
1155         results_text = (new_reg_text + all_reg_text + all_fail_text +
1156                         changes_text)
1157         if not results_text:
1158             results_text = 'Clean build with unchanged results.\n\n'
1159         versions_list = sorted(versions.keys())
1160         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1161                                           versions[k]['revision'])
1162                          for k in versions_list]
1163         versions_text = ('Component versions for this build:\n\n%s\n' %
1164                          '\n'.join(versions_list))
1165         body_text = results_text + versions_text
1166         msg = email.mime.text.MIMEText(body_text)
1167         msg['Subject'] = subject
1168         msg['From'] = self.bot_config['email-from']
1169         msg['To'] = self.bot_config['email-to']
1170         msg['Message-ID'] = email.utils.make_msgid()
1171         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1172         with smtplib.SMTP(self.bot_config['email-server']) as s:
1173             s.send_message(msg)
1174
1175     def bot_run_self(self, opts, action, check=True):
1176         """Run a copy of this script with given options."""
1177         cmd = [sys.executable, sys.argv[0], '--keep=none',
1178                '-j%d' % self.parallelism]
1179         if self.full_gcc:
1180             cmd.append('--full-gcc')
1181         cmd.extend(opts)
1182         cmd.extend([self.topdir, action])
1183         sys.stdout.flush()
1184         subprocess.run(cmd, check=check)
1185
1186     def bot(self):
1187         """Run repeated rounds of checkout and builds."""
1188         while True:
1189             self.load_bot_config_json()
1190             if not self.bot_config['run']:
1191                 print('Bot exiting by request.')
1192                 exit(0)
1193             self.bot_run_self([], 'bot-cycle', check=False)
1194             self.load_bot_config_json()
1195             if not self.bot_config['run']:
1196                 print('Bot exiting by request.')
1197                 exit(0)
1198             time.sleep(self.bot_config['delay'])
1199             if self.get_script_text() != self.script_text:
1200                 print('Script changed, bot re-execing.')
1201                 self.exec_self()
1202
1203 class LinuxHeadersPolicyForBuild(object):
1204     """Names and directories for installing Linux headers.  Build variant."""
1205
1206     def __init__(self, config):
1207         self.arch = config.arch
1208         self.srcdir = config.ctx.component_srcdir('linux')
1209         self.builddir = config.component_builddir('linux')
1210         self.headers_dir = os.path.join(config.sysroot, 'usr')
1211
1212 class LinuxHeadersPolicyForUpdateSyscalls(object):
1213     """Names and directories for Linux headers.  update-syscalls variant."""
1214
1215     def __init__(self, glibc, headers_dir):
1216         self.arch = glibc.compiler.arch
1217         self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1218         self.builddir = glibc.ctx.component_builddir(
1219             'update-syscalls', glibc.name, 'build-linux')
1220         self.headers_dir = headers_dir
1221
1222 def install_linux_headers(policy, cmdlist):
1223     """Install Linux kernel headers."""
1224     arch_map = {'aarch64': 'arm64',
1225                 'alpha': 'alpha',
1226                 'arm': 'arm',
1227                 'csky': 'csky',
1228                 'hppa': 'parisc',
1229                 'i486': 'x86',
1230                 'i586': 'x86',
1231                 'i686': 'x86',
1232                 'i786': 'x86',
1233                 'ia64': 'ia64',
1234                 'm68k': 'm68k',
1235                 'microblaze': 'microblaze',
1236                 'mips': 'mips',
1237                 'nios2': 'nios2',
1238                 'powerpc': 'powerpc',
1239                 's390': 's390',
1240                 'riscv32': 'riscv',
1241                 'riscv64': 'riscv',
1242                 'sh': 'sh',
1243                 'sparc': 'sparc',
1244                 'x86_64': 'x86'}
1245     linux_arch = None
1246     for k in arch_map:
1247         if policy.arch.startswith(k):
1248             linux_arch = arch_map[k]
1249             break
1250     assert linux_arch is not None
1251     cmdlist.push_subdesc('linux')
1252     cmdlist.create_use_dir(policy.builddir)
1253     cmdlist.add_command('install-headers',
1254                         ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1255                          'ARCH=%s' % linux_arch,
1256                          'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1257                          'headers_install'])
1258     cmdlist.cleanup_dir()
1259     cmdlist.pop_subdesc()
1260
1261 class Config(object):
1262     """A configuration for building a compiler and associated libraries."""
1263
1264     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1265                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1266         """Initialize a Config object."""
1267         self.ctx = ctx
1268         self.arch = arch
1269         self.os = os_name
1270         self.variant = variant
1271         if variant is None:
1272             self.name = '%s-%s' % (arch, os_name)
1273         else:
1274             self.name = '%s-%s-%s' % (arch, os_name, variant)
1275         self.triplet = '%s-glibc-%s' % (arch, os_name)
1276         if gcc_cfg is None:
1277             self.gcc_cfg = []
1278         else:
1279             self.gcc_cfg = gcc_cfg
1280         if first_gcc_cfg is None:
1281             self.first_gcc_cfg = []
1282         else:
1283             self.first_gcc_cfg = first_gcc_cfg
1284         if glibcs is None:
1285             glibcs = [{'variant': variant}]
1286         if extra_glibcs is None:
1287             extra_glibcs = []
1288         glibcs = [Glibc(self, **g) for g in glibcs]
1289         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1290         self.all_glibcs = glibcs + extra_glibcs
1291         self.compiler_glibcs = glibcs
1292         self.installdir = ctx.compiler_installdir(self.name)
1293         self.bindir = ctx.compiler_bindir(self.name)
1294         self.sysroot = ctx.compiler_sysroot(self.name)
1295         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1296         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1297
1298     def component_builddir(self, component):
1299         """Return the directory to use for a (non-glibc) build."""
1300         return self.ctx.component_builddir('compilers', self.name, component)
1301
1302     def build(self):
1303         """Generate commands to build this compiler."""
1304         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1305                                       self.logsdir)
1306         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1307         cmdlist.add_command('check-host-libraries',
1308                             ['test', '-f',
1309                              os.path.join(self.ctx.host_libraries_installdir,
1310                                           'ok')])
1311         cmdlist.use_path(self.bindir)
1312         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1313                               ['--disable-gdb',
1314                                '--disable-gdbserver',
1315                                '--disable-libdecnumber',
1316                                '--disable-readline',
1317                                '--disable-sim'])
1318         if self.os.startswith('linux'):
1319             install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1320         self.build_gcc(cmdlist, True)
1321         if self.os == 'gnu':
1322             self.install_gnumach_headers(cmdlist)
1323             self.build_cross_tool(cmdlist, 'mig', 'mig')
1324             self.install_hurd_headers(cmdlist)
1325         for g in self.compiler_glibcs:
1326             cmdlist.push_subdesc('glibc')
1327             cmdlist.push_subdesc(g.name)
1328             g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1329             cmdlist.pop_subdesc()
1330             cmdlist.pop_subdesc()
1331         self.build_gcc(cmdlist, False)
1332         cmdlist.add_command('done', ['touch',
1333                                      os.path.join(self.installdir, 'ok')])
1334         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1335                                       self.logsdir)
1336
1337     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1338         """Build one cross tool."""
1339         srcdir = self.ctx.component_srcdir(tool_src)
1340         builddir = self.component_builddir(tool_build)
1341         cmdlist.push_subdesc(tool_build)
1342         cmdlist.create_use_dir(builddir)
1343         cfg_cmd = [os.path.join(srcdir, 'configure'),
1344                    '--prefix=%s' % self.installdir,
1345                    '--build=%s' % self.ctx.build_triplet,
1346                    '--host=%s' % self.ctx.build_triplet,
1347                    '--target=%s' % self.triplet,
1348                    '--with-sysroot=%s' % self.sysroot]
1349         if extra_opts:
1350             cfg_cmd.extend(extra_opts)
1351         cmdlist.add_command('configure', cfg_cmd)
1352         cmdlist.add_command('build', ['make'])
1353         # Parallel "make install" for GCC has race conditions that can
1354         # cause it to fail; see
1355         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1356         # problems are not known for binutils, but doing the
1357         # installation in parallel within a particular toolchain build
1358         # (as opposed to installation of one toolchain from
1359         # build-many-glibcs.py running in parallel to the installation
1360         # of other toolchains being built) is not known to be
1361         # significantly beneficial, so it is simplest just to disable
1362         # parallel install for cross tools here.
1363         cmdlist.add_command('install', ['make', '-j1', 'install'])
1364         cmdlist.cleanup_dir()
1365         cmdlist.pop_subdesc()
1366
1367     def install_gnumach_headers(self, cmdlist):
1368         """Install GNU Mach headers."""
1369         srcdir = self.ctx.component_srcdir('gnumach')
1370         builddir = self.component_builddir('gnumach')
1371         cmdlist.push_subdesc('gnumach')
1372         cmdlist.create_use_dir(builddir)
1373         cmdlist.add_command('configure',
1374                             [os.path.join(srcdir, 'configure'),
1375                              '--build=%s' % self.ctx.build_triplet,
1376                              '--host=%s' % self.triplet,
1377                              '--prefix=',
1378                              'CC=%s-gcc -nostdlib' % self.triplet])
1379         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1380                                         'install-data'])
1381         cmdlist.cleanup_dir()
1382         cmdlist.pop_subdesc()
1383
1384     def install_hurd_headers(self, cmdlist):
1385         """Install Hurd headers."""
1386         srcdir = self.ctx.component_srcdir('hurd')
1387         builddir = self.component_builddir('hurd')
1388         cmdlist.push_subdesc('hurd')
1389         cmdlist.create_use_dir(builddir)
1390         cmdlist.add_command('configure',
1391                             [os.path.join(srcdir, 'configure'),
1392                              '--build=%s' % self.ctx.build_triplet,
1393                              '--host=%s' % self.triplet,
1394                              '--prefix=',
1395                              '--disable-profile', '--without-parted',
1396                              'CC=%s-gcc -nostdlib' % self.triplet])
1397         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1398                                         'no_deps=t', 'install-headers'])
1399         cmdlist.cleanup_dir()
1400         cmdlist.pop_subdesc()
1401
1402     def build_gcc(self, cmdlist, bootstrap):
1403         """Build GCC."""
1404         # libssp is of little relevance with glibc's own stack
1405         # checking support.  libcilkrts does not support GNU/Hurd (and
1406         # has been removed in GCC 8, so --disable-libcilkrts can be
1407         # removed once glibc no longer supports building with older
1408         # GCC versions).
1409         cfg_opts = list(self.gcc_cfg)
1410         cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1411         host_libs = self.ctx.host_libraries_installdir
1412         cfg_opts += ['--with-gmp=%s' % host_libs,
1413                      '--with-mpfr=%s' % host_libs,
1414                      '--with-mpc=%s' % host_libs]
1415         if bootstrap:
1416             tool_build = 'gcc-first'
1417             # Building a static-only, C-only compiler that is
1418             # sufficient to build glibc.  Various libraries and
1419             # features that may require libc headers must be disabled.
1420             # When configuring with a sysroot, --with-newlib is
1421             # required to define inhibit_libc (to stop some parts of
1422             # libgcc including libc headers); --without-headers is not
1423             # sufficient.
1424             cfg_opts += ['--enable-languages=c', '--disable-shared',
1425                          '--disable-threads',
1426                          '--disable-libatomic',
1427                          '--disable-decimal-float',
1428                          '--disable-libffi',
1429                          '--disable-libgomp',
1430                          '--disable-libitm',
1431                          '--disable-libmpx',
1432                          '--disable-libquadmath',
1433                          '--disable-libsanitizer',
1434                          '--without-headers', '--with-newlib',
1435                          '--with-glibc-version=%s' % self.ctx.glibc_version
1436                          ]
1437             cfg_opts += self.first_gcc_cfg
1438         else:
1439             tool_build = 'gcc'
1440             # libsanitizer commonly breaks because of glibc header
1441             # changes, or on unusual targets.  C++ pre-compiled
1442             # headers are not used during the glibc build and are
1443             # expensive to create.
1444             if not self.ctx.full_gcc:
1445                 cfg_opts += ['--disable-libsanitizer',
1446                              '--disable-libstdcxx-pch']
1447             langs = 'all' if self.ctx.full_gcc else 'c,c++'
1448             cfg_opts += ['--enable-languages=%s' % langs,
1449                          '--enable-shared', '--enable-threads']
1450         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1451
1452 class GlibcPolicyDefault(object):
1453     """Build policy for glibc: common defaults."""
1454
1455     def __init__(self, glibc):
1456         self.srcdir = glibc.ctx.component_srcdir('glibc')
1457         self.use_usr = glibc.os != 'gnu'
1458         self.prefix = '/usr' if self.use_usr else ''
1459         self.configure_args = [
1460             '--prefix=%s' % self.prefix,
1461             '--enable-profile',
1462             '--build=%s' % glibc.ctx.build_triplet,
1463             '--host=%s' % glibc.triplet,
1464             'CC=%s' % glibc.tool_name('gcc'),
1465             'CXX=%s' % glibc.tool_name('g++'),
1466             'AR=%s' % glibc.tool_name('ar'),
1467             'AS=%s' % glibc.tool_name('as'),
1468             'LD=%s' % glibc.tool_name('ld'),
1469             'NM=%s' % glibc.tool_name('nm'),
1470             'OBJCOPY=%s' % glibc.tool_name('objcopy'),
1471             'OBJDUMP=%s' % glibc.tool_name('objdump'),
1472             'RANLIB=%s' % glibc.tool_name('ranlib'),
1473             'READELF=%s' % glibc.tool_name('readelf'),
1474             'STRIP=%s' % glibc.tool_name('strip'),
1475         ]
1476         if glibc.os == 'gnu':
1477             self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1478         self.configure_args += glibc.cfg
1479
1480     def configure(self, cmdlist):
1481         """Invoked to add the configure command to the command list."""
1482         cmdlist.add_command('configure',
1483                             [os.path.join(self.srcdir, 'configure'),
1484                              *self.configure_args])
1485
1486     def extra_commands(self, cmdlist):
1487         """Invoked to inject additional commands (make check) after build."""
1488         pass
1489
1490 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1491     """Build policy for glibc during the compilers stage."""
1492
1493     def __init__(self, glibc):
1494         super().__init__(glibc)
1495         self.builddir = glibc.ctx.component_builddir(
1496             'compilers', glibc.compiler.name, 'glibc', glibc.name)
1497         self.installdir = glibc.compiler.sysroot
1498
1499 class GlibcPolicyForBuild(GlibcPolicyDefault):
1500     """Build policy for glibc during the glibcs stage."""
1501
1502     def __init__(self, glibc):
1503         super().__init__(glibc)
1504         self.builddir = glibc.ctx.component_builddir(
1505             'glibcs', glibc.name, 'glibc')
1506         self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1507         if glibc.ctx.strip:
1508             self.strip = glibc.tool_name('strip')
1509         else:
1510             self.strip = None
1511         self.save_logs = glibc.ctx.save_logs
1512
1513     def extra_commands(self, cmdlist):
1514         if self.strip:
1515             # Avoid picking up libc.so and libpthread.so, which are
1516             # linker scripts stored in /lib on Hurd.  libc and
1517             # libpthread are still stripped via their libc-X.YY.so
1518             # implementation files.
1519             find_command = (('find %s/lib* -name "*.so"'
1520                              + r' \! -name libc.so \! -name libpthread.so')
1521                             % self.installdir)
1522             cmdlist.add_command('strip', ['sh', '-c', ('%s $(%s)' %
1523                                   (self.strip, find_command))])
1524         cmdlist.add_command('check', ['make', 'check'])
1525         cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1526
1527 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1528     """Build policy for glibc during update-syscalls."""
1529
1530     def __init__(self, glibc):
1531         super().__init__(glibc)
1532         self.builddir = glibc.ctx.component_builddir(
1533             'update-syscalls', glibc.name, 'glibc')
1534         self.linuxdir = glibc.ctx.component_builddir(
1535             'update-syscalls', glibc.name, 'linux')
1536         self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1537             glibc, self.linuxdir)
1538         self.configure_args.insert(
1539             0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1540         # self.installdir not set because installation is not supported
1541
1542 class Glibc(object):
1543     """A configuration for building glibc."""
1544
1545     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1546                  cfg=None, ccopts=None):
1547         """Initialize a Glibc object."""
1548         self.ctx = compiler.ctx
1549         self.compiler = compiler
1550         if arch is None:
1551             self.arch = compiler.arch
1552         else:
1553             self.arch = arch
1554         if os_name is None:
1555             self.os = compiler.os
1556         else:
1557             self.os = os_name
1558         self.variant = variant
1559         if variant is None:
1560             self.name = '%s-%s' % (self.arch, self.os)
1561         else:
1562             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1563         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1564         if cfg is None:
1565             self.cfg = []
1566         else:
1567             self.cfg = cfg
1568         self.ccopts = ccopts
1569
1570     def tool_name(self, tool):
1571         """Return the name of a cross-compilation tool."""
1572         ctool = '%s-%s' % (self.compiler.triplet, tool)
1573         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1574             ctool = '%s %s' % (ctool, self.ccopts)
1575         return ctool
1576
1577     def build(self):
1578         """Generate commands to build this glibc."""
1579         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1580         installdir = self.ctx.glibc_installdir(self.name)
1581         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1582         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1583         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1584         cmdlist.add_command('check-compilers',
1585                             ['test', '-f',
1586                              os.path.join(self.compiler.installdir, 'ok')])
1587         cmdlist.use_path(self.compiler.bindir)
1588         self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1589         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1590                                       logsdir)
1591
1592     def build_glibc(self, cmdlist, policy):
1593         """Generate commands to build this glibc, either as part of a compiler
1594         build or with the bootstrapped compiler (and in the latter case, run
1595         tests as well)."""
1596         cmdlist.create_use_dir(policy.builddir)
1597         policy.configure(cmdlist)
1598         cmdlist.add_command('build', ['make'])
1599         cmdlist.add_command('install', ['make', 'install',
1600                                         'install_root=%s' % policy.installdir])
1601         # GCC uses paths such as lib/../lib64, so make sure lib
1602         # directories always exist.
1603         mkdir_cmd = ['mkdir', '-p',
1604                      os.path.join(policy.installdir, 'lib')]
1605         if policy.use_usr:
1606             mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1607         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1608         policy.extra_commands(cmdlist)
1609         cmdlist.cleanup_dir()
1610
1611     def update_syscalls(self):
1612         if self.os == 'gnu':
1613             # Hurd does not have system call tables that need updating.
1614             return
1615
1616         policy = GlibcPolicyForUpdateSyscalls(self)
1617         logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1618         self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1619         cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1620         cmdlist.add_command('check-compilers',
1621                             ['test', '-f',
1622                              os.path.join(self.compiler.installdir, 'ok')])
1623         cmdlist.use_path(self.compiler.bindir)
1624
1625         install_linux_headers(policy.linux_policy, cmdlist)
1626
1627         cmdlist.create_use_dir(policy.builddir)
1628         policy.configure(cmdlist)
1629         cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1630         cmdlist.cleanup_dir()
1631         self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1632                                       cmdlist, logsdir)
1633
1634 class Command(object):
1635     """A command run in the build process."""
1636
1637     def __init__(self, desc, num, dir, path, command, always_run=False):
1638         """Initialize a Command object."""
1639         self.dir = dir
1640         self.path = path
1641         self.desc = desc
1642         trans = str.maketrans({' ': '-'})
1643         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1644         self.command = command
1645         self.always_run = always_run
1646
1647     @staticmethod
1648     def shell_make_quote_string(s):
1649         """Given a string not containing a newline, quote it for use by the
1650         shell and make."""
1651         assert '\n' not in s
1652         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1653             return s
1654         strans = str.maketrans({"'": "'\\''"})
1655         s = "'%s'" % s.translate(strans)
1656         mtrans = str.maketrans({'$': '$$'})
1657         return s.translate(mtrans)
1658
1659     @staticmethod
1660     def shell_make_quote_list(l, translate_make):
1661         """Given a list of strings not containing newlines, quote them for use
1662         by the shell and make, returning a single string.  If translate_make
1663         is true and the first string is 'make', change it to $(MAKE)."""
1664         l = [Command.shell_make_quote_string(s) for s in l]
1665         if translate_make and l[0] == 'make':
1666             l[0] = '$(MAKE)'
1667         return ' '.join(l)
1668
1669     def shell_make_quote(self):
1670         """Return this command quoted for the shell and make."""
1671         return self.shell_make_quote_list(self.command, True)
1672
1673
1674 class CommandList(object):
1675     """A list of commands run in the build process."""
1676
1677     def __init__(self, desc, keep):
1678         """Initialize a CommandList object."""
1679         self.cmdlist = []
1680         self.dir = None
1681         self.path = None
1682         self.desc = [desc]
1683         self.keep = keep
1684
1685     def desc_txt(self, desc):
1686         """Return the description to use for a command."""
1687         return '%s %s' % (' '.join(self.desc), desc)
1688
1689     def use_dir(self, dir):
1690         """Set the default directory for subsequent commands."""
1691         self.dir = dir
1692
1693     def use_path(self, path):
1694         """Set a directory to be prepended to the PATH for subsequent
1695         commands."""
1696         self.path = path
1697
1698     def push_subdesc(self, subdesc):
1699         """Set the default subdescription for subsequent commands (e.g., the
1700         name of a component being built, within the series of commands
1701         building it)."""
1702         self.desc.append(subdesc)
1703
1704     def pop_subdesc(self):
1705         """Pop a subdescription from the list of descriptions."""
1706         self.desc.pop()
1707
1708     def create_use_dir(self, dir):
1709         """Remove and recreate a directory and use it for subsequent
1710         commands."""
1711         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1712         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1713         self.use_dir(dir)
1714
1715     def add_command_dir(self, desc, dir, command, always_run=False):
1716         """Add a command to run in a given directory."""
1717         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1718                       command, always_run)
1719         self.cmdlist.append(cmd)
1720
1721     def add_command(self, desc, command, always_run=False):
1722         """Add a command to run in the default directory."""
1723         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1724                       self.path, command, always_run)
1725         self.cmdlist.append(cmd)
1726
1727     def cleanup_dir(self, desc='cleanup', dir=None):
1728         """Clean up a build directory.  If no directory is specified, the
1729         default directory is cleaned up and ceases to be the default
1730         directory."""
1731         if dir is None:
1732             dir = self.dir
1733             self.use_dir(None)
1734         if self.keep != 'all':
1735             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1736                                  always_run=(self.keep == 'none'))
1737
1738     def makefile_commands(self, wrapper, logsdir):
1739         """Return the sequence of commands in the form of text for a Makefile.
1740         The given wrapper script takes arguments: base of logs for
1741         previous command, or empty; base of logs for this command;
1742         description; directory; PATH addition; the command itself."""
1743         # prev_base is the base of the name for logs of the previous
1744         # command that is not always-run (that is, a build command,
1745         # whose failure should stop subsequent build commands from
1746         # being run, as opposed to a cleanup command, which is run
1747         # even if previous commands failed).
1748         prev_base = ''
1749         cmds = []
1750         for c in self.cmdlist:
1751             ctxt = c.shell_make_quote()
1752             if prev_base and not c.always_run:
1753                 prev_log = os.path.join(logsdir, prev_base)
1754             else:
1755                 prev_log = ''
1756             this_log = os.path.join(logsdir, c.logbase)
1757             if not c.always_run:
1758                 prev_base = c.logbase
1759             if c.dir is None:
1760                 dir = ''
1761             else:
1762                 dir = c.dir
1763             if c.path is None:
1764                 path = ''
1765             else:
1766                 path = c.path
1767             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1768             prelim_txt = Command.shell_make_quote_list(prelims, False)
1769             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1770         return '\n'.join(cmds)
1771
1772     def status_logs(self, logsdir):
1773         """Return the list of log files with command status."""
1774         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1775                 for c in self.cmdlist]
1776
1777
1778 def get_parser():
1779     """Return an argument parser for this module."""
1780     parser = argparse.ArgumentParser(description=__doc__)
1781     parser.add_argument('-j', dest='parallelism',
1782                         help='Run this number of jobs in parallel',
1783                         type=int, default=os.cpu_count())
1784     parser.add_argument('--keep', dest='keep',
1785                         help='Whether to keep all build directories, '
1786                         'none or only those from failed builds',
1787                         default='none', choices=('none', 'all', 'failed'))
1788     parser.add_argument('--replace-sources', action='store_true',
1789                         help='Remove and replace source directories '
1790                         'with the wrong version of a component')
1791     parser.add_argument('--strip', action='store_true',
1792                         help='Strip installed glibc libraries')
1793     parser.add_argument('--full-gcc', action='store_true',
1794                         help='Build GCC with all languages and libsanitizer')
1795     parser.add_argument('--shallow', action='store_true',
1796                         help='Do not download Git history during checkout')
1797     parser.add_argument('topdir',
1798                         help='Toplevel working directory')
1799     parser.add_argument('action',
1800                         help='What to do',
1801                         choices=('checkout', 'bot-cycle', 'bot',
1802                                  'host-libraries', 'compilers', 'glibcs',
1803                                  'update-syscalls', 'list-compilers',
1804                                  'list-glibcs'))
1805     parser.add_argument('configs',
1806                         help='Versions to check out or configurations to build',
1807                         nargs='*')
1808     return parser
1809
1810
1811 def main(argv):
1812     """The main entry point."""
1813     parser = get_parser()
1814     opts = parser.parse_args(argv)
1815     topdir = os.path.abspath(opts.topdir)
1816     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1817                   opts.strip, opts.full_gcc, opts.action,
1818                   shallow=opts.shallow)
1819     ctx.run_builds(opts.action, opts.configs)
1820
1821
1822 if __name__ == '__main__':
1823     main(sys.argv[1:])