scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2024 Free Software Foundation, Inc.
   4 # Copyright The GNU Toolchain Authors.
   5 # This file is part of the GNU C Library.
   6 #
   7 # The GNU C Library is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU Lesser General Public
   9 # License as published by the Free Software Foundation; either
  10 # version 2.1 of the License, or (at your option) any later version.
  11 #
  12 # The GNU C Library is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 # Lesser General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Lesser General Public
  18 # License along with the GNU C Library; if not, see
  19 # <https://www.gnu.org/licenses/>.
  20
  21 """Build many configurations of glibc.
  22
  23 This script takes as arguments a directory name (containing a src
  24 subdirectory with sources of the relevant toolchain components) and a
  25 description of what to do: 'checkout', to check out sources into that
  26 directory, 'bot-cycle', to run a series of checkout and build steps,
  27 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  28 libraries required by the toolchain, 'compilers', to build
  29 cross-compilers for various configurations, or 'glibcs', to build
  30 glibc for various configurations and run the compilation parts of the
  31 testsuite.  Subsequent arguments name the versions of components to
  32 check out (<component>-<version), for 'checkout', or, for actions
  33 other than 'checkout' and 'bot-cycle', name configurations for which
  34 compilers or glibc are to be built.
  35
  36 The 'list-compilers' command prints the name of each available
  37 compiler configuration, without building anything.  The 'list-glibcs'
  38 command prints the name of each glibc compiler configuration, followed
  39 by the space, followed by the name of the compiler configuration used
  40 for building this glibc variant.
  41
  42 """
  43
  44 import argparse
  45 import datetime
  46 import email.mime.text
  47 import email.utils
  48 import json
  49 import os
  50 import re
  51 import shutil
  52 import smtplib
  53 import stat
  54 import subprocess
  55 import sys
  56 import time
  57 import urllib.request
  58
  59 # This is a list of system utilities that are expected to be available
  60 # to this script, and, if a non-zero version is included, the minimum
  61 # version required to work with this sccript.
  62 def get_list_of_required_tools():
  63     global REQUIRED_TOOLS
  64     REQUIRED_TOOLS = {
  65         'awk'      : (get_version_awk,   (0,0,0)),
  66         'bison'    : (get_version,       (0,0)),
  67         'flex'     : (get_version,       (0,0,0)),
  68         'git'      : (get_version,       (1,8,3)),
  69         'make'     : (get_version,       (4,0)),
  70         'makeinfo' : (get_version,       (0,0)),
  71         'patch'    : (get_version,       (0,0,0)),
  72         'sed'      : (get_version,       (0,0)),
  73         'tar'      : (get_version,       (0,0,0)),
  74         'gzip'     : (get_version,       (0,0)),
  75         'bzip2'    : (get_version_bzip2, (0,0,0)),
  76         'xz'       : (get_version,       (0,0,0)),
  77     }
  78
  79 try:
  80     subprocess.run
  81 except:
  82     class _CompletedProcess:
  83         def __init__(self, args, returncode, stdout=None, stderr=None):
  84             self.args = args
  85             self.returncode = returncode
  86             self.stdout = stdout
  87             self.stderr = stderr
  88
  89     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  90         assert(timeout is None)
  91         with subprocess.Popen(*popenargs, **kwargs) as process:
  92             try:
  93                 stdout, stderr = process.communicate(input)
  94             except:
  95                 process.kill()
  96                 process.wait()
  97                 raise
  98             returncode = process.poll()
  99             if check and returncode:
 100                 raise subprocess.CalledProcessError(returncode, popenargs)
 101         return _CompletedProcess(popenargs, returncode, stdout, stderr)
 102
 103     subprocess.run = _run
 104
 105
 106 class Context(object):
 107     """The global state associated with builds in a given directory."""
 108
 109     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
 110                  full_gcc, action, shallow=False):
 111         """Initialize the context."""
 112         self.topdir = topdir
 113         self.parallelism = parallelism
 114         self.keep = keep
 115         self.replace_sources = replace_sources
 116         self.strip = strip
 117         self.full_gcc = full_gcc
 118         self.shallow = shallow
 119         self.srcdir = os.path.join(topdir, 'src')
 120         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 121         self.build_state_json = os.path.join(topdir, 'build-state.json')
 122         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 123         self.installdir = os.path.join(topdir, 'install')
 124         self.host_libraries_installdir = os.path.join(self.installdir,
 125                                                       'host-libraries')
 126         self.builddir = os.path.join(topdir, 'build')
 127         self.logsdir = os.path.join(topdir, 'logs')
 128         self.logsdir_old = os.path.join(topdir, 'logs-old')
 129         self.makefile = os.path.join(self.builddir, 'Makefile')
 130         self.wrapper = os.path.join(self.builddir, 'wrapper')
 131         self.save_logs = os.path.join(self.builddir, 'save-logs')
 132         self.script_text = self.get_script_text()
 133         if action not in ('checkout', 'list-compilers', 'list-glibcs'):
 134             self.build_triplet = self.get_build_triplet()
 135             self.glibc_version = self.get_glibc_version()
 136         self.configs = {}
 137         self.glibc_configs = {}
 138         self.makefile_pieces = ['.PHONY: all\n']
 139         self.add_all_configs()
 140         self.load_versions_json()
 141         self.load_build_state_json()
 142         self.status_log_list = []
 143         self.email_warning = False
 144
 145     def get_script_text(self):
 146         """Return the text of this script."""
 147         with open(sys.argv[0], 'r') as f:
 148             return f.read()
 149
 150     def exec_self(self):
 151         """Re-execute this script with the same arguments."""
 152         sys.stdout.flush()
 153         os.execv(sys.executable, [sys.executable] + sys.argv)
 154
 155     def get_build_triplet(self):
 156         """Determine the build triplet with config.guess."""
 157         config_guess = os.path.join(self.component_srcdir('gcc'),
 158                                     'config.guess')
 159         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 160                                 check=True, universal_newlines=True).stdout
 161         return cg_out.rstrip()
 162
 163     def get_glibc_version(self):
 164         """Determine the glibc version number (major.minor)."""
 165         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 166         with open(version_h, 'r') as f:
 167             lines = f.readlines()
 168         starttext = '#define VERSION "'
 169         for l in lines:
 170             if l.startswith(starttext):
 171                 l = l[len(starttext):]
 172                 l = l.rstrip('"\n')
 173                 m = re.fullmatch(r'([0-9]+)\.([0-9]+)[.0-9]*', l)
 174                 return '%s.%s' % m.group(1, 2)
 175         print('error: could not determine glibc version')
 176         exit(1)
 177
 178     def add_all_configs(self):
 179         """Add all known glibc build configurations."""
 180         self.add_config(arch='aarch64',
 181                         os_name='linux-gnu',
 182                         extra_glibcs=[{'variant': 'disable-multi-arch',
 183                                        'cfg': ['--disable-multi-arch']}])
 184         self.add_config(arch='aarch64_be',
 185                         os_name='linux-gnu')
 186         self.add_config(arch='arc',
 187                         os_name='linux-gnu',
 188                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 189         self.add_config(arch='arc',
 190                         os_name='linux-gnuhf',
 191                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38_linux'])
 192         self.add_config(arch='arceb',
 193                         os_name='linux-gnu',
 194                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 195         self.add_config(arch='alpha',
 196                         os_name='linux-gnu')
 197         self.add_config(arch='arm',
 198                         os_name='linux-gnueabi',
 199                         extra_glibcs=[{'variant': 'v4t',
 200                                        'ccopts': '-march=armv4t'}])
 201         self.add_config(arch='armeb',
 202                         os_name='linux-gnueabi')
 203         self.add_config(arch='armeb',
 204                         os_name='linux-gnueabi',
 205                         variant='be8',
 206                         gcc_cfg=['--with-arch=armv7-a'])
 207         self.add_config(arch='arm',
 208                         os_name='linux-gnueabihf',
 209                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 210                         extra_glibcs=[{'variant': 'v7a',
 211                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 212                                       {'variant': 'thumb',
 213                                        'ccopts':
 214                                        '-mthumb -march=armv7-a -mfpu=vfpv3'},
 215                                       {'variant': 'v7a-disable-multi-arch',
 216                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 217                                        'cfg': ['--disable-multi-arch']}])
 218         self.add_config(arch='armeb',
 219                         os_name='linux-gnueabihf',
 220                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 221         self.add_config(arch='armeb',
 222                         os_name='linux-gnueabihf',
 223                         variant='be8',
 224                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 225                                  '--with-fpu=vfpv3'])
 226         self.add_config(arch='csky',
 227                         os_name='linux-gnuabiv2',
 228                         variant='soft',
 229                         gcc_cfg=['--disable-multilib'])
 230         self.add_config(arch='csky',
 231                         os_name='linux-gnuabiv2',
 232                         gcc_cfg=['--with-float=hard', '--disable-multilib'])
 233         self.add_config(arch='hppa',
 234                         os_name='linux-gnu')
 235         self.add_config(arch='i686',
 236                         os_name='gnu')
 237         self.add_config(arch='loongarch64',
 238                         os_name='linux-gnu',
 239                         variant='lp64d',
 240                         gcc_cfg=['--with-abi=lp64d','--disable-multilib'])
 241         self.add_config(arch='loongarch64',
 242                         os_name='linux-gnu',
 243                         variant='lp64s',
 244                         gcc_cfg=['--with-abi=lp64s','--disable-multilib'])
 245         self.add_config(arch='m68k',
 246                         os_name='linux-gnu',
 247                         gcc_cfg=['--disable-multilib'])
 248         self.add_config(arch='m68k',
 249                         os_name='linux-gnu',
 250                         variant='coldfire',
 251                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 252         self.add_config(arch='m68k',
 253                         os_name='linux-gnu',
 254                         variant='coldfire-soft',
 255                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 256                                  '--disable-multilib'])
 257         self.add_config(arch='microblaze',
 258                         os_name='linux-gnu',
 259                         gcc_cfg=['--disable-multilib'])
 260         self.add_config(arch='microblazeel',
 261                         os_name='linux-gnu',
 262                         gcc_cfg=['--disable-multilib'])
 263         self.add_config(arch='mips64',
 264                         os_name='linux-gnu',
 265                         gcc_cfg=['--with-mips-plt'],
 266                         glibcs=[{'variant': 'n32'},
 267                                 {'arch': 'mips',
 268                                  'ccopts': '-mabi=32'},
 269                                 {'variant': 'n64',
 270                                  'ccopts': '-mabi=64'}])
 271         self.add_config(arch='mips64',
 272                         os_name='linux-gnu',
 273                         variant='soft',
 274                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 275                         glibcs=[{'variant': 'n32-soft'},
 276                                 {'variant': 'soft',
 277                                  'arch': 'mips',
 278                                  'ccopts': '-mabi=32'},
 279                                 {'variant': 'n64-soft',
 280                                  'ccopts': '-mabi=64'}])
 281         self.add_config(arch='mips64',
 282                         os_name='linux-gnu',
 283                         variant='nan2008',
 284                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 285                                  '--with-arch-64=mips64r2',
 286                                  '--with-arch-32=mips32r2'],
 287                         glibcs=[{'variant': 'n32-nan2008'},
 288                                 {'variant': 'nan2008',
 289                                  'arch': 'mips',
 290                                  'ccopts': '-mabi=32'},
 291                                 {'variant': 'n64-nan2008',
 292                                  'ccopts': '-mabi=64'}])
 293         self.add_config(arch='mips64',
 294                         os_name='linux-gnu',
 295                         variant='nan2008-soft',
 296                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 297                                  '--with-arch-64=mips64r2',
 298                                  '--with-arch-32=mips32r2',
 299                                  '--with-float=soft'],
 300                         glibcs=[{'variant': 'n32-nan2008-soft'},
 301                                 {'variant': 'nan2008-soft',
 302                                  'arch': 'mips',
 303                                  'ccopts': '-mabi=32'},
 304                                 {'variant': 'n64-nan2008-soft',
 305                                  'ccopts': '-mabi=64'}])
 306         self.add_config(arch='mips64el',
 307                         os_name='linux-gnu',
 308                         gcc_cfg=['--with-mips-plt'],
 309                         glibcs=[{'variant': 'n32'},
 310                                 {'arch': 'mipsel',
 311                                  'ccopts': '-mabi=32'},
 312                                 {'variant': 'n64',
 313                                  'ccopts': '-mabi=64'}])
 314         self.add_config(arch='mips64el',
 315                         os_name='linux-gnu',
 316                         variant='soft',
 317                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 318                         glibcs=[{'variant': 'n32-soft'},
 319                                 {'variant': 'soft',
 320                                  'arch': 'mipsel',
 321                                  'ccopts': '-mabi=32'},
 322                                 {'variant': 'n64-soft',
 323                                  'ccopts': '-mabi=64'}])
 324         self.add_config(arch='mips64el',
 325                         os_name='linux-gnu',
 326                         variant='nan2008',
 327                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 328                                  '--with-arch-64=mips64r2',
 329                                  '--with-arch-32=mips32r2'],
 330                         glibcs=[{'variant': 'n32-nan2008'},
 331                                 {'variant': 'nan2008',
 332                                  'arch': 'mipsel',
 333                                  'ccopts': '-mabi=32'},
 334                                 {'variant': 'n64-nan2008',
 335                                  'ccopts': '-mabi=64'}])
 336         self.add_config(arch='mips64el',
 337                         os_name='linux-gnu',
 338                         variant='nan2008-soft',
 339                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 340                                  '--with-arch-64=mips64r2',
 341                                  '--with-arch-32=mips32r2',
 342                                  '--with-float=soft'],
 343                         glibcs=[{'variant': 'n32-nan2008-soft'},
 344                                 {'variant': 'nan2008-soft',
 345                                  'arch': 'mipsel',
 346                                  'ccopts': '-mabi=32'},
 347                                 {'variant': 'n64-nan2008-soft',
 348                                  'ccopts': '-mabi=64'}])
 349         self.add_config(arch='mipsisa64r6el',
 350                         os_name='linux-gnu',
 351                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 352                                  '--with-arch-64=mips64r6',
 353                                  '--with-arch-32=mips32r6',
 354                                  '--with-float=hard'],
 355                         glibcs=[{'variant': 'n32'},
 356                                 {'arch': 'mipsisa32r6el',
 357                                  'ccopts': '-mabi=32'},
 358                                 {'variant': 'n64',
 359                                  'ccopts': '-mabi=64'}])
 360         self.add_config(arch='nios2',
 361                         os_name='linux-gnu')
 362         self.add_config(arch='or1k',
 363                         os_name='linux-gnu',
 364                         variant='soft',
 365                         gcc_cfg=['--with-multilib-list=mcmov'])
 366         self.add_config(arch='powerpc',
 367                         os_name='linux-gnu',
 368                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 369                         extra_glibcs=[{'variant': 'power4',
 370                                        'ccopts': '-mcpu=power4',
 371                                        'cfg': ['--with-cpu=power4']}])
 372         self.add_config(arch='powerpc',
 373                         os_name='linux-gnu',
 374                         variant='soft',
 375                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 376                                  '--enable-secureplt'])
 377         self.add_config(arch='powerpc64',
 378                         os_name='linux-gnu',
 379                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 380         self.add_config(arch='powerpc64le',
 381                         os_name='linux-gnu',
 382                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 383                         extra_glibcs=[{'variant': 'disable-multi-arch',
 384                                        'cfg': ['--disable-multi-arch']}])
 385         self.add_config(arch='riscv32',
 386                         os_name='linux-gnu',
 387                         variant='rv32imac-ilp32',
 388                         gcc_cfg=['--with-arch=rv32imac', '--with-abi=ilp32',
 389                                  '--disable-multilib'])
 390         self.add_config(arch='riscv32',
 391                         os_name='linux-gnu',
 392                         variant='rv32imafdc-ilp32',
 393                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32',
 394                                  '--disable-multilib'])
 395         self.add_config(arch='riscv32',
 396                         os_name='linux-gnu',
 397                         variant='rv32imafdc-ilp32d',
 398                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
 399                                  '--disable-multilib'])
 400         self.add_config(arch='riscv64',
 401                         os_name='linux-gnu',
 402                         variant='rv64imac-lp64',
 403                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 404                                  '--disable-multilib'])
 405         self.add_config(arch='riscv64',
 406                         os_name='linux-gnu',
 407                         variant='rv64imafdc-lp64',
 408                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 409                                  '--disable-multilib'])
 410         self.add_config(arch='riscv64',
 411                         os_name='linux-gnu',
 412                         variant='rv64imafdc-lp64d',
 413                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 414                                  '--disable-multilib'])
 415         self.add_config(arch='s390x',
 416                         os_name='linux-gnu',
 417                         glibcs=[{},
 418                                 {'arch': 's390', 'ccopts': '-m31'}],
 419                         extra_glibcs=[{'variant': 'O3',
 420                                        'cflags': '-O3'}])
 421         self.add_config(arch='sh3',
 422                         os_name='linux-gnu')
 423         self.add_config(arch='sh3eb',
 424                         os_name='linux-gnu')
 425         self.add_config(arch='sh4',
 426                         os_name='linux-gnu')
 427         self.add_config(arch='sh4eb',
 428                         os_name='linux-gnu')
 429         self.add_config(arch='sh4',
 430                         os_name='linux-gnu',
 431                         variant='soft',
 432                         gcc_cfg=['--without-fp'])
 433         self.add_config(arch='sh4eb',
 434                         os_name='linux-gnu',
 435                         variant='soft',
 436                         gcc_cfg=['--without-fp'])
 437         self.add_config(arch='sparc64',
 438                         os_name='linux-gnu',
 439                         glibcs=[{'cfg' : ['--disable-default-pie']},
 440                                 {'cfg' : ['--disable-default-pie'],
 441                                  'arch': 'sparcv9',
 442                                  'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
 443                         extra_glibcs=[{'variant': 'leon3',
 444                                        'cfg' : ['--disable-default-pie'],
 445                                        'arch' : 'sparcv8',
 446                                        'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
 447                                       {'variant': 'disable-multi-arch',
 448                                        'cfg': ['--disable-multi-arch', '--disable-default-pie']},
 449                                       {'variant': 'disable-multi-arch',
 450                                        'arch': 'sparcv9',
 451                                        'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
 452                                        'cfg': ['--disable-multi-arch', '--disable-default-pie']}])
 453         self.add_config(arch='x86_64',
 454                         os_name='linux-gnu',
 455                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 456                         glibcs=[{},
 457                                 {'variant': 'x32', 'ccopts': '-mx32'},
 458                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 459                         extra_glibcs=[{'variant': 'disable-multi-arch',
 460                                        'cfg': ['--disable-multi-arch']},
 461                                       {'variant': 'minimal',
 462                                        'cfg': ['--disable-multi-arch',
 463                                                '--disable-profile',
 464                                                '--disable-timezone-tools',
 465                                                '--disable-mathvec',
 466                                                '--disable-build-nscd',
 467                                                '--disable-nscd']},
 468                                       {'variant': 'no-pie',
 469                                        'cfg': ['--disable-default-pie']},
 470                                       {'variant': 'x32-no-pie',
 471                                        'ccopts': '-mx32',
 472                                        'cfg': ['--disable-default-pie']},
 473                                       {'variant': 'no-pie',
 474                                        'arch': 'i686',
 475                                        'ccopts': '-m32 -march=i686',
 476                                        'cfg': ['--disable-default-pie']},
 477                                       {'variant': 'disable-multi-arch',
 478                                        'arch': 'i686',
 479                                        'ccopts': '-m32 -march=i686',
 480                                        'cfg': ['--disable-multi-arch']},
 481                                       {'arch': 'i486',
 482                                        'ccopts': '-m32 -march=i486'},
 483                                       {'arch': 'i586',
 484                                        'ccopts': '-m32 -march=i586'},
 485                                       {'variant': 'enable-fortify-source',
 486                                        'cfg': ['--enable-fortify-source']}])
 487         self.add_config(arch='x86_64',
 488                         os_name='gnu',
 489                         gcc_cfg=['--disable-multilib'])
 490
 491     def add_config(self, **args):
 492         """Add an individual build configuration."""
 493         cfg = Config(self, **args)
 494         if cfg.name in self.configs:
 495             print('error: duplicate config %s' % cfg.name)
 496             exit(1)
 497         self.configs[cfg.name] = cfg
 498         for c in cfg.all_glibcs:
 499             if c.name in self.glibc_configs:
 500                 print('error: duplicate glibc config %s' % c.name)
 501                 exit(1)
 502             self.glibc_configs[c.name] = c
 503
 504     def component_srcdir(self, component):
 505         """Return the source directory for a given component, e.g. gcc."""
 506         return os.path.join(self.srcdir, component)
 507
 508     def component_builddir(self, action, config, component, subconfig=None):
 509         """Return the directory to use for a build."""
 510         if config is None:
 511             # Host libraries.
 512             assert subconfig is None
 513             return os.path.join(self.builddir, action, component)
 514         if subconfig is None:
 515             return os.path.join(self.builddir, action, config, component)
 516         else:
 517             # glibc build as part of compiler build.
 518             return os.path.join(self.builddir, action, config, component,
 519                                 subconfig)
 520
 521     def compiler_installdir(self, config):
 522         """Return the directory in which to install a compiler."""
 523         return os.path.join(self.installdir, 'compilers', config)
 524
 525     def compiler_bindir(self, config):
 526         """Return the directory in which to find compiler binaries."""
 527         return os.path.join(self.compiler_installdir(config), 'bin')
 528
 529     def compiler_sysroot(self, config):
 530         """Return the sysroot directory for a compiler."""
 531         return os.path.join(self.compiler_installdir(config), 'sysroot')
 532
 533     def glibc_installdir(self, config):
 534         """Return the directory in which to install glibc."""
 535         return os.path.join(self.installdir, 'glibcs', config)
 536
 537     def run_builds(self, action, configs):
 538         """Run the requested builds."""
 539         if action == 'checkout':
 540             self.checkout(configs)
 541             return
 542         if action == 'bot-cycle':
 543             if configs:
 544                 print('error: configurations specified for bot-cycle')
 545                 exit(1)
 546             self.bot_cycle()
 547             return
 548         if action == 'bot':
 549             if configs:
 550                 print('error: configurations specified for bot')
 551                 exit(1)
 552             self.bot()
 553             return
 554         if action in ('host-libraries', 'list-compilers',
 555                       'list-glibcs') and configs:
 556             print('error: configurations specified for ' + action)
 557             exit(1)
 558         if action == 'list-compilers':
 559             for name in sorted(self.configs.keys()):
 560                 print(name)
 561             return
 562         if action == 'list-glibcs':
 563             for config in sorted(self.glibc_configs.values(),
 564                                  key=lambda c: c.name):
 565                 print(config.name, config.compiler.name)
 566             return
 567         self.clear_last_build_state(action)
 568         build_time = datetime.datetime.now(datetime.timezone.utc)
 569         if action == 'host-libraries':
 570             build_components = ('gmp', 'mpfr', 'mpc')
 571             old_components = ()
 572             old_versions = {}
 573             self.build_host_libraries()
 574         elif action == 'compilers':
 575             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 576                                 'gnumach', 'hurd')
 577             old_components = ('gmp', 'mpfr', 'mpc')
 578             old_versions = self.build_state['host-libraries']['build-versions']
 579             self.build_compilers(configs)
 580         else:
 581             build_components = ('glibc',)
 582             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 583                               'mig', 'gnumach', 'hurd')
 584             old_versions = self.build_state['compilers']['build-versions']
 585             if action == 'update-syscalls':
 586                 self.update_syscalls(configs)
 587             else:
 588                 self.build_glibcs(configs)
 589         self.write_files()
 590         self.do_build()
 591         if configs:
 592             # Partial build, do not update stored state.
 593             return
 594         build_versions = {}
 595         for k in build_components:
 596             if k in self.versions:
 597                 build_versions[k] = {'version': self.versions[k]['version'],
 598                                      'revision': self.versions[k]['revision']}
 599         for k in old_components:
 600             if k in old_versions:
 601                 build_versions[k] = {'version': old_versions[k]['version'],
 602                                      'revision': old_versions[k]['revision']}
 603         self.update_build_state(action, build_time, build_versions)
 604
 605     @staticmethod
 606     def remove_dirs(*args):
 607         """Remove directories and their contents if they exist."""
 608         for dir in args:
 609             shutil.rmtree(dir, ignore_errors=True)
 610
 611     @staticmethod
 612     def remove_recreate_dirs(*args):
 613         """Remove directories if they exist, and create them as empty."""
 614         Context.remove_dirs(*args)
 615         for dir in args:
 616             os.makedirs(dir, exist_ok=True)
 617
 618     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 619         """Add makefile text for a list of commands."""
 620         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 621         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 622                                     (target, target, target, commands))
 623         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 624
 625     def write_files(self):
 626         """Write out the Makefile and wrapper script."""
 627         mftext = ''.join(self.makefile_pieces)
 628         with open(self.makefile, 'w') as f:
 629             f.write(mftext)
 630         wrapper_text = (
 631             '#!/bin/sh\n'
 632             'prev_base=$1\n'
 633             'this_base=$2\n'
 634             'desc=$3\n'
 635             'dir=$4\n'
 636             'path=$5\n'
 637             'shift 5\n'
 638             'prev_status=$prev_base-status.txt\n'
 639             'this_status=$this_base-status.txt\n'
 640             'this_log=$this_base-log.txt\n'
 641             'date > "$this_log"\n'
 642             'echo >> "$this_log"\n'
 643             'echo "Description: $desc" >> "$this_log"\n'
 644             'printf "%s" "Command:" >> "$this_log"\n'
 645             'for word in "$@"; do\n'
 646             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 647             '    printf " %s" "$word"\n'
 648             '  else\n'
 649             '    printf " \'"\n'
 650             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 651             '    printf "\'"\n'
 652             '  fi\n'
 653             'done >> "$this_log"\n'
 654             'echo >> "$this_log"\n'
 655             'echo "Directory: $dir" >> "$this_log"\n'
 656             'echo "Path addition: $path" >> "$this_log"\n'
 657             'echo >> "$this_log"\n'
 658             'record_status ()\n'
 659             '{\n'
 660             '  echo >> "$this_log"\n'
 661             '  echo "$1: $desc" > "$this_status"\n'
 662             '  echo "$1: $desc" >> "$this_log"\n'
 663             '  echo >> "$this_log"\n'
 664             '  date >> "$this_log"\n'
 665             '  echo "$1: $desc"\n'
 666             '  exit 0\n'
 667             '}\n'
 668             'check_error ()\n'
 669             '{\n'
 670             '  if [ "$1" != "0" ]; then\n'
 671             '    record_status FAIL\n'
 672             '  fi\n'
 673             '}\n'
 674             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 675             '    record_status UNRESOLVED\n'
 676             'fi\n'
 677             'if [ "$dir" ]; then\n'
 678             '  cd "$dir"\n'
 679             '  check_error "$?"\n'
 680             'fi\n'
 681             'if [ "$path" ]; then\n'
 682             '  PATH=$path:$PATH\n'
 683             'fi\n'
 684             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 685             'check_error "$?"\n'
 686             'record_status PASS\n')
 687         with open(self.wrapper, 'w') as f:
 688             f.write(wrapper_text)
 689         # Mode 0o755.
 690         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 691                      stat.S_IROTH|stat.S_IXOTH)
 692         os.chmod(self.wrapper, mode_exec)
 693         save_logs_text = (
 694             '#!/bin/sh\n'
 695             'if ! [ -f tests.sum ]; then\n'
 696             '  echo "No test summary available."\n'
 697             '  exit 0\n'
 698             'fi\n'
 699             'save_file ()\n'
 700             '{\n'
 701             '  echo "Contents of $1:"\n'
 702             '  echo\n'
 703             '  cat "$1"\n'
 704             '  echo\n'
 705             '  echo "End of contents of $1."\n'
 706             '  echo\n'
 707             '}\n'
 708             'save_file tests.sum\n'
 709             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 710             'for t in $non_pass_tests; do\n'
 711             '  if [ -f "$t.out" ]; then\n'
 712             '    save_file "$t.out"\n'
 713             '  fi\n'
 714             'done\n')
 715         with open(self.save_logs, 'w') as f:
 716             f.write(save_logs_text)
 717         os.chmod(self.save_logs, mode_exec)
 718
 719     def do_build(self):
 720         """Do the actual build."""
 721         cmd = ['make', '-O', '-j%d' % self.parallelism]
 722         subprocess.run(cmd, cwd=self.builddir, check=True)
 723
 724     def build_host_libraries(self):
 725         """Build the host libraries."""
 726         installdir = self.host_libraries_installdir
 727         builddir = os.path.join(self.builddir, 'host-libraries')
 728         logsdir = os.path.join(self.logsdir, 'host-libraries')
 729         self.remove_recreate_dirs(installdir, builddir, logsdir)
 730         cmdlist = CommandList('host-libraries', self.keep)
 731         self.build_host_library(cmdlist, 'gmp')
 732         self.build_host_library(cmdlist, 'mpfr',
 733                                 ['--with-gmp=%s' % installdir])
 734         self.build_host_library(cmdlist, 'mpc',
 735                                 ['--with-gmp=%s' % installdir,
 736                                 '--with-mpfr=%s' % installdir])
 737         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 738         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 739
 740     def build_host_library(self, cmdlist, lib, extra_opts=None):
 741         """Build one host library."""
 742         srcdir = self.component_srcdir(lib)
 743         builddir = self.component_builddir('host-libraries', None, lib)
 744         installdir = self.host_libraries_installdir
 745         cmdlist.push_subdesc(lib)
 746         cmdlist.create_use_dir(builddir)
 747         cfg_cmd = [os.path.join(srcdir, 'configure'),
 748                    '--prefix=%s' % installdir,
 749                    '--disable-shared']
 750         if extra_opts:
 751             cfg_cmd.extend (extra_opts)
 752         cmdlist.add_command('configure', cfg_cmd)
 753         cmdlist.add_command('build', ['make'])
 754         cmdlist.add_command('check', ['make', 'check'])
 755         cmdlist.add_command('install', ['make', 'install'])
 756         cmdlist.cleanup_dir()
 757         cmdlist.pop_subdesc()
 758
 759     def build_compilers(self, configs):
 760         """Build the compilers."""
 761         if not configs:
 762             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 763             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 764             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 765             configs = sorted(self.configs.keys())
 766         for c in configs:
 767             self.configs[c].build()
 768
 769     def build_glibcs(self, configs):
 770         """Build the glibcs."""
 771         if not configs:
 772             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 773             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 774             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 775             configs = sorted(self.glibc_configs.keys())
 776         for c in configs:
 777             self.glibc_configs[c].build()
 778
 779     def update_syscalls(self, configs):
 780         """Update the glibc syscall lists."""
 781         if not configs:
 782             self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
 783             self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
 784             configs = sorted(self.glibc_configs.keys())
 785         for c in configs:
 786             self.glibc_configs[c].update_syscalls()
 787
 788     def load_versions_json(self):
 789         """Load information about source directory versions."""
 790         if not os.access(self.versions_json, os.F_OK):
 791             self.versions = {}
 792             return
 793         with open(self.versions_json, 'r') as f:
 794             self.versions = json.load(f)
 795
 796     def store_json(self, data, filename):
 797         """Store information in a JSON file."""
 798         filename_tmp = filename + '.tmp'
 799         with open(filename_tmp, 'w') as f:
 800             json.dump(data, f, indent=2, sort_keys=True)
 801         os.rename(filename_tmp, filename)
 802
 803     def store_versions_json(self):
 804         """Store information about source directory versions."""
 805         self.store_json(self.versions, self.versions_json)
 806
 807     def set_component_version(self, component, version, explicit, revision):
 808         """Set the version information for a component."""
 809         self.versions[component] = {'version': version,
 810                                     'explicit': explicit,
 811                                     'revision': revision}
 812         self.store_versions_json()
 813
 814     def checkout(self, versions):
 815         """Check out the desired component versions."""
 816         default_versions = {'binutils': 'vcs-2.42',
 817                             'gcc': 'vcs-13',
 818                             'glibc': 'vcs-mainline',
 819                             'gmp': '6.3.0',
 820                             'linux': '6.7',
 821                             'mpc': '1.3.1',
 822                             'mpfr': '4.2.1',
 823                             'mig': 'vcs-mainline',
 824                             'gnumach': 'vcs-mainline',
 825                             'hurd': 'vcs-mainline'}
 826         use_versions = {}
 827         explicit_versions = {}
 828         for v in versions:
 829             found_v = False
 830             for k in default_versions.keys():
 831                 kx = k + '-'
 832                 if v.startswith(kx):
 833                     vx = v[len(kx):]
 834                     if k in use_versions:
 835                         print('error: multiple versions for %s' % k)
 836                         exit(1)
 837                     use_versions[k] = vx
 838                     explicit_versions[k] = True
 839                     found_v = True
 840                     break
 841             if not found_v:
 842                 print('error: unknown component in %s' % v)
 843                 exit(1)
 844         for k in default_versions.keys():
 845             if k not in use_versions:
 846                 if k in self.versions and self.versions[k]['explicit']:
 847                     use_versions[k] = self.versions[k]['version']
 848                     explicit_versions[k] = True
 849                 else:
 850                     use_versions[k] = default_versions[k]
 851                     explicit_versions[k] = False
 852         os.makedirs(self.srcdir, exist_ok=True)
 853         for k in sorted(default_versions.keys()):
 854             update = os.access(self.component_srcdir(k), os.F_OK)
 855             v = use_versions[k]
 856             if (update and
 857                 k in self.versions and
 858                 v != self.versions[k]['version']):
 859                 if not self.replace_sources:
 860                     print('error: version of %s has changed from %s to %s, '
 861                           'use --replace-sources to check out again' %
 862                           (k, self.versions[k]['version'], v))
 863                     exit(1)
 864                 shutil.rmtree(self.component_srcdir(k))
 865                 update = False
 866             if v.startswith('vcs-'):
 867                 revision = self.checkout_vcs(k, v[4:], update)
 868             else:
 869                 self.checkout_tar(k, v, update)
 870                 revision = v
 871             self.set_component_version(k, v, explicit_versions[k], revision)
 872         if self.get_script_text() != self.script_text:
 873             # Rerun the checkout process in case the updated script
 874             # uses different default versions or new components.
 875             self.exec_self()
 876
 877     def checkout_vcs(self, component, version, update):
 878         """Check out the given version of the given component from version
 879         control.  Return a revision identifier."""
 880         if component == 'binutils':
 881             git_url = 'https://sourceware.org/git/binutils-gdb.git'
 882             if version == 'mainline':
 883                 git_branch = 'master'
 884             else:
 885                 trans = str.maketrans({'.': '_'})
 886                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 887             return self.git_checkout(component, git_url, git_branch, update)
 888         elif component == 'gcc':
 889             if version == 'mainline':
 890                 branch = 'master'
 891             else:
 892                 branch = 'releases/gcc-%s' % version
 893             return self.gcc_checkout(branch, update)
 894         elif component == 'glibc':
 895             git_url = 'https://sourceware.org/git/glibc.git'
 896             if version == 'mainline':
 897                 git_branch = 'master'
 898             else:
 899                 git_branch = 'release/%s/master' % version
 900             r = self.git_checkout(component, git_url, git_branch, update)
 901             self.fix_glibc_timestamps()
 902             return r
 903         elif component == 'gnumach':
 904             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 905             git_branch = 'master'
 906             r = self.git_checkout(component, git_url, git_branch, update)
 907             subprocess.run(['autoreconf', '-i'],
 908                            cwd=self.component_srcdir(component), check=True)
 909             return r
 910         elif component == 'mig':
 911             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 912             git_branch = 'master'
 913             r = self.git_checkout(component, git_url, git_branch, update)
 914             subprocess.run(['autoreconf', '-i'],
 915                            cwd=self.component_srcdir(component), check=True)
 916             return r
 917         elif component == 'hurd':
 918             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 919             git_branch = 'master'
 920             r = self.git_checkout(component, git_url, git_branch, update)
 921             subprocess.run(['autoconf'],
 922                            cwd=self.component_srcdir(component), check=True)
 923             return r
 924         else:
 925             print('error: component %s coming from VCS' % component)
 926             exit(1)
 927
 928     def git_checkout(self, component, git_url, git_branch, update):
 929         """Check out a component from git.  Return a commit identifier."""
 930         if update:
 931             subprocess.run(['git', 'remote', 'prune', 'origin'],
 932                            cwd=self.component_srcdir(component), check=True)
 933             if self.replace_sources:
 934                 subprocess.run(['git', 'clean', '-dxfq'],
 935                                cwd=self.component_srcdir(component), check=True)
 936             subprocess.run(['git', 'pull', '-q'],
 937                            cwd=self.component_srcdir(component), check=True)
 938         else:
 939             if self.shallow:
 940                 depth_arg = ('--depth', '1')
 941             else:
 942                 depth_arg = ()
 943             subprocess.run(['git', 'clone', '-q', '-b', git_branch,
 944                             *depth_arg, git_url,
 945                             self.component_srcdir(component)], check=True)
 946         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 947                            cwd=self.component_srcdir(component),
 948                            stdout=subprocess.PIPE,
 949                            check=True, universal_newlines=True).stdout
 950         return r.rstrip()
 951
 952     def fix_glibc_timestamps(self):
 953         """Fix timestamps in a glibc checkout."""
 954         # Ensure that builds do not try to regenerate generated files
 955         # in the source tree.
 956         srcdir = self.component_srcdir('glibc')
 957         # These files have Makefile dependencies to regenerate them in
 958         # the source tree that may be active during a normal build.
 959         # Some other files have such dependencies but do not need to
 960         # be touched because nothing in a build depends on the files
 961         # in question.
 962         for f in ('sysdeps/mach/hurd/bits/errno.h',):
 963             to_touch = os.path.join(srcdir, f)
 964             subprocess.run(['touch', '-c', to_touch], check=True)
 965         for dirpath, dirnames, filenames in os.walk(srcdir):
 966             for f in filenames:
 967                 if (f == 'configure' or
 968                     f == 'preconfigure' or
 969                     f.endswith('-kw.h')):
 970                     to_touch = os.path.join(dirpath, f)
 971                     subprocess.run(['touch', to_touch], check=True)
 972
 973     def gcc_checkout(self, branch, update):
 974         """Check out GCC from git.  Return the commit identifier."""
 975         if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
 976                      os.F_OK):
 977             if not self.replace_sources:
 978                 print('error: GCC has moved from SVN to git, use '
 979                       '--replace-sources to check out again')
 980                 exit(1)
 981             shutil.rmtree(self.component_srcdir('gcc'))
 982             update = False
 983         if not update:
 984             self.git_checkout('gcc', 'https://gcc.gnu.org/git/gcc.git',
 985                               branch, update)
 986         subprocess.run(['contrib/gcc_update', '--silent'],
 987                        cwd=self.component_srcdir('gcc'), check=True)
 988         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 989                            cwd=self.component_srcdir('gcc'),
 990                            stdout=subprocess.PIPE,
 991                            check=True, universal_newlines=True).stdout
 992         return r.rstrip()
 993
 994     def checkout_tar(self, component, version, update):
 995         """Check out the given version of the given component from a
 996         tarball."""
 997         if update:
 998             return
 999         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
1000                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
1001                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
1002                    'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
1003                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
1004                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
1005                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
1006                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
1007                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
1008         if component not in url_map:
1009             print('error: component %s coming from tarball' % component)
1010             exit(1)
1011         version_major = version.split('.')[0]
1012         url = url_map[component] % {'version': version, 'major': version_major}
1013         filename = os.path.join(self.srcdir, url.split('/')[-1])
1014         response = urllib.request.urlopen(url)
1015         data = response.read()
1016         with open(filename, 'wb') as f:
1017             f.write(data)
1018         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
1019                        check=True)
1020         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
1021                   self.component_srcdir(component))
1022         os.remove(filename)
1023
1024     def load_build_state_json(self):
1025         """Load information about the state of previous builds."""
1026         if os.access(self.build_state_json, os.F_OK):
1027             with open(self.build_state_json, 'r') as f:
1028                 self.build_state = json.load(f)
1029         else:
1030             self.build_state = {}
1031         for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
1032             if k not in self.build_state:
1033                 self.build_state[k] = {}
1034             if 'build-time' not in self.build_state[k]:
1035                 self.build_state[k]['build-time'] = ''
1036             if 'build-versions' not in self.build_state[k]:
1037                 self.build_state[k]['build-versions'] = {}
1038             if 'build-results' not in self.build_state[k]:
1039                 self.build_state[k]['build-results'] = {}
1040             if 'result-changes' not in self.build_state[k]:
1041                 self.build_state[k]['result-changes'] = {}
1042             if 'ever-passed' not in self.build_state[k]:
1043                 self.build_state[k]['ever-passed'] = []
1044
1045     def store_build_state_json(self):
1046         """Store information about the state of previous builds."""
1047         self.store_json(self.build_state, self.build_state_json)
1048
1049     def clear_last_build_state(self, action):
1050         """Clear information about the state of part of the build."""
1051         # We clear the last build time and versions when starting a
1052         # new build.  The results of the last build are kept around,
1053         # as comparison is still meaningful if this build is aborted
1054         # and a new one started.
1055         self.build_state[action]['build-time'] = ''
1056         self.build_state[action]['build-versions'] = {}
1057         self.store_build_state_json()
1058
1059     def update_build_state(self, action, build_time, build_versions):
1060         """Update the build state after a build."""
1061         build_time = build_time.replace(microsecond=0)
1062         self.build_state[action]['build-time'] = build_time.strftime(
1063             '%Y-%m-%d %H:%M:%S')
1064         self.build_state[action]['build-versions'] = build_versions
1065         build_results = {}
1066         for log in self.status_log_list:
1067             with open(log, 'r') as f:
1068                 log_text = f.read()
1069             log_text = log_text.rstrip()
1070             m = re.fullmatch('([A-Z]+): (.*)', log_text)
1071             result = m.group(1)
1072             test_name = m.group(2)
1073             assert test_name not in build_results
1074             build_results[test_name] = result
1075         old_build_results = self.build_state[action]['build-results']
1076         self.build_state[action]['build-results'] = build_results
1077         result_changes = {}
1078         all_tests = set(old_build_results.keys()) | set(build_results.keys())
1079         for t in all_tests:
1080             if t in old_build_results:
1081                 old_res = old_build_results[t]
1082             else:
1083                 old_res = '(New test)'
1084             if t in build_results:
1085                 new_res = build_results[t]
1086             else:
1087                 new_res = '(Test removed)'
1088             if old_res != new_res:
1089                 result_changes[t] = '%s -> %s' % (old_res, new_res)
1090         self.build_state[action]['result-changes'] = result_changes
1091         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1092                            if t in build_results}
1093         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1094         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1095                                                          new_passes)
1096         self.store_build_state_json()
1097
1098     def load_bot_config_json(self):
1099         """Load bot configuration."""
1100         with open(self.bot_config_json, 'r') as f:
1101             self.bot_config = json.load(f)
1102
1103     def part_build_old(self, action, delay):
1104         """Return whether the last build for a given action was at least a
1105         given number of seconds ago, or does not have a time recorded."""
1106         old_time_str = self.build_state[action]['build-time']
1107         if not old_time_str:
1108             return True
1109         old_time = datetime.datetime.strptime(
1110             old_time_str, '%Y-%m-%d %H:%M:%S').replace(
1111                 tzinfo=datetime.timezone.utc)
1112         new_time = datetime.datetime.now(datetime.timezone.utc)
1113         delta = new_time - old_time
1114         return delta.total_seconds() >= delay
1115
1116     def bot_cycle(self):
1117         """Run a single round of checkout and builds."""
1118         print('Bot cycle starting %s.'
1119               % str(datetime.datetime.now(datetime.timezone.utc)))
1120         self.load_bot_config_json()
1121         actions = ('host-libraries', 'compilers', 'glibcs')
1122         self.bot_run_self(['--replace-sources'], 'checkout')
1123         self.load_versions_json()
1124         if self.get_script_text() != self.script_text:
1125             print('Script changed, re-execing.')
1126             # On script change, all parts of the build should be rerun.
1127             for a in actions:
1128                 self.clear_last_build_state(a)
1129             self.exec_self()
1130         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1131                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1132                                           'mig', 'gnumach', 'hurd'),
1133                             'glibcs': ('glibc',)}
1134         must_build = {}
1135         for a in actions:
1136             build_vers = self.build_state[a]['build-versions']
1137             must_build[a] = False
1138             if not self.build_state[a]['build-time']:
1139                 must_build[a] = True
1140             old_vers = {}
1141             new_vers = {}
1142             for c in check_components[a]:
1143                 if c in build_vers:
1144                     old_vers[c] = build_vers[c]
1145                 new_vers[c] = {'version': self.versions[c]['version'],
1146                                'revision': self.versions[c]['revision']}
1147             if new_vers == old_vers:
1148                 print('Versions for %s unchanged.' % a)
1149             else:
1150                 print('Versions changed or rebuild forced for %s.' % a)
1151                 if a == 'compilers' and not self.part_build_old(
1152                         a, self.bot_config['compilers-rebuild-delay']):
1153                     print('Not requiring rebuild of compilers this soon.')
1154                 else:
1155                     must_build[a] = True
1156         if must_build['host-libraries']:
1157             must_build['compilers'] = True
1158         if must_build['compilers']:
1159             must_build['glibcs'] = True
1160         for a in actions:
1161             if must_build[a]:
1162                 print('Must rebuild %s.' % a)
1163                 self.clear_last_build_state(a)
1164             else:
1165                 print('No need to rebuild %s.' % a)
1166         if os.access(self.logsdir, os.F_OK):
1167             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1168             shutil.copytree(self.logsdir, self.logsdir_old)
1169         for a in actions:
1170             if must_build[a]:
1171                 build_time = datetime.datetime.now(datetime.timezone.utc)
1172                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1173                 self.bot_run_self([], a)
1174                 self.load_build_state_json()
1175                 self.bot_build_mail(a, build_time)
1176         print('Bot cycle done at %s.'
1177               % str(datetime.datetime.now(datetime.timezone.utc)))
1178
1179     def bot_build_mail(self, action, build_time):
1180         """Send email with the results of a build."""
1181         if not ('email-from' in self.bot_config and
1182                 'email-server' in self.bot_config and
1183                 'email-subject' in self.bot_config and
1184                 'email-to' in self.bot_config):
1185             if not self.email_warning:
1186                 print("Email not configured, not sending.")
1187                 self.email_warning = True
1188             return
1189
1190         build_time = build_time.replace(microsecond=0)
1191         subject = (self.bot_config['email-subject'] %
1192                    {'action': action,
1193                     'build-time': build_time.strftime('%Y-%m-%d %H:%M:%S')})
1194         results = self.build_state[action]['build-results']
1195         changes = self.build_state[action]['result-changes']
1196         ever_passed = set(self.build_state[action]['ever-passed'])
1197         versions = self.build_state[action]['build-versions']
1198         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1199         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1200         all_fails = {k for k in results if results[k] == 'FAIL'}
1201         if new_regressions:
1202             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1203             new_reg_text = ('New regressions:\n\n%s\n\n' %
1204                             '\n'.join(new_reg_list))
1205         else:
1206             new_reg_text = ''
1207         if all_regressions:
1208             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1209             all_reg_text = ('All regressions:\n\n%s\n\n' %
1210                             '\n'.join(all_reg_list))
1211         else:
1212             all_reg_text = ''
1213         if all_fails:
1214             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1215             all_fail_text = ('All failures:\n\n%s\n\n' %
1216                              '\n'.join(all_fail_list))
1217         else:
1218             all_fail_text = ''
1219         if changes:
1220             changes_list = sorted(changes.keys())
1221             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1222             changes_text = ('All changed results:\n\n%s\n\n' %
1223                             '\n'.join(changes_list))
1224         else:
1225             changes_text = ''
1226         results_text = (new_reg_text + all_reg_text + all_fail_text +
1227                         changes_text)
1228         if not results_text:
1229             results_text = 'Clean build with unchanged results.\n\n'
1230         versions_list = sorted(versions.keys())
1231         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1232                                           versions[k]['revision'])
1233                          for k in versions_list]
1234         versions_text = ('Component versions for this build:\n\n%s\n' %
1235                          '\n'.join(versions_list))
1236         body_text = results_text + versions_text
1237         msg = email.mime.text.MIMEText(body_text)
1238         msg['Subject'] = subject
1239         msg['From'] = self.bot_config['email-from']
1240         msg['To'] = self.bot_config['email-to']
1241         msg['Message-ID'] = email.utils.make_msgid()
1242         msg['Date'] = email.utils.format_datetime(
1243             datetime.datetime.now(datetime.timezone.utc))
1244         with smtplib.SMTP(self.bot_config['email-server']) as s:
1245             s.send_message(msg)
1246
1247     def bot_run_self(self, opts, action, check=True):
1248         """Run a copy of this script with given options."""
1249         cmd = [sys.executable, sys.argv[0], '--keep=none',
1250                '-j%d' % self.parallelism]
1251         if self.full_gcc:
1252             cmd.append('--full-gcc')
1253         cmd.extend(opts)
1254         cmd.extend([self.topdir, action])
1255         sys.stdout.flush()
1256         subprocess.run(cmd, check=check)
1257
1258     def bot(self):
1259         """Run repeated rounds of checkout and builds."""
1260         while True:
1261             self.load_bot_config_json()
1262             if not self.bot_config['run']:
1263                 print('Bot exiting by request.')
1264                 exit(0)
1265             self.bot_run_self([], 'bot-cycle', check=False)
1266             self.load_bot_config_json()
1267             if not self.bot_config['run']:
1268                 print('Bot exiting by request.')
1269                 exit(0)
1270             time.sleep(self.bot_config['delay'])
1271             if self.get_script_text() != self.script_text:
1272                 print('Script changed, bot re-execing.')
1273                 self.exec_self()
1274
1275 class LinuxHeadersPolicyForBuild(object):
1276     """Names and directories for installing Linux headers.  Build variant."""
1277
1278     def __init__(self, config):
1279         self.arch = config.arch
1280         self.srcdir = config.ctx.component_srcdir('linux')
1281         self.builddir = config.component_builddir('linux')
1282         self.headers_dir = os.path.join(config.sysroot, 'usr')
1283
1284 class LinuxHeadersPolicyForUpdateSyscalls(object):
1285     """Names and directories for Linux headers.  update-syscalls variant."""
1286
1287     def __init__(self, glibc, headers_dir):
1288         self.arch = glibc.compiler.arch
1289         self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1290         self.builddir = glibc.ctx.component_builddir(
1291             'update-syscalls', glibc.name, 'build-linux')
1292         self.headers_dir = headers_dir
1293
1294 def install_linux_headers(policy, cmdlist):
1295     """Install Linux kernel headers."""
1296     arch_map = {'aarch64': 'arm64',
1297                 'alpha': 'alpha',
1298                 'arc': 'arc',
1299                 'arm': 'arm',
1300                 'csky': 'csky',
1301                 'hppa': 'parisc',
1302                 'i486': 'x86',
1303                 'i586': 'x86',
1304                 'i686': 'x86',
1305                 'i786': 'x86',
1306                 'loongarch64': 'loongarch',
1307                 'm68k': 'm68k',
1308                 'microblaze': 'microblaze',
1309                 'mips': 'mips',
1310                 'nios2': 'nios2',
1311                 'or1k': 'openrisc',
1312                 'powerpc': 'powerpc',
1313                 's390': 's390',
1314                 'riscv32': 'riscv',
1315                 'riscv64': 'riscv',
1316                 'sh': 'sh',
1317                 'sparc': 'sparc',
1318                 'x86_64': 'x86'}
1319     linux_arch = None
1320     for k in arch_map:
1321         if policy.arch.startswith(k):
1322             linux_arch = arch_map[k]
1323             break
1324     assert linux_arch is not None
1325     cmdlist.push_subdesc('linux')
1326     cmdlist.create_use_dir(policy.builddir)
1327     cmdlist.add_command('install-headers',
1328                         ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1329                          'ARCH=%s' % linux_arch,
1330                          'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1331                          'headers_install'])
1332     cmdlist.cleanup_dir()
1333     cmdlist.pop_subdesc()
1334
1335 class Config(object):
1336     """A configuration for building a compiler and associated libraries."""
1337
1338     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1339                  first_gcc_cfg=None, binutils_cfg=None, glibcs=None,
1340                  extra_glibcs=None):
1341         """Initialize a Config object."""
1342         self.ctx = ctx
1343         self.arch = arch
1344         self.os = os_name
1345         self.variant = variant
1346         if variant is None:
1347             self.name = '%s-%s' % (arch, os_name)
1348         else:
1349             self.name = '%s-%s-%s' % (arch, os_name, variant)
1350         self.triplet = '%s-glibc-%s' % (arch, os_name)
1351         if gcc_cfg is None:
1352             self.gcc_cfg = []
1353         else:
1354             self.gcc_cfg = gcc_cfg
1355         if first_gcc_cfg is None:
1356             self.first_gcc_cfg = []
1357         else:
1358             self.first_gcc_cfg = first_gcc_cfg
1359         if binutils_cfg is None:
1360             self.binutils_cfg = []
1361         else:
1362             self.binutils_cfg = binutils_cfg
1363         if glibcs is None:
1364             glibcs = [{'variant': variant}]
1365         if extra_glibcs is None:
1366             extra_glibcs = []
1367         glibcs = [Glibc(self, **g) for g in glibcs]
1368         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1369         self.all_glibcs = glibcs + extra_glibcs
1370         self.compiler_glibcs = glibcs
1371         self.installdir = ctx.compiler_installdir(self.name)
1372         self.bindir = ctx.compiler_bindir(self.name)
1373         self.sysroot = ctx.compiler_sysroot(self.name)
1374         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1375         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1376
1377     def component_builddir(self, component):
1378         """Return the directory to use for a (non-glibc) build."""
1379         return self.ctx.component_builddir('compilers', self.name, component)
1380
1381     def build(self):
1382         """Generate commands to build this compiler."""
1383         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1384                                       self.logsdir)
1385         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1386         cmdlist.add_command('check-host-libraries',
1387                             ['test', '-f',
1388                              os.path.join(self.ctx.host_libraries_installdir,
1389                                           'ok')])
1390         cmdlist.use_path(self.bindir)
1391         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1392                               ['--disable-gdb',
1393                                '--disable-gdbserver',
1394                                '--disable-libdecnumber',
1395                                '--disable-readline',
1396                                '--disable-sim'] + self.binutils_cfg)
1397         if self.os.startswith('linux'):
1398             install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1399         self.build_gcc(cmdlist, True)
1400         if self.os == 'gnu':
1401             self.install_gnumach_headers(cmdlist)
1402             self.build_cross_tool(cmdlist, 'mig', 'mig')
1403             self.install_hurd_headers(cmdlist)
1404         for g in self.compiler_glibcs:
1405             cmdlist.push_subdesc('glibc')
1406             cmdlist.push_subdesc(g.name)
1407             g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1408             cmdlist.pop_subdesc()
1409             cmdlist.pop_subdesc()
1410         self.build_gcc(cmdlist, False)
1411         cmdlist.add_command('done', ['touch',
1412                                      os.path.join(self.installdir, 'ok')])
1413         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1414                                       self.logsdir)
1415
1416     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1417         """Build one cross tool."""
1418         srcdir = self.ctx.component_srcdir(tool_src)
1419         builddir = self.component_builddir(tool_build)
1420         cmdlist.push_subdesc(tool_build)
1421         cmdlist.create_use_dir(builddir)
1422         cfg_cmd = [os.path.join(srcdir, 'configure'),
1423                    '--prefix=%s' % self.installdir,
1424                    '--build=%s' % self.ctx.build_triplet,
1425                    '--host=%s' % self.ctx.build_triplet,
1426                    '--target=%s' % self.triplet,
1427                    '--with-sysroot=%s' % self.sysroot]
1428         if extra_opts:
1429             cfg_cmd.extend(extra_opts)
1430         cmdlist.add_command('configure', cfg_cmd)
1431         cmdlist.add_command('build', ['make'])
1432         # Parallel "make install" for GCC has race conditions that can
1433         # cause it to fail; see
1434         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1435         # problems are not known for binutils, but doing the
1436         # installation in parallel within a particular toolchain build
1437         # (as opposed to installation of one toolchain from
1438         # build-many-glibcs.py running in parallel to the installation
1439         # of other toolchains being built) is not known to be
1440         # significantly beneficial, so it is simplest just to disable
1441         # parallel install for cross tools here.
1442         cmdlist.add_command('install', ['make', '-j1', 'install'])
1443         cmdlist.cleanup_dir()
1444         cmdlist.pop_subdesc()
1445
1446     def install_gnumach_headers(self, cmdlist):
1447         """Install GNU Mach headers."""
1448         srcdir = self.ctx.component_srcdir('gnumach')
1449         builddir = self.component_builddir('gnumach')
1450         cmdlist.push_subdesc('gnumach')
1451         cmdlist.create_use_dir(builddir)
1452         cmdlist.add_command('configure',
1453                             [os.path.join(srcdir, 'configure'),
1454                              '--build=%s' % self.ctx.build_triplet,
1455                              '--host=%s' % self.triplet,
1456                              '--prefix=',
1457                              '--disable-user32',
1458                              'CC=%s-gcc -nostdlib' % self.triplet])
1459         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1460                                         'install-data'])
1461         cmdlist.cleanup_dir()
1462         cmdlist.pop_subdesc()
1463
1464     def install_hurd_headers(self, cmdlist):
1465         """Install Hurd headers."""
1466         srcdir = self.ctx.component_srcdir('hurd')
1467         builddir = self.component_builddir('hurd')
1468         cmdlist.push_subdesc('hurd')
1469         cmdlist.create_use_dir(builddir)
1470         cmdlist.add_command('configure',
1471                             [os.path.join(srcdir, 'configure'),
1472                              '--build=%s' % self.ctx.build_triplet,
1473                              '--host=%s' % self.triplet,
1474                              '--prefix=',
1475                              '--disable-profile', '--without-parted',
1476                              'CC=%s-gcc -nostdlib' % self.triplet])
1477         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1478                                         'no_deps=t', 'install-headers'])
1479         cmdlist.cleanup_dir()
1480         cmdlist.pop_subdesc()
1481
1482     def build_gcc(self, cmdlist, bootstrap):
1483         """Build GCC."""
1484         # libssp is of little relevance with glibc's own stack
1485         # checking support.  libcilkrts does not support GNU/Hurd (and
1486         # has been removed in GCC 8, so --disable-libcilkrts can be
1487         # removed once glibc no longer supports building with older
1488         # GCC versions).  --enable-initfini-array is enabled by default
1489         # in GCC 12, which can be removed when GCC 12 becomes the
1490         # minimum requirement.
1491         cfg_opts = list(self.gcc_cfg)
1492         cfg_opts += ['--enable-initfini-array']
1493         cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1494         host_libs = self.ctx.host_libraries_installdir
1495         cfg_opts += ['--with-gmp=%s' % host_libs,
1496                      '--with-mpfr=%s' % host_libs,
1497                      '--with-mpc=%s' % host_libs]
1498         if bootstrap:
1499             tool_build = 'gcc-first'
1500             # Building a static-only, C-only compiler that is
1501             # sufficient to build glibc.  Various libraries and
1502             # features that may require libc headers must be disabled.
1503             # When configuring with a sysroot, --with-newlib is
1504             # required to define inhibit_libc (to stop some parts of
1505             # libgcc including libc headers); --without-headers is not
1506             # sufficient.
1507             cfg_opts += ['--enable-languages=c', '--disable-shared',
1508                          '--disable-threads',
1509                          '--disable-libatomic',
1510                          '--disable-decimal-float',
1511                          '--disable-gcov',
1512                          '--disable-libffi',
1513                          '--disable-libgomp',
1514                          '--disable-libitm',
1515                          '--disable-libmpx',
1516                          '--disable-libquadmath',
1517                          '--disable-libsanitizer',
1518                          '--without-headers', '--with-newlib',
1519                          '--with-glibc-version=%s' % self.ctx.glibc_version
1520                          ]
1521             cfg_opts += self.first_gcc_cfg
1522         else:
1523             tool_build = 'gcc'
1524             # libsanitizer commonly breaks because of glibc header
1525             # changes, or on unusual targets.  C++ pre-compiled
1526             # headers are not used during the glibc build and are
1527             # expensive to create.
1528             if not self.ctx.full_gcc:
1529                 cfg_opts += ['--disable-libsanitizer',
1530                              '--disable-libstdcxx-pch']
1531             langs = 'all' if self.ctx.full_gcc else 'c,c++'
1532             cfg_opts += ['--enable-languages=%s' % langs,
1533                          '--enable-shared', '--enable-threads']
1534         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1535
1536 class GlibcPolicyDefault(object):
1537     """Build policy for glibc: common defaults."""
1538
1539     def __init__(self, glibc):
1540         self.srcdir = glibc.ctx.component_srcdir('glibc')
1541         self.use_usr = glibc.os != 'gnu'
1542         self.prefix = '/usr' if self.use_usr else ''
1543         self.configure_args = [
1544             '--prefix=%s' % self.prefix,
1545             '--enable-profile',
1546             '--build=%s' % glibc.ctx.build_triplet,
1547             '--host=%s' % glibc.triplet,
1548             'CC=%s' % glibc.tool_name('gcc'),
1549             'CXX=%s' % glibc.tool_name('g++'),
1550         ]
1551         if glibc.os == 'gnu':
1552             self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1553         if glibc.cflags:
1554             self.configure_args.append('CFLAGS=%s' % glibc.cflags)
1555             self.configure_args.append('CXXFLAGS=%s' % glibc.cflags)
1556         self.configure_args += glibc.cfg
1557
1558     def configure(self, cmdlist):
1559         """Invoked to add the configure command to the command list."""
1560         cmdlist.add_command('configure',
1561                             [os.path.join(self.srcdir, 'configure'),
1562                              *self.configure_args])
1563
1564     def extra_commands(self, cmdlist):
1565         """Invoked to inject additional commands (make check) after build."""
1566         pass
1567
1568 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1569     """Build policy for glibc during the compilers stage."""
1570
1571     def __init__(self, glibc):
1572         super().__init__(glibc)
1573         self.builddir = glibc.ctx.component_builddir(
1574             'compilers', glibc.compiler.name, 'glibc', glibc.name)
1575         self.installdir = glibc.compiler.sysroot
1576
1577 class GlibcPolicyForBuild(GlibcPolicyDefault):
1578     """Build policy for glibc during the glibcs stage."""
1579
1580     def __init__(self, glibc):
1581         super().__init__(glibc)
1582         self.builddir = glibc.ctx.component_builddir(
1583             'glibcs', glibc.name, 'glibc')
1584         self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1585         if glibc.ctx.strip:
1586             self.strip = glibc.tool_name('strip')
1587         else:
1588             self.strip = None
1589         self.save_logs = glibc.ctx.save_logs
1590
1591     def extra_commands(self, cmdlist):
1592         if self.strip:
1593             # Avoid stripping libc.so and libpthread.so, which are
1594             # linker scripts stored in /lib on Hurd.
1595             find_command = 'find %s/lib* -name "*.so*"' % self.installdir
1596             cmdlist.add_command('strip', ['sh', '-c', (
1597                 'set -e; for f in $(%s); do '
1598                 'if ! head -c16 $f | grep -q "GNU ld script"; then %s $f; fi; '
1599                 'done' % (find_command, self.strip))])
1600         cmdlist.add_command('check', ['make', 'check'])
1601         cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1602
1603 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1604     """Build policy for glibc during update-syscalls."""
1605
1606     def __init__(self, glibc):
1607         super().__init__(glibc)
1608         self.builddir = glibc.ctx.component_builddir(
1609             'update-syscalls', glibc.name, 'glibc')
1610         self.linuxdir = glibc.ctx.component_builddir(
1611             'update-syscalls', glibc.name, 'linux')
1612         self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1613             glibc, self.linuxdir)
1614         self.configure_args.insert(
1615             0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1616         # self.installdir not set because installation is not supported
1617
1618 class Glibc(object):
1619     """A configuration for building glibc."""
1620
1621     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1622                  cfg=None, ccopts=None, cflags=None):
1623         """Initialize a Glibc object."""
1624         self.ctx = compiler.ctx
1625         self.compiler = compiler
1626         if arch is None:
1627             self.arch = compiler.arch
1628         else:
1629             self.arch = arch
1630         if os_name is None:
1631             self.os = compiler.os
1632         else:
1633             self.os = os_name
1634         self.variant = variant
1635         if variant is None:
1636             self.name = '%s-%s' % (self.arch, self.os)
1637         else:
1638             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1639         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1640         if cfg is None:
1641             self.cfg = []
1642         else:
1643             self.cfg = cfg
1644         # ccopts contain ABI options and are passed to configure as CC / CXX.
1645         self.ccopts = ccopts
1646         # cflags contain non-ABI options like -g or -O and are passed to
1647         # configure as CFLAGS / CXXFLAGS.
1648         self.cflags = cflags
1649
1650     def tool_name(self, tool):
1651         """Return the name of a cross-compilation tool."""
1652         ctool = '%s-%s' % (self.compiler.triplet, tool)
1653         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1654             ctool = '%s %s' % (ctool, self.ccopts)
1655         return ctool
1656
1657     def build(self):
1658         """Generate commands to build this glibc."""
1659         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1660         installdir = self.ctx.glibc_installdir(self.name)
1661         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1662         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1663         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1664         cmdlist.add_command('check-compilers',
1665                             ['test', '-f',
1666                              os.path.join(self.compiler.installdir, 'ok')])
1667         cmdlist.use_path(self.compiler.bindir)
1668         self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1669         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1670                                       logsdir)
1671
1672     def build_glibc(self, cmdlist, policy):
1673         """Generate commands to build this glibc, either as part of a compiler
1674         build or with the bootstrapped compiler (and in the latter case, run
1675         tests as well)."""
1676         cmdlist.create_use_dir(policy.builddir)
1677         policy.configure(cmdlist)
1678         cmdlist.add_command('build', ['make'])
1679         cmdlist.add_command('install', ['make', 'install',
1680                                         'install_root=%s' % policy.installdir])
1681         # GCC uses paths such as lib/../lib64, so make sure lib
1682         # directories always exist.
1683         mkdir_cmd = ['mkdir', '-p',
1684                      os.path.join(policy.installdir, 'lib')]
1685         if policy.use_usr:
1686             mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1687         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1688         policy.extra_commands(cmdlist)
1689         cmdlist.cleanup_dir()
1690
1691     def update_syscalls(self):
1692         if self.os == 'gnu':
1693             # Hurd does not have system call tables that need updating.
1694             return
1695
1696         policy = GlibcPolicyForUpdateSyscalls(self)
1697         logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1698         self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1699         cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1700         cmdlist.add_command('check-compilers',
1701                             ['test', '-f',
1702                              os.path.join(self.compiler.installdir, 'ok')])
1703         cmdlist.use_path(self.compiler.bindir)
1704
1705         install_linux_headers(policy.linux_policy, cmdlist)
1706
1707         cmdlist.create_use_dir(policy.builddir)
1708         policy.configure(cmdlist)
1709         cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1710         cmdlist.cleanup_dir()
1711         self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1712                                       cmdlist, logsdir)
1713
1714 class Command(object):
1715     """A command run in the build process."""
1716
1717     def __init__(self, desc, num, dir, path, command, always_run=False):
1718         """Initialize a Command object."""
1719         self.dir = dir
1720         self.path = path
1721         self.desc = desc
1722         trans = str.maketrans({' ': '-'})
1723         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1724         self.command = command
1725         self.always_run = always_run
1726
1727     @staticmethod
1728     def shell_make_quote_string(s):
1729         """Given a string not containing a newline, quote it for use by the
1730         shell and make."""
1731         assert '\n' not in s
1732         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1733             return s
1734         strans = str.maketrans({"'": "'\\''"})
1735         s = "'%s'" % s.translate(strans)
1736         mtrans = str.maketrans({'$': '$$'})
1737         return s.translate(mtrans)
1738
1739     @staticmethod
1740     def shell_make_quote_list(l, translate_make):
1741         """Given a list of strings not containing newlines, quote them for use
1742         by the shell and make, returning a single string.  If translate_make
1743         is true and the first string is 'make', change it to $(MAKE)."""
1744         l = [Command.shell_make_quote_string(s) for s in l]
1745         if translate_make and l[0] == 'make':
1746             l[0] = '$(MAKE)'
1747         return ' '.join(l)
1748
1749     def shell_make_quote(self):
1750         """Return this command quoted for the shell and make."""
1751         return self.shell_make_quote_list(self.command, True)
1752
1753
1754 class CommandList(object):
1755     """A list of commands run in the build process."""
1756
1757     def __init__(self, desc, keep):
1758         """Initialize a CommandList object."""
1759         self.cmdlist = []
1760         self.dir = None
1761         self.path = None
1762         self.desc = [desc]
1763         self.keep = keep
1764
1765     def desc_txt(self, desc):
1766         """Return the description to use for a command."""
1767         return '%s %s' % (' '.join(self.desc), desc)
1768
1769     def use_dir(self, dir):
1770         """Set the default directory for subsequent commands."""
1771         self.dir = dir
1772
1773     def use_path(self, path):
1774         """Set a directory to be prepended to the PATH for subsequent
1775         commands."""
1776         self.path = path
1777
1778     def push_subdesc(self, subdesc):
1779         """Set the default subdescription for subsequent commands (e.g., the
1780         name of a component being built, within the series of commands
1781         building it)."""
1782         self.desc.append(subdesc)
1783
1784     def pop_subdesc(self):
1785         """Pop a subdescription from the list of descriptions."""
1786         self.desc.pop()
1787
1788     def create_use_dir(self, dir):
1789         """Remove and recreate a directory and use it for subsequent
1790         commands."""
1791         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1792         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1793         self.use_dir(dir)
1794
1795     def add_command_dir(self, desc, dir, command, always_run=False):
1796         """Add a command to run in a given directory."""
1797         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1798                       command, always_run)
1799         self.cmdlist.append(cmd)
1800
1801     def add_command(self, desc, command, always_run=False):
1802         """Add a command to run in the default directory."""
1803         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1804                       self.path, command, always_run)
1805         self.cmdlist.append(cmd)
1806
1807     def cleanup_dir(self, desc='cleanup', dir=None):
1808         """Clean up a build directory.  If no directory is specified, the
1809         default directory is cleaned up and ceases to be the default
1810         directory."""
1811         if dir is None:
1812             dir = self.dir
1813             self.use_dir(None)
1814         if self.keep != 'all':
1815             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1816                                  always_run=(self.keep == 'none'))
1817
1818     def makefile_commands(self, wrapper, logsdir):
1819         """Return the sequence of commands in the form of text for a Makefile.
1820         The given wrapper script takes arguments: base of logs for
1821         previous command, or empty; base of logs for this command;
1822         description; directory; PATH addition; the command itself."""
1823         # prev_base is the base of the name for logs of the previous
1824         # command that is not always-run (that is, a build command,
1825         # whose failure should stop subsequent build commands from
1826         # being run, as opposed to a cleanup command, which is run
1827         # even if previous commands failed).
1828         prev_base = ''
1829         cmds = []
1830         for c in self.cmdlist:
1831             ctxt = c.shell_make_quote()
1832             if prev_base and not c.always_run:
1833                 prev_log = os.path.join(logsdir, prev_base)
1834             else:
1835                 prev_log = ''
1836             this_log = os.path.join(logsdir, c.logbase)
1837             if not c.always_run:
1838                 prev_base = c.logbase
1839             if c.dir is None:
1840                 dir = ''
1841             else:
1842                 dir = c.dir
1843             if c.path is None:
1844                 path = ''
1845             else:
1846                 path = c.path
1847             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1848             prelim_txt = Command.shell_make_quote_list(prelims, False)
1849             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1850         return '\n'.join(cmds)
1851
1852     def status_logs(self, logsdir):
1853         """Return the list of log files with command status."""
1854         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1855                 for c in self.cmdlist]
1856
1857
1858 def get_parser():
1859     """Return an argument parser for this module."""
1860     parser = argparse.ArgumentParser(description=__doc__)
1861     parser.add_argument('-j', dest='parallelism',
1862                         help='Run this number of jobs in parallel',
1863                         type=int, default=os.cpu_count())
1864     parser.add_argument('--keep', dest='keep',
1865                         help='Whether to keep all build directories, '
1866                         'none or only those from failed builds',
1867                         default='none', choices=('none', 'all', 'failed'))
1868     parser.add_argument('--replace-sources', action='store_true',
1869                         help='Remove and replace source directories '
1870                         'with the wrong version of a component')
1871     parser.add_argument('--strip', action='store_true',
1872                         help='Strip installed glibc libraries')
1873     parser.add_argument('--full-gcc', action='store_true',
1874                         help='Build GCC with all languages and libsanitizer')
1875     parser.add_argument('--shallow', action='store_true',
1876                         help='Do not download Git history during checkout')
1877     parser.add_argument('topdir',
1878                         help='Toplevel working directory')
1879     parser.add_argument('action',
1880                         help='What to do',
1881                         choices=('checkout', 'bot-cycle', 'bot',
1882                                  'host-libraries', 'compilers', 'glibcs',
1883                                  'update-syscalls', 'list-compilers',
1884                                  'list-glibcs'))
1885     parser.add_argument('configs',
1886                         help='Versions to check out or configurations to build',
1887                         nargs='*')
1888     return parser
1889
1890
1891 def get_version_common(progname,line,word,delchars,arg1):
1892     try:
1893         out = subprocess.run([progname, arg1],
1894                              stdout=subprocess.PIPE,
1895                              stderr=subprocess.DEVNULL,
1896                              stdin=subprocess.DEVNULL,
1897                              check=True, universal_newlines=True)
1898         v = out.stdout.splitlines()[line].split()[word]
1899         if delchars:
1900             v = v.replace(delchars,'')
1901         return [int(x) for x in v.split('.')]
1902     except:
1903         return 'missing';
1904
1905 def get_version_common_stderr(progname,line,word,delchars,arg1):
1906     try:
1907         out = subprocess.run([progname, arg1],
1908                              stdout=subprocess.DEVNULL,
1909                              stderr=subprocess.PIPE,
1910                              stdin=subprocess.DEVNULL,
1911                              check=True, universal_newlines=True)
1912         v = out.stderr.splitlines()[line].split()[word]
1913         if delchars:
1914             v = v.replace(delchars,'')
1915         return [int(x) for x in v.split('.')]
1916     except:
1917         return 'missing';
1918
1919 def get_version(progname):
1920     return get_version_common (progname, 0, -1, None, '--version');
1921
1922 def get_version_awk(progname):
1923     return get_version_common (progname, 0, 2, ',', '--version');
1924
1925 def get_version_bzip2(progname):
1926     return get_version_common_stderr (progname, 0, 6, ',', '-h');
1927
1928 def check_version(ver, req):
1929     for v, r in zip(ver, req):
1930         if v > r:
1931             return True
1932         if v < r:
1933             return False
1934     return True
1935
1936 def version_str(ver):
1937     return '.'.join([str (x) for x in ver])
1938
1939 def check_for_required_tools():
1940     get_list_of_required_tools()
1941     count_old_tools = 0
1942     count_missing_tools = 0
1943
1944     for k, v in REQUIRED_TOOLS.items():
1945         version = v[0](k)
1946         if version == 'missing':
1947             ok = 'missing'
1948         else:
1949             ok = 'ok' if check_version (version, v[1]) else 'old'
1950         if ok == 'old':
1951             if count_old_tools == 0:
1952                 print("One or more required tools are too old:")
1953             count_old_tools = count_old_tools + 1
1954             print('{:9}: {:3} (obtained=\"{}\" required=\"{}\")'.format(k, ok,
1955                     version_str(version), version_str(v[1])))
1956         if ok == 'missing':
1957             if count_missing_tools == 0:
1958                 print("One or more required tools are missing:")
1959             count_missing_tools = count_missing_tools + 1
1960             print('{:9}: {:3} (required=\"{}\")'.format(k, ok,
1961                     version_str(v[1])))
1962
1963     if count_old_tools > 0 or count_missing_tools > 0:
1964         exit (1);
1965
1966 def main(argv):
1967     """The main entry point."""
1968     check_for_required_tools();
1969     parser = get_parser()
1970     opts = parser.parse_args(argv)
1971     topdir = os.path.abspath(opts.topdir)
1972     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1973                   opts.strip, opts.full_gcc, opts.action,
1974                   shallow=opts.shallow)
1975     ctx.run_builds(opts.action, opts.configs)
1976
1977
1978 if __name__ == '__main__':
1979     main(sys.argv[1:])