1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2012 The Chromium OS Authors.
7 from html.parser import HTMLParser
11 import urllib.request, urllib.error, urllib.parse
13 from buildman import bsettings
14 from u_boot_pylib import command
15 from u_boot_pylib import terminal
16 from u_boot_pylib import tools
18 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
19 PRIORITY_CALC) = list(range(4))
21 (VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)
23 # Simple class to collect links from a page
24 class MyHTMLParser(HTMLParser):
25 def __init__(self, arch):
26 """Create a new parser
28 After the parser runs, self.links will be set to a list of the links
29 to .xz archives found in the page, and self.arch_link will be set to
30 the one for the given architecture (or None if not found).
33 arch: Architecture to search for
35 HTMLParser.__init__(self)
38 self.re_arch = re.compile('[-_]%s-' % arch)
40 def handle_starttag(self, tag, attrs):
42 for tag, value in attrs:
44 if value and value.endswith('.xz'):
45 self.links.append(value)
46 if self.re_arch.search(value):
47 self.arch_link = value
54 gcc: Full path to C compiler
55 path: Directory path containing C compiler
56 cross: Cross compile string, e.g. 'arm-linux-'
57 arch: Architecture of toolchain as determined from the first
58 component of the filename. E.g. arm-linux-gcc becomes arm
59 priority: Toolchain priority (0=highest, 20=lowest)
60 override_toolchain: Toolchain to use for sandbox, overriding the normal
63 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
64 arch=None, override_toolchain=None):
65 """Create a new toolchain object.
68 fname: Filename of the gcc component
69 test: True to run the toolchain to test it
70 verbose: True to print out the information
71 priority: Priority to use for this toolchain, or PRIORITY_CALC to
75 self.path = os.path.dirname(fname)
76 self.override_toolchain = override_toolchain
78 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
79 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
80 basename = os.path.basename(fname)
81 pos = basename.rfind('-')
82 self.cross = basename[:pos + 1] if pos != -1 else ''
84 # The architecture is the first part of the name
85 pos = self.cross.find('-')
89 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
90 if self.arch == 'sandbox' and override_toolchain:
91 self.gcc = override_toolchain
93 env = self.MakeEnvironment(False)
95 # As a basic sanity check, run the C compiler with --version
96 cmd = [fname, '--version']
97 if priority == PRIORITY_CALC:
98 self.priority = self.GetPriority(fname)
100 self.priority = priority
102 result = command.run_pipe([cmd], capture=True, env=env,
103 raise_on_error=False)
104 self.ok = result.return_code == 0
106 print('Tool chain test: ', end=' ')
108 print("OK, arch='%s', priority %d" % (self.arch,
112 print('Command: ', cmd)
118 def GetPriority(self, fname):
119 """Return the priority of the toolchain.
121 Toolchains are ranked according to their suitability by their
125 fname: Filename of toolchain
127 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
129 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
130 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
131 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
132 '-linux-gnueabihf', '-le-linux', '-uclinux']
133 for prio in range(len(priority_list)):
134 if priority_list[prio] in fname:
135 return PRIORITY_CALC + prio
136 return PRIORITY_CALC + prio
138 def GetWrapper(self, show_warning=True):
139 """Get toolchain wrapper from the setting file.
142 for name, value in bsettings.get_items('toolchain-wrapper'):
144 print("Warning: Wrapper not found")
150 def GetEnvArgs(self, which):
151 """Get an environment variable/args value based on the the toolchain
154 which: VAR_... value to get
157 Value of that environment variable or arguments
159 if which == VAR_CROSS_COMPILE:
160 wrapper = self.GetWrapper()
161 base = '' if self.arch == 'sandbox' else self.path
162 return wrapper + os.path.join(base, self.cross)
163 elif which == VAR_PATH:
165 elif which == VAR_ARCH:
167 elif which == VAR_MAKE_ARGS:
168 args = self.MakeArgs()
170 return ' '.join(args)
173 raise ValueError('Unknown arg to GetEnvArgs (%d)' % which)
175 def MakeEnvironment(self, full_path):
176 """Returns an environment for using the toolchain.
178 Thie takes the current environment and adds CROSS_COMPILE so that
179 the tool chain will operate correctly. This also disables localized
180 output and possibly unicode encoded output of all build tools by
183 Note that os.environb is used to obtain the environment, since in some
184 cases the environment many contain non-ASCII characters and we see
187 UnicodeEncodeError: 'utf-8' codec can't encode characters in position
188 569-570: surrogates not allowed
191 full_path: Return the full path in CROSS_COMPILE and don't set
194 Dict containing the (bytes) environment to use. This is based on the
195 current environment, with changes as needed to CROSS_COMPILE, PATH
198 env = dict(os.environb)
199 wrapper = self.GetWrapper()
201 if self.override_toolchain:
202 # We'll use MakeArgs() to provide this
205 env[b'CROSS_COMPILE'] = tools.to_bytes(
206 wrapper + os.path.join(self.path, self.cross))
208 env[b'CROSS_COMPILE'] = tools.to_bytes(wrapper + self.cross)
209 env[b'PATH'] = tools.to_bytes(self.path) + b':' + env[b'PATH']
211 env[b'LC_ALL'] = b'C'
216 """Create the 'make' arguments for a toolchain
218 This is only used when the toolchain is being overridden. Since the
219 U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
220 environment (and MakeEnvironment()) to override these values. This
221 function returns the arguments to accomplish this.
224 List of arguments to pass to 'make'
226 if self.override_toolchain:
227 return ['HOSTCC=%s' % self.override_toolchain,
228 'CC=%s' % self.override_toolchain]
233 """Manage a list of toolchains for building U-Boot
235 We select one toolchain for each architecture type
238 toolchains: Dict of Toolchain objects, keyed by architecture name
239 prefixes: Dict of prefixes to check, keyed by architecture. This can
240 be a full path and toolchain prefix, for example
241 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
242 something on the search path, for example
243 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
244 paths: List of paths to check for toolchains (may contain wildcards)
247 def __init__(self, override_toolchain=None):
251 self.override_toolchain = override_toolchain
252 self._make_flags = dict(bsettings.get_items('make-flags'))
254 def GetPathList(self, show_warning=True):
255 """Get a list of available toolchain paths
258 show_warning: True to show a warning if there are no tool chains.
261 List of strings, each a path to a toolchain mentioned in the
262 [toolchain] section of the settings file.
264 toolchains = bsettings.get_items('toolchain')
265 if show_warning and not toolchains:
266 print(("Warning: No tool chains. Please run 'buildman "
267 "--fetch-arch all' to download all available toolchains, or "
268 "add a [toolchain] section to your buildman config file "
269 "%s. See buildman.rst for details" %
270 bsettings.config_fname))
273 for name, value in toolchains:
275 paths += glob.glob(value)
280 def GetSettings(self, show_warning=True):
281 """Get toolchain settings from the settings file.
284 show_warning: True to show a warning if there are no tool chains.
286 self.prefixes = bsettings.get_items('toolchain-prefix')
287 self.paths += self.GetPathList(show_warning)
289 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
291 """Add a toolchain to our list
293 We select the given toolchain as our preferred one for its
294 architecture if it is a higher priority than the others.
297 fname: Filename of toolchain's gcc driver
298 test: True to run the toolchain to test it
299 priority: Priority to use for this toolchain
300 arch: Toolchain architecture, or None if not known
302 toolchain = Toolchain(fname, test, verbose, priority, arch,
303 self.override_toolchain)
304 add_it = toolchain.ok
305 if toolchain.arch in self.toolchains:
306 add_it = (toolchain.priority <
307 self.toolchains[toolchain.arch].priority)
309 self.toolchains[toolchain.arch] = toolchain
311 print(("Toolchain '%s' at priority %d will be ignored because "
312 "another toolchain for arch '%s' has priority %d" %
313 (toolchain.gcc, toolchain.priority, toolchain.arch,
314 self.toolchains[toolchain.arch].priority)))
316 def ScanPath(self, path, verbose):
317 """Scan a path for a valid toolchain
321 verbose: True to print out progress information
323 Filename of C compiler if found, else None
326 for subdir in ['.', 'bin', 'usr/bin']:
327 dirname = os.path.join(path, subdir)
328 if verbose: print(" - looking in '%s'" % dirname)
329 for fname in glob.glob(dirname + '/*gcc'):
330 if verbose: print(" - found '%s'" % fname)
334 def ScanPathEnv(self, fname):
335 """Scan the PATH environment variable for a given filename.
338 fname: Filename to scan for
340 List of matching pathanames, or [] if none
343 for path in os.environ["PATH"].split(os.pathsep):
344 path = path.strip('"')
345 pathname = os.path.join(path, fname)
346 if os.path.exists(pathname):
347 pathname_list.append(pathname)
350 def Scan(self, verbose):
351 """Scan for available toolchains and select the best for each arch.
353 We look for all the toolchains we can file, figure out the
354 architecture for each, and whether it works. Then we select the
355 highest priority toolchain for each arch.
358 verbose: True to print out progress information
360 if verbose: print('Scanning for tool chains')
361 for name, value in self.prefixes:
362 if verbose: print(" - scanning prefix '%s'" % value)
363 if os.path.exists(value):
364 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
366 fname = value + 'gcc'
367 if os.path.exists(fname):
368 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
370 fname_list = self.ScanPathEnv(fname)
372 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
374 raise ValueError("No tool chain found for prefix '%s'" %
376 for path in self.paths:
377 if verbose: print(" - scanning path '%s'" % path)
378 fnames = self.ScanPath(path, verbose)
380 self.Add(fname, True, verbose)
383 """List out the selected toolchains for each architecture"""
384 col = terminal.Color()
385 print(col.build(col.BLUE, 'List of available toolchains (%d):' %
386 len(self.toolchains)))
387 if len(self.toolchains):
388 for key, value in sorted(self.toolchains.items()):
389 print('%-10s: %s' % (key, value.gcc))
393 def Select(self, arch):
394 """Returns the toolchain for a given architecture
397 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
400 toolchain object, or None if none found
402 for tag, value in bsettings.get_items('toolchain-alias'):
404 for alias in value.split():
405 if alias in self.toolchains:
406 return self.toolchains[alias]
408 if not arch in self.toolchains:
409 raise ValueError("No tool chain found for arch '%s'" % arch)
410 return self.toolchains[arch]
412 def ResolveReferences(self, var_dict, args):
413 """Resolve variable references in a string
415 This converts ${blah} within the string to the value of blah.
416 This function works recursively.
419 var_dict: Dictionary containing variables and their values
420 args: String containing make arguments
424 >>> bsettings.setup(None)
425 >>> tcs = Toolchains()
426 >>> tcs.Add('fred', False)
427 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
429 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
431 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
432 'this=OBLIQUE_setfi2ndrstnd'
434 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
437 m = re_var.search(args)
440 lookup = m.group(0)[2:-1]
441 value = var_dict.get(lookup, '')
442 args = args[:m.start(0)] + value + args[m.end(0):]
445 def GetMakeArguments(self, brd):
446 """Returns 'make' arguments for a given board
448 The flags are in a section called 'make-flags'. Flags are named
449 after the target they represent, for example snapper9260=TESTING=1
450 will pass TESTING=1 to make when building the snapper9260 board.
452 References to other boards can be added in the string also. For
456 at91-boards=ENABLE_AT91_TEST=1
457 snapper9260=${at91-boards} BUILD_TAG=442
458 snapper9g45=${at91-boards} BUILD_TAG=443
460 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
461 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
463 A special 'target' variable is set to the board target.
466 brd: Board object for the board to check.
468 'make' flags for that board, or '' if none
470 self._make_flags['target'] = brd.target
471 arg_str = self.ResolveReferences(self._make_flags,
472 self._make_flags.get(brd.target, ''))
473 args = re.findall("(?:\".*?\"|\S)+", arg_str)
476 args[i] = args[i].replace('"', '')
483 def LocateArchUrl(self, fetch_arch):
484 """Find a toolchain available online
486 Look in standard places for available toolchains. At present the
487 only standard place is at kernel.org.
490 arch: Architecture to look for, or 'list' for all
492 If fetch_arch is 'list', a tuple:
493 Machine architecture (e.g. x86_64)
496 URL containing this toolchain, if avaialble, else None
498 arch = command.output_one_line('uname', '-m')
499 if arch == 'aarch64':
501 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
502 versions = ['13.1.0', '12.2.0']
504 for version in versions:
505 url = '%s/%s/%s/' % (base, arch, version)
506 print('Checking: %s' % url)
507 response = urllib.request.urlopen(url)
508 html = tools.to_string(response.read())
509 parser = MyHTMLParser(fetch_arch)
511 if fetch_arch == 'list':
512 links += parser.links
513 elif parser.arch_link:
514 return url + parser.arch_link
515 if fetch_arch == 'list':
519 def Unpack(self, fname, dest):
523 fname: Filename to unpack
524 dest: Destination directory
526 Directory name of the first entry in the archive, without the
529 stdout = command.output('tar', 'xvfJ', fname, '-C', dest)
530 dirs = stdout.splitlines()[1].split('/')[:2]
531 return '/'.join(dirs)
533 def TestSettingsHasPath(self, path):
534 """Check if buildman will find this toolchain
537 True if the path is in settings, False if not
539 paths = self.GetPathList(False)
543 """List architectures with available toolchains to download"""
544 host_arch, archives = self.LocateArchUrl('list')
545 re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
547 for archive in archives:
548 # Remove the host architecture from the start
549 arch = re_arch.match(archive[len(host_arch):])
551 if arch.group(1) != '2.0' and arch.group(1) != '64':
552 arch_set.add(arch.group(1))
553 return sorted(arch_set)
555 def FetchAndInstall(self, arch):
556 """Fetch and install a new toolchain
559 Architecture to fetch, or 'list' to list
561 # Fist get the URL for this architecture
562 col = terminal.Color()
563 print(col.build(col.BLUE, "Downloading toolchain for arch '%s'" % arch))
564 url = self.LocateArchUrl(arch)
566 print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
569 home = os.environ['HOME']
570 dest = os.path.join(home, '.buildman-toolchains')
571 if not os.path.exists(dest):
574 # Download the tar file for this toolchain and unpack it
575 tarfile, tmpdir = tools.download(url, '.buildman')
578 print(col.build(col.GREEN, 'Unpacking to: %s' % dest), end=' ')
580 path = self.Unpack(tarfile, dest)
585 # Check that the toolchain works
586 print(col.build(col.GREEN, 'Testing'))
587 dirpath = os.path.join(dest, path)
588 compiler_fname_list = self.ScanPath(dirpath, True)
589 if not compiler_fname_list:
590 print('Could not locate C compiler - fetch failed.')
592 if len(compiler_fname_list) != 1:
593 print(col.build(col.RED, 'Warning, ambiguous toolchains: %s' %
594 ', '.join(compiler_fname_list)))
595 toolchain = Toolchain(compiler_fname_list[0], True, True)
597 # Make sure that it will be found by buildman
598 if not self.TestSettingsHasPath(dirpath):
599 print(("Adding 'download' to config file '%s'" %
600 bsettings.config_fname))
601 bsettings.set_item('toolchain', 'download', '%s/*/*' % dest)