1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2012 The Chromium OS Authors.
7 from html.parser import HTMLParser
11 import urllib.request, urllib.error, urllib.parse
13 from buildman import bsettings
14 from patman import command
15 from patman import terminal
16 from patman import tools
18 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
19 PRIORITY_CALC) = list(range(4))
21 (VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)
23 # Simple class to collect links from a page
24 class MyHTMLParser(HTMLParser):
25 def __init__(self, arch):
26 """Create a new parser
28 After the parser runs, self.links will be set to a list of the links
29 to .xz archives found in the page, and self.arch_link will be set to
30 the one for the given architecture (or None if not found).
33 arch: Architecture to search for
35 HTMLParser.__init__(self)
38 self.re_arch = re.compile('[-_]%s-' % arch)
40 def handle_starttag(self, tag, attrs):
42 for tag, value in attrs:
44 if value and value.endswith('.xz'):
45 self.links.append(value)
46 if self.re_arch.search(value):
47 self.arch_link = value
54 gcc: Full path to C compiler
55 path: Directory path containing C compiler
56 cross: Cross compile string, e.g. 'arm-linux-'
57 arch: Architecture of toolchain as determined from the first
58 component of the filename. E.g. arm-linux-gcc becomes arm
59 priority: Toolchain priority (0=highest, 20=lowest)
60 override_toolchain: Toolchain to use for sandbox, overriding the normal
63 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
64 arch=None, override_toolchain=None):
65 """Create a new toolchain object.
68 fname: Filename of the gcc component
69 test: True to run the toolchain to test it
70 verbose: True to print out the information
71 priority: Priority to use for this toolchain, or PRIORITY_CALC to
75 self.path = os.path.dirname(fname)
76 self.override_toolchain = override_toolchain
78 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
79 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
80 basename = os.path.basename(fname)
81 pos = basename.rfind('-')
82 self.cross = basename[:pos + 1] if pos != -1 else ''
84 # The architecture is the first part of the name
85 pos = self.cross.find('-')
89 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
90 if self.arch == 'sandbox' and override_toolchain:
91 self.gcc = override_toolchain
93 env = self.MakeEnvironment(False)
95 # As a basic sanity check, run the C compiler with --version
96 cmd = [fname, '--version']
97 if priority == PRIORITY_CALC:
98 self.priority = self.GetPriority(fname)
100 self.priority = priority
102 result = command.run_pipe([cmd], capture=True, env=env,
103 raise_on_error=False)
104 self.ok = result.return_code == 0
106 print('Tool chain test: ', end=' ')
108 print("OK, arch='%s', priority %d" % (self.arch,
112 print('Command: ', cmd)
118 def GetPriority(self, fname):
119 """Return the priority of the toolchain.
121 Toolchains are ranked according to their suitability by their
125 fname: Filename of toolchain
127 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
129 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
130 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
131 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
132 '-linux-gnueabihf', '-le-linux', '-uclinux']
133 for prio in range(len(priority_list)):
134 if priority_list[prio] in fname:
135 return PRIORITY_CALC + prio
136 return PRIORITY_CALC + prio
138 def GetWrapper(self, show_warning=True):
139 """Get toolchain wrapper from the setting file.
142 for name, value in bsettings.GetItems('toolchain-wrapper'):
144 print("Warning: Wrapper not found")
150 def GetEnvArgs(self, which):
151 """Get an environment variable/args value based on the the toolchain
154 which: VAR_... value to get
157 Value of that environment variable or arguments
159 wrapper = self.GetWrapper()
160 if which == VAR_CROSS_COMPILE:
161 return wrapper + os.path.join(self.path, self.cross)
162 elif which == VAR_PATH:
164 elif which == VAR_ARCH:
166 elif which == VAR_MAKE_ARGS:
167 args = self.MakeArgs()
169 return ' '.join(args)
172 raise ValueError('Unknown arg to GetEnvArgs (%d)' % which)
174 def MakeEnvironment(self, full_path):
175 """Returns an environment for using the toolchain.
177 Thie takes the current environment and adds CROSS_COMPILE so that
178 the tool chain will operate correctly. This also disables localized
179 output and possibly unicode encoded output of all build tools by
182 Note that os.environb is used to obtain the environment, since in some
183 cases the environment many contain non-ASCII characters and we see
186 UnicodeEncodeError: 'utf-8' codec can't encode characters in position
187 569-570: surrogates not allowed
190 full_path: Return the full path in CROSS_COMPILE and don't set
193 Dict containing the (bytes) environment to use. This is based on the
194 current environment, with changes as needed to CROSS_COMPILE, PATH
197 env = dict(os.environb)
198 wrapper = self.GetWrapper()
200 if self.override_toolchain:
201 # We'll use MakeArgs() to provide this
204 env[b'CROSS_COMPILE'] = tools.to_bytes(
205 wrapper + os.path.join(self.path, self.cross))
207 env[b'CROSS_COMPILE'] = tools.to_bytes(wrapper + self.cross)
208 env[b'PATH'] = tools.to_bytes(self.path) + b':' + env[b'PATH']
210 env[b'LC_ALL'] = b'C'
215 """Create the 'make' arguments for a toolchain
217 This is only used when the toolchain is being overridden. Since the
218 U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
219 environment (and MakeEnvironment()) to override these values. This
220 function returns the arguments to accomplish this.
223 List of arguments to pass to 'make'
225 if self.override_toolchain:
226 return ['HOSTCC=%s' % self.override_toolchain,
227 'CC=%s' % self.override_toolchain]
232 """Manage a list of toolchains for building U-Boot
234 We select one toolchain for each architecture type
237 toolchains: Dict of Toolchain objects, keyed by architecture name
238 prefixes: Dict of prefixes to check, keyed by architecture. This can
239 be a full path and toolchain prefix, for example
240 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
241 something on the search path, for example
242 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
243 paths: List of paths to check for toolchains (may contain wildcards)
246 def __init__(self, override_toolchain=None):
250 self.override_toolchain = override_toolchain
251 self._make_flags = dict(bsettings.GetItems('make-flags'))
253 def GetPathList(self, show_warning=True):
254 """Get a list of available toolchain paths
257 show_warning: True to show a warning if there are no tool chains.
260 List of strings, each a path to a toolchain mentioned in the
261 [toolchain] section of the settings file.
263 toolchains = bsettings.GetItems('toolchain')
264 if show_warning and not toolchains:
265 print(("Warning: No tool chains. Please run 'buildman "
266 "--fetch-arch all' to download all available toolchains, or "
267 "add a [toolchain] section to your buildman config file "
268 "%s. See README for details" %
269 bsettings.config_fname))
272 for name, value in toolchains:
274 paths += glob.glob(value)
279 def GetSettings(self, show_warning=True):
280 """Get toolchain settings from the settings file.
283 show_warning: True to show a warning if there are no tool chains.
285 self.prefixes = bsettings.GetItems('toolchain-prefix')
286 self.paths += self.GetPathList(show_warning)
288 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
290 """Add a toolchain to our list
292 We select the given toolchain as our preferred one for its
293 architecture if it is a higher priority than the others.
296 fname: Filename of toolchain's gcc driver
297 test: True to run the toolchain to test it
298 priority: Priority to use for this toolchain
299 arch: Toolchain architecture, or None if not known
301 toolchain = Toolchain(fname, test, verbose, priority, arch,
302 self.override_toolchain)
303 add_it = toolchain.ok
304 if toolchain.arch in self.toolchains:
305 add_it = (toolchain.priority <
306 self.toolchains[toolchain.arch].priority)
308 self.toolchains[toolchain.arch] = toolchain
310 print(("Toolchain '%s' at priority %d will be ignored because "
311 "another toolchain for arch '%s' has priority %d" %
312 (toolchain.gcc, toolchain.priority, toolchain.arch,
313 self.toolchains[toolchain.arch].priority)))
315 def ScanPath(self, path, verbose):
316 """Scan a path for a valid toolchain
320 verbose: True to print out progress information
322 Filename of C compiler if found, else None
325 for subdir in ['.', 'bin', 'usr/bin']:
326 dirname = os.path.join(path, subdir)
327 if verbose: print(" - looking in '%s'" % dirname)
328 for fname in glob.glob(dirname + '/*gcc'):
329 if verbose: print(" - found '%s'" % fname)
333 def ScanPathEnv(self, fname):
334 """Scan the PATH environment variable for a given filename.
337 fname: Filename to scan for
339 List of matching pathanames, or [] if none
342 for path in os.environ["PATH"].split(os.pathsep):
343 path = path.strip('"')
344 pathname = os.path.join(path, fname)
345 if os.path.exists(pathname):
346 pathname_list.append(pathname)
349 def Scan(self, verbose):
350 """Scan for available toolchains and select the best for each arch.
352 We look for all the toolchains we can file, figure out the
353 architecture for each, and whether it works. Then we select the
354 highest priority toolchain for each arch.
357 verbose: True to print out progress information
359 if verbose: print('Scanning for tool chains')
360 for name, value in self.prefixes:
361 if verbose: print(" - scanning prefix '%s'" % value)
362 if os.path.exists(value):
363 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
365 fname = value + 'gcc'
366 if os.path.exists(fname):
367 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
369 fname_list = self.ScanPathEnv(fname)
371 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
373 raise ValueError("No tool chain found for prefix '%s'" %
375 for path in self.paths:
376 if verbose: print(" - scanning path '%s'" % path)
377 fnames = self.ScanPath(path, verbose)
379 self.Add(fname, True, verbose)
382 """List out the selected toolchains for each architecture"""
383 col = terminal.Color()
384 print(col.Color(col.BLUE, 'List of available toolchains (%d):' %
385 len(self.toolchains)))
386 if len(self.toolchains):
387 for key, value in sorted(self.toolchains.items()):
388 print('%-10s: %s' % (key, value.gcc))
392 def Select(self, arch):
393 """Returns the toolchain for a given architecture
396 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
399 toolchain object, or None if none found
401 for tag, value in bsettings.GetItems('toolchain-alias'):
403 for alias in value.split():
404 if alias in self.toolchains:
405 return self.toolchains[alias]
407 if not arch in self.toolchains:
408 raise ValueError("No tool chain found for arch '%s'" % arch)
409 return self.toolchains[arch]
411 def ResolveReferences(self, var_dict, args):
412 """Resolve variable references in a string
414 This converts ${blah} within the string to the value of blah.
415 This function works recursively.
418 var_dict: Dictionary containing variables and their values
419 args: String containing make arguments
423 >>> bsettings.Setup()
424 >>> tcs = Toolchains()
425 >>> tcs.Add('fred', False)
426 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
428 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
430 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
431 'this=OBLIQUE_setfi2ndrstnd'
433 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
436 m = re_var.search(args)
439 lookup = m.group(0)[2:-1]
440 value = var_dict.get(lookup, '')
441 args = args[:m.start(0)] + value + args[m.end(0):]
444 def GetMakeArguments(self, board):
445 """Returns 'make' arguments for a given board
447 The flags are in a section called 'make-flags'. Flags are named
448 after the target they represent, for example snapper9260=TESTING=1
449 will pass TESTING=1 to make when building the snapper9260 board.
451 References to other boards can be added in the string also. For
455 at91-boards=ENABLE_AT91_TEST=1
456 snapper9260=${at91-boards} BUILD_TAG=442
457 snapper9g45=${at91-boards} BUILD_TAG=443
459 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
460 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
462 A special 'target' variable is set to the board target.
465 board: Board object for the board to check.
467 'make' flags for that board, or '' if none
469 self._make_flags['target'] = board.target
470 arg_str = self.ResolveReferences(self._make_flags,
471 self._make_flags.get(board.target, ''))
472 args = re.findall("(?:\".*?\"|\S)+", arg_str)
475 args[i] = args[i].replace('"', '')
482 def LocateArchUrl(self, fetch_arch):
483 """Find a toolchain available online
485 Look in standard places for available toolchains. At present the
486 only standard place is at kernel.org.
489 arch: Architecture to look for, or 'list' for all
491 If fetch_arch is 'list', a tuple:
492 Machine architecture (e.g. x86_64)
495 URL containing this toolchain, if avaialble, else None
497 arch = command.output_one_line('uname', '-m')
498 if arch == 'aarch64':
500 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
501 versions = ['11.1.0', '9.2.0', '7.3.0', '6.4.0', '4.9.4']
503 for version in versions:
504 url = '%s/%s/%s/' % (base, arch, version)
505 print('Checking: %s' % url)
506 response = urllib.request.urlopen(url)
507 html = tools.to_string(response.read())
508 parser = MyHTMLParser(fetch_arch)
510 if fetch_arch == 'list':
511 links += parser.links
512 elif parser.arch_link:
513 return url + parser.arch_link
514 if fetch_arch == 'list':
518 def Unpack(self, fname, dest):
522 fname: Filename to unpack
523 dest: Destination directory
525 Directory name of the first entry in the archive, without the
528 stdout = command.output('tar', 'xvfJ', fname, '-C', dest)
529 dirs = stdout.splitlines()[1].split('/')[:2]
530 return '/'.join(dirs)
532 def TestSettingsHasPath(self, path):
533 """Check if buildman will find this toolchain
536 True if the path is in settings, False if not
538 paths = self.GetPathList(False)
542 """List architectures with available toolchains to download"""
543 host_arch, archives = self.LocateArchUrl('list')
544 re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
546 for archive in archives:
547 # Remove the host architecture from the start
548 arch = re_arch.match(archive[len(host_arch):])
550 if arch.group(1) != '2.0' and arch.group(1) != '64':
551 arch_set.add(arch.group(1))
552 return sorted(arch_set)
554 def FetchAndInstall(self, arch):
555 """Fetch and install a new toolchain
558 Architecture to fetch, or 'list' to list
560 # Fist get the URL for this architecture
561 col = terminal.Color()
562 print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch))
563 url = self.LocateArchUrl(arch)
565 print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
568 home = os.environ['HOME']
569 dest = os.path.join(home, '.buildman-toolchains')
570 if not os.path.exists(dest):
573 # Download the tar file for this toolchain and unpack it
574 tarfile, tmpdir = tools.download(url, '.buildman')
577 print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ')
579 path = self.Unpack(tarfile, dest)
584 # Check that the toolchain works
585 print(col.Color(col.GREEN, 'Testing'))
586 dirpath = os.path.join(dest, path)
587 compiler_fname_list = self.ScanPath(dirpath, True)
588 if not compiler_fname_list:
589 print('Could not locate C compiler - fetch failed.')
591 if len(compiler_fname_list) != 1:
592 print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
593 ', '.join(compiler_fname_list)))
594 toolchain = Toolchain(compiler_fname_list[0], True, True)
596 # Make sure that it will be found by buildman
597 if not self.TestSettingsHasPath(dirpath):
598 print(("Adding 'download' to config file '%s'" %
599 bsettings.config_fname))
600 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)