1 # Copyright (c) 2012 The Chromium OS Authors.
3 # SPDX-License-Identifier: GPL-2.0+
8 from HTMLParser import HTMLParser
17 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
18 PRIORITY_CALC) = range(4)
20 # Simple class to collect links from a page
21 class MyHTMLParser(HTMLParser):
22 def __init__(self, arch):
23 """Create a new parser
25 After the parser runs, self.links will be set to a list of the links
26 to .xz archives found in the page, and self.arch_link will be set to
27 the one for the given architecture (or None if not found).
30 arch: Architecture to search for
32 HTMLParser.__init__(self)
35 self._match = '_%s-' % arch
37 def handle_starttag(self, tag, attrs):
39 for tag, value in attrs:
41 if value and value.endswith('.xz'):
42 self.links.append(value)
43 if self._match in value:
44 self.arch_link = value
51 gcc: Full path to C compiler
52 path: Directory path containing C compiler
53 cross: Cross compile string, e.g. 'arm-linux-'
54 arch: Architecture of toolchain as determined from the first
55 component of the filename. E.g. arm-linux-gcc becomes arm
56 priority: Toolchain priority (0=highest, 20=lowest)
58 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
60 """Create a new toolchain object.
63 fname: Filename of the gcc component
64 test: True to run the toolchain to test it
65 verbose: True to print out the information
66 priority: Priority to use for this toolchain, or PRIORITY_CALC to
70 self.path = os.path.dirname(fname)
72 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
73 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
74 basename = os.path.basename(fname)
75 pos = basename.rfind('-')
76 self.cross = basename[:pos + 1] if pos != -1 else ''
78 # The architecture is the first part of the name
79 pos = self.cross.find('-')
83 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
85 env = self.MakeEnvironment(False)
87 # As a basic sanity check, run the C compiler with --version
88 cmd = [fname, '--version']
89 if priority == PRIORITY_CALC:
90 self.priority = self.GetPriority(fname)
92 self.priority = priority
94 result = command.RunPipe([cmd], capture=True, env=env,
96 self.ok = result.return_code == 0
98 print 'Tool chain test: ',
100 print "OK, arch='%s', priority %d" % (self.arch,
104 print 'Command: ', cmd
110 def GetPriority(self, fname):
111 """Return the priority of the toolchain.
113 Toolchains are ranked according to their suitability by their
117 fname: Filename of toolchain
119 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
121 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
122 '-none-linux-gnueabi', '-uclinux', '-none-eabi',
123 '-gentoo-linux-gnu', '-linux-gnueabi', '-le-linux', '-uclinux']
124 for prio in range(len(priority_list)):
125 if priority_list[prio] in fname:
126 return PRIORITY_CALC + prio
127 return PRIORITY_CALC + prio
129 def MakeEnvironment(self, full_path):
130 """Returns an environment for using the toolchain.
132 Thie takes the current environment and adds CROSS_COMPILE so that
133 the tool chain will operate correctly.
136 full_path: Return the full path in CROSS_COMPILE and don't set
139 env = dict(os.environ)
141 env['CROSS_COMPILE'] = os.path.join(self.path, self.cross)
143 env['CROSS_COMPILE'] = self.cross
144 env['PATH'] = self.path + ':' + env['PATH']
150 """Manage a list of toolchains for building U-Boot
152 We select one toolchain for each architecture type
155 toolchains: Dict of Toolchain objects, keyed by architecture name
156 prefixes: Dict of prefixes to check, keyed by architecture. This can
157 be a full path and toolchain prefix, for example
158 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
159 something on the search path, for example
160 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
161 paths: List of paths to check for toolchains (may contain wildcards)
168 self._make_flags = dict(bsettings.GetItems('make-flags'))
170 def GetPathList(self, show_warning=True):
171 """Get a list of available toolchain paths
174 show_warning: True to show a warning if there are no tool chains.
177 List of strings, each a path to a toolchain mentioned in the
178 [toolchain] section of the settings file.
180 toolchains = bsettings.GetItems('toolchain')
181 if show_warning and not toolchains:
182 print ('Warning: No tool chains - please add a [toolchain] section'
183 ' to your buildman config file %s. See README for details' %
184 bsettings.config_fname)
187 for name, value in toolchains:
189 paths += glob.glob(value)
194 def GetSettings(self, show_warning=True):
195 """Get toolchain settings from the settings file.
198 show_warning: True to show a warning if there are no tool chains.
200 self.prefixes = bsettings.GetItems('toolchain-prefix')
201 self.paths += self.GetPathList(show_warning)
203 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
205 """Add a toolchain to our list
207 We select the given toolchain as our preferred one for its
208 architecture if it is a higher priority than the others.
211 fname: Filename of toolchain's gcc driver
212 test: True to run the toolchain to test it
213 priority: Priority to use for this toolchain
214 arch: Toolchain architecture, or None if not known
216 toolchain = Toolchain(fname, test, verbose, priority, arch)
217 add_it = toolchain.ok
218 if toolchain.arch in self.toolchains:
219 add_it = (toolchain.priority <
220 self.toolchains[toolchain.arch].priority)
222 self.toolchains[toolchain.arch] = toolchain
224 print ("Toolchain '%s' at priority %d will be ignored because "
225 "another toolchain for arch '%s' has priority %d" %
226 (toolchain.gcc, toolchain.priority, toolchain.arch,
227 self.toolchains[toolchain.arch].priority))
229 def ScanPath(self, path, verbose):
230 """Scan a path for a valid toolchain
234 verbose: True to print out progress information
236 Filename of C compiler if found, else None
239 for subdir in ['.', 'bin', 'usr/bin']:
240 dirname = os.path.join(path, subdir)
241 if verbose: print " - looking in '%s'" % dirname
242 for fname in glob.glob(dirname + '/*gcc'):
243 if verbose: print " - found '%s'" % fname
247 def ScanPathEnv(self, fname):
248 """Scan the PATH environment variable for a given filename.
251 fname: Filename to scan for
253 List of matching pathanames, or [] if none
256 for path in os.environ["PATH"].split(os.pathsep):
257 path = path.strip('"')
258 pathname = os.path.join(path, fname)
259 if os.path.exists(pathname):
260 pathname_list.append(pathname)
263 def Scan(self, verbose):
264 """Scan for available toolchains and select the best for each arch.
266 We look for all the toolchains we can file, figure out the
267 architecture for each, and whether it works. Then we select the
268 highest priority toolchain for each arch.
271 verbose: True to print out progress information
273 if verbose: print 'Scanning for tool chains'
274 for name, value in self.prefixes:
275 if verbose: print " - scanning prefix '%s'" % value
276 if os.path.exists(value):
277 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
279 fname = value + 'gcc'
280 if os.path.exists(fname):
281 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
283 fname_list = self.ScanPathEnv(fname)
285 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
287 raise ValueError, ("No tool chain found for prefix '%s'" %
289 for path in self.paths:
290 if verbose: print " - scanning path '%s'" % path
291 fnames = self.ScanPath(path, verbose)
293 self.Add(fname, True, verbose)
296 """List out the selected toolchains for each architecture"""
297 print 'List of available toolchains (%d):' % len(self.toolchains)
298 if len(self.toolchains):
299 for key, value in sorted(self.toolchains.iteritems()):
300 print '%-10s: %s' % (key, value.gcc)
304 def Select(self, arch):
305 """Returns the toolchain for a given architecture
308 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
311 toolchain object, or None if none found
313 for tag, value in bsettings.GetItems('toolchain-alias'):
315 for alias in value.split():
316 if alias in self.toolchains:
317 return self.toolchains[alias]
319 if not arch in self.toolchains:
320 raise ValueError, ("No tool chain found for arch '%s'" % arch)
321 return self.toolchains[arch]
323 def ResolveReferences(self, var_dict, args):
324 """Resolve variable references in a string
326 This converts ${blah} within the string to the value of blah.
327 This function works recursively.
330 var_dict: Dictionary containing variables and their values
331 args: String containing make arguments
335 >>> bsettings.Setup()
336 >>> tcs = Toolchains()
337 >>> tcs.Add('fred', False)
338 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
340 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
342 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
343 'this=OBLIQUE_setfi2ndrstnd'
345 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
348 m = re_var.search(args)
351 lookup = m.group(0)[2:-1]
352 value = var_dict.get(lookup, '')
353 args = args[:m.start(0)] + value + args[m.end(0):]
356 def GetMakeArguments(self, board):
357 """Returns 'make' arguments for a given board
359 The flags are in a section called 'make-flags'. Flags are named
360 after the target they represent, for example snapper9260=TESTING=1
361 will pass TESTING=1 to make when building the snapper9260 board.
363 References to other boards can be added in the string also. For
367 at91-boards=ENABLE_AT91_TEST=1
368 snapper9260=${at91-boards} BUILD_TAG=442
369 snapper9g45=${at91-boards} BUILD_TAG=443
371 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
372 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
374 A special 'target' variable is set to the board target.
377 board: Board object for the board to check.
379 'make' flags for that board, or '' if none
381 self._make_flags['target'] = board.target
382 arg_str = self.ResolveReferences(self._make_flags,
383 self._make_flags.get(board.target, ''))
384 args = arg_str.split(' ')
393 def LocateArchUrl(self, fetch_arch):
394 """Find a toolchain available online
396 Look in standard places for available toolchains. At present the
397 only standard place is at kernel.org.
400 arch: Architecture to look for, or 'list' for all
402 If fetch_arch is 'list', a tuple:
403 Machine architecture (e.g. x86_64)
406 URL containing this toolchain, if avaialble, else None
408 arch = command.OutputOneLine('uname', '-m')
409 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
410 versions = ['4.9.0', '4.6.3', '4.6.2', '4.5.1', '4.2.4']
412 for version in versions:
413 url = '%s/%s/%s/' % (base, arch, version)
414 print 'Checking: %s' % url
415 response = urllib2.urlopen(url)
416 html = response.read()
417 parser = MyHTMLParser(fetch_arch)
419 if fetch_arch == 'list':
420 links += parser.links
421 elif parser.arch_link:
422 return url + parser.arch_link
423 if fetch_arch == 'list':
427 def Download(self, url):
428 """Download a file to a temporary directory
434 Temporary directory name
435 Full path to the downloaded archive file in that directory,
436 or None if there was an error while downloading
438 print 'Downloading: %s' % url
439 leaf = url.split('/')[-1]
440 tmpdir = tempfile.mkdtemp('.buildman')
441 response = urllib2.urlopen(url)
442 fname = os.path.join(tmpdir, leaf)
443 fd = open(fname, 'wb')
444 meta = response.info()
445 size = int(meta.getheaders('Content-Length')[0])
450 # Read the file in chunks and show progress as we go
452 buffer = response.read(block_size)
454 print chr(8) * (len(status) + 1), '\r',
459 status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
461 status = status + chr(8) * (len(status) + 1)
466 print 'Error, failed to download'
471 def Unpack(self, fname, dest):
475 fname: Filename to unpack
476 dest: Destination directory
478 Directory name of the first entry in the archive, without the
481 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
482 return stdout.splitlines()[0][:-1]
484 def TestSettingsHasPath(self, path):
485 """Check if builmand will find this toolchain
488 True if the path is in settings, False if not
490 paths = self.GetPathList(False)
494 """List architectures with available toolchains to download"""
495 host_arch, archives = self.LocateArchUrl('list')
496 re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
498 for archive in archives:
499 # Remove the host architecture from the start
500 arch = re_arch.match(archive[len(host_arch):])
502 arch_set.add(arch.group(1))
503 return sorted(arch_set)
505 def FetchAndInstall(self, arch):
506 """Fetch and install a new toolchain
509 Architecture to fetch, or 'list' to list
511 # Fist get the URL for this architecture
512 url = self.LocateArchUrl(arch)
514 print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
517 home = os.environ['HOME']
518 dest = os.path.join(home, '.buildman-toolchains')
519 if not os.path.exists(dest):
522 # Download the tar file for this toolchain and unpack it
523 tmpdir, tarfile = self.Download(url)
526 print 'Unpacking to: %s' % dest,
528 path = self.Unpack(tarfile, dest)
533 # Check that the toolchain works
535 dirpath = os.path.join(dest, path)
536 compiler_fname_list = self.ScanPath(dirpath, True)
537 if not compiler_fname_list:
538 print 'Could not locate C compiler - fetch failed.'
540 if len(compiler_fname_list) != 1:
541 print ('Internal error, ambiguous toolchains: %s' %
542 (', '.join(compiler_fname)))
544 toolchain = Toolchain(compiler_fname_list[0], True, True)
546 # Make sure that it will be found by buildman
547 if not self.TestSettingsHasPath(dirpath):
548 print ("Adding 'download' to config file '%s'" %
549 bsettings.config_fname)
550 tools_dir = os.path.dirname(dirpath)
551 bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir)