1 # Copyright 2014 The Chromium OS Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """Module containing the various utilities to build Chrome with AFDO.
7 For a description of AFDO see gcc.gnu.org/wiki/AutoFDO.
10 from __future__ import print_function
16 from chromite.cbuildbot import failures_lib
17 from chromite.cbuildbot import constants
18 from chromite.lib import cros_build_lib
19 from chromite.lib import git
20 from chromite.lib import gs
21 from chromite.lib import osutils
22 from chromite.lib import timeout_util
25 # AFDO-specific constants.
26 # Chrome URL where AFDO data is stored.
27 AFDO_PROD_URL = 'gs://chromeos-prebuilt/afdo-job/canonicals/'
28 AFDO_TEST_URL = '%s/afdo-job/canonicals/' % constants.TRASH_BUCKET
29 AFDO_BASE_URL = AFDO_PROD_URL
30 AFDO_CHROOT_ROOT = os.path.join('%(build_root)s', constants.DEFAULT_CHROOT_DIR)
31 AFDO_LOCAL_DIR = os.path.join('%(root)s', 'tmp')
32 AFDO_BUILDROOT_LOCAL = AFDO_LOCAL_DIR % {'root': AFDO_CHROOT_ROOT}
33 CHROME_ARCH_VERSION = '%(package)s-%(arch)s-%(version)s'
34 CHROME_PERF_AFDO_FILE = '%s.perf.data' % CHROME_ARCH_VERSION
35 CHROME_PERF_AFDO_URL = '%s%s.bz2' % (AFDO_BASE_URL, CHROME_PERF_AFDO_FILE)
36 CHROME_AFDO_FILE = '%s.afdo' % CHROME_ARCH_VERSION
37 CHROME_AFDO_URL = '%s%s.bz2' % (AFDO_BASE_URL, CHROME_AFDO_FILE)
38 CHROME_ARCH_RELEASE = '%(package)s-%(arch)s-%(release)s'
39 LATEST_CHROME_AFDO_FILE = 'latest-%s.afdo' % CHROME_ARCH_RELEASE
40 LATEST_CHROME_AFDO_URL = AFDO_BASE_URL + LATEST_CHROME_AFDO_FILE
41 CHROME_DEBUG_BIN = os.path.join('%(root)s',
42 'build/%(board)s/usr/lib/debug',
43 'opt/google/chrome/chrome.debug')
44 CHROME_DEBUG_BIN_URL = '%s%s.debug.bz2' % (AFDO_BASE_URL, CHROME_ARCH_VERSION)
46 AFDO_GENERATE_GCOV_TOOL = '/usr/bin/create_gcov'
48 # regex to find AFDO file for specific architecture within the ebuild file.
49 CHROME_EBUILD_AFDO_EXP = r'^(?P<bef>AFDO_FILE\["%s"\]=")(?P<name>.*)(?P<aft>")'
50 # and corresponding replacement string.
51 CHROME_EBUILD_AFDO_REPL = r'\g<bef>%s\g<aft>'
53 # How old can the AFDO data be? (in days).
54 AFDO_ALLOWED_STALE = 7
56 # TODO(llozano): Currently using sandybridge boards. We should move to
57 # a more modern platform.
58 # Set of boards that can generate the AFDO profile (can generate 'perf'
59 # data with LBR events).
60 AFDO_DATA_GENERATORS = ('butterfly', 'lumpy', 'parrot', 'stumpy')
62 # For a given architecture, which architecture is used to generate
63 # the AFDO profile. Some architectures are not able to generate their
65 AFDO_ARCH_GENERATORS = {'amd64': 'amd64',
69 AFDO_ALERT_RECIPIENTS = ['chromeos-toolchain@google.com']
72 class MissingAFDOData(failures_lib.StepFailure):
73 """Exception thrown when necessary AFDO data is missing."""
76 class MissingAFDOMarkers(failures_lib.StepFailure):
77 """Exception thrown when necessary ebuild markers for AFDO are missing."""
80 def CompressAFDOFile(to_compress, buildroot):
81 """Compress file used by AFDO process.
84 to_compress: File to compress.
85 buildroot: buildroot where to store the compressed data.
88 Name of the compressed data file.
90 local_dir = AFDO_BUILDROOT_LOCAL % {'build_root': buildroot}
91 dest = os.path.join(local_dir, os.path.basename(to_compress)) + '.bz2'
92 cros_build_lib.CompressFile(to_compress, dest)
96 def UncompressAFDOFile(to_decompress, buildroot):
97 """Decompress file used by AFDO process.
100 to_decompress: File to decompress.
101 buildroot: buildroot where to store the decompressed data.
103 local_dir = AFDO_BUILDROOT_LOCAL % {'build_root': buildroot}
104 basename = os.path.basename(to_decompress)
105 dest_basename = basename.rsplit('.', 1)[0]
106 dest = os.path.join(local_dir, dest_basename)
107 cros_build_lib.UncompressFile(to_decompress, dest)
111 def GSUploadIfNotPresent(gs_context, src, dest):
112 """Upload a file to GS only if the file does not exist.
114 Will not generate an error if the file already exist in GS. It will
117 I could use GSContext.Copy(src,dest,version=0) here but it does not seem
118 to work for large files. Using GSContext.Exists(dest) instead. See
122 gs_context: GS context instance.
124 dest: Destination location.
127 True if file was uploaded. False otherwise.
129 if gs_context.Exists(dest):
130 cros_build_lib.Warning('File %s already in GS', dest)
133 gs_context.Copy(src, dest, acl='public-read')
137 def CheckAFDOPerfData(arch, cpv, buildroot, gs_context):
138 """Check whether AFDO perf data exists for the given architecture.
140 Check if 'perf' data file for this architecture and release is available
141 in GS. If so, copy it into a temp directory in the buildroot.
144 arch: architecture we're going to build Chrome for.
145 cpv: The portage_util.CPV object for chromeos-chrome.
146 buildroot: buildroot where AFDO data should be stored.
147 gs_context: GS context to retrieve data.
150 True if AFDO perf data is available. False otherwise.
152 # The file name of the perf data is based only in the chrome version.
153 # The test case that produces it does not know anything about the
155 # TODO(llozano): perf data filename should include the revision number.
156 version_number = cpv.version_no_rev.split('_')[0]
157 chrome_spec = {'package': cpv.package,
159 'version': version_number}
160 url = CHROME_PERF_AFDO_URL % chrome_spec
161 if not gs_context.Exists(url):
162 cros_build_lib.Info('Could not find AFDO perf data')
164 dest_dir = AFDO_BUILDROOT_LOCAL % {'build_root': buildroot}
165 dest_path = os.path.join(dest_dir, url.rsplit('/', 1)[1])
166 gs_context.Copy(url, dest_path)
168 UncompressAFDOFile(dest_path, buildroot)
169 cros_build_lib.Info('Found and retrieved AFDO perf data')
173 def WaitForAFDOPerfData(cpv, arch, buildroot, gs_context,
174 timeout=constants.AFDO_GENERATE_TIMEOUT):
175 """Wait for AFDO perf data to show up (with an appropriate timeout).
178 arch: architecture we're going to build Chrome for.
179 cpv: CPV object for Chrome.
180 buildroot: buildroot where AFDO data should be stored.
181 gs_context: GS context to retrieve data.
182 timeout: How long to wait total, in seconds.
185 True if found the AFDO perf data before the timeout expired.
189 timeout_util.WaitForReturnTrue(
191 func_args=(arch, cpv, buildroot, gs_context),
192 timeout=timeout, period=constants.SLEEP_TIMEOUT)
193 except timeout_util.TimeoutError:
198 def PatchChromeEbuildAFDOFile(ebuild_file, arch_profiles):
199 """Patch the Chrome ebuild with the dictionary of {arch: afdo_file} pairs.
202 ebuild_file: path of the ebuild file within the chroot.
203 arch_profiles: {arch: afdo_file} pairs to put into the ebuild.
205 original_ebuild = cros_build_lib.FromChrootPath(ebuild_file)
206 modified_ebuild = '%s.new' % original_ebuild
211 for arch in arch_profiles.keys():
212 arch_patterns[arch] = re.compile(CHROME_EBUILD_AFDO_EXP % arch)
213 arch_repls[arch] = CHROME_EBUILD_AFDO_REPL % arch_profiles[arch]
214 arch_markers[arch] = False
216 with open(original_ebuild, 'r') as original:
217 with open(modified_ebuild, 'w') as modified:
218 for line in original:
219 for arch in arch_profiles.keys():
220 matched = arch_patterns[arch].match(line)
222 arch_markers[arch] = True
223 modified.write(arch_patterns[arch].sub(arch_repls[arch], line))
225 else: # line without markers, just copy it.
228 for arch, found in arch_markers.iteritems():
230 raise MissingAFDOMarkers('Chrome ebuild file does not have appropriate '
231 'AFDO markers for arch %s' % arch)
233 os.rename(modified_ebuild, original_ebuild)
236 def UpdateChromeEbuildAFDOFile(board, arch_profiles):
237 """Update chrome ebuild with the dictionary of {arch: afdo_file} pairs.
239 Modifies the Chrome ebuild to set the appropriate AFDO file for each
240 given architecture. Regenerates the associated Manifest file and
241 commits the new ebuild and Manifest.
244 board: board we are building Chrome for.
245 arch_profiles: {arch: afdo_file} pairs to put into the ebuild.
247 # Find the Chrome ebuild file.
248 equery_prog = 'equery'
249 ebuild_prog = 'ebuild'
251 equery_prog += '-%s' % board
252 ebuild_prog += '-%s' % board
254 equery_cmd = [equery_prog, 'w', 'chromeos-chrome']
255 ebuild_file = cros_build_lib.RunCommand(equery_cmd,
257 redirect_stdout=True).output.rstrip()
259 # Patch the ebuild file with the names of the available afdo_files.
260 PatchChromeEbuildAFDOFile(ebuild_file, arch_profiles)
262 # Also patch the 9999 ebuild. This is necessary because the uprev
263 # process starts from the 9999 ebuild file and then compares to the
264 # current version to see if the uprev is really necessary. We dont
265 # want the names of the available afdo_files to show as differences.
266 # It also allows developers to do USE=afdo_use when using the 9999
268 ebuild_9999 = os.path.join(os.path.dirname(ebuild_file),
269 'chromeos-chrome-9999.ebuild')
270 PatchChromeEbuildAFDOFile(ebuild_9999, arch_profiles)
272 # Regenerate the Manifest file.
274 # If using the GS test location, pass this location to the
276 if AFDO_BASE_URL == AFDO_TEST_URL:
277 ebuild_gs_dir = {'AFDO_GS_DIRECTORY': AFDO_TEST_URL}
278 gen_manifest_cmd = [ebuild_prog, ebuild_file, 'manifest', '--force']
279 cros_build_lib.RunCommand(gen_manifest_cmd, enter_chroot=True,
280 extra_env=ebuild_gs_dir, print_cmd=True)
282 ebuild_dir = cros_build_lib.FromChrootPath(os.path.dirname(ebuild_file))
283 git.RunGit(ebuild_dir, ['add', 'Manifest'])
285 # Check if anything changed compared to the previous version.
286 mod_files = ['Manifest', os.path.basename(ebuild_file),
287 os.path.basename(ebuild_9999)]
288 modifications = git.RunGit(ebuild_dir,
289 ['status', '--porcelain', '--'] + mod_files,
290 capture_output=True, print_cmd=True).output
291 if not modifications:
292 cros_build_lib.Info('AFDO info for the Chrome ebuild did not change. '
296 # If there are changes to ebuild or Manifest, commit them.
297 commit_msg = ('"Set {arch: afdo_file} pairs %s and updated Manifest"'
299 git.RunGit(ebuild_dir,
300 ['commit', '-m', commit_msg, '--'] + mod_files,
304 def VerifyLatestAFDOFile(afdo_release_spec, buildroot, gs_context):
305 """Verify that the latest AFDO profile for a release is suitable.
307 Find the latest AFDO profile file for a particular release and check
308 that it is not too stale. The latest AFDO profile name for a release
309 can be found in a file in GS under the name
310 latest-chrome-<arch>-<release>.afdo.
313 afdo_release_spec: architecture and release to find the latest AFDO
315 buildroot: buildroot where AFDO data should be stored.
316 gs_context: GS context to retrieve data.
319 The name of the AFDO profile file if a suitable one was found.
322 latest_afdo_url = LATEST_CHROME_AFDO_URL % afdo_release_spec
324 # Check if latest-chrome-<arch>-<release>.afdo exists.
326 if gs_context.Exists(latest_afdo_url):
327 latest_detail = gs_context.LSWithDetails(latest_afdo_url)
328 if not latest_detail:
329 cros_build_lib.Info('Could not find latest AFDO info file %s' %
333 # Verify the AFDO profile file is not too stale.
334 mod_date = latest_detail[0][2]
335 curr_date = datetime.datetime.now()
336 allowed_stale_days = datetime.timedelta(days=AFDO_ALLOWED_STALE)
337 if (curr_date - mod_date) > allowed_stale_days:
338 cros_build_lib.Info('Found latest AFDO info file %s but it is too old' %
342 # Then get the name of the latest valid AFDO profile file.
343 local_dir = AFDO_BUILDROOT_LOCAL % {'build_root': buildroot }
344 latest_afdo_file = LATEST_CHROME_AFDO_FILE % afdo_release_spec
345 latest_afdo_path = os.path.join(local_dir, latest_afdo_file)
346 gs_context.Copy(latest_afdo_url, latest_afdo_path)
348 return osutils.ReadFile(latest_afdo_path).strip()
351 def GetLatestAFDOFile(cpv, arch, buildroot, gs_context):
352 """Try to find the latest suitable AFDO profile file.
354 Try to find the latest AFDO profile generated for current release
355 and architecture. If there is none, check the previous release (mostly
356 in case we have just branched).
359 cpv: cpv object for Chrome.
360 arch: architecture for which we are looking for AFDO profile.
361 buildroot: buildroot where AFDO data should be stored.
362 gs_context: GS context to retrieve data.
365 Name of latest suitable AFDO profile file if one is found.
368 generator_arch = AFDO_ARCH_GENERATORS[arch]
369 version_number = cpv.version
370 current_release = version_number.split('.')[0]
371 afdo_release_spec = {'package': cpv.package,
372 'arch': generator_arch,
373 'release': current_release}
374 afdo_file = VerifyLatestAFDOFile(afdo_release_spec, buildroot, gs_context)
378 # Could not find suitable AFDO file for the current release.
379 # Let's see if there is one from the previous release.
380 previous_release = str(int(current_release) - 1)
381 prev_release_spec = {'package': cpv.package,
382 'arch': generator_arch,
383 'release': previous_release}
384 return VerifyLatestAFDOFile(prev_release_spec, buildroot, gs_context)
387 def GenerateAFDOData(cpv, arch, board, buildroot, gs_context):
388 """Generate AFDO profile data from 'perf' data.
390 Given the 'perf' profile, generate an AFDO profile using create_gcov.
391 It also creates a latest-chrome-<arch>-<release>.afdo file pointing
392 to the generated AFDO profile.
393 Uploads the generated data to GS for retrieval by the chrome ebuild
394 file when doing an 'afdo_use' build.
395 It is possible the generated data has previously been uploaded to GS
396 in which case this routine will not upload the data again. Uploading
397 again may cause verication failures for the ebuild file referencing
398 the previous contents of the data.
401 cpv: cpv object for Chrome.
402 arch: architecture for which we are looking for AFDO profile.
403 board: board we are building for.
404 buildroot: buildroot where AFDO data should be stored.
405 gs_context: GS context to retrieve/store data.
408 Name of the AFDO profile file generated if successful.
410 CHROME_UNSTRIPPED_NAME = 'chrome.unstripped'
412 version_number = cpv.version
413 afdo_spec = {'package': cpv.package,
415 'version': version_number}
416 chroot_root = AFDO_CHROOT_ROOT % {'build_root': buildroot }
417 local_dir = AFDO_LOCAL_DIR % {'root': chroot_root }
418 in_chroot_local_dir = AFDO_LOCAL_DIR % {'root': '' }
420 # Upload compressed chrome debug binary to GS for triaging purposes.
421 # TODO(llozano): This simplifies things in case of need of triaging
422 # problems but is it really necessary?
423 debug_bin = CHROME_DEBUG_BIN % {'root': chroot_root,
425 comp_debug_bin_path = CompressAFDOFile(debug_bin, buildroot)
426 GSUploadIfNotPresent(gs_context, comp_debug_bin_path,
427 CHROME_DEBUG_BIN_URL % afdo_spec)
429 # create_gcov demands the name of the profiled binary exactly matches
430 # the name of the unstripped binary or it is named 'chrome.unstripped'.
431 # So create a symbolic link with the appropriate name.
432 local_debug_sym = os.path.join(local_dir, CHROME_UNSTRIPPED_NAME)
433 in_chroot_debug_bin = CHROME_DEBUG_BIN % {'root': '', 'board': board }
434 osutils.SafeUnlink(local_debug_sym)
435 os.symlink(in_chroot_debug_bin, local_debug_sym)
437 # Call create_gcov tool to generated AFDO profile from 'perf' profile
438 # and upload it to GS. Need to call from within chroot since this tool
439 # was built inside chroot.
440 debug_sym = os.path.join(in_chroot_local_dir, CHROME_UNSTRIPPED_NAME)
441 # The name of the 'perf' file is based only on the version of chrome. The
442 # revision number is not included.
443 afdo_spec_no_rev = {'package': cpv.package,
445 'version': cpv.version_no_rev.split('_')[0]}
446 perf_afdo_file = CHROME_PERF_AFDO_FILE % afdo_spec_no_rev
447 perf_afdo_path = os.path.join(in_chroot_local_dir, perf_afdo_file)
448 afdo_file = CHROME_AFDO_FILE % afdo_spec
449 afdo_path = os.path.join(in_chroot_local_dir, afdo_file)
450 afdo_cmd = [AFDO_GENERATE_GCOV_TOOL,
451 '--binary=%s' % debug_sym,
452 '--profile=%s' % perf_afdo_path,
453 '--gcov=%s' % afdo_path]
454 cros_build_lib.RunCommand(afdo_cmd, enter_chroot=True, capture_output=True,
457 afdo_local_path = os.path.join(local_dir, afdo_file)
458 comp_afdo_path = CompressAFDOFile(afdo_local_path, buildroot)
459 uploaded_afdo_file = GSUploadIfNotPresent(gs_context, comp_afdo_path,
460 CHROME_AFDO_URL % afdo_spec)
462 if uploaded_afdo_file:
463 # Create latest-chrome-<arch>-<release>.afdo pointing to the name
464 # of the AFDO profile file and upload to GS.
465 current_release = version_number.split('.')[0]
466 afdo_release_spec = {'package': cpv.package,
468 'release': current_release}
469 latest_afdo_file = LATEST_CHROME_AFDO_FILE % afdo_release_spec
470 latest_afdo_path = os.path.join(local_dir, latest_afdo_file)
471 osutils.WriteFile(latest_afdo_path, afdo_file)
472 gs_context.Copy(latest_afdo_path,
473 LATEST_CHROME_AFDO_URL % afdo_release_spec,
479 def CanGenerateAFDOData(board):
480 """Does this board has the capability of generating its own AFDO data?."""
481 return board in AFDO_DATA_GENERATORS
484 def GenerateOrFindAFDOData(cpv, arch, board, buildroot):
485 """Generate or find the appropriate AFDO profile for the given architecture.
487 For the architectures that can generate a 'perf' tool profile, wait for
488 it to be generated by the autotest AFDO_generate and generate an AFDO
489 profile for it. In the generation of the 'perf' profile failed, use
490 a previously generated AFDO profile that is not too stale.
491 For the architectures that cannot generate a 'perf' tool profile, use
492 the AFDO profile from another architecture (e.g.: ARM can use a profile
494 Once we have an adequate AFDO profile, put this information in the
498 cpv: cpv object for Chrome.
499 arch: architecture for which we are looking for AFDO profile.
500 board: board we are building for.
501 buildroot: buildroot where AFDO data should be stored.
503 gs_context = gs.GSContext()
505 if CanGenerateAFDOData(board):
506 # Generation of AFDO could fail for different reasons. Try to be
507 # resilient about this and in case of failure just find an
508 # older AFDO profile.
510 if WaitForAFDOPerfData(cpv, arch, buildroot, gs_context):
511 afdo_file = GenerateAFDOData(cpv, arch, board, buildroot, gs_context)
513 cros_build_lib.Info('Generated %s AFDO profile %s',
515 # Will let system-exiting exceptions through.
517 cros_build_lib.PrintBuildbotStepWarnings()
518 cros_build_lib.Warning('AFDO profile generation failed with exception ',
521 cros_build_lib.Info('Trying to find previous appropriate AFDO profile')
522 afdo_file = GetLatestAFDOFile(cpv, arch, buildroot, gs_context)
524 cros_build_lib.Info('Found previous %s AFDO profile %s',
527 raise MissingAFDOData('Could not generate or find appropriate AFDO profile')
529 # We found an AFDO profile. Lets put the info in the chrome ebuild.
530 UpdateChromeEbuildAFDOFile(board, {arch: afdo_file})