From a9246ddcd0697d96ca39b574d3a38daf2f58132e Mon Sep 17 00:00:00 2001 From: Kevin Lubick Date: Wed, 15 Feb 2017 10:20:30 -0500 Subject: [PATCH] Symbolize any stacktraces in DM and nanobench As a soft rollout, this only affects the Linux NUCS (on Ubuntu16). BUG=skia:6206 Change-Id: Ic314e85159cd3c0d9e55bbdce412a8d61adebb33 Reviewed-on: https://skia-review.googlesource.com/8276 Commit-Queue: Kevin Lubick Reviewed-by: Ravi Mistry --- .../core/resources/symbolize_stack_trace.py | 84 ++++++++++++++++++++++ infra/bots/recipe_modules/flavor/gn_flavor.py | 15 +++- ...g-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json | 6 +- ...g-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json | 6 +- 4 files changed, 108 insertions(+), 3 deletions(-) create mode 100755 infra/bots/recipe_modules/core/resources/symbolize_stack_trace.py diff --git a/infra/bots/recipe_modules/core/resources/symbolize_stack_trace.py b/infra/bots/recipe_modules/core/resources/symbolize_stack_trace.py new file mode 100755 index 0000000..ccd67c7 --- /dev/null +++ b/infra/bots/recipe_modules/core/resources/symbolize_stack_trace.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# pylint: disable=line-too-long + +import os +import re +import subprocess +import sys + +# Run a command and symbolize anything that looks like a stacktrace in the +# stdout/stderr. This will return with the same error code as the command. + +# First parameter is the current working directory, which will be stripped +# out of stacktraces. The rest of the parameters will be fed to +# subprocess.check_output() and should be the command and arguments that +# will be fed in. If any environment variables are set when running this +# script, they will be automatically used by the call to +# subprocess.check_output(). + +# This wrapper function is needed to make sure stdout and stderr stay properly +# interleaved, to assist in debugging. There are no clean ways to achieve +# this with recipes. For example, running the dm step with parameters like +# stdout=api.raw_io.output(), stderr=api.raw_io.output() ended up with +# stderr and stdout being separate files, which eliminated the interwoven logs. +# Aside from specifying stdout/stderr, there are no ways to capture or reason +# about the logs of previous steps without using a wrapper like this. + +def main(basedir, cmd): + logs = "" + retcode = 0 + try: + logs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip() + except subprocess.CalledProcessError as e: + retcode = e.returncode + logs = e.output + + # Stacktraces generally look like: + # /lib/x86_64-linux-gnu/libc.so.6(abort+0x16a) [0x7fa90e8d0c62] + # /b/s/w/irISUIyA/linux_vulkan_intel_driver_debug/./libvulkan_intel.so(+0x1f4d0a) [0x7fa909eead0a] + # /b/s/w/irISUIyA/out/Debug/dm() [0x17c3c5f] + # The stack_line regex splits those into three parts. Experimentation has + # shown that the address in () works best for external libraries, but our code + # doesn't have that. So, we capture both addresses and prefer using the first + # over the second, unless the first is blank or invalid. Relative offsets + # like abort+0x16a are ignored. + stack_line = r'^(?P.+)\(\+?(?P.*)\) \[(?P.+)\]' + # After performing addr2line, the result can be something obnoxious like: + # foo(bar) at /b/s/w/a39kd/Skia/out/Clang/../../src/gpu/Frobulator.cpp:13 + # The extra_path strips off the not-useful prefix and leaves just the + # important src/gpu/Frobulator.cpp:13 bit. + extra_path = r'/.*\.\./' + for line in logs.splitlines(): + line = line.strip() + + m = re.search(stack_line, line) + if m: + path = m.group('path') + addr = m.group('addr') + addr2 = m.group('addr2') + if os.path.exists(path): + if not addr or not addr.startswith('0x'): + addr = addr2 + sym = subprocess.check_output(['addr2line', '-Cfpe', path, addr]) + sym = sym.strip() + # If addr2line doesn't return anything useful, we don't replace the + # original address, so the human can see it. + if sym and not sym.startswith('?'): + if path.startswith(basedir): + path = path[len(basedir)+1:] + sym = re.sub(extra_path, '', sym) + line = path + ' ' + sym + + print line + + sys.exit(retcode) + + +if __name__ == '__main__': + if len(sys.argv) < 3: + print >> sys.stderr, 'USAGE: %s working_dir cmd_and_args...' % sys.argv[0] + sys.exit(1) + main(sys.argv[1], sys.argv[2:]) diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/gn_flavor.py index 27ec3f9..e7d8653 100644 --- a/infra/bots/recipe_modules/flavor/gn_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_flavor.py @@ -161,4 +161,17 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): # Find the MSAN-built libc++. env['LD_LIBRARY_PATH'] = clang_linux + '/msan' - self._run(name, cmd, env=env) + to_symbolize = ['dm', 'nanobench'] + if name in to_symbolize and 'Ubuntu16' == self.m.vars.builder_cfg['os']: + # Convert path objects or placeholders into strings such that they can + # be passed to symbolize_stack_trace.py + args = [self.m.vars.slave_dir] + [str(x) for x in cmd] + self._py('symbolized %s' % name, + self.m.vars.infrabots_dir.join('recipe_modules', 'core', + 'resources', 'symbolize_stack_trace.py'), + args=args, + env=env, + infra_step=False) + + else: + self._run(name, cmd, env=env) diff --git a/infra/bots/recipe_modules/perf/example.expected/Perf-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json b/infra/bots/recipe_modules/perf/example.expected/Perf-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json index 64f742b..53c5acf 100644 --- a/infra/bots/recipe_modules/perf/example.expected/Perf-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json +++ b/infra/bots/recipe_modules/perf/example.expected/Perf-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json @@ -61,6 +61,10 @@ }, { "cmd": [ + "python", + "-u", + "[START_DIR]/skia/infra/bots/recipe_modules/core/resources/symbolize_stack_trace.py", + "[START_DIR]", "[START_DIR]/out/Debug/nanobench", "--undefok", "-i", @@ -112,7 +116,7 @@ "SKIA_OUT": "[START_DIR]/out", "VK_ICD_FILENAMES": "[START_DIR]/linux_vulkan_intel_driver_debug/intel_icd.x86_64.json" }, - "name": "nanobench" + "name": "symbolized nanobench" }, { "name": "$result", diff --git a/infra/bots/recipe_modules/sktest/example.expected/Test-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json b/infra/bots/recipe_modules/sktest/example.expected/Test-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json index 4fe7ed4..416540a 100644 --- a/infra/bots/recipe_modules/sktest/example.expected/Test-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json +++ b/infra/bots/recipe_modules/sktest/example.expected/Test-Ubuntu16-Clang-NUC-GPU-IntelIris540-x86_64-Debug-Vulkan.json @@ -175,6 +175,10 @@ }, { "cmd": [ + "python", + "-u", + "[START_DIR]/skia/infra/bots/recipe_modules/core/resources/symbolize_stack_trace.py", + "[START_DIR]", "[START_DIR]/out/Debug/dm", "--undefok", "--resourcePath", @@ -331,7 +335,7 @@ "SKIA_OUT": "[START_DIR]/out", "VK_ICD_FILENAMES": "[START_DIR]/linux_vulkan_intel_driver_debug/intel_icd.x86_64.json" }, - "name": "dm" + "name": "symbolized dm" }, { "name": "$result", -- 2.7.4