GN: add sanitize arg
authormtklein <mtklein@chromium.org>
Thu, 8 Sep 2016 15:39:34 +0000 (08:39 -0700)
committerCommit bot <commit-bot@chromium.org>
Thu, 8 Sep 2016 15:39:34 +0000 (08:39 -0700)
Attempt to take over all *SAN builds.

MSAN has a lot of coordination required between gn/BUILD.gn and gn_flavor.py.
I'd like to follow up to move more of this into gn/BUILD.gn, to make it easier
to use locally.

The compile steps should be much faster now.  We no longer build CMake
and Clang for every run, instead using the clang_linux CIPD package.  This
removes the need for all the third_party/externals/llvm/... dependencies.

Similarly, since we're using the clang_linux package, we no longer depend
on Chrome's Clang, and thus no longer need to sync chromium on these bots.

Instead of packaging up MSAN libraries and llvm-symbolizer in the compile
output, I have the test / perf bots also depend on the clang_linux package.
These do not vary from build to build.

No more need for the xsan.blacklist -include hack: Clang, GN, and Ninja
all track changes to xsan.blacklist without our help.

This has the incidental effect of upgrading the compiler used by *SAN
bots from Clang 3.8 to Clang 3.9.

BUG=skia:
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2289343002

Review-Url: https://codereview.chromium.org/2289343002

22 files changed:
DEPS
gn/BUILD.gn
gn/BUILDCONFIG.gn
infra/bots/recipe_modules/flavor/api.py
infra/bots/recipe_modules/flavor/gn_flavor.py
infra/bots/recipe_modules/flavor/xsan_flavor.py [deleted file]
infra/bots/recipe_modules/vars/api.py
infra/bots/recipes/swarm_compile.expected/Build-Ubuntu-Clang-x86_64-Debug-ASAN.json [new file with mode: 0644]
infra/bots/recipes/swarm_compile.expected/Build-Ubuntu-GCC-x86_64-Debug-MSAN.json
infra/bots/recipes/swarm_compile.py
infra/bots/recipes/swarm_perf.expected/Perf-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release-GN.json
infra/bots/recipes/swarm_perf.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-GN.json [new file with mode: 0644]
infra/bots/recipes/swarm_perf.py
infra/bots/recipes/swarm_test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-ASAN.json [new file with mode: 0644]
infra/bots/recipes/swarm_test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-MSAN.json
infra/bots/recipes/swarm_test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-TSAN.json
infra/bots/recipes/swarm_test.py
infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN.json
infra/bots/recipes/swarm_trigger.py
third_party/freetype2/BUILD.gn
tools/xsan.blacklist
tools/xsan_build [deleted file]

diff --git a/DEPS b/DEPS
index 17a4fdb..d743cc9 100644 (file)
--- a/DEPS
+++ b/DEPS
@@ -53,14 +53,7 @@ deps = {
 }
 
 deps_os = {
-  'llvm': {
-    # xSAN bots build Clang from scratch.
-    "third_party/externals/llvm": "https://llvm.googlesource.com/llvm@release_38",
-    "third_party/externals/llvm/tools/clang": "https://llvm.googlesource.com/clang@release_38",
-    "third_party/externals/llvm/projects/compiler-rt": "https://llvm.googlesource.com/compiler-rt@release_38",
-    "third_party/externals/llvm/projects/libcxx": "https://llvm.googlesource.com/libcxx@release_38",
-    "third_party/externals/llvm/projects/libcxxabi": "https://llvm.googlesource.com/libcxxabi@release_38",
-  }
+  'llvm': { }
 }
 
 recursedeps = [ "common" ]
index a076aa9..c5db0a9 100644 (file)
@@ -23,7 +23,9 @@ declare_args() {
 }
 
 config("no_rtti") {
-  cflags_cc = [ "-fno-rtti" ]
+  if (sanitize != "ASAN") {  # -fsanitize=vptr requires RTTI
+    cflags_cc = [ "-fno-rtti" ]
+  }
 }
 
 config("default") {
@@ -53,6 +55,8 @@ config("default") {
 
     "-Wnon-virtual-dtor",
   ]
+  ldflags = []
+
   if (current_cpu == "arm") {
     cflags += [
       "-march=armv7-a",
@@ -80,7 +84,7 @@ config("default") {
       "-isystem$ndk/sources/android/support/include",
       "-isystem$ndk/sources/cxx-stl/llvm-libc++/libcxx/include",
     ]
-    ldflags = [
+    ldflags += [
       "--sysroot=$ndk/platforms/$ndk_platform",
       "--target=$ndk_target",
       "-B$ndk/toolchains/$ndk_gccdir-4.9/prebuilt/$ndk_host/$ndk_target/bin",
@@ -104,6 +108,31 @@ config("default") {
   if (is_linux) {
     libs = [ "pthread" ]
   }
+
+  if (sanitize != "") {
+    # You can either pass the sanitizers directly, e.g. "address,undefined",
+    # or pass one of the couple common aliases used by the bots.
+    sanitizers = sanitize
+    if (sanitize == "ASAN") {
+      sanitizers = "address,bool,function,integer-divide-by-zero,nonnull-attribute,null,object-size,return,returns-nonnull-attribute,shift,signed-integer-overflow,unreachable,vla-bound,vptr"
+    } else if (sanitize == "TSAN") {
+      sanitizers = "thread"
+    } else if (sanitize == "MSAN") {
+      sanitizers = "memory"
+    }
+
+    cflags += [
+      "-fsanitize=$sanitizers",
+      "-fno-sanitize-recover=$sanitizers",
+      "-fsanitize-blacklist=" + rebase_path("../tools/xsan.blacklist"),
+    ]
+    ldflags += [ "-fsanitize=$sanitizers" ]
+    if (sanitizers == "memory") {
+      cflags += [ "-fsanitize-memory-track-origins" ]
+      cflags_cc += [ "-stdlib=libc++" ]
+      ldflags += [ "-stdlib=libc++" ]
+    }
+  }
 }
 
 config("release") {
index 10f3244..1ef121a 100644 (file)
@@ -9,6 +9,7 @@ declare_args() {
   is_debug = true
   is_component_build = false
   ndk = ""
+  sanitize = ""
 }
 
 # Platform detection
index bff316a..21ee590 100644 (file)
@@ -17,7 +17,6 @@ from . import gn_flavor
 from . import ios_flavor
 from . import pdfium_flavor
 from . import valgrind_flavor
-from . import xsan_flavor
 
 
 TEST_EXPECTED_SKP_VERSION = '42'
@@ -54,12 +53,6 @@ def is_valgrind(builder_cfg):
   return 'Valgrind' in builder_cfg.get('extra_config', '')
 
 
-def is_xsan(builder_cfg):
-  return ('ASAN' in builder_cfg.get('extra_config', '') or
-          'MSAN' in builder_cfg.get('extra_config', '') or
-          'TSAN' in builder_cfg.get('extra_config', ''))
-
-
 class SkiaFlavorApi(recipe_api.RecipeApi):
   def get_flavor(self, builder_cfg):
     """Return a flavor utils object specific to the given builder."""
@@ -81,8 +74,6 @@ class SkiaFlavorApi(recipe_api.RecipeApi):
       return pdfium_flavor.PDFiumFlavorUtils(self.m)
     elif is_valgrind(builder_cfg):
       return valgrind_flavor.ValgrindFlavorUtils(self.m)
-    elif is_xsan(builder_cfg):
-      return xsan_flavor.XSanFlavorUtils(self.m)
     elif builder_cfg.get('configuration') == 'Coverage':
       return coverage_flavor.CoverageFlavorUtils(self.m)
     else:
index 3154ef3..61b12a1 100644 (file)
@@ -10,15 +10,17 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
     extra_config = self.m.vars.builder_cfg.get('extra_config', '')
 
     return any([
-      extra_config == 'GN',
+      'SAN' in extra_config,
       extra_config == 'Fast',
+      extra_config == 'GN',
       extra_config.startswith('SK')
     ])
 
-  def _run(self, title, cmd):
-    path = self.m.vars.default_env['PATH']
-    self.m.vars.default_env = {'PATH': path}
-    self.m.run(self.m.step, title, cmd=cmd, cwd=self.m.vars.skia_dir)
+  def _run(self, title, cmd, env=None):
+    self.m.vars.default_env = {k: v for (k,v)
+                               in self.m.vars.default_env.iteritems()
+                               if k in ['PATH']}
+    self.m.run(self.m.step, title, cmd=cmd, env=env, cwd=self.m.vars.skia_dir)
 
   def compile(self, unused_target, **kwargs):
     """Build Skia with GN."""
@@ -27,13 +29,15 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
     extra_config  = self.m.vars.builder_cfg.get('extra_config',  '')
     os            = self.m.vars.builder_cfg.get('os',            '')
 
+    clang_linux = str(self.m.vars.slave_dir.join('clang_linux'))
+
     cc, cxx = None, None
     extra_cflags = []
     extra_ldflags = []
 
     if compiler == 'Clang' and os == 'Ubuntu':
-      cc  = self.m.vars.slave_dir.join('clang_linux', 'bin', 'clang')
-      cxx = self.m.vars.slave_dir.join('clang_linux', 'bin', 'clang++')
+      cc  = clang_linux + '/bin/clang'
+      cxx = clang_linux + '/bin/clang++'
       extra_ldflags.append('-fuse-ld=lld')
     elif compiler == 'Clang':
       cc, cxx = 'clang', 'clang++'
@@ -44,17 +48,22 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
       extra_cflags.extend(['-march=native', '-fomit-frame-pointer', '-O3'])
     if extra_config.startswith('SK'):
       extra_cflags.append('-D' + extra_config)
+    if extra_config == 'MSAN':
+      extra_ldflags.append('-L' + clang_linux + '/msan')
 
     args = {}
 
     if configuration != 'Debug':
       args['is_debug'] = 'false'
+    if extra_config == 'MSAN':
+      args['skia_use_fontconfig'] = 'false'
 
     for (k,v) in {
       'cc':  cc,
       'cxx': cxx,
       'extra_cflags':  ' '.join(extra_cflags),
       'extra_ldflags': ' '.join(extra_ldflags),
+      'sanitize': extra_config if 'SAN' in extra_config else '',
     }.iteritems():
       if v:
         args[k] = '"%s"' % v
@@ -64,3 +73,28 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
     self._run('fetch-gn', [self.m.vars.skia_dir.join('bin', 'fetch-gn')])
     self._run('gn gen', ['gn', 'gen', self.out_dir, '--args=' + gn_args])
     self._run('ninja', ['ninja', '-C', self.out_dir])
+
+  def step(self, name, cmd, env=None, **kwargs):
+    app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0])
+    cmd = [app] + cmd[1:]
+    env = {}
+
+    clang_linux = str(self.m.vars.slave_dir.join('clang_linux'))
+    extra_config = self.m.vars.builder_cfg.get('extra_config', '')
+
+    if 'SAN' in extra_config:
+      # Sanitized binaries may want to run clang_linux/bin/llvm-symbolizer.
+      self.m.vars.default_env['PATH'] = '%%(PATH)s:%s' % clang_linux + '/bin'
+    elif 'Ubuntu' == self.m.vars.builder_cfg.get('os', ''):
+      cmd = ['catchsegv'] + cmd
+
+    if 'ASAN' == extra_config:
+      env[ 'ASAN_OPTIONS'] = 'symbolize=1 detect_leaks=1'
+      env[ 'LSAN_OPTIONS'] = 'symbolize=1 print_suppressions=1'
+      env['UBSAN_OPTIONS'] = 'symbolize=1 print_stacktrace=1'
+
+    if 'MSAN' == extra_config:
+      # Find the MSAN-built libc++.
+      env['LD_LIBRARY_PATH'] = clang_linux + '/msan'
+
+    self._run(name, cmd, env=env)
diff --git a/infra/bots/recipe_modules/flavor/xsan_flavor.py b/infra/bots/recipe_modules/flavor/xsan_flavor.py
deleted file mode 100644 (file)
index 9ecd4c9..0000000
+++ /dev/null
@@ -1,68 +0,0 @@
-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-"""Utils for running under *SAN"""
-
-
-import default_flavor
-
-
-class XSanFlavorUtils(default_flavor.DefaultFlavorUtils):
-  def __init__(self, m):
-    super(XSanFlavorUtils, self).__init__(m)
-    self._sanitizer = {
-      # We'd love to just pass 'address,undefined' and get all the checks, but
-      # we're not anywhere close to being able to do that.  Instead we start
-      # with a set of checks that we know pass or nearly pass.  See here for
-      # more information:
-      # http://clang.llvm.org/docs/UsersManual.html#controlling-code-generation
-      'ASAN': ('address,bool,function,integer-divide-by-zero,nonnull-attribute,'
-               'null,object-size,return,returns-nonnull-attribute,shift,'
-               'signed-integer-overflow,unreachable,vla-bound,vptr'),
-      # MSAN and TSAN can't run together with ASAN, so they're their own bots.
-      'MSAN': 'memory',
-      'TSAN': 'thread',
-    }[self.m.vars.builder_cfg['extra_config'].replace('Swarming', '')]
-
-  def compile(self, target, **kwargs):
-    cmd = [self.m.vars.skia_dir.join('tools', 'xsan_build'),
-           self._sanitizer, target]
-    self.m.run(self.m.step, 'build %s' % target, cmd=cmd,
-               cwd=self.m.vars.skia_dir, **kwargs)
-
-  def copy_extra_build_products(self, swarming_out_dir):
-    # Include msan_out if MSAN.
-    if 'MSAN' in self.m.vars.builder_cfg['extra_config']:
-      msan_out = self.m.path.join(
-          'third_party', 'externals', 'llvm', 'msan_out')
-      self.m.file.copytree(
-          'copy msan_out',
-          self.m.vars.skia_dir.join(msan_out),
-          swarming_out_dir.join(msan_out),
-          symlinks=True)
-    # Include llvm_symbolizer from the Chromium DEPS so that suppressions work
-    # by symbol name.
-    # TODO(benjaminwagner): Figure out how to add this to Skia DEPS for
-    # target_os 'llvm'.
-    self.m.file.copytree(
-        'copy llvm-build',
-        self.m.vars.checkout_root.join('src', 'third_party', 'llvm-build'),
-        swarming_out_dir.join('llvm-build'),
-        symlinks=True)
-
-  def step(self, name, cmd, env=None, **kwargs):
-    """Wrapper for the Step API; runs a step as appropriate for this flavor."""
-    env = dict(env or {})
-    env['ASAN_OPTIONS'] = 'symbolize=1 detect_leaks=1'
-    env['LSAN_OPTIONS'] = 'symbolize=1 print_suppressions=1'
-    self.m.vars.default_env['PATH'] = '%%(PATH)s:%s' % (
-        self.m.vars.slave_dir.join('llvm-build', 'Release+Asserts', 'bin'))
-    env['LD_LIBRARY_PATH'] = self.m.vars.slave_dir.join(
-        'third_party', 'externals', 'llvm', 'msan_out', 'lib')
-
-    path_to_app = self.out_dir.join(cmd[0])
-    new_cmd = [path_to_app]
-    new_cmd.extend(cmd[1:])
-    return self.m.run(self.m.step, name, cmd=new_cmd, env=env, **kwargs)
index 5418e26..54da3b3 100644 (file)
@@ -82,16 +82,14 @@ class SkiaVarsApi(recipe_api.RecipeApi):
     self.tmp_dir = self.m.path['slave_build'].join('tmp')
 
     # Some bots also require a checkout of chromium.
-    self.need_chromium_checkout = 'CommandBuffer' in self.builder_name
+    self.need_chromium_checkout = False
     if 'CommandBuffer' in self.builder_name:
+      self.need_chromium_checkout = True
       self.gclient_env['GYP_CHROMIUM_NO_ACTION'] = '0'
-    if ((self.is_compile_bot and
-         'SAN' in self.builder_name) or
-        'RecreateSKPs' in self.builder_name):
+    if 'RecreateSKPs' in self.builder_name:
       self.need_chromium_checkout = True
-      if 'RecreateSKPs' in self.builder_name:
-        self.gclient_env['CPPFLAGS'] = (
-            '-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1')
+      self.gclient_env['CPPFLAGS'] = (
+          '-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1')
 
     # Some bots also require a checkout of PDFium.
     self.need_pdfium_checkout = 'PDFium' in self.builder_name
diff --git a/infra/bots/recipes/swarm_compile.expected/Build-Ubuntu-Clang-x86_64-Debug-ASAN.json b/infra/bots/recipes/swarm_compile.expected/Build-Ubuntu-Clang-x86_64-Debug-ASAN.json
new file mode 100644 (file)
index 0000000..3ef0410
--- /dev/null
@@ -0,0 +1,189 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[CUSTOM_/_B_WORK]",
+      "511"
+    ],
+    "name": "makedirs checkout_path",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py",
+      "config",
+      "--spec",
+      "cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': 'DEPS', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]\ntarget_os = ['llvm']"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]",
+    "env": {
+      "BUILDTYPE": "Debug",
+      "CHROME_HEADLESS": "1",
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-Clang-x86_64-Debug-ASAN"
+    },
+    "name": "gclient setup"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py",
+      "sync",
+      "--nohooks",
+      "--force",
+      "--verbose",
+      "--delete_unversioned_trees",
+      "--revision",
+      "skia@abc123",
+      "--output-json",
+      "/path/to/tmp/json"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]",
+    "env": {
+      "BUILDTYPE": "Debug",
+      "CHROME_HEADLESS": "1",
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]",
+      "SKIA_OUT": "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-Clang-x86_64-Debug-ASAN"
+    },
+    "name": "gclient sync",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"solutions\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"skia/\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"revision\": 164710@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@got_revision@164710@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "[CUSTOM_/_B_WORK]/skia/bin/fetch-gn"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "name": "fetch-gn"
+  },
+  {
+    "cmd": [
+      "gn",
+      "gen",
+      "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-Clang-x86_64-Debug-ASAN/Debug",
+      "--args=cc=\"[SLAVE_BUILD]/clang_linux/bin/clang\" cxx=\"[SLAVE_BUILD]/clang_linux/bin/clang++\" extra_ldflags=\"-fuse-ld=lld\" sanitize=\"ASAN\""
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "name": "gn gen"
+  },
+  {
+    "cmd": [
+      "ninja",
+      "-C",
+      "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-Clang-x86_64-Debug-ASAN/Debug"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "name": "ninja"
+  },
+  {
+    "cmd": [
+      "[CUSTOM_/_B_WORK]/skia/bin/fetch-gn"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "name": "fetch-gn (2)"
+  },
+  {
+    "cmd": [
+      "gn",
+      "gen",
+      "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-Clang-x86_64-Debug-ASAN/Debug",
+      "--args=cc=\"[SLAVE_BUILD]/clang_linux/bin/clang\" cxx=\"[SLAVE_BUILD]/clang_linux/bin/clang++\" extra_ldflags=\"-fuse-ld=lld\" sanitize=\"ASAN\""
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "name": "gn gen (2)"
+  },
+  {
+    "cmd": [
+      "ninja",
+      "-C",
+      "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-Clang-x86_64-Debug-ASAN/Debug"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "name": "ninja (2)"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['dm', 'dm.exe', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']\n\ntry:\n  os.makedirs(dst)\nexcept OSError as e:\n  if e.errno != errno.EEXIST:\n    raise\n\nfor pattern in build_products_whitelist:\n  path = os.path.join(src, pattern)\n  for f in glob.glob(path):\n    dst_path = os.path.join(dst, os.path.relpath(f, src))\n    if not os.path.isdir(os.path.dirname(dst_path)):\n      os.makedirs(os.path.dirname(dst_path))\n    print 'Copying build product %s to %s' % (f, dst_path)\n    shutil.move(f, dst_path)\n",
+      "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-Clang-x86_64-Debug-ASAN/Debug",
+      "[CUSTOM_[SWARM_OUT_DIR]]/out/Debug"
+    ],
+    "name": "copy build products",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@import errno@@@",
+      "@@@STEP_LOG_LINE@python.inline@import glob@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import shutil@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
+      "@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['dm', 'dm.exe', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(dst)@@@",
+      "@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if e.errno != errno.EEXIST:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    raise@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  path = os.path.join(src, pattern)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  for f in glob.glob(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    dst_path = os.path.join(dst, os.path.relpath(f, src))@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if not os.path.isdir(os.path.dirname(dst_path)):@@@",
+      "@@@STEP_LOG_LINE@python.inline@      os.makedirs(os.path.dirname(dst_path))@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print 'Copying build product %s to %s' % (f, dst_path)@@@",
+      "@@@STEP_LOG_LINE@python.inline@    shutil.move(f, dst_path)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file
index 5cd108b..656a206 100644 (file)
@@ -28,7 +28,7 @@
       "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py",
       "config",
       "--spec",
-      "cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': 'DEPS', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}, {'deps_file': 'DEPS', 'managed': False, 'name': 'src', 'url': 'https://chromium.googlesource.com/chromium/src.git'}]\ntarget_os = ['llvm']"
+      "cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': 'DEPS', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]\ntarget_os = ['llvm']"
     ],
     "cwd": "[CUSTOM_/_B_WORK]",
     "env": {
@@ -51,8 +51,6 @@
       "--delete_unversioned_trees",
       "--revision",
       "skia@abc123",
-      "--revision",
-      "src@origin/lkgr",
       "--output-json",
       "/path/to/tmp/json"
     ],
@@ -69,9 +67,6 @@
       "@@@STEP_LOG_LINE@json.output@  \"solutions\": {@@@",
       "@@@STEP_LOG_LINE@json.output@    \"skia/\": {@@@",
       "@@@STEP_LOG_LINE@json.output@      \"revision\": 164710@@@",
-      "@@@STEP_LOG_LINE@json.output@    }, @@@",
-      "@@@STEP_LOG_LINE@json.output@    \"src/\": {@@@",
-      "@@@STEP_LOG_LINE@json.output@      \"revision\": 170242@@@",
       "@@@STEP_LOG_LINE@json.output@    }@@@",
       "@@@STEP_LOG_LINE@json.output@  }@@@",
       "@@@STEP_LOG_LINE@json.output@}@@@",
   },
   {
     "cmd": [
-      "python",
-      "-u",
-      "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py",
-      "runhooks"
+      "[CUSTOM_/_B_WORK]/skia/bin/fetch-gn"
     ],
-    "cwd": "[CUSTOM_/_B_WORK]",
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
     "env": {
-      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]"
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]"
     },
-    "name": "gclient runhooks"
+    "name": "fetch-gn"
   },
   {
     "cmd": [
-      "[CUSTOM_/_B_WORK]/skia/tools/xsan_build",
-      "memory",
-      "dm"
+      "gn",
+      "gen",
+      "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-GCC-x86_64-Debug-MSAN/Debug",
+      "--args=cc=\"gcc\" cxx=\"g++\" extra_ldflags=\"-L[SLAVE_BUILD]/clang_linux/msan\" sanitize=\"MSAN\" skia_use_fontconfig=false"
     ],
     "cwd": "[CUSTOM_/_B_WORK]/skia",
     "env": {
-      "BUILDTYPE": "Debug",
-      "CHROME_HEADLESS": "1",
-      "GYP_DEFINES": "skia_arch_type=x86_64 skia_warnings_as_errors=1",
-      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]",
-      "SKIA_OUT": "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-GCC-x86_64-Debug-MSAN"
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]"
     },
-    "name": "build dm"
+    "name": "gn gen"
   },
   {
     "cmd": [
-      "[CUSTOM_/_B_WORK]/skia/tools/xsan_build",
-      "memory",
-      "nanobench"
+      "ninja",
+      "-C",
+      "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-GCC-x86_64-Debug-MSAN/Debug"
     ],
     "cwd": "[CUSTOM_/_B_WORK]/skia",
     "env": {
-      "BUILDTYPE": "Debug",
-      "CHROME_HEADLESS": "1",
-      "GYP_DEFINES": "skia_arch_type=x86_64 skia_warnings_as_errors=1",
-      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]",
-      "SKIA_OUT": "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-GCC-x86_64-Debug-MSAN"
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "name": "ninja"
+  },
+  {
+    "cmd": [
+      "[CUSTOM_/_B_WORK]/skia/bin/fetch-gn"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]"
     },
-    "name": "build nanobench"
+    "name": "fetch-gn (2)"
+  },
+  {
+    "cmd": [
+      "gn",
+      "gen",
+      "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-GCC-x86_64-Debug-MSAN/Debug",
+      "--args=cc=\"gcc\" cxx=\"g++\" extra_ldflags=\"-L[SLAVE_BUILD]/clang_linux/msan\" sanitize=\"MSAN\" skia_use_fontconfig=false"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "name": "gn gen (2)"
+  },
+  {
+    "cmd": [
+      "ninja",
+      "-C",
+      "[CUSTOM_/_B_WORK]/skia/out/Build-Ubuntu-GCC-x86_64-Debug-MSAN/Debug"
+    ],
+    "cwd": "[CUSTOM_/_B_WORK]/skia",
+    "env": {
+      "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]"
+    },
+    "name": "ninja (2)"
   },
   {
     "cmd": [
     ]
   },
   {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
-      "[CUSTOM_/_B_WORK]/skia/third_party/externals/llvm/msan_out",
-      "[CUSTOM_[SWARM_OUT_DIR]]/third_party/externals/llvm/msan_out",
-      "1"
-    ],
-    "name": "copy msan_out"
-  },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
-      "[CUSTOM_/_B_WORK]/src/third_party/llvm-build",
-      "[CUSTOM_[SWARM_OUT_DIR]]/llvm-build",
-      "1"
-    ],
-    "name": "copy llvm-build"
-  },
-  {
     "name": "$result",
     "recipe_result": null,
     "status_code": 0
index 284f385..8c09084 100644 (file)
@@ -29,6 +29,7 @@ TEST_BUILDERS = {
       'Build-Mac-Clang-x86_64-Release-CMake',
       'Build-Mac-Clang-x86_64-Release-GN',
       'Build-Ubuntu-Clang-arm64-Release-GN_Android',
+      'Build-Ubuntu-Clang-x86_64-Debug-ASAN',
       'Build-Ubuntu-Clang-x86_64-Debug-GN',
       'Build-Ubuntu-GCC-Arm7-Debug-Android-Trybot',
       'Build-Ubuntu-GCC-Arm7-Debug-Android_FrameworkDefs',
index e34c1c4..498fe93 100644 (file)
   },
   {
     "cmd": [
-      "catchsegv",
       "[SLAVE_BUILD]/out/Release/nanobench",
       "--undefok",
       "-i",
       "os",
       "Mac"
     ],
-    "env": {
-      "BUILDTYPE": "Release",
-      "CHROME_HEADLESS": "1",
-      "SKIA_OUT": "[SLAVE_BUILD]/out"
-    },
+    "cwd": "[SLAVE_BUILD]/skia",
     "name": "nanobench"
   },
   {
diff --git a/infra/bots/recipes/swarm_perf.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-GN.json b/infra/bots/recipes/swarm_perf.expected/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-GN.json
new file mode 100644 (file)
index 0000000..99ed57f
--- /dev/null
@@ -0,0 +1,210 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/skp/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "Get downloaded SKP VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "42",
+      "[SLAVE_BUILD]/tmp/SKP_VERSION"
+    ],
+    "name": "write SKP_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/skimage/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "Get downloaded skimage VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "42",
+      "[SLAVE_BUILD]/tmp/SK_IMAGE_VERSION"
+    ],
+    "name": "write SK_IMAGE_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/svg/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "Get downloaded SVG VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "42",
+      "[SLAVE_BUILD]/tmp/SVG_VERSION"
+    ],
+    "name": "write SVG_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
+      "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-GN/data"
+    ],
+    "env": {
+      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
+    },
+    "name": "rmtree data",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-GN/data",
+      "511"
+    ],
+    "name": "makedirs data",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "catchsegv",
+      "[SLAVE_BUILD]/out/Release/nanobench",
+      "--undefok",
+      "-i",
+      "[SLAVE_BUILD]/skia/resources",
+      "--skps",
+      "[SLAVE_BUILD]/skp",
+      "--images",
+      "[SLAVE_BUILD]/skimage/nanobench",
+      "--nogpu",
+      "--pre_log",
+      "--scales",
+      "1.0",
+      "1.1",
+      "--config",
+      "565",
+      "8888",
+      "gpu",
+      "nonrendering",
+      "angle",
+      "hwui",
+      "f16",
+      "srgb",
+      "msaa16",
+      "nvpr16",
+      "nvprdit16",
+      "--match",
+      "~interlaced1.png",
+      "~interlaced2.png",
+      "~interlaced3.png",
+      "~inc0.gif",
+      "~inc1.gif",
+      "~incInterlaced.gif",
+      "~inc0.jpg",
+      "~incGray.jpg",
+      "~inc0.wbmp",
+      "~inc1.wbmp",
+      "~inc0.webp",
+      "~inc1.webp",
+      "~inc0.ico",
+      "~inc1.ico",
+      "~inc0.png",
+      "~inc1.png",
+      "~inc2.png",
+      "~inc12.png",
+      "~inc13.png",
+      "~inc14.png",
+      "~inc0.webp",
+      "~inc1.webp",
+      "--outResultsFile",
+      "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-GN/data/nanobench_abc123.json",
+      "--properties",
+      "gitHash",
+      "abc123",
+      "build_number",
+      "5",
+      "--key",
+      "arch",
+      "x86_64",
+      "compiler",
+      "Clang",
+      "cpu_or_gpu",
+      "CPU",
+      "cpu_or_gpu_value",
+      "AVX2",
+      "extra_config",
+      "GN",
+      "model",
+      "GCE",
+      "os",
+      "Ubuntu"
+    ],
+    "cwd": "[SLAVE_BUILD]/skia",
+    "name": "nanobench"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-GN/data",
+      "511"
+    ],
+    "name": "makedirs perf_dir",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file
index 0701809..8c63136 100644 (file)
@@ -24,18 +24,19 @@ TEST_BUILDERS = {
   'client.skia': {
     'skiabot-linux-swarm-000': [
       'Perf-Android-GCC-GalaxyS3-GPU-Mali400-Arm7-Release',
+      'Perf-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan',
       'Perf-Android-GCC-Nexus5-GPU-Adreno330-Arm7-Debug',
       'Perf-Android-GCC-Nexus6-GPU-Adreno420-Arm7-Release',
       'Perf-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release',
       'Perf-Android-GCC-NexusPlayer-GPU-PowerVR-x86-Release',
-      'Perf-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan',
-      'Perf-iOS-Clang-iPad4-GPU-SGX554-Arm7-Debug',
       'Perf-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release-GN',
+      'Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-GN',
       'Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind',
       'Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-VisualBench',
-      'Perf-Win-MSVC-GCE-CPU-AVX2-x86_64-Release',
       'Perf-Win-MSVC-GCE-CPU-AVX2-x86_64-Debug',
+      'Perf-Win-MSVC-GCE-CPU-AVX2-x86_64-Release',
       'Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release-Trybot',
+      'Perf-iOS-Clang-iPad4-GPU-SGX554-Arm7-Debug',
     ],
   },
 }
diff --git a/infra/bots/recipes/swarm_test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-ASAN.json b/infra/bots/recipes/swarm_test.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-ASAN.json
new file mode 100644 (file)
index 0000000..6bb52e3
--- /dev/null
@@ -0,0 +1,349 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/skp/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "Get downloaded SKP VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "42",
+      "[SLAVE_BUILD]/tmp/SKP_VERSION"
+    ],
+    "name": "write SKP_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/skimage/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "Get downloaded skimage VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "42",
+      "[SLAVE_BUILD]/tmp/SK_IMAGE_VERSION"
+    ],
+    "name": "write SK_IMAGE_VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/svg/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "Get downloaded SVG VERSION"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "42",
+      "[SLAVE_BUILD]/tmp/SVG_VERSION"
+    ],
+    "name": "write SVG_VERSION"
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/out/Debug/dm",
+      "--undefok",
+      "--resourcePath",
+      "[SLAVE_BUILD]/skia/resources",
+      "--skps",
+      "[SLAVE_BUILD]/skp",
+      "--images",
+      "[SLAVE_BUILD]/skimage/dm",
+      "--colorImages",
+      "[SLAVE_BUILD]/skimage/colorspace",
+      "--nameByHash",
+      "--properties",
+      "gitHash",
+      "abc123",
+      "master",
+      "client.skia",
+      "builder",
+      "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-ASAN",
+      "build_number",
+      "5",
+      "--svgs",
+      "[SLAVE_BUILD]/svg",
+      "--key",
+      "arch",
+      "x86_64",
+      "compiler",
+      "GCC",
+      "configuration",
+      "Debug",
+      "cpu_or_gpu",
+      "CPU",
+      "cpu_or_gpu_value",
+      "AVX2",
+      "extra_config",
+      "ASAN",
+      "model",
+      "GCE",
+      "os",
+      "Ubuntu",
+      "--nogpu",
+      "--config",
+      "565",
+      "8888",
+      "gpu",
+      "gpusrgb",
+      "pdf",
+      "msaa16",
+      "f16",
+      "srgb",
+      "sp-8888",
+      "2ndpic-8888",
+      "lite-8888",
+      "serialize-8888",
+      "tiles_rt-8888",
+      "pic-8888",
+      "--src",
+      "tests",
+      "gm",
+      "image",
+      "colorImage",
+      "svg",
+      "--blacklist",
+      "f16",
+      "_",
+      "_",
+      "dstreadshuffle",
+      "f16",
+      "image",
+      "_",
+      "_",
+      "srgb",
+      "image",
+      "_",
+      "_",
+      "gpusrgb",
+      "image",
+      "_",
+      "_",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bleed_image",
+      "serialize-8888",
+      "gm",
+      "_",
+      "c_gms",
+      "serialize-8888",
+      "gm",
+      "_",
+      "colortype",
+      "serialize-8888",
+      "gm",
+      "_",
+      "colortype_xfermodes",
+      "serialize-8888",
+      "gm",
+      "_",
+      "drawfilter",
+      "serialize-8888",
+      "gm",
+      "_",
+      "fontmgr_bounds_0.75_0",
+      "serialize-8888",
+      "gm",
+      "_",
+      "fontmgr_bounds_1_-0.25",
+      "serialize-8888",
+      "gm",
+      "_",
+      "fontmgr_bounds",
+      "serialize-8888",
+      "gm",
+      "_",
+      "fontmgr_match",
+      "serialize-8888",
+      "gm",
+      "_",
+      "fontmgr_iter",
+      "serialize-8888",
+      "gm",
+      "_",
+      "imagemasksubset",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bitmapfilters",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bitmapshaders",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bleed",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bleed_alpha_bmp",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bleed_alpha_bmp_shader",
+      "serialize-8888",
+      "gm",
+      "_",
+      "convex_poly_clip",
+      "serialize-8888",
+      "gm",
+      "_",
+      "extractalpha",
+      "serialize-8888",
+      "gm",
+      "_",
+      "filterbitmap_checkerboard_32_32_g8",
+      "serialize-8888",
+      "gm",
+      "_",
+      "filterbitmap_image_mandrill_64",
+      "serialize-8888",
+      "gm",
+      "_",
+      "shadows",
+      "serialize-8888",
+      "gm",
+      "_",
+      "simpleaaclip_aaclip",
+      "serialize-8888",
+      "gm",
+      "_",
+      "composeshader_bitmap",
+      "serialize-8888",
+      "gm",
+      "_",
+      "scaled_tilemodes_npot",
+      "serialize-8888",
+      "gm",
+      "_",
+      "scaled_tilemodes",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bleed_alpha_image",
+      "serialize-8888",
+      "gm",
+      "_",
+      "bleed_alpha_image_shader",
+      "sp-8888",
+      "gm",
+      "_",
+      "drawfilter",
+      "pic-8888",
+      "gm",
+      "_",
+      "drawfilter",
+      "2ndpic-8888",
+      "gm",
+      "_",
+      "drawfilter",
+      "lite-8888",
+      "gm",
+      "_",
+      "drawfilter",
+      "sp-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-picture",
+      "pic-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-picture",
+      "2ndpic-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-picture",
+      "serialize-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-picture",
+      "sp-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-raster",
+      "pic-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-raster",
+      "2ndpic-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-raster",
+      "serialize-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-raster",
+      "sp-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-ctable",
+      "pic-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-ctable",
+      "2ndpic-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-ctable",
+      "serialize-8888",
+      "gm",
+      "_",
+      "image-cacherator-from-ctable",
+      "sp-8888",
+      "gm",
+      "_",
+      "gamut",
+      "pic-8888",
+      "gm",
+      "_",
+      "gamut",
+      "lite-8888",
+      "gm",
+      "_",
+      "gamut",
+      "2ndpic-8888",
+      "gm",
+      "_",
+      "gamut",
+      "serialize-8888",
+      "gm",
+      "_",
+      "gamut"
+    ],
+    "cwd": "[SLAVE_BUILD]/skia",
+    "env": {
+      "ASAN_OPTIONS": "symbolize=1 detect_leaks=1",
+      "LSAN_OPTIONS": "symbolize=1 print_suppressions=1",
+      "PATH": "%(PATH)s:[SLAVE_BUILD]/clang_linux/bin",
+      "UBSAN_OPTIONS": "symbolize=1 print_stacktrace=1"
+    },
+    "name": "dm"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file
index 965f229..53bd035 100644 (file)
       "~Once",
       "~Shared"
     ],
+    "cwd": "[SLAVE_BUILD]/skia",
     "env": {
-      "ASAN_OPTIONS": "symbolize=1 detect_leaks=1",
-      "BUILDTYPE": "Debug",
-      "CHROME_HEADLESS": "1",
-      "LD_LIBRARY_PATH": "[SLAVE_BUILD]/third_party/externals/llvm/msan_out/lib",
-      "LSAN_OPTIONS": "symbolize=1 print_suppressions=1",
-      "PATH": "%(PATH)s:[SLAVE_BUILD]/llvm-build/Release+Asserts/bin",
-      "SKIA_OUT": "[SLAVE_BUILD]/out"
+      "LD_LIBRARY_PATH": "[SLAVE_BUILD]/clang_linux/msan",
+      "PATH": "%(PATH)s:[SLAVE_BUILD]/clang_linux/bin"
     },
     "name": "dm"
   },
index 818471b..487cab7 100644 (file)
       "--match",
       "~ReadWriteAlpha"
     ],
+    "cwd": "[SLAVE_BUILD]/skia",
     "env": {
-      "ASAN_OPTIONS": "symbolize=1 detect_leaks=1",
-      "BUILDTYPE": "Release",
-      "CHROME_HEADLESS": "1",
-      "LD_LIBRARY_PATH": "[SLAVE_BUILD]/third_party/externals/llvm/msan_out/lib",
-      "LSAN_OPTIONS": "symbolize=1 print_suppressions=1",
-      "PATH": "%(PATH)s:[SLAVE_BUILD]/llvm-build/Release+Asserts/bin",
-      "SKIA_OUT": "[SLAVE_BUILD]/out"
+      "PATH": "%(PATH)s:[SLAVE_BUILD]/clang_linux/bin"
     },
     "name": "dm"
   },
index 5809988..99fff3b 100644 (file)
@@ -39,6 +39,7 @@ TEST_BUILDERS = {
       'Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot',
       'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug',
       'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug',
+      'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-ASAN',
       'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-MSAN',
       'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared',
       'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-TSAN',
index 56758ed..5c39f43 100644 (file)
     "cmd": [
       "python",
       "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/clang_linux/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "read clang_linux VERSION (2)"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
       "[SLAVE_BUILD]/swarming_temp_dir",
       "511"
       "skimage:skia/bots/skimage:version:0",
       "--cipd-package",
       "svg:skia/bots/svg:version:0",
+      "--cipd-package",
+      "clang_linux:skia/bots/clang_linux:version:0",
       "def456",
       "--",
       "--workdir",
index 1f17584..060fa73 100644 (file)
@@ -670,26 +670,27 @@ def RunSteps(api):
 
   extra_hashes = []
 
+  builder_name = api.properties['buildername']
+
   # Get ready to compile.
   infrabots_dir = api.path['checkout'].join('infra', 'bots')
-  if 'Infra' in api.properties['buildername']:
+  if 'Infra' in builder_name:
     return infra_swarm(api, got_revision, infrabots_dir, extra_hashes)
 
-  builder_cfg = api.builder_name_schema.DictForBuilderName(
-      api.properties['buildername'])
+  builder_cfg = api.builder_name_schema.DictForBuilderName(builder_name)
 
-  if 'RecreateSKPs' in api.properties['buildername']:
+  if 'RecreateSKPs' in builder_name:
     recreate_skps_swarm(api, builder_cfg, got_revision, infrabots_dir,
                         extra_hashes)
     return
 
-  if '-CT_' in api.properties['buildername']:
+  if '-CT_' in builder_name:
     ct_skps_swarm(api, builder_cfg, got_revision, infrabots_dir, extra_hashes)
     return
 
   # Compile.
   do_compile_steps = True
-  if 'Coverage' in api.properties['buildername']:
+  if 'Coverage' in builder_name:
     do_compile_steps = False
   if do_compile_steps:
     extra_hashes.append(compile_steps_swarm(
@@ -718,6 +719,10 @@ def RunSteps(api):
   cipd_packages.append(cipd_pkg(api, infrabots_dir, 'skimage'))
   cipd_packages.append(cipd_pkg(api, infrabots_dir, 'svg'))
 
+  # To find llvm-symbolizer and/or MSAN-compiled libc++.
+  if 'Ubuntu' in builder_name and 'SAN' in builder_name:
+    cipd_packages.append(cipd_pkg(api, infrabots_dir, 'clang_linux'))
+
   # Trigger test and perf tasks.
   test_task = None
   perf_task = None
index 623c47e..3ac5dd2 100644 (file)
@@ -8,7 +8,8 @@ declare_args() {
 
 import("../third_party.gni")
 
-if (is_android) {
+# TODO: build from source all the time?
+if (is_android || sanitize == "MSAN") {
   third_party("freetype2") {
     public_include_dirs = [ "../externals/freetype/include" ]
 
index 57c8020..a917e2d 100644 (file)
@@ -1,12 +1,2 @@
-#if 0
-
-#  This file must be a no-op C #include header, and a valid *SAN blacklist file.
-#  Luckily, anything starting with # is a comment to *SAN blacklist files,
-#  and anything inside #if 0 is ignored by C.  Yippee!
-#
-#  If you want to type '*', type '.*' instead.  Don't make C comments!
-
 # Suppress third_party/externals.  We mostly care about our own code.
-src:.*third_party/externals.*
-
-#endif
+src:*third_party/externals*
diff --git a/tools/xsan_build b/tools/xsan_build
deleted file mode 100755 (executable)
index b569a7b..0000000
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/bin/bash
-
-# Build Skia with one of Clang's many sanitizers.
-#
-# $ tools/xsan_build {address,thread,undefined,etc.} [any other flags to pass to make...]
-#
-# This script assumes the use of Clang >=3.2.
-#
-# For more information, see:
-#   http://clang.llvm.org/docs/UsersManual.html#controlling-code-generation
-
-set -e
-set -x
-
-here=$(cd `dirname $0`; echo `pwd`)
-cores=48
-
-echo "Bootstrapping CMake"
-pushd $here/../third_party/externals/cmake
-./bootstrap --parallel=$cores
-make -j $cores cmake
-popd
-
-cmake=$here/../third_party/externals/cmake/bin/cmake
-
-echo "Building Clang"
-pushd $here/../third_party/externals/llvm
-mkdir -p out/
-cd out/
-rm -f CMakeCache.txt   # Force CMake to re-configure, in case DEPS has changed.
-$cmake -DCMAKE_BUILD_TYPE=Release -G Ninja ..
-ninja
-popd
-
-export CC=$here/../third_party/externals/llvm/out/bin/clang
-export CXX=$here/../third_party/externals/llvm/out/bin/clang++
-$CC --version
-
-if [[ "$1" == "memory" ]]; then
-    echo "Building libc++ with MSAN"
-    pushd $here/../third_party/externals/llvm
-    mkdir -p msan_out/
-    cd msan_out/
-    rm -f CMakeCache.txt   # Force CMake to re-configure, in case DEPS has changed.
-    $cmake -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_BUILD_TYPE=Release -G Ninja ..
-    ninja cxx cxxabi   # No need to build all of LLVM+Clang with MSAN, just libc++.
-    popd
-
-    msan_out=$here/../third_party/externals/llvm/msan_out
-
-    export GYP_DEFINES="skia_gpu=0 skia_no_fontconfig=1 skia_freetype_static=1 ${GYP_DEFINES}"
-    export CXXFLAGS="-stdlib=libc++ -I$msan_out/include ${CXX_FLAGS}"
-    export LDFLAGS="-stdlib=libc++ -L$msan_out/lib -Wl,-rpath,$msan_out/lib ${LDFLAGS}"
-fi
-export GYP_DEFINES="skia_sanitizer=$1 ${GYP_DEFINES}"
-
-shift
-make $@