Swarming bots: setup for skipping download of build products

Turns out it's pretty easy to pass the compile outputs to the test task by just adding the hash to the "includes" list in the .isolated file.  So the flow is:
1. Isolate skia repo
2. Run compile task, record hash of results
3. Isolate test inputs for DM. This writes a .isolated file
4. Edit the .isolated file from #3 to include the hash from #2
5. Upload the modified .isolated file to the isolate server
6. Trigger the swarming task for DM
7. Wait for DM task to finish, download results from isolate server
8. Upload results to GS as normal

I expect the swarming bots to break when this is committed due to the moved out directory.  The associated recipe change will fix them.

NOTRY=true
BUG=skia:4763
GOLD_TRYBOT_URL= https://gold.skia.org/search2?unt=true&query=source_type%3Dgm&master=false&issue=1759553003

Review URL: https://codereview.chromium.org/1759553003

diff --git a/infra/bots/add_isolated_input.py b/infra/bots/add_isolated_input.py
new file mode 100644 (file)
index 0000000..70bde6d
--- /dev/null
+++ b/infra/bots/add_isolated_input.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import argparse
+import json
+
+
+"""Add the given hash to the includes section of the given isolated file."""
+
+
+def add_isolated_hash(isolated_file, hash_str):
+  with open(isolated_file) as f:
+    isolated = json.load(f)
+  isolated['includes'].append(hash_str)
+  with open(isolated_file, 'w') as f:
+    json.dump(isolated, f, sort_keys=True)
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--isolated_file', required=True)
+  parser.add_argument('--hash', required=True)
+  args = parser.parse_args()
+  add_isolated_hash(args.isolated_file, args.hash)
+
+
+if __name__ == '__main__':
+  main()

diff --git a/infra/bots/common.py b/infra/bots/common.py
index b532d5db55eb1f81b82eb47b0e6cce13d0c455ef..31f1f824af42a65814aafd896ba6c547446c44dd 100644 (file)
--- a/infra/bots/common.py
+++ b/infra/bots/common.py
@@ -155,9 +155,9 @@ class BotInfo(object):
     self.spec = self.get_bot_spec(bot_name)
     self.bot_cfg = self.spec['builder_cfg']
     if self.bot_cfg['role'] == 'Build':
-      self.out_dir = swarm_out_dir
+      self.out_dir = os.path.join(swarm_out_dir, 'out')
     else:
-      self.out_dir = os.path.join(self.skia_dir, 'out', self.name)
+      self.out_dir = os.path.join(os.pardir, 'out')
     self.configuration = self.spec['configuration']
     self.default_env = {
       'SKIA_OUT': self.out_dir,

diff --git a/infra/bots/compile_skia.isolate b/infra/bots/compile_skia.isolate
index 5866f4c797595f4f671fc6a2553a1131173aeb8b..529ca151b9ae503d2faf1181cb0dd007340f2fff 100644 (file)
--- a/infra/bots/compile_skia.isolate
+++ b/infra/bots/compile_skia.isolate
@@ -4,7 +4,7 @@
   ],
   'variables': {
     'command': [
-      'python', 'compile_skia.py', '--builder_name', '<(BUILDER_NAME)', '--swarm_out_dir', '${ISOLATED_OUTDIR}/out',
+      'python', 'compile_skia.py', '--builder_name', '<(BUILDER_NAME)', '--swarm_out_dir', '${ISOLATED_OUTDIR}',
     ],
   },
 }

diff --git a/infra/bots/infrabots.isolate b/infra/bots/infrabots.isolate
new file mode 100644 (file)
index 0000000..4936c62
--- /dev/null
+++ b/infra/bots/infrabots.isolate
@@ -0,0 +1,13 @@
+{
+  'variables': {
+    'files': [
+      '../../tools/__init__.py',
+      '../../tools/buildbot_spec.py',
+      '../../tools/builder_name_schema.json',
+      '../../tools/builder_name_schema.py',
+      '../../tools/dm_flags.py',
+      '../../tools/nanobench_flags.py',
+      './',
+    ],
+  },
+}

diff --git a/infra/bots/perf_skia.isolate b/infra/bots/perf_skia.isolate
index 420a0d0db819c657a8aaf073eb9428488d7d2e54..dd442baf1d66013e979550f6098155521df0795a 100644 (file)
--- a/infra/bots/perf_skia.isolate
+++ b/infra/bots/perf_skia.isolate
@@ -1,7 +1,8 @@
 {
   'includes': [
     'images.isolate',
-    'skia_repo.isolate',
+    'infrabots.isolate',
+    'resources.isolate',
     'skps.isolate',
   ],
   'variables': {

diff --git a/infra/bots/resources.isolate b/infra/bots/resources.isolate
new file mode 100644 (file)
index 0000000..be73552
--- /dev/null
+++ b/infra/bots/resources.isolate
@@ -0,0 +1,7 @@
+{
+  'variables': {
+    'files': [
+      '../../resources/',
+    ],
+  },
+}

diff --git a/infra/bots/test_skia.isolate b/infra/bots/test_skia.isolate
index d9eab8cfd9655843f3fb915e1aa362c6885e7f10..d5bb98cad38e9e6f1b2f9c4cc90df294976810b8 100644 (file)
--- a/infra/bots/test_skia.isolate
+++ b/infra/bots/test_skia.isolate
@@ -1,7 +1,8 @@
 {
   'includes': [
     'images.isolate',
-    'skia_repo.isolate',
+    'infrabots.isolate',
+    'resources.isolate',
     'skps.isolate',
   ],
   'variables': {