Add extracting GM comments to the housekeeper bot.

BUG=skia:
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2237033002

Review-Url: https://codereview.chromium.org/2237033002

diff --git a/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit-Trybot.json b/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit-Trybot.json
index 56837514fbaffa798cad7eecc3274dbab1b6930f..2fdcad62fea686c8bd655661fcb564ae1be5863b 100644 (file)
--- a/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit-Trybot.json
+++ b/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit-Trybot.json
@@ -55,6 +55,38 @@
     },
     "name": "generate and upload binary size data"
   },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/go/go/bin/go",
+      "get",
+      "go.skia.org/infra/comments/go/extract_comments"
+    ],
+    "cwd": "[SLAVE_BUILD]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "GOPATH": "[SLAVE_BUILD]/tmp/golib",
+      "SKIA_OUT": "[SLAVE_BUILD]/out"
+    },
+    "name": "compile extract_comments"
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/tmp/golib/bin/extract_comments",
+      "--dir",
+      "gm",
+      "--dest",
+      "gs://skia-doc/gm/comments.json"
+    ],
+    "cwd": "[SLAVE_BUILD]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "GOPATH": "[SLAVE_BUILD]/tmp/golib",
+      "SKIA_OUT": "[SLAVE_BUILD]/out"
+    },
+    "name": "run extract_comments"
+  },
   {
     "name": "$result",
     "recipe_result": null,

diff --git a/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit.json b/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit.json
index 3f72a1f0aea2430f8e8d4ad25fa3c074926aa6b7..9be48d4af1ea23fb1d06fc407c786b89365e220e 100644 (file)
--- a/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit.json
+++ b/infra/bots/recipes/swarm_housekeeper.expected/Housekeeper-PerCommit.json
@@ -67,6 +67,38 @@
     },
     "name": "generate and upload binary size data"
   },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/go/go/bin/go",
+      "get",
+      "go.skia.org/infra/comments/go/extract_comments"
+    ],
+    "cwd": "[SLAVE_BUILD]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "GOPATH": "[SLAVE_BUILD]/tmp/golib",
+      "SKIA_OUT": "[SLAVE_BUILD]/out"
+    },
+    "name": "compile extract_comments"
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/tmp/golib/bin/extract_comments",
+      "--dir",
+      "gm",
+      "--dest",
+      "gs://skia-doc/gm/comments.json"
+    ],
+    "cwd": "[SLAVE_BUILD]/skia",
+    "env": {
+      "BUILDTYPE": "Release",
+      "CHROME_HEADLESS": "1",
+      "GOPATH": "[SLAVE_BUILD]/tmp/golib",
+      "SKIA_OUT": "[SLAVE_BUILD]/out"
+    },
+    "name": "run extract_comments"
+  },
   {
     "name": "$result",
     "recipe_result": null,

diff --git a/infra/bots/recipes/swarm_housekeeper.py b/infra/bots/recipes/swarm_housekeeper.py
index ae31611b1e673454a705cc0950020c79c6e4332c..43e694c3843e8069ac5f3956795cd63365081f56 100644 (file)
--- a/infra/bots/recipes/swarm_housekeeper.py
+++ b/infra/bots/recipes/swarm_housekeeper.py
@@ -5,7 +5,6 @@
 
 # Recipe for the Skia PerCommit Housekeeper.
 
-
 DEPS = [
   'core',
   'recipe_engine/path',
@@ -68,6 +67,30 @@ def RunSteps(api):
     cwd=cwd,
     abort_on_failure=False)
 
+  env = {}
+  env['GOPATH'] = api.vars.tmp_dir.join('golib')
+  extractexe = env['GOPATH'].join('bin', 'extract_comments')
+  goexe = api.vars.slave_dir.join('go', 'go', 'bin', 'go')
+
+  # Compile extract_comments.
+  api.run(
+    api.step,
+    'compile extract_comments',
+    cmd=[goexe, 'get', 'go.skia.org/infra/comments/go/extract_comments'],
+    cwd=cwd,
+    env=env,
+    abort_on_failure=True)
+
+  # Run extract_comments on the gm directory.
+  api.run(
+    api.step,
+    'run extract_comments',
+    cmd=[extractexe, '--dir', 'gm', '--dest', 'gs://skia-doc/gm/comments.json'],
+    cwd=cwd,
+    env=env,
+    abort_on_failure=True)
+
+
 def GenTests(api):
   for mastername, slaves in TEST_BUILDERS.iteritems():
     for slavename, builders_by_slave in slaves.iteritems():

diff --git a/infra/bots/recipes/swarm_trigger.expected/Housekeeper-PerCommit.json b/infra/bots/recipes/swarm_trigger.expected/Housekeeper-PerCommit.json
index 57d3a1637f7bfb58e919d9e933015cc622fef071..9c9eb6f02e33ce9b9a896ac60be2d8d8ce45ba20 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Housekeeper-PerCommit.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Housekeeper-PerCommit.json
@@ -456,6 +456,16 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
     ]
   },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[SLAVE_BUILD]/skia/infra/bots/assets/go/VERSION",
+      "/path/to/tmp/"
+    ],
+    "name": "read go VERSION"
+  },
   {
     "cmd": [
       "python",
@@ -601,6 +611,8 @@
       "slavename:skiabot-linux-swarm-000",
       "--tag",
       "stepname:housekeeper_skia on Ubuntu",
+      "--cipd-package",
+      "go:skia/bots/go:version:0",
       "def456",
       "--",
       "--workdir",

diff --git a/infra/bots/recipes/swarm_trigger.py b/infra/bots/recipes/swarm_trigger.py
index 1fdd57174f057fe44827ebbcc1b6fab125116d15..c4f1a54937281a15a60908d648050859f4ca4cd9 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.py
+++ b/infra/bots/recipes/swarm_trigger.py
@@ -271,7 +271,9 @@ def housekeeper_swarm(api, builder_cfg, got_revision, infrabots_dir,
       infrabots_dir,
       idempotent=False,
       store_output=False,
-      extra_isolate_hashes=extra_isolate_hashes)
+      extra_isolate_hashes=extra_isolate_hashes,
+      cipd_packages=[cipd_pkg(api, infrabots_dir, 'go')],
+      )
   return api.swarming.collect_swarming_task(task)