Reboot NexusPlayers with adb and shut them down if install fails

BUG=skia:6045

Change-Id: Ibd31e4cc44b71633377b05ad33c1bdab9024f78c
Reviewed-on: https://skia-review.googlesource.com/8152
Reviewed-by: Ravi Mistry <rmistry@google.com>
Commit-Queue: Kevin Lubick <kjlubick@google.com>

diff --git a/infra/bots/recipe_modules/flavor/gn_android_flavor.py b/infra/bots/recipe_modules/flavor/gn_android_flavor.py
index bf1510b..d99a257 100644 (file)
--- a/infra/bots/recipe_modules/flavor/gn_android_flavor.py
+++ b/infra/bots/recipe_modules/flavor/gn_android_flavor.py
@@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+from recipe_engine import recipe_api
+
 import default_flavor
 import subprocess
 
@@ -43,6 +45,34 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils):
       kwargs['infra_step'] = True
     return self._run(title, 'adb', *cmd, **kwargs)
 
+  # Waits for an android device to be available
+  def _wait_for_device(self):
+    self.m.python.inline('wait for device', """
+      import subprocess
+      import sys
+      import time
+
+      times = 0
+
+      while times < 30:
+        print 'Waiting for the device to be connected and ready.'
+        try:
+          output = subprocess.check_output(['adb', 'shell',
+                                            'getprop', 'sys.boot_completed'])
+          if '1' in output:
+            print 'Connected'
+            sys.exit(0)
+        except subprocess.CalledProcessError:
+          # no device connected/authorized yet
+          pass
+        time.sleep(5)
+
+      print 'Timed out waiting for device'
+      sys.exit(1)
+      """,
+      infra_step=True)
+
+
   def compile(self, unused_target, **kwargs):
     compiler      = self.m.vars.builder_cfg.get('compiler')
     configuration = self.m.vars.builder_cfg.get('configuration')
@@ -89,9 +119,13 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils):
     self._run('ninja', ninja, '-C', self.out_dir)
 
   def install(self):
+    if 'NexusPlayer' == self.m.vars.builder_cfg.get('model'):
+      self._adb('rebooting device', 'reboot')
+      self._wait_for_device()
     self._adb('mkdir ' + self.device_dirs.resource_dir,
               'shell', 'mkdir', '-p', self.device_dirs.resource_dir)
 
+
   def cleanup_steps(self):
     if self._ever_ran_adb:
       self.m.python.inline('dump log', """
@@ -112,6 +146,13 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils):
       """,
       args=[self.m.vars.skia_out.join(self.m.vars.configuration)],
       infra_step=True)
+
+    for f in self.m.run.failed_steps:
+      if isinstance(f, recipe_api.InfraFailure):
+        self._adb('shut down device to quarantine bot', 'shell', 'reboot', '-p')
+        break
+
+    if self._ever_ran_adb:
       self._adb('kill adb server', 'kill-server')
 
   def step(self, name, cmd, env=None, **kwargs):

diff --git a/infra/bots/recipe_modules/perf/api.py b/infra/bots/recipe_modules/perf/api.py
index b6544a1..98e08fc 100644 (file)
--- a/infra/bots/recipe_modules/perf/api.py
+++ b/infra/bots/recipe_modules/perf/api.py
@@ -235,6 +235,9 @@ class PerfApi(recipe_api.RecipeApi):
     try:
       self.m.flavor.install_everything()
       perf_steps(self.m)
+    except recipe_api.InfraFailure as e:
+      self.m.run.add_failed_step(e)
+      raise
     finally:
       self.m.flavor.cleanup_steps()
     self.m.run.check_failure()

diff --git a/infra/bots/recipe_modules/perf/example.expected/Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-GN_Android.json b/infra/bots/recipe_modules/perf/example.expected/Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-GN_Android.json
index c4228de..5004178 100644 (file)
--- a/infra/bots/recipe_modules/perf/example.expected/Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-GN_Android.json
+++ b/infra/bots/recipe_modules/perf/example.expected/Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-GN_Android.json
@@ -2,6 +2,47 @@
   {
     "cmd": [
       "adb",
+      "reboot"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "rebooting device"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport subprocess\nimport sys\nimport time\n\ntimes = 0\n\nwhile times < 30:\n  print 'Waiting for the device to be connected and ready.'\n  try:\n    output = subprocess.check_output(['adb', 'shell',\n                                      'getprop', 'sys.boot_completed'])\n    if '1' in output:\n      print 'Connected'\n      sys.exit(0)\n  except subprocess.CalledProcessError:\n    # no device connected/authorized yet\n    pass\n  time.sleep(5)\n\nprint 'Timed out waiting for device'\nsys.exit(1)\n"
+    ],
+    "name": "wait for device",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import subprocess@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@import time@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@times = 0@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@while times < 30:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  print 'Waiting for the device to be connected and ready.'@@@",
+      "@@@STEP_LOG_LINE@python.inline@  try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    output = subprocess.check_output(['adb', 'shell',@@@",
+      "@@@STEP_LOG_LINE@python.inline@                                      'getprop', 'sys.boot_completed'])@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if '1' in output:@@@",
+      "@@@STEP_LOG_LINE@python.inline@      print 'Connected'@@@",
+      "@@@STEP_LOG_LINE@python.inline@      sys.exit(0)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  except subprocess.CalledProcessError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    # no device connected/authorized yet@@@",
+      "@@@STEP_LOG_LINE@python.inline@    pass@@@",
+      "@@@STEP_LOG_LINE@python.inline@  time.sleep(5)@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print 'Timed out waiting for device'@@@",
+      "@@@STEP_LOG_LINE@python.inline@sys.exit(1)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "adb",
       "shell",
       "mkdir",
       "-p",

diff --git a/infra/bots/recipe_modules/perf/example.expected/Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-GN_Android_Vulkan.json b/infra/bots/recipe_modules/perf/example.expected/Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-GN_Android_Vulkan.json
index d33c8fe..7194496 100644 (file)
--- a/infra/bots/recipe_modules/perf/example.expected/Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-GN_Android_Vulkan.json
+++ b/infra/bots/recipe_modules/perf/example.expected/Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-GN_Android_Vulkan.json
@@ -2,6 +2,47 @@
   {
     "cmd": [
       "adb",
+      "reboot"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "rebooting device"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport subprocess\nimport sys\nimport time\n\ntimes = 0\n\nwhile times < 30:\n  print 'Waiting for the device to be connected and ready.'\n  try:\n    output = subprocess.check_output(['adb', 'shell',\n                                      'getprop', 'sys.boot_completed'])\n    if '1' in output:\n      print 'Connected'\n      sys.exit(0)\n  except subprocess.CalledProcessError:\n    # no device connected/authorized yet\n    pass\n  time.sleep(5)\n\nprint 'Timed out waiting for device'\nsys.exit(1)\n"
+    ],
+    "name": "wait for device",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import subprocess@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@import time@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@times = 0@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@while times < 30:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  print 'Waiting for the device to be connected and ready.'@@@",
+      "@@@STEP_LOG_LINE@python.inline@  try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    output = subprocess.check_output(['adb', 'shell',@@@",
+      "@@@STEP_LOG_LINE@python.inline@                                      'getprop', 'sys.boot_completed'])@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if '1' in output:@@@",
+      "@@@STEP_LOG_LINE@python.inline@      print 'Connected'@@@",
+      "@@@STEP_LOG_LINE@python.inline@      sys.exit(0)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  except subprocess.CalledProcessError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    # no device connected/authorized yet@@@",
+      "@@@STEP_LOG_LINE@python.inline@    pass@@@",
+      "@@@STEP_LOG_LINE@python.inline@  time.sleep(5)@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print 'Timed out waiting for device'@@@",
+      "@@@STEP_LOG_LINE@python.inline@sys.exit(1)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "adb",
       "shell",
       "mkdir",
       "-p",

diff --git a/infra/bots/recipe_modules/perf/example.expected/failed_push.json b/infra/bots/recipe_modules/perf/example.expected/failed_push.json
new file mode 100644 (file)
index 0000000..b47152e
--- /dev/null
+++ b/infra/bots/recipe_modules/perf/example.expected/failed_push.json
@@ -0,0 +1,136 @@
+[
+  {
+    "cmd": [
+      "adb",
+      "reboot"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "rebooting device"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport subprocess\nimport sys\nimport time\n\ntimes = 0\n\nwhile times < 30:\n  print 'Waiting for the device to be connected and ready.'\n  try:\n    output = subprocess.check_output(['adb', 'shell',\n                                      'getprop', 'sys.boot_completed'])\n    if '1' in output:\n      print 'Connected'\n      sys.exit(0)\n  except subprocess.CalledProcessError:\n    # no device connected/authorized yet\n    pass\n  time.sleep(5)\n\nprint 'Timed out waiting for device'\nsys.exit(1)\n"
+    ],
+    "name": "wait for device",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import subprocess@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@import time@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@times = 0@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@while times < 30:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  print 'Waiting for the device to be connected and ready.'@@@",
+      "@@@STEP_LOG_LINE@python.inline@  try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    output = subprocess.check_output(['adb', 'shell',@@@",
+      "@@@STEP_LOG_LINE@python.inline@                                      'getprop', 'sys.boot_completed'])@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if '1' in output:@@@",
+      "@@@STEP_LOG_LINE@python.inline@      print 'Connected'@@@",
+      "@@@STEP_LOG_LINE@python.inline@      sys.exit(0)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  except subprocess.CalledProcessError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    # no device connected/authorized yet@@@",
+      "@@@STEP_LOG_LINE@python.inline@    pass@@@",
+      "@@@STEP_LOG_LINE@python.inline@  time.sleep(5)@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print 'Timed out waiting for device'@@@",
+      "@@@STEP_LOG_LINE@python.inline@sys.exit(1)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "adb",
+      "shell",
+      "mkdir",
+      "-p",
+      "/sdcard/revenge_of_the_skiabot/resources"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "mkdir /sdcard/revenge_of_the_skiabot/resources"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os\nimport subprocess\nimport sys\nhost   = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n  p = os.path.relpath(d, host)\n  if p != '.' and p.startswith('.'):\n    continue\n  for f in fs:\n    print os.path.join(p,f)\n    subprocess.check_call(['adb', 'push',\n                           os.path.realpath(os.path.join(host, p, f)),\n                           os.path.join(device, p, f)])\n",
+      "[START_DIR]/skia/resources",
+      "/sdcard/revenge_of_the_skiabot/resources"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "push [START_DIR]/skia/resources/* /sdcard/revenge_of_the_skiabot/resources",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import subprocess@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@host   = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@device = sys.argv[2]@@@",
+      "@@@STEP_LOG_LINE@python.inline@for d, _, fs in os.walk(host):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  p = os.path.relpath(d, host)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if p != '.' and p.startswith('.'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    continue@@@",
+      "@@@STEP_LOG_LINE@python.inline@  for f in fs:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print os.path.join(p,f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@    subprocess.check_call(['adb', 'push',@@@",
+      "@@@STEP_LOG_LINE@python.inline@                           os.path.realpath(os.path.join(host, p, f)),@@@",
+      "@@@STEP_LOG_LINE@python.inline@                           os.path.join(device, p, f)])@@@",
+      "@@@STEP_LOG_END@python.inline@@@",
+      "@@@STEP_EXCEPTION@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os\nimport subprocess\nimport sys\nout = sys.argv[1]\nlog = subprocess.check_output(['adb', 'logcat', '-d'])\nfor line in log.split('\\n'):\n  tokens = line.split()\n  if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc':\n    addr, path = tokens[-2:]\n    local = os.path.join(out, os.path.basename(path))\n    if os.path.exists(local):\n      sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr])\n      line = line.replace(addr, addr + ' ' + sym.strip())\n  print line\n",
+      "[START_DIR]/out/Debug"
+    ],
+    "name": "dump log",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import subprocess@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@out = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@log = subprocess.check_output(['adb', 'logcat', '-d'])@@@",
+      "@@@STEP_LOG_LINE@python.inline@for line in log.split('\\n'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  tokens = line.split()@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc':@@@",
+      "@@@STEP_LOG_LINE@python.inline@    addr, path = tokens[-2:]@@@",
+      "@@@STEP_LOG_LINE@python.inline@    local = os.path.join(out, os.path.basename(path))@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if os.path.exists(local):@@@",
+      "@@@STEP_LOG_LINE@python.inline@      sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr])@@@",
+      "@@@STEP_LOG_LINE@python.inline@      line = line.replace(addr, addr + ' ' + sym.strip())@@@",
+      "@@@STEP_LOG_LINE@python.inline@  print line@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "adb",
+      "shell",
+      "reboot",
+      "-p"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "shut down device to quarantine bot"
+  },
+  {
+    "cmd": [
+      "adb",
+      "kill-server"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "kill adb server"
+  },
+  {
+    "name": "$result",
+    "reason": "Infra Failure: Step('push [START_DIR]/skia/resources/* /sdcard/revenge_of_the_skiabot/resources') returned 1",
+    "recipe_result": null,
+    "status_code": 1
+  }
+]
\ No newline at end of file

diff --git a/infra/bots/recipe_modules/perf/example.py b/infra/bots/recipe_modules/perf/example.py
index 2bcf6be..259852b 100644 (file)
--- a/infra/bots/recipe_modules/perf/example.py
+++ b/infra/bots/recipe_modules/perf/example.py
@@ -158,3 +158,28 @@ def GenTests(api):
           stdout=api.raw_io.output('skia-bot-123')) +
       api.step_data('get swarming task id', stdout=api.raw_io.output('123456'))
   )
+
+  builder = 'Perf-Android-Clang-NexusPlayer-CPU-SSE4-x86-Debug-GN_Android'
+  yield (
+    api.test('failed_push') +
+    api.properties(buildername=builder,
+                   mastername='client.skia',
+                   slavename='skiabot-linux-swarm-000',
+                   buildnumber=6,
+                   revision='abc123',
+                   path_config='kitchen',
+                   swarm_out_dir='[SWARM_OUT_DIR]') +
+    api.path.exists(
+        api.path['start_dir'].join('skia'),
+        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
+                                     'skimage', 'VERSION'),
+        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
+                                     'skp', 'VERSION'),
+        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
+                                     'svg', 'VERSION'),
+        api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt')
+    ) +
+    api.step_data('push [START_DIR]/skia/resources/* '+
+                  '/sdcard/revenge_of_the_skiabot/resources', retcode=1)
+  )
+

diff --git a/infra/bots/recipe_modules/run/api.py b/infra/bots/recipe_modules/run/api.py
index 11b5863..dc9d3db 100644 (file)
--- a/infra/bots/recipe_modules/run/api.py
+++ b/infra/bots/recipe_modules/run/api.py
@@ -47,6 +47,13 @@ class SkiaStepApi(recipe_api.RecipeApi):
       raise self.m.step.StepFailure('Failed build steps: %s' %
                                     ', '.join([f.name for f in self._failed]))
 
+  def add_failed_step(self, step):
+    self._failed.append(step)
+
+  @property
+  def failed_steps(self):
+    return self._failed[:]
+
   def run_once(self, fn, *args, **kwargs):
     if not fn.__name__ in self._already_ran:
       self._already_ran[fn.__name__] = fn(*args, **kwargs)

diff --git a/infra/bots/recipe_modules/sktest/api.py b/infra/bots/recipe_modules/sktest/api.py
index df97a15..3585279 100644 (file)
--- a/infra/bots/recipe_modules/sktest/api.py
+++ b/infra/bots/recipe_modules/sktest/api.py
@@ -584,6 +584,9 @@ class TestApi(recipe_api.RecipeApi):
     try:
       self.m.flavor.install_everything()
       test_steps(self.m)
+    except recipe_api.InfraFailure as e:
+      self.m.run.add_failed_step(e)
+      raise
     finally:
       self.m.flavor.cleanup_steps()
     self.m.run.check_failure()

diff --git a/infra/bots/recipe_modules/sktest/example.expected/Test-Android-Clang-NexusPlayer-CPU-SSE4-x86-Release-GN_Android.json b/infra/bots/recipe_modules/sktest/example.expected/Test-Android-Clang-NexusPlayer-CPU-SSE4-x86-Release-GN_Android.json
index 2b9b16d..30d4f8f 100644 (file)
--- a/infra/bots/recipe_modules/sktest/example.expected/Test-Android-Clang-NexusPlayer-CPU-SSE4-x86-Release-GN_Android.json
+++ b/infra/bots/recipe_modules/sktest/example.expected/Test-Android-Clang-NexusPlayer-CPU-SSE4-x86-Release-GN_Android.json
@@ -2,6 +2,47 @@
   {
     "cmd": [
       "adb",
+      "reboot"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "rebooting device"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport subprocess\nimport sys\nimport time\n\ntimes = 0\n\nwhile times < 30:\n  print 'Waiting for the device to be connected and ready.'\n  try:\n    output = subprocess.check_output(['adb', 'shell',\n                                      'getprop', 'sys.boot_completed'])\n    if '1' in output:\n      print 'Connected'\n      sys.exit(0)\n  except subprocess.CalledProcessError:\n    # no device connected/authorized yet\n    pass\n  time.sleep(5)\n\nprint 'Timed out waiting for device'\nsys.exit(1)\n"
+    ],
+    "name": "wait for device",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import subprocess@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@import time@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@times = 0@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@while times < 30:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  print 'Waiting for the device to be connected and ready.'@@@",
+      "@@@STEP_LOG_LINE@python.inline@  try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    output = subprocess.check_output(['adb', 'shell',@@@",
+      "@@@STEP_LOG_LINE@python.inline@                                      'getprop', 'sys.boot_completed'])@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if '1' in output:@@@",
+      "@@@STEP_LOG_LINE@python.inline@      print 'Connected'@@@",
+      "@@@STEP_LOG_LINE@python.inline@      sys.exit(0)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  except subprocess.CalledProcessError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    # no device connected/authorized yet@@@",
+      "@@@STEP_LOG_LINE@python.inline@    pass@@@",
+      "@@@STEP_LOG_LINE@python.inline@  time.sleep(5)@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print 'Timed out waiting for device'@@@",
+      "@@@STEP_LOG_LINE@python.inline@sys.exit(1)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "adb",
       "shell",
       "mkdir",
       "-p",

diff --git a/infra/bots/recipe_modules/sktest/example.expected/Test-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-GN_Android_Vulkan.json b/infra/bots/recipe_modules/sktest/example.expected/Test-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-GN_Android_Vulkan.json
index cbb6f62..006e3b8 100644 (file)
--- a/infra/bots/recipe_modules/sktest/example.expected/Test-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-GN_Android_Vulkan.json
+++ b/infra/bots/recipe_modules/sktest/example.expected/Test-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-GN_Android_Vulkan.json
@@ -2,6 +2,47 @@
   {
     "cmd": [
       "adb",
+      "reboot"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "rebooting device"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport subprocess\nimport sys\nimport time\n\ntimes = 0\n\nwhile times < 30:\n  print 'Waiting for the device to be connected and ready.'\n  try:\n    output = subprocess.check_output(['adb', 'shell',\n                                      'getprop', 'sys.boot_completed'])\n    if '1' in output:\n      print 'Connected'\n      sys.exit(0)\n  except subprocess.CalledProcessError:\n    # no device connected/authorized yet\n    pass\n  time.sleep(5)\n\nprint 'Timed out waiting for device'\nsys.exit(1)\n"
+    ],
+    "name": "wait for device",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import subprocess@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@import time@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@times = 0@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@while times < 30:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  print 'Waiting for the device to be connected and ready.'@@@",
+      "@@@STEP_LOG_LINE@python.inline@  try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    output = subprocess.check_output(['adb', 'shell',@@@",
+      "@@@STEP_LOG_LINE@python.inline@                                      'getprop', 'sys.boot_completed'])@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if '1' in output:@@@",
+      "@@@STEP_LOG_LINE@python.inline@      print 'Connected'@@@",
+      "@@@STEP_LOG_LINE@python.inline@      sys.exit(0)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  except subprocess.CalledProcessError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    # no device connected/authorized yet@@@",
+      "@@@STEP_LOG_LINE@python.inline@    pass@@@",
+      "@@@STEP_LOG_LINE@python.inline@  time.sleep(5)@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print 'Timed out waiting for device'@@@",
+      "@@@STEP_LOG_LINE@python.inline@sys.exit(1)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "adb",
       "shell",
       "mkdir",
       "-p",

diff --git a/infra/bots/recipe_modules/sktest/example.expected/failed_push.json b/infra/bots/recipe_modules/sktest/example.expected/failed_push.json
new file mode 100644 (file)
index 0000000..b47152e
--- /dev/null
+++ b/infra/bots/recipe_modules/sktest/example.expected/failed_push.json
@@ -0,0 +1,136 @@
+[
+  {
+    "cmd": [
+      "adb",
+      "reboot"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "rebooting device"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport subprocess\nimport sys\nimport time\n\ntimes = 0\n\nwhile times < 30:\n  print 'Waiting for the device to be connected and ready.'\n  try:\n    output = subprocess.check_output(['adb', 'shell',\n                                      'getprop', 'sys.boot_completed'])\n    if '1' in output:\n      print 'Connected'\n      sys.exit(0)\n  except subprocess.CalledProcessError:\n    # no device connected/authorized yet\n    pass\n  time.sleep(5)\n\nprint 'Timed out waiting for device'\nsys.exit(1)\n"
+    ],
+    "name": "wait for device",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import subprocess@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@import time@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@times = 0@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@while times < 30:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  print 'Waiting for the device to be connected and ready.'@@@",
+      "@@@STEP_LOG_LINE@python.inline@  try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    output = subprocess.check_output(['adb', 'shell',@@@",
+      "@@@STEP_LOG_LINE@python.inline@                                      'getprop', 'sys.boot_completed'])@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if '1' in output:@@@",
+      "@@@STEP_LOG_LINE@python.inline@      print 'Connected'@@@",
+      "@@@STEP_LOG_LINE@python.inline@      sys.exit(0)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  except subprocess.CalledProcessError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    # no device connected/authorized yet@@@",
+      "@@@STEP_LOG_LINE@python.inline@    pass@@@",
+      "@@@STEP_LOG_LINE@python.inline@  time.sleep(5)@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print 'Timed out waiting for device'@@@",
+      "@@@STEP_LOG_LINE@python.inline@sys.exit(1)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "adb",
+      "shell",
+      "mkdir",
+      "-p",
+      "/sdcard/revenge_of_the_skiabot/resources"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "mkdir /sdcard/revenge_of_the_skiabot/resources"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os\nimport subprocess\nimport sys\nhost   = sys.argv[1]\ndevice = sys.argv[2]\nfor d, _, fs in os.walk(host):\n  p = os.path.relpath(d, host)\n  if p != '.' and p.startswith('.'):\n    continue\n  for f in fs:\n    print os.path.join(p,f)\n    subprocess.check_call(['adb', 'push',\n                           os.path.realpath(os.path.join(host, p, f)),\n                           os.path.join(device, p, f)])\n",
+      "[START_DIR]/skia/resources",
+      "/sdcard/revenge_of_the_skiabot/resources"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "push [START_DIR]/skia/resources/* /sdcard/revenge_of_the_skiabot/resources",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import subprocess@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@host   = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@device = sys.argv[2]@@@",
+      "@@@STEP_LOG_LINE@python.inline@for d, _, fs in os.walk(host):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  p = os.path.relpath(d, host)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if p != '.' and p.startswith('.'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    continue@@@",
+      "@@@STEP_LOG_LINE@python.inline@  for f in fs:@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print os.path.join(p,f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@    subprocess.check_call(['adb', 'push',@@@",
+      "@@@STEP_LOG_LINE@python.inline@                           os.path.realpath(os.path.join(host, p, f)),@@@",
+      "@@@STEP_LOG_LINE@python.inline@                           os.path.join(device, p, f)])@@@",
+      "@@@STEP_LOG_END@python.inline@@@",
+      "@@@STEP_EXCEPTION@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os\nimport subprocess\nimport sys\nout = sys.argv[1]\nlog = subprocess.check_output(['adb', 'logcat', '-d'])\nfor line in log.split('\\n'):\n  tokens = line.split()\n  if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc':\n    addr, path = tokens[-2:]\n    local = os.path.join(out, os.path.basename(path))\n    if os.path.exists(local):\n      sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr])\n      line = line.replace(addr, addr + ' ' + sym.strip())\n  print line\n",
+      "[START_DIR]/out/Debug"
+    ],
+    "name": "dump log",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import subprocess@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@out = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@log = subprocess.check_output(['adb', 'logcat', '-d'])@@@",
+      "@@@STEP_LOG_LINE@python.inline@for line in log.split('\\n'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  tokens = line.split()@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc':@@@",
+      "@@@STEP_LOG_LINE@python.inline@    addr, path = tokens[-2:]@@@",
+      "@@@STEP_LOG_LINE@python.inline@    local = os.path.join(out, os.path.basename(path))@@@",
+      "@@@STEP_LOG_LINE@python.inline@    if os.path.exists(local):@@@",
+      "@@@STEP_LOG_LINE@python.inline@      sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr])@@@",
+      "@@@STEP_LOG_LINE@python.inline@      line = line.replace(addr, addr + ' ' + sym.strip())@@@",
+      "@@@STEP_LOG_LINE@python.inline@  print line@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "adb",
+      "shell",
+      "reboot",
+      "-p"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "shut down device to quarantine bot"
+  },
+  {
+    "cmd": [
+      "adb",
+      "kill-server"
+    ],
+    "cwd": "[START_DIR]/skia",
+    "name": "kill adb server"
+  },
+  {
+    "name": "$result",
+    "reason": "Infra Failure: Step('push [START_DIR]/skia/resources/* /sdcard/revenge_of_the_skiabot/resources') returned 1",
+    "recipe_result": null,
+    "status_code": 1
+  }
+]
\ No newline at end of file

diff --git a/infra/bots/recipe_modules/sktest/example.py b/infra/bots/recipe_modules/sktest/example.py
index 1172193..2401c99 100644 (file)
--- a/infra/bots/recipe_modules/sktest/example.py
+++ b/infra/bots/recipe_modules/sktest/example.py
@@ -203,3 +203,27 @@ def GenTests(api):
           stdout=api.raw_io.output('skia-bot-123')) +
       api.step_data('get swarming task id', stdout=api.raw_io.output('123456'))
   )
+
+  builder = 'Test-Android-Clang-NexusPlayer-CPU-SSE4-x86-Debug-GN_Android'
+  yield (
+    api.test('failed_push') +
+    api.properties(buildername=builder,
+                   mastername='client.skia',
+                   slavename='skiabot-linux-swarm-000',
+                   buildnumber=6,
+                   revision='abc123',
+                   path_config='kitchen',
+                   swarm_out_dir='[SWARM_OUT_DIR]') +
+    api.path.exists(
+        api.path['start_dir'].join('skia'),
+        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
+                                     'skimage', 'VERSION'),
+        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
+                                     'skp', 'VERSION'),
+        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
+                                     'svg', 'VERSION'),
+        api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt')
+    ) +
+    api.step_data('push [START_DIR]/skia/resources/* '+
+                  '/sdcard/revenge_of_the_skiabot/resources', retcode=1)
+  )