projects / platform / upstream / libSkiaSharp.git / commitdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | patch | inline | side by side (parent: 7bfc08b)
Add swarming task for upload_dm_results
authorborenet <borenet@chromium.org>
Tue, 27 Sep 2016 13:52:26 +0000 (06:52 -0700)
committerCommit bot <commit-bot@chromium.org>
Tue, 27 Sep 2016 13:52:26 +0000 (06:52 -0700)
BUG=skia:5719
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2360203004

Review-Url: https://codereview.chromium.org/2360203004

19 files changed:
infra/bots/recipes/swarm_trigger.expected/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android.json patch | blob | history
infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json patch | blob | history
infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json patch | blob | history
infra/bots/recipes/swarm_trigger.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json patch | blob | history
infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json patch | blob | history
infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN.json patch | blob | history
infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json patch | blob | history
infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared.json patch | blob | history
infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json patch | blob | history
infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release.json patch | blob | history
infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release.json patch | blob | history
infra/bots/recipes/swarm_trigger.expected/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release.json patch | blob | history
infra/bots/recipes/swarm_trigger.expected/recipe_with_gerrit_patch.json patch | blob | history
infra/bots/recipes/swarm_trigger.py patch | blob | history
infra/bots/recipes/upload_dm_results.expected/normal_bot.json [new file with mode: 0644] patch | blob
infra/bots/recipes/upload_dm_results.expected/trybot.json [new file with mode: 0644] patch | blob
infra/bots/recipes/upload_dm_results.py [new file with mode: 0644] patch | blob
infra/bots/recipes/upload_nano_results.py patch | blob | history
infra/bots/upload_dm_results.isolate [new file with mode: 0644] patch | blob

diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android.json b/infra/bots/recipes/swarm_trigger.expected/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android.json
index e2daf6cb5e56aaa07aeba34cc9b84d1cdb447172..1540c7ecff595a6787913f0310c580b2f3f6eb13 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android.json
@@ -703,28 +703,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -738,9 +716,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/Android/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5\", \"tasks\": {\"test_skia/Android/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on Android",
     "~followup_annotations": [
@@ -791,21 +767,78 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/dm"
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir",
+      "511"
     ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree dm",
+    "name": "makedirs swarming tmp dir (3)",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "{\n    \"args\": [\n        \"--isolate\", \n        \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n        \"--isolated\", \n        \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n        \"--config-variable\", \n        \"OS\", \n        \"Linux\", \n        \"--blacklist\", \n        \".git\", \n        \"--blacklist\", \n        \"out\", \n        \"--blacklist\", \n        \"*.pyc\", \n        \"--blacklist\", \n        \".recipe_deps\", \n        \"--extra-variable\", \n        \"WORKDIR\", \n        \"[SLAVE_BUILD]\"\n    ], \n    \"dir\": \"[SLAVE_BUILD]\", \n    \"version\": 1\n}",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "Write upload_dm_results_skia.isolated.gen.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::isolate]/resources/isolate.py",
+      "[SLAVE_BUILD]/swarming.client",
+      "batcharchive",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--verbose",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "isolate tests (3)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n  isolated = json.load(f)\nif not isolated.get('includes'):\n  isolated['includes'] = []\nfor h in sys.argv[2:]:\n  isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n  json.dump(isolated, f, sort_keys=True)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated",
+      "abc123"
+    ],
+    "name": "add_isolated_input (2)",
+    "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
+      "@@@STEP_LOG_LINE@python.inline@import json@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated = json.load(f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'] = []@@@",
+      "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'].append(h)@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  json.dump(isolated, f, sort_keys=True)@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
@@ -813,31 +846,161 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm",
-      "[SLAVE_BUILD]/dm",
-      "0"
+      "[SLAVE_BUILD]/swarming.client/isolateserver.py",
+      "archive",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated"
     ],
-    "name": "dm_dir"
+    "name": "upload new .isolated file for upload_dm_results_skia",
+    "stdout": "/path/to/tmp/"
   },
   {
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py",
-      "[SLAVE_BUILD]/dm",
-      "abc123",
-      "Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android",
-      "5",
-      "",
-      "[SLAVE_BUILD]/skia/common/py/utils"
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "trigger",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--priority",
+      "90",
+      "--shards",
+      "1",
+      "--task-name",
+      "upload_dm_results_skia/Linux/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--expiration",
+      "72000",
+      "--io-timeout",
+      "2400",
+      "--hard-timeout",
+      "14400",
+      "--dimension",
+      "cpu",
+      "x86-64-avx2",
+      "--dimension",
+      "gpu",
+      "none",
+      "--dimension",
+      "os",
+      "Linux",
+      "--dimension",
+      "pool",
+      "Skia",
+      "--tag",
+      "allow_milo:1",
+      "--tag",
+      "buildername:Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android",
+      "--tag",
+      "buildnumber:5",
+      "--tag",
+      "data:def456",
+      "--tag",
+      "master:client.skia",
+      "--tag",
+      "name:upload_dm_results_skia",
+      "--tag",
+      "os:Linux",
+      "--tag",
+      "revision:abc123",
+      "--tag",
+      "slavename:skiabot-linux-swarm-000",
+      "--tag",
+      "stepname:upload_dm_results_skia on Linux",
+      "--idempotent",
+      "def456",
+      "--",
+      "--workdir",
+      "../../..",
+      "upload_dm_results",
+      "buildername=Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android",
+      "mastername=client.skia",
+      "buildnumber=5",
+      "slavename=skiabot-linux-swarm-000",
+      "reason=Triggered by Skia swarm_trigger Recipe",
+      "swarm_out_dir=${ISOLATED_OUTDIR}",
+      "revision=abc123"
     ],
-    "cwd": "[SLAVE_BUILD]/skia",
-    "env": {
-      "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto",
-      "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto"
-    },
-    "name": "Upload DM Results"
+    "name": "[trigger] upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"tasks\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"upload_dm_results_skia/Linux/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 0, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "collect",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--decorate",
+      "--print-status-updates",
+      "--json",
+      "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-GN_Android/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
+      "--task-summary-json",
+      "/path/to/tmp/json"
+    ],
+    "name": "upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@swarming pending 71s@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"shards\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"abandoned_ts\": null, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"bot_id\": \"vm30\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"created_ts\": \"2014-09-25T01:41:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"durations\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        5.7, @@@",
+      "@@@STEP_LOG_LINE@json.output@        31.5@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"exit_codes\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        0, @@@",
+      "@@@STEP_LOG_LINE@json.output@        0@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"id\": \"148aa78d7aa0000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"internal_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"isolated_out\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolated\": \"abc123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"namespace\": \"default-gzip\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"view_url\": \"blah\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"modified_ts\": \"2014-09-25 01:42:00\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"outputs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Heart beat succeeded on win32.\\n\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Foo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"started_ts\": \"2014-09-25T01:42:11.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"state\": 112, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"try_number\": 1, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"user\": \"unknown\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0 isolated out@blah@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
+    ]
   },
   {
     "name": "$result",
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json b/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json
index 405be3f18eb5ce2c7ae9ec5062bb6ff68c484523..a3c0f2d6c0478eacf13bad78ee2585978426591a 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan.json
@@ -691,28 +691,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -726,9 +704,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/Android/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5\", \"tasks\": {\"test_skia/Android/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on Android",
     "~followup_annotations": [
@@ -779,21 +755,78 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/dm"
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir",
+      "511"
     ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree dm",
+    "name": "makedirs swarming tmp dir (3)",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "{\n    \"args\": [\n        \"--isolate\", \n        \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n        \"--isolated\", \n        \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n        \"--config-variable\", \n        \"OS\", \n        \"Linux\", \n        \"--blacklist\", \n        \".git\", \n        \"--blacklist\", \n        \"out\", \n        \"--blacklist\", \n        \"*.pyc\", \n        \"--blacklist\", \n        \".recipe_deps\", \n        \"--extra-variable\", \n        \"WORKDIR\", \n        \"[SLAVE_BUILD]\"\n    ], \n    \"dir\": \"[SLAVE_BUILD]\", \n    \"version\": 1\n}",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "Write upload_dm_results_skia.isolated.gen.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::isolate]/resources/isolate.py",
+      "[SLAVE_BUILD]/swarming.client",
+      "batcharchive",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--verbose",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "isolate tests (3)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n  isolated = json.load(f)\nif not isolated.get('includes'):\n  isolated['includes'] = []\nfor h in sys.argv[2:]:\n  isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n  json.dump(isolated, f, sort_keys=True)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated",
+      "abc123"
+    ],
+    "name": "add_isolated_input (2)",
+    "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
+      "@@@STEP_LOG_LINE@python.inline@import json@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated = json.load(f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'] = []@@@",
+      "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'].append(h)@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  json.dump(isolated, f, sort_keys=True)@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
@@ -801,31 +834,161 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm",
-      "[SLAVE_BUILD]/dm",
-      "0"
+      "[SLAVE_BUILD]/swarming.client/isolateserver.py",
+      "archive",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated"
     ],
-    "name": "dm_dir"
+    "name": "upload new .isolated file for upload_dm_results_skia",
+    "stdout": "/path/to/tmp/"
   },
   {
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py",
-      "[SLAVE_BUILD]/dm",
-      "abc123",
-      "Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan",
-      "5",
-      "",
-      "[SLAVE_BUILD]/skia/common/py/utils"
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "trigger",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--priority",
+      "90",
+      "--shards",
+      "1",
+      "--task-name",
+      "upload_dm_results_skia/Linux/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--expiration",
+      "72000",
+      "--io-timeout",
+      "2400",
+      "--hard-timeout",
+      "14400",
+      "--dimension",
+      "cpu",
+      "x86-64-avx2",
+      "--dimension",
+      "gpu",
+      "none",
+      "--dimension",
+      "os",
+      "Linux",
+      "--dimension",
+      "pool",
+      "Skia",
+      "--tag",
+      "allow_milo:1",
+      "--tag",
+      "buildername:Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan",
+      "--tag",
+      "buildnumber:5",
+      "--tag",
+      "data:def456",
+      "--tag",
+      "master:client.skia",
+      "--tag",
+      "name:upload_dm_results_skia",
+      "--tag",
+      "os:Linux",
+      "--tag",
+      "revision:abc123",
+      "--tag",
+      "slavename:skiabot-linux-swarm-000",
+      "--tag",
+      "stepname:upload_dm_results_skia on Linux",
+      "--idempotent",
+      "def456",
+      "--",
+      "--workdir",
+      "../../..",
+      "upload_dm_results",
+      "buildername=Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan",
+      "mastername=client.skia",
+      "buildnumber=5",
+      "slavename=skiabot-linux-swarm-000",
+      "reason=Triggered by Skia swarm_trigger Recipe",
+      "swarm_out_dir=${ISOLATED_OUTDIR}",
+      "revision=abc123"
     ],
-    "cwd": "[SLAVE_BUILD]/skia",
-    "env": {
-      "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto",
-      "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto"
-    },
-    "name": "Upload DM Results"
+    "name": "[trigger] upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"tasks\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"upload_dm_results_skia/Linux/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 0, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "collect",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--decorate",
+      "--print-status-updates",
+      "--json",
+      "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Android-GCC-NVIDIA_Shield-GPU-TegraX1-Arm64-Debug-Vulkan/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
+      "--task-summary-json",
+      "/path/to/tmp/json"
+    ],
+    "name": "upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@swarming pending 71s@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"shards\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"abandoned_ts\": null, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"bot_id\": \"vm30\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"created_ts\": \"2014-09-25T01:41:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"durations\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        5.7, @@@",
+      "@@@STEP_LOG_LINE@json.output@        31.5@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"exit_codes\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        0, @@@",
+      "@@@STEP_LOG_LINE@json.output@        0@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"id\": \"148aa78d7aa0000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"internal_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"isolated_out\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolated\": \"abc123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"namespace\": \"default-gzip\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"view_url\": \"blah\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"modified_ts\": \"2014-09-25 01:42:00\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"outputs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Heart beat succeeded on win32.\\n\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Foo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"started_ts\": \"2014-09-25T01:42:11.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"state\": 112, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"try_number\": 1, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"user\": \"unknown\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0 isolated out@blah@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
+    ]
   },
   {
     "name": "$result",
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json b/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json
index 780f02a1e9842e2c8d67154a0df0ee855c2f534e..1c130a455014f184af3c9019081ee95c21c8a7a9 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release.json
@@ -691,28 +691,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -726,9 +704,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/Android/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5\", \"tasks\": {\"test_skia/Android/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on Android",
     "~followup_annotations": [
@@ -779,21 +755,78 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/dm"
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir",
+      "511"
     ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree dm",
+    "name": "makedirs swarming tmp dir (3)",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "{\n    \"args\": [\n        \"--isolate\", \n        \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n        \"--isolated\", \n        \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n        \"--config-variable\", \n        \"OS\", \n        \"Linux\", \n        \"--blacklist\", \n        \".git\", \n        \"--blacklist\", \n        \"out\", \n        \"--blacklist\", \n        \"*.pyc\", \n        \"--blacklist\", \n        \".recipe_deps\", \n        \"--extra-variable\", \n        \"WORKDIR\", \n        \"[SLAVE_BUILD]\"\n    ], \n    \"dir\": \"[SLAVE_BUILD]\", \n    \"version\": 1\n}",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "Write upload_dm_results_skia.isolated.gen.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::isolate]/resources/isolate.py",
+      "[SLAVE_BUILD]/swarming.client",
+      "batcharchive",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--verbose",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "isolate tests (3)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n  isolated = json.load(f)\nif not isolated.get('includes'):\n  isolated['includes'] = []\nfor h in sys.argv[2:]:\n  isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n  json.dump(isolated, f, sort_keys=True)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated",
+      "abc123"
+    ],
+    "name": "add_isolated_input (2)",
+    "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
+      "@@@STEP_LOG_LINE@python.inline@import json@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated = json.load(f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'] = []@@@",
+      "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'].append(h)@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  json.dump(isolated, f, sort_keys=True)@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
@@ -801,31 +834,161 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm",
-      "[SLAVE_BUILD]/dm",
-      "0"
+      "[SLAVE_BUILD]/swarming.client/isolateserver.py",
+      "archive",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated"
     ],
-    "name": "dm_dir"
+    "name": "upload new .isolated file for upload_dm_results_skia",
+    "stdout": "/path/to/tmp/"
   },
   {
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py",
-      "[SLAVE_BUILD]/dm",
-      "abc123",
-      "Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release",
-      "5",
-      "",
-      "[SLAVE_BUILD]/skia/common/py/utils"
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "trigger",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--priority",
+      "90",
+      "--shards",
+      "1",
+      "--task-name",
+      "upload_dm_results_skia/Linux/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--expiration",
+      "72000",
+      "--io-timeout",
+      "2400",
+      "--hard-timeout",
+      "14400",
+      "--dimension",
+      "cpu",
+      "x86-64-avx2",
+      "--dimension",
+      "gpu",
+      "none",
+      "--dimension",
+      "os",
+      "Linux",
+      "--dimension",
+      "pool",
+      "Skia",
+      "--tag",
+      "allow_milo:1",
+      "--tag",
+      "buildername:Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release",
+      "--tag",
+      "buildnumber:5",
+      "--tag",
+      "data:def456",
+      "--tag",
+      "master:client.skia",
+      "--tag",
+      "name:upload_dm_results_skia",
+      "--tag",
+      "os:Linux",
+      "--tag",
+      "revision:abc123",
+      "--tag",
+      "slavename:skiabot-linux-swarm-000",
+      "--tag",
+      "stepname:upload_dm_results_skia on Linux",
+      "--idempotent",
+      "def456",
+      "--",
+      "--workdir",
+      "../../..",
+      "upload_dm_results",
+      "buildername=Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release",
+      "mastername=client.skia",
+      "buildnumber=5",
+      "slavename=skiabot-linux-swarm-000",
+      "reason=Triggered by Skia swarm_trigger Recipe",
+      "swarm_out_dir=${ISOLATED_OUTDIR}",
+      "revision=abc123"
     ],
-    "cwd": "[SLAVE_BUILD]/skia",
-    "env": {
-      "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto",
-      "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto"
-    },
-    "name": "Upload DM Results"
+    "name": "[trigger] upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"tasks\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"upload_dm_results_skia/Linux/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 0, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "collect",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--decorate",
+      "--print-status-updates",
+      "--json",
+      "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Android-GCC-Nexus7v2-GPU-Tegra3-Arm7-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
+      "--task-summary-json",
+      "/path/to/tmp/json"
+    ],
+    "name": "upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@swarming pending 71s@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"shards\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"abandoned_ts\": null, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"bot_id\": \"vm30\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"created_ts\": \"2014-09-25T01:41:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"durations\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        5.7, @@@",
+      "@@@STEP_LOG_LINE@json.output@        31.5@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"exit_codes\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        0, @@@",
+      "@@@STEP_LOG_LINE@json.output@        0@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"id\": \"148aa78d7aa0000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"internal_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"isolated_out\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolated\": \"abc123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"namespace\": \"default-gzip\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"view_url\": \"blah\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"modified_ts\": \"2014-09-25 01:42:00\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"outputs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Heart beat succeeded on win32.\\n\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Foo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"started_ts\": \"2014-09-25T01:42:11.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"state\": 112, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"try_number\": 1, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"user\": \"unknown\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0 isolated out@blah@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
+    ]
   },
   {
     "name": "$result",
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json b/infra/bots/recipes/swarm_trigger.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json
index f59ad2c300396ed866fdebf932aa86bbb6f77d20..eb7ffc54eeee879fc7deb79de891a67b27399d53 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release.json
@@ -670,28 +670,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -705,9 +683,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/Mac/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5\", \"tasks\": {\"test_skia/Mac/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on Mac",
     "~followup_annotations": [
@@ -758,21 +734,78 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/dm"
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir",
+      "511"
     ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree dm",
+    "name": "makedirs swarming tmp dir (3)",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "{\n    \"args\": [\n        \"--isolate\", \n        \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n        \"--isolated\", \n        \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n        \"--config-variable\", \n        \"OS\", \n        \"Linux\", \n        \"--blacklist\", \n        \".git\", \n        \"--blacklist\", \n        \"out\", \n        \"--blacklist\", \n        \"*.pyc\", \n        \"--blacklist\", \n        \".recipe_deps\", \n        \"--extra-variable\", \n        \"WORKDIR\", \n        \"[SLAVE_BUILD]\"\n    ], \n    \"dir\": \"[SLAVE_BUILD]\", \n    \"version\": 1\n}",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "Write upload_dm_results_skia.isolated.gen.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::isolate]/resources/isolate.py",
+      "[SLAVE_BUILD]/swarming.client",
+      "batcharchive",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--verbose",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "isolate tests (3)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n  isolated = json.load(f)\nif not isolated.get('includes'):\n  isolated['includes'] = []\nfor h in sys.argv[2:]:\n  isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n  json.dump(isolated, f, sort_keys=True)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated",
+      "abc123"
+    ],
+    "name": "add_isolated_input (2)",
+    "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
+      "@@@STEP_LOG_LINE@python.inline@import json@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated = json.load(f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'] = []@@@",
+      "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'].append(h)@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  json.dump(isolated, f, sort_keys=True)@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
@@ -780,31 +813,161 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm",
-      "[SLAVE_BUILD]/dm",
-      "0"
+      "[SLAVE_BUILD]/swarming.client/isolateserver.py",
+      "archive",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated"
     ],
-    "name": "dm_dir"
+    "name": "upload new .isolated file for upload_dm_results_skia",
+    "stdout": "/path/to/tmp/"
   },
   {
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py",
-      "[SLAVE_BUILD]/dm",
-      "abc123",
-      "Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release",
-      "5",
-      "",
-      "[SLAVE_BUILD]/skia/common/py/utils"
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "trigger",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--priority",
+      "90",
+      "--shards",
+      "1",
+      "--task-name",
+      "upload_dm_results_skia/Linux/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--expiration",
+      "72000",
+      "--io-timeout",
+      "2400",
+      "--hard-timeout",
+      "14400",
+      "--dimension",
+      "cpu",
+      "x86-64-avx2",
+      "--dimension",
+      "gpu",
+      "none",
+      "--dimension",
+      "os",
+      "Linux",
+      "--dimension",
+      "pool",
+      "Skia",
+      "--tag",
+      "allow_milo:1",
+      "--tag",
+      "buildername:Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release",
+      "--tag",
+      "buildnumber:5",
+      "--tag",
+      "data:def456",
+      "--tag",
+      "master:client.skia",
+      "--tag",
+      "name:upload_dm_results_skia",
+      "--tag",
+      "os:Linux",
+      "--tag",
+      "revision:abc123",
+      "--tag",
+      "slavename:skiabot-linux-swarm-000",
+      "--tag",
+      "stepname:upload_dm_results_skia on Linux",
+      "--idempotent",
+      "def456",
+      "--",
+      "--workdir",
+      "../../..",
+      "upload_dm_results",
+      "buildername=Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release",
+      "mastername=client.skia",
+      "buildnumber=5",
+      "slavename=skiabot-linux-swarm-000",
+      "reason=Triggered by Skia swarm_trigger Recipe",
+      "swarm_out_dir=${ISOLATED_OUTDIR}",
+      "revision=abc123"
     ],
-    "cwd": "[SLAVE_BUILD]/skia",
-    "env": {
-      "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto",
-      "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto"
-    },
-    "name": "Upload DM Results"
+    "name": "[trigger] upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"tasks\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"upload_dm_results_skia/Linux/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 0, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "collect",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--decorate",
+      "--print-status-updates",
+      "--json",
+      "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
+      "--task-summary-json",
+      "/path/to/tmp/json"
+    ],
+    "name": "upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@swarming pending 71s@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"shards\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"abandoned_ts\": null, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"bot_id\": \"vm30\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"created_ts\": \"2014-09-25T01:41:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"durations\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        5.7, @@@",
+      "@@@STEP_LOG_LINE@json.output@        31.5@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"exit_codes\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        0, @@@",
+      "@@@STEP_LOG_LINE@json.output@        0@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"id\": \"148aa78d7aa0000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"internal_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"isolated_out\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolated\": \"abc123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"namespace\": \"default-gzip\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"view_url\": \"blah\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"modified_ts\": \"2014-09-25 01:42:00\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"outputs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Heart beat succeeded on win32.\\n\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Foo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"started_ts\": \"2014-09-25T01:42:11.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"state\": 112, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"try_number\": 1, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"user\": \"unknown\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0 isolated out@blah@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
+    ]
   },
   {
     "name": "$result",
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json
index 78b61c5690cfae20199e82c8b805f89d4d28c799..0c02a22299f187f4d6dd1fd926622615aba2ca86 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json
@@ -439,28 +439,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -474,9 +452,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/Ubuntu/[dummy has/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot/5\", \"tasks\": {\"test_skia/Ubuntu/[dummy has/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on Ubuntu",
     "~followup_annotations": [
@@ -523,131 +499,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
     ]
   },
-  {
-    "cmd": [
-      "git",
-      "show",
-      "HEAD",
-      "--format=%at",
-      "-s"
-    ],
-    "cwd": "[SLAVE_BUILD]/skia",
-    "name": "git show",
-    "stdout": "/path/to/tmp/"
-  },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "RECIPE_MODULE[build::gsutil]/resources/gsutil_wrapper.py",
-      "--",
-      "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py",
-      "----",
-      "cp",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/abc123.cov",
-      "gs://skia-infra/coverage-raw-v1/trybot/2012/05/14/12/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot/5/500/abc123.cov"
-    ],
-    "env": {
-      "AWS_CREDENTIAL_FILE": null,
-      "BOTO_CONFIG": null
-    },
-    "name": "gsutil upload raw coverage data",
-    "~followup_annotations": [
-      "@@@STEP_LINK@gsutil.upload@https://storage.cloud.google.com/skia-infra/coverage-raw-v1/trybot/2012/05/14/12/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot/5/500/abc123.cov@@@"
-    ]
-  },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/nanobench_abc123.json",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/nanobench_abc123_1408633190.json"
-    ],
-    "name": "nanobench JSON"
-  },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/nanobench_abc123.json"
-    ],
-    "name": "old nanobench JSON",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os@@@",
-      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "RECIPE_MODULE[skia::core]/resources/upload_bench_results.py",
-      "Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot",
-      "5",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0",
-      "abc123",
-      "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py",
-      "500"
-    ],
-    "cwd": "[SLAVE_BUILD]/skia",
-    "env": {
-      "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto",
-      "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto"
-    },
-    "name": "upload nanobench coverage results"
-  },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/coverage_by_line_abc123.json",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/coverage_by_line_abc123_1408633190.json"
-    ],
-    "name": "Line-by-line coverage JSON"
-  },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/coverage_by_line_abc123.json"
-    ],
-    "name": "old line-by-line coverage JSON",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os@@@",
-      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "RECIPE_MODULE[build::gsutil]/resources/gsutil_wrapper.py",
-      "--",
-      "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py",
-      "----",
-      "cp",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/coverage_by_line_abc123_1408633190.json",
-      "gs://skia-infra/coverage-json-v1/trybot/2012/05/14/12/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot/5/500/coverage_by_line_abc123_1408633190.json"
-    ],
-    "env": {
-      "AWS_CREDENTIAL_FILE": null,
-      "BOTO_CONFIG": null
-    },
-    "name": "gsutil upload line-by-line coverage data",
-    "~followup_annotations": [
-      "@@@STEP_LINK@gsutil.upload@https://storage.cloud.google.com/skia-infra/coverage-json-v1/trybot/2012/05/14/12/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot/5/500/coverage_by_line_abc123_1408633190.json@@@"
-    ]
-  },
   {
     "name": "$result",
     "recipe_result": null,
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN.json b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN.json
index 5c39f43d8569284e5d5900abe266bdd45c498678..2d2e8d3ae9ef844e4a59dd8e0d8f17eec27a13c3 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN.json
@@ -694,28 +694,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -729,9 +707,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/Ubuntu/def456/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN/5\", \"tasks\": {\"test_skia/Ubuntu/def456/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on Ubuntu",
     "~followup_annotations": [
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json
index c4cad61129993e6e420642faf03d168fd091dd05..a34a5008907baaca04e6de667b204734e0d14fa6 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug.json
@@ -670,28 +670,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -705,9 +683,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5\", \"tasks\": {\"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on Ubuntu",
     "~followup_annotations": [
@@ -758,21 +734,78 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/dm"
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir",
+      "511"
     ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree dm",
+    "name": "makedirs swarming tmp dir (3)",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "{\n    \"args\": [\n        \"--isolate\", \n        \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n        \"--isolated\", \n        \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n        \"--config-variable\", \n        \"OS\", \n        \"Linux\", \n        \"--blacklist\", \n        \".git\", \n        \"--blacklist\", \n        \"out\", \n        \"--blacklist\", \n        \"*.pyc\", \n        \"--blacklist\", \n        \".recipe_deps\", \n        \"--extra-variable\", \n        \"WORKDIR\", \n        \"[SLAVE_BUILD]\"\n    ], \n    \"dir\": \"[SLAVE_BUILD]\", \n    \"version\": 1\n}",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "Write upload_dm_results_skia.isolated.gen.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::isolate]/resources/isolate.py",
+      "[SLAVE_BUILD]/swarming.client",
+      "batcharchive",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--verbose",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "isolate tests (3)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n  isolated = json.load(f)\nif not isolated.get('includes'):\n  isolated['includes'] = []\nfor h in sys.argv[2:]:\n  isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n  json.dump(isolated, f, sort_keys=True)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated",
+      "abc123"
+    ],
+    "name": "add_isolated_input (2)",
+    "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
+      "@@@STEP_LOG_LINE@python.inline@import json@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated = json.load(f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'] = []@@@",
+      "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'].append(h)@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  json.dump(isolated, f, sort_keys=True)@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
@@ -780,31 +813,161 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm",
-      "[SLAVE_BUILD]/dm",
-      "0"
+      "[SLAVE_BUILD]/swarming.client/isolateserver.py",
+      "archive",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated"
     ],
-    "name": "dm_dir"
+    "name": "upload new .isolated file for upload_dm_results_skia",
+    "stdout": "/path/to/tmp/"
   },
   {
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py",
-      "[SLAVE_BUILD]/dm",
-      "abc123",
-      "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug",
-      "5",
-      "",
-      "[SLAVE_BUILD]/skia/common/py/utils"
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "trigger",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--priority",
+      "90",
+      "--shards",
+      "1",
+      "--task-name",
+      "upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--expiration",
+      "72000",
+      "--io-timeout",
+      "2400",
+      "--hard-timeout",
+      "14400",
+      "--dimension",
+      "cpu",
+      "x86-64-avx2",
+      "--dimension",
+      "gpu",
+      "none",
+      "--dimension",
+      "os",
+      "Linux",
+      "--dimension",
+      "pool",
+      "Skia",
+      "--tag",
+      "allow_milo:1",
+      "--tag",
+      "buildername:Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug",
+      "--tag",
+      "buildnumber:5",
+      "--tag",
+      "data:def456",
+      "--tag",
+      "master:client.skia",
+      "--tag",
+      "name:upload_dm_results_skia",
+      "--tag",
+      "os:Linux",
+      "--tag",
+      "revision:abc123",
+      "--tag",
+      "slavename:skiabot-linux-swarm-000",
+      "--tag",
+      "stepname:upload_dm_results_skia on Linux",
+      "--idempotent",
+      "def456",
+      "--",
+      "--workdir",
+      "../../..",
+      "upload_dm_results",
+      "buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug",
+      "mastername=client.skia",
+      "buildnumber=5",
+      "slavename=skiabot-linux-swarm-000",
+      "reason=Triggered by Skia swarm_trigger Recipe",
+      "swarm_out_dir=${ISOLATED_OUTDIR}",
+      "revision=abc123"
     ],
-    "cwd": "[SLAVE_BUILD]/skia",
-    "env": {
-      "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto",
-      "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto"
-    },
-    "name": "Upload DM Results"
+    "name": "[trigger] upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"tasks\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 0, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "collect",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--decorate",
+      "--print-status-updates",
+      "--json",
+      "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
+      "--task-summary-json",
+      "/path/to/tmp/json"
+    ],
+    "name": "upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@swarming pending 71s@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"shards\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"abandoned_ts\": null, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"bot_id\": \"vm30\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"created_ts\": \"2014-09-25T01:41:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"durations\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        5.7, @@@",
+      "@@@STEP_LOG_LINE@json.output@        31.5@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"exit_codes\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        0, @@@",
+      "@@@STEP_LOG_LINE@json.output@        0@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"id\": \"148aa78d7aa0000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"internal_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"isolated_out\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolated\": \"abc123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"namespace\": \"default-gzip\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"view_url\": \"blah\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"modified_ts\": \"2014-09-25 01:42:00\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"outputs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Heart beat succeeded on win32.\\n\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Foo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"started_ts\": \"2014-09-25T01:42:11.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"state\": 112, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"try_number\": 1, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"user\": \"unknown\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0 isolated out@blah@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
+    ]
   },
   {
     "name": "$result",
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared.json b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared.json
index cdf2291ee63c4c44f9d7e8c409a694f54dc53f75..709afa38d3f59921a88ff5495512931fdc8e004f 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared.json
@@ -670,28 +670,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -705,9 +683,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5\", \"tasks\": {\"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on Ubuntu",
     "~followup_annotations": [
@@ -758,21 +734,78 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/dm"
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir",
+      "511"
     ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree dm",
+    "name": "makedirs swarming tmp dir (3)",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "{\n    \"args\": [\n        \"--isolate\", \n        \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n        \"--isolated\", \n        \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n        \"--config-variable\", \n        \"OS\", \n        \"Linux\", \n        \"--blacklist\", \n        \".git\", \n        \"--blacklist\", \n        \"out\", \n        \"--blacklist\", \n        \"*.pyc\", \n        \"--blacklist\", \n        \".recipe_deps\", \n        \"--extra-variable\", \n        \"WORKDIR\", \n        \"[SLAVE_BUILD]\"\n    ], \n    \"dir\": \"[SLAVE_BUILD]\", \n    \"version\": 1\n}",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "Write upload_dm_results_skia.isolated.gen.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::isolate]/resources/isolate.py",
+      "[SLAVE_BUILD]/swarming.client",
+      "batcharchive",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--verbose",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "isolate tests (3)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n  isolated = json.load(f)\nif not isolated.get('includes'):\n  isolated['includes'] = []\nfor h in sys.argv[2:]:\n  isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n  json.dump(isolated, f, sort_keys=True)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated",
+      "abc123"
+    ],
+    "name": "add_isolated_input (2)",
+    "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
+      "@@@STEP_LOG_LINE@python.inline@import json@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated = json.load(f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'] = []@@@",
+      "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'].append(h)@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  json.dump(isolated, f, sort_keys=True)@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
@@ -780,31 +813,161 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm",
-      "[SLAVE_BUILD]/dm",
-      "0"
+      "[SLAVE_BUILD]/swarming.client/isolateserver.py",
+      "archive",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated"
     ],
-    "name": "dm_dir"
+    "name": "upload new .isolated file for upload_dm_results_skia",
+    "stdout": "/path/to/tmp/"
   },
   {
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py",
-      "[SLAVE_BUILD]/dm",
-      "abc123",
-      "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared",
-      "5",
-      "",
-      "[SLAVE_BUILD]/skia/common/py/utils"
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "trigger",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--priority",
+      "90",
+      "--shards",
+      "1",
+      "--task-name",
+      "upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--expiration",
+      "72000",
+      "--io-timeout",
+      "2400",
+      "--hard-timeout",
+      "14400",
+      "--dimension",
+      "cpu",
+      "x86-64-avx2",
+      "--dimension",
+      "gpu",
+      "none",
+      "--dimension",
+      "os",
+      "Linux",
+      "--dimension",
+      "pool",
+      "Skia",
+      "--tag",
+      "allow_milo:1",
+      "--tag",
+      "buildername:Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared",
+      "--tag",
+      "buildnumber:5",
+      "--tag",
+      "data:def456",
+      "--tag",
+      "master:client.skia",
+      "--tag",
+      "name:upload_dm_results_skia",
+      "--tag",
+      "os:Linux",
+      "--tag",
+      "revision:abc123",
+      "--tag",
+      "slavename:skiabot-linux-swarm-000",
+      "--tag",
+      "stepname:upload_dm_results_skia on Linux",
+      "--idempotent",
+      "def456",
+      "--",
+      "--workdir",
+      "../../..",
+      "upload_dm_results",
+      "buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared",
+      "mastername=client.skia",
+      "buildnumber=5",
+      "slavename=skiabot-linux-swarm-000",
+      "reason=Triggered by Skia swarm_trigger Recipe",
+      "swarm_out_dir=${ISOLATED_OUTDIR}",
+      "revision=abc123"
     ],
-    "cwd": "[SLAVE_BUILD]/skia",
-    "env": {
-      "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto",
-      "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto"
-    },
-    "name": "Upload DM Results"
+    "name": "[trigger] upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"tasks\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 0, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "collect",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--decorate",
+      "--print-status-updates",
+      "--json",
+      "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
+      "--task-summary-json",
+      "/path/to/tmp/json"
+    ],
+    "name": "upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@swarming pending 71s@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"shards\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"abandoned_ts\": null, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"bot_id\": \"vm30\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"created_ts\": \"2014-09-25T01:41:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"durations\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        5.7, @@@",
+      "@@@STEP_LOG_LINE@json.output@        31.5@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"exit_codes\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        0, @@@",
+      "@@@STEP_LOG_LINE@json.output@        0@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"id\": \"148aa78d7aa0000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"internal_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"isolated_out\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolated\": \"abc123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"namespace\": \"default-gzip\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"view_url\": \"blah\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"modified_ts\": \"2014-09-25 01:42:00\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"outputs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Heart beat succeeded on win32.\\n\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Foo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"started_ts\": \"2014-09-25T01:42:11.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"state\": 112, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"try_number\": 1, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"user\": \"unknown\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0 isolated out@blah@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
+    ]
   },
   {
     "name": "$result",
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json
index 3eb69e4864ade5c20f844d9f9e4659e2a05f8c18..58c36bc47b709565d8b8bae9c61610604ac243f1 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind.json
@@ -667,28 +667,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -702,9 +680,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind/5\", \"tasks\": {\"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on Ubuntu",
     "~followup_annotations": [
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release.json b/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release.json
index 3524a6ecb73fd4552b3c7ec4da60331bb7dcf1f9..16526702edc6a7d963b9a03dc918a98c9629ae86 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release.json
@@ -679,28 +679,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -714,9 +692,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/Windows/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5\", \"tasks\": {\"test_skia/Windows/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on Windows",
     "~followup_annotations": [
@@ -767,21 +743,78 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/dm"
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir",
+      "511"
     ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree dm",
+    "name": "makedirs swarming tmp dir (3)",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "{\n    \"args\": [\n        \"--isolate\", \n        \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n        \"--isolated\", \n        \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n        \"--config-variable\", \n        \"OS\", \n        \"Linux\", \n        \"--blacklist\", \n        \".git\", \n        \"--blacklist\", \n        \"out\", \n        \"--blacklist\", \n        \"*.pyc\", \n        \"--blacklist\", \n        \".recipe_deps\", \n        \"--extra-variable\", \n        \"WORKDIR\", \n        \"[SLAVE_BUILD]\"\n    ], \n    \"dir\": \"[SLAVE_BUILD]\", \n    \"version\": 1\n}",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "Write upload_dm_results_skia.isolated.gen.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::isolate]/resources/isolate.py",
+      "[SLAVE_BUILD]/swarming.client",
+      "batcharchive",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--verbose",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "isolate tests (3)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n  isolated = json.load(f)\nif not isolated.get('includes'):\n  isolated['includes'] = []\nfor h in sys.argv[2:]:\n  isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n  json.dump(isolated, f, sort_keys=True)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated",
+      "abc123"
+    ],
+    "name": "add_isolated_input (2)",
+    "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
+      "@@@STEP_LOG_LINE@python.inline@import json@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated = json.load(f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'] = []@@@",
+      "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'].append(h)@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  json.dump(isolated, f, sort_keys=True)@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
@@ -789,31 +822,161 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm",
-      "[SLAVE_BUILD]/dm",
-      "0"
+      "[SLAVE_BUILD]/swarming.client/isolateserver.py",
+      "archive",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated"
     ],
-    "name": "dm_dir"
+    "name": "upload new .isolated file for upload_dm_results_skia",
+    "stdout": "/path/to/tmp/"
   },
   {
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py",
-      "[SLAVE_BUILD]/dm",
-      "abc123",
-      "Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release",
-      "5",
-      "",
-      "[SLAVE_BUILD]/skia/common/py/utils"
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "trigger",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--priority",
+      "90",
+      "--shards",
+      "1",
+      "--task-name",
+      "upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--expiration",
+      "72000",
+      "--io-timeout",
+      "2400",
+      "--hard-timeout",
+      "14400",
+      "--dimension",
+      "cpu",
+      "x86-64-avx2",
+      "--dimension",
+      "gpu",
+      "none",
+      "--dimension",
+      "os",
+      "Linux",
+      "--dimension",
+      "pool",
+      "Skia",
+      "--tag",
+      "allow_milo:1",
+      "--tag",
+      "buildername:Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release",
+      "--tag",
+      "buildnumber:5",
+      "--tag",
+      "data:def456",
+      "--tag",
+      "master:client.skia",
+      "--tag",
+      "name:upload_dm_results_skia",
+      "--tag",
+      "os:Linux",
+      "--tag",
+      "revision:abc123",
+      "--tag",
+      "slavename:skiabot-linux-swarm-000",
+      "--tag",
+      "stepname:upload_dm_results_skia on Linux",
+      "--idempotent",
+      "def456",
+      "--",
+      "--workdir",
+      "../../..",
+      "upload_dm_results",
+      "buildername=Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release",
+      "mastername=client.skia",
+      "buildnumber=5",
+      "slavename=skiabot-linux-swarm-000",
+      "reason=Triggered by Skia swarm_trigger Recipe",
+      "swarm_out_dir=${ISOLATED_OUTDIR}",
+      "revision=abc123"
     ],
-    "cwd": "[SLAVE_BUILD]/skia",
-    "env": {
-      "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto",
-      "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto"
-    },
-    "name": "Upload DM Results"
+    "name": "[trigger] upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"tasks\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 0, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "collect",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--decorate",
+      "--print-status-updates",
+      "--json",
+      "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
+      "--task-summary-json",
+      "/path/to/tmp/json"
+    ],
+    "name": "upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@swarming pending 71s@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"shards\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"abandoned_ts\": null, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"bot_id\": \"vm30\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"created_ts\": \"2014-09-25T01:41:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"durations\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        5.7, @@@",
+      "@@@STEP_LOG_LINE@json.output@        31.5@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"exit_codes\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        0, @@@",
+      "@@@STEP_LOG_LINE@json.output@        0@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"id\": \"148aa78d7aa0000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"internal_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"isolated_out\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolated\": \"abc123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"namespace\": \"default-gzip\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"view_url\": \"blah\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"modified_ts\": \"2014-09-25 01:42:00\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"outputs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Heart beat succeeded on win32.\\n\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Foo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"started_ts\": \"2014-09-25T01:42:11.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"state\": 112, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"try_number\": 1, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"user\": \"unknown\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0 isolated out@blah@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
+    ]
   },
   {
     "name": "$result",
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release.json b/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release.json
index 9626d20247f84714ebb33f46ab509969a1be27df..87428626252b623f5b59ac9c8592cd80f6bf96c0 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release.json
@@ -682,28 +682,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -717,9 +695,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/Windows-2008ServerR2-SP1/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5\", \"tasks\": {\"test_skia/Windows-2008ServerR2-SP1/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on Windows-2008ServerR2-SP1",
     "~followup_annotations": [
@@ -770,21 +746,78 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/dm"
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir",
+      "511"
     ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree dm",
+    "name": "makedirs swarming tmp dir (3)",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "{\n    \"args\": [\n        \"--isolate\", \n        \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n        \"--isolated\", \n        \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n        \"--config-variable\", \n        \"OS\", \n        \"Linux\", \n        \"--blacklist\", \n        \".git\", \n        \"--blacklist\", \n        \"out\", \n        \"--blacklist\", \n        \"*.pyc\", \n        \"--blacklist\", \n        \".recipe_deps\", \n        \"--extra-variable\", \n        \"WORKDIR\", \n        \"[SLAVE_BUILD]\"\n    ], \n    \"dir\": \"[SLAVE_BUILD]\", \n    \"version\": 1\n}",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "Write upload_dm_results_skia.isolated.gen.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::isolate]/resources/isolate.py",
+      "[SLAVE_BUILD]/swarming.client",
+      "batcharchive",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--verbose",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "isolate tests (3)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n  isolated = json.load(f)\nif not isolated.get('includes'):\n  isolated['includes'] = []\nfor h in sys.argv[2:]:\n  isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n  json.dump(isolated, f, sort_keys=True)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated",
+      "abc123"
+    ],
+    "name": "add_isolated_input (2)",
+    "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
+      "@@@STEP_LOG_LINE@python.inline@import json@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated = json.load(f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'] = []@@@",
+      "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'].append(h)@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  json.dump(isolated, f, sort_keys=True)@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
@@ -792,31 +825,161 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm",
-      "[SLAVE_BUILD]/dm",
-      "0"
+      "[SLAVE_BUILD]/swarming.client/isolateserver.py",
+      "archive",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated"
     ],
-    "name": "dm_dir"
+    "name": "upload new .isolated file for upload_dm_results_skia",
+    "stdout": "/path/to/tmp/"
   },
   {
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py",
-      "[SLAVE_BUILD]/dm",
-      "abc123",
-      "Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release",
-      "5",
-      "",
-      "[SLAVE_BUILD]/skia/common/py/utils"
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "trigger",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--priority",
+      "90",
+      "--shards",
+      "1",
+      "--task-name",
+      "upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--expiration",
+      "72000",
+      "--io-timeout",
+      "2400",
+      "--hard-timeout",
+      "14400",
+      "--dimension",
+      "cpu",
+      "x86-64-avx2",
+      "--dimension",
+      "gpu",
+      "none",
+      "--dimension",
+      "os",
+      "Linux",
+      "--dimension",
+      "pool",
+      "Skia",
+      "--tag",
+      "allow_milo:1",
+      "--tag",
+      "buildername:Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release",
+      "--tag",
+      "buildnumber:5",
+      "--tag",
+      "data:def456",
+      "--tag",
+      "master:client.skia",
+      "--tag",
+      "name:upload_dm_results_skia",
+      "--tag",
+      "os:Linux",
+      "--tag",
+      "revision:abc123",
+      "--tag",
+      "slavename:skiabot-linux-swarm-000",
+      "--tag",
+      "stepname:upload_dm_results_skia on Linux",
+      "--idempotent",
+      "def456",
+      "--",
+      "--workdir",
+      "../../..",
+      "upload_dm_results",
+      "buildername=Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release",
+      "mastername=client.skia",
+      "buildnumber=5",
+      "slavename=skiabot-linux-swarm-000",
+      "reason=Triggered by Skia swarm_trigger Recipe",
+      "swarm_out_dir=${ISOLATED_OUTDIR}",
+      "revision=abc123"
     ],
-    "cwd": "[SLAVE_BUILD]/skia",
-    "env": {
-      "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto",
-      "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto"
-    },
-    "name": "Upload DM Results"
+    "name": "[trigger] upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"tasks\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 0, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "collect",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--decorate",
+      "--print-status-updates",
+      "--json",
+      "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
+      "--task-summary-json",
+      "/path/to/tmp/json"
+    ],
+    "name": "upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@swarming pending 71s@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"shards\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"abandoned_ts\": null, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"bot_id\": \"vm30\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"created_ts\": \"2014-09-25T01:41:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"durations\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        5.7, @@@",
+      "@@@STEP_LOG_LINE@json.output@        31.5@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"exit_codes\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        0, @@@",
+      "@@@STEP_LOG_LINE@json.output@        0@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"id\": \"148aa78d7aa0000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"internal_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"isolated_out\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolated\": \"abc123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"namespace\": \"default-gzip\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"view_url\": \"blah\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"modified_ts\": \"2014-09-25 01:42:00\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"outputs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Heart beat succeeded on win32.\\n\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Foo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"started_ts\": \"2014-09-25T01:42:11.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"state\": 112, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"try_number\": 1, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"user\": \"unknown\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0 isolated out@blah@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
+    ]
   },
   {
     "name": "$result",
diff --git a/infra/bots/recipes/swarm_trigger.expected/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release.json b/infra/bots/recipes/swarm_trigger.expected/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release.json
index 3c2fa52e21a803805a9cf37dd8ee5e307144c497..cebeb0291fb1c1627f5e72bcabdfb237c59fdd1a 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release.json
+++ b/infra/bots/recipes/swarm_trigger.expected/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release.json
@@ -667,28 +667,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -702,9 +680,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/iOS-9.2/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5\", \"tasks\": {\"test_skia/iOS-9.2/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on iOS-9.2",
     "~followup_annotations": [
@@ -755,21 +731,78 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/dm"
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir",
+      "511"
     ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree dm",
+    "name": "makedirs swarming tmp dir (3)",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "{\n    \"args\": [\n        \"--isolate\", \n        \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n        \"--isolated\", \n        \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n        \"--config-variable\", \n        \"OS\", \n        \"Linux\", \n        \"--blacklist\", \n        \".git\", \n        \"--blacklist\", \n        \"out\", \n        \"--blacklist\", \n        \"*.pyc\", \n        \"--blacklist\", \n        \".recipe_deps\", \n        \"--extra-variable\", \n        \"WORKDIR\", \n        \"[SLAVE_BUILD]\"\n    ], \n    \"dir\": \"[SLAVE_BUILD]\", \n    \"version\": 1\n}",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "Write upload_dm_results_skia.isolated.gen.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::isolate]/resources/isolate.py",
+      "[SLAVE_BUILD]/swarming.client",
+      "batcharchive",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--verbose",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "isolate tests (3)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n  isolated = json.load(f)\nif not isolated.get('includes'):\n  isolated['includes'] = []\nfor h in sys.argv[2:]:\n  isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n  json.dump(isolated, f, sort_keys=True)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated",
+      "abc123"
+    ],
+    "name": "add_isolated_input (2)",
+    "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
+      "@@@STEP_LOG_LINE@python.inline@import json@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated = json.load(f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'] = []@@@",
+      "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'].append(h)@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  json.dump(isolated, f, sort_keys=True)@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
@@ -777,31 +810,161 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm",
-      "[SLAVE_BUILD]/dm",
-      "0"
+      "[SLAVE_BUILD]/swarming.client/isolateserver.py",
+      "archive",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated"
     ],
-    "name": "dm_dir"
+    "name": "upload new .isolated file for upload_dm_results_skia",
+    "stdout": "/path/to/tmp/"
   },
   {
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py",
-      "[SLAVE_BUILD]/dm",
-      "abc123",
-      "Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release",
-      "5",
-      "",
-      "[SLAVE_BUILD]/skia/common/py/utils"
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "trigger",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--priority",
+      "90",
+      "--shards",
+      "1",
+      "--task-name",
+      "upload_dm_results_skia/Linux/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--expiration",
+      "72000",
+      "--io-timeout",
+      "2400",
+      "--hard-timeout",
+      "14400",
+      "--dimension",
+      "cpu",
+      "x86-64-avx2",
+      "--dimension",
+      "gpu",
+      "none",
+      "--dimension",
+      "os",
+      "Linux",
+      "--dimension",
+      "pool",
+      "Skia",
+      "--tag",
+      "allow_milo:1",
+      "--tag",
+      "buildername:Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release",
+      "--tag",
+      "buildnumber:5",
+      "--tag",
+      "data:def456",
+      "--tag",
+      "master:client.skia",
+      "--tag",
+      "name:upload_dm_results_skia",
+      "--tag",
+      "os:Linux",
+      "--tag",
+      "revision:abc123",
+      "--tag",
+      "slavename:skiabot-linux-swarm-000",
+      "--tag",
+      "stepname:upload_dm_results_skia on Linux",
+      "--idempotent",
+      "def456",
+      "--",
+      "--workdir",
+      "../../..",
+      "upload_dm_results",
+      "buildername=Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release",
+      "mastername=client.skia",
+      "buildnumber=5",
+      "slavename=skiabot-linux-swarm-000",
+      "reason=Triggered by Skia swarm_trigger Recipe",
+      "swarm_out_dir=${ISOLATED_OUTDIR}",
+      "revision=abc123"
     ],
-    "cwd": "[SLAVE_BUILD]/skia",
-    "env": {
-      "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto",
-      "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto"
-    },
-    "name": "Upload DM Results"
+    "name": "[trigger] upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"tasks\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"upload_dm_results_skia/Linux/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 0, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "collect",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--decorate",
+      "--print-status-updates",
+      "--json",
+      "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Release/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
+      "--task-summary-json",
+      "/path/to/tmp/json"
+    ],
+    "name": "upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@swarming pending 71s@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"shards\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"abandoned_ts\": null, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"bot_id\": \"vm30\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"created_ts\": \"2014-09-25T01:41:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"durations\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        5.7, @@@",
+      "@@@STEP_LOG_LINE@json.output@        31.5@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"exit_codes\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        0, @@@",
+      "@@@STEP_LOG_LINE@json.output@        0@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"id\": \"148aa78d7aa0000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"internal_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"isolated_out\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolated\": \"abc123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"namespace\": \"default-gzip\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"view_url\": \"blah\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"modified_ts\": \"2014-09-25 01:42:00\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"outputs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Heart beat succeeded on win32.\\n\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Foo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"started_ts\": \"2014-09-25T01:42:11.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"state\": 112, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"try_number\": 1, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"user\": \"unknown\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0 isolated out@blah@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
+    ]
   },
   {
     "name": "$result",
diff --git a/infra/bots/recipes/swarm_trigger.expected/recipe_with_gerrit_patch.json b/infra/bots/recipes/swarm_trigger.expected/recipe_with_gerrit_patch.json
index 246f91d326cf606bc197eaa608b8bc1a46a73bf5..5ac2417d0c7edd8f0a15d19400d2c365c81bcf82 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.expected/recipe_with_gerrit_patch.json
+++ b/infra/bots/recipes/swarm_trigger.expected/recipe_with_gerrit_patch.json
@@ -678,28 +678,6 @@
       "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
     ]
   },
-  {
-    "cmd": [
-      "python",
-      "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
-    ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree test_skia",
-    "~followup_annotations": [
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
-      "@@@STEP_LOG_END@python.inline@@@"
-    ]
-  },
   {
     "cmd": [
       "python",
@@ -713,9 +691,7 @@
       "--json",
       "{\"base_task_name\": \"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\", \"tasks\": {\"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
       "--task-summary-json",
-      "/path/to/tmp/json",
-      "--task-output-dir",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia"
+      "/path/to/tmp/json"
     ],
     "name": "test_skia on Ubuntu",
     "~followup_annotations": [
@@ -766,21 +742,78 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n  chromium_utils.RemoveDirectory(sys.argv[1])\n",
-      "[SLAVE_BUILD]/dm"
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir",
+      "511"
     ],
-    "env": {
-      "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts"
-    },
-    "name": "rmtree dm",
+    "name": "makedirs swarming tmp dir (3)",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
-      "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
-      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "{\n    \"args\": [\n        \"--isolate\", \n        \"[SLAVE_BUILD]/skia/infra/bots/upload_dm_results.isolate\", \n        \"--isolated\", \n        \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated\", \n        \"--config-variable\", \n        \"OS\", \n        \"Linux\", \n        \"--blacklist\", \n        \".git\", \n        \"--blacklist\", \n        \"out\", \n        \"--blacklist\", \n        \"*.pyc\", \n        \"--blacklist\", \n        \".recipe_deps\", \n        \"--extra-variable\", \n        \"WORKDIR\", \n        \"[SLAVE_BUILD]\"\n    ], \n    \"dir\": \"[SLAVE_BUILD]\", \n    \"version\": 1\n}",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "Write upload_dm_results_skia.isolated.gen.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::isolate]/resources/isolate.py",
+      "[SLAVE_BUILD]/swarming.client",
+      "batcharchive",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--verbose",
+      "[SLAVE_BUILD]/swarming_temp_dir/upload_dm_results_skia.isolated.gen.json"
+    ],
+    "name": "isolate tests (3)",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"upload_dm_results_skia\": \"[dummy hash for upload_dm_results_skia]\"}@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n  isolated = json.load(f)\nif not isolated.get('includes'):\n  isolated['includes'] = []\nfor h in sys.argv[2:]:\n  isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n  json.dump(isolated, f, sort_keys=True)\n",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated",
+      "abc123"
+    ],
+    "name": "add_isolated_input (2)",
+    "~followup_annotations": [
       "@@@STEP_LOG_LINE@python.inline@@@@",
-      "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
-      "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
+      "@@@STEP_LOG_LINE@python.inline@import json@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated = json.load(f)@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'] = []@@@",
+      "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  isolated['includes'].append(h)@@@",
+      "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  json.dump(isolated, f, sort_keys=True)@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
@@ -788,31 +821,165 @@
     "cmd": [
       "python",
       "-u",
-      "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
-      "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm",
-      "[SLAVE_BUILD]/dm",
-      "0"
+      "[SLAVE_BUILD]/swarming.client/isolateserver.py",
+      "archive",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "[SLAVE_BUILD]/swarming_temp_dir/skia-task-upload_dm_results_skia.isolated"
     ],
-    "name": "dm_dir"
+    "name": "upload new .isolated file for upload_dm_results_skia",
+    "stdout": "/path/to/tmp/"
   },
   {
     "cmd": [
       "python",
       "-u",
-      "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py",
-      "[SLAVE_BUILD]/dm",
-      "abc123",
-      "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot",
-      "5",
-      "2100",
-      "[SLAVE_BUILD]/skia/common/py/utils"
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "trigger",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--isolate-server",
+      "https://isolateserver.appspot.com",
+      "--priority",
+      "90",
+      "--shards",
+      "1",
+      "--task-name",
+      "upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5",
+      "--dump-json",
+      "/path/to/tmp/json",
+      "--expiration",
+      "72000",
+      "--io-timeout",
+      "2400",
+      "--hard-timeout",
+      "14400",
+      "--dimension",
+      "cpu",
+      "x86-64-avx2",
+      "--dimension",
+      "gpu",
+      "none",
+      "--dimension",
+      "os",
+      "Linux",
+      "--dimension",
+      "pool",
+      "Skia",
+      "--tag",
+      "allow_milo:1",
+      "--tag",
+      "buildername:Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot",
+      "--tag",
+      "buildnumber:5",
+      "--tag",
+      "data:def456",
+      "--tag",
+      "master:client.skia",
+      "--tag",
+      "name:upload_dm_results_skia",
+      "--tag",
+      "os:Linux",
+      "--tag",
+      "revision:abc123",
+      "--tag",
+      "slavename:skiabot-linux-swarm-000",
+      "--tag",
+      "stepname:upload_dm_results_skia on Linux",
+      "--idempotent",
+      "def456",
+      "--",
+      "--workdir",
+      "../../..",
+      "upload_dm_results",
+      "event.change.number=2100",
+      "repository=skia",
+      "buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot",
+      "mastername=client.skia",
+      "patch_storage=gerrit",
+      "buildnumber=5",
+      "slavename=skiabot-linux-swarm-000",
+      "reason=Triggered by Skia swarm_trigger Recipe",
+      "swarm_out_dir=${ISOLATED_OUTDIR}",
+      "event.patchSet.ref=refs/changes/00/2100/2",
+      "revision=abc123"
     ],
-    "cwd": "[SLAVE_BUILD]/skia",
-    "env": {
-      "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto",
-      "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto"
-    },
-    "name": "Upload DM Results"
+    "name": "[trigger] upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"tasks\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@    \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"shard_index\": 0, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"task_id\": \"10000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  }@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[SLAVE_BUILD]/swarming.client/swarming.py",
+      "collect",
+      "--swarming",
+      "https://chromium-swarm.appspot.com",
+      "--decorate",
+      "--print-status-updates",
+      "--json",
+      "{\"base_task_name\": \"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\", \"tasks\": {\"upload_dm_results_skia/Linux/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}",
+      "--task-summary-json",
+      "/path/to/tmp/json"
+    ],
+    "name": "upload_dm_results_skia on Linux",
+    "~followup_annotations": [
+      "@@@STEP_TEXT@swarming pending 71s@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"shards\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"abandoned_ts\": null, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"bot_id\": \"vm30\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"created_ts\": \"2014-09-25T01:41:00.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"durations\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        5.7, @@@",
+      "@@@STEP_LOG_LINE@json.output@        31.5@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"exit_codes\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        0, @@@",
+      "@@@STEP_LOG_LINE@json.output@        0@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"id\": \"148aa78d7aa0000\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"internal_failure\": false, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"isolated_out\": {@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolated\": \"abc123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"namespace\": \"default-gzip\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"view_url\": \"blah\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      }, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"modified_ts\": \"2014-09-25 01:42:00\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"outputs\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Heart beat succeeded on win32.\\n\", @@@",
+      "@@@STEP_LOG_LINE@json.output@        \"Foo\"@@@",
+      "@@@STEP_LOG_LINE@json.output@      ], @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"started_ts\": \"2014-09-25T01:42:11.123\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"state\": 112, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"try_number\": 1, @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"user\": \"unknown\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@",
+      "@@@STEP_LINK@shard #0 isolated out@blah@@@",
+      "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@"
+    ]
   },
   {
     "name": "$result",
diff --git a/infra/bots/recipes/swarm_trigger.py b/infra/bots/recipes/swarm_trigger.py
index fdf81004e8bc8f145462c00713d82c98379cf4e2..8a9360cd317f933aeff02c0236c73b7cbfb12c2a 100644 (file)
--- a/infra/bots/recipes/swarm_trigger.py
+++ b/infra/bots/recipes/swarm_trigger.py
@@ -434,26 +434,6 @@ def get_timeouts(builder_cfg):
   return expiration, hard_timeout, io_timeout
 
 
-def gsutil_env(api, boto_file):
-  """Environment variables for gsutil."""
-  home_dir = os.path.expanduser('~')
-  if api.path._test_data.enabled:
-    home_dir = '[HOME]'
-
-  boto_path = None
-  if boto_file:
-    boto_path = api.path.join(home_dir, boto_file)
-  return {'AWS_CREDENTIAL_FILE': boto_path,
-          'BOTO_CONFIG': boto_path}
-
-
-def get_issue_num(api):
-  if api.properties.get('patch_storage') == 'gerrit':
-    return str(api.properties['event.change.number'])
-  else:
-    return str(api.properties['issue'])
-
-
 def perf_steps_trigger(api, builder_cfg, got_revision, infrabots_dir,
                        extra_hashes, cipd_packages):
   """Trigger perf tests via Swarming."""
@@ -526,110 +506,39 @@ def test_steps_trigger(api, builder_cfg, got_revision, infrabots_dir,
       expiration=expiration,
       hard_timeout=hard_timeout,
       io_timeout=io_timeout,
-      cipd_packages=cipd_packages)
+      cipd_packages=cipd_packages,
+      store_output=False)
 
 
-def test_steps_collect(api, task, got_revision, is_trybot, builder_cfg):
+def test_steps_collect(api, task, builder_cfg, got_revision, infrabots_dir):
   """Collect the test results from Swarming."""
   # Wait for tests to finish, download the results.
-  api.run.rmtree(task.task_output_dir)
-  api.swarming.collect_swarming_task(task)
+  if not api.vars.upload_dm_results:  # pragma: nocover
+    api.swarming.collect_swarming_task(task)
+    return
+
+  dm_hash = api.swarming.collect_swarming_task_isolate_hash(task)
 
   # Upload the results.
-  if api.vars.upload_dm_results:
-    dm_dir = api.path['slave_build'].join('dm')
-    dm_src = task.task_output_dir.join('0', 'dm')
-    api.run.rmtree(dm_dir)
-    api.file.copytree('dm_dir', dm_src, dm_dir, infra_step=True)
-
-    # Upload them to Google Storage.
-    api.python(
-        'Upload DM Results',
-        script=api.core.resource('upload_dm_results.py'),
-        args=[
-          dm_dir,
-          got_revision,
-          api.properties['buildername'],
-          api.properties['buildnumber'],
-          get_issue_num(api) if is_trybot else '',
-          api.path['slave_build'].join('skia', 'common', 'py', 'utils'),
-        ],
-        cwd=api.path['checkout'],
-        env=gsutil_env(api, 'chromium-skia-gm.boto'),
-        infra_step=True)
-
-  if builder_cfg['configuration']  == 'Coverage':
-    upload_coverage_results(api, task, got_revision, is_trybot)
-
-
-def upload_coverage_results(api, task, got_revision, is_trybot):
-  results_dir = task.task_output_dir.join('0')
-  git_timestamp = api.git.get_timestamp(test_data='1408633190',
-                                        infra_step=True)
-
-  # Upload raw coverage data.
-  cov_file_basename = '%s.cov' % got_revision
-  cov_file = results_dir.join(cov_file_basename)
-  now = api.time.utcnow()
-  gs_json_path = '/'.join((
-      str(now.year).zfill(4), str(now.month).zfill(2),
-      str(now.day).zfill(2), str(now.hour).zfill(2),
+  task = trigger_task(
+      api,
+      'upload_dm_results',
       api.properties['buildername'],
-      str(api.properties['buildnumber'])))
-  if is_trybot:
-    gs_json_path = '/'.join(('trybot', gs_json_path, get_issue_num(api)))
-  api.gsutil.upload(
-      name='upload raw coverage data',
-      source=cov_file,
-      bucket='skia-infra',
-      dest='/'.join(('coverage-raw-v1', gs_json_path,
-                     cov_file_basename)),
-      env={'AWS_CREDENTIAL_FILE': None, 'BOTO_CONFIG': None},
-  )
+      api.properties['mastername'],
+      api.properties['slavename'],
+      api.properties['buildnumber'],
+      builder_cfg,
+      got_revision,
+      infrabots_dir,
+      idempotent=True,
+      store_output=False,
+      cipd_packages=None,
+      extra_isolate_hashes=[dm_hash],
+      recipe_name='upload_dm_results',
+      isolate_file='upload_dm_results.isolate',
+      dimensions=UPLOAD_DIMENSIONS)
 
-  # Transform the nanobench_${git_hash}.json file received from swarming bot
-  # into the nanobench_${git_hash}_${timestamp}.json file
-  # upload_bench_results.py expects.
-  src_nano_file = results_dir.join('nanobench_%s.json' % got_revision)
-  dst_nano_file = results_dir.join(
-      'nanobench_%s_%s.json' % (got_revision, git_timestamp))
-  api.file.copy('nanobench JSON', src_nano_file, dst_nano_file,
-                infra_step=True)
-  api.file.remove('old nanobench JSON', src_nano_file)
-
-  # Upload nanobench JSON data.
-  gsutil_path = api.depot_tools.gsutil_py_path
-  upload_args = [api.properties['buildername'], api.properties['buildnumber'],
-                 results_dir, got_revision, gsutil_path]
-  if is_trybot:
-    upload_args.append(get_issue_num(api))
-  api.python(
-      'upload nanobench coverage results',
-      script=api.core.resource('upload_bench_results.py'),
-      args=upload_args,
-      cwd=api.path['checkout'],
-      env=gsutil_env(api, 'chromium-skia-gm.boto'),
-      infra_step=True)
-
-  # Transform the coverage_by_line_${git_hash}.json file received from
-  # swarming bot into a coverage_by_line_${git_hash}_${timestamp}.json file.
-  src_lbl_file = results_dir.join('coverage_by_line_%s.json' % got_revision)
-  dst_lbl_file_basename = 'coverage_by_line_%s_%s.json' % (
-      got_revision, git_timestamp)
-  dst_lbl_file = results_dir.join(dst_lbl_file_basename)
-  api.file.copy('Line-by-line coverage JSON', src_lbl_file, dst_lbl_file,
-                infra_step=True)
-  api.file.remove('old line-by-line coverage JSON', src_lbl_file)
-
-  # Upload line-by-line coverage data.
-  api.gsutil.upload(
-      name='upload line-by-line coverage data',
-      source=dst_lbl_file,
-      bucket='skia-infra',
-      dest='/'.join(('coverage-json-v1', gs_json_path,
-                     dst_lbl_file_basename)),
-      env={'AWS_CREDENTIAL_FILE': None, 'BOTO_CONFIG': None},
-  )
+  return api.swarming.collect_swarming_task(task)
 
 
 def cipd_pkg(api, infrabots_dir, asset_name):
@@ -738,13 +647,12 @@ def RunSteps(api):
   if do_perf_steps:
     perf_task = perf_steps_trigger(api, builder_cfg, got_revision,
                                    infrabots_dir, extra_hashes, cipd_packages)
-  is_trybot = builder_cfg['is_trybot']
 
   # Wait for results, then upload them if necessary.
 
   if test_task:
-    test_steps_collect(api, test_task,
-                       got_revision, is_trybot, builder_cfg)
+    test_steps_collect(api, test_task, builder_cfg,
+                       got_revision, infrabots_dir)
 
   if perf_task:
     perf_steps_collect(api, perf_task, builder_cfg,
@@ -778,6 +686,10 @@ def test_for_bot(api, builder, mastername, slavename, testname=None):
     test += api.step_data(
         'upload new .isolated file for test_skia',
         stdout=api.raw_io.output('def456 XYZ.isolated'))
+    if not 'Valgrind' in builder and not 'MSAN' in builder:
+      test += api.step_data(
+          'upload new .isolated file for upload_dm_results_skia',
+          stdout=api.raw_io.output('def456 XYZ.isolated'))
   if 'Perf' in builder and '-CT_' not in builder:
     test += api.step_data(
         'upload new .isolated file for perf_skia',
@@ -830,5 +742,8 @@ def GenTests(api):
           **gerrit_kwargs) +
       api.step_data(
           'upload new .isolated file for test_skia',
+          stdout=api.raw_io.output('def456 XYZ.isolated')) +
+      api.step_data(
+          'upload new .isolated file for upload_dm_results_skia',
           stdout=api.raw_io.output('def456 XYZ.isolated'))
   )
diff --git a/infra/bots/recipes/upload_dm_results.expected/normal_bot.json b/infra/bots/recipes/upload_dm_results.expected/normal_bot.json
new file mode 100644 (file)
index 0000000..8b35adc
--- /dev/null
+++ b/infra/bots/recipes/upload_dm_results.expected/normal_bot.json
@@ -0,0 +1,125 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[CWD]/dm/dm.json",
+      "/path/to/tmp/json"
+    ],
+    "name": "validate dm.json",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
+      "@@@STEP_LOG_END@json.output (invalid)@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[CWD]/tmp_upload",
+      "511"
+    ],
+    "name": "makedirs tmp dir",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[CWD]/dm/dm.json",
+      "[CWD]/tmp_upload"
+    ],
+    "name": "copy dm.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[CWD]/dm/verbose.log",
+      "[CWD]/tmp_upload"
+    ],
+    "name": "copy verbose.log"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
+      "[CWD]/dm/dm.json"
+    ],
+    "name": "rm old dm.json",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
+      "[CWD]/dm/verbose.log"
+    ],
+    "name": "rm old verbose.log",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n  f.write('\\n'.join(glob.glob(sys.argv[2])))\n",
+      "/path/to/tmp/",
+      "[CWD]/dm/*"
+    ],
+    "name": "find images"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "[CWD]/dm/*",
+      "gs://skia-infra-gm/dm-images-v1"
+    ],
+    "name": "upload images"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-z",
+      "json,log",
+      "[CWD]/tmp_upload/*",
+      "gs://skia-infra-gm/dm-json-v1/2012/05/14/12/abc123/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/1337018001"
+    ],
+    "name": "upload JSON and logs"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file
diff --git a/infra/bots/recipes/upload_dm_results.expected/trybot.json b/infra/bots/recipes/upload_dm_results.expected/trybot.json
new file mode 100644 (file)
index 0000000..ced83e6
--- /dev/null
+++ b/infra/bots/recipes/upload_dm_results.expected/trybot.json
@@ -0,0 +1,125 @@
+[
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[CWD]/dm/dm.json",
+      "/path/to/tmp/json"
+    ],
+    "name": "validate dm.json",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
+      "@@@STEP_LOG_END@json.output (invalid)@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n  if os.path.exists(path):\n    print \"%s exists but is not a dir\" % path\n    sys.exit(1)\n  os.makedirs(path, mode)\n",
+      "[CWD]/tmp_upload",
+      "511"
+    ],
+    "name": "makedirs tmp dir",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+      "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+      "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+      "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  if os.path.exists(path):@@@",
+      "@@@STEP_LOG_LINE@python.inline@    print \"%s exists but is not a dir\" % path@@@",
+      "@@@STEP_LOG_LINE@python.inline@    sys.exit(1)@@@",
+      "@@@STEP_LOG_LINE@python.inline@  os.makedirs(path, mode)@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[CWD]/dm/dm.json",
+      "[CWD]/tmp_upload"
+    ],
+    "name": "copy dm.json"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
+      "[CWD]/dm/verbose.log",
+      "[CWD]/tmp_upload"
+    ],
+    "name": "copy verbose.log"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
+      "[CWD]/dm/dm.json"
+    ],
+    "name": "rm old dm.json",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
+      "[CWD]/dm/verbose.log"
+    ],
+    "name": "rm old verbose.log",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import os@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n  f.write('\\n'.join(glob.glob(sys.argv[2])))\n",
+      "/path/to/tmp/",
+      "[CWD]/dm/*"
+    ],
+    "name": "find images"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "[CWD]/dm/*",
+      "gs://skia-infra-gm/dm-images-v1"
+    ],
+    "name": "upload images"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-z",
+      "json,log",
+      "[CWD]/tmp_upload/*",
+      "gs://skia-infra-gm/trybot/dm-json-v1/2012/05/14/12/abc123/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/1337018001/12345/1002"
+    ],
+    "name": "upload JSON and logs"
+  },
+  {
+    "name": "$result",
+    "recipe_result": null,
+    "status_code": 0
+  }
+]
\ No newline at end of file
diff --git a/infra/bots/recipes/upload_dm_results.py b/infra/bots/recipes/upload_dm_results.py
new file mode 100644 (file)
index 0000000..20d2c06
--- /dev/null
+++ b/infra/bots/recipes/upload_dm_results.py
@@ -0,0 +1,106 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+# Recipe for uploading DM results.
+
+
+DEPS = [
+  'build/file',
+  'recipe_engine/json',
+  'recipe_engine/path',
+  'recipe_engine/properties',
+  'recipe_engine/shutil',
+  'recipe_engine/step',
+  'recipe_engine/time',
+]
+
+
+import time
+
+
+DM_JSON = 'dm.json'
+GS_BUCKET = 'gs://skia-infra-gm'
+VERBOSE_LOG = 'verbose.log'
+
+
+def RunSteps(api):
+  builder_name = api.properties['buildername']
+  revision = api.properties['revision']
+  issue = str(api.properties.get('issue', ''))
+  patchset = str(api.properties.get('patchset', ''))
+
+  results_dir = api.path['cwd'].join('dm')
+
+  # Validate the JSON file.
+  json_file = results_dir.join(DM_JSON)
+  api.json.read('validate dm.json', json_file)
+
+  # Move dm.json and verbose.log to their own directory.
+  log_file = results_dir.join(VERBOSE_LOG)
+  tmp_dir = api.path['cwd'].join('tmp_upload')
+  api.shutil.makedirs('tmp dir', tmp_dir, infra_step=True)
+  api.shutil.copy('copy dm.json', json_file, tmp_dir)
+  api.shutil.copy('copy verbose.log', log_file, tmp_dir)
+  api.shutil.remove('rm old dm.json', json_file)
+  api.shutil.remove('rm old verbose.log', log_file)
+
+  # Upload the images.
+  image_dest_path = '/'.join((GS_BUCKET, 'dm-images-v1'))
+  files_to_upload = api.file.glob(
+      'find images',
+      results_dir.join('*'),
+      test_data=['someimage.png'],
+      infra_step=True)
+  if len(files_to_upload) > 0:
+    api.step(
+        'upload images',
+        cmd=['gsutil', 'cp', results_dir.join('*'), image_dest_path],
+     )
+
+  # Upload the JSON summary and verbose.log.
+  now = api.time.utcnow()
+  summary_dest_path = '/'.join([
+      'dm-json-v1',
+      str(now.year ).zfill(4),
+      str(now.month).zfill(2),
+      str(now.day  ).zfill(2),
+      str(now.hour ).zfill(2),
+      revision,
+      builder_name,
+      str(int(time.mktime(now.utctimetuple())))])
+
+  # Trybot results are further siloed by issue/patchset.
+  if builder_name.endswith('-Trybot'):
+    if not (issue and patchset):  # pragma: nocover
+      raise Exception('issue and patchset properties are required for trybots.')
+    summary_dest_path = '/'.join(('trybot', summary_dest_path, issue, patchset))
+
+  summary_dest_path = '/'.join((GS_BUCKET, summary_dest_path))
+
+  api.step(
+      'upload JSON and logs',
+      cmd=['gsutil', 'cp', '-z', 'json,log', tmp_dir.join('*'),
+           summary_dest_path],
+  )
+
+
+def GenTests(api):
+  builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug'
+  yield (
+    api.test('normal_bot') +
+    api.properties(buildername=builder,
+                   revision='abc123',
+                   path_config='kitchen')
+  )
+
+  builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot'
+  yield (
+    api.test('trybot') +
+    api.properties(buildername=builder,
+                   revision='abc123',
+                   path_config='kitchen',
+                   issue='12345',
+                   patchset='1002')
+  )
diff --git a/infra/bots/recipes/upload_nano_results.py b/infra/bots/recipes/upload_nano_results.py
index 8cc39be693582c6b810559927a0094f282645b3b..2abe388e856ca8f4d1dcb598b98b205d7a0f6019 100644 (file)
--- a/infra/bots/recipes/upload_nano_results.py
+++ b/infra/bots/recipes/upload_nano_results.py
@@ -18,8 +18,8 @@ DEPS = [
 def RunSteps(api):
   # Upload the nanobench resuls.
   builder_name = api.properties['buildername']
-  issue = str(api.properties.get('issue'))
-  patchset = str(api.properties.get('patchset'))
+  issue = str(api.properties.get('issue', ''))
+  patchset = str(api.properties.get('patchset', ''))
 
   now = api.time.utcnow()
 
diff --git a/infra/bots/upload_dm_results.isolate b/infra/bots/upload_dm_results.isolate
new file mode 100644 (file)
index 0000000..767fa3a
--- /dev/null
+++ b/infra/bots/upload_dm_results.isolate
@@ -0,0 +1,10 @@
+{
+  'includes': [
+    'swarm_recipe.isolate',
+  ],
+  'variables': {
+    'files': [
+      '../../../.gclient',
+    ],
+  },
+}
Domain: Dotnet / API;