From 94df1841ec695e01ba22c475c011a0a531a20262 Mon Sep 17 00:00:00 2001 From: Ravi Mistry Date: Thu, 18 May 2017 12:44:40 -0400 Subject: [PATCH] Use persistent checkout for UpdateMetaConfig bot Bot is failing without this. BUG=skia:6591 Change-Id: I29cf5e18a7fa240d3e45bdadf83c1f91060f45f5 Reviewed-on: https://skia-review.googlesource.com/17319 Reviewed-by: Eric Boren Commit-Queue: Ravi Mistry --- infra/bots/recipe_modules/vars/api.py | 1 + .../Housekeeper-Nightly-UpdateMetaConfig.json | 142 ++++++++++++++++++++- .../update_meta_config.expected/failed_update.json | 142 ++++++++++++++++++++- .../update_meta_config.expected/trybot_test.json | 140 ++++++++++++++++++++ infra/bots/recipes/update_meta_config.py | 4 +- 5 files changed, 421 insertions(+), 8 deletions(-) diff --git a/infra/bots/recipe_modules/vars/api.py b/infra/bots/recipe_modules/vars/api.py index e48de5b..2c23f7a 100644 --- a/infra/bots/recipe_modules/vars/api.py +++ b/infra/bots/recipe_modules/vars/api.py @@ -40,6 +40,7 @@ class SkiaVarsApi(recipe_api.RecipeApi): # Compile bots keep a persistent checkout. self.persistent_checkout = (self.is_compile_bot or 'RecreateSKPs' in self.builder_name or + 'UpdateMetaConfig' in self.builder_name or '-CT_' in self.builder_name or 'Presubmit' in self.builder_name or 'InfraTests' in self.builder_name or diff --git a/infra/bots/recipes/update_meta_config.expected/Housekeeper-Nightly-UpdateMetaConfig.json b/infra/bots/recipes/update_meta_config.expected/Housekeeper-Nightly-UpdateMetaConfig.json index 9a1a0ae..544b657 100644 --- a/infra/bots/recipes/update_meta_config.expected/Housekeeper-Nightly-UpdateMetaConfig.json +++ b/infra/bots/recipes/update_meta_config.expected/Housekeeper-Nightly-UpdateMetaConfig.json @@ -4,6 +4,142 @@ "python", "-u", "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CUSTOM_/_B_WORK]", + "511" + ], + "infra_step": true, + "name": "makedirs checkout_path", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "git", + "remote", + "set-url", + "origin", + "https://skia.googlesource.com/skia.git" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "infra_step": true, + "name": "git remote set-url" + }, + { + "cmd": [ + "git", + "fetch" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "infra_step": true, + "name": "git fetch" + }, + { + "cmd": [ + "git", + "reset", + "--hard", + "abc123" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "infra_step": true, + "name": "git reset" + }, + { + "cmd": [ + "git", + "clean", + "-d", + "-f" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "infra_step": true, + "name": "git clean" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[CUSTOM_/_B_WORK]/.gclient_entries" + ], + "infra_step": true, + "name": "remove [CUSTOM_/_B_WORK]/.gclient_entries", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec", + "cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]", + "--patch_root", + "skia", + "--revision_mapping_file", + "{\"got_revision\": \"skia\"}", + "--git-cache-dir", + "[CUSTOM_/_B_CACHE]", + "--output_json", + "/path/to/tmp/json", + "--revision", + "skia@abc123" + ], + "cwd": "[CUSTOM_/_B_WORK]", + "env": { + "GIT_HTTP_LOW_SPEED_LIMIT": "1000", + "GIT_HTTP_LOW_SPEED_TIME": "300", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "infra_step": true, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", "[START_DIR]/tmp", "511" ], @@ -51,15 +187,15 @@ { "cmd": [ "python", - "[START_DIR]/skia/infra/bots/update_meta_config.py", + "[CUSTOM_/_B_WORK]/skia/infra/bots/update_meta_config.py", "--repo_name", "skia", "--tasks_json", - "[START_DIR]/skia/infra/bots/tasks.json", + "[CUSTOM_/_B_WORK]/skia/infra/bots/tasks.json", "--gitcookies", "[HOME]/update_meta_config.git_cookies" ], - "cwd": "[START_DIR]/skia", + "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { "BUILDTYPE": "Release", "CHROME_HEADLESS": "1", diff --git a/infra/bots/recipes/update_meta_config.expected/failed_update.json b/infra/bots/recipes/update_meta_config.expected/failed_update.json index 1aaf64e..1d5a43d 100644 --- a/infra/bots/recipes/update_meta_config.expected/failed_update.json +++ b/infra/bots/recipes/update_meta_config.expected/failed_update.json @@ -4,6 +4,142 @@ "python", "-u", "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CUSTOM_/_B_WORK]", + "511" + ], + "infra_step": true, + "name": "makedirs checkout_path", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "git", + "remote", + "set-url", + "origin", + "https://skia.googlesource.com/skia.git" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "infra_step": true, + "name": "git remote set-url" + }, + { + "cmd": [ + "git", + "fetch" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "infra_step": true, + "name": "git fetch" + }, + { + "cmd": [ + "git", + "reset", + "--hard", + "abc123" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "infra_step": true, + "name": "git reset" + }, + { + "cmd": [ + "git", + "clean", + "-d", + "-f" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "infra_step": true, + "name": "git clean" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[CUSTOM_/_B_WORK]/.gclient_entries" + ], + "infra_step": true, + "name": "remove [CUSTOM_/_B_WORK]/.gclient_entries", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec", + "cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]", + "--patch_root", + "skia", + "--revision_mapping_file", + "{\"got_revision\": \"skia\"}", + "--git-cache-dir", + "[CUSTOM_/_B_CACHE]", + "--output_json", + "/path/to/tmp/json", + "--revision", + "skia@abc123" + ], + "cwd": "[CUSTOM_/_B_WORK]", + "env": { + "GIT_HTTP_LOW_SPEED_LIMIT": "1000", + "GIT_HTTP_LOW_SPEED_TIME": "300", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "infra_step": true, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", "[START_DIR]/tmp", "511" ], @@ -51,15 +187,15 @@ { "cmd": [ "python", - "[START_DIR]/skia/infra/bots/update_meta_config.py", + "[CUSTOM_/_B_WORK]/skia/infra/bots/update_meta_config.py", "--repo_name", "skia", "--tasks_json", - "[START_DIR]/skia/infra/bots/tasks.json", + "[CUSTOM_/_B_WORK]/skia/infra/bots/tasks.json", "--gitcookies", "[HOME]/update_meta_config.git_cookies" ], - "cwd": "[START_DIR]/skia", + "cwd": "[CUSTOM_/_B_WORK]/skia", "env": { "BUILDTYPE": "Release", "CHROME_HEADLESS": "1", diff --git a/infra/bots/recipes/update_meta_config.expected/trybot_test.json b/infra/bots/recipes/update_meta_config.expected/trybot_test.json index 5758c51..14438f6 100644 --- a/infra/bots/recipes/update_meta_config.expected/trybot_test.json +++ b/infra/bots/recipes/update_meta_config.expected/trybot_test.json @@ -4,6 +4,146 @@ "python", "-u", "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CUSTOM_/_B_WORK]", + "511" + ], + "infra_step": true, + "name": "makedirs checkout_path", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "git", + "remote", + "set-url", + "origin", + "https://skia.googlesource.com/skia.git" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "infra_step": true, + "name": "git remote set-url" + }, + { + "cmd": [ + "git", + "fetch" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "infra_step": true, + "name": "git fetch" + }, + { + "cmd": [ + "git", + "reset", + "--hard", + "abc123" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "infra_step": true, + "name": "git reset" + }, + { + "cmd": [ + "git", + "clean", + "-d", + "-f" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "infra_step": true, + "name": "git clean" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[CUSTOM_/_B_WORK]/.gclient_entries" + ], + "infra_step": true, + "name": "remove [CUSTOM_/_B_WORK]/.gclient_entries", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec", + "cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]", + "--patch_root", + "skia", + "--revision_mapping_file", + "{\"got_revision\": \"skia\"}", + "--git-cache-dir", + "[CUSTOM_/_B_CACHE]", + "--gerrit_repo", + "https://skia.googlesource.com/skia.git", + "--gerrit_ref", + "refs/changes/23/123/3", + "--output_json", + "/path/to/tmp/json", + "--revision", + "skia@abc123" + ], + "cwd": "[CUSTOM_/_B_WORK]", + "env": { + "GIT_HTTP_LOW_SPEED_LIMIT": "1000", + "GIT_HTTP_LOW_SPEED_TIME": "300", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "infra_step": true, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", "[START_DIR]/tmp", "511" ], diff --git a/infra/bots/recipes/update_meta_config.py b/infra/bots/recipes/update_meta_config.py index b3cbe23..3c6086c 100644 --- a/infra/bots/recipes/update_meta_config.py +++ b/infra/bots/recipes/update_meta_config.py @@ -82,7 +82,7 @@ def GenTests(api): revision='abc123', path_config='kitchen', swarm_out_dir='[SWARM_OUT_DIR]', - patch_issue='123', - patch_set='3') + + patch_issue=123, + patch_set=3) + api.expect_exception('Exception') ) -- 2.7.4