From 01b48e7f13c25af430b6c90aac8cefcd79c0d870 Mon Sep 17 00:00:00 2001 From: Ravi Mistry Date: Wed, 17 May 2017 14:28:06 -0400 Subject: [PATCH] Bot to update buildbucket.config daily Bug: skia:6591 Change-Id: Ic2ec5b033965ca073a00e272693b1c5e1d0ad69f Reviewed-on: https://skia-review.googlesource.com/16541 Commit-Queue: Ravi Mistry Reviewed-by: Eric Boren --- infra/bots/gen_tasks.go | 35 +++++++ infra/bots/jobs.json | 1 + infra/bots/meta_config.isolate | 5 + infra/bots/recipe_modules/infra/api.py | 45 +++++++++ .../infra/example.expected/failed_one_update.json | 48 ++++++++++ .../infra/example.expected/infra_tests.json | 48 ++++++++++ infra/bots/recipe_modules/infra/example.py | 3 + .../Housekeeper-Weekly-RecreateSKPs.json | 4 +- .../recreate_skps.expected/failed_upload.json | 4 +- infra/bots/recipes/recreate_skps.py | 50 +--------- .../Housekeeper-Nightly-UpdateMetaConfig.json | 98 ++++++++++++++++++++ .../update_meta_config.expected/failed_update.json | 103 +++++++++++++++++++++ .../update_meta_config.expected/trybot_test.json | 30 ++++++ infra/bots/recipes/update_meta_config.py | 88 ++++++++++++++++++ infra/bots/tasks.json | 30 ++++++ infra/bots/update_meta_config.py | 91 ++++++++++++++++++ 16 files changed, 631 insertions(+), 52 deletions(-) create mode 100644 infra/bots/meta_config.isolate create mode 100644 infra/bots/recipes/update_meta_config.expected/Housekeeper-Nightly-UpdateMetaConfig.json create mode 100644 infra/bots/recipes/update_meta_config.expected/failed_update.json create mode 100644 infra/bots/recipes/update_meta_config.expected/trybot_test.json create mode 100644 infra/bots/recipes/update_meta_config.py create mode 100644 infra/bots/update_meta_config.py diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go index 917cc50..f81b1ac 100644 --- a/infra/bots/gen_tasks.go +++ b/infra/bots/gen_tasks.go @@ -423,6 +423,32 @@ func recreateSKPs(b *specs.TasksCfgBuilder, name string) string { return name } + +// updateMetaConfig generates a UpdateMetaConfig task. Returns the name of the +// last task in the generated chain of tasks, which the Job should add as a +// dependency. +func updateMetaConfig(b *specs.TasksCfgBuilder, name string) string { + b.MustAddTask(name, &specs.TaskSpec{ + CipdPackages: []*specs.CipdPackage{}, + Dimensions: linuxGceDimensions(), + ExtraArgs: []string{ + "--workdir", "../../..", "update_meta_config", + fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), + fmt.Sprintf("buildername=%s", name), + fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), + fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), + fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), + fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), + fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), + fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), + }, + Isolate: "meta_config.isolate", + Priority: 0.8, + }) + return name +} + + // ctSKPs generates a CT SKPs task. Returns the name of the last task in the // generated chain of tasks, which the Job should add as a dependency. func ctSKPs(b *specs.TasksCfgBuilder, name string) string { @@ -690,6 +716,11 @@ func process(b *specs.TasksCfgBuilder, name string) { deps = append(deps, recreateSKPs(b, name)) } + // UpdateMetaConfig bot. + if strings.Contains(name, "UpdateMetaConfig") { + deps = append(deps, updateMetaConfig(b, name)) + } + // CT bots. if strings.Contains(name, "-CT_") { deps = append(deps, ctSKPs(b, name)) @@ -716,6 +747,7 @@ func process(b *specs.TasksCfgBuilder, name string) { name != "Housekeeper-PerCommit-BundleRecipes" && name != "Housekeeper-PerCommit-InfraTests" && !strings.Contains(name, "RecreateSKPs") && + !strings.Contains(name, "UpdateMetaConfig") && !strings.Contains(name, "-CT_") { compile(b, compileTaskName, compileTaskParts) } @@ -769,6 +801,9 @@ func process(b *specs.TasksCfgBuilder, name string) { if name == "Housekeeper-Nightly-RecreateSKPs_Canary" { j.Trigger = "nightly" } + if name == "Housekeeper-Nightly-UpdateMetaConfig" { + j.Trigger = "nightly" + } if name == "Housekeeper-Weekly-RecreateSKPs" { j.Trigger = "weekly" } diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json index 5bf6e52..45782a2 100644 --- a/infra/bots/jobs.json +++ b/infra/bots/jobs.json @@ -74,6 +74,7 @@ "Build-Win-MSVC-x86_64-Release-GDI", "Build-Win-MSVC-x86_64-Release-Vulkan", "Housekeeper-Nightly-RecreateSKPs_Canary", + "Housekeeper-Nightly-UpdateMetaConfig", "Housekeeper-PerCommit", "Housekeeper-PerCommit-BundleRecipes", "Housekeeper-PerCommit-InfraTests", diff --git a/infra/bots/meta_config.isolate b/infra/bots/meta_config.isolate new file mode 100644 index 0000000..d4e1484 --- /dev/null +++ b/infra/bots/meta_config.isolate @@ -0,0 +1,5 @@ +{ + 'includes': [ + 'swarm_recipe.isolate', + ], +} diff --git a/infra/bots/recipe_modules/infra/api.py b/infra/bots/recipe_modules/infra/api.py index 0a47842..5f37e48 100644 --- a/infra/bots/recipe_modules/infra/api.py +++ b/infra/bots/recipe_modules/infra/api.py @@ -63,3 +63,48 @@ class InfraApi(recipe_api.RecipeApi): 'update go pkgs', UPDATE_GO_ATTEMPTS, cmd=[self.go_exe, 'get', '-u', '-t', '%s/...' % INFRA_GO_PKG]) + + class MetadataFetch(): + def __init__(self, api, metadata_key, local_file, **kwargs): + self.m = api + self._key = metadata_key + self._local_file = local_file + + def __enter__(self): + return self.m.python.inline( + 'download ' + self._local_file, + """ +import os +import urllib2 + +TOKEN_FILE = '%s' +TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/%s' + +req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'}) +contents = urllib2.urlopen(req).read() + +home = os.path.expanduser('~') +token_file = os.path.join(home, TOKEN_FILE) + +with open(token_file, 'w') as f: + f.write(contents) + """ % (self._local_file, self._key), + ) + + def __exit__(self, t, v, tb): + self.m.python.inline( + 'cleanup ' + self._local_file, + """ +import os + + +TOKEN_FILE = '%s' + + +home = os.path.expanduser('~') +token_file = os.path.join(home, TOKEN_FILE) +if os.path.isfile(token_file): + os.remove(token_file) + """ % (self._local_file), + ) + return v is None diff --git a/infra/bots/recipe_modules/infra/example.expected/failed_one_update.json b/infra/bots/recipe_modules/infra/example.expected/failed_one_update.json index 3b8a04b..89d7042 100644 --- a/infra/bots/recipe_modules/infra/example.expected/failed_one_update.json +++ b/infra/bots/recipe_modules/infra/example.expected/failed_one_update.json @@ -70,6 +70,54 @@ "name": "update go pkgs (attempt 2)" }, { + "cmd": [ + "python", + "-u", + "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'file'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n" + ], + "name": "download file", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import urllib2@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'file'@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@", + "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@", + "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\n\n\nTOKEN_FILE = 'file'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n" + ], + "name": "cleanup file", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'file'@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@", + "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { "name": "$result", "recipe_result": null, "status_code": 0 diff --git a/infra/bots/recipe_modules/infra/example.expected/infra_tests.json b/infra/bots/recipe_modules/infra/example.expected/infra_tests.json index 169fc19..1e38e2e 100644 --- a/infra/bots/recipe_modules/infra/example.expected/infra_tests.json +++ b/infra/bots/recipe_modules/infra/example.expected/infra_tests.json @@ -48,6 +48,54 @@ "name": "update go pkgs" }, { + "cmd": [ + "python", + "-u", + "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'file'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n" + ], + "name": "download file", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import urllib2@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'file'@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@", + "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@", + "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\n\n\nTOKEN_FILE = 'file'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n" + ], + "name": "cleanup file", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'file'@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@", + "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { "name": "$result", "recipe_result": null, "status_code": 0 diff --git a/infra/bots/recipe_modules/infra/example.py b/infra/bots/recipe_modules/infra/example.py index d46a7ee..af1c9f2 100644 --- a/infra/bots/recipe_modules/infra/example.py +++ b/infra/bots/recipe_modules/infra/example.py @@ -11,6 +11,7 @@ DEPS = [ 'infra', 'recipe_engine/path', 'recipe_engine/properties', + 'recipe_engine/python', 'recipe_engine/step', 'run', 'vars', @@ -20,6 +21,8 @@ DEPS = [ def RunSteps(api): api.vars.setup() api.infra.update_go_deps() + with api.infra.MetadataFetch(api, 'key', 'file'): + pass def GenTests(api): diff --git a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json index 5d461f4..b731496 100644 --- a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json +++ b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json @@ -359,7 +359,7 @@ "-u", "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_skps.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n" ], - "name": "download update-skps.gitcookies", + "name": "download update_skps.git_cookies", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", "@@@STEP_LOG_LINE@python.inline@import os@@@", @@ -405,7 +405,7 @@ "-u", "\nimport os\n\n\nTOKEN_FILE = 'update_skps.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n" ], - "name": "cleanup update-skps.gitcookies", + "name": "cleanup update_skps.git_cookies", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", "@@@STEP_LOG_LINE@python.inline@import os@@@", diff --git a/infra/bots/recipes/recreate_skps.expected/failed_upload.json b/infra/bots/recipes/recreate_skps.expected/failed_upload.json index 5b1fbca..664077e 100644 --- a/infra/bots/recipes/recreate_skps.expected/failed_upload.json +++ b/infra/bots/recipes/recreate_skps.expected/failed_upload.json @@ -359,7 +359,7 @@ "-u", "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_skps.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n" ], - "name": "download update-skps.gitcookies", + "name": "download update_skps.git_cookies", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", "@@@STEP_LOG_LINE@python.inline@import os@@@", @@ -409,7 +409,7 @@ "-u", "\nimport os\n\n\nTOKEN_FILE = 'update_skps.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n" ], - "name": "cleanup update-skps.gitcookies", + "name": "cleanup update_skps.git_cookies", "~followup_annotations": [ "@@@STEP_LOG_LINE@python.inline@@@@", "@@@STEP_LOG_LINE@python.inline@import os@@@", diff --git a/infra/bots/recipes/recreate_skps.py b/infra/bots/recipes/recreate_skps.py index b789975..0da6fe2 100644 --- a/infra/bots/recipes/recreate_skps.py +++ b/infra/bots/recipes/recreate_skps.py @@ -36,53 +36,6 @@ UPDATE_SKPS_GITCOOKIES_FILE = 'update_skps.git_cookies' UPDATE_SKPS_KEY = 'update_skps_git_cookies' -class gitcookies_auth(object): - """Download update-skps@skia.org's .gitcookies.""" - def __init__(self, api, metadata_key): - self.m = api - self._key = metadata_key - - def __enter__(self): - return self.m.python.inline( - 'download update-skps.gitcookies', - """ -import os -import urllib2 - -TOKEN_FILE = '%s' -TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/%s' - -req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'}) -contents = urllib2.urlopen(req).read() - -home = os.path.expanduser('~') -token_file = os.path.join(home, TOKEN_FILE) - -with open(token_file, 'w') as f: - f.write(contents) - """ % (UPDATE_SKPS_GITCOOKIES_FILE, - self._key), - ) - - def __exit__(self, t, v, tb): - self.m.python.inline( - 'cleanup update-skps.gitcookies', - """ -import os - - -TOKEN_FILE = '%s' - - -home = os.path.expanduser('~') -token_file = os.path.join(home, TOKEN_FILE) -if os.path.isfile(token_file): - os.remove(token_file) - """ % (UPDATE_SKPS_GITCOOKIES_FILE), - ) - return v is None - - def RunSteps(api): # Check out Chrome. api.core.setup() @@ -128,7 +81,8 @@ def RunSteps(api): api.vars.skia_dir.join('infra', 'bots', 'upload_skps.py'), '--target_dir', output_dir, '--gitcookies', str(update_skps_gitcookies)] - with gitcookies_auth(api, UPDATE_SKPS_KEY): + with api.infra.MetadataFetch( + api, UPDATE_SKPS_KEY, UPDATE_SKPS_GITCOOKIES_FILE): with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env): api.run(api.step, 'Upload SKPs', cmd=cmd) diff --git a/infra/bots/recipes/update_meta_config.expected/Housekeeper-Nightly-UpdateMetaConfig.json b/infra/bots/recipes/update_meta_config.expected/Housekeeper-Nightly-UpdateMetaConfig.json new file mode 100644 index 0000000..9a1a0ae --- /dev/null +++ b/infra/bots/recipes/update_meta_config.expected/Housekeeper-Nightly-UpdateMetaConfig.json @@ -0,0 +1,98 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[START_DIR]/tmp", + "511" + ], + "infra_step": true, + "name": "makedirs tmp_dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_meta_config.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_meta_config_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n" + ], + "name": "download update_meta_config.git_cookies", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import urllib2@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_meta_config.git_cookies'@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_meta_config_git_cookies'@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@", + "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@", + "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "[START_DIR]/skia/infra/bots/update_meta_config.py", + "--repo_name", + "skia", + "--tasks_json", + "[START_DIR]/skia/infra/bots/tasks.json", + "--gitcookies", + "[HOME]/update_meta_config.git_cookies" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "Update meta/config" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\n\n\nTOKEN_FILE = 'update_meta_config.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n" + ], + "name": "cleanup update_meta_config.git_cookies", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_meta_config.git_cookies'@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@", + "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipes/update_meta_config.expected/failed_update.json b/infra/bots/recipes/update_meta_config.expected/failed_update.json new file mode 100644 index 0000000..1aaf64e --- /dev/null +++ b/infra/bots/recipes/update_meta_config.expected/failed_update.json @@ -0,0 +1,103 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[START_DIR]/tmp", + "511" + ], + "infra_step": true, + "name": "makedirs tmp_dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_meta_config.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_meta_config_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n" + ], + "name": "download update_meta_config.git_cookies", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import urllib2@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_meta_config.git_cookies'@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_meta_config_git_cookies'@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@", + "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@", + "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "[START_DIR]/skia/infra/bots/update_meta_config.py", + "--repo_name", + "skia", + "--tasks_json", + "[START_DIR]/skia/infra/bots/tasks.json", + "--gitcookies", + "[HOME]/update_meta_config.git_cookies" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "Update meta/config", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\n\n\nTOKEN_FILE = 'update_meta_config.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n" + ], + "name": "cleanup update_meta_config.git_cookies", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_meta_config.git_cookies'@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@", + "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "name": "$result", + "reason": "Step('Update meta/config') failed with return_code 1", + "recipe_result": null, + "status_code": 1 + } +] \ No newline at end of file diff --git a/infra/bots/recipes/update_meta_config.expected/trybot_test.json b/infra/bots/recipes/update_meta_config.expected/trybot_test.json new file mode 100644 index 0000000..5758c51 --- /dev/null +++ b/infra/bots/recipes/update_meta_config.expected/trybot_test.json @@ -0,0 +1,30 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[START_DIR]/tmp", + "511" + ], + "infra_step": true, + "name": "makedirs tmp_dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "name": "$result", + "reason": "Uncaught Exception: Exception('Cannot run update_meta_config recipe as a trybot',)", + "status_code": -1 + } +] \ No newline at end of file diff --git a/infra/bots/recipes/update_meta_config.py b/infra/bots/recipes/update_meta_config.py new file mode 100644 index 0000000..b3cbe23 --- /dev/null +++ b/infra/bots/recipes/update_meta_config.py @@ -0,0 +1,88 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Recipe for the Bot that updates meta config.""" + + +DEPS = [ + 'build/file', + 'depot_tools/gclient', + 'recipe_engine/context', + 'recipe_engine/path', + 'recipe_engine/properties', + 'recipe_engine/python', + 'recipe_engine/raw_io', + 'recipe_engine/step', + 'core', + 'infra', + 'run', + 'vars', +] + + +TEST_BUILDERS = { + 'client.skia.compile': { + 'skiabot-linux-swarm-000': [ + 'Housekeeper-Nightly-UpdateMetaConfig', + ], + }, +} + + +UPDATE_META_CONFIG_GITCOOKIES_FILE = 'update_meta_config.git_cookies' +UPDATE_META_CONFIG_KEY = 'update_meta_config_git_cookies' + + +def RunSteps(api): + api.core.setup() + + if api.vars.is_trybot: + raise Exception('Cannot run update_meta_config recipe as a trybot') + update_meta_config_gitcookies = api.path.join( + api.path.expanduser('~'), UPDATE_META_CONFIG_GITCOOKIES_FILE) + repo_name = api.properties.get('repository').split('/')[-1].rstrip('.git') + cmd = ['python', + api.vars.skia_dir.join('infra', 'bots', 'update_meta_config.py'), + '--repo_name', repo_name, + '--tasks_json', api.vars.skia_dir.join('infra', 'bots', 'tasks.json'), + '--gitcookies', str(update_meta_config_gitcookies)] + with api.infra.MetadataFetch( + api, UPDATE_META_CONFIG_KEY, UPDATE_META_CONFIG_GITCOOKIES_FILE): + with api.context(cwd=api.vars.skia_dir): + api.run(api.step, 'Update meta/config', cmd=cmd) + + +def GenTests(api): + builder = 'Housekeeper-Nightly-UpdateMetaConfig' + yield ( + api.test(builder) + + api.properties(buildername=builder, + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + ) + + yield ( + api.test('failed_update') + + api.properties(buildername=builder, + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + + api.step_data('Update meta/config', retcode=1) + ) + + yield ( + api.test('trybot_test') + + api.properties(buildername=builder, + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]', + patch_issue='123', + patch_set='3') + + api.expect_exception('Exception') + ) diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json index 6f3dd5f..da0c5a5 100644 --- a/infra/bots/tasks.json +++ b/infra/bots/tasks.json @@ -451,6 +451,13 @@ ], "trigger": "nightly" }, + "Housekeeper-Nightly-UpdateMetaConfig": { + "priority": 0.8, + "tasks": [ + "Housekeeper-Nightly-UpdateMetaConfig" + ], + "trigger": "nightly" + }, "Housekeeper-PerCommit": { "priority": 0.8, "tasks": [ @@ -4379,6 +4386,29 @@ "isolate": "compile_skia.isolate", "priority": 0.8 }, + "Housekeeper-Nightly-UpdateMetaConfig": { + "dimensions": [ + "cpu:x86-64-avx2", + "gpu:none", + "os:Ubuntu-14.04", + "pool:Skia" + ], + "extra_args": [ + "--workdir", + "../../..", + "update_meta_config", + "repository=<(REPO)", + "buildername=Housekeeper-Nightly-UpdateMetaConfig", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=<(REVISION)", + "patch_repo=<(PATCH_REPO)", + "patch_storage=<(PATCH_STORAGE)", + "patch_issue=<(ISSUE)", + "patch_set=<(PATCHSET)" + ], + "isolate": "meta_config.isolate", + "priority": 0.8 + }, "Housekeeper-PerCommit": { "cipd_packages": [ { diff --git a/infra/bots/update_meta_config.py b/infra/bots/update_meta_config.py new file mode 100644 index 0000000..663d170 --- /dev/null +++ b/infra/bots/update_meta_config.py @@ -0,0 +1,91 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Update meta/config of the specified Skia repo.""" + + +import argparse +import json +import os +import subprocess +import sys +import urllib2 + +import git_utils + + +SKIA_COMMITTER_EMAIL = 'update-meta-config@skia.org' +SKIA_COMMITTER_NAME = 'Update Meta Config' +SKIA_REPO_TEMPLATE = 'https://skia.googlesource.com/%s.git' + +CQ_INCLUDE_CHROMIUM_TRYBOTS = ( + 'master.tryserver.blink:linux_trusty_blink_rel,linux_trusty_blink_dbg;' + 'master.tryserver.chromium.linux:linux_optional_gpu_tests_rel;' + 'master.tryserver.chromium.mac:mac_optional_gpu_tests_rel;' + 'master.tryserver.chromium.win:win_optional_gpu_tests_rel;' + 'master.tryserver.chromium.android:android_optional_gpu_tests_rel' +) + + +def addChromiumTrybots(f): + for master_section in CQ_INCLUDE_CHROMIUM_TRYBOTS.split(';'): + master, bots = master_section.split(':') + f.write('[bucket "%s"]\n' % master) + for bot in bots.split(','): + f.write('\tbuilder = %s\n' % bot) + + +def main(gitcookies, repo_name, tasks_json): + skia_repo = SKIA_REPO_TEMPLATE % repo_name + with git_utils.NewGitCheckout(repository=skia_repo): + # Fetch and checkout the meta/config branch. + subprocess.check_call(['git', 'fetch', skia_repo, 'refs/meta/config:cfg']) + subprocess.check_call(['git', 'checkout', 'cfg']) + + # Create list of tryjobs from tasks_json. + tryjobs = [] + with open(tasks_json) as tasks_json: + data = json.load(tasks_json) + for job in data['jobs'].keys(): + if not job.startswith('Upload-'): + tryjobs.append(job) + tryjobs.sort() + + # Write to buildbucket.config. + buildbucket_config = os.path.join(os.getcwd(), 'buildbucket.config') + with open(buildbucket_config, 'w') as f: + + if repo_name == 'skia': + addChromiumTrybots(f) + + # Adding all Skia jobs. + f.write('[bucket "skia.primary"]\n') + for job in tryjobs: + f.write('\tbuilder = ' + job + '\n') + + # Push the change as the update-meta-config user. + config_dict = { + 'user.name': SKIA_COMMITTER_NAME, + 'user.email': SKIA_COMMITTER_EMAIL, + 'http.cookiefile': gitcookies, + } + with git_utils.GitLocalConfig(config_dict): + subprocess.check_call(['git', 'add', 'buildbucket.config']) + try: + subprocess.check_call( + ['git', 'commit', '-m', 'Update builders in buildbucket.config']) + except subprocess.CalledProcessError: + print 'No changes to buildbucket.config' + return + + subprocess.check_call(['git', 'push', skia_repo, 'cfg:refs/meta/config']) + + +if '__main__' == __name__: + parser = argparse.ArgumentParser() + parser.add_argument("--gitcookies") + parser.add_argument("--repo_name") + parser.add_argument("--tasks_json") + args = parser.parse_args() + main(args.gitcookies, args.repo_name, args.tasks_json) -- 2.7.4