return name
}
+
+// updateMetaConfig generates a UpdateMetaConfig task. Returns the name of the
+// last task in the generated chain of tasks, which the Job should add as a
+// dependency.
+func updateMetaConfig(b *specs.TasksCfgBuilder, name string) string {
+ b.MustAddTask(name, &specs.TaskSpec{
+ CipdPackages: []*specs.CipdPackage{},
+ Dimensions: linuxGceDimensions(),
+ ExtraArgs: []string{
+ "--workdir", "../../..", "update_meta_config",
+ fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
+ fmt.Sprintf("buildername=%s", name),
+ fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
+ fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
+ fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO),
+ fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
+ fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE),
+ fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET),
+ },
+ Isolate: "meta_config.isolate",
+ Priority: 0.8,
+ })
+ return name
+}
+
+
// ctSKPs generates a CT SKPs task. Returns the name of the last task in the
// generated chain of tasks, which the Job should add as a dependency.
func ctSKPs(b *specs.TasksCfgBuilder, name string) string {
deps = append(deps, recreateSKPs(b, name))
}
+ // UpdateMetaConfig bot.
+ if strings.Contains(name, "UpdateMetaConfig") {
+ deps = append(deps, updateMetaConfig(b, name))
+ }
+
// CT bots.
if strings.Contains(name, "-CT_") {
deps = append(deps, ctSKPs(b, name))
name != "Housekeeper-PerCommit-BundleRecipes" &&
name != "Housekeeper-PerCommit-InfraTests" &&
!strings.Contains(name, "RecreateSKPs") &&
+ !strings.Contains(name, "UpdateMetaConfig") &&
!strings.Contains(name, "-CT_") {
compile(b, compileTaskName, compileTaskParts)
}
if name == "Housekeeper-Nightly-RecreateSKPs_Canary" {
j.Trigger = "nightly"
}
+ if name == "Housekeeper-Nightly-UpdateMetaConfig" {
+ j.Trigger = "nightly"
+ }
if name == "Housekeeper-Weekly-RecreateSKPs" {
j.Trigger = "weekly"
}
"Build-Win-MSVC-x86_64-Release-GDI",
"Build-Win-MSVC-x86_64-Release-Vulkan",
"Housekeeper-Nightly-RecreateSKPs_Canary",
+ "Housekeeper-Nightly-UpdateMetaConfig",
"Housekeeper-PerCommit",
"Housekeeper-PerCommit-BundleRecipes",
"Housekeeper-PerCommit-InfraTests",
--- /dev/null
+{
+ 'includes': [
+ 'swarm_recipe.isolate',
+ ],
+}
'update go pkgs',
UPDATE_GO_ATTEMPTS,
cmd=[self.go_exe, 'get', '-u', '-t', '%s/...' % INFRA_GO_PKG])
+
+ class MetadataFetch():
+ def __init__(self, api, metadata_key, local_file, **kwargs):
+ self.m = api
+ self._key = metadata_key
+ self._local_file = local_file
+
+ def __enter__(self):
+ return self.m.python.inline(
+ 'download ' + self._local_file,
+ """
+import os
+import urllib2
+
+TOKEN_FILE = '%s'
+TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/%s'
+
+req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})
+contents = urllib2.urlopen(req).read()
+
+home = os.path.expanduser('~')
+token_file = os.path.join(home, TOKEN_FILE)
+
+with open(token_file, 'w') as f:
+ f.write(contents)
+ """ % (self._local_file, self._key),
+ )
+
+ def __exit__(self, t, v, tb):
+ self.m.python.inline(
+ 'cleanup ' + self._local_file,
+ """
+import os
+
+
+TOKEN_FILE = '%s'
+
+
+home = os.path.expanduser('~')
+token_file = os.path.join(home, TOKEN_FILE)
+if os.path.isfile(token_file):
+ os.remove(token_file)
+ """ % (self._local_file),
+ )
+ return v is None
"name": "update go pkgs (attempt 2)"
},
{
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'file'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
+ ],
+ "name": "download file",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'file'@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
+ "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\n\n\nTOKEN_FILE = 'file'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
+ ],
+ "name": "cleanup file",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'file'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
"name": "$result",
"recipe_result": null,
"status_code": 0
"name": "update go pkgs"
},
{
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'file'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
+ ],
+ "name": "download file",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'file'@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
+ "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\n\n\nTOKEN_FILE = 'file'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
+ ],
+ "name": "cleanup file",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'file'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
"name": "$result",
"recipe_result": null,
"status_code": 0
'infra',
'recipe_engine/path',
'recipe_engine/properties',
+ 'recipe_engine/python',
'recipe_engine/step',
'run',
'vars',
def RunSteps(api):
api.vars.setup()
api.infra.update_go_deps()
+ with api.infra.MetadataFetch(api, 'key', 'file'):
+ pass
def GenTests(api):
"-u",
"\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_skps.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
],
- "name": "download update-skps.gitcookies",
+ "name": "download update_skps.git_cookies",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"-u",
"\nimport os\n\n\nTOKEN_FILE = 'update_skps.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
],
- "name": "cleanup update-skps.gitcookies",
+ "name": "cleanup update_skps.git_cookies",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"-u",
"\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_skps.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
],
- "name": "download update-skps.gitcookies",
+ "name": "download update_skps.git_cookies",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"-u",
"\nimport os\n\n\nTOKEN_FILE = 'update_skps.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
],
- "name": "cleanup update-skps.gitcookies",
+ "name": "cleanup update_skps.git_cookies",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
UPDATE_SKPS_KEY = 'update_skps_git_cookies'
-class gitcookies_auth(object):
- """Download update-skps@skia.org's .gitcookies."""
- def __init__(self, api, metadata_key):
- self.m = api
- self._key = metadata_key
-
- def __enter__(self):
- return self.m.python.inline(
- 'download update-skps.gitcookies',
- """
-import os
-import urllib2
-
-TOKEN_FILE = '%s'
-TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/%s'
-
-req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})
-contents = urllib2.urlopen(req).read()
-
-home = os.path.expanduser('~')
-token_file = os.path.join(home, TOKEN_FILE)
-
-with open(token_file, 'w') as f:
- f.write(contents)
- """ % (UPDATE_SKPS_GITCOOKIES_FILE,
- self._key),
- )
-
- def __exit__(self, t, v, tb):
- self.m.python.inline(
- 'cleanup update-skps.gitcookies',
- """
-import os
-
-
-TOKEN_FILE = '%s'
-
-
-home = os.path.expanduser('~')
-token_file = os.path.join(home, TOKEN_FILE)
-if os.path.isfile(token_file):
- os.remove(token_file)
- """ % (UPDATE_SKPS_GITCOOKIES_FILE),
- )
- return v is None
-
-
def RunSteps(api):
# Check out Chrome.
api.core.setup()
api.vars.skia_dir.join('infra', 'bots', 'upload_skps.py'),
'--target_dir', output_dir,
'--gitcookies', str(update_skps_gitcookies)]
- with gitcookies_auth(api, UPDATE_SKPS_KEY):
+ with api.infra.MetadataFetch(
+ api, UPDATE_SKPS_KEY, UPDATE_SKPS_GITCOOKIES_FILE):
with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env):
api.run(api.step, 'Upload SKPs', cmd=cmd)
--- /dev/null
+[
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
+ "[START_DIR]/tmp",
+ "511"
+ ],
+ "infra_step": true,
+ "name": "makedirs tmp_dir",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+ "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+ "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+ "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
+ "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_meta_config.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_meta_config_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
+ ],
+ "name": "download update_meta_config.git_cookies",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_meta_config.git_cookies'@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_meta_config_git_cookies'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
+ "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "[START_DIR]/skia/infra/bots/update_meta_config.py",
+ "--repo_name",
+ "skia",
+ "--tasks_json",
+ "[START_DIR]/skia/infra/bots/tasks.json",
+ "--gitcookies",
+ "[HOME]/update_meta_config.git_cookies"
+ ],
+ "cwd": "[START_DIR]/skia",
+ "env": {
+ "BUILDTYPE": "Release",
+ "CHROME_HEADLESS": "1",
+ "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]",
+ "SKIA_OUT": "[START_DIR]/out"
+ },
+ "name": "Update meta/config"
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\n\n\nTOKEN_FILE = 'update_meta_config.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
+ ],
+ "name": "cleanup update_meta_config.git_cookies",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_meta_config.git_cookies'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "name": "$result",
+ "recipe_result": null,
+ "status_code": 0
+ }
+]
\ No newline at end of file
--- /dev/null
+[
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
+ "[START_DIR]/tmp",
+ "511"
+ ],
+ "infra_step": true,
+ "name": "makedirs tmp_dir",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+ "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+ "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+ "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
+ "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_meta_config.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_meta_config_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
+ ],
+ "name": "download update_meta_config.git_cookies",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_meta_config.git_cookies'@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_meta_config_git_cookies'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
+ "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "[START_DIR]/skia/infra/bots/update_meta_config.py",
+ "--repo_name",
+ "skia",
+ "--tasks_json",
+ "[START_DIR]/skia/infra/bots/tasks.json",
+ "--gitcookies",
+ "[HOME]/update_meta_config.git_cookies"
+ ],
+ "cwd": "[START_DIR]/skia",
+ "env": {
+ "BUILDTYPE": "Release",
+ "CHROME_HEADLESS": "1",
+ "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]",
+ "SKIA_OUT": "[START_DIR]/out"
+ },
+ "name": "Update meta/config",
+ "~followup_annotations": [
+ "step returned non-zero exit code: 1",
+ "@@@STEP_FAILURE@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\n\n\nTOKEN_FILE = 'update_meta_config.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
+ ],
+ "name": "cleanup update_meta_config.git_cookies",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_meta_config.git_cookies'@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
+ "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
+ "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "name": "$result",
+ "reason": "Step('Update meta/config') failed with return_code 1",
+ "recipe_result": null,
+ "status_code": 1
+ }
+]
\ No newline at end of file
--- /dev/null
+[
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
+ "[START_DIR]/tmp",
+ "511"
+ ],
+ "infra_step": true,
+ "name": "makedirs tmp_dir",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
+ "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
+ "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
+ "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
+ "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "name": "$result",
+ "reason": "Uncaught Exception: Exception('Cannot run update_meta_config recipe as a trybot',)",
+ "status_code": -1
+ }
+]
\ No newline at end of file
--- /dev/null
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Recipe for the Bot that updates meta config."""
+
+
+DEPS = [
+ 'build/file',
+ 'depot_tools/gclient',
+ 'recipe_engine/context',
+ 'recipe_engine/path',
+ 'recipe_engine/properties',
+ 'recipe_engine/python',
+ 'recipe_engine/raw_io',
+ 'recipe_engine/step',
+ 'core',
+ 'infra',
+ 'run',
+ 'vars',
+]
+
+
+TEST_BUILDERS = {
+ 'client.skia.compile': {
+ 'skiabot-linux-swarm-000': [
+ 'Housekeeper-Nightly-UpdateMetaConfig',
+ ],
+ },
+}
+
+
+UPDATE_META_CONFIG_GITCOOKIES_FILE = 'update_meta_config.git_cookies'
+UPDATE_META_CONFIG_KEY = 'update_meta_config_git_cookies'
+
+
+def RunSteps(api):
+ api.core.setup()
+
+ if api.vars.is_trybot:
+ raise Exception('Cannot run update_meta_config recipe as a trybot')
+ update_meta_config_gitcookies = api.path.join(
+ api.path.expanduser('~'), UPDATE_META_CONFIG_GITCOOKIES_FILE)
+ repo_name = api.properties.get('repository').split('/')[-1].rstrip('.git')
+ cmd = ['python',
+ api.vars.skia_dir.join('infra', 'bots', 'update_meta_config.py'),
+ '--repo_name', repo_name,
+ '--tasks_json', api.vars.skia_dir.join('infra', 'bots', 'tasks.json'),
+ '--gitcookies', str(update_meta_config_gitcookies)]
+ with api.infra.MetadataFetch(
+ api, UPDATE_META_CONFIG_KEY, UPDATE_META_CONFIG_GITCOOKIES_FILE):
+ with api.context(cwd=api.vars.skia_dir):
+ api.run(api.step, 'Update meta/config', cmd=cmd)
+
+
+def GenTests(api):
+ builder = 'Housekeeper-Nightly-UpdateMetaConfig'
+ yield (
+ api.test(builder) +
+ api.properties(buildername=builder,
+ repository='https://skia.googlesource.com/skia.git',
+ revision='abc123',
+ path_config='kitchen',
+ swarm_out_dir='[SWARM_OUT_DIR]')
+ )
+
+ yield (
+ api.test('failed_update') +
+ api.properties(buildername=builder,
+ repository='https://skia.googlesource.com/skia.git',
+ revision='abc123',
+ path_config='kitchen',
+ swarm_out_dir='[SWARM_OUT_DIR]') +
+ api.step_data('Update meta/config', retcode=1)
+ )
+
+ yield (
+ api.test('trybot_test') +
+ api.properties(buildername=builder,
+ repository='https://skia.googlesource.com/skia.git',
+ revision='abc123',
+ path_config='kitchen',
+ swarm_out_dir='[SWARM_OUT_DIR]',
+ patch_issue='123',
+ patch_set='3') +
+ api.expect_exception('Exception')
+ )
],
"trigger": "nightly"
},
+ "Housekeeper-Nightly-UpdateMetaConfig": {
+ "priority": 0.8,
+ "tasks": [
+ "Housekeeper-Nightly-UpdateMetaConfig"
+ ],
+ "trigger": "nightly"
+ },
"Housekeeper-PerCommit": {
"priority": 0.8,
"tasks": [
"isolate": "compile_skia.isolate",
"priority": 0.8
},
+ "Housekeeper-Nightly-UpdateMetaConfig": {
+ "dimensions": [
+ "cpu:x86-64-avx2",
+ "gpu:none",
+ "os:Ubuntu-14.04",
+ "pool:Skia"
+ ],
+ "extra_args": [
+ "--workdir",
+ "../../..",
+ "update_meta_config",
+ "repository=<(REPO)",
+ "buildername=Housekeeper-Nightly-UpdateMetaConfig",
+ "swarm_out_dir=${ISOLATED_OUTDIR}",
+ "revision=<(REVISION)",
+ "patch_repo=<(PATCH_REPO)",
+ "patch_storage=<(PATCH_STORAGE)",
+ "patch_issue=<(ISSUE)",
+ "patch_set=<(PATCHSET)"
+ ],
+ "isolate": "meta_config.isolate",
+ "priority": 0.8
+ },
"Housekeeper-PerCommit": {
"cipd_packages": [
{
--- /dev/null
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Update meta/config of the specified Skia repo."""
+
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import urllib2
+
+import git_utils
+
+
+SKIA_COMMITTER_EMAIL = 'update-meta-config@skia.org'
+SKIA_COMMITTER_NAME = 'Update Meta Config'
+SKIA_REPO_TEMPLATE = 'https://skia.googlesource.com/%s.git'
+
+CQ_INCLUDE_CHROMIUM_TRYBOTS = (
+ 'master.tryserver.blink:linux_trusty_blink_rel,linux_trusty_blink_dbg;'
+ 'master.tryserver.chromium.linux:linux_optional_gpu_tests_rel;'
+ 'master.tryserver.chromium.mac:mac_optional_gpu_tests_rel;'
+ 'master.tryserver.chromium.win:win_optional_gpu_tests_rel;'
+ 'master.tryserver.chromium.android:android_optional_gpu_tests_rel'
+)
+
+
+def addChromiumTrybots(f):
+ for master_section in CQ_INCLUDE_CHROMIUM_TRYBOTS.split(';'):
+ master, bots = master_section.split(':')
+ f.write('[bucket "%s"]\n' % master)
+ for bot in bots.split(','):
+ f.write('\tbuilder = %s\n' % bot)
+
+
+def main(gitcookies, repo_name, tasks_json):
+ skia_repo = SKIA_REPO_TEMPLATE % repo_name
+ with git_utils.NewGitCheckout(repository=skia_repo):
+ # Fetch and checkout the meta/config branch.
+ subprocess.check_call(['git', 'fetch', skia_repo, 'refs/meta/config:cfg'])
+ subprocess.check_call(['git', 'checkout', 'cfg'])
+
+ # Create list of tryjobs from tasks_json.
+ tryjobs = []
+ with open(tasks_json) as tasks_json:
+ data = json.load(tasks_json)
+ for job in data['jobs'].keys():
+ if not job.startswith('Upload-'):
+ tryjobs.append(job)
+ tryjobs.sort()
+
+ # Write to buildbucket.config.
+ buildbucket_config = os.path.join(os.getcwd(), 'buildbucket.config')
+ with open(buildbucket_config, 'w') as f:
+
+ if repo_name == 'skia':
+ addChromiumTrybots(f)
+
+ # Adding all Skia jobs.
+ f.write('[bucket "skia.primary"]\n')
+ for job in tryjobs:
+ f.write('\tbuilder = ' + job + '\n')
+
+ # Push the change as the update-meta-config user.
+ config_dict = {
+ 'user.name': SKIA_COMMITTER_NAME,
+ 'user.email': SKIA_COMMITTER_EMAIL,
+ 'http.cookiefile': gitcookies,
+ }
+ with git_utils.GitLocalConfig(config_dict):
+ subprocess.check_call(['git', 'add', 'buildbucket.config'])
+ try:
+ subprocess.check_call(
+ ['git', 'commit', '-m', 'Update builders in buildbucket.config'])
+ except subprocess.CalledProcessError:
+ print 'No changes to buildbucket.config'
+ return
+
+ subprocess.check_call(['git', 'push', skia_repo, 'cfg:refs/meta/config'])
+
+
+if '__main__' == __name__:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--gitcookies")
+ parser.add_argument("--repo_name")
+ parser.add_argument("--tasks_json")
+ args = parser.parse_args()
+ main(args.gitcookies, args.repo_name, args.tasks_json)
projects / platform / upstream / libSkiaSharp.git / commitdiff