From: Robert Iannucci Date: Tue, 9 May 2017 18:06:48 +0000 (-0700) Subject: Manually roll recipes. X-Git-Tag: accepted/tizen/5.0/unified/20181102.025319~36^2~354 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=3734c7d9e39cce682c5095a8f187d41223955f5c;p=platform%2Fupstream%2FlibSkiaSharp.git Manually roll recipes. Include new --package option for recipes.py. build: https://crrev.com/380216596a663dc45d6562b29bce3719e2620ac0 Bump bot environment "vpython". (dnj@chromium.org) https://crrev.com/226d6eadceee58fe6832679e39a040a46d5df989 Bump Kitchen canary, clear PYTHONPATH. (dnj@chromium.org) https://crrev.com/45f9beb12883c6b509c699c1dd1694be218c4740 Write uploaded log url to file in goma module (tikuta@google.com) https://crrev.com/5c6b5807ee3de9277b8c6e3182f621e4216e2eea WebRTC: Rename tools-webrtc -> tools_webrtc (kjellander@chromium.org) https://crrev.com/3090e9f2c42f7254fc9b42b21ed792e19d77e667 Manually roll depot_tools 69a239e:e2f9fee (phajdan.jr@chromium.org) https://crrev.com/e845e79d0cd39af876b4fff648c68bd023633c8a chromium_checkout: add non-fatal gclient validate step (phajdan.jr@chromium.org) https://crrev.com/3bed2fc03bc435464880411740b20031f80ed41c Remove old GYP compatibility targets from chromium_tests. (RELAND) (jbudorick@chromium.org) https://crrev.com/9b44962a1bfb8134ffac7f5606a11cb8ff8489e8 Flutter: Rev Android build tools to 23.0.3 (mit@google.com) https://crrev.com/6fa4d503d7ef77a951b446ca01b5b18a3c9fb79c remote_run: promote Kitchen canary to stable. (dnj@chromium.org) depot_tools: https://crrev.com/4ebaaf9d9c1e24548f3c9f9f748e8631d091ecd0 Bump "vpython" version. (dnj@chromium.org) https://crrev.com/0bbd1c28d5827379f363f2a9170eeda27fd7b758 git-cl: upload to merge-base with master, not tip of origin/master (agable@chromium.org) https://crrev.com/e2f9feecaf4326cef699624d5f52b9f31ecbc104 Add validate command to gclient (phajdan.jr@chromium.org) recipe_engine: https://crrev.com/66338449f4d727da096422c1df1b6654224d6512 [bootstrap] don't change directories when bootstrapping. (iannucci@chromium.org) https://crrev.com/487e5e371bf8140b0467c6366f94042581f5c63c [recipes.py] refactor PackageContext to remove unused --deps-path arg. (iannucci@chromium.org) https://crrev.com/92c5b73ff81fb30c80a33bab91be51551d5e67b7 Bump "vpython" bootstrap version. (dnj@chromium.org) https://crrev.com/d6c4a597be2da8be7971b4d5f66e3d95e8fc325c [recipes.py] refactor PackageContext to remove --no-fetch logic. (iannucci@chromium.org) https://crrev.com/cfeeb53051e02856a35395a69b09f822ad35e2e5 [doc/recipes.py] Improve copypasta bootstrap script. (iannucci@chromium.org) https://crrev.com/e90c11af854574dd68edd0e7c7a5e04ceb49ffd0 [recipes.cfg] remove support for api_version 1. (iannucci@chromium.org) https://crrev.com/497f5b7f92cea8e5273d56cc3cba876fb18b42d9 [doc/recipes.py] fix issue when running recipes.py from . (iannucci@chromium.org) https://crrev.com/e7bbbf6cfac944157d80cc92fd78837165e69d3f Fix a race crash when recipe expectations directory exists (phajdan.jr@chromium.org) https://crrev.com/bf2ff828987549b946135919bd1bffd6e42fd8fd [doc/recipes.py] Add option to recipes.py to allow it to work without .git (iannucci@chromium.org) R=borenet@google.com Bug: skia: Change-Id: Ic7c20a7beb6165ea79194053016759722fea6a51 Reviewed-on: https://skia-review.googlesource.com/16201 Reviewed-by: Eric Boren Commit-Queue: Eric Boren --- diff --git a/infra/bots/bundle_recipes.isolate b/infra/bots/bundle_recipes.isolate index d873fc3..3faa6f7 100644 --- a/infra/bots/bundle_recipes.isolate +++ b/infra/bots/bundle_recipes.isolate @@ -4,7 +4,7 @@ ], 'variables': { 'command': [ - 'python', 'recipes.py', 'run', '--timestamps', + 'python', 'recipes.py', '--package', '../config/recipes.cfg', 'run', '--timestamps', ], 'files': [ '../../../.gclient', diff --git a/infra/bots/recipes.py b/infra/bots/recipes.py index 7bb2c1f..2bd462a 100755 --- a/infra/bots/recipes.py +++ b/infra/bots/recipes.py @@ -1,27 +1,21 @@ #!/usr/bin/env python -# Copyright 2016 The LUCI Authors. All rights reserved. +# Copyright 2017 The LUCI Authors. All rights reserved. # Use of this source code is governed under the Apache License, Version 2.0 # that can be found in the LICENSE file. """Bootstrap script to clone and forward to the recipe engine tool. -*********************************************************************** -** DO NOT MODIFY EXCEPT IN THE PER-REPO CONFIGURATION SECTION BELOW. ** -*********************************************************************** +******************* +** DO NOT MODIFY ** +******************* This is a copy of https://github.com/luci/recipes-py/blob/master/doc/recipes.py. -To fix bugs, fix in the github repo then copy it back to here and fix the -PER-REPO CONFIGURATION section to look like this one. +To fix bugs, fix in the github repo then run the autoroller. """ import os -# IMPORTANT: Do not alter the header or footer line for the -# "PER-REPO CONFIGURATION" section below, or the autoroller will not be able -# to automatically update this file! All lines between the header and footer -# lines will be retained verbatim by the autoroller. - #### PER-REPO CONFIGURATION (editable) #### # The root of the repository relative to the directory of this file. REPO_ROOT = os.path.join(os.pardir, os.pardir) @@ -29,8 +23,6 @@ REPO_ROOT = os.path.join(os.pardir, os.pardir) RECIPES_CFG = os.path.join('infra', 'config', 'recipes.cfg') #### END PER-REPO CONFIGURATION #### -BOOTSTRAP_VERSION = 1 - import argparse import json import logging @@ -40,12 +32,34 @@ import sys import time import urlparse +from collections import namedtuple + from cStringIO import StringIO +# The dependency entry for the recipe_engine in the client repo's recipes.cfg +# +# url (str) - the url to the engine repo we want to use. +# revision (str) - the git revision for the engine to get. +# path_override (str) - the subdirectory in the engine repo we should use to +# find it's recipes.py entrypoint. This is here for completeness, but will +# essentially always be empty. It would be used if the recipes-py repo was +# merged as a subdirectory of some other repo and you depended on that +# subdirectory. +# branch (str) - the branch to fetch for the engine as an absolute ref (e.g. +# refs/heads/master) +# repo_type ("GIT"|"GITILES") - An ignored enum which will be removed soon. +EngineDep = namedtuple('EngineDep', + 'url revision path_override branch repo_type') + + +class MalformedRecipesCfg(Exception): + def __init__(self, msg, path): + super(MalformedRecipesCfg, self).__init__('malformed recipes.cfg: %s: %r' + % (msg, path)) + def parse(repo_root, recipes_cfg_path): - """Parse is transitional code which parses a recipes.cfg file as either jsonpb - or as textpb. + """Parse is a lightweight a recipes.cfg file parser. Args: repo_root (str) - native path to the root of the repo we're trying to run @@ -53,13 +67,7 @@ def parse(repo_root, recipes_cfg_path): recipes_cfg_path (str) - native path to the recipes.cfg file to process. Returns (as tuple): - engine_url (str) - the url to the engine repo we want to use. - engine_revision (str) - the git revision for the engine to get. - engine_subpath (str) - the subdirectory in the engine repo we should use to - find it's recipes.py entrypoint. This is here for completeness, but will - essentially always be empty. It would be used if the recipes-py repo was - merged as a subdirectory of some other repo and you depended on that - subdirectory. + engine_dep (EngineDep): The recipe_engine dependency. recipes_path (str) - native path to where the recipes live inside of the current repo (i.e. the folder containing `recipes/` and/or `recipe_modules`) @@ -67,22 +75,41 @@ def parse(repo_root, recipes_cfg_path): with open(recipes_cfg_path, 'rU') as fh: pb = json.load(fh) - if pb['api_version'] == 1: - # TODO(iannucci): remove when we only support version 2 - engine = next( - (d for d in pb['deps'] if d['project_id'] == 'recipe_engine'), None) - if engine is None: - raise ValueError('could not find recipe_engine dep in %r' - % recipes_cfg_path) - else: + try: + if pb['api_version'] != 2: + raise MalformedRecipesCfg('unknown version %d' % pb['api_version'], + recipes_cfg_path) + engine = pb['deps']['recipe_engine'] - engine_url = engine['url'] - engine_revision = engine.get('revision', '') - engine_subpath = engine.get('path_override', '') - recipes_path = pb.get('recipes_path', '') - recipes_path = os.path.join(repo_root, recipes_path.replace('/', os.path.sep)) - return engine_url, engine_revision, engine_subpath, recipes_path + if 'url' not in engine: + raise MalformedRecipesCfg( + 'Required field "url" in dependency "recipe_engine" not found', + recipes_cfg_path) + + engine.setdefault('revision', '') + engine.setdefault('path_override', '') + engine.setdefault('branch', 'refs/heads/master') + recipes_path = pb.get('recipes_path', '') + + # TODO(iannucci): only support absolute refs + if not engine['branch'].startswith('refs/'): + engine['branch'] = 'refs/heads/' + engine['branch'] + + engine.setdefault('repo_type', 'GIT') + if engine['repo_type'] not in ('GIT', 'GITILES'): + raise MalformedRecipesCfg( + 'Unsupported "repo_type" value in dependency "recipe_engine"', + recipes_cfg_path) + + recipes_path = os.path.join( + repo_root, recipes_path.replace('/', os.path.sep)) + return EngineDep(**engine), recipes_path + except KeyError as ex: + raise MalformedRecipesCfg(ex.message, recipes_cfg_path) + + +GIT = 'git.bat' if sys.platform.startswith(('win', 'cygwin')) else 'git' def _subprocess_call(argv, **kwargs): @@ -90,86 +117,99 @@ def _subprocess_call(argv, **kwargs): return subprocess.call(argv, **kwargs) -def _subprocess_check_call(argv, **kwargs): +def _git_check_call(argv, **kwargs): + argv = [GIT]+argv logging.info('Running %r', argv) subprocess.check_call(argv, **kwargs) -def find_engine_override(argv): - """Since the bootstrap process attempts to defer all logic to the recipes-py - repo, we need to be aware if the user is overriding the recipe_engine - dependency. This looks for and returns the overridden recipe_engine path, if - any, or None if the user didn't override it.""" +def _git_output(argv, **kwargs): + argv = [GIT]+argv + logging.info('Running %r', argv) + return subprocess.check_output(argv, **kwargs) + + +def parse_args(argv): + """This extracts a subset of the arguments that this bootstrap script cares + about. Currently this consists of: + * an override for the recipe engine in the form of `-O recipe_engin=/path` + * the --package option. + """ PREFIX = 'recipe_engine=' p = argparse.ArgumentParser(add_help=False) p.add_argument('-O', '--project-override', action='append') + p.add_argument('--package', type=os.path.abspath) args, _ = p.parse_known_args(argv) for override in args.project_override or (): if override.startswith(PREFIX): - return override[len(PREFIX):] - return None + return override[len(PREFIX):], args.package + return None, args.package -def main(): - if '--verbose' in sys.argv: - logging.getLogger().setLevel(logging.INFO) +def checkout_engine(engine_path, repo_root, recipes_cfg_path): + dep, recipes_path = parse(repo_root, recipes_cfg_path) - if REPO_ROOT is None or RECIPES_CFG is None: - logging.error( - 'In order to use this script, please copy it to your repo and ' - 'replace the REPO_ROOT and RECIPES_CFG values with approprite paths.') - sys.exit(1) + url = dep.url - if sys.platform.startswith(('win', 'cygwin')): - git = 'git.bat' - else: - git = 'git' + if not engine_path and url.startswith('file://'): + engine_path = urlparse.urlparse(url).path - # Find the repository and config file to operate on. - repo_root = os.path.abspath( - os.path.join(os.path.dirname(__file__), REPO_ROOT)) - recipes_cfg_path = os.path.join(repo_root, RECIPES_CFG) + if not engine_path: + revision = dep.revision + subpath = dep.path_override + branch = dep.branch - engine_url, engine_revision, engine_subpath, recipes_path = parse( - repo_root, recipes_cfg_path) + # Ensure that we have the recipe engine cloned. + engine = os.path.join(recipes_path, '.recipe_deps', 'recipe_engine') + engine_path = os.path.join(engine, subpath) - engine_path = find_engine_override(sys.argv[1:]) - if not engine_path and engine_url.startswith('file://'): - engine_path = urlparse.urlparse(engine_url).path + with open(os.devnull, 'w') as NUL: + # Note: this logic mirrors the logic in recipe_engine/fetch.py + _git_check_call(['init', engine], stdout=NUL) - if not engine_path: - deps_path = os.path.join(recipes_path, '.recipe_deps') - # Ensure that we have the recipe engine cloned. - engine_root_path = os.path.join(deps_path, 'recipe_engine') - engine_path = os.path.join(engine_root_path, engine_subpath) - def ensure_engine(): - if not os.path.exists(deps_path): - os.makedirs(deps_path) - if not os.path.exists(engine_root_path): - _subprocess_check_call([git, 'clone', engine_url, engine_root_path]) - - needs_fetch = _subprocess_call( - [git, 'rev-parse', '--verify', '%s^{commit}' % engine_revision], - cwd=engine_root_path, stdout=open(os.devnull, 'w')) - if needs_fetch: - _subprocess_check_call([git, 'fetch'], cwd=engine_root_path) - _subprocess_check_call( - [git, 'checkout', '--quiet', engine_revision], cwd=engine_root_path) + try: + _git_check_call(['rev-parse', '--verify', '%s^{commit}' % revision], + cwd=engine, stdout=NUL, stderr=NUL) + except subprocess.CalledProcessError: + _git_check_call(['fetch', url, branch], cwd=engine, stdout=NUL, + stderr=NUL) try: - ensure_engine() + _git_check_call(['diff', '--quiet', revision], cwd=engine) except subprocess.CalledProcessError: - logging.exception('ensure_engine failed') + _git_check_call(['reset', '-q', '--hard', revision], cwd=engine) + + return engine_path - # Retry errors. - time.sleep(random.uniform(2,5)) - ensure_engine() - args = ['--package', recipes_cfg_path] + sys.argv[1:] +def main(): + if '--verbose' in sys.argv: + logging.getLogger().setLevel(logging.INFO) + + args = sys.argv[1:] + engine_override, recipes_cfg_path = parse_args(args) + + if recipes_cfg_path: + # calculate repo_root from recipes_cfg_path + repo_root = os.path.dirname( + os.path.dirname( + os.path.dirname(recipes_cfg_path))) + else: + # find repo_root with git and calculate recipes_cfg_path + repo_root = (_git_output( + ['rev-parse', '--show-toplevel'], + cwd=os.path.abspath(os.path.dirname(__file__))).strip()) + repo_root = os.path.abspath(repo_root) + recipes_cfg_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg') + args = ['--package', recipes_cfg_path] + args + + engine_path = checkout_engine(engine_override, repo_root, recipes_cfg_path) + return _subprocess_call([ sys.executable, '-u', os.path.join(engine_path, 'recipes.py')] + args) + if __name__ == '__main__': sys.exit(main()) diff --git a/infra/bots/swarm_recipe.isolate b/infra/bots/swarm_recipe.isolate index 9e8f516..c21a321 100644 --- a/infra/bots/swarm_recipe.isolate +++ b/infra/bots/swarm_recipe.isolate @@ -4,7 +4,7 @@ ], 'variables': { 'command': [ - 'python', 'recipes.py', 'run', '--timestamps', + 'python', 'recipes.py', '--package', '../config/recipes.cfg', 'run', '--timestamps', ], 'files': [ '../config/recipes.cfg', diff --git a/infra/config/recipes.cfg b/infra/config/recipes.cfg index cbcc5dc..32a15e0 100644 --- a/infra/config/recipes.cfg +++ b/infra/config/recipes.cfg @@ -14,17 +14,17 @@ "deps": { "build": { "branch": "master", - "revision": "671dc2edd993a46cb4d34621ed5e4c01b269d3a8", + "revision": "8aaa8f98e2b99de8419261adb5efc89a220a7a6a", "url": "https://chromium.googlesource.com/chromium/tools/build.git" }, "depot_tools": { "branch": "master", - "revision": "b4a79690367881c427c9c5adf614823a01fc9990", + "revision": "070c2e3eee08c7df7bcb3b36d53173050813865b", "url": "https://chromium.googlesource.com/chromium/tools/depot_tools.git" }, "recipe_engine": { "branch": "master", - "revision": "5cdf9803d55420f3ae8d2e0dd524bc2de9c7284b", + "revision": "bf2ff828987549b946135919bd1bffd6e42fd8fd", "url": "https://chromium.googlesource.com/external/github.com/luci/recipes-py.git" } },