1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """This module contains constants used by cbuildbot and related code."""
10 """Try and find the root check out of the chromiumos tree"""
11 source_root = path = os.path.realpath(os.path.join(
12 os.path.abspath(__file__), '..', '..', '..'))
14 if os.path.isdir(os.path.join(path, '.repo')):
18 path = os.path.dirname(path)
21 SOURCE_ROOT = _FindSourceRoot()
22 CHROOT_SOURCE_ROOT = '/mnt/host/source'
24 CROSUTILS_DIR = os.path.join(SOURCE_ROOT, 'src/scripts')
25 CHROMITE_DIR = os.path.join(SOURCE_ROOT, 'chromite')
26 CHROMITE_BIN_SUBDIR = 'chromite/bin'
27 CHROMITE_BIN_DIR = os.path.join(SOURCE_ROOT, CHROMITE_BIN_SUBDIR)
28 PATH_TO_CBUILDBOT = os.path.join(CHROMITE_BIN_SUBDIR, 'cbuildbot')
29 DEFAULT_CHROOT_DIR = 'chroot'
30 SDK_TOOLCHAINS_OUTPUT = 'tmp/toolchain-pkgs'
31 AUTOTEST_BUILD_PATH = 'usr/local/build/autotest'
33 HOME_DIRECTORY = os.path.expanduser('~')
35 # If cbuiltbot is running on a bot, then the cidb access credentials will be
36 # available here. This directory will not exist otherwise.
37 CIDB_PROD_BOT_CREDS = os.path.join(HOME_DIRECTORY, '.cidb_creds',
39 CIDB_DEBUG_BOT_CREDS = os.path.join(HOME_DIRECTORY, '.cidb_creds',
43 # TODO: Eliminate these or merge with manifest_version.py:STATUS_PASSED
45 FINAL_STATUS_PASSED = 'passed'
46 FINAL_STATUS_FAILED = 'failed'
48 # Re-execution API constants.
49 # Used by --resume and --bootstrap to decipher which options they
50 # can pass to the target cbuildbot (since it may not have that
52 # Format is Major:Minor. Minor is used for tracking new options added
53 # that aren't critical to the older version if it's not ran.
54 # Major is used for tracking heavy API breakage- for example, no longer
55 # supporting the --resume option.
58 REEXEC_API_VERSION = '%i.%i' % (REEXEC_API_MAJOR, REEXEC_API_MINOR)
60 # Minor version 3 is the first to support --master-build-id
61 REEXEC_API_MASTER_BUILD_ID = 3
63 # We rely on the (waterfall, builder name, build number) to uniquely identify
64 # a build. However, future migrations or state wipes of the buildbot master may
65 # cause it to reset its build number counter. When that happens, this value
66 # should be incremented, ensuring that (waterfall, builder name, build number,
67 # buildbot generation) is a unique identifier of builds.
68 BUILDBOT_GENERATION = 1
70 ISOLATESERVER = 'https://isolateserver.appspot.com'
72 GOOGLE_EMAIL = '@google.com'
73 CHROMIUM_EMAIL = '@chromium.org'
75 CORP_DOMAIN = 'corp.google.com'
76 GOLO_DOMAIN = 'golo.chromium.org'
78 GOB_HOST = '%s.googlesource.com'
80 EXTERNAL_GOB_INSTANCE = 'chromium'
81 EXTERNAL_GERRIT_INSTANCE = 'chromium-review'
82 EXTERNAL_GOB_HOST = GOB_HOST % EXTERNAL_GOB_INSTANCE
83 EXTERNAL_GERRIT_HOST = GOB_HOST % EXTERNAL_GERRIT_INSTANCE
84 EXTERNAL_GOB_URL = 'https://%s' % EXTERNAL_GOB_HOST
85 EXTERNAL_GERRIT_URL = 'https://%s' % EXTERNAL_GERRIT_HOST
87 INTERNAL_GOB_INSTANCE = 'chrome-internal'
88 INTERNAL_GERRIT_INSTANCE = 'chrome-internal-review'
89 INTERNAL_GOB_HOST = GOB_HOST % INTERNAL_GOB_INSTANCE
90 INTERNAL_GERRIT_HOST = GOB_HOST % INTERNAL_GERRIT_INSTANCE
91 INTERNAL_GOB_URL = 'https://%s' % INTERNAL_GOB_HOST
92 INTERNAL_GERRIT_URL = 'https://%s' % INTERNAL_GERRIT_HOST
94 REPO_PROJECT = 'external/repo'
95 REPO_URL = '%s/%s' % (EXTERNAL_GOB_URL, REPO_PROJECT)
97 CHROMITE_PROJECT = 'chromiumos/chromite'
98 CHROMITE_URL = '%s/%s' % (EXTERNAL_GOB_URL, CHROMITE_PROJECT)
99 CHROMIUM_SRC_PROJECT = 'chromium/src'
100 CHROMIUM_GOB_URL = '%s/%s.git' % (EXTERNAL_GOB_URL, CHROMIUM_SRC_PROJECT)
102 MANIFEST_PROJECT = 'chromiumos/manifest'
103 MANIFEST_INT_PROJECT = 'chromeos/manifest-internal'
104 MANIFEST_PROJECTS = (MANIFEST_PROJECT, MANIFEST_INT_PROJECT)
106 MANIFEST_URL = '%s/%s' % (EXTERNAL_GOB_URL, MANIFEST_PROJECT)
107 MANIFEST_INT_URL = '%s/%s' % (INTERNAL_GERRIT_URL, MANIFEST_INT_PROJECT)
109 DEFAULT_MANIFEST = 'default.xml'
110 OFFICIAL_MANIFEST = 'official.xml'
111 SHARED_CACHE_ENVVAR = 'CROS_CACHEDIR'
113 # CrOS remotes specified in the manifests.
114 EXTERNAL_REMOTE = 'cros'
115 INTERNAL_REMOTE = 'cros-internal'
116 CHROMIUM_REMOTE = 'chromium'
117 CHROME_REMOTE = 'chrome'
120 EXTERNAL_REMOTE: EXTERNAL_GERRIT_HOST,
121 INTERNAL_REMOTE: INTERNAL_GERRIT_HOST,
125 EXTERNAL_REMOTE: EXTERNAL_GOB_URL,
126 INTERNAL_REMOTE: INTERNAL_GOB_URL,
130 CHROMIUM_REMOTE: EXTERNAL_GOB_URL,
131 CHROME_REMOTE: INTERNAL_GOB_URL,
133 GIT_REMOTES.update(CROS_REMOTES)
135 # Prefix to distinguish internal and external changes. This is used
136 # when user specifies a patch with "-g", when generating a key for
137 # a patch to used in our PatchCache, and when display a custom string
139 INTERNAL_CHANGE_PREFIX = '*'
140 EXTERNAL_CHANGE_PREFIX = ''
143 INTERNAL_REMOTE: INTERNAL_CHANGE_PREFIX,
144 EXTERNAL_REMOTE: EXTERNAL_CHANGE_PREFIX,
147 # List of remotes that are ok to include in the external manifest.
148 EXTERNAL_REMOTES = (EXTERNAL_REMOTE, CHROMIUM_REMOTE)
150 # Mapping 'remote name' -> regexp that matches names of repositories on that
151 # remote that can be branched when creating CrOS branch. Branching script will
152 # actually create a new git ref when branching these projects. It won't attempt
153 # to create a git ref for other projects that may be mentioned in a manifest.
154 BRANCHABLE_PROJECTS = {
155 EXTERNAL_REMOTE: r'chromiumos/(.+)',
156 INTERNAL_REMOTE: r'chromeos/(.+)',
159 # TODO(sosa): Move to manifest-versions-external once its created
160 MANIFEST_VERSIONS_SUFFIX = '/chromiumos/manifest-versions'
161 MANIFEST_VERSIONS_INT_SUFFIX = '/chromeos/manifest-versions'
162 MANIFEST_VERSIONS_GS_URL = 'gs://chromeos-manifest-versions'
163 TRASH_BUCKET = 'gs://chromeos-throw-away-bucket'
165 STREAK_COUNTERS = 'streak_counters'
167 PATCH_BRANCH = 'patch_branch'
168 STABLE_EBUILD_BRANCH = 'stabilizing_branch'
169 MERGE_BRANCH = 'merge_branch'
171 # These branches are deleted at the beginning of every buildbot run.
174 STABLE_EBUILD_BRANCH,
178 # Constants for uprevving Chrome
180 # Portage category and package name for Chrome.
181 CHROME_PN = 'chromeos-chrome'
182 CHROME_CP = 'chromeos-base/%s' % CHROME_PN
186 USE_CHROME_INTERNAL = 'chrome_internal'
187 USE_AFDO_USE = 'afdo_use'
190 # Builds and validates _alpha ebuilds. These builds sync to the latest
191 # revsion of the Chromium src tree and build with that checkout.
192 CHROME_REV_TOT = 'tot'
194 # Builds and validates chrome at a given revision through cbuildbot
196 CHROME_REV_SPEC = 'spec'
198 # Builds and validates the latest Chromium release as defined by
199 # ~/trunk/releases in the Chrome src tree. These ebuilds are suffixed with rc.
200 CHROME_REV_LATEST = 'latest_release'
202 # Builds and validates the latest Chromium release for a specific Chromium
203 # branch that we want to watch. These ebuilds are suffixed with rc.
204 CHROME_REV_STICKY = 'stable_release'
206 # Builds and validates Chromium for a pre-populated directory.
207 # Also uses _alpha, since portage doesn't have anything lower.
208 CHROME_REV_LOCAL = 'local'
209 VALID_CHROME_REVISIONS = [CHROME_REV_TOT, CHROME_REV_LATEST,
210 CHROME_REV_STICKY, CHROME_REV_LOCAL, CHROME_REV_SPEC]
213 # Build types supported.
215 # TODO(sosa): Deprecate PFQ type.
216 # Incremental builds that are built using binary packages when available.
217 # These builds have less validation than other build types.
218 INCREMENTAL_TYPE = 'binary'
220 # These builds serve as PFQ builders. This is being deprecated.
223 # Hybrid Commit and PFQ type. Ultimate protection. Commonly referred to
224 # as simply "commit queue" now.
225 PALADIN_TYPE = 'paladin'
227 # A builder that kicks off Pre-CQ builders that bless the purest CLs.
228 PRE_CQ_LAUNCHER_TYPE = 'priest'
230 # A builder that cuts and prunes branches.
231 CREATE_BRANCH_TYPE = 'gardener'
233 # Chrome PFQ type. Incremental build type that builds and validates new
234 # versions of Chrome. Only valid if set with CHROME_REV. See
235 # VALID_CHROME_REVISIONS for more information.
236 CHROME_PFQ_TYPE = 'chrome'
238 # Builds from source and non-incremental. This builds fully wipe their
239 # chroot before the start of every build and no not use a BINHOST.
240 BUILD_FROM_SOURCE_TYPE = 'full'
242 # Full but with versioned logic.
243 CANARY_TYPE = 'canary'
245 # Generate payloads for an already built build/version.
246 PAYLOADS_TYPE = 'payloads'
248 BRANCH_UTIL_CONFIG = 'branch-util'
250 # Special build type for Chroot builders. These builds focus on building
251 # toolchains and validate that they work.
252 CHROOT_BUILDER_TYPE = 'chroot'
253 CHROOT_BUILDER_BOARD = 'amd64-host'
255 # Build that refreshes the online Portage package status spreadsheet.
256 REFRESH_PACKAGES_TYPE = 'refresh_packages'
258 VALID_BUILD_TYPES = (
261 BUILD_FROM_SOURCE_TYPE,
264 CHROOT_BUILDER_BOARD,
267 PRE_CQ_LAUNCHER_TYPE,
268 REFRESH_PACKAGES_TYPE,
273 # The name of the builder used to launch the pre-CQ.
274 PRE_CQ_BUILDER_NAME = 'pre-cq-group'
276 # The name of the Pre-CQ launcher on the waterfall.
277 PRE_CQ_LAUNCHER_NAME = 'Pre-CQ Launcher'
279 # Define pool of machines for Hardware tests.
280 HWTEST_DEFAULT_NUM = 6
281 HWTEST_TRYBOT_NUM = 3
282 HWTEST_MACH_POOL = 'bvt'
283 HWTEST_PALADIN_POOL = 'cq'
284 HWTEST_TOT_PALADIN_POOL = 'tot-cq'
285 HWTEST_PFQ_POOL = 'pfq'
286 HWTEST_SUITES_POOL = 'suites'
287 HWTEST_CHROME_PERF_POOL = 'chromeperf'
288 HWTEST_TRYBOT_POOL = 'try-bot'
291 # Defines for the various hardware test suites:
292 # AU: Blocking suite run against all canaries; tests basic AU
294 # BVT: Basic blocking suite to be run against any build that
295 # requires a HWTest phase.
296 # COMMIT: Suite of basic tests required for commits to the source
297 # tree. Runs as a blocking suite on the CQ and PFQ; runs as
298 # a non-blocking suite on canaries.
299 # CANARY: Non-blocking suite run only against the canaries.
300 # AFDO: Non-blocking suite run only AFDO builders.
301 HWTEST_AU_SUITE = 'au'
302 HWTEST_BVT_SUITE = 'bvt-inline'
303 HWTEST_COMMIT_SUITE = 'bvt-cq'
304 HWTEST_CANARY_SUITE = 'bvt-perbuild'
305 HWTEST_AFDO_SUITE = 'AFDO_record'
308 # Additional timeout to wait for autotest to abort a suite if the test takes
309 # too long to run. This is meant to be overly conservative as a timeout may
310 # indicate that autotest is at capacity.
311 HWTEST_TIMEOUT_EXTENSION = 10 * 60
313 HWTEST_DEFAULT_PRIORITY = 'DEFAULT'
314 HWTEST_CQ_PRIORITY = 'CQ'
315 HWTEST_BUILD_PRIORITY = 'Build'
316 HWTEST_PFQ_PRIORITY = 'PFQ'
318 # Ordered by priority (first item being lowest).
319 HWTEST_VALID_PRIORITIES = ['Weekly',
322 HWTEST_DEFAULT_PRIORITY,
323 HWTEST_BUILD_PRIORITY,
327 # Creates a mapping of priorities to make easy comparsions.
328 HWTEST_PRIORITIES_MAP = dict(zip(HWTEST_VALID_PRIORITIES,
329 range(len(HWTEST_VALID_PRIORITIES))))
331 # Defines VM Test types.
332 FULL_AU_TEST_TYPE = 'full_suite'
333 SIMPLE_AU_TEST_TYPE = 'pfq_suite'
334 SMOKE_SUITE_TEST_TYPE = 'smoke_suite'
335 TELEMETRY_SUITE_TEST_TYPE = 'telemetry_suite'
336 CROS_VM_TEST_TYPE = 'cros_vm_test'
337 DEV_MODE_TEST_TYPE = 'dev_mode_test'
338 VALID_VM_TEST_TYPES = [FULL_AU_TEST_TYPE, SIMPLE_AU_TEST_TYPE,
339 SMOKE_SUITE_TEST_TYPE, TELEMETRY_SUITE_TEST_TYPE,
340 CROS_VM_TEST_TYPE, DEV_MODE_TEST_TYPE]
342 CHROMIUMOS_OVERLAY_DIR = 'src/third_party/chromiumos-overlay'
343 VERSION_FILE = os.path.join(CHROMIUMOS_OVERLAY_DIR,
344 'chromeos/config/chromeos_version.sh')
345 SDK_VERSION_FILE = os.path.join(CHROMIUMOS_OVERLAY_DIR,
346 'chromeos/binhost/host/sdk_version.conf')
347 SDK_GS_BUCKET = 'chromiumos-sdk'
352 BOTH_OVERLAYS = 'both'
353 PUBLIC_OVERLAYS = PUBLIC
354 PRIVATE_OVERLAYS = PRIVATE
355 VALID_OVERLAYS = [BOTH_OVERLAYS, PUBLIC_OVERLAYS, PRIVATE_OVERLAYS, None]
357 # Common default logging settings for use with the logging module.
358 LOGGER_FMT = '%(asctime)s: %(levelname)s: %(message)s'
359 LOGGER_DATE_FMT = '%H:%M:%S'
361 # Used by remote patch serialization/deserialzation.
362 INTERNAL_PATCH_TAG = 'i'
363 EXTERNAL_PATCH_TAG = 'e'
364 PATCH_TAGS = (INTERNAL_PATCH_TAG, EXTERNAL_PATCH_TAG)
366 # Tree status strings
368 TREE_THROTTLED = 'throttled'
369 TREE_CLOSED = 'closed'
370 VALID_TREE_STATUSES = (TREE_OPEN, TREE_THROTTLED, TREE_CLOSED)
372 _GERRIT_QUERY_TEMPLATE = ('status:open AND '
373 'label:Code-Review=+2 AND '
374 'label:Verified=+1 AND '
375 'label:Commit-Queue>=%+i AND '
376 'NOT ( label:CodeReview=-2 OR label:Verified=-1 OR '
379 # Default gerrit query used to find changes for CQ.
380 # Permits CQ+1 or CQ+2 changes.
381 DEFAULT_CQ_READY_QUERY = _GERRIT_QUERY_TEMPLATE % 1
383 # Gerrit query used to find changes for CQ when tree is throttled.
384 # Permits only CQ+2 changes.
385 THROTTLED_CQ_READY_QUERY = _GERRIT_QUERY_TEMPLATE % 2
387 # Default filter rules for verifying that Gerrit returned results that matched
388 # our query. This used for working around Gerrit bugs.
389 DEFAULT_CQ_READY_FIELDS = {
395 DEFAULT_CQ_SHOULD_REJECT_FIELDS = {
400 GERRIT_ON_BORG_LABELS = {
401 'Code-Review': 'CRVW',
402 'Commit-Queue': 'COMR',
404 'Trybot-Verified': 'TBVF',
407 # Actions that a CQ run can take on a CL
408 CL_ACTION_PICKED_UP = 'picked_up' # CL picked up in CommitQueueSync
409 CL_ACTION_SUBMITTED = 'submitted' # CL submitted successfully
410 CL_ACTION_KICKED_OUT = 'kicked_out' # CL CQ-Ready value set to zero
411 CL_ACTION_SUBMIT_FAILED = 'submit_failed' # CL submitted but submit failed
413 CL_ACTIONS = [CL_ACTION_PICKED_UP,
415 CL_ACTION_KICKED_OUT,
416 CL_ACTION_SUBMIT_FAILED]
422 # Environment variables that should be exposed to all children processes
423 # invoked via cros_build_lib.RunCommand.
424 ENV_PASSTHRU = ('CROS_SUDO_KEEP_ALIVE', SHARED_CACHE_ENVVAR)
426 # List of variables to proxy into the chroot from the host, and to
427 # have sudo export if existent. Anytime this list is modified, a new
428 # chroot_version_hooks.d upgrade script that symlinks to 45_rewrite_sudoers.d
430 CHROOT_ENVIRONMENT_WHITELIST = (
432 'CHROMEOS_VERSION_AUSERVER',
433 'CHROMEOS_VERSION_DEVSERVER',
434 'CHROMEOS_VERSION_TRACK',
438 'GIT_COMMITTER_EMAIL',
439 'GIT_COMMITTER_NAME',
453 # Paths for Chrome LKGM which are relative to the Chromium base url.
454 CHROME_LKGM_FILE = 'CHROMEOS_LKGM'
455 PATH_TO_CHROME_LKGM = 'chromeos/%s' % CHROME_LKGM_FILE
456 SVN_CHROME_LKGM = 'trunk/src/%s' % PATH_TO_CHROME_LKGM
459 COMMON_CACHE = 'common'
461 # Artifact constants.
462 def _SlashToUnderscore(string):
463 return string.replace('/', '_')
465 DEFAULT_ARCHIVE_BUCKET = 'gs://chromeos-image-archive'
466 RELEASE_BUCKET = 'gs://chromeos-releases'
467 TRASH_BUCKET = 'gs://chromeos-throw-away-bucket'
468 CHROME_SYSROOT_TAR = 'sysroot_%s.tar.xz' % _SlashToUnderscore(CHROME_CP)
469 CHROME_ENV_TAR = 'environment_%s.tar.xz' % _SlashToUnderscore(CHROME_CP)
470 CHROME_ENV_FILE = 'environment'
471 BASE_IMAGE_NAME = 'chromiumos_base_image'
472 BASE_IMAGE_TAR = '%s.tar.xz' % BASE_IMAGE_NAME
473 BASE_IMAGE_BIN = '%s.bin' % BASE_IMAGE_NAME
474 IMAGE_SCRIPTS_NAME = 'image_scripts'
475 IMAGE_SCRIPTS_TAR = '%s.tar.xz' % IMAGE_SCRIPTS_NAME
476 VM_IMAGE_NAME = 'chromiumos_qemu_image'
477 VM_IMAGE_BIN = '%s.bin' % VM_IMAGE_NAME
478 VM_DISK_PREFIX = 'chromiumos_qemu_disk.bin'
479 VM_MEM_PREFIX = 'chromiumos_qemu_mem.bin'
480 VM_TEST_RESULTS = 'vm_test_results_%(attempt)s'
482 METADATA_JSON = 'metadata.json'
483 PARTIAL_METADATA_JSON = 'partial-metadata.json'
484 DELTA_SYSROOT_TAR = 'delta_sysroot.tar.xz'
485 DELTA_SYSROOT_BATCH = 'batch'
487 # Global configuration constants.
488 CHROMITE_CONFIG_DIR = os.path.expanduser('~/.chromite')
489 CHROME_SDK_BASHRC = os.path.join(CHROMITE_CONFIG_DIR, 'chrome_sdk.bashrc')
494 LAB_STATUS_URL = 'http://chromiumos-lab.appspot.com/current?format=json'
496 GOLO_SMTP_SERVER = 'mail.golo.chromium.org'
498 # URLs to the various waterfalls.
499 BUILD_DASHBOARD = 'http://build.chromium.org/p/chromiumos'
500 BUILD_INT_DASHBOARD = 'https://uberchromegw.corp.google.com/i/chromeos'
501 TRYBOT_DASHBOARD = 'https://uberchromegw.corp.google.com/i/chromiumos.tryserver'
503 # Useful config targets.
504 CQ_MASTER = 'master-paladin'
505 PRE_CQ_GROUP = 'trybot-pre-cq-group'
507 # Email validation regex. Not quite fully compliant with RFC 2822, but good
509 EMAIL_REGEX = r'[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4}'
511 # Blacklist of files not allowed to be uploaded into the Partner Project Google
513 # debug.tgz contains debug symbols.
514 # manifest.xml exposes all of our repo names.
515 # vm_test_results can contain symbolicated crash dumps.
516 EXTRA_BUCKETS_FILES_BLACKLIST = [
522 # AFDO common constants.
523 # How long does the AFDO_record autotest have to generate the AFDO perf data.
524 AFDO_GENERATE_TIMEOUT = 90 * 60