import os
import re
import tempfile
-import time
from xml.dom import minidom
from chromite.cbuildbot import cbuildbot_config
from chromite.cbuildbot import manifest_version
from chromite.lib import cros_build_lib
from chromite.lib import git
+from chromite.lib import timeout_util
# Paladin constants for manifest names.
PALADIN_PASS_COUNT_ATTR = 'pass_count'
PALADIN_TOTAL_FAIL_COUNT_ATTR = 'total_fail_count'
+CHROME_ELEMENT = 'chrome'
+CHROME_VERSION_ATTR = 'version'
MANIFEST_ELEMENT = 'manifest'
DEFAULT_ELEMENT = 'default'
Vars:
lkgm_subdir: Subdirectory within manifest repo to store candidates.
"""
- # Max timeout before assuming other builders have failed.
- LONG_MAX_TIMEOUT_SECONDS = 1200
-
- # Max timeout before assuming other builders have failed for Chrome PFQ.
- # Longer as there is little to lose for Chrome PFQ waiting and arm
- # has been slower often.
- CHROME_LONG_MAX_TIMEOUT_SECONDS = 3 * 60 * 60
- # Max timeout before assuming other builders have failed.
- MAX_TIMEOUT_SECONDS = 300
- # Polling timeout for checking git repo for other build statuses.
- SLEEP_TIMEOUT = constants.SLEEP_TIMEOUT
-
# Sub-directories for LKGM and Chrome LKGM's.
LKGM_SUBDIR = 'LKGM-candidates'
CHROME_PFQ_SUBDIR = 'chrome-LKGM-candidates'
assert cbuildbot_config.IsPFQType(self.build_type)
self.rel_working_dir = self.LKGM_SUBDIR
- def _GetMaxLongTimeout(self):
- """Get the long "max timeout" for this builder.
-
- Returns:
- Timeout in seconds.
- """
- if self.build_type == constants.PFQ_TYPE:
- return self.LONG_MAX_TIMEOUT_SECONDS
- else:
- return self.CHROME_LONG_MAX_TIMEOUT_SECONDS
-
- def _RunLambdaWithTimeout(self, function_to_run, max_timeout):
- """Runs function_to_run until it returns a value or timeout is reached.
-
- function_to_run is called with one argument equal to the number of seconds
- left until timing out.
-
- Args:
- function_to_run: Function to run until it returns a value or the timeout
- is reached.
- max_timeout: Time to wait for function_to_run to return a value.
- use_long_timeout: Adjusts the timeout to use. See logic below.
- """
- function_success = False
- start_time = time.time()
-
- # Monitor the repo until all builders report in or we've waited too long.
- seconds_left = max_timeout - (time.time() - start_time)
- while seconds_left > 0:
- function_success = function_to_run(seconds_left)
- if function_success:
- break
- else:
- time.sleep(self.SLEEP_TIMEOUT)
-
- seconds_left = max_timeout - (time.time() - start_time)
-
- return function_success
-
def GetCurrentVersionInfo(self):
"""Returns the lkgm version info from the version file."""
version_info = super(LKGMManager, self).GetCurrentVersionInfo()
chrome_branch=version_info.chrome_branch,
incr_type=self.incr_type)
+ def _AddChromeVersionToManifest(self, manifest, chrome_version):
+ """Adds the chrome element with version |chrome_version| to |manifest|.
+
+ The manifest file should contain the Chrome version to build for
+ PFQ slaves.
+
+ Args:
+ manifest: Path to the manifest
+ chrome_version: A string representing the version of Chrome
+ (e.g. 35.0.1863.0).
+ """
+ manifest_dom = minidom.parse(manifest)
+ chrome = manifest_dom.createElement(CHROME_ELEMENT)
+ chrome.setAttribute(CHROME_VERSION_ATTR, chrome_version)
+ manifest_dom.documentElement.appendChild(chrome)
+ with open(manifest, 'w+') as manifest_file:
+ manifest_dom.writexml(manifest_file)
+
def _AddPatchesToManifest(self, manifest, patches):
"""Adds list of |patches| to given |manifest|.
return new_path
def CreateNewCandidate(self, validation_pool=None,
- retries=manifest_version.NUM_RETRIES):
+ chrome_version=None,
+ retries=manifest_version.NUM_RETRIES,
+ build_id=None):
"""Creates, syncs to, and returns the next candidate manifest.
Args:
validation_pool: Validation pool to apply to the manifest before
publishing.
- retries: Number of retries for updating the status.
+ chrome_version: The Chrome version to write in the manifest. Defaults
+ to None, in which case no version is written.
+ retries: Number of retries for updating the status. Defaults to
+ manifest_version.NUM_RETRIES.
+ build_id: Optional integer cidb id of the build that is creating
+ this candidate.
Raises:
GenerateBuildSpecException in case of failure to generate a buildspec
self._GenerateBlameListSinceLKGM()
new_manifest = self.CreateManifest()
+
+ # For Chrome PFQ, add the version of Chrome to use.
+ if chrome_version:
+ self._AddChromeVersionToManifest(new_manifest, chrome_version)
+
# For the Commit Queue, apply the validation pool as part of checkout.
if validation_pool:
# If we have nothing that could apply from the validation pool and
git.CreatePushBranch(manifest_version.PUSH_BRANCH, self.manifest_dir,
sync=False)
version = self.GetNextVersion(version_info)
- self.PublishManifest(new_manifest, version)
+ self.PublishManifest(new_manifest, version, build_id=build_id)
self.current_version = version
return self.GetLocalManifest(version)
except cros_build_lib.RunCommandError as e:
raise manifest_version.GenerateBuildSpecException(last_error)
def CreateFromManifest(self, manifest, retries=manifest_version.NUM_RETRIES,
- dashboard_url=None):
+ dashboard_url=None, build_id=None):
"""Sets up an lkgm_manager from the given manifest.
This method sets up an LKGM manager and publishes a new manifest to the
i.e R20-1920.0.1-rc7.xml where R20-1920.0.1-rc7 is the version.
retries: Number of retries for updating the status.
dashboard_url: Optional url linking to builder dashboard for this build.
+ build_id: Optional integer cidb build id of the build publishing the
+ manifest.
Raises:
GenerateBuildSpecException in case of failure to check-in the new
sync=False)
version = os.path.splitext(os.path.basename(manifest))[0]
logging.info('Publishing filtered build spec')
- self.PublishManifest(new_manifest, version)
+ self.PublishManifest(new_manifest, version, build_id=build_id)
self.SetInFlight(version, dashboard_url=dashboard_url)
self.current_version = version
return self.GetLocalManifest(version)
else:
raise manifest_version.GenerateBuildSpecException(last_error)
- def GetLatestCandidate(self, dashboard_url=None):
+ def GetLatestCandidate(self, dashboard_url=None, timeout=3 * 60):
"""Gets and syncs to the next candiate manifest.
Args:
retries: Number of retries for updating the status
dashboard_url: Optional url linking to builder dashboard for this build.
+ timeout: The timeout in seconds.
Returns:
Local path to manifest to build or None in case of no need to build.
Raises:
GenerateBuildSpecException in case of failure to generate a buildspec
"""
- def _AttemptToGetLatestCandidate(seconds_left):
+ def _AttemptToGetLatestCandidate():
"""Attempts to acquire latest candidate using manifest repo."""
self.RefreshManifestCheckout()
self.InitializeManifestVariables(self.GetCurrentVersionInfo())
return self.latest_unprocessed
elif self.dry_run and self.latest:
return self.latest
- else:
- minutes_left = int((seconds_left / 60) + 0.5)
- logging.info('Found nothing new to build, will keep trying for %d more'
- ' minutes.', minutes_left)
- logging.info('If this is a PFQ, then you should have forced the master'
- ', which runs cbuildbot_master')
- return None
+
+ def _PrintRemainingTime(minutes_left):
+ logging.info('Found nothing new to build, will keep trying for %d more'
+ ' minutes.', minutes_left)
+ logging.info('If this is a PFQ, then you should have forced the master'
+ ', which runs cbuildbot_master')
# TODO(sosa): We only really need the overlay for the version info but we
# do a full checkout here because we have no way of refining it currently.
self.CheckoutSourceCode()
- version_to_build = self._RunLambdaWithTimeout(
- _AttemptToGetLatestCandidate, self._GetMaxLongTimeout())
+ try:
+ version_to_build = timeout_util.WaitForSuccess(
+ lambda x: x is None,
+ _AttemptToGetLatestCandidate,
+ timeout,
+ period=self.SLEEP_TIMEOUT,
+ side_effect_func=_PrintRemainingTime)
+ except timeout_util.TimeoutError:
+ version_to_build = None
if version_to_build:
logging.info('Starting build spec: %s', version_to_build)
else:
return None
- def GetBuildersStatus(self, builders_array, wait_for_results=True):
- """Returns a build-names->status dictionary of build statuses.
-
- Args:
- builders_array: A list of the names of the builders to check.
- wait_for_results: If True, keep trying until all builder statuses are
- available or until timeout is reached.
- """
- builders_completed = set()
- builder_statuses = {}
-
- def _CheckStatusOfBuildersArray(seconds_left):
- """Helper function that iterates through current statuses."""
- for builder_name in builders_array:
- cached_status = builder_statuses.get(builder_name)
- if not cached_status or not cached_status.Completed():
- logging.debug("Checking for builder %s's status", builder_name)
- builder_status = self.GetBuildStatus(builder_name,
- self.current_version)
- builder_statuses[builder_name] = builder_status
- if builder_status.Missing():
- logging.warn('No status found for builder %s.', builder_name)
- elif builder_status.Completed():
- builders_completed.add(builder_name)
- logging.info('Builder %s completed with status "%s".',
- builder_name, builder_status.status)
-
- if len(builders_completed) < len(builders_array):
- minutes_left = int((seconds_left / 60) + 0.5)
- logging.info('Still waiting (up to %d more minutes) for the following'
- ' builds to complete: %r', minutes_left,
- sorted(set(builders_array).difference(builders_completed)))
- return None
- else:
- return 'Builds completed.'
-
- # Even if we do not want to wait for results long, wait a little bit
- # just to exercise the same code.
- max_timeout = self._GetMaxLongTimeout() if wait_for_results else 3 * 60
-
- # Check for build completion until all builders report in.
- builds_succeeded = self._RunLambdaWithTimeout(
- _CheckStatusOfBuildersArray, max_timeout)
- if not builds_succeeded:
- logging.error('Not all builds finished before timeout (%d minutes)'
- ' reached.', int((max_timeout / 60) + 0.5))
-
- return builder_statuses
-
def PromoteCandidate(self, retries=manifest_version.NUM_RETRIES):
"""Promotes the current LKGM candidate to be a real versioned LKGM."""
assert self.current_version, 'No current manifest exists.'
projects / platform / framework / web / crosswalk.git / blobdiff