"""Module containing the report stages."""
+from __future__ import print_function
+
import logging
import os
import sys
from chromite.cbuildbot import commands
+from chromite.cbuildbot import constants
from chromite.cbuildbot import failures_lib
+from chromite.cbuildbot import manifest_version
from chromite.cbuildbot import metadata_lib
from chromite.cbuildbot import results_lib
-from chromite.cbuildbot import constants
+from chromite.cbuildbot import tree_status
from chromite.cbuildbot.stages import completion_stages
from chromite.cbuildbot.stages import generic_stages
from chromite.cbuildbot.stages import sync_stages
from chromite.lib import alerts
+from chromite.lib import cidb
from chromite.lib import cros_build_lib
from chromite.lib import gs
from chromite.lib import osutils
from chromite.lib import toolchain
-class ReportBuildStartStage(generic_stages.BuilderStage,
- generic_stages.ArchivingStageMixin):
- """Uploads partial metadata artifact describing what will be built.
+def WriteBasicMetadata(builder_run):
+ """Writes basic metadata that should be known at start of execution.
+
+ This method writes to |build_run|'s metadata instance the basic metadata
+ values that should be known at the beginning of the first cbuildbot
+ execution, prior to any reexecutions.
- This stage should be the first stage run after the final cbuildbot
- bootstrap/reexecution. By the time this stage is run, all sync stages
- are complete and version numbers of chrome and chromeos are known.
+ In particular, this method does not write any metadata values that depend
+ on the builder config, as the config may be modified by patches that are
+ applied before the final reexectuion.
+
+ This method is safe to run more than once (for instance, once per cbuildbot
+ execution) because it will write the same data each time.
+
+ Args:
+ builder_run: The BuilderRun instance for this build.
+ """
+ start_time = results_lib.Results.start_time
+ start_time_stamp = cros_build_lib.UserDateTimeFormat(timeval=start_time)
+
+ metadata = {
+ # Data for this build.
+ 'bot-hostname': cros_build_lib.GetHostName(fully_qualified=True),
+ 'build-number': builder_run.buildnumber,
+ 'builder-name': os.environ.get('BUILDBOT_BUILDERNAME', ''),
+ 'buildbot-url': os.environ.get('BUILDBOT_BUILDBOTURL', ''),
+ 'buildbot-master-name':
+ os.environ.get('BUILDBOT_MASTERNAME', ''),
+ 'bot-config': builder_run.config['name'],
+ 'time': {
+ 'start': start_time_stamp,
+ },
+ 'master_build_id': builder_run.options.master_build_id,
+ }
+
+ builder_run.attrs.metadata.UpdateWithDict(metadata)
+
+
+class BuildStartStage(generic_stages.BuilderStage):
+ """The first stage to run.
+
+ This stage writes a few basic metadata values that are known at the start of
+ build, and inserts the build into the database, if appropriate.
+ """
+ @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
+ def PerformStage(self):
+ WriteBasicMetadata(self._run)
+ d = self._run.attrs.metadata.GetDict()
+
+ # BuildStartStage should only run once per build. But just in case it
+ # is somehow running a second time, we do not want to insert an additional
+ # database entry. Detect if a database entry has been inserted already
+ # and if so quit the stage.
+ if 'build_id' in d:
+ logging.info('Already have build_id %s, not inserting an entry.',
+ d['build_id'])
+ return
+
+ if cidb.CIDBConnectionFactory.IsCIDBSetup():
+ db_type = cidb.CIDBConnectionFactory.GetCIDBConnectionType()
+ db = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
+ if db:
+ waterfall = d['buildbot-master-name']
+ assert waterfall in ('chromeos', 'chromiumos', 'chromiumos.tryserver')
+ build_id = db.InsertBuild(
+ builder_name=d['builder-name'],
+ waterfall=waterfall,
+ build_number=d['build-number'],
+ build_config=d['bot-config'],
+ bot_hostname=d['bot-hostname'],
+ master_build_id=d['master_build_id'])
+ self._run.attrs.metadata.UpdateWithDict({'build_id': build_id,
+ 'db_type': db_type})
+ logging.info('Inserted build_id %s into cidb database.', build_id)
+
+
+ def HandleSkip(self):
+ """Ensure that re-executions use the same db instance as initial db."""
+ metadata_dict = self._run.attrs.metadata.GetDict()
+ if 'build_id' in metadata_dict:
+ db_type = cidb.CIDBConnectionFactory.GetCIDBConnectionType()
+ if not 'db_type' in metadata_dict:
+ # This will only execute while this CL is in the commit queue. After
+ # this CL lands, this block can be removed.
+ self._run.attrs.metadata.UpdateWithDict({'db_type': db_type})
+ return
+
+ if db_type != metadata_dict['db_type']:
+ cidb.CIDBConnectionFactory.InvalidateCIDBSetup()
+ raise AssertionError('Invalid attempt to switch from database %s to '
+ '%s.' % (metadata_dict['db_type'], db_type))
+
+
+class BuildReexecutionFinishedStage(generic_stages.BuilderStage,
+ generic_stages.ArchivingStageMixin):
+ """The first stage to run after the final cbuildbot reexecution.
+
+ This stage is the first stage run after the final cbuildbot
+ bootstrap/reexecution. By the time this stage is run, the sync stages
+ are complete and version numbers of chromeos are known (though chrome
+ version may not be known until SyncChrome).
+
+ This stage writes metadata values that are first known after the final
+ reexecution (such as those that come from the config). This stage also
+ updates the build's cidb entry if appropriate.
Where possible, metadata that is already known at this time should be
written at this time rather than in ReportStage.
"""
- def init(self, builder_run, **kwargs):
- super(ReportBuildStartStage, self).__init__(builder_run, **kwargs)
-
@failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
def PerformStage(self):
config = self._run.config
os.path.join(build_root, constants.SDK_VERSION_FILE),
ignore_missing=True)
- start_time = results_lib.Results.start_time
- start_time_stamp = cros_build_lib.UserDateTimeFormat(timeval=start_time)
-
verinfo = self._run.GetVersionInfo(build_root)
platform_tag = getattr(self._run.attrs, 'release_tag')
if not platform_tag:
metadata = {
# Version of the metadata format.
'metadata-version': '2',
- # Data for this build.
- 'bot-config': config['name'],
- 'bot-hostname': cros_build_lib.GetHostName(fully_qualified=True),
'boards': config['boards'],
- 'build-number': self._run.buildnumber,
- 'builder-name': os.environ.get('BUILDBOT_BUILDERNAME', ''),
'child-configs': child_configs,
- 'time': {
- 'start': start_time_stamp,
- },
- 'version': version,
+ 'build_type': config['build_type'],
# Data for the toolchain used.
'sdk-version': sdk_verinfo.get('SDK_LATEST_VERSION', '<unknown>'),
logging.info('Metadata being written: %s', metadata)
self._run.attrs.metadata.UpdateWithDict(metadata)
+ # Update 'version' separately to avoid overwriting the existing
+ # entries in it (e.g. PFQ builders may have written the Chrome
+ # version to uprev).
+ logging.info("Metadata 'version' being written: %s", version)
+ self._run.attrs.metadata.UpdateKeyDictWithDict('version', version)
+
+ # Ensure that all boards and child config boards have a per-board
+ # metadata subdict.
+ for b in config['boards']:
+ self._run.attrs.metadata.UpdateBoardDictWithDict(b, {})
+
+ for cc in child_configs:
+ for b in cc['boards']:
+ self._run.attrs.metadata.UpdateBoardDictWithDict(b, {})
+
+ # Upload build metadata (and write it to database if necessary)
self.UploadMetadata(filename=constants.PARTIAL_METADATA_JSON)
+ # Write child-per-build and board-per-build rows to database
+ if cidb.CIDBConnectionFactory.IsCIDBSetup():
+ db = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
+ if db:
+ build_id = self._run.attrs.metadata.GetValue('build_id')
+ # TODO(akeshet): replace this with a GetValue call once crbug.com/406522
+ # is resolved
+ per_board_dict = self._run.attrs.metadata.GetDict()['board-metadata']
+ for board, board_metadata in per_board_dict.items():
+ db.InsertBoardPerBuild(build_id, board)
+ if board_metadata:
+ db.UpdateBoardPerBuildMetadata(build_id, board, board_metadata)
+ for child_config in self._run.attrs.metadata.GetValue('child-configs'):
+ db.InsertChildConfigPerBuild(build_id, child_config['name'])
+
class ReportStage(generic_stages.BuilderStage,
generic_stages.ArchivingStageMixin):
builder_run.config.name, verb, abs(streak_value))
# See if updated streak should trigger a notification email.
if (builder_run.config.health_alert_recipients and
+ builder_run.config.health_threshold > 0 and
streak_value <= -builder_run.config.health_threshold):
cros_build_lib.Info(
'Builder failed %i consecutive times, sending health alert email '
if not self._run.debug:
alerts.SendEmail('%s health alert' % builder_run.config.name,
- builder_run.config.health_alert_recipients,
+ tree_status.GetHealthAlertRecipients(builder_run),
message=self._HealthAlertMessage(-streak_value),
smtp_server=constants.GOLO_SMTP_SERVER,
extra_fields={'X-cbuildbot-alert': 'cq-health'})
results_lib.Results.Report(sys.stdout, archive_urls=archive_urls,
current_version=version)
+ if cidb.CIDBConnectionFactory.IsCIDBSetup():
+ db = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
+ if db:
+ build_id = self._run.attrs.metadata.GetValue('build_id')
+ # TODO(akeshet): Eliminate this status string translate once
+ # these differing status strings are merged, crbug.com/318930
+ if final_status == constants.FINAL_STATUS_PASSED:
+ status_for_db = manifest_version.BuilderStatus.STATUS_PASSED
+ else:
+ status_for_db = manifest_version.BuilderStatus.STATUS_FAILED
+
+ # TODO(akeshet): Consider uploading the status pickle to the database,
+ # (by specifying that argument to FinishBuild), or come up with a
+ # pickle-free mechanism to describe failure details in database.
+ # TODO(akeshet): Find a clearer way to get the "primary upload url" for
+ # the metadata.json file. One alternative is _GetUploadUrls(...)[0].
+ # Today it seems that element 0 of its return list is the primary upload
+ # url, but there is no guarantee or unit test coverage of that.
+ metadata_url = os.path.join(self.upload_url, constants.METADATA_JSON)
+ db.FinishBuild(build_id, status=status_for_db,
+ metadata_url=metadata_url)
+
class RefreshPackageStatusStage(generic_stages.BuilderStage):
"""Stage for refreshing Portage package status in online spreadsheet."""
projects / platform / framework / web / crosswalk.git / blobdiff