X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=src%2Ftools%2Ftelemetry%2Ftelemetry%2Fresults%2Fpage_test_results.py;h=676e003414de3f2f5ddecf2d26540316287d4189;hb=4a1a0bdd01eef90b0826a0e761d3379d3715c10f;hp=f6314d9768e9d89fb6f307f1907ee36b1a23eec3;hpb=b1be5ca53587d23e7aeb77b26861fdc0a181ffd8;p=platform%2Fframework%2Fweb%2Fcrosswalk.git diff --git a/src/tools/telemetry/telemetry/results/page_test_results.py b/src/tools/telemetry/telemetry/results/page_test_results.py index f6314d9..676e003 100644 --- a/src/tools/telemetry/telemetry/results/page_test_results.py +++ b/src/tools/telemetry/telemetry/results/page_test_results.py @@ -4,21 +4,44 @@ import collections import copy -import logging -import sys import traceback +from telemetry import value as value_module +from telemetry.results import page_run +from telemetry.results import progress_reporter as progress_reporter_module +from telemetry.value import failure +from telemetry.value import skip + class PageTestResults(object): - def __init__(self, output_stream=None): + def __init__(self, output_stream=None, output_formatters=None, + progress_reporter=None, trace_tag=''): + """ + Args: + output_stream: The output stream to use to write test results. + output_formatters: A list of output formatters. The output + formatters are typically used to format the test results, such + as CsvOutputFormatter, which output the test results as CSV. + progress_reporter: An instance of progress_reporter.ProgressReporter, + to be used to output test status/results progressively. + trace_tag: A string to append to the buildbot trace + name. Currently only used for buildbot. + """ + # TODO(chrishenry): Figure out if trace_tag is still necessary. + super(PageTestResults, self).__init__() self._output_stream = output_stream - self.pages_that_had_errors = set() - self.pages_that_had_failures = set() - self.successes = [] - self.errors = [] - self.failures = [] - self.skipped = [] + self._progress_reporter = ( + progress_reporter if progress_reporter is not None + else progress_reporter_module.ProgressReporter()) + self._output_formatters = ( + output_formatters if output_formatters is not None else []) + self._trace_tag = trace_tag + + self._current_page_run = None + self._all_page_runs = [] + self._representative_value_for_each_value_name = {} + self._all_summary_values = [] def __copy__(self): cls = self.__class__ @@ -30,54 +53,131 @@ class PageTestResults(object): return result @property - def pages_that_had_errors_or_failures(self): - return self.pages_that_had_errors.union( - self.pages_that_had_failures) - - def _GetStringFromExcInfo(self, err): - return ''.join(traceback.format_exception(*err)) + def all_page_specific_values(self): + values = [] + for run in self._all_page_runs: + values += run.values + if self._current_page_run: + values += self._current_page_run.values + return values - def StartTest(self, page): - pass + @property + def all_summary_values(self): + return self._all_summary_values - def StopTest(self, page): - pass + @property + def current_page(self): + assert self._current_page_run, 'Not currently running test.' + return self._current_page_run.page - def AddError(self, page, err): - self.pages_that_had_errors.add(page) - self.errors.append((page, self._GetStringFromExcInfo(err))) + @property + def current_page_run(self): + assert self._current_page_run, 'Not currently running test.' + return self._current_page_run - def AddFailure(self, page, err): - self.pages_that_had_failures.add(page) - self.failures.append((page, self._GetStringFromExcInfo(err))) + @property + def all_page_runs(self): + return self._all_page_runs - def AddSkip(self, page, reason): - self.skipped.append((page, reason)) + @property + def pages_that_succeeded(self): + """Returns the set of pages that succeeded.""" + pages = set(run.page for run in self.all_page_runs) + pages.difference_update(self.pages_that_failed) + return pages - def AddSuccess(self, page): - self.successes.append(page) + @property + def pages_that_failed(self): + """Returns the set of failed pages.""" + failed_pages = set() + for run in self.all_page_runs: + if run.failed: + failed_pages.add(run.page) + return failed_pages - def AddFailureMessage(self, page, message): - try: - raise Exception(message) - except Exception: - self.AddFailure(page, sys.exc_info()) + @property + def failures(self): + values = self.all_page_specific_values + return [v for v in values if isinstance(v, failure.FailureValue)] - def AddErrorMessage(self, page, message): - try: - raise Exception(message) - except Exception: - self.AddError(page, sys.exc_info()) + @property + def skipped_values(self): + values = self.all_page_specific_values + return [v for v in values if isinstance(v, skip.SkipValue)] - def PrintSummary(self): - if self.failures: - logging.error('Failed pages:\n%s', '\n'.join( - p.display_name for p in zip(*self.failures)[0])) + def _GetStringFromExcInfo(self, err): + return ''.join(traceback.format_exception(*err)) - if self.errors: - logging.error('Errored pages:\n%s', '\n'.join( - p.display_name for p in zip(*self.errors)[0])) + def WillRunPage(self, page): + assert not self._current_page_run, 'Did not call DidRunPage.' + self._current_page_run = page_run.PageRun(page) + self._progress_reporter.WillRunPage(self) + + def DidRunPage(self, page, discard_run=False): # pylint: disable=W0613 + """ + Args: + page: The current page under test. + discard_run: Whether to discard the entire run and all of its + associated results. + """ + assert self._current_page_run, 'Did not call WillRunPage.' + self._progress_reporter.DidRunPage(self) + if not discard_run: + self._all_page_runs.append(self._current_page_run) + self._current_page_run = None + + def WillAttemptPageRun(self, attempt_count, max_attempts): + """To be called when a single attempt on a page run is starting. + + This is called between WillRunPage and DidRunPage and can be + called multiple times, one for each attempt. + + Args: + attempt_count: The current attempt number, start at 1 + (attempt_count == 1 for the first attempt, 2 for second + attempt, and so on). + max_attempts: Maximum number of page run attempts before failing. + """ + self._progress_reporter.WillAttemptPageRun( + self, attempt_count, max_attempts) + # Clear any values from previous attempts for this page run. + self._current_page_run.ClearValues() + + def AddValue(self, value): + assert self._current_page_run, 'Not currently running test.' + self._ValidateValue(value) + # TODO(eakuefner/chrishenry): Add only one skip per pagerun assert here + self._current_page_run.AddValue(value) + self._progress_reporter.DidAddValue(value) + + def AddSummaryValue(self, value): + assert value.page is None + self._ValidateValue(value) + self._all_summary_values.append(value) + + def _ValidateValue(self, value): + assert isinstance(value, value_module.Value) + if value.name not in self._representative_value_for_each_value_name: + self._representative_value_for_each_value_name[value.name] = value + representative_value = self._representative_value_for_each_value_name[ + value.name] + assert value.IsMergableWith(representative_value) - if self.skipped: - logging.warning('Skipped pages:\n%s', '\n'.join( - p.display_name for p in zip(*self.skipped)[0])) + def PrintSummary(self): + self._progress_reporter.DidFinishAllTests(self) + for output_formatter in self._output_formatters: + output_formatter.Format(self) + + def FindPageSpecificValuesForPage(self, page, value_name): + values = [] + for value in self.all_page_specific_values: + if value.page == page and value.name == value_name: + values.append(value) + return values + + def FindAllPageSpecificValuesNamed(self, value_name): + values = [] + for value in self.all_page_specific_values: + if value.name == value_name: + values.append(value) + return values