src/build/android/pylib/utils/flakiness_dashboard_results_uploader.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Uploads the results to the flakiness dashboard server."""
   6 # pylint: disable=E1002,R0201
   7
   8 import logging
   9 import os
  10 import shutil
  11 import tempfile
  12 import xml
  13
  14
  15 #TODO(craigdh): pylib/utils/ should not depend on pylib/.
  16 from pylib import cmd_helper
  17 from pylib import constants
  18 from pylib.utils import json_results_generator
  19 from pylib.utils import repo_utils
  20
  21
  22
  23 class JSONResultsGenerator(json_results_generator.JSONResultsGeneratorBase):
  24   """Writes test results to a JSON file and handles uploading that file to
  25   the test results server.
  26   """
  27   def __init__(self, builder_name, build_name, build_number, tmp_folder,
  28                test_results_map, test_results_server, test_type, master_name):
  29     super(JSONResultsGenerator, self).__init__(
  30         builder_name=builder_name,
  31         build_name=build_name,
  32         build_number=build_number,
  33         results_file_base_path=tmp_folder,
  34         builder_base_url=None,
  35         test_results_map=test_results_map,
  36         svn_repositories=(('webkit', 'third_party/WebKit'),
  37                           ('chrome', '.')),
  38         test_results_server=test_results_server,
  39         test_type=test_type,
  40         master_name=master_name)
  41
  42   #override
  43   def _GetModifierChar(self, test_name):
  44     if test_name not in self._test_results_map:
  45       return self.__class__.NO_DATA_RESULT
  46
  47     return self._test_results_map[test_name].modifier
  48
  49   #override
  50   def _GetSVNRevision(self, in_directory):
  51     """Returns the git/svn revision for the given directory.
  52
  53     Args:
  54       in_directory: The directory relative to src.
  55     """
  56     def _is_git_directory(in_directory):
  57       """Returns true if the given directory is in a git repository.
  58
  59       Args:
  60         in_directory: The directory path to be tested.
  61       """
  62       if os.path.exists(os.path.join(in_directory, '.git')):
  63         return True
  64       parent = os.path.dirname(in_directory)
  65       if parent == constants.DIR_SOURCE_ROOT or parent == in_directory:
  66         return False
  67       return _is_git_directory(parent)
  68
  69     in_directory = os.path.join(constants.DIR_SOURCE_ROOT, in_directory)
  70
  71     if not os.path.exists(os.path.join(in_directory, '.svn')):
  72       if _is_git_directory(in_directory):
  73         return repo_utils.GetGitHeadSHA1(in_directory)
  74       else:
  75         return ''
  76
  77     output = cmd_helper.GetCmdOutput(['svn', 'info', '--xml'], cwd=in_directory)
  78     try:
  79       dom = xml.dom.minidom.parseString(output)
  80       return dom.getElementsByTagName('entry')[0].getAttribute('revision')
  81     except xml.parsers.expat.ExpatError:
  82       return ''
  83     return ''
  84
  85
  86 class ResultsUploader(object):
  87   """Handles uploading buildbot tests results to the flakiness dashboard."""
  88   def __init__(self, tests_type):
  89     self._build_number = os.environ.get('BUILDBOT_BUILDNUMBER')
  90     self._builder_name = os.environ.get('BUILDBOT_BUILDERNAME')
  91     self._tests_type = tests_type
  92
  93     if not self._build_number or not self._builder_name:
  94       raise Exception('You should not be uploading tests results to the server'
  95                       'from your local machine.')
  96
  97     upstream = (tests_type != 'Chromium_Android_Instrumentation')
  98     if upstream:
  99       # TODO(frankf): Use factory properties (see buildbot/bb_device_steps.py)
 100       # This requires passing the actual master name (e.g. 'ChromiumFYI' not
 101       # 'chromium.fyi').
 102       from slave import slave_utils # pylint: disable=F0401
 103       self._build_name = slave_utils.SlaveBuildName(constants.DIR_SOURCE_ROOT)
 104       self._master_name = slave_utils.GetActiveMaster()
 105     else:
 106       self._build_name = 'chromium-android'
 107       buildbot_branch = os.environ.get('BUILDBOT_BRANCH')
 108       if not buildbot_branch:
 109         buildbot_branch = 'master'
 110       self._master_name = '%s-%s' % (self._build_name, buildbot_branch)
 111
 112     self._test_results_map = {}
 113
 114   def AddResults(self, test_results):
 115     # TODO(frankf): Differentiate between fail/crash/timeouts.
 116     conversion_map = [
 117         (test_results.GetPass(), False,
 118             json_results_generator.JSONResultsGeneratorBase.PASS_RESULT),
 119         (test_results.GetFail(), True,
 120             json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
 121         (test_results.GetCrash(), True,
 122             json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
 123         (test_results.GetTimeout(), True,
 124             json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
 125         (test_results.GetUnknown(), True,
 126             json_results_generator.JSONResultsGeneratorBase.NO_DATA_RESULT),
 127         ]
 128
 129     for results_list, failed, modifier in conversion_map:
 130       for single_test_result in results_list:
 131         test_result = json_results_generator.TestResult(
 132             test=single_test_result.GetName(),
 133             failed=failed,
 134             elapsed_time=single_test_result.GetDur() / 1000)
 135         # The WebKit TestResult object sets the modifier it based on test name.
 136         # Since we don't use the same test naming convention as WebKit the
 137         # modifier will be wrong, so we need to overwrite it.
 138         test_result.modifier = modifier
 139
 140         self._test_results_map[single_test_result.GetName()] = test_result
 141
 142   def Upload(self, test_results_server):
 143     if not self._test_results_map:
 144       return
 145
 146     tmp_folder = tempfile.mkdtemp()
 147
 148     try:
 149       results_generator = JSONResultsGenerator(
 150           builder_name=self._builder_name,
 151           build_name=self._build_name,
 152           build_number=self._build_number,
 153           tmp_folder=tmp_folder,
 154           test_results_map=self._test_results_map,
 155           test_results_server=test_results_server,
 156           test_type=self._tests_type,
 157           master_name=self._master_name)
 158
 159       json_files = ["incremental_results.json", "times_ms.json"]
 160       results_generator.GenerateJSONOutput()
 161       results_generator.GenerateTimesMSFile()
 162       results_generator.UploadJSONFiles(json_files)
 163     except Exception as e:
 164       logging.error("Uploading results to test server failed: %s." % e)
 165     finally:
 166       shutil.rmtree(tmp_folder)
 167
 168
 169 def Upload(results, flakiness_dashboard_server, test_type):
 170   """Reports test results to the flakiness dashboard for Chrome for Android.
 171
 172   Args:
 173     results: test results.
 174     flakiness_dashboard_server: the server to upload the results to.
 175     test_type: the type of the tests (as displayed by the flakiness dashboard).
 176   """
 177   uploader = ResultsUploader(test_type)
 178   uploader.AddResults(results)
 179   uploader.Upload(flakiness_dashboard_server)