src/build/android/pylib/utils/flakiness_dashboard_results_uploader.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Uploads the results to the flakiness dashboard server."""
   6 # pylint: disable=E1002,R0201
   7
   8 import logging
   9 import os
  10 import shutil
  11 import sys
  12 import tempfile
  13 import xml
  14
  15
  16 # Include path when ran from a Chromium checkout.
  17 sys.path.append(
  18     os.path.abspath(os.path.join(os.path.dirname(__file__),
  19                                  os.pardir, os.pardir, os.pardir, os.pardir,
  20                                  'third_party', 'WebKit', 'Tools', 'Scripts')))
  21
  22 # Include path when ran from a WebKit checkout.
  23 sys.path.append(
  24     os.path.abspath(os.path.join(os.path.dirname(__file__),
  25                                  os.pardir, os.pardir, os.pardir, os.pardir,
  26                                  os.pardir, os.pardir, os.pardir,
  27                                  'Tools', 'Scripts')))
  28
  29 # pylint: disable=F0401
  30 from webkitpy.common.system import executive, filesystem
  31 from webkitpy.layout_tests.layout_package import json_results_generator
  32 # pylint: enable=F0401
  33
  34 #TODO(craigdh): pylib/utils/ should not depend on pylib/.
  35 from pylib import cmd_helper
  36 from pylib import constants
  37 from pylib.utils import repo_utils
  38
  39
  40 # The JSONResultsGenerator gets the filesystem.join operation from the Port
  41 # object. Creating a Port object requires specifying information that only
  42 # makes sense for running WebKit layout tests, so we provide a dummy object
  43 # that contains the fields required by the generator.
  44 class PortDummy(object):
  45   def __init__(self):
  46     self._executive = executive.Executive()
  47     self._filesystem = filesystem.FileSystem()
  48
  49
  50 class JSONResultsGenerator(json_results_generator.JSONResultsGeneratorBase):
  51   """Writes test results to a JSON file and handles uploading that file to
  52   the test results server.
  53   """
  54   def __init__(self, port, builder_name, build_name, build_number, tmp_folder,
  55                test_results_map, test_results_server, test_type, master_name):
  56     super(JSONResultsGenerator, self).__init__(
  57         port=port,
  58         builder_name=builder_name,
  59         build_name=build_name,
  60         build_number=build_number,
  61         results_file_base_path=tmp_folder,
  62         builder_base_url=None,
  63         test_results_map=test_results_map,
  64         svn_repositories=(('webkit', 'third_party/WebKit'),
  65                           ('chrome', '.')),
  66         test_results_server=test_results_server,
  67         test_type=test_type,
  68         master_name=master_name)
  69
  70   #override
  71   def _get_modifier_char(self, test_name):
  72     if test_name not in self._test_results_map:
  73       return self.__class__.NO_DATA_RESULT
  74
  75     return self._test_results_map[test_name].modifier
  76
  77   #override
  78   def _get_svn_revision(self, in_directory):
  79     """Returns the git/svn revision for the given directory.
  80
  81     Args:
  82       in_directory: The directory relative to src.
  83     """
  84     def _is_git_directory(in_directory):
  85       """Returns true if the given directory is in a git repository.
  86
  87       Args:
  88         in_directory: The directory path to be tested.
  89       """
  90       if os.path.exists(os.path.join(in_directory, '.git')):
  91         return True
  92       parent = os.path.dirname(in_directory)
  93       if parent == constants.DIR_SOURCE_ROOT or parent == in_directory:
  94         return False
  95       return _is_git_directory(parent)
  96
  97     in_directory = os.path.join(constants.DIR_SOURCE_ROOT, in_directory)
  98
  99     if not os.path.exists(os.path.join(in_directory, '.svn')):
 100       if _is_git_directory(in_directory):
 101         return repo_utils.GetGitHeadSHA1(in_directory)
 102       else:
 103         return ''
 104
 105     output = cmd_helper.GetCmdOutput(['svn', 'info', '--xml'], cwd=in_directory)
 106     try:
 107       dom = xml.dom.minidom.parseString(output)
 108       return dom.getElementsByTagName('entry')[0].getAttribute('revision')
 109     except xml.parsers.expat.ExpatError:
 110       return ''
 111     return ''
 112
 113
 114 class ResultsUploader(object):
 115   """Handles uploading buildbot tests results to the flakiness dashboard."""
 116   def __init__(self, tests_type):
 117     self._build_number = os.environ.get('BUILDBOT_BUILDNUMBER')
 118     self._builder_name = os.environ.get('BUILDBOT_BUILDERNAME')
 119     self._tests_type = tests_type
 120
 121     if not self._build_number or not self._builder_name:
 122       raise Exception('You should not be uploading tests results to the server'
 123                       'from your local machine.')
 124
 125     upstream = (tests_type != 'Chromium_Android_Instrumentation')
 126     if upstream:
 127       # TODO(frankf): Use factory properties (see buildbot/bb_device_steps.py)
 128       # This requires passing the actual master name (e.g. 'ChromiumFYI' not
 129       # 'chromium.fyi').
 130       from slave import slave_utils # pylint: disable=F0401
 131       self._build_name = slave_utils.SlaveBuildName(constants.DIR_SOURCE_ROOT)
 132       self._master_name = slave_utils.GetActiveMaster()
 133     else:
 134       self._build_name = 'chromium-android'
 135       buildbot_branch = os.environ.get('BUILDBOT_BRANCH')
 136       if not buildbot_branch:
 137         buildbot_branch = 'master'
 138       self._master_name = '%s-%s' % (self._build_name, buildbot_branch)
 139
 140     self._test_results_map = {}
 141
 142   def AddResults(self, test_results):
 143     # TODO(frankf): Differentiate between fail/crash/timeouts.
 144     conversion_map = [
 145         (test_results.GetPass(), False,
 146             json_results_generator.JSONResultsGeneratorBase.PASS_RESULT),
 147         (test_results.GetFail(), True,
 148             json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
 149         (test_results.GetCrash(), True,
 150             json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
 151         (test_results.GetTimeout(), True,
 152             json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
 153         (test_results.GetUnknown(), True,
 154             json_results_generator.JSONResultsGeneratorBase.NO_DATA_RESULT),
 155         ]
 156
 157     for results_list, failed, modifier in conversion_map:
 158       for single_test_result in results_list:
 159         test_result = json_results_generator.TestResult(
 160             test=single_test_result.GetName(),
 161             failed=failed,
 162             elapsed_time=single_test_result.GetDur() / 1000)
 163         # The WebKit TestResult object sets the modifier it based on test name.
 164         # Since we don't use the same test naming convention as WebKit the
 165         # modifier will be wrong, so we need to overwrite it.
 166         test_result.modifier = modifier
 167
 168         self._test_results_map[single_test_result.GetName()] = test_result
 169
 170   def Upload(self, test_results_server):
 171     if not self._test_results_map:
 172       return
 173
 174     tmp_folder = tempfile.mkdtemp()
 175
 176     try:
 177       results_generator = JSONResultsGenerator(
 178           port=PortDummy(),
 179           builder_name=self._builder_name,
 180           build_name=self._build_name,
 181           build_number=self._build_number,
 182           tmp_folder=tmp_folder,
 183           test_results_map=self._test_results_map,
 184           test_results_server=test_results_server,
 185           test_type=self._tests_type,
 186           master_name=self._master_name)
 187
 188       json_files = ["incremental_results.json", "times_ms.json"]
 189       results_generator.generate_json_output()
 190       results_generator.generate_times_ms_file()
 191       results_generator.upload_json_files(json_files)
 192     except Exception as e:
 193       logging.error("Uploading results to test server failed: %s." % e)
 194     finally:
 195       shutil.rmtree(tmp_folder)
 196
 197
 198 def Upload(results, flakiness_dashboard_server, test_type):
 199   """Reports test results to the flakiness dashboard for Chrome for Android.
 200
 201   Args:
 202     results: test results.
 203     flakiness_dashboard_server: the server to upload the results to.
 204     test_type: the type of the tests (as displayed by the flakiness dashboard).
 205   """
 206   uploader = ResultsUploader(test_type)
 207   uploader.AddResults(results)
 208   uploader.Upload(flakiness_dashboard_server)