src/build/android/pylib/utils/flakiness_dashboard_results_uploader.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Uploads the results to the flakiness dashboard server."""
   6
   7 import logging
   8 import os
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tempfile
  13 import xml
  14
  15
  16 # Include path when ran from a Chromium checkout.
  17 sys.path.append(
  18     os.path.abspath(os.path.join(os.path.dirname(__file__),
  19                                  os.pardir, os.pardir, os.pardir, os.pardir,
  20                                  'third_party', 'WebKit', 'Tools', 'Scripts')))
  21
  22 # Include path when ran from a WebKit checkout.
  23 sys.path.append(
  24     os.path.abspath(os.path.join(os.path.dirname(__file__),
  25                                  os.pardir, os.pardir, os.pardir, os.pardir,
  26                                  os.pardir, os.pardir, os.pardir,
  27                                  'Tools', 'Scripts')))
  28
  29 from webkitpy.common.system import executive, filesystem
  30 from webkitpy.layout_tests.layout_package import json_results_generator
  31
  32 #TODO(craigdh): pylib/utils/ should not depend on pylib/.
  33 from pylib import cmd_helper
  34 from pylib import constants
  35 from pylib.utils import repo_utils
  36
  37
  38 # The JSONResultsGenerator gets the filesystem.join operation from the Port
  39 # object. Creating a Port object requires specifying information that only
  40 # makes sense for running WebKit layout tests, so we provide a dummy object
  41 # that contains the fields required by the generator.
  42 class PortDummy(object):
  43   def __init__(self):
  44     self._executive = executive.Executive()
  45     self._filesystem = filesystem.FileSystem()
  46
  47
  48 class JSONResultsGenerator(json_results_generator.JSONResultsGeneratorBase):
  49   """Writes test results to a JSON file and handles uploading that file to
  50   the test results server.
  51   """
  52   def __init__(self, port, builder_name, build_name, build_number, tmp_folder,
  53                test_results_map, test_results_server, test_type, master_name):
  54     super(JSONResultsGenerator, self).__init__(
  55         port=port,
  56         builder_name=builder_name,
  57         build_name=build_name,
  58         build_number=build_number,
  59         results_file_base_path=tmp_folder,
  60         builder_base_url=None,
  61         test_results_map=test_results_map,
  62         svn_repositories=(('webkit', 'third_party/WebKit'),
  63                           ('chrome', '.')),
  64         test_results_server=test_results_server,
  65         test_type=test_type,
  66         master_name=master_name)
  67
  68   #override
  69   def _get_modifier_char(self, test_name):
  70     if test_name not in self._test_results_map:
  71       return self.__class__.NO_DATA_RESULT
  72
  73     return self._test_results_map[test_name].modifier
  74
  75   #override
  76   def _get_svn_revision(self, in_directory):
  77     """Returns the git/svn revision for the given directory.
  78
  79     Args:
  80       in_directory: The directory relative to src.
  81     """
  82     def _is_git_directory(in_directory):
  83       """Returns true if the given directory is in a git repository.
  84
  85       Args:
  86         in_directory: The directory path to be tested.
  87       """
  88       if os.path.exists(os.path.join(in_directory, '.git')):
  89         return True
  90       parent = os.path.dirname(in_directory)
  91       if parent == constants.DIR_SOURCE_ROOT or parent == in_directory:
  92         return False
  93       return _is_git_directory(parent)
  94
  95     in_directory = os.path.join(constants.DIR_SOURCE_ROOT, in_directory)
  96
  97     if not os.path.exists(os.path.join(in_directory, '.svn')):
  98       if _is_git_directory(in_directory):
  99         return repo_utils.GetGitHeadSHA1(in_directory)
 100       else:
 101         return ''
 102
 103     output = cmd_helper.GetCmdOutput(['svn', 'info', '--xml'], cwd=in_directory)
 104     try:
 105       dom = xml.dom.minidom.parseString(output)
 106       return dom.getElementsByTagName('entry')[0].getAttribute('revision')
 107     except xml.parsers.expat.ExpatError:
 108       return ''
 109     return ''
 110
 111
 112 class ResultsUploader(object):
 113   """Handles uploading buildbot tests results to the flakiness dashboard."""
 114   def __init__(self, tests_type):
 115     self._build_number = os.environ.get('BUILDBOT_BUILDNUMBER')
 116     self._builder_name = os.environ.get('BUILDBOT_BUILDERNAME')
 117     self._tests_type = tests_type
 118
 119     if not self._build_number or not self._builder_name:
 120       raise Exception('You should not be uploading tests results to the server'
 121                       'from your local machine.')
 122
 123     upstream = (tests_type != 'Chromium_Android_Instrumentation')
 124     if upstream:
 125       # TODO(frankf): Use factory properties (see buildbot/bb_device_steps.py)
 126       # This requires passing the actual master name (e.g. 'ChromiumFYI' not
 127       # 'chromium.fyi').
 128       from slave import slave_utils
 129       self._build_name = slave_utils.SlaveBuildName(constants.DIR_SOURCE_ROOT)
 130       self._master_name = slave_utils.GetActiveMaster()
 131     else:
 132       self._build_name = 'chromium-android'
 133       buildbot_branch = os.environ.get('BUILDBOT_BRANCH')
 134       if not buildbot_branch:
 135         buildbot_branch = 'master'
 136       self._master_name = '%s-%s' % (self._build_name, buildbot_branch)
 137
 138     self._test_results_map = {}
 139
 140   def AddResults(self, test_results):
 141     # TODO(frankf): Differentiate between fail/crash/timeouts.
 142     conversion_map = [
 143         (test_results.GetPass(), False,
 144             json_results_generator.JSONResultsGeneratorBase.PASS_RESULT),
 145         (test_results.GetFail(), True,
 146             json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
 147         (test_results.GetCrash(), True,
 148             json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
 149         (test_results.GetTimeout(), True,
 150             json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
 151         (test_results.GetUnknown(), True,
 152             json_results_generator.JSONResultsGeneratorBase.NO_DATA_RESULT),
 153         ]
 154
 155     for results_list, failed, modifier in conversion_map:
 156       for single_test_result in results_list:
 157         test_result = json_results_generator.TestResult(
 158             test=single_test_result.GetName(),
 159             failed=failed,
 160             elapsed_time=single_test_result.GetDur() / 1000)
 161         # The WebKit TestResult object sets the modifier it based on test name.
 162         # Since we don't use the same test naming convention as WebKit the
 163         # modifier will be wrong, so we need to overwrite it.
 164         test_result.modifier = modifier
 165
 166         self._test_results_map[single_test_result.GetName()] = test_result
 167
 168   def Upload(self, test_results_server):
 169     if not self._test_results_map:
 170       return
 171
 172     tmp_folder = tempfile.mkdtemp()
 173
 174     try:
 175       results_generator = JSONResultsGenerator(
 176           port=PortDummy(),
 177           builder_name=self._builder_name,
 178           build_name=self._build_name,
 179           build_number=self._build_number,
 180           tmp_folder=tmp_folder,
 181           test_results_map=self._test_results_map,
 182           test_results_server=test_results_server,
 183           test_type=self._tests_type,
 184           master_name=self._master_name)
 185
 186       json_files = ["incremental_results.json", "times_ms.json"]
 187       results_generator.generate_json_output()
 188       results_generator.generate_times_ms_file()
 189       results_generator.upload_json_files(json_files)
 190     except Exception as e:
 191       logging.error("Uploading results to test server failed: %s." % e);
 192     finally:
 193       shutil.rmtree(tmp_folder)
 194
 195
 196 def Upload(results, flakiness_dashboard_server, test_type):
 197   """Reports test results to the flakiness dashboard for Chrome for Android.
 198
 199   Args:
 200     results: test results.
 201     flakiness_dashboard_server: the server to upload the results to.
 202     test_type: the type of the tests (as displayed by the flakiness dashboard).
 203   """
 204   uploader = ResultsUploader(test_type)
 205   uploader.AddResults(results)
 206   uploader.Upload(flakiness_dashboard_server)