src/third_party/chromite/scripts/gather_builder_stats_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Unit tests for gather_builder_stats."""
   7
   8 from __future__ import print_function
   9
  10 import datetime
  11 import itertools
  12 import os
  13 import random
  14 import sys
  15 import unittest
  16
  17 sys.path.insert(0, os.path.abspath('%s/../..' % os.path.dirname(__file__)))
  18 from chromite.lib import cros_build_lib
  19 from chromite.lib import cros_test_lib
  20 from chromite.scripts import gather_builder_stats
  21 from chromite.cbuildbot import metadata_lib
  22 from chromite.cbuildbot import constants
  23
  24 import mock
  25
  26
  27 REASON_BAD_CL = gather_builder_stats.CLStats.REASON_BAD_CL
  28 CQ = constants.CQ
  29 PRE_CQ = constants.PRE_CQ
  30
  31
  32 class TestCLActionLogic(unittest.TestCase):
  33   """Ensures that CL action analysis logic is correct."""
  34
  35   def _getTestBuildData(self, cq):
  36     """Generate a return test data.
  37
  38     Args:
  39       cq: Whether this is a CQ run. If False, this is a Pre-CQ run.
  40
  41     Returns:
  42       A list of metadata_lib.BuildData objects to use as
  43       test data for CL action summary logic.
  44     """
  45     # Mock patches for test data.
  46     c1p1 = metadata_lib.GerritPatchTuple(1, 1, False)
  47     c2p1 = metadata_lib.GerritPatchTuple(2, 1, True)
  48     c2p2 = metadata_lib.GerritPatchTuple(2, 2, True)
  49     c3p1 = metadata_lib.GerritPatchTuple(3, 1, True)
  50     c3p2 = metadata_lib.GerritPatchTuple(3, 2, True)
  51     c4p1 = metadata_lib.GerritPatchTuple(4, 1, True)
  52     c4p2 = metadata_lib.GerritPatchTuple(4, 2, True)
  53
  54     # Mock builder status dictionaries
  55     passed_status = {'status' : constants.FINAL_STATUS_PASSED}
  56     failed_status = {'status' : constants.FINAL_STATUS_FAILED}
  57
  58     t = itertools.count()
  59     bot_config = (constants.CQ_MASTER if cq
  60                   else constants.PRE_CQ_GROUP_GS_LOCATION)
  61
  62     # pylint: disable=W0212
  63     TEST_METADATA = [
  64       # Build 1 picks up no patches.
  65       metadata_lib.CBuildbotMetadata(
  66           ).UpdateWithDict({'build-number' : 1,
  67                             'bot-config' : bot_config,
  68                             'results' : [],
  69                             'status' : passed_status}),
  70       # Build 2 picks up c1p1 and does nothing.
  71       metadata_lib.CBuildbotMetadata(
  72           ).UpdateWithDict({'build-number' : 2,
  73                             'bot-config' : bot_config,
  74                             'results' : [],
  75                             'status' : failed_status,
  76                             'changes': [c1p1._asdict()]}
  77           ).RecordCLAction(c1p1, constants.CL_ACTION_PICKED_UP, t.next()),
  78       # Build 3 picks up c1p1 and c2p1 and rejects both.
  79       # c3p1 is not included in the run because it fails to apply.
  80       metadata_lib.CBuildbotMetadata(
  81           ).UpdateWithDict({'build-number' : 3,
  82                             'bot-config' : bot_config,
  83                             'results' : [],
  84                             'status' : failed_status,
  85                             'changes': [c1p1._asdict(),
  86                                         c2p1._asdict()]}
  87           ).RecordCLAction(c1p1, constants.CL_ACTION_PICKED_UP, t.next()
  88           ).RecordCLAction(c2p1, constants.CL_ACTION_PICKED_UP, t.next()
  89           ).RecordCLAction(c1p1, constants.CL_ACTION_KICKED_OUT, t.next()
  90           ).RecordCLAction(c2p1, constants.CL_ACTION_KICKED_OUT, t.next()
  91           ).RecordCLAction(c3p1, constants.CL_ACTION_KICKED_OUT, t.next()),
  92       # Build 4 picks up c4p1 and rejects it.
  93       metadata_lib.CBuildbotMetadata(
  94           ).UpdateWithDict({'build-number' : 3,
  95                             'bot-config' : bot_config,
  96                             'results' : [],
  97                             'status' : failed_status,
  98                             'changes': [c4p1._asdict()]}
  99           ).RecordCLAction(c4p2, constants.CL_ACTION_PICKED_UP, t.next()
 100           ).RecordCLAction(c4p2, constants.CL_ACTION_KICKED_OUT, t.next()),
 101     ]
 102     if cq:
 103       TEST_METADATA += [
 104         # Build 4 picks up c1p1 and c2p2 and submits both.
 105         # So  c1p1 should be detected as a 1-time rejected good patch,
 106         # and c2p1 should be detected as a possibly bad patch.
 107         metadata_lib.CBuildbotMetadata(
 108             ).UpdateWithDict({'build-number' : 4,
 109                               'bot-config' : bot_config,
 110                               'results' : [],
 111                               'status' : passed_status,
 112                               'changes': [c1p1._asdict(),
 113                                           c2p2._asdict()]}
 114             ).RecordCLAction(c1p1, constants.CL_ACTION_PICKED_UP, t.next()
 115             ).RecordCLAction(c2p2, constants.CL_ACTION_PICKED_UP, t.next()
 116             ).RecordCLAction(c3p2, constants.CL_ACTION_PICKED_UP, t.next()
 117             ).RecordCLAction(c4p1, constants.CL_ACTION_PICKED_UP, t.next()
 118             ).RecordCLAction(c1p1, constants.CL_ACTION_SUBMITTED, t.next()
 119             ).RecordCLAction(c2p2, constants.CL_ACTION_SUBMITTED, t.next()
 120             ).RecordCLAction(c3p2, constants.CL_ACTION_SUBMITTED, t.next()
 121             ).RecordCLAction(c4p2, constants.CL_ACTION_SUBMITTED, t.next()),
 122       ]
 123     else:
 124       TEST_METADATA += [
 125         metadata_lib.CBuildbotMetadata(
 126             ).UpdateWithDict({'build-number' : 5,
 127                               'bot-config' : bot_config,
 128                               'results' : [],
 129                               'status' : failed_status,
 130                               'changes': [c4p1._asdict()]}
 131             ).RecordCLAction(c4p1, constants.CL_ACTION_PICKED_UP, t.next()
 132             ).RecordCLAction(c4p1, constants.CL_ACTION_KICKED_OUT, t.next()),
 133         metadata_lib.CBuildbotMetadata(
 134             ).UpdateWithDict({'build-number' : 6,
 135                               'bot-config' : bot_config,
 136                               'results' : [],
 137                               'status' : failed_status,
 138                               'changes': [c4p1._asdict()]}
 139             ).RecordCLAction(c1p1, constants.CL_ACTION_PICKED_UP, t.next()
 140             ).RecordCLAction(c1p1, constants.CL_ACTION_KICKED_OUT, t.next())
 141       ]
 142     # pylint: enable=W0212
 143
 144     # TEST_METADATA should not be guaranteed to be ordered by build number
 145     # so shuffle it, but use the same seed each time so that unit test is
 146     # deterministic.
 147     random.seed(0)
 148     random.shuffle(TEST_METADATA)
 149
 150     # Wrap the test metadata into BuildData objects.
 151     TEST_BUILDDATA = [metadata_lib.BuildData('', d.GetDict())
 152                       for d in TEST_METADATA]
 153
 154     return TEST_BUILDDATA
 155
 156
 157   def testCLStatsSummary(self):
 158     with cros_build_lib.ContextManagerStack() as stack:
 159       pre_cq_builddata = self._getTestBuildData(cq=False)
 160       cq_builddata = self._getTestBuildData(cq=True)
 161       stack.Add(mock.patch.object, gather_builder_stats.StatsManager,
 162                 '_FetchBuildData', side_effect=[cq_builddata, pre_cq_builddata])
 163       stack.Add(mock.patch.object, gather_builder_stats, '_PrepareCreds')
 164       stack.Add(mock.patch.object, gather_builder_stats.CLStats,
 165                 'GatherFailureReasons')
 166       cl_stats = gather_builder_stats.CLStats('foo@bar.com')
 167       cl_stats.Gather(datetime.date.today())
 168       cl_stats.reasons = {1: '', 2: '', 3: REASON_BAD_CL, 4: REASON_BAD_CL}
 169       cl_stats.blames =  {1: '', 2: '', 3: 'crosreview.com/1',
 170                           4: 'crosreview.com/1'}
 171       summary = cl_stats.Summarize()
 172
 173       expected = {
 174           'mean_good_patch_rejections': 0.5,
 175           'unique_patches': 7,
 176           'unique_blames_change_count': 0,
 177           'total_cl_actions': 28,
 178           'good_patch_rejection_breakdown': [(0, 3), (1, 0), (2, 1)],
 179           'good_patch_rejection_count': {CQ: 1, PRE_CQ: 1},
 180           'good_patch_rejections': 2,
 181           'false_rejection_rate': {CQ: 20., PRE_CQ: 20., 'combined': 100./3,},
 182           'submitted_patches': 4,
 183           'submit_fails': 0,
 184           'unique_cls': 4,
 185           'median_handling_time': -1, # This will be ignored in comparison
 186           'patch_handling_time': -1,  # This will be ignored in comparison
 187           'bad_cl_candidates': {
 188               CQ: [metadata_lib.GerritChangeTuple(gerrit_number=2,
 189                                                   internal=True)],
 190               PRE_CQ: [metadata_lib.GerritChangeTuple(gerrit_number=2,
 191                                                       internal=True),
 192                        metadata_lib.GerritChangeTuple(gerrit_number=4,
 193                                                       internal=True)],
 194           },
 195           'correctly_rejected_by_stage': {CQ: {}, PRE_CQ: {}},
 196           'incorrectly_rejected_by_stage': {PRE_CQ: {}},
 197           'rejections': 10}
 198       # Ignore handling times in comparison, since these are not fully
 199       # reproducible from run to run of the unit test.
 200       summary['median_handling_time'] = expected['median_handling_time']
 201       summary['patch_handling_time'] = expected['patch_handling_time']
 202       self.maxDiff = None
 203       self.assertEqual(summary, expected)
 204
 205   def testProcessBlameString(self):
 206     """Tests that bug and CL links are correctly parsed."""
 207     blame = ('some words then crbug.com/1234, then other junk and '
 208              'https://code.google.com/p/chromium/issues/detail?id=4321 '
 209              'then some stuff and other stuff and b/2345 and also '
 210              'https://b.corp.google.com/issue?id=5432&query=5432 '
 211              'and then some crosreview.com/3456 or some '
 212              'https://chromium-review.googlesource.com/#/c/6543/ and '
 213              'then crosreview.com/i/9876 followed by '
 214              'https://chrome-internal-review.googlesource.com/#/c/6789/ '
 215              'blah https://gutsv3.corp.google.com/#ticket/1234 t/4321')
 216     expected = ['crbug.com/1234',
 217                 'crbug.com/4321',
 218                 'b/2345',
 219                 'b/5432',
 220                 'crosreview.com/3456',
 221                 'crosreview.com/6543',
 222                 'crosreview.com/i/9876',
 223                 'crosreview.com/i/6789',
 224                 't/1234',
 225                 't/4321']
 226     self.assertEqual(gather_builder_stats.CLStats.ProcessBlameString(blame),
 227                      expected)
 228
 229
 230 if __name__ == '__main__':
 231   cros_test_lib.main()