tools/profiling/microbenchmarks/bm_diff/bm_diff.py

   1 #!/usr/bin/env python2.7
   2 #
   3 # Copyright 2017 gRPC authors.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 """ Computes the diff between two bm runs and outputs significant results """
  17
  18 import bm_constants
  19 import bm_speedup
  20
  21 import sys
  22 import os
  23
  24 sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
  25 import bm_json
  26
  27 import json
  28 import tabulate
  29 import argparse
  30 import collections
  31 import subprocess
  32
  33 verbose = False
  34
  35
  36 def _median(ary):
  37     assert (len(ary))
  38     ary = sorted(ary)
  39     n = len(ary)
  40     if n % 2 == 0:
  41         return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
  42     else:
  43         return ary[n / 2]
  44
  45
  46 def _args():
  47     argp = argparse.ArgumentParser(
  48         description='Perform diff on microbenchmarks')
  49     argp.add_argument('-t',
  50                       '--track',
  51                       choices=sorted(bm_constants._INTERESTING),
  52                       nargs='+',
  53                       default=sorted(bm_constants._INTERESTING),
  54                       help='Which metrics to track')
  55     argp.add_argument('-b',
  56                       '--benchmarks',
  57                       nargs='+',
  58                       choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
  59                       default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
  60                       help='Which benchmarks to run')
  61     argp.add_argument(
  62         '-l',
  63         '--loops',
  64         type=int,
  65         default=20,
  66         help=
  67         'Number of times to loops the benchmarks. Must match what was passed to bm_run.py'
  68     )
  69     argp.add_argument('-r',
  70                       '--regex',
  71                       type=str,
  72                       default="",
  73                       help='Regex to filter benchmarks run')
  74     argp.add_argument('--counters', dest='counters', action='store_true')
  75     argp.add_argument('--no-counters', dest='counters', action='store_false')
  76     argp.set_defaults(counters=True)
  77     argp.add_argument('-n', '--new', type=str, help='New benchmark name')
  78     argp.add_argument('-o', '--old', type=str, help='Old benchmark name')
  79     argp.add_argument('-v',
  80                       '--verbose',
  81                       type=bool,
  82                       help='Print details of before/after')
  83     args = argp.parse_args()
  84     global verbose
  85     if args.verbose: verbose = True
  86     assert args.new
  87     assert args.old
  88     return args
  89
  90
  91 def _maybe_print(str):
  92     if verbose: print str
  93
  94
  95 class Benchmark:
  96
  97     def __init__(self):
  98         self.samples = {
  99             True: collections.defaultdict(list),
 100             False: collections.defaultdict(list)
 101         }
 102         self.final = {}
 103
 104     def add_sample(self, track, data, new):
 105         for f in track:
 106             if f in data:
 107                 self.samples[new][f].append(float(data[f]))
 108
 109     def process(self, track, new_name, old_name):
 110         for f in sorted(track):
 111             new = self.samples[True][f]
 112             old = self.samples[False][f]
 113             if not new or not old: continue
 114             mdn_diff = abs(_median(new) - _median(old))
 115             _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' %
 116                          (f, new_name, new, old_name, old, mdn_diff))
 117             s = bm_speedup.speedup(new, old, 1e-5)
 118             if abs(s) > 3:
 119                 if mdn_diff > 0.5 or 'trickle' in f:
 120                     self.final[f] = '%+d%%' % s
 121         return self.final.keys()
 122
 123     def skip(self):
 124         return not self.final
 125
 126     def row(self, flds):
 127         return [self.final[f] if f in self.final else '' for f in flds]
 128
 129
 130 def _read_json(filename, badjson_files, nonexistant_files):
 131     stripped = ".".join(filename.split(".")[:-2])
 132     try:
 133         with open(filename) as f:
 134             r = f.read()
 135             return json.loads(r)
 136     except IOError, e:
 137         if stripped in nonexistant_files:
 138             nonexistant_files[stripped] += 1
 139         else:
 140             nonexistant_files[stripped] = 1
 141         return None
 142     except ValueError, e:
 143         print r
 144         if stripped in badjson_files:
 145             badjson_files[stripped] += 1
 146         else:
 147             badjson_files[stripped] = 1
 148         return None
 149
 150
 151 def fmt_dict(d):
 152     return ''.join(["    " + k + ": " + str(d[k]) + "\n" for k in d])
 153
 154
 155 def diff(bms, loops, regex, track, old, new, counters):
 156     benchmarks = collections.defaultdict(Benchmark)
 157
 158     badjson_files = {}
 159     nonexistant_files = {}
 160     for bm in bms:
 161         for loop in range(0, loops):
 162             for line in subprocess.check_output([
 163                     'bm_diff_%s/opt/%s' % (old, bm), '--benchmark_list_tests',
 164                     '--benchmark_filter=%s' % regex
 165             ]).splitlines():
 166                 stripped_line = line.strip().replace("/", "_").replace(
 167                     "<", "_").replace(">", "_").replace(", ", "_")
 168                 js_new_opt = _read_json(
 169                     '%s.%s.opt.%s.%d.json' % (bm, stripped_line, new, loop),
 170                     badjson_files, nonexistant_files)
 171                 js_old_opt = _read_json(
 172                     '%s.%s.opt.%s.%d.json' % (bm, stripped_line, old, loop),
 173                     badjson_files, nonexistant_files)
 174                 if counters:
 175                     js_new_ctr = _read_json(
 176                         '%s.%s.counters.%s.%d.json' %
 177                         (bm, stripped_line, new, loop), badjson_files,
 178                         nonexistant_files)
 179                     js_old_ctr = _read_json(
 180                         '%s.%s.counters.%s.%d.json' %
 181                         (bm, stripped_line, old, loop), badjson_files,
 182                         nonexistant_files)
 183                 else:
 184                     js_new_ctr = None
 185                     js_old_ctr = None
 186
 187                 for row in bm_json.expand_json(js_new_ctr, js_new_opt):
 188                     name = row['cpp_name']
 189                     if name.endswith('_mean') or name.endswith('_stddev'):
 190                         continue
 191                     benchmarks[name].add_sample(track, row, True)
 192                 for row in bm_json.expand_json(js_old_ctr, js_old_opt):
 193                     name = row['cpp_name']
 194                     if name.endswith('_mean') or name.endswith('_stddev'):
 195                         continue
 196                     benchmarks[name].add_sample(track, row, False)
 197
 198     really_interesting = set()
 199     for name, bm in benchmarks.items():
 200         _maybe_print(name)
 201         really_interesting.update(bm.process(track, new, old))
 202     fields = [f for f in track if f in really_interesting]
 203
 204     headers = ['Benchmark'] + fields
 205     rows = []
 206     for name in sorted(benchmarks.keys()):
 207         if benchmarks[name].skip(): continue
 208         rows.append([name] + benchmarks[name].row(fields))
 209     note = None
 210     if len(badjson_files):
 211         note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict(
 212             badjson_files)
 213     if len(nonexistant_files):
 214         if note:
 215             note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(
 216                 nonexistant_files)
 217         else:
 218             note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(
 219                 nonexistant_files)
 220     if rows:
 221         return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note
 222     else:
 223         return None, note
 224
 225
 226 if __name__ == '__main__':
 227     args = _args()
 228     diff, note = diff(args.benchmarks, args.loops, args.regex, args.track,
 229                       args.old, args.new, args.counters)
 230     print('%s\n%s' % (note, diff if diff else "No performance differences"))