Tools/Scripts/bencher

   1 #!/usr/bin/env ruby
   2
   3 # Copyright (C) 2011 Apple Inc. All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions
   7 # are met:
   8 # 1. Redistributions of source code must retain the above copyright
   9 #    notice, this list of conditions and the following disclaimer.
  10 # 2. Redistributions in binary form must reproduce the above copyright
  11 #    notice, this list of conditions and the following disclaimer in the
  12 #    documentation and/or other materials provided with the distribution.
  13 #
  14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
  15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
  18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  24 # THE POSSIBILITY OF SUCH DAMAGE.
  25
  26 require 'rubygems'
  27
  28 require 'getoptlong'
  29 require 'pathname'
  30 require 'tempfile'
  31 require 'socket'
  32
  33 begin
  34   require 'json'
  35 rescue LoadError => e
  36   $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
  37   exit 1
  38 end
  39
  40 # Configuration
  41
  42 CONFIGURATION_FLNM = ENV["HOME"]+"/.bencher"
  43
  44 unless FileTest.exist? CONFIGURATION_FLNM
  45   $stderr.puts "Error: no configuration file at ~/.bencher."
  46   $stderr.puts "This file should contain paths to SunSpider, V8, and Kraken, as well as a"
  47   $stderr.puts "temporary directory that bencher can use for its remote mode. It should be"
  48   $stderr.puts "formatted in JSON.  For example:"
  49   $stderr.puts "{"
  50   $stderr.puts "    \"sunSpiderPath\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/sunspider-1.0\","
  51   $stderr.puts "    \"v8Path\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/v8-v6\","
  52   $stderr.puts "    \"krakenPath\": \"/Volumes/Data/pizlo/kraken/kraken-e119421cb325/tests/kraken-1.1\","
  53   $stderr.puts "    \"tempPath\": \"/Volumes/Data/pizlo/bencher/temp\""
  54   $stderr.puts "}"
  55   exit 1
  56 end
  57
  58 CONFIGURATION = JSON.parse(File::read(CONFIGURATION_FLNM))
  59
  60 SUNSPIDER_PATH = CONFIGURATION["sunSpiderPath"]
  61 V8_PATH = CONFIGURATION["v8Path"]
  62 KRAKEN_PATH = CONFIGURATION["krakenPath"]
  63 TEMP_PATH = CONFIGURATION["tempPath"]
  64
  65 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933,
  66             0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684,
  67             0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227,
  68             0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674,
  69             0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652,
  70             0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604,
  71             0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338,
  72             0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538,
  73             0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185,
  74             0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694,
  75             0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244,
  76             0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067,
  77             0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323,
  78             0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129,
  79             0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574,
  80             0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722,
  81             0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624,
  82             0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321,
  83             0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843,
  84             0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217,
  85             0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464,
  86             0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599,
  87             0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638,
  88             0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592,
  89             0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471,
  90             0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283,
  91             0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037,
  92             0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738,
  93             0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481,
  94             0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085,
  95             0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651,
  96             0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183,
  97             0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683,
  98             0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154,
  99             0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599,
 100             0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019,
 101             0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417,
 102             0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795,
 103             0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153,
 104             0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494,
 105             0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818,
 106             0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128,
 107             0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423,
 108             0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704,
 109             0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974,
 110             0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232,
 111             0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479,
 112             0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716,
 113             0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943,
 114             0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192,
 115             0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401,
 116             0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963,
 117             0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822,
 118             0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007,
 119             0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186,
 120             0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358,
 121             0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525,
 122             0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686,
 123             0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841,
 124             0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991,
 125             0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137,
 126             0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278,
 127             0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414,
 128             0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547,
 129             0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675,
 130             0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799,
 131             0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192,
 132             0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037,
 133             0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151,
 134             0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262,
 135             0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369,
 136             0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489,
 137             0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259,
 138             0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689,
 139             0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785,
 140             0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879,
 141             0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297,
 142             0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059,
 143             0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145,
 144             0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242,
 145             0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324,
 146             0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404,
 147             0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483,
 148             0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559,
 149             0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634,
 150             0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707,
 151             0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778,
 152             0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848,
 153             0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916,
 154             0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983,
 155             0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057,
 156             0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121,
 157             0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183,
 158             0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244,
 159             0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304,
 160             0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363,
 161             0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442,
 162             0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476,
 163             0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539,
 164             0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946,
 165             0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653,
 166             0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712,
 167             0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762,
 168             0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812,
 169             0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867,
 170             0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915,
 171             0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962,
 172             0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007,
 173             0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059,
 174             0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103,
 175             0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146,
 176             0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189,
 177             0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231,
 178             0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272,
 179             0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312,
 180             0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352,
 181             0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391,
 182             0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429,
 183             0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467,
 184             0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504,
 185             0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546,
 186             0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582,
 187             0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617,
 188             0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652,
 189             0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686,
 190             0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724,
 191             0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757,
 192             0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579,
 193             0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822,
 194             0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853,
 195             0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885,
 196             0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915,
 197             0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595,
 198             0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979,
 199             0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013,
 200             0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041,
 201             0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074,
 202             0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102,
 203             0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129,
 204             0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156,
 205             0.99616, 0.996164]
 206
 207 # Run-time configuration parameters (can be set with command-line options)
 208
 209 $rerun=1
 210 $inner=3
 211 $warmup=1
 212 $outer=4
 213 $includeSunSpider=true
 214 $includeV8=true
 215 $includeKraken=true
 216 $measureGC=false
 217 $benchmarkPattern=nil
 218 $verbosity=0
 219 $innerMode=:reload
 220 $timeMode=:auto
 221 $keepFiles=false
 222 $forceVMKind=nil
 223 $brief=false
 224 $silent=false
 225 $remoteHosts=[]
 226 $sshOptions=[]
 227 $slave=false
 228 $vms = []
 229
 230 # Helpful functions and classes
 231
 232 def smallUsage
 233   puts "Use the --help option to get basic usage information."
 234   exit 1
 235 end
 236
 237 def usage
 238   puts "bencher [options] <vm1> [<vm2> ...]"
 239   puts
 240   puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
 241   puts "benchmarks, and reports detailed statistics.  What makes bencher special is"
 242   puts "that each benchmark/VM configuration is run in a single VM invocation, and"
 243   puts "the invocations are run in random order.  This minimizes systematics due to"
 244   puts "one benchmark polluting the running time of another.  The fine-grained"
 245   puts "interleaving of VM invocations further minimizes systematics due to changes in"
 246   puts "the performance or behavior of your machine."
 247   puts
 248   puts "Bencher is highly configurable.  You can compare as many VMs as you like.  You"
 249   puts "can change the amount of warm-up iterations, number of iterations executed per"
 250   puts "VM invocation, and the number of VM invocations per benchmark.  By default,"
 251   puts "SunSpider, VM, and Kraken are all run; but you can run any combination of these"
 252   puts "suites."
 253   puts
 254   puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
 255   puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
 256   puts "the executable and <name> is the name that you would like to give the"
 257   puts "configuration for the purposeof reporting.  If no name is given, a generic name"
 258   puts "of the form Conf#<n> will be ascribed to the configuration automatically."
 259   puts
 260   puts "Options:"
 261   puts "--rerun <n>          Set the number of iterations of the benchmark that"
 262   puts "                     contribute to the measured run time.  Default is #{$rerun}."
 263   puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
 264   puts "                     iterations.  Default is #{$inner}."
 265   puts "--outer <n>          Set the number of runtime invocations for each benchmark."
 266   puts "                     Default is #{$outer}."
 267   puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
 268   puts "                     is #{$warmup}."
 269   puts "--timing-mode        Set the way that bencher measures time.  Possible values"
 270   puts "                     are 'preciseTime', 'date', and 'auto'.  Default is"
 271   puts "                     'auto', which automatically detects the best way."
 272   puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
 273   puts "                     the one specified.  Valid arguments are 'jsc' or"
 274   puts "                     'DumpRenderTree'."
 275   puts "--v8-only            Only run V8."
 276   puts "--sunspider-only     Only run SunSpider."
 277   puts "--kraken-only        Only run Kraken."
 278   puts "--exclude-v8         Exclude V8 (only run SunSpider and Kraken)."
 279   puts "--exclude-sunspider  Exclude SunSpider (only run V8 and Kraken)."
 280   puts "--exclude-kraken     Exclude Kraken (only run SunSpider and V8)."
 281   puts "--benchmarks         Only run benchmarks matching the given regular expression."
 282   puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
 283   puts "                     Works best with large values of --inner.  You can also say"
 284   puts "                     --measure-gc <conf>, which turns this on for one"
 285   puts "                     configuration only."
 286   puts "--keep-files         Keep temporary files.  Useful for debugging."
 287   puts "--verbose or -v      Print more stuff."
 288   puts "--brief              Print only the final result for each VM."
 289   puts "--silent             Don't print progress. This might slightly reduce some"
 290   puts "                     performance perturbation."
 291   puts "--remote <sshhosts>  Performance performance measurements remotely, on the given"
 292   puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
 293   puts "                     user@host string. However, you can also supply a comma-"
 294   puts "                     separated list of SSH hosts. Alternatively, you can use this"
 295   puts "                     option multiple times to specify multiple hosts. This"
 296   puts "                     automatically copies the WebKit release builds of the VMs"
 297   puts "                     you specified to all of the hosts."
 298   puts "--ssh-options        Pass additional options to SSH."
 299   puts "--help or -h         Display this message."
 300   puts
 301   puts "Example:"
 302   puts "bencher TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
 303   exit 1
 304 end
 305
 306 def fail(reason)
 307   if reason.respond_to? :backtrace
 308     puts "FAILED: #{reason}"
 309     puts "Stack trace:"
 310     puts reason.backtrace.join("\n")
 311   else
 312     puts "FAILED: #{reason}"
 313   end
 314   smallUsage
 315 end
 316
 317 def quickFail(r1,r2)
 318   $stderr.puts "#{$0}: #{r1}"
 319   puts
 320   fail(r2)
 321 end
 322
 323 def intArg(argName,arg,min,max)
 324   result=arg.to_i
 325   unless result.to_s == arg
 326     quickFail("Expected an integer value for #{argName}, but got #{arg}.",
 327               "Invalid argument for command-line option")
 328   end
 329   if min and result<min
 330     quickFail("Argument for #{argName} cannot be smaller than #{min}.",
 331               "Invalid argument for command-line option")
 332   end
 333   if max and result>max
 334     quickFail("Argument for #{argName} cannot be greater than #{max}.",
 335               "Invalid argument for command-line option")
 336   end
 337   result
 338 end
 339
 340 def computeMean(array)
 341   sum=0.0
 342   array.each {
 343     | value |
 344     sum += value
 345   }
 346   sum/array.length
 347 end
 348
 349 def computeGeometricMean(array)
 350   mult=1.0
 351   array.each {
 352     | value |
 353     mult*=value
 354   }
 355   mult**(1.0/array.length)
 356 end
 357
 358 def computeHarmonicMean(array)
 359   1.0 / computeMean(array.collect{ | value | 1.0 / value })
 360 end
 361
 362 def computeStdDev(array)
 363   case array.length
 364   when 0
 365     0.0/0.0
 366   when 1
 367     0.0
 368   else
 369     begin
 370       mean=computeMean(array)
 371       sum=0.0
 372       array.each {
 373         | value |
 374         sum += (value-mean)**2
 375       }
 376       Math.sqrt(sum/(array.length-1))
 377     rescue
 378       0.0/0.0
 379     end
 380   end
 381 end
 382
 383 class Array
 384   def shuffle!
 385     size.downto(1) { |n| push delete_at(rand(n)) }
 386     self
 387   end
 388 end
 389
 390 def inverseBetaRegularized(n)
 391   IBR_LOOKUP[n-1]
 392 end
 393
 394 def numToStr(num)
 395   "%.4f"%(num.to_f)
 396 end
 397
 398 class NoChange
 399   attr_reader :amountFaster
 400
 401   def initialize(amountFaster)
 402     @amountFaster = amountFaster
 403   end
 404
 405   def shortForm
 406     " "
 407   end
 408
 409   def to_s
 410     if @amountFaster < 1.01
 411       ""
 412     else
 413       "  might be #{numToStr(@amountFaster)}x faster"
 414     end
 415   end
 416 end
 417
 418 class Faster
 419   attr_reader :amountFaster
 420
 421   def initialize(amountFaster)
 422     @amountFaster = amountFaster
 423   end
 424
 425   def shortForm
 426     "^"
 427   end
 428
 429   def to_s
 430     "^ definitely #{numToStr(@amountFaster)}x faster"
 431   end
 432 end
 433
 434 class Slower
 435   attr_reader :amountSlower
 436
 437   def initialize(amountSlower)
 438     @amountSlower = amountSlower
 439   end
 440
 441   def shortForm
 442     "!"
 443   end
 444
 445   def to_s
 446     "! definitely #{numToStr(@amountSlower)}x slower"
 447   end
 448 end
 449
 450 class MayBeSlower
 451   attr_reader :amountSlower
 452
 453   def initialize(amountSlower)
 454     @amountSlower = amountSlower
 455   end
 456
 457   def shortForm
 458     "?"
 459   end
 460
 461   def to_s
 462     if @amountSlower < 1.01
 463       "?"
 464     else
 465       "? might be #{numToStr(@amountSlower)}x slower"
 466     end
 467   end
 468 end
 469
 470 class Stats
 471   def initialize
 472     @array = []
 473   end
 474
 475   def add(value)
 476     if value.is_a? Stats
 477       add(value.array)
 478     elsif value.respond_to? :each
 479       value.each {
 480         | v |
 481         add(v)
 482       }
 483     else
 484       @array << value.to_f
 485     end
 486   end
 487
 488   def array
 489     @array
 490   end
 491
 492   def sum
 493     result=0
 494     @array.each {
 495       | value |
 496       result += value
 497     }
 498     result
 499   end
 500
 501   def min
 502     @array.min
 503   end
 504
 505   def max
 506     @array.max
 507   end
 508
 509   def size
 510     @array.length
 511   end
 512
 513   def mean
 514     computeMean(array)
 515   end
 516
 517   def arithmeticMean
 518     mean
 519   end
 520
 521   def stdDev
 522     computeStdDev(array)
 523   end
 524
 525   def stdErr
 526     stdDev/Math.sqrt(size)
 527   end
 528
 529   # Computes a 95% Student's t distribution confidence interval
 530   def confInt
 531     if size < 2
 532       0.0/0.0
 533     else
 534       raise if size > 1000
 535       Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
 536     end
 537   end
 538
 539   def lower
 540     mean-confInt
 541   end
 542
 543   def upper
 544     mean+confInt
 545   end
 546
 547   def geometricMean
 548     computeGeometricMean(array)
 549   end
 550
 551   def harmonicMean
 552     computeHarmonicMean(array)
 553   end
 554
 555   def compareTo(other)
 556     if upper < other.lower
 557       Faster.new(other.mean/mean)
 558     elsif lower > other.upper
 559       Slower.new(mean/other.mean)
 560     elsif mean > other.mean
 561       MayBeSlower.new(mean/other.mean)
 562     else
 563       NoChange.new(other.mean/mean)
 564     end
 565   end
 566
 567   def to_s
 568     "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
 569   end
 570 end
 571
 572 def doublePuts(out1,out2,msg)
 573   out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
 574   out2.puts msg
 575 end
 576
 577 def benchRunHarness(vm, benchpath)
 578   $stderr.puts "running #{benchpath} with #{vm}..." if $verbosity>=1
 579   result=nil
 580   Tempfile.open("bencher") {
 581     | file |
 582     yield(file)
 583     file.flush
 584     begin
 585       result=vm.runAndReport(file.path)
 586     rescue => e
 587       $stderr.puts "Could not run #{file.path} because #{e}:"
 588       File.open(file.path,"r") {
 589         | inp |
 590         inp.each_line {
 591           | line |
 592           $stderr.puts "#{line}"
 593         }
 594       }
 595     end
 596     file.unlink unless $keepFiles
 597   }
 598   raise unless result and result.size == $inner
 599   result
 600 end
 601
 602 def emitTimeHelpers(file)
 603   case $timeMode
 604   when :preciseTime
 605     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
 606     doublePuts($stderr,file,"   return preciseTime()*1000")
 607     doublePuts($stderr,file,"}")
 608   when :date
 609     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
 610     doublePuts($stderr,file,"   return Date.now()")
 611     doublePuts($stderr,file,"}")
 612   else
 613     raise
 614   end
 615   doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
 616   doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
 617   $rerun.times {
 618     doublePuts($stderr,file,"   run(__bencher_what);")
 619   }
 620   doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
 621   doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
 622   doublePuts($stderr,file,"}")
 623 end
 624
 625 def emitBenchRunCode(vm, file, benchpath)
 626   emitTimeHelpers(file)
 627   case $innerMode
 628   when :reload
 629     case vm.vmType
 630     when :jsc
 631       $warmup.times {
 632         doublePuts($stderr,file,"__bencher_run(#{benchpath.inspect})")
 633         doublePuts($stderr,file,"gc();") unless vm.shouldMeasureGC
 634       }
 635       $inner.times {
 636         doublePuts($stderr,file,"print(\"Time: \"+__bencher_run(#{benchpath.inspect}));")
 637         doublePuts($stderr,file,"gc();") unless vm.shouldMeasureGC
 638       }
 639     when :dumpRenderTree
 640       doublePuts($stderr,file,"__bencher_count = 0;")
 641       doublePuts($stderr,file,"function __bencher_doNext(result) {")
 642       doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
 643       doublePuts($stderr,file,"        debug(\"Time: \"+result);")
 644       doublePuts($stderr,file,"    __bencher_count++;")
 645       doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
 646       doublePuts($stderr,file,"        __bencher_runImpl(#{benchpath.inspect}, __bencher_doNext);")
 647       doublePuts($stderr,file,"    else")
 648       doublePuts($stderr,file,"        quit();")
 649       doublePuts($stderr,file,"}")
 650       doublePuts($stderr,file,"__bencher_runImpl(#{benchpath.inspect}, __bencher_doNext);")
 651     else
 652       raise vm.vmType
 653     end
 654   when :loadOnce
 655     doublePuts($stderr,file,"function runit() {")
 656     File.open(benchpath,'r') {
 657       | inp |
 658       inp.each_line {
 659         | line |
 660         doublePuts($stderr,file,line)
 661       }
 662     }
 663     doublePuts($stderr,file,"}")
 664     $warmup.times {
 665       doublePuts($stderr,file,"runit();")
 666       doublePuts($stderr,file,"gc();") unless vm.shouldMeasureGC
 667     }
 668     $inner.times {
 669       doublePuts($stderr,file,"before = __bencher_curTimeMS();")
 670       $rerun.times {
 671         doublePuts($stderr,file,"runit();")
 672       }
 673       doublePuts($stderr,file,"after = __bencher_curTimeMS();")
 674       doublePuts($stderr,file,"print(\"Time: \"+(after-before));")
 675       doublePuts($stderr,file,"gc();") unless vm.shouldMeasureGC
 676     }
 677     if vm.vmType == :dumpRenderTree
 678       doublePuts($stderr,file,"quit();")
 679     end
 680   else
 681     raise "bad $innerMode: #{$innerMode}"
 682   end
 683 end
 684
 685 def runBenchmarkAndReport(vm, benchpath)
 686   benchRunHarness(vm, benchpath) {
 687     | file |
 688     emitBenchRunCode(vm, file, benchpath)
 689   }
 690 end
 691
 692 class StatsAccumulator
 693   def initialize
 694     @stats = []
 695     ($outer*$inner).times {
 696       @stats << Stats.new
 697     }
 698   end
 699
 700   def statsForIteration(outerIteration, innerIteration)
 701     @stats[outerIteration*$inner + innerIteration]
 702   end
 703
 704   def stats
 705     result = Stats.new
 706     @stats.each {
 707       | stat |
 708       result.add(yield stat)
 709     }
 710     result
 711   end
 712
 713   def geometricMeanStats
 714     stats {
 715       | stat |
 716       stat.geometricMean
 717     }
 718   end
 719
 720   def arithmeticMeanStats
 721     stats {
 722       | stat |
 723       stat.arithmeticMean
 724     }
 725   end
 726 end
 727
 728 class VM < StatsAccumulator
 729   def initialize(origPath, path, name, nameKind, svnRevision)
 730     super()
 731     @origPath = origPath
 732     @path = path
 733     @name = name
 734     @nameKind = nameKind
 735
 736     if $forceVMKind
 737       @vmType = $forceVMKind
 738     else
 739       Tempfile.open("bencher-vmtest") {
 740         | file |
 741         file.puts "print(\"here\");"
 742         file.flush
 743
 744         result = nil
 745         @vmType = :jsc
 746         run(file.path) {
 747           | inp |
 748           result = inp.read
 749           $stderr.puts "stdout: #{result}" if $verbosity>=2
 750         }
 751
 752         if result.chomp == "here"
 753           $stderr.puts "#{@name} is definitely a jsc-style VM." if $verbosity>=1
 754           @vmType = :jsc
 755         else
 756           $stderr.puts "Assuming that #{@name} is a DumpRenderTree-style VM." if $verbosity>=1
 757           @vmType = :dumpRenderTree
 758         end
 759       }
 760     end
 761
 762     @svnRevision = svnRevision
 763
 764     unless $slave
 765       # Try to detect information about the VM.
 766       if path =~ /\/WebKitBuild\/Release\/([a-zA-Z]+)$/
 767         @checkoutPath = $~.pre_match
 768         unless @svnRevision
 769           begin
 770             Dir.chdir(@checkoutPath) {
 771               $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
 772               IO.popen("svn info", "r") {
 773                 | inp |
 774                 inp.each_line {
 775                   | line |
 776                   if line =~ /Revision: ([0-9]+)/
 777                     @svnRevision = $1
 778                   end
 779                 }
 780               }
 781             }
 782             unless @svnRevision
 783               $stderr.puts "Warning: running svn info for #{name} silently failed."
 784             end
 785           rescue => e
 786             # Failed to detect svn revision.
 787             $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
 788           end
 789         end
 790       else
 791         $stderr.puts "Warning: could not identify checkout location for #{name}"
 792       end
 793     end
 794   end
 795
 796   def to_s
 797     @name
 798   end
 799
 800   def name
 801     @name
 802   end
 803
 804   def shouldMeasureGC
 805     $measureGC == true or ($measureGC == name)
 806   end
 807
 808   def origPath
 809     @origPath
 810   end
 811
 812   def path
 813     @path
 814   end
 815
 816   def nameKind
 817     @nameKind
 818   end
 819
 820   def vmType
 821     @vmType
 822   end
 823
 824   def checkoutPath
 825     @checkoutPath
 826   end
 827
 828   def svnRevision
 829     @svnRevision
 830   end
 831
 832   def printFunction
 833     case @vmType
 834     when :jsc
 835       "print"
 836     when :dumpRenderTree
 837       "debug"
 838     else
 839       raise @vmType
 840     end
 841   end
 842
 843   def run(filename)
 844     case @vmType
 845     when :jsc
 846       if @path =~ /\/Release\/([a-zA-Z]+)$/
 847         raise unless @path =~ /\/([a-zA-Z]+)$/
 848         libPath = $~.pre_match
 849         ENV["DYLD_LIBRARY_PATH"]=libPath
 850         ENV["DYLD_FRAMEWORK_PATH"]=libPath
 851       else
 852         ENV["DYLD_LIBRARY_PATH"]=""
 853         ENV["DYLD_FRAMEWORK_PATH"]=""
 854       end
 855       cmd = "#{@path} #{filename}"
 856       $stderr.puts ">> #{cmd}" if $verbosity>=2
 857       IO.popen(cmd,"r") {
 858         | inp |
 859         yield inp
 860       }
 861       $?.success?
 862     when :dumpRenderTree
 863       result = nil
 864       Tempfile.open(["bencher-htmldoc",".html"]) {
 865         | htmlFile |
 866         Tempfile.open("bencher-css") {
 867           | cssFile |
 868           doublePuts($stderr,cssFile,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
 869           Tempfile.open("bencher-pre") {
 870             | preFile |
 871             doublePuts($stderr,preFile,
 872                        "if (window.layoutTestController) {\n"+
 873                        "    layoutTestController.dumpAsText(window.enablePixelTesting);\n"+
 874                        "    layoutTestController.waitUntilDone();\n"+
 875                        "}\n"+
 876                        "\n"+
 877                        "function debug(msg)\n"+
 878                        "{\n"+
 879                        "    var span = document.createElement(\"span\");\n"+
 880                        "    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace\n"+
 881                        "    span.innerHTML = msg + '<br />';\n"+
 882                        "}\n"+
 883                        "\n"+
 884                        "function quit() {\n"+
 885                        "    layoutTestController.notifyDone();\n"+
 886                        "}\n"+
 887                        "\n"+
 888                        "__bencher_continuation=null;\n"+
 889                        "\n"+
 890                        "function reportResult(result) {\n"+
 891                        "    __bencher_continuation(result);\n"+
 892                        "}\n"+
 893                        "\n"+
 894                        "function __bencher_runImpl(filename, continuation) {\n"+
 895                        "    function doit() {\n"+
 896                        "        document.getElementById(\"frameparent\").innerHTML = \"\";\n"+
 897                        "        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";\n"+
 898                        "        var testFrame = document.getElementById(\"testframe\");\n"+
 899                        "        testFrame.contentDocument.open();\n"+
 900                        "        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div><script type=\\\"text/javascript\\\">__bencher_before = Date.now();</script><script src=\\\"file://\"+filename+\"\\\"></script><script type=\\\"text/javascript\\\">window.parent.reportResult(Date.now() - __bencher_before);</script></body></html>\");\n"+
 901                        "        testFrame.contentDocument.close();\n"+
 902                        "    }\n"+
 903                        "    __bencher_continuation = continuation;\n"+
 904                        "    window.setTimeout(doit, 10);\n"+
 905                        "}\n")
 906             doublePuts($stderr,htmlFile,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"file://#{cssFile.path}\"><script src=\"file://#{preFile.path}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"file://#{filename}\"></script></body></html>")
 907
 908             htmlFile.flush
 909             preFile.flush
 910             cssFile.flush
 911
 912             cmd = "#{path} #{htmlFile.path}"
 913             $stderr.puts ">> #{cmd}" if $verbosity>=2
 914             IO.popen(cmd, "r") {
 915               | inp |
 916               yield inp
 917             }
 918             result = $?
 919
 920             #sleep 10000
 921
 922             htmlFile.unlink unless $keepFiles
 923             preFile.unlink unless $keepFiles
 924             cssFile.unlink unless $keepFiles
 925           }
 926         }
 927       }
 928       result.success?
 929     else
 930       raise @vmType
 931     end
 932   end
 933
 934   def runAndReport(filename)
 935     result = nil
 936     5.times {
 937       result = Stats.new
 938       run(filename) {
 939         | inp |
 940         inp.each_line {
 941           | line |
 942           $stderr.puts "stdout: #{line}" if $verbosity>=2
 943           if line =~ /^Time: /
 944             result.add($~.post_match.to_f)
 945           end
 946         }
 947       }
 948       break if result.size == $inner
 949       if $verbosity == 0
 950         $stderr.print "   "
 951       end
 952       $stderr.puts "Failed to run #{filename} on #{self}, retrying..."
 953     }
 954     raise "Failed to run #{filename} on #{self}" unless result and result.size == $inner
 955     result
 956   end
 957 end
 958
 959 class SunSpiderBenchmark
 960   attr_accessor :benchmarkSuite
 961
 962   def initialize(name)
 963     @name = name
 964   end
 965
 966   def to_s
 967     @name
 968   end
 969
 970   def runAndReport(vm)
 971     runBenchmarkAndReport(vm, "#{SUNSPIDER_PATH}/#{@name}.js")
 972   end
 973 end
 974
 975 class V8Benchmark
 976   attr_accessor :benchmarkSuite
 977
 978   def initialize(name)
 979     @name = name
 980   end
 981
 982   def to_s
 983     @name
 984   end
 985
 986   def runAndReport(vm)
 987     runBenchmarkAndReport(vm, "#{V8_PATH}/v8-#{@name}.js")
 988   end
 989 end
 990
 991 class KrakenBenchmark
 992   attr_accessor :benchmarkSuite
 993
 994   def initialize(name)
 995     @name = name
 996   end
 997
 998   def to_s
 999     @name
1000   end
1001
1002   def runAndReport(vm)
1003     # Kraken requires some special-casing
1004
1005     dataPath="#{KRAKEN_PATH}/#{@name}-data.js"
1006     benchPath="#{KRAKEN_PATH}/#{@name}.js"
1007
1008     benchRunHarness(vm, benchPath) {
1009       | file |
1010       emitTimeHelpers(file)
1011       doublePuts($stderr,file,"load(#{dataPath.inspect});")
1012       doublePuts($stderr,file,"gc();") unless vm.shouldMeasureGC
1013       if $innerMode == :reload
1014         case vm.vmType
1015         when :jsc
1016           doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
1017           doublePuts($stderr,file,"   before = __bencher_curTimeMS();")
1018           $rerun.times {
1019             doublePuts($stderr,file,"   load(#{benchPath.inspect});")
1020           }
1021           doublePuts($stderr,file,"   after = __bencher_curTimeMS();")
1022           doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) #{vm.printFunction}(\"Time: \"+(after-before));");
1023           doublePuts($stderr,file,"   gc();") unless vm.shouldMeasureGC
1024           doublePuts($stderr,file,"}")
1025         when :dumpRenderTree
1026           raise "Kraken in DumpRenderTree is currently unsupported."
1027         else
1028           raise vm.vmType
1029         end
1030       else
1031         emitBenchRunCode(vm, file, benchPath)
1032       end
1033     }
1034   end
1035 end
1036
1037 class BenchmarkSuite
1038   def initialize(name, path, preferredMean)
1039     @name = name
1040     @path = path
1041     @preferredMean = preferredMean
1042     @benchmarks = []
1043   end
1044
1045   def name
1046     @name
1047   end
1048
1049   def to_s
1050     @name
1051   end
1052
1053   def path
1054     @path
1055   end
1056
1057   def add(benchmark)
1058     if not $benchmarkPattern or "#{@name}/#{benchmark.to_s}" =~ $benchmarkPattern
1059       benchmark.benchmarkSuite = self
1060       @benchmarks << benchmark
1061     end
1062   end
1063
1064   def benchmarks
1065     @benchmarks
1066   end
1067
1068   def empty?
1069     @benchmarks.empty?
1070   end
1071
1072   def retain_if
1073     @benchmarks.delete_if {
1074       | benchmark |
1075       not yield benchmark
1076     }
1077   end
1078
1079   def preferredMean
1080     @preferredMean
1081   end
1082
1083   def computeMean(stat)
1084     stat.send @preferredMean
1085   end
1086 end
1087
1088 class BenchmarkOnVM
1089   def initialize(vm, benchmark, suiteOnVM)
1090     @vm = vm
1091     @benchmark = benchmark
1092     @suiteOnVM = suiteOnVM
1093     @stats = Stats.new
1094   end
1095
1096   def to_s
1097     "#{@benchmark} on #{@vm}"
1098   end
1099
1100   def benchmark
1101     @benchmark
1102   end
1103
1104   def vm
1105     @vm
1106   end
1107
1108   def suite
1109     @benchmark.benchmarkSuite
1110   end
1111
1112   def suiteOnVM
1113     @suiteOnVM
1114   end
1115
1116   def stats
1117     @stats
1118   end
1119
1120   def runRecordAndReport
1121     result = @benchmark.runAndReport(@vm)
1122     @stats.add(result)
1123     result
1124   end
1125 end
1126
1127 class SuiteOnVM < StatsAccumulator
1128   def initialize(vm, suite)
1129     super()
1130     @vm = vm
1131     @suite = suite
1132   end
1133
1134   def to_s
1135     "#{@suite} on #{@vm}"
1136   end
1137
1138   def suite
1139     @suite
1140   end
1141
1142   def vm
1143     @vm
1144   end
1145 end
1146
1147 class BenchPlan
1148   def initialize(benchmarkOnVM, iteration)
1149     @benchmarkOnVM = benchmarkOnVM
1150     @iteration = iteration
1151   end
1152
1153   def to_s
1154     "#{@benchmarkOnVM} \##{@iteration+1}"
1155   end
1156
1157   def benchmarkOnVM
1158     @benchmarkOnVM
1159   end
1160
1161   def benchmark
1162     @benchmarkOnVM.benchmark
1163   end
1164
1165   def suite
1166     @benchmarkOnVM.suite
1167   end
1168
1169   def vm
1170     @benchmarkOnVM.vm
1171   end
1172
1173   def iteration
1174     @iteration
1175   end
1176
1177   def runAndRecord
1178     @benchmarkOnVM.runRecordAndReport.array.each_with_index {
1179       | value, index |
1180       @benchmarkOnVM.vm.statsForIteration(@iteration, index).add(value)
1181       @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, index).add(value)
1182     }
1183   end
1184 end
1185
1186 def lpad(str,chars)
1187   if str.length>chars
1188     str
1189   else
1190     "%#{chars}s"%(str)
1191   end
1192 end
1193
1194 def rpad(str,chars)
1195   while str.length<chars
1196     str+=" "
1197   end
1198   str
1199 end
1200
1201 def center(str,chars)
1202   while str.length<chars
1203     str+=" "
1204     if str.length<chars
1205       str=" "+str
1206     end
1207   end
1208   str
1209 end
1210
1211 def statsToStr(stats)
1212   if $inner*$outer == 1
1213     string = numToStr(stats.mean)
1214     raise unless string =~ /\./
1215     left = $~.pre_match
1216     right = $~.post_match
1217     lpad(left,12)+"."+rpad(right,9)
1218   else
1219     lpad(numToStr(stats.mean),11)+"+-"+rpad(numToStr(stats.confInt),9)
1220   end
1221 end
1222
1223 begin
1224   $sawBenchOptions = false
1225
1226   def resetBenchOptionsIfNecessary
1227     unless $sawBenchOptions
1228       $includeSunSpider = false
1229       $includeV8 = false
1230       $includeKraken = false
1231       $sawBenchOptions = true
1232     end
1233   end
1234
1235   GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
1236                  ['--inner', GetoptLong::REQUIRED_ARGUMENT],
1237                  ['--outer', GetoptLong::REQUIRED_ARGUMENT],
1238                  ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
1239                  ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
1240                  ['--sunspider-only', GetoptLong::NO_ARGUMENT],
1241                  ['--v8-only', GetoptLong::NO_ARGUMENT],
1242                  ['--kraken-only', GetoptLong::NO_ARGUMENT],
1243                  ['--exclude-sunspider', GetoptLong::NO_ARGUMENT],
1244                  ['--exclude-v8', GetoptLong::NO_ARGUMENT],
1245                  ['--exclude-kraken', GetoptLong::NO_ARGUMENT],
1246                  ['--sunspider', GetoptLong::NO_ARGUMENT],
1247                  ['--v8', GetoptLong::NO_ARGUMENT],
1248                  ['--kraken', GetoptLong::NO_ARGUMENT],
1249                  ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
1250                  ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
1251                  ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
1252                  ['--load-once', GetoptLong::NO_ARGUMENT],
1253                  ['--keep-files', GetoptLong::NO_ARGUMENT],
1254                  ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
1255                  ['--brief', GetoptLong::NO_ARGUMENT],
1256                  ['--silent', GetoptLong::NO_ARGUMENT],
1257                  ['--remote', GetoptLong::REQUIRED_ARGUMENT],
1258                  ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
1259                  ['--slave', GetoptLong::NO_ARGUMENT],
1260                  ['--vms', GetoptLong::REQUIRED_ARGUMENT],
1261                  ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
1262     | opt, arg |
1263     case opt
1264     when '--rerun'
1265       $rerun = intArg(opt,arg,1,nil)
1266     when '--inner'
1267       $inner = intArg(opt,arg,1,nil)
1268     when '--outer'
1269       $outer = intArg(opt,arg,1,nil)
1270     when '--warmup'
1271       $warmup = intArg(opt,arg,0,nil)
1272     when '--timing-mode'
1273       if arg.upcase == "PRECISETIME"
1274         $timeMode = :preciseTime
1275       elsif arg.upcase == "DATE"
1276         $timeMode = :date
1277       elsif arg.upcase == "AUTO"
1278         $timeMode = :auto
1279       else
1280         quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
1281                   "Invalid argument for command-line option")
1282       end
1283     when '--force-vm-kind'
1284       if arg.upcase == "JSC"
1285         $forceVMKind = :jsc
1286       elsif arg.upcase == "DUMPRENDERTREE"
1287         $forceVMKind = :dumpRenderTree
1288       elsif arg.upcase == "AUTO"
1289         $forceVMKind = nil
1290       else
1291         quickFail("Expected either 'jsc' or 'DumpRenderTree' for --force-vm-kind, but got '#{arg}'.",
1292                   "Invalid argument for command-line option")
1293       end
1294     when '--sunspider-only'
1295       $includeV8 = false
1296       $includeKraken = false
1297     when '--v8-only'
1298       $includeSunSpider = false
1299       $includeKraken = false
1300     when '--kraken-only'
1301       $includeSunSpider = false
1302       $includeV8 = false
1303     when '--exclude-sunspider'
1304       $includeSunSpider = false
1305     when '--exclude-v8'
1306       $includeV8 = false
1307     when '--exclude-kraken'
1308       $includeKraken = false
1309     when '--sunspider'
1310       resetBenchOptionsIfNecessary
1311       $includeSunSpider = true
1312     when '--v8'
1313       resetBenchOptionsIfNecessary
1314       $includeV8 = true
1315     when '--kraken'
1316       resetBenchOptionsIfNecessary
1317       $includeKraken = true
1318     when '--benchmarks'
1319       $benchmarkPattern = Regexp.new(arg)
1320     when '--measure-gc'
1321       if arg == ''
1322         $measureGC = true
1323       else
1324         $measureGC = arg
1325       end
1326     when '--load-once'
1327       $innerMode = :loadOnce
1328     when '--keep-files'
1329       $keepFiles = true
1330     when '--verbose'
1331       $verbosity += 1
1332     when '--brief'
1333       $brief = true
1334     when '--silent'
1335       $silent = true
1336     when '--remote'
1337       $remoteHosts += arg.split(',')
1338     when '--ssh-options'
1339       $sshOptions << arg
1340     when '--slave'
1341       $slave = true
1342     when '--vms'
1343       JSON.parse(File::read(arg)).each {
1344         | vmDescriptor |
1345         $vms << VM.new(vmDescriptor["origPath"],
1346                        vmDescriptor["path"],
1347                        vmDescriptor["name"],
1348                        vmDescriptor["nameKind"].to_sym,
1349                        vmDescriptor["svnRevision"])
1350       }
1351     when '--help'
1352       usage
1353     else
1354       raise "bad option: #{opt}"
1355     end
1356   }
1357
1358   SUNSPIDER = BenchmarkSuite.new("SunSpider", SUNSPIDER_PATH, :arithmeticMean)
1359   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
1360    "access-fannkuch", "access-nbody", "access-nsieve",
1361    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
1362    "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
1363    "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
1364    "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
1365    "string-base64", "string-fasta", "string-tagcloud",
1366    "string-unpack-code", "string-validate-input"].each {
1367     | name |
1368     SUNSPIDER.add SunSpiderBenchmark.new(name)
1369   }
1370
1371   V8 = BenchmarkSuite.new("V8", V8_PATH, :geometricMean)
1372   ["crypto", "deltablue", "earley-boyer", "raytrace",
1373    "regexp", "richards", "splay"].each {
1374     | name |
1375     V8.add V8Benchmark.new(name)
1376   }
1377
1378   KRAKEN = BenchmarkSuite.new("Kraken", KRAKEN_PATH, :arithmeticMean)
1379   ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
1380    "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
1381    "imaging-gaussian-blur", "json-parse-financial",
1382    "json-stringify-tinderbox", "stanford-crypto-aes",
1383    "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
1384    "stanford-crypto-sha256-iterative"].each {
1385     | name |
1386     KRAKEN.add KrakenBenchmark.new(name)
1387   }
1388
1389   ARGV.each {
1390     | vm |
1391     if vm =~ /([a-zA-Z0-9_ ]+):/
1392       name = $1
1393       nameKind = :given
1394       vm = $~.post_match
1395     else
1396       name = "Conf\##{$vms.length+1}"
1397       nameKind = :auto
1398     end
1399     $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
1400     $vms << VM.new(Pathname.new(vm).realpath, vm, name, nameKind, nil)
1401   }
1402
1403   if $vms.empty?
1404     quickFail("Please specify at least on configuraiton on the command line.",
1405               "Insufficient arguments")
1406   end
1407
1408   if $measureGC and $measureGC != true
1409     found = false
1410     $vms.each {
1411       | vm |
1412       if vm.name == $measureGC
1413         found = true
1414       end
1415     }
1416     unless found
1417       $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
1418     end
1419   end
1420
1421   if $timeMode == :auto
1422     havePreciseTime = true
1423     $vms.each {
1424       | vm |
1425       if vm.vmType == :dumpRenderTree
1426         $stderr.puts "Warning: #{vm} does not have preciseTime() because it is DumpRenderTree; using Date.now instead."
1427         if $rerun != 1
1428           quickFail("Cannot use --rerun with a DumpRenderTree style VM, because this support has not yet been implemented.", "Wrong option for VM type.")
1429         end
1430         havePreciseTime = false
1431       else
1432         Tempfile.open("bencher-timetest") {
1433           | file |
1434           doublePuts($stderr,file,"#{vm.printFunction}(\"Time: \"+preciseTime());")
1435           file.flush
1436           result = nil
1437           vm.run(file.path) {
1438             | inp |
1439             result = inp.read
1440           }
1441           thisVMHasPreciseTime = false
1442           if result =~ /Time: /
1443             secs = $~.post_match.to_i
1444             if (secs - Time.now.to_i).abs < 5
1445               thisVMHasPreciseTime = true
1446             end
1447           end
1448           unless thisVMHasPreciseTime
1449             $stderr.puts "Warning: #{vm} does not have preciseTime(); using Date.now instead."
1450             havePreciseTime = false
1451           end
1452           file.unlink unless $keepFiles
1453         }
1454       end
1455     }
1456
1457     if havePreciseTime
1458       $timeMode = :preciseTime
1459     else
1460       $timeMode = :date
1461     end
1462   end
1463
1464   if $outer*$inner == 1
1465     $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
1466   end
1467
1468   $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
1469
1470   $suites = []
1471
1472   if $includeSunSpider and not SUNSPIDER.empty?
1473     $suites << SUNSPIDER
1474   end
1475
1476   if $includeV8 and not V8.empty?
1477     $suites << V8
1478   end
1479
1480   if $includeKraken and not KRAKEN.empty?
1481     $suites << KRAKEN
1482   end
1483
1484   $benchmarks = []
1485   $suites.each {
1486     | suite |
1487     $benchmarks += suite.benchmarks
1488   }
1489
1490   $suitesOnVMs = []
1491   $suitesOnVMsForSuite = {}
1492   $suites.each {
1493     | suite |
1494     $suitesOnVMsForSuite[suite] = []
1495   }
1496   $suitesOnVMsForVM = {}
1497   $vms.each {
1498     | vm |
1499     $suitesOnVMsForVM[vm] = []
1500   }
1501
1502   $benchmarksOnVMs = []
1503   $benchmarksOnVMsForBenchmark = {}
1504   $benchmarks.each {
1505     | benchmark |
1506     $benchmarksOnVMsForBenchmark[benchmark] = []
1507   }
1508
1509   $vms.each {
1510     | vm |
1511     $suites.each {
1512       | suite |
1513       suiteOnVM = SuiteOnVM.new(vm, suite)
1514       $suitesOnVMs << suiteOnVM
1515       $suitesOnVMsForSuite[suite] << suiteOnVM
1516       $suitesOnVMsForVM[vm] << suiteOnVM
1517       suite.benchmarks.each {
1518         | benchmark |
1519         benchmarkOnVM = BenchmarkOnVM.new(vm, benchmark, suiteOnVM)
1520         $benchmarksOnVMs << benchmarkOnVM
1521         $benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
1522       }
1523     }
1524   }
1525
1526   unless $remoteHosts.empty?
1527     # Handle the remote benchmarking case.
1528
1529     raise unless TEMP_PATH
1530
1531     $stderr.puts "Packaging VM builds for remote hosts..." if $verbosity==0
1532
1533     # First package all of the release builds.
1534     $vms.each_with_index {
1535       | vm, index |
1536       unless vm.checkoutPath
1537         fail("Could not figure out how to package #{vm.name}, because the VM type was not recognized", "Bad VM kind for --remote")
1538       end
1539
1540       Dir.chdir(vm.checkoutPath + "/WebKitBuild") {
1541         cmd = "tar -czf #{TEMP_PATH}/vm#{index}.tar.gz Release"
1542         $stderr.puts ">> #{cmd}" if $verbosity>=2
1543         raise unless system(cmd)
1544       }
1545     }
1546
1547     # Now actually go talk to the remote hosts.
1548
1549     def sshRead(host, command)
1550       cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{host.inspect} #{command.inspect}"
1551       $stderr.puts ">> #{cmd}" if $verbosity>=2
1552       result = ""
1553       IO.popen(cmd, "r") {
1554         | inp |
1555         inp.each_line {
1556           | line |
1557           $stderr.puts "#{host}: #{line}" if $verbosity>=2
1558           result += line
1559         }
1560       }
1561       raise "#{$?}" unless $?.success?
1562       result
1563     end
1564
1565     def sshWrite(host, command, data)
1566       cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{host.inspect} #{command.inspect}"
1567       $stderr.puts ">> #{cmd}" if $verbosity>=2
1568       IO.popen(cmd, "w") {
1569         | outp |
1570         outp.write(data)
1571       }
1572       raise "#{$?}" unless $?.success?
1573     end
1574
1575     $remoteHosts.each {
1576       | host |
1577
1578       $stderr.puts "Sending VM builds to #{host}..." if $verbosity==0
1579
1580       remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
1581       raise unless remoteTempPath
1582
1583       sshWrite(host, "cd #{remoteTempPath.inspect} && cat > bencher && chmod 700 bencher", IO::read($0))
1584
1585       $vms.size.times {
1586         | index |
1587         sshWrite(host, "cd #{remoteTempPath.inspect} && rm -rf vm#{index} && mkdir vm#{index} && cd vm#{index} && tar -xzf /dev/stdin", IO::read("#{TEMP_PATH}/vm#{index}.tar.gz"))
1588       }
1589
1590       config=[]
1591       $vms.each_with_index {
1592         | vm, index |
1593         config << {
1594           "origPath" => vm.origPath,
1595           "path" => "#{remoteTempPath}/vm#{index}/Release/jsc",
1596           "name" => vm.name,
1597           "nameKind" => vm.nameKind.to_s,
1598           "svnRevision" => vm.svnRevision
1599         }
1600       }
1601
1602       sshWrite(host, "cd #{remoteTempPath.inspect} && (cat > vms.conf)", config.to_json)
1603
1604       $stderr.puts "Running on #{host}..." if $verbosity==0
1605
1606       cmd = "cd #{remoteTempPath.inspect} && ./bencher"
1607       cmd += " --rerun #{$rerun}"
1608       cmd += " --inner #{$inner}"
1609       cmd += " --outer #{$outer}"
1610       cmd += " --warmup #{$warmup}"
1611       cmd += " --timing-mode #{$timeMode.to_s}"
1612       cmd += " --sunspider" if $includeSunSpider
1613       cmd += " --v8" if $includeV8
1614       cmd += " --kraken" if $includeKraken
1615       if $measureGC
1616         if $measureGC == true
1617           cmd += " --measure-gc"
1618         else
1619           cmd += " --measure-gc #{$measureGC.inspect}"
1620         end
1621       end
1622       if $verbosity>=1
1623         cmd += " -"
1624         $verbosity.times {
1625           cmd += "v"
1626         }
1627       end
1628       cmd += " --brief" if $brief
1629       cmd += " --silent" if $silent
1630       cmd += " --slave"
1631       cmd += " --vms #{(remoteTempPath+'/vms.conf').inspect}"
1632       result = sshRead(host, cmd)
1633       puts result
1634       if result =~ /Generating benchmark report at (.*)\n\n/
1635         reportName = $1
1636         report = $~.post_match
1637         File.open(reportName, "w") {
1638           | outp |
1639           outp.puts report
1640         }
1641       end
1642     }
1643
1644     exit 0
1645   end
1646
1647   $plans = []
1648   $benchmarksOnVMs.each {
1649     | benchmarkOnVM |
1650     $outer.times {
1651       | iteration |
1652       $plans << BenchPlan.new(benchmarkOnVM, iteration)
1653     }
1654   }
1655
1656   $plans.shuffle!
1657
1658   $suitepad = $suites.collect {
1659     | suite |
1660     suite.to_s.size
1661   }.max + 1
1662
1663   $benchpad = ($benchmarks +
1664                ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
1665     | benchmark |
1666     benchmark.to_s.size
1667   }.max + 1
1668
1669   $vmpad = $vms.collect {
1670     | vm |
1671     vm.to_s.size
1672   }.max + 1
1673
1674   $plans.each_with_index {
1675     | plan, idx |
1676     if $verbosity == 0 and not $silent
1677       text1 = lpad(idx.to_s,$plans.size.to_s.size)+"/"+$plans.size.to_s
1678       text2 = plan.suite.to_s+"/"+plan.benchmark.to_s+"/"+plan.vm.to_s
1679       $stderr.print "\r#{text1} #{rpad(text2,$suitepad+1+$benchpad+1+$vmpad)}"
1680       $stderr.print "\r#{text1} #{text2}"
1681       $stderr.flush
1682     end
1683     plan.runAndRecord
1684   }
1685
1686   if $verbosity == 0 and not $silent
1687     $stderr.print "\r#{$plans.size}/#{$plans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}"
1688     $stderr.puts "\r#{$plans.size}/#{$plans.size}"
1689   end
1690
1691   # Compute the geomean of the preferred means of results on a SuiteOnVM
1692   $overallResults = []
1693   $vms.each {
1694     | vm |
1695     result = Stats.new
1696     $outer.times {
1697       | outerIndex |
1698       $inner.times {
1699         | innerIndex |
1700         curResult = Stats.new
1701         $suitesOnVMsForVM[vm].each {
1702           | suiteOnVM |
1703           # For a given iteration, suite, and VM, compute the suite's preferred mean
1704           # over the data collected for all benchmarks in that suite. We'll have one
1705           # sample per benchmark. For example on V8 this will be the geomean of 1
1706           # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
1707           # splay.
1708           curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
1709         }
1710
1711         # curResult now holds 1 sample for each of the means computed in the above
1712         # loop. Compute the geomean over this, and store it.
1713         result.add(curResult.geometricMean)
1714       }
1715     }
1716
1717     # $overallResults will have a Stats for each VM. That Stats object will hold
1718     # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
1719     # confidence interval of the geomeans of preferred means. Convoluted, but
1720     # useful and probably sound.
1721     $overallResults << result
1722   }
1723
1724   if $verbosity >= 2
1725     $benchmarksOnVMs.each {
1726       | benchmarkOnVM |
1727       $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
1728     }
1729
1730     $vms.each {
1731       | vm |
1732       $stderr.puts "#{vm} (arithmeticMean): #{vm.arithmeticMeanStats}"
1733       $stderr.puts "#{vm} (geometricMean): #{vm.geometricMeanStats}"
1734     }
1735   end
1736
1737   reportName =
1738     (if ($vms.collect {
1739            | vm |
1740            vm.nameKind
1741          }.index :auto)
1742        ""
1743      else
1744        $vms.collect {
1745          | vm |
1746          vm.to_s
1747        }.join("_") + "_"
1748      end) +
1749     ($suites.collect {
1750        | suite |
1751        suite.to_s
1752      }.join("")) + "_" +
1753     (begin
1754        time = Time.now
1755        "%04d%02d%02d_%02d%02d" %
1756          [ time.year, time.month, time.day,
1757            time.hour, time.min ]
1758      end) +
1759     "_benchReport.txt"
1760
1761   unless $brief
1762     puts "Generating benchmark report at #{reportName}"
1763   end
1764
1765   outp = $stdout
1766   begin
1767     outp = File.open(reportName,"w")
1768   rescue => e
1769     $stderr.puts "Error: could not save report to #{reportName}: #{e}"
1770     $stderr.puts
1771   end
1772
1773   def createVMsString
1774     result = ""
1775     result += "   " if $suites.size > 1
1776     result += rpad("", $benchpad)
1777     result += " "
1778     $vms.size.times {
1779       | index |
1780       if index != 0
1781         result += " "+NoChange.new(0).shortForm
1782       end
1783       result += lpad(center($vms[index].name, 9+9+2), 11+9+2)
1784     }
1785     result += "    "
1786     if $vms.size >= 3
1787       result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
1788     else
1789       result += " "*26
1790     end
1791     result
1792   end
1793
1794   $columns = createVMsString.size
1795
1796   def plural(num)
1797     if num == 1
1798       ""
1799     else
1800       "s"
1801     end
1802   end
1803
1804   def wrap(str)
1805     array = str.split
1806     result = ""
1807     curLine = array.shift
1808     array.each {
1809       | curStr |
1810       if (curLine + " " + curStr).size > $columns
1811         result += curLine + "\n"
1812         curLine = curStr
1813       else
1814         curLine += " " + curStr
1815       end
1816     }
1817     result + curLine + "\n"
1818   end
1819
1820   outp.print "Benchmark report for "
1821   if $suites.size == 1
1822     outp.print $suites[0].to_s
1823   elsif $suites.size == 2
1824     outp.print "#{$suites[0]} and #{$suites[1]}"
1825   else
1826     outp.print "#{$suites[0..-2].join(', ')}, and #{$suites[-1]}"
1827   end
1828   outp.print " on #{Socket.gethostname}"
1829   begin
1830     IO.popen("sysctl hw.model", "r") {
1831       | inp |
1832       inp.each_line {
1833         | line |
1834         if line =~ /hw.model: /
1835           outp.print " (#{$~.post_match.chomp})"
1836           break
1837         end
1838       }
1839     }
1840   rescue => e
1841     # Silently fail since this isn't critical.
1842   end
1843   outp.puts "."
1844   outp.puts
1845
1846   # This looks stupid; revisit later.
1847   if false
1848     $suites.each {
1849       | suite |
1850       outp.puts "#{suite} at #{suite.path}"
1851     }
1852
1853     outp.puts
1854   end
1855
1856   outp.puts "VMs tested:"
1857   $vms.each {
1858     | vm |
1859     outp.print "\"#{vm.name}\" at #{vm.origPath}"
1860     if vm.svnRevision
1861       outp.print " (r#{vm.svnRevision})"
1862     end
1863     outp.puts
1864   }
1865
1866   outp.puts
1867
1868   outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
1869                  "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
1870                  (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
1871                                       "total time of those iterations, for each sample.")
1872                   else "" end)+
1873                  (if $measureGC == true then (" No manual garbage collection invocations were "+
1874                                               "emitted.")
1875                   elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
1876                                          "all VMs except #{$measureGC}.")
1877                   else (" Emitted a call to gc() between sample measurements.") end)+
1878                  (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
1879                                         "began with the very first iteration.")
1880                   else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
1881                         "invocation for warm-up.") end)+
1882                  (case $timeMode
1883                   when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
1884                                           "microsecond-level timing.")
1885                   when :date then (" Used the portable Date.now() method to get millisecond-"+
1886                                    "level timing.")
1887                   else raise end)+
1888                  (case $innerMode
1889                   when :reload then "" # nothing interesting to say
1890                   when :loadOnce then (" Ran benchmarks using an experimental mode that "+
1891                                        "ensures that the benchmark code is loaded only once "+
1892                                        "during each VM invocation, which enables more VM "+
1893                                        "warm-up.")
1894                   else raise end)+
1895                  " Reporting benchmark execution times with 95% confidence "+
1896                  "intervals in milliseconds.")
1897
1898   outp.puts
1899
1900   def printVMs(outp)
1901     outp.puts createVMsString
1902   end
1903
1904   def summaryStats(outp, accumulators, name, &proc)
1905     outp.print "   " if $suites.size > 1
1906     outp.print rpad(name, $benchpad)
1907     outp.print " "
1908     accumulators.size.times {
1909       | index |
1910       if index != 0
1911         outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
1912       end
1913       outp.print statsToStr(accumulators[index].stats(&proc))
1914     }
1915     if accumulators.size>=2
1916       outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).to_s)
1917     end
1918     outp.puts
1919   end
1920
1921   def meanName(currentMean, preferredMean)
1922     result = "<#{currentMean}>"
1923     if "#{currentMean}Mean" == preferredMean.to_s
1924       result += " *"
1925     end
1926     result
1927   end
1928
1929   def allSummaryStats(outp, accumulators, preferredMean)
1930     summaryStats(outp, accumulators, meanName("arithmetic", preferredMean)) {
1931       | stat |
1932       stat.arithmeticMean
1933     }
1934
1935     summaryStats(outp, accumulators, meanName("geometric", preferredMean)) {
1936       | stat |
1937       stat.geometricMean
1938     }
1939
1940     summaryStats(outp, accumulators, meanName("harmonic", preferredMean)) {
1941       | stat |
1942       stat.harmonicMean
1943     }
1944   end
1945
1946   $suites.each {
1947     | suite |
1948     printVMs(outp)
1949     if $suites.size > 1
1950       outp.puts "#{suite.name}:"
1951     else
1952       outp.puts
1953     end
1954     suite.benchmarks.each {
1955       | benchmark |
1956       outp.print "   " if $suites.size > 1
1957       outp.print rpad(benchmark.to_s, $benchpad)
1958       outp.print " "
1959       myConfigs = $benchmarksOnVMsForBenchmark[benchmark]
1960       myConfigs.size.times {
1961         | index |
1962         if index != 0
1963           outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
1964         end
1965         outp.print statsToStr(myConfigs[index].stats)
1966       }
1967       if $vms.size>=2
1968         outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
1969       end
1970       outp.puts
1971     }
1972     outp.puts
1973     allSummaryStats(outp, $suitesOnVMsForSuite[suite], suite.preferredMean)
1974     outp.puts if $suites.size > 1
1975   }
1976
1977   if $suites.size > 1
1978     printVMs(outp)
1979     outp.puts "All benchmarks:"
1980     allSummaryStats(outp, $vms, nil)
1981
1982     outp.puts
1983     printVMs(outp)
1984     outp.puts "Geomean of preferred means:"
1985     outp.print "   "
1986     outp.print rpad("<scaled-result>", $benchpad)
1987     outp.print " "
1988     $vms.size.times {
1989       | index |
1990       if index != 0
1991         outp.print " "+$overallResults[index].compareTo($overallResults[index-1]).shortForm
1992       end
1993       outp.print statsToStr($overallResults[index])
1994     }
1995     if $overallResults.size>=2
1996       outp.print("    "+$overallResults[-1].compareTo($overallResults[0]).to_s)
1997     end
1998     outp.puts
1999   end
2000   outp.puts
2001
2002   if outp != $stdout
2003     outp.close
2004   end
2005
2006   if outp != $stdout and not $brief
2007     puts
2008     File.open(reportName) {
2009       | inp |
2010       puts inp.read
2011     }
2012   end
2013
2014   if $brief
2015     puts($overallResults.collect{|stats| stats.mean}.join("\t"))
2016     puts($overallResults.collect{|stats| stats.confInt}.join("\t"))
2017   end
2018
2019 rescue => e
2020   fail(e)
2021 end
2022
2023