Tools/Scripts/bencher

   1 #!/usr/bin/env ruby
   2
   3 # Copyright (C) 2011 Apple Inc. All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions
   7 # are met:
   8 # 1. Redistributions of source code must retain the above copyright
   9 #    notice, this list of conditions and the following disclaimer.
  10 # 2. Redistributions in binary form must reproduce the above copyright
  11 #    notice, this list of conditions and the following disclaimer in the
  12 #    documentation and/or other materials provided with the distribution.
  13 #
  14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
  15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
  18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  24 # THE POSSIBILITY OF SUCH DAMAGE.
  25
  26 require 'rubygems'
  27
  28 require 'getoptlong'
  29 require 'pathname'
  30 require 'tempfile'
  31 require 'socket'
  32
  33 begin
  34   require 'json'
  35 rescue LoadError => e
  36   $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
  37   exit 1
  38 end
  39
  40 # Configuration
  41
  42 CONFIGURATION_FLNM = ENV["HOME"]+"/.bencher"
  43
  44 unless FileTest.exist? CONFIGURATION_FLNM
  45   $stderr.puts "Error: no configuration file at ~/.bencher."
  46   $stderr.puts "This file should contain paths to SunSpider, V8, and Kraken, as well as a"
  47   $stderr.puts "temporary directory that bencher can use for its remote mode. It should be"
  48   $stderr.puts "formatted in JSON.  For example:"
  49   $stderr.puts "{"
  50   $stderr.puts "    \"sunSpiderPath\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/sunspider-1.0\","
  51   $stderr.puts "    \"v8Path\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/v8-v6\","
  52   $stderr.puts "    \"krakenPath\": \"/Volumes/Data/pizlo/kraken/kraken-e119421cb325/tests/kraken-1.1\","
  53   $stderr.puts "    \"tempPath\": \"/Volumes/Data/pizlo/bencher/temp\""
  54   $stderr.puts "}"
  55   exit 1
  56 end
  57
  58 CONFIGURATION = JSON.parse(File::read(CONFIGURATION_FLNM))
  59
  60 SUNSPIDER_PATH = CONFIGURATION["sunSpiderPath"]
  61 V8_PATH = CONFIGURATION["v8Path"]
  62 KRAKEN_PATH = CONFIGURATION["krakenPath"]
  63 TEMP_PATH = CONFIGURATION["tempPath"]
  64 BENCH_DATA_PATH = TEMP_PATH + "/benchdata"
  65
  66 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933,
  67             0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684,
  68             0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227,
  69             0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674,
  70             0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652,
  71             0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604,
  72             0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338,
  73             0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538,
  74             0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185,
  75             0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694,
  76             0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244,
  77             0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067,
  78             0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323,
  79             0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129,
  80             0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574,
  81             0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722,
  82             0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624,
  83             0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321,
  84             0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843,
  85             0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217,
  86             0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464,
  87             0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599,
  88             0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638,
  89             0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592,
  90             0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471,
  91             0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283,
  92             0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037,
  93             0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738,
  94             0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481,
  95             0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085,
  96             0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651,
  97             0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183,
  98             0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683,
  99             0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154,
 100             0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599,
 101             0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019,
 102             0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417,
 103             0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795,
 104             0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153,
 105             0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494,
 106             0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818,
 107             0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128,
 108             0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423,
 109             0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704,
 110             0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974,
 111             0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232,
 112             0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479,
 113             0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716,
 114             0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943,
 115             0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192,
 116             0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401,
 117             0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963,
 118             0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822,
 119             0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007,
 120             0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186,
 121             0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358,
 122             0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525,
 123             0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686,
 124             0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841,
 125             0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991,
 126             0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137,
 127             0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278,
 128             0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414,
 129             0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547,
 130             0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675,
 131             0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799,
 132             0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192,
 133             0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037,
 134             0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151,
 135             0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262,
 136             0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369,
 137             0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489,
 138             0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259,
 139             0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689,
 140             0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785,
 141             0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879,
 142             0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297,
 143             0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059,
 144             0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145,
 145             0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242,
 146             0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324,
 147             0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404,
 148             0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483,
 149             0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559,
 150             0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634,
 151             0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707,
 152             0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778,
 153             0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848,
 154             0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916,
 155             0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983,
 156             0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057,
 157             0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121,
 158             0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183,
 159             0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244,
 160             0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304,
 161             0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363,
 162             0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442,
 163             0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476,
 164             0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539,
 165             0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946,
 166             0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653,
 167             0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712,
 168             0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762,
 169             0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812,
 170             0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867,
 171             0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915,
 172             0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962,
 173             0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007,
 174             0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059,
 175             0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103,
 176             0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146,
 177             0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189,
 178             0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231,
 179             0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272,
 180             0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312,
 181             0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352,
 182             0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391,
 183             0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429,
 184             0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467,
 185             0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504,
 186             0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546,
 187             0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582,
 188             0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617,
 189             0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652,
 190             0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686,
 191             0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724,
 192             0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757,
 193             0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579,
 194             0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822,
 195             0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853,
 196             0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885,
 197             0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915,
 198             0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595,
 199             0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979,
 200             0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013,
 201             0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041,
 202             0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074,
 203             0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102,
 204             0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129,
 205             0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156,
 206             0.99616, 0.996164]
 207
 208 # Run-time configuration parameters (can be set with command-line options)
 209
 210 $rerun=1
 211 $inner=3
 212 $warmup=1
 213 $outer=4
 214 $includeSunSpider=true
 215 $includeV8=true
 216 $includeKraken=true
 217 $measureGC=false
 218 $benchmarkPattern=nil
 219 $verbosity=0
 220 $timeMode=:preciseTime
 221 $forceVMKind=nil
 222 $brief=false
 223 $silent=false
 224 $remoteHosts=[]
 225 $alsoLocal=false
 226 $sshOptions=[]
 227 $vms = []
 228 $needToCopyVMs = false
 229 $dontCopyVMs = false
 230
 231 $prepare = true
 232 $run = true
 233 $analyze = []
 234
 235 # Helpful functions and classes
 236
 237 def smallUsage
 238   puts "Use the --help option to get basic usage information."
 239   exit 1
 240 end
 241
 242 def usage
 243   puts "bencher [options] <vm1> [<vm2> ...]"
 244   puts
 245   puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
 246   puts "benchmarks, and reports detailed statistics.  What makes bencher special is"
 247   puts "that each benchmark/VM configuration is run in a single VM invocation, and"
 248   puts "the invocations are run in random order.  This minimizes systematics due to"
 249   puts "one benchmark polluting the running time of another.  The fine-grained"
 250   puts "interleaving of VM invocations further minimizes systematics due to changes in"
 251   puts "the performance or behavior of your machine."
 252   puts
 253   puts "Bencher is highly configurable.  You can compare as many VMs as you like.  You"
 254   puts "can change the amount of warm-up iterations, number of iterations executed per"
 255   puts "VM invocation, and the number of VM invocations per benchmark.  By default,"
 256   puts "SunSpider, VM, and Kraken are all run; but you can run any combination of these"
 257   puts "suites."
 258   puts
 259   puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
 260   puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
 261   puts "the executable and <name> is the name that you would like to give the"
 262   puts "configuration for the purposeof reporting.  If no name is given, a generic name"
 263   puts "of the form Conf#<n> will be ascribed to the configuration automatically."
 264   puts
 265   puts "Options:"
 266   puts "--rerun <n>          Set the number of iterations of the benchmark that"
 267   puts "                     contribute to the measured run time.  Default is #{$rerun}."
 268   puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
 269   puts "                     iterations.  Default is #{$inner}."
 270   puts "--outer <n>          Set the number of runtime invocations for each benchmark."
 271   puts "                     Default is #{$outer}."
 272   puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
 273   puts "                     is #{$warmup}."
 274   puts "--timing-mode        Set the way that bencher measures time.  Possible values"
 275   puts "                     are 'preciseTime' and 'date'.  Default is 'preciseTime'."
 276   puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
 277   puts "                     the one specified.  Valid arguments are 'jsc' or"
 278   puts "                     'DumpRenderTree'."
 279   puts "--force-vm-copy      Force VM builds to be copied to bencher's working directory."
 280   puts "                     This may reduce pathologies resulting from path names."
 281   puts "--dont-copy-vms      Don't copy VMs even when doing a remote benchmarking run;"
 282   puts "                     instead assume that they are already there."
 283   puts "--v8-only            Only run V8."
 284   puts "--sunspider-only     Only run SunSpider."
 285   puts "--kraken-only        Only run Kraken."
 286   puts "--exclude-v8         Exclude V8 (only run SunSpider and Kraken)."
 287   puts "--exclude-sunspider  Exclude SunSpider (only run V8 and Kraken)."
 288   puts "--exclude-kraken     Exclude Kraken (only run SunSpider and V8)."
 289   puts "--benchmarks         Only run benchmarks matching the given regular expression."
 290   puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
 291   puts "                     Works best with large values of --inner.  You can also say"
 292   puts "                     --measure-gc <conf>, which turns this on for one"
 293   puts "                     configuration only."
 294   puts "--verbose or -v      Print more stuff."
 295   puts "--brief              Print only the final result for each VM."
 296   puts "--silent             Don't print progress. This might slightly reduce some"
 297   puts "                     performance perturbation."
 298   puts "--remote <sshhosts>  Performance performance measurements remotely, on the given"
 299   puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
 300   puts "                     user@host string. However, you can also supply a comma-"
 301   puts "                     separated list of SSH hosts. Alternatively, you can use this"
 302   puts "                     option multiple times to specify multiple hosts. This"
 303   puts "                     automatically copies the WebKit release builds of the VMs"
 304   puts "                     you specified to all of the hosts."
 305   puts "--ssh-options        Pass additional options to SSH."
 306   puts "--local              Also do a local benchmark run even when doing --remote."
 307   puts "--prepare-only       Only prepare the bencher runscript (a shell script that"
 308   puts "                     invokes the VMs to run benchmarks) but don't run it."
 309   puts "--analyze            Only read the output of the runscript but don't do anything"
 310   puts "                     else. This requires passing the same arguments to bencher"
 311   puts "                     that you passed when running --prepare-only."
 312   puts "--help or -h         Display this message."
 313   puts
 314   puts "Example:"
 315   puts "bencher TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
 316   exit 1
 317 end
 318
 319 def fail(reason)
 320   if reason.respond_to? :backtrace
 321     puts "FAILED: #{reason}"
 322     puts "Stack trace:"
 323     puts reason.backtrace.join("\n")
 324   else
 325     puts "FAILED: #{reason}"
 326   end
 327   smallUsage
 328 end
 329
 330 def quickFail(r1,r2)
 331   $stderr.puts "#{$0}: #{r1}"
 332   puts
 333   fail(r2)
 334 end
 335
 336 def intArg(argName,arg,min,max)
 337   result=arg.to_i
 338   unless result.to_s == arg
 339     quickFail("Expected an integer value for #{argName}, but got #{arg}.",
 340               "Invalid argument for command-line option")
 341   end
 342   if min and result<min
 343     quickFail("Argument for #{argName} cannot be smaller than #{min}.",
 344               "Invalid argument for command-line option")
 345   end
 346   if max and result>max
 347     quickFail("Argument for #{argName} cannot be greater than #{max}.",
 348               "Invalid argument for command-line option")
 349   end
 350   result
 351 end
 352
 353 def computeMean(array)
 354   sum=0.0
 355   array.each {
 356     | value |
 357     sum += value
 358   }
 359   sum/array.length
 360 end
 361
 362 def computeGeometricMean(array)
 363   mult=1.0
 364   array.each {
 365     | value |
 366     mult*=value
 367   }
 368   mult**(1.0/array.length)
 369 end
 370
 371 def computeHarmonicMean(array)
 372   1.0 / computeMean(array.collect{ | value | 1.0 / value })
 373 end
 374
 375 def computeStdDev(array)
 376   case array.length
 377   when 0
 378     0.0/0.0
 379   when 1
 380     0.0
 381   else
 382     begin
 383       mean=computeMean(array)
 384       sum=0.0
 385       array.each {
 386         | value |
 387         sum += (value-mean)**2
 388       }
 389       Math.sqrt(sum/(array.length-1))
 390     rescue
 391       0.0/0.0
 392     end
 393   end
 394 end
 395
 396 class Array
 397   def shuffle!
 398     size.downto(1) { |n| push delete_at(rand(n)) }
 399     self
 400   end
 401 end
 402
 403 def inverseBetaRegularized(n)
 404   IBR_LOOKUP[n-1]
 405 end
 406
 407 def numToStr(num)
 408   "%.4f"%(num.to_f)
 409 end
 410
 411 class NoChange
 412   attr_reader :amountFaster
 413
 414   def initialize(amountFaster)
 415     @amountFaster = amountFaster
 416   end
 417
 418   def shortForm
 419     " "
 420   end
 421
 422   def longForm
 423     "  might be #{numToStr(@amountFaster)}x faster"
 424   end
 425
 426   def to_s
 427     if @amountFaster < 1.01
 428       ""
 429     else
 430       longForm
 431     end
 432   end
 433 end
 434
 435 class Faster
 436   attr_reader :amountFaster
 437
 438   def initialize(amountFaster)
 439     @amountFaster = amountFaster
 440   end
 441
 442   def shortForm
 443     "^"
 444   end
 445
 446   def longForm
 447     "^ definitely #{numToStr(@amountFaster)}x faster"
 448   end
 449
 450   def to_s
 451     longForm
 452   end
 453 end
 454
 455 class Slower
 456   attr_reader :amountSlower
 457
 458   def initialize(amountSlower)
 459     @amountSlower = amountSlower
 460   end
 461
 462   def shortForm
 463     "!"
 464   end
 465
 466   def longForm
 467     "! definitely #{numToStr(@amountSlower)}x slower"
 468   end
 469
 470   def to_s
 471     longForm
 472   end
 473 end
 474
 475 class MayBeSlower
 476   attr_reader :amountSlower
 477
 478   def initialize(amountSlower)
 479     @amountSlower = amountSlower
 480   end
 481
 482   def shortForm
 483     "?"
 484   end
 485
 486   def longForm
 487     "? might be #{numToStr(@amountSlower)}x slower"
 488   end
 489
 490   def to_s
 491     if @amountSlower < 1.01
 492       "?"
 493     else
 494       longForm
 495     end
 496   end
 497 end
 498
 499 class Stats
 500   def initialize
 501     @array = []
 502   end
 503
 504   def add(value)
 505     if value.is_a? Stats
 506       add(value.array)
 507     elsif value.respond_to? :each
 508       value.each {
 509         | v |
 510         add(v)
 511       }
 512     else
 513       @array << value.to_f
 514     end
 515   end
 516
 517   def array
 518     @array
 519   end
 520
 521   def sum
 522     result=0
 523     @array.each {
 524       | value |
 525       result += value
 526     }
 527     result
 528   end
 529
 530   def min
 531     @array.min
 532   end
 533
 534   def max
 535     @array.max
 536   end
 537
 538   def size
 539     @array.length
 540   end
 541
 542   def mean
 543     computeMean(array)
 544   end
 545
 546   def arithmeticMean
 547     mean
 548   end
 549
 550   def stdDev
 551     computeStdDev(array)
 552   end
 553
 554   def stdErr
 555     stdDev/Math.sqrt(size)
 556   end
 557
 558   # Computes a 95% Student's t distribution confidence interval
 559   def confInt
 560     if size < 2
 561       0.0/0.0
 562     else
 563       raise if size > 1000
 564       Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
 565     end
 566   end
 567
 568   def lower
 569     mean-confInt
 570   end
 571
 572   def upper
 573     mean+confInt
 574   end
 575
 576   def geometricMean
 577     computeGeometricMean(array)
 578   end
 579
 580   def harmonicMean
 581     computeHarmonicMean(array)
 582   end
 583
 584   def compareTo(other)
 585     if upper < other.lower
 586       Faster.new(other.mean/mean)
 587     elsif lower > other.upper
 588       Slower.new(mean/other.mean)
 589     elsif mean > other.mean
 590       MayBeSlower.new(mean/other.mean)
 591     else
 592       NoChange.new(other.mean/mean)
 593     end
 594   end
 595
 596   def to_s
 597     "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
 598   end
 599 end
 600
 601 def doublePuts(out1,out2,msg)
 602   out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
 603   out2.puts msg
 604 end
 605
 606 class Benchfile < File
 607   @@counter = 0
 608
 609   attr_reader :filename, :basename
 610
 611   def initialize(name)
 612     @basename, @filename = Benchfile.uniqueFilename(name)
 613     super(@filename, "w")
 614   end
 615
 616   def self.uniqueFilename(name)
 617     if name.is_a? Array
 618       basename = name[0] + @@counter.to_s + name[1]
 619     else
 620       basename = name + @@counter.to_s
 621     end
 622     filename = BENCH_DATA_PATH + "/" + basename
 623     @@counter += 1
 624     raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
 625     [basename, filename]
 626   end
 627
 628   def self.create(name)
 629     file = Benchfile.new(name)
 630     yield file
 631     file.close
 632     file.basename
 633   end
 634 end
 635
 636 $dataFiles={}
 637 def ensureFile(key, filename)
 638   unless $dataFiles[key]
 639     $dataFiles[key] = Benchfile.create(key) {
 640       | outp |
 641       doublePuts($stderr,outp,IO::read(filename))
 642     }
 643   end
 644   $dataFiles[key]
 645 end
 646
 647 def emitBenchRunCodeFile(name, plan, benchDataPath, benchPath)
 648   case plan.vm.vmType
 649   when :jsc
 650     Benchfile.create("bencher") {
 651       | file |
 652       case $timeMode
 653       when :preciseTime
 654         doublePuts($stderr,file,"function __bencher_curTimeMS() {")
 655         doublePuts($stderr,file,"   return preciseTime()*1000")
 656         doublePuts($stderr,file,"}")
 657       when :date
 658         doublePuts($stderr,file,"function __bencher_curTimeMS() {")
 659         doublePuts($stderr,file,"   return Date.now()")
 660         doublePuts($stderr,file,"}")
 661       else
 662         raise
 663       end
 664
 665       if benchDataPath
 666         doublePuts($stderr,file,"load(#{benchDataPath.inspect});")
 667         doublePuts($stderr,file,"gc();")
 668         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
 669         doublePuts($stderr,file,"   before = __bencher_curTimeMS();")
 670         $rerun.times {
 671           doublePuts($stderr,file,"   load(#{benchPath.inspect});")
 672         }
 673         doublePuts($stderr,file,"   after = __bencher_curTimeMS();")
 674         doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(after-before));");
 675         doublePuts($stderr,file,"   gc();") unless plan.vm.shouldMeasureGC
 676         doublePuts($stderr,file,"}")
 677       else
 678         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
 679         doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
 680         $rerun.times {
 681           doublePuts($stderr,file,"   run(__bencher_what);")
 682         }
 683         doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
 684         doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
 685         doublePuts($stderr,file,"}")
 686         $warmup.times {
 687           doublePuts($stderr,file,"__bencher_run(#{benchPath.inspect})")
 688           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
 689         }
 690         $inner.times {
 691           | innerIndex |
 692           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchPath.inspect}));")
 693           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
 694         }
 695       end
 696     }
 697   when :dumpRenderTree
 698     mainCode = Benchfile.create("bencher") {
 699       | file |
 700       doublePuts($stderr,file,"__bencher_count = 0;")
 701       doublePuts($stderr,file,"function __bencher_doNext(result) {")
 702       doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
 703       doublePuts($stderr,file,"        debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
 704       doublePuts($stderr,file,"    __bencher_count++;")
 705       doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
 706       doublePuts($stderr,file,"        __bencher_runImpl(__bencher_doNext);")
 707       doublePuts($stderr,file,"    else")
 708       doublePuts($stderr,file,"        quit();")
 709       doublePuts($stderr,file,"}")
 710       doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
 711     }
 712
 713     cssCode = Benchfile.create("bencher-css") {
 714       | file |
 715       doublePuts($stderr,file,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
 716     }
 717
 718     preCode = Benchfile.create("bencher-pre") {
 719       | file |
 720       doublePuts($stderr,file,"if (window.testRunner) {")
 721       doublePuts($stderr,file,"    testRunner.dumpAsText(window.enablePixelTesting);")
 722       doublePuts($stderr,file,"    testRunner.waitUntilDone();")
 723       doublePuts($stderr,file,"}")
 724       doublePuts($stderr,file,"")
 725       doublePuts($stderr,file,"function debug(msg)")
 726       doublePuts($stderr,file,"{")
 727       doublePuts($stderr,file,"    var span = document.createElement(\"span\");")
 728       doublePuts($stderr,file,"    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
 729       doublePuts($stderr,file,"    span.innerHTML = msg + '<br />';")
 730       doublePuts($stderr,file,"}")
 731       doublePuts($stderr,file,"")
 732       doublePuts($stderr,file,"function quit() {")
 733       doublePuts($stderr,file,"    testRunner.notifyDone();")
 734       doublePuts($stderr,file,"}")
 735       doublePuts($stderr,file,"")
 736       doublePuts($stderr,file,"__bencher_continuation=null;")
 737       doublePuts($stderr,file,"")
 738       doublePuts($stderr,file,"function reportResult(result) {")
 739       doublePuts($stderr,file,"    __bencher_continuation(result);")
 740       doublePuts($stderr,file,"}")
 741       doublePuts($stderr,file,"")
 742       doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
 743       doublePuts($stderr,file,"    function doit() {")
 744       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"\";")
 745       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
 746       doublePuts($stderr,file,"        var testFrame = document.getElementById(\"testframe\");")
 747       doublePuts($stderr,file,"        testFrame.contentDocument.open();")
 748       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
 749       if benchDataPath
 750         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=\\\"#{benchDataPath}\\\"></script>\");")
 751       end
 752       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">__bencher_before = Date.now();</script><script src=\\\"#{benchPath}\\\"></script><script type=\\\"text/javascript\\\">window.parent.reportResult(Date.now() - __bencher_before);</script></body></html>\");")
 753       doublePuts($stderr,file,"        testFrame.contentDocument.close();")
 754       doublePuts($stderr,file,"    }")
 755       doublePuts($stderr,file,"    __bencher_continuation = continuation;")
 756       doublePuts($stderr,file,"    window.setTimeout(doit, 10);")
 757       doublePuts($stderr,file,"}")
 758     }
 759
 760     Benchfile.create(["bencher-htmldoc",".html"]) {
 761       | file |
 762       doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
 763     }
 764   else
 765     raise
 766   end
 767 end
 768
 769 def emitBenchRunCode(name, plan, benchDataPath, benchPath)
 770   plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchDataPath, benchPath))
 771 end
 772
 773 def planForDescription(plans, benchFullname, vmName, iteration)
 774   raise unless benchFullname =~ /\//
 775   suiteName = $~.pre_match
 776   benchName = $~.post_match
 777   result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
 778   raise unless result.size == 1
 779   result[0]
 780 end
 781
 782 class ParsedResult
 783   attr_reader :plan, :innerIndex, :time
 784
 785   def initialize(plan, innerIndex, time)
 786     @plan = plan
 787     @innerIndex = innerIndex
 788     @time = time
 789
 790     raise unless @plan.is_a? BenchPlan
 791     raise unless @innerIndex.is_a? Integer
 792     raise unless @time.is_a? Numeric
 793   end
 794
 795   def benchmark
 796     plan.benchmark
 797   end
 798
 799   def suite
 800     plan.suite
 801   end
 802
 803   def vm
 804     plan.vm
 805   end
 806
 807   def outerIndex
 808     plan.iteration
 809   end
 810
 811   def self.parse(plans, string)
 812     if string =~ /([a-zA-Z0-9\/-]+): ([a-zA-Z0-9_# ]+): ([0-9]+): ([0-9]+): Time: /
 813       benchFullname = $1
 814       vmName = $2
 815       outerIndex = $3.to_i
 816       innerIndex = $4.to_i
 817       time = $~.post_match.to_f
 818       ParsedResult.new(planForDescription(plans, benchFullname, vmName, outerIndex), innerIndex, time)
 819     else
 820       nil
 821     end
 822   end
 823 end
 824
 825 class VM
 826   def initialize(origPath, name, nameKind, svnRevision)
 827     @origPath = origPath.to_s
 828     @path = origPath.to_s
 829     @name = name
 830     @nameKind = nameKind
 831
 832     if $forceVMKind
 833       @vmType = $forceVMKind
 834     else
 835       if @origPath =~ /DumpRenderTree$/
 836         @vmType = :dumpRenderTree
 837       else
 838         @vmType = :jsc
 839       end
 840     end
 841
 842     @svnRevision = svnRevision
 843
 844     # Try to detect information about the VM.
 845     if path =~ /\/WebKitBuild\/Release\/([a-zA-Z]+)$/
 846       @checkoutPath = $~.pre_match
 847       # FIXME: Use some variant of this:
 848       # <bdash>   def retrieve_revision
 849       # <bdash>     `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
 850       # <bdash>   end
 851       unless @svnRevision
 852         begin
 853           Dir.chdir(@checkoutPath) {
 854             $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
 855             IO.popen("svn info", "r") {
 856               | inp |
 857               inp.each_line {
 858                 | line |
 859                 if line =~ /Revision: ([0-9]+)/
 860                   @svnRevision = $1
 861                 end
 862               }
 863             }
 864           }
 865           unless @svnRevision
 866             $stderr.puts "Warning: running svn info for #{name} silently failed."
 867           end
 868         rescue => e
 869           # Failed to detect svn revision.
 870           $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
 871         end
 872       end
 873     else
 874       $stderr.puts "Warning: could not identify checkout location for #{name}"
 875     end
 876
 877     if @path =~ /\/Release\/([a-zA-Z]+)$/
 878       @libPath, @relativeBinPath = $~.pre_match+"/Release", "./#{$1}"
 879     elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
 880       @libPath = $~.pre_match
 881     elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
 882       @libPath, @relativeBinPath = $~.pre_match, $&[1..-1]
 883     end
 884   end
 885
 886   def canCopyIntoBenchPath
 887     if @libPath and @relativeBinPath
 888       true
 889     else
 890       false
 891     end
 892   end
 893
 894   def copyIntoBenchPath
 895     raise unless canCopyIntoBenchPath
 896     basename, filename = Benchfile.uniqueFilename("vm")
 897     raise unless Dir.mkdir(filename)
 898     cmd = "cp -a #{@libPath.inspect}/* #{filename.inspect}"
 899     $stderr.puts ">> #{cmd}" if $verbosity>=2
 900     raise unless system(cmd)
 901     @path = "#{basename}/#{@relativeBinPath}"
 902     @libPath = basename
 903   end
 904
 905   def to_s
 906     @name
 907   end
 908
 909   def name
 910     @name
 911   end
 912
 913   def shouldMeasureGC
 914     $measureGC == true or ($measureGC == name)
 915   end
 916
 917   def origPath
 918     @origPath
 919   end
 920
 921   def path
 922     @path
 923   end
 924
 925   def nameKind
 926     @nameKind
 927   end
 928
 929   def vmType
 930     @vmType
 931   end
 932
 933   def checkoutPath
 934     @checkoutPath
 935   end
 936
 937   def svnRevision
 938     @svnRevision
 939   end
 940
 941   def printFunction
 942     case @vmType
 943     when :jsc
 944       "print"
 945     when :dumpRenderTree
 946       "debug"
 947     else
 948       raise @vmType
 949     end
 950   end
 951
 952   def emitRunCode(fileToRun)
 953     myLibPath = @libPath
 954     myLibPath = "" unless myLibPath
 955     $script.puts "export DYLD_LIBRARY_PATH=#{myLibPath.to_s.inspect}"
 956     $script.puts "export DYLD_FRAMEWORK_PATH=#{myLibPath.to_s.inspect}"
 957     $script.puts "#{path} #{fileToRun}"
 958   end
 959 end
 960
 961 class StatsAccumulator
 962   def initialize
 963     @stats = []
 964     ($outer*$inner).times {
 965       @stats << Stats.new
 966     }
 967   end
 968
 969   def statsForIteration(outerIteration, innerIteration)
 970     @stats[outerIteration*$inner + innerIteration]
 971   end
 972
 973   def stats
 974     result = Stats.new
 975     @stats.each {
 976       | stat |
 977       result.add(yield stat)
 978     }
 979     result
 980   end
 981
 982   def geometricMeanStats
 983     stats {
 984       | stat |
 985       stat.geometricMean
 986     }
 987   end
 988
 989   def arithmeticMeanStats
 990     stats {
 991       | stat |
 992       stat.arithmeticMean
 993     }
 994   end
 995 end
 996
 997 module Benchmark
 998   attr_accessor :benchmarkSuite
 999   attr_reader :name
1000
1001   def fullname
1002     benchmarkSuite.name + "/" + name
1003   end
1004
1005   def to_s
1006     fullname
1007   end
1008 end
1009
1010 class SunSpiderBenchmark
1011   include Benchmark
1012
1013   def initialize(name)
1014     @name = name
1015   end
1016
1017   def emitRunCode(plan)
1018     emitBenchRunCode(fullname, plan, nil, ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js"))
1019   end
1020 end
1021
1022 class V8Benchmark
1023   include Benchmark
1024
1025   def initialize(name)
1026     @name = name
1027   end
1028
1029   def emitRunCode(plan)
1030     emitBenchRunCode(fullname, plan, nil, ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js"))
1031   end
1032 end
1033
1034 class KrakenBenchmark
1035   include Benchmark
1036
1037   def initialize(name)
1038     @name = name
1039   end
1040
1041   def emitRunCode(plan)
1042     emitBenchRunCode(fullname, plan, ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js"), ensureFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js"))
1043   end
1044 end
1045
1046 class BenchmarkSuite
1047   def initialize(name, path, preferredMean)
1048     @name = name
1049     @path = path
1050     @preferredMean = preferredMean
1051     @benchmarks = []
1052   end
1053
1054   def name
1055     @name
1056   end
1057
1058   def to_s
1059     @name
1060   end
1061
1062   def path
1063     @path
1064   end
1065
1066   def add(benchmark)
1067     if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
1068       benchmark.benchmarkSuite = self
1069       @benchmarks << benchmark
1070     end
1071   end
1072
1073   def benchmarks
1074     @benchmarks
1075   end
1076
1077   def benchmarkForName(name)
1078     result = @benchmarks.select{|v| v.name == name}
1079     raise unless result.length == 1
1080     result[0]
1081   end
1082
1083   def empty?
1084     @benchmarks.empty?
1085   end
1086
1087   def retain_if
1088     @benchmarks.delete_if {
1089       | benchmark |
1090       not yield benchmark
1091     }
1092   end
1093
1094   def preferredMean
1095     @preferredMean
1096   end
1097
1098   def computeMean(stat)
1099     stat.send @preferredMean
1100   end
1101 end
1102
1103 class BenchRunPlan
1104   def initialize(benchmark, vm, iteration)
1105     @benchmark = benchmark
1106     @vm = vm
1107     @iteration = iteration
1108   end
1109
1110   def benchmark
1111     @benchmark
1112   end
1113
1114   def suite
1115     @benchmark.benchmarkSuite
1116   end
1117
1118   def vm
1119     @vm
1120   end
1121
1122   def iteration
1123     @iteration
1124   end
1125
1126   def emitRunCode
1127     @benchmark.emitRunCode(self)
1128   end
1129 end
1130
1131 class BenchmarkOnVM
1132   def initialize(benchmark, suiteOnVM)
1133     @benchmark = benchmark
1134     @suiteOnVM = suiteOnVM
1135     @stats = Stats.new
1136   end
1137
1138   def to_s
1139     "#{@benchmark} on #{@suiteOnVM.vm}"
1140   end
1141
1142   def benchmark
1143     @benchmark
1144   end
1145
1146   def vm
1147     @suiteOnVM.vm
1148   end
1149
1150   def vmStats
1151     @suiteOnVM.vmStats
1152   end
1153
1154   def suite
1155     @benchmark.benchmarkSuite
1156   end
1157
1158   def suiteOnVM
1159     @suiteOnVM
1160   end
1161
1162   def stats
1163     @stats
1164   end
1165
1166   def parseResult(result)
1167     raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
1168     raise unless result.benchmark == @benchmark
1169     @stats.add(result.time)
1170   end
1171 end
1172
1173 class SuiteOnVM < StatsAccumulator
1174   def initialize(vm, vmStats, suite)
1175     super()
1176     @vm = vm
1177     @vmStats = vmStats
1178     @suite = suite
1179
1180     raise unless @vm.is_a? VM
1181     raise unless @vmStats.is_a? StatsAccumulator
1182     raise unless @suite.is_a? BenchmarkSuite
1183   end
1184
1185   def to_s
1186     "#{@suite} on #{@vm}"
1187   end
1188
1189   def suite
1190     @suite
1191   end
1192
1193   def vm
1194     @vm
1195   end
1196
1197   def vmStats
1198     raise unless @vmStats
1199     @vmStats
1200   end
1201 end
1202
1203 class BenchPlan
1204   def initialize(benchmarkOnVM, iteration)
1205     @benchmarkOnVM = benchmarkOnVM
1206     @iteration = iteration
1207   end
1208
1209   def to_s
1210     "#{@benchmarkOnVM} \##{@iteration+1}"
1211   end
1212
1213   def benchmarkOnVM
1214     @benchmarkOnVM
1215   end
1216
1217   def benchmark
1218     @benchmarkOnVM.benchmark
1219   end
1220
1221   def suite
1222     @benchmarkOnVM.suite
1223   end
1224
1225   def vm
1226     @benchmarkOnVM.vm
1227   end
1228
1229   def iteration
1230     @iteration
1231   end
1232
1233   def parseResult(result)
1234     raise unless result.plan == self
1235     @benchmarkOnVM.parseResult(result)
1236     @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
1237     @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
1238   end
1239 end
1240
1241 def lpad(str,chars)
1242   if str.length>chars
1243     str
1244   else
1245     "%#{chars}s"%(str)
1246   end
1247 end
1248
1249 def rpad(str,chars)
1250   while str.length<chars
1251     str+=" "
1252   end
1253   str
1254 end
1255
1256 def center(str,chars)
1257   while str.length<chars
1258     str+=" "
1259     if str.length<chars
1260       str=" "+str
1261     end
1262   end
1263   str
1264 end
1265
1266 def statsToStr(stats)
1267   if $inner*$outer == 1
1268     string = numToStr(stats.mean)
1269     raise unless string =~ /\./
1270     left = $~.pre_match
1271     right = $~.post_match
1272     lpad(left,12)+"."+rpad(right,9)
1273   else
1274     lpad(numToStr(stats.mean),11)+"+-"+rpad(numToStr(stats.confInt),9)
1275   end
1276 end
1277
1278 def plural(num)
1279   if num == 1
1280     ""
1281   else
1282     "s"
1283   end
1284 end
1285
1286 def wrap(str, columns)
1287   array = str.split
1288   result = ""
1289   curLine = array.shift
1290   array.each {
1291     | curStr |
1292     if (curLine + " " + curStr).size > columns
1293       result += curLine + "\n"
1294       curLine = curStr
1295     else
1296       curLine += " " + curStr
1297     end
1298   }
1299   result + curLine + "\n"
1300 end
1301
1302 def runAndGetResults
1303   results = nil
1304   Dir.chdir(BENCH_DATA_PATH) {
1305     IO.popen("sh ./runscript", "r") {
1306       | inp |
1307       results = inp.read
1308     }
1309     raise "Script did not complete correctly: #{$?}" unless $?.success?
1310   }
1311   raise unless results
1312   results
1313 end
1314
1315 def parseAndDisplayResults(results)
1316   vmStatses = []
1317   $vms.each {
1318     vmStatses << StatsAccumulator.new
1319   }
1320
1321   suitesOnVMs = []
1322   suitesOnVMsForSuite = {}
1323   $suites.each {
1324     | suite |
1325     suitesOnVMsForSuite[suite] = []
1326   }
1327   suitesOnVMsForVM = {}
1328   $vms.each {
1329     | vm |
1330     suitesOnVMsForVM[vm] = []
1331   }
1332
1333   benchmarksOnVMs = []
1334   benchmarksOnVMsForBenchmark = {}
1335   $benchmarks.each {
1336     | benchmark |
1337     benchmarksOnVMsForBenchmark[benchmark] = []
1338   }
1339
1340   $vms.each_with_index {
1341     | vm, vmIndex |
1342     vmStats = vmStatses[vmIndex]
1343     $suites.each {
1344       | suite |
1345       suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
1346       suitesOnVMs << suiteOnVM
1347       suitesOnVMsForSuite[suite] << suiteOnVM
1348       suitesOnVMsForVM[vm] << suiteOnVM
1349       suite.benchmarks.each {
1350         | benchmark |
1351         benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM)
1352         benchmarksOnVMs << benchmarkOnVM
1353         benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
1354       }
1355     }
1356   }
1357
1358   plans = []
1359   benchmarksOnVMs.each {
1360     | benchmarkOnVM |
1361     $outer.times {
1362       | iteration |
1363       plans << BenchPlan.new(benchmarkOnVM, iteration)
1364     }
1365   }
1366
1367   hostname = nil
1368   hwmodel = nil
1369   results.each_line {
1370     | line |
1371     line.chomp!
1372     if line =~ /HOSTNAME:([^.]+)/
1373       hostname = $1
1374     elsif line =~ /HARDWARE:hw\.model: /
1375       hwmodel = $~.post_match.chomp
1376     else
1377       result = ParsedResult.parse(plans, line.chomp)
1378       if result
1379         result.plan.parseResult(result)
1380       end
1381     end
1382   }
1383
1384   # Compute the geomean of the preferred means of results on a SuiteOnVM
1385   overallResults = []
1386   $vms.each {
1387     | vm |
1388     result = Stats.new
1389     $outer.times {
1390       | outerIndex |
1391       $inner.times {
1392         | innerIndex |
1393         curResult = Stats.new
1394         suitesOnVMsForVM[vm].each {
1395           | suiteOnVM |
1396           # For a given iteration, suite, and VM, compute the suite's preferred mean
1397           # over the data collected for all benchmarks in that suite. We'll have one
1398           # sample per benchmark. For example on V8 this will be the geomean of 1
1399           # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
1400           # splay.
1401           curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
1402         }
1403
1404         # curResult now holds 1 sample for each of the means computed in the above
1405         # loop. Compute the geomean over this, and store it.
1406         result.add(curResult.geometricMean)
1407       }
1408     }
1409
1410     # $overallResults will have a Stats for each VM. That Stats object will hold
1411     # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
1412     # confidence interval of the geomeans of preferred means. Convoluted, but
1413     # useful and probably sound.
1414     overallResults << result
1415   }
1416
1417   if $verbosity >= 2
1418     benchmarksOnVMs.each {
1419       | benchmarkOnVM |
1420       $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
1421     }
1422
1423     $vms.each_with_index {
1424       | vm, vmIndex |
1425       vmStats = vmStatses[vmIndex]
1426       $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
1427       $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
1428     }
1429   end
1430
1431   reportName =
1432     (if ($vms.collect {
1433            | vm |
1434            vm.nameKind
1435          }.index :auto)
1436        ""
1437      else
1438        $vms.collect {
1439          | vm |
1440          vm.to_s
1441        }.join("_") + "_"
1442      end) +
1443     ($suites.collect {
1444        | suite |
1445        suite.to_s
1446      }.join("")) + "_" +
1447     (if hostname
1448        hostname + "_"
1449      else
1450        ""
1451      end)+
1452     (begin
1453        time = Time.now
1454        "%04d%02d%02d_%02d%02d" %
1455          [ time.year, time.month, time.day,
1456            time.hour, time.min ]
1457      end) +
1458     "_benchReport.txt"
1459
1460   unless $brief
1461     puts "Generating benchmark report at #{reportName}"
1462   end
1463
1464   outp = $stdout
1465   begin
1466     outp = File.open(reportName,"w")
1467   rescue => e
1468     $stderr.puts "Error: could not save report to #{reportName}: #{e}"
1469     $stderr.puts
1470   end
1471
1472   def createVMsString
1473     result = ""
1474     result += "   " if $suites.size > 1
1475     result += rpad("", $benchpad)
1476     result += " "
1477     $vms.size.times {
1478       | index |
1479       if index != 0
1480         result += " "+NoChange.new(0).shortForm
1481       end
1482       result += lpad(center($vms[index].name, 9+9+2), 11+9+2)
1483     }
1484     result += "    "
1485     if $vms.size >= 3
1486       result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
1487     elsif $vms.size >= 2
1488       result += " "*26
1489     end
1490     result
1491   end
1492
1493   columns = [createVMsString.size, 78].max
1494
1495   outp.print "Benchmark report for "
1496   if $suites.size == 1
1497     outp.print $suites[0].to_s
1498   elsif $suites.size == 2
1499     outp.print "#{$suites[0]} and #{$suites[1]}"
1500   else
1501     outp.print "#{$suites[0..-2].join(', ')}, and #{$suites[-1]}"
1502   end
1503   if hostname
1504     outp.print " on #{hostname}"
1505   end
1506   if hwmodel
1507     outp.print " (#{hwmodel})"
1508   end
1509   outp.puts "."
1510   outp.puts
1511
1512   # This looks stupid; revisit later.
1513   if false
1514     $suites.each {
1515       | suite |
1516       outp.puts "#{suite} at #{suite.path}"
1517     }
1518
1519     outp.puts
1520   end
1521
1522   outp.puts "VMs tested:"
1523   $vms.each {
1524     | vm |
1525     outp.print "\"#{vm.name}\" at #{vm.origPath}"
1526     if vm.svnRevision
1527       outp.print " (r#{vm.svnRevision})"
1528     end
1529     outp.puts
1530   }
1531
1532   outp.puts
1533
1534   outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
1535                  "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
1536                  (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
1537                                       "total time of those iterations, for each sample.")
1538                   else "" end)+
1539                  (if $measureGC == true then (" No manual garbage collection invocations were "+
1540                                               "emitted.")
1541                   elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
1542                                          "all VMs except #{$measureGC}.")
1543                   else (" Emitted a call to gc() between sample measurements.") end)+
1544                  (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
1545                                         "began with the very first iteration.")
1546                   else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
1547                         "invocation for warm-up.") end)+
1548                  (case $timeMode
1549                   when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
1550                                           "microsecond-level timing.")
1551                   when :date then (" Used the portable Date.now() method to get millisecond-"+
1552                                    "level timing.")
1553                   else raise end)+
1554                  " Reporting benchmark execution times with 95% confidence "+
1555                  "intervals in milliseconds.",
1556                  columns)
1557
1558   outp.puts
1559
1560   def printVMs(outp)
1561     outp.puts createVMsString
1562   end
1563
1564   def summaryStats(outp, accumulators, name, &proc)
1565     outp.print "   " if $suites.size > 1
1566     outp.print rpad(name, $benchpad)
1567     outp.print " "
1568     accumulators.size.times {
1569       | index |
1570       if index != 0
1571         outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
1572       end
1573       outp.print statsToStr(accumulators[index].stats(&proc))
1574     }
1575     if accumulators.size>=2
1576       outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
1577     end
1578     outp.puts
1579   end
1580
1581   def meanName(currentMean, preferredMean)
1582     result = "<#{currentMean}>"
1583     if "#{currentMean}Mean" == preferredMean.to_s
1584       result += " *"
1585     end
1586     result
1587   end
1588
1589   def allSummaryStats(outp, accumulators, preferredMean)
1590     summaryStats(outp, accumulators, meanName("arithmetic", preferredMean)) {
1591       | stat |
1592       stat.arithmeticMean
1593     }
1594
1595     summaryStats(outp, accumulators, meanName("geometric", preferredMean)) {
1596       | stat |
1597       stat.geometricMean
1598     }
1599
1600     summaryStats(outp, accumulators, meanName("harmonic", preferredMean)) {
1601       | stat |
1602       stat.harmonicMean
1603     }
1604   end
1605
1606   $suites.each {
1607     | suite |
1608     printVMs(outp)
1609     if $suites.size > 1
1610       outp.puts "#{suite.name}:"
1611     else
1612       outp.puts
1613     end
1614     suite.benchmarks.each {
1615       | benchmark |
1616       outp.print "   " if $suites.size > 1
1617       outp.print rpad(benchmark.name, $benchpad)
1618       outp.print " "
1619       myConfigs = benchmarksOnVMsForBenchmark[benchmark]
1620       myConfigs.size.times {
1621         | index |
1622         if index != 0
1623           outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
1624         end
1625         outp.print statsToStr(myConfigs[index].stats)
1626       }
1627       if $vms.size>=2
1628         outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
1629       end
1630       outp.puts
1631     }
1632     outp.puts
1633     allSummaryStats(outp, suitesOnVMsForSuite[suite], suite.preferredMean)
1634     outp.puts if $suites.size > 1
1635   }
1636
1637   if $suites.size > 1
1638     printVMs(outp)
1639     outp.puts "All benchmarks:"
1640     allSummaryStats(outp, vmStatses, nil)
1641
1642     outp.puts
1643     printVMs(outp)
1644     outp.puts "Geomean of preferred means:"
1645     outp.print "   "
1646     outp.print rpad("<scaled-result>", $benchpad)
1647     outp.print " "
1648     $vms.size.times {
1649       | index |
1650       if index != 0
1651         outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
1652       end
1653       outp.print statsToStr(overallResults[index])
1654     }
1655     if overallResults.size>=2
1656       outp.print("    "+overallResults[-1].compareTo(overallResults[0]).longForm)
1657     end
1658     outp.puts
1659   end
1660   outp.puts
1661
1662   if outp != $stdout
1663     outp.close
1664   end
1665
1666   if outp != $stdout and not $brief
1667     puts
1668     File.open(reportName) {
1669       | inp |
1670       puts inp.read
1671     }
1672   end
1673
1674   if $brief
1675     puts(overallResults.collect{|stats| stats.mean}.join("\t"))
1676     puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
1677   end
1678
1679
1680 end
1681
1682 begin
1683   $sawBenchOptions = false
1684
1685   def resetBenchOptionsIfNecessary
1686     unless $sawBenchOptions
1687       $includeSunSpider = false
1688       $includeV8 = false
1689       $includeKraken = false
1690       $sawBenchOptions = true
1691     end
1692   end
1693
1694   GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
1695                  ['--inner', GetoptLong::REQUIRED_ARGUMENT],
1696                  ['--outer', GetoptLong::REQUIRED_ARGUMENT],
1697                  ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
1698                  ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
1699                  ['--sunspider-only', GetoptLong::NO_ARGUMENT],
1700                  ['--v8-only', GetoptLong::NO_ARGUMENT],
1701                  ['--kraken-only', GetoptLong::NO_ARGUMENT],
1702                  ['--exclude-sunspider', GetoptLong::NO_ARGUMENT],
1703                  ['--exclude-v8', GetoptLong::NO_ARGUMENT],
1704                  ['--exclude-kraken', GetoptLong::NO_ARGUMENT],
1705                  ['--sunspider', GetoptLong::NO_ARGUMENT],
1706                  ['--v8', GetoptLong::NO_ARGUMENT],
1707                  ['--kraken', GetoptLong::NO_ARGUMENT],
1708                  ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
1709                  ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
1710                  ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
1711                  ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
1712                  ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
1713                  ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
1714                  ['--brief', GetoptLong::NO_ARGUMENT],
1715                  ['--silent', GetoptLong::NO_ARGUMENT],
1716                  ['--remote', GetoptLong::REQUIRED_ARGUMENT],
1717                  ['--local', GetoptLong::NO_ARGUMENT],
1718                  ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
1719                  ['--slave', GetoptLong::NO_ARGUMENT],
1720                  ['--prepare-only', GetoptLong::NO_ARGUMENT],
1721                  ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
1722                  ['--vms', GetoptLong::REQUIRED_ARGUMENT],
1723                  ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
1724     | opt, arg |
1725     case opt
1726     when '--rerun'
1727       $rerun = intArg(opt,arg,1,nil)
1728     when '--inner'
1729       $inner = intArg(opt,arg,1,nil)
1730     when '--outer'
1731       $outer = intArg(opt,arg,1,nil)
1732     when '--warmup'
1733       $warmup = intArg(opt,arg,0,nil)
1734     when '--timing-mode'
1735       if arg.upcase == "PRECISETIME"
1736         $timeMode = :preciseTime
1737       elsif arg.upcase == "DATE"
1738         $timeMode = :date
1739       elsif arg.upcase == "AUTO"
1740         $timeMode = :auto
1741       else
1742         quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
1743                   "Invalid argument for command-line option")
1744       end
1745     when '--force-vm-kind'
1746       if arg.upcase == "JSC"
1747         $forceVMKind = :jsc
1748       elsif arg.upcase == "DUMPRENDERTREE"
1749         $forceVMKind = :dumpRenderTree
1750       elsif arg.upcase == "AUTO"
1751         $forceVMKind = nil
1752       else
1753         quickFail("Expected either 'jsc' or 'DumpRenderTree' for --force-vm-kind, but got '#{arg}'.",
1754                   "Invalid argument for command-line option")
1755       end
1756     when '--force-vm-copy'
1757       $needToCopyVMs = true
1758     when '--dont-copy-vms'
1759       $dontCopyVMs = true
1760     when '--sunspider-only'
1761       $includeV8 = false
1762       $includeKraken = false
1763     when '--v8-only'
1764       $includeSunSpider = false
1765       $includeKraken = false
1766     when '--kraken-only'
1767       $includeSunSpider = false
1768       $includeV8 = false
1769     when '--exclude-sunspider'
1770       $includeSunSpider = false
1771     when '--exclude-v8'
1772       $includeV8 = false
1773     when '--exclude-kraken'
1774       $includeKraken = false
1775     when '--sunspider'
1776       resetBenchOptionsIfNecessary
1777       $includeSunSpider = true
1778     when '--v8'
1779       resetBenchOptionsIfNecessary
1780       $includeV8 = true
1781     when '--kraken'
1782       resetBenchOptionsIfNecessary
1783       $includeKraken = true
1784     when '--benchmarks'
1785       $benchmarkPattern = Regexp.new(arg)
1786     when '--measure-gc'
1787       if arg == ''
1788         $measureGC = true
1789       else
1790         $measureGC = arg
1791       end
1792     when '--verbose'
1793       $verbosity += 1
1794     when '--brief'
1795       $brief = true
1796     when '--silent'
1797       $silent = true
1798     when '--remote'
1799       $remoteHosts += arg.split(',')
1800       $needToCopyVMs = true
1801     when '--ssh-options'
1802       $sshOptions << arg
1803     when '--local'
1804       $alsoLocal = true
1805     when '--prepare-only'
1806       $run = false
1807     when '--analyze'
1808       $prepare = false
1809       $run = false
1810       $analyze << arg
1811     when '--help'
1812       usage
1813     else
1814       raise "bad option: #{opt}"
1815     end
1816   }
1817
1818   # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
1819   if $dontCopyVMs
1820     $needToCopyVMs = false
1821   end
1822
1823   SUNSPIDER = BenchmarkSuite.new("SunSpider", SUNSPIDER_PATH, :arithmeticMean)
1824   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
1825    "access-fannkuch", "access-nbody", "access-nsieve",
1826    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
1827    "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
1828    "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
1829    "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
1830    "string-base64", "string-fasta", "string-tagcloud",
1831    "string-unpack-code", "string-validate-input"].each {
1832     | name |
1833     SUNSPIDER.add SunSpiderBenchmark.new(name)
1834   }
1835
1836   V8 = BenchmarkSuite.new("V8", V8_PATH, :geometricMean)
1837   ["crypto", "deltablue", "earley-boyer", "raytrace",
1838    "regexp", "richards", "splay"].each {
1839     | name |
1840     V8.add V8Benchmark.new(name)
1841   }
1842
1843   KRAKEN = BenchmarkSuite.new("Kraken", KRAKEN_PATH, :arithmeticMean)
1844   ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
1845    "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
1846    "imaging-gaussian-blur", "json-parse-financial",
1847    "json-stringify-tinderbox", "stanford-crypto-aes",
1848    "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
1849    "stanford-crypto-sha256-iterative"].each {
1850     | name |
1851     KRAKEN.add KrakenBenchmark.new(name)
1852   }
1853
1854   ARGV.each {
1855     | vm |
1856     if vm =~ /([a-zA-Z0-9_ ]+):/
1857       name = $1
1858       nameKind = :given
1859       vm = $~.post_match
1860     else
1861       name = "Conf\##{$vms.length+1}"
1862       nameKind = :auto
1863     end
1864     $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
1865     $vms << VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
1866   }
1867
1868   if $vms.empty?
1869     quickFail("Please specify at least on configuraiton on the command line.",
1870               "Insufficient arguments")
1871   end
1872
1873   $vms.each {
1874     | vm |
1875     if vm.vmType != :jsc and $timeMode != :date
1876       $timeMode = :date
1877       $stderr.puts "Warning: using Date.now() instead of preciseTime() because #{vm} doesn't support the latter."
1878     end
1879   }
1880
1881   if FileTest.exist? BENCH_DATA_PATH
1882     cmd = "rm -rf #{BENCH_DATA_PATH}"
1883     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1884     raise unless system cmd
1885   end
1886
1887   Dir.mkdir BENCH_DATA_PATH
1888
1889   if $needToCopyVMs
1890     canCopyIntoBenchPath = true
1891     $vms.each {
1892       | vm |
1893       canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
1894     }
1895
1896     if canCopyIntoBenchPath
1897       $vms.each {
1898         | vm |
1899         $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
1900         vm.copyIntoBenchPath
1901       }
1902       $stderr.puts "All VMs are in place."
1903     else
1904       $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
1905     end
1906   end
1907
1908   if $measureGC and $measureGC != true
1909     found = false
1910     $vms.each {
1911       | vm |
1912       if vm.name == $measureGC
1913         found = true
1914       end
1915     }
1916     unless found
1917       $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
1918     end
1919   end
1920
1921   if $outer*$inner == 1
1922     $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
1923   end
1924
1925   $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
1926
1927   $suites = []
1928
1929   if $includeSunSpider and not SUNSPIDER.empty?
1930     $suites << SUNSPIDER
1931   end
1932
1933   if $includeV8 and not V8.empty?
1934     $suites << V8
1935   end
1936
1937   if $includeKraken and not KRAKEN.empty?
1938     $suites << KRAKEN
1939   end
1940
1941   $benchmarks = []
1942   $suites.each {
1943     | suite |
1944     $benchmarks += suite.benchmarks
1945   }
1946
1947   $runPlans = []
1948   $vms.each {
1949     | vm |
1950     $benchmarks.each {
1951       | benchmark |
1952       $outer.times {
1953         | iteration |
1954         $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
1955       }
1956     }
1957   }
1958
1959   $runPlans.shuffle!
1960
1961   $suitepad = $suites.collect {
1962     | suite |
1963     suite.to_s.size
1964   }.max + 1
1965
1966   $benchpad = ($benchmarks +
1967                ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
1968     | benchmark |
1969     if benchmark.respond_to? :name
1970       benchmark.name.size
1971     else
1972       benchmark.size
1973     end
1974   }.max + 1
1975
1976   $vmpad = $vms.collect {
1977     | vm |
1978     vm.to_s.size
1979   }.max + 1
1980
1981   if $prepare
1982     File.open("#{BENCH_DATA_PATH}/runscript", "w") {
1983       | file |
1984       file.puts "echo \"HOSTNAME:\\c\""
1985       file.puts "hostname"
1986       file.puts "echo"
1987       file.puts "echo \"HARDWARE:\\c\""
1988       file.puts "/usr/sbin/sysctl hw.model"
1989       file.puts "echo"
1990       file.puts "set -e"
1991       $script = file
1992       $runPlans.each_with_index {
1993         | plan, idx |
1994         if $verbosity == 0 and not $silent
1995           text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
1996           text2 = plan.benchmark.to_s+"/"+plan.vm.to_s
1997           file.puts("echo "+("\r#{text1} #{rpad(text2,$suitepad+1+$benchpad+1+$vmpad)}".inspect)[0..-2]+"\\c\" 1>&2")
1998           file.puts("echo "+("\r#{text1} #{text2}".inspect)[0..-2]+"\\c\" 1>&2")
1999         end
2000         plan.emitRunCode
2001       }
2002       if $verbosity == 0 and not $silent
2003         file.puts("echo "+("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}".inspect)[0..-2]+"\\c\" 1>&2")
2004         file.puts("echo "+("\r#{$runPlans.size}/#{$runPlans.size}".inspect)+" 1>&2")
2005       end
2006     }
2007   end
2008
2009   if $run
2010     unless $remoteHosts.empty?
2011       $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
2012       Dir.chdir(TEMP_PATH) {
2013         cmd = "tar -czf payload.tar.gz benchdata"
2014         $stderr.puts ">> #{cmd}" if $verbosity>=2
2015         raise unless system(cmd)
2016       }
2017
2018       def grokHost(host)
2019         if host =~ /:([0-9]+)$/
2020           "-p " + $1 + " " + $~.pre_match.inspect
2021         else
2022           host.inspect
2023         end
2024       end
2025
2026       def sshRead(host, command)
2027         cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{grokHost(host)} #{command.inspect}"
2028         $stderr.puts ">> #{cmd}" if $verbosity>=2
2029         result = ""
2030         IO.popen(cmd, "r") {
2031           | inp |
2032           inp.each_line {
2033             | line |
2034             $stderr.puts "#{host}: #{line}" if $verbosity>=2
2035             result += line
2036           }
2037         }
2038         raise "#{$?}" unless $?.success?
2039         result
2040       end
2041
2042       def sshWrite(host, command, data)
2043         cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{grokHost(host)} #{command.inspect}"
2044         $stderr.puts ">> #{cmd}" if $verbosity>=2
2045         IO.popen(cmd, "w") {
2046           | outp |
2047           outp.write(data)
2048         }
2049         raise "#{$?}" unless $?.success?
2050       end
2051
2052       $remoteHosts.each {
2053         | host |
2054         $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
2055
2056         remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
2057         raise unless remoteTempPath
2058
2059         sshWrite(host, "cd #{remoteTempPath.inspect} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
2060
2061         $stderr.puts "Running on #{host}..." if $verbosity==0
2062
2063         parseAndDisplayResults(sshRead(host, "cd #{(remoteTempPath+'/benchdata').inspect} && sh runscript"))
2064       }
2065     end
2066
2067     if not $remoteHosts.empty? and $alsoLocal
2068       $stderr.puts "Running locally..."
2069     end
2070
2071     if $remoteHosts.empty? or $alsoLocal
2072       parseAndDisplayResults(runAndGetResults)
2073     end
2074   end
2075
2076   $analyze.each_with_index {
2077     | filename, index |
2078     if index >= 1
2079       puts
2080     end
2081     parseAndDisplayResults(IO::read(filename))
2082   }
2083
2084   if $prepare and not $run and $analyze.empty?
2085     puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
2086               "the benchmarks and get the results by doing:", 78)
2087     puts
2088     puts "cd #{BENCH_DATA_PATH}"
2089     puts "sh runscript > results.txt"
2090     puts
2091     puts wrap("Then you can analyze the results by running bencher with the same arguments "+
2092               "as now, but replacing --prepare-only with --analyze results.txt.", 78)
2093   end
2094 rescue => e
2095   fail(e)
2096 end
2097
2098