#!/usr/bin/env ruby

# Copyright (C) 2011 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.

require 'rubygems'

require 'getoptlong'
require 'pathname'
require 'tempfile'
require 'socket'

begin
  require 'json'
rescue LoadError => e
  $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
  exit 1
end

# Configuration

CONFIGURATION_FLNM = ENV["HOME"]+"/.bencher"

unless FileTest.exist? CONFIGURATION_FLNM
  $stderr.puts "Error: no configuration file at ~/.bencher."
  $stderr.puts "This file should contain paths to SunSpider, V8, and Kraken, as well as a"
  $stderr.puts "temporary directory that bencher can use for its remote mode. It should be"
  $stderr.puts "formatted in JSON.  For example:"
  $stderr.puts "{"
  $stderr.puts "    \"sunSpiderPath\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/sunspider-1.0\","
  $stderr.puts "    \"v8Path\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/v8-v6\","
  $stderr.puts "    \"krakenPath\": \"/Volumes/Data/pizlo/kraken/kraken-e119421cb325/tests/kraken-1.1\","
  $stderr.puts "    \"tempPath\": \"/Volumes/Data/pizlo/bencher/temp\""
  $stderr.puts "}"
  exit 1
end

CONFIGURATION = JSON.parse(File::read(CONFIGURATION_FLNM))

SUNSPIDER_PATH = CONFIGURATION["sunSpiderPath"]
V8_PATH = CONFIGURATION["v8Path"]
KRAKEN_PATH = CONFIGURATION["krakenPath"]
TEMP_PATH = CONFIGURATION["tempPath"]

IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933, 
            0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684, 
            0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227, 
            0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674, 
            0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652, 
            0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604, 
            0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338, 
            0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538, 
            0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185, 
            0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694, 
            0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244, 
            0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067, 
            0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323, 
            0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129, 
            0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574, 
            0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722, 
            0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624, 
            0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321, 
            0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843, 
            0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217, 
            0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464, 
            0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599, 
            0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638, 
            0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592, 
            0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471, 
            0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283, 
            0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037, 
            0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738, 
            0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481, 
            0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085, 
            0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651, 
            0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183, 
            0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683, 
            0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154, 
            0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599, 
            0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019, 
            0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417, 
            0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795, 
            0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153, 
            0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494, 
            0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818, 
            0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128, 
            0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423, 
            0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704, 
            0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974, 
            0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232, 
            0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479, 
            0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716, 
            0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943, 
            0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192, 
            0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401, 
            0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963, 
            0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822, 
            0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007, 
            0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186, 
            0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358, 
            0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525, 
            0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686, 
            0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841, 
            0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991, 
            0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137, 
            0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278, 
            0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414, 
            0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547, 
            0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675, 
            0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799, 
            0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192, 
            0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037, 
            0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151, 
            0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262, 
            0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369, 
            0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489, 
            0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259, 
            0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689, 
            0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785, 
            0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879, 
            0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297, 
            0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059, 
            0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145, 
            0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242, 
            0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324, 
            0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404, 
            0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483, 
            0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559, 
            0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634, 
            0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707, 
            0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778, 
            0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848, 
            0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916, 
            0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983, 
            0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057, 
            0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121, 
            0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183, 
            0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244, 
            0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304, 
            0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363, 
            0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442, 
            0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476, 
            0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539, 
            0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946, 
            0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653, 
            0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712, 
            0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762, 
            0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812, 
            0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867, 
            0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915, 
            0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962, 
            0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007, 
            0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059, 
            0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103, 
            0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146, 
            0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189, 
            0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231, 
            0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272, 
            0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312, 
            0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352, 
            0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391, 
            0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429, 
            0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467, 
            0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504, 
            0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546, 
            0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582, 
            0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617, 
            0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652, 
            0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686, 
            0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724, 
            0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757, 
            0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579, 
            0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822, 
            0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853, 
            0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885, 
            0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915, 
            0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595, 
            0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979, 
            0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013, 
            0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041, 
            0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074, 
            0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102, 
            0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129, 
            0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156, 
            0.99616, 0.996164]

# Run-time configuration parameters (can be set with command-line options)

$rerun=1
$inner=3
$warmup=1
$outer=4
$includeSunSpider=true
$includeV8=true
$includeKraken=true
$measureGC=false
$benchmarkPattern=nil
$verbosity=0
$innerMode=:reload
$timeMode=:auto
$keepFiles=false
$forceVMKind=nil
$brief=false
$silent=false
$remoteHosts=[]
$sshOptions=[]
$slave=false
$vms = []

# Helpful functions and classes

def smallUsage
  puts "Use the --help option to get basic usage information."
  exit 1
end

def usage
  puts "bencher [options] <vm1> [<vm2> ...]"
  puts
  puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
  puts "benchmarks, and reports detailed statistics.  What makes bencher special is"
  puts "that each benchmark/VM configuration is run in a single VM invocation, and"
  puts "the invocations are run in random order.  This minimizes systematics due to"
  puts "one benchmark polluting the running time of another.  The fine-grained"
  puts "interleaving of VM invocations further minimizes systematics due to changes in"
  puts "the performance or behavior of your machine."
  puts 
  puts "Bencher is highly configurable.  You can compare as many VMs as you like.  You"
  puts "can change the amount of warm-up iterations, number of iterations executed per"
  puts "VM invocation, and the number of VM invocations per benchmark.  By default,"
  puts "SunSpider, VM, and Kraken are all run; but you can run any combination of these"
  puts "suites."
  puts
  puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
  puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
  puts "the executable and <name> is the name that you would like to give the"
  puts "configuration for the purposeof reporting.  If no name is given, a generic name"
  puts "of the form Conf#<n> will be ascribed to the configuration automatically."
  puts
  puts "Options:"
  puts "--rerun <n>          Set the number of iterations of the benchmark that"
  puts "                     contribute to the measured run time.  Default is #{$rerun}."
  puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
  puts "                     iterations.  Default is #{$inner}."
  puts "--outer <n>          Set the number of runtime invocations for each benchmark."
  puts "                     Default is #{$outer}."
  puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
  puts "                     is #{$warmup}."
  puts "--timing-mode        Set the way that bencher measures time.  Possible values"
  puts "                     are 'preciseTime', 'date', and 'auto'.  Default is"
  puts "                     'auto', which automatically detects the best way."
  puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
  puts "                     the one specified.  Valid arguments are 'jsc' or"
  puts "                     'DumpRenderTree'."
  puts "--v8-only            Only run V8."
  puts "--sunspider-only     Only run SunSpider."
  puts "--kraken-only        Only run Kraken."
  puts "--exclude-v8         Exclude V8 (only run SunSpider and Kraken)."
  puts "--exclude-sunspider  Exclude SunSpider (only run V8 and Kraken)."
  puts "--exclude-kraken     Exclude Kraken (only run SunSpider and V8)."
  puts "--benchmarks         Only run benchmarks matching the given regular expression."
  puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
  puts "                     Works best with large values of --inner.  You can also say"
  puts "                     --measure-gc <conf>, which turns this on for one"
  puts "                     configuration only."
  puts "--keep-files         Keep temporary files.  Useful for debugging."
  puts "--verbose or -v      Print more stuff."
  puts "--brief              Print only the final result for each VM."
  puts "--silent             Don't print progress. This might slightly reduce some"
  puts "                     performance perturbation."
  puts "--remote <sshhosts>  Performance performance measurements remotely, on the given"
  puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
  puts "                     user@host string. However, you can also supply a comma-"
  puts "                     separated list of SSH hosts. Alternatively, you can use this"
  puts "                     option multiple times to specify multiple hosts. This"
  puts "                     automatically copies the WebKit release builds of the VMs"
  puts "                     you specified to all of the hosts."
  puts "--ssh-options        Pass additional options to SSH."
  puts "--help or -h         Display this message."
  puts
  puts "Example:"
  puts "bencher TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
  exit 1
end

def fail(reason)
  if reason.respond_to? :backtrace
    puts "FAILED: #{reason}"
    puts "Stack trace:"
    puts reason.backtrace.join("\n")
  else
    puts "FAILED: #{reason}"
  end
  smallUsage
end

def quickFail(r1,r2)
  $stderr.puts "#{$0}: #{r1}"
  puts
  fail(r2)
end

def intArg(argName,arg,min,max)
  result=arg.to_i
  unless result.to_s == arg
    quickFail("Expected an integer value for #{argName}, but got #{arg}.",
              "Invalid argument for command-line option")
  end
  if min and result<min
    quickFail("Argument for #{argName} cannot be smaller than #{min}.",
              "Invalid argument for command-line option")
  end
  if max and result>max
    quickFail("Argument for #{argName} cannot be greater than #{max}.",
              "Invalid argument for command-line option")
  end
  result
end

def computeMean(array)
  sum=0.0
  array.each {
    | value |
    sum += value
  }
  sum/array.length
end

def computeGeometricMean(array)
  mult=1.0
  array.each {
    | value |
    mult*=value
  }
  mult**(1.0/array.length)
end

def computeHarmonicMean(array)
  1.0 / computeMean(array.collect{ | value | 1.0 / value })
end

def computeStdDev(array)
  case array.length
  when 0
    0.0/0.0
  when 1
    0.0
  else
    begin
      mean=computeMean(array)
      sum=0.0
      array.each {
        | value |
        sum += (value-mean)**2
      }
      Math.sqrt(sum/(array.length-1))
    rescue
      0.0/0.0
    end
  end
end

class Array
  def shuffle!
    size.downto(1) { |n| push delete_at(rand(n)) }
    self
  end
end

def inverseBetaRegularized(n)
  IBR_LOOKUP[n-1]
end

def numToStr(num)
  "%.4f"%(num.to_f)
end
  
class NoChange
  attr_reader :amountFaster
  
  def initialize(amountFaster)
    @amountFaster = amountFaster
  end
  
  def shortForm
    " "
  end
  
  def to_s
    if @amountFaster < 1.01
      ""
    else
      "  might be #{numToStr(@amountFaster)}x faster"
    end
  end
end

class Faster
  attr_reader :amountFaster
  
  def initialize(amountFaster)
    @amountFaster = amountFaster
  end
  
  def shortForm
    "^"
  end
  
  def to_s
    "^ definitely #{numToStr(@amountFaster)}x faster"
  end
end

class Slower
  attr_reader :amountSlower
  
  def initialize(amountSlower)
    @amountSlower = amountSlower
  end
  
  def shortForm
    "!"
  end
  
  def to_s
    "! definitely #{numToStr(@amountSlower)}x slower"
  end
end

class MayBeSlower
  attr_reader :amountSlower
  
  def initialize(amountSlower)
    @amountSlower = amountSlower
  end
  
  def shortForm
    "?"
  end
  
  def to_s
    if @amountSlower < 1.01
      "?"
    else
      "? might be #{numToStr(@amountSlower)}x slower"
    end
  end
end

class Stats
  def initialize
    @array = []
  end
  
  def add(value)
    if value.is_a? Stats
      add(value.array)
    elsif value.respond_to? :each
      value.each {
        | v |
        add(v)
      }
    else
      @array << value.to_f
    end
  end
    
  def array
    @array
  end
  
  def sum
    result=0
    @array.each {
      | value |
      result += value
    }
    result
  end
  
  def min
    @array.min
  end
  
  def max
    @array.max
  end
  
  def size
    @array.length
  end
  
  def mean
    computeMean(array)
  end
  
  def arithmeticMean
    mean
  end
  
  def stdDev
    computeStdDev(array)
  end

  def stdErr
    stdDev/Math.sqrt(size)
  end
  
  # Computes a 95% Student's t distribution confidence interval
  def confInt
    if size < 2
      0.0/0.0
    else
      raise if size > 1000
      Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
    end
  end
  
  def lower
    mean-confInt
  end
  
  def upper
    mean+confInt
  end
  
  def geometricMean
    computeGeometricMean(array)
  end
  
  def harmonicMean
    computeHarmonicMean(array)
  end
  
  def compareTo(other)
    if upper < other.lower
      Faster.new(other.mean/mean)
    elsif lower > other.upper
      Slower.new(mean/other.mean)
    elsif mean > other.mean
      MayBeSlower.new(mean/other.mean)
    else
      NoChange.new(other.mean/mean)
    end
  end
  
  def to_s
    "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
  end
end

def doublePuts(out1,out2,msg)
  out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
  out2.puts msg
end

def benchRunHarness(vm, benchpath)
  $stderr.puts "running #{benchpath} with #{vm}..." if $verbosity>=1
  result=nil
  Tempfile.open("bencher") {
    | file |
    yield(file)
    file.flush
    begin
      result=vm.runAndReport(file.path)
    rescue => e
      $stderr.puts "Could not run #{file.path} because #{e}:"
      File.open(file.path,"r") {
        | inp |
        inp.each_line {
          | line |
          $stderr.puts "#{line}"
        }
      }
    end
    file.unlink unless $keepFiles
  }
  raise unless result and result.size == $inner
  result
end

def emitTimeHelpers(file)
  case $timeMode
  when :preciseTime
    doublePuts($stderr,file,"function __bencher_curTimeMS() {")
    doublePuts($stderr,file,"   return preciseTime()*1000")
    doublePuts($stderr,file,"}")
  when :date
    doublePuts($stderr,file,"function __bencher_curTimeMS() {")
    doublePuts($stderr,file,"   return Date.now()")
    doublePuts($stderr,file,"}")
  else
    raise
  end
  doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
  doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
  $rerun.times {
    doublePuts($stderr,file,"   run(__bencher_what);")
  }
  doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
  doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
  doublePuts($stderr,file,"}")
end

def emitBenchRunCode(vm, file, benchpath)
  emitTimeHelpers(file)
  case $innerMode
  when :reload
    case vm.vmType
    when :jsc
      $warmup.times {
        doublePuts($stderr,file,"__bencher_run(#{benchpath.inspect})")
        doublePuts($stderr,file,"gc();") unless vm.shouldMeasureGC
      }
      $inner.times {
        doublePuts($stderr,file,"print(\"Time: \"+__bencher_run(#{benchpath.inspect}));")
        doublePuts($stderr,file,"gc();") unless vm.shouldMeasureGC
      }
    when :dumpRenderTree
      doublePuts($stderr,file,"__bencher_count = 0;")
      doublePuts($stderr,file,"function __bencher_doNext(result) {")
      doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
      doublePuts($stderr,file,"        debug(\"Time: \"+result);")
      doublePuts($stderr,file,"    __bencher_count++;")
      doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
      doublePuts($stderr,file,"        __bencher_runImpl(#{benchpath.inspect}, __bencher_doNext);")
      doublePuts($stderr,file,"    else")
      doublePuts($stderr,file,"        quit();")
      doublePuts($stderr,file,"}")
      doublePuts($stderr,file,"__bencher_runImpl(#{benchpath.inspect}, __bencher_doNext);")
    else
      raise vm.vmType
    end
  when :loadOnce
    doublePuts($stderr,file,"function runit() {")
    File.open(benchpath,'r') {
      | inp |
      inp.each_line {
        | line |
        doublePuts($stderr,file,line)
      }
    }
    doublePuts($stderr,file,"}")
    $warmup.times {
      doublePuts($stderr,file,"runit();")
      doublePuts($stderr,file,"gc();") unless vm.shouldMeasureGC
    }
    $inner.times {
      doublePuts($stderr,file,"before = __bencher_curTimeMS();")
      $rerun.times {
        doublePuts($stderr,file,"runit();")
      }
      doublePuts($stderr,file,"after = __bencher_curTimeMS();")
      doublePuts($stderr,file,"print(\"Time: \"+(after-before));")
      doublePuts($stderr,file,"gc();") unless vm.shouldMeasureGC
    }
    if vm.vmType == :dumpRenderTree
      doublePuts($stderr,file,"quit();")
    end
  else
    raise "bad $innerMode: #{$innerMode}"
  end
end

def runBenchmarkAndReport(vm, benchpath)
  benchRunHarness(vm, benchpath) {
    | file |
    emitBenchRunCode(vm, file, benchpath)
  }
end

class StatsAccumulator
  def initialize
    @stats = []
    ($outer*$inner).times {
      @stats << Stats.new
    }
  end
  
  def statsForIteration(outerIteration, innerIteration)
    @stats[outerIteration*$inner + innerIteration]
  end
  
  def stats
    result = Stats.new
    @stats.each {
      | stat |
      result.add(yield stat)
    }
    result
  end
  
  def geometricMeanStats
    stats {
      | stat |
      stat.geometricMean
    }
  end
  
  def arithmeticMeanStats
    stats {
      | stat |
      stat.arithmeticMean
    }
  end
end

class VM < StatsAccumulator
  def initialize(origPath, path, name, nameKind, svnRevision)
    super()
    @origPath = origPath
    @path = path
    @name = name
    @nameKind = nameKind
    
    if $forceVMKind
      @vmType = $forceVMKind
    else
      Tempfile.open("bencher-vmtest") {
        | file |
        file.puts "print(\"here\");"
        file.flush
        
        result = nil
        @vmType = :jsc
        run(file.path) {
          | inp |
          result = inp.read
          $stderr.puts "stdout: #{result}" if $verbosity>=2
        }
        
        if result.chomp == "here"
          $stderr.puts "#{@name} is definitely a jsc-style VM." if $verbosity>=1
          @vmType = :jsc
        else
          $stderr.puts "Assuming that #{@name} is a DumpRenderTree-style VM." if $verbosity>=1
          @vmType = :dumpRenderTree
        end
      }
    end
    
    @svnRevision = svnRevision
    
    unless $slave
      # Try to detect information about the VM.
      if path =~ /\/WebKitBuild\/Release\/([a-zA-Z]+)$/
        @checkoutPath = $~.pre_match
        unless @svnRevision
          begin
            Dir.chdir(@checkoutPath) {
              $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
              IO.popen("svn info", "r") {
                | inp |
                inp.each_line {
                  | line |
                  if line =~ /Revision: ([0-9]+)/
                    @svnRevision = $1
                  end
                }
              }
            }
            unless @svnRevision
              $stderr.puts "Warning: running svn info for #{name} silently failed."
            end
          rescue => e
            # Failed to detect svn revision.
            $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
          end
        end
      else
        $stderr.puts "Warning: could not identify checkout location for #{name}"
      end
    end
  end
  
  def to_s
    @name
  end
  
  def name
    @name
  end
  
  def shouldMeasureGC
    $measureGC == true or ($measureGC == name)
  end
  
  def origPath
    @origPath
  end
  
  def path
    @path
  end
  
  def nameKind
    @nameKind
  end
  
  def vmType
    @vmType
  end
  
  def checkoutPath
    @checkoutPath
  end
  
  def svnRevision
    @svnRevision
  end
  
  def printFunction
    case @vmType
    when :jsc
      "print"
    when :dumpRenderTree
      "debug"
    else
      raise @vmType
    end
  end
  
  def run(filename)
    case @vmType
    when :jsc
      if @path =~ /\/Release\/([a-zA-Z]+)$/
        raise unless @path =~ /\/([a-zA-Z]+)$/
        libPath = $~.pre_match
        ENV["DYLD_LIBRARY_PATH"]=libPath
        ENV["DYLD_FRAMEWORK_PATH"]=libPath
      else
        ENV["DYLD_LIBRARY_PATH"]=""
        ENV["DYLD_FRAMEWORK_PATH"]=""
      end
      cmd = "#{@path} #{filename}"
      $stderr.puts ">> #{cmd}" if $verbosity>=2
      IO.popen(cmd,"r") {
        | inp |
        yield inp
      }
      $?.success?
    when :dumpRenderTree
      result = nil
      Tempfile.open(["bencher-htmldoc",".html"]) {
        | htmlFile |
        Tempfile.open("bencher-css") {
          | cssFile |
          doublePuts($stderr,cssFile,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
          Tempfile.open("bencher-pre") {
            | preFile |
            doublePuts($stderr,preFile,
                       "if (window.layoutTestController) {\n"+
                       "    layoutTestController.dumpAsText(window.enablePixelTesting);\n"+
                       "    layoutTestController.waitUntilDone();\n"+
                       "}\n"+
                       "\n"+
                       "function debug(msg)\n"+
                       "{\n"+
                       "    var span = document.createElement(\"span\");\n"+
                       "    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace\n"+
                       "    span.innerHTML = msg + '<br />';\n"+
                       "}\n"+
                       "\n"+
                       "function quit() {\n"+
                       "    layoutTestController.notifyDone();\n"+
                       "}\n"+
                       "\n"+
                       "__bencher_continuation=null;\n"+
                       "\n"+
                       "function reportResult(result) {\n"+
                       "    __bencher_continuation(result);\n"+
                       "}\n"+
                       "\n"+
                       "function __bencher_runImpl(filename, continuation) {\n"+
                       "    function doit() {\n"+
                       "        document.getElementById(\"frameparent\").innerHTML = \"\";\n"+
                       "        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";\n"+
                       "        var testFrame = document.getElementById(\"testframe\");\n"+
                       "        testFrame.contentDocument.open();\n"+
                       "        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div><script type=\\\"text/javascript\\\">__bencher_before = Date.now();</script><script src=\\\"file://\"+filename+\"\\\"></script><script type=\\\"text/javascript\\\">window.parent.reportResult(Date.now() - __bencher_before);</script></body></html>\");\n"+
                       "        testFrame.contentDocument.close();\n"+
                       "    }\n"+
                       "    __bencher_continuation = continuation;\n"+
                       "    window.setTimeout(doit, 10);\n"+
                       "}\n")
            doublePuts($stderr,htmlFile,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"file://#{cssFile.path}\"><script src=\"file://#{preFile.path}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"file://#{filename}\"></script></body></html>")
            
            htmlFile.flush
            preFile.flush
            cssFile.flush
            
            cmd = "#{path} #{htmlFile.path}"
            $stderr.puts ">> #{cmd}" if $verbosity>=2
            IO.popen(cmd, "r") {
              | inp |
              yield inp
            }
            result = $?
            
            #sleep 10000
            
            htmlFile.unlink unless $keepFiles
            preFile.unlink unless $keepFiles
            cssFile.unlink unless $keepFiles
          }
        }
      }
      result.success?
    else
      raise @vmType
    end
  end
  
  def runAndReport(filename)
    result = nil
    5.times {
      result = Stats.new
      run(filename) {
        | inp |
        inp.each_line {
          | line |
          $stderr.puts "stdout: #{line}" if $verbosity>=2
          if line =~ /^Time: /
            result.add($~.post_match.to_f)
          end
        }
      }
      break if result.size == $inner
      if $verbosity == 0
        $stderr.print "   "
      end
      $stderr.puts "Failed to run #{filename} on #{self}, retrying..."
    }
    raise "Failed to run #{filename} on #{self}" unless result and result.size == $inner
    result
  end
end

class SunSpiderBenchmark
  attr_accessor :benchmarkSuite
  
  def initialize(name)
    @name = name
  end
  
  def to_s
    @name
  end
  
  def runAndReport(vm)
    runBenchmarkAndReport(vm, "#{SUNSPIDER_PATH}/#{@name}.js")
  end
end

class V8Benchmark
  attr_accessor :benchmarkSuite
  
  def initialize(name)
    @name = name
  end
  
  def to_s
    @name
  end
  
  def runAndReport(vm)
    runBenchmarkAndReport(vm, "#{V8_PATH}/v8-#{@name}.js")
  end
end

class KrakenBenchmark
  attr_accessor :benchmarkSuite
  
  def initialize(name)
    @name = name
  end
  
  def to_s
    @name
  end
  
  def runAndReport(vm)
    # Kraken requires some special-casing
    
    dataPath="#{KRAKEN_PATH}/#{@name}-data.js"
    benchPath="#{KRAKEN_PATH}/#{@name}.js"
    
    benchRunHarness(vm, benchPath) {
      | file |
      emitTimeHelpers(file)
      doublePuts($stderr,file,"load(#{dataPath.inspect});")
      doublePuts($stderr,file,"gc();") unless vm.shouldMeasureGC
      if $innerMode == :reload
        case vm.vmType
        when :jsc
          doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
          doublePuts($stderr,file,"   before = __bencher_curTimeMS();")
          $rerun.times {
            doublePuts($stderr,file,"   load(#{benchPath.inspect});")
          }
          doublePuts($stderr,file,"   after = __bencher_curTimeMS();")
          doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) #{vm.printFunction}(\"Time: \"+(after-before));");
          doublePuts($stderr,file,"   gc();") unless vm.shouldMeasureGC
          doublePuts($stderr,file,"}")
        when :dumpRenderTree
          raise "Kraken in DumpRenderTree is currently unsupported."
        else
          raise vm.vmType
        end
      else
        emitBenchRunCode(vm, file, benchPath)
      end
    }
  end
end

class BenchmarkSuite
  def initialize(name, path, preferredMean)
    @name = name
    @path = path
    @preferredMean = preferredMean
    @benchmarks = []
  end
  
  def name
    @name
  end
  
  def to_s
    @name
  end
  
  def path
    @path
  end
  
  def add(benchmark)
    if not $benchmarkPattern or "#{@name}/#{benchmark.to_s}" =~ $benchmarkPattern
      benchmark.benchmarkSuite = self
      @benchmarks << benchmark
    end
  end
  
  def benchmarks
    @benchmarks
  end
  
  def empty?
    @benchmarks.empty?
  end
  
  def retain_if
    @benchmarks.delete_if {
      | benchmark |
      not yield benchmark
    }
  end
  
  def preferredMean
    @preferredMean
  end
  
  def computeMean(stat)
    stat.send @preferredMean
  end
end

class BenchmarkOnVM
  def initialize(vm, benchmark, suiteOnVM)
    @vm = vm
    @benchmark = benchmark
    @suiteOnVM = suiteOnVM
    @stats = Stats.new
  end
  
  def to_s
    "#{@benchmark} on #{@vm}"
  end
  
  def benchmark
    @benchmark
  end
  
  def vm
    @vm
  end
  
  def suite
    @benchmark.benchmarkSuite
  end
  
  def suiteOnVM
    @suiteOnVM
  end
  
  def stats
    @stats
  end
  
  def runRecordAndReport
    result = @benchmark.runAndReport(@vm)
    @stats.add(result)
    result
  end
end

class SuiteOnVM < StatsAccumulator
  def initialize(vm, suite)
    super()
    @vm = vm
    @suite = suite
  end
  
  def to_s
    "#{@suite} on #{@vm}"
  end
  
  def suite
    @suite
  end
  
  def vm
    @vm
  end
end

class BenchPlan
  def initialize(benchmarkOnVM, iteration)
    @benchmarkOnVM = benchmarkOnVM
    @iteration = iteration
  end
  
  def to_s
    "#{@benchmarkOnVM} \##{@iteration+1}"
  end
  
  def benchmarkOnVM
    @benchmarkOnVM
  end
  
  def benchmark
    @benchmarkOnVM.benchmark
  end
  
  def suite
    @benchmarkOnVM.suite
  end
  
  def vm
    @benchmarkOnVM.vm
  end
  
  def iteration
    @iteration
  end
  
  def runAndRecord
    @benchmarkOnVM.runRecordAndReport.array.each_with_index {
      | value, index |
      @benchmarkOnVM.vm.statsForIteration(@iteration, index).add(value)
      @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, index).add(value)
    }
  end
end

def lpad(str,chars)
  if str.length>chars
    str
  else
    "%#{chars}s"%(str)
  end
end

def rpad(str,chars)
  while str.length<chars
    str+=" "
  end
  str
end

def center(str,chars)
  while str.length<chars
    str+=" "
    if str.length<chars
      str=" "+str
    end
  end
  str
end

def statsToStr(stats)
  if $inner*$outer == 1
    string = numToStr(stats.mean)
    raise unless string =~ /\./
    left = $~.pre_match
    right = $~.post_match
    lpad(left,12)+"."+rpad(right,9)
  else
    lpad(numToStr(stats.mean),11)+"+-"+rpad(numToStr(stats.confInt),9)
  end
end
  
begin
  $sawBenchOptions = false
  
  def resetBenchOptionsIfNecessary
    unless $sawBenchOptions
      $includeSunSpider = false
      $includeV8 = false
      $includeKraken = false
      $sawBenchOptions = true
    end
  end
  
  GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
                 ['--inner', GetoptLong::REQUIRED_ARGUMENT],
                 ['--outer', GetoptLong::REQUIRED_ARGUMENT],
                 ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
                 ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
                 ['--sunspider-only', GetoptLong::NO_ARGUMENT],
                 ['--v8-only', GetoptLong::NO_ARGUMENT],
                 ['--kraken-only', GetoptLong::NO_ARGUMENT],
                 ['--exclude-sunspider', GetoptLong::NO_ARGUMENT],
                 ['--exclude-v8', GetoptLong::NO_ARGUMENT],
                 ['--exclude-kraken', GetoptLong::NO_ARGUMENT],
                 ['--sunspider', GetoptLong::NO_ARGUMENT],
                 ['--v8', GetoptLong::NO_ARGUMENT],
                 ['--kraken', GetoptLong::NO_ARGUMENT],
                 ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
                 ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
                 ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
                 ['--load-once', GetoptLong::NO_ARGUMENT],
                 ['--keep-files', GetoptLong::NO_ARGUMENT],
                 ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
                 ['--brief', GetoptLong::NO_ARGUMENT],
                 ['--silent', GetoptLong::NO_ARGUMENT],
                 ['--remote', GetoptLong::REQUIRED_ARGUMENT],
                 ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
                 ['--slave', GetoptLong::NO_ARGUMENT],
                 ['--vms', GetoptLong::REQUIRED_ARGUMENT],
                 ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
    | opt, arg |
    case opt
    when '--rerun'
      $rerun = intArg(opt,arg,1,nil)
    when '--inner'
      $inner = intArg(opt,arg,1,nil)
    when '--outer'
      $outer = intArg(opt,arg,1,nil)
    when '--warmup'
      $warmup = intArg(opt,arg,0,nil)
    when '--timing-mode'
      if arg.upcase == "PRECISETIME"
        $timeMode = :preciseTime
      elsif arg.upcase == "DATE"
        $timeMode = :date
      elsif arg.upcase == "AUTO"
        $timeMode = :auto
      else
        quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
                  "Invalid argument for command-line option")
      end
    when '--force-vm-kind'
      if arg.upcase == "JSC"
        $forceVMKind = :jsc
      elsif arg.upcase == "DUMPRENDERTREE"
        $forceVMKind = :dumpRenderTree
      elsif arg.upcase == "AUTO"
        $forceVMKind = nil
      else
        quickFail("Expected either 'jsc' or 'DumpRenderTree' for --force-vm-kind, but got '#{arg}'.",
                  "Invalid argument for command-line option")
      end
    when '--sunspider-only'
      $includeV8 = false
      $includeKraken = false
    when '--v8-only'
      $includeSunSpider = false
      $includeKraken = false
    when '--kraken-only'
      $includeSunSpider = false
      $includeV8 = false
    when '--exclude-sunspider'
      $includeSunSpider = false
    when '--exclude-v8'
      $includeV8 = false
    when '--exclude-kraken'
      $includeKraken = false
    when '--sunspider'
      resetBenchOptionsIfNecessary
      $includeSunSpider = true
    when '--v8'
      resetBenchOptionsIfNecessary
      $includeV8 = true
    when '--kraken'
      resetBenchOptionsIfNecessary
      $includeKraken = true
    when '--benchmarks'
      $benchmarkPattern = Regexp.new(arg)
    when '--measure-gc'
      if arg == ''
        $measureGC = true
      else
        $measureGC = arg
      end
    when '--load-once'
      $innerMode = :loadOnce
    when '--keep-files'
      $keepFiles = true
    when '--verbose'
      $verbosity += 1
    when '--brief'
      $brief = true
    when '--silent'
      $silent = true
    when '--remote'
      $remoteHosts += arg.split(',')
    when '--ssh-options'
      $sshOptions << arg
    when '--slave'
      $slave = true
    when '--vms'
      JSON.parse(File::read(arg)).each {
        | vmDescriptor |
        $vms << VM.new(vmDescriptor["origPath"],
                       vmDescriptor["path"],
                       vmDescriptor["name"],
                       vmDescriptor["nameKind"].to_sym,
                       vmDescriptor["svnRevision"])
      }
    when '--help'
      usage
    else
      raise "bad option: #{opt}"
    end
  }
  
  SUNSPIDER = BenchmarkSuite.new("SunSpider", SUNSPIDER_PATH, :arithmeticMean)
  ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
   "access-fannkuch", "access-nbody", "access-nsieve",
   "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
   "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
   "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
   "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
   "string-base64", "string-fasta", "string-tagcloud",
   "string-unpack-code", "string-validate-input"].each {
    | name |
    SUNSPIDER.add SunSpiderBenchmark.new(name)
  }

  V8 = BenchmarkSuite.new("V8", V8_PATH, :geometricMean)
  ["crypto", "deltablue", "earley-boyer", "raytrace",
   "regexp", "richards", "splay"].each {
    | name |
    V8.add V8Benchmark.new(name)
  }

  KRAKEN = BenchmarkSuite.new("Kraken", KRAKEN_PATH, :arithmeticMean)
  ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
   "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
   "imaging-gaussian-blur", "json-parse-financial",
   "json-stringify-tinderbox", "stanford-crypto-aes",
   "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
   "stanford-crypto-sha256-iterative"].each {
    | name |
    KRAKEN.add KrakenBenchmark.new(name)
  }

  ARGV.each {
    | vm |
    if vm =~ /([a-zA-Z0-9_ ]+):/
      name = $1
      nameKind = :given
      vm = $~.post_match
    else
      name = "Conf\##{$vms.length+1}"
      nameKind = :auto
    end
    $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
    $vms << VM.new(Pathname.new(vm).realpath, vm, name, nameKind, nil)
  }
  
  if $vms.empty?
    quickFail("Please specify at least on configuraiton on the command line.",
              "Insufficient arguments")
  end
  
  if $measureGC and $measureGC != true
    found = false
    $vms.each {
      | vm |
      if vm.name == $measureGC
        found = true
      end
    }
    unless found
      $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
    end
  end
  
  if $timeMode == :auto
    havePreciseTime = true
    $vms.each {
      | vm |
      if vm.vmType == :dumpRenderTree
        $stderr.puts "Warning: #{vm} does not have preciseTime() because it is DumpRenderTree; using Date.now instead."
        if $rerun != 1
          quickFail("Cannot use --rerun with a DumpRenderTree style VM, because this support has not yet been implemented.", "Wrong option for VM type.")
        end
        havePreciseTime = false
      else
        Tempfile.open("bencher-timetest") {
          | file |
          doublePuts($stderr,file,"#{vm.printFunction}(\"Time: \"+preciseTime());")
          file.flush
          result = nil
          vm.run(file.path) {
            | inp |
            result = inp.read
          }
          thisVMHasPreciseTime = false
          if result =~ /Time: /
            secs = $~.post_match.to_i
            if (secs - Time.now.to_i).abs < 5
              thisVMHasPreciseTime = true
            end
          end
          unless thisVMHasPreciseTime
            $stderr.puts "Warning: #{vm} does not have preciseTime(); using Date.now instead."
            havePreciseTime = false
          end
          file.unlink unless $keepFiles
        }
      end
    }
    
    if havePreciseTime
      $timeMode = :preciseTime
    else
      $timeMode = :date
    end
  end
  
  if $outer*$inner == 1
    $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
  end
  
  $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
  
  $suites = []
  
  if $includeSunSpider and not SUNSPIDER.empty?
    $suites << SUNSPIDER
  end
  
  if $includeV8 and not V8.empty?
    $suites << V8
  end
  
  if $includeKraken and not KRAKEN.empty?
    $suites << KRAKEN
  end
  
  $benchmarks = []
  $suites.each {
    | suite |
    $benchmarks += suite.benchmarks
  }
  
  $suitesOnVMs = []
  $suitesOnVMsForSuite = {}
  $suites.each {
    | suite |
    $suitesOnVMsForSuite[suite] = []
  }
  $suitesOnVMsForVM = {}
  $vms.each {
    | vm |
    $suitesOnVMsForVM[vm] = []
  }
  
  $benchmarksOnVMs = []
  $benchmarksOnVMsForBenchmark = {}
  $benchmarks.each {
    | benchmark |
    $benchmarksOnVMsForBenchmark[benchmark] = []
  }
  
  $vms.each {
    | vm |
    $suites.each {
      | suite |
      suiteOnVM = SuiteOnVM.new(vm, suite)
      $suitesOnVMs << suiteOnVM
      $suitesOnVMsForSuite[suite] << suiteOnVM
      $suitesOnVMsForVM[vm] << suiteOnVM
      suite.benchmarks.each {
        | benchmark |
        benchmarkOnVM = BenchmarkOnVM.new(vm, benchmark, suiteOnVM)
        $benchmarksOnVMs << benchmarkOnVM
        $benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
      }
    }
  }
  
  unless $remoteHosts.empty?
    # Handle the remote benchmarking case.
    
    raise unless TEMP_PATH
    
    $stderr.puts "Packaging VM builds for remote hosts..." if $verbosity==0
    
    # First package all of the release builds.
    $vms.each_with_index {
      | vm, index |
      unless vm.checkoutPath
        fail("Could not figure out how to package #{vm.name}, because the VM type was not recognized", "Bad VM kind for --remote")
      end
      
      Dir.chdir(vm.checkoutPath + "/WebKitBuild") {
        cmd = "tar -czf #{TEMP_PATH}/vm#{index}.tar.gz Release"
        $stderr.puts ">> #{cmd}" if $verbosity>=2
        raise unless system(cmd)
      }
    }
    
    # Now actually go talk to the remote hosts.
    
    def sshRead(host, command)
      cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{host.inspect} #{command.inspect}"
      $stderr.puts ">> #{cmd}" if $verbosity>=2
      result = ""
      IO.popen(cmd, "r") {
        | inp |
        inp.each_line {
          | line |
          $stderr.puts "#{host}: #{line}" if $verbosity>=2
          result += line
        }
      }
      raise "#{$?}" unless $?.success?
      result
    end
    
    def sshWrite(host, command, data)
      cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{host.inspect} #{command.inspect}"
      $stderr.puts ">> #{cmd}" if $verbosity>=2
      IO.popen(cmd, "w") {
        | outp |
        outp.write(data)
      }
      raise "#{$?}" unless $?.success?
    end
    
    $remoteHosts.each {
      | host |
      
      $stderr.puts "Sending VM builds to #{host}..." if $verbosity==0
      
      remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
      raise unless remoteTempPath
      
      sshWrite(host, "cd #{remoteTempPath.inspect} && cat > bencher && chmod 700 bencher", IO::read($0))
      
      $vms.size.times {
        | index |
        sshWrite(host, "cd #{remoteTempPath.inspect} && rm -rf vm#{index} && mkdir vm#{index} && cd vm#{index} && tar -xzf /dev/stdin", IO::read("#{TEMP_PATH}/vm#{index}.tar.gz"))
      }
      
      config=[]
      $vms.each_with_index {
        | vm, index |
        config << {
          "origPath" => vm.origPath,
          "path" => "#{remoteTempPath}/vm#{index}/Release/jsc",
          "name" => vm.name,
          "nameKind" => vm.nameKind.to_s,
          "svnRevision" => vm.svnRevision
        }
      }
      
      sshWrite(host, "cd #{remoteTempPath.inspect} && (cat > vms.conf)", config.to_json)
      
      $stderr.puts "Running on #{host}..." if $verbosity==0
      
      cmd = "cd #{remoteTempPath.inspect} && ./bencher"
      cmd += " --rerun #{$rerun}"
      cmd += " --inner #{$inner}"
      cmd += " --outer #{$outer}"
      cmd += " --warmup #{$warmup}"
      cmd += " --timing-mode #{$timeMode.to_s}"
      cmd += " --sunspider" if $includeSunSpider
      cmd += " --v8" if $includeV8
      cmd += " --kraken" if $includeKraken
      if $measureGC
        if $measureGC == true
          cmd += " --measure-gc"
        else
          cmd += " --measure-gc #{$measureGC.inspect}"
        end
      end
      if $verbosity>=1
        cmd += " -"
        $verbosity.times {
          cmd += "v"
        }
      end
      cmd += " --brief" if $brief
      cmd += " --silent" if $silent
      cmd += " --slave"
      cmd += " --vms #{(remoteTempPath+'/vms.conf').inspect}"
      result = sshRead(host, cmd)
      puts result
      if result =~ /Generating benchmark report at (.*)\n\n/
        reportName = $1
        report = $~.post_match
        File.open(reportName, "w") {
          | outp |
          outp.puts report
        }
      end
    }
    
    exit 0
  end

  $plans = []
  $benchmarksOnVMs.each {
    | benchmarkOnVM |
    $outer.times {
      | iteration |
      $plans << BenchPlan.new(benchmarkOnVM, iteration)
    }
  }
  
  $plans.shuffle!
  
  $suitepad = $suites.collect {
    | suite |
    suite.to_s.size
  }.max + 1
  
  $benchpad = ($benchmarks +
               ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
    | benchmark |
    benchmark.to_s.size
  }.max + 1

  $vmpad = $vms.collect {
    | vm |
    vm.to_s.size
  }.max + 1

  $plans.each_with_index {
    | plan, idx |
    if $verbosity == 0 and not $silent
      text1 = lpad(idx.to_s,$plans.size.to_s.size)+"/"+$plans.size.to_s
      text2 = plan.suite.to_s+"/"+plan.benchmark.to_s+"/"+plan.vm.to_s
      $stderr.print "\r#{text1} #{rpad(text2,$suitepad+1+$benchpad+1+$vmpad)}"
      $stderr.print "\r#{text1} #{text2}"
      $stderr.flush
    end
    plan.runAndRecord
  }
  
  if $verbosity == 0 and not $silent
    $stderr.print "\r#{$plans.size}/#{$plans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}"
    $stderr.puts "\r#{$plans.size}/#{$plans.size}"
  end
  
  # Compute the geomean of the preferred means of results on a SuiteOnVM
  $overallResults = []
  $vms.each {
    | vm |
    result = Stats.new
    $outer.times {
      | outerIndex |
      $inner.times {
        | innerIndex |
        curResult = Stats.new
        $suitesOnVMsForVM[vm].each {
          | suiteOnVM |
          # For a given iteration, suite, and VM, compute the suite's preferred mean
          # over the data collected for all benchmarks in that suite. We'll have one
          # sample per benchmark. For example on V8 this will be the geomean of 1
          # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
          # splay.
          curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
        }
        
        # curResult now holds 1 sample for each of the means computed in the above
        # loop. Compute the geomean over this, and store it.
        result.add(curResult.geometricMean)
      }
    }

    # $overallResults will have a Stats for each VM. That Stats object will hold
    # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
    # confidence interval of the geomeans of preferred means. Convoluted, but
    # useful and probably sound.
    $overallResults << result
  }
  
  if $verbosity >= 2
    $benchmarksOnVMs.each {
      | benchmarkOnVM |
      $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
    }
    
    $vms.each {
      | vm |
      $stderr.puts "#{vm} (arithmeticMean): #{vm.arithmeticMeanStats}"
      $stderr.puts "#{vm} (geometricMean): #{vm.geometricMeanStats}"
    }
  end

  reportName =
    (if ($vms.collect {
           | vm |
           vm.nameKind
         }.index :auto)
       ""
     else
       $vms.collect {
         | vm |
         vm.to_s
       }.join("_") + "_"
     end) +
    ($suites.collect {
       | suite |
       suite.to_s
     }.join("")) + "_" +
    (begin
       time = Time.now
       "%04d%02d%02d_%02d%02d" %
         [ time.year, time.month, time.day,
           time.hour, time.min ]
     end) +
    "_benchReport.txt"

  unless $brief
    puts "Generating benchmark report at #{reportName}"
  end
  
  outp = $stdout
  begin
    outp = File.open(reportName,"w")
  rescue => e
    $stderr.puts "Error: could not save report to #{reportName}: #{e}"
    $stderr.puts
  end
  
  def createVMsString
    result = ""
    result += "   " if $suites.size > 1
    result += rpad("", $benchpad)
    result += " "
    $vms.size.times {
      | index |
      if index != 0
        result += " "+NoChange.new(0).shortForm
      end
      result += lpad(center($vms[index].name, 9+9+2), 11+9+2)
    }
    result += "    "
    if $vms.size >= 3
      result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
    else
      result += " "*26
    end
    result
  end
  
  $columns = createVMsString.size
  
  def plural(num)
    if num == 1
      ""
    else
      "s"
    end
  end
  
  def wrap(str)
    array = str.split
    result = ""
    curLine = array.shift
    array.each {
      | curStr |
      if (curLine + " " + curStr).size > $columns
        result += curLine + "\n"
        curLine = curStr
      else
        curLine += " " + curStr
      end
    }
    result + curLine + "\n"
  end
  
  outp.print "Benchmark report for "
  if $suites.size == 1
    outp.print $suites[0].to_s
  elsif $suites.size == 2
    outp.print "#{$suites[0]} and #{$suites[1]}"
  else
    outp.print "#{$suites[0..-2].join(', ')}, and #{$suites[-1]}"
  end
  outp.print " on #{Socket.gethostname}"
  begin
    IO.popen("sysctl hw.model", "r") {
      | inp |
      inp.each_line {
        | line |
        if line =~ /hw.model: /
          outp.print " (#{$~.post_match.chomp})"
          break
        end
      }
    }
  rescue => e
    # Silently fail since this isn't critical.
  end
  outp.puts "."
  outp.puts
  
  # This looks stupid; revisit later.
  if false
    $suites.each {
      | suite |
      outp.puts "#{suite} at #{suite.path}"
    }
    
    outp.puts
  end
  
  outp.puts "VMs tested:"
  $vms.each {
    | vm |
    outp.print "\"#{vm.name}\" at #{vm.origPath}"
    if vm.svnRevision
      outp.print " (r#{vm.svnRevision})"
    end
    outp.puts
  }
  
  outp.puts
  
  outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
                 "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
                 (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
                                      "total time of those iterations, for each sample.")
                  else "" end)+
                 (if $measureGC == true then (" No manual garbage collection invocations were "+
                                              "emitted.")
                  elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
                                         "all VMs except #{$measureGC}.")
                  else (" Emitted a call to gc() between sample measurements.") end)+
                 (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
                                        "began with the very first iteration.")
                  else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
                        "invocation for warm-up.") end)+
                 (case $timeMode
                  when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
                                          "microsecond-level timing.")
                  when :date then (" Used the portable Date.now() method to get millisecond-"+
                                   "level timing.")
                  else raise end)+
                 (case $innerMode
                  when :reload then "" # nothing interesting to say
                  when :loadOnce then (" Ran benchmarks using an experimental mode that "+
                                       "ensures that the benchmark code is loaded only once "+
                                       "during each VM invocation, which enables more VM "+
                                       "warm-up.")
                  else raise end)+
                 " Reporting benchmark execution times with 95% confidence "+
                 "intervals in milliseconds.")
  
  outp.puts
  
  def printVMs(outp)
    outp.puts createVMsString
  end
  
  def summaryStats(outp, accumulators, name, &proc)
    outp.print "   " if $suites.size > 1
    outp.print rpad(name, $benchpad)
    outp.print " "
    accumulators.size.times {
      | index |
      if index != 0
        outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
      end
      outp.print statsToStr(accumulators[index].stats(&proc))
    }
    if accumulators.size>=2
      outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).to_s)
    end
    outp.puts
  end
  
  def meanName(currentMean, preferredMean)
    result = "<#{currentMean}>"
    if "#{currentMean}Mean" == preferredMean.to_s
      result += " *"
    end
    result
  end
  
  def allSummaryStats(outp, accumulators, preferredMean)
    summaryStats(outp, accumulators, meanName("arithmetic", preferredMean)) {
      | stat |
      stat.arithmeticMean
    }
    
    summaryStats(outp, accumulators, meanName("geometric", preferredMean)) {
      | stat |
      stat.geometricMean
    }
    
    summaryStats(outp, accumulators, meanName("harmonic", preferredMean)) {
      | stat |
      stat.harmonicMean
    }
  end
  
  $suites.each {
    | suite |
    printVMs(outp)
    if $suites.size > 1
      outp.puts "#{suite.name}:"
    else
      outp.puts
    end
    suite.benchmarks.each {
      | benchmark |
      outp.print "   " if $suites.size > 1
      outp.print rpad(benchmark.to_s, $benchpad)
      outp.print " "
      myConfigs = $benchmarksOnVMsForBenchmark[benchmark]
      myConfigs.size.times {
        | index |
        if index != 0
          outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
        end
        outp.print statsToStr(myConfigs[index].stats)
      }
      if $vms.size>=2
        outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
      end
      outp.puts
    }
    outp.puts
    allSummaryStats(outp, $suitesOnVMsForSuite[suite], suite.preferredMean)
    outp.puts if $suites.size > 1
  }
  
  if $suites.size > 1
    printVMs(outp)
    outp.puts "All benchmarks:"
    allSummaryStats(outp, $vms, nil)
    
    outp.puts
    printVMs(outp)
    outp.puts "Geomean of preferred means:"
    outp.print "   "
    outp.print rpad("<scaled-result>", $benchpad)
    outp.print " "
    $vms.size.times {
      | index |
      if index != 0
        outp.print " "+$overallResults[index].compareTo($overallResults[index-1]).shortForm
      end
      outp.print statsToStr($overallResults[index])
    }
    if $overallResults.size>=2
      outp.print("    "+$overallResults[-1].compareTo($overallResults[0]).to_s)
    end
    outp.puts
  end
  outp.puts
  
  if outp != $stdout
    outp.close
  end
  
  if outp != $stdout and not $brief
    puts
    File.open(reportName) {
      | inp |
      puts inp.read
    }
  end
  
  if $brief
    puts($overallResults.collect{|stats| stats.mean}.join("\t"))
    puts($overallResults.collect{|stats| stats.confInt}.join("\t"))
  end
  
rescue => e
  fail(e)
end