3 # Copyright (C) 2011 Apple Inc. All rights reserved.
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
8 # 1. Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # 2. Redistributions in binary form must reproduce the above copyright
11 # notice, this list of conditions and the following disclaimer in the
12 # documentation and/or other materials provided with the distribution.
14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24 # THE POSSIBILITY OF SUCH DAMAGE.
36 $stderr.puts "It does not appear that you have the 'json' package installed. Try running 'sudo gem install json'."
42 CONFIGURATION_FLNM = ENV["HOME"]+"/.bencher"
44 unless FileTest.exist? CONFIGURATION_FLNM
45 $stderr.puts "Error: no configuration file at ~/.bencher."
46 $stderr.puts "This file should contain paths to SunSpider, V8, and Kraken, as well as a"
47 $stderr.puts "temporary directory that bencher can use for its remote mode. It should be"
48 $stderr.puts "formatted in JSON. For example:"
50 $stderr.puts " \"sunSpiderPath\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/sunspider-1.0\","
51 $stderr.puts " \"v8Path\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/v8-v6\","
52 $stderr.puts " \"krakenPath\": \"/Volumes/Data/pizlo/kraken/kraken-e119421cb325/tests/kraken-1.1\","
53 $stderr.puts " \"tempPath\": \"/Volumes/Data/pizlo/bencher/temp\""
58 CONFIGURATION = JSON.parse(File::read(CONFIGURATION_FLNM))
60 SUNSPIDER_PATH = CONFIGURATION["sunSpiderPath"]
61 V8_PATH = CONFIGURATION["v8Path"]
62 KRAKEN_PATH = CONFIGURATION["krakenPath"]
63 TEMP_PATH = CONFIGURATION["tempPath"]
64 BENCH_DATA_PATH = TEMP_PATH + "/benchdata"
66 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933,
67 0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684,
68 0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227,
69 0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674,
70 0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652,
71 0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604,
72 0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338,
73 0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538,
74 0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185,
75 0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694,
76 0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244,
77 0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067,
78 0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323,
79 0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129,
80 0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574,
81 0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722,
82 0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624,
83 0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321,
84 0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843,
85 0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217,
86 0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464,
87 0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599,
88 0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638,
89 0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592,
90 0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471,
91 0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283,
92 0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037,
93 0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738,
94 0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481,
95 0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085,
96 0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651,
97 0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183,
98 0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683,
99 0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154,
100 0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599,
101 0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019,
102 0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417,
103 0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795,
104 0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153,
105 0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494,
106 0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818,
107 0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128,
108 0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423,
109 0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704,
110 0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974,
111 0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232,
112 0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479,
113 0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716,
114 0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943,
115 0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192,
116 0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401,
117 0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963,
118 0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822,
119 0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007,
120 0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186,
121 0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358,
122 0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525,
123 0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686,
124 0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841,
125 0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991,
126 0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137,
127 0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278,
128 0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414,
129 0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547,
130 0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675,
131 0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799,
132 0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192,
133 0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037,
134 0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151,
135 0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262,
136 0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369,
137 0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489,
138 0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259,
139 0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689,
140 0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785,
141 0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879,
142 0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297,
143 0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059,
144 0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145,
145 0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242,
146 0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324,
147 0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404,
148 0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483,
149 0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559,
150 0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634,
151 0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707,
152 0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778,
153 0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848,
154 0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916,
155 0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983,
156 0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057,
157 0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121,
158 0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183,
159 0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244,
160 0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304,
161 0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363,
162 0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442,
163 0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476,
164 0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539,
165 0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946,
166 0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653,
167 0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712,
168 0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762,
169 0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812,
170 0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867,
171 0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915,
172 0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962,
173 0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007,
174 0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059,
175 0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103,
176 0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146,
177 0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189,
178 0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231,
179 0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272,
180 0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312,
181 0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352,
182 0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391,
183 0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429,
184 0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467,
185 0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504,
186 0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546,
187 0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582,
188 0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617,
189 0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652,
190 0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686,
191 0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724,
192 0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757,
193 0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579,
194 0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822,
195 0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853,
196 0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885,
197 0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915,
198 0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595,
199 0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979,
200 0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013,
201 0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041,
202 0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074,
203 0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102,
204 0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129,
205 0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156,
208 # Run-time configuration parameters (can be set with command-line options)
214 $includeSunSpider=true
218 $benchmarkPattern=nil
220 $timeMode=:preciseTime
228 $needToCopyVMs = false
235 # Helpful functions and classes
238 puts "Use the --help option to get basic usage information."
243 puts "bencher [options] <vm1> [<vm2> ...]"
245 puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
246 puts "benchmarks, and reports detailed statistics. What makes bencher special is"
247 puts "that each benchmark/VM configuration is run in a single VM invocation, and"
248 puts "the invocations are run in random order. This minimizes systematics due to"
249 puts "one benchmark polluting the running time of another. The fine-grained"
250 puts "interleaving of VM invocations further minimizes systematics due to changes in"
251 puts "the performance or behavior of your machine."
253 puts "Bencher is highly configurable. You can compare as many VMs as you like. You"
254 puts "can change the amount of warm-up iterations, number of iterations executed per"
255 puts "VM invocation, and the number of VM invocations per benchmark. By default,"
256 puts "SunSpider, VM, and Kraken are all run; but you can run any combination of these"
259 puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
260 puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
261 puts "the executable and <name> is the name that you would like to give the"
262 puts "configuration for the purposeof reporting. If no name is given, a generic name"
263 puts "of the form Conf#<n> will be ascribed to the configuration automatically."
266 puts "--rerun <n> Set the number of iterations of the benchmark that"
267 puts " contribute to the measured run time. Default is #{$rerun}."
268 puts "--inner <n> Set the number of inner (per-runtime-invocation)"
269 puts " iterations. Default is #{$inner}."
270 puts "--outer <n> Set the number of runtime invocations for each benchmark."
271 puts " Default is #{$outer}."
272 puts "--warmup <n> Set the number of warm-up runs per invocation. Default"
273 puts " is #{$warmup}."
274 puts "--timing-mode Set the way that bencher measures time. Possible values"
275 puts " are 'preciseTime' and 'date'. Default is 'preciseTime'."
276 puts "--force-vm-kind Turn off auto-detection of VM kind, and assume that it is"
277 puts " the one specified. Valid arguments are 'jsc' or"
278 puts " 'DumpRenderTree'."
279 puts "--force-vm-copy Force VM builds to be copied to bencher's working directory."
280 puts " This may reduce pathologies resulting from path names."
281 puts "--dont-copy-vms Don't copy VMs even when doing a remote benchmarking run;"
282 puts " instead assume that they are already there."
283 puts "--v8-only Only run V8."
284 puts "--sunspider-only Only run SunSpider."
285 puts "--kraken-only Only run Kraken."
286 puts "--exclude-v8 Exclude V8 (only run SunSpider and Kraken)."
287 puts "--exclude-sunspider Exclude SunSpider (only run V8 and Kraken)."
288 puts "--exclude-kraken Exclude Kraken (only run SunSpider and V8)."
289 puts "--benchmarks Only run benchmarks matching the given regular expression."
290 puts "--measure-gc Turn off manual calls to gc(), so that GC time is measured."
291 puts " Works best with large values of --inner. You can also say"
292 puts " --measure-gc <conf>, which turns this on for one"
293 puts " configuration only."
294 puts "--verbose or -v Print more stuff."
295 puts "--brief Print only the final result for each VM."
296 puts "--silent Don't print progress. This might slightly reduce some"
297 puts " performance perturbation."
298 puts "--remote <sshhosts> Performance performance measurements remotely, on the given"
299 puts " SSH host(s). Easiest way to use this is to specify the SSH"
300 puts " user@host string. However, you can also supply a comma-"
301 puts " separated list of SSH hosts. Alternatively, you can use this"
302 puts " option multiple times to specify multiple hosts. This"
303 puts " automatically copies the WebKit release builds of the VMs"
304 puts " you specified to all of the hosts."
305 puts "--ssh-options Pass additional options to SSH."
306 puts "--local Also do a local benchmark run even when doing --remote."
307 puts "--prepare-only Only prepare the bencher runscript (a shell script that"
308 puts " invokes the VMs to run benchmarks) but don't run it."
309 puts "--analyze Only read the output of the runscript but don't do anything"
310 puts " else. This requires passing the same arguments to bencher"
311 puts " that you passed when running --prepare-only."
312 puts "--help or -h Display this message."
315 puts "bencher TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
320 if reason.respond_to? :backtrace
321 puts "FAILED: #{reason}"
323 puts reason.backtrace.join("\n")
325 puts "FAILED: #{reason}"
331 $stderr.puts "#{$0}: #{r1}"
336 def intArg(argName,arg,min,max)
338 unless result.to_s == arg
339 quickFail("Expected an integer value for #{argName}, but got #{arg}.",
340 "Invalid argument for command-line option")
342 if min and result<min
343 quickFail("Argument for #{argName} cannot be smaller than #{min}.",
344 "Invalid argument for command-line option")
346 if max and result>max
347 quickFail("Argument for #{argName} cannot be greater than #{max}.",
348 "Invalid argument for command-line option")
353 def computeMean(array)
362 def computeGeometricMean(array)
368 mult**(1.0/array.length)
371 def computeHarmonicMean(array)
372 1.0 / computeMean(array.collect{ | value | 1.0 / value })
375 def computeStdDev(array)
383 mean=computeMean(array)
387 sum += (value-mean)**2
389 Math.sqrt(sum/(array.length-1))
398 size.downto(1) { |n| push delete_at(rand(n)) }
403 def inverseBetaRegularized(n)
412 attr_reader :amountFaster
414 def initialize(amountFaster)
415 @amountFaster = amountFaster
423 " might be #{numToStr(@amountFaster)}x faster"
427 if @amountFaster < 1.01
436 attr_reader :amountFaster
438 def initialize(amountFaster)
439 @amountFaster = amountFaster
447 "^ definitely #{numToStr(@amountFaster)}x faster"
456 attr_reader :amountSlower
458 def initialize(amountSlower)
459 @amountSlower = amountSlower
467 "! definitely #{numToStr(@amountSlower)}x slower"
476 attr_reader :amountSlower
478 def initialize(amountSlower)
479 @amountSlower = amountSlower
487 "? might be #{numToStr(@amountSlower)}x slower"
491 if @amountSlower < 1.01
507 elsif value.respond_to? :each
555 stdDev/Math.sqrt(size)
558 # Computes a 95% Student's t distribution confidence interval
564 Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
577 computeGeometricMean(array)
581 computeHarmonicMean(array)
585 if upper < other.lower
586 Faster.new(other.mean/mean)
587 elsif lower > other.upper
588 Slower.new(mean/other.mean)
589 elsif mean > other.mean
590 MayBeSlower.new(mean/other.mean)
592 NoChange.new(other.mean/mean)
597 "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
601 def doublePuts(out1,out2,msg)
602 out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
606 class Benchfile < File
609 attr_reader :filename, :basename
612 @basename, @filename = Benchfile.uniqueFilename(name)
613 super(@filename, "w")
616 def self.uniqueFilename(name)
618 basename = name[0] + @@counter.to_s + name[1]
620 basename = name + @@counter.to_s
622 filename = BENCH_DATA_PATH + "/" + basename
624 raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
628 def self.create(name)
629 file = Benchfile.new(name)
637 def ensureFile(key, filename)
638 unless $dataFiles[key]
639 $dataFiles[key] = Benchfile.create(key) {
641 doublePuts($stderr,outp,IO::read(filename))
647 def emitBenchRunCodeFile(name, plan, benchDataPath, benchPath)
650 Benchfile.create("bencher") {
654 doublePuts($stderr,file,"function __bencher_curTimeMS() {")
655 doublePuts($stderr,file," return preciseTime()*1000")
656 doublePuts($stderr,file,"}")
658 doublePuts($stderr,file,"function __bencher_curTimeMS() {")
659 doublePuts($stderr,file," return Date.now()")
660 doublePuts($stderr,file,"}")
666 doublePuts($stderr,file,"load(#{benchDataPath.inspect});")
667 doublePuts($stderr,file,"gc();")
668 doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
669 doublePuts($stderr,file," before = __bencher_curTimeMS();")
671 doublePuts($stderr,file," load(#{benchPath.inspect});")
673 doublePuts($stderr,file," after = __bencher_curTimeMS();")
674 doublePuts($stderr,file," if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(after-before));");
675 doublePuts($stderr,file," gc();") unless plan.vm.shouldMeasureGC
676 doublePuts($stderr,file,"}")
678 doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
679 doublePuts($stderr,file," var __bencher_before = __bencher_curTimeMS();")
681 doublePuts($stderr,file," run(__bencher_what);")
683 doublePuts($stderr,file," var __bencher_after = __bencher_curTimeMS();")
684 doublePuts($stderr,file," return __bencher_after - __bencher_before;")
685 doublePuts($stderr,file,"}")
687 doublePuts($stderr,file,"__bencher_run(#{benchPath.inspect})")
688 doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
692 doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchPath.inspect}));")
693 doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
698 mainCode = Benchfile.create("bencher") {
700 doublePuts($stderr,file,"__bencher_count = 0;")
701 doublePuts($stderr,file,"function __bencher_doNext(result) {")
702 doublePuts($stderr,file," if (__bencher_count >= #{$warmup})")
703 doublePuts($stderr,file," debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
704 doublePuts($stderr,file," __bencher_count++;")
705 doublePuts($stderr,file," if (__bencher_count < #{$inner+$warmup})")
706 doublePuts($stderr,file," __bencher_runImpl(__bencher_doNext);")
707 doublePuts($stderr,file," else")
708 doublePuts($stderr,file," quit();")
709 doublePuts($stderr,file,"}")
710 doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
713 cssCode = Benchfile.create("bencher-css") {
715 doublePuts($stderr,file,".pass {\n font-weight: bold;\n color: green;\n}\n.fail {\n font-weight: bold;\n color: red;\n}\n\#console {\n white-space: pre-wrap;\n font-family: monospace;\n}")
718 preCode = Benchfile.create("bencher-pre") {
720 doublePuts($stderr,file,"if (window.layoutTestController) {")
721 doublePuts($stderr,file," layoutTestController.dumpAsText(window.enablePixelTesting);")
722 doublePuts($stderr,file," layoutTestController.waitUntilDone();")
723 doublePuts($stderr,file,"}")
724 doublePuts($stderr,file,"")
725 doublePuts($stderr,file,"function debug(msg)")
726 doublePuts($stderr,file,"{")
727 doublePuts($stderr,file," var span = document.createElement(\"span\");")
728 doublePuts($stderr,file," document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
729 doublePuts($stderr,file," span.innerHTML = msg + '<br />';")
730 doublePuts($stderr,file,"}")
731 doublePuts($stderr,file,"")
732 doublePuts($stderr,file,"function quit() {")
733 doublePuts($stderr,file," layoutTestController.notifyDone();")
734 doublePuts($stderr,file,"}")
735 doublePuts($stderr,file,"")
736 doublePuts($stderr,file,"__bencher_continuation=null;")
737 doublePuts($stderr,file,"")
738 doublePuts($stderr,file,"function reportResult(result) {")
739 doublePuts($stderr,file," __bencher_continuation(result);")
740 doublePuts($stderr,file,"}")
741 doublePuts($stderr,file,"")
742 doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
743 doublePuts($stderr,file," function doit() {")
744 doublePuts($stderr,file," document.getElementById(\"frameparent\").innerHTML = \"\";")
745 doublePuts($stderr,file," document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
746 doublePuts($stderr,file," var testFrame = document.getElementById(\"testframe\");")
747 doublePuts($stderr,file," testFrame.contentDocument.open();")
748 doublePuts($stderr,file," testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
750 doublePuts($stderr,file," testFrame.contentDocument.write(\"<script src=\\\"#{benchDataPath}\\\"></script>\");")
752 doublePuts($stderr,file," testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">__bencher_before = Date.now();</script><script src=\\\"#{benchPath}\\\"></script><script type=\\\"text/javascript\\\">window.parent.reportResult(Date.now() - __bencher_before);</script></body></html>\");")
753 doublePuts($stderr,file," testFrame.contentDocument.close();")
754 doublePuts($stderr,file," }")
755 doublePuts($stderr,file," __bencher_continuation = continuation;")
756 doublePuts($stderr,file," window.setTimeout(doit, 10);")
757 doublePuts($stderr,file,"}")
760 Benchfile.create(["bencher-htmldoc",".html"]) {
762 doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
769 def emitBenchRunCode(name, plan, benchDataPath, benchPath)
770 plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchDataPath, benchPath))
773 def planForDescription(plans, benchFullname, vmName, iteration)
774 raise unless benchFullname =~ /\//
775 suiteName = $~.pre_match
776 benchName = $~.post_match
777 result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
778 raise unless result.size == 1
783 attr_reader :plan, :innerIndex, :time
785 def initialize(plan, innerIndex, time)
787 @innerIndex = innerIndex
790 raise unless @plan.is_a? BenchPlan
791 raise unless @innerIndex.is_a? Integer
792 raise unless @time.is_a? Numeric
811 def self.parse(plans, string)
812 if string =~ /([a-zA-Z0-9\/-]+): ([a-zA-Z0-9_# ]+): ([0-9]+): ([0-9]+): Time: /
817 time = $~.post_match.to_f
818 ParsedResult.new(planForDescription(plans, benchFullname, vmName, outerIndex), innerIndex, time)
826 def initialize(origPath, name, nameKind, svnRevision)
827 @origPath = origPath.to_s
828 @path = origPath.to_s
833 @vmType = $forceVMKind
835 if @origPath =~ /DumpRenderTree$/
836 @vmType = :dumpRenderTree
842 @svnRevision = svnRevision
844 # Try to detect information about the VM.
845 if path =~ /\/WebKitBuild\/Release\/([a-zA-Z]+)$/
846 @checkoutPath = $~.pre_match
847 # FIXME: Use some variant of this:
848 # <bdash> def retrieve_revision
849 # <bdash> `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
853 Dir.chdir(@checkoutPath) {
854 $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
855 IO.popen("svn info", "r") {
859 if line =~ /Revision: ([0-9]+)/
866 $stderr.puts "Warning: running svn info for #{name} silently failed."
869 # Failed to detect svn revision.
870 $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
874 $stderr.puts "Warning: could not identify checkout location for #{name}"
877 if @path =~ /\/Release\/([a-zA-Z]+)$/
878 @libPath, @relativeBinPath = $~.pre_match+"/Release", "./#{$1}"
879 elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
880 @libPath = $~.pre_match
881 elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
882 @libPath, @relativeBinPath = $~.pre_match, $&[1..-1]
886 def canCopyIntoBenchPath
887 if @libPath and @relativeBinPath
894 def copyIntoBenchPath
895 raise unless canCopyIntoBenchPath
896 basename, filename = Benchfile.uniqueFilename("vm")
897 raise unless Dir.mkdir(filename)
898 cmd = "cp -a #{@libPath.inspect}/* #{filename.inspect}"
899 $stderr.puts ">> #{cmd}" if $verbosity>=2
900 raise unless system(cmd)
901 @path = "#{basename}/#{@relativeBinPath}"
914 $measureGC == true or ($measureGC == name)
952 def emitRunCode(fileToRun)
954 myLibPath = "" unless myLibPath
955 $script.puts "export DYLD_LIBRARY_PATH=#{myLibPath.to_s.inspect}"
956 $script.puts "export DYLD_FRAMEWORK_PATH=#{myLibPath.to_s.inspect}"
957 $script.puts "#{path} #{fileToRun}"
961 class StatsAccumulator
964 ($outer*$inner).times {
969 def statsForIteration(outerIteration, innerIteration)
970 @stats[outerIteration*$inner + innerIteration]
977 result.add(yield stat)
982 def geometricMeanStats
989 def arithmeticMeanStats
998 attr_accessor :benchmarkSuite
1002 benchmarkSuite.name + "/" + name
1010 class SunSpiderBenchmark
1013 def initialize(name)
1017 def emitRunCode(plan)
1018 emitBenchRunCode(fullname, plan, nil, ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js"))
1025 def initialize(name)
1029 def emitRunCode(plan)
1030 emitBenchRunCode(fullname, plan, nil, ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js"))
1034 class KrakenBenchmark
1037 def initialize(name)
1041 def emitRunCode(plan)
1042 emitBenchRunCode(fullname, plan, ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js"), ensureFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js"))
1046 class BenchmarkSuite
1047 def initialize(name, path, preferredMean)
1050 @preferredMean = preferredMean
1067 if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
1068 benchmark.benchmarkSuite = self
1069 @benchmarks << benchmark
1077 def benchmarkForName(name)
1078 result = @benchmarks.select{|v| v.name == name}
1079 raise unless result.length == 1
1088 @benchmarks.delete_if {
1098 def computeMean(stat)
1099 stat.send @preferredMean
1104 def initialize(benchmark, vm, iteration)
1105 @benchmark = benchmark
1107 @iteration = iteration
1115 @benchmark.benchmarkSuite
1127 @benchmark.emitRunCode(self)
1132 def initialize(benchmark, suiteOnVM)
1133 @benchmark = benchmark
1134 @suiteOnVM = suiteOnVM
1139 "#{@benchmark} on #{@suiteOnVM.vm}"
1155 @benchmark.benchmarkSuite
1166 def parseResult(result)
1167 raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
1168 raise unless result.benchmark == @benchmark
1169 @stats.add(result.time)
1173 class SuiteOnVM < StatsAccumulator
1174 def initialize(vm, vmStats, suite)
1180 raise unless @vm.is_a? VM
1181 raise unless @vmStats.is_a? StatsAccumulator
1182 raise unless @suite.is_a? BenchmarkSuite
1186 "#{@suite} on #{@vm}"
1198 raise unless @vmStats
1204 def initialize(benchmarkOnVM, iteration)
1205 @benchmarkOnVM = benchmarkOnVM
1206 @iteration = iteration
1210 "#{@benchmarkOnVM} \##{@iteration+1}"
1218 @benchmarkOnVM.benchmark
1222 @benchmarkOnVM.suite
1233 def parseResult(result)
1234 raise unless result.plan == self
1235 @benchmarkOnVM.parseResult(result)
1236 @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
1237 @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
1250 while str.length<chars
1256 def center(str,chars)
1257 while str.length<chars
1266 def statsToStr(stats)
1267 if $inner*$outer == 1
1268 string = numToStr(stats.mean)
1269 raise unless string =~ /\./
1271 right = $~.post_match
1272 lpad(left,12)+"."+rpad(right,9)
1274 lpad(numToStr(stats.mean),11)+"+-"+rpad(numToStr(stats.confInt),9)
1286 def wrap(str, columns)
1289 curLine = array.shift
1292 if (curLine + " " + curStr).size > columns
1293 result += curLine + "\n"
1296 curLine += " " + curStr
1299 result + curLine + "\n"
1302 def runAndGetResults
1304 Dir.chdir(BENCH_DATA_PATH) {
1305 IO.popen("sh ./runscript", "r") {
1309 raise "Script did not complete correctly: #{$?}" unless $?.success?
1311 raise unless results
1315 def parseAndDisplayResults(results)
1318 vmStatses << StatsAccumulator.new
1322 suitesOnVMsForSuite = {}
1325 suitesOnVMsForSuite[suite] = []
1327 suitesOnVMsForVM = {}
1330 suitesOnVMsForVM[vm] = []
1333 benchmarksOnVMs = []
1334 benchmarksOnVMsForBenchmark = {}
1337 benchmarksOnVMsForBenchmark[benchmark] = []
1340 $vms.each_with_index {
1342 vmStats = vmStatses[vmIndex]
1345 suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
1346 suitesOnVMs << suiteOnVM
1347 suitesOnVMsForSuite[suite] << suiteOnVM
1348 suitesOnVMsForVM[vm] << suiteOnVM
1349 suite.benchmarks.each {
1351 benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM)
1352 benchmarksOnVMs << benchmarkOnVM
1353 benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
1359 benchmarksOnVMs.each {
1363 plans << BenchPlan.new(benchmarkOnVM, iteration)
1372 if line =~ /HOSTNAME:([^.]+)/
1374 elsif line =~ /HARDWARE:hw\.model: /
1375 hwmodel = $~.post_match.chomp
1377 result = ParsedResult.parse(plans, line.chomp)
1379 result.plan.parseResult(result)
1384 # Compute the geomean of the preferred means of results on a SuiteOnVM
1393 curResult = Stats.new
1394 suitesOnVMsForVM[vm].each {
1396 # For a given iteration, suite, and VM, compute the suite's preferred mean
1397 # over the data collected for all benchmarks in that suite. We'll have one
1398 # sample per benchmark. For example on V8 this will be the geomean of 1
1399 # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
1401 curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
1404 # curResult now holds 1 sample for each of the means computed in the above
1405 # loop. Compute the geomean over this, and store it.
1406 result.add(curResult.geometricMean)
1410 # $overallResults will have a Stats for each VM. That Stats object will hold
1411 # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
1412 # confidence interval of the geomeans of preferred means. Convoluted, but
1413 # useful and probably sound.
1414 overallResults << result
1418 benchmarksOnVMs.each {
1420 $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
1423 $vms.each_with_index {
1425 vmStats = vmStatses[vmIndex]
1426 $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
1427 $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
1454 "%04d%02d%02d_%02d%02d" %
1455 [ time.year, time.month, time.day,
1456 time.hour, time.min ]
1461 puts "Generating benchmark report at #{reportName}"
1466 outp = File.open(reportName,"w")
1468 $stderr.puts "Error: could not save report to #{reportName}: #{e}"
1474 result += " " if $suites.size > 1
1475 result += rpad("", $benchpad)
1480 result += " "+NoChange.new(0).shortForm
1482 result += lpad(center($vms[index].name, 9+9+2), 11+9+2)
1486 result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
1487 elsif $vms.size >= 2
1493 columns = [createVMsString.size, 78].max
1495 outp.print "Benchmark report for "
1496 if $suites.size == 1
1497 outp.print $suites[0].to_s
1498 elsif $suites.size == 2
1499 outp.print "#{$suites[0]} and #{$suites[1]}"
1501 outp.print "#{$suites[0..-2].join(', ')}, and #{$suites[-1]}"
1504 outp.print " on #{hostname}"
1507 outp.print " (#{hwmodel})"
1512 # This looks stupid; revisit later.
1516 outp.puts "#{suite} at #{suite.path}"
1522 outp.puts "VMs tested:"
1525 outp.print "\"#{vm.name}\" at #{vm.origPath}"
1527 outp.print " (r#{vm.svnRevision})"
1534 outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
1535 "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
1536 (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
1537 "total time of those iterations, for each sample.")
1539 (if $measureGC == true then (" No manual garbage collection invocations were "+
1541 elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
1542 "all VMs except #{$measureGC}.")
1543 else (" Emitted a call to gc() between sample measurements.") end)+
1544 (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
1545 "began with the very first iteration.")
1546 else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
1547 "invocation for warm-up.") end)+
1549 when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
1550 "microsecond-level timing.")
1551 when :date then (" Used the portable Date.now() method to get millisecond-"+
1554 " Reporting benchmark execution times with 95% confidence "+
1555 "intervals in milliseconds.",
1561 outp.puts createVMsString
1564 def summaryStats(outp, accumulators, name, &proc)
1565 outp.print " " if $suites.size > 1
1566 outp.print rpad(name, $benchpad)
1568 accumulators.size.times {
1571 outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
1573 outp.print statsToStr(accumulators[index].stats(&proc))
1575 if accumulators.size>=2
1576 outp.print(" "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
1581 def meanName(currentMean, preferredMean)
1582 result = "<#{currentMean}>"
1583 if "#{currentMean}Mean" == preferredMean.to_s
1589 def allSummaryStats(outp, accumulators, preferredMean)
1590 summaryStats(outp, accumulators, meanName("arithmetic", preferredMean)) {
1595 summaryStats(outp, accumulators, meanName("geometric", preferredMean)) {
1600 summaryStats(outp, accumulators, meanName("harmonic", preferredMean)) {
1610 outp.puts "#{suite.name}:"
1614 suite.benchmarks.each {
1616 outp.print " " if $suites.size > 1
1617 outp.print rpad(benchmark.name, $benchpad)
1619 myConfigs = benchmarksOnVMsForBenchmark[benchmark]
1620 myConfigs.size.times {
1623 outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
1625 outp.print statsToStr(myConfigs[index].stats)
1628 outp.print(" "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
1633 allSummaryStats(outp, suitesOnVMsForSuite[suite], suite.preferredMean)
1634 outp.puts if $suites.size > 1
1639 outp.puts "All benchmarks:"
1640 allSummaryStats(outp, vmStatses, nil)
1644 outp.puts "Geomean of preferred means:"
1646 outp.print rpad("<scaled-result>", $benchpad)
1651 outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
1653 outp.print statsToStr(overallResults[index])
1655 if overallResults.size>=2
1656 outp.print(" "+overallResults[-1].compareTo(overallResults[0]).longForm)
1666 if outp != $stdout and not $brief
1668 File.open(reportName) {
1675 puts(overallResults.collect{|stats| stats.mean}.join("\t"))
1676 puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
1683 $sawBenchOptions = false
1685 def resetBenchOptionsIfNecessary
1686 unless $sawBenchOptions
1687 $includeSunSpider = false
1689 $includeKraken = false
1690 $sawBenchOptions = true
1694 GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
1695 ['--inner', GetoptLong::REQUIRED_ARGUMENT],
1696 ['--outer', GetoptLong::REQUIRED_ARGUMENT],
1697 ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
1698 ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
1699 ['--sunspider-only', GetoptLong::NO_ARGUMENT],
1700 ['--v8-only', GetoptLong::NO_ARGUMENT],
1701 ['--kraken-only', GetoptLong::NO_ARGUMENT],
1702 ['--exclude-sunspider', GetoptLong::NO_ARGUMENT],
1703 ['--exclude-v8', GetoptLong::NO_ARGUMENT],
1704 ['--exclude-kraken', GetoptLong::NO_ARGUMENT],
1705 ['--sunspider', GetoptLong::NO_ARGUMENT],
1706 ['--v8', GetoptLong::NO_ARGUMENT],
1707 ['--kraken', GetoptLong::NO_ARGUMENT],
1708 ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
1709 ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
1710 ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
1711 ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
1712 ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
1713 ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
1714 ['--brief', GetoptLong::NO_ARGUMENT],
1715 ['--silent', GetoptLong::NO_ARGUMENT],
1716 ['--remote', GetoptLong::REQUIRED_ARGUMENT],
1717 ['--local', GetoptLong::NO_ARGUMENT],
1718 ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
1719 ['--slave', GetoptLong::NO_ARGUMENT],
1720 ['--prepare-only', GetoptLong::NO_ARGUMENT],
1721 ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
1722 ['--vms', GetoptLong::REQUIRED_ARGUMENT],
1723 ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
1727 $rerun = intArg(opt,arg,1,nil)
1729 $inner = intArg(opt,arg,1,nil)
1731 $outer = intArg(opt,arg,1,nil)
1733 $warmup = intArg(opt,arg,0,nil)
1734 when '--timing-mode'
1735 if arg.upcase == "PRECISETIME"
1736 $timeMode = :preciseTime
1737 elsif arg.upcase == "DATE"
1739 elsif arg.upcase == "AUTO"
1742 quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
1743 "Invalid argument for command-line option")
1745 when '--force-vm-kind'
1746 if arg.upcase == "JSC"
1748 elsif arg.upcase == "DUMPRENDERTREE"
1749 $forceVMKind = :dumpRenderTree
1750 elsif arg.upcase == "AUTO"
1753 quickFail("Expected either 'jsc' or 'DumpRenderTree' for --force-vm-kind, but got '#{arg}'.",
1754 "Invalid argument for command-line option")
1756 when '--force-vm-copy'
1757 $needToCopyVMs = true
1758 when '--dont-copy-vms'
1760 when '--sunspider-only'
1762 $includeKraken = false
1764 $includeSunSpider = false
1765 $includeKraken = false
1766 when '--kraken-only'
1767 $includeSunSpider = false
1769 when '--exclude-sunspider'
1770 $includeSunSpider = false
1773 when '--exclude-kraken'
1774 $includeKraken = false
1776 resetBenchOptionsIfNecessary
1777 $includeSunSpider = true
1779 resetBenchOptionsIfNecessary
1782 resetBenchOptionsIfNecessary
1783 $includeKraken = true
1785 $benchmarkPattern = Regexp.new(arg)
1799 $remoteHosts += arg.split(',')
1800 $needToCopyVMs = true
1801 when '--ssh-options'
1805 when '--prepare-only'
1814 raise "bad option: #{opt}"
1818 # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
1820 $needToCopyVMs = false
1823 SUNSPIDER = BenchmarkSuite.new("SunSpider", SUNSPIDER_PATH, :arithmeticMean)
1824 ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
1825 "access-fannkuch", "access-nbody", "access-nsieve",
1826 "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
1827 "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
1828 "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
1829 "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
1830 "string-base64", "string-fasta", "string-tagcloud",
1831 "string-unpack-code", "string-validate-input"].each {
1833 SUNSPIDER.add SunSpiderBenchmark.new(name)
1836 V8 = BenchmarkSuite.new("V8", V8_PATH, :geometricMean)
1837 ["crypto", "deltablue", "earley-boyer", "raytrace",
1838 "regexp", "richards", "splay"].each {
1840 V8.add V8Benchmark.new(name)
1843 KRAKEN = BenchmarkSuite.new("Kraken", KRAKEN_PATH, :arithmeticMean)
1844 ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
1845 "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
1846 "imaging-gaussian-blur", "json-parse-financial",
1847 "json-stringify-tinderbox", "stanford-crypto-aes",
1848 "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
1849 "stanford-crypto-sha256-iterative"].each {
1851 KRAKEN.add KrakenBenchmark.new(name)
1856 if vm =~ /([a-zA-Z0-9_ ]+):/
1861 name = "Conf\##{$vms.length+1}"
1864 $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
1865 $vms << VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
1869 quickFail("Please specify at least on configuraiton on the command line.",
1870 "Insufficient arguments")
1875 if vm.vmType != :jsc and $timeMode != :date
1877 $stderr.puts "Warning: using Date.now() instead of preciseTime() because #{vm} doesn't support the latter."
1881 if FileTest.exist? BENCH_DATA_PATH
1882 cmd = "rm -rf #{BENCH_DATA_PATH}"
1883 $stderr.puts ">> #{cmd}" if $verbosity >= 2
1884 raise unless system cmd
1887 Dir.mkdir BENCH_DATA_PATH
1890 canCopyIntoBenchPath = true
1893 canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
1896 if canCopyIntoBenchPath
1899 $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
1900 vm.copyIntoBenchPath
1902 $stderr.puts "All VMs are in place."
1904 $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
1908 if $measureGC and $measureGC != true
1912 if vm.name == $measureGC
1917 $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
1921 if $outer*$inner == 1
1922 $stderr.puts "Warning: will only collect one sample per benchmark/VM. Confidence interval calculation will fail."
1925 $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
1929 if $includeSunSpider and not SUNSPIDER.empty?
1930 $suites << SUNSPIDER
1933 if $includeV8 and not V8.empty?
1937 if $includeKraken and not KRAKEN.empty?
1944 $benchmarks += suite.benchmarks
1954 $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
1961 $suitepad = $suites.collect {
1966 $benchpad = ($benchmarks +
1967 ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
1969 if benchmark.respond_to? :name
1976 $vmpad = $vms.collect {
1982 File.open("#{BENCH_DATA_PATH}/runscript", "w") {
1984 file.puts "echo \"HOSTNAME:\\c\""
1985 file.puts "hostname"
1987 file.puts "echo \"HARDWARE:\\c\""
1988 file.puts "/usr/sbin/sysctl hw.model"
1992 $runPlans.each_with_index {
1994 if $verbosity == 0 and not $silent
1995 text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
1996 text2 = plan.benchmark.to_s+"/"+plan.vm.to_s
1997 file.puts("echo "+("\r#{text1} #{rpad(text2,$suitepad+1+$benchpad+1+$vmpad)}".inspect)[0..-2]+"\\c\" 1>&2")
1998 file.puts("echo "+("\r#{text1} #{text2}".inspect)[0..-2]+"\\c\" 1>&2")
2002 if $verbosity == 0 and not $silent
2003 file.puts("echo "+("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}".inspect)[0..-2]+"\\c\" 1>&2")
2004 file.puts("echo "+("\r#{$runPlans.size}/#{$runPlans.size}".inspect)+" 1>&2")
2010 unless $remoteHosts.empty?
2011 $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
2012 Dir.chdir(TEMP_PATH) {
2013 cmd = "tar -czf payload.tar.gz benchdata"
2014 $stderr.puts ">> #{cmd}" if $verbosity>=2
2015 raise unless system(cmd)
2019 if host =~ /:([0-9]+)$/
2020 "-p " + $1 + " " + $~.pre_match.inspect
2026 def sshRead(host, command)
2027 cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{grokHost(host)} #{command.inspect}"
2028 $stderr.puts ">> #{cmd}" if $verbosity>=2
2030 IO.popen(cmd, "r") {
2034 $stderr.puts "#{host}: #{line}" if $verbosity>=2
2038 raise "#{$?}" unless $?.success?
2042 def sshWrite(host, command, data)
2043 cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{grokHost(host)} #{command.inspect}"
2044 $stderr.puts ">> #{cmd}" if $verbosity>=2
2045 IO.popen(cmd, "w") {
2049 raise "#{$?}" unless $?.success?
2054 $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
2056 remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
2057 raise unless remoteTempPath
2059 sshWrite(host, "cd #{remoteTempPath.inspect} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
2061 $stderr.puts "Running on #{host}..." if $verbosity==0
2063 parseAndDisplayResults(sshRead(host, "cd #{(remoteTempPath+'/benchdata').inspect} && sh runscript"))
2067 if not $remoteHosts.empty? and $alsoLocal
2068 $stderr.puts "Running locally..."
2071 if $remoteHosts.empty? or $alsoLocal
2072 parseAndDisplayResults(runAndGetResults)
2076 $analyze.each_with_index {
2081 parseAndDisplayResults(IO::read(filename))
2084 if $prepare and not $run and $analyze.empty?
2085 puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
2086 "the benchmarks and get the results by doing:", 78)
2088 puts "cd #{BENCH_DATA_PATH}"
2089 puts "sh runscript > results.txt"
2091 puts wrap("Then you can analyze the results by running bencher with the same arguments "+
2092 "as now, but replacing --prepare-only with --analyze results.txt.", 78)