2 * Copyright (C) 2007 Apple Inc. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 function sunspiderCompareResults(output1, output2)
28 var count1 = output1.length;
29 var count2 = output2.length;
32 itemTotals1.length = count1;
35 var categoryTotals1 = {};
36 var testTotalsByCategory1 = {};
39 var categoryMeans1 = {};
40 var testMeansByCategory1 = {};
43 var categoryStdDevs1 = {};
44 var testStdDevsByCategory1 = {};
47 var categoryStdErrs1 = {};
48 var testStdErrsByCategory1 = {};
51 itemTotals2.length = count2;
54 var categoryTotals2 = {};
55 var testTotalsByCategory2 = {};
58 var categoryMeans2 = {};
59 var testMeansByCategory2 = {};
62 var categoryStdDevs2 = {};
63 var testStdDevsByCategory2 = {};
66 var categoryStdErrs2 = {};
67 var testStdErrsByCategory2 = {};
71 itemTotals1 = {total: []};
73 for (var i = 0; i < categories.length; i++) {
74 var category = categories[i];
75 itemTotals1[category] = [];
76 categoryTotals1[category] = 0;
77 testTotalsByCategory1[category] = {};
78 categoryMeans1[category] = 0;
79 testMeansByCategory1[category] = {};
80 categoryStdDevs1[category] = 0;
81 testStdDevsByCategory1[category] = {};
82 categoryStdErrs1[category] = 0;
83 testStdErrsByCategory1[category] = {};
86 for (var i = 0; i < tests.length; i++) {
88 itemTotals1[test] = [];
89 var category = test.replace(/-.*/, "");
90 testTotalsByCategory1[category][test] = 0;
91 testMeansByCategory1[category][test] = 0;
92 testStdDevsByCategory1[category][test] = 0;
93 testStdErrsByCategory1[category][test] = 0;
96 for (var i = 0; i < count1; i++) {
97 itemTotals1["total"][i] = 0;
98 for (var category in categoryTotals1) {
99 itemTotals1[category][i] = 0;
100 for (var test in testTotalsByCategory1[category]) {
101 itemTotals1[test][i] = 0;
106 itemTotals2 = {total: []};
108 for (var i = 0; i < categories.length; i++) {
109 var category = categories[i];
110 itemTotals2[category] = [];
111 categoryTotals2[category] = 0;
112 testTotalsByCategory2[category] = {};
113 categoryMeans2[category] = 0;
114 testMeansByCategory2[category] = {};
115 categoryStdDevs2[category] = 0;
116 testStdDevsByCategory2[category] = {};
117 categoryStdErrs2[category] = 0;
118 testStdErrsByCategory2[category] = {};
121 for (var i = 0; i < tests.length; i++) {
123 itemTotals2[test] = [];
124 var category = test.replace(/-.*/, "");
125 testTotalsByCategory2[category][test] = 0;
126 testMeansByCategory2[category][test] = 0;
127 testStdDevsByCategory2[category][test] = 0;
128 testStdErrsByCategory2[category][test] = 0;
131 for (var i = 0; i < count2; i++) {
132 itemTotals2["total"][i] = 0;
133 for (var category in categoryTotals2) {
134 itemTotals2[category][i] = 0;
135 for (var test in testTotalsByCategory2[category]) {
136 itemTotals2[test][i] = 0;
143 function computeItemTotals(output, itemTotals)
145 for (var i = 0; i < output.length; i++) {
146 var result = output[i];
147 for (var test in result) {
148 var time = result[test];
149 var category = test.replace(/-.*/, "");
150 itemTotals["total"][i] += time;
151 itemTotals[category][i] += time;
152 itemTotals[test][i] += time;
157 function computeTotals(output, categoryTotals, testTotalsByCategory)
161 for (var i = 0; i < output.length; i++) {
162 var result = output[i];
163 for (var test in result) {
164 var time = result[test];
165 var category = test.replace(/-.*/, "");
167 categoryTotals[category] += time;
168 testTotalsByCategory[category][test] += time;
175 function computeMeans(count, total, categoryTotals, categoryMeans, testTotalsByCategory, testMeansByCategory)
177 var mean = total / count;
178 for (var category in categoryTotals) {
179 categoryMeans[category] = categoryTotals[category] / count;
180 for (var test in testTotalsByCategory[category]) {
181 testMeansByCategory[category][test] = testTotalsByCategory[category][test] / count;
187 function standardDeviation(mean, items)
189 var deltaSquaredSum = 0;
190 for (var i = 0; i < items.length; i++) {
191 var delta = items[i] - mean;
192 deltaSquaredSum += delta * delta;
194 variance = deltaSquaredSum / (items.length - 1);
195 return Math.sqrt(variance);
198 function computeStdDevs(mean, itemTotals, categoryStdDevs, categoryMeans, testStdDevsByCategory, testMeansByCategory)
200 var stdDev = standardDeviation(mean, itemTotals["total"]);
201 for (var category in categoryStdDevs) {
202 categoryStdDevs[category] = standardDeviation(categoryMeans[category], itemTotals[category]);
204 for (var category in categoryStdDevs) {
205 for (var test in testStdDevsByCategory[category]) {
206 testStdDevsByCategory[category][test] = standardDeviation(testMeansByCategory[category][test], itemTotals[test]);
212 function computeStdErrors(count, stdDev, categoryStdErrs, categoryStdDevs, testStdErrsByCategory, testStdDevsByCategory)
214 var sqrtCount = Math.sqrt(count);
216 var stdErr = stdDev / sqrtCount;
217 for (var category in categoryStdErrs) {
218 categoryStdErrs[category] = categoryStdDevs[category] / sqrtCount;
220 for (var category in categoryStdDevs) {
221 for (var test in testStdErrsByCategory[category]) {
222 testStdErrsByCategory[category][test] = testStdDevsByCategory[category][test] / sqrtCount;
229 var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09, 2.09, 2.08, 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04, 2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96];
230 var tMax = tDistribution.length;
237 return tDistribution[n];
241 function formatMean(meanWidth, mean, stdErr, count)
243 var meanString = mean.toFixed(1).toString();
244 while (meanString.length < meanWidth) {
245 meanString = " " + meanString;
248 var error = "+/- " + ((tDist(count) * stdErr / mean) * 100).toFixed(1) + "% ";
250 return meanString + "ms " + error;
253 function computeLabelWidth()
255 var width = "Total".length;
256 for (var category in categoryMeans1) {
257 if (category.length + 2 > width)
258 width = category.length + 2;
260 for (var i = 0; i < tests.length; i++) {
261 var shortName = tests[i].replace(/^[^-]*-/, "");
262 if (shortName.length + 4 > width)
263 width = shortName.length + 4;
269 function computeMeanWidth(mean, categoryMeans, testMeansByCategory)
271 var width = mean.toFixed(1).toString().length;
272 for (var category in categoryMeans) {
273 var candidate = categoryMeans[category].toFixed(1).toString().length;
274 if (candidate > width)
276 for (var test in testMeansByCategory[category]) {
277 var candidate = testMeansByCategory[category][test].toFixed(1).toString().length;
278 if (candidate > width)
288 while (str.length < n) {
294 function resultLine(labelWidth, indent, label, meanWidth1, mean1, stdErr1, meanWidth2, mean2, stdErr2)
296 result = pad("", indent);
297 result += label + ": ";
298 result = pad(result, labelWidth + 2);
300 var t = (mean1 - mean2) / (Math.sqrt((stdErr1 * stdErr1) + (stdErr2 * stdErr2)));
301 var df = count1 + count2 - 2;
303 var statisticallySignificant = (Math.abs(t) > tDist(df+1));
304 var diff = mean2 - mean1;
305 var percentage = 100 * diff / mean1;
306 var isFaster = diff < 0;
307 var probablySame = (percentage < 0.1) && !statisticallySignificant;
308 var ratio = isFaster ? (mean1 / mean2) : (mean2 / mean1);
309 var fixedRatio = (ratio < 1.2) ? ratio.toFixed(3).toString() : ((ratio < 10) ? ratio.toFixed(2).toString() : ratio.toFixed(1).toString());
310 var formattedRatio = isFaster ? fixedRatio + "x as fast" : "*" + fixedRatio + "x as slow*";
318 } else if (!statisticallySignificant) {
320 diffDetail = " not conclusive: might be " + formattedRatio;
322 diffSummary = formattedRatio;
323 diffDetail = " significant";
326 return result + pad(diffSummary, 18) + formatMean(meanWidth1, mean1, stdErr1, count1) + " " + formatMean(meanWidth2, mean2, stdErr2, count2) + diffDetail;
329 function printOutput()
331 var labelWidth = computeLabelWidth();
332 var meanWidth1 = computeMeanWidth(mean1, categoryMeans1, testMeansByCategory1);
333 var meanWidth2 = computeMeanWidth(mean2, categoryMeans2, testMeansByCategory2);
337 while (header.length < labelWidth)
339 header += " COMPARISON FROM TO DETAILS";
342 print("=============================================================================");
344 print(resultLine(labelWidth, 0, "** TOTAL **", meanWidth1, mean1, stdErr1, meanWidth2, mean2, stdErr2));
346 print("=============================================================================");
348 for (var category in categoryMeans1) {
350 print(resultLine(labelWidth, 2, category,
351 meanWidth1, categoryMeans1[category], categoryStdErrs1[category],
352 meanWidth2, categoryMeans2[category], categoryStdErrs2[category]));
353 for (var test in testMeansByCategory1[category]) {
354 var shortName = test.replace(/^[^-]*-/, "");
355 print(resultLine(labelWidth, 4, shortName,
356 meanWidth1, testMeansByCategory1[category][test], testStdErrsByCategory1[category][test],
357 meanWidth2, testMeansByCategory2[category][test], testStdErrsByCategory2[category][test]));
364 computeItemTotals(output1, itemTotals1);
365 computeItemTotals(output2, itemTotals2);
367 total1 = computeTotals(output1, categoryTotals1, testTotalsByCategory1);
368 total2 = computeTotals(output2, categoryTotals2, testTotalsByCategory2);
370 mean1 = computeMeans(count1, total1, categoryTotals1, categoryMeans1, testTotalsByCategory1, testMeansByCategory1);
371 mean2 = computeMeans(count2, total2, categoryTotals2, categoryMeans2, testTotalsByCategory2, testMeansByCategory2);
373 stdDev1 = computeStdDevs(mean1, itemTotals1, categoryStdDevs1, categoryMeans1, testStdDevsByCategory1, testMeansByCategory1);
374 stdDev2 = computeStdDevs(mean2, itemTotals2, categoryStdDevs2, categoryMeans2, testStdDevsByCategory2, testMeansByCategory2);
376 stdErr1 = computeStdErrors(count1, stdDev1, categoryStdErrs1, categoryStdDevs1, testStdErrsByCategory1, testStdDevsByCategory1);
377 stdErr2 = computeStdErrors(count2, stdDev2, categoryStdErrs2, categoryStdDevs2, testStdErrsByCategory2, testStdDevsByCategory2);