src/third_party/WebKit/Tools/GardeningServer/scripts/results.js

   1 /*
   2  * Copyright (C) 2011 Google Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
  14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
  17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  23  * THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 var results = results || {};
  27
  28 (function() {
  29
  30 var kResultsName = 'failing_results.json';
  31
  32 var PASS = 'PASS';
  33 var TIMEOUT = 'TIMEOUT';
  34 var TEXT = 'TEXT';
  35 var CRASH = 'CRASH';
  36 var IMAGE = 'IMAGE';
  37 var IMAGE_TEXT = 'IMAGE+TEXT';
  38 var AUDIO = 'AUDIO';
  39 var MISSING = 'MISSING';
  40
  41 var kFailingResults = [TEXT, IMAGE_TEXT, AUDIO];
  42
  43 var kExpectedImageSuffix = '-expected.png';
  44 var kActualImageSuffix = '-actual.png';
  45 var kImageDiffSuffix = '-diff.png';
  46 var kExpectedAudioSuffix = '-expected.wav';
  47 var kActualAudioSuffix = '-actual.wav';
  48 var kExpectedTextSuffix = '-expected.txt';
  49 var kActualTextSuffix = '-actual.txt';
  50 var kDiffTextSuffix = '-diff.txt';
  51 var kCrashLogSuffix = '-crash-log.txt';
  52
  53 var kPNGExtension = 'png';
  54 var kTXTExtension = 'txt';
  55 var kWAVExtension = 'wav';
  56
  57 var kPreferredSuffixOrder = [
  58     kExpectedImageSuffix,
  59     kActualImageSuffix,
  60     kImageDiffSuffix,
  61     kExpectedTextSuffix,
  62     kActualTextSuffix,
  63     kDiffTextSuffix,
  64     kCrashLogSuffix,
  65     kExpectedAudioSuffix,
  66     kActualAudioSuffix,
  67     // FIXME: Add support for the rest of the result types.
  68 ];
  69
  70 // Kinds of results.
  71 results.kActualKind = 'actual';
  72 results.kExpectedKind = 'expected';
  73 results.kDiffKind = 'diff';
  74 results.kUnknownKind = 'unknown';
  75
  76 // Types of tests.
  77 results.kImageType = 'image'
  78 results.kAudioType = 'audio'
  79 results.kTextType = 'text'
  80 // FIXME: There are more types of tests.
  81
  82 function possibleSuffixListFor(failureTypeList)
  83 {
  84     var suffixList = [];
  85
  86     function pushImageSuffixes()
  87     {
  88         suffixList.push(kExpectedImageSuffix);
  89         suffixList.push(kActualImageSuffix);
  90         suffixList.push(kImageDiffSuffix);
  91     }
  92
  93     function pushAudioSuffixes()
  94     {
  95         suffixList.push(kExpectedAudioSuffix);
  96         suffixList.push(kActualAudioSuffix);
  97     }
  98
  99     function pushTextSuffixes()
 100     {
 101         suffixList.push(kActualTextSuffix);
 102         suffixList.push(kExpectedTextSuffix);
 103         suffixList.push(kDiffTextSuffix);
 104         // '-wdiff.html',
 105         // '-pretty-diff.html',
 106     }
 107
 108     $.each(failureTypeList, function(index, failureType) {
 109         switch(failureType) {
 110         case IMAGE:
 111             pushImageSuffixes();
 112             break;
 113         case TEXT:
 114             pushTextSuffixes();
 115             break;
 116         case AUDIO:
 117             pushAudioSuffixes();
 118             break;
 119         case IMAGE_TEXT:
 120             pushImageSuffixes();
 121             pushTextSuffixes();
 122             break;
 123         case CRASH:
 124             suffixList.push(kCrashLogSuffix);
 125             break;
 126         case MISSING:
 127             pushImageSuffixes();
 128             pushTextSuffixes();
 129             break;
 130         default:
 131             // FIXME: Add support for the rest of the result types.
 132             // '-expected.html',
 133             // '-expected-mismatch.html',
 134             // ... and possibly more.
 135             break;
 136         }
 137     });
 138
 139     return base.uniquifyArray(suffixList);
 140 }
 141
 142 results.failureTypeToExtensionList = function(failureType)
 143 {
 144     switch(failureType) {
 145     case IMAGE:
 146         return [kPNGExtension];
 147     case AUDIO:
 148         return [kWAVExtension];
 149     case TEXT:
 150         return [kTXTExtension];
 151     case MISSING:
 152     case IMAGE_TEXT:
 153         return [kTXTExtension, kPNGExtension];
 154     default:
 155         // FIXME: Add support for the rest of the result types.
 156         // '-expected.html',
 157         // '-expected-mismatch.html',
 158         // ... and possibly more.
 159         return [];
 160     }
 161 };
 162
 163 results.failureTypeList = function(failureBlob)
 164 {
 165     return failureBlob.split(' ');
 166 };
 167
 168 function resultsDirectoryURL(builderName)
 169 {
 170     if (config.useLocalResults)
 171         return '/localresult?path=';
 172     return config.layoutTestResultsURL + '/' + config.resultsDirectoryNameFromBuilderName(builderName) + '/results/layout-test-results/';
 173 }
 174
 175 function resultsPrefixListingURL(builderName, marker)
 176 {
 177     var url =  config.layoutTestResultsURL + '/?prefix=' + config.resultsDirectoryNameFromBuilderName(builderName) + '/&delimiter=/';
 178     if (marker)
 179         return url + '&marker=' + marker;
 180     return url;
 181 }
 182
 183 function resultsDirectoryURLForBuildNumber(builderName, buildNumber)
 184 {
 185     return config.layoutTestResultsURL + '/' + config.resultsDirectoryNameFromBuilderName(builderName) + '/' + buildNumber + '/' ;
 186 }
 187
 188 function resultsSummaryURL(builderName)
 189 {
 190     return resultsDirectoryURL(builderName) + kResultsName;
 191 }
 192
 193 function resultsSummaryURLForBuildNumber(builderName, buildNumber)
 194 {
 195     return resultsDirectoryURLForBuildNumber(builderName, buildNumber) + kResultsName;
 196 }
 197
 198 var g_resultsCache = new base.AsynchronousCache(function (key, callback) {
 199     net.jsonp(key, callback);
 200 });
 201
 202 results.ResultAnalyzer = base.extends(Object, {
 203     init: function(resultNode)
 204     {
 205         this._isUnexpected = resultNode.is_unexpected;
 206         this._actual = resultNode ? results.failureTypeList(resultNode.actual) : [];
 207         this._expected = resultNode ? this._addImpliedExpectations(results.failureTypeList(resultNode.expected)) : [];
 208     },
 209     _addImpliedExpectations: function(resultsList)
 210     {
 211         if (resultsList.indexOf('FAIL') == -1)
 212             return resultsList;
 213         return resultsList.concat(kFailingResults);
 214     },
 215     _hasPass: function(results)
 216     {
 217         return results.indexOf(PASS) != -1;
 218     },
 219     unexpectedResults: function()
 220     {
 221         return this._actual.filter(function(result) {
 222             return this._expected.indexOf(result) == -1;
 223         }, this);
 224     },
 225     succeeded: function()
 226     {
 227         return this._hasPass(this._actual);
 228     },
 229     flaky: function()
 230     {
 231         return this._actual.length > 1;
 232     },
 233     wontfix: function()
 234     {
 235         return this._expected.indexOf('WONTFIX') != -1;
 236     },
 237     hasUnexpectedFailures: function()
 238     {
 239         return this._isUnexpected;
 240     }
 241 });
 242
 243 function isExpectedFailure(resultNode)
 244 {
 245     var analyzer = new results.ResultAnalyzer(resultNode);
 246     return !analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyzer.flaky() && !analyzer.wontfix();
 247 }
 248
 249 function isUnexpectedFailure(resultNode)
 250 {
 251     var analyzer = new results.ResultAnalyzer(resultNode);
 252     return analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyzer.flaky() && !analyzer.wontfix();
 253 }
 254
 255 function isResultNode(node)
 256 {
 257     return !!node.actual;
 258 }
 259
 260 results.expectedFailures = function(resultsTree)
 261 {
 262     return base.filterTree(resultsTree.tests, isResultNode, isExpectedFailure);
 263 };
 264
 265 results.unexpectedFailures = function(resultsTree)
 266 {
 267     return base.filterTree(resultsTree.tests, isResultNode, isUnexpectedFailure);
 268 };
 269
 270 function resultsByTest(resultsByBuilder, filter)
 271 {
 272     var resultsByTest = {};
 273
 274     $.each(resultsByBuilder, function(builderName, resultsTree) {
 275         $.each(filter(resultsTree), function(testName, resultNode) {
 276             resultsByTest[testName] = resultsByTest[testName] || {};
 277             resultsByTest[testName][builderName] = resultNode;
 278         });
 279     });
 280
 281     return resultsByTest;
 282 }
 283
 284 results.expectedFailuresByTest = function(resultsByBuilder)
 285 {
 286     return resultsByTest(resultsByBuilder, results.expectedFailures);
 287 };
 288
 289 results.unexpectedFailuresByTest = function(resultsByBuilder)
 290 {
 291     return resultsByTest(resultsByBuilder, results.unexpectedFailures);
 292 };
 293
 294 results.failureInfoForTestAndBuilder = function(resultsByTest, testName, builderName)
 295 {
 296     var failureInfoForTest = {
 297         'testName': testName,
 298         'builderName': builderName,
 299         'failureTypeList': results.failureTypeList(resultsByTest[testName][builderName].actual),
 300     };
 301
 302     return failureInfoForTest;
 303 };
 304
 305 results.collectUnexpectedResults = function(dictionaryOfResultNodes)
 306 {
 307     var collectedResults = [];
 308     $.each(dictionaryOfResultNodes, function(key, resultNode) {
 309         var analyzer = new results.ResultAnalyzer(resultNode);
 310         collectedResults = collectedResults.concat(analyzer.unexpectedResults());
 311     });
 312     return base.uniquifyArray(collectedResults);
 313 };
 314
 315 // Callback data is [{ buildNumber:, url: }]
 316 function historicalResultsLocations(builderName, callback)
 317 {
 318     var historicalResultsData = [];
 319
 320     function parseListingDocument(prefixListingDocument) {
 321         $(prefixListingDocument).find("Prefix").each(function() {
 322             var buildString = this.textContent.replace(config.resultsDirectoryNameFromBuilderName(builderName) + '/', '');
 323             if (buildString.match(/\d+\//)) {
 324                 var buildNumber = parseInt(buildString);
 325                 var resultsData = {
 326                     'buildNumber': buildNumber,
 327                     'url': resultsSummaryURLForBuildNumber(builderName, buildNumber)
 328                 };
 329                 historicalResultsData.unshift(resultsData);
 330            }
 331         });
 332         var nextMarker = $(prefixListingDocument).find('NextMarker').get();
 333         if (nextMarker.length) {
 334             var nextListingURL = resultsPrefixListingURL(builderName, nextMarker[0].textContent);
 335             net.xml(nextListingURL, parseListingDocument);
 336         } else {
 337             callback(historicalResultsData);
 338         }
 339     }
 340
 341     builders.mostRecentBuildForBuilder(builderName, function (mostRecentBuildNumber) {
 342         var marker = config.resultsDirectoryNameFromBuilderName(builderName) + "/" + (mostRecentBuildNumber - 100) + "/";
 343         var listingURL = resultsPrefixListingURL(builderName, marker);
 344         net.xml(listingURL, parseListingDocument);
 345     });
 346 }
 347
 348 function walkHistory(builderName, testName, callback)
 349 {
 350     var indexOfNextKeyToFetch = 0;
 351     var keyList = [];
 352
 353     function continueWalk()
 354     {
 355         if (indexOfNextKeyToFetch >= keyList.length) {
 356             processResultNode(0, null);
 357             return;
 358         }
 359
 360         var resultsURL = keyList[indexOfNextKeyToFetch].url;
 361         ++indexOfNextKeyToFetch;
 362         g_resultsCache.get(resultsURL, function(resultsTree) {
 363             if ($.isEmptyObject(resultsTree)) {
 364                 continueWalk();
 365                 return;
 366             }
 367             var resultNode = results.resultNodeForTest(resultsTree, testName);
 368             var revision = parseInt(resultsTree['blink_revision'])
 369             if (isNaN(revision))
 370                 revision = 0;
 371             processResultNode(revision, resultNode);
 372         });
 373     }
 374
 375     function processResultNode(revision, resultNode)
 376     {
 377         var shouldContinue = callback(revision, resultNode);
 378         if (!shouldContinue)
 379             return;
 380         continueWalk();
 381     }
 382
 383     historicalResultsLocations(builderName, function(resultsLocations) {
 384         keyList = resultsLocations;
 385         continueWalk();
 386     });
 387 }
 388
 389 results.regressionRangeForFailure = function(builderName, testName, callback)
 390 {
 391     var oldestFailingRevision = 0;
 392     var newestPassingRevision = 0;
 393
 394     walkHistory(builderName, testName, function(revision, resultNode) {
 395         if (!revision) {
 396             callback(oldestFailingRevision, newestPassingRevision);
 397             return false;
 398         }
 399         if (!resultNode) {
 400             newestPassingRevision = revision;
 401             callback(oldestFailingRevision, newestPassingRevision);
 402             return false;
 403         }
 404         if (isUnexpectedFailure(resultNode)) {
 405             oldestFailingRevision = revision;
 406             return true;
 407         }
 408         if (!oldestFailingRevision)
 409             return true;  // We need to keep looking for a failing revision.
 410         newestPassingRevision = revision;
 411         callback(oldestFailingRevision, newestPassingRevision);
 412         return false;
 413     });
 414 };
 415
 416 function mergeRegressionRanges(regressionRanges)
 417 {
 418     var mergedRange = {};
 419
 420     mergedRange.oldestFailingRevision = 0;
 421     mergedRange.newestPassingRevision = 0;
 422
 423     $.each(regressionRanges, function(builderName, range) {
 424         if (!range.oldestFailingRevision && !range.newestPassingRevision)
 425             return
 426
 427         if (!mergedRange.oldestFailingRevision)
 428             mergedRange.oldestFailingRevision = range.oldestFailingRevision;
 429         if (!mergedRange.newestPassingRevision)
 430             mergedRange.newestPassingRevision = range.newestPassingRevision;
 431
 432         if (range.oldestFailingRevision && range.oldestFailingRevision < mergedRange.oldestFailingRevision)
 433             mergedRange.oldestFailingRevision = range.oldestFailingRevision;
 434         if (range.newestPassingRevision > mergedRange.newestPassingRevision)
 435             mergedRange.newestPassingRevision = range.newestPassingRevision;
 436     });
 437
 438     return mergedRange;
 439 }
 440
 441 results.unifyRegressionRanges = function(builderNameList, testName, callback)
 442 {
 443     var regressionRanges = {};
 444
 445     var tracker = new base.RequestTracker(builderNameList.length, function() {
 446         var mergedRange = mergeRegressionRanges(regressionRanges);
 447         callback(mergedRange.oldestFailingRevision, mergedRange.newestPassingRevision);
 448     });
 449
 450     $.each(builderNameList, function(index, builderName) {
 451         results.regressionRangeForFailure(builderName, testName, function(oldestFailingRevision, newestPassingRevision) {
 452             var range = {};
 453             range.oldestFailingRevision = oldestFailingRevision;
 454             range.newestPassingRevision = newestPassingRevision;
 455             regressionRanges[builderName] = range;
 456             tracker.requestComplete();
 457         });
 458     });
 459 };
 460
 461 results.resultNodeForTest = function(resultsTree, testName)
 462 {
 463     var testNamePath = testName.split('/');
 464     var currentNode = resultsTree['tests'];
 465     $.each(testNamePath, function(index, segmentName) {
 466         if (!currentNode)
 467             return;
 468         currentNode = (segmentName in currentNode) ? currentNode[segmentName] : null;
 469     });
 470     return currentNode;
 471 };
 472
 473 results.resultKind = function(url)
 474 {
 475     if (/-actual\.[a-z]+$/.test(url))
 476         return results.kActualKind;
 477     else if (/-expected\.[a-z]+$/.test(url))
 478         return results.kExpectedKind;
 479     else if (/diff\.[a-z]+$/.test(url))
 480         return results.kDiffKind;
 481     return results.kUnknownKind;
 482 }
 483
 484 results.resultType = function(url)
 485 {
 486     if (/\.png$/.test(url))
 487         return results.kImageType;
 488     if (/\.wav$/.test(url))
 489         return results.kAudioType;
 490     return results.kTextType;
 491 }
 492
 493 function sortResultURLsBySuffix(urls)
 494 {
 495     var sortedURLs = [];
 496     $.each(kPreferredSuffixOrder, function(i, suffix) {
 497         $.each(urls, function(j, url) {
 498             if (!base.endsWith(url, suffix))
 499                 return;
 500             sortedURLs.push(url);
 501         });
 502     });
 503     if (sortedURLs.length != urls.length)
 504         throw "sortResultURLsBySuffix failed to return the same number of URLs."
 505     return sortedURLs;
 506 }
 507
 508 results.fetchResultsURLs = function(failureInfo, callback)
 509 {
 510     var testNameStem = base.trimExtension(failureInfo.testName);
 511     var urlStem = resultsDirectoryURL(failureInfo.builderName);
 512
 513     var suffixList = possibleSuffixListFor(failureInfo.failureTypeList);
 514     var resultURLs = [];
 515     var tracker = new base.RequestTracker(suffixList.length, function() {
 516         callback(sortResultURLsBySuffix(resultURLs));
 517     });
 518     $.each(suffixList, function(index, suffix) {
 519         var url = urlStem + testNameStem + suffix;
 520         net.probe(url, {
 521             success: function() {
 522                 resultURLs.push(url);
 523                 tracker.requestComplete();
 524             },
 525             error: function() {
 526                 tracker.requestComplete();
 527             },
 528         });
 529     });
 530 };
 531
 532 results.fetchResultsByBuilder = function(builderNameList, callback)
 533 {
 534     var resultsByBuilder = {};
 535     var tracker = new base.RequestTracker(builderNameList.length, function() {
 536         callback(resultsByBuilder);
 537     });
 538     $.each(builderNameList, function(index, builderName) {
 539         var resultsURL = resultsSummaryURL(builderName);
 540         net.jsonp(resultsURL, function(resultsTree) {
 541             resultsByBuilder[builderName] = resultsTree;
 542             tracker.requestComplete();
 543         });
 544     });
 545 };
 546
 547 })();