2 ** Copyright (c) 2012 The Khronos Group Inc.
4 ** Permission is hereby granted, free of charge, to any person obtaining a
5 ** copy of this software and/or associated documentation files (the
6 ** "Materials"), to deal in the Materials without restriction, including
7 ** without limitation the rights to use, copy, modify, merge, publish,
8 ** distribute, sublicense, and/or sell copies of the Materials, and to
9 ** permit persons to whom the Materials are furnished to do so, subject to
10 ** the following conditions:
12 ** The above copyright notice and this permission notice shall be included
13 ** in all copies or substantial portions of the Materials.
15 ** THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 ** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17 ** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18 ** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19 ** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20 ** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21 ** MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
24 // This is a test harness for running javascript tests in the browser.
25 // The only identifier exposed by this harness is WebGLTestHarnessModule.
27 // To use it make an HTML page with an iframe. Then call the harness like this
29 // function reportResults(type, msg, success) {
34 // var fileListURL = '00_test_list.txt';
35 // var testHarness = new WebGLTestHarnessModule.TestHarness(
41 // The harness will load the fileListURL and parse it for the URLs, one URL
42 // per line preceded by options, see below. URLs should be on the same domain
43 // and at the same folder level or below the main html file. If any URL ends
44 // in .txt it will be parsed as well so you can nest .txt files. URLs inside a
45 // .txt file should be relative to that text file.
47 // During startup, for each page found the reportFunction will be called with
48 // WebGLTestHarnessModule.TestHarness.reportType.ADD_PAGE and msg will be
49 // the URL of the test.
51 // Each test is required to call testHarness.reportResults. This is most easily
52 // accomplished by storing that value on the main window with
54 // window.webglTestHarness = testHarness
56 // and then adding these to functions to your tests.
58 // function reportTestResultsToHarness(success, msg) {
59 // if (window.parent.webglTestHarness) {
60 // window.parent.webglTestHarness.reportResults(success, msg);
64 // function notifyFinishedToHarness() {
65 // if (window.parent.webglTestHarness) {
66 // window.parent.webglTestHarness.notifyFinished();
70 // This way your tests will still run without the harness and you can use
71 // any testing framework you want.
73 // Each test should call reportTestResultsToHarness with true for success if it
74 // succeeded and false if it fail followed and any message it wants to
75 // associate with the test. If your testing framework supports checking for
76 // timeout you can call it with success equal to undefined in that case.
78 // To run the tests, call testHarness.runTests(options);
80 // For each test run, before the page is loaded the reportFunction will be
81 // called with WebGLTestHarnessModule.TestHarness.reportType.START_PAGE and msg
82 // will be the URL of the test. You may return false if you want the test to be
85 // For each test completed the reportFunction will be called with
86 // with WebGLTestHarnessModule.TestHarness.reportType.TEST_RESULT,
87 // success = true on success, false on failure, undefined on timeout
88 // and msg is any message the test choose to pass on.
90 // When all the tests on the page have finished your page must call
91 // notifyFinishedToHarness. If notifyFinishedToHarness is not called
92 // the harness will assume the test timed out.
94 // When all the tests on a page have finished OR the page as timed out the
95 // reportFunction will be called with
96 // WebGLTestHarnessModule.TestHarness.reportType.FINISH_PAGE
97 // where success = true if the page has completed or undefined if the page timed
100 // Finally, when all the tests have completed the reportFunction will be called
101 // with WebGLTestHarnessModule.TestHarness.reportType.FINISHED_ALL_TESTS.
105 // These are passed in to the TestHarness as a JavaScript object
107 // version: (required!)
109 // Specifies a version used to filter tests. Tests marked as requiring
110 // a version greater than this version will not be included.
112 // example: new TestHarness(...., {version: "3.1.2"});
116 // Specifies the minimum version a test must require to be included.
117 // This basically flips the filter so that only tests marked with
118 // --min-version will be included if they are at this minVersion or
121 // example: new TestHarness(...., {minVersion: "2.3.1"});
125 // Specifies the maximum version a test must require to be included.
126 // This basically flips the filter so that only tests marked with
127 // --max-version will be included if they are at this maxVersion or
130 // example: new TestHarness(...., {maxVersion: "2.3.1"});
134 // Specifies to skip any tests marked as slow.
136 // example: new TestHarness(..., {fast: true});
140 // Any test URL or .txt file can be prefixed by the following options
144 // Sets the minimum version required to include this test. A version is
145 // passed into the harness options. Any test marked as requiring a
146 // min-version greater than the version passed to the harness is skipped.
147 // This allows you to add new tests to a suite of tests for a future
148 // version of the suite without including the test in the current version.
149 // If no -min-version is specified it is inheriited from the .txt file
150 // including it. The default is 1.0.0
152 // example: --min-version 2.1.3 sometest.html
156 // Sets the maximum version required to include this test. A version is
157 // passed into the harness options. Any test marked as requiring a
158 // max-version less than the version passed to the harness is skipped.
159 // This allows you to test functionality that has been removed from later
160 // versions of the suite.
161 // If no -max-version is specified it is inherited from the .txt file
164 // example: --max-version 1.9.9 sometest.html
168 // Marks a test as slow. Slow tests can be skipped by passing fastOnly: true
169 // to the TestHarness. Of course you need to pass all tests but sometimes
170 // you'd like to test quickly and run only the fast subset of tests.
172 // example: --slow some-test-that-takes-2-mins.html
175 WebGLTestHarnessModule = function() {
178 * Wrapped logging function.
180 var log = function(msg) {
181 if (window.console && window.console.log) {
182 window.console.log(msg);
187 * Loads text from an external file. This function is synchronous.
188 * @param {string} url The url of the external file.
189 * @param {!function(bool, string): void} callback that is sent a bool for
190 * success and the string.
192 var loadTextFileAsynchronous = function(url, callback) {
193 log ("loading: " + url);
194 var error = 'loadTextFileSynchronous failed to load url "' + url + '"';
196 if (window.XMLHttpRequest) {
197 request = new XMLHttpRequest();
198 if (request.overrideMimeType) {
199 request.overrideMimeType('text/plain');
202 throw 'XMLHttpRequest is disabled';
205 request.open('GET', url, true);
206 request.onreadystatechange = function() {
207 if (request.readyState == 4) {
209 // HTTP reports success with a 200 status. The file protocol reports
210 // success with zero. HTTP does not use zero as a status code (they
212 // https://developer.mozilla.org/En/Using_XMLHttpRequest
213 var success = request.status == 200 || request.status == 0;
215 text = request.responseText;
217 log("loaded: " + url);
218 callback(success, text);
223 log("failed to load: " + url);
229 * @param {string} versionString WebGL version string.
230 * @return {number} Integer containing the WebGL major version.
232 var getMajorVersion = function(versionString) {
233 if (!versionString) {
236 return parseInt(versionString.split(" ")[0].split(".")[0], 10);
240 * @param {string} url Base URL of the test.
241 * @param {number} webglVersion Integer containing the WebGL major version.
242 * @return {string} URL that will run the test with the given WebGL version.
244 var getURLWithVersion = function(url, webglVersion) {
245 return url + "?webglVersion=" + webglVersion;
249 * Compare version strings.
251 var greaterThanOrEqualToVersion = function(have, want) {
252 have = have.split(" ")[0].split(".");
253 want = want.split(" ")[0].split(".");
255 //have 1.2.3 want 1.1
256 //have 1.1.1 want 1.1
257 //have 1.0.9 want 1.1
258 //have 1.1 want 1.1.1
260 for (var ii = 0; ii < want.length; ++ii) {
261 var wantNum = parseInt(want[ii]);
262 var haveNum = have[ii] ? parseInt(have[ii]) : 0
263 if (haveNum > wantNum) {
264 return true; // 2.0.0 is greater than 1.2.3
266 if (haveNum < wantNum) {
274 * Reads a file, recursively adding files referenced inside.
276 * Each line of URL is parsed, comments starting with '#' or ';'
277 * or '//' are stripped.
279 * arguments beginning with -- are extracted
281 * lines that end in .txt are recursively scanned for more files
282 * other lines are added to the list of files.
284 * @param {string} url The url of the file to read.
285 * @param {void function(boolean, !Array.<string>)} callback.
286 * Callback that is called with true for success and an
287 * array of filenames.
288 * @param {Object} options. Optional options
291 * version: {string} The version of the conformance test.
292 * Tests with the argument --min-version <version> will
293 * be ignored version is less then <version>
296 var getFileList = function(url, callback, options) {
299 var copyObject = function(obj) {
300 return JSON.parse(JSON.stringify(obj));
303 var toCamelCase = function(str) {
304 return str.replace(/-([a-z])/g, function (g) { return g[1].toUpperCase() });
307 var globalOptions = copyObject(options);
308 globalOptions.defaultVersion = "1.0";
309 globalOptions.defaultMaxVersion = null;
311 var getFileListImpl = function(prefix, line, lineNum, hierarchicalOptions, callback) {
314 var args = line.split(/\s+/);
317 var testOptions = {};
318 for (var jj = 0; jj < args.length; ++jj) {
322 throw ("bad option at in " + url + ":" + lineNum + ": " + arg);
324 var option = arg.substring(2);
326 // no argument options.
328 testOptions[toCamelCase(option)] = true;
330 // one argument options.
334 testOptions[toCamelCase(option)] = args[jj];
337 throw ("bad unknown option '" + option + "' at in " + url + ":" + lineNum + ": " + arg);
340 nonOptions.push(arg);
343 var url = prefix + nonOptions.join(" ");
345 if (url.substr(url.length - 4) != '.txt') {
346 var minVersion = testOptions.minVersion;
348 minVersion = hierarchicalOptions.defaultVersion;
350 var maxVersion = testOptions.maxVersion;
352 maxVersion = hierarchicalOptions.defaultMaxVersion;
354 var slow = testOptions.slow;
356 slow = hierarchicalOptions.defaultSlow;
359 if (globalOptions.fast && slow) {
361 } else if (globalOptions.minVersion) {
362 useTest = greaterThanOrEqualToVersion(minVersion, globalOptions.minVersion);
363 } else if (globalOptions.maxVersion && maxVersion) {
364 useTest = greaterThanOrEqualToVersion(globalOptions.maxVersion, maxVersion);
366 useTest = greaterThanOrEqualToVersion(globalOptions.version, minVersion);
368 useTest = useTest && greaterThanOrEqualToVersion(maxVersion, globalOptions.version);
378 if (url.substr(url.length - 4) == '.txt') {
379 // If a version was explicity specified pass it down.
380 if (testOptions.minVersion) {
381 hierarchicalOptions.defaultVersion = testOptions.minVersion;
383 if (testOptions.maxVersion) {
384 hierarchicalOptions.defaultMaxVersion = testOptions.maxVersion;
386 if (testOptions.slow) {
387 hierarchicalOptions.defaultSlow = testOptions.slow;
389 loadTextFileAsynchronous(url, function() {
390 return function(success, text) {
395 var lines = text.split('\n');
397 var lastSlash = url.lastIndexOf('/');
398 if (lastSlash >= 0) {
399 prefix = url.substr(0, lastSlash + 1);
404 for (var ii = 0; ii < lines.length; ++ii) {
405 var str = lines[ii].replace(/^\s\s*/, '').replace(/\s\s*$/, '');
406 if (str.length > 4 &&
409 str.substr(0, 2) != "//") {
411 getFileListImpl(prefix, str, ii + 1, copyObject(hierarchicalOptions), function(index) {
412 return function(success, new_files) {
413 //log("got files: " + new_files.length);
415 files[index] = new_files;
424 function finish(success) {
429 //log("count: " + count);
431 callback(!fail, files);
438 callback(true, files);
442 getFileListImpl('', url, 1, globalOptions, function(success, files) {
446 function flatten(files) {
447 for (var ii = 0; ii < files.length; ++ii) {
448 var value = files[ii];
449 if (typeof(value) == "string") {
456 callback(success, flat);
460 var FilterURL = (function() {
461 var prefix = window.location.pathname;
462 prefix = prefix.substring(0, prefix.lastIndexOf("/") + 1);
463 return function(url) {
464 if (url.substring(0, prefix.length) == prefix) {
465 url = url.substring(prefix.length);
471 var TestFile = function(url) {
475 var Test = function(file) {
479 var TestHarness = function(iframe, filelistUrl, reportFunc, options) {
480 this.window = window;
481 this.iframes = iframe.length ? iframe : [iframe];
482 this.reportFunc = reportFunc;
483 this.timeoutDelay = 20000;
485 this.allowSkip = options.allowSkip;
486 this.webglVersion = getMajorVersion(options.version);
489 getFileList(filelistUrl, function() {
490 return function(success, files) {
491 that.addFiles_(success, files);
497 TestHarness.reportType = {
503 FINISHED_ALL_TESTS: 6
506 TestHarness.prototype.addFiles_ = function(success, files) {
509 TestHarness.reportType.FINISHED_ALL_TESTS,
511 'Unable to load tests. Are you running locally?\n' +
512 'You need to run from a server or configure your\n' +
513 'browser to allow access to local files (not recommended).\n\n' +
514 'Note: An easy way to run from a server:\n\n' +
515 '\tcd path_to_tests\n' +
516 '\tpython -m SimpleHTTPServer\n\n' +
517 'then point your browser to ' +
518 '<a href="http://localhost:8000/webgl-conformance-tests.html">' +
519 'http://localhost:8000/webgl-conformance-tests.html</a>',
523 log("total files: " + files.length);
524 for (var ii = 0; ii < files.length; ++ii) {
525 log("" + ii + ": " + files[ii]);
526 this.files.push(new TestFile(files[ii]));
527 this.reportFunc(TestHarness.reportType.ADD_PAGE, '', files[ii], undefined);
529 this.reportFunc(TestHarness.reportType.READY, '', undefined, undefined);
532 TestHarness.prototype.runTests = function(opt_options) {
533 var options = opt_options || { };
534 options.start = options.start || 0;
535 options.count = options.count || this.files.length;
537 this.idleIFrames = this.iframes.slice(0);
538 this.runningTests = {};
540 for (var ii = 0; ii < options.count; ++ii) {
541 testsToRun.push(ii + options.start);
543 this.numTestsRemaining = options.count;
544 this.testsToRun = testsToRun;
545 this.startNextTest();
548 TestHarness.prototype.setTimeout = function(test) {
550 test.timeoutId = this.window.setTimeout(function() {
552 }, this.timeoutDelay);
555 TestHarness.prototype.clearTimeout = function(test) {
556 this.window.clearTimeout(test.timeoutId);
559 TestHarness.prototype.startNextTest = function() {
560 if (this.numTestsRemaining == 0) {
562 this.reportFunc(TestHarness.reportType.FINISHED_ALL_TESTS,
565 while (this.testsToRun.length > 0 && this.idleIFrames.length > 0) {
566 var testId = this.testsToRun.shift();
567 var iframe = this.idleIFrames.shift();
568 this.startTest(iframe, this.files[testId], this.webglVersion);
573 TestHarness.prototype.startTest = function(iframe, testFile, webglVersion) {
578 var url = testFile.url;
579 this.runningTests[url] = test;
580 log("loading: " + url);
581 if (this.reportFunc(TestHarness.reportType.START_PAGE, url, url, undefined)) {
582 iframe.src = getURLWithVersion(url, webglVersion);
583 this.setTimeout(test);
585 this.reportResults(url, !!this.allowSkip, "skipped", true);
586 this.notifyFinished(url);
590 TestHarness.prototype.getTest = function(url) {
591 var test = this.runningTests[FilterURL(url)];
593 throw("unknown test:" + url);
598 TestHarness.prototype.reportResults = function(url, success, msg, skipped) {
599 url = FilterURL(url);
600 var test = this.getTest(url);
601 this.clearTimeout(test);
602 log(success ? "PASS" : "FAIL", msg);
603 this.reportFunc(TestHarness.reportType.TEST_RESULT, url, msg, success, skipped);
604 // For each result we get, reset the timeout
605 this.setTimeout(test);
608 TestHarness.prototype.dequeTest = function(test) {
609 this.clearTimeout(test);
610 this.idleIFrames.push(test.iframe);
611 delete this.runningTests[test.testFile.url];
612 --this.numTestsRemaining;
615 TestHarness.prototype.notifyFinished = function(url) {
616 url = FilterURL(url);
617 var test = this.getTest(url);
618 log(url + ": finished");
619 this.dequeTest(test);
620 this.reportFunc(TestHarness.reportType.FINISH_PAGE, url, url, true);
621 this.startNextTest();
624 TestHarness.prototype.timeout = function(test) {
625 this.dequeTest(test);
626 var url = test.testFile.url;
627 log(url + ": timeout");
628 this.reportFunc(TestHarness.reportType.FINISH_PAGE, url, url, undefined);
629 this.startNextTest();
632 TestHarness.prototype.setTimeoutDelay = function(x) {
633 this.timeoutDelay = x;
637 'TestHarness': TestHarness,
638 'getMajorVersion': getMajorVersion,
639 'getURLWithVersion': getURLWithVersion