3 # Copyright (C) 2010, 2011, 2012 Apple Inc. All rights reserved.
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
8 # 1. Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # 2. Redistributions in binary form must reproduce the above copyright
11 # notice, this list of conditions and the following disclaimer in the
12 # documentation and/or other materials provided with the distribution.
14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24 # THE POSSIBILITY OF SUCH DAMAGE.
31 use Getopt::Long qw(:config pass_through);
33 use lib $FindBin::Bin;
38 sub dumpTestsBySuite(\@);
41 sub runTestsBySuite(\@$);
42 sub prepareEnvironmentForRunningTestTool();
45 # Defined in VCSUtils.
46 sub possiblyColored($$);
48 # Timeout for individual test, in sec
55 my $buildDefault = $build ? "build" : "do not build";
59 my $programName = basename($0);
61 Usage: $programName [options] [suite or test prefixes]
62 --help Show this help message
63 -v|--verbose Verbose output
64 -d|--dump-tests Dump the names of testcases without running them
65 --[no-]build Build (or do not build) unit tests prior to running (default: $buildDefault)
66 --chromium Run the Chromium port on Mac/Win/Linux
71 'verbose|v' => \$verbose,
72 'dump|d' => \$dumpTests,
82 buildTestTool() if $build;
83 setPathForRunningWebKitApp(\%ENV);
84 my @testsToRun = listAllTests();
86 @testsToRun = grep { my $test = $_; grep { $test =~ m/^\Q$_\E/ } @ARGV; } @testsToRun if @ARGV;
89 dumpTestsBySuite(@testsToRun);
93 exit runTestsBySuite(@testsToRun, $verbose);
95 sub isSupportedPlatform()
97 return isAppleMacWebKit() || isAppleWinWebKit() || isChromium();
100 sub dumpTestsBySuite(\@)
103 print "Dumping test cases\n";
104 print "------------------\n";
106 for my $suiteAndTest (sort @$tests) {
107 my ($suite, $test) = split(/\./, $suiteAndTest);
108 if ($lastSuite ne $suite) {
114 print "------------------\n";
117 sub runTestsBySuite(\@$)
119 my ($tests, $verbose) = @_;
122 for my $suiteAndTest (sort @$tests) {
123 my ($suite, $test) = split(/\./, $suiteAndTest);
124 if ($lastSuite ne $suite) {
126 print "Suite: $suite\n" unless $verbose;
128 my $failed = runTest($suite, $test, $verbose);
129 $anyFailures ||= $failed;
134 print "Tests that failed:\n";
135 for my $test (@testsFailed) {
139 if (@testsTimedOut) {
140 print "Tests that timed out:\n";
141 for my $test (@testsTimedOut) {
151 my ($suite, $testName, $verbose) = @_;
152 my $test = $suite . "." . $testName;
154 my $gtestArg = "--gtest_filter=" . $test;
156 print " Test: $testName -> " unless $verbose;
161 die "run-api-tests is not supported on this platform.\n" unless isSupportedPlatform();
163 prepareEnvironmentForRunningTestTool();
166 my ($childIn, $childOut, $childErr);
168 $childOut = ">&STDERR";
169 $childErr = ">&STDERR";
171 open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
172 $childOut = ">&DEVNULL";
173 $childErr = ">&DEVNULL";
177 if (isAppleMacWebKit() && architecture()) {
178 $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), testToolPath(), $gtestArg, @ARGV) or die "Failed to run test: $test.";
180 $pid = open3($childIn, $childOut, $childErr, testToolPath(), $gtestArg, @ARGV) or die "Failed to run test: $test.";
186 close(DEVNULL) unless ($verbose);
188 local $SIG{ALRM} = sub { die "alarm\n" };
195 die unless $@ eq "alarm\n";
196 kill SIGTERM, $pid or kill SIGKILL, $pid;
201 push @testsFailed, $test;
204 push @testsTimedOut, $test;
205 print possiblyColored("bold yellow", "Timeout"), "\n";
206 } elsif (!$verbose) {
208 print possiblyColored("bold red", "Failed"), "\n";
210 print possiblyColored("bold green", "Passed"), "\n";
214 return $timedOut || $result;
222 die "run-api-tests is not supported on this platform.\n" unless isSupportedPlatform();
224 prepareEnvironmentForRunningTestTool();
227 my ($childIn, $childOut, $childErr);
229 $childErr = ">&STDERR";
231 open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
232 $childErr = ">&DEVNULL";
236 if (isAppleMacWebKit() && architecture()) {
237 $pid = open3($childIn, $childOut, $childErr, "arch", "-" . architecture(), testToolPath(), "--gtest_list_tests") or die "Failed to build list of tests!";
239 $pid = open3($childIn, $childOut, $childErr, testToolPath(), "--gtest_list_tests") or die "Failed to build list of tests!";
243 @toolOutput = <$childOut>;
246 close(DEVNULL) unless ($verbose);
252 print STDERR "Failed to build list of tests!\n";
253 exit exitStatus($result);
258 for my $line (@toolOutput) {
259 $line =~ s/[\r\n]*$//;
260 if ($line =~ m/\.$/) {
261 $suite = $line; # "SuiteName."
263 $line =~ s/^\s*//; # "TestName"
264 push @tests, $suite . $line; # "SuiteName.TestName"
273 my $originalCwd = getcwd();
277 my $buildTestTool = "build-api-tests";
278 print STDERR "Running $buildTestTool\n";
281 my ($childIn, $childOut, $childErr);
283 # When not quiet, let the child use our stdout/stderr.
284 $childOut = ">&STDOUT";
285 $childErr = ">&STDERR";
287 open(DEVNULL, ">", File::Spec->devnull()) or die "Failed to open /dev/null";
288 $childOut = ">&DEVNULL";
289 $childErr = ">&DEVNULL";
292 my @args = argumentsForConfiguration();
293 my $pathToBuildTestTool = File::Spec->catfile("Tools", "Scripts", $buildTestTool);
294 my $buildProcess = open3($childIn, $childOut, $childErr, "perl", $pathToBuildTestTool, @args) or die "Failed to run " . $buildTestTool;
299 close(DEVNULL) unless ($verbose);
301 waitpid($buildProcess, 0);
302 my $buildResult = $?;
305 print STDERR "Compiling TestWebKitAPI failed!\n";
306 exit exitStatus($buildResult);
312 sub prepareEnvironmentForRunningTestTool()
314 return unless isAppleMacWebKit();
316 $ENV{DYLD_FRAMEWORK_PATH} = productDir();
317 $ENV{WEBKIT_UNSET_DYLD_FRAMEWORK_PATH} = "YES";
322 my $path = File::Spec->catfile(productDir(), "TestWebKitAPI");
323 return $path unless isAppleWinWebKit();
326 if (configurationForVisualStudio() eq "Debug_All") {
331 return "$path$suffix.exe";