2 # -*- Mode: Perl; tab-width: 4; indent-tabs-mode: nil; -*-
4 # ***** BEGIN LICENSE BLOCK *****
5 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
7 # The contents of this file are subject to the Mozilla Public License Version
8 # 1.1 (the "License"); you may not use this file except in compliance with
9 # the License. You may obtain a copy of the License at
10 # http://www.mozilla.org/MPL/
12 # Software distributed under the License is distributed on an "AS IS" basis,
13 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
14 # for the specific language governing rights and limitations under the
17 # The Original Code is Mozilla JavaScript Testing Utilities
19 # The Initial Developer of the Original Code is
20 # Mozilla Corporation.
21 # Portions created by the Initial Developer are Copyright (C) 2007
22 # the Initial Developer. All Rights Reserved.
24 # Contributor(s): Bob Clary <bclary@bclary.com>
26 # Alternatively, the contents of this file may be used under the terms of
27 # either the GNU General Public License Version 2 or later (the "GPL"), or
28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
29 # in which case the provisions of the GPL or the LGPL are applicable instead
30 # of those above. If you wish to allow use of your version of this file only
31 # under the terms of either the GPL or the LGPL, and not to allow others to
32 # use your version of this file under the terms of the MPL, indicate your
33 # decision by deleting the provisions above and replace them with the notice
34 # and other provisions required by the GPL or the LGPL. If you do not delete
35 # the provisions above, a recipient may use your version of this file under
36 # the terms of any one of the MPL, the GPL or the LGPL.
38 # ***** END LICENSE BLOCK *****
41 use Getopt::Mixed "nextOption";
53 my $option_desc = "b=s branch>b T=s buildtype>T R=s repo>R t=s testtype>t o=s os>o K=s kernel>K A=s arch>A M=s memory>M z=s timezone>z J=s jsoptions>J l=s rawlogfile>l f=s failurelogfile>f r=s patterns>r O=s outputprefix>O D debug>D";
70 my $debug = $ENV{DEBUG};
74 my $knownfailurebranchpattern;
75 my $failurebranchpattern;
76 my $knownfailureospattern;
78 my $knownfailurerepopattern;
79 my $failurerepopattern;
80 my $knownfailurebuildtypepattern;
81 my $failurebuildtypepattern;
82 my $knownfailuretesttypepattern;
83 my $failuretesttypepattern;
84 my $knownfailuretimezonepattern;
85 my $failuretimezonepattern;
86 my $knownfailurejsoptionspattern;
87 my $failurejsoptionspattern;
88 my $knownfailurearchpattern;
89 my $failurearchpattern;
90 my $knownfailurekernelpattern;
91 my $failurekernelpattern;
92 my $knownfailurememorypattern;
93 my $failurememorypattern;
102 my %includedtests = {};
106 my $regchars = '\[\^\-\]\|\{\}\?\*\+\.\<\>\$\(\)';
111 my $jsdir = $ENV{TEST_JSDIR};
113 if (!defined($jsdir)) {
114 $jsdir = "/work/mozilla/mozilla.com/test.mozilla.com/www/tests/mozilla.org/js";
117 my @excludedfiles = ("excluded-$branch-$testtype-$buildtype.tests");
118 my @includedfiles = ("included-$branch-$testtype-$buildtype.tests");
120 # create working patterns file consisting of matches to users selection
121 # and which has the test description patterns escaped
123 # remove the excluded tests from the possible fixes log
126 foreach $excludedfile ( @excludedfiles ) {
127 open EXCLUDED, "<$jsdir/$excludedfile" or die "Unable to open excluded file $jsdir/$excludedfile: $!\n";
131 next if ($_ =~ /^\#/);
135 push @excludedtests, ($_);
140 @excludedtests = sort @excludedtests;
142 foreach $includedfile ( @includedfiles ) {
143 open INCLUDED, "<$jsdir/$includedfile" or die "Unable to open included file $jsdir/$includedfile: $!\n";
147 next if ($_ =~ /^\#/);
151 $includedtests{$_} = 1;
156 debug "loading patterns $patterns";
157 debug "pattern filter: ^TEST_ID=[^,]*, TEST_BRANCH=$knownfailurebranchpattern, TEST_REPO=$knownfailurerepopattern, TEST_BUILDTYPE=$knownfailurebuildtypepattern, TEST_TYPE=$knownfailuretesttypepattern, TEST_OS=$knownfailureospattern, TEST_KERNEL=$knownfailurekernelpattern, TEST_PROCESSORTYPE=$knownfailurearchpattern, TEST_MEMORY=$knownfailurememorypattern, TEST_TIMEZONE=$knownfailuretimezonepattern, TEST_OPTIONS=$knownfailurejsoptionspattern,";
159 open PATTERNS, "<$patterns" or die "Unable to open known failure patterns file $patterns: $!\n";
165 ($testid) = $_ =~ /^TEST_ID=([^,]*),/;
167 if (!$includedtests{$testid})
169 debug "test $testid was not included during this run";
171 elsif ($_ =~ /^TEST_ID=[^,]*, TEST_BRANCH=$knownfailurebranchpattern, TEST_REPO=$knownfailurerepopattern, TEST_BUILDTYPE=$knownfailurebuildtypepattern, TEST_TYPE=$knownfailuretesttypepattern, TEST_OS=$knownfailureospattern, TEST_KERNEL=$knownfailurekernelpattern, TEST_PROCESSORTYPE=$knownfailurearchpattern, TEST_MEMORY=$knownfailurememorypattern, TEST_TIMEZONE=$knownfailuretimezonepattern, TEST_OPTIONS=$knownfailurejsoptionspattern,/) {
172 debug "adding pattern : $_";
173 push @patterns, (escape_pattern($_));
176 debug "skipping pattern: $_";
182 # create a working copy of the current failures which match the users selection
184 debug "failure filter: ^TEST_ID=[^,]*, TEST_BRANCH=$failurebranchpattern, TEST_REPO=$failurerepopattern, TEST_BUILDTYPE=$failurebuildtypepattern, TEST_TYPE=$failuretesttypepattern, TEST_OS=$failureospattern, TEST_KERNEL=$failurekernelpattern, TEST_PROCESSORTYPE=$failurearchpattern, TEST_MEMORY=$failurememorypattern, TEST_TIMEZONE=$failuretimezonepattern, TEST_OPTIONS=$failurejsoptionspattern, TEST_RESULT=FAIL[^,]*,/";
186 if (defined($rawlogfile)) {
188 $failurelogfile = "$outputprefix-results-failures.log";
189 my $alllog = "$outputprefix-results-all.log";
191 debug "writing failures $failurelogfile";
193 open INPUTLOG, "$jsdir/post-process-logs.pl $rawlogfile |" or die "Unable to open $rawlogfile $!\n";
194 open ALLLOG, ">$alllog" or die "Unable to open $alllog $!\n";
195 open FAILURELOG, ">$failurelogfile" or die "Unable to open $failurelogfile $!\n";
202 if ($_ =~ /^TEST_ID=[^,]*, TEST_BRANCH=$failurebranchpattern, TEST_REPO=$failurerepopattern, TEST_BUILDTYPE=$failurebuildtypepattern, TEST_TYPE=$failuretesttypepattern, TEST_OS=$failureospattern, TEST_KERNEL=$failurekernelpattern, TEST_PROCESSORTYPE=$failurearchpattern, TEST_MEMORY=$failurememorypattern, TEST_TIMEZONE=$failuretimezonepattern, TEST_OPTIONS=$failurejsoptionspattern, TEST_RESULT=FAIL[^,]*,/) {
204 push @failures, ($_);
205 print FAILURELOG "$_\n";
213 die "FATAL ERROR in post-process-logs.pl" if $inputrc != 0;
217 debug "loading failures $failurelogfile";
219 my $failurelogfilemode;
221 if ($failurelogfile =~ /\.bz2$/)
223 $failurelogfilemode = "bzcat $failurelogfile|";
225 elsif ($failurelogfile =~ /\.gz$/)
227 $failurelogfilemode = "zcat $failurelogfile|";
231 $failurelogfilemode = "<$failurelogfile";
234 open FAILURES, "$failurelogfilemode" or die "Unable to open current failure log $failurelogfile: $!\n";
238 if ($_ =~ /^TEST_ID=[^,]*, TEST_BRANCH=$failurebranchpattern, TEST_REPO=$failurerepopattern, TEST_BUILDTYPE=$failurebuildtypepattern, TEST_TYPE=$failuretesttypepattern, TEST_OS=$failureospattern, TEST_KERNEL=$failurekernelpattern, TEST_PROCESSORTYPE=$failurearchpattern, TEST_MEMORY=$failurememorypattern, TEST_TIMEZONE=$failuretimezonepattern, TEST_OPTIONS=$failurejsoptionspattern, TEST_RESULT=FAIL[^,]*,/) {
240 push @failures, ($_);
246 debug "finding fixed bugs";
248 unlink "$outputprefix-results-possible-fixes.log";
250 foreach $pattern (@patterns) {
251 # look for known failure patterns that don't have matches in the
252 # the current failures selected by the user.
254 debug "searching for matches to $pattern\n";
256 @results = grep m@^$pattern@, @failures;
260 foreach $failure (@failures) {
261 if ($failure =~ $pattern) {
262 debug "MATCH: $pattern - $failure\n";
265 debug "NOMATCH: $pattern - $failure\n";
269 if ($#results == -1) {
270 debug "fix: '$pattern'";
271 push @fixes, ($pattern)
275 foreach $excludedtest ( @excludedtests ) {
276 # remove any potential fixes which are due to the test being excluded
279 @results = grep m@$excludedtest@, @fixes;
280 if ($#results > -1) {
281 print "excluding: " . (join ', ', @results) . "\n";
285 @results = grep !m@$excludedtest@, @fixes;
291 open OUTPUT, ">$outputprefix-results-possible-fixes.log" or die "Unable to open $outputprefix-results-possible-fixes.log: $!";
292 foreach $fix (@fixes) {
293 print OUTPUT unescape_pattern($fix) . "\n";
300 print STDOUT "log: $outputprefix-results-possible-fixes.log\n";
302 debug "finding regressions";
305 my $changed = ($#patterns != -1);
307 debug "changed=$changed, \$#patterns=$#patterns, \$#failures=$#failures";
317 foreach $pattern (@patterns) {
319 debug "Pattern: $pattern";
321 my @nomatches = grep !m@^$pattern@, @failures;
322 my @matches = grep m@^$pattern@, @failures;
325 my $temp = join ', ', @nomatches;
326 debug "nomatches: $#nomatches $temp";
327 $temp = join ', ', @matches;
328 debug "matches: $#matches $temp";
331 @failures = @nomatches;
333 if ($#matches > -1) {
337 debug "*****************************************";
342 debug "\$#excludedtests=$#excludedtests, \$#failures=$#failures";
344 foreach $excludedtest ( @excludedtests ) {
347 @results = grep m@$excludedtest@, @failures;
348 if ($#results > -1) {
349 print "excluding: " . (join ', ', @results) . "\n";
353 @results = grep !m@$excludedtest@, @failures;
355 debug "\$#results=$#results, \$excludedtest=$excludedtest, \$#failures=$#failures";
357 @failures = @results;
360 debug "possible regressions: \$#failures=$#failures";
362 open OUTPUT, ">$outputprefix-results-possible-regressions.log" or die "Unable to open $outputprefix-results-possible-regressions.log: $!";
365 foreach $failure (@failures) {
366 print OUTPUT "$failure\n";
368 debug "regression: $failure";
373 print STDOUT "log: $outputprefix-results-possible-regressions.log\n";
379 print STDERR "DEBUG: $msg\n";
391 known-failures.pl [-b|--branch] branch
392 [-T|--buildtype] buildtype
393 [-t|--testtype] testtype
398 [-z|--timezone] timezone
399 [-J|--jsoptions] jsoptions
400 [-r|--patterns] patterns
401 ([-f|--failurelogfile] failurelogfile|[-l|--logfile] rawlogfile])
402 [-O|--outputprefix] outputprefix
406 =============== ============================================================
407 -b branch branch 1.8.0, 1.8.1, 1.9.0, all
408 -R repository CVS for 1.8.0, 1.8.1, 1.9.0 branches,
409 mercurial repository name for 1.9.1 and later branches
410 (\`basename http://hg.mozilla.org/repository\`)
411 -T buildtype build type opt, debug, all
412 -t testtype test type browser, shell, all
413 -o os operating system nt, darwin, linux, all
414 -K kernel kernel, all or a specific pattern
415 -A arch architecture, all or a specific pattern
416 -M memory memory in Gigabytes, all or a specific pattern
417 -z timezone -0400, -0700, etc. default to user\'s zone
418 -J jsoptions JavaScript options
419 -l rawlogfile raw logfile
420 -f failurelogfile failure logfile
421 -r patterns known failure patterns
422 -O outputprefix output files will be generated with this prefix
423 -D turn on debugging output
430 my ($option, $value);
432 Getopt::Mixed::init ($option_desc);
433 $Getopt::Mixed::order = $Getopt::Mixed::RETURN_IN_ORDER;
435 while (($option, $value) = nextOption()) {
437 if ($option eq "b") {
440 elsif ($option eq "R") {
443 elsif ($option eq "T") {
446 elsif ($option eq "t") {
449 elsif ($option eq "o") {
452 elsif ($option eq "K") {
455 elsif ($option eq "A") {
458 elsif ($option eq "M") {
461 elsif ($option eq "z") {
464 elsif ($option eq "J") {
471 $value =~ s/(-\w) (\w)/$1$2/g;
472 @s = sort split / /, $value;
474 $j =~ s/(-\w)(\w)/$1 $2/g;
478 elsif ($option eq "r") {
481 elsif ($option eq "l") {
482 $rawlogfile = $value;
484 elsif ($option eq "f") {
485 $failurelogfile = $value;
487 elsif ($option eq "O") {
488 $outputprefix = $value;
490 elsif ($option eq "D") {
497 print "branch=$branch, buildtype=$buildtype, testtype=$testtype, os=$os, kernel=$kernel, arch=$arch, memory=$memory, timezone=$timezone, jsoptions=$jsoptions, patterns=$patterns, rawlogfile=$rawlogfile failurelogfile=$failurelogfile, outputprefix=$outputprefix\n";
499 Getopt::Mixed::cleanup();
501 if ( !defined($branch) ) {
502 usage "missing branch";
505 if (!defined($buildtype)) {
506 usage "missing buildtype";
509 if (!defined($testtype)) {
510 usage "missing testtype";
517 if (!defined($memory)) {
521 if (!defined($timezone)) {
522 usage "missing timezone";
525 if (!defined($jsoptions)) {
529 if (!defined($patterns)) {
530 usage "missing patterns";
533 if (!defined($rawlogfile) && !defined($failurelogfile)) {
534 usage "missing logfile";
537 if (!defined($outputprefix)) {
538 usage "missing outputprefix";
541 if ($branch eq "all") {
542 $knownfailurebranchpattern = "[^,]*";
543 $failurebranchpattern = "[^,]*";
546 $knownfailurebranchpattern = "($branch|.*)";
547 $knownfailurebranchpattern =~ s/\./\\./g;
549 $failurebranchpattern = "$branch";
550 $failurebranchpattern =~ s/\./\\./g;
553 if ($repo eq "all" || $repo eq ".*") {
554 $knownfailurerepopattern = "[^,]*";
555 $failurerepopattern = "[^,]*";
558 $knownfailurerepopattern = "($repo|\\.\\*)";
559 $failurerepopattern = "$repo";
562 if ($buildtype eq "opt") {
563 $knownfailurebuildtypepattern = "(opt|\\.\\*)";
564 $failurebuildtypepattern = "opt";
566 elsif ($buildtype eq "debug") {
567 $knownfailurebuildtypepattern = "(debug|\\.\\*)";
568 $failurebuildtypepattern = "debug";
570 elsif ($buildtype eq "all") {
571 $knownfailurebuildtypepattern = "[^,]*";
572 $failurebuildtypepattern = "[^,]*";
575 if ($testtype eq "shell") {
576 $knownfailuretesttypepattern = "(shell|\\.\\*)";
577 $failuretesttypepattern = "shell";
579 elsif ($testtype eq "browser") {
580 $knownfailuretesttypepattern = "(browser|\\.\\*)";
581 $failuretesttypepattern = "browser";
583 elsif ($testtype eq "all") {
584 $knownfailuretesttypepattern = "[^,]*";
585 $failuretesttypepattern = "[^,]*";
589 $knownfailureospattern = "(nt|\\.\\*)";
590 $failureospattern = "nt";
592 elsif ($os eq "darwin") {
593 $knownfailureospattern = "(darwin|\\.\\*)";
594 $failureospattern = "darwin";
596 elsif ($os eq "linux") {
597 $knownfailureospattern = "(linux|\\.\\*)";
598 $failureospattern = "linux";
600 elsif ($os eq "all") {
601 $knownfailureospattern = "[^,]*";
602 $failureospattern = "[^,]*";
605 if ($kernel ne "all") {
606 $knownfailurekernelpattern = "(" . $kernel . "|\\.\\*)";
607 $failurekernelpattern = "$kernel";
610 $knownfailurekernelpattern = "[^,]*";
611 $failurekernelpattern = "[^,]*";
614 if ($arch ne "all") {
615 $knownfailurearchpattern = "(" . $arch . "|\\.\\*)";
616 $failurearchpattern = "$arch";
619 $knownfailurearchpattern = "[^,]*";
620 $failurearchpattern = "[^,]*";
623 if ($memory ne "all") {
624 $knownfailurememorypattern = "(" . $memory . "|\\.\\*)";
625 $failurememorypattern = "$memory";
628 $knownfailurememorypattern = "[^,]*";
629 $failurememorypattern = "[^,]*";
632 if ($timezone eq "all") {
633 $knownfailuretimezonepattern = "[^,]*";
634 $failuretimezonepattern = "[^,]*";
637 $knownfailuretimezonepattern = "(" . escape_string($timezone) . "|\\.\\*)";
638 $failuretimezonepattern = escape_string("$timezone");
641 if ($jsoptions eq "all") {
642 $knownfailurejsoptionspattern = "[^,]*";
643 $failurejsoptionspattern = "[^,]*";
646 $knownfailurejsoptionspattern = "(" . escape_string($jsoptions) . "|\\.\\*)";
647 $failurejsoptionspattern = escape_string("$jsoptions");
655 # replace unescaped regular expression characters in the
656 # string so they are not interpreted as regexp chars
657 # when matching descriptions. leave the escaped regexp chars
658 # `regexp` alone so they can be unescaped later and used in
665 # escape non word chars that aren't surrounded by ``
666 $s =~ s/(?<!`)([$regchars])(?!`)/\\$1/g;
667 $s =~ s/(?<!`)([$regchars])(?=`)/\\$1/g;
668 $s =~ s/(?<=`)([$regchars])(?!`)/\\$1/g;
670 # unquote the regchars
671 $s =~ s/\`([^\`])\`/$1/g;
673 debug "escape_string : $s";
685 my ($leading, $trailing) = $line =~ /(.*TEST_DESCRIPTION=)(.*)/;
687 # debug "escape_pattern: before: $leading$trailing";
689 $trailing = escape_string($trailing);
691 debug "escape_pattern : $leading$trailing";
693 return "$leading$trailing";
697 sub unescape_pattern {
702 my ($leading, $trailing) = $line =~ /(.*TEST_DESCRIPTION=)(.*)/;
704 # quote the unescaped non word chars
705 $trailing =~ s/(?<!\\)([$regchars])/`$1`/g;
707 # unescape the escaped non word chars
708 $trailing =~ s/\\([$regchars])/$1/g;
710 $trailing =~ s/\\\\/\\/g;
712 debug "unescape_pattern: after: $leading$trailing";
714 return "$leading$trailing";