4 # Test driver for the Make test suite
6 # Usage: run_make_tests [testname]
14 # Copyright (C) 1992-2022 Free Software Foundation, Inc.
15 # This file is part of GNU Make.
17 # GNU Make is free software; you can redistribute it and/or modify it under
18 # the terms of the GNU General Public License as published by the Free Software
19 # Foundation; either version 3 of the License, or (at your option) any later
22 # GNU Make is distributed in the hope that it will be useful, but WITHOUT ANY
23 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
24 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
27 # You should have received a copy of the GNU General Public License along with
28 # this program. If not, see <https://www.gnu.org/licenses/>.
30 # Add the working directory to @INC and load the test driver
32 use lib "$FindBin::Bin";
34 our $testsroot = $FindBin::Bin;
36 require "test_driver.pl";
42 ($cwdvol, $cwddir, $_) = File::Spec->splitpath($cwdpath, 1);
44 # Some target systems might not have the POSIX module...
45 $has_POSIX = eval { require "POSIX.pm" };
53 $valgrind = 0; # invoke make with valgrind
55 $memcheck_args = '--num-callers=15 --tool=memcheck --leak-check=full --suppressions=guile.supp';
56 $massif_args = '--num-callers=15 --tool=massif --alloc-fn=xmalloc --alloc-fn=xcalloc --alloc-fn=xrealloc --alloc-fn=xstrdup --alloc-fn=xstrndup';
59 # The location of the GNU make source directory
64 # The location of the build directory
81 $CMD_rmfile = 'rm -f';
83 # rmdir broken in some Perls on VMS.
86 require VMS::Filespec;
87 VMS::Filespec->import();
90 my $vms_file = vmspath($_[0]);
91 $vms_file = fileify($vms_file);
92 my $ret = unlink(vmsify($vms_file));
96 *CORE::GLOBAL::rmdir = \&vms_rmdir;
98 $CMD_rmfile = 'delete_file -no_ask';
103 # Find the strings that will be generated for various error codes.
104 # We want them from the C locale regardless of our current locale.
106 $ERR_no_such_file = undef;
107 $ERR_read_only_file = undef;
108 $ERR_unreadable_file = undef;
109 $ERR_nonexe_file = undef;
110 $ERR_exe_dir = undef;
111 $ERR_command_not_found = undef;
113 #$SIG{INT} = sub { print STDERR "Caught a signal!\n"; die @_; };
119 if ($option =~ /^-make([-_]?path)?$/i) {
120 $make_path = shift @argv;
121 if (! -f $make_path) {
122 print "$option $make_path: Not found.\n";
128 if ($option =~ /^-srcdir$/i) {
129 $srcdir = shift @argv;
130 if (! -f File::Spec->catfile($srcdir, 'src', 'gnumake.h')) {
131 print "$option $srcdir: Not a valid GNU make source directory.\n";
137 if ($option =~ /^-all([-_]?tests)?$/i) {
142 if ($option =~ /^-(valgrind|memcheck)$/i) {
144 $valgrind_args = $memcheck_args;
148 if ($option =~ /^-massif$/i) {
150 $valgrind_args = $massif_args;
154 # This doesn't work--it _should_! Someone badly needs to fix this.
156 # elsif ($option =~ /^-work([-_]?dir)?$/)
158 # $workdir = shift @argv;
166 # This is an "all-in-one" function. Arguments are as follows:
168 # [0] (string): The makefile to be tested. undef means use the last one.
169 # [1] (string): Arguments to pass to make.
170 # [2] (string): Answer we should get back.
171 # [3] (integer): Exit code we expect. A missing code means 0 (success)
174 $old_makefile = undef;
179 sub subst_make_string
182 $makefile and s/#MAKEFILE#/$makefile/g;
183 s/#MAKEPATH#/$mkpath/g;
184 s/#MAKE#/$make_name/g;
185 s/#PERL#/$perl_name/g;
188 # If we're using a shell
189 s/#HELPER#/$perl_name $helptool/g;
195 local ($makestring, $options, $answer, $err_code, $timeout) = @_;
198 # If the user specified a makefile string, create a new makefile to contain
199 # it. If the first value is not defined, use the last one (if there is
202 if (! defined $makestring) {
203 defined $old_makefile
204 or die "run_make_test(undef) invoked before run_make_test('...')\n";
205 $makefile = $old_makefile;
206 } elsif ($makestring) {
207 if (! defined($makefile)) {
208 $makefile = &get_tmpfile();
211 # Make sure it ends in a newline and substitute any special tokens.
212 $makestring && $makestring !~ /\n$/s and $makestring .= "\n";
213 $makestring = subst_make_string($makestring);
215 # Populate the makefile!
216 open(MAKEFILE, "> $makefile") or die "Failed to open $makefile: $!\n";
217 print MAKEFILE $makestring;
218 close(MAKEFILE) or die "Failed to write $makefile: $!\n";
221 # Do the same processing on $answer as we did on $makestring.
222 if (defined $answer) {
223 $answer && $answer !~ /\n$/s and $answer .= "\n";
224 $answer = subst_make_string($answer);
227 run_make_with_options($makefile, $options, &get_logfile(0),
228 $err_code, $timeout, @call);
229 &compare_output($answer, &get_logfile(1));
231 $old_makefile = $makefile;
240 push(@$cmd, ref($_) ? @$_ : $_);
242 $cmd .= ' '.(ref($_) ? "@$_" : $_);
250 return !$_[0] || ref($_[0]) ? [@make_command] : join(' ', @make_command);
253 # The old-fashioned way...
254 # $options can be a scalar (string) or a ref to an array of options
255 # If it's a scalar the entire argument is passed to system/exec etc. as
256 # a single string. If it's a ref then the array is passed to system/exec.
257 # Using a ref should be preferred as it's more portable but all the older
258 # invocations use strings.
259 sub run_make_with_options {
260 my ($filename, $options, $logname, $expected_code, $timeout, @call) = @_;
261 @call = caller unless @call;
263 my $command = create_command($options);
265 $expected_code = 0 unless defined($expected_code);
267 # Reset to reflect this one test.
271 $command = add_options($command, '-f', $filename);
275 if (!ref($options) && $^O eq 'VMS') {
276 # Try to make sure arguments are properly quoted.
277 # This does not handle all cases.
278 # We should convert the tests to use array refs not strings
280 # VMS uses double quotes instead of single quotes.
281 $options =~ s/\'/\"/g;
283 # If the leading quote is inside non-whitespace, then the
284 # quote must be doubled, because it will be enclosed in another
286 $options =~ s/(\S)(\".*\")/$1\"$2\"/g;
288 # Options must be quoted to preserve case if not already quoted.
289 $options =~ s/(\S+)/\"$1\"/g;
291 # Special fixup for embedded quotes.
292 $options =~ s/(\"\".+)\"(\s+)\"(.+\"\")/$1$2$3/g;
294 $options =~ s/(\A)(?:\"\")(.+)(?:\"\")/$1\"$2\"/g;
296 # Special fixup for misc/general4 test.
297 $options =~ s/""\@echo" "cc""/\@echo cc"/;
298 $options =~ s/"\@echo link"""/\@echo link"/;
300 # Remove shell escapes expected to be removed by bash
301 if ($options !~ /path=pre/) {
305 # special fixup for options/eval
306 $options =~ s/"--eval=\$\(info" "eval/"--eval=\$\(info eval/;
308 print ("Options fixup = -$options-\n") if $debug;
311 $command = add_options($command, $options);
314 my $cmdstr = cmd2str($command);
315 $command_string = "# $call[1]:$call[2]:\n$cmdstr\n";
318 print VALGRIND "\n\nExecuting: $cmdstr\n";
322 my $old_timeout = $test_timeout;
323 $timeout and $test_timeout = $timeout;
325 # If valgrind is enabled, turn off the timeout check
326 $valgrind and $test_timeout = 0;
329 $code = run_command_with_output($logname, @$command);
331 $code = run_command_with_output($logname, $command);
333 $test_timeout = $old_timeout;
336 # Check to see if we have Purify errors. If so, keep the logfile.
337 # For this to work you need to build with the Purify flag -exit-status=yes
339 if ($pure_log && -f $pure_log) {
340 if ($code & 0x7000) {
343 # If we have a purify log, save it
344 $tn = $pure_testname . ($num_of_logfiles ? ".$num_of_logfiles" : "");
345 print("Renaming purify log file to $tn\n") if $debug;
346 rename($pure_log, "$tn") or die "Can't rename $pure_log to $tn: $!\n";
353 if ($code != $expected_code) {
354 print "Error running @make_command (expected $expected_code; got $code)\n$call[1]:$call[2]: $cmdstr\n";
356 &create_file(get_runfile(), $command_string);
357 # If it's a SIGINT, stop here
359 print STDERR "\nCaught signal ".($code & 127)."!\n";
360 ($code & 127) == 2 and exit($code);
365 if ($profile & $vos) {
366 system "add_profile @make_command";
374 &print_standard_usage ("run_make_tests",
375 "[-make MAKE_PATHNAME] [-srcdir SRCDIR] [-memcheck] [-massif]",);
380 &print_standard_help (
382 "\tYou may specify the pathname of the copy of make to run.",
384 "\tSpecify the make source directory.",
387 "\tRun the test suite under valgrind's memcheck tool.",
388 "\tChange the default valgrind args with the VALGRIND_ARGS env var.",
390 "\tRun the test suite under valgrind's massif tool.",
391 "\tChange the default valgrind args with the VALGRIND_ARGS env var."
398 $testee = "GNU Make";
400 $tmpfilesuffix = "mk";
401 if ($port_type eq 'UNIX') {
402 $scriptsuffix = '.sh';
403 } elsif ($port_type eq 'VMS') {
404 $scriptsuffix = '.com';
406 $scriptsuffix = '.bat';
409 $ENV{LC_ALL} = $makeENV{LC_ALL};
410 $ENV{LANG} = $makeENV{LANG};
411 $ENV{LANGUAGE} = $makeENV{LANGUAGE};
417 POSIX->import(qw(locale_h));
418 # Windows has POSIX locale, but only LC_ALL not LC_MESSAGES
419 $loc = POSIX::setlocale(&POSIX::LC_ALL);
420 POSIX::setlocale(&POSIX::LC_ALL, 'C');
423 if (open(my $F, '<', 'file.none')) {
424 print "Opened non-existent file! Skipping related tests.\n";
426 $ERR_no_such_file = "$!";
432 chmod(0444, 'file.out');
433 if (open(my $F, '>', 'file.out')) {
434 print "Opened read-only file! Skipping related tests.\n";
437 $ERR_read_only_file = "$!";
440 $_ = `./file.out 2>&1`;
442 print "Executed non-executable file! Skipping related tests.\n";
444 $ERR_nonexe_file = "$!";
447 if ($^O =~ /cygwin/i) {
448 # For some reason the execute here gives a different answer than make's
449 print "Skipping directory execution on $^O\n";
453 print "Executed directory! Skipping related tests.\n";
459 chmod(0000, 'file.out');
460 if (open(my $F, '<', 'file.out')) {
461 print "Opened unreadable file! Skipping related tests.\n";
464 $ERR_unreadable_file = "$!";
467 unlink('file.out') or die "Failed to delete file.out: $!\n";
469 $_ = `/bin/sh -c 'bad-command 2>&1'`;
471 print "Invoked invalid file! Skipping related tests.\n";
474 s/bad-command/#CMDNAME#/g;
475 $ERR_command_not_found = $_;
478 $loc and POSIX::setlocale(&POSIX::LC_ALL, $loc);
480 $ENV{LC_ALL} = $origENV{LC_ALL};
481 $ENV{LANG} = $origENV{LANG};
482 $ENV{LANGUAGE} = $origENV{LANGUAGE};
485 # This is no longer used: we import config-flags.pm instead
488 # if (open(my $fh, '<', "$_[0]/config.status")) {
489 # while (my $line = <$fh>) {
490 # $line =~ m/^[SD]\["([^\"]+)"\]=" *(.*)"/ and $CONFIG_FLAGS{$1} = $2;
500 my ($v, $d, $f) = File::Spec->splitpath($prog);
502 # If there's no directory then we need to search the PATH
504 foreach my $e (File::Spec->path()) {
505 $prog = File::Spec->catfile($e, $f);
507 ($v, $d, $f) = File::Spec->splitpath($prog);
518 return exists($CONFIG_FLAGS{$_[0]}) ? $CONFIG_FLAGS{$_[0]} : '';
521 sub set_more_defaults
525 # Now that we have located make_path, locate the srcdir and blddir
526 my ($mpv, $mpd, $mpf) = find_prog($make_path);
528 # We have a make program so try to compute the blddir.
530 my $f = File::Spec->catpath($mpv, File::Spec->catdir($mpd, 'tests'), 'config-flags.pm');
537 # If srcdir wasn't provided on the command line, try to find it.
538 if (! $srcdir && $blddir) {
539 # See if the blddir is the srcdir
540 my $f = File::Spec->catpath($bldvol, File::Spec->catdir($blddir, 'src'), 'gnumake.h');
548 # Not found, see if our parent is the source dir
549 my $f = File::Spec->catpath($cwdvol, File::Spec->catdir(File::Spec->updir(), 'src'), 'gnumake.h');
551 $srcdir = File::Spec->updir();
556 # If we have srcdir but not blddir, set them equal
557 if ($srcdir && !$blddir) {
562 # Load the config flags
564 warn "Cannot locate config-flags.pm (no blddir)\n";
566 my $f = File::Spec->catpath($bldvol, File::Spec->catdir($blddir, 'tests'), 'config-flags.pm');
568 warn "Cannot locate $f\n";
570 unshift(@INC, File::Spec->catpath($bldvol, File::Spec->catdir($blddir, 'tests'), ''));
571 require "config-flags.pm";
575 # Find the full pathname of Make. For DOS systems this is more
576 # complicated, so we ask make itself.
577 if ($osname eq 'VMS') {
578 $port_type = 'VMS-DCL' unless defined $ENV{"SHELL"};
579 # On VMS pre-setup make to be found with simply 'make'.
582 create_file('make.mk', 'all:;$(info $(MAKE))');
583 my $mk = `$make_path -sf make.mk`;
586 $mk or die "FATAL ERROR: Cannot determine the value of \$(MAKE)\n";
589 ($mpv, $mpd, $mpf) = File::Spec->splitpath($make_path);
591 # Ask make what shell to use
592 create_file('shell.mk', 'all:;$(info $(SHELL))');
593 $sh_name = `$make_path -sf shell.mk`;
595 $sh_name =~ s/\r?\n$//;
597 print "Cannot determine shell\n";
600 my $o = `$sh_name -c ': do nothing' 2>&1`;
601 $is_posix_sh = $? == 0 && $o eq '';
604 $string = `$make_path -v`;
605 $string =~ /^(GNU Make [^,\n]*)/ or die "$make_path is not GNU make. Version:\n$string";
606 $testee_version = "$1\n";
608 create_file('null.mk', '');
611 $redir = '' if os_name eq 'VMS';
612 $string = `$make_path -f null.mk $redir`;
613 if ($string =~ /(.*): \*\*\* No targets\. Stop\./) {
619 # prepend pwd if this is a relative path (ie, does not
620 # start with a slash, but contains one). Thanks for the
623 if ($mpd && !File::Spec->file_name_is_absolute($make_path) && $cwdvol == $mpv) {
624 $mkpath = File::Spec->catpath($cwdvol, File::Spec->catdir($cwd, $mpd), $mpf);
626 $mkpath = $make_path;
629 # Not with the make program, so see if we can get it out of the makefile
630 if (! $srcdir && open(MF, '<', File::Spec->catfile(File::Spec->updir(), 'Makefile'))) {
634 /^abs_srcdir\s*=\s*(.*?)\s*$/m;
635 -f File::Spec->catfile($1, 'src', 'gnumake.h') and $srcdir = $1;
638 # At this point we should have srcdir and blddir: get fq versions
639 $fqsrcdir = File::Spec->rel2abs($srcdir);
640 $fqblddir = File::Spec->rel2abs($blddir);
642 # Find the helper tool
643 $helptool = File::Spec->catfile($fqsrcdir, 'tests', 'thelp.pl');
645 # It's difficult to quote this properly in all the places it's used so
646 # ensure it doesn't need to be quoted.
647 $helptool =~ s,\\,/,g if $port_type eq 'W32';
648 $helptool =~ s, ,\\ ,g;
650 # Get Purify log info--if any.
652 if (exists $ENV{PURIFYOPTIONS}
653 && $ENV{PURIFYOPTIONS} =~ /.*-logfile=([^ ]+)/) {
654 $pure_log = $1 || '';
655 $pure_log =~ s/%v/$make_name/;
659 $string = `$make_path -j 2 -f null.mk $redir`;
660 if ($string =~ /not supported/) {
669 create_file('features.mk', 'all:;$(info $(.FEATURES))');
670 %FEATURES = map { $_ => 1 } split /\s+/, `$make_path -sf features.mk`;
671 unlink('features.mk');
673 # Find the default values for different built-in variables
675 foreach (keys %DEFVARS) {
676 $s .= "\$(info $_=\$($_))\n";
678 create_file('defvars.mk', $s);
679 foreach (split "\n", `$make_path -sf defvars.mk`) {
680 my @e = split /=/, $_, 2;
681 $DEFVARS{$e[0]} = $e[1];
683 unlink('defvars.mk');
685 # Set up for valgrind, if requested.
687 @make_command = ($make_path);
690 my $args = $valgrind_args;
691 open(VALGRIND, "> valgrind.out") or die "Cannot open valgrind.out: $!\n";
692 # -q --leak-check=yes
693 exists $ENV{VALGRIND_ARGS} and $args = $ENV{VALGRIND_ARGS};
694 @make_command = ('valgrind', '--log-fd='.fileno(VALGRIND));
695 push(@make_command, split(' ', $args));
696 push(@make_command, $make_path);
698 fcntl(VALGRIND, 2, 0) or die "fcntl(setfd) failed: $!\n";
699 system("echo Starting on `date` 1>&".fileno(VALGRIND));
700 print "Enabled valgrind support.\n";
704 print "Port type: $port_type\n";
705 print "Make command: @make_command\n";
706 print "Shell path: $sh_name".($is_posix_sh ? ' (POSIX)' : '')."\n";
707 print "#PWD#: $cwdpath\n";
708 print "#PERL#: $perl_name\n";
709 print "#MAKEPATH#: $mkpath\n";
710 print "#MAKE#: $make_name\n";
716 $makefile = &get_tmpfile;
721 # Get rid of any Purify logs.
723 ($pure_testname = $testname) =~ tr,/,_,;
724 $pure_testname = "$pure_log.$pure_testname";
725 system("rm -f $pure_testname*");
726 print("Purify testfiles are: $pure_testname*\n") if $debug;