2008-10-26 Ralf Wildenhues <Ralf.Wildenhues@gmx.de>
+ Parallel automake --add-missing: serialized file installs.
+ * automake.in (QUEUE_CONF_FILE, QUEUE_LOCATION, QUEUE_STRING):
+ New serialization keys.
+ ($required_conf_file_queue): New file global.
+ (queue_required_conf_file, require_queued_conf_file): New
+ functions, to queue and dequeue requirements for aux dir files.
+ (require_conf_file): Enqueue if needed.
+ (get_number_of_threads): Can do threads with --add-missing now.
+ (handle_makefiles_threaded): Let worker threads enqueue, let
+ master attend to queued requirements at the right time.
+ * tests/parallel-am.test: Explain the purpose of the include
+ chain used here.
+ * tests/parallel-am2.test: Also cope with --add-missing.
+ * tests/parallel-am3.test: New test, test absence of races with
+ concurrent same-file installs stemming from --add-missing.
+ * tests/Makefile.am: Adjust.
+
Implement serialization for Locations.
* lib/Automake/Location.pm (serialize, deserialize): New
functions. They allows to serialize a Location in an array, and
# Serialization keys for message queues.
use constant {
QUEUE_MESSAGE => "msg",
+ QUEUE_CONF_FILE => "conf file",
+ QUEUE_LOCATION => "location",
+ QUEUE_STRING => "string"
};
\f
}
}
+# Queue to push require_conf_file requirements to.
+my $required_conf_file_queue;
+
+# &queue_required_conf_file ($QUEUE, $KEY, $DIR, $WHERE, $MYSTRICT, @FILES)
+# -------------------------------------------------------------------------
+sub queue_required_conf_file ($$$$@)
+{
+ my ($queue, $key, $dir, $where, $mystrict, @files) = @_;
+ my @serial_loc;
+ if (ref $where)
+ {
+ @serial_loc = (QUEUE_LOCATION, $where->serialize ());
+ }
+ else
+ {
+ @serial_loc = (QUEUE_STRING, $where);
+ }
+ $queue->enqueue ($key, $dir, @serial_loc, $mystrict, 0 + @files, @files);
+}
+
+# &require_queued_conf_file ($QUEUE)
+# ----------------------------------
+sub require_queued_conf_file ($)
+{
+ my ($queue) = @_;
+ my $where;
+ my $dir = $queue->dequeue ();
+ my $loc_key = $queue->dequeue ();
+ if ($loc_key eq QUEUE_LOCATION)
+ {
+ $where = Automake::Location::deserialize ($queue);
+ }
+ elsif ($loc_key eq QUEUE_STRING)
+ {
+ $where = $queue->dequeue ();
+ }
+ else
+ {
+ prog_error "unexpected key $loc_key";
+ }
+ my $mystrict = $queue->dequeue ();
+ my $nfiles = $queue->dequeue ();
+ my @files;
+ push @files, $queue->dequeue ()
+ foreach (1 .. $nfiles);
+
+ # Dequeuing happens outside of per-makefile context, so we have to
+ # set the variables used by require_file_internal and the functions
+ # it calls. Gross!
+ $relative_dir = $dir;
+ require_file_internal ($where, $mystrict, $config_aux_dir, @files);
+}
+
# &require_conf_file ($WHERE, $MYSTRICT, @FILES)
# ----------------------------------------------
-# Looks in configuration path, as specified by AC_CONFIG_AUX_DIR.
+# Looks in configuration path, as specified by AC_CONFIG_AUX_DIR;
+# worker threads may queue up the action to be serialized by the master.
+#
+# FIXME: this seriously relies on the semantics of require_file_internal
+# and maybe_push_required_file, in that we exploit the fact that only the
+# contents of the last handled output file may be impacted (which in turn
+# is dealt with by the master thread).
sub require_conf_file ($$@)
{
my ($where, $mystrict, @files) = @_;
- require_file_internal ($where, $mystrict, $config_aux_dir, @files);
+ if (defined $required_conf_file_queue)
+ {
+ queue_required_conf_file ($required_conf_file_queue, QUEUE_CONF_FILE,
+ $relative_dir, $where, $mystrict, @files);
+ }
+ else
+ {
+ require_file_internal ($where, $mystrict, $config_aux_dir, @files);
+ }
}
{
$nthreads = $max_threads;
}
-
- # We cannot deal with --add-missing (yet).
- $nthreads = 0
- if ($add_missing);
-
return $nthreads;
}
# worker threads push back everything that needs serialization:
# * warning and (normal) error messages, for stable stderr output
# order and content (avoiding duplicates, for example),
+# * races when installing aux files (and respective messages),
# * races when collecting aux files for distribution.
#
# The latter requires that the makefile that deals with the aux dir
verb "handling $file";
my $queue = $msg_queues{$file};
setup_channel_queue ($queue, QUEUE_MESSAGE);
+ $required_conf_file_queue = $queue;
handle_makefile ($file);
$queue->enqueue (undef);
setup_channel_queue (undef, undef);
+ $required_conf_file_queue = undef;
}
return $exit_code;
});
{
pop_channel_queue ($queue);
}
+ elsif ($key eq QUEUE_CONF_FILE)
+ {
+ require_queued_conf_file ($queue);
+ }
else
{
prog_error "unexpected key $key";
--- /dev/null
+#! /bin/sh
+# Copyright (C) 2008 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test parallel automake execution.
+
+# This tests:
+# 3) normal automake output should be identical and ordered in the same way
+# with --add-missing, even with concurrent file requirements, and the
+# installation of aux files should be race-free,
+
+. ./defs || Exit 1
+
+set -e
+
+cat > configure.in << 'END'
+AC_INIT([parallel-am], [1.0])
+AC_CONFIG_AUX_DIR([build-aux])
+AM_INIT_AUTOMAKE
+AC_PROG_CC
+AM_PATH_LISPDIR
+AM_PATH_PYTHON
+AC_CONFIG_FILES([Makefile])
+END
+
+cat > Makefile.am << 'END'
+SUBDIRS =
+END
+
+list='1 2 3'
+for i in $list; do
+ echo "AC_CONFIG_FILES([sub$i/Makefile])" >> configure.in
+ echo "SUBDIRS += sub$i" >> Makefile.am
+ mkdir sub$i
+ cat > sub$i/Makefile.am <<END
+python_PYTHON = foo$i.py
+lisp_LISP = foo$i.el
+bin_PROGRAMS = p$i
+END
+done
+
+rm -f install-sh missing depcomp
+mkdir build-aux
+
+$ACLOCAL
+
+# Generate expected output using the non-threaded code.
+unset AUTOMAKE_JOBS
+AUTOMAKE_run 0 --add-missing
+mv stderr expected
+mv Makefile.in Makefile.in.exp
+
+AUTOMAKE_JOBS=3
+export AUTOMAKE_JOBS
+
+for run in 1 2 3 4 5 6 7; do
+ rm -f build-aux/* sub*/Makefile.in
+ AUTOMAKE_run 0 --add-missing
+ diff stderr expected
+ diff Makefile.in Makefile.in.exp
+done
+
+: