/doc/manual.html
/doc/doxygen
/gtest-1.6.0
+
+# Eclipse project files
+.project
+.cproject
+
### Via mingw on Linux (not well supported)
+Setup on Ubuntu Lucid:
* `sudo apt-get install gcc-mingw32 wine`
* `export CC=i586-mingw32msvc-cc CXX=i586-mingw32msvc-c++ AR=i586-mingw32msvc-ar`
+
+Setup on Ubuntu Precise:
+* `sudo apt-get install gcc-mingw-w64-i686 g++-mingw-w64-i686 wine`
+* `export CC=i686-w64-mingw32-gcc CXX=i686-w64-mingw32-g++ AR=i686-w64-mingw32-ar`
+
+Then run:
* `./configure.py --platform=mingw --host=linux`
* Build `ninja.exe` using a Linux ninja binary: `/path/to/linux/ninja`
* Run: `./ninja.exe` (implicitly runs through wine(!))
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import print_function
+
from optparse import OptionParser
import sys
import os
help='enable verbose build',)
parser.add_option('--x64', action='store_true',
help='force 64-bit build (Windows)',)
+# TODO: make this --platform to match configure.py.
+parser.add_option('--windows', action='store_true',
+ help='force native Windows build (when using Cygwin Python)',
+ default=sys.platform.startswith('win32'))
(options, conf_args) = parser.parse_args()
def run(*args, **kwargs):
cflags.append('-I/usr/local/include')
ldflags.append('-L/usr/local/lib')
-print 'Building ninja manually...'
+print('Building ninja manually...')
try:
os.mkdir('build')
-except OSError, e:
+except OSError:
+ e = sys.exc_info()[1]
if e.errno != errno.EEXIST:
raise
if filename == 'browse.cc': # Depends on generated header.
continue
- if sys.platform.startswith('win32'):
+ if options.windows:
if src.endswith('-posix.cc'):
continue
else:
sources.append(src)
-if sys.platform.startswith('win32'):
+if options.windows:
sources.append('src/getopt.c')
vcdir = os.environ.get('VCINSTALLDIR')
cflags.extend(['-Wno-deprecated',
'-DNINJA_PYTHON="' + sys.executable + '"',
'-DNINJA_BOOTSTRAP'])
- if sys.platform.startswith('win32'):
+ if options.windows:
cflags.append('-D_WIN32_WINNT=0x0501')
if options.x64:
cflags.append('-m64')
args.extend(cflags)
args.extend(ldflags)
binary = 'ninja.bootstrap'
-if sys.platform.startswith('win32'):
+if options.windows:
binary = 'ninja.bootstrap.exe'
args.extend(sources)
if vcdir:
args.extend(['-o', binary])
if options.verbose:
- print ' '.join(args)
+ print(' '.join(args))
-run(args)
+try:
+ run(args)
+except:
+ print('Failure running:', args)
+ raise
verbose = []
if options.verbose:
verbose = ['-v']
-if sys.platform.startswith('win32'):
- print 'Building ninja using itself...'
+if options.windows:
+ print('Building ninja using itself...')
run([sys.executable, 'configure.py', '--with-ninja=%s' % binary] +
conf_args)
run(['./' + binary] + verbose)
for obj in glob.glob('*.obj'):
os.unlink(obj)
- print """
+ print("""
Done!
Note: to work around Windows file locking, where you can't rebuild an
in-use binary, to run ninja after making any changes to build ninja itself
you should run ninja.bootstrap instead. Your build is also configured to
use ninja.bootstrap.exe as the MSVC helper; see the --with-ninja flag of
-the --help output of configure.py."""
+the --help output of configure.py.""")
else:
- print 'Building ninja using itself...'
+ print('Building ninja using itself...')
run([sys.executable, 'configure.py'] + conf_args)
run(['./' + binary] + verbose)
os.unlink(binary)
- print 'Done!'
+ print('Done!')
Projects that use ninja themselves should either write a similar script
or use a meta-build system that supports Ninja output."""
+from __future__ import print_function
+
from optparse import OptionParser
import os
import sys
default="ninja")
(options, args) = parser.parse_args()
if args:
- print 'ERROR: extra unparsed command-line arguments:', args
+ print('ERROR: extra unparsed command-line arguments:', args)
sys.exit(1)
platform = options.platform
'-fno-rtti',
'-fno-exceptions',
'-fvisibility=hidden', '-pipe',
+ '-Wno-missing-field-initializers',
'-DNINJA_PYTHON="%s"' % options.with_python]
if options.debug:
cflags += ['-D_GLIBCXX_DEBUG', '-D_GLIBCXX_DEBUG_PEDANTIC']
libs.append('-lprofiler')
def shell_escape(str):
- """Escape str such that it's interpreted as a single argument by the shell."""
+ """Escape str such that it's interpreted as a single argument by
+ the shell."""
+
# This isn't complete, but it's just enough to make NINJA_PYTHON work.
if platform in ('windows', 'mingw'):
return str
n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc'))
n.build(src('lexer.cc'), 're2c', src('lexer.in.cc'))
else:
- print ("warning: A compatible version of re2c (>= 0.11.3) was not found; "
+ print("warning: A compatible version of re2c (>= 0.11.3) was not found; "
"changes to src/*.in.cc will not affect your build.")
n.newline()
'util']:
objs += cxx(name)
if platform in ('mingw', 'windows'):
- objs += cxx('subprocess-win32')
+ for name in ['subprocess-win32',
+ 'includes_normalize-win32',
+ 'msvc_helper-win32',
+ 'msvc_helper_main-win32']:
+ objs += cxx(name)
if platform == 'windows':
- objs += cxx('includes_normalize-win32')
- objs += cxx('msvc_helper-win32')
- objs += cxx('msvc_helper_main-win32')
objs += cxx('minidump-win32')
objs += cc('getopt')
else:
n.comment('Tests all build into ninja_test executable.')
variables = []
-test_cflags = None
+test_cflags = cflags[:]
test_ldflags = None
test_libs = libs
objs = []
os.path.join(path, 'src', 'gtest_main.cc'),
variables=[('cflags', gtest_cflags)])
- test_cflags = cflags + ['-DGTEST_HAS_RTTI=0',
- '-I%s' % os.path.join(path, 'include')]
+ test_cflags.append('-I%s' % os.path.join(path, 'include'))
elif platform == 'windows':
test_libs.extend(['gtest_main.lib', 'gtest.lib'])
else:
+ test_cflags.append('-DGTEST_HAS_RTTI=0')
test_libs.extend(['-lgtest_main', '-lgtest'])
+if test_cflags == cflags:
+ test_cflags = None
+
+n.variable('test_cflags', test_cflags)
for name in ['build_log_test',
'build_test',
'clean_test',
'subprocess_test',
'test',
'util_test']:
- objs += cxx(name, variables=[('cflags', test_cflags)])
-if platform == 'windows':
+ objs += cxx(name, variables=[('cflags', '$test_cflags')])
+if platform in ('windows', 'mingw'):
for name in ['includes_normalize_test', 'msvc_helper_test']:
objs += cxx(name, variables=[('cflags', test_cflags)])
all_targets += ninja_test
-n.comment('Ancilliary executables.')
+n.comment('Ancillary executables.')
objs = cxx('parser_perftest')
all_targets += n.build(binary('parser_perftest'), 'link', objs,
implicit=ninja_lib, variables=[('libs', libs)])
if host == 'linux':
n.comment('Packaging')
n.rule('rpmbuild',
- command="rpmbuild \
- --define 'ver git' \
- --define \"rel `git rev-parse --short HEAD`\" \
- --define '_topdir %(pwd)/rpm-build' \
- --define '_builddir %{_topdir}' \
- --define '_rpmdir %{_topdir}' \
- --define '_srcrpmdir %{_topdir}' \
- --define '_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm' \
- --define '_specdir %{_topdir}' \
- --define '_sourcedir %{_topdir}' \
- --quiet \
- -bb misc/packaging/ninja.spec",
- description='Building RPM..')
- n.build('rpm', 'rpmbuild',
- implicit=['ninja','README', 'COPYING', doc('manual.html')])
+ command="misc/packaging/rpmbuild.sh",
+ description='Building rpms..')
+ n.build('rpm', 'rpmbuild')
n.newline()
n.build('all', 'phony', all_targets)
-print 'wrote %s.' % BUILD_FILENAME
+print('wrote %s.' % BUILD_FILENAME)
Several placeholders are available:
* `%s`: The number of started edges.
* `%t`: The total number of edges that must be run to complete the build.
+* `%p`: The percentage of started edges.
* `%r`: The number of currently running edges.
* `%u`: The number of remaining edges to start.
* `%f`: The number of finished edges.
statement and it is out of date, Ninja will rebuild and reload it
before building the targets requested by the user.
+Pools
+~~~~~
+
+Pools allow you to allocate one or more rules or edges a finite number
+of concurrent jobs which is more tightly restricted than the default
+parallelism.
+
+This can be useful, for example, to restrict a particular expensive rule
+(like link steps for huge executables), or to restrict particular build
+statements which you know perform poorly when run concurrently.
+
+Each pool has a `depth` variable which is specified in the build file.
+The pool is then referred to with the `pool` variable on either a rule
+or a build statement.
+
+No matter what pools you specify, ninja will never run more concurrent jobs
+than the default parallelism, or the number of jobs specified on the command
+line (with -j).
+
+----------------
+# No more than 4 links at a time.
+pool link_pool
+ depth = 4
+
+# No more than 1 heavy object at a time.
+pool heavy_object_pool
+ depth = 1
+
+rule link
+ ...
+ pool = link_pool
+
+rule cc
+ ...
+
+# The link_pool is used here. Only 4 links will run concurrently.
+build foo.exe: link input.obj
+
+# A build statement can be exempted from its rule's pool by setting an
+# empty pool. This effectively puts the build statement back into the default
+# pool, which has infinite depth.
+build other.exe: link input.obj
+ pool =
+
+# A build statement can specify a pool directly.
+# Only one of these builds will run at a time.
+build heavy_object1.obj: cc heavy_obj1.cc
+ pool = heavy_object_pool
+build heavy_object2.obj: cc heavy_obj2.cc
+ pool = heavy_object_pool
+
+----------------
+
Generating Ninja files from code
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# . path/to/ninja/misc/bash-completion
_ninja_target() {
- local cur targets
+ local cur targets dir line targets_command OPTIND
cur="${COMP_WORDS[COMP_CWORD]}"
- targets=$((ninja -t targets all 2>/dev/null) | awk -F: '{print $1}')
+ dir="."
+ line=$(echo ${COMP_LINE} | cut -d" " -f 2-)
+ while getopts C: opt "${line[@]}"; do
+ case $opt in
+ C) dir="$OPTARG" ;;
+ esac
+ done;
+ targets_command="ninja -C ${dir} -t targets all"
+ targets=$((${targets_command} 2>/dev/null) | awk -F: '{print $1}')
COMPREPLY=($(compgen -W "$targets" -- "$cur"))
return 0
}
(setq ninja-keywords
(list
'("^#.*" . font-lock-comment-face)
- (cons (concat "^" (regexp-opt '("rule" "build" "subninja" "include")
+ (cons (concat "^" (regexp-opt '("rule" "build" "subninja" "include"
+ "pool" "default")
'words))
font-lock-keyword-face)
'("\\([[:alnum:]_]+\\) =" . (1 font-lock-variable-name-face))
" ninja build file syntax.
" Language: ninja build file as described at
" http://martine.github.com/ninja/manual.html
-" Version: 1.2
-" Last Change: 2012/06/01
+" Version: 1.3
+" Last Change: 2012/12/14
" Maintainer: Nicolas Weber <nicolasweber@gmx.de>
" Version 1.2 of this script is in the upstream vim repository and will be
" included in the next vim release. If you change this, please send your change
" lexer.in.cc, ReadToken() and manifest_parser.cc, Parse()
syn match ninjaKeyword "^build\>"
syn match ninjaKeyword "^rule\>"
+syn match ninjaKeyword "^pool\>"
syn match ninjaKeyword "^default\>"
syn match ninjaKeyword "^include\>"
syn match ninjaKeyword "^subninja\>"
" let assignments.
" manifest_parser.cc, ParseRule()
syn region ninjaRule start="^rule" end="^\ze\S" contains=ALL transparent
-syn keyword ninjaRuleCommand contained command depfile description generator restat
+syn keyword ninjaRuleCommand contained command depfile description generator
+ \ pool restat rspfile rspfile_content
+
+syn region ninjaPool start="^pool" end="^\ze\S" contains=ALL transparent
+syn keyword ninjaPoolCommand contained depth
" Strings are parsed as follows:
" lexer.in.cc, ReadEvalString()
hi def link ninjaComment Comment
hi def link ninjaKeyword Keyword
hi def link ninjaRuleCommand Statement
+hi def link ninjaPoolCommand Statement
hi def link ninjaWrapLineOperator ninjaOperator
hi def link ninjaOperator Operator
hi def link ninjaSimpleVar ninjaVar
value = ' '.join(filter(None, value)) # Filter out empty strings.
self._line('%s = %s' % (key, value), indent)
+ def pool(self, name, depth):
+ self._line('pool %s' % name)
+ self.variable('depth', depth, indent=1)
+
def rule(self, name, command, description=None, depfile=None,
- generator=False, restat=False, rspfile=None, rspfile_content=None):
+ generator=False, pool=None, restat=False, rspfile=None,
+ rspfile_content=None):
self._line('rule %s' % name)
self.variable('command', command, indent=1)
if description:
self.variable('depfile', depfile, indent=1)
if generator:
self.variable('generator', '1', indent=1)
+ if pool:
+ self.variable('pool', pool, indent=1)
if restat:
self.variable('restat', '1', indent=1)
if rspfile:
all_inputs.append('||')
all_inputs.extend(order_only)
- self._line('build %s: %s %s' % (' '.join(out_outputs),
- rule,
- ' '.join(all_inputs)))
+ self._line('build %s: %s' % (' '.join(out_outputs),
+ ' '.join([rule] + all_inputs)))
if variables:
if isinstance(variables, dict):
- iterator = variables.iteritems()
+ iterator = iter(variables.items())
else:
iterator = iter(variables)
# limitations under the License.
import unittest
-from StringIO import StringIO
+
+try:
+ from StringIO import StringIO
+except ImportError:
+ from io import StringIO
import ninja_syntax
Group: Development/Tools
License: Apache 2.0
URL: https://github.com/martine/ninja
+Source0: %{name}-%{version}-%{release}.tar.gz
+BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}
%description
Ninja is yet another build system. It takes as input the interdependencies of files (typically source code and output executables) and
which has over 30,000 source files and whose other build systems (including one built from custom non-recursive Makefiles) can take ten
seconds to start building after changing one file. Ninja is under a second.
+%prep
+%setup -q -n %{name}-%{version}-%{release}
+
%build
-# Assuming we've bootstrapped already..
-../ninja manual ninja -C ..
+echo Building..
+./bootstrap.py
+./ninja manual
%install
mkdir -p %{buildroot}%{_bindir} %{buildroot}%{_docdir}
-cp -p ../ninja %{buildroot}%{_bindir}/
-git log --oneline --pretty=format:'%h: %s (%an, %cd)' --abbrev-commit --all > GITLOG
+cp -p ninja %{buildroot}%{_bindir}/
%files
%defattr(-, root, root)
-%doc GITLOG ../COPYING ../README ../doc/manual.html
+%doc COPYING README doc/manual.html
%{_bindir}/*
%clean
-mv %{_topdir}/*.rpm ..
-rm -rf %{_topdir}
+rm -rf %{buildroot}
+
+#The changelog is built automatically from Git history
+%changelog
--- /dev/null
+#!/bin/bash
+
+echo Building ninja RPMs..
+GITROOT=$(git rev-parse --show-toplevel)
+cd $GITROOT
+
+VER=1.0
+REL=$(git rev-parse --short HEAD)git
+RPMTOPDIR=$GITROOT/rpm-build
+echo "Ver: $VER, Release: $REL"
+
+# Create tarball
+mkdir -p $RPMTOPDIR/{SOURCES,SPECS}
+git archive --format=tar --prefix=ninja-${VER}-${REL}/ HEAD | gzip -c > $RPMTOPDIR/SOURCES/ninja-${VER}-${REL}.tar.gz
+
+# Convert git log to RPM's ChangeLog format (shown with rpm -qp --changelog <rpm file>)
+sed -e "s/%{ver}/$VER/" -e "s/%{rel}/$REL/" misc/packaging/ninja.spec > $RPMTOPDIR/SPECS/ninja.spec
+git log --format="* %cd %aN%n- (%h) %s%d%n" --date=local | sed -r 's/[0-9]+:[0-9]+:[0-9]+ //' >> $RPMTOPDIR/SPECS/ninja.spec
+
+# Build SRC and binary RPMs
+rpmbuild --quiet \
+ --define "_topdir $RPMTOPDIR" \
+ --define "_rpmdir $PWD" \
+ --define "_srcrpmdir $PWD" \
+ --define '_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm' \
+ -ba $RPMTOPDIR/SPECS/ninja.spec &&
+
+rm -rf $RPMTOPDIR &&
+echo Done
it when needed.
"""
-import BaseHTTPServer
+from __future__ import print_function
+
+try:
+ import http.server as httpserver
+except ImportError:
+ import BaseHTTPServer as httpserver
import subprocess
import sys
import webbrowser
outputs = []
try:
- target = lines.next()[:-1] # strip trailing colon
+ target = next(lines)[:-1] # strip trailing colon
- line = lines.next()
+ line = next(lines)
(match, rule) = match_strip(line, ' input: ')
if match:
- (match, line) = match_strip(lines.next(), ' ')
+ (match, line) = match_strip(next(lines), ' ')
while match:
type = None
(match, line) = match_strip(line, '| ')
if match:
type = 'order-only'
inputs.append((line, type))
- (match, line) = match_strip(lines.next(), ' ')
+ (match, line) = match_strip(next(lines), ' ')
match, _ = match_strip(line, ' outputs:')
if match:
- (match, line) = match_strip(lines.next(), ' ')
+ (match, line) = match_strip(next(lines), ' ')
while match:
outputs.append(line)
- (match, line) = match_strip(lines.next(), ' ')
+ (match, line) = match_strip(next(lines), ' ')
except StopIteration:
pass
return Node(inputs, rule, target, outputs)
-def generate_html(node):
- print '''<!DOCTYPE html>
+def create_page(body):
+ return '''<!DOCTYPE html>
<style>
body {
font-family: sans;
.filelist {
-webkit-columns: auto 2;
}
-</style>'''
+</style>
+''' + body
- print '<h1><tt>%s</tt></h1>' % node.target
+def generate_html(node):
+ document = ['<h1><tt>%s</tt></h1>' % node.target]
if node.inputs:
- print '<h2>target is built using rule <tt>%s</tt> of</h2>' % node.rule
+ document.append('<h2>target is built using rule <tt>%s</tt> of</h2>' %
+ node.rule)
if len(node.inputs) > 0:
- print '<div class=filelist>'
+ document.append('<div class=filelist>')
for input, type in sorted(node.inputs):
extra = ''
if type:
extra = ' (%s)' % type
- print '<tt><a href="?%s">%s</a>%s</tt><br>' % (input, input, extra)
- print '</div>'
+ document.append('<tt><a href="?%s">%s</a>%s</tt><br>' %
+ (input, input, extra))
+ document.append('</div>')
if node.outputs:
- print '<h2>dependent edges build:</h2>'
- print '<div class=filelist>'
+ document.append('<h2>dependent edges build:</h2>')
+ document.append('<div class=filelist>')
for output in sorted(node.outputs):
- print '<tt><a href="?%s">%s</a></tt><br>' % (output, output)
- print '</div>'
+ document.append('<tt><a href="?%s">%s</a></tt><br>' %
+ (output, output))
+ document.append('</div>')
+
+ return '\n'.join(document)
def ninja_dump(target):
proc = subprocess.Popen([sys.argv[1], '-t', 'query', target],
- stdout=subprocess.PIPE)
- return proc.communicate()[0]
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ universal_newlines=True)
+ return proc.communicate() + (proc.returncode,)
-class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+class RequestHandler(httpserver.BaseHTTPRequestHandler):
def do_GET(self):
assert self.path[0] == '/'
target = self.path[1:]
return
target = target[1:]
- input = ninja_dump(target)
+ ninja_output, ninja_error, exit_code = ninja_dump(target)
+ if exit_code == 0:
+ page_body = generate_html(parse(ninja_output.strip()))
+ else:
+ # Relay ninja's error message.
+ page_body = '<h1><tt>%s</tt></h1>' % ninja_error
self.send_response(200)
self.end_headers()
- stdout = sys.stdout
- sys.stdout = self.wfile
- try:
- generate_html(parse(input.strip()))
- finally:
- sys.stdout = stdout
+ self.wfile.write(create_page(page_body).encode('utf-8'))
def log_message(self, format, *args):
pass # Swallow console spam.
port = 8000
-httpd = BaseHTTPServer.HTTPServer(('',port), RequestHandler)
+httpd = httpserver.HTTPServer(('',port), RequestHandler)
try:
- print 'Web server running on port %d, ctl-C to abort...' % port
+ print('Web server running on port %d, ctl-C to abort...' % port)
webbrowser.open_new('http://localhost:%s' % port)
httpd.serve_forever()
except KeyboardInterrupt:
- print
+ print()
pass # Swallow console spam.
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
+#include <functional>
#ifdef _WIN32
#include <windows.h>
#include "subprocess.h"
#include "util.h"
+namespace {
+
+/// A CommandRunner that doesn't actually run the commands.
+struct DryRunCommandRunner : public CommandRunner {
+ virtual ~DryRunCommandRunner() {}
+
+ // Overridden from CommandRunner:
+ virtual bool CanRunMore();
+ virtual bool StartCommand(Edge* edge);
+ virtual Edge* WaitForCommand(ExitStatus* status, string* /* output */);
+
+ private:
+ queue<Edge*> finished_;
+};
+
+bool DryRunCommandRunner::CanRunMore() {
+ return true;
+}
+
+bool DryRunCommandRunner::StartCommand(Edge* edge) {
+ finished_.push(edge);
+ return true;
+}
+
+Edge* DryRunCommandRunner::WaitForCommand(ExitStatus* status,
+ string* /*output*/) {
+ if (finished_.empty()) {
+ *status = ExitFailure;
+ return NULL;
+ }
+ *status = ExitSuccess;
+ Edge* edge = finished_.front();
+ finished_.pop();
+ return edge;
+}
+
+} // namespace
+
BuildStatus::BuildStatus(const BuildConfig& config)
: config_(config),
start_time_millis_(GetTimeMillis()),
started_edges_(0), finished_edges_(0), total_edges_(0),
have_blank_line_(true), progress_status_format_(NULL),
- overall_rate_(), current_rate_(),
- current_rate_average_count_(config.parallelism) {
+ overall_rate_(), current_rate_(config.parallelism) {
#ifndef _WIN32
const char* term = getenv("TERM");
smart_terminal_ = isatty(1) && term && string(term) != "dumb";
printf("\n");
}
-string BuildStatus::FormatProgressStatus(const char* progress_status_format) const {
+string BuildStatus::FormatProgressStatus(
+ const char* progress_status_format) const {
string out;
char buf[32];
+ int percent;
for (const char* s = progress_status_format; *s != '\0'; ++s) {
if (*s == '%') {
++s;
out += buf;
break;
- // Overall finished edges per second.
+ // Overall finished edges per second.
case 'o':
- overall_rate_.UpdateRate(finished_edges_, finished_edges_);
- overall_rate_.snprinfRate(buf, "%.1f");
+ overall_rate_.UpdateRate(finished_edges_);
+ snprinfRate(overall_rate_.rate(), buf, "%.1f");
out += buf;
break;
- // Current rate, average over the last '-j' jobs.
+ // Current rate, average over the last '-j' jobs.
case 'c':
- // TODO use sliding window?
- if (finished_edges_ > current_rate_.last_update() &&
- finished_edges_ - current_rate_.last_update() == current_rate_average_count_) {
- current_rate_.UpdateRate(current_rate_average_count_, finished_edges_);
- current_rate_.Restart();
- }
- current_rate_.snprinfRate(buf, "%.0f");
+ current_rate_.UpdateRate(finished_edges_);
+ snprinfRate(current_rate_.rate(), buf, "%.1f");
+ out += buf;
+ break;
+
+ // Percentage
+ case 'p':
+ percent = (100 * started_edges_) / total_edges_;
+ snprintf(buf, sizeof(buf), "%3i%%", percent);
out += buf;
break;
- default: {
+ default:
Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s);
return "";
}
- }
} else {
out.push_back(*s);
}
want = true;
++wanted_edges_;
if (edge->AllInputsReady())
- ready_.insert(edge);
+ ScheduleWork(edge);
if (!edge->is_phony())
++command_edges_;
}
return edge;
}
+void Plan::ScheduleWork(Edge* edge) {
+ Pool* pool = edge->pool();
+ if (pool->ShouldDelayEdge()) {
+ pool->DelayEdge(edge);
+ pool->RetrieveReadyEdges(&ready_);
+ } else {
+ pool->EdgeScheduled(*edge);
+ ready_.insert(edge);
+ }
+}
+
+void Plan::ResumeDelayedJobs(Edge* edge) {
+ edge->pool()->EdgeFinished(*edge);
+ edge->pool()->RetrieveReadyEdges(&ready_);
+}
+
void Plan::EdgeFinished(Edge* edge) {
map<Edge*, bool>::iterator i = want_.find(edge);
assert(i != want_.end());
want_.erase(i);
edge->outputs_ready_ = true;
+ // See if this job frees up any delayed jobs
+ ResumeDelayedJobs(edge);
+
// Check off any nodes we were waiting for with this edge.
for (vector<Node*>::iterator i = edge->outputs_.begin();
i != edge->outputs_.end(); ++i) {
// See if the edge is now ready.
if ((*i)->AllInputsReady()) {
if (want_i->second) {
- ready_.insert(*i);
+ ScheduleWork(*i);
} else {
// We do not need to build this edge, but we might need to build one of
// its dependents.
// If all non-order-only inputs for this edge are now clean,
// we might have changed the dirty state of the outputs.
- vector<Node*>::iterator begin = (*ei)->inputs_.begin(),
- end = (*ei)->inputs_.end() - (*ei)->order_only_deps_;
+ vector<Node*>::iterator
+ begin = (*ei)->inputs_.begin(),
+ end = (*ei)->inputs_.end() - (*ei)->order_only_deps_;
if (find_if(begin, end, mem_fun(&Node::dirty)) == end) {
// Recompute most_recent_input and command.
Node* most_recent_input = NULL;
return edge;
}
-/// A CommandRunner that doesn't actually run the commands.
-struct DryRunCommandRunner : public CommandRunner {
- virtual ~DryRunCommandRunner() {}
- virtual bool CanRunMore() {
- return true;
- }
- virtual bool StartCommand(Edge* edge) {
- finished_.push(edge);
- return true;
- }
- virtual Edge* WaitForCommand(ExitStatus* status, string* /* output */) {
- if (finished_.empty()) {
- *status = ExitFailure;
- return NULL;
- }
- *status = ExitSuccess;
- Edge* edge = finished_.front();
- finished_.pop();
- return edge;
- }
-
- queue<Edge*> finished_;
-};
-
Builder::Builder(State* state, const BuildConfig& config,
BuildLog* log, DiskInterface* disk_interface)
: state_(state), config_(config), disk_interface_(disk_interface),
}
bool Builder::StartEdge(Edge* edge, string* err) {
+ METRIC_RECORD("StartEdge");
if (edge->is_phony())
return true;
// Create response file, if needed
// XXX: this may also block; do we care?
if (edge->HasRspFile()) {
- if (!disk_interface_->WriteFile(edge->GetRspFile(), edge->GetRspFileContent()))
+ if (!disk_interface_->WriteFile(edge->GetRspFile(),
+ edge->GetRspFileContent())) {
return false;
+ }
}
// start command computing and run it
}
void Builder::FinishEdge(Edge* edge, bool success, const string& output) {
+ METRIC_RECORD("FinishEdge");
TimeStamp restat_mtime = 0;
if (success) {
}
if (restat_mtime != 0 && !edge->rule().depfile().empty()) {
- TimeStamp depfile_mtime = disk_interface_->Stat(edge->EvaluateDepFile());
+ TimeStamp depfile_mtime =
+ disk_interface_->Stat(edge->EvaluateDepFile());
if (depfile_mtime > restat_mtime)
restat_mtime = depfile_mtime;
}
#ifndef NINJA_BUILD_H_
#define NINJA_BUILD_H_
+#include <cstdio>
#include <map>
+#include <memory>
+#include <queue>
#include <set>
#include <string>
-#include <queue>
#include <vector>
-#include <memory>
-#include <cstdio>
#include "graph.h" // XXX needed for DependencyScan; should rearrange.
#include "exit_status.h"
bool CheckDependencyCycle(Node* node, vector<Node*>* stack, string* err);
void NodeFinished(Node* node);
+ /// Submits a ready edge as a candidate for execution.
+ /// The edge may be delayed from running, for example if it's a member of a
+ /// currently-full pool.
+ void ScheduleWork(Edge* edge);
+
+ /// Allows jobs blocking on |edge| to potentially resume.
+ /// For example, if |edge| is a member of a pool, calling this may schedule
+ /// previously pending jobs in that pool.
+ void ResumeDelayedJobs(Edge* edge);
+
/// Keep track of which edges we want to build in this plan. If this map does
/// not contain an entry for an edge, we do not want to build the entry or its
/// dependents. If an entry maps to false, we do not want to build it, but we
/// Format the progress status string by replacing the placeholders.
/// See the user manual for more information about the available
/// placeholders.
- /// @param progress_status_format_ The format of the progress status.
+ /// @param progress_status_format The format of the progress status.
string FormatProgressStatus(const char* progress_status_format) const;
private:
/// The custom progress status format to use.
const char* progress_status_format_;
+ template<size_t S>
+ void snprinfRate(double rate, char(&buf)[S], const char* format) const {
+ if (rate == -1) snprintf(buf, S, "?");
+ else snprintf(buf, S, format, rate);
+ }
+
struct RateInfo {
- RateInfo() : last_update_(0), rate_(-1) {}
-
- double rate() const { return rate_; }
- int last_update() const { return last_update_; }
- void Restart() { return stopwatch_.Restart(); }
-
- double UpdateRate(int edges, int update_hint) {
- if (update_hint != last_update_) {
- rate_ = edges / stopwatch_.Elapsed() + 0.5;
- last_update_ = update_hint;
- }
- return rate_;
+ RateInfo() : rate_(-1) {}
+
+ void Restart() { stopwatch_.Restart(); }
+ double rate() { return rate_; }
+
+ void UpdateRate(int edges) {
+ if (edges && stopwatch_.Elapsed())
+ rate_ = edges / stopwatch_.Elapsed();
}
- template<class T>
- void snprinfRate(T buf, const char* format) {
- if (rate_ == -1)
- snprintf(buf, sizeof(buf), "?");
- else
- snprintf(buf, sizeof(buf), format, rate_);
+ private:
+ double rate_;
+ Stopwatch stopwatch_;
+ };
+
+ struct SlidingRateInfo {
+ SlidingRateInfo(int n) : rate_(-1), N(n), last_update_(-1) {}
+
+ void Restart() { stopwatch_.Restart(); }
+ double rate() { return rate_; }
+
+ void UpdateRate(int update_hint) {
+ if (update_hint == last_update_)
+ return;
+ last_update_ = update_hint;
+
+ if (times_.size() == N)
+ times_.pop();
+ times_.push(stopwatch_.Elapsed());
+ if (times_.back() != times_.front())
+ rate_ = times_.size() / (times_.back() - times_.front());
}
private:
+ double rate_;
Stopwatch stopwatch_;
+ const size_t N;
+ std::queue<double> times_;
int last_update_;
- double rate_;
};
mutable RateInfo overall_rate_;
- mutable RateInfo current_rate_;
- const int current_rate_average_count_;
+ mutable SlidingRateInfo current_rate_;
#ifdef _WIN32
void* console_;
uint64_t h = seed ^ (len * m);
const uint64_t * data = (const uint64_t *)key;
const uint64_t * end = data + (len/8);
- while(data != end) {
+ while (data != end) {
uint64_t k = *data++;
k *= m;
k ^= k >> r;
h *= m;
}
const unsigned char* data2 = (const unsigned char*)data;
- switch(len & 7)
+ switch (len & 7)
{
case 7: h ^= uint64_t(data2[6]) << 48;
case 6: h ^= uint64_t(data2[5]) << 40;
return MurmurHash64A(command.str_, command.len_);
}
+BuildLog::LogEntry::LogEntry(const string& output)
+ : output(output) {}
+
+BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash,
+ int start_time, int end_time, TimeStamp restat_mtime)
+ : output(output), command_hash(command_hash),
+ start_time(start_time), end_time(end_time), restat_mtime(restat_mtime)
+{}
+
BuildLog::BuildLog()
: log_file_(NULL), needs_recompaction_(false) {}
void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time,
TimeStamp restat_mtime) {
string command = edge->EvaluateCommand(true);
+ uint64_t command_hash = LogEntry::HashCommand(command);
for (vector<Node*>::iterator out = edge->outputs_.begin();
out != edge->outputs_.end(); ++out) {
const string& path = (*out)->path();
if (i != entries_.end()) {
log_entry = i->second;
} else {
- log_entry = new LogEntry;
- log_entry->output = path;
+ log_entry = new LogEntry(path);
entries_.insert(Entries::value_type(log_entry->output, log_entry));
}
- log_entry->command_hash = LogEntry::HashCommand(command);
+ log_entry->command_hash = command_hash;
log_entry->start_time = start_time;
log_entry->end_time = end_time;
log_entry->restat_mtime = restat_mtime;
log_file_ = NULL;
}
-class LineReader {
- public:
+struct LineReader {
explicit LineReader(FILE* file)
: file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {
memset(buf_, 0, sizeof(buf_));
if (i != entries_.end()) {
entry = i->second;
} else {
- entry = new LogEntry;
- entry->output = output;
+ entry = new LogEntry(output);
entries_.insert(Entries::value_type(entry->output, entry));
++unique_entry_count;
}
}
bool BuildLog::Recompact(const string& path, string* err) {
+ METRIC_RECORD(".ninja_log recompact");
printf("Recompacting log...\n");
string temp_path = path + ".recompact";
start_time == o.start_time && end_time == o.end_time &&
restat_mtime == o.restat_mtime;
}
+
+ explicit LogEntry(const string& output);
+ LogEntry(const string& output, uint64_t command_hash,
+ int start_time, int end_time, TimeStamp restat_mtime);
};
/// Lookup a previously-run command by its output path.
#include <unistd.h>
#endif
+namespace {
+
const char kTestFilename[] = "BuildLogTest-tempfile";
struct BuildLogTest : public StateTestWithBuiltinRules {
ASSERT_EQ(0, truncate(kTestFilename, size));
#else
int fh;
- fh = _sopen(kTestFilename, _O_RDWR | _O_CREAT, _SH_DENYNO, _S_IREAD | _S_IWRITE);
+ fh = _sopen(kTestFilename, _O_RDWR | _O_CREAT, _SH_DENYNO,
+ _S_IREAD | _S_IWRITE);
ASSERT_EQ(0, _chsize(fh, size));
_close(fh);
#endif
ASSERT_EQ(789, e->restat_mtime);
ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash));
}
+
+TEST_F(BuildLogTest, MultiTargetEdge) {
+ AssertParse(&state_,
+"build out out.d: cat\n");
+
+ BuildLog log;
+ log.RecordCommand(state_.edges_[0], 21, 22);
+
+ ASSERT_EQ(2u, log.entries().size());
+ BuildLog::LogEntry* e1 = log.LookupByOutput("out");
+ ASSERT_TRUE(e1);
+ BuildLog::LogEntry* e2 = log.LookupByOutput("out.d");
+ ASSERT_TRUE(e2);
+ ASSERT_EQ("out", e1->output);
+ ASSERT_EQ("out.d", e2->output);
+ ASSERT_EQ(21, e1->start_time);
+ ASSERT_EQ(21, e2->start_time);
+ ASSERT_EQ(22, e2->end_time);
+ ASSERT_EQ(22, e2->end_time);
+}
+
+} // anonymous namespace
ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err);
}
+TEST_F(PlanTest, PoolWithDepthOne) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"pool foobar\n"
+" depth = 1\n"
+"rule poolcat\n"
+" command = cat $in > $out\n"
+" pool = foobar\n"
+"build out1: poolcat in\n"
+"build out2: poolcat in\n"));
+ GetNode("out1")->MarkDirty();
+ GetNode("out2")->MarkDirty();
+ string err;
+ EXPECT_TRUE(plan_.AddTarget(GetNode("out1"), &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(plan_.AddTarget(GetNode("out2"), &err));
+ ASSERT_EQ("", err);
+ ASSERT_TRUE(plan_.more_to_do());
+
+ Edge* edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ ASSERT_EQ("out1", edge->outputs_[0]->path());
+
+ // This will be false since poolcat is serialized
+ ASSERT_FALSE(plan_.FindWork());
+
+ plan_.EdgeFinished(edge);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ ASSERT_EQ("out2", edge->outputs_[0]->path());
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ plan_.EdgeFinished(edge);
+
+ ASSERT_FALSE(plan_.more_to_do());
+ edge = plan_.FindWork();
+ ASSERT_EQ(0, edge);
+}
+
+TEST_F(PlanTest, PoolsWithDepthTwo) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"pool foobar\n"
+" depth = 2\n"
+"pool bazbin\n"
+" depth = 2\n"
+"rule foocat\n"
+" command = cat $in > $out\n"
+" pool = foobar\n"
+"rule bazcat\n"
+" command = cat $in > $out\n"
+" pool = bazbin\n"
+"build out1: foocat in\n"
+"build out2: foocat in\n"
+"build out3: foocat in\n"
+"build outb1: bazcat in\n"
+"build outb2: bazcat in\n"
+"build outb3: bazcat in\n"
+" pool =\n"
+"build allTheThings: cat out1 out2 out3 outb1 outb2 outb3\n"
+));
+ // Mark all the out* nodes dirty
+ for (int i = 0; i < 3; ++i) {
+ GetNode("out" + string(1, '1' + i))->MarkDirty();
+ GetNode("outb" + string(1, '1' + i))->MarkDirty();
+ }
+ GetNode("allTheThings")->MarkDirty();
+
+ string err;
+ EXPECT_TRUE(plan_.AddTarget(GetNode("allTheThings"), &err));
+ ASSERT_EQ("", err);
+
+ // Grab the first 4 edges, out1 out2 outb1 outb2
+ deque<Edge*> edges;
+ for (int i = 0; i < 4; ++i) {
+ ASSERT_TRUE(plan_.more_to_do());
+ Edge* edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ string base_name(i < 2 ? "out" : "outb");
+ ASSERT_EQ(base_name + string(1, '1' + (i % 2)), edge->outputs_[0]->path());
+ edges.push_back(edge);
+ }
+
+ // outb3 is exempt because it has an empty pool
+ ASSERT_TRUE(plan_.more_to_do());
+ Edge* edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ ASSERT_EQ("outb3", edge->outputs_[0]->path());
+ edges.push_back(edge);
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ // finish out1
+ plan_.EdgeFinished(edges.front());
+ edges.pop_front();
+
+ // out3 should be available
+ Edge* out3 = plan_.FindWork();
+ ASSERT_TRUE(out3);
+ ASSERT_EQ("in", out3->inputs_[0]->path());
+ ASSERT_EQ("out3", out3->outputs_[0]->path());
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ plan_.EdgeFinished(out3);
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ for (deque<Edge*>::iterator it = edges.begin(); it != edges.end(); ++it) {
+ plan_.EdgeFinished(*it);
+ }
+
+ Edge* final = plan_.FindWork();
+ ASSERT_TRUE(final);
+ ASSERT_EQ("allTheThings", final->outputs_[0]->path());
+
+ plan_.EdgeFinished(final);
+
+ ASSERT_FALSE(plan_.more_to_do());
+ ASSERT_FALSE(plan_.FindWork());
+}
+
struct BuildTest : public StateTestWithBuiltinRules,
public CommandRunner {
BuildTest() : config_(MakeConfig()),
string err;
#ifdef _WIN32
- ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build subdir\\dir2\\file: cat in1\n"));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "build subdir\\dir2\\file: cat in1\n"));
EXPECT_TRUE(builder_.AddTarget("subdir\\dir2\\file", &err));
#else
- ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build subdir/dir2/file: cat in1\n"));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "build subdir/dir2/file: cat in1\n"));
EXPECT_TRUE(builder_.AddTarget("subdir/dir2/file", &err));
#endif
// Create all necessary files
fs_.Create("in", now_, "");
- // The implicit dependencies and the depfile itself
+ // The implicit dependencies and the depfile itself
// are newer than the output
TimeStamp restat_mtime = ++now_;
fs_.Create("out1.d", now_, "out1: will.be.deleted restat.file\n");
ASSERT_TRUE(NULL != log_entry);
ASSERT_EQ(restat_mtime, log_entry->restat_mtime);
- // Now remove a file, referenced from depfile, so that target becomes
+ // Now remove a file, referenced from depfile, so that target becomes
// dirty, but the output does not change
fs_.RemoveFile("will.be.deleted");
-
+
// Trigger the build again - only out1 gets built
commands_ran_.clear();
state_.Reset();
}
// Test that RSP files are created when & where appropriate and deleted after
-// succesful execution.
+// successful execution.
TEST_F(BuildTest, RspFileSuccess)
{
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
EXPECT_EQ("[%/s0/t0/r0/u0/f0]",
status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]"));
}
+
: state_(state),
config_(config),
removed_(),
+ cleaned_(),
cleaned_files_count_(0),
disk_interface_(new RealDiskInterface),
status_(0) {
: state_(state),
config_(config),
removed_(),
+ cleaned_(),
cleaned_files_count_(0),
disk_interface_(disk_interface),
status_(0) {
return (i != removed_.end());
}
+void Cleaner::RemoveEdgeFiles(Edge* edge) {
+ string depfile = edge->EvaluateDepFile();
+ if (!depfile.empty())
+ Remove(depfile);
+
+ string rspfile = edge->GetRspFile();
+ if (!rspfile.empty())
+ Remove(rspfile);
+}
+
void Cleaner::PrintHeader() {
if (config_.verbosity == BuildConfig::QUIET)
return;
out_node != (*e)->outputs_.end(); ++out_node) {
Remove((*out_node)->path());
}
- // Remove the depfile
- if (!(*e)->rule().depfile().empty())
- Remove((*e)->EvaluateDepFile());
- // Remove the response file
- if ((*e)->HasRspFile())
- Remove((*e)->GetRspFile());
+
+ RemoveEdgeFiles(*e);
}
PrintFooter();
return status_;
// Do not try to remove phony targets
if (!e->is_phony()) {
Remove(target->path());
- if (!target->in_edge()->rule().depfile().empty())
- Remove(target->in_edge()->EvaluateDepFile());
- if (e->HasRspFile())
- Remove(e->GetRspFile());
+ RemoveEdgeFiles(e);
}
for (vector<Node*>::iterator n = e->inputs_.begin(); n != e->inputs_.end();
++n) {
- DoCleanTarget(*n);
+ Node* next = *n;
+ // call DoCleanTarget recursively if this node has not been visited
+ if (cleaned_.count(next) == 0) {
+ DoCleanTarget(next);
+ }
}
}
+
+ // mark this target to be cleaned already
+ cleaned_.insert(target);
}
int Cleaner::CleanTarget(Node* target) {
for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
out_node != (*e)->outputs_.end(); ++out_node) {
Remove((*out_node)->path());
- if (!(*e)->rule().depfile().empty())
- Remove((*e)->EvaluateDepFile());
- if ((*e)->HasRspFile())
- Remove((*e)->GetRspFile());
+ RemoveEdgeFiles(*e);
}
}
}
status_ = 0;
cleaned_files_count_ = 0;
removed_.clear();
+ cleaned_.clear();
}
struct Rule;
struct DiskInterface;
-class Cleaner {
- public:
+struct Cleaner {
/// Build a cleaner object with a real disk interface.
Cleaner(State* state, const BuildConfig& config);
/// @returns whether the file @a path exists.
bool FileExists(const string& path);
void Report(const string& path);
+
/// Remove the given @a path file only if it has not been already removed.
void Remove(const string& path);
/// @return whether the given @a path has already been removed.
bool IsAlreadyRemoved(const string& path);
+ /// Remove the depfile and rspfile for an Edge.
+ void RemoveEdgeFiles(Edge* edge);
+
/// Helper recursive method for CleanTarget().
void DoCleanTarget(Node* target);
void PrintHeader();
State* state_;
const BuildConfig& config_;
set<string> removed_;
+ set<Node*> cleaned_;
int cleaned_files_count_;
DiskInterface* disk_interface_;
int status_;
// MSDN: "Naming Files, Paths, and Namespaces"
// http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
if (!path.empty() && path[0] != '\\' && path.size() > MAX_PATH) {
- Error("Stat(%s): Filename longer than %i characters", path.c_str(), MAX_PATH);
+ Error("Stat(%s): Filename longer than %i characters",
+ path.c_str(), MAX_PATH);
return -1;
}
WIN32_FILE_ATTRIBUTE_DATA attrs;
bool RealDiskInterface::WriteFile(const string& path, const string& contents) {
FILE * fp = fopen(path.c_str(), "w");
if (fp == NULL) {
- Error("WriteFile(%s): Unable to create file. %s", path.c_str(), strerror(errno));
+ Error("WriteFile(%s): Unable to create file. %s",
+ path.c_str(), strerror(errno));
return false;
}
if (fwrite(contents.data(), 1, contents.length(), fp) < contents.length()) {
- Error("WriteFile(%s): Unable to write to the file. %s", path.c_str(), strerror(errno));
+ Error("WriteFile(%s): Unable to write to the file. %s",
+ path.c_str(), strerror(errno));
fclose(fp);
return false;
}
if (fclose(fp) == EOF) {
- Error("WriteFile(%s): Unable to close the file. %s", path.c_str(), strerror(errno));
+ Error("WriteFile(%s): Unable to close the file. %s",
+ path.c_str(), strerror(errno));
return false;
}
namespace {
-class DiskInterfaceTest : public testing::Test {
- public:
+struct DiskInterfaceTest : public testing::Test {
virtual void SetUp() {
// These tests do real disk accesses, so create a temp dir.
temp_dir_.CreateAndEnter("Ninja-DiskInterfaceTest");
if (edge->rule_->restat() && build_log() &&
(entry = build_log()->LookupByOutput(output->path()))) {
if (entry->restat_mtime < most_recent_stamp) {
- EXPLAIN("restat of output %s older than most recent input %s (%d vs %d)",
- output->path().c_str(), most_recent_input->path().c_str(),
- entry->restat_mtime, most_recent_stamp);
+ EXPLAIN("restat of output %s older than most recent input %s "
+ "(%d vs %d)",
+ output->path().c_str(), most_recent_input->path().c_str(),
+ entry->restat_mtime, most_recent_stamp);
return true;
}
} else {
virtual string LookupVariable(const string& var);
/// Given a span of Nodes, construct a list of paths suitable for a command
- /// line. XXX here is where shell-escaping of e.g spaces should happen.
+ /// line.
string MakePathList(vector<Node*>::iterator begin,
vector<Node*>::iterator end,
char sep);
} else if (edge_->env_) {
return edge_->env_->LookupVariable(var);
} else {
- // XXX should we warn here?
return string();
}
}
string Edge::EvaluateCommand(bool incl_rsp_file) {
EdgeEnv env(this);
string command = rule_->command().Evaluate(&env);
- if (incl_rsp_file && HasRspFile())
+ if (incl_rsp_file && HasRspFile())
command += ";rspfile=" + GetRspFileContent();
return command;
}
// create one; this makes us not abort if the input is missing,
// but instead will rebuild in that circumstance.
if (!node->in_edge()) {
- Edge* phony_edge = state_->AddEdge(&State::kPhonyRule);
+ Edge* phony_edge = state_->AddEdge(&State::kPhonyRule,
+ &State::kDefaultPool);
node->set_in_edge(phony_edge);
phony_edge->outputs_.push_back(node);
i != outputs_.end() && *i != NULL; ++i) {
printf("%s ", (*i)->path().c_str());
}
+ if (pool_) {
+ if (!pool_->name().empty()) {
+ printf("(in pool '%s')", pool_->name().c_str());
+ }
+ } else {
+ printf("(null pool?)");
+ }
printf("] 0x%p\n", this);
}
}
void Node::Dump(const char* prefix) const {
- printf("%s <%s 0x%p> mtime: %d%s, (:%s), ",
- prefix, path().c_str(), this,
- mtime(), mtime()?"":" (:missing)",
- dirty()?" dirty":" clean");
- if (in_edge()) {
- in_edge()->Dump("in-edge: ");
- }else{
- printf("no in-edge\n");
- }
- printf(" out edges:\n");
- for (vector<Edge*>::const_iterator e = out_edges().begin();
- e != out_edges().end() && *e != NULL; ++e) {
- (*e)->Dump(" +- ");
- }
+ printf("%s <%s 0x%p> mtime: %d%s, (:%s), ",
+ prefix, path().c_str(), this,
+ mtime(), mtime() ? "" : " (:missing)",
+ dirty() ? " dirty" : " clean");
+ if (in_edge()) {
+ in_edge()->Dump("in-edge: ");
+ } else {
+ printf("no in-edge\n");
+ }
+ printf(" out edges:\n");
+ for (vector<Edge*>::const_iterator e = out_edges().begin();
+ e != out_edges().end() && *e != NULL; ++e) {
+ (*e)->Dump(" +- ");
+ }
}
EvalString command_;
EvalString description_;
EvalString depfile_;
+ EvalString pool_;
EvalString rspfile_;
EvalString rspfile_content_;
};
struct BuildLog;
struct Node;
struct State;
+struct Pool;
/// An edge in the dependency graph; links between Nodes using Rules.
struct Edge {
/// Expand all variables in a command and return it as a string.
/// If incl_rsp_file is enabled, the string will also contain the
/// full contents of a response file (if applicable)
- string EvaluateCommand(bool incl_rsp_file = false); // XXX move to env, take env ptr
+ string EvaluateCommand(bool incl_rsp_file = false);
string EvaluateDepFile();
string GetDescription();
void Dump(const char* prefix="") const;
const Rule* rule_;
+ Pool* pool_;
vector<Node*> inputs_;
vector<Node*> outputs_;
Env* env_;
bool outputs_ready_;
const Rule& rule() const { return *rule_; }
+ Pool* pool() const { return pool_; }
+ int weight() const { return 1; }
bool outputs_ready() const { return outputs_ready_; }
- // XXX There are three types of inputs.
+ // There are three types of inputs.
// 1) explicit deps, which show up as $in on the command line;
// 2) implicit deps, which the target depends on implicitly (e.g. C headers),
// and changes in them cause the target to rebuild;
// 3) order-only deps, which are needed before the target builds but which
// don't cause the target to rebuild.
- // Currently we stuff all of these into inputs_ and keep counts of #2 and #3
- // when we need to compute subsets. This is suboptimal; should think of a
- // better representation. (Could make each pointer into a pair of a pointer
- // and a type of input, or if memory matters could use the low bits of the
- // pointer...)
+ // These are stored in inputs_ in that order, and we keep counts of
+ // #2 and #3 when we need to access the various subsets.
int implicit_deps_;
int order_only_deps_;
bool is_implicit(size_t index) {
const int r = 24;
unsigned int h = seed ^ len;
const unsigned char * data = (const unsigned char *)key;
- while(len >= 4) {
+ while (len >= 4) {
unsigned int k = *(unsigned int *)data;
k *= m;
k ^= k >> r;
data += 4;
len -= 4;
}
- switch(len) {
+ switch (len) {
case 3: h ^= data[2] << 16;
case 2: h ^= data[1] << 8;
case 1: h ^= data[0];
TEST(IncludesNormalize, WithRelative) {
string currentdir = IncludesNormalize::ToLower(GetCurDir());
EXPECT_EQ("c", IncludesNormalize::Normalize("a/b/c", "a/b"));
- EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"), NULL));
+ EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"),
+ NULL));
EXPECT_EQ(string("..\\") + currentdir + string("\\a"),
IncludesNormalize::Normalize("a", "../b"));
EXPECT_EQ(string("..\\") + currentdir + string("\\a\\b"),
}
TEST(IncludesNormalize, Split) {
- EXPECT_EQ("", IncludesNormalize::Join(IncludesNormalize::Split("", '/'), ':'));
- EXPECT_EQ("a", IncludesNormalize::Join(IncludesNormalize::Split("a", '/'), ':'));
- EXPECT_EQ("a:b:c", IncludesNormalize::Join(IncludesNormalize::Split("a/b/c", '/'), ':'));
+ EXPECT_EQ("", IncludesNormalize::Join(IncludesNormalize::Split("", '/'),
+ ':'));
+ EXPECT_EQ("a", IncludesNormalize::Join(IncludesNormalize::Split("a", '/'),
+ ':'));
+ EXPECT_EQ("a:b:c",
+ IncludesNormalize::Join(
+ IncludesNormalize::Split("a/b/c", '/'), ':'));
}
TEST(IncludesNormalize, ToLower) {
EXPECT_EQ("", IncludesNormalize::ToLower(""));
EXPECT_EQ("stuff", IncludesNormalize::ToLower("Stuff"));
EXPECT_EQ("stuff and things", IncludesNormalize::ToLower("Stuff AND thINGS"));
- EXPECT_EQ("stuff 3and thin43gs", IncludesNormalize::ToLower("Stuff 3AND thIN43GS"));
+ EXPECT_EQ("stuff 3and thin43gs",
+ IncludesNormalize::ToLower("Stuff 3AND thIN43GS"));
}
TEST(IncludesNormalize, DifferentDrive) {
case NEWLINE: return "newline";
case PIPE2: return "'||'";
case PIPE: return "'|'";
+ case POOL: return "'pool'";
case RULE: return "'rule'";
case SUBNINJA: return "'subninja'";
case TEOF: return "eof";
};
yych = *p;
- if (yych <= 'Z') {
+ if (yych <= '^') {
if (yych <= ',') {
if (yych <= 0x1F) {
- if (yych <= 0x00) goto yy21;
+ if (yych <= 0x00) goto yy22;
if (yych == '\n') goto yy6;
- goto yy23;
+ goto yy24;
} else {
if (yych <= ' ') goto yy2;
if (yych == '#') goto yy4;
- goto yy23;
+ goto yy24;
}
} else {
if (yych <= ':') {
- if (yych == '/') goto yy23;
- if (yych <= '9') goto yy20;
- goto yy14;
+ if (yych == '/') goto yy24;
+ if (yych <= '9') goto yy21;
+ goto yy15;
} else {
- if (yych == '=') goto yy12;
- if (yych <= '@') goto yy23;
- goto yy20;
+ if (yych <= '=') {
+ if (yych <= '<') goto yy24;
+ goto yy13;
+ } else {
+ if (yych <= '@') goto yy24;
+ if (yych <= 'Z') goto yy21;
+ goto yy24;
+ }
}
}
} else {
- if (yych <= 'h') {
- if (yych <= 'a') {
- if (yych == '_') goto yy20;
- if (yych <= '`') goto yy23;
- goto yy20;
+ if (yych <= 'i') {
+ if (yych <= 'b') {
+ if (yych == '`') goto yy24;
+ if (yych <= 'a') goto yy21;
+ goto yy8;
} else {
- if (yych <= 'b') goto yy8;
- if (yych == 'd') goto yy11;
- goto yy20;
+ if (yych == 'd') goto yy12;
+ if (yych <= 'h') goto yy21;
+ goto yy19;
}
} else {
- if (yych <= 's') {
- if (yych <= 'i') goto yy18;
- if (yych <= 'q') goto yy20;
- if (yych <= 'r') goto yy10;
- goto yy19;
+ if (yych <= 'r') {
+ if (yych == 'p') goto yy10;
+ if (yych <= 'q') goto yy21;
+ goto yy11;
} else {
- if (yych <= 'z') goto yy20;
- if (yych == '|') goto yy16;
- goto yy23;
+ if (yych <= 'z') {
+ if (yych <= 's') goto yy20;
+ goto yy21;
+ } else {
+ if (yych == '|') goto yy17;
+ goto yy24;
+ }
}
}
}
yy2:
yyaccept = 0;
yych = *(q = ++p);
- goto yy65;
+ goto yy70;
yy3:
{ token = INDENT; break; }
yy4:
yyaccept = 1;
yych = *(q = ++p);
if (yych <= 0x00) goto yy5;
- if (yych != '\r') goto yy60;
+ if (yych != '\r') goto yy65;
yy5:
{ token = ERROR; break; }
yy6:
{ token = NEWLINE; break; }
yy8:
++p;
- if ((yych = *p) == 'u') goto yy54;
- goto yy25;
+ if ((yych = *p) == 'u') goto yy59;
+ goto yy26;
yy9:
{ token = IDENT; break; }
yy10:
yych = *++p;
- if (yych == 'u') goto yy50;
- goto yy25;
+ if (yych == 'o') goto yy55;
+ goto yy26;
yy11:
yych = *++p;
- if (yych == 'e') goto yy43;
- goto yy25;
+ if (yych == 'u') goto yy51;
+ goto yy26;
yy12:
+ yych = *++p;
+ if (yych == 'e') goto yy44;
+ goto yy26;
+yy13:
++p;
{ token = EQUALS; break; }
-yy14:
+yy15:
++p;
{ token = COLON; break; }
-yy16:
+yy17:
++p;
- if ((yych = *p) == '|') goto yy41;
+ if ((yych = *p) == '|') goto yy42;
{ token = PIPE; break; }
-yy18:
- yych = *++p;
- if (yych == 'n') goto yy34;
- goto yy25;
yy19:
yych = *++p;
- if (yych == 'u') goto yy26;
- goto yy25;
+ if (yych == 'n') goto yy35;
+ goto yy26;
yy20:
yych = *++p;
- goto yy25;
+ if (yych == 'u') goto yy27;
+ goto yy26;
yy21:
+ yych = *++p;
+ goto yy26;
+yy22:
++p;
{ token = TEOF; break; }
-yy23:
+yy24:
yych = *++p;
goto yy5;
-yy24:
+yy25:
++p;
yych = *p;
-yy25:
+yy26:
if (yybm[0+yych] & 32) {
- goto yy24;
+ goto yy25;
}
goto yy9;
-yy26:
+yy27:
yych = *++p;
- if (yych != 'b') goto yy25;
+ if (yych != 'b') goto yy26;
yych = *++p;
- if (yych != 'n') goto yy25;
+ if (yych != 'n') goto yy26;
yych = *++p;
- if (yych != 'i') goto yy25;
+ if (yych != 'i') goto yy26;
yych = *++p;
- if (yych != 'n') goto yy25;
+ if (yych != 'n') goto yy26;
yych = *++p;
- if (yych != 'j') goto yy25;
+ if (yych != 'j') goto yy26;
yych = *++p;
- if (yych != 'a') goto yy25;
+ if (yych != 'a') goto yy26;
++p;
if (yybm[0+(yych = *p)] & 32) {
- goto yy24;
+ goto yy25;
}
{ token = SUBNINJA; break; }
-yy34:
+yy35:
yych = *++p;
- if (yych != 'c') goto yy25;
+ if (yych != 'c') goto yy26;
yych = *++p;
- if (yych != 'l') goto yy25;
+ if (yych != 'l') goto yy26;
yych = *++p;
- if (yych != 'u') goto yy25;
+ if (yych != 'u') goto yy26;
yych = *++p;
- if (yych != 'd') goto yy25;
+ if (yych != 'd') goto yy26;
yych = *++p;
- if (yych != 'e') goto yy25;
+ if (yych != 'e') goto yy26;
++p;
if (yybm[0+(yych = *p)] & 32) {
- goto yy24;
+ goto yy25;
}
{ token = INCLUDE; break; }
-yy41:
+yy42:
++p;
{ token = PIPE2; break; }
-yy43:
+yy44:
yych = *++p;
- if (yych != 'f') goto yy25;
+ if (yych != 'f') goto yy26;
yych = *++p;
- if (yych != 'a') goto yy25;
+ if (yych != 'a') goto yy26;
yych = *++p;
- if (yych != 'u') goto yy25;
+ if (yych != 'u') goto yy26;
yych = *++p;
- if (yych != 'l') goto yy25;
+ if (yych != 'l') goto yy26;
yych = *++p;
- if (yych != 't') goto yy25;
+ if (yych != 't') goto yy26;
++p;
if (yybm[0+(yych = *p)] & 32) {
- goto yy24;
+ goto yy25;
}
{ token = DEFAULT; break; }
-yy50:
+yy51:
yych = *++p;
- if (yych != 'l') goto yy25;
+ if (yych != 'l') goto yy26;
yych = *++p;
- if (yych != 'e') goto yy25;
+ if (yych != 'e') goto yy26;
++p;
if (yybm[0+(yych = *p)] & 32) {
- goto yy24;
+ goto yy25;
}
{ token = RULE; break; }
-yy54:
+yy55:
yych = *++p;
- if (yych != 'i') goto yy25;
+ if (yych != 'o') goto yy26;
yych = *++p;
- if (yych != 'l') goto yy25;
+ if (yych != 'l') goto yy26;
+ ++p;
+ if (yybm[0+(yych = *p)] & 32) {
+ goto yy25;
+ }
+ { token = POOL; break; }
+yy59:
+ yych = *++p;
+ if (yych != 'i') goto yy26;
yych = *++p;
- if (yych != 'd') goto yy25;
+ if (yych != 'l') goto yy26;
+ yych = *++p;
+ if (yych != 'd') goto yy26;
++p;
if (yybm[0+(yych = *p)] & 32) {
- goto yy24;
+ goto yy25;
}
{ token = BUILD; break; }
-yy59:
+yy64:
++p;
yych = *p;
-yy60:
+yy65:
if (yybm[0+yych] & 64) {
- goto yy59;
+ goto yy64;
}
- if (yych <= 0x00) goto yy61;
- if (yych <= '\f') goto yy62;
-yy61:
+ if (yych <= 0x00) goto yy66;
+ if (yych <= '\f') goto yy67;
+yy66:
p = q;
if (yyaccept <= 0) {
goto yy3;
} else {
goto yy5;
}
-yy62:
+yy67:
++p;
{ continue; }
-yy64:
+yy69:
yyaccept = 0;
q = ++p;
yych = *p;
-yy65:
+yy70:
if (yybm[0+yych] & 128) {
- goto yy64;
+ goto yy69;
}
- if (yych == '\n') goto yy66;
- if (yych == '#') goto yy59;
+ if (yych == '\n') goto yy71;
+ if (yych == '#') goto yy64;
goto yy3;
-yy66:
+yy71:
++p;
yych = *p;
goto yy7;
};
yych = *p;
if (yych <= ' ') {
- if (yych <= 0x00) goto yy73;
- if (yych <= 0x1F) goto yy75;
+ if (yych <= 0x00) goto yy78;
+ if (yych <= 0x1F) goto yy80;
} else {
- if (yych == '$') goto yy71;
- goto yy75;
+ if (yych == '$') goto yy76;
+ goto yy80;
}
++p;
yych = *p;
- goto yy79;
-yy70:
+ goto yy84;
+yy75:
{ continue; }
-yy71:
+yy76:
++p;
- if ((yych = *p) == '\n') goto yy76;
-yy72:
+ if ((yych = *p) == '\n') goto yy81;
+yy77:
{ break; }
-yy73:
+yy78:
++p;
{ break; }
-yy75:
+yy80:
yych = *++p;
- goto yy72;
-yy76:
+ goto yy77;
+yy81:
++p;
{ continue; }
-yy78:
+yy83:
++p;
yych = *p;
-yy79:
+yy84:
if (yybm[0+yych] & 128) {
- goto yy78;
+ goto yy83;
}
- goto yy70;
+ goto yy75;
}
}
yych = *p;
if (yych <= '@') {
if (yych <= '.') {
- if (yych <= ',') goto yy84;
+ if (yych <= ',') goto yy89;
} else {
- if (yych <= '/') goto yy84;
- if (yych >= ':') goto yy84;
+ if (yych <= '/') goto yy89;
+ if (yych >= ':') goto yy89;
}
} else {
if (yych <= '_') {
- if (yych <= 'Z') goto yy82;
- if (yych <= '^') goto yy84;
+ if (yych <= 'Z') goto yy87;
+ if (yych <= '^') goto yy89;
} else {
- if (yych <= '`') goto yy84;
- if (yych >= '{') goto yy84;
+ if (yych <= '`') goto yy89;
+ if (yych >= '{') goto yy89;
}
}
-yy82:
+yy87:
++p;
yych = *p;
- goto yy87;
-yy83:
+ goto yy92;
+yy88:
{
out->assign(start, p - start);
break;
}
-yy84:
+yy89:
++p;
{ return false; }
-yy86:
+yy91:
++p;
yych = *p;
-yy87:
+yy92:
if (yybm[0+yych] & 128) {
- goto yy86;
+ goto yy91;
}
- goto yy83;
+ goto yy88;
}
}
yych = *p;
if (yych <= ' ') {
if (yych <= '\n') {
- if (yych <= 0x00) goto yy96;
- if (yych >= '\n') goto yy92;
+ if (yych <= 0x00) goto yy101;
+ if (yych >= '\n') goto yy97;
} else {
- if (yych == '\r') goto yy98;
- if (yych >= ' ') goto yy92;
+ if (yych == '\r') goto yy103;
+ if (yych >= ' ') goto yy97;
}
} else {
if (yych <= '9') {
- if (yych == '$') goto yy94;
+ if (yych == '$') goto yy99;
} else {
- if (yych <= ':') goto yy92;
- if (yych == '|') goto yy92;
+ if (yych <= ':') goto yy97;
+ if (yych == '|') goto yy97;
}
}
++p;
yych = *p;
- goto yy121;
-yy91:
+ goto yy126;
+yy96:
{
eval->AddText(StringPiece(start, p - start));
continue;
}
-yy92:
+yy97:
++p;
{
if (path) {
continue;
}
}
-yy94:
+yy99:
++p;
if ((yych = *p) <= '/') {
if (yych <= ' ') {
- if (yych == '\n') goto yy110;
- if (yych <= 0x1F) goto yy99;
- goto yy101;
+ if (yych == '\n') goto yy115;
+ if (yych <= 0x1F) goto yy104;
+ goto yy106;
} else {
if (yych <= '$') {
- if (yych <= '#') goto yy99;
- goto yy103;
+ if (yych <= '#') goto yy104;
+ goto yy108;
} else {
- if (yych == '-') goto yy105;
- goto yy99;
+ if (yych == '-') goto yy110;
+ goto yy104;
}
}
} else {
if (yych <= '^') {
if (yych <= ':') {
- if (yych <= '9') goto yy105;
- goto yy107;
+ if (yych <= '9') goto yy110;
+ goto yy112;
} else {
- if (yych <= '@') goto yy99;
- if (yych <= 'Z') goto yy105;
- goto yy99;
+ if (yych <= '@') goto yy104;
+ if (yych <= 'Z') goto yy110;
+ goto yy104;
}
} else {
if (yych <= '`') {
- if (yych <= '_') goto yy105;
- goto yy99;
+ if (yych <= '_') goto yy110;
+ goto yy104;
} else {
- if (yych <= 'z') goto yy105;
- if (yych <= '{') goto yy109;
- goto yy99;
+ if (yych <= 'z') goto yy110;
+ if (yych <= '{') goto yy114;
+ goto yy104;
}
}
}
-yy95:
+yy100:
{
last_token_ = start;
return Error(DescribeLastError(), err);
}
-yy96:
+yy101:
++p;
{
last_token_ = start;
return Error("unexpected EOF", err);
}
-yy98:
+yy103:
yych = *++p;
- goto yy95;
-yy99:
+ goto yy100;
+yy104:
++p;
-yy100:
+yy105:
{
last_token_ = start;
return Error("bad $-escape (literal $ must be written as $$)", err);
}
-yy101:
+yy106:
++p;
{
eval->AddText(StringPiece(" ", 1));
continue;
}
-yy103:
+yy108:
++p;
{
eval->AddText(StringPiece("$", 1));
continue;
}
-yy105:
+yy110:
++p;
yych = *p;
- goto yy119;
-yy106:
+ goto yy124;
+yy111:
{
eval->AddSpecial(StringPiece(start + 1, p - start - 1));
continue;
}
-yy107:
+yy112:
++p;
{
eval->AddText(StringPiece(":", 1));
continue;
}
-yy109:
+yy114:
yych = *(q = ++p);
if (yybm[0+yych] & 32) {
- goto yy113;
+ goto yy118;
}
- goto yy100;
-yy110:
+ goto yy105;
+yy115:
++p;
yych = *p;
if (yybm[0+yych] & 16) {
- goto yy110;
+ goto yy115;
}
{
continue;
}
-yy113:
+yy118:
++p;
yych = *p;
if (yybm[0+yych] & 32) {
- goto yy113;
+ goto yy118;
}
- if (yych == '}') goto yy116;
+ if (yych == '}') goto yy121;
p = q;
- goto yy100;
-yy116:
+ goto yy105;
+yy121:
++p;
{
eval->AddSpecial(StringPiece(start + 2, p - start - 3));
continue;
}
-yy118:
+yy123:
++p;
yych = *p;
-yy119:
+yy124:
if (yybm[0+yych] & 64) {
- goto yy118;
+ goto yy123;
}
- goto yy106;
-yy120:
+ goto yy111;
+yy125:
++p;
yych = *p;
-yy121:
+yy126:
if (yybm[0+yych] & 128) {
- goto yy120;
+ goto yy125;
}
- goto yy91;
+ goto yy96;
}
}
NEWLINE,
PIPE,
PIPE2,
+ POOL,
RULE,
SUBNINJA,
TEOF,
case NEWLINE: return "newline";
case PIPE2: return "'||'";
case PIPE: return "'|'";
+ case POOL: return "'pool'";
case RULE: return "'rule'";
case SUBNINJA: return "'subninja'";
case TEOF: return "eof";
[ ]*[\n] { token = NEWLINE; break; }
[ ]+ { token = INDENT; break; }
"build" { token = BUILD; break; }
+ "pool" { token = POOL; break; }
"rule" { token = RULE; break; }
"default" { token = DEFAULT; break; }
"=" { token = EQUALS; break; }
for (;;) {
Lexer::Token token = lexer_.ReadToken();
switch (token) {
+ case Lexer::POOL:
+ if (!ParsePool(err))
+ return false;
+ break;
case Lexer::BUILD:
if (!ParseEdge(err))
return false;
return false; // not reached
}
+
+bool ManifestParser::ParsePool(string* err) {
+ string name;
+ if (!lexer_.ReadIdent(&name))
+ return lexer_.Error("expected pool name", err);
+
+ if (!ExpectToken(Lexer::NEWLINE, err))
+ return false;
+
+ if (state_->LookupPool(name) != NULL)
+ return lexer_.Error("duplicate pool '" + name + "'", err);
+
+ int depth = -1;
+
+ while (lexer_.PeekToken(Lexer::INDENT)) {
+ string key;
+ EvalString value;
+ if (!ParseLet(&key, &value, err))
+ return false;
+
+ if (key == "depth") {
+ string depth_string = value.Evaluate(env_);
+ depth = atol(depth_string.c_str());
+ if (depth < 0)
+ return lexer_.Error("invalid pool depth", err);
+ } else {
+ return lexer_.Error("unexpected variable '" + key + "'", err);
+ }
+ }
+
+ if (depth < 0)
+ return lexer_.Error("expected 'depth =' line", err);
+
+ state_->AddPool(new Pool(name, depth));
+ return true;
+}
+
+
bool ManifestParser::ParseRule(string* err) {
string name;
if (!lexer_.ReadIdent(&name))
rule->rspfile_ = value;
} else if (key == "rspfile_content") {
rule->rspfile_content_ = value;
+ } else if (key == "pool") {
+ rule->pool_ = value;
} else {
// Die on other keyvals for now; revisit if we want to add a
// scope here.
}
}
- if (rule->rspfile_.empty() != rule->rspfile_content_.empty())
- return lexer_.Error("rspfile and rspfile_content need to be both specified", err);
+ if (rule->rspfile_.empty() != rule->rspfile_content_.empty()) {
+ return lexer_.Error("rspfile and rspfile_content need to be both specified",
+ err);
+ }
if (rule->command_.empty())
return lexer_.Error("expected 'command =' line", err);
// Default to using outer env.
BindingEnv* env = env_;
+ Pool* pool = NULL;
// But create and fill a nested env if there are variables in scope.
if (lexer_.PeekToken(Lexer::INDENT)) {
EvalString val;
if (!ParseLet(&key, &val, err))
return false;
- env->AddBinding(key, val.Evaluate(env_));
+ if (key == "pool") {
+ string pool_name = val.Evaluate(env_);
+ pool = state_->LookupPool(pool_name);
+ if (pool == NULL)
+ return lexer_.Error("undefined pool '" + pool_name + "'", err);
+ } else {
+ env->AddBinding(key, val.Evaluate(env_));
+ }
} while (lexer_.PeekToken(Lexer::INDENT));
}
- Edge* edge = state_->AddEdge(rule);
+ if (pool == NULL) {
+ if (!rule->pool_.empty()) {
+ pool = state_->LookupPool(rule->pool_.Evaluate(env_));
+ if (pool == NULL)
+ return lexer_.Error("cannot resolve pool for this edge.", err);
+ } else {
+ pool = &State::kDefaultPool;
+ }
+ }
+
+ Edge* edge = state_->AddEdge(rule, pool);
edge->env_ = env;
for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) {
string path = i->Evaluate(env);
bool Parse(const string& filename, const string& input, string* err);
/// Parse various statement types.
+ bool ParsePool(string* err);
bool ParseRule(string* err);
bool ParseLet(string* key, EvalString* val, string* err);
bool ParseEdge(string* err);
};
/// Get the current time as relative to some epoch.
-/// Epoch varies between platforms; only useful for measuring elapsed
-/// time.
+/// Epoch varies between platforms; only useful for measuring elapsed time.
int64_t GetTimeMillis();
-
-/// A simple stopwatch which retruns the time
-// in seconds since Restart() was called
-class Stopwatch
-{
-public:
+/// A simple stopwatch which returns the time
+/// in seconds since Restart() was called.
+struct Stopwatch {
+ public:
Stopwatch() : started_(0) {}
- /// Seconds since Restart() call
- double Elapsed() const { return 1e-6 * static_cast<double>(Now() - started_); }
+ /// Seconds since Restart() call.
+ double Elapsed() const {
+ return 1e-6 * static_cast<double>(Now() - started_);
+ }
void Restart() { started_ = Now(); }
-private:
+ private:
uint64_t started_;
uint64_t Now() const;
};
-
/// The primary interface to metrics. Use METRIC_RECORD("foobar") at the top
/// of a function to get timing stats recorded for each call of the function.
#define METRIC_RECORD(name) \
#include "msvc_helper.h"
+#include <stdio.h>
#include <string.h>
#include <windows.h>
input.substr(input.size() - needle.size()) == needle);
}
+string Replace(const string& input, const string& find, const string& replace) {
+ string result = input;
+ size_t start_pos = 0;
+ while ((start_pos = result.find(find, start_pos)) != string::npos) {
+ result.replace(start_pos, find.length(), replace);
+ start_pos += replace.length();
+ }
+ return result;
+}
+
+string EscapeForDepfile(const string& path) {
+ // Depfiles don't escape single \.
+ return Replace(path, " ", "\\ ");
+}
+
} // anonymous namespace
// static
if (!include.empty()) {
include = IncludesNormalize::Normalize(include, NULL);
if (!IsSystemInclude(include))
- includes_.push_back(include);
+ includes_.insert(include);
} else if (FilterInputFilename(line)) {
// Drop it.
// TODO: if we support compiling multiple output files in a single
return exit_code;
}
+
+vector<string> CLWrapper::GetEscapedResult() {
+ vector<string> result;
+ for (set<string>::iterator i = includes_.begin(); i != includes_.end(); ++i) {
+ result.push_back(EscapeForDepfile(*i));
+ }
+ return result;
+}
// limitations under the License.
#include <string>
+#include <set>
#include <vector>
using namespace std;
/// Exposed for testing.
static bool FilterInputFilename(const string& line);
+ /// Fill a vector with the unique'd headers, escaped for output as a .d
+ /// file.
+ vector<string> GetEscapedResult();
+
void* env_block_;
- vector<string> includes_;
+ set<string> includes_;
};
#include "msvc_helper.h"
+#include <stdio.h>
#include <windows.h>
#include "util.h"
"usage: ninja -t msvc [options] -- cl.exe /showIncludes /otherArgs\n"
"options:\n"
" -e ENVFILE load environment block from ENVFILE as environment\n"
-" -r BASE normalize paths and make relative to BASE before output\n"
" -o FILE write output dependency information to FILE.d\n"
);
}
int MSVCHelperMain(int argc, char** argv) {
const char* output_filename = NULL;
- const char* relative_to = NULL;
const char* envfile = NULL;
const option kLongOptions[] = {
{ NULL, 0, NULL, 0 }
};
int opt;
- while ((opt = getopt_long(argc, argv, "e:o:r:h", kLongOptions, NULL)) != -1) {
+ while ((opt = getopt_long(argc, argv, "e:o:h", kLongOptions, NULL)) != -1) {
switch (opt) {
case 'e':
envfile = optarg;
case 'o':
output_filename = optarg;
break;
- case 'r':
- relative_to = optarg;
- break;
case 'h':
default:
Usage();
Fatal("opening %s: %s", depfile.c_str(), GetLastErrorString().c_str());
}
fprintf(output, "%s: ", output_filename);
- for (vector<string>::iterator i = cl.includes_.begin();
- i != cl.includes_.end(); ++i) {
+ vector<string> headers = cl.GetEscapedResult();
+ for (vector<string>::iterator i = headers.begin(); i != headers.end(); ++i) {
fprintf(output, "%s\n", i->c_str());
}
fclose(output);
&output);
ASSERT_EQ("foo\nbar\n", output);
ASSERT_EQ(1u, cl.includes_.size());
- ASSERT_EQ("foo.h", cl.includes_[0]);
+ ASSERT_EQ("foo.h", *cl.includes_.begin());
}
TEST(MSVCHelperTest, RunFilenameFilter) {
// system headers.
ASSERT_EQ("", output);
ASSERT_EQ(1u, cl.includes_.size());
- ASSERT_EQ("path.h", cl.includes_[0]);
+ ASSERT_EQ("path.h", *cl.includes_.begin());
}
TEST(MSVCHelperTest, EnvBlock) {
cl.Run("cmd /c \"echo foo is %foo%", &output);
ASSERT_EQ("foo is bar\n", output);
}
+
+TEST(MSVCHelperTest, DuplicatedHeader) {
+ CLWrapper cl;
+ string output;
+ cl.Run("cmd /c \"echo Note: including file: foo.h&&"
+ "echo Note: including file: bar.h&&"
+ "echo Note: including file: foo.h\"",
+ &output);
+ // We should have dropped one copy of foo.h.
+ ASSERT_EQ("", output);
+ ASSERT_EQ(2u, cl.includes_.size());
+}
+
+TEST(MSVCHelperTest, DuplicatedHeaderPathConverted) {
+ CLWrapper cl;
+ string output;
+ cl.Run("cmd /c \"echo Note: including file: sub/foo.h&&"
+ "echo Note: including file: bar.h&&"
+ "echo Note: including file: sub\\foo.h\"",
+ &output);
+ // We should have dropped one copy of foo.h.
+ ASSERT_EQ("", output);
+ ASSERT_EQ(2u, cl.includes_.size());
+}
+
+TEST(MSVCHelperTest, SpacesInFilename) {
+ CLWrapper cl;
+ string output;
+ cl.Run("cmd /c \"echo Note: including file: sub\\some sdk\\foo.h",
+ &output);
+ ASSERT_EQ("", output);
+ vector<string> headers = cl.GetEscapedResult();
+ ASSERT_EQ(1u, headers.size());
+ ASSERT_EQ("sub\\some\\ sdk\\foo.h", headers[0]);
+}
/// The version number of the current Ninja release. This will always
/// be "git" on trunk.
-const char* kVersion = "1.0.0";
+const char* kVersion = "1.1.0";
/// Global information passed into subtools.
struct Globals {
/// An implementation of ManifestParser::FileReader that actually reads
/// the file.
struct RealFileReader : public ManifestParser::FileReader {
- bool ReadFile(const string& path, string* content, string* err) {
+ virtual bool ReadFile(const string& path, string* content, string* err) {
return ::ReadFile(path, content, err) == 0;
}
};
return node->dirty();
}
+Node* CollectTarget(State* state, const char* cpath, string* err) {
+ string path = cpath;
+ if (!CanonicalizePath(&path, err))
+ return NULL;
+
+ // Special syntax: "foo.cc^" means "the first output of foo.cc".
+ bool first_dependent = false;
+ if (!path.empty() && path[path.size() - 1] == '^') {
+ path.resize(path.size() - 1);
+ first_dependent = true;
+ }
+
+ Node* node = state->LookupNode(path);
+ if (node) {
+ if (first_dependent) {
+ if (node->out_edges().empty()) {
+ *err = "'" + path + "' has no out edge";
+ return NULL;
+ }
+ Edge* edge = node->out_edges()[0];
+ if (edge->outputs_.empty()) {
+ edge->Dump();
+ Fatal("edge has no outputs");
+ }
+ node = edge->outputs_[0];
+ }
+ return node;
+ } else {
+ *err = "unknown target '" + path + "'";
+
+ if (path == "clean") {
+ *err += ", did you mean 'ninja -t clean'?";
+ } else if (path == "help") {
+ *err += ", did you mean 'ninja -h'?";
+ } else {
+ Node* suggestion = state->SpellcheckNode(path);
+ if (suggestion) {
+ *err += ", did you mean '" + suggestion->path() + "'?";
+ }
+ }
+ return NULL;
+ }
+}
+
bool CollectTargetsFromArgs(State* state, int argc, char* argv[],
vector<Node*>* targets, string* err) {
if (argc == 0) {
}
for (int i = 0; i < argc; ++i) {
- string path = argv[i];
- if (!CanonicalizePath(&path, err))
+ Node* node = CollectTarget(state, argv[i], err);
+ if (node == NULL)
return false;
-
- // Special syntax: "foo.cc^" means "the first output of foo.cc".
- bool first_dependent = false;
- if (!path.empty() && path[path.size() - 1] == '^') {
- path.resize(path.size() - 1);
- first_dependent = true;
- }
-
- Node* node = state->LookupNode(path);
- if (node) {
- if (first_dependent) {
- if (node->out_edges().empty()) {
- *err = "'" + path + "' has no out edge";
- return false;
- }
- Edge* edge = node->out_edges()[0];
- if (edge->outputs_.empty()) {
- edge->Dump();
- Fatal("edge has no outputs");
- }
- node = edge->outputs_[0];
- }
- targets->push_back(node);
- } else {
- *err = "unknown target '" + path + "'";
-
- if (path == "clean") {
- *err += ", did you mean 'ninja -t clean'?";
- } else if (path == "help") {
- *err += ", did you mean 'ninja -h'?";
- } else {
- Node* suggestion = state->SpellcheckNode(path);
- if (suggestion) {
- *err += ", did you mean '" + suggestion->path() + "'?";
- }
- }
- return false;
- }
+ targets->push_back(node);
}
return true;
}
return 1;
}
for (int i = 0; i < argc; ++i) {
- Node* node = globals->state->LookupNode(argv[i]);
+ string err;
+ Node* node = CollectTarget(globals->state, argv[i], &err);
if (!node) {
- Node* suggestion = globals->state->SpellcheckNode(argv[i]);
- if (suggestion) {
- printf("%s unknown, did you mean %s?\n",
- argv[i], suggestion->path().c_str());
- } else {
- printf("%s unknown\n", argv[i]);
- }
+ Error("%s", err.c_str());
return 1;
}
- printf("%s:\n", argv[i]);
+ printf("%s:\n", node->path().c_str());
if (Edge* edge = node->in_edge()) {
printf(" input: %s\n", edge->rule_->name().c_str());
for (int in = 0; in < (int)edge->inputs_.size(); in++) {
}
#endif // _WIN32
-#if defined(WIN32)
+#if defined(_WIN32)
int ToolMSVC(Globals* globals, int argc, char* argv[]) {
// Reset getopt: push one argument onto the front of argv, reset optind.
argc++;
{ "browse", "browse dependency graph in a web browser",
Tool::RUN_AFTER_LOAD, ToolBrowse },
#endif
-#if defined(WIN32)
+#if defined(_WIN32)
{ "msvc", "build helper for MSVC cl.exe (EXPERIMENTAL)",
Tool::RUN_AFTER_FLAGS, ToolMSVC },
#endif
if (!builder->Build(&err)) {
printf("ninja: build stopped: %s.\n", err.c_str());
+ if (err.find("interrupted by user") != string::npos) {
+ return 2;
+ }
return 1;
}
int opt;
while (tool_name.empty() &&
- (opt = getopt_long(argc, argv, "d:f:hj:k:l:nt:vC:V", kLongOptions,
+ (opt = getopt_long(argc, argv, "d:f:j:k:l:nt:vC:h", kLongOptions,
NULL)) != -1) {
switch (opt) {
case 'd':
case 'j':
config.parallelism = atoi(optarg);
break;
- case 'l': {
- char* end;
- double value = strtod(optarg, &end);
- if (end == optarg)
- Fatal("-l parameter not numeric: did you mean -l 0.0?");
- config.max_load_average = value;
- break;
- }
case 'k': {
char* end;
int value = strtol(optarg, &end, 10);
config.failures_allowed = value > 0 ? value : INT_MAX;
break;
}
+ case 'l': {
+ char* end;
+ double value = strtod(optarg, &end);
+ if (end == optarg)
+ Fatal("-l parameter not numeric: did you mean -l 0.0?");
+ config.max_load_average = value;
+ break;
+ }
case 'n':
config.dry_run = true;
break;
- case 'v':
- config.verbosity = BuildConfig::VERBOSE;
- break;
case 't':
tool_name = optarg;
break;
+ case 'v':
+ config.verbosity = BuildConfig::VERBOSE;
+ break;
case 'C':
working_dir = optarg;
break;
bool rebuilt_manifest = false;
reload:
- RealDiskInterface disk_interface;
RealFileReader file_reader;
ManifestParser parser(globals.state, &file_reader);
string err;
return tool->func(&globals, argc, argv);
BuildLog build_log;
+ RealDiskInterface disk_interface;
if (!OpenLog(&build_log, &globals, &disk_interface))
return 1;
#include "metrics.h"
#include "util.h"
+
+void Pool::EdgeScheduled(const Edge& edge) {
+ if (depth_ != 0)
+ current_use_ += edge.weight();
+}
+
+void Pool::EdgeFinished(const Edge& edge) {
+ if (depth_ != 0)
+ current_use_ -= edge.weight();
+}
+
+void Pool::DelayEdge(Edge* edge) {
+ assert(depth_ != 0);
+ delayed_.push_back(edge);
+}
+
+void Pool::RetrieveReadyEdges(set<Edge*>* ready_queue) {
+ while (!delayed_.empty()) {
+ Edge* edge = delayed_.front();
+ if (current_use_ + edge->weight() > depth_)
+ break;
+ delayed_.pop_front();
+ ready_queue->insert(edge);
+ EdgeScheduled(*edge);
+ }
+}
+
+void Pool::Dump() const {
+ printf("%s (%d/%d) ->\n", name_.c_str(), current_use_, depth_);
+ for (deque<Edge*>::const_iterator it = delayed_.begin();
+ it != delayed_.end(); ++it)
+ {
+ printf("\t");
+ (*it)->Dump();
+ }
+}
+
+Pool State::kDefaultPool("", 0);
const Rule State::kPhonyRule("phony");
State::State() {
AddRule(&kPhonyRule);
+ AddPool(&kDefaultPool);
}
void State::AddRule(const Rule* rule) {
return i->second;
}
-Edge* State::AddEdge(const Rule* rule) {
+void State::AddPool(Pool* pool) {
+ assert(LookupPool(pool->name()) == NULL);
+ pools_[pool->name()] = pool;
+}
+
+Pool* State::LookupPool(const string& pool_name) {
+ map<string, Pool*>::iterator i = pools_.find(pool_name);
+ if (i == pools_.end())
+ return NULL;
+ return i->second;
+}
+
+Edge* State::AddEdge(const Rule* rule, Pool* pool) {
Edge* edge = new Edge();
edge->rule_ = rule;
+ edge->pool_ = pool;
edge->env_ = &bindings_;
edges_.push_back(edge);
return edge;
node->status_known() ? (node->dirty() ? "dirty" : "clean")
: "unknown");
}
+ if (!pools_.empty()) {
+ printf("resource_pools:\n");
+ for (map<string, Pool*>::const_iterator it = pools_.begin();
+ it != pools_.end(); ++it)
+ {
+ if (!it->second->name().empty()) {
+ it->second->Dump();
+ }
+ }
+ }
}
#define NINJA_STATE_H_
#include <map>
+#include <deque>
+#include <set>
#include <string>
#include <vector>
using namespace std;
struct Node;
struct Rule;
+/// A pool for delayed edges.
+/// Pools are scoped to a State. Edges within a State will share Pools. A Pool
+/// will keep a count of the total 'weight' of the currently scheduled edges. If
+/// a Plan attempts to schedule an Edge which would cause the total weight to
+/// exceed the depth of the Pool, the Pool will enque the Edge instead of
+/// allowing the Plan to schedule it. The Pool will relinquish queued Edges when
+/// the total scheduled weight diminishes enough (i.e. when a scheduled edge
+/// completes).
+struct Pool {
+ explicit Pool(const string& name, int depth)
+ : name_(name), current_use_(0), depth_(depth) { }
+
+ // A depth of 0 is infinite
+ bool is_valid() const { return depth_ >= 0; }
+ int depth() const { return depth_; }
+ const string& name() const { return name_; }
+
+ /// true if the Pool might delay this edge
+ bool ShouldDelayEdge() const { return depth_ != 0; }
+
+ /// informs this Pool that the given edge is committed to be run.
+ /// Pool will count this edge as using resources from this pool.
+ void EdgeScheduled(const Edge& edge);
+
+ /// informs this Pool that the given edge is no longer runnable, and should
+ /// relinquish its resources back to the pool
+ void EdgeFinished(const Edge& edge);
+
+ /// adds the given edge to this Pool to be delayed.
+ void DelayEdge(Edge* edge);
+
+ /// Pool will add zero or more edges to the ready_queue
+ void RetrieveReadyEdges(set<Edge*>* ready_queue);
+
+ /// Dump the Pool and its edges (useful for debugging).
+ void Dump() const;
+
+private:
+ int UnitsWaiting() { return delayed_.size(); }
+
+ string name_;
+
+ /// |current_use_| is the total of the weights of the edges which are
+ /// currently scheduled in the Plan (i.e. the edges in Plan::ready_).
+ int current_use_;
+ int depth_;
+
+ deque<Edge*> delayed_;
+};
+
/// Global state (file status, loaded rules) for a single run.
struct State {
+ static Pool kDefaultPool;
static const Rule kPhonyRule;
State();
void AddRule(const Rule* rule);
const Rule* LookupRule(const string& rule_name);
- Edge* AddEdge(const Rule* rule);
+ void AddPool(Pool* pool);
+ Pool* LookupPool(const string& pool_name);
+
+ Edge* AddEdge(const Rule* rule, Pool* pool);
Node* GetNode(StringPiece path);
Node* LookupNode(StringPiece path);
/// state where we haven't yet examined the disk for dirty state.
void Reset();
- /// Dump the nodes (useful for debugging).
+ /// Dump the nodes and Pools (useful for debugging).
void Dump();
/// @return the root node(s) of the graph. (Root nodes have no output edges).
/// All the rules used in the graph.
map<string, const Rule*> rules_;
+ /// All the pools used in the graph.
+ map<string, Pool*> pools_;
+
/// All the edges of the graph.
vector<Edge*> edges_;
rule->set_command(command);
state.AddRule(rule);
- Edge* edge = state.AddEdge(rule);
+ Edge* edge = state.AddEdge(rule, &State::kDefaultPool);
state.AddIn(edge, "in1");
state.AddIn(edge, "in2");
state.AddOut(edge, "out");
break;
// Open /dev/null over stdin.
- int devnull = open("/dev/null", O_WRONLY);
+ int devnull = open("/dev/null", O_RDONLY);
if (devnull < 0)
break;
if (dup2(devnull, 0) < 0)
NULL, NULL,
&startup_info, &process_info)) {
DWORD error = GetLastError();
- if (error == ERROR_FILE_NOT_FOUND) { // file (program) not found error is treated as a normal build action failure
+ if (error == ERROR_FILE_NOT_FOUND) {
+ // File (program) not found error is treated as a normal build
+ // action failure.
if (child_pipe)
CloseHandle(child_pipe);
CloseHandle(pipe_);
CloseHandle(nul);
pipe_ = NULL;
// child_ is already NULL;
- buf_ = "CreateProcess failed: The system cannot find the file specified.\n";
+ buf_ = "CreateProcess failed: The system cannot find the file "
+ "specified.\n";
return true;
} else {
Win32Fatal("CreateProcess"); // pass all other errors to Win32Fatal
EXPECT_EQ(ExitFailure, subproc->Finish());
EXPECT_NE("", subproc->GetOutput());
#ifdef _WIN32
- ASSERT_EQ("CreateProcess failed: The system cannot find the file specified.\n", subproc->GetOutput());
+ ASSERT_EQ("CreateProcess failed: The system cannot find the file "
+ "specified.\n", subproc->GetOutput());
#endif
}
ASSERT_EQ(kNumProcs, subprocs_.finished_.size());
}
#endif // linux
+
+// TODO: this test could work on Windows, just not sure how to simply
+// read stdin.
+#ifndef _WIN32
+// Verify that a command that attempts to read stdin correctly thinks
+// that stdin is closed.
+TEST_F(SubprocessTest, ReadStdin) {
+ Subprocess* subproc = subprocs_.Add("cat -");
+ while (!subproc->Done()) {
+ subprocs_.DoWork();
+ }
+ ASSERT_EQ(ExitSuccess, subproc->Finish());
+ ASSERT_EQ(1u, subprocs_.finished_.size());
+}
+#endif // _WIN32
}
if (component_count == kMaxPathComponents)
- Fatal("path has too many components");
+ Fatal("path has too many components : %s", path);
components[component_count] = dst;
++component_count;
/// Given a misspelled string and a list of correct spellings, returns
/// the closest match or NULL if there is no close enough match.
-const char* SpellcheckStringV(const string& text, const vector<const char*>& words);
+const char* SpellcheckStringV(const string& text,
+ const vector<const char*>& words);
/// Like SpellcheckStringV, but takes a NULL-terminated list.
const char* SpellcheckString(const string& text, ...);
projects / platform / upstream / ninja.git / commitdiff