Merge pull request #614 from nico/pooldoc

Mention pools in the discussion of ninja's toplevel declarations.

diff --git a/bootstrap.py b/bootstrap.py
index 5682bf1..66ec85b 100755 (executable)
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -37,7 +37,7 @@ parser.add_option('--platform',
                   help='target platform (' + '/'.join(platform_helper.platforms()) + ')',
                   choices=platform_helper.platforms())
 parser.add_option('--force-pselect', action='store_true',
-                  help="ppoll() is used by default on Linux and OpenBSD, but older versions might need to use pselect instead",)
+                  help="ppoll() is used by default on Linux, OpenBSD and Bitrig, but older versions might need to use pselect instead",)
 (options, conf_args) = parser.parse_args()
 
 
@@ -53,7 +53,7 @@ def run(*args, **kwargs):
 # g++ call as well as in the later configure.py.
 cflags = os.environ.get('CFLAGS', '').split()
 ldflags = os.environ.get('LDFLAGS', '').split()
-if platform.is_freebsd() or platform.is_openbsd():
+if platform.is_freebsd() or platform.is_openbsd() or platform.is_bitrig():
     cflags.append('-I/usr/local/include')
     ldflags.append('-L/usr/local/lib')
 
@@ -109,7 +109,7 @@ else:
         cflags.append('-D_WIN32_WINNT=0x0501')
     if options.x64:
         cflags.append('-m64')
-if (platform.is_linux() or platform.is_openbsd()) and not options.force_pselect:
+if (platform.is_linux() or platform.is_openbsd() or platform.is_bitrig()) and not options.force_pselect:
     cflags.append('-DUSE_PPOLL')
 if options.force_pselect:
     conf_args.append("--force-pselect")

diff --git a/configure.py b/configure.py
index 22eb1e5..c838392 100755 (executable)
--- a/configure.py
+++ b/configure.py
@@ -48,7 +48,7 @@ parser.add_option('--with-python', metavar='EXE',
                   help='use EXE as the Python interpreter',
                   default=os.path.basename(sys.executable))
 parser.add_option('--force-pselect', action='store_true',
-                  help="ppoll() is used by default on Linux and OpenBSD, but older versions might need to use pselect instead",)
+                  help="ppoll() is used by default where available, but some platforms may need to use pselect instead",)
 (options, args) = parser.parse_args()
 if args:
     print('ERROR: extra unparsed command-line arguments:', args)
@@ -165,7 +165,7 @@ else:
         cflags.append('-fno-omit-frame-pointer')
         libs.extend(['-Wl,--no-as-needed', '-lprofiler'])
 
-if (platform.is_linux() or platform.is_openbsd()) and not options.force_pselect:
+if (platform.is_linux() or platform.is_openbsd() or platform.is_bitrig()) and not options.force_pselect:
     cflags.append('-DUSE_PPOLL')
 
 def shell_escape(str):

diff --git a/platform_helper.py b/platform_helper.py
index 5097f49..e615660 100644 (file)
--- a/platform_helper.py
+++ b/platform_helper.py
@@ -19,7 +19,7 @@ import sys
 
 def platforms():
     return ['linux', 'darwin', 'freebsd', 'openbsd', 'solaris', 'sunos5',
-            'mingw', 'msvc', 'gnukfreebsd8']
+            'mingw', 'msvc', 'gnukfreebsd8', 'bitrig']
 
 class Platform( object ):
     def __init__( self, platform):
@@ -41,6 +41,8 @@ class Platform( object ):
             self._platform = 'mingw'
         elif self._platform.startswith('win'):
             self._platform = 'msvc'
+       elif self._platform.startswith('bitrig'):
+           self._platform = 'bitrig'
 
 
     def platform(self):
@@ -69,3 +71,6 @@ class Platform( object ):
 
     def is_sunos5(self):
         return self._platform == 'sunos5'
+
+    def is_bitrig(self):
+        return self._platform == 'bitrig'

diff --git a/src/build_log.cc b/src/build_log.cc
index 6b73002..a040ce2 100644 (file)
--- a/src/build_log.cc
+++ b/src/build_log.cc
@@ -54,26 +54,27 @@ uint64_t MurmurHash64A(const void* key, size_t len) {
   const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995);
   const int r = 47;
   uint64_t h = seed ^ (len * m);
-  const uint64_t * data = (const uint64_t *)key;
-  const uint64_t * end = data + (len/8);
-  while (data != end) {
-    uint64_t k = *data++;
+  const unsigned char * data = (const unsigned char *)key;
+  while (len >= 8) {
+    uint64_t k;
+    memcpy(&k, data, sizeof k);
     k *= m;
     k ^= k >> r;
     k *= m;
     h ^= k;
     h *= m;
+    data += 8;
+    len -= 8;
   }
-  const unsigned char* data2 = (const unsigned char*)data;
   switch (len & 7)
   {
-  case 7: h ^= uint64_t(data2[6]) << 48;
-  case 6: h ^= uint64_t(data2[5]) << 40;
-  case 5: h ^= uint64_t(data2[4]) << 32;
-  case 4: h ^= uint64_t(data2[3]) << 24;
-  case 3: h ^= uint64_t(data2[2]) << 16;
-  case 2: h ^= uint64_t(data2[1]) << 8;
-  case 1: h ^= uint64_t(data2[0]);
+  case 7: h ^= uint64_t(data[6]) << 48;
+  case 6: h ^= uint64_t(data[5]) << 40;
+  case 5: h ^= uint64_t(data[4]) << 32;
+  case 4: h ^= uint64_t(data[3]) << 24;
+  case 3: h ^= uint64_t(data[2]) << 16;
+  case 2: h ^= uint64_t(data[1]) << 8;
+  case 1: h ^= uint64_t(data[0]);
           h *= m;
   };
   h ^= h >> r;

diff --git a/src/deps_log.cc b/src/deps_log.cc
index 931cc77..ce9bf06 100644 (file)
--- a/src/deps_log.cc
+++ b/src/deps_log.cc
@@ -32,6 +32,11 @@
 const char kFileSignature[] = "# ninjadeps\n";
 const int kCurrentVersion = 1;
 
+// Since the size field is 2 bytes and the top bit marks deps entries, a single
+// record can be at most 32 kB. Set the buffer size to this and flush the file
+// buffer after every record to make sure records aren't written partially.
+const int kMaxBufferSize = 1 << 15;
+
 DepsLog::~DepsLog() {
   Close();
 }
@@ -48,6 +53,7 @@ bool DepsLog::OpenForWrite(const string& path, string* err) {
     *err = strerror(errno);
     return false;
   }
+  setvbuf(file_, NULL, _IOFBF, kMaxBufferSize);
   SetCloseOnExec(fileno(file_));
 
   // Opening a file in append mode doesn't set the file pointer to the file's
@@ -64,6 +70,7 @@ bool DepsLog::OpenForWrite(const string& path, string* err) {
       return false;
     }
   }
+  fflush(file_);
 
   return true;
 }
@@ -124,6 +131,7 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime,
     id = nodes[i]->id();
     fwrite(&id, 4, 1, file_);
   }
+  fflush(file_);
 
   // Update in-memory representation.
   Deps* deps = new Deps(mtime, node_count);
@@ -318,6 +326,7 @@ bool DepsLog::RecordId(Node* node) {
   uint16_t size = (uint16_t)node->path().size();
   fwrite(&size, 2, 1, file_);
   fwrite(node->path().data(), node->path().size(), 1, file_);
+  fflush(file_);
 
   node->set_id(nodes_.size());
   nodes_.push_back(node);

diff --git a/src/hash_map.h b/src/hash_map.h
index 076f6c0..919b6fc 100644 (file)
--- a/src/hash_map.h
+++ b/src/hash_map.h
@@ -15,6 +15,7 @@
 #ifndef NINJA_MAP_H_
 #define NINJA_MAP_H_
 
+#include <string.h>
 #include "string_piece.h"
 
 // MurmurHash2, by Austin Appleby
@@ -26,7 +27,8 @@ unsigned int MurmurHash2(const void* key, size_t len) {
   unsigned int h = seed ^ len;
   const unsigned char * data = (const unsigned char *)key;
   while (len >= 4) {
-    unsigned int k = *(unsigned int *)data;
+    unsigned int k;
+    memcpy(&k, data, sizeof k);
     k *= m;
     k ^= k >> r;
     k *= m;

diff --git a/src/line_printer.cc b/src/line_printer.cc
index a75eb05..3537e88 100644 (file)
--- a/src/line_printer.cc
+++ b/src/line_printer.cc
@@ -104,6 +104,10 @@ void LinePrinter::Print(string to_print, LineType type) {
 void LinePrinter::PrintOnNewLine(const string& to_print) {
   if (!have_blank_line_)
     printf("\n");
-  printf("%s", to_print.c_str());
+  if (!to_print.empty()) {
+    // Avoid printf and C strings, since the actual output might contain null
+    // bytes like UTF-16 does (yuck).
+    fwrite(&to_print[0], sizeof(char), to_print.size(), stdout);
+  }
   have_blank_line_ = to_print.empty() || *to_print.rbegin() == '\n';
 }

diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc
index 3593567..d4f0007 100644 (file)
--- a/src/manifest_parser.cc
+++ b/src/manifest_parser.cc
@@ -15,6 +15,7 @@
 #include "manifest_parser.h"
 
 #include <stdio.h>
+#include <stdlib.h>
 #include <vector>
 
 #include "graph.h"
@@ -145,10 +146,8 @@ bool ManifestParser::ParseRule(string* err) {
   if (!ExpectToken(Lexer::NEWLINE, err))
     return false;
 
-  if (state_->LookupRule(name) != NULL) {
-    *err = "duplicate rule '" + name + "'";
-    return false;
-  }
+  if (state_->LookupRule(name) != NULL)
+    return lexer_.Error("duplicate rule '" + name + "'", err);
 
   Rule* rule = new Rule(name);  // XXX scoped_ptr
 
@@ -306,7 +305,7 @@ bool ManifestParser::ParseEdge(string* err) {
   if (!pool_name.empty()) {
     Pool* pool = state_->LookupPool(pool_name);
     if (pool == NULL)
-      return lexer_.Error("unknown pool name", err);
+      return lexer_.Error("unknown pool name '" + pool_name + "'", err);
     edge->pool_ = pool;
   }
 

diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc
index 2638edc..b333549 100644 (file)
--- a/src/manifest_parser_test.cc
+++ b/src/manifest_parser_test.cc
@@ -377,6 +377,17 @@ TEST_F(ParserTest, Errors) {
     State state;
     ManifestParser parser(&state, NULL);
     string err;
+    EXPECT_FALSE(parser.ParseTest("build\n", &err));
+    EXPECT_EQ("input:1: expected path\n"
+              "build\n"
+              "     ^ near here"
+              , err);
+  }
+
+  {
+    State state;
+    ManifestParser parser(&state, NULL);
+    string err;
     EXPECT_FALSE(parser.ParseTest("build x: y z\n", &err));
     EXPECT_EQ("input:1: unknown build rule 'y'\n"
               "build x: y z\n"
@@ -422,6 +433,32 @@ TEST_F(ParserTest, Errors) {
     ManifestParser parser(&state, NULL);
     string err;
     EXPECT_FALSE(parser.ParseTest("rule cat\n"
+                                  "  command = echo\n"
+                                  "rule cat\n"
+                                  "  command = echo\n", &err));
+    EXPECT_EQ("input:3: duplicate rule 'cat'\n"
+              "rule cat\n"
+              "        ^ near here"
+              , err);
+  }
+
+  {
+    State state;
+    ManifestParser parser(&state, NULL);
+    string err;
+    EXPECT_FALSE(parser.ParseTest("rule cat\n"
+                                  "  command = echo\n"
+                                  "  rspfile = cat.rsp\n", &err));
+    EXPECT_EQ(
+        "input:4: rspfile and rspfile_content need to be both specified\n",
+        err);
+  }
+
+  {
+    State state;
+    ManifestParser parser(&state, NULL);
+    string err;
+    EXPECT_FALSE(parser.ParseTest("rule cat\n"
                                   "  command = ${fafsd\n"
                                   "foo = bar\n",
                                   &err));
@@ -583,6 +620,71 @@ TEST_F(ParserTest, Errors) {
                                   "  generator = 1\n", &err));
     EXPECT_EQ("input:4: unexpected indent\n", err);
   }
+
+  {
+    State state;
+    ManifestParser parser(&state, NULL);
+    string err;
+    EXPECT_FALSE(parser.ParseTest("pool\n", &err));
+    EXPECT_EQ("input:1: expected pool name\n", err);
+  }
+
+  {
+    State state;
+    ManifestParser parser(&state, NULL);
+    string err;
+    EXPECT_FALSE(parser.ParseTest("pool foo\n", &err));
+    EXPECT_EQ("input:2: expected 'depth =' line\n", err);
+  }
+
+  {
+    State state;
+    ManifestParser parser(&state, NULL);
+    string err;
+    EXPECT_FALSE(parser.ParseTest("pool foo\n"
+                                  "  depth = 4\n"
+                                  "pool foo\n", &err));
+    EXPECT_EQ("input:3: duplicate pool 'foo'\n"
+              "pool foo\n"
+              "        ^ near here"
+              , err);
+  }
+
+  {
+    State state;
+    ManifestParser parser(&state, NULL);
+    string err;
+    EXPECT_FALSE(parser.ParseTest("pool foo\n"
+                                  "  depth = -1\n", &err));
+    EXPECT_EQ("input:2: invalid pool depth\n"
+              "  depth = -1\n"
+              "            ^ near here"
+              , err);
+  }
+
+  {
+    State state;
+    ManifestParser parser(&state, NULL);
+    string err;
+    EXPECT_FALSE(parser.ParseTest("pool foo\n"
+                                  "  bar = 1\n", &err));
+    EXPECT_EQ("input:2: unexpected variable 'bar'\n"
+              "  bar = 1\n"
+              "         ^ near here"
+              , err);
+  }
+
+  {
+    State state;
+    ManifestParser parser(&state, NULL);
+    string err;
+    // Pool names are dereferenced at edge parsing time.
+    EXPECT_FALSE(parser.ParseTest("rule run\n"
+                                  "  command = echo\n"
+                                  "  pool = unnamed_pool\n"
+                                  "build out: run in\n", &err));
+    EXPECT_EQ("input:5: unknown pool name 'unnamed_pool'\n", err);
+  }
 }
 
 TEST_F(ParserTest, MissingInput) {

diff --git a/src/ninja.cc b/src/ninja.cc
index 3b381b7..1e5590d 100644 (file)
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -104,6 +104,7 @@ struct NinjaMain {
   // The various subcommands, run via "-t XXX".
   int ToolGraph(int argc, char* argv[]);
   int ToolQuery(int argc, char* argv[]);
+  int ToolDeps(int argc, char* argv[]);
   int ToolBrowse(int argc, char* argv[]);
   int ToolMSVC(int argc, char* argv[]);
   int ToolTargets(int argc, char* argv[]);
@@ -437,6 +438,45 @@ int ToolTargetsList(State* state) {
   return 0;
 }
 
+int NinjaMain::ToolDeps(int argc, char** argv) {
+  vector<Node*> nodes;
+  if (argc == 0) {
+    for (vector<Node*>::const_iterator ni = deps_log_.nodes().begin();
+         ni != deps_log_.nodes().end(); ++ni) {
+      // Only query for targets with an incoming edge and deps
+      Edge* e = (*ni)->in_edge();
+      if (e && !e->GetBinding("deps").empty())
+        nodes.push_back(*ni);
+    }
+  } else {
+    string err;
+    if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) {
+      Error("%s", err.c_str());
+      return 1;
+    }
+  }
+
+  RealDiskInterface disk_interface;
+  for (vector<Node*>::iterator it = nodes.begin(), end = nodes.end();
+       it != end; ++it) {
+    DepsLog::Deps* deps = deps_log_.GetDeps(*it);
+    if (!deps) {
+      printf("%s: deps not found\n", (*it)->path().c_str());
+      continue;
+    }
+
+    TimeStamp mtime = disk_interface.Stat((*it)->path());
+    printf("%s: #deps %d, deps mtime %d (%s)\n",
+           (*it)->path().c_str(), deps->node_count, deps->mtime,
+           (!mtime || mtime > deps->mtime ? "STALE":"VALID"));
+    for (int i = 0; i < deps->node_count; ++i)
+      printf("    %s\n", deps->nodes[i]->path().c_str());
+    printf("\n");
+  }
+
+  return 0;
+}
+
 int NinjaMain::ToolTargets(int argc, char* argv[]) {
   int depth = 1;
   if (argc >= 1) {
@@ -644,6 +684,8 @@ const Tool* ChooseTool(const string& tool_name) {
       Tool::RUN_AFTER_LOAD, &NinjaMain::ToolClean },
     { "commands", "list all commands required to rebuild given targets",
       Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCommands },
+    { "deps", "show dependencies stored in the deps log",
+      Tool::RUN_AFTER_LOGS, &NinjaMain::ToolDeps },
     { "graph", "output graphviz dot file for targets",
       Tool::RUN_AFTER_LOAD, &NinjaMain::ToolGraph },
     { "query", "show inputs/outputs for a path",

diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc
index b396f84..a9af756 100644 (file)
--- a/src/subprocess-posix.cc
+++ b/src/subprocess-posix.cc
@@ -41,7 +41,7 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) {
     Fatal("pipe: %s", strerror(errno));
   fd_ = output_pipe[0];
 #if !defined(USE_PPOLL)
-  // On Linux and OpenBSD, we use ppoll in DoWork(); elsewhere we use pselect
+  // If available, we use ppoll in DoWork(); otherwise we use pselect
   // and so must avoid overly-large FDs.
   if (fd_ >= static_cast<int>(FD_SETSIZE))
     Fatal("pipe: %s", strerror(EMFILE));
@@ -224,7 +224,7 @@ bool SubprocessSet::DoWork() {
   return interrupted_;
 }
 
-#else  // linux || __OpenBSD__
+#else  // !defined(USE_PPOLL)
 bool SubprocessSet::DoWork() {
   fd_set set;
   int nfds = 0;
@@ -266,7 +266,7 @@ bool SubprocessSet::DoWork() {
 
   return interrupted_;
 }
-#endif  // linux || __OpenBSD__
+#endif  // !defined(USE_PPOLL)
 
 Subprocess* SubprocessSet::NextFinished() {
   if (finished_.empty())

diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc
index afd9008..9f8dcea 100644 (file)
--- a/src/subprocess_test.cc
+++ b/src/subprocess_test.cc
@@ -152,7 +152,7 @@ TEST_F(SubprocessTest, SetWithMulti) {
 
 // OS X's process limit is less than 1025 by default
 // (|sysctl kern.maxprocperuid| is 709 on 10.7 and 10.8 and less prior to that).
-#if defined(linux) || defined(__OpenBSD__)
+#if !defined(__APPLE__) && !defined(_WIN32)
 TEST_F(SubprocessTest, SetWithLots) {
   // Arbitrary big number; needs to be over 1024 to confirm we're no longer
   // hostage to pselect.
@@ -179,7 +179,7 @@ TEST_F(SubprocessTest, SetWithLots) {
   }
   ASSERT_EQ(kNumProcs, subprocs_.finished_.size());
 }
-#endif  // linux || __OpenBSD__
+#endif  // !__APPLE__ && !_WIN32 
 
 // TODO: this test could work on Windows, just not sure how to simply
 // read stdin.