From ec92fe3da6e792f9e14a490675aebc132ec37ef6 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Tue, 8 Jan 2013 08:43:39 -0800 Subject: [PATCH] add recompaction to depslog Not done automatically yet, just an implementation and a test. --- src/deps_log.cc | 63 ++++++++++++++++++++++++++++++++++++++-------- src/deps_log.h | 4 +++ src/deps_log_test.cc | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 10 deletions(-) diff --git a/src/deps_log.cc b/src/deps_log.cc index 081fcf0..5031515 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -54,6 +54,11 @@ bool DepsLog::OpenForWrite(const string& path, string* err) { bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, const vector& nodes) { + return RecordDeps(node, mtime, nodes.size(), (Node**)&nodes.front()); +} + +bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, + int node_count, Node** nodes) { // Track whether there's any new data to be recorded. bool made_change = false; @@ -62,10 +67,9 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, RecordId(node); made_change = true; } - for (vector::const_iterator i = nodes.begin(); - i != nodes.end(); ++i) { - if ((*i)->id() < 0) { - RecordId(*i); + for (int i = 0; i < node_count; ++i) { + if (nodes[i]->id() < 0) { + RecordId(nodes[i]); made_change = true; } } @@ -75,10 +79,10 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, Deps* deps = GetDeps(node); if (!deps || deps->mtime != mtime || - deps->node_count != (int)nodes.size()) { + deps->node_count != node_count) { made_change = true; } else { - for (int i = 0; i < (int)nodes.size(); ++i) { + for (int i = 0; i < node_count; ++i) { if (deps->nodes[i] != nodes[i]) { made_change = true; break; @@ -91,16 +95,15 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, if (!made_change) return true; - uint16_t size = 4 * (1 + 1 + (uint16_t)nodes.size()); + uint16_t size = 4 * (1 + 1 + (uint16_t)node_count); size |= 0x8000; // Deps record: set high bit. fwrite(&size, 2, 1, file_); int id = node->id(); fwrite(&id, 4, 1, file_); int timestamp = mtime; fwrite(×tamp, 4, 1, file_); - for (vector::const_iterator i = nodes.begin(); - i != nodes.end(); ++i) { - id = (*i)->id(); + for (int i = 0; i < node_count; ++i) { + id = nodes[i]->id(); fwrite(&id, 4, 1, file_); } @@ -193,6 +196,46 @@ DepsLog::Deps* DepsLog::GetDeps(Node* node) { return deps_[node->id()]; } +bool DepsLog::Recompact(const string& path, string* err) { + METRIC_RECORD(".ninja_deps recompact"); + printf("Recompacting deps...\n"); + + string temp_path = path + ".recompact"; + DepsLog new_log; + if (!new_log.OpenForWrite(temp_path, err)) + return false; + + // Clear all known ids so that new ones can be reassigned. + for (vector::iterator i = nodes_.begin(); + i != nodes_.end(); ++i) { + (*i)->set_id(-1); + } + + // Write out all deps again. + for (int old_id = 0; old_id < (int)deps_.size(); ++old_id) { + Deps* deps = deps_[old_id]; + if (!new_log.RecordDeps(nodes_[old_id], deps->mtime, + deps->node_count, deps->nodes)) { + new_log.Close(); + return false; + } + } + + new_log.Close(); + + if (unlink(path.c_str()) < 0) { + *err = strerror(errno); + return false; + } + + if (rename(temp_path.c_str(), path.c_str()) < 0) { + *err = strerror(errno); + return false; + } + + return true; +} + bool DepsLog::RecordId(Node* node) { uint16_t size = (uint16_t)node->path().size(); fwrite(&size, 2, 1, file_); diff --git a/src/deps_log.h b/src/deps_log.h index 56f9590..d763a1d 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -65,6 +65,7 @@ struct DepsLog { // Writing (build-time) interface. bool OpenForWrite(const string& path, string* err); bool RecordDeps(Node* node, TimeStamp mtime, const vector& nodes); + bool RecordDeps(Node* node, TimeStamp mtime, int node_count, Node** nodes); void Close(); // Reading (startup-time) interface. @@ -78,6 +79,9 @@ struct DepsLog { bool Load(const string& path, State* state, string* err); Deps* GetDeps(Node* node); + /// Rewrite the known log entries, throwing away old data. + bool Recompact(const string& path, string* err); + /// Used for tests. const vector& nodes() const { return nodes_; } diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc index e411e12..2d91c0e 100644 --- a/src/deps_log_test.cc +++ b/src/deps_log_test.cc @@ -122,4 +122,74 @@ TEST_F(DepsLogTest, DoubleEntry) { } } +// Verify that adding the new deps works and can be compacted away. +TEST_F(DepsLogTest, Recompact) { + // Write some deps to the file and grab its size. + int file_size; + { + State state; + DepsLog log; + string err; + ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + + vector deps; + deps.push_back(state.GetNode("foo.h")); + deps.push_back(state.GetNode("bar.h")); + log.RecordDeps(state.GetNode("out.o"), 1, deps); + log.Close(); + + struct stat st; + ASSERT_EQ(0, stat(kTestFilename, &st)); + file_size = (int)st.st_size; + ASSERT_GT(file_size, 0); + } + + // Now reload the file, and add slighly different deps. + int file_size_2; + { + State state; + DepsLog log; + string err; + ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); + + ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + + vector deps; + deps.push_back(state.GetNode("foo.h")); + log.RecordDeps(state.GetNode("out.o"), 1, deps); + log.Close(); + + struct stat st; + ASSERT_EQ(0, stat(kTestFilename, &st)); + file_size_2 = (int)st.st_size; + // The file should grow to record the new deps. + ASSERT_GT(file_size_2, file_size); + } + + // Now reload the file, verify the new deps have replaced the old, then + // recompact. + { + State state; + DepsLog log; + string err; + ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); + + DepsLog::Deps* deps = log.GetDeps(state.GetNode("out.o")); + ASSERT_TRUE(deps); + ASSERT_EQ(1, deps->mtime); + ASSERT_EQ(1, deps->node_count); + ASSERT_EQ("foo.h", deps->nodes[0]->path()); + + ASSERT_TRUE(log.Recompact(kTestFilename, &err)); + + struct stat st; + ASSERT_EQ(0, stat(kTestFilename, &st)); + int file_size_3 = (int)st.st_size; + // The file should have shrunk a bit for the smaller deps. + ASSERT_LT(file_size_3, file_size_2); + } +} + } // anonymous namespace -- 2.7.4