#include <string.h>
#ifndef _WIN32
#include <unistd.h>
+#elif defined(_MSC_VER) && (_MSC_VER < 1900)
+typedef __int32 int32_t;
+typedef unsigned __int32 uint32_t;
#endif
#include "graph.h"
// The version is stored as 4 bytes after the signature and also serves as a
// byte order mark. Signature and version combined are 16 bytes long.
const char kFileSignature[] = "# ninjadeps\n";
-const int kCurrentVersion = 2;
+const int kCurrentVersion = 4;
-// Since the size field is 2 bytes and the top bit marks deps entries, a single
-// record can be at most 32 kB. Set the buffer size to this and flush the file
-// buffer after every record to make sure records aren't written partially.
-const int kMaxBufferSize = 1 << 15;
-
-// Record size is currently limited to 15 bit
-const size_t kMaxRecordSize = (1 << 15) - 1;
+// Record size is currently limited to less than the full 32 bit, due to
+// internal buffers having to have this size.
+const unsigned kMaxRecordSize = (1 << 19) - 1;
DepsLog::~DepsLog() {
Close();
*err = strerror(errno);
return false;
}
- setvbuf(file_, NULL, _IOFBF, kMaxBufferSize);
+ // Set the buffer size to this and flush the file buffer after every record
+ // to make sure records aren't written partially.
+ setvbuf(file_, NULL, _IOFBF, kMaxRecordSize + 1);
SetCloseOnExec(fileno(file_));
// Opening a file in append mode doesn't set the file pointer to the file's
return true;
// Update on-disk representation.
- size_t size = 4 * (1 + 1 + (uint16_t)node_count);
+ unsigned size = 4 * (1 + 2 + node_count);
if (size > kMaxRecordSize) {
errno = ERANGE;
return false;
}
- size |= 0x8000; // Deps record: set high bit.
- uint16_t size16 = (uint16_t)size;
- if (fwrite(&size16, 2, 1, file_) < 1)
+ size |= 0x80000000; // Deps record: set high bit.
+ if (fwrite(&size, 4, 1, file_) < 1)
return false;
int id = node->id();
if (fwrite(&id, 4, 1, file_) < 1)
return false;
- int timestamp = mtime;
- if (fwrite(×tamp, 4, 1, file_) < 1)
+ uint32_t mtime_part = static_cast<uint32_t>(mtime & 0xffffffff);
+ if (fwrite(&mtime_part, 4, 1, file_) < 1)
+ return false;
+ mtime_part = static_cast<uint32_t>((mtime >> 32) & 0xffffffff);
+ if (fwrite(&mtime_part, 4, 1, file_) < 1)
return false;
for (int i = 0; i < node_count; ++i) {
id = nodes[i]->id();
return false;
}
if (fflush(file_) != 0)
- return false;
+ return false;
// Update in-memory representation.
Deps* deps = new Deps(mtime, node_count);
bool DepsLog::Load(const string& path, State* state, string* err) {
METRIC_RECORD(".ninja_deps load");
- char buf[32 << 10];
+ char buf[kMaxRecordSize + 1];
FILE* f = fopen(path.c_str(), "rb");
if (!f) {
if (errno == ENOENT)
valid_header = false;
// Note: For version differences, this should migrate to the new format.
// But the v1 format could sometimes (rarely) end up with invalid data, so
- // don't migrate v1 to v2 to force a rebuild.
+ // don't migrate v1 to v3 to force a rebuild. (v2 only existed for a few days,
+ // and there was no release with it, so pretend that it never happened.)
if (!valid_header || strcmp(buf, kFileSignature) != 0 ||
version != kCurrentVersion) {
if (version == 1)
- *err = "deps log potentially corrupt; rebuilding";
+ *err = "deps log version change; rebuilding";
else
*err = "bad deps log signature or version; starting over";
fclose(f);
for (;;) {
offset = ftell(f);
- uint16_t size;
- if (fread(&size, 2, 1, f) < 1) {
+ unsigned size;
+ if (fread(&size, 4, 1, f) < 1) {
if (!feof(f))
read_failed = true;
break;
}
- bool is_deps = (size >> 15) != 0;
- size = size & 0x7FFF;
+ bool is_deps = (size >> 31) != 0;
+ size = size & 0x7FFFFFFF;
- if (fread(buf, size, 1, f) < 1) {
+ if (size > kMaxRecordSize || fread(buf, size, 1, f) < 1) {
read_failed = true;
break;
}
assert(size % 4 == 0);
int* deps_data = reinterpret_cast<int*>(buf);
int out_id = deps_data[0];
- int mtime = deps_data[1];
- deps_data += 2;
- int deps_count = (size / 4) - 2;
+ TimeStamp mtime;
+ mtime = (TimeStamp)(((uint64_t)(unsigned int)deps_data[2] << 32) |
+ (uint64_t)(unsigned int)deps_data[1]);
+ deps_data += 3;
+ int deps_count = (size / 4) - 3;
Deps* deps = new Deps(mtime, deps_count);
for (int i = 0; i < deps_count; ++i) {
++unique_dep_record_count;
} else {
int path_size = size - 4;
- StringPiece path(buf, path_size);
- Node* node = state->GetNode(path);
+ assert(path_size > 0); // CanonicalizePath() rejects empty paths.
+ // There can be up to 3 bytes of padding.
+ if (buf[path_size - 1] == '\0') --path_size;
+ if (buf[path_size - 1] == '\0') --path_size;
+ if (buf[path_size - 1] == '\0') --path_size;
+ StringPiece subpath(buf, path_size);
+ // It is not necessary to pass in a correct slash_bits here. It will
+ // either be a Node that's in the manifest (in which case it will already
+ // have a correct slash_bits that GetNode will look up), or it is an
+ // implicit dependency from a .d which does not affect the build command
+ // (and so need not have its slashes maintained).
+ Node* node = state->GetNode(subpath, 0);
// Check that the expected index matches the actual index. This can only
// happen if two ninja processes write to the same deps log concurrently.
// (This uses unary complement to make the checksum look less like a
// dependency record entry.)
- unsigned checksum;
- memcpy(&checksum, buf + path_size, sizeof checksum);
+ unsigned checksum = *reinterpret_cast<unsigned*>(buf + size - 4);
int expected_id = ~checksum;
int id = nodes_.size();
if (id != expected_id) {
}
fclose(f);
- if (!Truncate(path.c_str(), offset, err))
+ if (!Truncate(path, offset, err))
return false;
// The truncate succeeded; we'll just report the load error as a
bool DepsLog::Recompact(const string& path, string* err) {
METRIC_RECORD(".ninja_deps recompact");
- printf("Recompacting deps...\n");
Close();
string temp_path = path + ".recompact";
Deps* deps = deps_[old_id];
if (!deps) continue; // If nodes_[old_id] is a leaf, it has no deps.
+ if (!IsDepsEntryLiveFor(nodes_[old_id]))
+ continue;
+
if (!new_log.RecordDeps(nodes_[old_id], deps->mtime,
deps->node_count, deps->nodes)) {
new_log.Close();
return true;
}
+bool DepsLog::IsDepsEntryLiveFor(Node* node) {
+ // Skip entries that don't have in-edges or whose edges don't have a
+ // "deps" attribute. They were in the deps log from previous builds, but
+ // the the files they were for were removed from the build and their deps
+ // entries are no longer needed.
+ // (Without the check for "deps", a chain of two or more nodes that each
+ // had deps wouldn't be collected in a single recompaction.)
+ return node->in_edge() && !node->in_edge()->GetBinding("deps").empty();
+}
+
bool DepsLog::UpdateDeps(int out_id, Deps* deps) {
if (out_id >= (int)deps_.size())
deps_.resize(out_id + 1);
}
bool DepsLog::RecordId(Node* node) {
- size_t size = node->path().size() + 4;
+ int path_size = node->path().size();
+ int padding = (4 - path_size % 4) % 4; // Pad path to 4 byte boundary.
+
+ unsigned size = path_size + padding + 4;
if (size > kMaxRecordSize) {
errno = ERANGE;
return false;
}
- uint16_t size16 = (uint16_t)size;
- if (fwrite(&size16, 2, 1, file_) < 1)
+ if (fwrite(&size, 4, 1, file_) < 1)
return false;
- if (fwrite(node->path().data(), node->path().size(), 1, file_) < 1) {
+ if (fwrite(node->path().data(), path_size, 1, file_) < 1) {
assert(node->path().size() > 0);
return false;
}
+ if (padding && fwrite("\0\0", padding, 1, file_) < 1)
+ return false;
int id = nodes_.size();
unsigned checksum = ~(unsigned)id;
if (fwrite(&checksum, 4, 1, file_) < 1)
projects / platform / upstream / ninja.git / blobdiff