1 // Copyright 2011 Google Inc. All Rights Reserved.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 // On AIX, inttypes.h gets indirectly included by build_log.h.
16 // It's easiest just to ask for the printf format macros right away.
18 #ifndef __STDC_FORMAT_MACROS
19 #define __STDC_FORMAT_MACROS
23 #include "build_log.h"
39 // Implementation details:
40 // Each run's log appends to the log file.
41 // To load, we run through all log entries in series, throwing away
43 // Once the number of redundant entries exceeds a threshold, we write
44 // out a new file and replace the existing one with it.
48 const char kFileSignature[] = "# ninja log v%d\n";
49 const int kOldestSupportedVersion = 4;
50 const int kCurrentVersion = 5;
52 // 64bit MurmurHash2, by Austin Appleby
54 #define BIG_CONSTANT(x) (x)
55 #else // defined(_MSC_VER)
56 #define BIG_CONSTANT(x) (x##LLU)
57 #endif // !defined(_MSC_VER)
59 uint64_t MurmurHash64A(const void* key, size_t len) {
60 static const uint64_t seed = 0xDECAFBADDECAFBADull;
61 const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995);
63 uint64_t h = seed ^ (len * m);
64 const unsigned char* data = (const unsigned char*)key;
67 memcpy(&k, data, sizeof k);
78 case 7: h ^= uint64_t(data[6]) << 48;
79 case 6: h ^= uint64_t(data[5]) << 40;
80 case 5: h ^= uint64_t(data[4]) << 32;
81 case 4: h ^= uint64_t(data[3]) << 24;
82 case 3: h ^= uint64_t(data[2]) << 16;
83 case 2: h ^= uint64_t(data[1]) << 8;
84 case 1: h ^= uint64_t(data[0]);
98 uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) {
99 return MurmurHash64A(command.str_, command.len_);
102 BuildLog::LogEntry::LogEntry(const string& output)
105 BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash,
106 int start_time, int end_time, TimeStamp restat_mtime)
107 : output(output), command_hash(command_hash),
108 start_time(start_time), end_time(end_time), mtime(restat_mtime)
112 : log_file_(NULL), needs_recompaction_(false) {}
114 BuildLog::~BuildLog() {
118 bool BuildLog::OpenForWrite(const string& path, const BuildLogUser& user,
120 if (needs_recompaction_) {
121 if (!Recompact(path, user, err))
125 log_file_ = fopen(path.c_str(), "ab");
127 *err = strerror(errno);
130 setvbuf(log_file_, NULL, _IOLBF, BUFSIZ);
131 SetCloseOnExec(fileno(log_file_));
133 // Opening a file in append mode doesn't set the file pointer to the file's
134 // end on Windows. Do that explicitly.
135 fseek(log_file_, 0, SEEK_END);
137 if (ftell(log_file_) == 0) {
138 if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) {
139 *err = strerror(errno);
147 bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time,
149 string command = edge->EvaluateCommand(true);
150 uint64_t command_hash = LogEntry::HashCommand(command);
151 for (vector<Node*>::iterator out = edge->outputs_.begin();
152 out != edge->outputs_.end(); ++out) {
153 const string& path = (*out)->path();
154 Entries::iterator i = entries_.find(path);
156 if (i != entries_.end()) {
157 log_entry = i->second;
159 log_entry = new LogEntry(path);
160 entries_.insert(Entries::value_type(log_entry->output, log_entry));
162 log_entry->command_hash = command_hash;
163 log_entry->start_time = start_time;
164 log_entry->end_time = end_time;
165 log_entry->mtime = mtime;
168 if (!WriteEntry(log_file_, *log_entry))
175 void BuildLog::Close() {
182 explicit LineReader(FILE* file)
183 : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {
184 memset(buf_, 0, sizeof(buf_));
187 // Reads a \n-terminated line from the file passed to the constructor.
188 // On return, *line_start points to the beginning of the next line, and
189 // *line_end points to the \n at the end of the line. If no newline is seen
190 // in a fixed buffer size, *line_end is set to NULL. Returns false on EOF.
191 bool ReadLine(char** line_start, char** line_end) {
192 if (line_start_ >= buf_end_ || !line_end_) {
193 // Buffer empty, refill.
194 size_t size_read = fread(buf_, 1, sizeof(buf_), file_);
198 buf_end_ = buf_ + size_read;
200 // Advance to next line in buffer.
201 line_start_ = line_end_ + 1;
204 line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
206 // No newline. Move rest of data to start of buffer, fill rest.
207 size_t already_consumed = line_start_ - buf_;
208 size_t size_rest = (buf_end_ - buf_) - already_consumed;
209 memmove(buf_, line_start_, size_rest);
211 size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_);
212 buf_end_ = buf_ + size_rest + read;
214 line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
217 *line_start = line_start_;
218 *line_end = line_end_;
224 char buf_[256 << 10];
225 char* buf_end_; // Points one past the last valid byte in |buf_|.
228 // Points at the next \n in buf_ after line_start, or NULL.
232 bool BuildLog::Load(const string& path, string* err) {
233 METRIC_RECORD(".ninja_log load");
234 FILE* file = fopen(path.c_str(), "r");
238 *err = strerror(errno);
243 int unique_entry_count = 0;
244 int total_entry_count = 0;
246 LineReader reader(file);
247 char* line_start = 0;
249 while (reader.ReadLine(&line_start, &line_end)) {
251 sscanf(line_start, kFileSignature, &log_version);
253 if (log_version < kOldestSupportedVersion) {
254 *err = ("build log version invalid, perhaps due to being too old; "
257 unlink(path.c_str());
258 // Don't report this as a failure. An empty build log will cause
259 // us to rebuild the outputs anyway.
264 // If no newline was found in this chunk, read the next.
268 const char kFieldSeparator = '\t';
270 char* start = line_start;
271 char* end = (char*)memchr(start, kFieldSeparator, line_end - start);
276 int start_time = 0, end_time = 0;
277 TimeStamp restat_mtime = 0;
279 start_time = atoi(start);
282 end = (char*)memchr(start, kFieldSeparator, line_end - start);
286 end_time = atoi(start);
289 end = (char*)memchr(start, kFieldSeparator, line_end - start);
293 restat_mtime = atol(start);
296 end = (char*)memchr(start, kFieldSeparator, line_end - start);
299 string output = string(start, end - start);
305 Entries::iterator i = entries_.find(output);
306 if (i != entries_.end()) {
309 entry = new LogEntry(output);
310 entries_.insert(Entries::value_type(entry->output, entry));
311 ++unique_entry_count;
315 entry->start_time = start_time;
316 entry->end_time = end_time;
317 entry->mtime = restat_mtime;
318 if (log_version >= 5) {
319 char c = *end; *end = '\0';
320 entry->command_hash = (uint64_t)strtoull(start, NULL, 16);
323 entry->command_hash = LogEntry::HashCommand(StringPiece(start,
330 return true; // file was empty
333 // Decide whether it's time to rebuild the log:
334 // - if we're upgrading versions
335 // - if it's getting large
336 int kMinCompactionEntryCount = 100;
337 int kCompactionRatio = 3;
338 if (log_version < kCurrentVersion) {
339 needs_recompaction_ = true;
340 } else if (total_entry_count > kMinCompactionEntryCount &&
341 total_entry_count > unique_entry_count * kCompactionRatio) {
342 needs_recompaction_ = true;
348 BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) {
349 Entries::iterator i = entries_.find(path);
350 if (i != entries_.end())
355 bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) {
356 return fprintf(f, "%d\t%d\t%d\t%s\t%" PRIx64 "\n",
357 entry.start_time, entry.end_time, entry.mtime,
358 entry.output.c_str(), entry.command_hash) > 0;
361 bool BuildLog::Recompact(const string& path, const BuildLogUser& user,
363 METRIC_RECORD(".ninja_log recompact");
366 string temp_path = path + ".recompact";
367 FILE* f = fopen(temp_path.c_str(), "wb");
369 *err = strerror(errno);
373 if (fprintf(f, kFileSignature, kCurrentVersion) < 0) {
374 *err = strerror(errno);
379 vector<StringPiece> dead_outputs;
380 for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) {
381 if (user.IsPathDead(i->first)) {
382 dead_outputs.push_back(i->first);
386 if (!WriteEntry(f, *i->second)) {
387 *err = strerror(errno);
393 for (size_t i = 0; i < dead_outputs.size(); ++i)
394 entries_.erase(dead_outputs[i]);
397 if (unlink(path.c_str()) < 0) {
398 *err = strerror(errno);
402 if (rename(temp_path.c_str(), path.c_str()) < 0) {
403 *err = strerror(errno);