src/tools/gn/input_file_manager.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "tools/gn/input_file_manager.h"
   6
   7 #include "base/bind.h"
   8 #include "base/stl_util.h"
   9 #include "tools/gn/filesystem_utils.h"
  10 #include "tools/gn/parser.h"
  11 #include "tools/gn/scheduler.h"
  12 #include "tools/gn/scope_per_file_provider.h"
  13 #include "tools/gn/tokenizer.h"
  14 #include "tools/gn/trace.h"
  15
  16 namespace {
  17
  18 void InvokeFileLoadCallback(const InputFileManager::FileLoadCallback& cb,
  19                             const ParseNode* node) {
  20   cb.Run(node);
  21 }
  22
  23 bool DoLoadFile(const LocationRange& origin,
  24                 const BuildSettings* build_settings,
  25                 const SourceFile& name,
  26                 InputFile* file,
  27                 std::vector<Token>* tokens,
  28                 scoped_ptr<ParseNode>* root,
  29                 Err* err) {
  30   // Do all of this stuff outside the lock. We should not give out file
  31   // pointers until the read is complete.
  32   if (g_scheduler->verbose_logging()) {
  33     std::string logmsg = name.value();
  34     if (origin.begin().file())
  35       logmsg += " (referenced from " + origin.begin().Describe(false) + ")";
  36     g_scheduler->Log("Loading", logmsg);
  37   }
  38
  39   // Read.
  40   base::FilePath primary_path = build_settings->GetFullPath(name);
  41   ScopedTrace load_trace(TraceItem::TRACE_FILE_LOAD, name.value());
  42   if (!file->Load(primary_path)) {
  43     if (!build_settings->secondary_source_path().empty()) {
  44       // Fall back to secondary source tree.
  45       base::FilePath secondary_path =
  46           build_settings->GetFullPathSecondary(name);
  47       if (!file->Load(secondary_path)) {
  48         *err = Err(origin, "Can't load input file.",
  49                    "Unable to load either \n" +
  50                    FilePathToUTF8(primary_path) + " or \n" +
  51                    FilePathToUTF8(secondary_path));
  52         return false;
  53       }
  54     } else {
  55       *err = Err(origin,
  56                  "Unable to load \"" + FilePathToUTF8(primary_path) + "\".");
  57       return false;
  58     }
  59   }
  60   load_trace.Done();
  61
  62   ScopedTrace exec_trace(TraceItem::TRACE_FILE_PARSE, name.value());
  63
  64   // Tokenize.
  65   *tokens = Tokenizer::Tokenize(file, err);
  66   if (err->has_error())
  67     return false;
  68
  69   // Parse.
  70   *root = Parser::Parse(*tokens, err);
  71   if (err->has_error())
  72     return false;
  73
  74   exec_trace.Done();
  75   return true;
  76 }
  77
  78 }  // namespace
  79
  80 InputFileManager::InputFileData::InputFileData(const SourceFile& file_name)
  81     : file(file_name),
  82       loaded(false),
  83       sync_invocation(false) {
  84 }
  85
  86 InputFileManager::InputFileData::~InputFileData() {
  87 }
  88
  89 InputFileManager::InputFileManager() {
  90 }
  91
  92 InputFileManager::~InputFileManager() {
  93   // Should be single-threaded by now.
  94   STLDeleteContainerPairSecondPointers(input_files_.begin(),
  95                                        input_files_.end());
  96   STLDeleteContainerPointers(dynamic_inputs_.begin(), dynamic_inputs_.end());
  97 }
  98
  99 bool InputFileManager::AsyncLoadFile(const LocationRange& origin,
 100                                      const BuildSettings* build_settings,
 101                                      const SourceFile& file_name,
 102                                      const FileLoadCallback& callback,
 103                                      Err* err) {
 104   // Try not to schedule callbacks while holding the lock. All cases that don't
 105   // want to schedule should return early. Otherwise, this will be scheduled
 106   // after we leave the lock.
 107   base::Closure schedule_this;
 108   {
 109     base::AutoLock lock(lock_);
 110
 111     InputFileMap::const_iterator found = input_files_.find(file_name);
 112     if (found == input_files_.end()) {
 113       // New file, schedule load.
 114       InputFileData* data = new InputFileData(file_name);
 115       data->scheduled_callbacks.push_back(callback);
 116       input_files_[file_name] = data;
 117
 118       schedule_this = base::Bind(&InputFileManager::BackgroundLoadFile,
 119                                  this,
 120                                  origin,
 121                                  build_settings,
 122                                  file_name,
 123                                  &data->file);
 124     } else {
 125       InputFileData* data = found->second;
 126
 127       // Prevent mixing async and sync loads. See SyncLoadFile for discussion.
 128       if (data->sync_invocation) {
 129         g_scheduler->FailWithError(Err(
 130             origin, "Load type mismatch.",
 131             "The file \"" + file_name.value() + "\" was previously loaded\n"
 132             "synchronously (via an import) and now you're trying to load it "
 133             "asynchronously\n(via a deps rule). This is a class 2 misdemeanor: "
 134             "a single input file must\nbe loaded the same way each time to "
 135             "avoid blowing my tiny, tiny mind."));
 136         return false;
 137       }
 138
 139       if (data->loaded) {
 140         // Can just directly issue the callback on the background thread.
 141         schedule_this = base::Bind(&InvokeFileLoadCallback, callback,
 142                                    data->parsed_root.get());
 143       } else {
 144         // Load is pending on this file, schedule the invoke.
 145         data->scheduled_callbacks.push_back(callback);
 146         return true;
 147       }
 148     }
 149   }
 150   g_scheduler->pool()->PostWorkerTaskWithShutdownBehavior(
 151       FROM_HERE, schedule_this,
 152       base::SequencedWorkerPool::BLOCK_SHUTDOWN);
 153   return true;
 154 }
 155
 156 const ParseNode* InputFileManager::SyncLoadFile(
 157     const LocationRange& origin,
 158     const BuildSettings* build_settings,
 159     const SourceFile& file_name,
 160     Err* err) {
 161   base::AutoLock lock(lock_);
 162
 163   InputFileData* data = NULL;
 164   InputFileMap::iterator found = input_files_.find(file_name);
 165   if (found == input_files_.end()) {
 166     // Haven't seen this file yet, start loading right now.
 167     data = new InputFileData(file_name);
 168     data->sync_invocation = true;
 169     input_files_[file_name] = data;
 170
 171     base::AutoUnlock unlock(lock_);
 172     if (!LoadFile(origin, build_settings, file_name, &data->file, err))
 173       return NULL;
 174   } else {
 175     // This file has either been loaded or is pending loading.
 176     data = found->second;
 177
 178     if (!data->sync_invocation) {
 179       // Don't allow mixing of sync and async loads. If an async load is
 180       // scheduled and then a bunch of threads need to load it synchronously
 181       // and block on it loading, it could deadlock or at least cause a lot
 182       // of wasted CPU while those threads wait for the load to complete (which
 183       // may be far back in the input queue).
 184       //
 185       // We could work around this by promoting the load to a sync load. This
 186       // requires a bunch of extra code to either check flags and likely do
 187       // extra locking (bad) or to just do both types of load on the file and
 188       // deal with the race condition.
 189       //
 190       // I have no practical way to test this, and generally we should have
 191       // all include files processed synchronously and all build files
 192       // processed asynchronously, so it doesn't happen in practice.
 193       *err = Err(
 194           origin, "Load type mismatch.",
 195           "The file \"" + file_name.value() + "\" was previously loaded\n"
 196           "asynchronously (via a deps rule) and now you're trying to load it "
 197           "synchronously.\nThis is a class 2 misdemeanor: a single input file "
 198           "must be loaded the same way\neach time to avoid blowing my tiny, "
 199           "tiny mind.");
 200       return NULL;
 201     }
 202
 203     if (!data->loaded) {
 204       // Wait for the already-pending sync load to complete.
 205       if (!data->completion_event)
 206         data->completion_event.reset(new base::WaitableEvent(false, false));
 207       {
 208         base::AutoUnlock unlock(lock_);
 209         data->completion_event->Wait();
 210       }
 211       // If there were multiple waiters on the same event, we now need to wake
 212       // up the next one.
 213       data->completion_event->Signal();
 214     }
 215   }
 216
 217   // The other load could have failed. In this case that error will be printed
 218   // to the console, but we need to return something here, so make up a
 219   // dummy error.
 220   if (!data->parsed_root)
 221     *err = Err(origin, "File parse failed");
 222   return data->parsed_root.get();
 223 }
 224
 225 void InputFileManager::AddDynamicInput(const SourceFile& name,
 226                                        InputFile** file,
 227                                        std::vector<Token>** tokens,
 228                                        scoped_ptr<ParseNode>** parse_root) {
 229   InputFileData* data = new InputFileData(name);
 230   {
 231     base::AutoLock lock(lock_);
 232     dynamic_inputs_.push_back(data);
 233   }
 234   *file = &data->file;
 235   *tokens = &data->tokens;
 236   *parse_root = &data->parsed_root;
 237 }
 238
 239 int InputFileManager::GetInputFileCount() const {
 240   base::AutoLock lock(lock_);
 241   return static_cast<int>(input_files_.size());
 242 }
 243
 244 void InputFileManager::GetAllPhysicalInputFileNames(
 245     std::vector<base::FilePath>* result) const {
 246   base::AutoLock lock(lock_);
 247   result->reserve(input_files_.size());
 248   for (InputFileMap::const_iterator i = input_files_.begin();
 249        i != input_files_.end(); ++i) {
 250     if (!i->second->file.physical_name().empty())
 251       result->push_back(i->second->file.physical_name());
 252   }
 253 }
 254
 255 void InputFileManager::BackgroundLoadFile(const LocationRange& origin,
 256                                           const BuildSettings* build_settings,
 257                                           const SourceFile& name,
 258                                           InputFile* file) {
 259   Err err;
 260   if (!LoadFile(origin, build_settings, name, file, &err))
 261     g_scheduler->FailWithError(err);
 262 }
 263
 264 bool InputFileManager::LoadFile(const LocationRange& origin,
 265                                 const BuildSettings* build_settings,
 266                                 const SourceFile& name,
 267                                 InputFile* file,
 268                                 Err* err) {
 269   std::vector<Token> tokens;
 270   scoped_ptr<ParseNode> root;
 271   bool success = DoLoadFile(origin, build_settings, name, file,
 272                             &tokens, &root, err);
 273   // Can't return early. We have to ensure that the completion event is
 274   // signaled in all cases bacause another thread could be blocked on this one.
 275
 276   // Save this pointer for running the callbacks below, which happens after the
 277   // scoped ptr ownership is taken away inside the lock.
 278   ParseNode* unowned_root = root.get();
 279
 280   std::vector<FileLoadCallback> callbacks;
 281   {
 282     base::AutoLock lock(lock_);
 283     DCHECK(input_files_.find(name) != input_files_.end());
 284
 285     InputFileData* data = input_files_[name];
 286     data->loaded = true;
 287     if (success) {
 288       data->tokens.swap(tokens);
 289       data->parsed_root = root.Pass();
 290     }
 291
 292     // Unblock waiters on this event.
 293     //
 294     // It's somewhat bad to signal this inside the lock. When it's used, it's
 295     // lazily created inside the lock. So we need to do the check and signal
 296     // inside the lock to avoid race conditions on the lazy creation of the
 297     // lock.
 298     //
 299     // We could avoid this by creating the lock every time, but the lock is
 300     // very seldom used and will generally be NULL, so my current theory is that
 301     // several signals of a completion event inside a lock is better than
 302     // creating about 1000 extra locks (one for each file).
 303     if (data->completion_event)
 304       data->completion_event->Signal();
 305
 306     callbacks.swap(data->scheduled_callbacks);
 307   }
 308
 309   // Run pending invocations. Theoretically we could schedule each of these
 310   // separately to get some parallelism. But normally there will only be one
 311   // item in the list, so that's extra overhead and complexity for no gain.
 312   if (success) {
 313     for (size_t i = 0; i < callbacks.size(); i++)
 314       callbacks[i].Run(unowned_root);
 315   }
 316   return success;
 317 }