1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "tools/gn/toolchain_manager.h"
10 #include "build/build_config.h"
11 #include "tools/gn/err.h"
12 #include "tools/gn/item.h"
13 #include "tools/gn/item_node.h"
14 #include "tools/gn/item_tree.h"
15 #include "tools/gn/parse_tree.h"
16 #include "tools/gn/scheduler.h"
17 #include "tools/gn/scope.h"
18 #include "tools/gn/scope_per_file_provider.h"
19 #include "tools/gn/trace.h"
21 // How toolchain loading works
22 // ---------------------------
23 // When we start loading a build, we'll load the build config file and that
24 // will call set_default_toolchain. We'll schedule a load of the file
25 // containing the default toolchain definition, and can do this in parallel
26 // with all other build files. Targets will have an implicit dependency on the
27 // toolchain so we won't write out any files until we've loaded the toolchain
30 // When we see a reference to a target using a different toolchain, it gets
31 // more complicated. In this case, the toolchain definition contains arguments
32 // to pass into the build config file when it is invoked in the context of that
33 // toolchain. As a result, we have to actually see the definition of the
34 // toolchain before doing anything else.
36 // So when we see a reference to a non-default toolchain we do the following:
38 // 1. Schedule a load of the file containing the toolchain definition, if it
39 // isn't loaded already.
40 // 2. When the toolchain definition is loaded, we schedule a load of the
41 // build config file in the context of that toolchain. We'll use the
42 // arguments from the toolchain definition to execute it.
43 // 3. When the build config is set up, then we can load all of the individual
44 // buildfiles in the context of that config that we require.
49 // Toolchain settings have not requested to be loaded. This means we
50 // haven't seen any targets that require this toolchain yet. This means that
51 // we have seen a toolchain definition, but no targets that use it. Not
52 // loading the settings automatically allows you to define a bunch of
53 // toolchains and potentially not use them without much overhead.
56 // The toolchain definition for non-default toolchains has been scheduled
57 // to be loaded but has not completed. When this is done, we can load the
58 // settings file. This is needed to get the arguments out of the toolchain
59 // definition. This is skipped for the default toolchain which has no
60 // arguments (see summary above).
61 TOOLCHAIN_DEFINITION_LOADING,
63 // The settings have been scheduled to be loaded but have not completed.
64 TOOLCHAIN_SETTINGS_LOADING,
66 // The settings are done being loaded.
67 TOOLCHAIN_SETTINGS_LOADED
70 SourceFile DirToBuildFile(const SourceDir& dir) {
71 return SourceFile(dir.value() + "BUILD.gn");
76 struct ToolchainManager::Info {
77 Info(const BuildSettings* build_settings,
78 const Label& toolchain_name,
79 const std::string& output_subdir_name)
80 : state(TOOLCHAIN_NOT_LOADED),
81 settings(build_settings, output_subdir_name),
82 toolchain(new Toolchain(&settings, toolchain_name)),
84 toolchain_file_loaded(false),
86 settings.set_toolchain_label(toolchain_name);
90 if (!item_node) // See toolchain definition for more.
94 // Makes sure that an ItemNode is created for the toolchain, which lets
95 // targets depend on the (potentially future) loading of the toolchain.
97 // We can't always do this at the beginning since when doing the default
98 // build config, we don't know the toolchain name yet. We also need to go
99 // through some effort to avoid doing this inside the toolchain manager's
100 // lock (to avoid holding two locks at once).
101 void EnsureItemNode() {
103 ItemTree& tree = settings.build_settings()->item_tree();
104 item_node = new ItemNode(toolchain);
105 tree.AddNodeLocked(item_node);
109 ToolchainState state;
111 // The first place in the build that we saw a reference for this toolchain.
112 // This allows us to report errors if it can't be loaded and blame some
113 // reasonable place of the code. This will be empty for the default toolchain.
114 LocationRange specified_from;
116 // When the state is TOOLCHAIN_SETTINGS_LOADED, the settings should be
117 // considered read-only and can be read without locking. Otherwise, they
118 // should not be accessed at all except to load them (which can therefore
119 // also be done outside of the lock). This works as long as the state flag
120 // is only ever read or written inside the lock.
123 // When we create an item node, this pointer will be owned by that node
124 // so it's lifetime is managed by the dependency graph. Before we've created
125 // the ItemNode, this class has to takre responsibility for this pointer.
126 Toolchain* toolchain;
128 LocationRange toolchain_definition_location;
130 // Set when the file corresponding to the toolchain definition is loaded.
131 // This will normally be set right after "toolchain_set". However, if the
132 // toolchain definition is missing, the file might be marked loaded but the
133 // toolchain definition could still be unset.
134 bool toolchain_file_loaded;
136 // While state == TOOLCHAIN_SETTINGS_LOADING || TOOLCHAIN_DEFINITION_LOADING,
137 // this will collect all scheduled invocations using this toolchain. They'll
138 // be issued once the settings file has been interpreted.
140 // The map maps the source file to "some" location it was invoked from (so
141 // we can give good error messages). It does NOT map to the root of the
142 // file to be invoked (the file still needs loading). This will be NULL
143 // for internally invoked files.
144 typedef std::map<SourceFile, LocationRange> ScheduledInvocationMap;
145 ScheduledInvocationMap scheduled_invocations;
147 // Tracks all scheduled and executed invocations for this toolchain. This
148 // is used to avoid invoking a file more than once for a toolchain.
149 std::set<SourceFile> all_invocations;
151 // Filled in by EnsureItemNode, see that for more.
155 ToolchainManager::ToolchainManager(const BuildSettings* build_settings)
156 : build_settings_(build_settings) {
159 ToolchainManager::~ToolchainManager() {
160 for (ToolchainMap::iterator i = toolchains_.begin();
161 i != toolchains_.end(); ++i)
166 void ToolchainManager::StartLoadingUnlocked(const SourceFile& build_file_name) {
167 // How the default build config works: Initially we don't have a toolchain
168 // name to call the settings for the default build config. So we create one
169 // with an empty toolchain name and execute the default build config file.
170 // When that's done, we'll go and fix up the name to the default build config
171 // that the script set.
172 base::AutoLock lock(GetLock());
174 Info* info = LoadNewToolchainLocked(LocationRange(), Label(), &err);
176 g_scheduler->FailWithError(err);
178 info->scheduled_invocations[build_file_name] = LocationRange();
179 info->all_invocations.insert(build_file_name);
181 if (!ScheduleBuildConfigLoadLocked(info, true, &err))
182 g_scheduler->FailWithError(err);
185 const Settings* ToolchainManager::GetSettingsForToolchainLocked(
186 const LocationRange& from_here,
187 const Label& toolchain_name,
189 GetLock().AssertAcquired();
190 ToolchainMap::iterator found = toolchains_.find(toolchain_name);
192 if (found == toolchains_.end()) {
193 info = LoadNewToolchainLocked(from_here, toolchain_name, err);
197 info = found->second;
199 info->EnsureItemNode();
201 return &info->settings;
204 const Toolchain* ToolchainManager::GetToolchainDefinitionUnlocked(
205 const Label& toolchain_name) {
206 base::AutoLock lock(GetLock());
207 ToolchainMap::iterator found = toolchains_.find(toolchain_name);
208 if (found == toolchains_.end() || !found->second->toolchain_set)
211 // Since we don't allow defining a toolchain more than once, we know that
212 // once it's set it won't be mutated, so we can safely return this pointer
213 // for reading outside the lock.
214 return found->second->toolchain;
217 bool ToolchainManager::SetDefaultToolchainUnlocked(
218 const Label& default_toolchain,
219 const LocationRange& defined_here,
221 base::AutoLock lock(GetLock());
222 if (!default_toolchain_.is_null()) {
223 *err = Err(defined_here, "Default toolchain already set.");
224 err->AppendSubErr(Err(default_toolchain_defined_here_,
225 "Previously defined here.",
226 "You can only set this once."));
230 if (default_toolchain.is_null()) {
231 *err = Err(defined_here, "Bad default toolchain name.",
232 "You can't set the default toolchain name to nothing.");
235 if (!default_toolchain.toolchain_dir().is_null() ||
236 !default_toolchain.toolchain_name().empty()) {
237 *err = Err(defined_here, "Toolchain name has toolchain.",
238 "You can't specify a toolchain (inside the parens) for a toolchain "
239 "name. I got:\n" + default_toolchain.GetUserVisibleName(true));
243 default_toolchain_ = default_toolchain;
244 default_toolchain_defined_here_ = defined_here;
248 Label ToolchainManager::GetDefaultToolchainUnlocked() const {
249 base::AutoLock lock(GetLock());
250 return default_toolchain_;
253 bool ToolchainManager::SetToolchainDefinitionLocked(
255 const LocationRange& defined_from,
257 GetLock().AssertAcquired();
259 ToolchainMap::iterator found = toolchains_.find(tc.label());
261 if (found == toolchains_.end()) {
263 info = LoadNewToolchainLocked(defined_from, tc.label(), err);
267 // It's important to preserve the exact Toolchain object in our tree since
268 // it will be in the ItemTree and targets may have dependencies on it.
269 info = found->second;
272 // The labels should match or else we're setting the wrong one!
273 CHECK(info->toolchain->label() == tc.label());
275 // Save the toolchain. We can just overwrite our definition.
276 *info->toolchain = tc;
278 if (info->toolchain_set) {
279 *err = Err(defined_from, "Duplicate toolchain definition.");
280 err->AppendSubErr(Err(
281 info->toolchain_definition_location,
282 "Previously defined here.",
283 "A toolchain can only be defined once. One tricky way that this could\n"
284 "happen is if your definition is itself in a file that's interpreted\n"
285 "under different toolchains, which would result in multiple\n"
286 "definitions as the file is loaded multiple times. So be sure your\n"
287 "toolchain definitions are in files that either don't define any\n"
288 "targets (probably best) or at least don't contain targets executed\n"
289 "with more than one toolchain."));
293 info->EnsureItemNode();
295 info->toolchain_set = true;
296 info->toolchain_definition_location = defined_from;
300 bool ToolchainManager::ScheduleInvocationLocked(
301 const LocationRange& specified_from,
302 const Label& toolchain_name,
303 const SourceDir& dir,
305 GetLock().AssertAcquired();
306 SourceFile build_file(DirToBuildFile(dir));
308 // If there's no specified toolchain name, use the default.
309 ToolchainMap::iterator found;
310 if (toolchain_name.is_null())
311 found = toolchains_.find(default_toolchain_);
313 found = toolchains_.find(toolchain_name);
316 if (found == toolchains_.end()) {
318 info = LoadNewToolchainLocked(specified_from, toolchain_name, err);
323 info = found->second;
324 if (info->all_invocations.find(build_file) !=
325 info->all_invocations.end()) {
326 // We've already seen this source file for this toolchain, don't need
332 info->all_invocations.insert(build_file);
334 switch (info->state) {
335 case TOOLCHAIN_NOT_LOADED:
336 // Toolchain needs to be loaded. Start loading it and push this buildfile
337 // on the wait queue. The actual toolchain build file will have been
338 // scheduled to be loaded by LoadNewToolchainLocked() above, so we just
339 // need to request that the build settings be loaded when that toolchain
340 // file is done executing (recall toolchain files are executed in the
341 // context of the default toolchain, which is why we need to do the extra
342 // Info lookup in the make_pair).
343 DCHECK(!default_toolchain_.is_null());
344 pending_build_config_map_[
345 std::make_pair(DirToBuildFile(toolchain_name.dir()),
346 toolchains_[default_toolchain_])] =
348 info->scheduled_invocations[build_file] = specified_from;
350 // Transition to the loading state.
351 info->specified_from = specified_from;
352 info->state = TOOLCHAIN_DEFINITION_LOADING;
355 case TOOLCHAIN_DEFINITION_LOADING:
356 case TOOLCHAIN_SETTINGS_LOADING:
357 // Toolchain is in the process of loading, push this buildfile on the
358 // wait queue to run when the config is ready.
359 info->scheduled_invocations[build_file] = specified_from;
362 case TOOLCHAIN_SETTINGS_LOADED:
363 // Everything is ready, just schedule the build file to load.
364 return ScheduleBackgroundInvoke(info, specified_from, build_file, err);
373 std::string ToolchainManager::ToolchainToOutputSubdir(
374 const Label& toolchain_name) {
375 // For now just assume the toolchain name is always a valid dir name. We may
376 // want to clean up the in the future.
377 return toolchain_name.name();
380 ToolchainManager::Info* ToolchainManager::LoadNewToolchainLocked(
381 const LocationRange& specified_from,
382 const Label& toolchain_name,
384 GetLock().AssertAcquired();
385 Info* info = new Info(build_settings_,
387 ToolchainToOutputSubdir(toolchain_name));
389 toolchains_[toolchain_name] = info;
391 // Invoke the file containing the toolchain definition so that it gets
392 // defined. The default one (label is empty) will be done spearately.
393 if (!toolchain_name.is_null()) {
394 // The default toolchain should be specified whenever we're requesting
395 // another one. This is how we know under what context we should execute
396 // the invoke for the toolchain file.
397 CHECK(!default_toolchain_.is_null());
398 ScheduleInvocationLocked(specified_from, default_toolchain_,
399 toolchain_name.dir(), err);
404 void ToolchainManager::FixupDefaultToolchainLocked() {
405 // Now that we've run the default build config, we should know the
406 // default toolchain name. Fix up our reference.
407 // See Start() for more.
408 GetLock().AssertAcquired();
409 if (default_toolchain_.is_null()) {
410 g_scheduler->FailWithError(Err(Location(),
411 "Default toolchain not set.",
412 "Your build config file \"" +
413 build_settings_->build_config_file().value() +
414 "\"\ndid not call set_default_toolchain(). This is needed so "
415 "I know how to actually\ncompile your code."));
419 ToolchainMap::iterator old_default = toolchains_.find(Label());
420 CHECK(old_default != toolchains_.end());
421 Info* info = old_default->second;
422 toolchains_[default_toolchain_] = info;
423 toolchains_.erase(old_default);
425 // Toolchain should not have been loaded in the build config file.
426 CHECK(!info->toolchain_set);
428 // We need to set the toolchain label now that we know it. There's no way
429 // to set the label, but we can assign the toolchain to a new one. Loading
430 // the build config can not change the toolchain, so we won't be overwriting
432 *info->toolchain = Toolchain(&info->settings, default_toolchain_);
433 info->settings.set_is_default(true);
434 info->settings.set_toolchain_label(default_toolchain_);
435 info->EnsureItemNode();
437 // The default toolchain is loaded in greedy mode so all targets we
438 // encounter are generated. Non-default toolchain settings stay in non-greedy
439 // so we only generate the minimally required set.
440 info->settings.set_greedy_target_generation(true);
442 // Schedule a load of the toolchain build file.
444 ScheduleInvocationLocked(LocationRange(), default_toolchain_,
445 default_toolchain_.dir(), &err);
447 g_scheduler->FailWithError(err);
450 bool ToolchainManager::ScheduleBackgroundInvoke(
452 const LocationRange& specified_from,
453 const SourceFile& build_file,
455 g_scheduler->IncrementWorkCount();
456 if (!g_scheduler->input_file_manager()->AsyncLoadFile(
457 specified_from, build_settings_, build_file,
458 base::Bind(&ToolchainManager::BackgroundInvoke,
459 base::Unretained(this), info, build_file),
461 g_scheduler->DecrementWorkCount();
467 bool ToolchainManager::ScheduleBuildConfigLoadLocked(Info* info,
470 GetLock().AssertAcquired();
472 g_scheduler->IncrementWorkCount();
473 if (!g_scheduler->input_file_manager()->AsyncLoadFile(
474 info->specified_from, build_settings_,
475 build_settings_->build_config_file(),
476 base::Bind(&ToolchainManager::BackgroundLoadBuildConfig,
477 base::Unretained(this), info, is_default),
479 g_scheduler->DecrementWorkCount();
482 info->state = TOOLCHAIN_SETTINGS_LOADING;
486 void ToolchainManager::BackgroundLoadBuildConfig(Info* info,
488 const ParseNode* root) {
489 // Danger: No early returns without decrementing the work count.
490 if (root && !g_scheduler->is_failed()) {
491 // Nobody should be accessing settings at this point other than us since we
492 // haven't marked it loaded, so we can do it outside the lock.
493 Scope* base_config = info->settings.base_config();
494 base_config->set_source_dir(SourceDir("//"));
496 info->settings.build_settings()->build_args().SetupRootScope(
497 base_config, info->toolchain->args());
499 base_config->SetProcessingBuildConfig();
501 base_config->SetProcessingDefaultBuildConfig();
503 ScopedTrace trace(TraceItem::TRACE_FILE_EXECUTE,
504 info->settings.build_settings()->build_config_file().value());
505 trace.SetToolchain(info->settings.toolchain_label());
507 const BlockNode* root_block = root->AsBlock();
509 root_block->ExecuteBlockInScope(base_config, &err);
513 base_config->ClearProcessingBuildConfig();
515 base_config->ClearProcessingDefaultBuildConfig();
517 if (err.has_error()) {
518 g_scheduler->FailWithError(err);
520 // Base config processing succeeded.
521 Info::ScheduledInvocationMap schedule_these;
523 base::AutoLock lock(GetLock());
524 schedule_these.swap(info->scheduled_invocations);
525 info->state = TOOLCHAIN_SETTINGS_LOADED;
527 FixupDefaultToolchainLocked();
530 // Schedule build files waiting on this settings. There can be many so we
531 // want to load them in parallel on the pool.
532 for (Info::ScheduledInvocationMap::iterator i = schedule_these.begin();
533 i != schedule_these.end() && !g_scheduler->is_failed(); ++i) {
534 if (!ScheduleBackgroundInvoke(info, i->second, i->first, &err)) {
535 g_scheduler->FailWithError(err);
541 g_scheduler->DecrementWorkCount();
544 void ToolchainManager::BackgroundInvoke(const Info* info,
545 const SourceFile& file_name,
546 const ParseNode* root) {
547 if (root && !g_scheduler->is_failed()) {
548 if (g_scheduler->verbose_logging()) {
549 g_scheduler->Log("Running", file_name.value() + " with toolchain " +
550 info->toolchain->label().GetUserVisibleName(false));
553 Scope our_scope(info->settings.base_config());
554 ScopePerFileProvider per_file_provider(&our_scope);
555 our_scope.set_source_dir(file_name.GetDir());
557 ScopedTrace trace(TraceItem::TRACE_FILE_EXECUTE, file_name.value());
558 trace.SetToolchain(info->settings.toolchain_label());
561 root->Execute(&our_scope, &err);
563 g_scheduler->FailWithError(err);
568 // Check to see if any build config invocations depend on this file and
570 base::AutoLock lock(GetLock());
571 BuildConfigInvokeMap::iterator found_file =
572 pending_build_config_map_.find(std::make_pair(file_name, info));
573 if (found_file != pending_build_config_map_.end()) {
574 // The toolchain state should be waiting on the definition, which
575 // should be the thing we just loaded.
576 Info* info_to_load = found_file->second;
577 DCHECK(info_to_load->state == TOOLCHAIN_DEFINITION_LOADING);
578 DCHECK(!info_to_load->toolchain_file_loaded);
579 info_to_load->toolchain_file_loaded = true;
581 if (!ScheduleBuildConfigLoadLocked(info_to_load, false, &err))
582 g_scheduler->FailWithError(err);
583 pending_build_config_map_.erase(found_file);
588 g_scheduler->DecrementWorkCount();
591 base::Lock& ToolchainManager::GetLock() const {
592 return build_settings_->item_tree().lock();