1 # Copyright (c) 2012 Google Inc. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 from compiler.ast import Const
6 from compiler.ast import Dict
7 from compiler.ast import Discard
8 from compiler.ast import List
9 from compiler.ast import Module
10 from compiler.ast import Node
11 from compiler.ast import Stmt
14 import gyp.simple_copy
15 import multiprocessing
26 from gyp.common import GypError
27 from gyp.common import OrderedSet
30 # A list of types that are treated as linkable.
31 linkable_types = ['executable', 'shared_library', 'loadable_module']
33 # A list of sections that contain links to other targets.
34 dependency_sections = ['dependencies', 'export_dependent_settings']
36 # base_path_sections is a list of sections defined by GYP that contain
37 # pathnames. The generators can provide more keys, the two lists are merged
38 # into path_sections, but you should call IsPathSection instead of using either
40 base_path_sections = [
51 def IsPathSection(section):
52 # If section ends in one of the '=+?!' characters, it's applied to a section
53 # without the trailing characters. '/' is notably absent from this list,
54 # because there's no way for a regular expression to be treated as a path.
55 while section[-1:] in '=+?!':
56 section = section[:-1]
58 if section in path_sections:
61 # Sections mathing the regexp '_(dir|file|path)s?$' are also
62 # considered PathSections. Using manual string matching since that
63 # is much faster than the regexp and this can be called hundreds of
64 # thousands of times so micro performance matters.
69 if tail[-5:] in ('_file', '_path'):
71 return tail[-4:] == '_dir'
75 # base_non_configuration_keys is a list of key names that belong in the target
76 # itself and should not be propagated into its configurations. It is merged
77 # with a list that can come from the generator to
78 # create non_configuration_keys.
79 base_non_configuration_keys = [
80 # Sections that must exist inside targets and not configurations.
84 'default_configuration',
86 'dependencies_original',
96 'standalone_static_library',
103 # Sections that can be found inside targets or configurations, but that
104 # should not be propagated from targets into their configurations.
107 non_configuration_keys = []
109 # Keys that do not belong inside a configuration dictionary.
110 invalid_configuration_keys = [
112 'all_dependent_settings',
115 'direct_dependent_settings',
119 'standalone_static_library',
124 # Controls whether or not the generator supports multiple toolsets.
125 multiple_toolsets = False
127 # Paths for converting filelist paths to output paths: {
129 # qualified_output_dir,
131 generator_filelist_paths = None
133 def GetIncludedBuildFiles(build_file_path, aux_data, included=None):
134 """Return a list of all build files included into build_file_path.
136 The returned list will contain build_file_path as well as all other files
137 that it included, either directly or indirectly. Note that the list may
138 contain files that were included into a conditional section that evaluated
139 to false and was not merged into build_file_path's dict.
141 aux_data is a dict containing a key for each build file or included build
142 file. Those keys provide access to dicts whose "included" keys contain
143 lists of all other files included by the build file.
145 included should be left at its default None value by external callers. It
146 is used for recursion.
148 The returned list will not contain any duplicate entries. Each build file
149 in the list will be relative to the current directory.
155 if build_file_path in included:
158 included.append(build_file_path)
160 for included_build_file in aux_data[build_file_path].get('included', []):
161 GetIncludedBuildFiles(included_build_file, aux_data, included)
166 def CheckedEval(file_contents):
167 """Return the eval of a gyp file.
169 The gyp file is restricted to dictionaries and lists only, and
170 repeated keys are not allowed.
172 Note that this is slower than eval() is.
175 ast = compiler.parse(file_contents)
176 assert isinstance(ast, Module)
177 c1 = ast.getChildren()
179 assert isinstance(c1[1], Stmt)
180 c2 = c1[1].getChildren()
181 assert isinstance(c2[0], Discard)
182 c3 = c2[0].getChildren()
184 return CheckNode(c3[0], [])
187 def CheckNode(node, keypath):
188 if isinstance(node, Dict):
189 c = node.getChildren()
191 for n in range(0, len(c), 2):
192 assert isinstance(c[n], Const)
193 key = c[n].getChildren()[0]
195 raise GypError("Key '" + key + "' repeated at level " +
196 repr(len(keypath) + 1) + " with key path '" +
197 '.'.join(keypath) + "'")
198 kp = list(keypath) # Make a copy of the list for descending this node.
200 dict[key] = CheckNode(c[n + 1], kp)
202 elif isinstance(node, List):
203 c = node.getChildren()
205 for index, child in enumerate(c):
206 kp = list(keypath) # Copy list.
207 kp.append(repr(index))
208 children.append(CheckNode(child, kp))
210 elif isinstance(node, Const):
211 return node.getChildren()[0]
213 raise TypeError, "Unknown AST node at key path '" + '.'.join(keypath) + \
217 def LoadOneBuildFile(build_file_path, data, aux_data, includes,
219 if build_file_path in data:
220 return data[build_file_path]
222 if os.path.exists(build_file_path):
223 build_file_contents = open(build_file_path).read()
225 raise GypError("%s not found (cwd: %s)" % (build_file_path, os.getcwd()))
227 build_file_data = None
230 build_file_data = CheckedEval(build_file_contents)
232 build_file_data = eval(build_file_contents, {'__builtins__': None},
234 except SyntaxError, e:
235 e.filename = build_file_path
238 gyp.common.ExceptionAppend(e, 'while reading ' + build_file_path)
241 if type(build_file_data) is not dict:
242 raise GypError("%s does not evaluate to a dictionary." % build_file_path)
244 data[build_file_path] = build_file_data
245 aux_data[build_file_path] = {}
247 # Scan for includes and merge them in.
248 if ('skip_includes' not in build_file_data or
249 not build_file_data['skip_includes']):
252 LoadBuildFileIncludesIntoDict(build_file_data, build_file_path, data,
253 aux_data, includes, check)
255 LoadBuildFileIncludesIntoDict(build_file_data, build_file_path, data,
256 aux_data, None, check)
258 gyp.common.ExceptionAppend(e,
259 'while reading includes of ' + build_file_path)
262 return build_file_data
265 def LoadBuildFileIncludesIntoDict(subdict, subdict_path, data, aux_data,
269 includes_list.extend(includes)
270 if 'includes' in subdict:
271 for include in subdict['includes']:
272 # "include" is specified relative to subdict_path, so compute the real
273 # path to include by appending the provided "include" to the directory
274 # in which subdict_path resides.
276 os.path.normpath(os.path.join(os.path.dirname(subdict_path), include))
277 includes_list.append(relative_include)
278 # Unhook the includes list, it's no longer needed.
279 del subdict['includes']
281 # Merge in the included files.
282 for include in includes_list:
283 if not 'included' in aux_data[subdict_path]:
284 aux_data[subdict_path]['included'] = []
285 aux_data[subdict_path]['included'].append(include)
287 gyp.DebugOutput(gyp.DEBUG_INCLUDES, "Loading Included File: '%s'", include)
290 LoadOneBuildFile(include, data, aux_data, None, False, check),
291 subdict_path, include)
293 # Recurse into subdictionaries.
294 for k, v in subdict.iteritems():
296 LoadBuildFileIncludesIntoDict(v, subdict_path, data, aux_data,
298 elif type(v) is list:
299 LoadBuildFileIncludesIntoList(v, subdict_path, data, aux_data,
303 # This recurses into lists so that it can look for dicts.
304 def LoadBuildFileIncludesIntoList(sublist, sublist_path, data, aux_data, check):
306 if type(item) is dict:
307 LoadBuildFileIncludesIntoDict(item, sublist_path, data, aux_data,
309 elif type(item) is list:
310 LoadBuildFileIncludesIntoList(item, sublist_path, data, aux_data, check)
312 # Processes toolsets in all the targets. This recurses into condition entries
313 # since they can contain toolsets as well.
314 def ProcessToolsetsInDict(data):
315 if 'targets' in data:
316 target_list = data['targets']
318 for target in target_list:
319 # If this target already has an explicit 'toolset', and no 'toolsets'
320 # list, don't modify it further.
321 if 'toolset' in target and 'toolsets' not in target:
322 new_target_list.append(target)
324 if multiple_toolsets:
325 toolsets = target.get('toolsets', ['target'])
327 toolsets = ['target']
328 # Make sure this 'toolsets' definition is only processed once.
329 if 'toolsets' in target:
330 del target['toolsets']
331 if len(toolsets) > 0:
332 # Optimization: only do copies if more than one toolset is specified.
333 for build in toolsets[1:]:
334 new_target = gyp.simple_copy.deepcopy(target)
335 new_target['toolset'] = build
336 new_target_list.append(new_target)
337 target['toolset'] = toolsets[0]
338 new_target_list.append(target)
339 data['targets'] = new_target_list
340 if 'conditions' in data:
341 for condition in data['conditions']:
342 if type(condition) is list:
343 for condition_dict in condition[1:]:
344 ProcessToolsetsInDict(condition_dict)
347 # TODO(mark): I don't love this name. It just means that it's going to load
348 # a build file that contains targets and is expected to provide a targets dict
349 # that contains the targets...
350 def LoadTargetBuildFile(build_file_path, data, aux_data, variables, includes,
351 depth, check, load_dependencies):
352 # If depth is set, predefine the DEPTH variable to be a relative path from
353 # this build file's directory to the directory identified by depth.
355 # TODO(dglazkov) The backslash/forward-slash replacement at the end is a
356 # temporary measure. This should really be addressed by keeping all paths
357 # in POSIX until actual project generation.
358 d = gyp.common.RelativePath(depth, os.path.dirname(build_file_path))
360 variables['DEPTH'] = '.'
362 variables['DEPTH'] = d.replace('\\', '/')
364 if build_file_path in data['target_build_files']:
367 data['target_build_files'].add(build_file_path)
369 gyp.DebugOutput(gyp.DEBUG_INCLUDES,
370 "Loading Target Build File '%s'", build_file_path)
372 build_file_data = LoadOneBuildFile(build_file_path, data, aux_data,
373 includes, True, check)
375 # Store DEPTH for later use in generators.
376 build_file_data['_DEPTH'] = depth
378 # Set up the included_files key indicating which .gyp files contributed to
380 if 'included_files' in build_file_data:
381 raise GypError(build_file_path + ' must not contain included_files key')
383 included = GetIncludedBuildFiles(build_file_path, aux_data)
384 build_file_data['included_files'] = []
385 for included_file in included:
386 # included_file is relative to the current directory, but it needs to
387 # be made relative to build_file_path's directory.
388 included_relative = \
389 gyp.common.RelativePath(included_file,
390 os.path.dirname(build_file_path))
391 build_file_data['included_files'].append(included_relative)
393 # Do a first round of toolsets expansion so that conditions can be defined
395 ProcessToolsetsInDict(build_file_data)
397 # Apply "pre"/"early" variable expansions and condition evaluations.
398 ProcessVariablesAndConditionsInDict(
399 build_file_data, PHASE_EARLY, variables, build_file_path)
401 # Since some toolsets might have been defined conditionally, perform
402 # a second round of toolsets expansion now.
403 ProcessToolsetsInDict(build_file_data)
405 # Look at each project's target_defaults dict, and merge settings into
407 if 'target_defaults' in build_file_data:
408 if 'targets' not in build_file_data:
409 raise GypError("Unable to find targets in build file %s" %
413 while index < len(build_file_data['targets']):
414 # This procedure needs to give the impression that target_defaults is
415 # used as defaults, and the individual targets inherit from that.
416 # The individual targets need to be merged into the defaults. Make
417 # a deep copy of the defaults for each target, merge the target dict
418 # as found in the input file into that copy, and then hook up the
419 # copy with the target-specific data merged into it as the replacement
421 old_target_dict = build_file_data['targets'][index]
422 new_target_dict = gyp.simple_copy.deepcopy(
423 build_file_data['target_defaults'])
424 MergeDicts(new_target_dict, old_target_dict,
425 build_file_path, build_file_path)
426 build_file_data['targets'][index] = new_target_dict
430 del build_file_data['target_defaults']
432 # Look for dependencies. This means that dependency resolution occurs
433 # after "pre" conditionals and variable expansion, but before "post" -
434 # in other words, you can't put a "dependencies" section inside a "post"
435 # conditional within a target.
438 if 'targets' in build_file_data:
439 for target_dict in build_file_data['targets']:
440 if 'dependencies' not in target_dict:
442 for dependency in target_dict['dependencies']:
444 gyp.common.ResolveTarget(build_file_path, dependency, None)[0])
446 if load_dependencies:
447 for dependency in dependencies:
449 LoadTargetBuildFile(dependency, data, aux_data, variables,
450 includes, depth, check, load_dependencies)
452 gyp.common.ExceptionAppend(
453 e, 'while loading dependencies of %s' % build_file_path)
456 return (build_file_path, dependencies)
459 def CallLoadTargetBuildFile(global_flags,
460 build_file_path, data,
462 includes, depth, check,
463 generator_input_info):
464 """Wrapper around LoadTargetBuildFile for parallel processing.
466 This wrapper is used when LoadTargetBuildFile is executed in
471 signal.signal(signal.SIGINT, signal.SIG_IGN)
473 # Apply globals so that the worker process behaves the same.
474 for key, value in global_flags.iteritems():
475 globals()[key] = value
477 # Save the keys so we can return data that changed.
478 data_keys = set(data)
479 aux_data_keys = set(aux_data)
481 SetGeneratorGlobals(generator_input_info)
482 result = LoadTargetBuildFile(build_file_path, data,
484 includes, depth, check, False)
488 (build_file_path, dependencies) = result
492 if key == 'target_build_files':
494 if key not in data_keys:
495 data_out[key] = data[key]
498 if key not in aux_data_keys:
499 aux_data_out[key] = aux_data[key]
501 # This gets serialized and sent back to the main process via a pipe.
502 # It's handled in LoadTargetBuildFileCallback.
503 return (build_file_path,
508 sys.stderr.write("gyp: %s\n" % e)
511 print >>sys.stderr, 'Exception:', e
512 print >>sys.stderr, traceback.format_exc()
516 class ParallelProcessingError(Exception):
520 class ParallelState(object):
521 """Class to keep track of state when processing input files in parallel.
523 If build files are loaded in parallel, use this to keep track of
524 state during farming out and processing parallel jobs. It's stored
525 in a global so that the callback function can have access to it.
529 # The multiprocessing pool.
531 # The condition variable used to protect this object and notify
532 # the main loop when there might be more data to process.
533 self.condition = None
534 # The "data" dict that was passed to LoadTargetBuildFileParallel
536 # The "aux_data" dict that was passed to LoadTargetBuildFileParallel
538 # The number of parallel calls outstanding; decremented when a response
541 # The set of all build files that have been scheduled, so we don't
542 # schedule the same one twice.
543 self.scheduled = set()
544 # A list of dependency build file paths that haven't been scheduled yet.
545 self.dependencies = []
546 # Flag to indicate if there was an error in a child process.
549 def LoadTargetBuildFileCallback(self, result):
550 """Handle the results of running LoadTargetBuildFile in another process.
552 self.condition.acquire()
555 self.condition.notify()
556 self.condition.release()
558 (build_file_path0, data0, aux_data0, dependencies0) = result
559 self.data['target_build_files'].add(build_file_path0)
561 self.data[key] = data0[key]
562 for key in aux_data0:
563 self.aux_data[key] = aux_data0[key]
564 for new_dependency in dependencies0:
565 if new_dependency not in self.scheduled:
566 self.scheduled.add(new_dependency)
567 self.dependencies.append(new_dependency)
569 self.condition.notify()
570 self.condition.release()
573 def LoadTargetBuildFilesParallel(build_files, data, aux_data,
574 variables, includes, depth, check,
575 generator_input_info):
576 parallel_state = ParallelState()
577 parallel_state.condition = threading.Condition()
578 # Make copies of the build_files argument that we can modify while working.
579 parallel_state.dependencies = list(build_files)
580 parallel_state.scheduled = set(build_files)
581 parallel_state.pending = 0
582 parallel_state.data = data
583 parallel_state.aux_data = aux_data
586 parallel_state.condition.acquire()
587 while parallel_state.dependencies or parallel_state.pending:
588 if parallel_state.error:
590 if not parallel_state.dependencies:
591 parallel_state.condition.wait()
594 dependency = parallel_state.dependencies.pop()
596 parallel_state.pending += 1
598 data_in['target_build_files'] = data['target_build_files']
601 'path_sections': globals()['path_sections'],
602 'non_configuration_keys': globals()['non_configuration_keys'],
603 'multiple_toolsets': globals()['multiple_toolsets']}
605 if not parallel_state.pool:
606 parallel_state.pool = multiprocessing.Pool(multiprocessing.cpu_count())
607 parallel_state.pool.apply_async(
608 CallLoadTargetBuildFile,
609 args = (global_flags, dependency,
610 data_in, aux_data_in,
611 variables, includes, depth, check, generator_input_info),
612 callback = parallel_state.LoadTargetBuildFileCallback)
613 except KeyboardInterrupt, e:
614 parallel_state.pool.terminate()
617 parallel_state.condition.release()
619 parallel_state.pool.close()
620 parallel_state.pool.join()
621 parallel_state.pool = None
623 if parallel_state.error:
626 # Look for the bracket that matches the first bracket seen in a
627 # string, and return the start and end as a tuple. For example, if
628 # the input is something like "<(foo <(bar)) blah", then it would
629 # return (1, 13), indicating the entire string except for the leading
630 # "<" and trailing " blah".
631 LBRACKETS= set('{[(')
632 BRACKETS = {'}': '{', ']': '[', ')': '('}
633 def FindEnclosingBracketGroup(input_str):
636 for index, char in enumerate(input_str):
637 if char in LBRACKETS:
641 elif char in BRACKETS:
644 if stack.pop() != BRACKETS[char]:
647 return (start, index + 1)
651 def IsStrCanonicalInt(string):
652 """Returns True if |string| is in its canonical integer form.
654 The canonical form is such that str(int(string)) == string.
656 if type(string) is str:
657 # This function is called a lot so for maximum performance, avoid
658 # involving regexps which would otherwise make the code much
659 # shorter. Regexps would need twice the time of this function.
667 if '1' <= string[0] <= '9':
668 return string.isdigit()
673 # This matches things like "<(asdf)", "<!(cmd)", "<!@(cmd)", "<|(list)",
674 # "<!interpreter(arguments)", "<([list])", and even "<([)" and "<(<())".
675 # In the last case, the inner "<()" is captured in match['content'].
676 early_variable_re = re.compile(
677 '(?P<replace>(?P<type><(?:(?:!?@?)|\|)?)'
678 '(?P<command_string>[-a-zA-Z0-9_.]+)?'
679 '\((?P<is_array>\s*\[?)'
680 '(?P<content>.*?)(\]?)\))')
682 # This matches the same as early_variable_re, but with '>' instead of '<'.
683 late_variable_re = re.compile(
684 '(?P<replace>(?P<type>>(?:(?:!?@?)|\|)?)'
685 '(?P<command_string>[-a-zA-Z0-9_.]+)?'
686 '\((?P<is_array>\s*\[?)'
687 '(?P<content>.*?)(\]?)\))')
689 # This matches the same as early_variable_re, but with '^' instead of '<'.
690 latelate_variable_re = re.compile(
691 '(?P<replace>(?P<type>[\^](?:(?:!?@?)|\|)?)'
692 '(?P<command_string>[-a-zA-Z0-9_.]+)?'
693 '\((?P<is_array>\s*\[?)'
694 '(?P<content>.*?)(\]?)\))')
696 # Global cache of results from running commands so they don't have to be run
698 cached_command_results = {}
701 def FixupPlatformCommand(cmd):
702 if sys.platform == 'win32':
703 if type(cmd) is list:
704 cmd = [re.sub('^cat ', 'type ', cmd[0])] + cmd[1:]
706 cmd = re.sub('^cat ', 'type ', cmd)
715 def ExpandVariables(input, phase, variables, build_file):
716 # Look for the pattern that gets expanded into variables
717 if phase == PHASE_EARLY:
718 variable_re = early_variable_re
719 expansion_symbol = '<'
720 elif phase == PHASE_LATE:
721 variable_re = late_variable_re
722 expansion_symbol = '>'
723 elif phase == PHASE_LATELATE:
724 variable_re = latelate_variable_re
725 expansion_symbol = '^'
729 input_str = str(input)
730 if IsStrCanonicalInt(input_str):
731 return int(input_str)
733 # Do a quick scan to determine if an expensive regex search is warranted.
734 if expansion_symbol not in input_str:
737 # Get the entire list of matches as a list of MatchObject instances.
738 # (using findall here would return strings instead of MatchObjects).
739 matches = list(variable_re.finditer(input_str))
744 # Reverse the list of matches so that replacements are done right-to-left.
745 # That ensures that earlier replacements won't mess up the string in a
746 # way that causes later calls to find the earlier substituted text instead
747 # of what's intended for replacement.
749 for match_group in matches:
750 match = match_group.groupdict()
751 gyp.DebugOutput(gyp.DEBUG_VARIABLES, "Matches: %r", match)
752 # match['replace'] is the substring to look for, match['type']
753 # is the character code for the replacement type (< > <! >! <| >| <@
754 # >@ <!@ >!@), match['is_array'] contains a '[' for command
755 # arrays, and match['content'] is the name of the variable (< >)
756 # or command to run (<! >!). match['command_string'] is an optional
757 # command string. Currently, only 'pymod_do_main' is supported.
759 # run_command is true if a ! variant is used.
760 run_command = '!' in match['type']
761 command_string = match['command_string']
763 # file_list is true if a | variant is used.
764 file_list = '|' in match['type']
766 # Capture these now so we can adjust them later.
767 replace_start = match_group.start('replace')
768 replace_end = match_group.end('replace')
770 # Find the ending paren, and re-evaluate the contained string.
771 (c_start, c_end) = FindEnclosingBracketGroup(input_str[replace_start:])
773 # Adjust the replacement range to match the entire command
774 # found by FindEnclosingBracketGroup (since the variable_re
775 # probably doesn't match the entire command if it contained
777 replace_end = replace_start + c_end
779 # Find the "real" replacement, matching the appropriate closing
780 # paren, and adjust the replacement start and end.
781 replacement = input_str[replace_start:replace_end]
783 # Figure out what the contents of the variable parens are.
784 contents_start = replace_start + c_start + 1
785 contents_end = replace_end - 1
786 contents = input_str[contents_start:contents_end]
788 # Do filter substitution now for <|().
789 # Admittedly, this is different than the evaluation order in other
790 # contexts. However, since filtration has no chance to run on <|(),
791 # this seems like the only obvious way to give them access to filters.
793 processed_variables = gyp.simple_copy.deepcopy(variables)
794 ProcessListFiltersInDict(contents, processed_variables)
795 # Recurse to expand variables in the contents
796 contents = ExpandVariables(contents, phase,
797 processed_variables, build_file)
799 # Recurse to expand variables in the contents
800 contents = ExpandVariables(contents, phase, variables, build_file)
802 # Strip off leading/trailing whitespace so that variable matches are
803 # simpler below (and because they are rarely needed).
804 contents = contents.strip()
806 # expand_to_list is true if an @ variant is used. In that case,
807 # the expansion should result in a list. Note that the caller
808 # is to be expecting a list in return, and not all callers do
809 # because not all are working in list context. Also, for list
810 # expansions, there can be no other text besides the variable
811 # expansion in the input string.
812 expand_to_list = '@' in match['type'] and input_str == replacement
814 if run_command or file_list:
815 # Find the build file's directory, so commands can be run or file lists
816 # generated relative to it.
817 build_file_dir = os.path.dirname(build_file)
818 if build_file_dir == '' and not file_list:
819 # If build_file is just a leaf filename indicating a file in the
820 # current directory, build_file_dir might be an empty string. Set
821 # it to None to signal to subprocess.Popen that it should run the
822 # command in the current directory.
823 build_file_dir = None
825 # Support <|(listfile.txt ...) which generates a file
826 # containing items from a gyp list, generated at gyp time.
827 # This works around actions/rules which have more inputs than will
828 # fit on the command line.
830 if type(contents) is list:
831 contents_list = contents
833 contents_list = contents.split(' ')
834 replacement = contents_list[0]
835 if os.path.isabs(replacement):
836 raise GypError('| cannot handle absolute paths, got "%s"' % replacement)
838 if not generator_filelist_paths:
839 path = os.path.join(build_file_dir, replacement)
841 if os.path.isabs(build_file_dir):
842 toplevel = generator_filelist_paths['toplevel']
843 rel_build_file_dir = gyp.common.RelativePath(build_file_dir, toplevel)
845 rel_build_file_dir = build_file_dir
846 qualified_out_dir = generator_filelist_paths['qualified_out_dir']
847 path = os.path.join(qualified_out_dir, rel_build_file_dir, replacement)
848 gyp.common.EnsureDirExists(path)
850 replacement = gyp.common.RelativePath(path, build_file_dir)
851 f = gyp.common.WriteOnDiff(path)
852 for i in contents_list[1:]:
858 if match['is_array']:
859 contents = eval(contents)
862 # Check for a cached value to avoid executing commands, or generating
863 # file lists more than once. The cache key contains the command to be
864 # run as well as the directory to run it from, to account for commands
865 # that depend on their current directory.
866 # TODO(http://code.google.com/p/gyp/issues/detail?id=111): In theory,
867 # someone could author a set of GYP files where each time the command
868 # is invoked it produces different output by design. When the need
869 # arises, the syntax should be extended to support no caching off a
870 # command's output so it is run every time.
871 cache_key = (str(contents), build_file_dir)
872 cached_value = cached_command_results.get(cache_key, None)
873 if cached_value is None:
874 gyp.DebugOutput(gyp.DEBUG_VARIABLES,
875 "Executing command '%s' in directory '%s'",
876 contents, build_file_dir)
880 if command_string == 'pymod_do_main':
881 # <!pymod_do_main(modulename param eters) loads |modulename| as a
882 # python module and then calls that module's DoMain() function,
883 # passing ["param", "eters"] as a single list argument. For modules
884 # that don't load quickly, this can be faster than
885 # <!(python modulename param eters). Do this in |build_file_dir|.
886 oldwd = os.getcwd() # Python doesn't like os.open('.'): no fchdir.
887 if build_file_dir: # build_file_dir may be None (see above).
888 os.chdir(build_file_dir)
891 parsed_contents = shlex.split(contents)
893 py_module = __import__(parsed_contents[0])
894 except ImportError as e:
895 raise GypError("Error importing pymod_do_main"
896 "module (%s): %s" % (parsed_contents[0], e))
897 replacement = str(py_module.DoMain(parsed_contents[1:])).rstrip()
900 assert replacement != None
902 raise GypError("Unknown command string '%s' in '%s'." %
903 (command_string, contents))
905 # Fix up command with platform specific workarounds.
906 contents = FixupPlatformCommand(contents)
907 p = subprocess.Popen(contents, shell=use_shell,
908 stdout=subprocess.PIPE,
909 stderr=subprocess.PIPE,
910 stdin=subprocess.PIPE,
913 p_stdout, p_stderr = p.communicate('')
915 if p.wait() != 0 or p_stderr:
916 sys.stderr.write(p_stderr)
917 # Simulate check_call behavior, since check_call only exists
918 # in python 2.5 and later.
919 raise GypError("Call to '%s' returned exit status %d." %
920 (contents, p.returncode))
921 replacement = p_stdout.rstrip()
923 cached_command_results[cache_key] = replacement
925 gyp.DebugOutput(gyp.DEBUG_VARIABLES,
926 "Had cache value for command '%s' in directory '%s'",
927 contents,build_file_dir)
928 replacement = cached_value
931 if not contents in variables:
932 if contents[-1] in ['!', '/']:
933 # In order to allow cross-compiles (nacl) to happen more naturally,
934 # we will allow references to >(sources/) etc. to resolve to
935 # and empty list if undefined. This allows actions to:
944 raise GypError('Undefined variable ' + contents +
947 replacement = variables[contents]
949 if type(replacement) is list:
950 for item in replacement:
951 if not contents[-1] == '/' and type(item) not in (str, int):
952 raise GypError('Variable ' + contents +
953 ' must expand to a string or list of strings; ' +
955 item.__class__.__name__)
956 # Run through the list and handle variable expansions in it. Since
957 # the list is guaranteed not to contain dicts, this won't do anything
958 # with conditions sections.
959 ProcessVariablesAndConditionsInList(replacement, phase, variables,
961 elif type(replacement) not in (str, int):
962 raise GypError('Variable ' + contents +
963 ' must expand to a string or list of strings; ' +
964 'found a ' + replacement.__class__.__name__)
967 # Expanding in list context. It's guaranteed that there's only one
968 # replacement to do in |input_str| and that it's this replacement. See
970 if type(replacement) is list:
971 # If it's already a list, make a copy.
972 output = replacement[:]
974 # Split it the same way sh would split arguments.
975 output = shlex.split(str(replacement))
977 # Expanding in string context.
978 encoded_replacement = ''
979 if type(replacement) is list:
980 # When expanding a list into string context, turn the list items
981 # into a string in a way that will work with a subprocess call.
983 # TODO(mark): This isn't completely correct. This should
984 # call a generator-provided function that observes the
985 # proper list-to-argument quoting rules on a specific
986 # platform instead of just calling the POSIX encoding
988 encoded_replacement = gyp.common.EncodePOSIXShellList(replacement)
990 encoded_replacement = replacement
992 output = output[:replace_start] + str(encoded_replacement) + \
994 # Prepare for the next match iteration.
998 gyp.DebugOutput(gyp.DEBUG_VARIABLES,
999 "Found only identity matches on %r, avoiding infinite "
1003 # Look for more matches now that we've replaced some, to deal with
1004 # expanding local variables (variables defined in the same
1005 # variables block as this one).
1006 gyp.DebugOutput(gyp.DEBUG_VARIABLES, "Found output %r, recursing.", output)
1007 if type(output) is list:
1008 if output and type(output[0]) is list:
1009 # Leave output alone if it's a list of lists.
1010 # We don't want such lists to be stringified.
1016 ExpandVariables(item, phase, variables, build_file))
1019 output = ExpandVariables(output, phase, variables, build_file)
1021 # Convert all strings that are canonically-represented integers into integers.
1022 if type(output) is list:
1023 for index in xrange(0, len(output)):
1024 if IsStrCanonicalInt(output[index]):
1025 output[index] = int(output[index])
1026 elif IsStrCanonicalInt(output):
1027 output = int(output)
1031 # The same condition is often evaluated over and over again so it
1032 # makes sense to cache as much as possible between evaluations.
1033 cached_conditions_asts = {}
1035 def EvalCondition(condition, conditions_key, phase, variables, build_file):
1036 """Returns the dict that should be used or None if the result was
1037 that nothing should be used."""
1038 if type(condition) is not list:
1039 raise GypError(conditions_key + ' must be a list')
1040 if len(condition) != 2 and len(condition) != 3:
1041 # It's possible that condition[0] won't work in which case this
1042 # attempt will raise its own IndexError. That's probably fine.
1043 raise GypError(conditions_key + ' ' + condition[0] +
1044 ' must be length 2 or 3, not ' + str(len(condition)))
1046 [cond_expr, true_dict] = condition[0:2]
1048 if len(condition) == 3:
1049 false_dict = condition[2]
1051 # Do expansions on the condition itself. Since the conditon can naturally
1052 # contain variable references without needing to resort to GYP expansion
1053 # syntax, this is of dubious value for variables, but someone might want to
1054 # use a command expansion directly inside a condition.
1055 cond_expr_expanded = ExpandVariables(cond_expr, phase, variables,
1057 if type(cond_expr_expanded) not in (str, int):
1059 'Variable expansion in this context permits str and int ' + \
1060 'only, found ' + cond_expr_expanded.__class__.__name__
1063 if cond_expr_expanded in cached_conditions_asts:
1064 ast_code = cached_conditions_asts[cond_expr_expanded]
1066 ast_code = compile(cond_expr_expanded, '<string>', 'eval')
1067 cached_conditions_asts[cond_expr_expanded] = ast_code
1068 if eval(ast_code, {'__builtins__': None}, variables):
1071 except SyntaxError, e:
1072 syntax_error = SyntaxError('%s while evaluating condition \'%s\' in %s '
1073 'at character %d.' %
1074 (str(e.args[0]), e.text, build_file, e.offset),
1075 e.filename, e.lineno, e.offset, e.text)
1077 except NameError, e:
1078 gyp.common.ExceptionAppend(e, 'while evaluating condition \'%s\' in %s' %
1079 (cond_expr_expanded, build_file))
1083 def ProcessConditionsInDict(the_dict, phase, variables, build_file):
1084 # Process a 'conditions' or 'target_conditions' section in the_dict,
1085 # depending on phase.
1086 # early -> conditions
1087 # late -> target_conditions
1088 # latelate -> no conditions
1090 # Each item in a conditions list consists of cond_expr, a string expression
1091 # evaluated as the condition, and true_dict, a dict that will be merged into
1092 # the_dict if cond_expr evaluates to true. Optionally, a third item,
1093 # false_dict, may be present. false_dict is merged into the_dict if
1094 # cond_expr evaluates to false.
1096 # Any dict merged into the_dict will be recursively processed for nested
1097 # conditionals and other expansions, also according to phase, immediately
1098 # prior to being merged.
1100 if phase == PHASE_EARLY:
1101 conditions_key = 'conditions'
1102 elif phase == PHASE_LATE:
1103 conditions_key = 'target_conditions'
1104 elif phase == PHASE_LATELATE:
1109 if not conditions_key in the_dict:
1112 conditions_list = the_dict[conditions_key]
1113 # Unhook the conditions list, it's no longer needed.
1114 del the_dict[conditions_key]
1116 for condition in conditions_list:
1117 merge_dict = EvalCondition(condition, conditions_key, phase, variables,
1120 if merge_dict != None:
1121 # Expand variables and nested conditinals in the merge_dict before
1123 ProcessVariablesAndConditionsInDict(merge_dict, phase,
1124 variables, build_file)
1126 MergeDicts(the_dict, merge_dict, build_file, build_file)
1129 def LoadAutomaticVariablesFromDict(variables, the_dict):
1130 # Any keys with plain string values in the_dict become automatic variables.
1131 # The variable name is the key name with a "_" character prepended.
1132 for key, value in the_dict.iteritems():
1133 if type(value) in (str, int, list):
1134 variables['_' + key] = value
1137 def LoadVariablesFromVariablesDict(variables, the_dict, the_dict_key):
1138 # Any keys in the_dict's "variables" dict, if it has one, becomes a
1139 # variable. The variable name is the key name in the "variables" dict.
1140 # Variables that end with the % character are set only if they are unset in
1141 # the variables dict. the_dict_key is the name of the key that accesses
1142 # the_dict in the_dict's parent dict. If the_dict's parent is not a dict
1143 # (it could be a list or it could be parentless because it is a root dict),
1144 # the_dict_key will be None.
1145 for key, value in the_dict.get('variables', {}).iteritems():
1146 if type(value) not in (str, int, list):
1149 if key.endswith('%'):
1150 variable_name = key[:-1]
1151 if variable_name in variables:
1152 # If the variable is already set, don't set it.
1154 if the_dict_key is 'variables' and variable_name in the_dict:
1155 # If the variable is set without a % in the_dict, and the_dict is a
1156 # variables dict (making |variables| a varaibles sub-dict of a
1157 # variables dict), use the_dict's definition.
1158 value = the_dict[variable_name]
1162 variables[variable_name] = value
1165 def ProcessVariablesAndConditionsInDict(the_dict, phase, variables_in,
1166 build_file, the_dict_key=None):
1167 """Handle all variable and command expansion and conditional evaluation.
1169 This function is the public entry point for all variable expansions and
1170 conditional evaluations. The variables_in dictionary will not be modified
1174 # Make a copy of the variables_in dict that can be modified during the
1175 # loading of automatics and the loading of the variables dict.
1176 variables = variables_in.copy()
1177 LoadAutomaticVariablesFromDict(variables, the_dict)
1179 if 'variables' in the_dict:
1180 # Make sure all the local variables are added to the variables
1181 # list before we process them so that you can reference one
1182 # variable from another. They will be fully expanded by recursion
1183 # in ExpandVariables.
1184 for key, value in the_dict['variables'].iteritems():
1185 variables[key] = value
1187 # Handle the associated variables dict first, so that any variable
1188 # references within can be resolved prior to using them as variables.
1189 # Pass a copy of the variables dict to avoid having it be tainted.
1190 # Otherwise, it would have extra automatics added for everything that
1191 # should just be an ordinary variable in this scope.
1192 ProcessVariablesAndConditionsInDict(the_dict['variables'], phase,
1193 variables, build_file, 'variables')
1195 LoadVariablesFromVariablesDict(variables, the_dict, the_dict_key)
1197 for key, value in the_dict.iteritems():
1198 # Skip "variables", which was already processed if present.
1199 if key != 'variables' and type(value) is str:
1200 expanded = ExpandVariables(value, phase, variables, build_file)
1201 if type(expanded) not in (str, int):
1203 'Variable expansion in this context permits str and int ' + \
1204 'only, found ' + expanded.__class__.__name__ + ' for ' + key
1205 the_dict[key] = expanded
1207 # Variable expansion may have resulted in changes to automatics. Reload.
1208 # TODO(mark): Optimization: only reload if no changes were made.
1209 variables = variables_in.copy()
1210 LoadAutomaticVariablesFromDict(variables, the_dict)
1211 LoadVariablesFromVariablesDict(variables, the_dict, the_dict_key)
1213 # Process conditions in this dict. This is done after variable expansion
1214 # so that conditions may take advantage of expanded variables. For example,
1215 # if the_dict contains:
1216 # {'type': '<(library_type)',
1217 # 'conditions': [['_type=="static_library"', { ... }]]},
1218 # _type, as used in the condition, will only be set to the value of
1219 # library_type if variable expansion is performed before condition
1220 # processing. However, condition processing should occur prior to recursion
1221 # so that variables (both automatic and "variables" dict type) may be
1222 # adjusted by conditions sections, merged into the_dict, and have the
1223 # intended impact on contained dicts.
1225 # This arrangement means that a "conditions" section containing a "variables"
1226 # section will only have those variables effective in subdicts, not in
1227 # the_dict. The workaround is to put a "conditions" section within a
1228 # "variables" section. For example:
1229 # {'conditions': [['os=="mac"', {'variables': {'define': 'IS_MAC'}}]],
1230 # 'defines': ['<(define)'],
1231 # 'my_subdict': {'defines': ['<(define)']}},
1232 # will not result in "IS_MAC" being appended to the "defines" list in the
1233 # current scope but would result in it being appended to the "defines" list
1234 # within "my_subdict". By comparison:
1235 # {'variables': {'conditions': [['os=="mac"', {'define': 'IS_MAC'}]]},
1236 # 'defines': ['<(define)'],
1237 # 'my_subdict': {'defines': ['<(define)']}},
1238 # will append "IS_MAC" to both "defines" lists.
1240 # Evaluate conditions sections, allowing variable expansions within them
1241 # as well as nested conditionals. This will process a 'conditions' or
1242 # 'target_conditions' section, perform appropriate merging and recursive
1243 # conditional and variable processing, and then remove the conditions section
1244 # from the_dict if it is present.
1245 ProcessConditionsInDict(the_dict, phase, variables, build_file)
1247 # Conditional processing may have resulted in changes to automatics or the
1248 # variables dict. Reload.
1249 variables = variables_in.copy()
1250 LoadAutomaticVariablesFromDict(variables, the_dict)
1251 LoadVariablesFromVariablesDict(variables, the_dict, the_dict_key)
1253 # Recurse into child dicts, or process child lists which may result in
1254 # further recursion into descendant dicts.
1255 for key, value in the_dict.iteritems():
1256 # Skip "variables" and string values, which were already processed if
1258 if key == 'variables' or type(value) is str:
1260 if type(value) is dict:
1261 # Pass a copy of the variables dict so that subdicts can't influence
1263 ProcessVariablesAndConditionsInDict(value, phase, variables,
1265 elif type(value) is list:
1266 # The list itself can't influence the variables dict, and
1267 # ProcessVariablesAndConditionsInList will make copies of the variables
1268 # dict if it needs to pass it to something that can influence it. No
1269 # copy is necessary here.
1270 ProcessVariablesAndConditionsInList(value, phase, variables,
1272 elif type(value) is not int:
1273 raise TypeError, 'Unknown type ' + value.__class__.__name__ + \
1277 def ProcessVariablesAndConditionsInList(the_list, phase, variables,
1279 # Iterate using an index so that new values can be assigned into the_list.
1281 while index < len(the_list):
1282 item = the_list[index]
1283 if type(item) is dict:
1284 # Make a copy of the variables dict so that it won't influence anything
1285 # outside of its own scope.
1286 ProcessVariablesAndConditionsInDict(item, phase, variables, build_file)
1287 elif type(item) is list:
1288 ProcessVariablesAndConditionsInList(item, phase, variables, build_file)
1289 elif type(item) is str:
1290 expanded = ExpandVariables(item, phase, variables, build_file)
1291 if type(expanded) in (str, int):
1292 the_list[index] = expanded
1293 elif type(expanded) is list:
1294 the_list[index:index+1] = expanded
1295 index += len(expanded)
1297 # index now identifies the next item to examine. Continue right now
1298 # without falling into the index increment below.
1302 'Variable expansion in this context permits strings and ' + \
1303 'lists only, found ' + expanded.__class__.__name__ + ' at ' + \
1305 elif type(item) is not int:
1306 raise TypeError, 'Unknown type ' + item.__class__.__name__ + \
1307 ' at index ' + index
1311 def BuildTargetsDict(data):
1312 """Builds a dict mapping fully-qualified target names to their target dicts.
1314 |data| is a dict mapping loaded build files by pathname relative to the
1315 current directory. Values in |data| are build file contents. For each
1316 |data| value with a "targets" key, the value of the "targets" key is taken
1317 as a list containing target dicts. Each target's fully-qualified name is
1318 constructed from the pathname of the build file (|data| key) and its
1319 "target_name" property. These fully-qualified names are used as the keys
1320 in the returned dict. These keys provide access to the target dicts,
1321 the dicts in the "targets" lists.
1325 for build_file in data['target_build_files']:
1326 for target in data[build_file].get('targets', []):
1327 target_name = gyp.common.QualifiedTarget(build_file,
1328 target['target_name'],
1330 if target_name in targets:
1331 raise GypError('Duplicate target definitions for ' + target_name)
1332 targets[target_name] = target
1337 def QualifyDependencies(targets):
1338 """Make dependency links fully-qualified relative to the current directory.
1340 |targets| is a dict mapping fully-qualified target names to their target
1341 dicts. For each target in this dict, keys known to contain dependency
1342 links are examined, and any dependencies referenced will be rewritten
1343 so that they are fully-qualified and relative to the current directory.
1344 All rewritten dependencies are suitable for use as keys to |targets| or a
1348 all_dependency_sections = [dep + op
1349 for dep in dependency_sections
1350 for op in ('', '!', '/')]
1352 for target, target_dict in targets.iteritems():
1353 target_build_file = gyp.common.BuildFile(target)
1354 toolset = target_dict['toolset']
1355 for dependency_key in all_dependency_sections:
1356 dependencies = target_dict.get(dependency_key, [])
1357 for index in xrange(0, len(dependencies)):
1358 dep_file, dep_target, dep_toolset = gyp.common.ResolveTarget(
1359 target_build_file, dependencies[index], toolset)
1360 if not multiple_toolsets:
1361 # Ignore toolset specification in the dependency if it is specified.
1362 dep_toolset = toolset
1363 dependency = gyp.common.QualifiedTarget(dep_file,
1366 dependencies[index] = dependency
1368 # Make sure anything appearing in a list other than "dependencies" also
1369 # appears in the "dependencies" list.
1370 if dependency_key != 'dependencies' and \
1371 dependency not in target_dict['dependencies']:
1372 raise GypError('Found ' + dependency + ' in ' + dependency_key +
1373 ' of ' + target + ', but not in dependencies')
1376 def ExpandWildcardDependencies(targets, data):
1377 """Expands dependencies specified as build_file:*.
1379 For each target in |targets|, examines sections containing links to other
1380 targets. If any such section contains a link of the form build_file:*, it
1381 is taken as a wildcard link, and is expanded to list each target in
1382 build_file. The |data| dict provides access to build file dicts.
1384 Any target that does not wish to be included by wildcard can provide an
1385 optional "suppress_wildcard" key in its target dict. When present and
1386 true, a wildcard dependency link will not include such targets.
1388 All dependency names, including the keys to |targets| and the values in each
1389 dependency list, must be qualified when this function is called.
1392 for target, target_dict in targets.iteritems():
1393 toolset = target_dict['toolset']
1394 target_build_file = gyp.common.BuildFile(target)
1395 for dependency_key in dependency_sections:
1396 dependencies = target_dict.get(dependency_key, [])
1398 # Loop this way instead of "for dependency in" or "for index in xrange"
1399 # because the dependencies list will be modified within the loop body.
1401 while index < len(dependencies):
1402 (dependency_build_file, dependency_target, dependency_toolset) = \
1403 gyp.common.ParseQualifiedTarget(dependencies[index])
1404 if dependency_target != '*' and dependency_toolset != '*':
1405 # Not a wildcard. Keep it moving.
1409 if dependency_build_file == target_build_file:
1410 # It's an error for a target to depend on all other targets in
1411 # the same file, because a target cannot depend on itself.
1412 raise GypError('Found wildcard in ' + dependency_key + ' of ' +
1413 target + ' referring to same build file')
1415 # Take the wildcard out and adjust the index so that the next
1416 # dependency in the list will be processed the next time through the
1418 del dependencies[index]
1421 # Loop through the targets in the other build file, adding them to
1422 # this target's list of dependencies in place of the removed
1424 dependency_target_dicts = data[dependency_build_file]['targets']
1425 for dependency_target_dict in dependency_target_dicts:
1426 if int(dependency_target_dict.get('suppress_wildcard', False)):
1428 dependency_target_name = dependency_target_dict['target_name']
1429 if (dependency_target != '*' and
1430 dependency_target != dependency_target_name):
1432 dependency_target_toolset = dependency_target_dict['toolset']
1433 if (dependency_toolset != '*' and
1434 dependency_toolset != dependency_target_toolset):
1436 dependency = gyp.common.QualifiedTarget(dependency_build_file,
1437 dependency_target_name,
1438 dependency_target_toolset)
1440 dependencies.insert(index, dependency)
1446 """Removes duplicate elements from l, keeping the first element."""
1448 return [seen.setdefault(e, e) for e in l if e not in seen]
1451 def RemoveDuplicateDependencies(targets):
1452 """Makes sure every dependency appears only once in all targets's dependency
1454 for target_name, target_dict in targets.iteritems():
1455 for dependency_key in dependency_sections:
1456 dependencies = target_dict.get(dependency_key, [])
1458 target_dict[dependency_key] = Unify(dependencies)
1461 def Filter(l, item):
1462 """Removes item from l."""
1464 return [res.setdefault(e, e) for e in l if e != item]
1467 def RemoveSelfDependencies(targets):
1468 """Remove self dependencies from targets that have the prune_self_dependency
1470 for target_name, target_dict in targets.iteritems():
1471 for dependency_key in dependency_sections:
1472 dependencies = target_dict.get(dependency_key, [])
1474 for t in dependencies:
1475 if t == target_name:
1476 if targets[t].get('variables', {}).get('prune_self_dependency', 0):
1477 target_dict[dependency_key] = Filter(dependencies, target_name)
1480 def RemoveLinkDependenciesFromNoneTargets(targets):
1481 """Remove dependencies having the 'link_dependency' attribute from the 'none'
1483 for target_name, target_dict in targets.iteritems():
1484 for dependency_key in dependency_sections:
1485 dependencies = target_dict.get(dependency_key, [])
1487 for t in dependencies:
1488 if target_dict.get('type', None) == 'none':
1489 if targets[t].get('variables', {}).get('link_dependency', 0):
1490 target_dict[dependency_key] = \
1491 Filter(target_dict[dependency_key], t)
1494 class DependencyGraphNode(object):
1498 ref: A reference to an object that this DependencyGraphNode represents.
1499 dependencies: List of DependencyGraphNodes on which this one depends.
1500 dependents: List of DependencyGraphNodes that depend on this one.
1503 class CircularException(GypError):
1506 def __init__(self, ref):
1508 self.dependencies = []
1509 self.dependents = []
1512 return '<DependencyGraphNode: %r>' % self.ref
1514 def FlattenToList(self):
1515 # flat_list is the sorted list of dependencies - actually, the list items
1516 # are the "ref" attributes of DependencyGraphNodes. Every target will
1517 # appear in flat_list after all of its dependencies, and before all of its
1519 flat_list = OrderedSet()
1521 # in_degree_zeros is the list of DependencyGraphNodes that have no
1522 # dependencies not in flat_list. Initially, it is a copy of the children
1523 # of this node, because when the graph was built, nodes with no
1524 # dependencies were made implicit dependents of the root node.
1525 in_degree_zeros = set(self.dependents[:])
1527 while in_degree_zeros:
1528 # Nodes in in_degree_zeros have no dependencies not in flat_list, so they
1529 # can be appended to flat_list. Take these nodes out of in_degree_zeros
1530 # as work progresses, so that the next node to process from the list can
1531 # always be accessed at a consistent position.
1532 node = in_degree_zeros.pop()
1533 flat_list.add(node.ref)
1535 # Look at dependents of the node just added to flat_list. Some of them
1536 # may now belong in in_degree_zeros.
1537 for node_dependent in node.dependents:
1538 is_in_degree_zero = True
1539 # TODO: We want to check through the
1540 # node_dependent.dependencies list but if it's long and we
1541 # always start at the beginning, then we get O(n^2) behaviour.
1542 for node_dependent_dependency in node_dependent.dependencies:
1543 if not node_dependent_dependency.ref in flat_list:
1544 # The dependent one or more dependencies not in flat_list. There
1545 # will be more chances to add it to flat_list when examining
1546 # it again as a dependent of those other dependencies, provided
1547 # that there are no cycles.
1548 is_in_degree_zero = False
1551 if is_in_degree_zero:
1552 # All of the dependent's dependencies are already in flat_list. Add
1553 # it to in_degree_zeros where it will be processed in a future
1554 # iteration of the outer loop.
1555 in_degree_zeros.add(node_dependent)
1557 return list(flat_list)
1559 def FindCycles(self):
1561 Returns a list of cycles in the graph, where each cycle is its own list.
1566 def Visit(node, path):
1567 for child in node.dependents:
1569 results.append([child] + path[:path.index(child) + 1])
1570 elif not child in visited:
1572 Visit(child, [child] + path)
1579 def DirectDependencies(self, dependencies=None):
1580 """Returns a list of just direct dependencies."""
1581 if dependencies == None:
1584 for dependency in self.dependencies:
1585 # Check for None, corresponding to the root node.
1586 if dependency.ref != None and dependency.ref not in dependencies:
1587 dependencies.append(dependency.ref)
1591 def _AddImportedDependencies(self, targets, dependencies=None):
1592 """Given a list of direct dependencies, adds indirect dependencies that
1593 other dependencies have declared to export their settings.
1595 This method does not operate on self. Rather, it operates on the list
1596 of dependencies in the |dependencies| argument. For each dependency in
1597 that list, if any declares that it exports the settings of one of its
1598 own dependencies, those dependencies whose settings are "passed through"
1599 are added to the list. As new items are added to the list, they too will
1600 be processed, so it is possible to import settings through multiple levels
1603 This method is not terribly useful on its own, it depends on being
1604 "primed" with a list of direct dependencies such as one provided by
1605 DirectDependencies. DirectAndImportedDependencies is intended to be the
1609 if dependencies == None:
1613 while index < len(dependencies):
1614 dependency = dependencies[index]
1615 dependency_dict = targets[dependency]
1616 # Add any dependencies whose settings should be imported to the list
1617 # if not already present. Newly-added items will be checked for
1618 # their own imports when the list iteration reaches them.
1619 # Rather than simply appending new items, insert them after the
1620 # dependency that exported them. This is done to more closely match
1621 # the depth-first method used by DeepDependencies.
1623 for imported_dependency in \
1624 dependency_dict.get('export_dependent_settings', []):
1625 if imported_dependency not in dependencies:
1626 dependencies.insert(index + add_index, imported_dependency)
1627 add_index = add_index + 1
1632 def DirectAndImportedDependencies(self, targets, dependencies=None):
1633 """Returns a list of a target's direct dependencies and all indirect
1634 dependencies that a dependency has advertised settings should be exported
1635 through the dependency for.
1638 dependencies = self.DirectDependencies(dependencies)
1639 return self._AddImportedDependencies(targets, dependencies)
1641 def DeepDependencies(self, dependencies=None):
1642 """Returns an OrderedSet of all of a target's dependencies, recursively."""
1643 if dependencies is None:
1644 # Using a list to get ordered output and a set to do fast "is it
1645 # already added" checks.
1646 dependencies = OrderedSet()
1648 for dependency in self.dependencies:
1649 # Check for None, corresponding to the root node.
1650 if dependency.ref is None:
1652 if dependency.ref not in dependencies:
1653 dependencies.add(dependency.ref)
1654 dependency.DeepDependencies(dependencies)
1658 def _LinkDependenciesInternal(self, targets, include_shared_libraries,
1659 dependencies=None, initial=True):
1660 """Returns an OrderedSet of dependency targets that are linked
1663 This function has a split personality, depending on the setting of
1664 |initial|. Outside callers should always leave |initial| at its default
1667 When adding a target to the list of dependencies, this function will
1668 recurse into itself with |initial| set to False, to collect dependencies
1669 that are linked into the linkable target for which the list is being built.
1671 If |include_shared_libraries| is False, the resulting dependencies will not
1672 include shared_library targets that are linked into this target.
1674 if dependencies is None:
1675 # Using a list to get ordered output and a set to do fast "is it
1676 # already added" checks.
1677 dependencies = OrderedSet()
1679 # Check for None, corresponding to the root node.
1680 if self.ref is None:
1683 # It's kind of sucky that |targets| has to be passed into this function,
1684 # but that's presently the easiest way to access the target dicts so that
1685 # this function can find target types.
1687 if 'target_name' not in targets[self.ref]:
1688 raise GypError("Missing 'target_name' field in target.")
1690 if 'type' not in targets[self.ref]:
1691 raise GypError("Missing 'type' field in target %s" %
1692 targets[self.ref]['target_name'])
1694 target_type = targets[self.ref]['type']
1696 is_linkable = target_type in linkable_types
1698 if initial and not is_linkable:
1699 # If this is the first target being examined and it's not linkable,
1700 # return an empty list of link dependencies, because the link
1701 # dependencies are intended to apply to the target itself (initial is
1702 # True) and this target won't be linked.
1705 # Don't traverse 'none' targets if explicitly excluded.
1706 if (target_type == 'none' and
1707 not targets[self.ref].get('dependencies_traverse', True)):
1708 dependencies.add(self.ref)
1711 # Executables and loadable modules are already fully and finally linked.
1712 # Nothing else can be a link dependency of them, there can only be
1713 # dependencies in the sense that a dependent target might run an
1714 # executable or load the loadable_module.
1715 if not initial and target_type in ('executable', 'loadable_module'):
1718 # Shared libraries are already fully linked. They should only be included
1719 # in |dependencies| when adjusting static library dependencies (in order to
1720 # link against the shared_library's import lib), but should not be included
1721 # in |dependencies| when propagating link_settings.
1722 # The |include_shared_libraries| flag controls which of these two cases we
1724 if (not initial and target_type == 'shared_library' and
1725 not include_shared_libraries):
1728 # The target is linkable, add it to the list of link dependencies.
1729 if self.ref not in dependencies:
1730 dependencies.add(self.ref)
1731 if initial or not is_linkable:
1732 # If this is a subsequent target and it's linkable, don't look any
1733 # further for linkable dependencies, as they'll already be linked into
1734 # this target linkable. Always look at dependencies of the initial
1735 # target, and always look at dependencies of non-linkables.
1736 for dependency in self.dependencies:
1737 dependency._LinkDependenciesInternal(targets,
1738 include_shared_libraries,
1739 dependencies, False)
1743 def DependenciesForLinkSettings(self, targets):
1745 Returns a list of dependency targets whose link_settings should be merged
1749 # TODO(sbaig) Currently, chrome depends on the bug that shared libraries'
1750 # link_settings are propagated. So for now, we will allow it, unless the
1751 # 'allow_sharedlib_linksettings_propagation' flag is explicitly set to
1752 # False. Once chrome is fixed, we can remove this flag.
1753 include_shared_libraries = \
1754 targets[self.ref].get('allow_sharedlib_linksettings_propagation', True)
1755 return self._LinkDependenciesInternal(targets, include_shared_libraries)
1757 def DependenciesToLinkAgainst(self, targets):
1759 Returns a list of dependency targets that are linked into this target.
1761 return self._LinkDependenciesInternal(targets, True)
1764 def BuildDependencyList(targets):
1765 # Create a DependencyGraphNode for each target. Put it into a dict for easy
1767 dependency_nodes = {}
1768 for target, spec in targets.iteritems():
1769 if target not in dependency_nodes:
1770 dependency_nodes[target] = DependencyGraphNode(target)
1772 # Set up the dependency links. Targets that have no dependencies are treated
1773 # as dependent on root_node.
1774 root_node = DependencyGraphNode(None)
1775 for target, spec in targets.iteritems():
1776 target_node = dependency_nodes[target]
1777 target_build_file = gyp.common.BuildFile(target)
1778 dependencies = spec.get('dependencies')
1779 if not dependencies:
1780 target_node.dependencies = [root_node]
1781 root_node.dependents.append(target_node)
1783 for dependency in dependencies:
1784 dependency_node = dependency_nodes.get(dependency)
1785 if not dependency_node:
1786 raise GypError("Dependency '%s' not found while "
1787 "trying to load target %s" % (dependency, target))
1788 target_node.dependencies.append(dependency_node)
1789 dependency_node.dependents.append(target_node)
1791 flat_list = root_node.FlattenToList()
1793 # If there's anything left unvisited, there must be a circular dependency
1795 if len(flat_list) != len(targets):
1796 if not root_node.dependents:
1797 # If all targets have dependencies, add the first target as a dependent
1798 # of root_node so that the cycle can be discovered from root_node.
1799 target = targets.keys()[0]
1800 target_node = dependency_nodes[target]
1801 target_node.dependencies.append(root_node)
1802 root_node.dependents.append(target_node)
1805 for cycle in root_node.FindCycles():
1806 paths = [node.ref for node in cycle]
1807 cycles.append('Cycle: %s' % ' -> '.join(paths))
1808 raise DependencyGraphNode.CircularException(
1809 'Cycles in dependency graph detected:\n' + '\n'.join(cycles))
1811 return [dependency_nodes, flat_list]
1814 def VerifyNoGYPFileCircularDependencies(targets):
1815 # Create a DependencyGraphNode for each gyp file containing a target. Put
1816 # it into a dict for easy access.
1817 dependency_nodes = {}
1818 for target in targets.iterkeys():
1819 build_file = gyp.common.BuildFile(target)
1820 if not build_file in dependency_nodes:
1821 dependency_nodes[build_file] = DependencyGraphNode(build_file)
1823 # Set up the dependency links.
1824 for target, spec in targets.iteritems():
1825 build_file = gyp.common.BuildFile(target)
1826 build_file_node = dependency_nodes[build_file]
1827 target_dependencies = spec.get('dependencies', [])
1828 for dependency in target_dependencies:
1830 dependency_build_file = gyp.common.BuildFile(dependency)
1832 gyp.common.ExceptionAppend(
1833 e, 'while computing dependencies of .gyp file %s' % build_file)
1836 if dependency_build_file == build_file:
1837 # A .gyp file is allowed to refer back to itself.
1839 dependency_node = dependency_nodes.get(dependency_build_file)
1840 if not dependency_node:
1841 raise GypError("Dependancy '%s' not found" % dependency_build_file)
1842 if dependency_node not in build_file_node.dependencies:
1843 build_file_node.dependencies.append(dependency_node)
1844 dependency_node.dependents.append(build_file_node)
1847 # Files that have no dependencies are treated as dependent on root_node.
1848 root_node = DependencyGraphNode(None)
1849 for build_file_node in dependency_nodes.itervalues():
1850 if len(build_file_node.dependencies) == 0:
1851 build_file_node.dependencies.append(root_node)
1852 root_node.dependents.append(build_file_node)
1854 flat_list = root_node.FlattenToList()
1856 # If there's anything left unvisited, there must be a circular dependency
1858 if len(flat_list) != len(dependency_nodes):
1859 if not root_node.dependents:
1860 # If all files have dependencies, add the first file as a dependent
1861 # of root_node so that the cycle can be discovered from root_node.
1862 file_node = dependency_nodes.values()[0]
1863 file_node.dependencies.append(root_node)
1864 root_node.dependents.append(file_node)
1866 for cycle in root_node.FindCycles():
1867 paths = [node.ref for node in cycle]
1868 cycles.append('Cycle: %s' % ' -> '.join(paths))
1869 raise DependencyGraphNode.CircularException(
1870 'Cycles in .gyp file dependency graph detected:\n' + '\n'.join(cycles))
1873 def DoDependentSettings(key, flat_list, targets, dependency_nodes):
1874 # key should be one of all_dependent_settings, direct_dependent_settings,
1877 for target in flat_list:
1878 target_dict = targets[target]
1879 build_file = gyp.common.BuildFile(target)
1881 if key == 'all_dependent_settings':
1882 dependencies = dependency_nodes[target].DeepDependencies()
1883 elif key == 'direct_dependent_settings':
1885 dependency_nodes[target].DirectAndImportedDependencies(targets)
1886 elif key == 'link_settings':
1888 dependency_nodes[target].DependenciesForLinkSettings(targets)
1890 raise GypError("DoDependentSettings doesn't know how to determine "
1891 'dependencies for ' + key)
1893 for dependency in dependencies:
1894 dependency_dict = targets[dependency]
1895 if not key in dependency_dict:
1897 dependency_build_file = gyp.common.BuildFile(dependency)
1898 MergeDicts(target_dict, dependency_dict[key],
1899 build_file, dependency_build_file)
1902 def AdjustStaticLibraryDependencies(flat_list, targets, dependency_nodes,
1904 # Recompute target "dependencies" properties. For each static library
1905 # target, remove "dependencies" entries referring to other static libraries,
1906 # unless the dependency has the "hard_dependency" attribute set. For each
1907 # linkable target, add a "dependencies" entry referring to all of the
1908 # target's computed list of link dependencies (including static libraries
1909 # if no such entry is already present.
1910 for target in flat_list:
1911 target_dict = targets[target]
1912 target_type = target_dict['type']
1914 if target_type == 'static_library':
1915 if not 'dependencies' in target_dict:
1918 target_dict['dependencies_original'] = target_dict.get(
1919 'dependencies', [])[:]
1921 # A static library should not depend on another static library unless
1922 # the dependency relationship is "hard," which should only be done when
1923 # a dependent relies on some side effect other than just the build
1924 # product, like a rule or action output. Further, if a target has a
1925 # non-hard dependency, but that dependency exports a hard dependency,
1926 # the non-hard dependency can safely be removed, but the exported hard
1927 # dependency must be added to the target to keep the same dependency
1930 dependency_nodes[target].DirectAndImportedDependencies(targets)
1932 while index < len(dependencies):
1933 dependency = dependencies[index]
1934 dependency_dict = targets[dependency]
1936 # Remove every non-hard static library dependency and remove every
1937 # non-static library dependency that isn't a direct dependency.
1938 if (dependency_dict['type'] == 'static_library' and \
1939 not dependency_dict.get('hard_dependency', False)) or \
1940 (dependency_dict['type'] != 'static_library' and \
1941 not dependency in target_dict['dependencies']):
1942 # Take the dependency out of the list, and don't increment index
1943 # because the next dependency to analyze will shift into the index
1944 # formerly occupied by the one being removed.
1945 del dependencies[index]
1949 # Update the dependencies. If the dependencies list is empty, it's not
1950 # needed, so unhook it.
1951 if len(dependencies) > 0:
1952 target_dict['dependencies'] = dependencies
1954 del target_dict['dependencies']
1956 elif target_type in linkable_types:
1957 # Get a list of dependency targets that should be linked into this
1958 # target. Add them to the dependencies list if they're not already
1961 link_dependencies = \
1962 dependency_nodes[target].DependenciesToLinkAgainst(targets)
1963 for dependency in link_dependencies:
1964 if dependency == target:
1966 if not 'dependencies' in target_dict:
1967 target_dict['dependencies'] = []
1968 if not dependency in target_dict['dependencies']:
1969 target_dict['dependencies'].append(dependency)
1970 # Sort the dependencies list in the order from dependents to dependencies.
1971 # e.g. If A and B depend on C and C depends on D, sort them in A, B, C, D.
1972 # Note: flat_list is already sorted in the order from dependencies to
1974 if sort_dependencies and 'dependencies' in target_dict:
1975 target_dict['dependencies'] = [dep for dep in reversed(flat_list)
1976 if dep in target_dict['dependencies']]
1979 # Initialize this here to speed up MakePathRelative.
1980 exception_re = re.compile(r'''["']?[-/$<>^]''')
1983 def MakePathRelative(to_file, fro_file, item):
1984 # If item is a relative path, it's relative to the build file dict that it's
1985 # coming from. Fix it up to make it relative to the build file dict that
1987 # Exception: any |item| that begins with these special characters is
1988 # returned without modification.
1989 # / Used when a path is already absolute (shortcut optimization;
1990 # such paths would be returned as absolute anyway)
1991 # $ Used for build environment variables
1992 # - Used for some build environment flags (such as -lapr-1 in a
1993 # "libraries" section)
1994 # < Used for our own variable and command expansions (see ExpandVariables)
1995 # > Used for our own variable and command expansions (see ExpandVariables)
1996 # ^ Used for our own variable and command expansions (see ExpandVariables)
1998 # "/' Used when a value is quoted. If these are present, then we
1999 # check the second character instead.
2001 if to_file == fro_file or exception_re.match(item):
2004 # TODO(dglazkov) The backslash/forward-slash replacement at the end is a
2005 # temporary measure. This should really be addressed by keeping all paths
2006 # in POSIX until actual project generation.
2007 ret = os.path.normpath(os.path.join(
2008 gyp.common.RelativePath(os.path.dirname(fro_file),
2009 os.path.dirname(to_file)),
2010 item)).replace('\\', '/')
2015 def MergeLists(to, fro, to_file, fro_file, is_paths=False, append=True):
2016 # Python documentation recommends objects which do not support hash
2017 # set this value to None. Python library objects follow this rule.
2018 is_hashable = lambda val: val.__hash__
2020 # If x is hashable, returns whether x is in s. Else returns whether x is in l.
2021 def is_in_set_or_list(x, s, l):
2028 # Make membership testing of hashables in |to| (in particular, strings)
2030 hashable_to_set = set(x for x in to if is_hashable(x))
2033 if type(item) in (str, int):
2034 # The cheap and easy case.
2036 to_item = MakePathRelative(to_file, fro_file, item)
2040 if not (type(item) is str and item.startswith('-')):
2041 # Any string that doesn't begin with a "-" is a singleton - it can
2042 # only appear once in a list, to be enforced by the list merge append
2045 elif type(item) is dict:
2046 # Make a copy of the dictionary, continuing to look for paths to fix.
2047 # The other intelligent aspects of merge processing won't apply because
2048 # item is being merged into an empty dict.
2050 MergeDicts(to_item, item, to_file, fro_file)
2051 elif type(item) is list:
2052 # Recurse, making a copy of the list. If the list contains any
2053 # descendant dicts, path fixing will occur. Note that here, custom
2054 # values for is_paths and append are dropped; those are only to be
2055 # applied to |to| and |fro|, not sublists of |fro|. append shouldn't
2056 # matter anyway because the new |to_item| list is empty.
2058 MergeLists(to_item, item, to_file, fro_file)
2061 'Attempt to merge list item of unsupported type ' + \
2062 item.__class__.__name__
2065 # If appending a singleton that's already in the list, don't append.
2066 # This ensures that the earliest occurrence of the item will stay put.
2067 if not singleton or not is_in_set_or_list(to_item, hashable_to_set, to):
2069 if is_hashable(to_item):
2070 hashable_to_set.add(to_item)
2072 # If prepending a singleton that's already in the list, remove the
2073 # existing instance and proceed with the prepend. This ensures that the
2074 # item appears at the earliest possible position in the list.
2075 while singleton and to_item in to:
2078 # Don't just insert everything at index 0. That would prepend the new
2079 # items to the list in reverse order, which would be an unwelcome
2081 to.insert(prepend_index, to_item)
2082 if is_hashable(to_item):
2083 hashable_to_set.add(to_item)
2084 prepend_index = prepend_index + 1
2087 def MergeDicts(to, fro, to_file, fro_file):
2088 # I wanted to name the parameter "from" but it's a Python keyword...
2089 for k, v in fro.iteritems():
2090 # It would be nice to do "if not k in to: to[k] = v" but that wouldn't give
2091 # copy semantics. Something else may want to merge from the |fro| dict
2092 # later, and having the same dict ref pointed to twice in the tree isn't
2093 # what anyone wants considering that the dicts may subsequently be
2097 if type(v) in (str, int):
2098 if type(to[k]) not in (str, int):
2100 elif type(v) is not type(to[k]):
2105 'Attempt to merge dict value of type ' + v.__class__.__name__ + \
2106 ' into incompatible type ' + to[k].__class__.__name__ + \
2108 if type(v) in (str, int):
2109 # Overwrite the existing value, if any. Cheap and easy.
2110 is_path = IsPathSection(k)
2112 to[k] = MakePathRelative(to_file, fro_file, v)
2115 elif type(v) is dict:
2116 # Recurse, guaranteeing copies will be made of objects that require it.
2119 MergeDicts(to[k], v, to_file, fro_file)
2120 elif type(v) is list:
2121 # Lists in dicts can be merged with different policies, depending on
2122 # how the key in the "from" dict (k, the from-key) is written.
2124 # If the from-key has ...the to-list will have this action
2125 # this character appended:... applied when receiving the from-list:
2128 # ? set, only if to-list does not yet exist
2131 # This logic is list-specific, but since it relies on the associated
2132 # dict key, it's checked in this dict-oriented function.
2137 lists_incompatible = [list_base, list_base + '?']
2141 lists_incompatible = [list_base + '=', list_base + '?']
2145 lists_incompatible = [list_base, list_base + '=', list_base + '+']
2148 lists_incompatible = [list_base + '=', list_base + '?']
2150 # Some combinations of merge policies appearing together are meaningless.
2151 # It's stupid to replace and append simultaneously, for example. Append
2152 # and prepend are the only policies that can coexist.
2153 for list_incompatible in lists_incompatible:
2154 if list_incompatible in fro:
2155 raise GypError('Incompatible list policies ' + k + ' and ' +
2160 # If the key ends in "?", the list will only be merged if it doesn't
2163 elif type(to[list_base]) is not list:
2164 # This may not have been checked above if merging in a list with an
2165 # extension character.
2167 'Attempt to merge dict value of type ' + v.__class__.__name__ + \
2168 ' into incompatible type ' + to[list_base].__class__.__name__ + \
2169 ' for key ' + list_base + '(' + k + ')'
2173 # Call MergeLists, which will make copies of objects that require it.
2174 # MergeLists can recurse back into MergeDicts, although this will be
2175 # to make copies of dicts (with paths fixed), there will be no
2176 # subsequent dict "merging" once entering a list because lists are
2177 # always replaced, appended to, or prepended to.
2178 is_paths = IsPathSection(list_base)
2179 MergeLists(to[list_base], v, to_file, fro_file, is_paths, append)
2182 'Attempt to merge dict value of unsupported type ' + \
2183 v.__class__.__name__ + ' for key ' + k
2186 def MergeConfigWithInheritance(new_configuration_dict, build_file,
2187 target_dict, configuration, visited):
2188 # Skip if previously visted.
2189 if configuration in visited:
2192 # Look at this configuration.
2193 configuration_dict = target_dict['configurations'][configuration]
2196 for parent in configuration_dict.get('inherit_from', []):
2197 MergeConfigWithInheritance(new_configuration_dict, build_file,
2198 target_dict, parent, visited + [configuration])
2200 # Merge it into the new config.
2201 MergeDicts(new_configuration_dict, configuration_dict,
2202 build_file, build_file)
2205 if 'abstract' in new_configuration_dict:
2206 del new_configuration_dict['abstract']
2209 def SetUpConfigurations(target, target_dict):
2210 # key_suffixes is a list of key suffixes that might appear on key names.
2211 # These suffixes are handled in conditional evaluations (for =, +, and ?)
2212 # and rules/exclude processing (for ! and /). Keys with these suffixes
2213 # should be treated the same as keys without.
2214 key_suffixes = ['=', '+', '?', '!', '/']
2216 build_file = gyp.common.BuildFile(target)
2218 # Provide a single configuration by default if none exists.
2219 # TODO(mark): Signal an error if default_configurations exists but
2220 # configurations does not.
2221 if not 'configurations' in target_dict:
2222 target_dict['configurations'] = {'Default': {}}
2223 if not 'default_configuration' in target_dict:
2224 concrete = [i for (i, config) in target_dict['configurations'].iteritems()
2225 if not config.get('abstract')]
2226 target_dict['default_configuration'] = sorted(concrete)[0]
2228 merged_configurations = {}
2229 configs = target_dict['configurations']
2230 for (configuration, old_configuration_dict) in configs.iteritems():
2231 # Skip abstract configurations (saves work only).
2232 if old_configuration_dict.get('abstract'):
2234 # Configurations inherit (most) settings from the enclosing target scope.
2235 # Get the inheritance relationship right by making a copy of the target
2237 new_configuration_dict = {}
2238 for (key, target_val) in target_dict.iteritems():
2240 if key_ext in key_suffixes:
2244 if not key_base in non_configuration_keys:
2245 new_configuration_dict[key] = gyp.simple_copy.deepcopy(target_val)
2247 # Merge in configuration (with all its parents first).
2248 MergeConfigWithInheritance(new_configuration_dict, build_file,
2249 target_dict, configuration, [])
2251 merged_configurations[configuration] = new_configuration_dict
2253 # Put the new configurations back into the target dict as a configuration.
2254 for configuration in merged_configurations.keys():
2255 target_dict['configurations'][configuration] = (
2256 merged_configurations[configuration])
2258 # Now drop all the abstract ones.
2259 for configuration in target_dict['configurations'].keys():
2260 old_configuration_dict = target_dict['configurations'][configuration]
2261 if old_configuration_dict.get('abstract'):
2262 del target_dict['configurations'][configuration]
2264 # Now that all of the target's configurations have been built, go through
2265 # the target dict's keys and remove everything that's been moved into a
2266 # "configurations" section.
2268 for key in target_dict:
2270 if key_ext in key_suffixes:
2274 if not key_base in non_configuration_keys:
2275 delete_keys.append(key)
2276 for key in delete_keys:
2277 del target_dict[key]
2279 # Check the configurations to see if they contain invalid keys.
2280 for configuration in target_dict['configurations'].keys():
2281 configuration_dict = target_dict['configurations'][configuration]
2282 for key in configuration_dict.keys():
2283 if key in invalid_configuration_keys:
2284 raise GypError('%s not allowed in the %s configuration, found in '
2285 'target %s' % (key, configuration, target))
2289 def ProcessListFiltersInDict(name, the_dict):
2290 """Process regular expression and exclusion-based filters on lists.
2292 An exclusion list is in a dict key named with a trailing "!", like
2293 "sources!". Every item in such a list is removed from the associated
2294 main list, which in this example, would be "sources". Removed items are
2295 placed into a "sources_excluded" list in the dict.
2297 Regular expression (regex) filters are contained in dict keys named with a
2298 trailing "/", such as "sources/" to operate on the "sources" list. Regex
2299 filters in a dict take the form:
2300 'sources/': [ ['exclude', '_(linux|mac|win)\\.cc$'],
2301 ['include', '_mac\\.cc$'] ],
2302 The first filter says to exclude all files ending in _linux.cc, _mac.cc, and
2303 _win.cc. The second filter then includes all files ending in _mac.cc that
2304 are now or were once in the "sources" list. Items matching an "exclude"
2305 filter are subject to the same processing as would occur if they were listed
2306 by name in an exclusion list (ending in "!"). Items matching an "include"
2307 filter are brought back into the main list if previously excluded by an
2308 exclusion list or exclusion regex filter. Subsequent matching "exclude"
2309 patterns can still cause items to be excluded after matching an "include".
2312 # Look through the dictionary for any lists whose keys end in "!" or "/".
2313 # These are lists that will be treated as exclude lists and regular
2314 # expression-based exclude/include lists. Collect the lists that are
2315 # needed first, looking for the lists that they operate on, and assemble
2316 # then into |lists|. This is done in a separate loop up front, because
2317 # the _included and _excluded keys need to be added to the_dict, and that
2318 # can't be done while iterating through it.
2322 for key, value in the_dict.iteritems():
2324 if operation != '!' and operation != '/':
2327 if type(value) is not list:
2328 raise ValueError, name + ' key ' + key + ' must be list, not ' + \
2329 value.__class__.__name__
2332 if list_key not in the_dict:
2333 # This happens when there's a list like "sources!" but no corresponding
2334 # "sources" list. Since there's nothing for it to operate on, queue up
2335 # the "sources!" list for deletion now.
2336 del_lists.append(key)
2339 if type(the_dict[list_key]) is not list:
2340 value = the_dict[list_key]
2341 raise ValueError, name + ' key ' + list_key + \
2342 ' must be list, not ' + \
2343 value.__class__.__name__ + ' when applying ' + \
2344 {'!': 'exclusion', '/': 'regex'}[operation]
2346 if not list_key in lists:
2347 lists.append(list_key)
2349 # Delete the lists that are known to be unneeded at this point.
2350 for del_list in del_lists:
2351 del the_dict[del_list]
2353 for list_key in lists:
2354 the_list = the_dict[list_key]
2356 # Initialize the list_actions list, which is parallel to the_list. Each
2357 # item in list_actions identifies whether the corresponding item in
2358 # the_list should be excluded, unconditionally preserved (included), or
2359 # whether no exclusion or inclusion has been applied. Items for which
2360 # no exclusion or inclusion has been applied (yet) have value -1, items
2361 # excluded have value 0, and items included have value 1. Includes and
2362 # excludes override previous actions. All items in list_actions are
2363 # initialized to -1 because no excludes or includes have been processed
2365 list_actions = list((-1,) * len(the_list))
2367 exclude_key = list_key + '!'
2368 if exclude_key in the_dict:
2369 for exclude_item in the_dict[exclude_key]:
2370 for index in xrange(0, len(the_list)):
2371 if exclude_item == the_list[index]:
2372 # This item matches the exclude_item, so set its action to 0
2374 list_actions[index] = 0
2376 # The "whatever!" list is no longer needed, dump it.
2377 del the_dict[exclude_key]
2379 regex_key = list_key + '/'
2380 if regex_key in the_dict:
2381 for regex_item in the_dict[regex_key]:
2382 [action, pattern] = regex_item
2383 pattern_re = re.compile(pattern)
2385 if action == 'exclude':
2386 # This item matches an exclude regex, so set its value to 0 (exclude).
2388 elif action == 'include':
2389 # This item matches an include regex, so set its value to 1 (include).
2392 # This is an action that doesn't make any sense.
2393 raise ValueError, 'Unrecognized action ' + action + ' in ' + name + \
2396 for index in xrange(0, len(the_list)):
2397 list_item = the_list[index]
2398 if list_actions[index] == action_value:
2399 # Even if the regex matches, nothing will change so continue (regex
2400 # searches are expensive).
2402 if pattern_re.search(list_item):
2403 # Regular expression match.
2404 list_actions[index] = action_value
2406 # The "whatever/" list is no longer needed, dump it.
2407 del the_dict[regex_key]
2409 # Add excluded items to the excluded list.
2411 # Note that exclude_key ("sources!") is different from excluded_key
2412 # ("sources_excluded"). The exclude_key list is input and it was already
2413 # processed and deleted; the excluded_key list is output and it's about
2415 excluded_key = list_key + '_excluded'
2416 if excluded_key in the_dict:
2417 raise GypError(name + ' key ' + excluded_key +
2418 ' must not be present prior '
2419 ' to applying exclusion/regex filters for ' + list_key)
2423 # Go backwards through the list_actions list so that as items are deleted,
2424 # the indices of items that haven't been seen yet don't shift. That means
2425 # that things need to be prepended to excluded_list to maintain them in the
2426 # same order that they existed in the_list.
2427 for index in xrange(len(list_actions) - 1, -1, -1):
2428 if list_actions[index] == 0:
2429 # Dump anything with action 0 (exclude). Keep anything with action 1
2430 # (include) or -1 (no include or exclude seen for the item).
2431 excluded_list.insert(0, the_list[index])
2434 # If anything was excluded, put the excluded list into the_dict at
2436 if len(excluded_list) > 0:
2437 the_dict[excluded_key] = excluded_list
2439 # Now recurse into subdicts and lists that may contain dicts.
2440 for key, value in the_dict.iteritems():
2441 if type(value) is dict:
2442 ProcessListFiltersInDict(key, value)
2443 elif type(value) is list:
2444 ProcessListFiltersInList(key, value)
2447 def ProcessListFiltersInList(name, the_list):
2448 for item in the_list:
2449 if type(item) is dict:
2450 ProcessListFiltersInDict(name, item)
2451 elif type(item) is list:
2452 ProcessListFiltersInList(name, item)
2455 def ValidateTargetType(target, target_dict):
2456 """Ensures the 'type' field on the target is one of the known types.
2459 target: string, name of target.
2460 target_dict: dict, target spec.
2462 Raises an exception on error.
2464 VALID_TARGET_TYPES = ('executable', 'loadable_module',
2465 'static_library', 'shared_library',
2467 target_type = target_dict.get('type', None)
2468 if target_type not in VALID_TARGET_TYPES:
2469 raise GypError("Target %s has an invalid target type '%s'. "
2470 "Must be one of %s." %
2471 (target, target_type, '/'.join(VALID_TARGET_TYPES)))
2472 if (target_dict.get('standalone_static_library', 0) and
2473 not target_type == 'static_library'):
2474 raise GypError('Target %s has type %s but standalone_static_library flag is'
2475 ' only valid for static_library type.' % (target,
2479 def ValidateRulesInTarget(target, target_dict, extra_sources_for_rules):
2480 """Ensures that the rules sections in target_dict are valid and consistent,
2481 and determines which sources they apply to.
2484 target: string, name of target.
2485 target_dict: dict, target spec containing "rules" and "sources" lists.
2486 extra_sources_for_rules: a list of keys to scan for rule matches in
2487 addition to 'sources'.
2490 # Dicts to map between values found in rules' 'rule_name' and 'extension'
2491 # keys and the rule dicts themselves.
2493 rule_extensions = {}
2495 rules = target_dict.get('rules', [])
2497 # Make sure that there's no conflict among rule names and extensions.
2498 rule_name = rule['rule_name']
2499 if rule_name in rule_names:
2500 raise GypError('rule %s exists in duplicate, target %s' %
2501 (rule_name, target))
2502 rule_names[rule_name] = rule
2504 rule_extension = rule['extension']
2505 if rule_extension.startswith('.'):
2506 rule_extension = rule_extension[1:]
2507 if rule_extension in rule_extensions:
2508 raise GypError(('extension %s associated with multiple rules, ' +
2509 'target %s rules %s and %s') %
2510 (rule_extension, target,
2511 rule_extensions[rule_extension]['rule_name'],
2513 rule_extensions[rule_extension] = rule
2515 # Make sure rule_sources isn't already there. It's going to be
2516 # created below if needed.
2517 if 'rule_sources' in rule:
2519 'rule_sources must not exist in input, target %s rule %s' %
2520 (target, rule_name))
2523 source_keys = ['sources']
2524 source_keys.extend(extra_sources_for_rules)
2525 for source_key in source_keys:
2526 for source in target_dict.get(source_key, []):
2527 (source_root, source_extension) = os.path.splitext(source)
2528 if source_extension.startswith('.'):
2529 source_extension = source_extension[1:]
2530 if source_extension == rule_extension:
2531 rule_sources.append(source)
2533 if len(rule_sources) > 0:
2534 rule['rule_sources'] = rule_sources
2537 def ValidateRunAsInTarget(target, target_dict, build_file):
2538 target_name = target_dict.get('target_name')
2539 run_as = target_dict.get('run_as')
2542 if type(run_as) is not dict:
2543 raise GypError("The 'run_as' in target %s from file %s should be a "
2545 (target_name, build_file))
2546 action = run_as.get('action')
2548 raise GypError("The 'run_as' in target %s from file %s must have an "
2549 "'action' section." %
2550 (target_name, build_file))
2551 if type(action) is not list:
2552 raise GypError("The 'action' for 'run_as' in target %s from file %s "
2554 (target_name, build_file))
2555 working_directory = run_as.get('working_directory')
2556 if working_directory and type(working_directory) is not str:
2557 raise GypError("The 'working_directory' for 'run_as' in target %s "
2558 "in file %s should be a string." %
2559 (target_name, build_file))
2560 environment = run_as.get('environment')
2561 if environment and type(environment) is not dict:
2562 raise GypError("The 'environment' for 'run_as' in target %s "
2563 "in file %s should be a dictionary." %
2564 (target_name, build_file))
2567 def ValidateActionsInTarget(target, target_dict, build_file):
2568 '''Validates the inputs to the actions in a target.'''
2569 target_name = target_dict.get('target_name')
2570 actions = target_dict.get('actions', [])
2571 for action in actions:
2572 action_name = action.get('action_name')
2574 raise GypError("Anonymous action in target %s. "
2575 "An action must have an 'action_name' field." %
2577 inputs = action.get('inputs', None)
2579 raise GypError('Action in target %s has no inputs.' % target_name)
2580 action_command = action.get('action')
2581 if action_command and not action_command[0]:
2582 raise GypError("Empty action as command in target %s." % target_name)
2585 def TurnIntIntoStrInDict(the_dict):
2586 """Given dict the_dict, recursively converts all integers into strings.
2588 # Use items instead of iteritems because there's no need to try to look at
2589 # reinserted keys and their associated values.
2590 for k, v in the_dict.items():
2594 elif type(v) is dict:
2595 TurnIntIntoStrInDict(v)
2596 elif type(v) is list:
2597 TurnIntIntoStrInList(v)
2601 the_dict[str(k)] = v
2604 def TurnIntIntoStrInList(the_list):
2605 """Given list the_list, recursively converts all integers into strings.
2607 for index in xrange(0, len(the_list)):
2608 item = the_list[index]
2609 if type(item) is int:
2610 the_list[index] = str(item)
2611 elif type(item) is dict:
2612 TurnIntIntoStrInDict(item)
2613 elif type(item) is list:
2614 TurnIntIntoStrInList(item)
2617 def PruneUnwantedTargets(targets, flat_list, dependency_nodes, root_targets,
2619 """Return only the targets that are deep dependencies of |root_targets|."""
2620 qualified_root_targets = []
2621 for target in root_targets:
2622 target = target.strip()
2623 qualified_targets = gyp.common.FindQualifiedTargets(target, flat_list)
2624 if not qualified_targets:
2625 raise GypError("Could not find target %s" % target)
2626 qualified_root_targets.extend(qualified_targets)
2629 for target in qualified_root_targets:
2630 wanted_targets[target] = targets[target]
2631 for dependency in dependency_nodes[target].DeepDependencies():
2632 wanted_targets[dependency] = targets[dependency]
2634 wanted_flat_list = [t for t in flat_list if t in wanted_targets]
2636 # Prune unwanted targets from each build_file's data dict.
2637 for build_file in data['target_build_files']:
2638 if not 'targets' in data[build_file]:
2641 for target in data[build_file]['targets']:
2642 qualified_name = gyp.common.QualifiedTarget(build_file,
2643 target['target_name'],
2645 if qualified_name in wanted_targets:
2646 new_targets.append(target)
2647 data[build_file]['targets'] = new_targets
2649 return wanted_targets, wanted_flat_list
2652 def VerifyNoCollidingTargets(targets):
2653 """Verify that no two targets in the same directory share the same name.
2656 targets: A list of targets in the form 'path/to/file.gyp:target_name'.
2658 # Keep a dict going from 'subdirectory:target_name' to 'foo.gyp'.
2660 for target in targets:
2661 # Separate out 'path/to/file.gyp, 'target_name' from
2662 # 'path/to/file.gyp:target_name'.
2663 path, name = target.rsplit(':', 1)
2664 # Separate out 'path/to', 'file.gyp' from 'path/to/file.gyp'.
2665 subdir, gyp = os.path.split(path)
2666 # Use '.' for the current directory '', so that the error messages make
2670 # Prepare a key like 'path/to:target_name'.
2671 key = subdir + ':' + name
2673 # Complain if this target is already used.
2674 raise GypError('Duplicate target name "%s" in directory "%s" used both '
2675 'in "%s" and "%s".' % (name, subdir, gyp, used[key]))
2679 def SetGeneratorGlobals(generator_input_info):
2680 # Set up path_sections and non_configuration_keys with the default data plus
2681 # the generator-specific data.
2682 global path_sections
2683 path_sections = set(base_path_sections)
2684 path_sections.update(generator_input_info['path_sections'])
2686 global non_configuration_keys
2687 non_configuration_keys = base_non_configuration_keys[:]
2688 non_configuration_keys.extend(generator_input_info['non_configuration_keys'])
2690 global multiple_toolsets
2691 multiple_toolsets = generator_input_info[
2692 'generator_supports_multiple_toolsets']
2694 global generator_filelist_paths
2695 generator_filelist_paths = generator_input_info['generator_filelist_paths']
2698 def Load(build_files, variables, includes, depth, generator_input_info, check,
2699 circular_check, parallel, root_targets):
2700 SetGeneratorGlobals(generator_input_info)
2701 # A generator can have other lists (in addition to sources) be processed
2703 extra_sources_for_rules = generator_input_info['extra_sources_for_rules']
2705 # Load build files. This loads every target-containing build file into
2706 # the |data| dictionary such that the keys to |data| are build file names,
2707 # and the values are the entire build file contents after "early" or "pre"
2708 # processing has been done and includes have been resolved.
2709 # NOTE: data contains both "target" files (.gyp) and "includes" (.gypi), as
2710 # well as meta-data (e.g. 'included_files' key). 'target_build_files' keeps
2711 # track of the keys corresponding to "target" files.
2712 data = {'target_build_files': set()}
2714 # Normalize paths everywhere. This is important because paths will be
2715 # used as keys to the data dict and for references between input files.
2716 build_files = set(map(os.path.normpath, build_files))
2718 LoadTargetBuildFilesParallel(build_files, data, aux_data,
2719 variables, includes, depth, check,
2720 generator_input_info)
2722 for build_file in build_files:
2724 LoadTargetBuildFile(build_file, data, aux_data,
2725 variables, includes, depth, check, True)
2726 except Exception, e:
2727 gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file)
2730 # Build a dict to access each target's subdict by qualified name.
2731 targets = BuildTargetsDict(data)
2733 # Fully qualify all dependency links.
2734 QualifyDependencies(targets)
2736 # Remove self-dependencies from targets that have 'prune_self_dependencies'
2738 RemoveSelfDependencies(targets)
2740 # Expand dependencies specified as build_file:*.
2741 ExpandWildcardDependencies(targets, data)
2743 # Remove all dependencies marked as 'link_dependency' from the targets of
2745 RemoveLinkDependenciesFromNoneTargets(targets)
2747 # Apply exclude (!) and regex (/) list filters only for dependency_sections.
2748 for target_name, target_dict in targets.iteritems():
2750 for key_base in dependency_sections:
2751 for op in ('', '!', '/'):
2753 if key in target_dict:
2754 tmp_dict[key] = target_dict[key]
2755 del target_dict[key]
2756 ProcessListFiltersInDict(target_name, tmp_dict)
2757 # Write the results back to |target_dict|.
2758 for key in tmp_dict:
2759 target_dict[key] = tmp_dict[key]
2761 # Make sure every dependency appears at most once.
2762 RemoveDuplicateDependencies(targets)
2765 # Make sure that any targets in a.gyp don't contain dependencies in other
2766 # .gyp files that further depend on a.gyp.
2767 VerifyNoGYPFileCircularDependencies(targets)
2769 [dependency_nodes, flat_list] = BuildDependencyList(targets)
2772 # Remove, from |targets| and |flat_list|, the targets that are not deep
2773 # dependencies of the targets specified in |root_targets|.
2774 targets, flat_list = PruneUnwantedTargets(
2775 targets, flat_list, dependency_nodes, root_targets, data)
2777 # Check that no two targets in the same directory have the same name.
2778 VerifyNoCollidingTargets(flat_list)
2780 # Handle dependent settings of various types.
2781 for settings_type in ['all_dependent_settings',
2782 'direct_dependent_settings',
2784 DoDependentSettings(settings_type, flat_list, targets, dependency_nodes)
2786 # Take out the dependent settings now that they've been published to all
2787 # of the targets that require them.
2788 for target in flat_list:
2789 if settings_type in targets[target]:
2790 del targets[target][settings_type]
2792 # Make sure static libraries don't declare dependencies on other static
2793 # libraries, but that linkables depend on all unlinked static libraries
2794 # that they need so that their link steps will be correct.
2795 gii = generator_input_info
2796 if gii['generator_wants_static_library_dependencies_adjusted']:
2797 AdjustStaticLibraryDependencies(flat_list, targets, dependency_nodes,
2798 gii['generator_wants_sorted_dependencies'])
2800 # Apply "post"/"late"/"target" variable expansions and condition evaluations.
2801 for target in flat_list:
2802 target_dict = targets[target]
2803 build_file = gyp.common.BuildFile(target)
2804 ProcessVariablesAndConditionsInDict(
2805 target_dict, PHASE_LATE, variables, build_file)
2807 # Move everything that can go into a "configurations" section into one.
2808 for target in flat_list:
2809 target_dict = targets[target]
2810 SetUpConfigurations(target, target_dict)
2812 # Apply exclude (!) and regex (/) list filters.
2813 for target in flat_list:
2814 target_dict = targets[target]
2815 ProcessListFiltersInDict(target, target_dict)
2817 # Apply "latelate" variable expansions and condition evaluations.
2818 for target in flat_list:
2819 target_dict = targets[target]
2820 build_file = gyp.common.BuildFile(target)
2821 ProcessVariablesAndConditionsInDict(
2822 target_dict, PHASE_LATELATE, variables, build_file)
2824 # Make sure that the rules make sense, and build up rule_sources lists as
2825 # needed. Not all generators will need to use the rule_sources lists, but
2826 # some may, and it seems best to build the list in a common spot.
2827 # Also validate actions and run_as elements in targets.
2828 for target in flat_list:
2829 target_dict = targets[target]
2830 build_file = gyp.common.BuildFile(target)
2831 ValidateTargetType(target, target_dict)
2832 ValidateRulesInTarget(target, target_dict, extra_sources_for_rules)
2833 ValidateRunAsInTarget(target, target_dict, build_file)
2834 ValidateActionsInTarget(target, target_dict, build_file)
2836 # Generators might not expect ints. Turn them into strs.
2837 TurnIntIntoStrInDict(data)
2839 # TODO(mark): Return |data| for now because the generator needs a list of
2840 # build files that came in. In the future, maybe it should just accept
2841 # a list, and not the whole data dict.
2842 return [flat_list, targets, data]