1 # Copyright (c) 2012 Google Inc. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 from compiler.ast import Const
6 from compiler.ast import Dict
7 from compiler.ast import Discard
8 from compiler.ast import List
9 from compiler.ast import Module
10 from compiler.ast import Node
11 from compiler.ast import Stmt
14 import gyp.simple_copy
15 import multiprocessing
26 from gyp.common import GypError
27 from gyp.common import OrderedSet
30 # A list of types that are treated as linkable.
31 linkable_types = ['executable', 'shared_library', 'loadable_module']
33 # A list of sections that contain links to other targets.
34 dependency_sections = ['dependencies', 'export_dependent_settings']
36 # base_path_sections is a list of sections defined by GYP that contain
37 # pathnames. The generators can provide more keys, the two lists are merged
38 # into path_sections, but you should call IsPathSection instead of using either
40 base_path_sections = [
51 def IsPathSection(section):
52 # If section ends in one of the '=+?!' characters, it's applied to a section
53 # without the trailing characters. '/' is notably absent from this list,
54 # because there's no way for a regular expression to be treated as a path.
55 while section[-1:] in '=+?!':
56 section = section[:-1]
58 if section in path_sections:
61 # Sections mathing the regexp '_(dir|file|path)s?$' are also
62 # considered PathSections. Using manual string matching since that
63 # is much faster than the regexp and this can be called hundreds of
64 # thousands of times so micro performance matters.
69 if tail[-5:] in ('_file', '_path'):
71 return tail[-4:] == '_dir'
75 # base_non_configuration_keys is a list of key names that belong in the target
76 # itself and should not be propagated into its configurations. It is merged
77 # with a list that can come from the generator to
78 # create non_configuration_keys.
79 base_non_configuration_keys = [
80 # Sections that must exist inside targets and not configurations.
84 'default_configuration',
86 'dependencies_original',
96 'standalone_static_library',
103 # Sections that can be found inside targets or configurations, but that
104 # should not be propagated from targets into their configurations.
107 non_configuration_keys = []
109 # Keys that do not belong inside a configuration dictionary.
110 invalid_configuration_keys = [
112 'all_dependent_settings',
115 'direct_dependent_settings',
119 'standalone_static_library',
124 # Controls whether or not the generator supports multiple toolsets.
125 multiple_toolsets = False
127 # Paths for converting filelist paths to output paths: {
129 # qualified_output_dir,
131 generator_filelist_paths = None
133 def GetIncludedBuildFiles(build_file_path, aux_data, included=None):
134 """Return a list of all build files included into build_file_path.
136 The returned list will contain build_file_path as well as all other files
137 that it included, either directly or indirectly. Note that the list may
138 contain files that were included into a conditional section that evaluated
139 to false and was not merged into build_file_path's dict.
141 aux_data is a dict containing a key for each build file or included build
142 file. Those keys provide access to dicts whose "included" keys contain
143 lists of all other files included by the build file.
145 included should be left at its default None value by external callers. It
146 is used for recursion.
148 The returned list will not contain any duplicate entries. Each build file
149 in the list will be relative to the current directory.
155 if build_file_path in included:
158 included.append(build_file_path)
160 for included_build_file in aux_data[build_file_path].get('included', []):
161 GetIncludedBuildFiles(included_build_file, aux_data, included)
166 def CheckedEval(file_contents):
167 """Return the eval of a gyp file.
169 The gyp file is restricted to dictionaries and lists only, and
170 repeated keys are not allowed.
172 Note that this is slower than eval() is.
175 ast = compiler.parse(file_contents)
176 assert isinstance(ast, Module)
177 c1 = ast.getChildren()
179 assert isinstance(c1[1], Stmt)
180 c2 = c1[1].getChildren()
181 assert isinstance(c2[0], Discard)
182 c3 = c2[0].getChildren()
184 return CheckNode(c3[0], [])
187 def CheckNode(node, keypath):
188 if isinstance(node, Dict):
189 c = node.getChildren()
191 for n in range(0, len(c), 2):
192 assert isinstance(c[n], Const)
193 key = c[n].getChildren()[0]
195 raise GypError("Key '" + key + "' repeated at level " +
196 repr(len(keypath) + 1) + " with key path '" +
197 '.'.join(keypath) + "'")
198 kp = list(keypath) # Make a copy of the list for descending this node.
200 dict[key] = CheckNode(c[n + 1], kp)
202 elif isinstance(node, List):
203 c = node.getChildren()
205 for index, child in enumerate(c):
206 kp = list(keypath) # Copy list.
207 kp.append(repr(index))
208 children.append(CheckNode(child, kp))
210 elif isinstance(node, Const):
211 return node.getChildren()[0]
213 raise TypeError, "Unknown AST node at key path '" + '.'.join(keypath) + \
217 def LoadOneBuildFile(build_file_path, data, aux_data, includes,
219 if build_file_path in data:
220 return data[build_file_path]
222 if os.path.exists(build_file_path):
223 build_file_contents = open(build_file_path).read()
225 raise GypError("%s not found (cwd: %s)" % (build_file_path, os.getcwd()))
227 build_file_data = None
230 build_file_data = CheckedEval(build_file_contents)
232 build_file_data = eval(build_file_contents, {'__builtins__': None},
234 except SyntaxError, e:
235 e.filename = build_file_path
238 gyp.common.ExceptionAppend(e, 'while reading ' + build_file_path)
241 if type(build_file_data) is not dict:
242 raise GypError("%s does not evaluate to a dictionary." % build_file_path)
244 data[build_file_path] = build_file_data
245 aux_data[build_file_path] = {}
247 # Scan for includes and merge them in.
248 if ('skip_includes' not in build_file_data or
249 not build_file_data['skip_includes']):
252 LoadBuildFileIncludesIntoDict(build_file_data, build_file_path, data,
253 aux_data, includes, check)
255 LoadBuildFileIncludesIntoDict(build_file_data, build_file_path, data,
256 aux_data, None, check)
258 gyp.common.ExceptionAppend(e,
259 'while reading includes of ' + build_file_path)
262 return build_file_data
265 def LoadBuildFileIncludesIntoDict(subdict, subdict_path, data, aux_data,
269 includes_list.extend(includes)
270 if 'includes' in subdict:
271 for include in subdict['includes']:
272 # "include" is specified relative to subdict_path, so compute the real
273 # path to include by appending the provided "include" to the directory
274 # in which subdict_path resides.
276 os.path.normpath(os.path.join(os.path.dirname(subdict_path), include))
277 includes_list.append(relative_include)
278 # Unhook the includes list, it's no longer needed.
279 del subdict['includes']
281 # Merge in the included files.
282 for include in includes_list:
283 if not 'included' in aux_data[subdict_path]:
284 aux_data[subdict_path]['included'] = []
285 aux_data[subdict_path]['included'].append(include)
287 gyp.DebugOutput(gyp.DEBUG_INCLUDES, "Loading Included File: '%s'", include)
290 LoadOneBuildFile(include, data, aux_data, None, False, check),
291 subdict_path, include)
293 # Recurse into subdictionaries.
294 for k, v in subdict.iteritems():
296 LoadBuildFileIncludesIntoDict(v, subdict_path, data, aux_data,
298 elif type(v) is list:
299 LoadBuildFileIncludesIntoList(v, subdict_path, data, aux_data,
303 # This recurses into lists so that it can look for dicts.
304 def LoadBuildFileIncludesIntoList(sublist, sublist_path, data, aux_data, check):
306 if type(item) is dict:
307 LoadBuildFileIncludesIntoDict(item, sublist_path, data, aux_data,
309 elif type(item) is list:
310 LoadBuildFileIncludesIntoList(item, sublist_path, data, aux_data, check)
312 # Processes toolsets in all the targets. This recurses into condition entries
313 # since they can contain toolsets as well.
314 def ProcessToolsetsInDict(data):
315 if 'targets' in data:
316 target_list = data['targets']
318 for target in target_list:
319 # If this target already has an explicit 'toolset', and no 'toolsets'
320 # list, don't modify it further.
321 if 'toolset' in target and 'toolsets' not in target:
322 new_target_list.append(target)
324 if multiple_toolsets:
325 toolsets = target.get('toolsets', ['target'])
327 toolsets = ['target']
328 # Make sure this 'toolsets' definition is only processed once.
329 if 'toolsets' in target:
330 del target['toolsets']
331 if len(toolsets) > 0:
332 # Optimization: only do copies if more than one toolset is specified.
333 for build in toolsets[1:]:
334 new_target = gyp.simple_copy.deepcopy(target)
335 new_target['toolset'] = build
336 new_target_list.append(new_target)
337 target['toolset'] = toolsets[0]
338 new_target_list.append(target)
339 data['targets'] = new_target_list
340 if 'conditions' in data:
341 for condition in data['conditions']:
342 if type(condition) is list:
343 for condition_dict in condition[1:]:
344 ProcessToolsetsInDict(condition_dict)
347 # TODO(mark): I don't love this name. It just means that it's going to load
348 # a build file that contains targets and is expected to provide a targets dict
349 # that contains the targets...
350 def LoadTargetBuildFile(build_file_path, data, aux_data, variables, includes,
351 depth, check, load_dependencies):
352 # If depth is set, predefine the DEPTH variable to be a relative path from
353 # this build file's directory to the directory identified by depth.
355 # TODO(dglazkov) The backslash/forward-slash replacement at the end is a
356 # temporary measure. This should really be addressed by keeping all paths
357 # in POSIX until actual project generation.
358 d = gyp.common.RelativePath(depth, os.path.dirname(build_file_path))
360 variables['DEPTH'] = '.'
362 variables['DEPTH'] = d.replace('\\', '/')
364 if build_file_path in data['target_build_files']:
367 data['target_build_files'].add(build_file_path)
369 gyp.DebugOutput(gyp.DEBUG_INCLUDES,
370 "Loading Target Build File '%s'", build_file_path)
372 build_file_data = LoadOneBuildFile(build_file_path, data, aux_data,
373 includes, True, check)
375 # Store DEPTH for later use in generators.
376 build_file_data['_DEPTH'] = depth
378 # Set up the included_files key indicating which .gyp files contributed to
380 if 'included_files' in build_file_data:
381 raise GypError(build_file_path + ' must not contain included_files key')
383 included = GetIncludedBuildFiles(build_file_path, aux_data)
384 build_file_data['included_files'] = []
385 for included_file in included:
386 # included_file is relative to the current directory, but it needs to
387 # be made relative to build_file_path's directory.
388 included_relative = \
389 gyp.common.RelativePath(included_file,
390 os.path.dirname(build_file_path))
391 build_file_data['included_files'].append(included_relative)
393 # Do a first round of toolsets expansion so that conditions can be defined
395 ProcessToolsetsInDict(build_file_data)
397 # Apply "pre"/"early" variable expansions and condition evaluations.
398 ProcessVariablesAndConditionsInDict(
399 build_file_data, PHASE_EARLY, variables, build_file_path)
401 # Since some toolsets might have been defined conditionally, perform
402 # a second round of toolsets expansion now.
403 ProcessToolsetsInDict(build_file_data)
405 # Look at each project's target_defaults dict, and merge settings into
407 if 'target_defaults' in build_file_data:
408 if 'targets' not in build_file_data:
409 raise GypError("Unable to find targets in build file %s" %
413 while index < len(build_file_data['targets']):
414 # This procedure needs to give the impression that target_defaults is
415 # used as defaults, and the individual targets inherit from that.
416 # The individual targets need to be merged into the defaults. Make
417 # a deep copy of the defaults for each target, merge the target dict
418 # as found in the input file into that copy, and then hook up the
419 # copy with the target-specific data merged into it as the replacement
421 old_target_dict = build_file_data['targets'][index]
422 new_target_dict = gyp.simple_copy.deepcopy(
423 build_file_data['target_defaults'])
424 MergeDicts(new_target_dict, old_target_dict,
425 build_file_path, build_file_path)
426 build_file_data['targets'][index] = new_target_dict
430 del build_file_data['target_defaults']
432 # Look for dependencies. This means that dependency resolution occurs
433 # after "pre" conditionals and variable expansion, but before "post" -
434 # in other words, you can't put a "dependencies" section inside a "post"
435 # conditional within a target.
438 if 'targets' in build_file_data:
439 for target_dict in build_file_data['targets']:
440 if 'dependencies' not in target_dict:
442 for dependency in target_dict['dependencies']:
444 gyp.common.ResolveTarget(build_file_path, dependency, None)[0])
446 if load_dependencies:
447 for dependency in dependencies:
449 LoadTargetBuildFile(dependency, data, aux_data, variables,
450 includes, depth, check, load_dependencies)
452 gyp.common.ExceptionAppend(
453 e, 'while loading dependencies of %s' % build_file_path)
456 return (build_file_path, dependencies)
459 def CallLoadTargetBuildFile(global_flags,
460 build_file_path, data,
462 includes, depth, check,
463 generator_input_info):
464 """Wrapper around LoadTargetBuildFile for parallel processing.
466 This wrapper is used when LoadTargetBuildFile is executed in
471 signal.signal(signal.SIGINT, signal.SIG_IGN)
473 # Apply globals so that the worker process behaves the same.
474 for key, value in global_flags.iteritems():
475 globals()[key] = value
477 # Save the keys so we can return data that changed.
478 data_keys = set(data)
479 aux_data_keys = set(aux_data)
481 SetGeneratorGlobals(generator_input_info)
482 result = LoadTargetBuildFile(build_file_path, data,
484 includes, depth, check, False)
488 (build_file_path, dependencies) = result
492 if key == 'target_build_files':
494 if key not in data_keys:
495 data_out[key] = data[key]
498 if key not in aux_data_keys:
499 aux_data_out[key] = aux_data[key]
501 # This gets serialized and sent back to the main process via a pipe.
502 # It's handled in LoadTargetBuildFileCallback.
503 return (build_file_path,
508 sys.stderr.write("gyp: %s\n" % e)
511 print >>sys.stderr, 'Exception:', e
512 print >>sys.stderr, traceback.format_exc()
516 class ParallelProcessingError(Exception):
520 class ParallelState(object):
521 """Class to keep track of state when processing input files in parallel.
523 If build files are loaded in parallel, use this to keep track of
524 state during farming out and processing parallel jobs. It's stored
525 in a global so that the callback function can have access to it.
529 # The multiprocessing pool.
531 # The condition variable used to protect this object and notify
532 # the main loop when there might be more data to process.
533 self.condition = None
534 # The "data" dict that was passed to LoadTargetBuildFileParallel
536 # The "aux_data" dict that was passed to LoadTargetBuildFileParallel
538 # The number of parallel calls outstanding; decremented when a response
541 # The set of all build files that have been scheduled, so we don't
542 # schedule the same one twice.
543 self.scheduled = set()
544 # A list of dependency build file paths that haven't been scheduled yet.
545 self.dependencies = []
546 # Flag to indicate if there was an error in a child process.
549 def LoadTargetBuildFileCallback(self, result):
550 """Handle the results of running LoadTargetBuildFile in another process.
552 self.condition.acquire()
555 self.condition.notify()
556 self.condition.release()
558 (build_file_path0, data0, aux_data0, dependencies0) = result
559 self.data['target_build_files'].add(build_file_path0)
561 self.data[key] = data0[key]
562 for key in aux_data0:
563 self.aux_data[key] = aux_data0[key]
564 for new_dependency in dependencies0:
565 if new_dependency not in self.scheduled:
566 self.scheduled.add(new_dependency)
567 self.dependencies.append(new_dependency)
569 self.condition.notify()
570 self.condition.release()
573 def LoadTargetBuildFilesParallel(build_files, data, aux_data,
574 variables, includes, depth, check,
575 generator_input_info):
576 parallel_state = ParallelState()
577 parallel_state.condition = threading.Condition()
578 # Make copies of the build_files argument that we can modify while working.
579 parallel_state.dependencies = list(build_files)
580 parallel_state.scheduled = set(build_files)
581 parallel_state.pending = 0
582 parallel_state.data = data
583 parallel_state.aux_data = aux_data
586 parallel_state.condition.acquire()
587 while parallel_state.dependencies or parallel_state.pending:
588 if parallel_state.error:
590 if not parallel_state.dependencies:
591 parallel_state.condition.wait()
594 dependency = parallel_state.dependencies.pop()
596 parallel_state.pending += 1
598 data_in['target_build_files'] = data['target_build_files']
601 'path_sections': globals()['path_sections'],
602 'non_configuration_keys': globals()['non_configuration_keys'],
603 'multiple_toolsets': globals()['multiple_toolsets']}
605 if not parallel_state.pool:
606 parallel_state.pool = multiprocessing.Pool(multiprocessing.cpu_count())
607 parallel_state.pool.apply_async(
608 CallLoadTargetBuildFile,
609 args = (global_flags, dependency,
610 data_in, aux_data_in,
611 variables, includes, depth, check, generator_input_info),
612 callback = parallel_state.LoadTargetBuildFileCallback)
613 except KeyboardInterrupt, e:
614 parallel_state.pool.terminate()
617 parallel_state.condition.release()
619 parallel_state.pool.close()
620 parallel_state.pool.join()
621 parallel_state.pool = None
623 if parallel_state.error:
626 # Look for the bracket that matches the first bracket seen in a
627 # string, and return the start and end as a tuple. For example, if
628 # the input is something like "<(foo <(bar)) blah", then it would
629 # return (1, 13), indicating the entire string except for the leading
630 # "<" and trailing " blah".
631 LBRACKETS= set('{[(')
632 BRACKETS = {'}': '{', ']': '[', ')': '('}
633 def FindEnclosingBracketGroup(input_str):
636 for index, char in enumerate(input_str):
637 if char in LBRACKETS:
641 elif char in BRACKETS:
644 if stack.pop() != BRACKETS[char]:
647 return (start, index + 1)
651 def IsStrCanonicalInt(string):
652 """Returns True if |string| is in its canonical integer form.
654 The canonical form is such that str(int(string)) == string.
656 if type(string) is str:
657 # This function is called a lot so for maximum performance, avoid
658 # involving regexps which would otherwise make the code much
659 # shorter. Regexps would need twice the time of this function.
667 if '1' <= string[0] <= '9':
668 return string.isdigit()
673 # This matches things like "<(asdf)", "<!(cmd)", "<!@(cmd)", "<|(list)",
674 # "<!interpreter(arguments)", "<([list])", and even "<([)" and "<(<())".
675 # In the last case, the inner "<()" is captured in match['content'].
676 early_variable_re = re.compile(
677 '(?P<replace>(?P<type><(?:(?:!?@?)|\|)?)'
678 '(?P<command_string>[-a-zA-Z0-9_.]+)?'
679 '\((?P<is_array>\s*\[?)'
680 '(?P<content>.*?)(\]?)\))')
682 # This matches the same as early_variable_re, but with '>' instead of '<'.
683 late_variable_re = re.compile(
684 '(?P<replace>(?P<type>>(?:(?:!?@?)|\|)?)'
685 '(?P<command_string>[-a-zA-Z0-9_.]+)?'
686 '\((?P<is_array>\s*\[?)'
687 '(?P<content>.*?)(\]?)\))')
689 # This matches the same as early_variable_re, but with '^' instead of '<'.
690 latelate_variable_re = re.compile(
691 '(?P<replace>(?P<type>[\^](?:(?:!?@?)|\|)?)'
692 '(?P<command_string>[-a-zA-Z0-9_.]+)?'
693 '\((?P<is_array>\s*\[?)'
694 '(?P<content>.*?)(\]?)\))')
696 # Global cache of results from running commands so they don't have to be run
698 cached_command_results = {}
701 def FixupPlatformCommand(cmd):
702 if sys.platform == 'win32':
703 if type(cmd) is list:
704 cmd = [re.sub('^cat ', 'type ', cmd[0])] + cmd[1:]
706 cmd = re.sub('^cat ', 'type ', cmd)
715 def ExpandVariables(input, phase, variables, build_file):
716 # Look for the pattern that gets expanded into variables
717 if phase == PHASE_EARLY:
718 variable_re = early_variable_re
719 expansion_symbol = '<'
720 elif phase == PHASE_LATE:
721 variable_re = late_variable_re
722 expansion_symbol = '>'
723 elif phase == PHASE_LATELATE:
724 variable_re = latelate_variable_re
725 expansion_symbol = '^'
729 input_str = str(input)
730 if IsStrCanonicalInt(input_str):
731 return int(input_str)
733 # Do a quick scan to determine if an expensive regex search is warranted.
734 if expansion_symbol not in input_str:
737 # Get the entire list of matches as a list of MatchObject instances.
738 # (using findall here would return strings instead of MatchObjects).
739 matches = list(variable_re.finditer(input_str))
744 # Reverse the list of matches so that replacements are done right-to-left.
745 # That ensures that earlier replacements won't mess up the string in a
746 # way that causes later calls to find the earlier substituted text instead
747 # of what's intended for replacement.
749 for match_group in matches:
750 match = match_group.groupdict()
751 gyp.DebugOutput(gyp.DEBUG_VARIABLES, "Matches: %r", match)
752 # match['replace'] is the substring to look for, match['type']
753 # is the character code for the replacement type (< > <! >! <| >| <@
754 # >@ <!@ >!@), match['is_array'] contains a '[' for command
755 # arrays, and match['content'] is the name of the variable (< >)
756 # or command to run (<! >!). match['command_string'] is an optional
757 # command string. Currently, only 'pymod_do_main' is supported.
759 # run_command is true if a ! variant is used.
760 run_command = '!' in match['type']
761 command_string = match['command_string']
763 # file_list is true if a | variant is used.
764 file_list = '|' in match['type']
766 # Capture these now so we can adjust them later.
767 replace_start = match_group.start('replace')
768 replace_end = match_group.end('replace')
770 # Find the ending paren, and re-evaluate the contained string.
771 (c_start, c_end) = FindEnclosingBracketGroup(input_str[replace_start:])
773 # Adjust the replacement range to match the entire command
774 # found by FindEnclosingBracketGroup (since the variable_re
775 # probably doesn't match the entire command if it contained
777 replace_end = replace_start + c_end
779 # Find the "real" replacement, matching the appropriate closing
780 # paren, and adjust the replacement start and end.
781 replacement = input_str[replace_start:replace_end]
783 # Figure out what the contents of the variable parens are.
784 contents_start = replace_start + c_start + 1
785 contents_end = replace_end - 1
786 contents = input_str[contents_start:contents_end]
788 # Do filter substitution now for <|().
789 # Admittedly, this is different than the evaluation order in other
790 # contexts. However, since filtration has no chance to run on <|(),
791 # this seems like the only obvious way to give them access to filters.
793 processed_variables = gyp.simple_copy.deepcopy(variables)
794 ProcessListFiltersInDict(contents, processed_variables)
795 # Recurse to expand variables in the contents
796 contents = ExpandVariables(contents, phase,
797 processed_variables, build_file)
799 # Recurse to expand variables in the contents
800 contents = ExpandVariables(contents, phase, variables, build_file)
802 # Strip off leading/trailing whitespace so that variable matches are
803 # simpler below (and because they are rarely needed).
804 contents = contents.strip()
806 # expand_to_list is true if an @ variant is used. In that case,
807 # the expansion should result in a list. Note that the caller
808 # is to be expecting a list in return, and not all callers do
809 # because not all are working in list context. Also, for list
810 # expansions, there can be no other text besides the variable
811 # expansion in the input string.
812 expand_to_list = '@' in match['type'] and input_str == replacement
814 if run_command or file_list:
815 # Find the build file's directory, so commands can be run or file lists
816 # generated relative to it.
817 build_file_dir = os.path.dirname(build_file)
818 if build_file_dir == '' and not file_list:
819 # If build_file is just a leaf filename indicating a file in the
820 # current directory, build_file_dir might be an empty string. Set
821 # it to None to signal to subprocess.Popen that it should run the
822 # command in the current directory.
823 build_file_dir = None
825 # Support <|(listfile.txt ...) which generates a file
826 # containing items from a gyp list, generated at gyp time.
827 # This works around actions/rules which have more inputs than will
828 # fit on the command line.
830 if type(contents) is list:
831 contents_list = contents
833 contents_list = contents.split(' ')
834 replacement = contents_list[0]
835 if os.path.isabs(replacement):
836 raise GypError('| cannot handle absolute paths, got "%s"' % replacement)
838 if not generator_filelist_paths:
839 path = os.path.join(build_file_dir, replacement)
841 if os.path.isabs(build_file_dir):
842 toplevel = generator_filelist_paths['toplevel']
843 rel_build_file_dir = gyp.common.RelativePath(build_file_dir, toplevel)
845 rel_build_file_dir = build_file_dir
846 qualified_out_dir = generator_filelist_paths['qualified_out_dir']
847 path = os.path.join(qualified_out_dir, rel_build_file_dir, replacement)
848 gyp.common.EnsureDirExists(path)
850 replacement = gyp.common.RelativePath(path, build_file_dir)
851 f = gyp.common.WriteOnDiff(path)
852 for i in contents_list[1:]:
858 if match['is_array']:
859 contents = eval(contents)
862 # Check for a cached value to avoid executing commands, or generating
863 # file lists more than once. The cache key contains the command to be
864 # run as well as the directory to run it from, to account for commands
865 # that depend on their current directory.
866 # TODO(http://code.google.com/p/gyp/issues/detail?id=111): In theory,
867 # someone could author a set of GYP files where each time the command
868 # is invoked it produces different output by design. When the need
869 # arises, the syntax should be extended to support no caching off a
870 # command's output so it is run every time.
871 cache_key = (str(contents), build_file_dir)
872 cached_value = cached_command_results.get(cache_key, None)
873 if cached_value is None:
874 gyp.DebugOutput(gyp.DEBUG_VARIABLES,
875 "Executing command '%s' in directory '%s'",
876 contents, build_file_dir)
880 if command_string == 'pymod_do_main':
881 # <!pymod_do_main(modulename param eters) loads |modulename| as a
882 # python module and then calls that module's DoMain() function,
883 # passing ["param", "eters"] as a single list argument. For modules
884 # that don't load quickly, this can be faster than
885 # <!(python modulename param eters). Do this in |build_file_dir|.
886 oldwd = os.getcwd() # Python doesn't like os.open('.'): no fchdir.
887 if build_file_dir: # build_file_dir may be None (see above).
888 os.chdir(build_file_dir)
891 parsed_contents = shlex.split(contents)
893 py_module = __import__(parsed_contents[0])
894 except ImportError as e:
895 raise GypError("Error importing pymod_do_main"
896 "module (%s): %s" % (parsed_contents[0], e))
897 replacement = str(py_module.DoMain(parsed_contents[1:])).rstrip()
900 assert replacement != None
902 raise GypError("Unknown command string '%s' in '%s'." %
903 (command_string, contents))
905 # Fix up command with platform specific workarounds.
906 contents = FixupPlatformCommand(contents)
907 p = subprocess.Popen(contents, shell=use_shell,
908 stdout=subprocess.PIPE,
909 stderr=subprocess.PIPE,
910 stdin=subprocess.PIPE,
913 p_stdout, p_stderr = p.communicate('')
915 if p.wait() != 0 or p_stderr:
916 sys.stderr.write(p_stderr)
917 # Simulate check_call behavior, since check_call only exists
918 # in python 2.5 and later.
919 raise GypError("Call to '%s' returned exit status %d." %
920 (contents, p.returncode))
921 replacement = p_stdout.rstrip()
923 cached_command_results[cache_key] = replacement
925 gyp.DebugOutput(gyp.DEBUG_VARIABLES,
926 "Had cache value for command '%s' in directory '%s'",
927 contents,build_file_dir)
928 replacement = cached_value
931 if not contents in variables:
932 if contents[-1] in ['!', '/']:
933 # In order to allow cross-compiles (nacl) to happen more naturally,
934 # we will allow references to >(sources/) etc. to resolve to
935 # and empty list if undefined. This allows actions to:
944 raise GypError('Undefined variable ' + contents +
947 replacement = variables[contents]
949 if type(replacement) is list:
950 for item in replacement:
951 if not contents[-1] == '/' and type(item) not in (str, int):
952 raise GypError('Variable ' + contents +
953 ' must expand to a string or list of strings; ' +
955 item.__class__.__name__)
956 # Run through the list and handle variable expansions in it. Since
957 # the list is guaranteed not to contain dicts, this won't do anything
958 # with conditions sections.
959 ProcessVariablesAndConditionsInList(replacement, phase, variables,
961 elif type(replacement) not in (str, int):
962 raise GypError('Variable ' + contents +
963 ' must expand to a string or list of strings; ' +
964 'found a ' + replacement.__class__.__name__)
967 # Expanding in list context. It's guaranteed that there's only one
968 # replacement to do in |input_str| and that it's this replacement. See
970 if type(replacement) is list:
971 # If it's already a list, make a copy.
972 output = replacement[:]
974 # Split it the same way sh would split arguments.
975 output = shlex.split(str(replacement))
977 # Expanding in string context.
978 encoded_replacement = ''
979 if type(replacement) is list:
980 # When expanding a list into string context, turn the list items
981 # into a string in a way that will work with a subprocess call.
983 # TODO(mark): This isn't completely correct. This should
984 # call a generator-provided function that observes the
985 # proper list-to-argument quoting rules on a specific
986 # platform instead of just calling the POSIX encoding
988 encoded_replacement = gyp.common.EncodePOSIXShellList(replacement)
990 encoded_replacement = replacement
992 output = output[:replace_start] + str(encoded_replacement) + \
994 # Prepare for the next match iteration.
998 gyp.DebugOutput(gyp.DEBUG_VARIABLES,
999 "Found only identity matches on %r, avoiding infinite "
1003 # Look for more matches now that we've replaced some, to deal with
1004 # expanding local variables (variables defined in the same
1005 # variables block as this one).
1006 gyp.DebugOutput(gyp.DEBUG_VARIABLES, "Found output %r, recursing.", output)
1007 if type(output) is list:
1008 if output and type(output[0]) is list:
1009 # Leave output alone if it's a list of lists.
1010 # We don't want such lists to be stringified.
1016 ExpandVariables(item, phase, variables, build_file))
1019 output = ExpandVariables(output, phase, variables, build_file)
1021 # Convert all strings that are canonically-represented integers into integers.
1022 if type(output) is list:
1023 for index in xrange(0, len(output)):
1024 if IsStrCanonicalInt(output[index]):
1025 output[index] = int(output[index])
1026 elif IsStrCanonicalInt(output):
1027 output = int(output)
1031 # The same condition is often evaluated over and over again so it
1032 # makes sense to cache as much as possible between evaluations.
1033 cached_conditions_asts = {}
1035 def EvalCondition(condition, conditions_key, phase, variables, build_file):
1036 """Returns the dict that should be used or None if the result was
1037 that nothing should be used."""
1038 if type(condition) is not list:
1039 raise GypError(conditions_key + ' must be a list')
1040 if len(condition) != 2 and len(condition) != 3:
1041 # It's possible that condition[0] won't work in which case this
1042 # attempt will raise its own IndexError. That's probably fine.
1043 raise GypError(conditions_key + ' ' + condition[0] +
1044 ' must be length 2 or 3, not ' + str(len(condition)))
1046 [cond_expr, true_dict] = condition[0:2]
1048 if len(condition) == 3:
1049 false_dict = condition[2]
1051 # Do expansions on the condition itself. Since the conditon can naturally
1052 # contain variable references without needing to resort to GYP expansion
1053 # syntax, this is of dubious value for variables, but someone might want to
1054 # use a command expansion directly inside a condition.
1055 cond_expr_expanded = ExpandVariables(cond_expr, phase, variables,
1057 if type(cond_expr_expanded) not in (str, int):
1059 'Variable expansion in this context permits str and int ' + \
1060 'only, found ' + cond_expr_expanded.__class__.__name__
1063 if cond_expr_expanded in cached_conditions_asts:
1064 ast_code = cached_conditions_asts[cond_expr_expanded]
1066 ast_code = compile(cond_expr_expanded, '<string>', 'eval')
1067 cached_conditions_asts[cond_expr_expanded] = ast_code
1068 if eval(ast_code, {'__builtins__': None}, variables):
1071 except SyntaxError, e:
1072 syntax_error = SyntaxError('%s while evaluating condition \'%s\' in %s '
1073 'at character %d.' %
1074 (str(e.args[0]), e.text, build_file, e.offset),
1075 e.filename, e.lineno, e.offset, e.text)
1077 except NameError, e:
1078 gyp.common.ExceptionAppend(e, 'while evaluating condition \'%s\' in %s' %
1079 (cond_expr_expanded, build_file))
1083 def ProcessConditionsInDict(the_dict, phase, variables, build_file):
1084 # Process a 'conditions' or 'target_conditions' section in the_dict,
1085 # depending on phase.
1086 # early -> conditions
1087 # late -> target_conditions
1088 # latelate -> no conditions
1090 # Each item in a conditions list consists of cond_expr, a string expression
1091 # evaluated as the condition, and true_dict, a dict that will be merged into
1092 # the_dict if cond_expr evaluates to true. Optionally, a third item,
1093 # false_dict, may be present. false_dict is merged into the_dict if
1094 # cond_expr evaluates to false.
1096 # Any dict merged into the_dict will be recursively processed for nested
1097 # conditionals and other expansions, also according to phase, immediately
1098 # prior to being merged.
1100 if phase == PHASE_EARLY:
1101 conditions_key = 'conditions'
1102 elif phase == PHASE_LATE:
1103 conditions_key = 'target_conditions'
1104 elif phase == PHASE_LATELATE:
1109 if not conditions_key in the_dict:
1112 conditions_list = the_dict[conditions_key]
1113 # Unhook the conditions list, it's no longer needed.
1114 del the_dict[conditions_key]
1116 for condition in conditions_list:
1117 merge_dict = EvalCondition(condition, conditions_key, phase, variables,
1120 if merge_dict != None:
1121 # Expand variables and nested conditinals in the merge_dict before
1123 ProcessVariablesAndConditionsInDict(merge_dict, phase,
1124 variables, build_file)
1126 MergeDicts(the_dict, merge_dict, build_file, build_file)
1129 def LoadAutomaticVariablesFromDict(variables, the_dict):
1130 # Any keys with plain string values in the_dict become automatic variables.
1131 # The variable name is the key name with a "_" character prepended.
1132 for key, value in the_dict.iteritems():
1133 if type(value) in (str, int, list):
1134 variables['_' + key] = value
1137 def LoadVariablesFromVariablesDict(variables, the_dict, the_dict_key):
1138 # Any keys in the_dict's "variables" dict, if it has one, becomes a
1139 # variable. The variable name is the key name in the "variables" dict.
1140 # Variables that end with the % character are set only if they are unset in
1141 # the variables dict. the_dict_key is the name of the key that accesses
1142 # the_dict in the_dict's parent dict. If the_dict's parent is not a dict
1143 # (it could be a list or it could be parentless because it is a root dict),
1144 # the_dict_key will be None.
1145 for key, value in the_dict.get('variables', {}).iteritems():
1146 if type(value) not in (str, int, list):
1149 if key.endswith('%'):
1150 variable_name = key[:-1]
1151 if variable_name in variables:
1152 # If the variable is already set, don't set it.
1154 if the_dict_key is 'variables' and variable_name in the_dict:
1155 # If the variable is set without a % in the_dict, and the_dict is a
1156 # variables dict (making |variables| a varaibles sub-dict of a
1157 # variables dict), use the_dict's definition.
1158 value = the_dict[variable_name]
1162 variables[variable_name] = value
1165 def ProcessVariablesAndConditionsInDict(the_dict, phase, variables_in,
1166 build_file, the_dict_key=None):
1167 """Handle all variable and command expansion and conditional evaluation.
1169 This function is the public entry point for all variable expansions and
1170 conditional evaluations. The variables_in dictionary will not be modified
1174 # Make a copy of the variables_in dict that can be modified during the
1175 # loading of automatics and the loading of the variables dict.
1176 variables = variables_in.copy()
1177 LoadAutomaticVariablesFromDict(variables, the_dict)
1179 if 'variables' in the_dict:
1180 # Make sure all the local variables are added to the variables
1181 # list before we process them so that you can reference one
1182 # variable from another. They will be fully expanded by recursion
1183 # in ExpandVariables.
1184 for key, value in the_dict['variables'].iteritems():
1185 variables[key] = value
1187 # Handle the associated variables dict first, so that any variable
1188 # references within can be resolved prior to using them as variables.
1189 # Pass a copy of the variables dict to avoid having it be tainted.
1190 # Otherwise, it would have extra automatics added for everything that
1191 # should just be an ordinary variable in this scope.
1192 ProcessVariablesAndConditionsInDict(the_dict['variables'], phase,
1193 variables, build_file, 'variables')
1195 LoadVariablesFromVariablesDict(variables, the_dict, the_dict_key)
1197 for key, value in the_dict.iteritems():
1198 # Skip "variables", which was already processed if present.
1199 if key != 'variables' and type(value) is str:
1200 expanded = ExpandVariables(value, phase, variables, build_file)
1201 if type(expanded) not in (str, int):
1203 'Variable expansion in this context permits str and int ' + \
1204 'only, found ' + expanded.__class__.__name__ + ' for ' + key
1205 the_dict[key] = expanded
1207 # Variable expansion may have resulted in changes to automatics. Reload.
1208 # TODO(mark): Optimization: only reload if no changes were made.
1209 variables = variables_in.copy()
1210 LoadAutomaticVariablesFromDict(variables, the_dict)
1211 LoadVariablesFromVariablesDict(variables, the_dict, the_dict_key)
1213 # Process conditions in this dict. This is done after variable expansion
1214 # so that conditions may take advantage of expanded variables. For example,
1215 # if the_dict contains:
1216 # {'type': '<(library_type)',
1217 # 'conditions': [['_type=="static_library"', { ... }]]},
1218 # _type, as used in the condition, will only be set to the value of
1219 # library_type if variable expansion is performed before condition
1220 # processing. However, condition processing should occur prior to recursion
1221 # so that variables (both automatic and "variables" dict type) may be
1222 # adjusted by conditions sections, merged into the_dict, and have the
1223 # intended impact on contained dicts.
1225 # This arrangement means that a "conditions" section containing a "variables"
1226 # section will only have those variables effective in subdicts, not in
1227 # the_dict. The workaround is to put a "conditions" section within a
1228 # "variables" section. For example:
1229 # {'conditions': [['os=="mac"', {'variables': {'define': 'IS_MAC'}}]],
1230 # 'defines': ['<(define)'],
1231 # 'my_subdict': {'defines': ['<(define)']}},
1232 # will not result in "IS_MAC" being appended to the "defines" list in the
1233 # current scope but would result in it being appended to the "defines" list
1234 # within "my_subdict". By comparison:
1235 # {'variables': {'conditions': [['os=="mac"', {'define': 'IS_MAC'}]]},
1236 # 'defines': ['<(define)'],
1237 # 'my_subdict': {'defines': ['<(define)']}},
1238 # will append "IS_MAC" to both "defines" lists.
1240 # Evaluate conditions sections, allowing variable expansions within them
1241 # as well as nested conditionals. This will process a 'conditions' or
1242 # 'target_conditions' section, perform appropriate merging and recursive
1243 # conditional and variable processing, and then remove the conditions section
1244 # from the_dict if it is present.
1245 ProcessConditionsInDict(the_dict, phase, variables, build_file)
1247 # Conditional processing may have resulted in changes to automatics or the
1248 # variables dict. Reload.
1249 variables = variables_in.copy()
1250 LoadAutomaticVariablesFromDict(variables, the_dict)
1251 LoadVariablesFromVariablesDict(variables, the_dict, the_dict_key)
1253 # Recurse into child dicts, or process child lists which may result in
1254 # further recursion into descendant dicts.
1255 for key, value in the_dict.iteritems():
1256 # Skip "variables" and string values, which were already processed if
1258 if key == 'variables' or type(value) is str:
1260 if type(value) is dict:
1261 # Pass a copy of the variables dict so that subdicts can't influence
1263 ProcessVariablesAndConditionsInDict(value, phase, variables,
1265 elif type(value) is list:
1266 # The list itself can't influence the variables dict, and
1267 # ProcessVariablesAndConditionsInList will make copies of the variables
1268 # dict if it needs to pass it to something that can influence it. No
1269 # copy is necessary here.
1270 ProcessVariablesAndConditionsInList(value, phase, variables,
1272 elif type(value) is not int:
1273 raise TypeError, 'Unknown type ' + value.__class__.__name__ + \
1277 def ProcessVariablesAndConditionsInList(the_list, phase, variables,
1279 # Iterate using an index so that new values can be assigned into the_list.
1281 while index < len(the_list):
1282 item = the_list[index]
1283 if type(item) is dict:
1284 # Make a copy of the variables dict so that it won't influence anything
1285 # outside of its own scope.
1286 ProcessVariablesAndConditionsInDict(item, phase, variables, build_file)
1287 elif type(item) is list:
1288 ProcessVariablesAndConditionsInList(item, phase, variables, build_file)
1289 elif type(item) is str:
1290 expanded = ExpandVariables(item, phase, variables, build_file)
1291 if type(expanded) in (str, int):
1292 the_list[index] = expanded
1293 elif type(expanded) is list:
1294 the_list[index:index+1] = expanded
1295 index += len(expanded)
1297 # index now identifies the next item to examine. Continue right now
1298 # without falling into the index increment below.
1302 'Variable expansion in this context permits strings and ' + \
1303 'lists only, found ' + expanded.__class__.__name__ + ' at ' + \
1305 elif type(item) is not int:
1306 raise TypeError, 'Unknown type ' + item.__class__.__name__ + \
1307 ' at index ' + index
1311 def BuildTargetsDict(data):
1312 """Builds a dict mapping fully-qualified target names to their target dicts.
1314 |data| is a dict mapping loaded build files by pathname relative to the
1315 current directory. Values in |data| are build file contents. For each
1316 |data| value with a "targets" key, the value of the "targets" key is taken
1317 as a list containing target dicts. Each target's fully-qualified name is
1318 constructed from the pathname of the build file (|data| key) and its
1319 "target_name" property. These fully-qualified names are used as the keys
1320 in the returned dict. These keys provide access to the target dicts,
1321 the dicts in the "targets" lists.
1325 for build_file in data['target_build_files']:
1326 for target in data[build_file].get('targets', []):
1327 target_name = gyp.common.QualifiedTarget(build_file,
1328 target['target_name'],
1330 if target_name in targets:
1331 raise GypError('Duplicate target definitions for ' + target_name)
1332 targets[target_name] = target
1337 def QualifyDependencies(targets):
1338 """Make dependency links fully-qualified relative to the current directory.
1340 |targets| is a dict mapping fully-qualified target names to their target
1341 dicts. For each target in this dict, keys known to contain dependency
1342 links are examined, and any dependencies referenced will be rewritten
1343 so that they are fully-qualified and relative to the current directory.
1344 All rewritten dependencies are suitable for use as keys to |targets| or a
1348 all_dependency_sections = [dep + op
1349 for dep in dependency_sections
1350 for op in ('', '!', '/')]
1352 for target, target_dict in targets.iteritems():
1353 target_build_file = gyp.common.BuildFile(target)
1354 toolset = target_dict['toolset']
1355 for dependency_key in all_dependency_sections:
1356 dependencies = target_dict.get(dependency_key, [])
1357 for index in xrange(0, len(dependencies)):
1358 dep_file, dep_target, dep_toolset = gyp.common.ResolveTarget(
1359 target_build_file, dependencies[index], toolset)
1360 if not multiple_toolsets:
1361 # Ignore toolset specification in the dependency if it is specified.
1362 dep_toolset = toolset
1363 dependency = gyp.common.QualifiedTarget(dep_file,
1366 dependencies[index] = dependency
1368 # Make sure anything appearing in a list other than "dependencies" also
1369 # appears in the "dependencies" list.
1370 if dependency_key != 'dependencies' and \
1371 dependency not in target_dict['dependencies']:
1372 raise GypError('Found ' + dependency + ' in ' + dependency_key +
1373 ' of ' + target + ', but not in dependencies')
1376 def ExpandWildcardDependencies(targets, data):
1377 """Expands dependencies specified as build_file:*.
1379 For each target in |targets|, examines sections containing links to other
1380 targets. If any such section contains a link of the form build_file:*, it
1381 is taken as a wildcard link, and is expanded to list each target in
1382 build_file. The |data| dict provides access to build file dicts.
1384 Any target that does not wish to be included by wildcard can provide an
1385 optional "suppress_wildcard" key in its target dict. When present and
1386 true, a wildcard dependency link will not include such targets.
1388 All dependency names, including the keys to |targets| and the values in each
1389 dependency list, must be qualified when this function is called.
1392 for target, target_dict in targets.iteritems():
1393 toolset = target_dict['toolset']
1394 target_build_file = gyp.common.BuildFile(target)
1395 for dependency_key in dependency_sections:
1396 dependencies = target_dict.get(dependency_key, [])
1398 # Loop this way instead of "for dependency in" or "for index in xrange"
1399 # because the dependencies list will be modified within the loop body.
1401 while index < len(dependencies):
1402 (dependency_build_file, dependency_target, dependency_toolset) = \
1403 gyp.common.ParseQualifiedTarget(dependencies[index])
1404 if dependency_target != '*' and dependency_toolset != '*':
1405 # Not a wildcard. Keep it moving.
1409 if dependency_build_file == target_build_file:
1410 # It's an error for a target to depend on all other targets in
1411 # the same file, because a target cannot depend on itself.
1412 raise GypError('Found wildcard in ' + dependency_key + ' of ' +
1413 target + ' referring to same build file')
1415 # Take the wildcard out and adjust the index so that the next
1416 # dependency in the list will be processed the next time through the
1418 del dependencies[index]
1421 # Loop through the targets in the other build file, adding them to
1422 # this target's list of dependencies in place of the removed
1424 dependency_target_dicts = data[dependency_build_file]['targets']
1425 for dependency_target_dict in dependency_target_dicts:
1426 if int(dependency_target_dict.get('suppress_wildcard', False)):
1428 dependency_target_name = dependency_target_dict['target_name']
1429 if (dependency_target != '*' and
1430 dependency_target != dependency_target_name):
1432 dependency_target_toolset = dependency_target_dict['toolset']
1433 if (dependency_toolset != '*' and
1434 dependency_toolset != dependency_target_toolset):
1436 dependency = gyp.common.QualifiedTarget(dependency_build_file,
1437 dependency_target_name,
1438 dependency_target_toolset)
1440 dependencies.insert(index, dependency)
1446 """Removes duplicate elements from l, keeping the first element."""
1448 return [seen.setdefault(e, e) for e in l if e not in seen]
1451 def RemoveDuplicateDependencies(targets):
1452 """Makes sure every dependency appears only once in all targets's dependency
1454 for target_name, target_dict in targets.iteritems():
1455 for dependency_key in dependency_sections:
1456 dependencies = target_dict.get(dependency_key, [])
1458 target_dict[dependency_key] = Unify(dependencies)
1461 def Filter(l, item):
1462 """Removes item from l."""
1464 return [res.setdefault(e, e) for e in l if e != item]
1467 def RemoveSelfDependencies(targets):
1468 """Remove self dependencies from targets that have the prune_self_dependency
1470 for target_name, target_dict in targets.iteritems():
1471 for dependency_key in dependency_sections:
1472 dependencies = target_dict.get(dependency_key, [])
1474 for t in dependencies:
1475 if t == target_name:
1476 if targets[t].get('variables', {}).get('prune_self_dependency', 0):
1477 target_dict[dependency_key] = Filter(dependencies, target_name)
1480 def RemoveLinkDependenciesFromNoneTargets(targets):
1481 """Remove dependencies having the 'link_dependency' attribute from the 'none'
1483 for target_name, target_dict in targets.iteritems():
1484 for dependency_key in dependency_sections:
1485 dependencies = target_dict.get(dependency_key, [])
1487 for t in dependencies:
1488 if target_dict.get('type', None) == 'none':
1489 if targets[t].get('variables', {}).get('link_dependency', 0):
1490 target_dict[dependency_key] = \
1491 Filter(target_dict[dependency_key], t)
1494 class DependencyGraphNode(object):
1498 ref: A reference to an object that this DependencyGraphNode represents.
1499 dependencies: List of DependencyGraphNodes on which this one depends.
1500 dependents: List of DependencyGraphNodes that depend on this one.
1503 class CircularException(GypError):
1506 def __init__(self, ref):
1508 self.dependencies = []
1509 self.dependents = []
1512 return '<DependencyGraphNode: %r>' % self.ref
1514 def FlattenToList(self):
1515 # flat_list is the sorted list of dependencies - actually, the list items
1516 # are the "ref" attributes of DependencyGraphNodes. Every target will
1517 # appear in flat_list after all of its dependencies, and before all of its
1519 flat_list = OrderedSet()
1521 # in_degree_zeros is the list of DependencyGraphNodes that have no
1522 # dependencies not in flat_list. Initially, it is a copy of the children
1523 # of this node, because when the graph was built, nodes with no
1524 # dependencies were made implicit dependents of the root node.
1525 in_degree_zeros = set(self.dependents[:])
1527 while in_degree_zeros:
1528 # Nodes in in_degree_zeros have no dependencies not in flat_list, so they
1529 # can be appended to flat_list. Take these nodes out of in_degree_zeros
1530 # as work progresses, so that the next node to process from the list can
1531 # always be accessed at a consistent position.
1532 node = in_degree_zeros.pop()
1533 flat_list.add(node.ref)
1535 # Look at dependents of the node just added to flat_list. Some of them
1536 # may now belong in in_degree_zeros.
1537 for node_dependent in node.dependents:
1538 is_in_degree_zero = True
1539 # TODO: We want to check through the
1540 # node_dependent.dependencies list but if it's long and we
1541 # always start at the beginning, then we get O(n^2) behaviour.
1542 for node_dependent_dependency in node_dependent.dependencies:
1543 if not node_dependent_dependency.ref in flat_list:
1544 # The dependent one or more dependencies not in flat_list. There
1545 # will be more chances to add it to flat_list when examining
1546 # it again as a dependent of those other dependencies, provided
1547 # that there are no cycles.
1548 is_in_degree_zero = False
1551 if is_in_degree_zero:
1552 # All of the dependent's dependencies are already in flat_list. Add
1553 # it to in_degree_zeros where it will be processed in a future
1554 # iteration of the outer loop.
1555 in_degree_zeros.add(node_dependent)
1557 return list(flat_list)
1559 def FindCycles(self, path=None):
1561 Returns a list of cycles in the graph, where each cycle is its own list.
1567 for node in self.dependents:
1574 results.append(tuple(cycle))
1576 results.extend(node.FindCycles([node] + path))
1578 return list(set(results))
1580 def DirectDependencies(self, dependencies=None):
1581 """Returns a list of just direct dependencies."""
1582 if dependencies == None:
1585 for dependency in self.dependencies:
1586 # Check for None, corresponding to the root node.
1587 if dependency.ref != None and dependency.ref not in dependencies:
1588 dependencies.append(dependency.ref)
1592 def _AddImportedDependencies(self, targets, dependencies=None):
1593 """Given a list of direct dependencies, adds indirect dependencies that
1594 other dependencies have declared to export their settings.
1596 This method does not operate on self. Rather, it operates on the list
1597 of dependencies in the |dependencies| argument. For each dependency in
1598 that list, if any declares that it exports the settings of one of its
1599 own dependencies, those dependencies whose settings are "passed through"
1600 are added to the list. As new items are added to the list, they too will
1601 be processed, so it is possible to import settings through multiple levels
1604 This method is not terribly useful on its own, it depends on being
1605 "primed" with a list of direct dependencies such as one provided by
1606 DirectDependencies. DirectAndImportedDependencies is intended to be the
1610 if dependencies == None:
1614 while index < len(dependencies):
1615 dependency = dependencies[index]
1616 dependency_dict = targets[dependency]
1617 # Add any dependencies whose settings should be imported to the list
1618 # if not already present. Newly-added items will be checked for
1619 # their own imports when the list iteration reaches them.
1620 # Rather than simply appending new items, insert them after the
1621 # dependency that exported them. This is done to more closely match
1622 # the depth-first method used by DeepDependencies.
1624 for imported_dependency in \
1625 dependency_dict.get('export_dependent_settings', []):
1626 if imported_dependency not in dependencies:
1627 dependencies.insert(index + add_index, imported_dependency)
1628 add_index = add_index + 1
1633 def DirectAndImportedDependencies(self, targets, dependencies=None):
1634 """Returns a list of a target's direct dependencies and all indirect
1635 dependencies that a dependency has advertised settings should be exported
1636 through the dependency for.
1639 dependencies = self.DirectDependencies(dependencies)
1640 return self._AddImportedDependencies(targets, dependencies)
1642 def DeepDependencies(self, dependencies=None):
1643 """Returns an OrderedSet of all of a target's dependencies, recursively."""
1644 if dependencies is None:
1645 # Using a list to get ordered output and a set to do fast "is it
1646 # already added" checks.
1647 dependencies = OrderedSet()
1649 for dependency in self.dependencies:
1650 # Check for None, corresponding to the root node.
1651 if dependency.ref is None:
1653 if dependency.ref not in dependencies:
1654 dependencies.add(dependency.ref)
1655 dependency.DeepDependencies(dependencies)
1659 def _LinkDependenciesInternal(self, targets, include_shared_libraries,
1660 dependencies=None, initial=True):
1661 """Returns an OrderedSet of dependency targets that are linked
1664 This function has a split personality, depending on the setting of
1665 |initial|. Outside callers should always leave |initial| at its default
1668 When adding a target to the list of dependencies, this function will
1669 recurse into itself with |initial| set to False, to collect dependencies
1670 that are linked into the linkable target for which the list is being built.
1672 If |include_shared_libraries| is False, the resulting dependencies will not
1673 include shared_library targets that are linked into this target.
1675 if dependencies is None:
1676 # Using a list to get ordered output and a set to do fast "is it
1677 # already added" checks.
1678 dependencies = OrderedSet()
1680 # Check for None, corresponding to the root node.
1681 if self.ref is None:
1684 # It's kind of sucky that |targets| has to be passed into this function,
1685 # but that's presently the easiest way to access the target dicts so that
1686 # this function can find target types.
1688 if 'target_name' not in targets[self.ref]:
1689 raise GypError("Missing 'target_name' field in target.")
1691 if 'type' not in targets[self.ref]:
1692 raise GypError("Missing 'type' field in target %s" %
1693 targets[self.ref]['target_name'])
1695 target_type = targets[self.ref]['type']
1697 is_linkable = target_type in linkable_types
1699 if initial and not is_linkable:
1700 # If this is the first target being examined and it's not linkable,
1701 # return an empty list of link dependencies, because the link
1702 # dependencies are intended to apply to the target itself (initial is
1703 # True) and this target won't be linked.
1706 # Don't traverse 'none' targets if explicitly excluded.
1707 if (target_type == 'none' and
1708 not targets[self.ref].get('dependencies_traverse', True)):
1709 dependencies.add(self.ref)
1712 # Executables and loadable modules are already fully and finally linked.
1713 # Nothing else can be a link dependency of them, there can only be
1714 # dependencies in the sense that a dependent target might run an
1715 # executable or load the loadable_module.
1716 if not initial and target_type in ('executable', 'loadable_module'):
1719 # Shared libraries are already fully linked. They should only be included
1720 # in |dependencies| when adjusting static library dependencies (in order to
1721 # link against the shared_library's import lib), but should not be included
1722 # in |dependencies| when propagating link_settings.
1723 # The |include_shared_libraries| flag controls which of these two cases we
1725 if (not initial and target_type == 'shared_library' and
1726 not include_shared_libraries):
1729 # The target is linkable, add it to the list of link dependencies.
1730 if self.ref not in dependencies:
1731 dependencies.add(self.ref)
1732 if initial or not is_linkable:
1733 # If this is a subsequent target and it's linkable, don't look any
1734 # further for linkable dependencies, as they'll already be linked into
1735 # this target linkable. Always look at dependencies of the initial
1736 # target, and always look at dependencies of non-linkables.
1737 for dependency in self.dependencies:
1738 dependency._LinkDependenciesInternal(targets,
1739 include_shared_libraries,
1740 dependencies, False)
1744 def DependenciesForLinkSettings(self, targets):
1746 Returns a list of dependency targets whose link_settings should be merged
1750 # TODO(sbaig) Currently, chrome depends on the bug that shared libraries'
1751 # link_settings are propagated. So for now, we will allow it, unless the
1752 # 'allow_sharedlib_linksettings_propagation' flag is explicitly set to
1753 # False. Once chrome is fixed, we can remove this flag.
1754 include_shared_libraries = \
1755 targets[self.ref].get('allow_sharedlib_linksettings_propagation', True)
1756 return self._LinkDependenciesInternal(targets, include_shared_libraries)
1758 def DependenciesToLinkAgainst(self, targets):
1760 Returns a list of dependency targets that are linked into this target.
1762 return self._LinkDependenciesInternal(targets, True)
1765 def BuildDependencyList(targets):
1766 # Create a DependencyGraphNode for each target. Put it into a dict for easy
1768 dependency_nodes = {}
1769 for target, spec in targets.iteritems():
1770 if target not in dependency_nodes:
1771 dependency_nodes[target] = DependencyGraphNode(target)
1773 # Set up the dependency links. Targets that have no dependencies are treated
1774 # as dependent on root_node.
1775 root_node = DependencyGraphNode(None)
1776 for target, spec in targets.iteritems():
1777 target_node = dependency_nodes[target]
1778 target_build_file = gyp.common.BuildFile(target)
1779 dependencies = spec.get('dependencies')
1780 if not dependencies:
1781 target_node.dependencies = [root_node]
1782 root_node.dependents.append(target_node)
1784 for dependency in dependencies:
1785 dependency_node = dependency_nodes.get(dependency)
1786 if not dependency_node:
1787 raise GypError("Dependency '%s' not found while "
1788 "trying to load target %s" % (dependency, target))
1789 target_node.dependencies.append(dependency_node)
1790 dependency_node.dependents.append(target_node)
1792 flat_list = root_node.FlattenToList()
1794 # If there's anything left unvisited, there must be a circular dependency
1795 # (cycle). If you need to figure out what's wrong, look for elements of
1796 # targets that are not in flat_list.
1797 if len(flat_list) != len(targets):
1798 raise DependencyGraphNode.CircularException(
1799 'Some targets not reachable, cycle in dependency graph detected: ' +
1800 ' '.join(set(flat_list) ^ set(targets)))
1802 return [dependency_nodes, flat_list]
1805 def VerifyNoGYPFileCircularDependencies(targets):
1806 # Create a DependencyGraphNode for each gyp file containing a target. Put
1807 # it into a dict for easy access.
1808 dependency_nodes = {}
1809 for target in targets.iterkeys():
1810 build_file = gyp.common.BuildFile(target)
1811 if not build_file in dependency_nodes:
1812 dependency_nodes[build_file] = DependencyGraphNode(build_file)
1814 # Set up the dependency links.
1815 for target, spec in targets.iteritems():
1816 build_file = gyp.common.BuildFile(target)
1817 build_file_node = dependency_nodes[build_file]
1818 target_dependencies = spec.get('dependencies', [])
1819 for dependency in target_dependencies:
1821 dependency_build_file = gyp.common.BuildFile(dependency)
1823 gyp.common.ExceptionAppend(
1824 e, 'while computing dependencies of .gyp file %s' % build_file)
1827 if dependency_build_file == build_file:
1828 # A .gyp file is allowed to refer back to itself.
1830 dependency_node = dependency_nodes.get(dependency_build_file)
1831 if not dependency_node:
1832 raise GypError("Dependancy '%s' not found" % dependency_build_file)
1833 if dependency_node not in build_file_node.dependencies:
1834 build_file_node.dependencies.append(dependency_node)
1835 dependency_node.dependents.append(build_file_node)
1838 # Files that have no dependencies are treated as dependent on root_node.
1839 root_node = DependencyGraphNode(None)
1840 for build_file_node in dependency_nodes.itervalues():
1841 if len(build_file_node.dependencies) == 0:
1842 build_file_node.dependencies.append(root_node)
1843 root_node.dependents.append(build_file_node)
1845 flat_list = root_node.FlattenToList()
1847 # If there's anything left unvisited, there must be a circular dependency
1849 if len(flat_list) != len(dependency_nodes):
1851 for file in dependency_nodes.iterkeys():
1852 if not file in flat_list:
1853 bad_files.append(file)
1854 common_path_prefix = os.path.commonprefix(dependency_nodes)
1856 for cycle in root_node.FindCycles():
1857 simplified_paths = []
1859 assert(node.ref.startswith(common_path_prefix))
1860 simplified_paths.append(node.ref[len(common_path_prefix):])
1861 cycles.append('Cycle: %s' % ' -> '.join(simplified_paths))
1862 raise DependencyGraphNode.CircularException, \
1863 'Cycles in .gyp file dependency graph detected:\n' + '\n'.join(cycles)
1866 def DoDependentSettings(key, flat_list, targets, dependency_nodes):
1867 # key should be one of all_dependent_settings, direct_dependent_settings,
1870 for target in flat_list:
1871 target_dict = targets[target]
1872 build_file = gyp.common.BuildFile(target)
1874 if key == 'all_dependent_settings':
1875 dependencies = dependency_nodes[target].DeepDependencies()
1876 elif key == 'direct_dependent_settings':
1878 dependency_nodes[target].DirectAndImportedDependencies(targets)
1879 elif key == 'link_settings':
1881 dependency_nodes[target].DependenciesForLinkSettings(targets)
1883 raise GypError("DoDependentSettings doesn't know how to determine "
1884 'dependencies for ' + key)
1886 for dependency in dependencies:
1887 dependency_dict = targets[dependency]
1888 if not key in dependency_dict:
1890 dependency_build_file = gyp.common.BuildFile(dependency)
1891 MergeDicts(target_dict, dependency_dict[key],
1892 build_file, dependency_build_file)
1895 def AdjustStaticLibraryDependencies(flat_list, targets, dependency_nodes,
1897 # Recompute target "dependencies" properties. For each static library
1898 # target, remove "dependencies" entries referring to other static libraries,
1899 # unless the dependency has the "hard_dependency" attribute set. For each
1900 # linkable target, add a "dependencies" entry referring to all of the
1901 # target's computed list of link dependencies (including static libraries
1902 # if no such entry is already present.
1903 for target in flat_list:
1904 target_dict = targets[target]
1905 target_type = target_dict['type']
1907 if target_type == 'static_library':
1908 if not 'dependencies' in target_dict:
1911 target_dict['dependencies_original'] = target_dict.get(
1912 'dependencies', [])[:]
1914 # A static library should not depend on another static library unless
1915 # the dependency relationship is "hard," which should only be done when
1916 # a dependent relies on some side effect other than just the build
1917 # product, like a rule or action output. Further, if a target has a
1918 # non-hard dependency, but that dependency exports a hard dependency,
1919 # the non-hard dependency can safely be removed, but the exported hard
1920 # dependency must be added to the target to keep the same dependency
1923 dependency_nodes[target].DirectAndImportedDependencies(targets)
1925 while index < len(dependencies):
1926 dependency = dependencies[index]
1927 dependency_dict = targets[dependency]
1929 # Remove every non-hard static library dependency and remove every
1930 # non-static library dependency that isn't a direct dependency.
1931 if (dependency_dict['type'] == 'static_library' and \
1932 not dependency_dict.get('hard_dependency', False)) or \
1933 (dependency_dict['type'] != 'static_library' and \
1934 not dependency in target_dict['dependencies']):
1935 # Take the dependency out of the list, and don't increment index
1936 # because the next dependency to analyze will shift into the index
1937 # formerly occupied by the one being removed.
1938 del dependencies[index]
1942 # Update the dependencies. If the dependencies list is empty, it's not
1943 # needed, so unhook it.
1944 if len(dependencies) > 0:
1945 target_dict['dependencies'] = dependencies
1947 del target_dict['dependencies']
1949 elif target_type in linkable_types:
1950 # Get a list of dependency targets that should be linked into this
1951 # target. Add them to the dependencies list if they're not already
1954 link_dependencies = \
1955 dependency_nodes[target].DependenciesToLinkAgainst(targets)
1956 for dependency in link_dependencies:
1957 if dependency == target:
1959 if not 'dependencies' in target_dict:
1960 target_dict['dependencies'] = []
1961 if not dependency in target_dict['dependencies']:
1962 target_dict['dependencies'].append(dependency)
1963 # Sort the dependencies list in the order from dependents to dependencies.
1964 # e.g. If A and B depend on C and C depends on D, sort them in A, B, C, D.
1965 # Note: flat_list is already sorted in the order from dependencies to
1967 if sort_dependencies and 'dependencies' in target_dict:
1968 target_dict['dependencies'] = [dep for dep in reversed(flat_list)
1969 if dep in target_dict['dependencies']]
1972 # Initialize this here to speed up MakePathRelative.
1973 exception_re = re.compile(r'''["']?[-/$<>^]''')
1976 def MakePathRelative(to_file, fro_file, item):
1977 # If item is a relative path, it's relative to the build file dict that it's
1978 # coming from. Fix it up to make it relative to the build file dict that
1980 # Exception: any |item| that begins with these special characters is
1981 # returned without modification.
1982 # / Used when a path is already absolute (shortcut optimization;
1983 # such paths would be returned as absolute anyway)
1984 # $ Used for build environment variables
1985 # - Used for some build environment flags (such as -lapr-1 in a
1986 # "libraries" section)
1987 # < Used for our own variable and command expansions (see ExpandVariables)
1988 # > Used for our own variable and command expansions (see ExpandVariables)
1989 # ^ Used for our own variable and command expansions (see ExpandVariables)
1991 # "/' Used when a value is quoted. If these are present, then we
1992 # check the second character instead.
1994 if to_file == fro_file or exception_re.match(item):
1997 # TODO(dglazkov) The backslash/forward-slash replacement at the end is a
1998 # temporary measure. This should really be addressed by keeping all paths
1999 # in POSIX until actual project generation.
2000 ret = os.path.normpath(os.path.join(
2001 gyp.common.RelativePath(os.path.dirname(fro_file),
2002 os.path.dirname(to_file)),
2003 item)).replace('\\', '/')
2008 def MergeLists(to, fro, to_file, fro_file, is_paths=False, append=True):
2009 # Python documentation recommends objects which do not support hash
2010 # set this value to None. Python library objects follow this rule.
2011 is_hashable = lambda val: val.__hash__
2013 # If x is hashable, returns whether x is in s. Else returns whether x is in l.
2014 def is_in_set_or_list(x, s, l):
2021 # Make membership testing of hashables in |to| (in particular, strings)
2023 hashable_to_set = set(x for x in to if is_hashable(x))
2026 if type(item) in (str, int):
2027 # The cheap and easy case.
2029 to_item = MakePathRelative(to_file, fro_file, item)
2033 if not (type(item) is str and item.startswith('-')):
2034 # Any string that doesn't begin with a "-" is a singleton - it can
2035 # only appear once in a list, to be enforced by the list merge append
2038 elif type(item) is dict:
2039 # Make a copy of the dictionary, continuing to look for paths to fix.
2040 # The other intelligent aspects of merge processing won't apply because
2041 # item is being merged into an empty dict.
2043 MergeDicts(to_item, item, to_file, fro_file)
2044 elif type(item) is list:
2045 # Recurse, making a copy of the list. If the list contains any
2046 # descendant dicts, path fixing will occur. Note that here, custom
2047 # values for is_paths and append are dropped; those are only to be
2048 # applied to |to| and |fro|, not sublists of |fro|. append shouldn't
2049 # matter anyway because the new |to_item| list is empty.
2051 MergeLists(to_item, item, to_file, fro_file)
2054 'Attempt to merge list item of unsupported type ' + \
2055 item.__class__.__name__
2058 # If appending a singleton that's already in the list, don't append.
2059 # This ensures that the earliest occurrence of the item will stay put.
2060 if not singleton or not is_in_set_or_list(to_item, hashable_to_set, to):
2062 if is_hashable(to_item):
2063 hashable_to_set.add(to_item)
2065 # If prepending a singleton that's already in the list, remove the
2066 # existing instance and proceed with the prepend. This ensures that the
2067 # item appears at the earliest possible position in the list.
2068 while singleton and to_item in to:
2071 # Don't just insert everything at index 0. That would prepend the new
2072 # items to the list in reverse order, which would be an unwelcome
2074 to.insert(prepend_index, to_item)
2075 if is_hashable(to_item):
2076 hashable_to_set.add(to_item)
2077 prepend_index = prepend_index + 1
2080 def MergeDicts(to, fro, to_file, fro_file):
2081 # I wanted to name the parameter "from" but it's a Python keyword...
2082 for k, v in fro.iteritems():
2083 # It would be nice to do "if not k in to: to[k] = v" but that wouldn't give
2084 # copy semantics. Something else may want to merge from the |fro| dict
2085 # later, and having the same dict ref pointed to twice in the tree isn't
2086 # what anyone wants considering that the dicts may subsequently be
2090 if type(v) in (str, int):
2091 if type(to[k]) not in (str, int):
2093 elif type(v) is not type(to[k]):
2098 'Attempt to merge dict value of type ' + v.__class__.__name__ + \
2099 ' into incompatible type ' + to[k].__class__.__name__ + \
2101 if type(v) in (str, int):
2102 # Overwrite the existing value, if any. Cheap and easy.
2103 is_path = IsPathSection(k)
2105 to[k] = MakePathRelative(to_file, fro_file, v)
2108 elif type(v) is dict:
2109 # Recurse, guaranteeing copies will be made of objects that require it.
2112 MergeDicts(to[k], v, to_file, fro_file)
2113 elif type(v) is list:
2114 # Lists in dicts can be merged with different policies, depending on
2115 # how the key in the "from" dict (k, the from-key) is written.
2117 # If the from-key has ...the to-list will have this action
2118 # this character appended:... applied when receiving the from-list:
2121 # ? set, only if to-list does not yet exist
2124 # This logic is list-specific, but since it relies on the associated
2125 # dict key, it's checked in this dict-oriented function.
2130 lists_incompatible = [list_base, list_base + '?']
2134 lists_incompatible = [list_base + '=', list_base + '?']
2138 lists_incompatible = [list_base, list_base + '=', list_base + '+']
2141 lists_incompatible = [list_base + '=', list_base + '?']
2143 # Some combinations of merge policies appearing together are meaningless.
2144 # It's stupid to replace and append simultaneously, for example. Append
2145 # and prepend are the only policies that can coexist.
2146 for list_incompatible in lists_incompatible:
2147 if list_incompatible in fro:
2148 raise GypError('Incompatible list policies ' + k + ' and ' +
2153 # If the key ends in "?", the list will only be merged if it doesn't
2156 elif type(to[list_base]) is not list:
2157 # This may not have been checked above if merging in a list with an
2158 # extension character.
2160 'Attempt to merge dict value of type ' + v.__class__.__name__ + \
2161 ' into incompatible type ' + to[list_base].__class__.__name__ + \
2162 ' for key ' + list_base + '(' + k + ')'
2166 # Call MergeLists, which will make copies of objects that require it.
2167 # MergeLists can recurse back into MergeDicts, although this will be
2168 # to make copies of dicts (with paths fixed), there will be no
2169 # subsequent dict "merging" once entering a list because lists are
2170 # always replaced, appended to, or prepended to.
2171 is_paths = IsPathSection(list_base)
2172 MergeLists(to[list_base], v, to_file, fro_file, is_paths, append)
2175 'Attempt to merge dict value of unsupported type ' + \
2176 v.__class__.__name__ + ' for key ' + k
2179 def MergeConfigWithInheritance(new_configuration_dict, build_file,
2180 target_dict, configuration, visited):
2181 # Skip if previously visted.
2182 if configuration in visited:
2185 # Look at this configuration.
2186 configuration_dict = target_dict['configurations'][configuration]
2189 for parent in configuration_dict.get('inherit_from', []):
2190 MergeConfigWithInheritance(new_configuration_dict, build_file,
2191 target_dict, parent, visited + [configuration])
2193 # Merge it into the new config.
2194 MergeDicts(new_configuration_dict, configuration_dict,
2195 build_file, build_file)
2198 if 'abstract' in new_configuration_dict:
2199 del new_configuration_dict['abstract']
2202 def SetUpConfigurations(target, target_dict):
2203 # key_suffixes is a list of key suffixes that might appear on key names.
2204 # These suffixes are handled in conditional evaluations (for =, +, and ?)
2205 # and rules/exclude processing (for ! and /). Keys with these suffixes
2206 # should be treated the same as keys without.
2207 key_suffixes = ['=', '+', '?', '!', '/']
2209 build_file = gyp.common.BuildFile(target)
2211 # Provide a single configuration by default if none exists.
2212 # TODO(mark): Signal an error if default_configurations exists but
2213 # configurations does not.
2214 if not 'configurations' in target_dict:
2215 target_dict['configurations'] = {'Default': {}}
2216 if not 'default_configuration' in target_dict:
2217 concrete = [i for (i, config) in target_dict['configurations'].iteritems()
2218 if not config.get('abstract')]
2219 target_dict['default_configuration'] = sorted(concrete)[0]
2221 merged_configurations = {}
2222 configs = target_dict['configurations']
2223 for (configuration, old_configuration_dict) in configs.iteritems():
2224 # Skip abstract configurations (saves work only).
2225 if old_configuration_dict.get('abstract'):
2227 # Configurations inherit (most) settings from the enclosing target scope.
2228 # Get the inheritance relationship right by making a copy of the target
2230 new_configuration_dict = {}
2231 for (key, target_val) in target_dict.iteritems():
2233 if key_ext in key_suffixes:
2237 if not key_base in non_configuration_keys:
2238 new_configuration_dict[key] = gyp.simple_copy.deepcopy(target_val)
2240 # Merge in configuration (with all its parents first).
2241 MergeConfigWithInheritance(new_configuration_dict, build_file,
2242 target_dict, configuration, [])
2244 merged_configurations[configuration] = new_configuration_dict
2246 # Put the new configurations back into the target dict as a configuration.
2247 for configuration in merged_configurations.keys():
2248 target_dict['configurations'][configuration] = (
2249 merged_configurations[configuration])
2251 # Now drop all the abstract ones.
2252 for configuration in target_dict['configurations'].keys():
2253 old_configuration_dict = target_dict['configurations'][configuration]
2254 if old_configuration_dict.get('abstract'):
2255 del target_dict['configurations'][configuration]
2257 # Now that all of the target's configurations have been built, go through
2258 # the target dict's keys and remove everything that's been moved into a
2259 # "configurations" section.
2261 for key in target_dict:
2263 if key_ext in key_suffixes:
2267 if not key_base in non_configuration_keys:
2268 delete_keys.append(key)
2269 for key in delete_keys:
2270 del target_dict[key]
2272 # Check the configurations to see if they contain invalid keys.
2273 for configuration in target_dict['configurations'].keys():
2274 configuration_dict = target_dict['configurations'][configuration]
2275 for key in configuration_dict.keys():
2276 if key in invalid_configuration_keys:
2277 raise GypError('%s not allowed in the %s configuration, found in '
2278 'target %s' % (key, configuration, target))
2282 def ProcessListFiltersInDict(name, the_dict):
2283 """Process regular expression and exclusion-based filters on lists.
2285 An exclusion list is in a dict key named with a trailing "!", like
2286 "sources!". Every item in such a list is removed from the associated
2287 main list, which in this example, would be "sources". Removed items are
2288 placed into a "sources_excluded" list in the dict.
2290 Regular expression (regex) filters are contained in dict keys named with a
2291 trailing "/", such as "sources/" to operate on the "sources" list. Regex
2292 filters in a dict take the form:
2293 'sources/': [ ['exclude', '_(linux|mac|win)\\.cc$'],
2294 ['include', '_mac\\.cc$'] ],
2295 The first filter says to exclude all files ending in _linux.cc, _mac.cc, and
2296 _win.cc. The second filter then includes all files ending in _mac.cc that
2297 are now or were once in the "sources" list. Items matching an "exclude"
2298 filter are subject to the same processing as would occur if they were listed
2299 by name in an exclusion list (ending in "!"). Items matching an "include"
2300 filter are brought back into the main list if previously excluded by an
2301 exclusion list or exclusion regex filter. Subsequent matching "exclude"
2302 patterns can still cause items to be excluded after matching an "include".
2305 # Look through the dictionary for any lists whose keys end in "!" or "/".
2306 # These are lists that will be treated as exclude lists and regular
2307 # expression-based exclude/include lists. Collect the lists that are
2308 # needed first, looking for the lists that they operate on, and assemble
2309 # then into |lists|. This is done in a separate loop up front, because
2310 # the _included and _excluded keys need to be added to the_dict, and that
2311 # can't be done while iterating through it.
2315 for key, value in the_dict.iteritems():
2317 if operation != '!' and operation != '/':
2320 if type(value) is not list:
2321 raise ValueError, name + ' key ' + key + ' must be list, not ' + \
2322 value.__class__.__name__
2325 if list_key not in the_dict:
2326 # This happens when there's a list like "sources!" but no corresponding
2327 # "sources" list. Since there's nothing for it to operate on, queue up
2328 # the "sources!" list for deletion now.
2329 del_lists.append(key)
2332 if type(the_dict[list_key]) is not list:
2333 value = the_dict[list_key]
2334 raise ValueError, name + ' key ' + list_key + \
2335 ' must be list, not ' + \
2336 value.__class__.__name__ + ' when applying ' + \
2337 {'!': 'exclusion', '/': 'regex'}[operation]
2339 if not list_key in lists:
2340 lists.append(list_key)
2342 # Delete the lists that are known to be unneeded at this point.
2343 for del_list in del_lists:
2344 del the_dict[del_list]
2346 for list_key in lists:
2347 the_list = the_dict[list_key]
2349 # Initialize the list_actions list, which is parallel to the_list. Each
2350 # item in list_actions identifies whether the corresponding item in
2351 # the_list should be excluded, unconditionally preserved (included), or
2352 # whether no exclusion or inclusion has been applied. Items for which
2353 # no exclusion or inclusion has been applied (yet) have value -1, items
2354 # excluded have value 0, and items included have value 1. Includes and
2355 # excludes override previous actions. All items in list_actions are
2356 # initialized to -1 because no excludes or includes have been processed
2358 list_actions = list((-1,) * len(the_list))
2360 exclude_key = list_key + '!'
2361 if exclude_key in the_dict:
2362 for exclude_item in the_dict[exclude_key]:
2363 for index in xrange(0, len(the_list)):
2364 if exclude_item == the_list[index]:
2365 # This item matches the exclude_item, so set its action to 0
2367 list_actions[index] = 0
2369 # The "whatever!" list is no longer needed, dump it.
2370 del the_dict[exclude_key]
2372 regex_key = list_key + '/'
2373 if regex_key in the_dict:
2374 for regex_item in the_dict[regex_key]:
2375 [action, pattern] = regex_item
2376 pattern_re = re.compile(pattern)
2378 if action == 'exclude':
2379 # This item matches an exclude regex, so set its value to 0 (exclude).
2381 elif action == 'include':
2382 # This item matches an include regex, so set its value to 1 (include).
2385 # This is an action that doesn't make any sense.
2386 raise ValueError, 'Unrecognized action ' + action + ' in ' + name + \
2389 for index in xrange(0, len(the_list)):
2390 list_item = the_list[index]
2391 if list_actions[index] == action_value:
2392 # Even if the regex matches, nothing will change so continue (regex
2393 # searches are expensive).
2395 if pattern_re.search(list_item):
2396 # Regular expression match.
2397 list_actions[index] = action_value
2399 # The "whatever/" list is no longer needed, dump it.
2400 del the_dict[regex_key]
2402 # Add excluded items to the excluded list.
2404 # Note that exclude_key ("sources!") is different from excluded_key
2405 # ("sources_excluded"). The exclude_key list is input and it was already
2406 # processed and deleted; the excluded_key list is output and it's about
2408 excluded_key = list_key + '_excluded'
2409 if excluded_key in the_dict:
2410 raise GypError(name + ' key ' + excluded_key +
2411 ' must not be present prior '
2412 ' to applying exclusion/regex filters for ' + list_key)
2416 # Go backwards through the list_actions list so that as items are deleted,
2417 # the indices of items that haven't been seen yet don't shift. That means
2418 # that things need to be prepended to excluded_list to maintain them in the
2419 # same order that they existed in the_list.
2420 for index in xrange(len(list_actions) - 1, -1, -1):
2421 if list_actions[index] == 0:
2422 # Dump anything with action 0 (exclude). Keep anything with action 1
2423 # (include) or -1 (no include or exclude seen for the item).
2424 excluded_list.insert(0, the_list[index])
2427 # If anything was excluded, put the excluded list into the_dict at
2429 if len(excluded_list) > 0:
2430 the_dict[excluded_key] = excluded_list
2432 # Now recurse into subdicts and lists that may contain dicts.
2433 for key, value in the_dict.iteritems():
2434 if type(value) is dict:
2435 ProcessListFiltersInDict(key, value)
2436 elif type(value) is list:
2437 ProcessListFiltersInList(key, value)
2440 def ProcessListFiltersInList(name, the_list):
2441 for item in the_list:
2442 if type(item) is dict:
2443 ProcessListFiltersInDict(name, item)
2444 elif type(item) is list:
2445 ProcessListFiltersInList(name, item)
2448 def ValidateTargetType(target, target_dict):
2449 """Ensures the 'type' field on the target is one of the known types.
2452 target: string, name of target.
2453 target_dict: dict, target spec.
2455 Raises an exception on error.
2457 VALID_TARGET_TYPES = ('executable', 'loadable_module',
2458 'static_library', 'shared_library',
2460 target_type = target_dict.get('type', None)
2461 if target_type not in VALID_TARGET_TYPES:
2462 raise GypError("Target %s has an invalid target type '%s'. "
2463 "Must be one of %s." %
2464 (target, target_type, '/'.join(VALID_TARGET_TYPES)))
2465 if (target_dict.get('standalone_static_library', 0) and
2466 not target_type == 'static_library'):
2467 raise GypError('Target %s has type %s but standalone_static_library flag is'
2468 ' only valid for static_library type.' % (target,
2472 def ValidateSourcesInTarget(target, target_dict, build_file,
2473 duplicate_basename_check):
2474 if not duplicate_basename_check:
2476 # TODO: Check if MSVC allows this for loadable_module targets.
2477 if target_dict.get('type', None) not in ('static_library', 'shared_library'):
2479 sources = target_dict.get('sources', [])
2481 for source in sources:
2482 name, ext = os.path.splitext(source)
2483 is_compiled_file = ext in [
2484 '.c', '.cc', '.cpp', '.cxx', '.m', '.mm', '.s', '.S']
2485 if not is_compiled_file:
2487 basename = os.path.basename(name) # Don't include extension.
2488 basenames.setdefault(basename, []).append(source)
2491 for basename, files in basenames.iteritems():
2493 error += ' %s: %s\n' % (basename, ' '.join(files))
2496 print('static library %s has several files with the same basename:\n' %
2497 target + error + 'Some build systems, e.g. MSVC08 and Make generator '
2498 'for Mac, cannot handle that. Use --no-duplicate-basename-check to'
2499 'disable this validation.')
2500 raise GypError('Duplicate basenames in sources section, see list above')
2503 def ValidateRulesInTarget(target, target_dict, extra_sources_for_rules):
2504 """Ensures that the rules sections in target_dict are valid and consistent,
2505 and determines which sources they apply to.
2508 target: string, name of target.
2509 target_dict: dict, target spec containing "rules" and "sources" lists.
2510 extra_sources_for_rules: a list of keys to scan for rule matches in
2511 addition to 'sources'.
2514 # Dicts to map between values found in rules' 'rule_name' and 'extension'
2515 # keys and the rule dicts themselves.
2517 rule_extensions = {}
2519 rules = target_dict.get('rules', [])
2521 # Make sure that there's no conflict among rule names and extensions.
2522 rule_name = rule['rule_name']
2523 if rule_name in rule_names:
2524 raise GypError('rule %s exists in duplicate, target %s' %
2525 (rule_name, target))
2526 rule_names[rule_name] = rule
2528 rule_extension = rule['extension']
2529 if rule_extension.startswith('.'):
2530 rule_extension = rule_extension[1:]
2531 if rule_extension in rule_extensions:
2532 raise GypError(('extension %s associated with multiple rules, ' +
2533 'target %s rules %s and %s') %
2534 (rule_extension, target,
2535 rule_extensions[rule_extension]['rule_name'],
2537 rule_extensions[rule_extension] = rule
2539 # Make sure rule_sources isn't already there. It's going to be
2540 # created below if needed.
2541 if 'rule_sources' in rule:
2543 'rule_sources must not exist in input, target %s rule %s' %
2544 (target, rule_name))
2547 source_keys = ['sources']
2548 source_keys.extend(extra_sources_for_rules)
2549 for source_key in source_keys:
2550 for source in target_dict.get(source_key, []):
2551 (source_root, source_extension) = os.path.splitext(source)
2552 if source_extension.startswith('.'):
2553 source_extension = source_extension[1:]
2554 if source_extension == rule_extension:
2555 rule_sources.append(source)
2557 if len(rule_sources) > 0:
2558 rule['rule_sources'] = rule_sources
2561 def ValidateRunAsInTarget(target, target_dict, build_file):
2562 target_name = target_dict.get('target_name')
2563 run_as = target_dict.get('run_as')
2566 if type(run_as) is not dict:
2567 raise GypError("The 'run_as' in target %s from file %s should be a "
2569 (target_name, build_file))
2570 action = run_as.get('action')
2572 raise GypError("The 'run_as' in target %s from file %s must have an "
2573 "'action' section." %
2574 (target_name, build_file))
2575 if type(action) is not list:
2576 raise GypError("The 'action' for 'run_as' in target %s from file %s "
2578 (target_name, build_file))
2579 working_directory = run_as.get('working_directory')
2580 if working_directory and type(working_directory) is not str:
2581 raise GypError("The 'working_directory' for 'run_as' in target %s "
2582 "in file %s should be a string." %
2583 (target_name, build_file))
2584 environment = run_as.get('environment')
2585 if environment and type(environment) is not dict:
2586 raise GypError("The 'environment' for 'run_as' in target %s "
2587 "in file %s should be a dictionary." %
2588 (target_name, build_file))
2591 def ValidateActionsInTarget(target, target_dict, build_file):
2592 '''Validates the inputs to the actions in a target.'''
2593 target_name = target_dict.get('target_name')
2594 actions = target_dict.get('actions', [])
2595 for action in actions:
2596 action_name = action.get('action_name')
2598 raise GypError("Anonymous action in target %s. "
2599 "An action must have an 'action_name' field." %
2601 inputs = action.get('inputs', None)
2603 raise GypError('Action in target %s has no inputs.' % target_name)
2604 action_command = action.get('action')
2605 if action_command and not action_command[0]:
2606 raise GypError("Empty action as command in target %s." % target_name)
2609 def TurnIntIntoStrInDict(the_dict):
2610 """Given dict the_dict, recursively converts all integers into strings.
2612 # Use items instead of iteritems because there's no need to try to look at
2613 # reinserted keys and their associated values.
2614 for k, v in the_dict.items():
2618 elif type(v) is dict:
2619 TurnIntIntoStrInDict(v)
2620 elif type(v) is list:
2621 TurnIntIntoStrInList(v)
2625 the_dict[str(k)] = v
2628 def TurnIntIntoStrInList(the_list):
2629 """Given list the_list, recursively converts all integers into strings.
2631 for index in xrange(0, len(the_list)):
2632 item = the_list[index]
2633 if type(item) is int:
2634 the_list[index] = str(item)
2635 elif type(item) is dict:
2636 TurnIntIntoStrInDict(item)
2637 elif type(item) is list:
2638 TurnIntIntoStrInList(item)
2641 def PruneUnwantedTargets(targets, flat_list, dependency_nodes, root_targets,
2643 """Return only the targets that are deep dependencies of |root_targets|."""
2644 qualified_root_targets = []
2645 for target in root_targets:
2646 target = target.strip()
2647 qualified_targets = gyp.common.FindQualifiedTargets(target, flat_list)
2648 if not qualified_targets:
2649 raise GypError("Could not find target %s" % target)
2650 qualified_root_targets.extend(qualified_targets)
2653 for target in qualified_root_targets:
2654 wanted_targets[target] = targets[target]
2655 for dependency in dependency_nodes[target].DeepDependencies():
2656 wanted_targets[dependency] = targets[dependency]
2658 wanted_flat_list = [t for t in flat_list if t in wanted_targets]
2660 # Prune unwanted targets from each build_file's data dict.
2661 for build_file in data['target_build_files']:
2662 if not 'targets' in data[build_file]:
2665 for target in data[build_file]['targets']:
2666 qualified_name = gyp.common.QualifiedTarget(build_file,
2667 target['target_name'],
2669 if qualified_name in wanted_targets:
2670 new_targets.append(target)
2671 data[build_file]['targets'] = new_targets
2673 return wanted_targets, wanted_flat_list
2676 def VerifyNoCollidingTargets(targets):
2677 """Verify that no two targets in the same directory share the same name.
2680 targets: A list of targets in the form 'path/to/file.gyp:target_name'.
2682 # Keep a dict going from 'subdirectory:target_name' to 'foo.gyp'.
2684 for target in targets:
2685 # Separate out 'path/to/file.gyp, 'target_name' from
2686 # 'path/to/file.gyp:target_name'.
2687 path, name = target.rsplit(':', 1)
2688 # Separate out 'path/to', 'file.gyp' from 'path/to/file.gyp'.
2689 subdir, gyp = os.path.split(path)
2690 # Use '.' for the current directory '', so that the error messages make
2694 # Prepare a key like 'path/to:target_name'.
2695 key = subdir + ':' + name
2697 # Complain if this target is already used.
2698 raise GypError('Duplicate target name "%s" in directory "%s" used both '
2699 'in "%s" and "%s".' % (name, subdir, gyp, used[key]))
2703 def SetGeneratorGlobals(generator_input_info):
2704 # Set up path_sections and non_configuration_keys with the default data plus
2705 # the generator-specific data.
2706 global path_sections
2707 path_sections = set(base_path_sections)
2708 path_sections.update(generator_input_info['path_sections'])
2710 global non_configuration_keys
2711 non_configuration_keys = base_non_configuration_keys[:]
2712 non_configuration_keys.extend(generator_input_info['non_configuration_keys'])
2714 global multiple_toolsets
2715 multiple_toolsets = generator_input_info[
2716 'generator_supports_multiple_toolsets']
2718 global generator_filelist_paths
2719 generator_filelist_paths = generator_input_info['generator_filelist_paths']
2722 def Load(build_files, variables, includes, depth, generator_input_info, check,
2723 circular_check, duplicate_basename_check, parallel, root_targets):
2724 SetGeneratorGlobals(generator_input_info)
2725 # A generator can have other lists (in addition to sources) be processed
2727 extra_sources_for_rules = generator_input_info['extra_sources_for_rules']
2729 # Load build files. This loads every target-containing build file into
2730 # the |data| dictionary such that the keys to |data| are build file names,
2731 # and the values are the entire build file contents after "early" or "pre"
2732 # processing has been done and includes have been resolved.
2733 # NOTE: data contains both "target" files (.gyp) and "includes" (.gypi), as
2734 # well as meta-data (e.g. 'included_files' key). 'target_build_files' keeps
2735 # track of the keys corresponding to "target" files.
2736 data = {'target_build_files': set()}
2738 # Normalize paths everywhere. This is important because paths will be
2739 # used as keys to the data dict and for references between input files.
2740 build_files = set(map(os.path.normpath, build_files))
2742 LoadTargetBuildFilesParallel(build_files, data, aux_data,
2743 variables, includes, depth, check,
2744 generator_input_info)
2746 for build_file in build_files:
2748 LoadTargetBuildFile(build_file, data, aux_data,
2749 variables, includes, depth, check, True)
2750 except Exception, e:
2751 gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file)
2754 # Build a dict to access each target's subdict by qualified name.
2755 targets = BuildTargetsDict(data)
2757 # Fully qualify all dependency links.
2758 QualifyDependencies(targets)
2760 # Remove self-dependencies from targets that have 'prune_self_dependencies'
2762 RemoveSelfDependencies(targets)
2764 # Expand dependencies specified as build_file:*.
2765 ExpandWildcardDependencies(targets, data)
2767 # Remove all dependencies marked as 'link_dependency' from the targets of
2769 RemoveLinkDependenciesFromNoneTargets(targets)
2771 # Apply exclude (!) and regex (/) list filters only for dependency_sections.
2772 for target_name, target_dict in targets.iteritems():
2774 for key_base in dependency_sections:
2775 for op in ('', '!', '/'):
2777 if key in target_dict:
2778 tmp_dict[key] = target_dict[key]
2779 del target_dict[key]
2780 ProcessListFiltersInDict(target_name, tmp_dict)
2781 # Write the results back to |target_dict|.
2782 for key in tmp_dict:
2783 target_dict[key] = tmp_dict[key]
2785 # Make sure every dependency appears at most once.
2786 RemoveDuplicateDependencies(targets)
2789 # Make sure that any targets in a.gyp don't contain dependencies in other
2790 # .gyp files that further depend on a.gyp.
2791 VerifyNoGYPFileCircularDependencies(targets)
2793 [dependency_nodes, flat_list] = BuildDependencyList(targets)
2796 # Remove, from |targets| and |flat_list|, the targets that are not deep
2797 # dependencies of the targets specified in |root_targets|.
2798 targets, flat_list = PruneUnwantedTargets(
2799 targets, flat_list, dependency_nodes, root_targets, data)
2801 # Check that no two targets in the same directory have the same name.
2802 VerifyNoCollidingTargets(flat_list)
2804 # Handle dependent settings of various types.
2805 for settings_type in ['all_dependent_settings',
2806 'direct_dependent_settings',
2808 DoDependentSettings(settings_type, flat_list, targets, dependency_nodes)
2810 # Take out the dependent settings now that they've been published to all
2811 # of the targets that require them.
2812 for target in flat_list:
2813 if settings_type in targets[target]:
2814 del targets[target][settings_type]
2816 # Make sure static libraries don't declare dependencies on other static
2817 # libraries, but that linkables depend on all unlinked static libraries
2818 # that they need so that their link steps will be correct.
2819 gii = generator_input_info
2820 if gii['generator_wants_static_library_dependencies_adjusted']:
2821 AdjustStaticLibraryDependencies(flat_list, targets, dependency_nodes,
2822 gii['generator_wants_sorted_dependencies'])
2824 # Apply "post"/"late"/"target" variable expansions and condition evaluations.
2825 for target in flat_list:
2826 target_dict = targets[target]
2827 build_file = gyp.common.BuildFile(target)
2828 ProcessVariablesAndConditionsInDict(
2829 target_dict, PHASE_LATE, variables, build_file)
2831 # Move everything that can go into a "configurations" section into one.
2832 for target in flat_list:
2833 target_dict = targets[target]
2834 SetUpConfigurations(target, target_dict)
2836 # Apply exclude (!) and regex (/) list filters.
2837 for target in flat_list:
2838 target_dict = targets[target]
2839 ProcessListFiltersInDict(target, target_dict)
2841 # Apply "latelate" variable expansions and condition evaluations.
2842 for target in flat_list:
2843 target_dict = targets[target]
2844 build_file = gyp.common.BuildFile(target)
2845 ProcessVariablesAndConditionsInDict(
2846 target_dict, PHASE_LATELATE, variables, build_file)
2848 # TODO(thakis): Get vpx_scale/arm/scalesystemdependent.c to be renamed to
2849 # scalesystemdependent_arm_additions.c or similar.
2850 if 'arm' in variables.get('target_arch', ''):
2851 duplicate_basename_check = False
2853 # Make sure that the rules make sense, and build up rule_sources lists as
2854 # needed. Not all generators will need to use the rule_sources lists, but
2855 # some may, and it seems best to build the list in a common spot.
2856 # Also validate actions and run_as elements in targets.
2857 for target in flat_list:
2858 target_dict = targets[target]
2859 build_file = gyp.common.BuildFile(target)
2860 ValidateTargetType(target, target_dict)
2861 ValidateSourcesInTarget(target, target_dict, build_file,
2862 duplicate_basename_check)
2863 ValidateRulesInTarget(target, target_dict, extra_sources_for_rules)
2864 ValidateRunAsInTarget(target, target_dict, build_file)
2865 ValidateActionsInTarget(target, target_dict, build_file)
2867 # Generators might not expect ints. Turn them into strs.
2868 TurnIntIntoStrInDict(data)
2870 # TODO(mark): Return |data| for now because the generator needs a list of
2871 # build files that came in. In the future, maybe it should just accept
2872 # a list, and not the whole data dict.
2873 return [flat_list, targets, data]