2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Meta checkout manager supporting both Subversion and GIT."""
8 # .gclient : Current client configuration, written by 'config' command.
9 # Format is a Python script defining 'solutions', a list whose
10 # entries each are maps binding the strings "name" and "url"
11 # to strings specifying the name and location of the client
12 # module, as well as "custom_deps" to a map similar to the
13 # deps section of the DEPS file below, as well as
14 # "custom_hooks" to a list similar to the hooks sections of
15 # the DEPS file below.
16 # .gclient_entries : A cache constructed by 'update' command. Format is a
17 # Python script defining 'entries', a list of the names
18 # of all modules in the client
19 # <module>/DEPS : Python script defining var 'deps' as a map from each
20 # requisite submodule name to a URL where it can be found (via
24 # .gclient and DEPS files may optionally contain a list named "hooks" to
25 # allow custom actions to be performed based on files that have changed in the
26 # working copy as a result of a "sync"/"update" or "revert" operation. This
27 # can be prevented by using --nohooks (hooks run by default). Hooks can also
28 # be forced to run with the "runhooks" operation. If "sync" is run with
29 # --force, all known but not suppressed hooks will run regardless of the state
30 # of the working copy.
32 # Each item in a "hooks" list is a dict, containing these two keys:
33 # "pattern" The associated value is a string containing a regular
34 # expression. When a file whose pathname matches the expression
35 # is checked out, updated, or reverted, the hook's "action" will
37 # "action" A list describing a command to run along with its arguments, if
38 # any. An action command will run at most one time per gclient
39 # invocation, regardless of how many files matched the pattern.
40 # The action is executed in the same directory as the .gclient
41 # file. If the first item in the list is the string "python",
42 # the current Python interpreter (sys.executable) will be used
43 # to run the command. If the list contains string
44 # "$matching_files" it will be removed from the list and the list
45 # will be extended by the list of matching files.
46 # "name" An optional string specifying the group to which a hook belongs
47 # for overriding and organizing.
51 # { "pattern": "\\.(gif|jpe?g|pr0n|png)$",
52 # "action": ["python", "image_indexer.py", "--all"]},
55 # "action": ["python", "src/build/gyp_chromium"]},
59 # DEPS files may optionally contain a list named "pre_deps_hooks". These are
60 # the same as normal hooks, except that they run before the DEPS are
61 # processed. Pre-DEPS run with "sync" and "revert" unless the --noprehooks
64 # Specifying a target OS
65 # An optional key named "target_os" may be added to a gclient file to specify
66 # one or more additional operating systems that should be considered when
67 # processing the deps_os dict of a DEPS file.
70 # target_os = [ "android" ]
72 # If the "target_os_only" key is also present and true, then *only* the
73 # operating systems listed in "target_os" will be used.
76 # target_os = [ "ios" ]
77 # target_os_only = True
96 import breakpad # pylint: disable=W0611
102 from third_party.repo.progress import Progress
105 from third_party import colorama
107 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src.git'
110 def ast_dict_index(dnode, key):
111 """Search an ast.Dict for the argument key, and return its index."""
112 idx = [i for i in range(len(dnode.keys)) if (
113 type(dnode.keys[i]) is ast.Str and dnode.keys[i].s == key)]
117 raise gclient_utils.Error('Multiple dict entries with same key in AST')
120 def ast2str(node, indent=0):
121 """Return a pretty-printed rendition of an ast.Node."""
124 return '\n'.join([ast2str(x, indent) for x in node.body])
125 elif t is ast.Assign:
126 return ((' ' * indent) +
127 ' = '.join([ast2str(x) for x in node.targets] +
128 [ast2str(node.value, indent)]) + '\n')
134 elif len(node.elts) == 1:
135 return '[' + ast2str(node.elts[0], indent) + ']'
136 return ('[\n' + (' ' * (indent + 1)) +
137 (',\n' + (' ' * (indent + 1))).join(
138 [ast2str(x, indent + 1) for x in node.elts]) +
139 '\n' + (' ' * indent) + ']')
143 elif len(node.keys) == 1:
144 return '{%s: %s}' % (ast2str(node.keys[0]),
145 ast2str(node.values[0], indent + 1))
146 return ('{\n' + (' ' * (indent + 1)) +
147 (',\n' + (' ' * (indent + 1))).join(
148 ['%s: %s' % (ast2str(node.keys[i]),
149 ast2str(node.values[i], indent + 1))
150 for i in range(len(node.keys))]) +
151 '\n' + (' ' * indent) + '}')
153 return "'%s'" % node.s
155 raise gclient_utils.Error("Unexpected AST node at line %d, column %d: %s"
156 % (node.lineno, node.col_offset, t))
159 class GClientKeywords(object):
160 class FromImpl(object):
161 """Used to implement the From() syntax."""
163 def __init__(self, module_name, sub_target_name=None):
164 """module_name is the dep module we want to include from. It can also be
165 the name of a subdirectory to include from.
167 sub_target_name is an optional parameter if the module name in the other
168 DEPS file is different. E.g., you might want to map src/net to net."""
169 self.module_name = module_name
170 self.sub_target_name = sub_target_name
173 return 'From(%s, %s)' % (repr(self.module_name),
174 repr(self.sub_target_name))
176 class FileImpl(object):
177 """Used to implement the File('') syntax which lets you sync a single file
180 def __init__(self, file_location):
181 self.file_location = file_location
184 return 'File("%s")' % self.file_location
187 return os.path.split(self.file_location)[0]
189 def GetFilename(self):
190 rev_tokens = self.file_location.split('@')
191 return os.path.split(rev_tokens[0])[1]
193 def GetRevision(self):
194 rev_tokens = self.file_location.split('@')
195 if len(rev_tokens) > 1:
199 class VarImpl(object):
200 def __init__(self, custom_vars, local_scope):
201 self._custom_vars = custom_vars
202 self._local_scope = local_scope
204 def Lookup(self, var_name):
205 """Implements the Var syntax."""
206 if var_name in self._custom_vars:
207 return self._custom_vars[var_name]
208 elif var_name in self._local_scope.get("vars", {}):
209 return self._local_scope["vars"][var_name]
210 raise gclient_utils.Error("Var is not defined: %s" % var_name)
213 class DependencySettings(GClientKeywords):
214 """Immutable configuration settings."""
216 self, parent, url, safesync_url, managed, custom_deps, custom_vars,
217 custom_hooks, deps_file, should_process):
218 GClientKeywords.__init__(self)
220 # These are not mutable:
221 self._parent = parent
222 self._safesync_url = safesync_url
223 self._deps_file = deps_file
225 # 'managed' determines whether or not this dependency is synced/updated by
226 # gclient after gclient checks it out initially. The difference between
227 # 'managed' and 'should_process' is that the user specifies 'managed' via
228 # the --unmanaged command-line flag or a .gclient config, where
229 # 'should_process' is dynamically set by gclient if it goes over its
230 # recursion limit and controls gclient's behavior so it does not misbehave.
231 self._managed = managed
232 self._should_process = should_process
233 # This is a mutable value which has the list of 'target_os' OSes listed in
234 # the current deps file.
235 self.local_target_os = None
237 # These are only set in .gclient and not in DEPS files.
238 self._custom_vars = custom_vars or {}
239 self._custom_deps = custom_deps or {}
240 self._custom_hooks = custom_hooks or []
242 # TODO(iannucci): Remove this when all masters are correctly substituting
244 if (self._custom_vars.get('webkit_trunk', '') ==
245 'svn://svn-mirror.golo.chromium.org/webkit-readonly/trunk'):
246 new_url = 'svn://svn-mirror.golo.chromium.org/blink/trunk'
247 print 'Overwriting Var("webkit_trunk") with %s' % new_url
248 self._custom_vars['webkit_trunk'] = new_url
250 # Post process the url to remove trailing slashes.
251 if isinstance(self._url, basestring):
252 # urls are sometime incorrectly written as proto://host/path/@rev. Replace
253 # it to proto://host/path@rev.
254 self._url = self._url.replace('/@', '@')
255 elif not isinstance(self._url,
256 (self.FromImpl, self.FileImpl, None.__class__)):
257 raise gclient_utils.Error(
258 ('dependency url must be either a string, None, '
259 'File() or From() instead of %s') % self._url.__class__.__name__)
260 # Make any deps_file path platform-appropriate.
261 for sep in ['/', '\\']:
262 self._deps_file = self._deps_file.replace(sep, os.sep)
266 return self._deps_file
278 """Returns the root node, a GClient object."""
280 # This line is to signal pylint that it could be a GClient instance.
281 return self or GClient(None, None)
282 return self.parent.root
285 def safesync_url(self):
286 return self._safesync_url
289 def should_process(self):
290 """True if this dependency should be processed, i.e. checked out."""
291 return self._should_process
294 def custom_vars(self):
295 return self._custom_vars.copy()
298 def custom_deps(self):
299 return self._custom_deps.copy()
302 def custom_hooks(self):
303 return self._custom_hooks[:]
311 if self.local_target_os is not None:
312 return tuple(set(self.local_target_os).union(self.parent.target_os))
314 return self.parent.target_os
316 def get_custom_deps(self, name, url):
317 """Returns a custom deps if applicable."""
319 url = self.parent.get_custom_deps(name, url)
320 # None is a valid return value to disable a dependency.
321 return self.custom_deps.get(name, url)
324 class Dependency(gclient_utils.WorkItem, DependencySettings):
325 """Object that represents a dependency checkout."""
327 def __init__(self, parent, name, url, safesync_url, managed, custom_deps,
328 custom_vars, custom_hooks, deps_file, should_process):
329 gclient_utils.WorkItem.__init__(self, name)
330 DependencySettings.__init__(
331 self, parent, url, safesync_url, managed, custom_deps, custom_vars,
332 custom_hooks, deps_file, should_process)
334 # This is in both .gclient and DEPS files:
335 self._deps_hooks = []
337 self._pre_deps_hooks = []
339 # Calculates properties:
340 self._parsed_url = None
341 self._dependencies = []
342 # A cache of the files affected by the current operation, necessary for
345 # List of host names from which dependencies are allowed.
346 # Default is an empty set, meaning unspecified in DEPS file, and hence all
347 # hosts will be allowed. Non-empty set means whitelist of hosts.
348 # allowed_hosts var is scoped to its DEPS file, and so it isn't recursive.
349 self._allowed_hosts = frozenset()
350 # If it is not set to True, the dependency wasn't processed for its child
351 # dependency, i.e. its DEPS wasn't read.
352 self._deps_parsed = False
353 # This dependency has been processed, i.e. checked out
354 self._processed = False
355 # This dependency had its pre-DEPS hooks run
356 self._pre_deps_hooks_ran = False
357 # This dependency had its hook run
358 self._hooks_ran = False
359 # This is the scm used to checkout self.url. It may be used by dependencies
360 # to get the datetime of the revision we checked out.
361 self._used_scm = None
362 self._used_revision = None
363 # The actual revision we ended up getting, or None if that information is
365 self._got_revision = None
367 # This is a mutable value that overrides the normal recursion limit for this
368 # dependency. It is read from the actual DEPS file so cannot be set on
369 # class instantiation.
370 self.recursion_override = None
371 # recursedeps is a mutable value that selectively overrides the default
372 # 'no recursion' setting on a dep-by-dep basis. It will replace
373 # recursion_override.
374 self.recursedeps = None
376 if not self.name and self.parent:
377 raise gclient_utils.Error('Dependency without name')
380 def requirements(self):
381 """Calculate the list of requirements."""
383 # self.parent is implicitly a requirement. This will be recursive by
385 if self.parent and self.parent.name:
386 requirements.add(self.parent.name)
388 # For a tree with at least 2 levels*, the leaf node needs to depend
389 # on the level higher up in an orderly way.
390 # This becomes messy for >2 depth as the DEPS file format is a dictionary,
391 # thus unsorted, while the .gclient format is a list thus sorted.
393 # * _recursion_limit is hard coded 2 and there is no hope to change this
396 # Interestingly enough, the following condition only works in the case we
397 # want: self is a 2nd level node. 3nd level node wouldn't need this since
398 # they already have their parent as a requirement.
399 if self.parent and self.parent.parent and not self.parent.parent.parent:
400 requirements |= set(i.name for i in self.root.dependencies if i.name)
402 if isinstance(self.url, self.FromImpl):
403 requirements.add(self.url.module_name)
407 obj.name for obj in self.root.subtree(False)
410 self.name.startswith(posixpath.join(obj.name, ''))))
411 requirements = tuple(sorted(requirements))
412 logging.info('Dependency(%s).requirements = %s' % (self.name, requirements))
416 def try_recursedeps(self):
417 """Returns False if recursion_override is ever specified."""
418 if self.recursion_override is not None:
420 return self.parent.try_recursedeps
423 def recursion_limit(self):
424 """Returns > 0 if this dependency is not too recursed to be processed."""
425 # We continue to support the absence of recursedeps until tools and DEPS
426 # using recursion_override are updated.
427 if self.try_recursedeps and self.parent.recursedeps != None:
428 if self.name in self.parent.recursedeps:
431 if self.recursion_override is not None:
432 return self.recursion_override
433 return max(self.parent.recursion_limit - 1, 0)
435 def verify_validity(self):
436 """Verifies that this Dependency is fine to add as a child of another one.
438 Returns True if this entry should be added, False if it is a duplicate of
441 logging.info('Dependency(%s).verify_validity()' % self.name)
442 if self.name in [s.name for s in self.parent.dependencies]:
443 raise gclient_utils.Error(
444 'The same name "%s" appears multiple times in the deps section' %
446 if not self.should_process:
447 # Return early, no need to set requirements.
450 # This require a full tree traversal with locks.
451 siblings = [d for d in self.root.subtree(False) if d.name == self.name]
452 for sibling in siblings:
453 self_url = self.LateOverride(self.url)
454 sibling_url = sibling.LateOverride(sibling.url)
455 # Allow to have only one to be None or ''.
456 if self_url != sibling_url and bool(self_url) == bool(sibling_url):
457 raise gclient_utils.Error(
458 ('Dependency %s specified more than once:\n'
467 # In theory we could keep it as a shadow of the other one. In
468 # practice, simply ignore it.
469 logging.warn('Won\'t process duplicate dependency %s' % sibling)
473 def LateOverride(self, url):
474 """Resolves the parsed url from url.
476 Manages From() keyword accordingly. Do not touch self.parsed_url nor
477 self.url because it may called with other urls due to From()."""
478 assert self.parsed_url == None or not self.should_process, self.parsed_url
479 parsed_url = self.get_custom_deps(self.name, url)
480 if parsed_url != url:
482 'Dependency(%s).LateOverride(%s) -> %s' %
483 (self.name, url, parsed_url))
486 if isinstance(url, self.FromImpl):
487 # Requires tree traversal.
489 dep for dep in self.root.subtree(True) if url.module_name == dep.name
492 raise gclient_utils.Error('Failed to find one reference to %s. %s' % (
493 url.module_name, ref))
494 # It may happen that len(ref) > 1 but it's no big deal.
496 sub_target = url.sub_target_name or self.name
497 found_deps = [d for d in ref.dependencies if d.name == sub_target]
498 if len(found_deps) != 1:
499 raise gclient_utils.Error(
500 'Couldn\'t find %s in %s, referenced by %s (parent: %s)\n%s' % (
501 sub_target, ref.name, self.name, self.parent.name,
504 # Call LateOverride() again.
505 found_dep = found_deps[0]
506 parsed_url = found_dep.LateOverride(found_dep.url)
508 'Dependency(%s).LateOverride(%s) -> %s (From)' %
509 (self.name, url, parsed_url))
512 if isinstance(url, basestring):
513 parsed_url = urlparse.urlparse(url)
514 if (not parsed_url[0] and
515 not re.match(r'^\w+\@[\w\.-]+\:[\w\/]+', parsed_url[2])):
516 # A relative url. Fetch the real base.
518 if not path.startswith('/'):
519 raise gclient_utils.Error(
520 'relative DEPS entry \'%s\' must begin with a slash' % url)
521 # Create a scm just to query the full url.
522 parent_url = self.parent.parsed_url
523 if isinstance(parent_url, self.FileImpl):
524 parent_url = parent_url.file_location
525 scm = gclient_scm.CreateSCM(
526 parent_url, self.root.root_dir, None, self.outbuf)
527 parsed_url = scm.FullUrlForRelativeUrl(url)
531 'Dependency(%s).LateOverride(%s) -> %s' %
532 (self.name, url, parsed_url))
535 if isinstance(url, self.FileImpl):
537 'Dependency(%s).LateOverride(%s) -> %s (File)' %
538 (self.name, url, url))
543 'Dependency(%s).LateOverride(%s) -> %s' % (self.name, url, url))
546 raise gclient_utils.Error('Unknown url type')
549 def MergeWithOsDeps(deps, deps_os, target_os_list):
550 """Returns a new "deps" structure that is the deps sent in updated
551 with information from deps_os (the deps_os section of the DEPS
552 file) that matches the list of target os."""
554 for the_target_os in target_os_list:
555 the_target_os_deps = deps_os.get(the_target_os, {})
556 for os_dep_key, os_dep_value in the_target_os_deps.iteritems():
557 overrides = os_overrides.setdefault(os_dep_key, [])
558 overrides.append((the_target_os, os_dep_value))
560 # If any os didn't specify a value (we have fewer value entries
561 # than in the os list), then it wants to use the default value.
562 for os_dep_key, os_dep_value in os_overrides.iteritems():
563 if len(os_dep_value) != len(target_os_list):
564 # Record the default value too so that we don't accidently
565 # set it to None or miss a conflicting DEPS.
566 if os_dep_key in deps:
567 os_dep_value.append(('default', deps[os_dep_key]))
570 for os_dep_key, os_dep_value in os_overrides.iteritems():
571 # os_dep_value is a list of (os, value) pairs.
572 possible_values = set(x[1] for x in os_dep_value if x[1] is not None)
573 if not possible_values:
574 target_os_deps[os_dep_key] = None
576 if len(possible_values) > 1:
577 # It would be possible to abort here but it would be
578 # unfortunate if we end up preventing any kind of checkout.
579 logging.error('Conflicting dependencies for %s: %s. (target_os=%s)',
580 os_dep_key, os_dep_value, target_os_list)
581 # Sorting to get the same result every time in case of conflicts.
582 target_os_deps[os_dep_key] = sorted(possible_values)[0]
584 new_deps = deps.copy()
585 new_deps.update(target_os_deps)
588 def ParseDepsFile(self):
589 """Parses the DEPS file for this dependency."""
590 assert not self.deps_parsed
591 assert not self.dependencies
596 # First try to locate the configured deps file. If it's missing, fallback
598 deps_files = [self.deps_file]
599 if 'DEPS' not in deps_files:
600 deps_files.append('DEPS')
601 for deps_file in deps_files:
602 filepath = os.path.join(self.root.root_dir, self.name, deps_file)
603 if os.path.isfile(filepath):
605 'ParseDepsFile(%s): %s file found at %s', self.name, deps_file,
609 'ParseDepsFile(%s): No %s file found at %s', self.name, deps_file,
612 if os.path.isfile(filepath):
613 deps_content = gclient_utils.FileRead(filepath)
614 logging.debug('ParseDepsFile(%s) read:\n%s', self.name, deps_content)
615 use_strict = 'use strict' in deps_content.splitlines()[0]
619 # One thing is unintuitive, vars = {} must happen before Var() use.
620 var = self.VarImpl(self.custom_vars, local_scope)
623 'ParseDepsFile(%s): Strict Mode Enabled', self.name)
625 '__builtins__': {'None': None},
631 'File': self.FileImpl,
632 'From': self.FromImpl,
638 exec(deps_content, global_scope, local_scope)
639 except SyntaxError, e:
640 gclient_utils.SyntaxErrorToError(filepath, e)
642 for key, val in local_scope.iteritems():
643 if not isinstance(val, (dict, list, tuple, str)):
644 raise gclient_utils.Error(
645 'ParseDepsFile(%s): Strict mode disallows %r -> %r' %
646 (self.name, key, val))
648 deps = local_scope.get('deps', {})
649 if 'recursion' in local_scope:
650 self.recursion_override = local_scope.get('recursion')
652 'Setting %s recursion to %d.', self.name, self.recursion_limit)
653 self.recursedeps = local_scope.get('recursedeps', None)
654 if 'recursedeps' in local_scope:
655 self.recursedeps = set(self.recursedeps)
656 logging.warning('Found recursedeps %r.', repr(self.recursedeps))
657 # If present, save 'target_os' in the local_target_os property.
658 if 'target_os' in local_scope:
659 self.local_target_os = local_scope['target_os']
660 # load os specific dependencies if defined. these dependencies may
661 # override or extend the values defined by the 'deps' member.
662 target_os_list = self.target_os
663 if 'deps_os' in local_scope and target_os_list:
664 deps = self.MergeWithOsDeps(deps, local_scope['deps_os'], target_os_list)
666 # If a line is in custom_deps, but not in the solution, we want to append
667 # this line to the solution.
668 for d in self.custom_deps:
670 deps[d] = self.custom_deps[d]
672 # If use_relative_paths is set in the DEPS file, regenerate
673 # the dictionary using paths relative to the directory containing
674 # the DEPS file. Also update recursedeps if use_relative_paths is
676 use_relative_paths = local_scope.get('use_relative_paths', False)
677 if use_relative_paths:
678 logging.warning('use_relative_paths enabled.')
680 for d, url in deps.items():
681 # normpath is required to allow DEPS to use .. in their
682 # dependency local path.
683 rel_deps[os.path.normpath(os.path.join(self.name, d))] = url
684 logging.warning('Updating deps by prepending %s.', self.name)
687 # Update recursedeps if it's set.
688 if self.recursedeps is not None:
689 logging.warning('Updating recursedeps by prepending %s.', self.name)
691 for d in self.recursedeps:
692 rel_deps.add(os.path.normpath(os.path.join(self.name, d)))
693 self.recursedeps = rel_deps
695 if 'allowed_hosts' in local_scope:
697 self._allowed_hosts = frozenset(local_scope.get('allowed_hosts'))
698 except TypeError: # raised if non-iterable
700 if not self._allowed_hosts:
701 logging.warning("allowed_hosts is specified but empty %s",
703 raise gclient_utils.Error(
704 'ParseDepsFile(%s): allowed_hosts must be absent '
705 'or a non-empty iterable' % self.name)
707 # Convert the deps into real Dependency.
709 for name, url in deps.iteritems():
710 should_process = self.recursion_limit and self.should_process
711 deps_to_add.append(Dependency(
712 self, name, url, None, None, None, None, None,
713 self.deps_file, should_process))
714 deps_to_add.sort(key=lambda x: x.name)
716 # override named sets of hooks by the custom hooks
718 hook_names_to_suppress = [c.get('name', '') for c in self.custom_hooks]
719 for hook in local_scope.get('hooks', []):
720 if hook.get('name', '') not in hook_names_to_suppress:
721 hooks_to_run.append(hook)
723 # add the replacements and any additions
724 for hook in self.custom_hooks:
726 hooks_to_run.append(hook)
728 self._pre_deps_hooks = [self.GetHookAction(hook, []) for hook in
729 local_scope.get('pre_deps_hooks', [])]
731 self.add_dependencies_and_close(deps_to_add, hooks_to_run)
732 logging.info('ParseDepsFile(%s) done' % self.name)
734 def add_dependencies_and_close(self, deps_to_add, hooks):
735 """Adds the dependencies, hooks and mark the parsing as done."""
736 for dep in deps_to_add:
737 if dep.verify_validity():
738 self.add_dependency(dep)
739 self._mark_as_parsed(hooks)
741 def maybeGetParentRevision(self, command, options, parsed_url, parent):
742 """Uses revision/timestamp of parent if no explicit revision was specified.
744 If we are performing an update and --transitive is set, use
745 - the parent's revision if 'self.url' is in the same repository
746 - the parent's timestamp otherwise
747 to update 'self.url'. The used revision/timestamp will be set in
749 If we have an explicit revision do nothing.
751 if command == 'update' and options.transitive and not options.revision:
752 _, revision = gclient_utils.SplitUrlRevision(parsed_url)
754 options.revision = getattr(parent, '_used_revision', None)
755 if (options.revision and
756 not gclient_utils.IsDateRevision(options.revision)):
757 assert self.parent and self.parent.used_scm
758 # If this dependency is in the same repository as parent it's url will
759 # start with a slash. If so we take the parent revision instead of
761 # (The timestamps of commits in google code are broken -- which can
762 # result in dependencies to be checked out at the wrong revision)
763 if self.url.startswith('/'):
765 print('Using parent\'s revision %s since we are in the same '
766 'repository.' % options.revision)
768 parent_revision_date = self.parent.used_scm.GetRevisionDate(
770 options.revision = gclient_utils.MakeDateRevision(
771 parent_revision_date)
773 print('Using parent\'s revision date %s since we are in a '
774 'different repository.' % options.revision)
776 def findDepsFromNotAllowedHosts(self):
777 """Returns a list of depenecies from not allowed hosts.
779 If allowed_hosts is not set, allows all hosts and returns empty list.
781 if not self._allowed_hosts:
784 for dep in self._dependencies:
785 # Don't enforce this for custom_deps.
786 if dep.name in self._custom_deps:
788 if isinstance(dep.url, basestring):
789 parsed_url = urlparse.urlparse(dep.url)
790 if parsed_url.netloc and parsed_url.netloc not in self._allowed_hosts:
794 # Arguments number differs from overridden method
795 # pylint: disable=W0221
796 def run(self, revision_overrides, command, args, work_queue, options):
797 """Runs |command| then parse the DEPS file."""
798 logging.info('Dependency(%s).run()' % self.name)
799 assert self._file_list == []
800 if not self.should_process:
802 # When running runhooks, there's no need to consult the SCM.
803 # All known hooks are expected to run unconditionally regardless of working
804 # copy state, so skip the SCM status check.
805 run_scm = command not in ('runhooks', 'recurse', None)
806 parsed_url = self.LateOverride(self.url)
807 file_list = [] if not options.nohooks else None
808 revision_override = revision_overrides.pop(self.name, None)
809 if run_scm and parsed_url:
810 if isinstance(parsed_url, self.FileImpl):
811 # Special support for single-file checkout.
812 if not command in (None, 'cleanup', 'diff', 'pack', 'status'):
813 # Sadly, pylint doesn't realize that parsed_url is of FileImpl.
814 # pylint: disable=E1103
815 options.revision = parsed_url.GetRevision()
816 self._used_scm = gclient_scm.SVNWrapper(
817 parsed_url.GetPath(), self.root.root_dir, self.name,
818 out_cb=work_queue.out_cb)
819 self._used_scm.RunCommand('updatesingle',
820 options, args + [parsed_url.GetFilename()], file_list)
822 # Create a shallow copy to mutate revision.
823 options = copy.copy(options)
824 options.revision = revision_override
825 self.maybeGetParentRevision(
826 command, options, parsed_url, self.parent)
827 self._used_revision = options.revision
828 self._used_scm = gclient_scm.CreateSCM(
829 parsed_url, self.root.root_dir, self.name, self.outbuf,
830 out_cb=work_queue.out_cb)
831 self._got_revision = self._used_scm.RunCommand(command, options, args,
834 file_list = [os.path.join(self.name, f.strip()) for f in file_list]
836 # TODO(phajdan.jr): We should know exactly when the paths are absolute.
837 # Convert all absolute paths to relative.
838 for i in range(len(file_list or [])):
839 # It depends on the command being executed (like runhooks vs sync).
840 if not os.path.isabs(file_list[i]):
842 prefix = os.path.commonprefix(
843 [self.root.root_dir.lower(), file_list[i].lower()])
844 file_list[i] = file_list[i][len(prefix):]
845 # Strip any leading path separators.
846 while file_list[i].startswith(('\\', '/')):
847 file_list[i] = file_list[i][1:]
849 # Always parse the DEPS file.
851 self._run_is_done(file_list or [], parsed_url)
852 if command in ('update', 'revert') and not options.noprehooks:
853 self.RunPreDepsHooks()
855 if self.recursion_limit:
856 # Parse the dependencies of this dependency.
857 for s in self.dependencies:
858 work_queue.enqueue(s)
860 if command == 'recurse':
861 if not isinstance(parsed_url, self.FileImpl):
862 # Skip file only checkout.
863 scm = gclient_scm.GetScmName(parsed_url)
864 if not options.scm or scm in options.scm:
865 cwd = os.path.normpath(os.path.join(self.root.root_dir, self.name))
866 # Pass in the SCM type as an env variable. Make sure we don't put
867 # unicode strings in the environment.
868 env = os.environ.copy()
870 env['GCLIENT_SCM'] = str(scm)
872 env['GCLIENT_URL'] = str(parsed_url)
873 env['GCLIENT_DEP_PATH'] = str(self.name)
874 if options.prepend_dir and scm == 'git':
877 """Git-specific path marshaling. It is optimized for git-grep."""
879 def mod_path(git_pathspec):
880 match = re.match('^(\\S+?:)?([^\0]+)$', git_pathspec)
881 modified_path = os.path.join(self.name, match.group(2))
882 branch = match.group(1) or ''
883 return '%s%s' % (branch, modified_path)
885 match = re.match('^Binary file ([^\0]+) matches$', line)
887 print 'Binary file %s matches\n' % mod_path(match.group(1))
890 items = line.split('\0')
891 if len(items) == 2 and items[1]:
892 print '%s : %s' % (mod_path(items[0]), items[1])
893 elif len(items) >= 2:
894 # Multiple null bytes or a single trailing null byte indicate
895 # git is likely displaying filenames only (such as with -l)
896 print '\n'.join(mod_path(path) for path in items if path)
903 if parsed_url is None:
904 print >> sys.stderr, 'Skipped omitted dependency %s' % cwd
905 elif os.path.isdir(cwd):
907 gclient_utils.CheckCallAndFilter(
908 args, cwd=cwd, env=env, print_stdout=print_stdout,
911 except subprocess2.CalledProcessError:
912 if not options.ignore:
915 print >> sys.stderr, 'Skipped missing %s' % cwd
918 @gclient_utils.lockedmethod
919 def _run_is_done(self, file_list, parsed_url):
920 # Both these are kept for hooks that are run as a separate tree traversal.
921 self._file_list = file_list
922 self._parsed_url = parsed_url
923 self._processed = True
926 def GetHookAction(hook_dict, matching_file_list):
927 """Turns a parsed 'hook' dict into an executable command."""
928 logging.debug(hook_dict)
929 logging.debug(matching_file_list)
930 command = hook_dict['action'][:]
931 if command[0] == 'python':
932 # If the hook specified "python" as the first item, the action is a
933 # Python script. Run it by starting a new copy of the same
935 command[0] = sys.executable
936 if '$matching_files' in command:
937 splice_index = command.index('$matching_files')
938 command[splice_index:splice_index + 1] = matching_file_list
941 def GetHooks(self, options):
942 """Evaluates all hooks, and return them in a flat list.
944 RunOnDeps() must have been called before to load the DEPS.
947 if not self.should_process or not self.recursion_limit:
948 # Don't run the hook when it is above recursion_limit.
950 # If "--force" was specified, run all hooks regardless of what files have
953 # TODO(maruel): If the user is using git or git-svn, then we don't know
954 # what files have changed so we always run all hooks. It'd be nice to fix
957 isinstance(self.parsed_url, self.FileImpl) or
958 gclient_scm.GetScmName(self.parsed_url) in ('git', None) or
959 os.path.isdir(os.path.join(self.root.root_dir, self.name, '.git'))):
960 for hook_dict in self.deps_hooks:
961 result.append(self.GetHookAction(hook_dict, []))
963 # Run hooks on the basis of whether the files from the gclient operation
964 # match each hook's pattern.
965 for hook_dict in self.deps_hooks:
966 pattern = re.compile(hook_dict['pattern'])
967 matching_file_list = [
968 f for f in self.file_list_and_children if pattern.search(f)
970 if matching_file_list:
971 result.append(self.GetHookAction(hook_dict, matching_file_list))
972 for s in self.dependencies:
973 result.extend(s.GetHooks(options))
976 def RunHooksRecursively(self, options):
977 assert self.hooks_ran == False
978 self._hooks_ran = True
979 for hook in self.GetHooks(options):
981 start_time = time.time()
982 gclient_utils.CheckCallAndFilterAndHeader(
983 hook, cwd=self.root.root_dir, always=True)
984 except (gclient_utils.Error, subprocess2.CalledProcessError), e:
985 # Use a discrete exit status code of 2 to indicate that a hook action
986 # failed. Users of this script may wish to treat hook action failures
987 # differently from VC failures.
988 print >> sys.stderr, 'Error: %s' % str(e)
991 elapsed_time = time.time() - start_time
992 if elapsed_time > 10:
993 print "Hook '%s' took %.2f secs" % (
994 gclient_utils.CommandToStr(hook), elapsed_time)
996 def RunPreDepsHooks(self):
997 assert self.processed
998 assert self.deps_parsed
999 assert not self.pre_deps_hooks_ran
1000 assert not self.hooks_ran
1001 for s in self.dependencies:
1002 assert not s.processed
1003 self._pre_deps_hooks_ran = True
1004 for hook in self.pre_deps_hooks:
1006 start_time = time.time()
1007 gclient_utils.CheckCallAndFilterAndHeader(
1008 hook, cwd=self.root.root_dir, always=True)
1009 except (gclient_utils.Error, subprocess2.CalledProcessError), e:
1010 # Use a discrete exit status code of 2 to indicate that a hook action
1011 # failed. Users of this script may wish to treat hook action failures
1012 # differently from VC failures.
1013 print >> sys.stderr, 'Error: %s' % str(e)
1016 elapsed_time = time.time() - start_time
1017 if elapsed_time > 10:
1018 print "Hook '%s' took %.2f secs" % (
1019 gclient_utils.CommandToStr(hook), elapsed_time)
1022 def subtree(self, include_all):
1023 """Breadth first recursion excluding root node."""
1024 dependencies = self.dependencies
1025 for d in dependencies:
1026 if d.should_process or include_all:
1028 for d in dependencies:
1029 for i in d.subtree(include_all):
1032 def depth_first_tree(self):
1033 """Depth-first recursion including the root node."""
1035 for i in self.dependencies:
1036 for j in i.depth_first_tree():
1037 if j.should_process:
1040 @gclient_utils.lockedmethod
1041 def add_dependency(self, new_dep):
1042 self._dependencies.append(new_dep)
1044 @gclient_utils.lockedmethod
1045 def _mark_as_parsed(self, new_hooks):
1046 self._deps_hooks.extend(new_hooks)
1047 self._deps_parsed = True
1050 @gclient_utils.lockedmethod
1051 def dependencies(self):
1052 return tuple(self._dependencies)
1055 @gclient_utils.lockedmethod
1056 def deps_hooks(self):
1057 return tuple(self._deps_hooks)
1060 @gclient_utils.lockedmethod
1061 def pre_deps_hooks(self):
1062 return tuple(self._pre_deps_hooks)
1065 @gclient_utils.lockedmethod
1066 def parsed_url(self):
1067 return self._parsed_url
1070 @gclient_utils.lockedmethod
1071 def deps_parsed(self):
1072 """This is purely for debugging purposes. It's not used anywhere."""
1073 return self._deps_parsed
1076 @gclient_utils.lockedmethod
1077 def processed(self):
1078 return self._processed
1081 @gclient_utils.lockedmethod
1082 def pre_deps_hooks_ran(self):
1083 return self._pre_deps_hooks_ran
1086 @gclient_utils.lockedmethod
1087 def hooks_ran(self):
1088 return self._hooks_ran
1091 @gclient_utils.lockedmethod
1092 def allowed_hosts(self):
1093 return self._allowed_hosts
1096 @gclient_utils.lockedmethod
1097 def file_list(self):
1098 return tuple(self._file_list)
1102 """SCMWrapper instance for this dependency or None if not processed yet."""
1103 return self._used_scm
1106 @gclient_utils.lockedmethod
1107 def got_revision(self):
1108 return self._got_revision
1111 def file_list_and_children(self):
1112 result = list(self.file_list)
1113 for d in self.dependencies:
1114 result.extend(d.file_list_and_children)
1115 return tuple(result)
1119 for i in ('name', 'url', 'parsed_url', 'safesync_url', 'custom_deps',
1120 'custom_vars', 'deps_hooks', 'file_list', 'should_process',
1121 'processed', 'hooks_ran', 'deps_parsed', 'requirements',
1123 # First try the native property if it exists.
1124 if hasattr(self, '_' + i):
1125 value = getattr(self, '_' + i, False)
1127 value = getattr(self, i, False)
1129 out.append('%s: %s' % (i, value))
1131 for d in self.dependencies:
1132 out.extend([' ' + x for x in str(d).splitlines()])
1134 return '\n'.join(out)
1137 return '%s: %s' % (self.name, self.url)
1139 def hierarchy(self):
1140 """Returns a human-readable hierarchical reference to a Dependency."""
1141 out = '%s(%s)' % (self.name, self.url)
1144 out = '%s(%s) -> %s' % (i.name, i.url, out)
1149 class GClient(Dependency):
1150 """Object that represent a gclient checkout. A tree of Dependency(), one per
1151 solution or DEPS entry."""
1163 "android": "android",
1166 DEFAULT_CLIENT_FILE_TEXT = ("""\
1168 { "name" : "%(solution_name)s",
1169 "url" : "%(solution_url)s",
1170 "deps_file" : "%(deps_file)s",
1171 "managed" : %(managed)s,
1174 "safesync_url": "%(safesync_url)s",
1177 cache_dir = %(cache_dir)r
1180 DEFAULT_SNAPSHOT_SOLUTION_TEXT = ("""\
1181 { "name" : "%(solution_name)s",
1182 "url" : "%(solution_url)s",
1183 "deps_file" : "%(deps_file)s",
1184 "managed" : %(managed)s,
1186 %(solution_deps)s },
1187 "safesync_url": "%(safesync_url)s",
1191 DEFAULT_SNAPSHOT_FILE_TEXT = ("""\
1192 # Snapshot generated with gclient revinfo --snapshot
1197 def __init__(self, root_dir, options):
1198 # Do not change previous behavior. Only solution level and immediate DEPS
1200 self._recursion_limit = 2
1201 Dependency.__init__(self, None, None, None, None, True, None, None, None,
1203 self._options = options
1205 enforced_os = options.deps_os.split(',')
1207 enforced_os = [self.DEPS_OS_CHOICES.get(sys.platform, 'unix')]
1208 if 'all' in enforced_os:
1209 enforced_os = self.DEPS_OS_CHOICES.itervalues()
1210 self._enforced_os = tuple(set(enforced_os))
1211 self._root_dir = root_dir
1212 self.config_content = None
1214 def _CheckConfig(self):
1215 """Verify that the config matches the state of the existing checked-out
1217 for dep in self.dependencies:
1218 if dep.managed and dep.url:
1219 scm = gclient_scm.CreateSCM(
1220 dep.url, self.root_dir, dep.name, self.outbuf)
1221 actual_url = scm.GetActualRemoteURL(self._options)
1222 if actual_url and not scm.DoesRemoteURLMatch(self._options):
1223 raise gclient_utils.Error('''
1224 Your .gclient file seems to be broken. The requested URL is different from what
1225 is actually checked out in %(checkout_path)s.
1227 The .gclient file contains:
1228 %(expected_url)s (%(expected_scm)s)
1230 The local checkout in %(checkout_path)s reports:
1231 %(actual_url)s (%(actual_scm)s)
1233 You should ensure that the URL listed in .gclient is correct and either change
1234 it or fix the checkout. If you're managing your own git checkout in
1235 %(checkout_path)s but the URL in .gclient is for an svn repository, you probably
1236 want to set 'managed': False in .gclient.
1237 ''' % {'checkout_path': os.path.join(self.root_dir, dep.name),
1238 'expected_url': dep.url,
1239 'expected_scm': gclient_scm.GetScmName(dep.url),
1240 'actual_url': actual_url,
1241 'actual_scm': gclient_scm.GetScmName(actual_url)})
1243 def SetConfig(self, content):
1244 assert not self.dependencies
1246 self.config_content = content
1248 exec(content, config_dict)
1249 except SyntaxError, e:
1250 gclient_utils.SyntaxErrorToError('.gclient', e)
1252 # Append any target OS that is not already being enforced to the tuple.
1253 target_os = config_dict.get('target_os', [])
1254 if config_dict.get('target_os_only', False):
1255 self._enforced_os = tuple(set(target_os))
1257 self._enforced_os = tuple(set(self._enforced_os).union(target_os))
1259 cache_dir = config_dict.get('cache_dir')
1261 cache_dir = os.path.join(self.root_dir, cache_dir)
1262 cache_dir = os.path.abspath(cache_dir)
1263 # If running on a bot, force break any stale git cache locks.
1264 if os.path.exists(cache_dir) and os.environ.get('CHROME_HEADLESS'):
1265 subprocess2.check_call(['git', 'cache', 'unlock', '--cache-dir',
1266 cache_dir, '--force', '--all'])
1267 gclient_scm.GitWrapper.cache_dir = cache_dir
1268 git_cache.Mirror.SetCachePath(cache_dir)
1270 if not target_os and config_dict.get('target_os_only', False):
1271 raise gclient_utils.Error('Can\'t use target_os_only if target_os is '
1275 for s in config_dict.get('solutions', []):
1277 deps_to_add.append(Dependency(
1278 self, s['name'], s['url'],
1279 s.get('safesync_url', None),
1280 s.get('managed', True),
1281 s.get('custom_deps', {}),
1282 s.get('custom_vars', {}),
1283 s.get('custom_hooks', []),
1284 s.get('deps_file', 'DEPS'),
1287 raise gclient_utils.Error('Invalid .gclient file. Solution is '
1288 'incomplete: %s' % s)
1289 self.add_dependencies_and_close(deps_to_add, config_dict.get('hooks', []))
1290 logging.info('SetConfig() done')
1292 def SaveConfig(self):
1293 gclient_utils.FileWrite(os.path.join(self.root_dir,
1294 self._options.config_filename),
1295 self.config_content)
1297 def MigrateConfigToGit(self, path, options):
1298 svn_url_re = re.compile('^(https?://src\.chromium\.org/svn|'
1299 'svn://svn\.chromium\.org/chrome)/'
1300 '(trunk|branches/[^/]+)/src')
1301 old_git_re = re.compile('^(https?://git\.chromium\.org|'
1302 'ssh://([a-zA-Z_][a-zA-Z0-9_-]*@)?'
1303 'gerrit\.chromium\.org(:2941[89])?)/'
1304 'chromium/src\.git')
1305 # Scan existing .gclient file for obsolete settings. It would be simpler
1306 # to traverse self.dependencies, but working with the AST allows the code to
1307 # dump an updated .gclient file that preserves the ordering of the original.
1308 a = ast.parse(self.config_content, options.config_filename, 'exec')
1310 solutions = [elem for elem in a.body if 'solutions' in
1311 [target.id for target in elem.targets]]
1314 solutions = solutions[-1]
1315 for solution in solutions.value.elts:
1316 # Check for obsolete URL's
1317 url_idx = ast_dict_index(solution, 'url')
1320 url_val = solution.values[url_idx]
1321 if type(url_val) is not ast.Str:
1323 if (svn_url_re.match(url_val.s.strip())):
1324 raise gclient_utils.Error(
1326 The chromium code repository has migrated completely to git.
1327 Your SVN-based checkout is now obsolete; you need to create a brand-new
1328 git checkout by following these instructions:
1330 http://www.chromium.org/developers/how-tos/get-the-code
1332 if (old_git_re.match(url_val.s.strip())):
1333 url_val.s = CHROMIUM_SRC_URL
1336 # Ensure deps_file is set to .DEPS.git. We enforce this here to smooth
1337 # over switching between pre-git-migration and post-git-migration
1339 # - For pre-migration revisions, .DEPS.git must be explicitly set.
1340 # - For post-migration revisions, .DEPS.git is not present, so gclient
1341 # will correctly fall back to DEPS.
1342 if url_val.s == CHROMIUM_SRC_URL:
1343 deps_file_idx = ast_dict_index(solution, 'deps_file')
1344 if deps_file_idx != -1:
1346 solution.keys.append(ast.Str('deps_file'))
1347 solution.values.append(ast.Str('.DEPS.git'))
1355 WARNING: gclient detected an obsolete setting in your %s file. The file has
1356 been automagically updated. The previous version is available at %s.old.
1357 """ % (options.config_filename, options.config_filename))
1359 # Replace existing .gclient with the updated version.
1360 # Return a new GClient instance based on the new content.
1361 new_content = ast2str(a)
1362 dot_gclient_fn = os.path.join(path, options.config_filename)
1364 os.rename(dot_gclient_fn, dot_gclient_fn + '.old')
1367 with open(dot_gclient_fn, 'w') as fh:
1368 fh.write(new_content)
1369 client = GClient(path, options)
1370 client.SetConfig(new_content)
1374 def LoadCurrentConfig(options):
1375 """Searches for and loads a .gclient file relative to the current working
1376 dir. Returns a GClient object."""
1378 client = GClient('.', options)
1379 client.SetConfig(options.spec)
1381 path = gclient_utils.FindGclientRoot(os.getcwd(), options.config_filename)
1384 client = GClient(path, options)
1385 client.SetConfig(gclient_utils.FileRead(
1386 os.path.join(path, options.config_filename)))
1387 client = client.MigrateConfigToGit(path, options)
1389 if (options.revisions and
1390 len(client.dependencies) > 1 and
1391 any('@' not in r for r in options.revisions)):
1392 print >> sys.stderr, (
1393 'You must specify the full solution name like --revision %s@%s\n'
1394 'when you have multiple solutions setup in your .gclient file.\n'
1395 'Other solutions present are: %s.') % (
1396 client.dependencies[0].name,
1397 options.revisions[0],
1398 ', '.join(s.name for s in client.dependencies[1:]))
1401 def SetDefaultConfig(self, solution_name, deps_file, solution_url,
1402 safesync_url, managed=True, cache_dir=None):
1403 self.SetConfig(self.DEFAULT_CLIENT_FILE_TEXT % {
1404 'solution_name': solution_name,
1405 'solution_url': solution_url,
1406 'deps_file': deps_file,
1407 'safesync_url' : safesync_url,
1409 'cache_dir': cache_dir,
1412 def _SaveEntries(self):
1413 """Creates a .gclient_entries file to record the list of unique checkouts.
1415 The .gclient_entries file lives in the same directory as .gclient.
1417 # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It
1418 # makes testing a bit too fun.
1419 result = 'entries = {\n'
1420 for entry in self.root.subtree(False):
1421 # Skip over File() dependencies as we can't version them.
1422 if not isinstance(entry.parsed_url, self.FileImpl):
1423 result += ' %s: %s,\n' % (pprint.pformat(entry.name),
1424 pprint.pformat(entry.parsed_url))
1426 file_path = os.path.join(self.root_dir, self._options.entries_filename)
1427 logging.debug(result)
1428 gclient_utils.FileWrite(file_path, result)
1430 def _ReadEntries(self):
1431 """Read the .gclient_entries file for the given client.
1434 A sequence of solution names, which will be empty if there is the
1435 entries file hasn't been created yet.
1438 filename = os.path.join(self.root_dir, self._options.entries_filename)
1439 if not os.path.exists(filename):
1442 exec(gclient_utils.FileRead(filename), scope)
1443 except SyntaxError, e:
1444 gclient_utils.SyntaxErrorToError(filename, e)
1445 return scope['entries']
1447 def _EnforceRevisions(self):
1448 """Checks for revision overrides."""
1449 revision_overrides = {}
1450 if self._options.head:
1451 return revision_overrides
1452 # Do not check safesync_url if one or more --revision flag is specified.
1453 if not self._options.revisions:
1454 for s in self.dependencies:
1456 self._options.revisions.append('%s@unmanaged' % s.name)
1457 elif s.safesync_url:
1458 self._ApplySafeSyncRev(dep=s)
1459 if not self._options.revisions:
1460 return revision_overrides
1461 solutions_names = [s.name for s in self.dependencies]
1463 for revision in self._options.revisions:
1464 if not '@' in revision:
1465 # Support for --revision 123
1466 revision = '%s@%s' % (solutions_names[index], revision)
1467 name, rev = revision.split('@', 1)
1468 revision_overrides[name] = rev
1470 return revision_overrides
1472 def _ApplySafeSyncRev(self, dep):
1473 """Finds a valid revision from the content of the safesync_url and apply it
1474 by appending revisions to the revision list. Throws if revision appears to
1475 be invalid for the given |dep|."""
1476 assert len(dep.safesync_url) > 0
1477 handle = urllib.urlopen(dep.safesync_url)
1478 rev = handle.read().strip()
1481 raise gclient_utils.Error(
1482 'It appears your safesync_url (%s) is not working properly\n'
1483 '(as it returned an empty response). Check your config.' %
1485 scm = gclient_scm.CreateSCM(
1486 dep.url, dep.root.root_dir, dep.name, self.outbuf)
1487 safe_rev = scm.GetUsableRev(rev, self._options)
1488 if self._options.verbose:
1489 print('Using safesync_url revision: %s.\n' % safe_rev)
1490 self._options.revisions.append('%s@%s' % (dep.name, safe_rev))
1492 def RunOnDeps(self, command, args, ignore_requirements=False, progress=True):
1493 """Runs a command on each dependency in a client and its dependencies.
1496 command: The command to use (e.g., 'status' or 'diff')
1497 args: list of str - extra arguments to add to the command line.
1499 if not self.dependencies:
1500 raise gclient_utils.Error('No solution specified')
1502 revision_overrides = {}
1503 # It's unnecessary to check for revision overrides for 'recurse'.
1504 # Save a few seconds by not calling _EnforceRevisions() in that case.
1505 if command not in ('diff', 'recurse', 'runhooks', 'status'):
1507 revision_overrides = self._EnforceRevisions()
1509 # Disable progress for non-tty stdout.
1510 if (sys.stdout.isatty() and not self._options.verbose and progress):
1511 if command in ('update', 'revert'):
1512 pm = Progress('Syncing projects', 1)
1513 elif command == 'recurse':
1514 pm = Progress(' '.join(args), 1)
1515 work_queue = gclient_utils.ExecutionQueue(
1516 self._options.jobs, pm, ignore_requirements=ignore_requirements,
1517 verbose=self._options.verbose)
1518 for s in self.dependencies:
1519 work_queue.enqueue(s)
1520 work_queue.flush(revision_overrides, command, args, options=self._options)
1521 if revision_overrides:
1522 print >> sys.stderr, ('Please fix your script, having invalid '
1523 '--revision flags will soon considered an error.')
1525 # Once all the dependencies have been processed, it's now safe to run the
1527 if not self._options.nohooks:
1528 self.RunHooksRecursively(self._options)
1530 if command == 'update':
1531 # Notify the user if there is an orphaned entry in their working copy.
1532 # Only delete the directory if there are no changes in it, and
1533 # delete_unversioned_trees is set to true.
1534 entries = [i.name for i in self.root.subtree(False) if i.url]
1535 full_entries = [os.path.join(self.root_dir, e.replace('/', os.path.sep))
1538 for entry, prev_url in self._ReadEntries().iteritems():
1540 # entry must have been overridden via .gclient custom_deps
1542 # Fix path separator on Windows.
1543 entry_fixed = entry.replace('/', os.path.sep)
1544 e_dir = os.path.join(self.root_dir, entry_fixed)
1546 def _IsParentOfAny(parent, path_list):
1547 parent_plus_slash = parent + '/'
1549 path[:len(parent_plus_slash)] == parent_plus_slash
1550 for path in path_list)
1552 # Use entry and not entry_fixed there.
1553 if (entry not in entries and
1554 (not any(path.startswith(entry + '/') for path in entries)) and
1555 os.path.exists(e_dir)):
1556 scm = gclient_scm.CreateSCM(
1557 prev_url, self.root_dir, entry_fixed, self.outbuf)
1559 # Check to see if this directory is now part of a higher-up checkout.
1560 # The directory might be part of a git OR svn checkout.
1562 for scm_class in (gclient_scm.scm.GIT, gclient_scm.scm.SVN):
1564 scm_root = scm_class.GetCheckoutRoot(scm.checkout_path)
1565 except subprocess2.CalledProcessError:
1570 logging.warning('Could not find checkout root for %s. Unable to '
1571 'determine whether it is part of a higher-level '
1572 'checkout, so not removing.' % entry)
1574 if scm_root in full_entries:
1575 logging.info('%s is part of a higher level checkout, not '
1576 'removing.', scm.GetCheckoutRoot())
1580 scm.status(self._options, [], file_list)
1581 modified_files = file_list != []
1582 if (not self._options.delete_unversioned_trees or
1583 (modified_files and not self._options.force)):
1584 # There are modified files in this entry. Keep warning until
1586 print(('\nWARNING: \'%s\' is no longer part of this client. '
1587 'It is recommended that you manually remove it.\n') %
1591 print('\n________ deleting \'%s\' in \'%s\'' % (
1592 entry_fixed, self.root_dir))
1593 gclient_utils.rmtree(e_dir)
1594 # record the current list of entries for next time
1598 def PrintRevInfo(self):
1599 if not self.dependencies:
1600 raise gclient_utils.Error('No solution specified')
1601 # Load all the settings.
1602 work_queue = gclient_utils.ExecutionQueue(
1603 self._options.jobs, None, False, verbose=self._options.verbose)
1604 for s in self.dependencies:
1605 work_queue.enqueue(s)
1606 work_queue.flush({}, None, [], options=self._options)
1608 def GetURLAndRev(dep):
1609 """Returns the revision-qualified SCM url for a Dependency."""
1610 if dep.parsed_url is None:
1612 if isinstance(dep.parsed_url, self.FileImpl):
1613 original_url = dep.parsed_url.file_location
1615 original_url = dep.parsed_url
1616 url, _ = gclient_utils.SplitUrlRevision(original_url)
1617 scm = gclient_scm.CreateSCM(
1618 original_url, self.root_dir, dep.name, self.outbuf)
1619 if not os.path.isdir(scm.checkout_path):
1621 return '%s@%s' % (url, scm.revinfo(self._options, [], None))
1623 if self._options.snapshot:
1625 # First level at .gclient
1626 for d in self.dependencies:
1629 """Recursively grab dependencies."""
1630 for d in dep.dependencies:
1631 entries[d.name] = GetURLAndRev(d)
1635 for k in sorted(entries.keys()):
1637 # Quotes aren't escaped...
1638 custom_deps.append(' \"%s\": \'%s\',\n' % (k, entries[k]))
1640 custom_deps.append(' \"%s\": None,\n' % k)
1641 new_gclient += self.DEFAULT_SNAPSHOT_SOLUTION_TEXT % {
1642 'solution_name': d.name,
1643 'solution_url': d.url,
1644 'deps_file': d.deps_file,
1645 'safesync_url' : d.safesync_url or '',
1646 'managed': d.managed,
1647 'solution_deps': ''.join(custom_deps),
1649 # Print the snapshot configuration file
1650 print(self.DEFAULT_SNAPSHOT_FILE_TEXT % {'solution_list': new_gclient})
1653 for d in self.root.subtree(False):
1654 if self._options.actual:
1655 entries[d.name] = GetURLAndRev(d)
1657 entries[d.name] = d.parsed_url
1658 keys = sorted(entries.keys())
1660 print('%s: %s' % (x, entries[x]))
1661 logging.info(str(self))
1663 def ParseDepsFile(self):
1664 """No DEPS to parse for a .gclient file."""
1665 raise gclient_utils.Error('Internal error')
1669 """Root directory of gclient checkout."""
1670 return self._root_dir
1673 def enforced_os(self):
1674 """What deps_os entries that are to be parsed."""
1675 return self._enforced_os
1678 def recursion_limit(self):
1679 """How recursive can each dependencies in DEPS file can load DEPS file."""
1680 return self._recursion_limit
1683 def try_recursedeps(self):
1684 """Whether to attempt using recursedeps-style recursion processing."""
1688 def target_os(self):
1689 return self._enforced_os
1692 #### gclient commands.
1695 def CMDcleanup(parser, args):
1696 """Cleans up all working copies.
1698 Mostly svn-specific. Simply runs 'svn cleanup' for each module.
1700 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
1701 help='override deps for the specified (comma-separated) '
1702 'platform(s); \'all\' will process all deps_os '
1704 (options, args) = parser.parse_args(args)
1705 client = GClient.LoadCurrentConfig(options)
1707 raise gclient_utils.Error('client not configured; see \'gclient config\'')
1709 # Print out the .gclient file. This is longer than if we just printed the
1710 # client dict, but more legible, and it might contain helpful comments.
1711 print(client.config_content)
1712 return client.RunOnDeps('cleanup', args)
1715 @subcommand.usage('[command] [args ...]')
1716 def CMDrecurse(parser, args):
1717 """Operates [command args ...] on all the dependencies.
1719 Runs a shell command on all entries.
1720 Sets GCLIENT_DEP_PATH enviroment variable as the dep's relative location to
1721 root directory of the checkout.
1723 # Stop parsing at the first non-arg so that these go through to the command
1724 parser.disable_interspersed_args()
1725 parser.add_option('-s', '--scm', action='append', default=[],
1726 help='Choose scm types to operate upon.')
1727 parser.add_option('-i', '--ignore', action='store_true',
1728 help='Ignore non-zero return codes from subcommands.')
1729 parser.add_option('--prepend-dir', action='store_true',
1730 help='Prepend relative dir for use with git <cmd> --null.')
1731 parser.add_option('--no-progress', action='store_true',
1732 help='Disable progress bar that shows sub-command updates')
1733 options, args = parser.parse_args(args)
1735 print >> sys.stderr, 'Need to supply a command!'
1737 root_and_entries = gclient_utils.GetGClientRootAndEntries()
1738 if not root_and_entries:
1739 print >> sys.stderr, (
1740 'You need to run gclient sync at least once to use \'recurse\'.\n'
1741 'This is because .gclient_entries needs to exist and be up to date.')
1744 # Normalize options.scm to a set()
1746 for scm in options.scm:
1747 scm_set.update(scm.split(','))
1748 options.scm = scm_set
1750 options.nohooks = True
1751 client = GClient.LoadCurrentConfig(options)
1752 return client.RunOnDeps('recurse', args, ignore_requirements=True,
1753 progress=not options.no_progress)
1756 @subcommand.usage('[args ...]')
1757 def CMDfetch(parser, args):
1758 """Fetches upstream commits for all modules.
1760 Completely git-specific. Simply runs 'git fetch [args ...]' for each module.
1762 (options, args) = parser.parse_args(args)
1763 return CMDrecurse(OptionParser(), [
1764 '--jobs=%d' % options.jobs, '--scm=git', 'git', 'fetch'] + args)
1767 def CMDgrep(parser, args):
1768 """Greps through git repos managed by gclient.
1770 Runs 'git grep [args...]' for each module.
1772 # We can't use optparse because it will try to parse arguments sent
1773 # to git grep and throw an error. :-(
1774 if not args or re.match('(-h|--help)$', args[0]):
1775 print >> sys.stderr, (
1776 'Usage: gclient grep [-j <N>] git-grep-args...\n\n'
1777 'Example: "gclient grep -j10 -A2 RefCountedBase" runs\n"git grep '
1778 '-A2 RefCountedBase" on each of gclient\'s git\nrepos with up to '
1779 '10 jobs.\n\nBonus: page output by appending "|& less -FRSX" to the'
1780 ' end of your query.'
1784 jobs_arg = ['--jobs=1']
1785 if re.match(r'(-j|--jobs=)\d+$', args[0]):
1786 jobs_arg, args = args[:1], args[1:]
1787 elif re.match(r'(-j|--jobs)$', args[0]):
1788 jobs_arg, args = args[:2], args[2:]
1792 jobs_arg + ['--ignore', '--prepend-dir', '--no-progress', '--scm=git',
1793 'git', 'grep', '--null', '--color=Always'] + args)
1796 @subcommand.usage('[url] [safesync url]')
1797 def CMDconfig(parser, args):
1798 """Creates a .gclient file in the current directory.
1800 This specifies the configuration for further commands. After update/sync,
1801 top-level DEPS files in each module are read to determine dependent
1802 modules to operate on as well. If optional [url] parameter is
1803 provided, then configuration is read from a specified Subversion server
1806 # We do a little dance with the --gclientfile option. 'gclient config' is the
1807 # only command where it's acceptable to have both '--gclientfile' and '--spec'
1808 # arguments. So, we temporarily stash any --gclientfile parameter into
1809 # options.output_config_file until after the (gclientfile xor spec) error
1811 parser.remove_option('--gclientfile')
1812 parser.add_option('--gclientfile', dest='output_config_file',
1813 help='Specify an alternate .gclient file')
1814 parser.add_option('--name',
1815 help='overrides the default name for the solution')
1816 parser.add_option('--deps-file', default='DEPS',
1817 help='overrides the default name for the DEPS file for the'
1818 'main solutions and all sub-dependencies')
1819 parser.add_option('--unmanaged', action='store_true', default=False,
1820 help='overrides the default behavior to make it possible '
1821 'to have the main solution untouched by gclient '
1822 '(gclient will check out unmanaged dependencies but '
1823 'will never sync them)')
1824 parser.add_option('--cache-dir',
1825 help='(git only) Cache all git repos into this dir and do '
1826 'shared clones from the cache, instead of cloning '
1827 'directly from the remote. (experimental)')
1828 parser.set_defaults(config_filename=None)
1829 (options, args) = parser.parse_args(args)
1830 if options.output_config_file:
1831 setattr(options, 'config_filename', getattr(options, 'output_config_file'))
1832 if ((options.spec and args) or len(args) > 2 or
1833 (not options.spec and not args)):
1834 parser.error('Inconsistent arguments. Use either --spec or one or 2 args')
1836 client = GClient('.', options)
1838 client.SetConfig(options.spec)
1840 base_url = args[0].rstrip('/')
1841 if not options.name:
1842 name = base_url.split('/')[-1]
1843 if name.endswith('.git'):
1846 # specify an alternate relpath for the given URL.
1848 deps_file = options.deps_file
1851 safesync_url = args[1]
1852 client.SetDefaultConfig(name, deps_file, base_url, safesync_url,
1853 managed=not options.unmanaged,
1854 cache_dir=options.cache_dir)
1859 @subcommand.epilog("""Example:
1860 gclient pack > patch.txt
1861 generate simple patch for configured client and dependences
1863 def CMDpack(parser, args):
1864 """Generates a patch which can be applied at the root of the tree.
1866 Internally, runs 'svn diff'/'git diff' on each checked out module and
1867 dependencies, and performs minimal postprocessing of the output. The
1868 resulting patch is printed to stdout and can be applied to a freshly
1869 checked out tree via 'patch -p0 < patchfile'.
1871 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
1872 help='override deps for the specified (comma-separated) '
1873 'platform(s); \'all\' will process all deps_os '
1875 parser.remove_option('--jobs')
1876 (options, args) = parser.parse_args(args)
1877 # Force jobs to 1 so the stdout is not annotated with the thread ids
1879 client = GClient.LoadCurrentConfig(options)
1881 raise gclient_utils.Error('client not configured; see \'gclient config\'')
1883 # Print out the .gclient file. This is longer than if we just printed the
1884 # client dict, but more legible, and it might contain helpful comments.
1885 print(client.config_content)
1886 return client.RunOnDeps('pack', args)
1889 def CMDstatus(parser, args):
1890 """Shows modification status for every dependencies."""
1891 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
1892 help='override deps for the specified (comma-separated) '
1893 'platform(s); \'all\' will process all deps_os '
1895 (options, args) = parser.parse_args(args)
1896 client = GClient.LoadCurrentConfig(options)
1898 raise gclient_utils.Error('client not configured; see \'gclient config\'')
1900 # Print out the .gclient file. This is longer than if we just printed the
1901 # client dict, but more legible, and it might contain helpful comments.
1902 print(client.config_content)
1903 return client.RunOnDeps('status', args)
1906 @subcommand.epilog("""Examples:
1908 update files from SCM according to current configuration,
1909 *for modules which have changed since last update or sync*
1910 gclient sync --force
1911 update files from SCM according to current configuration, for
1912 all modules (useful for recovering files deleted from local copy)
1913 gclient sync --revision src@31000
1914 update src directory to r31000
1917 If the --output-json option is specified, the following document structure will
1918 be emitted to the provided file. 'null' entries may occur for subprojects which
1919 are present in the gclient solution, but were not processed (due to custom_deps,
1924 "<name>": { # <name> is the posix-normalized path to the solution.
1925 "revision": [<svn rev int>|<git id hex string>|null],
1926 "scm": ["svn"|"git"|null],
1931 def CMDsync(parser, args):
1932 """Checkout/update all modules."""
1933 parser.add_option('-f', '--force', action='store_true',
1934 help='force update even for unchanged modules')
1935 parser.add_option('-n', '--nohooks', action='store_true',
1936 help='don\'t run hooks after the update is complete')
1937 parser.add_option('-p', '--noprehooks', action='store_true',
1938 help='don\'t run pre-DEPS hooks', default=False)
1939 parser.add_option('-r', '--revision', action='append',
1940 dest='revisions', metavar='REV', default=[],
1941 help='Enforces revision/hash for the solutions with the '
1942 'format src@rev. The src@ part is optional and can be '
1943 'skipped. -r can be used multiple times when .gclient '
1944 'has multiple solutions configured and will work even '
1945 'if the src@ part is skipped. Note that specifying '
1946 '--revision means your safesync_url gets ignored.')
1947 parser.add_option('--with_branch_heads', action='store_true',
1948 help='Clone git "branch_heads" refspecs in addition to '
1949 'the default refspecs. This adds about 1/2GB to a '
1950 'full checkout. (git only)')
1951 parser.add_option('--with_tags', action='store_true',
1952 help='Clone git tags in addition to the default refspecs.')
1953 parser.add_option('-t', '--transitive', action='store_true',
1954 help='When a revision is specified (in the DEPS file or '
1955 'with the command-line flag), transitively update '
1956 'the dependencies to the date of the given revision. '
1957 'Only supported for SVN repositories.')
1958 parser.add_option('-H', '--head', action='store_true',
1959 help='skips any safesync_urls specified in '
1960 'configured solutions and sync to head instead')
1961 parser.add_option('-D', '--delete_unversioned_trees', action='store_true',
1962 help='Deletes from the working copy any dependencies that '
1963 'have been removed since the last sync, as long as '
1964 'there are no local modifications. When used with '
1965 '--force, such dependencies are removed even if they '
1966 'have local modifications. When used with --reset, '
1967 'all untracked directories are removed from the '
1968 'working copy, excluding those which are explicitly '
1969 'ignored in the repository.')
1970 parser.add_option('-R', '--reset', action='store_true',
1971 help='resets any local changes before updating (git only)')
1972 parser.add_option('-M', '--merge', action='store_true',
1973 help='merge upstream changes instead of trying to '
1974 'fast-forward or rebase')
1975 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
1976 help='override deps for the specified (comma-separated) '
1977 'platform(s); \'all\' will process all deps_os '
1979 parser.add_option('-m', '--manually_grab_svn_rev', action='store_true',
1980 help='Skip svn up whenever possible by requesting '
1981 'actual HEAD revision from the repository')
1982 parser.add_option('--upstream', action='store_true',
1983 help='Make repo state match upstream branch.')
1984 parser.add_option('--output-json',
1985 help='Output a json document to this path containing '
1986 'summary information about the sync.')
1987 parser.add_option('--no-history', action='store_true',
1988 help='GIT ONLY - Reduces the size/time of the checkout at '
1989 'the cost of no history. Requires Git 1.9+')
1990 parser.add_option('--shallow', action='store_true',
1991 help='GIT ONLY - Do a shallow clone into the cache dir. '
1992 'Requires Git 1.9+')
1993 parser.add_option('--ignore_locks', action='store_true',
1994 help='GIT ONLY - Ignore cache locks.')
1995 (options, args) = parser.parse_args(args)
1996 client = GClient.LoadCurrentConfig(options)
1999 raise gclient_utils.Error('client not configured; see \'gclient config\'')
2001 if options.revisions and options.head:
2002 # TODO(maruel): Make it a parser.error if it doesn't break any builder.
2003 print('Warning: you cannot use both --head and --revision')
2006 # Print out the .gclient file. This is longer than if we just printed the
2007 # client dict, but more legible, and it might contain helpful comments.
2008 print(client.config_content)
2009 ret = client.RunOnDeps('update', args)
2010 if options.output_json:
2012 for d in client.subtree(True):
2013 normed = d.name.replace('\\', '/').rstrip('/') + '/'
2015 'revision': d.got_revision,
2016 'scm': d.used_scm.name if d.used_scm else None,
2017 'url': str(d.url) if d.url else None,
2019 with open(options.output_json, 'wb') as f:
2020 json.dump({'solutions': slns}, f)
2027 def CMDdiff(parser, args):
2028 """Displays local diff for every dependencies."""
2029 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2030 help='override deps for the specified (comma-separated) '
2031 'platform(s); \'all\' will process all deps_os '
2033 (options, args) = parser.parse_args(args)
2034 client = GClient.LoadCurrentConfig(options)
2036 raise gclient_utils.Error('client not configured; see \'gclient config\'')
2038 # Print out the .gclient file. This is longer than if we just printed the
2039 # client dict, but more legible, and it might contain helpful comments.
2040 print(client.config_content)
2041 return client.RunOnDeps('diff', args)
2044 def CMDrevert(parser, args):
2045 """Reverts all modifications in every dependencies.
2047 That's the nuclear option to get back to a 'clean' state. It removes anything
2048 that shows up in svn status."""
2049 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2050 help='override deps for the specified (comma-separated) '
2051 'platform(s); \'all\' will process all deps_os '
2053 parser.add_option('-n', '--nohooks', action='store_true',
2054 help='don\'t run hooks after the revert is complete')
2055 parser.add_option('-p', '--noprehooks', action='store_true',
2056 help='don\'t run pre-DEPS hooks', default=False)
2057 parser.add_option('--upstream', action='store_true',
2058 help='Make repo state match upstream branch.')
2059 (options, args) = parser.parse_args(args)
2060 # --force is implied.
2061 options.force = True
2062 options.reset = False
2063 options.delete_unversioned_trees = False
2064 client = GClient.LoadCurrentConfig(options)
2066 raise gclient_utils.Error('client not configured; see \'gclient config\'')
2067 return client.RunOnDeps('revert', args)
2070 def CMDrunhooks(parser, args):
2071 """Runs hooks for files that have been modified in the local working copy."""
2072 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2073 help='override deps for the specified (comma-separated) '
2074 'platform(s); \'all\' will process all deps_os '
2076 parser.add_option('-f', '--force', action='store_true', default=True,
2077 help='Deprecated. No effect.')
2078 (options, args) = parser.parse_args(args)
2079 client = GClient.LoadCurrentConfig(options)
2081 raise gclient_utils.Error('client not configured; see \'gclient config\'')
2083 # Print out the .gclient file. This is longer than if we just printed the
2084 # client dict, but more legible, and it might contain helpful comments.
2085 print(client.config_content)
2086 options.force = True
2087 options.nohooks = False
2088 return client.RunOnDeps('runhooks', args)
2091 def CMDrevinfo(parser, args):
2092 """Outputs revision info mapping for the client and its dependencies.
2094 This allows the capture of an overall 'revision' for the source tree that
2095 can be used to reproduce the same tree in the future. It is only useful for
2096 'unpinned dependencies', i.e. DEPS/deps references without a svn revision
2097 number or a git hash. A git branch name isn't 'pinned' since the actual
2100 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2101 help='override deps for the specified (comma-separated) '
2102 'platform(s); \'all\' will process all deps_os '
2104 parser.add_option('-a', '--actual', action='store_true',
2105 help='gets the actual checked out revisions instead of the '
2106 'ones specified in the DEPS and .gclient files')
2107 parser.add_option('-s', '--snapshot', action='store_true',
2108 help='creates a snapshot .gclient file of the current '
2109 'version of all repositories to reproduce the tree, '
2111 (options, args) = parser.parse_args(args)
2112 client = GClient.LoadCurrentConfig(options)
2114 raise gclient_utils.Error('client not configured; see \'gclient config\'')
2115 client.PrintRevInfo()
2119 def CMDhookinfo(parser, args):
2120 """Outputs the hooks that would be run by `gclient runhooks`."""
2121 (options, args) = parser.parse_args(args)
2122 options.force = True
2123 client = GClient.LoadCurrentConfig(options)
2125 raise gclient_utils.Error('client not configured; see \'gclient config\'')
2126 client.RunOnDeps(None, [])
2127 print '; '.join(' '.join(hook) for hook in client.GetHooks(options))
2131 def CMDverify(parser, args):
2132 """Verifies the DEPS file deps are only from allowed_hosts."""
2133 (options, args) = parser.parse_args(args)
2134 client = GClient.LoadCurrentConfig(options)
2136 raise gclient_utils.Error('client not configured; see \'gclient config\'')
2137 client.RunOnDeps(None, [])
2138 # Look at each first-level dependency of this gclient only.
2139 for dep in client.dependencies:
2140 bad_deps = dep.findDepsFromNotAllowedHosts()
2143 print "There are deps from not allowed hosts in file %s" % dep.deps_file
2144 for bad_dep in bad_deps:
2145 print "\t%s at %s" % (bad_dep.name, bad_dep.url)
2146 print "allowed_hosts:", ', '.join(dep.allowed_hosts)
2148 raise gclient_utils.Error(
2149 'dependencies from disallowed hosts; check your DEPS file.')
2152 class OptionParser(optparse.OptionParser):
2153 gclientfile_default = os.environ.get('GCLIENT_FILE', '.gclient')
2155 def __init__(self, **kwargs):
2156 optparse.OptionParser.__init__(
2157 self, version='%prog ' + __version__, **kwargs)
2159 # Some arm boards have issues with parallel sync.
2160 if platform.machine().startswith('arm'):
2163 jobs = max(8, gclient_utils.NumLocalCpus())
2165 # Temporary workaround to lower bot-load on SVN server.
2166 # Bypassed if a bot_update flag is detected.
2167 if (os.environ.get('CHROME_HEADLESS') == '1' and
2168 not os.path.exists('update.flag')):
2172 '-j', '--jobs', default=jobs, type='int',
2173 help='Specify how many SCM commands can run in parallel; defaults to '
2174 '%default on this machine')
2176 '-v', '--verbose', action='count', default=0,
2177 help='Produces additional output for diagnostics. Can be used up to '
2178 'three times for more logging info.')
2180 '--gclientfile', dest='config_filename',
2181 help='Specify an alternate %s file' % self.gclientfile_default)
2184 help='create a gclient file containing the provided string. Due to '
2185 'Cygwin/Python brokenness, it can\'t contain any newlines.')
2187 '--no-nag-max', default=False, action='store_true',
2188 help='Ignored for backwards compatibility.')
2190 def parse_args(self, args=None, values=None):
2191 """Integrates standard options processing."""
2192 options, args = optparse.OptionParser.parse_args(self, args, values)
2193 levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG]
2194 logging.basicConfig(
2195 level=levels[min(options.verbose, len(levels) - 1)],
2196 format='%(module)s(%(lineno)d) %(funcName)s:%(message)s')
2197 if options.config_filename and options.spec:
2198 self.error('Cannot specifiy both --gclientfile and --spec')
2199 if (options.config_filename and
2200 options.config_filename != os.path.basename(options.config_filename)):
2201 self.error('--gclientfile target must be a filename, not a path')
2202 if not options.config_filename:
2203 options.config_filename = self.gclientfile_default
2204 options.entries_filename = options.config_filename + '_entries'
2205 if options.jobs < 1:
2206 self.error('--jobs must be 1 or higher')
2208 # These hacks need to die.
2209 if not hasattr(options, 'revisions'):
2210 # GClient.RunOnDeps expects it even if not applicable.
2211 options.revisions = []
2212 if not hasattr(options, 'head'):
2214 if not hasattr(options, 'nohooks'):
2215 options.nohooks = True
2216 if not hasattr(options, 'noprehooks'):
2217 options.noprehooks = True
2218 if not hasattr(options, 'deps_os'):
2219 options.deps_os = None
2220 if not hasattr(options, 'manually_grab_svn_rev'):
2221 options.manually_grab_svn_rev = None
2222 if not hasattr(options, 'force'):
2223 options.force = None
2224 return (options, args)
2227 def disable_buffering():
2228 # Make stdout auto-flush so buildbot doesn't kill us during lengthy
2229 # operations. Python as a strong tendency to buffer sys.stdout.
2230 sys.stdout = gclient_utils.MakeFileAutoFlush(sys.stdout)
2231 # Make stdout annotated with the thread ids.
2232 sys.stdout = gclient_utils.MakeFileAnnotated(sys.stdout)
2236 """Doesn't parse the arguments here, just find the right subcommand to
2238 if sys.hexversion < 0x02060000:
2239 print >> sys.stderr, (
2240 '\nYour python version %s is unsupported, please upgrade.\n' %
2241 sys.version.split(' ', 1)[0])
2243 if not sys.executable:
2244 print >> sys.stderr, (
2245 '\nPython cannot find the location of it\'s own executable.\n')
2247 fix_encoding.fix_encoding()
2250 dispatcher = subcommand.CommandDispatcher(__name__)
2252 return dispatcher.execute(OptionParser(), argv)
2253 except KeyboardInterrupt:
2254 gclient_utils.GClientChildren.KillAllRemainingChildren()
2256 except (gclient_utils.Error, subprocess2.CalledProcessError), e:
2257 print >> sys.stderr, 'Error: %s' % str(e)
2260 gclient_utils.PrintWarnings()
2263 if '__main__' == __name__:
2264 sys.exit(Main(sys.argv[1:]))
2266 # vim: ts=2:sw=2:tw=80:et: