1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """Common python commands used by various build scripts."""
9 from datetime import datetime
24 # TODO(build): Fix this.
25 # This should be absolute import, but that requires fixing all
26 # relative imports first.
27 _path = os.path.realpath(__file__)
28 _path = os.path.normpath(os.path.join(os.path.dirname(_path), '..', '..'))
29 sys.path.insert(0, _path)
30 from chromite.cbuildbot import constants
31 from chromite.lib import signals
32 # Now restore it so that relative scripts don't get cranky.
39 logger = logging.getLogger('chromite')
41 # For use by ShellQuote. Match all characters that the shell might treat
42 # specially. This means a number of things:
43 # - Reserved characters.
44 # - Characters used in expansions (brace, variable, path, globs, etc...).
45 # - Characters that an interactive shell might use (like !).
46 # - Whitespace so that one arg turns into multiple.
47 # See the bash man page as well as the POSIX shell documentation for more info:
48 # http://www.gnu.org/software/bash/manual/bashref.html
49 # http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html
50 _SHELL_QUOTABLE_CHARS = frozenset('[|&;()<> \t!{}[]=*?~$"\'\\#^')
51 # The chars that, when used inside of double quotes, need escaping.
52 # Order here matters as we need to escape backslashes first.
53 _SHELL_ESCAPE_CHARS = r'\"`$'
57 """Quote |s| in a way that is safe for use in a shell.
59 We aim to be safe, but also to produce "nice" output. That means we don't
60 use quotes when we don't need to, and we prefer to use less quotes (like
61 putting it all in single quotes) than more (using double quotes and escaping
62 a bunch of stuff, or mixing the quotes).
64 While python does provide a number of alternatives like:
67 They suffer from various problems like:
68 - Not widely available in different python versions.
69 - Do not produce pretty output in many cases.
70 - Are in modules that rarely otherwise get used.
72 Note: We don't handle reserved shell words like "for" or "case". This is
73 because those only matter when they're the first element in a command, and
74 there is no use case for that. When we want to run commands, we tend to
75 run real programs and not shell ones.
78 s: The string to quote.
81 A safely (possibly quoted) string.
85 # See if no quoting is needed so we can return the string as-is.
87 if c in _SHELL_QUOTABLE_CHARS:
95 # See if we can use single quotes first. Output is nicer.
99 # Have to use double quotes. Escape the few chars that still expand when
100 # used inside of double quotes.
101 for c in _SHELL_ESCAPE_CHARS:
103 s = s.replace(c, r'\%s' % c)
108 """Do the opposite of ShellQuote.
109 This function assumes that the input is a valid escaped string. The behaviour
110 is undefined on malformed strings.
113 s: An escaped string.
116 The unescaped version of the string.
130 while i < len(s) - 1:
131 # Skip the backslash when it makes sense.
132 if s[i] == '\\' and s[i + 1] in _SHELL_ESCAPE_CHARS:
136 return output + s[i] if i < len(s) else output
140 """Translate a command list into a space-separated string.
142 The resulting string should be suitable for logging messages and for
143 pasting into a terminal to run. Command arguments are surrounded by
144 quotes to keep them grouped, even if an argument has spaces in it.
147 ['a', 'b'] ==> "'a' 'b'"
148 ['a b', 'c'] ==> "'a b' 'c'"
149 ['a', 'b\'c'] ==> '\'a\' "b\'c"'
150 [u'a', "/'$b"] ==> '\'a\' "/\'$b"'
152 See unittest for additional (tested) examples.
155 cmd: List of command arguments.
158 String representing full command.
160 # Use str before repr to translate unicode strings to regular strings.
161 return ' '.join(ShellQuote(arg) for arg in cmd)
164 class CommandResult(object):
165 """An object to store various attributes of a child process."""
167 def __init__(self, cmd=None, error=None, output=None, returncode=None):
171 self.returncode = returncode
175 """Return self.cmd as a space-separated string, useful for log messages."""
176 return CmdToStr(self.cmd)
179 class RunCommandError(Exception):
180 """Error caught in RunCommand() method."""
182 def __init__(self, msg, result, exception=None):
183 self.msg, self.result, self.exception = msg, result, exception
184 if exception is not None and not isinstance(exception, Exception):
185 raise ValueError('exception must be an exception instance; got %r'
187 Exception.__init__(self, msg)
188 self.args = (msg, result, exception)
190 def Stringify(self, error=True, output=True):
191 """Custom method for controlling what is included in stringifying this.
193 Each individual argument is the literal name of an attribute
194 on the result object; if False, that value is ignored for adding
195 to this string content. If true, it'll be incorporated.
198 error: See comment about individual arguments above.
199 output: See comment about individual arguments above.
201 items = ['return code: %s' % (self.result.returncode,)]
202 if error and self.result.error:
203 items.append(self.result.error)
204 if output and self.result.output:
205 items.append(self.result.output)
206 items.append(self.msg)
207 return '\n'.join(items)
210 # __str__ needs to return ascii, thus force a conversion to be safe.
211 return self.Stringify().decode('utf-8').encode('ascii', 'xmlcharrefreplace')
213 def __eq__(self, other):
214 return (type(self) == type(other) and
215 self.args == other.args)
217 def __ne__(self, other):
218 return not self.__eq__(other)
221 class TerminateRunCommandError(RunCommandError):
222 """We were signaled to shutdown while running a command.
224 Client code shouldn't generally know, nor care about this class. It's
225 used internally to suppress retry attempts when we're signaled to die.
229 def SudoRunCommand(cmd, user='root', **kwargs):
230 """Run a command via sudo.
232 Client code must use this rather than coming up with their own RunCommand
233 invocation that jams sudo in- this function is used to enforce certain
234 rules in our code about sudo usage, and as a potential auditing point.
237 cmd: The command to run. See RunCommand for rules of this argument-
238 SudoRunCommand purely prefixes it with sudo.
239 user: The user to run the command as.
240 kwargs: See RunCommand options, it's a direct pass thru to it.
241 Note that this supports a 'strict' keyword that defaults to True.
242 If set to False, it'll suppress strict sudo behavior.
245 See RunCommand documentation.
248 This function may immediately raise RunCommandError if we're operating
249 in a strict sudo context and the API is being misused.
250 Barring that, see RunCommand's documentation- it can raise the same things
255 strict = kwargs.pop('strict', True)
257 if user == 'root' and os.geteuid() == 0:
258 return RunCommand(cmd, **kwargs)
260 if strict and STRICT_SUDO:
261 if 'CROS_SUDO_KEEP_ALIVE' not in os.environ:
262 raise RunCommandError(
263 'We were invoked in a strict sudo non - interactive context, but no '
264 'sudo keep alive daemon is running. This is a bug in the code.',
265 CommandResult(cmd=cmd, returncode=126))
269 sudo_cmd += ['-u', user]
271 # Pass these values down into the sudo environment, since sudo will
272 # just strip them normally.
273 extra_env = kwargs.pop('extra_env', None)
274 extra_env = {} if extra_env is None else extra_env.copy()
276 for var in constants.ENV_PASSTHRU:
277 if var not in extra_env and var in os.environ:
278 extra_env[var] = os.environ[var]
280 sudo_cmd.extend('%s=%s' % (k, v) for k, v in extra_env.iteritems())
282 # Finally, block people from passing options to sudo.
283 sudo_cmd.append('--')
285 if isinstance(cmd, basestring):
286 # We need to handle shell ourselves so the order is correct:
287 # $ sudo [sudo args] -- bash -c '[shell command]'
288 # If we let RunCommand take care of it, we'd end up with:
289 # $ bash -c 'sudo [sudo args] -- [shell command]'
290 shell = kwargs.pop('shell', False)
292 raise Exception('Cannot run a string command without a shell')
293 sudo_cmd.extend(['/bin/bash', '-c', cmd])
297 return RunCommand(sudo_cmd, **kwargs)
300 def _KillChildProcess(proc, kill_timeout, cmd, original_handler, signum, frame):
301 """Functor that when curried w/ the appropriate arguments, is used as a signal
302 handler by RunCommand.
304 This is internal to Runcommand. No other code should use this.
307 # If we've been invoked because of a signal, ignore delivery of that signal
308 # from this point forward. The invoking context of _KillChildProcess
309 # restores signal delivery to what it was prior; we suppress future delivery
310 # till then since this code handles SIGINT/SIGTERM fully including
311 # delivering the signal to the original handler on the way out.
312 signal.signal(signum, signal.SIG_IGN)
314 # Do not trust Popen's returncode alone; we can be invoked from contexts where
315 # the Popen instance was created, but no process was generated.
316 if proc.returncode is None and proc.pid is not None:
319 while proc.poll() is None and kill_timeout >= 0:
323 if proc.poll() is None:
324 # Still doesn't want to die. Too bad, so sad, time to die.
326 except EnvironmentError as e:
327 Warning('Ignoring unhandled exception in _KillChildProcess: %s', e)
329 # Ensure our child process has been reaped.
332 if not signals.RelaySignal(original_handler, signum, frame):
333 # Mock up our own, matching exit code for signaling.
334 cmd_result = CommandResult(cmd=cmd, returncode=signum << 8)
335 raise TerminateRunCommandError('Received signal %i' % signum, cmd_result)
338 class _Popen(subprocess.Popen):
339 """subprocess.Popen derivative customized for our usage.
341 Specifically, we fix terminate/send_signal/kill to work if the child process
342 was a setuid binary; on vanilla kernels, the parent can wax the child
343 regardless, on goobuntu this apparently isn't allowed, thus we fall back
344 to the sudo machinery we have.
346 While we're overriding send_signal, we also suppress ESRCH being raised
347 if the process has exited, and suppress signaling all together if the process
348 has knowingly been waitpid'd already.
351 def send_signal(self, signum):
352 if self.returncode is not None:
353 # The original implementation in Popen would allow signaling whatever
354 # process now occupies this pid, even if the Popen object had waitpid'd.
355 # Since we can escalate to sudo kill, we do not want to allow that.
356 # Fixing this addresses that angle, and makes the API less sucky in the
361 os.kill(self.pid, signum)
362 except EnvironmentError as e:
363 if e.errno == errno.EPERM:
364 # Kill returns either 0 (signal delivered), or 1 (signal wasn't
365 # delivered). This isn't particularly informative, but we still
366 # need that info to decide what to do, thus the error_code_ok=True.
367 ret = SudoRunCommand(['kill', '-%i' % signum, str(self.pid)],
368 print_cmd=False, redirect_stdout=True,
369 redirect_stderr=True, error_code_ok=True)
370 if ret.returncode == 1:
371 # The kill binary doesn't distinguish between permission denied,
372 # and the pid is missing. Denied can only occur under weird
373 # grsec/selinux policies. We ignore that potential and just
374 # assume the pid was already dead and try to reap it.
376 elif e.errno == errno.ESRCH:
377 # Since we know the process is dead, reap it now.
378 # Normally Popen would throw this error- we suppress it since frankly
379 # that's a misfeature and we're already overriding this method.
385 # pylint: disable=W0622
386 def RunCommand(cmd, print_cmd=True, error_message=None, redirect_stdout=False,
387 redirect_stderr=False, cwd=None, input=None, enter_chroot=False,
388 shell=False, env=None, extra_env=None, ignore_sigint=False,
389 combine_stdout_stderr=False, log_stdout_to_file=None,
390 chroot_args=None, debug_level=logging.INFO,
391 error_code_ok=False, kill_timeout=1, log_output=False,
392 stdout_to_pipe=False, capture_output=False, quiet=False):
396 cmd: cmd to run. Should be input to subprocess.Popen. If a string, shell
397 must be true. Otherwise the command must be an array of arguments, and
399 print_cmd: prints the command before running it.
400 error_message: prints out this message when an error occurs.
401 redirect_stdout: returns the stdout.
402 redirect_stderr: holds stderr output until input is communicated.
403 cwd: the working directory to run this cmd.
404 input: input to pipe into this command through stdin.
405 enter_chroot: this command should be run from within the chroot. If set,
406 cwd must point to the scripts directory. If we are already inside the
407 chroot, this command will be run as if |enter_chroot| is False.
408 shell: Controls whether we add a shell as a command interpreter. See cmd
409 since it has to agree as to the type.
410 env: If non-None, this is the environment for the new process. If
411 enter_chroot is true then this is the environment of the enter_chroot,
412 most of which gets removed from the cmd run.
413 extra_env: If set, this is added to the environment for the new process.
414 In enter_chroot=True case, these are specified on the post-entry
415 side, and so are often more useful. This dictionary is not used to
416 clear any entries though.
417 ignore_sigint: If True, we'll ignore signal.SIGINT before calling the
418 child. This is the desired behavior if we know our child will handle
419 Ctrl-C. If we don't do this, I think we and the child will both get
420 Ctrl-C at the same time, which means we'll forcefully kill the child.
421 combine_stdout_stderr: Combines stdout and stderr streams into stdout.
422 log_stdout_to_file: If set, redirects stdout to file specified by this path.
423 If |combine_stdout_stderr| is set to True, then stderr will also be logged
424 to the specified file.
425 chroot_args: An array of arguments for the chroot environment wrapper.
426 debug_level: The debug level of RunCommand's output - applies to output
427 coming from subprocess as well.
428 error_code_ok: Does not raise an exception when command returns a non-zero
429 exit code. Instead, returns the CommandResult object
430 containing the exit code. Note: will still raise an
431 exception if the cmd file does not exist.
432 kill_timeout: If we're interrupted, how long should we give the invoked
433 process to shutdown from a SIGTERM before we SIGKILL it.
434 Specified in seconds.
435 log_output: Log the command and its output automatically.
436 stdout_to_pipe: Redirect stdout to pipe.
437 capture_output: Set |redirect_stdout| and |redirect_stderr| to True.
438 quiet: Set |print_cmd| to False, |stdout_to_pipe| and
439 |combine_stdout_stderr| to True.
442 A CommandResult object.
445 RunCommandError: Raises exception on error with optional error_message.
448 redirect_stdout, redirect_stderr = True, True
452 stdout_to_pipe, combine_stdout_stderr = True, True
454 # Set default for variables.
458 cmd_result = CommandResult()
460 mute_output = logger.getEffectiveLevel() > debug_level
462 # Force the timeout to float; in the process, if it's not convertible,
463 # a self-explanatory exception will be thrown.
464 kill_timeout = float(kill_timeout)
468 return tempfile.TemporaryFile(bufsize=0)
469 except EnvironmentError as e:
470 if e.errno != errno.ENOENT:
472 # This can occur if we were pointed at a specific location for our
473 # TMP, but that location has since been deleted. Suppress that issue
474 # in this particular case since our usage gurantees deletion,
475 # and since this is primarily triggered during hard cgroups shutdown.
476 return tempfile.TemporaryFile(bufsize=0, dir='/tmp')
478 # Modify defaults based on parameters.
479 # Note that tempfiles must be unbuffered else attempts to read
480 # what a separate process did to that file can result in a bad
482 if log_stdout_to_file:
483 stdout = open(log_stdout_to_file, 'w+')
485 stdout = subprocess.PIPE
486 elif redirect_stdout or mute_output or log_output:
487 stdout = _get_tempfile()
489 if combine_stdout_stderr:
490 stderr = subprocess.STDOUT
491 elif redirect_stderr or mute_output or log_output:
492 stderr = _get_tempfile()
494 # If subprocesses have direct access to stdout or stderr, they can bypass
495 # our buffers, so we need to flush to ensure that output is not interleaved.
496 if stdout is None or stderr is None:
501 stdin = subprocess.PIPE
503 if isinstance(cmd, basestring):
505 raise Exception('Cannot run a string command without a shell')
506 cmd = ['/bin/bash', '-c', cmd]
509 raise Exception('Cannot run an array command with a shell')
511 # If we are using enter_chroot we need to use enterchroot pass env through
512 # to the final command.
513 env = env.copy() if env is not None else os.environ.copy()
514 if enter_chroot and not IsInsideChroot():
515 wrapper = ['cros_sdk']
518 wrapper += chroot_args
521 wrapper.extend('%s=%s' % (k, v) for k, v in extra_env.iteritems())
523 cmd = wrapper + ['--'] + cmd
526 env.update(extra_env)
528 for var in constants.ENV_PASSTHRU:
529 if var not in env and var in os.environ:
530 env[var] = os.environ[var]
532 # Print out the command before running.
533 if print_cmd or log_output:
535 logger.log(debug_level, 'RunCommand: %s in %s', CmdToStr(cmd), cwd)
537 logger.log(debug_level, 'RunCommand: %s', CmdToStr(cmd))
542 # Verify that the signals modules is actually usable, and won't segfault
543 # upon invocation of getsignal. See signals.SignalModuleUsable for the
544 # details and upstream python bug.
545 use_signals = signals.SignalModuleUsable()
547 proc = _Popen(cmd, cwd=cwd, stdin=stdin, stdout=stdout,
548 stderr=stderr, shell=False, env=env,
553 old_sigint = signal.signal(signal.SIGINT, signal.SIG_IGN)
555 old_sigint = signal.getsignal(signal.SIGINT)
556 signal.signal(signal.SIGINT,
557 functools.partial(_KillChildProcess, proc, kill_timeout,
560 old_sigterm = signal.getsignal(signal.SIGTERM)
561 signal.signal(signal.SIGTERM,
562 functools.partial(_KillChildProcess, proc, kill_timeout,
566 (cmd_result.output, cmd_result.error) = proc.communicate(input)
569 signal.signal(signal.SIGINT, old_sigint)
570 signal.signal(signal.SIGTERM, old_sigterm)
572 if stdout and not log_stdout_to_file and not stdout_to_pipe:
574 cmd_result.output = stdout.read()
577 if stderr and stderr != subprocess.STDOUT:
579 cmd_result.error = stderr.read()
582 cmd_result.returncode = proc.returncode
585 if cmd_result.output:
586 logger.log(debug_level, '(stdout):\n%s' % cmd_result.output)
588 logger.log(debug_level, '(stderr):\n%s' % cmd_result.error)
590 if not error_code_ok and proc.returncode:
591 msg = ('Failed command "%s", cwd=%s, extra env=%r'
592 % (CmdToStr(cmd), cwd, extra_env))
594 msg += '\n%s' % error_message
595 raise RunCommandError(msg, cmd_result)
598 if e.errno == errno.EACCES:
599 estr += '; does the program need `chmod a+x`?'
600 raise RunCommandError(estr, CommandResult(cmd=cmd), exception=e)
603 # Ensure the process is dead.
604 _KillChildProcess(proc, kill_timeout, cmd, None, None, None)
609 # Convenience RunCommand methods.
611 # We don't use functools.partial because it binds the methods at import time,
612 # which doesn't work well with unit tests, since it bypasses the mock that may
613 # be set up for RunCommand.
615 def DebugRunCommand(*args, **kwargs):
616 kwargs.setdefault('debug_level', logging.DEBUG)
617 return RunCommand(*args, **kwargs)
620 class DieSystemExit(SystemExit):
621 """Custom Exception used so we can intercept this if necessary."""
624 def Die(message, *args):
625 """Emits an error message with a stack trace and halts execution.
628 message: The message to be emitted before exiting.
630 logger.error(message, *args)
631 raise DieSystemExit(1)
634 def Error(message, *args, **kwargs):
635 """Emits a red warning message using the logging module."""
636 logger.error(message, *args, **kwargs)
639 # pylint: disable=W0622
640 def Warning(message, *args, **kwargs):
641 """Emits a warning message using the logging module."""
642 logger.warn(message, *args, **kwargs)
645 def Info(message, *args, **kwargs):
646 """Emits an info message using the logging module."""
647 logger.info(message, *args, **kwargs)
650 def Debug(message, *args, **kwargs):
651 """Emits a debugging message using the logging module."""
652 logger.debug(message, *args, **kwargs)
655 def PrintBuildbotLink(text, url, handle=None):
656 """Prints out a link to buildbot."""
657 text = ' '.join(text.split())
658 (handle or sys.stderr).write('\n@@@STEP_LINK@%s@%s@@@\n' % (text, url))
661 def PrintBuildbotStepText(text, handle=None):
662 """Prints out stage text to buildbot."""
663 text = ' '.join(text.split())
664 (handle or sys.stderr).write('\n@@@STEP_TEXT@%s@@@\n' % (text,))
667 def PrintBuildbotStepWarnings(handle=None):
668 """Marks a stage as having warnings."""
669 (handle or sys.stderr).write('\n@@@STEP_WARNINGS@@@\n')
672 def PrintBuildbotStepFailure(handle=None):
673 """Marks a stage as having failures."""
674 (handle or sys.stderr).write('\n@@@STEP_FAILURE@@@\n')
677 def PrintBuildbotStepName(name, handle=None):
678 """Marks a step name for buildbot to display."""
679 (handle or sys.stderr).write('\n@@@BUILD_STEP %s@@@\n' % name)
682 def ListFiles(base_dir):
683 """Recursively list files in a directory.
686 base_dir: directory to start recursively listing in.
689 A list of files relative to the base_dir path or
690 An empty list of there are no files in the directories.
692 directories = [base_dir]
695 directory = directories.pop()
696 for name in os.listdir(directory):
697 fullpath = os.path.join(directory, name)
698 if os.path.isfile(fullpath):
699 files_list.append(fullpath)
700 elif os.path.isdir(fullpath):
701 directories.append(fullpath)
706 def IsInsideChroot():
707 """Returns True if we are inside chroot."""
708 return os.path.exists('/etc/cros_chroot_version')
711 def AssertInsideChroot():
712 """Die if we are outside the chroot"""
713 if not IsInsideChroot():
714 Die('%s: please run inside the chroot', os.path.basename(sys.argv[0]))
717 def AssertOutsideChroot():
718 """Die if we are inside the chroot"""
720 Die('%s: please run outside the chroot', os.path.basename(sys.argv[0]))
723 def GetChromeosVersion(str_obj):
724 """Helper method to parse output for CHROMEOS_VERSION_STRING.
727 str_obj: a string, which may contain Chrome OS version info.
730 A string, value of CHROMEOS_VERSION_STRING environment variable set by
731 chromeos_version.sh. Or None if not found.
733 if str_obj is not None:
734 match = re.search(r'CHROMEOS_VERSION_STRING=([0-9_.]+)', str_obj)
735 if match and match.group(1):
736 Info('CHROMEOS_VERSION_STRING = %s' % match.group(1))
737 return match.group(1)
739 Info('CHROMEOS_VERSION_STRING NOT found')
743 def GetHostName(fully_qualified=False):
744 """Return hostname of current machine, with domain if |fully_qualified|."""
745 hostname = socket.gethostbyaddr(socket.gethostname())[0]
750 return hostname.partition('.')[0]
754 """Return domain of current machine.
756 If there is no domain, return 'localdomain'.
759 hostname = GetHostName(fully_qualified=True)
760 domain = hostname.partition('.')[2]
761 return domain if domain else 'localdomain'
764 def TimedCommand(functor, *args, **kwargs):
765 """Wrapper for simple log timing of other python functions.
767 If you want to log info about how long it took to run an arbitrary command,
768 you would do something like:
769 TimedCommand(RunCommand, ['wget', 'http://foo'])
772 functor: The function to run.
773 args: The args to pass to the function.
774 kwargs: Optional args to pass to the function.
775 timed_log_level: The log level to use (defaults to info).
776 timed_log_msg: The message to log with timing info appended (defaults to
777 details about the call made). It must include a %s to hold
778 the time delta details.
780 log_msg = kwargs.pop('timed_log_msg', '%s(*%r, **%r) took: %%s'
781 % (functor.__name__, args, kwargs))
782 log_level = kwargs.pop('timed_log_level', logging.INFO)
783 start = datetime.now()
784 ret = functor(*args, **kwargs)
785 logger.log(log_level, log_msg, datetime.now() - start)
795 def FindCompressor(compression, chroot=None):
796 """Locate a compressor utility program (possibly in a chroot).
798 Since we compress/decompress a lot, make it easy to locate a
799 suitable utility program in a variety of locations. We favor
800 the one in the chroot over /, and the parallel implementation
801 over the single threaded one.
804 compression: The type of compression desired.
805 chroot: Optional path to a chroot to search.
808 Path to a compressor.
811 ValueError: If compression is unknown.
813 if compression == COMP_GZIP:
816 elif compression == COMP_BZIP2:
819 elif compression == COMP_XZ:
822 elif compression == COMP_NONE:
825 raise ValueError('unknown compression')
832 for prog in [para, std]:
834 for subdir in ['', 'usr']:
835 path = os.path.join(root, subdir, 'bin', prog)
836 if os.path.exists(path):
842 def CompressionStrToType(s):
843 """Convert a compression string type to a constant.
849 A constant, or None if the compression type is unknown.
857 return _COMP_STR.get(s)
862 def CompressFile(infile, outfile):
863 """Compress a file using compressor specified by |outfile| suffix.
866 infile: File to compress.
867 outfile: Name of output file. Compression used is based on the
868 type of suffix of the name specified (e.g.: .bz2).
870 comp_str = outfile.rsplit('.', 1)[-1]
871 comp_type = CompressionStrToType(comp_str)
872 assert comp_type and comp_type != COMP_NONE
873 comp = FindCompressor(comp_type)
874 cmd = [comp, '-c', infile]
875 RunCommand(cmd, log_stdout_to_file=outfile)
878 def UncompressFile(infile, outfile):
879 """Uncompress a file using compressor specified by |infile| suffix.
882 infile: File to uncompress. Compression used is based on the
883 type of suffix of the name specified (e.g.: .bz2).
884 outfile: Name of output file.
886 comp_str = infile.rsplit('.', 1)[-1]
887 comp_type = CompressionStrToType(comp_str)
888 assert comp_type and comp_type != COMP_NONE
889 comp = FindCompressor(comp_type)
890 cmd = [comp, '-dc', infile]
891 RunCommand(cmd, log_stdout_to_file=outfile)
894 def CreateTarball(target, cwd, sudo=False, compression=COMP_XZ, chroot=None,
895 inputs=None, extra_args=None, **kwargs):
896 """Create a tarball. Executes 'tar' on the commandline.
899 target: The path of the tar file to generate.
900 cwd: The directory to run the tar command.
901 sudo: Whether to run with "sudo".
902 compression: The type of compression desired. See the FindCompressor
903 function for details.
904 chroot: See FindCompressor().
905 inputs: A list of files or directories to add to the tarball. If unset,
907 extra_args: A list of extra args to pass to "tar".
908 kwargs: Any RunCommand options/overrides to use.
911 The cmd_result object returned by the RunCommand invocation.
915 if extra_args is None:
917 kwargs.setdefault('debug_level', logging.DEBUG)
919 comp = FindCompressor(compression, chroot=chroot)
922 ['--sparse', '-I', comp, '-cf', target] +
924 rc_func = SudoRunCommand if sudo else RunCommand
925 return rc_func(cmd, cwd=cwd, **kwargs)
928 def GetInput(prompt):
929 """Helper function to grab input from a user. Makes testing easier."""
930 return raw_input(prompt)
933 def GetChoice(prompt, options):
934 """Ask user to choose an option from the list.
937 prompt: The text to display before listing options.
938 options: The list of options to display.
945 for opt, i in zip(options, xrange(len(options))):
946 prompt += '\n [%d]: %s' % (i, opt)
948 prompt = '%s\nEnter your choice to continue [0-%d]: ' % (
949 prompt, len(options) - 1)
953 choice = int(GetInput(prompt))
955 print 'Input value is not an integer'
958 if choice < 0 or choice >= len(options):
959 print 'Input value is out of range'
966 def BooleanPrompt(prompt='Do you want to continue?', default=True,
967 true_value='yes', false_value='no', prolog=None):
968 """Helper function for processing boolean choice prompts.
971 prompt: The question to present to the user.
972 default: Boolean to return if the user just presses enter.
973 true_value: The text to display that represents a True returned.
974 false_value: The text to display that represents a False returned.
975 prolog: The text to display before prompt.
980 true_value, false_value = true_value.lower(), false_value.lower()
981 true_text, false_text = true_value, false_value
982 if true_value == false_value:
983 raise ValueError('true_value and false_value must differ: got %r'
987 true_text = true_text[0].upper() + true_text[1:]
989 false_text = false_text[0].upper() + false_text[1:]
991 prompt = ('\n%s (%s/%s)? ' % (prompt, true_text, false_text))
994 prompt = ('\n%s\n%s' % (prolog, prompt))
998 response = GetInput(prompt).lower()
1000 # If the user hits CTRL+D, or stdin is disabled, use the default.
1003 except KeyboardInterrupt:
1004 # If the user hits CTRL+C, just exit the process.
1006 Die('CTRL+C detected; exiting')
1010 if true_value.startswith(response):
1011 if not false_value.startswith(response):
1013 # common prefix between the two...
1014 elif false_value.startswith(response):
1018 def BooleanShellValue(sval, default, msg=None):
1019 """See if the string value is a value users typically consider as boolean
1021 Often times people set shell variables to different values to mean "true"
1022 or "false". For example, they can do:
1026 Handle all that user ugliness here.
1028 If the user picks an invalid value, you can use |msg| to display a non-fatal
1029 warning rather than raising an exception.
1032 sval: The string value we got from the user.
1033 default: If we can't figure out if the value is true or false, use this.
1034 msg: If |sval| is an unknown value, use |msg| to warn the user that we
1035 could not decode the input. Otherwise, raise ValueError().
1038 The interpreted boolean value of |sval|.
1041 ValueError() if |sval| is an unknown value and |msg| is not set.
1046 if isinstance(sval, basestring):
1048 if s in ('yes', 'y', '1', 'true'):
1050 elif s in ('no', 'n', '0', 'false'):
1054 Warning('%s: %r' % (msg, sval))
1057 raise ValueError('Could not decode as a boolean value: %r' % sval)
1060 # Suppress whacked complaints about abstract class being unused.
1061 # pylint: disable=R0921
1062 class MasterPidContextManager(object):
1063 """Allow context managers to restrict their exit to within the same PID."""
1065 # In certain cases we actually want this ran outside
1066 # of the main pid- specifically in backup processes
1068 ALTERNATE_MASTER_PID = None
1071 self._invoking_pid = None
1073 def __enter__(self):
1074 self._invoking_pid = os.getpid()
1075 return self._enter()
1077 def __exit__(self, exc_type, exc, traceback):
1078 curpid = os.getpid()
1079 if curpid == self.ALTERNATE_MASTER_PID:
1080 self._invoking_pid = curpid
1081 if curpid == self._invoking_pid:
1082 return self._exit(exc_type, exc, traceback)
1085 raise NotImplementedError(self, '_enter')
1087 def _exit(self, exc_type, exc, traceback):
1088 raise NotImplementedError(self, '_exit')
1091 @contextlib.contextmanager
1092 def NoOpContextManager():
1096 def AllowDisabling(enabled, functor, *args, **kwargs):
1097 """Context Manager wrapper that can be used to enable/disable usage.
1099 This is mainly useful to control whether or not a given Context Manager
1104 with AllowDisabling(options.timeout <= 0, Timeout, options.timeout):
1105 ... do code w/in a timeout context..
1107 If options.timeout is a positive integer, then the_Timeout context manager is
1108 created and ran. If it's zero or negative, then the timeout code is disabled.
1110 While Timeout *could* handle this itself, it's redundant having each
1111 implementation do this, thus the generic wrapper.
1114 return functor(*args, **kwargs)
1115 return NoOpContextManager()
1118 class ContextManagerStack(object):
1119 """Context manager that is designed to safely allow nesting and stacking.
1121 Python2.7 directly supports a with syntax removing the need for this,
1122 although this form avoids indentation hell if there is a lot of context
1125 For Python2.6, see http://docs.python.org/library/contextlib.html; the short
1126 version is that there is a race in the available stdlib/language rules under
1127 2.6 when dealing w/ multiple context managers, thus this safe version was
1130 For each context manager added to this instance, it will unwind them,
1131 invoking them as if it had been constructed as a set of manually nested
1138 def Add(self, functor, *args, **kwargs):
1139 """Add a context manager onto the stack.
1141 Usage of this is essentially the following:
1142 >>> stack.add(Timeout, 60)
1144 It must be done in this fashion, else there is a mild race that exists
1145 between context manager instantiation and initial __enter__.
1147 Invoking it in the form specified eliminates that race.
1150 functor: A callable to instantiate a context manager.
1151 args and kwargs: positional and optional args to functor.
1154 The newly created (and __enter__'d) context manager.
1158 obj = functor(*args, **kwargs)
1163 self._stack.append(obj)
1165 def __enter__(self):
1166 # Nothing to do in this case. The individual __enter__'s are done
1167 # when the context managers are added, which will likely be after
1168 # the __enter__ method of this stack is called.
1171 def __exit__(self, exc_type, exc, traceback):
1172 # Exit each context manager in stack in reverse order, tracking the results
1173 # to know whether or not to suppress the exception raised (or to switch that
1174 # exception to a new one triggered by an individual handler's __exit__).
1175 for handler in reversed(self._stack):
1176 # pylint: disable=W0702
1178 if handler.__exit__(exc_type, exc, traceback):
1179 exc_type = exc = traceback = None
1181 exc_type, exc, traceback = sys.exc_info()
1185 # Return True if any exception was handled.
1186 if all(x is None for x in (exc_type, exc, traceback)):
1189 # Raise any exception that is left over from exiting all context managers.
1190 # Normally a single context manager would return False to allow caller to
1191 # re-raise the exception itself, but here the exception might have been
1192 # raised during the exiting of one of the individual context managers.
1193 raise exc_type, exc, traceback
1196 def SetupBasicLogging(level=logging.DEBUG):
1197 """Sets up basic logging to use format from constants."""
1198 logging_format = '%(asctime)s - %(filename)s - %(levelname)-8s: %(message)s'
1199 date_format = constants.LOGGER_DATE_FMT
1200 logging.basicConfig(level=level, format=logging_format,
1201 datefmt=date_format)
1204 class ApiMismatchError(Exception):
1205 """Raised by GetTargetChromiteApiVersion."""
1208 class NoChromiteError(Exception):
1209 """Raised when an expected chromite installation was missing."""
1212 def GetTargetChromiteApiVersion(buildroot, validate_version=True):
1213 """Get the re-exec API version of the target chromite.
1216 buildroot: The directory containing the chromite to check.
1217 validate_version: If set to true, checks the target chromite for
1218 compatibility, and raises an ApiMismatchError when there is an
1222 The version number in (major, minor) tuple.
1225 May raise an ApiMismatchError if validate_version is set.
1229 [constants.PATH_TO_CBUILDBOT, '--reexec-api-version'],
1230 cwd=buildroot, error_code_ok=True, capture_output=True)
1231 except RunCommandError:
1232 # Although error_code_ok=True was used, this exception will still be raised
1233 # if the executible did not exist.
1234 full_cbuildbot_path = os.path.join(buildroot, constants.PATH_TO_CBUILDBOT)
1235 if not os.path.exists(full_cbuildbot_path):
1236 raise NoChromiteError('No cbuildbot found in buildroot %s, expected to '
1237 'find %s. ' % (buildroot, full_cbuildbot_path))
1240 # If the command failed, then we're targeting a cbuildbot that lacks the
1241 # option; assume 0:0 (ie, initial state).
1243 if api.returncode == 0:
1244 major, minor = map(int, api.output.strip().split('.', 1))
1246 if validate_version and major != constants.REEXEC_API_MAJOR:
1247 raise ApiMismatchError(
1248 'The targeted version of chromite in buildroot %s requires '
1249 'api version %i, but we are api version %i. We cannot proceed.'
1250 % (buildroot, major, constants.REEXEC_API_MAJOR))
1255 def GetChrootVersion(chroot=None, buildroot=None):
1256 """Extract the version of the chroot.
1259 chroot: Full path to the chroot to examine.
1260 buildroot: If |chroot| is not set, find it relative to |buildroot|.
1263 The version of the chroot dir.
1265 if chroot is None and buildroot is None:
1266 raise ValueError('need either |chroot| or |buildroot| to search')
1268 from chromite.lib import osutils
1270 chroot = os.path.join(buildroot, constants.DEFAULT_CHROOT_DIR)
1271 ver_path = os.path.join(chroot, 'etc', 'cros_chroot_version')
1273 return osutils.ReadFile(ver_path).strip()
1275 Warning('could not read %s', ver_path)
1279 def iflatten_instance(iterable, terminate_on_kls=(basestring,)):
1280 """Derivative of snakeoil.lists.iflatten_instance; flatten an object.
1282 Given an object, flatten it into a single depth iterable-
1283 stopping descent on objects that either aren't iterable, or match
1284 isinstance(obj, terminate_on_kls).
1287 >>> print list(iflatten_instance([1, 2, "as", ["4", 5]))
1288 [1, 2, "as", "4", 5]
1290 def descend_into(item):
1291 if isinstance(item, terminate_on_kls):
1297 # Note strings can be infinitely descended through- thus this
1298 # recursion limiter.
1299 return not isinstance(item, basestring) or len(item) > 1
1301 if not descend_into(iterable):
1304 for item in iterable:
1305 if not descend_into(item):
1308 for subitem in iflatten_instance(item, terminate_on_kls):
1312 # TODO: Remove this once we move to snakeoil.
1313 def load_module(name):
1317 name: python dotted namespace path of the module to import
1323 FailedImport if importing fails
1325 m = __import__(name)
1326 # __import__('foo.bar') returns foo, so...
1327 for bit in name.split('.')[1:]:
1332 def PredicateSplit(func, iterable):
1333 """Splits an iterable into two groups based on a predicate return value.
1336 func: A functor that takes an item as its argument and returns a boolean
1337 value indicating which group the item belongs.
1338 iterable: The collection to split.
1341 A tuple containing two lists, the first containing items that func()
1342 returned True for, and the second containing items that func() returned
1345 trues, falses = [], []
1347 (trues if func(x) else falses).append(x)
1348 return trues, falses
1351 @contextlib.contextmanager
1352 def Open(input, mode='r'):
1353 """Convenience ctx that accepts a file path or an already open file object."""
1354 if isinstance(input, basestring):
1355 with open(input, mode=mode) as f:
1361 def LoadKeyValueFile(input, ignore_missing=False, multiline=False):
1362 """Turn a key=value file into a dict
1364 Note: If you're designing a new data store, please use json rather than
1365 this format. This func is designed to work with legacy/external files
1366 where json isn't an option.
1369 input: The file to read. Can be a path or an open file object.
1370 ignore_missing: If the file does not exist, return an empty dict.
1371 multiline: Allow a value enclosed by quotes to span multiple lines.
1374 a dict of all the key=value pairs found in the file.
1379 with Open(input) as f:
1383 line = raw_line.split('#')[0]
1384 if not line.strip():
1387 # Continue processing a multiline value.
1388 if multiline and in_quotes and key:
1389 if line.rstrip()[-1] == in_quotes:
1390 # Wrap up the multiline value if the line ends with a quote.
1391 d[key] += line.rstrip()[:-1]
1397 chunks = line.split('=', 1)
1398 if len(chunks) != 2:
1399 raise ValueError('Malformed key=value file %r; line %r'
1400 % (input, raw_line))
1401 key = chunks[0].strip()
1402 val = chunks[1].strip()
1403 if len(val) >= 2 and val[0] in "\"'" and val[0] == val[-1]:
1404 # Strip matching quotes on the same line.
1406 elif val and multiline and val[0] in "\"'":
1407 # Unmatched quote here indicates a multiline value. Do not
1408 # strip the '\n' at the end of the line.
1410 val = chunks[1].lstrip()[1:]
1412 except EnvironmentError as e:
1413 if not (ignore_missing and e.errno == errno.ENOENT):
1419 def MemoizedSingleCall(functor):
1420 """Decorator for simple functor targets, caching the results
1422 The functor must accept no arguments beyond either a class or self (depending
1423 on if this is used in a classmethod/instancemethod context). Results of the
1424 wrapped method will be written to the class/instance namespace in a specially
1425 named cached value. All future invocations will just reuse that value.
1427 Note that this cache is per-process, so sibling and parent processes won't
1428 notice updates to the cache.
1430 # TODO(build): Should we rebase to snakeoil.klass.cached* functionality?
1432 # pylint: disable=W0212
1434 val = getattr(obj, key, None)
1437 setattr(obj, key, val)
1440 # Dummy up our wrapper to make it look like what we're wrapping,
1441 # and expose the underlying docstrings.
1442 f.__name__ = functor.__name__
1443 f.__module__ = functor.__module__
1444 f.__doc__ = functor.__doc__
1445 f._cache_key = '_%s_cached' % (functor.__name__.lstrip('_'),)
1449 def SafeRun(functors, combine_exceptions=False):
1450 """Executes a list of functors, continuing on exceptions.
1453 functors: An iterable of functors to call.
1454 combine_exceptions: If set, and multiple exceptions are encountered,
1455 SafeRun will raise a RuntimeError containing a list of all the exceptions.
1456 If only one exception is encountered, then the default behavior of
1457 re-raising the original exception with unmodified stack trace will be
1461 The first exception encountered, with corresponding backtrace, unless
1462 |combine_exceptions| is specified and there is more than one exception
1463 encountered, in which case a RuntimeError containing a list of all the
1464 exceptions that were encountered is raised.
1471 except Exception as e:
1472 # Append the exception object and the traceback.
1473 errors.append((e, sys.exc_info()[2]))
1476 if len(errors) == 1 or not combine_exceptions:
1477 # To preserve the traceback.
1478 inst, tb = errors[0]
1479 raise inst, None, tb
1481 raise RuntimeError([e[0] for e in errors])
1484 def ParseDurationToSeconds(duration):
1485 """Parses a string duration of the form HH:MM:SS into seconds.
1488 duration: A string such as '12:43:12' (representing in this case
1489 12 hours, 43 minutes, 12 seconds).
1492 An integer number of seconds.
1494 h, m, s = [int(t) for t in duration.split(':')]
1495 return s + 60 * m + 3600 * h
1498 def UserDateTimeFormat(timeval=None):
1499 """Format a date meant to be viewed by a user
1501 The focus here is to have a format that is easily readable by humans,
1502 but still easy (and unambiguous) for a machine to parse. Hence, we
1503 use the RFC 2822 date format (with timezone name appended).
1506 timeval: Either a datetime object or a floating point time value as accepted
1507 by gmtime()/localtime(). If None, the current time is used.
1510 A string format such as 'Wed, 20 Feb 2013 15:25:15 -0500 (EST)'
1512 if isinstance(timeval, datetime):
1513 timeval = time.mktime(timeval.timetuple())
1514 return '%s (%s)' % (email.utils.formatdate(timeval=timeval, localtime=True),
1515 time.strftime('%Z', time.localtime(timeval)))
1518 def ParseUserDateTimeFormat(time_string):
1519 """Parse a time string into a floating point time value.
1521 This function is essentially the inverse of UserDateTimeFormat.
1524 time_string: A string datetime represetation in RFC 2822 format, such as
1525 'Wed, 20 Feb 2013 15:25:15 -0500 (EST)'.
1528 Floating point Unix timestamp (seconds since epoch).
1530 return email.utils.mktime_tz(email.utils.parsedate_tz(time_string))
1533 def GetDefaultBoard():
1534 """Gets the default board.
1537 The default board (as a string), or None if either the default board
1538 file was missing or malformed.
1540 default_board_file_name = os.path.join(constants.SOURCE_ROOT, 'src',
1541 'scripts', '.default_board')
1543 with open(default_board_file_name) as default_board_file:
1544 default_board = default_board_file.read().strip()
1545 # Check for user typos like whitespace
1546 if not re.match('[a-zA-Z0-9-_]*$', default_board):
1547 Warning('Noticed invalid default board: |%s|. '
1548 'Ignoring this default.', default_board)
1549 default_board = None
1553 return default_board
1556 def GetBoard(device_board, override_board=None, force=False):
1557 """Gets the board name to use.
1559 Ask user to confirm when |override_board| and |device_board| are
1563 device_board: The board detected on the device.
1564 override_board: Overrides the board.
1565 force: Force using the default board if |device_board| is None.
1568 Returns the first non-None board in the following order:
1569 |override_board|, |device_board|, and GetDefaultBoard().
1572 DieSystemExit: If user enters no.
1575 return override_board
1577 board = device_board or GetDefaultBoard()
1578 if not device_board:
1579 msg = 'Cannot detect board name; using default board %s.' % board
1580 if not force and not BooleanPrompt(default=False, prolog=msg):
1588 class AttributeFrozenError(Exception):
1589 """Raised when frozen attribute value is modified."""
1592 class FrozenAttributesClass(type):
1593 """Metaclass for any class to support freezing attribute values.
1595 This metaclass can be used by any class to add the ability to
1596 freeze attribute values with the Freeze method.
1598 Use by adding this line in a class:
1599 __metaclass__ = FrozenAttributesClass
1601 _FROZEN_ERR_MSG = 'Attribute values are frozen, cannot alter %s.'
1603 def __new__(mcs, clsname, bases, scope):
1604 # Create Freeze method that freezes current attributes.
1605 # pylint: disable=E1003
1606 if 'Freeze' in scope:
1607 raise TypeError('Class %s has its own Freeze method, cannot use with'
1608 ' the FrozenAttributesClass metaclass.' % clsname)
1610 # Make sure cls will have _FROZEN_ERR_MSG set.
1611 scope.setdefault('_FROZEN_ERR_MSG', mcs._FROZEN_ERR_MSG)
1614 cls = super(FrozenAttributesClass, mcs).__new__(mcs, clsname, bases, scope)
1616 # Replace cls.__setattr__ with the one that honors freezing.
1617 orig_setattr = cls.__setattr__
1619 def SetAttr(obj, name, value):
1620 """If the object is frozen then abort."""
1621 # pylint: disable=W0212
1622 if getattr(obj, '_frozen', False):
1623 raise AttributeFrozenError(obj._FROZEN_ERR_MSG % name)
1624 if isinstance(orig_setattr, types.MethodType):
1625 orig_setattr(obj, name, value)
1627 super(cls, obj).__setattr__(name, value)
1628 cls.__setattr__ = SetAttr
1630 # Add new cls.Freeze method.
1638 class FrozenAttributesMixin(object):
1639 """Alternate mechanism for freezing attributes in a class.
1641 If an existing class is not a new-style class then it will be unable to
1642 use the FrozenAttributesClass metaclass directly. Simply use this class
1643 as a mixin instead to accomplish the same thing.
1645 __metaclass__ = FrozenAttributesClass
1648 def GetIPv4Address(dev=None, global_ip=True):
1649 """Returns any global/host IP address or the IP address of the given device.
1651 socket.gethostname() is insufficient for machines where the host files are
1652 not set up "correctly." Since some of our builders may have this issue,
1653 this method gives you a generic way to get the address so you are reachable
1654 either via a VM or remote machine on the same network.
1657 dev: Get the IP address of the device (e.g. 'eth0').
1658 global_ip: If set True, returns a globally valid IP address. Otherwise,
1659 returns a local IP address (default: True).
1661 cmd = ['ip', 'addr', 'show']
1662 cmd += ['scope', 'global' if global_ip else 'host']
1663 cmd += [] if dev is None else ['dev', dev]
1665 result = RunCommand(cmd, print_cmd=False, capture_output=True)
1666 matches = re.findall(r'\binet (\d+\.\d+\.\d+\.\d+).*', result.output)
1669 Warning('Failed to find ip address in %r', result.output)
1673 def GetSysroot(board=None):
1674 """Returns the sysroot for |board| or '/' if |board| is None."""
1675 return '/' if board is None else os.path.join('/build', board)
1678 # Chroot helper methods; assume default 'chroot' directory name.
1679 def ToChrootPath(path):
1680 """Reinterprets |path| to be used inside of chroot.
1683 A reinterpreted path if currently outside chroot or |path| if
1686 from chromite.lib import osutils
1687 from chromite.lib import git
1688 full_path = osutils.ExpandPath(path)
1689 if IsInsideChroot():
1693 return git.ReinterpretPathForChroot(full_path)
1695 raise ValueError('path %s is outside of your source tree' % path)
1698 def FromChrootPath(path):
1699 """Interprets a chroot |path| to be used inside or outside chroot.
1702 If currently outside chroot, returns the reinterpreted |path| to
1703 be used outside chroot. Otherwise, returns |path|.
1705 from chromite.lib import osutils
1706 full_path = osutils.ExpandPath(path)
1707 if IsInsideChroot():
1710 # Replace chroot source root with current source root, if applicable.
1711 if full_path.startswith(constants.CHROOT_SOURCE_ROOT):
1712 return os.path.join(
1713 constants.SOURCE_ROOT,
1714 full_path[len(constants.CHROOT_SOURCE_ROOT):].strip(os.path.sep))
1716 return os.path.join(constants.SOURCE_ROOT, constants.DEFAULT_CHROOT_DIR,
1717 path.strip(os.path.sep))
1720 def Collection(classname, **kwargs):
1721 """Create a new class with mutable named members.
1723 This is like collections.namedtuple, but mutable. Also similar to the
1724 python 3.3 types.SimpleNamespace.
1727 # Declare default values for this new class.
1728 Foo = cros_build_lib.Collection('Foo', a=0, b=10)
1729 # Create a new class but set b to 4.
1731 # Print out a (will be the default 0) and b (will be 4).
1732 print('a = %i, b = %i' % (foo.a, foo.b))
1735 def sn_init(self, **kwargs):
1736 """The new class's __init__ function."""
1737 # First verify the kwargs don't have excess settings.
1738 valid_keys = set(self.__slots__[1:])
1739 these_keys = set(kwargs.keys())
1740 invalid_keys = these_keys - valid_keys
1742 raise TypeError('invalid keyword arguments for this object: %r' %
1745 # Now initialize this object.
1746 for k in valid_keys:
1747 setattr(self, k, kwargs.get(k, self.__defaults__[k]))
1750 """The new class's __repr__ function."""
1751 return '%s(%s)' % (classname, ', '.join(
1752 '%s=%r' % (k, getattr(self, k)) for k in self.__slots__[1:]))
1754 # Give the new class a unique name and then generate the code for it.
1755 classname = 'Collection_%s' % classname
1757 'class %(classname)s(object):',
1758 ' __slots__ = ["__defaults__", "%(slots)s"]',
1759 ' __defaults__ = {}',
1761 'classname': classname,
1762 'slots': '", "'.join(sorted(str(k) for k in kwargs)),
1765 # Create the class in a local namespace as exec requires.
1767 exec expr in namespace
1768 new_class = namespace[classname]
1771 new_class.__defaults__ = kwargs.copy()
1772 new_class.__init__ = sn_init
1773 new_class.__repr__ = sn_repr
1778 PartitionInfo = collections.namedtuple(
1780 ['number', 'start', 'end', 'size', 'file_system', 'name', 'flags']
1784 def GetImageDiskPartitionInfo(image_path, unit='MB', key_selector='name'):
1785 """Returns the disk partition table of an image.
1788 image_path: Path to the image file.
1789 unit: The unit to display (e.g., 'B', 'KiB', 'KB', 'MB').
1790 See `parted` documentation for more info.
1791 key_selector: The value of the partition that will be used as the key for
1792 that partition in this function's returned dictionary.
1795 A dictionary of ParitionInfo items keyed by |key_selector|.
1798 # Inside chroot, parted is in /usr/sbin. Outside, it is in /sbin.
1799 parted_path = 'parted'
1800 if IsInsideChroot():
1801 # Inside chroot, parted is in /usr/sbin, but is not included in $PATH.
1802 parted_path = '/usr/sbin/parted'
1805 [parted_path, '-m', image_path, 'unit', unit, 'print'],
1806 capture_output=True).output.splitlines()
1808 # Sample output (partition #, start, end, size, file system, name, flags):
1809 # /foo/chromiumos_qemu_image.bin:3360MB:file:512:512:gpt:;
1810 # 11:0.03MB:8.42MB:8.39MB::RWFW:;
1811 # 6:8.42MB:8.42MB:0.00MB::KERN-C:;
1812 # 7:8.42MB:8.42MB:0.00MB::ROOT-C:;
1813 # 9:8.42MB:8.42MB:0.00MB::reserved:;
1814 # 10:8.42MB:8.42MB:0.00MB::reserved:;
1815 # 2:10.5MB:27.3MB:16.8MB::KERN-A:;
1816 # 4:27.3MB:44.0MB:16.8MB::KERN-B:;
1817 # 8:44.0MB:60.8MB:16.8MB:ext4:OEM:;
1818 # 12:128MB:145MB:16.8MB:fat16:EFI-SYSTEM:boot;
1819 # 5:145MB:2292MB:2147MB::ROOT-B:;
1820 # 3:2292MB:4440MB:2147MB:ext2:ROOT-A:;
1821 # 1:4440MB:7661MB:3221MB:ext4:STATE:;
1824 match = re.match(r'(.*:.*:.*:.*:.*:.*:.*);', line)
1826 # pylint: disable=W0212
1827 d = dict(zip(PartitionInfo._fields, match.group(1).split(':')))
1828 # pylint: enable=W0212
1829 # Disregard any non-numeric partition number (e.g. the file path).
1830 if d['number'].isdigit():
1831 d['number'] = int(d['number'])
1832 for key in ['start', 'end', 'size']:
1833 d[key] = float(d[key][:-len(unit)])
1835 table[d[key_selector]] = PartitionInfo(**d)