2 # Copyright 2012 The Swarming Authors. All rights reserved.
3 # Use of this source code is governed under the Apache License, Version 2.0 that
4 # can be found in the LICENSE file.
6 """Front end tool to operate on .isolate files.
8 This includes creating, merging or compiling them to generate a .isolated file.
10 See more information at
11 https://code.google.com/p/swarming/wiki/IsolateDesign
12 https://code.google.com/p/swarming/wiki/IsolateUserGuide
14 # Run ./isolate.py --help for more detailed information.
31 from third_party import colorama
32 from third_party.depot_tools import fix_encoding
33 from third_party.depot_tools import subcommand
35 from utils import file_path
36 from utils import tools
42 class ExecutionError(Exception):
43 """A generic error occurred."""
48 ### Path handling code.
51 def expand_directories_and_symlinks(indir, infiles, blacklist,
52 follow_symlinks, ignore_broken_items):
53 """Expands the directories and the symlinks, applies the blacklist and
56 Files are specified in os native path separator.
59 for relfile in infiles:
62 isolateserver.expand_directory_and_symlink(
63 indir, relfile, blacklist, follow_symlinks))
64 except isolateserver.MappingError as e:
65 if ignore_broken_items:
66 logging.info('warning: %s', e)
72 def recreate_tree(outdir, indir, infiles, action, as_hash):
73 """Creates a new tree with only the input files in it.
76 outdir: Output directory to create the files in.
77 indir: Root directory the infiles are based in.
78 infiles: dict of files to map from |indir| to |outdir|.
79 action: One of accepted action of run_isolated.link_file().
80 as_hash: Output filename is the hash instead of relfile.
83 'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)' %
84 (outdir, indir, len(infiles), action, as_hash))
86 assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir
87 if not os.path.isdir(outdir):
88 logging.info('Creating %s' % outdir)
91 for relfile, metadata in infiles.iteritems():
92 infile = os.path.join(indir, relfile)
94 # Do the hashtable specific checks.
96 # Skip links when storing a hashtable.
98 outfile = os.path.join(outdir, metadata['h'])
99 if os.path.isfile(outfile):
100 # Just do a quick check that the file size matches. No need to stat()
101 # again the input file, grab the value from the dict.
102 if not 's' in metadata:
103 raise isolateserver.MappingError(
104 'Misconfigured item %s: %s' % (relfile, metadata))
105 if metadata['s'] == os.stat(outfile).st_size:
108 logging.warn('Overwritting %s' % metadata['h'])
111 outfile = os.path.join(outdir, relfile)
112 outsubdir = os.path.dirname(outfile)
113 if not os.path.isdir(outsubdir):
114 os.makedirs(outsubdir)
116 # TODO(csharp): Fix crbug.com/150823 and enable the touched logic again.
117 # if metadata.get('T') == True:
118 # open(outfile, 'ab').close()
120 pointed = metadata['l']
121 logging.debug('Symlink: %s -> %s' % (outfile, pointed))
122 # symlink doesn't exist on Windows.
123 os.symlink(pointed, outfile) # pylint: disable=E1101
125 run_isolated.link_file(outfile, infile, action)
131 def _normalize_path_variable(cwd, relative_base_dir, key, value):
132 """Normalizes a path variable into a relative directory.
134 # Variables could contain / or \ on windows. Always normalize to
136 x = os.path.join(cwd, value.strip().replace('/', os.path.sep))
137 normalized = file_path.get_native_path_case(os.path.normpath(x))
138 if not os.path.isdir(normalized):
139 raise ExecutionError('%s=%s is not a directory' % (key, normalized))
141 # All variables are relative to the .isolate file.
142 normalized = os.path.relpath(normalized, relative_base_dir)
144 'Translated variable %s from %s to %s', key, value, normalized)
148 def normalize_path_variables(cwd, path_variables, relative_base_dir):
149 """Processes path variables as a special case and returns a copy of the dict.
151 For each 'path' variable: first normalizes it based on |cwd|, verifies it
152 exists then sets it as relative to relative_base_dir.
155 'normalize_path_variables(%s, %s, %s)', cwd, path_variables,
157 assert isinstance(cwd, unicode), cwd
158 assert isinstance(relative_base_dir, unicode), relative_base_dir
159 relative_base_dir = file_path.get_native_path_case(relative_base_dir)
161 (k, _normalize_path_variable(cwd, relative_base_dir, k, v))
162 for k, v in path_variables.iteritems())
165 ### Internal state files.
168 def isolatedfile_to_state(filename):
169 """For a '.isolate' file, returns the path to the saved '.state' file."""
170 return filename + '.state'
173 def classify_files(root_dir, tracked, untracked):
174 """Converts the list of files into a .isolate 'variables' dictionary.
177 - tracked: list of files names to generate a dictionary out of that should
179 - untracked: list of files names that must not be tracked.
182 new_untracked = list(untracked)
184 def should_be_tracked(filepath):
185 """Returns True if it is a file without whitespace in a non-optional
186 directory that has no symlink in its path.
188 if filepath.endswith('/'):
192 # Look if any element in the path is a symlink.
193 split = filepath.split('/')
194 for i in range(len(split)):
195 if os.path.islink(os.path.join(root_dir, '/'.join(split[:i+1]))):
199 for filepath in sorted(tracked):
200 if should_be_tracked(filepath):
201 new_tracked.append(filepath)
204 new_untracked.append(filepath)
208 variables[isolate_format.KEY_TRACKED] = sorted(new_tracked)
210 variables[isolate_format.KEY_UNTRACKED] = sorted(new_untracked)
214 def chromium_fix(f, variables):
215 """Fixes an isolate dependency with Chromium-specific fixes."""
216 # Blacklist logs and other unimportant files.
217 # - 'First Run' is not created by the compile but by the test itself.
218 # - Skip log in PRODUCT_DIR. Note that these are applied on '/' style path
219 # separator at this point.
220 if (re.match(r'^\<\(PRODUCT_DIR\)\/[^\/]+\.log$', f) or
221 f == '<(PRODUCT_DIR)/First Run'):
222 logging.debug('Ignoring %s', f)
225 EXECUTABLE = re.compile(
226 r'^(\<\(PRODUCT_DIR\)\/[^\/\.]+)' +
227 re.escape(variables.get('EXECUTABLE_SUFFIX', '')) +
229 match = EXECUTABLE.match(f)
231 return match.group(1) + '<(EXECUTABLE_SUFFIX)'
233 if sys.platform == 'darwin':
234 # On OSX, the name of the output is dependent on gyp define, it can be
235 # 'Google Chrome.app' or 'Chromium.app', same for 'XXX
236 # Framework.framework'. Furthermore, they are versioned with a gyp
237 # variable. To lower the complexity of the .isolate file, remove all the
238 # individual entries that show up under any of the 4 entries and replace
239 # them with the directory itself. Overall, this results in a bit more
240 # files than strictly necessary.
242 '<(PRODUCT_DIR)/Chromium Framework.framework/',
243 '<(PRODUCT_DIR)/Chromium.app/',
244 '<(PRODUCT_DIR)/Google Chrome Framework.framework/',
245 '<(PRODUCT_DIR)/Google Chrome.app/',
247 for prefix in OSX_BUNDLES:
248 if f.startswith(prefix):
249 # Note this result in duplicate values, so the a set() must be used to
255 def generate_simplified(
256 tracked, untracked, touched, root_dir, path_variables, extra_variables,
257 relative_cwd, trace_blacklist):
258 """Generates a clean and complete .isolate 'variables' dictionary.
260 Cleans up and extracts only files from within root_dir then processes
261 variables and relative_cwd.
263 root_dir = os.path.realpath(root_dir)
265 'generate_simplified(%d files, %s, %s, %s, %s)' %
266 (len(tracked) + len(untracked) + len(touched),
267 root_dir, path_variables, extra_variables, relative_cwd))
270 relative_cwd = file_path.cleanup_path(relative_cwd)
271 assert not os.path.isabs(relative_cwd), relative_cwd
273 # Normalizes to posix path. .isolate files are using posix paths on all OSes
275 path_variables = dict(
276 (k, v.replace(os.path.sep, '/')) for k, v in path_variables.iteritems())
277 # Contains normalized path_variables plus extra_variables.
278 total_variables = path_variables.copy()
279 total_variables.update(extra_variables)
281 # Actual work: Process the files.
282 # TODO(maruel): if all the files in a directory are in part tracked and in
283 # part untracked, the directory will not be extracted. Tracked files should be
284 # 'promoted' to be untracked as needed.
285 tracked = trace_inputs.extract_directories(
286 root_dir, tracked, trace_blacklist)
287 untracked = trace_inputs.extract_directories(
288 root_dir, untracked, trace_blacklist)
289 # touched is not compressed, otherwise it would result in files to be archived
290 # that we don't need.
292 root_dir_posix = root_dir.replace(os.path.sep, '/')
294 """Bases the file on the most restrictive variable."""
295 # Important, GYP stores the files with / and not \.
296 f = f.replace(os.path.sep, '/')
297 logging.debug('fix(%s)' % f)
298 # If it's not already a variable.
299 if not f.startswith('<'):
300 # relative_cwd is usually the directory containing the gyp file. It may be
301 # empty if the whole directory containing the gyp file is needed.
302 # Use absolute paths in case cwd_dir is outside of root_dir.
303 # Convert the whole thing to / since it's isolate's speak.
304 f = file_path.posix_relpath(
305 posixpath.join(root_dir_posix, f),
306 posixpath.join(root_dir_posix, relative_cwd)) or './'
308 # Use the longest value first.
309 for key, value in sorted(
310 path_variables.iteritems(), key=lambda x: -len(x[1])):
311 if f.startswith(value):
312 f = '<(%s)%s' % (key, f[len(value):])
313 logging.debug('Converted to %s' % f)
318 """Reduces the items to convert variables, removes unneeded items, apply
319 chromium-specific fixes and only return unique items.
321 variables_converted = (fix(f.path) for f in items)
323 chromium_fix(f, total_variables) for f in variables_converted)
324 return set(f for f in chromium_fixed if f)
326 tracked = fix_all(tracked)
327 untracked = fix_all(untracked)
328 touched = fix_all(touched)
329 out = classify_files(root_dir, tracked, untracked)
331 out[isolate_format.KEY_TOUCHED] = sorted(touched)
335 def generate_isolate(
336 tracked, untracked, touched, root_dir, path_variables, config_variables,
337 extra_variables, relative_cwd, trace_blacklist):
338 """Generates a clean and complete .isolate file."""
339 dependencies = generate_simplified(
340 tracked, untracked, touched, root_dir, path_variables, extra_variables,
341 relative_cwd, trace_blacklist)
342 config_variable_names, config_values = zip(
343 *sorted(config_variables.iteritems()))
344 out = isolate_format.Configs(None, config_variable_names)
347 isolate_format.ConfigSettings(
348 dependencies, os.path.abspath(relative_cwd)))
349 return out.make_isolate_file()
352 def chromium_save_isolated(isolated, data, path_variables, algo):
353 """Writes one or many .isolated files.
355 This slightly increases the cold cache cost but greatly reduce the warm cache
356 cost by splitting low-churn files off the master .isolated file. It also
357 reduces overall isolateserver memcache consumption.
361 def extract_into_included_isolated(prefix):
363 'algo': data['algo'],
365 'version': data['version'],
367 for f in data['files'].keys():
368 if f.startswith(prefix):
369 new_slave['files'][f] = data['files'].pop(f)
370 if new_slave['files']:
371 slaves.append(new_slave)
373 # Split test/data/ in its own .isolated file.
374 extract_into_included_isolated(os.path.join('test', 'data', ''))
376 # Split everything out of PRODUCT_DIR in its own .isolated file.
377 if path_variables.get('PRODUCT_DIR'):
378 extract_into_included_isolated(path_variables['PRODUCT_DIR'])
381 for index, f in enumerate(slaves):
382 slavepath = isolated[:-len('.isolated')] + '.%d.isolated' % index
383 tools.write_json(slavepath, f, True)
384 data.setdefault('includes', []).append(
385 isolateserver.hash_file(slavepath, algo))
386 files.append(os.path.basename(slavepath))
388 files.extend(isolateserver.save_isolated(isolated, data))
392 class Flattenable(object):
393 """Represents data that can be represented as a json file."""
397 """Returns a json-serializable version of itself.
401 items = ((member, getattr(self, member)) for member in self.MEMBERS)
402 return dict((member, value) for member, value in items if value is not None)
405 def load(cls, data, *args, **kwargs):
406 """Loads a flattened version."""
408 out = cls(*args, **kwargs)
409 for member in out.MEMBERS:
411 # Access to a protected member XXX of a client class
412 # pylint: disable=W0212
413 out._load_member(member, data.pop(member))
416 'Found unexpected entry %s while constructing an object %s' %
417 (data, cls.__name__), data, cls.__name__)
420 def _load_member(self, member, value):
421 """Loads a member into self."""
422 setattr(self, member, value)
425 def load_file(cls, filename, *args, **kwargs):
426 """Loads the data from a file or return an empty instance."""
428 out = cls.load(tools.read_json(filename), *args, **kwargs)
429 logging.debug('Loaded %s(%s)', cls.__name__, filename)
430 except (IOError, ValueError) as e:
431 # On failure, loads the default instance.
432 out = cls(*args, **kwargs)
433 logging.warn('Failed to load %s: %s', filename, e)
437 class SavedState(Flattenable):
438 """Describes the content of a .state file.
440 This file caches the items calculated by this script and is used to increase
441 the performance of the script. This file is not loaded by run_isolated.py.
442 This file can always be safely removed.
444 It is important to note that the 'files' dict keys are using native OS path
445 separator instead of '/' used in .isolate file.
448 # Value of sys.platform so that the file is rejected if loaded from a
449 # different OS. While this should never happen in practice, users are ...
452 # Algorithm used to generate the hash. The only supported value is at the
453 # time of writting 'sha-1'.
455 # List of included .isolated files. Used to support/remember 'slave'
456 # .isolated files. Relative path to isolated_basedir.
457 'child_isolated_files',
458 # Cache of the processed command. This value is saved because .isolated
459 # files are never loaded by isolate.py so it's the only way to load the
462 # GYP variables that are used to generate conditions. The most frequent
465 # GYP variables that will be replaced in 'command' and paths but will not be
466 # considered a relative directory.
468 # Cache of the files found so the next run can skip hash calculation.
470 # Path of the original .isolate file. Relative path to isolated_basedir.
472 # GYP variables used to generate the .isolated files paths based on path
473 # variables. Frequent examples are DEPTH and PRODUCT_DIR.
475 # If the generated directory tree should be read-only.
477 # Relative cwd to use to start the command.
479 # Root directory the files are mapped from.
481 # Version of the saved state file format. Any breaking change must update
486 # Bump this version whenever the saved state changes. It is also keyed on the
487 # .isolated file version so any change in the generator will invalidate .state
489 EXPECTED_VERSION = isolateserver.ISOLATED_FILE_VERSION + '.2'
491 def __init__(self, isolated_basedir):
492 """Creates an empty SavedState.
495 isolated_basedir: the directory where the .isolated and .isolated.state
498 super(SavedState, self).__init__()
499 assert os.path.isabs(isolated_basedir), isolated_basedir
500 assert os.path.isdir(isolated_basedir), isolated_basedir
501 self.isolated_basedir = isolated_basedir
503 # The default algorithm used.
504 self.OS = sys.platform
505 self.algo = isolateserver.SUPPORTED_ALGOS['sha-1']
506 self.child_isolated_files = []
508 self.config_variables = {}
509 self.extra_variables = {}
511 self.isolate_file = None
512 self.path_variables = {}
513 self.read_only = None
514 self.relative_cwd = None
516 self.version = self.EXPECTED_VERSION
518 def update_config(self, config_variables):
519 """Updates the saved state with only config variables."""
520 self.config_variables.update(config_variables)
522 def update(self, isolate_file, path_variables, extra_variables):
523 """Updates the saved state with new data to keep GYP variables and internal
524 reference to the original .isolate file.
526 assert os.path.isabs(isolate_file)
527 # Convert back to a relative path. On Windows, if the isolate and
528 # isolated files are on different drives, isolate_file will stay an absolute
530 isolate_file = file_path.safe_relpath(isolate_file, self.isolated_basedir)
532 # The same .isolate file should always be used to generate the .isolated and
534 assert isolate_file == self.isolate_file or not self.isolate_file, (
535 isolate_file, self.isolate_file)
536 self.extra_variables.update(extra_variables)
537 self.isolate_file = isolate_file
538 self.path_variables.update(path_variables)
540 def update_isolated(self, command, infiles, touched, read_only, relative_cwd):
541 """Updates the saved state with data necessary to generate a .isolated file.
543 The new files in |infiles| are added to self.files dict but their hash is
546 self.command = command
549 self.files.setdefault(f, {})
551 self.files.setdefault(f, {})['T'] = True
552 # Prune extraneous files that are not a dependency anymore.
553 for f in set(self.files).difference(set(infiles).union(touched)):
555 if read_only is not None:
556 self.read_only = read_only
557 self.relative_cwd = relative_cwd
559 def to_isolated(self):
560 """Creates a .isolated dictionary out of the saved state.
562 https://code.google.com/p/swarming/wiki/IsolatedDesign
565 """Returns a 'files' entry with only the whitelisted keys."""
566 return dict((k, data[k]) for k in ('h', 'l', 'm', 's') if k in data)
569 'algo': isolateserver.SUPPORTED_ALGOS_REVERSE[self.algo],
571 (filepath, strip(data)) for filepath, data in self.files.iteritems()),
572 # The version of the .state file is different than the one of the
574 'version': isolateserver.ISOLATED_FILE_VERSION,
577 out['command'] = self.command
578 if self.read_only is not None:
579 out['read_only'] = self.read_only
580 if self.relative_cwd:
581 out['relative_cwd'] = self.relative_cwd
585 def isolate_filepath(self):
586 """Returns the absolute path of self.isolate_file."""
587 return os.path.normpath(
588 os.path.join(self.isolated_basedir, self.isolate_file))
590 # Arguments number differs from overridden method
592 def load(cls, data, isolated_basedir): # pylint: disable=W0221
593 """Special case loading to disallow different OS.
595 It is not possible to load a .isolated.state files from a different OS, this
596 file is saved in OS-specific format.
598 out = super(SavedState, cls).load(data, isolated_basedir)
599 if data.get('OS') != sys.platform:
600 raise isolateserver.ConfigError('Unexpected OS %s', data.get('OS'))
602 # Converts human readable form back into the proper class type.
603 algo = data.get('algo')
604 if not algo in isolateserver.SUPPORTED_ALGOS:
605 raise isolateserver.ConfigError('Unknown algo \'%s\'' % out.algo)
606 out.algo = isolateserver.SUPPORTED_ALGOS[algo]
608 # Refuse the load non-exact version, even minor difference. This is unlike
609 # isolateserver.load_isolated(). This is because .isolated.state could have
610 # changed significantly even in minor version difference.
611 if out.version != cls.EXPECTED_VERSION:
612 raise isolateserver.ConfigError(
613 'Unsupported version \'%s\'' % out.version)
615 # The .isolate file must be valid. If it is not present anymore, zap the
616 # value as if it was not noted, so .isolate_file can safely be overriden
618 if out.isolate_file and not os.path.isfile(out.isolate_filepath):
619 out.isolate_file = None
621 # It could be absolute on Windows if the drive containing the .isolate and
622 # the drive containing the .isolated files differ, .e.g .isolate is on
623 # C:\\ and .isolated is on D:\\ .
624 assert not os.path.isabs(out.isolate_file) or sys.platform == 'win32'
625 assert os.path.isfile(out.isolate_filepath), out.isolate_filepath
629 """Makes sure 'algo' is in human readable form."""
630 out = super(SavedState, self).flatten()
631 out['algo'] = isolateserver.SUPPORTED_ALGOS_REVERSE[out['algo']]
636 return ''.join('\n %s=%s' % (k, d[k]) for k in sorted(d))
638 out = '%s(\n' % self.__class__.__name__
639 out += ' command: %s\n' % self.command
640 out += ' files: %d\n' % len(self.files)
641 out += ' isolate_file: %s\n' % self.isolate_file
642 out += ' read_only: %s\n' % self.read_only
643 out += ' relative_cwd: %s\n' % self.relative_cwd
644 out += ' child_isolated_files: %s\n' % self.child_isolated_files
645 out += ' path_variables: %s\n' % dict_to_str(self.path_variables)
646 out += ' config_variables: %s\n' % dict_to_str(self.config_variables)
647 out += ' extra_variables: %s\n' % dict_to_str(self.extra_variables)
651 class CompleteState(object):
652 """Contains all the state to run the task at hand."""
653 def __init__(self, isolated_filepath, saved_state):
654 super(CompleteState, self).__init__()
655 assert isolated_filepath is None or os.path.isabs(isolated_filepath)
656 self.isolated_filepath = isolated_filepath
657 # Contains the data to ease developer's use-case but that is not strictly
659 self.saved_state = saved_state
662 def load_files(cls, isolated_filepath):
663 """Loads state from disk."""
664 assert os.path.isabs(isolated_filepath), isolated_filepath
665 isolated_basedir = os.path.dirname(isolated_filepath)
668 SavedState.load_file(
669 isolatedfile_to_state(isolated_filepath), isolated_basedir))
672 self, cwd, isolate_file, path_variables, config_variables,
673 extra_variables, ignore_broken_items):
674 """Updates self.isolated and self.saved_state with information loaded from a
677 Processes the loaded data, deduce root_dir, relative_cwd.
679 # Make sure to not depend on os.getcwd().
680 assert os.path.isabs(isolate_file), isolate_file
681 isolate_file = file_path.get_native_path_case(isolate_file)
683 'CompleteState.load_isolate(%s, %s, %s, %s, %s, %s)',
684 cwd, isolate_file, path_variables, config_variables, extra_variables,
687 # Config variables are not affected by the paths and must be used to
688 # retrieve the paths, so update them first.
689 self.saved_state.update_config(config_variables)
691 with open(isolate_file, 'r') as f:
692 # At that point, variables are not replaced yet in command and infiles.
693 # infiles may contain directory entries and is in posix style.
694 command, infiles, touched, read_only, isolate_cmd_dir = (
695 isolate_format.load_isolate_for_config(
696 os.path.dirname(isolate_file), f.read(),
697 self.saved_state.config_variables))
699 # Processes the variables with the new found relative root. Note that 'cwd'
700 # is used when path variables are used.
701 path_variables = normalize_path_variables(
702 cwd, path_variables, isolate_cmd_dir)
703 # Update the rest of the saved state.
704 self.saved_state.update(isolate_file, path_variables, extra_variables)
706 total_variables = self.saved_state.path_variables.copy()
707 total_variables.update(self.saved_state.config_variables)
708 total_variables.update(self.saved_state.extra_variables)
710 isolate_format.eval_variables(i, total_variables) for i in command
713 total_variables = self.saved_state.path_variables.copy()
714 total_variables.update(self.saved_state.extra_variables)
716 isolate_format.eval_variables(f, total_variables) for f in infiles
719 isolate_format.eval_variables(f, total_variables) for f in touched
721 # root_dir is automatically determined by the deepest root accessed with the
722 # form '../../foo/bar'. Note that path variables must be taken in account
723 # too, add them as if they were input files.
724 self.saved_state.root_dir = isolate_format.determine_root_dir(
725 isolate_cmd_dir, infiles + touched +
726 self.saved_state.path_variables.values())
727 # The relative directory is automatically determined by the relative path
728 # between root_dir and the directory containing the .isolate file,
730 relative_cwd = os.path.relpath(isolate_cmd_dir, self.saved_state.root_dir)
731 # Now that we know where the root is, check that the path_variables point
733 for k, v in self.saved_state.path_variables.iteritems():
734 dest = os.path.join(isolate_cmd_dir, relative_cwd, v)
735 if not file_path.path_starts_with(self.saved_state.root_dir, dest):
736 raise isolateserver.MappingError(
737 'Path variable %s=%r points outside the inferred root directory '
739 % (k, v, self.saved_state.root_dir, dest))
740 # Normalize the files based to self.saved_state.root_dir. It is important to
741 # keep the trailing os.path.sep at that step.
744 file_path.normpath(os.path.join(isolate_cmd_dir, f)),
745 self.saved_state.root_dir)
750 file_path.normpath(os.path.join(isolate_cmd_dir, f)),
751 self.saved_state.root_dir)
754 follow_symlinks = sys.platform != 'win32'
755 # Expand the directories by listing each file inside. Up to now, trailing
756 # os.path.sep must be kept. Do not expand 'touched'.
757 infiles = expand_directories_and_symlinks(
758 self.saved_state.root_dir,
760 lambda x: re.match(r'.*\.(git|svn|pyc)$', x),
764 # If we ignore broken items then remove any missing touched items.
765 if ignore_broken_items:
766 original_touched_count = len(touched)
767 touched = [touch for touch in touched if os.path.exists(touch)]
769 if len(touched) != original_touched_count:
770 logging.info('Removed %d invalid touched entries',
771 len(touched) - original_touched_count)
773 # Finally, update the new data to be able to generate the foo.isolated file,
774 # the file that is used by run_isolated.py.
775 self.saved_state.update_isolated(
776 command, infiles, touched, read_only, relative_cwd)
779 def process_inputs(self, subdir):
780 """Updates self.saved_state.files with the files' mode and hash.
782 If |subdir| is specified, filters to a subdirectory. The resulting .isolated
785 See isolateserver.process_input() for more information.
787 for infile in sorted(self.saved_state.files):
788 if subdir and not infile.startswith(subdir):
789 self.saved_state.files.pop(infile)
791 filepath = os.path.join(self.root_dir, infile)
792 self.saved_state.files[infile] = isolateserver.process_input(
794 self.saved_state.files[infile],
795 self.saved_state.read_only,
796 self.saved_state.algo)
798 def save_files(self):
799 """Saves self.saved_state and creates a .isolated file."""
800 logging.debug('Dumping to %s' % self.isolated_filepath)
801 self.saved_state.child_isolated_files = chromium_save_isolated(
802 self.isolated_filepath,
803 self.saved_state.to_isolated(),
804 self.saved_state.path_variables,
805 self.saved_state.algo)
807 i.get('s', 0) for i in self.saved_state.files.itervalues())
809 # TODO(maruel): Stats are missing the .isolated files.
810 logging.debug('Total size: %d bytes' % total_bytes)
811 saved_state_file = isolatedfile_to_state(self.isolated_filepath)
812 logging.debug('Dumping to %s' % saved_state_file)
813 tools.write_json(saved_state_file, self.saved_state.flatten(), True)
817 return self.saved_state.root_dir
820 def indent(data, indent_length):
822 spacing = ' ' * indent_length
823 return ''.join(spacing + l for l in str(data).splitlines(True))
825 out = '%s(\n' % self.__class__.__name__
826 out += ' root_dir: %s\n' % self.root_dir
827 out += ' saved_state: %s)' % indent(self.saved_state, 2)
831 def load_complete_state(options, cwd, subdir, skip_update):
832 """Loads a CompleteState.
834 This includes data from .isolate and .isolated.state files. Never reads the
838 options: Options instance generated with OptionParserIsolate. For either
839 options.isolate and options.isolated, if the value is set, it is an
841 cwd: base directory to be used when loading the .isolate file.
842 subdir: optional argument to only process file in the subdirectory, relative
843 to CompleteState.root_dir.
844 skip_update: Skip trying to load the .isolate file and processing the
845 dependencies. It is useful when not needed, like when tracing.
847 assert not options.isolate or os.path.isabs(options.isolate)
848 assert not options.isolated or os.path.isabs(options.isolated)
849 cwd = file_path.get_native_path_case(unicode(cwd))
851 # Load the previous state if it was present. Namely, "foo.isolated.state".
852 # Note: this call doesn't load the .isolate file.
853 complete_state = CompleteState.load_files(options.isolated)
855 # Constructs a dummy object that cannot be saved. Useful for temporary
856 # commands like 'run'. There is no directory containing a .isolated file so
857 # specify the current working directory as a valid directory.
858 complete_state = CompleteState(None, SavedState(os.getcwd()))
860 if not options.isolate:
861 if not complete_state.saved_state.isolate_file:
863 raise ExecutionError('A .isolate file is required.')
866 isolate = complete_state.saved_state.isolate_filepath
868 isolate = options.isolate
869 if complete_state.saved_state.isolate_file:
870 rel_isolate = file_path.safe_relpath(
871 options.isolate, complete_state.saved_state.isolated_basedir)
872 if rel_isolate != complete_state.saved_state.isolate_file:
873 # This happens if the .isolate file was moved for example. In this case,
874 # discard the saved state.
876 '--isolated %s != %s as saved in %s. Discarding saved state',
878 complete_state.saved_state.isolate_file,
879 isolatedfile_to_state(options.isolated))
880 complete_state = CompleteState(
882 SavedState(complete_state.saved_state.isolated_basedir))
885 # Then load the .isolate and expands directories.
886 complete_state.load_isolate(
887 cwd, isolate, options.path_variables, options.config_variables,
888 options.extra_variables, options.ignore_broken_items)
890 # Regenerate complete_state.saved_state.files.
892 subdir = unicode(subdir)
893 # This is tricky here. If it is a path, take it from the root_dir. If
894 # it is a variable, it must be keyed from the directory containing the
895 # .isolate file. So translate all variables first.
896 translated_path_variables = dict(
898 os.path.normpath(os.path.join(complete_state.saved_state.relative_cwd,
900 for k, v in complete_state.saved_state.path_variables.iteritems())
901 subdir = isolate_format.eval_variables(subdir, translated_path_variables)
902 subdir = subdir.replace('/', os.path.sep)
905 complete_state.process_inputs(subdir)
906 return complete_state
909 def read_trace_as_isolate_dict(complete_state, trace_blacklist):
910 """Reads a trace and returns the .isolate dictionary.
912 Returns exceptions during the log parsing so it can be re-raised.
914 api = trace_inputs.get_api()
915 logfile = complete_state.isolated_filepath + '.log'
916 if not os.path.isfile(logfile):
917 raise ExecutionError(
918 'No log file \'%s\' to read, did you forget to \'trace\'?' % logfile)
920 data = api.parse_log(logfile, trace_blacklist, None)
921 exceptions = [i['exception'] for i in data if 'exception' in i]
922 results = (i['results'] for i in data if 'results' in i)
923 results_stripped = (i.strip_root(complete_state.root_dir) for i in results)
924 files = set(sum((result.existent for result in results_stripped), []))
925 tracked, touched = isolate_format.split_touched(files)
926 value = generate_isolate(
930 complete_state.root_dir,
931 complete_state.saved_state.path_variables,
932 complete_state.saved_state.config_variables,
933 complete_state.saved_state.extra_variables,
934 complete_state.saved_state.relative_cwd,
936 return value, exceptions
937 except trace_inputs.TracingFailure, e:
938 raise ExecutionError(
939 'Reading traces failed for: %s\n%s' %
940 (' '.join(complete_state.saved_state.command), str(e)))
943 def merge(complete_state, trace_blacklist):
944 """Reads a trace and merges it back into the source .isolate file."""
945 value, exceptions = read_trace_as_isolate_dict(
946 complete_state, trace_blacklist)
948 # Now take that data and union it into the original .isolate file.
949 with open(complete_state.saved_state.isolate_filepath, 'r') as f:
950 prev_content = f.read()
951 isolate_dir = os.path.dirname(complete_state.saved_state.isolate_filepath)
952 prev_config = isolate_format.load_isolate_as_config(
954 isolate_format.eval_content(prev_content),
955 isolate_format.extract_comment(prev_content))
956 new_config = isolate_format.load_isolate_as_config(isolate_dir, value, '')
957 config = prev_config.union(new_config)
958 data = config.make_isolate_file()
959 print('Updating %s' % complete_state.saved_state.isolate_file)
960 with open(complete_state.saved_state.isolate_filepath, 'wb') as f:
961 isolate_format.print_all(config.file_comment, data, f)
963 # It got an exception, raise the first one.
970 def create_isolate_tree(outdir, root_dir, files, relative_cwd, read_only):
971 """Creates a isolated tree usable for test execution.
973 Returns the current working directory where the isolated command should be
976 # Forcibly copy when the tree has to be read only. Otherwise the inode is
977 # modified, and this cause real problems because the user's source tree
978 # becomes read only. On the other hand, the cost of doing file copy is huge.
979 if read_only not in (0, None):
980 action = run_isolated.COPY
982 action = run_isolated.HARDLINK_WITH_FALLBACK
990 cwd = os.path.normpath(os.path.join(outdir, relative_cwd))
991 if not os.path.isdir(cwd):
992 # It can happen when no files are mapped from the directory containing the
993 # .isolate file. But the directory must exist to be the current working
996 run_isolated.change_tree_read_only(outdir, read_only)
1000 def prepare_for_archival(options, cwd):
1001 """Loads the isolated file and create 'infiles' for archival."""
1002 complete_state = load_complete_state(
1003 options, cwd, options.subdir, False)
1004 # Make sure that complete_state isn't modified until save_files() is
1005 # called, because any changes made to it here will propagate to the files
1006 # created (which is probably not intended).
1007 complete_state.save_files()
1009 infiles = complete_state.saved_state.files
1010 # Add all the .isolated files.
1014 ] + complete_state.saved_state.child_isolated_files
1015 for item in isolated_files:
1016 item_path = os.path.join(
1017 os.path.dirname(complete_state.isolated_filepath), item)
1018 # Do not use isolateserver.hash_file() here because the file is
1019 # likely smallish (under 500kb) and its file size is needed.
1020 with open(item_path, 'rb') as f:
1022 isolated_hash.append(
1023 complete_state.saved_state.algo(content).hexdigest())
1024 isolated_metadata = {
1025 'h': isolated_hash[-1],
1029 infiles[item_path] = isolated_metadata
1030 return complete_state, infiles, isolated_hash
1036 def CMDarchive(parser, args):
1037 """Creates a .isolated file and uploads the tree to an isolate server.
1039 All the files listed in the .isolated file are put in the isolate server
1040 cache via isolateserver.py.
1042 add_subdir_option(parser)
1043 isolateserver.add_isolate_server_options(parser, False)
1044 auth.add_auth_options(parser)
1045 options, args = parser.parse_args(args)
1046 auth.process_auth_options(parser, options)
1047 isolateserver.process_isolate_server_options(parser, options)
1049 parser.error('Unsupported argument: %s' % args)
1051 with tools.Profiler('GenerateHashtable'):
1054 complete_state, infiles, isolated_hash = prepare_for_archival(
1056 logging.info('Creating content addressed object store with %d item',
1059 isolateserver.upload_tree(
1060 base_url=options.isolate_server,
1061 indir=complete_state.root_dir,
1063 namespace=options.namespace)
1065 print('%s %s' % (isolated_hash[0], os.path.basename(options.isolated)))
1067 # If the command failed, delete the .isolated file if it exists. This is
1068 # important so no stale swarm job is executed.
1069 if not success and os.path.isfile(options.isolated):
1070 os.remove(options.isolated)
1071 return int(not success)
1074 def CMDcheck(parser, args):
1075 """Checks that all the inputs are present and generates .isolated."""
1076 add_subdir_option(parser)
1077 options, args = parser.parse_args(args)
1079 parser.error('Unsupported argument: %s' % args)
1081 complete_state = load_complete_state(
1082 options, os.getcwd(), options.subdir, False)
1084 # Nothing is done specifically. Just store the result and state.
1085 complete_state.save_files()
1089 def CMDhashtable(parser, args):
1090 """Creates a .isolated file and stores the contains in a directory.
1092 All the files listed in the .isolated file are put in the directory with their
1093 sha-1 as their file name. When using an NFS/CIFS server, the files can then be
1094 shared accross slaves without an isolate server.
1096 add_subdir_option(parser)
1097 isolateserver.add_outdir_options(parser)
1098 add_skip_refresh_option(parser)
1099 options, args = parser.parse_args(args)
1101 parser.error('Unsupported argument: %s' % args)
1103 isolateserver.process_outdir_options(parser, options, cwd)
1107 complete_state, infiles, isolated_hash = prepare_for_archival(options, cwd)
1108 logging.info('Creating content addressed object store with %d item',
1110 if not os.path.isdir(options.outdir):
1111 os.makedirs(options.outdir)
1113 # TODO(maruel): Make the files read-only?
1115 outdir=options.outdir,
1116 indir=complete_state.root_dir,
1118 action=run_isolated.HARDLINK_WITH_FALLBACK,
1121 print('%s %s' % (isolated_hash[0], os.path.basename(options.isolated)))
1123 # If the command failed, delete the .isolated file if it exists. This is
1124 # important so no stale swarm job is executed.
1125 if not success and os.path.isfile(options.isolated):
1126 os.remove(options.isolated)
1127 return int(not success)
1130 def CMDmerge(parser, args):
1131 """Reads and merges the data from the trace back into the original .isolate.
1133 parser.require_isolated = False
1134 add_trace_option(parser)
1135 options, args = parser.parse_args(args)
1137 parser.error('Unsupported argument: %s' % args)
1139 complete_state = load_complete_state(options, os.getcwd(), None, False)
1140 blacklist = tools.gen_blacklist(options.trace_blacklist)
1141 merge(complete_state, blacklist)
1145 def CMDread(parser, args):
1146 """Reads the trace file generated with command 'trace'."""
1147 parser.require_isolated = False
1148 add_trace_option(parser)
1149 add_skip_refresh_option(parser)
1151 '-m', '--merge', action='store_true',
1152 help='merge the results back in the .isolate file instead of printing')
1153 options, args = parser.parse_args(args)
1155 parser.error('Unsupported argument: %s' % args)
1157 complete_state = load_complete_state(
1158 options, os.getcwd(), None, options.skip_refresh)
1159 blacklist = tools.gen_blacklist(options.trace_blacklist)
1160 value, exceptions = read_trace_as_isolate_dict(complete_state, blacklist)
1162 merge(complete_state, blacklist)
1164 isolate_format.pretty_print(value, sys.stdout)
1167 # It got an exception, raise the first one.
1175 def CMDremap(parser, args):
1176 """Creates a directory with all the dependencies mapped into it.
1178 Useful to test manually why a test is failing. The target executable is not
1181 parser.require_isolated = False
1182 isolateserver.add_outdir_options(parser)
1183 add_skip_refresh_option(parser)
1184 options, args = parser.parse_args(args)
1186 parser.error('Unsupported argument: %s' % args)
1188 isolateserver.process_outdir_options(parser, options, cwd)
1189 complete_state = load_complete_state(options, cwd, None, options.skip_refresh)
1191 if not os.path.isdir(options.outdir):
1192 os.makedirs(options.outdir)
1193 print('Remapping into %s' % options.outdir)
1194 if os.listdir(options.outdir):
1195 raise ExecutionError('Can\'t remap in a non-empty directory')
1197 create_isolate_tree(
1198 options.outdir, complete_state.root_dir, complete_state.saved_state.files,
1199 complete_state.saved_state.relative_cwd,
1200 complete_state.saved_state.read_only)
1201 if complete_state.isolated_filepath:
1202 complete_state.save_files()
1206 def CMDrewrite(parser, args):
1207 """Rewrites a .isolate file into the canonical format."""
1208 parser.require_isolated = False
1209 options, args = parser.parse_args(args)
1211 parser.error('Unsupported argument: %s' % args)
1213 if options.isolated:
1214 # Load the previous state if it was present. Namely, "foo.isolated.state".
1215 complete_state = CompleteState.load_files(options.isolated)
1216 isolate = options.isolate or complete_state.saved_state.isolate_filepath
1218 isolate = options.isolate
1220 parser.error('--isolate is required.')
1222 with open(isolate, 'r') as f:
1224 config = isolate_format.load_isolate_as_config(
1225 os.path.dirname(os.path.abspath(isolate)),
1226 isolate_format.eval_content(content),
1227 isolate_format.extract_comment(content))
1228 data = config.make_isolate_file()
1229 print('Updating %s' % isolate)
1230 with open(isolate, 'wb') as f:
1231 isolate_format.print_all(config.file_comment, data, f)
1235 @subcommand.usage('-- [extra arguments]')
1236 def CMDrun(parser, args):
1237 """Runs the test executable in an isolated (temporary) directory.
1239 All the dependencies are mapped into the temporary directory and the
1240 directory is cleaned up after the target exits.
1242 Argument processing stops at -- and these arguments are appended to the
1243 command line of the target to run. For example, use:
1244 isolate.py run --isolated foo.isolated -- --gtest_filter=Foo.Bar
1246 parser.require_isolated = False
1247 add_skip_refresh_option(parser)
1248 options, args = parser.parse_args(args)
1250 complete_state = load_complete_state(
1251 options, os.getcwd(), None, options.skip_refresh)
1252 cmd = complete_state.saved_state.command + args
1254 raise ExecutionError('No command to run.')
1255 cmd = tools.fix_python_path(cmd)
1257 outdir = run_isolated.make_temp_dir(
1258 'isolate-%s' % datetime.date.today(),
1259 os.path.dirname(complete_state.root_dir))
1261 # TODO(maruel): Use run_isolated.run_tha_test().
1262 cwd = create_isolate_tree(
1263 outdir, complete_state.root_dir, complete_state.saved_state.files,
1264 complete_state.saved_state.relative_cwd,
1265 complete_state.saved_state.read_only)
1266 logging.info('Running %s, cwd=%s' % (cmd, cwd))
1267 result = subprocess.call(cmd, cwd=cwd)
1269 run_isolated.rmtree(outdir)
1271 if complete_state.isolated_filepath:
1272 complete_state.save_files()
1276 @subcommand.usage('-- [extra arguments]')
1277 def CMDtrace(parser, args):
1278 """Traces the target using trace_inputs.py.
1280 It runs the executable without remapping it, and traces all the files it and
1281 its child processes access. Then the 'merge' command can be used to generate
1282 an updated .isolate file out of it or the 'read' command to print it out to
1285 Argument processing stops at -- and these arguments are appended to the
1286 command line of the target to run. For example, use:
1287 isolate.py trace --isolated foo.isolated -- --gtest_filter=Foo.Bar
1289 add_trace_option(parser)
1291 '-m', '--merge', action='store_true',
1292 help='After tracing, merge the results back in the .isolate file')
1293 add_skip_refresh_option(parser)
1294 options, args = parser.parse_args(args)
1296 complete_state = load_complete_state(
1297 options, os.getcwd(), None, options.skip_refresh)
1298 cmd = complete_state.saved_state.command + args
1300 raise ExecutionError('No command to run.')
1301 cmd = tools.fix_python_path(cmd)
1302 cwd = os.path.normpath(os.path.join(
1303 unicode(complete_state.root_dir),
1304 complete_state.saved_state.relative_cwd))
1305 cmd[0] = os.path.normpath(os.path.join(cwd, cmd[0]))
1306 if not os.path.isfile(cmd[0]):
1307 raise ExecutionError(
1308 'Tracing failed for: %s\nIt doesn\'t exit' % ' '.join(cmd))
1309 logging.info('Running %s, cwd=%s' % (cmd, cwd))
1310 api = trace_inputs.get_api()
1311 logfile = complete_state.isolated_filepath + '.log'
1312 api.clean_trace(logfile)
1315 with api.get_tracer(logfile) as tracer:
1316 result, out = tracer.trace(
1321 except trace_inputs.TracingFailure, e:
1322 raise ExecutionError('Tracing failed for: %s\n%s' % (' '.join(cmd), str(e)))
1326 'Tracer exited with %d, which means the tests probably failed so the '
1327 'trace is probably incomplete.', result)
1330 complete_state.save_files()
1333 blacklist = tools.gen_blacklist(options.trace_blacklist)
1334 merge(complete_state, blacklist)
1339 def _process_variable_arg(option, opt, _value, parser):
1340 """Called by OptionParser to process a --<foo>-variable argument."""
1341 if not parser.rargs:
1342 raise optparse.OptionValueError(
1343 'Please use %s FOO=BAR or %s FOO BAR' % (opt, opt))
1344 k = parser.rargs.pop(0)
1345 variables = getattr(parser.values, option.dest)
1347 k, v = k.split('=', 1)
1349 if not parser.rargs:
1350 raise optparse.OptionValueError(
1351 'Please use %s FOO=BAR or %s FOO BAR' % (opt, opt))
1352 v = parser.rargs.pop(0)
1353 if not re.match('^' + isolate_format.VALID_VARIABLE + '$', k):
1354 raise optparse.OptionValueError(
1355 'Variable \'%s\' doesn\'t respect format \'%s\'' %
1356 (k, isolate_format.VALID_VARIABLE))
1357 variables.append((k, v.decode('utf-8')))
1360 def add_variable_option(parser):
1361 """Adds --isolated and --<foo>-variable to an OptionParser."""
1365 help='.isolated file to generate or read')
1366 # Keep for compatibility. TODO(maruel): Remove once not used anymore.
1370 help=optparse.SUPPRESS_HELP)
1371 is_win = sys.platform in ('win32', 'cygwin')
1372 # There is really 3 kind of variables:
1373 # - path variables, like DEPTH or PRODUCT_DIR that should be
1374 # replaced opportunistically when tracing tests.
1375 # - extraneous things like EXECUTABE_SUFFIX.
1376 # - configuration variables that are to be used in deducing the matrix to
1378 # - unrelated variables that are used as command flags for example.
1380 '--config-variable',
1382 callback=_process_variable_arg,
1384 dest='config_variables',
1386 help='Config variables are used to determine which conditions should be '
1387 'matched when loading a .isolate file, default: %default. '
1388 'All 3 kinds of variables are persistent accross calls, they are '
1389 'saved inside <.isolated>.state')
1393 callback=_process_variable_arg,
1395 dest='path_variables',
1397 help='Path variables are used to replace file paths when loading a '
1398 '.isolate file, default: %default')
1402 callback=_process_variable_arg,
1403 default=[('EXECUTABLE_SUFFIX', '.exe' if is_win else '')],
1404 dest='extra_variables',
1406 help='Extraneous variables are replaced on the \'command\' entry and on '
1407 'paths in the .isolate file but are not considered relative paths.')
1410 def add_subdir_option(parser):
1413 help='Filters to a subdirectory. Its behavior changes depending if it '
1414 'is a relative path as a string or as a path variable. Path '
1415 'variables are always keyed from the directory containing the '
1416 '.isolate file. Anything else is keyed on the root directory.')
1419 def add_trace_option(parser):
1420 """Adds --trace-blacklist to the parser."""
1422 '--trace-blacklist',
1423 action='append', default=list(isolateserver.DEFAULT_BLACKLIST),
1424 help='List of regexp to use as blacklist filter for files to consider '
1425 'important, not to be confused with --blacklist which blacklists '
1429 def add_skip_refresh_option(parser):
1431 '--skip-refresh', action='store_true',
1432 help='Skip reading .isolate file and do not refresh the hash of '
1436 def parse_isolated_option(parser, options, cwd, require_isolated):
1437 """Processes --isolated."""
1438 if options.isolated:
1439 options.isolated = os.path.normpath(
1440 os.path.join(cwd, options.isolated.replace('/', os.path.sep)))
1441 if require_isolated and not options.isolated:
1442 parser.error('--isolated is required.')
1443 if options.isolated and not options.isolated.endswith('.isolated'):
1444 parser.error('--isolated value must end with \'.isolated\'')
1447 def parse_variable_option(options):
1448 """Processes all the --<foo>-variable flags."""
1449 # TODO(benrg): Maybe we should use a copy of gyp's NameValueListToDict here,
1450 # but it wouldn't be backward compatible.
1451 def try_make_int(s):
1452 """Converts a value to int if possible, converts to unicode otherwise."""
1456 return s.decode('utf-8')
1457 options.config_variables = dict(
1458 (k, try_make_int(v)) for k, v in options.config_variables)
1459 options.path_variables = dict(options.path_variables)
1460 options.extra_variables = dict(options.extra_variables)
1463 class OptionParserIsolate(tools.OptionParserWithLogging):
1464 """Adds automatic --isolate, --isolated, --out and --<foo>-variable handling.
1466 # Set it to False if it is not required, e.g. it can be passed on but do not
1467 # fail if not given.
1468 require_isolated = True
1470 def __init__(self, **kwargs):
1471 tools.OptionParserWithLogging.__init__(
1473 verbose=int(os.environ.get('ISOLATE_DEBUG', 0)),
1475 group = optparse.OptionGroup(self, "Common options")
1479 help='.isolate file to load the dependency data from')
1480 add_variable_option(group)
1482 '--ignore_broken_items', action='store_true',
1483 default=bool(os.environ.get('ISOLATE_IGNORE_BROKEN_ITEMS')),
1484 help='Indicates that invalid entries in the isolated file to be '
1485 'only be logged and not stop processing. Defaults to True if '
1486 'env var ISOLATE_IGNORE_BROKEN_ITEMS is set')
1487 self.add_option_group(group)
1489 def parse_args(self, *args, **kwargs):
1490 """Makes sure the paths make sense.
1492 On Windows, / and \ are often mixed together in a path.
1494 options, args = tools.OptionParserWithLogging.parse_args(
1495 self, *args, **kwargs)
1496 if not self.allow_interspersed_args and args:
1497 self.error('Unsupported argument: %s' % args)
1499 cwd = file_path.get_native_path_case(unicode(os.getcwd()))
1500 parse_isolated_option(self, options, cwd, self.require_isolated)
1501 parse_variable_option(options)
1504 # TODO(maruel): Work with non-ASCII.
1505 # The path must be in native path case for tracing purposes.
1506 options.isolate = unicode(options.isolate).replace('/', os.path.sep)
1507 options.isolate = os.path.normpath(os.path.join(cwd, options.isolate))
1508 options.isolate = file_path.get_native_path_case(options.isolate)
1510 return options, args
1514 dispatcher = subcommand.CommandDispatcher(__name__)
1516 return dispatcher.execute(OptionParserIsolate(version=__version__), argv)
1517 except Exception as e:
1518 tools.report_error(e)
1522 if __name__ == '__main__':
1523 fix_encoding.fix_encoding()
1524 tools.disable_buffering()
1526 sys.exit(main(sys.argv[1:]))