X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=src%2Ftools%2Fswarming_client%2Fisolate_format.py;h=80624dbcdd0a11739d41dd41b1f6c97f146581ca;hb=004985e17e624662a4c85c76a7654039dc83f028;hp=1208e4803f10bd0647815ac54d066c7717a5a165;hpb=2f108dbacb161091e42a3479f4e171339b7e7623;p=platform%2Fframework%2Fweb%2Fcrosswalk.git diff --git a/src/tools/swarming_client/isolate_format.py b/src/tools/swarming_client/isolate_format.py index 1208e48..80624db 100644 --- a/src/tools/swarming_client/isolate_format.py +++ b/src/tools/swarming_client/isolate_format.py @@ -13,11 +13,12 @@ See more information at """ import ast -import copy import itertools import logging import os +import posixpath import re +import sys import isolateserver @@ -51,9 +52,9 @@ def determine_root_dir(relative_root, infiles): x = os.path.dirname(x) if deepest_root.startswith(x): deepest_root = x - logging.debug( - 'determine_root_dir(%s, %d files) -> %s' % ( - relative_root, len(infiles), deepest_root)) + logging.info( + 'determine_root_dir(%s, %d files) -> %s', + relative_root, len(infiles), deepest_root) return deepest_root @@ -91,13 +92,15 @@ def split_touched(files): def pretty_print(variables, stdout): - """Outputs a gyp compatible list from the decoded variables. + """Outputs a .isolate file from the decoded variables. + + The .isolate format is GYP compatible. Similar to pprint.print() but with NIH syndrome. """ # Order the dictionary keys by these keys in priority. ORDER = ( - 'variables', 'condition', 'command', 'relative_cwd', 'read_only', + 'variables', 'condition', 'command', 'read_only', KEY_TRACKED, KEY_UNTRACKED) def sorting_key(x): @@ -151,7 +154,7 @@ def pretty_print(variables, stdout): stdout.write( '\'%s\',\n' % item.replace('\\', '\\\\').replace('\'', '\\\'')) elif isinstance(item, (int, bool)) or item is None: - stdout.write('%s\n' % item) + stdout.write('%s,\n' % item) else: assert False, item @@ -169,25 +172,6 @@ def print_all(comment, data, stream): pretty_print(data, stream) -def union(lhs, rhs): - """Merges two compatible datastructures composed of dict/list/set.""" - assert lhs is not None or rhs is not None - if lhs is None: - return copy.deepcopy(rhs) - if rhs is None: - return copy.deepcopy(lhs) - assert type(lhs) == type(rhs), (lhs, rhs) - if hasattr(lhs, 'union'): - # Includes set, ConfigSettings and Configs. - return lhs.union(rhs) - if isinstance(lhs, dict): - return dict((k, union(lhs.get(k), rhs.get(k))) for k in set(lhs).union(rhs)) - elif isinstance(lhs, list): - # Do not go inside the list. - return lhs + rhs - assert False, type(lhs) - - def extract_comment(content): """Extracts file level comment.""" out = [] @@ -324,6 +308,7 @@ def verify_condition(condition, variables_and_values): def verify_root(value, variables_and_values): """Verifies that |value| is the parsed form of a valid .isolate file. + See verify_ast() for the meaning of |variables_and_values|. """ VALID_ROOTS = ['includes', 'conditions', 'variables'] @@ -504,31 +489,88 @@ def convert_map_to_isolate_dict(values, config_variables): class ConfigSettings(object): """Represents the dependency variables for a single build configuration. + The structure is immutable. + + .touch, .tracked and .untracked are the list of dependencies. The items in + these lists use '/' as a path separator. + .command and .isolate_dir describe how to run the command. .isolate_dir uses + the OS' native path separator. It must be an absolute path, it's the path + where to start the command from. + .read_only describe how to map the files. """ - def __init__(self, values): + def __init__(self, values, isolate_dir): verify_variables(values) + if isolate_dir is None: + # It must be an empty object if isolate_dir is None. + assert values == {}, values + else: + # Otherwise, the path must be absolute. + assert os.path.isabs(isolate_dir), isolate_dir self.touched = sorted(values.get(KEY_TOUCHED, [])) self.tracked = sorted(values.get(KEY_TRACKED, [])) self.untracked = sorted(values.get(KEY_UNTRACKED, [])) self.command = values.get('command', [])[:] + self.isolate_dir = isolate_dir self.read_only = values.get('read_only') def union(self, rhs): - """Merges two config settings together. + """Merges two config settings together into a new instance. + + A new instance is not created and self or rhs is returned if the other + object is the empty object. + + self has priority over rhs for .command. Use the same .isolate_dir as the + one having a .command. - self has priority over rhs for 'command' variable. + Dependencies listed in rhs are patch adjusted ONLY if they don't start with + a path variable, e.g. the characters '<('. """ + # When an object has .isolate_dir == None, it means it is the empty object. + if rhs.isolate_dir is None: + return self + if self.isolate_dir is None: + return rhs + + if sys.platform == 'win32': + assert self.isolate_dir[0].lower() == rhs.isolate_dir[0].lower() + + # Takes the difference between the two isolate_dir. Note that while + # isolate_dir is in native path case, all other references are in posix. + l_rel_cwd, r_rel_cwd = self.isolate_dir, rhs.isolate_dir + if self.command or rhs.command: + use_rhs = bool(not self.command and rhs.command) + else: + # If self doesn't define any file, use rhs. + use_rhs = not bool(self.touched or self.tracked or self.untracked) + if use_rhs: + # Rebase files in rhs. + l_rel_cwd, r_rel_cwd = r_rel_cwd, l_rel_cwd + + rebase_path = os.path.relpath(r_rel_cwd, l_rel_cwd).replace( + os.path.sep, '/') + def rebase_item(f): + if f.startswith('<(') or rebase_path == '.': + return f + return posixpath.join(rebase_path, f) + + def map_both(l, r): + """Rebase items in either lhs or rhs, as needed.""" + if use_rhs: + l, r = r, l + return sorted(l + map(rebase_item, r)) + var = { - KEY_TOUCHED: sorted(self.touched + rhs.touched), - KEY_TRACKED: sorted(self.tracked + rhs.tracked), - KEY_UNTRACKED: sorted(self.untracked + rhs.untracked), + KEY_TOUCHED: map_both(self.touched, rhs.touched), + KEY_TRACKED: map_both(self.tracked, rhs.tracked), + KEY_UNTRACKED: map_both(self.untracked, rhs.untracked), 'command': self.command or rhs.command, 'read_only': rhs.read_only if self.read_only is None else self.read_only, } - return ConfigSettings(var) + return ConfigSettings(var, l_rel_cwd) def flatten(self): + """Converts the object into a dict.""" out = {} if self.command: out['command'] = self.command @@ -540,8 +582,21 @@ class ConfigSettings(object): out[KEY_UNTRACKED] = self.untracked if self.read_only is not None: out['read_only'] = self.read_only + # TODO(maruel): Probably better to not output it if command is None? + if self.isolate_dir is not None: + out['isolate_dir'] = self.isolate_dir return out + def __str__(self): + """Returns a short representation useful for debugging.""" + files = ''.join( + '\n ' + f for f in (self.touched + self.tracked + self.untracked)) + return 'ConfigSettings(%s, %s, %s, %s)' % ( + self.command, + self.isolate_dir, + self.read_only, + files or '[]') + def _safe_index(l, k): try: @@ -575,6 +630,10 @@ class Configs(object): At this point, we don't know all the possibilities. So mount a partial view that we have. + + This class doesn't hold isolate_dir, since it is dependent on the final + configuration selected. It is implicitly dependent on which .isolate defines + the 'command' that will take effect. """ def __init__(self, file_comment, config_variables): self.file_comment = file_comment @@ -584,6 +643,8 @@ class Configs(object): assert isinstance(config_variables, tuple) assert all(isinstance(c, basestring) for c in config_variables), ( config_variables) + config_variables = tuple(config_variables) + assert tuple(sorted(config_variables)) == config_variables, config_variables self._config_variables = config_variables # The keys of _by_config are tuples of values for each of the items in # self._config_variables. A None item in the list of the key means the value @@ -596,9 +657,14 @@ class Configs(object): def get_config(self, config): """Returns all configs that matches this config as a single ConfigSettings. + + Returns an empty ConfigSettings if none apply. """ - out = ConfigSettings({}) - for k, v in self._by_config.iteritems(): + # TODO(maruel): Fix ordering based on the bounded values. The keys are not + # necessarily sorted in the way that makes sense, they are alphabetically + # sorted. It is important because the left-most takes predescence. + out = ConfigSettings({}, None) + for k, v in sorted(self._by_config.iteritems()): if all(i == j or j is None for i, j in zip(config, k)): out = out.union(v) return out @@ -621,6 +687,7 @@ class Configs(object): """Returns a new Configs instance, the union of variables from self and rhs. Uses self.file_comment if available, otherwise rhs.file_comment. + It keeps config_variables sorted in the output. """ # Merge the keys of config_variables for each Configs instances. All the new # variables will become unbounded. This requires realigning the keys. @@ -636,7 +703,9 @@ class Configs(object): (_map_keys(mapping_rhs, k), v) for k, v in rhs._by_config.iteritems()) for key in set(lhs_config) | set(rhs_config): - out.set_config(key, union(lhs_config.get(key), rhs_config.get(key))) + l = lhs_config.get(key) + r = rhs_config.get(key) + out.set_config(key, l.union(r) if (l and r) else (l or r)) return out def flatten(self): @@ -652,6 +721,11 @@ class Configs(object): return convert_map_to_isolate_dict(configs_by_dependency, self.config_variables) + def __str__(self): + return 'Configs(%s,%s)' % ( + self._config_variables, + ''.join('\n %s' % str(f) for f in self._by_config)) + def load_isolate_as_config(isolate_dir, value, file_comment): """Parses one .isolate file and returns a Configs() instance. @@ -690,6 +764,7 @@ def load_isolate_as_config(isolate_dir, value, file_comment): }, } """ + assert os.path.isabs(isolate_dir), isolate_dir if any(len(cond) == 3 for cond in value.get('conditions', [])): raise isolateserver.ConfigError('Using \'else\' is not supported anymore.') variables_and_values = {} @@ -707,14 +782,14 @@ def load_isolate_as_config(isolate_dir, value, file_comment): # Add global variables. The global variables are on the empty tuple key. isolate.set_config( (None,) * len(config_variables), - ConfigSettings(value.get('variables', {}))) + ConfigSettings(value.get('variables', {}), isolate_dir)) # Add configuration-specific variables. for expr, then in value.get('conditions', []): configs = match_configs(expr, config_variables, all_configs) new = Configs(None, config_variables) for config in configs: - new.set_config(config, ConfigSettings(then['variables'])) + new.set_config(config, ConfigSettings(then['variables'], isolate_dir)) isolate = isolate.union(new) # Load the includes. Process them in reverse so the last one take precedence. @@ -724,12 +799,16 @@ def load_isolate_as_config(isolate_dir, value, file_comment): 'Failed to load configuration; absolute include path \'%s\'' % include) included_isolate = os.path.normpath(os.path.join(isolate_dir, include)) + if sys.platform == 'win32': + if included_isolate[0].lower() != isolate_dir[0].lower(): + raise isolateserver.ConfigError( + 'Can\'t reference a .isolate file from another drive') with open(included_isolate, 'r') as f: included_isolate = load_isolate_as_config( os.path.dirname(included_isolate), eval_content(f.read()), None) - isolate = union(isolate, included_isolate) + isolate = isolate.union(included_isolate) return isolate @@ -738,8 +817,9 @@ def load_isolate_for_config(isolate_dir, content, config_variables): """Loads the .isolate file and returns the information unprocessed but filtered for the specific OS. - Returns the command, dependencies and read_only flag. The dependencies are - fixed to use os.path.sep. + Returns: + tuple of command, dependencies, touched, read_only flag, isolate_dir. + The dependencies are fixed to use os.path.sep. """ # Load the .isolate file, process its conditions, retrieve the command and # dependencies. @@ -758,8 +838,10 @@ def load_isolate_for_config(isolate_dir, content, config_variables): config = isolate.get_config(config_name) # Merge tracked and untracked variables, isolate.py doesn't care about the # trackability of the variables, only the build tool does. - dependencies = [ + dependencies = sorted( f.replace('/', os.path.sep) for f in config.tracked + config.untracked - ] - touched = [f.replace('/', os.path.sep) for f in config.touched] - return config.command, dependencies, touched, config.read_only + ) + touched = sorted(f.replace('/', os.path.sep) for f in config.touched) + return ( + config.command, dependencies, touched, config.read_only, + config.isolate_dir)