X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=giscanner%2Ftransformer.py;h=bcabdedc01a2ad6c422efbac1559108c280011c4;hb=cff0d034c2e3efa66bd0947152dd4137bbc7fecf;hp=a131e3c143a0515bd27ff7f5453fc194419ef831;hpb=ab127091d84336a851e0cec9cfdd0435065d9e2a;p=platform%2Fupstream%2Fgobject-introspection.git diff --git a/giscanner/transformer.py b/giscanner/transformer.py index a131e3c..bcabded 100644 --- a/giscanner/transformer.py +++ b/giscanner/transformer.py @@ -19,198 +19,377 @@ # import os +import sys +import subprocess -from .ast import (Bitfield, Callback, Enum, Function, Namespace, Member, - Parameter, Return, Struct, Field, - Type, Array, Alias, Interface, Class, Node, Union, - Varargs, Constant, type_name_from_ctype, - type_names, TYPE_STRING, BASIC_GIR_TYPES) -from .config import DATADIR, GIR_DIR, GIR_SUFFIX -from .glibast import GLibBoxed +from . import ast +from . import message +from . import utils +from .cachestore import CacheStore from .girparser import GIRParser -from .odict import odict from .sourcescanner import ( SourceSymbol, ctype_name, CTYPE_POINTER, CTYPE_BASIC_TYPE, CTYPE_UNION, CTYPE_ARRAY, CTYPE_TYPEDEF, CTYPE_VOID, CTYPE_ENUM, CTYPE_FUNCTION, CTYPE_STRUCT, - CSYMBOL_TYPE_FUNCTION, CSYMBOL_TYPE_TYPEDEF, CSYMBOL_TYPE_STRUCT, + CSYMBOL_TYPE_FUNCTION, CSYMBOL_TYPE_FUNCTION_MACRO, CSYMBOL_TYPE_TYPEDEF, CSYMBOL_TYPE_STRUCT, CSYMBOL_TYPE_ENUM, CSYMBOL_TYPE_UNION, CSYMBOL_TYPE_OBJECT, CSYMBOL_TYPE_MEMBER, CSYMBOL_TYPE_ELLIPSIS, CSYMBOL_TYPE_CONST, - TYPE_QUALIFIER_CONST) -from .utils import to_underscores + TYPE_QUALIFIER_CONST, TYPE_QUALIFIER_VOLATILE) -_xdg_data_dirs = [x for x in os.environ.get('XDG_DATA_DIRS', '').split(':') \ - + [DATADIR, '/usr/share'] if x] - -class SkipError(Exception): +class TransformerException(Exception): pass -class Names(object): - names = property(lambda self: self._names) - aliases = property(lambda self: self._aliases) - type_names = property(lambda self: self._type_names) - ctypes = property(lambda self: self._ctypes) - - def __init__(self): - super(Names, self).__init__() - self._names = odict() # Maps from GIName -> (namespace, node) - self._aliases = {} # Maps from GIName -> GIName - self._type_names = {} # Maps from GTName -> (namespace, node) - self._ctypes = {} # Maps from CType -> (namespace, node) - - class Transformer(object): + namespace = property(lambda self: self._namespace) - def __init__(self, cachestore, namespace_name, namespace_version): - self._cachestore = cachestore - self.generator = None - self._namespace = Namespace(namespace_name, namespace_version) - self._names = Names() + def __init__(self, namespace, accept_unprefixed=False, + identifier_filter_cmd=None, symbol_filter_cmd=None): + self._cachestore = CacheStore() + self._accept_unprefixed = accept_unprefixed + self._namespace = namespace self._pkg_config_packages = set() self._typedefs_ns = {} - self._strip_prefix = '' - self._includes = set() + self._parsed_includes = {} # Namespace> self._includepaths = [] + self._passthrough_mode = False + self._identifier_filter_cmd = identifier_filter_cmd + self._symbol_filter_cmd = symbol_filter_cmd - def get_names(self): - return self._names - - def get_includes(self): - return self._includes - - def set_strip_prefix(self, strip_prefix): - self._strip_prefix = strip_prefix - - def get_strip_prefix(self): - return self._strip_prefix + # Cache a list of struct/unions in C's "tag namespace". This helps + # manage various orderings of typedefs and structs. See: + # https://bugzilla.gnome.org/show_bug.cgi?id=581525 + self._tag_ns = {} def get_pkgconfig_packages(self): return self._pkg_config_packages - def set_source_ast(self, src_ast): - self.generator = src_ast + def disable_cache(self): + self._cachestore = None + + def set_passthrough_mode(self): + self._passthrough_mode = True + + def _append_new_node(self, node): + original = self._namespace.get(node.name) + # Special case constants here; we allow duplication to sort-of + # handle #ifdef. But this introduces an arch-dependency in the .gir + # file. So far this has only come up scanning glib - in theory, other + # modules will just depend on that. + if original and\ + (isinstance(original, ast.FunctionMacro) or isinstance(node, + ast.FunctionMacro)): + pass + elif isinstance(original, ast.Constant) and isinstance(node, ast.Constant): + pass + elif original is node: + # Ignore attempts to add the same node to the namespace. This can + # happen when parsing typedefs and structs in particular orderings: + # typedef struct _Foo Foo; + # struct _Foo {...}; + pass + elif original: + positions = set() + positions.update(original.file_positions) + positions.update(node.file_positions) + message.fatal("Namespace conflict for '%s'" % (node.name, ), + positions) + else: + self._namespace.append(node) + + def parse(self, symbols): + for symbol in symbols: + # WORKAROUND + # https://bugzilla.gnome.org/show_bug.cgi?id=550616 + if symbol.ident in ['gst_g_error_get_type']: + continue - def parse(self): - nodes = [] - for symbol in self.generator.get_symbols(): try: node = self._traverse_one(symbol) - except SkipError: + except TransformerException as e: + message.warn_symbol(symbol, e) continue - self._add_node(node) - return self._namespace + + if node and node.name: + self._append_new_node(node) + if isinstance(node, ast.Compound) and node.tag_name and \ + node.tag_name not in self._tag_ns: + self._tag_ns[node.tag_name] = node + + # Run through the tag namespace looking for structs that have not been + # promoted into the main namespace. In this case we simply promote them + # with their struct tag. + for tag_name, struct in self._tag_ns.items(): + if not struct.name: + try: + name = self.strip_identifier(tag_name) + struct.name = name + self._append_new_node(struct) + except TransformerException as e: + message.warn_node(node, e) def set_include_paths(self, paths): self._includepaths = list(paths) def register_include(self, include): - if include in self._includes: + if include in self._namespace.includes: return + self._namespace.includes.add(include) filename = self._find_include(include) self._parse_include(filename) - self._includes.add(include) + + def register_include_uninstalled(self, include_path): + basename = os.path.basename(include_path) + if not basename.endswith('.gir'): + raise SystemExit("Include path '%s' must be a filename path " + "ending in .gir" % (include_path, )) + girname = basename[:-4] + include = ast.Include.from_string(girname) + if include in self._namespace.includes: + return + self._namespace.includes.add(include) + self._parse_include(include_path, uninstalled=True) + + def lookup_giname(self, name): + """Given a name of the form Foo or Bar.Foo, +return the corresponding ast.Node, or None if none +available. Will throw KeyError however for unknown +namespaces.""" + if '.' not in name: + return self._namespace.get(name) + else: + (ns, giname) = name.split('.', 1) + if ns == self._namespace.name: + return self._namespace.get(giname) + # Fallback to the main namespace if not a dependency and matches a prefix + if ns in self._namespace.identifier_prefixes and ns not in self._parsed_includes: + message.warn(("Deprecated reference to identifier " + + "prefix %s in GIName %s") % (ns, name)) + return self._namespace.get(giname) + include = self._parsed_includes[ns] + return include.get(giname) + + def lookup_typenode(self, typeobj): + """Given a Type object, if it points to a giname, +calls lookup_giname() on the name. Otherwise return +None.""" + if typeobj.target_giname: + return self.lookup_giname(typeobj.target_giname) + return None # Private + def _get_gi_data_dirs(self): + data_dirs = utils.get_system_data_dirs() + data_dirs.append(DATADIR) + if os.name != 'nt': + # For backwards compatibility, was always unconditionally added to the list. + data_dirs.append('/usr/share') + return data_dirs + def _find_include(self, include): searchdirs = self._includepaths[:] - for path in _xdg_data_dirs: - searchdirs.append(os.path.join(path, GIR_SUFFIX)) - searchdirs.append(GIR_DIR) + searchdirs.extend(GIRDIR) + for path in self._get_gi_data_dirs(): + searchdirs.append(os.path.join(path, 'gir-1.0')) girname = '%s-%s.gir' % (include.name, include.version) for d in searchdirs: path = os.path.join(d, girname) if os.path.exists(path): return path - raise ValueError("Couldn't find include %r (search path: %r)"\ - % (girname, searchdirs)) - - def _parse_include(self, filename): - parser = self._cachestore.load(filename) + sys.stderr.write("Couldn't find include '%s' (search path: '%s')\n" % + (girname, searchdirs)) + sys.exit(1) + + @classmethod + def parse_from_gir(cls, filename, extra_include_dirs=None): + self = cls(None) + if extra_include_dirs is not None: + self.set_include_paths(extra_include_dirs) + self.set_passthrough_mode() + parser = self._parse_include(filename) + self._namespace = parser.get_namespace() + del self._parsed_includes[self._namespace.name] + return self + + def _parse_include(self, filename, uninstalled=False): + parser = None + if self._cachestore is not None: + parser = self._cachestore.load(filename) if parser is None: - parser = GIRParser() - parser.set_include_parsing(True) + parser = GIRParser(types_only=not self._passthrough_mode) parser.parse(filename) - self._cachestore.store(filename, parser) + if self._cachestore is not None: + self._cachestore.store(filename, parser) - for include in parser.get_includes(): - self.register_include(include) + for include in parser.get_namespace().includes: + if include.name not in self._parsed_includes: + dep_filename = self._find_include(include) + self._parse_include(dep_filename) - for pkg in parser.get_pkgconfig_packages(): - self._pkg_config_packages.add(pkg) + if not uninstalled: + for pkg in parser.get_namespace().exported_packages: + self._pkg_config_packages.add(pkg) namespace = parser.get_namespace() - nsname = namespace.name - for node in namespace.nodes: - if isinstance(node, Alias): - self._names.aliases[node.name] = (nsname, node) - elif isinstance(node, (GLibBoxed, Interface, Class)): - self._names.type_names[node.type_name] = (nsname, node) - giname = '%s.%s' % (nsname, node.name) - self._names.names[giname] = (nsname, node) - if hasattr(node, 'ctype'): - self._names.ctypes[node.ctype] = (nsname, node) - elif hasattr(node, 'symbol'): - self._names.ctypes[node.symbol] = (nsname, node) - - def _add_node(self, node): - if node is None: - return - if node.name.startswith('_'): - return - self._namespace.nodes.append(node) - self._names.names[node.name] = (None, node) - - def _strip_namespace_func(self, name): - prefix = self._namespace.name.lower() + '_' - if name.lower().startswith(prefix): - name = name[len(prefix):] + self._parsed_includes[namespace.name] = namespace + return parser + + def _iter_namespaces(self): + """Return an iterator over all included namespaces; the +currently-scanned namespace is first.""" + yield self._namespace + for ns in self._parsed_includes.values(): + yield ns + + def _sort_matches(self, val): + """Key sort which ensures items in self._namespace are last by returning + a tuple key starting with 1 for self._namespace entries and 0 for + everythin else. + """ + if val[0] == self._namespace: + return 1, val[2] else: - prefix = to_underscores(self._namespace.name).lower() + '_' - if name.lower().startswith(prefix): - name = name[len(prefix):] - return self.remove_prefix(name, isfunction=True) - - def remove_prefix(self, name, isfunction=False): - # when --strip-prefix=g: - # GHashTable -> HashTable - # g_hash_table_new -> hash_table_new - prefix = self._strip_prefix.lower() - if isfunction: - prefix += '_' - if len(name) > len(prefix) and name.lower().startswith(prefix): - name = name[len(prefix):] - - while name.startswith('_'): - name = name[1:] + return 0, val[2] + + def _split_c_string_for_namespace_matches(self, name, is_identifier=False): + if not is_identifier and self._symbol_filter_cmd: + proc = subprocess.Popen(self._symbol_filter_cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc_name, err = proc.communicate(name.encode()) + proc_name = proc_name.strip() + if proc.returncode: + raise ValueError('filter: %r exited: %d with error: %s' % + (self._symbol_filter_cmd, proc.returncode, err)) + name = proc_name.decode('ascii') + name = name.strip() + + matches = [] # Namespaces which might contain this name + unprefixed_namespaces = [] # Namespaces with no prefix, last resort + for ns in self._iter_namespaces(): + if is_identifier: + prefixes = ns.identifier_prefixes + elif name[0].isupper(): + prefixes = ns._ucase_symbol_prefixes + else: + prefixes = ns.symbol_prefixes + if prefixes: + for prefix in prefixes: + if (not is_identifier) and (not prefix.endswith('_')): + prefix = prefix + '_' + if name.startswith(prefix): + matches.append((ns, name[len(prefix):], len(prefix))) + break + else: + unprefixed_namespaces.append(ns) + if matches: + matches.sort(key=self._sort_matches) + return list(map(lambda x: (x[0], x[1]), matches)) + elif self._accept_unprefixed: + return [(self._namespace, name)] + elif unprefixed_namespaces: + # A bit of a hack; this function ideally shouldn't look through the + # contents of namespaces; but since we aren't scanning anything + # without a prefix, it's not too bad. + for ns in unprefixed_namespaces: + if name in ns: + return [(ns, name)] + raise ValueError("Unknown namespace for %s '%s'" + % ('identifier' if is_identifier else 'symbol', name, )) + + def split_ctype_namespaces(self, ident): + """Given a StudlyCaps string identifier like FooBar, return a +list of (namespace, stripped_identifier) sorted by namespace length, +or raise ValueError. As a special case, if the current namespace matches, +it is always biggest (i.e. last).""" + return self._split_c_string_for_namespace_matches(ident, is_identifier=True) + + def split_csymbol_namespaces(self, symbol): + """Given a C symbol like foo_bar_do_baz, return a list of +(namespace, stripped_symbol) sorted by namespace match probablity, or +raise ValueError.""" + return self._split_c_string_for_namespace_matches(symbol, is_identifier=False) + + def split_csymbol(self, symbol): + """Given a C symbol like foo_bar_do_baz, return the most probable +(namespace, stripped_symbol) match, or raise ValueError.""" + matches = self._split_c_string_for_namespace_matches(symbol, is_identifier=False) + return matches[-1] + + def strip_identifier(self, ident): + if self._identifier_filter_cmd: + proc = subprocess.Popen(self._identifier_filter_cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc_ident, err = proc.communicate(ident.encode()) + if proc.returncode: + raise ValueError('filter: %r exited: %d with error: %s' % + (self._identifier_filter_cmd, proc.returncode, err)) + ident = proc_ident.decode('ascii').strip() + + hidden = ident.startswith('_') + if hidden: + ident = ident[1:] + try: + matches = self.split_ctype_namespaces(ident) + except ValueError as e: + raise TransformerException(str(e)) + for ns, name in matches: + if ns is self._namespace: + if hidden: + return '_' + name + return name + (ns, name) = matches[-1] + raise TransformerException( + "Skipping foreign identifier '%s' from namespace %s" % (ident, ns.name, )) + return None + + def _strip_symbol(self, symbol): + ident = symbol.ident + hidden = ident.startswith('_') + if hidden: + ident = ident[1:] + try: + (ns, name) = self.split_csymbol(ident) + except ValueError as e: + raise TransformerException(str(e)) + if ns != self._namespace: + raise TransformerException( + "Skipping foreign symbol from namespace %s" % (ns.name, )) + if hidden: + return '_' + name return name - def _traverse_one(self, symbol, stype=None): + def _traverse_one(self, symbol, stype=None, parent_symbol=None): assert isinstance(symbol, SourceSymbol), symbol if stype is None: stype = symbol.type if stype == CSYMBOL_TYPE_FUNCTION: return self._create_function(symbol) + elif stype == CSYMBOL_TYPE_FUNCTION_MACRO: + return self._create_function_macro(symbol) elif stype == CSYMBOL_TYPE_TYPEDEF: return self._create_typedef(symbol) elif stype == CSYMBOL_TYPE_STRUCT: - return self._create_struct(symbol) + return self._create_tag_ns_compound(ast.Record, symbol) elif stype == CSYMBOL_TYPE_ENUM: return self._create_enum(symbol) - elif stype == CSYMBOL_TYPE_OBJECT: - return self._create_object(symbol) elif stype == CSYMBOL_TYPE_MEMBER: - return self._create_member(symbol) + return self._create_member(symbol, parent_symbol) elif stype == CSYMBOL_TYPE_UNION: - return self._create_union(symbol) + return self._create_tag_ns_compound(ast.Union, symbol) elif stype == CSYMBOL_TYPE_CONST: return self._create_const(symbol) + # Ignore variable declarations in the header + elif stype == CSYMBOL_TYPE_OBJECT: + pass else: - raise NotImplementedError( - 'Transformer: unhandled symbol: %r' % (symbol, )) + print("transformer: unhandled symbol: '%s'" % (symbol, )) def _enum_common_prefix(self, symbol): def common_prefix(a, b): @@ -242,196 +421,243 @@ class Transformer(object): prefixlen = 0 members = [] for child in symbol.base_type.child_list: + if child.private: + continue if prefixlen > 0: name = child.ident[prefixlen:] else: # Ok, the enum members don't have a consistent prefix # among them, so let's just remove the global namespace # prefix. - name = self.remove_prefix(child.ident) - members.append(Member(name.lower(), - child.const_int, - child.ident)) + name = self._strip_symbol(child) + members.append(ast.Member(name.lower(), + child.const_int, + child.ident, + None)) - enum_name = self.remove_prefix(symbol.ident) + enum_name = self.strip_identifier(symbol.ident) if symbol.base_type.is_bitfield: - klass = Bitfield + klass = ast.Bitfield else: - klass = Enum - node = klass(enum_name, symbol.ident, members) - self._names.type_names[symbol.ident] = (None, node) + klass = ast.Enum + node = klass(enum_name, symbol.ident, members=members) + node.add_symbol_reference(symbol) return node - def _create_object(self, symbol): - return Member(symbol.ident, symbol.base_type.name, - symbol.ident) - - def _type_is_callback(self, type): - if (isinstance(type, Callback) or - isinstance(self._typedefs_ns.get(type.name), Callback)): - return True - return False - - def _handle_closure(self, param, closure_idx, closure_param): - if (closure_param.type.name == 'any' and - closure_param.name.endswith('data')): - param.closure_name = closure_param.name - param.closure_index = closure_idx - return True - return False - - def _handle_destroy(self, param, destroy_idx, destroy_param): - if ((self._namespace.name == 'GLib' and - destroy_param.type.name == 'DestroyNotify') or - destroy_param.type.name == 'GLib.DestroyNotify'): - param.destroy_name = destroy_param.name - param.destroy_index = destroy_idx - return True - return False - - def _augment_callback_params(self, params): - for i, param in enumerate(params): - if not self._type_is_callback(param.type): - continue - - # j is the index where we look for closure/destroy to - # group with the callback param - j = i + 1 - if j == len(params): - continue # no more args -> nothing to group - # look at the param directly following for either a - # closure or a destroy; only one of these will fire - had_closure = self._handle_closure(param, j, params[j]) - had_destroy = self._handle_destroy(param, j, params[j]) - j += 1 - # are we out of params, or did we find neither? - if j == len(params) or (not had_closure and not had_destroy): - continue - # we found either a closure or a destroy; check the - # parameter following for the other - if not had_closure: - self._handle_closure(param, j, params[j]) - if not had_destroy: - self._handle_destroy(param, j, params[j]) - def _create_function(self, symbol): - parameters = list(self._create_parameters(symbol.base_type)) + # Drop functions that start with _ very early on here + if symbol.ident.startswith('_'): + return None + parameters = list(self._create_parameters(symbol, symbol.base_type)) return_ = self._create_return(symbol.base_type.base_type) - self._augment_callback_params(parameters) - name = self._strip_namespace_func(symbol.ident) - func = Function(name, return_, parameters, symbol.ident) + name = self._strip_symbol(symbol) + func = ast.Function(name, return_, parameters, False, symbol.ident) + func.add_symbol_reference(symbol) return func - def _create_source_type(self, source_type): - if source_type is None: - return 'None' + def _create_function_macro(self, symbol): + if symbol.ident.startswith('_'): + return None + + if (symbol.source_filename is None or not symbol.source_filename.endswith('.h')): + return None + + parameters = list(self._create_parameters(symbol, symbol.base_type)) + name = self._strip_symbol(symbol) + macro = ast.FunctionMacro(name, parameters, symbol.ident) + macro.add_symbol_reference(symbol) + return macro + + def _create_source_type(self, source_type, is_parameter=False): + assert source_type is not None if source_type.type == CTYPE_VOID: value = 'void' elif source_type.type == CTYPE_BASIC_TYPE: value = source_type.name elif source_type.type == CTYPE_TYPEDEF: value = source_type.name + elif (source_type.type == CTYPE_POINTER or + # Array to pointer adjustment as per 6.7.6.3. + # This is performed only on the outermost array, + # so we don't forward is_parameter. + (source_type.type == CTYPE_ARRAY and is_parameter)): + value = self._create_source_type(source_type.base_type) + '*' elif source_type.type == CTYPE_ARRAY: return self._create_source_type(source_type.base_type) - elif source_type.type == CTYPE_POINTER: - value = self._create_source_type(source_type.base_type) + '*' else: - value = 'any' + value = 'gpointer' return value - def _create_parameters(self, base_type): + def _create_complete_source_type(self, source_type, is_parameter=False): + assert source_type is not None - # warn if we see annotations for unknown parameters - param_names = set(child.ident for child in base_type.child_list) - for child in base_type.child_list: - yield self._create_parameter(child) + const = (source_type.type_qualifier & TYPE_QUALIFIER_CONST) + volatile = (source_type.type_qualifier & TYPE_QUALIFIER_VOLATILE) - def _create_member(self, symbol): + if source_type.type == CTYPE_VOID: + return 'void' + elif source_type.type in [CTYPE_BASIC_TYPE, + CTYPE_TYPEDEF, + CTYPE_STRUCT, + CTYPE_UNION, + CTYPE_ENUM]: + value = source_type.name + if const: + value = 'const ' + value + if volatile: + value = 'volatile ' + value + return value + elif (source_type.type == CTYPE_POINTER or + # Array to pointer adjustment as per 6.7.6.3. + # This is performed only on the outermost array, + # so we don't forward is_parameter. + (source_type.type == CTYPE_ARRAY and is_parameter)): + value = self._create_complete_source_type(source_type.base_type) + '*' + # TODO: handle pointer to function as a special case? + if const: + value += ' const' + if volatile: + value += ' volatile' + return value + elif source_type.type == CTYPE_ARRAY: + return self._create_complete_source_type(source_type.base_type) + else: + if const: + value = 'gconstpointer' + else: + value = 'gpointer' + if volatile: + value = 'volatile ' + value + return value + + def _create_parameters(self, symbol, base_type): + for i, child in enumerate(base_type.child_list): + yield self._create_parameter(symbol, i, child) + + def _synthesize_union_type(self, symbol, parent_symbol): + # Synthesize a named union so that it can be referenced. + parent_ident = parent_symbol.ident + # FIXME: Should split_ctype_namespaces handle the hidden case? + hidden = parent_ident.startswith('_') + if hidden: + parent_ident = parent_ident[1:] + matches = self.split_ctype_namespaces(parent_ident) + (namespace, parent_name) = matches[-1] + assert namespace and parent_name + if hidden: + parent_name = '_' + parent_name + fake_union = ast.Union("%s__%s__union" % (parent_name, symbol.ident)) + # _parse_fields accesses .base_type.child_list, so we have to + # pass symbol.base_type even though that refers to the array, not the + # union. + self._parse_fields(symbol.base_type, fake_union) + self._append_new_node(fake_union) + fake_type = ast.Type( + target_giname="%s.%s" % (namespace.name, fake_union.name)) + return fake_type + + def _create_member(self, symbol, parent_symbol=None): source_type = symbol.base_type - if (source_type.type == CTYPE_POINTER and - symbol.base_type.base_type.type == CTYPE_FUNCTION): - node = self._create_callback(symbol) + if (source_type.type == CTYPE_POINTER + and symbol.base_type.base_type.type == CTYPE_FUNCTION): + node = self._create_callback(symbol, member=True) elif source_type.type == CTYPE_STRUCT and source_type.name is None: - node = self._create_struct(symbol, anonymous=True) + node = self._create_member_compound(ast.Record, symbol) elif source_type.type == CTYPE_UNION and source_type.name is None: - node = self._create_union(symbol, anonymous=True) + node = self._create_member_compound(ast.Union, symbol) else: # Special handling for fields; we don't have annotations on them # to apply later, yet. if source_type.type == CTYPE_ARRAY: - ctype = self._create_source_type(source_type) - canonical_ctype = self._canonicalize_ctype(ctype) - if canonical_ctype[-1] == '*': - derefed_name = canonical_ctype[:-1] + # Determine flattened array size and its element type. + flattened_size = 1 + while source_type.type == CTYPE_ARRAY: + for child in source_type.child_list: + if flattened_size is not None: + flattened_size *= child.const_int + break + else: + flattened_size = None + source_type = source_type.base_type + + # If the array contains anonymous unions, like in the GValue + # struct, we need to handle this specially. This is necessary + # to be able to properly calculate the size of the compound + # type (e.g. GValue) that contains this array, see + # . + if source_type.type == CTYPE_UNION and source_type.name is None: + element_type = self._synthesize_union_type(symbol, parent_symbol) else: - derefed_name = canonical_ctype - derefed_name = self.resolve_param_type(derefed_name) - ftype = Array(ctype, self.parse_ctype(derefed_name)) - child_list = list(symbol.base_type.child_list) + ctype = self._create_source_type(source_type) + complete_ctype = self._create_complete_source_type(source_type) + element_type = self.create_type_from_ctype_string(ctype, + complete_ctype=complete_ctype) + ftype = ast.Array(None, element_type) ftype.zeroterminated = False - if child_list: - ftype.size = '%d' % (child_list[0].const_int, ) + ftype.size = flattened_size else: - ftype = self._create_type(symbol.base_type, - is_param=False, is_retval=False) - ftype = self.resolve_param_type(ftype) - # Fields are assumed to be read-write + ftype = self._create_type_from_base(symbol.base_type) + # ast.Fields are assumed to be read-write # (except for Objects, see also glibtransformer.py) - node = Field(symbol.ident, ftype, ftype.name, - readable=True, writable=True, bits=symbol.const_int) + node = ast.Field(symbol.ident, ftype, + readable=True, writable=True, + bits=symbol.const_int) + if symbol.private: + node.readable = False + node.writable = False + node.private = True return node def _create_typedef(self, symbol): ctype = symbol.base_type.type - if (ctype == CTYPE_POINTER and - symbol.base_type.base_type.type == CTYPE_FUNCTION): + if (ctype == CTYPE_POINTER and symbol.base_type.base_type.type == CTYPE_FUNCTION): node = self._create_typedef_callback(symbol) - elif (ctype == CTYPE_POINTER and - symbol.base_type.base_type.type == CTYPE_STRUCT): - node = self._create_typedef_struct(symbol, disguised=True) + elif (ctype == CTYPE_FUNCTION): + node = self._create_typedef_callback(symbol) + elif (ctype == CTYPE_POINTER and symbol.base_type.base_type.type == CTYPE_STRUCT): + node = self._create_typedef_compound(ast.Record, symbol, disguised=True) elif ctype == CTYPE_STRUCT: - node = self._create_typedef_struct(symbol) + node = self._create_typedef_compound(ast.Record, symbol) elif ctype == CTYPE_UNION: - node = self._create_typedef_union(symbol) + node = self._create_typedef_compound(ast.Union, symbol) elif ctype == CTYPE_ENUM: return self._create_enum(symbol) elif ctype in (CTYPE_TYPEDEF, CTYPE_POINTER, CTYPE_BASIC_TYPE, CTYPE_VOID): - name = self.remove_prefix(symbol.ident) - if symbol.base_type.name: - target = self.remove_prefix(symbol.base_type.name) - else: - target = 'none' - if name in type_names: + name = self.strip_identifier(symbol.ident) + target = self._create_type_from_base(symbol.base_type) + if name in ast.type_names: return None - return Alias(name, target, ctype=symbol.ident) + # https://bugzilla.gnome.org/show_bug.cgi?id=755882 + if name.endswith('_autoptr'): + return None + node = ast.Alias(name, target, ctype=symbol.ident) + node.add_symbol_reference(symbol) else: raise NotImplementedError( - "symbol %r of type %s" % (symbol.ident, ctype_name(ctype))) + "symbol '%s' of type %s" % (symbol.ident, ctype_name(ctype))) return node def _canonicalize_ctype(self, ctype): # First look up the ctype including any pointers; # a few type names like 'char*' have their own aliases # and we need pointer information for those. - firstpass = type_name_from_ctype(ctype) + firstpass = ast.type_names.get(ctype) # If we have a particular alias for this, skip deep # canonicalization to prevent changing # e.g. char* -> int8* - if firstpass != ctype: - return firstpass + if firstpass: + return firstpass.target_fundamental - # We're also done if the type is already a fundamental - # known type, or there are no pointers. - if ctype in type_names or not firstpass.endswith('*'): - return firstpass + if not ctype.endswith('*'): + return ctype # We have a pointer type. # Strip the end pointer, canonicalize our base type - base = firstpass[:-1] + base = ctype[:-1] canonical_base = self._canonicalize_ctype(base) # Append the pointer again @@ -439,249 +665,368 @@ class Transformer(object): return canonical - def parse_ctype(self, ctype, is_member=False): + def _create_type_from_base(self, source_type, is_parameter=False, is_return=False): + ctype = self._create_source_type(source_type, is_parameter=is_parameter) + complete_ctype = self._create_complete_source_type(source_type, is_parameter=is_parameter) + const = ((source_type.type == CTYPE_POINTER) and + (source_type.base_type.type_qualifier & TYPE_QUALIFIER_CONST)) + return self.create_type_from_ctype_string(ctype, is_const=const, + is_parameter=is_parameter, is_return=is_return, + complete_ctype=complete_ctype) + + def _create_bare_container_type(self, base, ctype=None, + is_const=False, complete_ctype=None): + if base in ('GList', 'GSList', 'GLib.List', 'GLib.SList'): + if base in ('GList', 'GSList'): + name = 'GLib.' + base[1:] + else: + name = base + return ast.List(name, ast.TYPE_ANY, ctype=ctype, + is_const=is_const, complete_ctype=complete_ctype) + elif base in ('GByteArray', 'GLib.ByteArray', 'GObject.ByteArray'): + return ast.Array('GLib.ByteArray', ast.TYPE_UINT8, ctype=ctype, + is_const=is_const, complete_ctype=complete_ctype) + elif base in ('GArray', 'GPtrArray', + 'GLib.Array', 'GLib.PtrArray', + 'GObject.Array', 'GObject.PtrArray'): + if '.' in base: + name = 'GLib.' + base.split('.', 1)[1] + else: + name = 'GLib.' + base[1:] + return ast.Array(name, ast.TYPE_ANY, ctype=ctype, + is_const=is_const, complete_ctype=complete_ctype) + elif base in ('GHashTable', 'GLib.HashTable', 'GObject.HashTable'): + return ast.Map(ast.TYPE_ANY, ast.TYPE_ANY, ctype=ctype, is_const=is_const, + complete_ctype=complete_ctype) + return None + + def create_type_from_ctype_string(self, ctype, is_const=False, + is_parameter=False, is_return=False, + complete_ctype=None): canonical = self._canonicalize_ctype(ctype) - - # Remove all pointers - we require standard calling - # conventions. For example, an 'int' is always passed by - # value (unless it's out or inout). - derefed_typename = canonical.replace('*', '') - - # Preserve "pointerness" of struct/union members - if (is_member and canonical.endswith('*') and - derefed_typename in BASIC_GIR_TYPES): - return 'any' - else: - return derefed_typename - - def _create_type(self, source_type, is_param, is_retval): - ctype = self._create_source_type(source_type) - if ctype.startswith('va_list'): - raise SkipError() - # FIXME: FILE* should not be skipped, it should be handled - # properly instead - elif ctype == 'FILE*': - raise SkipError - - is_member = not (is_param or is_retval) - # Here we handle basic type parsing; most of the heavy lifting - # and inference comes in annotationparser.py when we merge - # in annotation data. - derefed_name = self.parse_ctype(ctype, is_member) - rettype = Type(derefed_name, ctype) - rettype.canonical = self._canonicalize_ctype(ctype) - derefed_ctype = ctype.replace('*', '') - rettype.derefed_canonical = self._canonicalize_ctype(derefed_ctype) - - canontype = type_name_from_ctype(ctype) - # Is it a const char * or a const gpointer? - if ((canontype == TYPE_STRING or source_type.type == CTYPE_POINTER) and - (source_type.base_type.type_qualifier & TYPE_QUALIFIER_CONST)): - rettype.is_const = True - return rettype - - def _create_parameter(self, symbol): + base = canonical.replace('*', '') + + # While gboolean and _Bool are distinct types, they used to be treated + # by scanner as exactly the same one. In general this is incorrect + # because of different ABI, but this usually works fine, + # so for backward compatibility lets continue for now: + # https://gitlab.gnome.org/GNOME/gobject-introspection/merge_requests/24#note_92792 + if canonical in ('_Bool', 'bool'): + canonical = 'gboolean' + base = canonical + + # Special default: char ** -> ast.Array, same for GStrv + if (is_return and canonical == 'utf8*') or base == 'GStrv': + bare_utf8 = ast.TYPE_STRING.clone() + bare_utf8.ctype = None + return ast.Array(None, bare_utf8, ctype=ctype, + is_const=is_const, complete_ctype=complete_ctype) + + fundamental = ast.type_names.get(base) + if fundamental is not None: + return ast.Type(target_fundamental=fundamental.target_fundamental, + ctype=ctype, + is_const=is_const, complete_ctype=complete_ctype) + container = self._create_bare_container_type(base, ctype=ctype, is_const=is_const, + complete_ctype=complete_ctype) + if container: + return container + return ast.Type(ctype=ctype, is_const=is_const, complete_ctype=complete_ctype) + + def _create_parameter(self, parent_symbol, index, symbol): if symbol.type == CSYMBOL_TYPE_ELLIPSIS: - ptype = Varargs() + return ast.Parameter('...', ast.Varargs()) else: - ptype = self._create_type(symbol.base_type, - is_param=True, is_retval=False) - ptype = self.resolve_param_type(ptype) - return Parameter(symbol.ident, ptype) + if symbol.base_type: + ptype = self._create_type_from_base(symbol.base_type, is_parameter=True) + else: + ptype = None + + if symbol.ident is None: + if symbol.base_type and symbol.base_type.type != CTYPE_VOID: + message.warn_symbol(parent_symbol, "missing parameter name; undocumentable") + ident = 'arg%d' % (index, ) + else: + ident = symbol.ident + + return ast.Parameter(ident, ptype) def _create_return(self, source_type): - rtype = self._create_type(source_type, - is_param=False, is_retval=True) - rtype = self.resolve_param_type(rtype) - return_ = Return(rtype) - return return_ + typeval = self._create_type_from_base(source_type, is_return=True) + return ast.Return(typeval) def _create_const(self, symbol): + if symbol.ident.startswith('_'): + return None + # Don't create constants for non-public things # http://bugzilla.gnome.org/show_bug.cgi?id=572790 - if (symbol.source_filename is None or - not symbol.source_filename.endswith('.h')): + if (symbol.source_filename is None or not symbol.source_filename.endswith('.h')): return None - name = self.remove_prefix(symbol.ident) + name = self._strip_symbol(symbol) if symbol.const_string is not None: - type_name = 'utf8' + typeval = ast.TYPE_STRING value = symbol.const_string elif symbol.const_int is not None: - type_name = 'int' - value = symbol.const_int + if symbol.base_type is not None: + typeval = self._create_type_from_base(symbol.base_type) + else: + typeval = ast.TYPE_INT + unaliased = typeval + self._resolve_type_from_ctype(unaliased) + if typeval.target_giname and typeval.ctype: + target = self.lookup_giname(typeval.target_giname) + target = self.resolve_aliases(target) + if isinstance(target, ast.Type): + unaliased = target + if unaliased == ast.TYPE_UINT64: + value = str(symbol.const_int % 2 ** 64) + elif unaliased == ast.TYPE_UINT32: + value = str(symbol.const_int % 2 ** 32) + elif unaliased == ast.TYPE_UINT16: + value = str(symbol.const_int % 2 ** 16) + elif unaliased == ast.TYPE_UINT8: + value = str(symbol.const_int % 2 ** 16) + else: + value = str(symbol.const_int) + elif symbol.const_boolean is not None: + typeval = ast.TYPE_BOOLEAN + value = "true" if symbol.const_boolean else "false" elif symbol.const_double is not None: - type_name = 'double' - value = symbol.const_double + typeval = ast.TYPE_DOUBLE + value = '%f' % (symbol.const_double, ) else: raise AssertionError() - const = Constant(name, type_name, value) + const = ast.Constant(name, typeval, value, + symbol.ident) + const.add_symbol_reference(symbol) return const - def _create_typedef_struct(self, symbol, disguised=False): - name = self.remove_prefix(symbol.ident) - struct = Struct(name, symbol.ident, disguised) - self._typedefs_ns[symbol.ident] = struct - self._create_struct(symbol) - return struct - - def _create_typedef_union(self, symbol): - name = self.remove_prefix(symbol.ident) - union = Union(name, symbol.ident) - self._typedefs_ns[symbol.ident] = union - self._create_union(symbol) - return union - - def _create_typedef_callback(self, symbol): - callback = self._create_callback(symbol) - self._typedefs_ns[callback.name] = callback - return callback - - def _create_compound(self, klass, symbol, anonymous): - if symbol.ident is None: - # the compound is an anonymous member of another union or a struct - assert anonymous - compound = klass(None, None) + def _create_typedef_compound(self, compound_class, symbol, disguised=False): + name = self.strip_identifier(symbol.ident) + assert symbol.base_type + if symbol.base_type.name: + tag_name = symbol.base_type.name else: - compound = self._typedefs_ns.get(symbol.ident, None) - - if compound is None: - # This is a bit of a hack; really we should try - # to resolve through the typedefs to find the real - # name - if symbol.ident.startswith('_'): - name = symbol.ident[1:] - compound = self._typedefs_ns.get(name, None) + tag_name = None + + # If the struct already exists in the tag namespace, use it. + if tag_name in self._tag_ns: + compound = self._tag_ns[tag_name] + if compound.name: + # If the struct name is set it means the struct has already been + # promoted from the tag namespace to the main namespace by a + # prior typedef struct. If we get here it means this is another + # typedef of that struct. Instead of creating an alias to the + # primary typedef that has been promoted, we create a new Record + # with shared fields. This handles the case where we want to + # give structs like GInitiallyUnowned its own Record: + # typedef struct _GObject GObject; + # typedef struct _GObject GInitiallyUnowned; + # See: http://bugzilla.gnome.org/show_bug.cgi?id=569408 + new_compound = compound_class(name, symbol.ident, tag_name=tag_name) + new_compound.fields = compound.fields + new_compound.add_symbol_reference(symbol) + return new_compound else: - name = symbol.ident - if compound is None: - name = self.remove_prefix(name) - compound = klass(name, symbol.ident) + # If the struct does not have its name set, it exists only in + # the tag namespace. Set it here and return it which will + # promote it to the main namespace. Essentially the first + # typedef for a struct clobbers its name and ctype which is what + # will be visible to GI. + compound.name = name + compound.ctype = symbol.ident + else: + # Create a new struct with a typedef name and tag name when available. + # Structs with a typedef name are promoted into the main namespace + # by it being returned to the "parse" function and are also added to + # the tag namespace if it has a tag_name set. + compound = compound_class(name, symbol.ident, disguised=disguised, tag_name=tag_name) + if tag_name: + # Force the struct as disguised for now since we do not yet know + # if it has fields that will be parsed. Note that this is using + # an erroneous definition of disguised and we should eventually + # only look at the field count when needed. + compound.disguised = True + else: + # Case where we have an anonymous struct which is typedef'd: + # typedef struct {...} Struct; + # we need to parse the fields because we never get a struct + # in the tag namespace which is normally where fields are parsed. + self._parse_fields(symbol, compound) - for child in symbol.base_type.child_list: - field = self._traverse_one(child) - if field: - compound.fields.append(field) + compound.add_symbol_reference(symbol) + return compound + def _create_tag_ns_compound(self, compound_class, symbol): + # Get or create a struct from C's tag namespace + if symbol.ident in self._tag_ns: + compound = self._tag_ns[symbol.ident] + else: + compound = compound_class(None, symbol.ident, tag_name=symbol.ident) + + # Make sure disguised is False as we are now about to parse the + # fields of the real struct. + compound.disguised = False + # Fields may need to be parsed in either of the above cases because the + # Record can be created with a typedef prior to the struct definition. + self._parse_fields(symbol, compound) + compound.add_symbol_reference(symbol) return compound - def _create_struct(self, symbol, anonymous=False): - return self._create_compound(Struct, symbol, anonymous) + def _create_member_compound(self, compound_class, symbol): + compound = compound_class(symbol.ident, symbol.ident) + self._parse_fields(symbol, compound) + compound.add_symbol_reference(symbol) + return compound - def _create_union(self, symbol, anonymous=False): - return self._create_compound(Union, symbol, anonymous) + def _create_typedef_callback(self, symbol): + callback = self._create_callback(symbol) + if not callback: + return None + return callback - def _create_callback(self, symbol): - parameters = list(self._create_parameters(symbol.base_type.base_type)) - retval = self._create_return(symbol.base_type.base_type.base_type) + def _parse_fields(self, symbol, compound): + for child in symbol.base_type.child_list: + child_node = self._traverse_one(child, parent_symbol=symbol) + if not child_node: + continue + if isinstance(child_node, ast.Field): + field = child_node + else: + field = ast.Field(child.ident, None, True, False, + anonymous_node=child_node) + compound.fields.append(field) + + def _create_callback(self, symbol, member=False): + if (symbol.base_type.type == CTYPE_FUNCTION): # function + paramtype = symbol.base_type + retvaltype = symbol.base_type.base_type + elif (symbol.base_type.type == CTYPE_POINTER): # function pointer + paramtype = symbol.base_type.base_type + retvaltype = symbol.base_type.base_type.base_type + parameters = list(self._create_parameters(symbol, paramtype)) + retval = self._create_return(retvaltype) # Mark the 'user_data' arguments for i, param in enumerate(parameters): - if (param.type.name == 'any' and - param.name == 'user_data'): - param.closure_index = i + if (param.type.target_fundamental == 'gpointer' and param.argname == 'user_data'): + param.closure_name = param.argname - if symbol.ident.find('_') > 0: - name = self.remove_prefix(symbol.ident, True) + if member: + name = symbol.ident + elif symbol.ident.find('_') > 0: + name = self._strip_symbol(symbol) else: - name = self.remove_prefix(symbol.ident) - callback = Callback(name, retval, parameters, symbol.ident) + name = self.strip_identifier(symbol.ident) + callback = ast.Callback(name, retval, parameters, False, + ctype=symbol.ident) + callback.add_symbol_reference(symbol) return callback - def _typepair_to_str(self, item): - nsname, item = item - if nsname is None: - return item.name - return '%s.%s' % (nsname, item.name) + def create_type_from_user_string(self, typestr): + """Parse a C type string (as might be given from an + annotation) and resolve it. For compatibility, we can consume +both GI type string (utf8, Foo.Bar) style, as well as C (char *, FooBar) style. - def _resolve_type_name_1(self, type_name, ctype, names): - # First look using the built-in names - if ctype: - try: - return type_names[ctype] - except KeyError, e: - pass - try: - return type_names[type_name] - except KeyError, e: - pass +Note that type resolution may not succeed.""" + if '.' in typestr: + container = self._create_bare_container_type(typestr) + if container: + typeval = container + else: + typeval = self._namespace.type_from_name(typestr) + else: + typeval = self.create_type_from_ctype_string(typestr) + + self.resolve_type(typeval) + if typeval.resolved: + # Explicitly clear out the c_type; there isn't one in this case. + typeval.ctype = None + return typeval + + def _resolve_type_from_ctype_all_namespaces(self, typeval, pointer_stripped): + # If we can't determine the namespace from the type name, + # fall back to trying all of our includes. An example of this is mutter, + # which has nominal namespace of "Meta", but a few classes are + # "Mutter". We don't export that data in introspection currently. + # Basically the library should be fixed, but we'll hack around it here. + for namespace in self._parsed_includes.values(): + target = namespace.get_by_ctype(pointer_stripped) + if target: + typeval.target_giname = '%s.%s' % (namespace.name, target.name) + return True + return False - if ctype: - ctype = ctype.replace('*', '') - resolved = names.ctypes.get(ctype) - if resolved: - return self._typepair_to_str(resolved) - type_name = self.remove_prefix(type_name) - resolved = names.aliases.get(type_name) - if resolved: - return self._typepair_to_str(resolved) - resolved = names.names.get(type_name) - if resolved: - return self._typepair_to_str(resolved) - resolved = names.type_names.get(type_name) - if resolved: - return self._typepair_to_str(resolved) - raise KeyError("failed to find %r" % (type_name, )) - - def resolve_type_name_full(self, type_name, ctype, - names, allow_invalid=True): + def _resolve_type_from_ctype(self, typeval): + assert typeval.ctype is not None + pointer_stripped = typeval.ctype.replace('*', '') try: - return self._resolve_type_name_1(type_name, ctype, names) - except KeyError, e: - try: - return self._resolve_type_name_1(type_name, ctype, self._names) - except KeyError, e: - if not allow_invalid: - raise - return type_name + matches = self.split_ctype_namespaces(pointer_stripped) + except ValueError: + return self._resolve_type_from_ctype_all_namespaces(typeval, pointer_stripped) + for namespace, name in matches: + target = namespace.get(name) + if not target: + target = namespace.get_by_ctype(pointer_stripped) + if target: + typeval.target_giname = '%s.%s' % (namespace.name, target.name) + return True + return False - def resolve_type_name(self, type_name, ctype=None): - try: - return self.resolve_type_name_full(type_name, ctype, self._names) - except KeyError, e: - return type_name - - def gtypename_to_giname(self, gtname, names): - resolved = names.type_names.get(gtname) - if resolved: - return self._typepair_to_str(resolved) - resolved = self._names.type_names.get(gtname) - if resolved: - return self._typepair_to_str(resolved) - raise KeyError("Failed to resolve GType name: %r" % (gtname, )) - - def ctype_of(self, obj): - if hasattr(obj, 'ctype'): - return obj.ctype - elif hasattr(obj, 'symbol'): - return obj.symbol - else: - return None + def _resolve_type_from_gtype_name(self, typeval): + assert typeval.gtype_name is not None + for ns in self._iter_namespaces(): + node = ns.type_names.get(typeval.gtype_name, None) + if node is not None: + typeval.target_giname = '%s.%s' % (ns.name, node.name) + return True + return False - def resolve_param_type_full(self, ptype, names, **kwargs): - if isinstance(ptype, Node): - ptype.name = self.resolve_type_name_full(ptype.name, - self.ctype_of(ptype), - names, **kwargs) - elif isinstance(ptype, basestring): - return self.resolve_type_name_full(ptype, ptype, names, **kwargs) - else: - raise AssertionError("Unhandled param: %r" % (ptype, )) - return ptype + def _resolve_type_internal(self, typeval): + if isinstance(typeval, (ast.Array, ast.List)): + return self.resolve_type(typeval.element_type) + elif isinstance(typeval, ast.Map): + key_resolved = self.resolve_type(typeval.key_type) + value_resolved = self.resolve_type(typeval.value_type) + return key_resolved and value_resolved + elif typeval.resolved: + return True + elif typeval.ctype: + return self._resolve_type_from_ctype(typeval) + elif typeval.gtype_name: + return self._resolve_type_from_gtype_name(typeval) + + def resolve_type(self, typeval): + if not self._resolve_type_internal(typeval): + return False + + if typeval.target_fundamental or typeval.target_foreign: + return True + + assert typeval.target_giname is not None - def resolve_param_type(self, ptype): try: - return self.resolve_param_type_full(ptype, self._names) - except KeyError, e: - return ptype - - def follow_aliases(self, type_name, names): - while True: - resolved = names.aliases.get(type_name) - if resolved: - (ns, alias) = resolved - type_name = alias.target + type_ = self.lookup_giname(typeval.target_giname) + except KeyError: + type_ = None + + if type_ is None: + typeval.target_giname = None + + return typeval.resolved + + def resolve_aliases(self, typenode): + """Removes all aliases from typenode, returns first non-alias + in the typenode alias chain. Returns typenode argument if it + is not an alias.""" + while isinstance(typenode, ast.Alias): + if typenode.target.target_giname is not None: + typenode = self.lookup_giname(typenode.target.target_giname) + elif typenode.target.target_fundamental is not None: + typenode = typenode.target else: break - return type_name + return typenode