giscanner/transformer.py

   1 # -*- Mode: Python -*-
   2 # GObject-Introspection - a framework for introspecting GObject libraries
   3 # Copyright (C) 2008  Johan Dahlin
   4 #
   5 # This library is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU Lesser General Public
   7 # License as published by the Free Software Foundation; either
   8 # version 2 of the License, or (at your option) any later version.
   9 #
  10 # This library is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 # Lesser General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU Lesser General Public
  16 # License along with this library; if not, write to the
  17 # Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18 # Boston, MA 02111-1307, USA.
  19 #
  20
  21 import os
  22 import sys
  23
  24 from . import ast
  25 from . import message
  26 from .cachestore import CacheStore
  27 from .girparser import GIRParser
  28 from .sourcescanner import (
  29     SourceSymbol, ctype_name, CTYPE_POINTER,
  30     CTYPE_BASIC_TYPE, CTYPE_UNION, CTYPE_ARRAY, CTYPE_TYPEDEF,
  31     CTYPE_VOID, CTYPE_ENUM, CTYPE_FUNCTION, CTYPE_STRUCT,
  32     CSYMBOL_TYPE_FUNCTION, CSYMBOL_TYPE_TYPEDEF, CSYMBOL_TYPE_STRUCT,
  33     CSYMBOL_TYPE_ENUM, CSYMBOL_TYPE_UNION, CSYMBOL_TYPE_OBJECT,
  34     CSYMBOL_TYPE_MEMBER, CSYMBOL_TYPE_ELLIPSIS, CSYMBOL_TYPE_CONST,
  35     TYPE_QUALIFIER_CONST, TYPE_QUALIFIER_VOLATILE)
  36
  37 class TransformerException(Exception):
  38     pass
  39
  40
  41 _xdg_data_dirs = [x for x in os.environ.get('XDG_DATA_DIRS', '').split(os.pathsep)]
  42 _xdg_data_dirs.append(DATADIR)
  43
  44 if os.name != 'nt':
  45     _xdg_data_dirs.append('/usr/share')
  46
  47
  48 class Transformer(object):
  49     namespace = property(lambda self: self._namespace)
  50
  51     def __init__(self, namespace, accept_unprefixed=False):
  52         self._cachestore = CacheStore()
  53         self._accept_unprefixed = accept_unprefixed
  54         self._namespace = namespace
  55         self._pkg_config_packages = set()
  56         self._typedefs_ns = {}
  57         self._includes = {} # <string namespace -> Namespace>
  58         self._include_names = set() # string namespace
  59         self._includepaths = []
  60         self._passthrough_mode = False
  61         self._annotations = {}
  62
  63     def get_includes(self):
  64         return self._include_names
  65
  66     def get_pkgconfig_packages(self):
  67         return self._pkg_config_packages
  68
  69     def disable_cache(self):
  70         self._cachestore = None
  71
  72     def set_passthrough_mode(self):
  73         self._passthrough_mode = True
  74
  75     def set_annotations(self, annotations):
  76         self._annotations = annotations
  77
  78     def _append_new_node(self, node):
  79         original = self._namespace.get(node.name)
  80         # Special case constants here; we allow duplication to sort-of
  81         # handle #ifdef.  But this introduces an arch-dependency in the .gir
  82         # file.  So far this has only come up scanning glib - in theory, other
  83         # modules will just depend on that.
  84         if isinstance(original, ast.Constant) and isinstance(node, ast.Constant):
  85             pass
  86         elif original:
  87             positions = set()
  88             positions.update(original.file_positions)
  89             positions.update(node.file_positions)
  90             message.fatal("Namespace conflict for '%s'" % (node.name, ),
  91                           positions)
  92         else:
  93             self._namespace.append(node)
  94
  95     def parse(self, symbols):
  96         for symbol in symbols:
  97             ## WORKAROUND ##
  98             # https://bugzilla.gnome.org/show_bug.cgi?id=550616
  99             if symbol.ident in ['gst_g_error_get_type']:
 100                 continue
 101             node = self._traverse_one(symbol)
 102             if node:
 103                 self._append_new_node(node)
 104
 105         # Now look through the namespace for things like
 106         # typedef struct _Foo Foo;
 107         # where we've never seen the struct _Foo.  Just create
 108         # an empty structure for these as "disguised"
 109         # If we do have a class/interface, merge fields
 110         for typedef, compound in self._typedefs_ns.iteritems():
 111             ns_compound = self._namespace.get(compound.name)
 112             if not ns_compound:
 113                 ns_compound = self._namespace.get('_' + compound.name)
 114             if (not ns_compound and isinstance(compound, (ast.Record, ast.Union))
 115                 and len(compound.fields) == 0):
 116                 disguised = ast.Record(compound.name, typedef, disguised=True)
 117                 self._namespace.append(disguised)
 118             elif not ns_compound:
 119                 self._namespace.append(compound)
 120             elif isinstance(ns_compound, (ast.Record, ast.Union)) and len(ns_compound.fields) == 0:
 121                 ns_compound.fields = compound.fields
 122         self._typedefs_ns = None
 123
 124     def set_include_paths(self, paths):
 125         self._includepaths = list(paths)
 126
 127     def register_include(self, include):
 128         if include in self._include_names:
 129             return
 130         filename = self._find_include(include)
 131         self._parse_include(filename)
 132         self._include_names.add(include)
 133
 134     def register_include_uninstalled(self, include_path):
 135         basename = os.path.basename(include_path)
 136         if not basename.endswith('.gir'):
 137             raise SystemExit(
 138 "Include path %r must be a filename path ending in .gir" % (include_path, ))
 139         girname = basename[:-4]
 140         include = ast.Include.from_string(girname)
 141         if include in self._include_names:
 142             return
 143         self._parse_include(include_path, uninstalled=True)
 144         self._include_names.add(include)
 145
 146     def lookup_giname(self, name):
 147         """Given a name of the form Foo or Bar.Foo,
 148 return the corresponding ast.Node, or None if none
 149 available.  Will throw KeyError however for unknown
 150 namespaces."""
 151         if '.' not in name:
 152             return self._namespace.get(name)
 153         else:
 154             (ns, giname) = name.split('.', 1)
 155             if ns == self._namespace.name:
 156                 return self._namespace.get(giname)
 157             # Fallback to the main namespace if not a dependency and matches a prefix
 158             if ns in self._namespace.identifier_prefixes and not ns in self._includes:
 159                 message.warn(("Deprecated reference to identifier " +
 160                               "prefix %s in GIName %s") % (ns, name))
 161                 return self._namespace.get(giname)
 162             include = self._includes[ns]
 163             return include.get(giname)
 164
 165     def lookup_typenode(self, typeobj):
 166         """Given a Type object, if it points to a giname,
 167 calls lookup_giname() on the name.  Otherwise return
 168 None."""
 169         if typeobj.target_giname:
 170             return self.lookup_giname(typeobj.target_giname)
 171         return None
 172
 173     # Private
 174
 175     def _find_include(self, include):
 176         searchdirs = self._includepaths[:]
 177         for path in _xdg_data_dirs:
 178             searchdirs.append(os.path.join(path, 'gir-1.0'))
 179         searchdirs.append(os.path.join(DATADIR, 'gir-1.0'))
 180
 181         girname = '%s-%s.gir' % (include.name, include.version)
 182         for d in searchdirs:
 183             path = os.path.join(d, girname)
 184             if os.path.exists(path):
 185                 return path
 186         sys.stderr.write("Couldn't find include %r (search path: %r)\n"\
 187                          % (girname, searchdirs))
 188         sys.exit(1)
 189
 190     @classmethod
 191     def parse_from_gir(cls, filename, extra_include_dirs=None):
 192         self = cls(None)
 193         if extra_include_dirs is not None:
 194             self.set_include_paths(extra_include_dirs)
 195         self.set_passthrough_mode()
 196         self._parse_include(filename)
 197         parser = self._cachestore.load(filename)
 198         self._namespace = parser.get_namespace()
 199         del self._includes[self._namespace.name]
 200         return self
 201
 202     def _parse_include(self, filename, uninstalled=False):
 203         parser = None
 204         if self._cachestore is not None:
 205             parser = self._cachestore.load(filename)
 206         if parser is None:
 207             parser = GIRParser(types_only=not self._passthrough_mode)
 208             parser.parse(filename)
 209             if self._cachestore is not None:
 210                 self._cachestore.store(filename, parser)
 211
 212         for include in parser.get_includes():
 213             self.register_include(include)
 214
 215         if not uninstalled:
 216             for pkg in parser.get_pkgconfig_packages():
 217                 self._pkg_config_packages.add(pkg)
 218         namespace = parser.get_namespace()
 219         self._includes[namespace.name] = namespace
 220
 221     def _iter_namespaces(self):
 222         """Return an iterator over all included namespaces; the
 223 currently-scanned namespace is first."""
 224         yield self._namespace
 225         for ns in self._includes.itervalues():
 226             yield ns
 227
 228     def _sort_matches(self, x, y):
 229         if x[0] is self._namespace:
 230             return 1
 231         elif y[0] is self._namespace:
 232             return -1
 233         return cmp(x[2], y[2])
 234
 235     def _split_c_string_for_namespace_matches(self, name, is_identifier=False):
 236         matches = []  # Namespaces which might contain this name
 237         unprefixed_namespaces = [] # Namespaces with no prefix, last resort
 238         for ns in self._iter_namespaces():
 239             if is_identifier:
 240                 prefixes = ns.identifier_prefixes
 241             elif name[0].isupper():
 242                 prefixes = ns._ucase_symbol_prefixes
 243             else:
 244                 prefixes = ns.symbol_prefixes
 245             if prefixes:
 246                 for prefix in prefixes:
 247                     if (not is_identifier) and (not prefix.endswith('_')):
 248                         prefix = prefix + '_'
 249                     if name.startswith(prefix):
 250                         matches.append((ns, name[len(prefix):], len(prefix)))
 251                         break
 252             else:
 253                 unprefixed_namespaces.append(ns)
 254         if matches:
 255             matches.sort(self._sort_matches)
 256             return map(lambda x: (x[0], x[1]), matches)
 257         elif self._accept_unprefixed:
 258             return [(self._namespace, name)]
 259         elif unprefixed_namespaces:
 260             # A bit of a hack; this function ideally shouldn't look through the
 261             # contents of namespaces; but since we aren't scanning anything
 262             # without a prefix, it's not too bad.
 263             for ns in unprefixed_namespaces:
 264                 if name in ns:
 265                     return [(ns, name)]
 266         raise ValueError("Unknown namespace for %s %r"
 267                          % ('identifier' if is_identifier else 'symbol', name, ))
 268
 269     def split_ctype_namespaces(self, ident):
 270         """Given a StudlyCaps string identifier like FooBar, return a
 271 list of (namespace, stripped_identifier) sorted by namespace length,
 272 or raise ValueError.  As a special case, if the current namespace matches,
 273 it is always biggest (i.e. last)."""
 274         return self._split_c_string_for_namespace_matches(ident, is_identifier=True)
 275
 276     def split_csymbol_namespaces(self, symbol):
 277         """Given a C symbol like foo_bar_do_baz, return a list of
 278 (namespace, stripped_symbol) sorted by namespace match probablity, or
 279 raise ValueError."""
 280         return self._split_c_string_for_namespace_matches(symbol, is_identifier=False)
 281
 282     def split_csymbol(self, symbol):
 283         """Given a C symbol like foo_bar_do_baz, return the most probable
 284 (namespace, stripped_symbol) match, or raise ValueError."""
 285         matches = self._split_c_string_for_namespace_matches(symbol, is_identifier=False)
 286         return matches[-1]
 287
 288     def strip_identifier(self, ident):
 289         hidden = ident.startswith('_')
 290         if hidden:
 291             ident = ident[1:]
 292         try:
 293             matches = self.split_ctype_namespaces(ident)
 294         except ValueError, e:
 295             raise TransformerException(str(e))
 296         for ns, name in matches:
 297             if ns is self._namespace:
 298                 if hidden:
 299                     return '_' + name
 300                 return name
 301         (ns, name) = matches[-1]
 302         raise TransformerException(
 303             "Skipping foreign identifier %r from namespace %s" % (
 304             ident, ns.name, ))
 305         return None
 306
 307     def _strip_symbol(self, symbol):
 308         ident = symbol.ident
 309         hidden = ident.startswith('_')
 310         if hidden:
 311             ident = ident[1:]
 312         try:
 313             (ns, name) = self.split_csymbol(ident)
 314         except ValueError, e:
 315             raise TransformerException(str(e))
 316         if ns != self._namespace:
 317             raise TransformerException(
 318                 "Skipping foreign symbol from namespace %s" % (ns.name, ))
 319         if hidden:
 320             return '_' + name
 321         return name
 322
 323     def _traverse_one(self, symbol, stype=None, parent_symbol=None):
 324         assert isinstance(symbol, SourceSymbol), symbol
 325
 326         if stype is None:
 327             stype = symbol.type
 328         if stype == CSYMBOL_TYPE_FUNCTION:
 329             return self._create_function(symbol)
 330         elif stype == CSYMBOL_TYPE_TYPEDEF:
 331             return self._create_typedef(symbol)
 332         elif stype == CSYMBOL_TYPE_STRUCT:
 333             return self._create_struct(symbol)
 334         elif stype == CSYMBOL_TYPE_ENUM:
 335             return self._create_enum(symbol)
 336         elif stype == CSYMBOL_TYPE_MEMBER:
 337             return self._create_member(symbol, parent_symbol)
 338         elif stype == CSYMBOL_TYPE_UNION:
 339             return self._create_union(symbol)
 340         elif stype == CSYMBOL_TYPE_CONST:
 341             # Don't parse constants which are marked (skip)
 342             docblock = self._annotations.get(symbol.ident)
 343             if not docblock or not 'skip' in docblock.options:
 344                 return self._create_const(symbol)
 345         # Ignore variable declarations in the header
 346         elif stype == CSYMBOL_TYPE_OBJECT:
 347             pass
 348         else:
 349             print 'transformer: unhandled symbol: %r' % (symbol, )
 350
 351     def _enum_common_prefix(self, symbol):
 352         def common_prefix(a, b):
 353             commonparts = []
 354             for aword, bword in zip(a.split('_'), b.split('_')):
 355                 if aword != bword:
 356                     return '_'.join(commonparts) + '_'
 357                 commonparts.append(aword)
 358             return min(a, b)
 359
 360         # Nothing less than 2 has a common prefix
 361         if len(list(symbol.base_type.child_list)) < 2:
 362             return None
 363         prefix = None
 364         for child in symbol.base_type.child_list:
 365             if prefix is None:
 366                 prefix = child.ident
 367             else:
 368                 prefix = common_prefix(prefix, child.ident)
 369                 if prefix == '':
 370                     return None
 371         return prefix
 372
 373     def _create_enum(self, symbol):
 374         prefix = self._enum_common_prefix(symbol)
 375         if prefix:
 376             prefixlen = len(prefix)
 377         else:
 378             prefixlen = 0
 379         members = []
 380         for child in symbol.base_type.child_list:
 381             if child.private:
 382                 continue
 383             if prefixlen > 0:
 384                 name = child.ident[prefixlen:]
 385             else:
 386                 # Ok, the enum members don't have a consistent prefix
 387                 # among them, so let's just remove the global namespace
 388                 # prefix.
 389                 try:
 390                     name = self._strip_symbol(child)
 391                 except TransformerException, e:
 392                     message.warn_symbol(symbol, e)
 393                     return None
 394             members.append(ast.Member(name.lower(),
 395                                       child.const_int,
 396                                       child.ident,
 397                                       None))
 398
 399         try:
 400             enum_name = self.strip_identifier(symbol.ident)
 401         except TransformerException, e:
 402             message.warn_symbol(symbol, e)
 403             return None
 404         if symbol.base_type.is_bitfield:
 405             klass = ast.Bitfield
 406         else:
 407             klass = ast.Enum
 408         node = klass(enum_name, symbol.ident, members=members)
 409         node.add_symbol_reference(symbol)
 410         return node
 411
 412     def _create_function(self, symbol):
 413         # Drop functions that start with _ very early on here
 414         if symbol.ident.startswith('_'):
 415             return None
 416         parameters = list(self._create_parameters(symbol.base_type))
 417         return_ = self._create_return(symbol.base_type.base_type)
 418         try:
 419             name = self._strip_symbol(symbol)
 420         except TransformerException, e:
 421             message.warn_symbol(symbol, e)
 422             return None
 423         func = ast.Function(name, return_, parameters, False, symbol.ident)
 424         func.add_symbol_reference(symbol)
 425         return func
 426
 427     def _create_source_type(self, source_type):
 428         assert source_type is not None
 429         if source_type.type == CTYPE_VOID:
 430             value = 'void'
 431         elif source_type.type == CTYPE_BASIC_TYPE:
 432             value = source_type.name
 433         elif source_type.type == CTYPE_TYPEDEF:
 434             value = source_type.name
 435         elif source_type.type == CTYPE_ARRAY:
 436             return self._create_source_type(source_type.base_type)
 437         elif source_type.type == CTYPE_POINTER:
 438             value = self._create_source_type(source_type.base_type) + '*'
 439         else:
 440             value = 'gpointer'
 441         return value
 442
 443     def _create_complete_source_type(self, source_type):
 444         assert source_type is not None
 445
 446         const = (source_type.type_qualifier & TYPE_QUALIFIER_CONST)
 447         volatile = (source_type.type_qualifier & TYPE_QUALIFIER_VOLATILE)
 448
 449         if source_type.type == CTYPE_VOID:
 450             return 'void'
 451         elif source_type.type in [CTYPE_BASIC_TYPE,
 452                                   CTYPE_TYPEDEF,
 453                                   CTYPE_STRUCT,
 454                                   CTYPE_UNION,
 455                                   CTYPE_ENUM]:
 456             value = source_type.name
 457             if not value:
 458                 value = 'gpointer'
 459             if const:
 460                 value = 'const ' + value
 461             if volatile:
 462                 value = 'volatile ' + value
 463         elif source_type.type == CTYPE_ARRAY:
 464             return self._create_complete_source_type(source_type.base_type)
 465         elif source_type.type == CTYPE_POINTER:
 466             value = self._create_complete_source_type(source_type.base_type) + '*'
 467             # TODO: handle pointer to function as a special case?
 468             if const:
 469                 value += ' const'
 470             if volatile:
 471                 value += ' volatile'
 472
 473         else:
 474             if const:
 475                 value = 'gconstpointer'
 476             else:
 477                 value = 'gpointer'
 478             if volatile:
 479                 value = 'volatile ' + value
 480             return value
 481
 482         return value
 483
 484     def _create_parameters(self, base_type):
 485         # warn if we see annotations for unknown parameters
 486         param_names = set(child.ident for child in base_type.child_list)
 487         for child in base_type.child_list:
 488             yield self._create_parameter(child)
 489
 490     def _synthesize_union_type(self, symbol, parent_symbol):
 491         # Synthesize a named union so that it can be referenced.
 492         parent_ident = parent_symbol.ident
 493         # FIXME: Should split_ctype_namespaces handle the hidden case?
 494         hidden = parent_ident.startswith('_')
 495         if hidden:
 496             parent_ident = parent_ident[1:]
 497         matches = self.split_ctype_namespaces(parent_ident)
 498         (namespace, parent_name) = matches[-1]
 499         assert namespace and parent_name
 500         if hidden:
 501             parent_name = '_' + parent_name
 502         fake_union = ast.Union("%s__%s__union" % (parent_name, symbol.ident))
 503         # _parse_fields accesses <type>.base_type.child_list, so we have to
 504         # pass symbol.base_type even though that refers to the array, not the
 505         # union.
 506         self._parse_fields(symbol.base_type, fake_union)
 507         self._append_new_node(fake_union)
 508         fake_type = ast.Type(
 509             target_giname="%s.%s" % (namespace.name, fake_union.name))
 510         return fake_type
 511
 512     def _create_member(self, symbol, parent_symbol=None):
 513         source_type = symbol.base_type
 514         if (source_type.type == CTYPE_POINTER and
 515             symbol.base_type.base_type.type == CTYPE_FUNCTION):
 516             node = self._create_callback(symbol, member=True)
 517         elif source_type.type == CTYPE_STRUCT and source_type.name is None:
 518             node = self._create_struct(symbol, anonymous=True)
 519         elif source_type.type == CTYPE_UNION and source_type.name is None:
 520             node = self._create_union(symbol, anonymous=True)
 521         else:
 522             # Special handling for fields; we don't have annotations on them
 523             # to apply later, yet.
 524             if source_type.type == CTYPE_ARRAY:
 525                 complete_ctype = self._create_complete_source_type(source_type)
 526                 # If the array contains anonymous unions, like in the GValue
 527                 # struct, we need to handle this specially.  This is necessary
 528                 # to be able to properly calculate the size of the compound
 529                 # type (e.g. GValue) that contains this array, see
 530                 # <https://bugzilla.gnome.org/show_bug.cgi?id=657040>.
 531                 if (source_type.base_type.type == CTYPE_UNION and
 532                     source_type.base_type.name is None):
 533                     synthesized_type = self._synthesize_union_type(symbol, parent_symbol)
 534                     ftype = ast.Array(None, synthesized_type, complete_ctype=complete_ctype)
 535                 else:
 536                     ctype = self._create_source_type(source_type)
 537                     canonical_ctype = self._canonicalize_ctype(ctype)
 538                     if canonical_ctype[-1] == '*':
 539                         derefed_name = canonical_ctype[:-1]
 540                     else:
 541                         derefed_name = canonical_ctype
 542                     if complete_ctype[-1] == '*':
 543                         derefed_complete_ctype = complete_ctype[:-1]
 544                     else:
 545                         derefed_complete_ctype = complete_ctype
 546                     from_ctype = self.create_type_from_ctype_string(ctype,
 547                                                                     complete_ctype=complete_ctype)
 548                     ftype = ast.Array(None, from_ctype,
 549                                       ctype=derefed_name,
 550                                       complete_ctype=derefed_complete_ctype)
 551                 child_list = list(symbol.base_type.child_list)
 552                 ftype.zeroterminated = False
 553                 if child_list:
 554                     ftype.size = child_list[0].const_int
 555             else:
 556                 ftype = self._create_type_from_base(symbol.base_type)
 557             # ast.Fields are assumed to be read-write
 558             # (except for Objects, see also glibtransformer.py)
 559             node = ast.Field(symbol.ident, ftype,
 560                              readable=True, writable=True,
 561                              bits=symbol.const_int)
 562             if symbol.private:
 563                 node.readable = False
 564                 node.writable = False
 565                 node.private = True
 566         return node
 567
 568     def _create_typedef(self, symbol):
 569         ctype = symbol.base_type.type
 570         if (ctype == CTYPE_POINTER and
 571             symbol.base_type.base_type.type == CTYPE_FUNCTION):
 572             node = self._create_typedef_callback(symbol)
 573         elif (ctype == CTYPE_POINTER and
 574             symbol.base_type.base_type.type == CTYPE_STRUCT):
 575             node = self._create_typedef_struct(symbol, disguised=True)
 576         elif ctype == CTYPE_STRUCT:
 577             node = self._create_typedef_struct(symbol)
 578         elif ctype == CTYPE_UNION:
 579             node = self._create_typedef_union(symbol)
 580         elif ctype == CTYPE_ENUM:
 581             return self._create_enum(symbol)
 582         elif ctype in (CTYPE_TYPEDEF,
 583                        CTYPE_POINTER,
 584                        CTYPE_BASIC_TYPE,
 585                        CTYPE_VOID):
 586             try:
 587                 name = self.strip_identifier(symbol.ident)
 588             except TransformerException, e:
 589                 message.warn(e)
 590                 return None
 591             if symbol.base_type.name:
 592                 complete_ctype = self._create_complete_source_type(symbol.base_type)
 593                 target = self.create_type_from_ctype_string(symbol.base_type.name,
 594                                                             complete_ctype=complete_ctype)
 595             else:
 596                 target = ast.TYPE_ANY
 597             if name in ast.type_names:
 598                 return None
 599             return ast.Alias(name, target, ctype=symbol.ident)
 600         else:
 601             raise NotImplementedError(
 602                 "symbol %r of type %s" % (symbol.ident, ctype_name(ctype)))
 603         return node
 604
 605     def _canonicalize_ctype(self, ctype):
 606         # First look up the ctype including any pointers;
 607         # a few type names like 'char*' have their own aliases
 608         # and we need pointer information for those.
 609         firstpass = ast.type_names.get(ctype)
 610
 611         # If we have a particular alias for this, skip deep
 612         # canonicalization to prevent changing
 613         # e.g. char* -> int8*
 614         if firstpass:
 615             return firstpass.target_fundamental
 616
 617         if not ctype.endswith('*'):
 618             return ctype
 619
 620         # We have a pointer type.
 621         # Strip the end pointer, canonicalize our base type
 622         base = ctype[:-1]
 623         canonical_base = self._canonicalize_ctype(base)
 624
 625         # Append the pointer again
 626         canonical = canonical_base + '*'
 627
 628         return canonical
 629
 630     def parse_ctype(self, ctype, is_member=False):
 631         canonical = self._canonicalize_ctype(ctype)
 632
 633         # Remove all pointers - we require standard calling
 634         # conventions.  For example, an 'int' is always passed by
 635         # value (unless it's out or inout).
 636         derefed_typename = canonical.replace('*', '')
 637
 638         # Preserve "pointerness" of struct/union members
 639         if (is_member and canonical.endswith('*') and
 640             derefed_typename in ast.basic_type_names):
 641             return 'gpointer'
 642         else:
 643             return derefed_typename
 644
 645     def _create_type_from_base(self, source_type, is_parameter=False, is_return=False):
 646         ctype = self._create_source_type(source_type)
 647         complete_ctype = self._create_complete_source_type(source_type)
 648         const = ((source_type.type == CTYPE_POINTER) and
 649                  (source_type.base_type.type_qualifier & TYPE_QUALIFIER_CONST))
 650         return self.create_type_from_ctype_string(ctype, is_const=const,
 651                                                   is_parameter=is_parameter, is_return=is_return,
 652                                                   complete_ctype=complete_ctype)
 653
 654     def _create_bare_container_type(self, base, ctype=None,
 655                                     is_const=False, complete_ctype=None):
 656         if base in ('GList', 'GSList', 'GLib.List', 'GLib.SList'):
 657             if base in ('GList', 'GSList'):
 658                 name = 'GLib.' + base[1:]
 659             else:
 660                 name = base
 661             return ast.List(name, ast.TYPE_ANY, ctype=ctype,
 662                         is_const=is_const, complete_ctype=complete_ctype)
 663         elif base in ('GArray', 'GPtrArray', 'GByteArray',
 664                       'GLib.Array', 'GLib.PtrArray', 'GLib.ByteArray',
 665                       'GObject.Array', 'GObject.PtrArray', 'GObject.ByteArray'):
 666             if '.' in base:
 667                 name = 'GLib.' + base.split('.', 1)[1]
 668             else:
 669                 name = 'GLib.' + base[1:]
 670             return ast.Array(name, ast.TYPE_ANY, ctype=ctype,
 671                          is_const=is_const, complete_ctype=complete_ctype)
 672         elif base in ('GHashTable', 'GLib.HashTable', 'GObject.HashTable'):
 673             return ast.Map(ast.TYPE_ANY, ast.TYPE_ANY, ctype=ctype, is_const=is_const,
 674                            complete_ctype=complete_ctype)
 675         return None
 676
 677     def create_type_from_ctype_string(self, ctype, is_const=False,
 678                                       is_parameter=False, is_return=False,
 679                                       complete_ctype=None):
 680         canonical = self._canonicalize_ctype(ctype)
 681         base = canonical.replace('*', '')
 682
 683         # Special default: char ** -> ast.Array, same for GStrv
 684         if (is_return and canonical == 'utf8*') or base == 'GStrv':
 685             bare_utf8 = ast.TYPE_STRING.clone()
 686             bare_utf8.ctype = None
 687             return ast.Array(None, bare_utf8, ctype=ctype,
 688                              is_const=is_const, complete_ctype=complete_ctype)
 689
 690         fundamental = ast.type_names.get(base)
 691         if fundamental is not None:
 692             return ast.Type(target_fundamental=fundamental.target_fundamental,
 693                         ctype=ctype,
 694                         is_const=is_const, complete_ctype=complete_ctype)
 695         container = self._create_bare_container_type(base, ctype=ctype, is_const=is_const,
 696                                                      complete_ctype=complete_ctype)
 697         if container:
 698             return container
 699         return ast.Type(ctype=ctype, is_const=is_const, complete_ctype=complete_ctype)
 700
 701     def _create_parameter(self, symbol):
 702         if symbol.type == CSYMBOL_TYPE_ELLIPSIS:
 703             ptype = ast.Varargs()
 704         else:
 705             ptype = self._create_type_from_base(symbol.base_type, is_parameter=True)
 706         return ast.Parameter(symbol.ident, ptype)
 707
 708     def _create_return(self, source_type):
 709         typeval = self._create_type_from_base(source_type, is_return=True)
 710         return ast.Return(typeval)
 711
 712     def _create_const(self, symbol):
 713         if symbol.ident.startswith('_'):
 714             return None
 715
 716         # Don't create constants for non-public things
 717         # http://bugzilla.gnome.org/show_bug.cgi?id=572790
 718         if (symbol.source_filename is None or
 719             not symbol.source_filename.endswith('.h')):
 720             return None
 721         try:
 722             name = self._strip_symbol(symbol)
 723         except TransformerException, e:
 724             message.warn_symbol(symbol, e)
 725             return None
 726         if symbol.const_string is not None:
 727             typeval = ast.TYPE_STRING
 728             value = unicode(symbol.const_string, 'utf-8')
 729         elif symbol.const_int is not None:
 730             if symbol.base_type is not None:
 731                 typeval = self._create_type_from_base(symbol.base_type)
 732             else:
 733                 typeval = ast.TYPE_INT
 734             unaliased = typeval
 735             self._resolve_type_from_ctype(unaliased)
 736             if typeval.target_giname and typeval.ctype:
 737                 target = self.lookup_giname(typeval.target_giname)
 738                 target = self.resolve_aliases(target)
 739                 if isinstance(target, ast.Type):
 740                     unaliased = target
 741             if unaliased == ast.TYPE_UINT64:
 742                 value = str(symbol.const_int % 2**64)
 743             elif unaliased == ast.TYPE_UINT32:
 744                 value = str(symbol.const_int % 2**32)
 745             elif unaliased == ast.TYPE_UINT16:
 746                 value = str(symbol.const_int % 2**16)
 747             elif unaliased == ast.TYPE_UINT8:
 748                 value = str(symbol.const_int % 2**16)
 749             else:
 750                 value = str(symbol.const_int)
 751         elif symbol.const_double is not None:
 752             typeval = ast.TYPE_DOUBLE
 753             value = '%f' % (symbol.const_double, )
 754         else:
 755             raise AssertionError()
 756
 757         const = ast.Constant(name, typeval, value,
 758                              symbol.ident)
 759         const.add_symbol_reference(symbol)
 760         return const
 761
 762     def _create_typedef_struct(self, symbol, disguised=False):
 763         try:
 764             name = self.strip_identifier(symbol.ident)
 765         except TransformerException, e:
 766             message.warn_symbol(symbol, e)
 767             return None
 768         struct = ast.Record(name, symbol.ident, disguised=disguised)
 769         self._parse_fields(symbol, struct)
 770         struct.add_symbol_reference(symbol)
 771         self._typedefs_ns[symbol.ident] = struct
 772         return None
 773
 774     def _create_typedef_union(self, symbol):
 775         try:
 776             name = self.strip_identifier(symbol.ident)
 777         except TransformerException, e:
 778             message.warn(e)
 779             return None
 780         union = ast.Union(name, symbol.ident)
 781         self._parse_fields(symbol, union)
 782         union.add_symbol_reference(symbol)
 783         self._typedefs_ns[symbol.ident] = union
 784         return None
 785
 786     def _create_typedef_callback(self, symbol):
 787         callback = self._create_callback(symbol)
 788         if not callback:
 789             return None
 790         self._typedefs_ns[callback.name] = callback
 791         return callback
 792
 793     def _parse_fields(self, symbol, compound):
 794         for child in symbol.base_type.child_list:
 795             child_node = self._traverse_one(child, parent_symbol=symbol)
 796             if not child_node:
 797                 continue
 798             if isinstance(child_node, ast.Field):
 799                 field = child_node
 800             else:
 801                 field = ast.Field(child.ident, None, True, False,
 802                               anonymous_node=child_node)
 803             compound.fields.append(field)
 804
 805     def _create_compound(self, klass, symbol, anonymous):
 806         if symbol.ident is None:
 807             # the compound is an anonymous member of another union or a struct
 808             assert anonymous
 809             compound = klass(None, None)
 810         else:
 811             compound = self._typedefs_ns.get(symbol.ident, None)
 812
 813         if compound is None:
 814             # This is a bit of a hack; really we should try
 815             # to resolve through the typedefs to find the real
 816             # name
 817             if symbol.ident.startswith('_'):
 818                 compound = self._typedefs_ns.get(symbol.ident[1:], None)
 819             if compound is None:
 820                 if anonymous:
 821                     name = symbol.ident
 822                 else:
 823                     try:
 824                         name = self.strip_identifier(symbol.ident)
 825                     except TransformerException, e:
 826                         message.warn(e)
 827                         return None
 828                 compound = klass(name, symbol.ident)
 829
 830         self._parse_fields(symbol, compound)
 831         compound.add_symbol_reference(symbol)
 832         return compound
 833
 834     def _create_struct(self, symbol, anonymous=False):
 835         return self._create_compound(ast.Record, symbol, anonymous)
 836
 837     def _create_union(self, symbol, anonymous=False):
 838         return self._create_compound(ast.Union, symbol, anonymous)
 839
 840     def _create_callback(self, symbol, member=False):
 841         parameters = list(self._create_parameters(symbol.base_type.base_type))
 842         retval = self._create_return(symbol.base_type.base_type.base_type)
 843
 844         # Mark the 'user_data' arguments
 845         for i, param in enumerate(parameters):
 846             if (param.type.target_fundamental == 'gpointer' and
 847                 param.argname == 'user_data'):
 848                 param.closure_name = param.argname
 849
 850         if member:
 851             name = symbol.ident
 852         elif symbol.ident.find('_') > 0:
 853             try:
 854                 name = self._strip_symbol(symbol)
 855             except TransformerException, e:
 856                 message.warn_symbol(symbol, e)
 857                 return None
 858         else:
 859             try:
 860                 name = self.strip_identifier(symbol.ident)
 861             except TransformerException, e:
 862                 message.warn(e)
 863                 return None
 864         callback = ast.Callback(name, retval, parameters, False,
 865                                 ctype=symbol.ident)
 866         callback.add_symbol_reference(symbol)
 867
 868         return callback
 869
 870     def create_type_from_user_string(self, typestr):
 871         """Parse a C type string (as might be given from an
 872         annotation) and resolve it.  For compatibility, we can consume
 873 both GI type string (utf8, Foo.Bar) style, as well as C (char *, FooBar) style.
 874
 875 Note that type resolution may not succeed."""
 876         if '.' in typestr:
 877             container = self._create_bare_container_type(typestr)
 878             if container:
 879                 return container
 880             return self._namespace.type_from_name(typestr)
 881         typeval = self.create_type_from_ctype_string(typestr)
 882         self.resolve_type(typeval)
 883         if typeval.resolved:
 884             # Explicitly clear out the c_type; there isn't one in this case.
 885             typeval.ctype = None
 886         return typeval
 887
 888     def _resolve_type_from_ctype_all_namespaces(self, typeval, pointer_stripped):
 889         # If we can't determine the namespace from the type name,
 890         # fall back to trying all of our includes.  An example of this is mutter,
 891         # which has nominal namespace of "Meta", but a few classes are
 892         # "Mutter".  We don't export that data in introspection currently.
 893         # Basically the library should be fixed, but we'll hack around it here.
 894         for namespace in self._includes.itervalues():
 895             target = namespace.get_by_ctype(pointer_stripped)
 896             if target:
 897                 typeval.target_giname = '%s.%s' % (namespace.name, target.name)
 898                 return True
 899         return False
 900
 901     def _resolve_type_from_ctype(self, typeval):
 902         assert typeval.ctype is not None
 903         pointer_stripped = typeval.ctype.replace('*', '')
 904         try:
 905             matches = self.split_ctype_namespaces(pointer_stripped)
 906         except ValueError:
 907             return self._resolve_type_from_ctype_all_namespaces(typeval, pointer_stripped)
 908         for namespace, name in matches:
 909             target = namespace.get(name)
 910             if not target:
 911                 target = namespace.get_by_ctype(pointer_stripped)
 912             if target:
 913                 typeval.target_giname = '%s.%s' % (namespace.name, target.name)
 914                 return True
 915         return False
 916
 917     def _resolve_type_from_gtype_name(self, typeval):
 918         assert typeval.gtype_name is not None
 919         for ns in self._iter_namespaces():
 920             node = ns.type_names.get(typeval.gtype_name, None)
 921             if node is not None:
 922                 typeval.target_giname = '%s.%s' % (ns.name, node.name)
 923                 return True
 924         return False
 925
 926     def resolve_type(self, typeval):
 927         if isinstance(typeval, (ast.Array, ast.List)):
 928             return self.resolve_type(typeval.element_type)
 929         elif isinstance(typeval, ast.Map):
 930             key_resolved = self.resolve_type(typeval.key_type)
 931             value_resolved = self.resolve_type(typeval.value_type)
 932             return key_resolved and value_resolved
 933         elif typeval.resolved:
 934             return True
 935         elif typeval.ctype:
 936             return self._resolve_type_from_ctype(typeval)
 937         elif typeval.gtype_name:
 938             return self._resolve_type_from_gtype_name(typeval)
 939
 940     def _typepair_to_str(self, item):
 941         nsname, item = item
 942         if nsname is None:
 943             return item.name
 944         return '%s.%s' % (nsname, item.name)
 945
 946     def gtypename_to_giname(self, gtname, names):
 947         resolved = names.type_names.get(gtname)
 948         if resolved:
 949             return self._typepair_to_str(resolved)
 950         resolved = self._names.type_names.get(gtname)
 951         if resolved:
 952             return self._typepair_to_str(resolved)
 953         raise KeyError("Failed to resolve GType name: %r" % (gtname, ))
 954
 955     def ctype_of(self, obj):
 956         if hasattr(obj, 'ctype'):
 957             return obj.ctype
 958         elif hasattr(obj, 'symbol'):
 959             return obj.symbol
 960         else:
 961             return None
 962
 963     def resolve_aliases(self, typenode):
 964         """Removes all aliases from typenode, returns first non-alias
 965         in the typenode alias chain.  Returns typenode argument if it
 966         is not an alias."""
 967         while isinstance(typenode, ast.Alias):
 968             if typenode.target.target_giname is not None:
 969                 typenode = self.lookup_giname(typenode.target.target_giname)
 970             elif typenode.target.target_fundamental is not None:
 971                 typenode = ast.type_names[typenode.target.target_fundamental]
 972             else:
 973                 break
 974         return typenode