giscanner/transformer.py

   1 # -*- Mode: Python -*-
   2 # GObject-Introspection - a framework for introspecting GObject libraries
   3 # Copyright (C) 2008  Johan Dahlin
   4 #
   5 # This library is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU Lesser General Public
   7 # License as published by the Free Software Foundation; either
   8 # version 2 of the License, or (at your option) any later version.
   9 #
  10 # This library is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 # Lesser General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU Lesser General Public
  16 # License along with this library; if not, write to the
  17 # Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18 # Boston, MA 02111-1307, USA.
  19 #
  20
  21 import os
  22 import sys
  23
  24 from . import ast
  25 from . import message
  26 from .cachestore import CacheStore
  27 from .girparser import GIRParser
  28 from .sourcescanner import (
  29     SourceSymbol, ctype_name, CTYPE_POINTER,
  30     CTYPE_BASIC_TYPE, CTYPE_UNION, CTYPE_ARRAY, CTYPE_TYPEDEF,
  31     CTYPE_VOID, CTYPE_ENUM, CTYPE_FUNCTION, CTYPE_STRUCT,
  32     CSYMBOL_TYPE_FUNCTION, CSYMBOL_TYPE_TYPEDEF, CSYMBOL_TYPE_STRUCT,
  33     CSYMBOL_TYPE_ENUM, CSYMBOL_TYPE_UNION, CSYMBOL_TYPE_OBJECT,
  34     CSYMBOL_TYPE_MEMBER, CSYMBOL_TYPE_ELLIPSIS, CSYMBOL_TYPE_CONST,
  35     TYPE_QUALIFIER_CONST, TYPE_QUALIFIER_VOLATILE)
  36
  37 class TransformerException(Exception):
  38     pass
  39
  40
  41 _xdg_data_dirs = [x for x in os.environ.get('XDG_DATA_DIRS', '').split(os.pathsep)]
  42 _xdg_data_dirs.append(DATADIR)
  43
  44 if os.name != 'nt':
  45     _xdg_data_dirs.append('/usr/share')
  46
  47
  48 class Transformer(object):
  49     namespace = property(lambda self: self._namespace)
  50
  51     def __init__(self, namespace, accept_unprefixed=False):
  52         self._cachestore = CacheStore()
  53         self._accept_unprefixed = accept_unprefixed
  54         self._namespace = namespace
  55         self._pkg_config_packages = set()
  56         self._typedefs_ns = {}
  57         self._parsed_includes = {} # <string namespace -> Namespace>
  58         self._includepaths = []
  59         self._passthrough_mode = False
  60         self._annotations = {}
  61
  62     def get_pkgconfig_packages(self):
  63         return self._pkg_config_packages
  64
  65     def disable_cache(self):
  66         self._cachestore = None
  67
  68     def set_passthrough_mode(self):
  69         self._passthrough_mode = True
  70
  71     def set_annotations(self, annotations):
  72         self._annotations = annotations
  73
  74     def _append_new_node(self, node):
  75         original = self._namespace.get(node.name)
  76         # Special case constants here; we allow duplication to sort-of
  77         # handle #ifdef.  But this introduces an arch-dependency in the .gir
  78         # file.  So far this has only come up scanning glib - in theory, other
  79         # modules will just depend on that.
  80         if isinstance(original, ast.Constant) and isinstance(node, ast.Constant):
  81             pass
  82         elif original:
  83             positions = set()
  84             positions.update(original.file_positions)
  85             positions.update(node.file_positions)
  86             message.fatal("Namespace conflict for '%s'" % (node.name, ),
  87                           positions)
  88         else:
  89             self._namespace.append(node)
  90
  91     def parse(self, symbols):
  92         for symbol in symbols:
  93             ## WORKAROUND ##
  94             # https://bugzilla.gnome.org/show_bug.cgi?id=550616
  95             if symbol.ident in ['gst_g_error_get_type']:
  96                 continue
  97             node = self._traverse_one(symbol)
  98             if node:
  99                 self._append_new_node(node)
 100
 101         # Now look through the namespace for things like
 102         # typedef struct _Foo Foo;
 103         # where we've never seen the struct _Foo.  Just create
 104         # an empty structure for these as "disguised"
 105         # If we do have a class/interface, merge fields
 106         for typedef, compound in self._typedefs_ns.iteritems():
 107             ns_compound = self._namespace.get(compound.name)
 108             if not ns_compound:
 109                 ns_compound = self._namespace.get('_' + compound.name)
 110             if (not ns_compound and isinstance(compound, (ast.Record, ast.Union))
 111                 and len(compound.fields) == 0):
 112                 disguised = ast.Record(compound.name, typedef, disguised=True)
 113                 self._namespace.append(disguised)
 114             elif not ns_compound:
 115                 self._namespace.append(compound)
 116             elif isinstance(ns_compound, (ast.Record, ast.Union)) and len(ns_compound.fields) == 0:
 117                 ns_compound.fields = compound.fields
 118         self._typedefs_ns = None
 119
 120     def set_include_paths(self, paths):
 121         self._includepaths = list(paths)
 122
 123     def register_include(self, include):
 124         if include in self._namespace.includes:
 125             return
 126         self._namespace.includes.add(include)
 127         filename = self._find_include(include)
 128         self._parse_include(filename)
 129
 130     def register_include_uninstalled(self, include_path):
 131         basename = os.path.basename(include_path)
 132         if not basename.endswith('.gir'):
 133             raise SystemExit(
 134 "Include path %r must be a filename path ending in .gir" % (include_path, ))
 135         girname = basename[:-4]
 136         include = ast.Include.from_string(girname)
 137         if include in self._namespace.includes:
 138             return
 139         self._namespace.includes.add(include)
 140         self._parse_include(include_path, uninstalled=True)
 141
 142     def lookup_giname(self, name):
 143         """Given a name of the form Foo or Bar.Foo,
 144 return the corresponding ast.Node, or None if none
 145 available.  Will throw KeyError however for unknown
 146 namespaces."""
 147         if '.' not in name:
 148             return self._namespace.get(name)
 149         else:
 150             (ns, giname) = name.split('.', 1)
 151             if ns == self._namespace.name:
 152                 return self._namespace.get(giname)
 153             # Fallback to the main namespace if not a dependency and matches a prefix
 154             if ns in self._namespace.identifier_prefixes and not ns in self._parsed_includes:
 155                 message.warn(("Deprecated reference to identifier " +
 156                               "prefix %s in GIName %s") % (ns, name))
 157                 return self._namespace.get(giname)
 158             include = self._parsed_includes[ns]
 159             return include.get(giname)
 160
 161     def lookup_typenode(self, typeobj):
 162         """Given a Type object, if it points to a giname,
 163 calls lookup_giname() on the name.  Otherwise return
 164 None."""
 165         if typeobj.target_giname:
 166             return self.lookup_giname(typeobj.target_giname)
 167         return None
 168
 169     # Private
 170
 171     def _find_include(self, include):
 172         searchdirs = self._includepaths[:]
 173         for path in _xdg_data_dirs:
 174             searchdirs.append(os.path.join(path, 'gir-1.0'))
 175         searchdirs.append(os.path.join(DATADIR, 'gir-1.0'))
 176
 177         girname = '%s-%s.gir' % (include.name, include.version)
 178         for d in searchdirs:
 179             path = os.path.join(d, girname)
 180             if os.path.exists(path):
 181                 return path
 182         sys.stderr.write("Couldn't find include %r (search path: %r)\n"\
 183                          % (girname, searchdirs))
 184         sys.exit(1)
 185
 186     @classmethod
 187     def parse_from_gir(cls, filename, extra_include_dirs=None):
 188         self = cls(None)
 189         if extra_include_dirs is not None:
 190             self.set_include_paths(extra_include_dirs)
 191         self.set_passthrough_mode()
 192         self._parse_include(filename)
 193         parser = self._cachestore.load(filename)
 194         self._namespace = parser.get_namespace()
 195         del self._parsed_includes[self._namespace.name]
 196         return self
 197
 198     def _parse_include(self, filename, uninstalled=False):
 199         parser = None
 200         if self._cachestore is not None:
 201             parser = self._cachestore.load(filename)
 202         if parser is None:
 203             parser = GIRParser(types_only=not self._passthrough_mode)
 204             parser.parse(filename)
 205             if self._cachestore is not None:
 206                 self._cachestore.store(filename, parser)
 207
 208         for include in parser.get_namespace().includes:
 209             if include.name not in self._parsed_includes:
 210                 dep_filename = self._find_include(include)
 211                 self._parse_include(dep_filename)
 212
 213         if not uninstalled:
 214             for pkg in parser.get_namespace().exported_packages:
 215                 self._pkg_config_packages.add(pkg)
 216         namespace = parser.get_namespace()
 217         self._parsed_includes[namespace.name] = namespace
 218
 219     def _iter_namespaces(self):
 220         """Return an iterator over all included namespaces; the
 221 currently-scanned namespace is first."""
 222         yield self._namespace
 223         for ns in self._parsed_includes.itervalues():
 224             yield ns
 225
 226     def _sort_matches(self, x, y):
 227         if x[0] is self._namespace:
 228             return 1
 229         elif y[0] is self._namespace:
 230             return -1
 231         return cmp(x[2], y[2])
 232
 233     def _split_c_string_for_namespace_matches(self, name, is_identifier=False):
 234         matches = []  # Namespaces which might contain this name
 235         unprefixed_namespaces = [] # Namespaces with no prefix, last resort
 236         for ns in self._iter_namespaces():
 237             if is_identifier:
 238                 prefixes = ns.identifier_prefixes
 239             elif name[0].isupper():
 240                 prefixes = ns._ucase_symbol_prefixes
 241             else:
 242                 prefixes = ns.symbol_prefixes
 243             if prefixes:
 244                 for prefix in prefixes:
 245                     if (not is_identifier) and (not prefix.endswith('_')):
 246                         prefix = prefix + '_'
 247                     if name.startswith(prefix):
 248                         matches.append((ns, name[len(prefix):], len(prefix)))
 249                         break
 250             else:
 251                 unprefixed_namespaces.append(ns)
 252         if matches:
 253             matches.sort(self._sort_matches)
 254             return map(lambda x: (x[0], x[1]), matches)
 255         elif self._accept_unprefixed:
 256             return [(self._namespace, name)]
 257         elif unprefixed_namespaces:
 258             # A bit of a hack; this function ideally shouldn't look through the
 259             # contents of namespaces; but since we aren't scanning anything
 260             # without a prefix, it's not too bad.
 261             for ns in unprefixed_namespaces:
 262                 if name in ns:
 263                     return [(ns, name)]
 264         raise ValueError("Unknown namespace for %s %r"
 265                          % ('identifier' if is_identifier else 'symbol', name, ))
 266
 267     def split_ctype_namespaces(self, ident):
 268         """Given a StudlyCaps string identifier like FooBar, return a
 269 list of (namespace, stripped_identifier) sorted by namespace length,
 270 or raise ValueError.  As a special case, if the current namespace matches,
 271 it is always biggest (i.e. last)."""
 272         return self._split_c_string_for_namespace_matches(ident, is_identifier=True)
 273
 274     def split_csymbol_namespaces(self, symbol):
 275         """Given a C symbol like foo_bar_do_baz, return a list of
 276 (namespace, stripped_symbol) sorted by namespace match probablity, or
 277 raise ValueError."""
 278         return self._split_c_string_for_namespace_matches(symbol, is_identifier=False)
 279
 280     def split_csymbol(self, symbol):
 281         """Given a C symbol like foo_bar_do_baz, return the most probable
 282 (namespace, stripped_symbol) match, or raise ValueError."""
 283         matches = self._split_c_string_for_namespace_matches(symbol, is_identifier=False)
 284         return matches[-1]
 285
 286     def strip_identifier(self, ident):
 287         hidden = ident.startswith('_')
 288         if hidden:
 289             ident = ident[1:]
 290         try:
 291             matches = self.split_ctype_namespaces(ident)
 292         except ValueError, e:
 293             raise TransformerException(str(e))
 294         for ns, name in matches:
 295             if ns is self._namespace:
 296                 if hidden:
 297                     return '_' + name
 298                 return name
 299         (ns, name) = matches[-1]
 300         raise TransformerException(
 301             "Skipping foreign identifier %r from namespace %s" % (
 302             ident, ns.name, ))
 303         return None
 304
 305     def _strip_symbol(self, symbol):
 306         ident = symbol.ident
 307         hidden = ident.startswith('_')
 308         if hidden:
 309             ident = ident[1:]
 310         try:
 311             (ns, name) = self.split_csymbol(ident)
 312         except ValueError, e:
 313             raise TransformerException(str(e))
 314         if ns != self._namespace:
 315             raise TransformerException(
 316                 "Skipping foreign symbol from namespace %s" % (ns.name, ))
 317         if hidden:
 318             return '_' + name
 319         return name
 320
 321     def _traverse_one(self, symbol, stype=None, parent_symbol=None):
 322         assert isinstance(symbol, SourceSymbol), symbol
 323
 324         if stype is None:
 325             stype = symbol.type
 326         if stype == CSYMBOL_TYPE_FUNCTION:
 327             return self._create_function(symbol)
 328         elif stype == CSYMBOL_TYPE_TYPEDEF:
 329             return self._create_typedef(symbol)
 330         elif stype == CSYMBOL_TYPE_STRUCT:
 331             return self._create_struct(symbol)
 332         elif stype == CSYMBOL_TYPE_ENUM:
 333             return self._create_enum(symbol)
 334         elif stype == CSYMBOL_TYPE_MEMBER:
 335             return self._create_member(symbol, parent_symbol)
 336         elif stype == CSYMBOL_TYPE_UNION:
 337             return self._create_union(symbol)
 338         elif stype == CSYMBOL_TYPE_CONST:
 339             return self._create_const(symbol)
 340         # Ignore variable declarations in the header
 341         elif stype == CSYMBOL_TYPE_OBJECT:
 342             pass
 343         else:
 344             print 'transformer: unhandled symbol: %r' % (symbol, )
 345
 346     def _enum_common_prefix(self, symbol):
 347         def common_prefix(a, b):
 348             commonparts = []
 349             for aword, bword in zip(a.split('_'), b.split('_')):
 350                 if aword != bword:
 351                     return '_'.join(commonparts) + '_'
 352                 commonparts.append(aword)
 353             return min(a, b)
 354
 355         # Nothing less than 2 has a common prefix
 356         if len(list(symbol.base_type.child_list)) < 2:
 357             return None
 358         prefix = None
 359         for child in symbol.base_type.child_list:
 360             if prefix is None:
 361                 prefix = child.ident
 362             else:
 363                 prefix = common_prefix(prefix, child.ident)
 364                 if prefix == '':
 365                     return None
 366         return prefix
 367
 368     def _create_enum(self, symbol):
 369         prefix = self._enum_common_prefix(symbol)
 370         if prefix:
 371             prefixlen = len(prefix)
 372         else:
 373             prefixlen = 0
 374         members = []
 375         for child in symbol.base_type.child_list:
 376             if child.private:
 377                 continue
 378             if prefixlen > 0:
 379                 name = child.ident[prefixlen:]
 380             else:
 381                 # Ok, the enum members don't have a consistent prefix
 382                 # among them, so let's just remove the global namespace
 383                 # prefix.
 384                 try:
 385                     name = self._strip_symbol(child)
 386                 except TransformerException, e:
 387                     message.warn_symbol(symbol, e)
 388                     return None
 389             members.append(ast.Member(name.lower(),
 390                                       child.const_int,
 391                                       child.ident,
 392                                       None))
 393
 394         try:
 395             enum_name = self.strip_identifier(symbol.ident)
 396         except TransformerException, e:
 397             message.warn_symbol(symbol, e)
 398             return None
 399         if symbol.base_type.is_bitfield:
 400             klass = ast.Bitfield
 401         else:
 402             klass = ast.Enum
 403         node = klass(enum_name, symbol.ident, members=members)
 404         node.add_symbol_reference(symbol)
 405         return node
 406
 407     def _create_function(self, symbol):
 408         # Drop functions that start with _ very early on here
 409         if symbol.ident.startswith('_'):
 410             return None
 411         parameters = list(self._create_parameters(symbol, symbol.base_type))
 412         return_ = self._create_return(symbol.base_type.base_type)
 413         try:
 414             name = self._strip_symbol(symbol)
 415         except TransformerException, e:
 416             message.warn_symbol(symbol, e)
 417             return None
 418         func = ast.Function(name, return_, parameters, False, symbol.ident)
 419         func.add_symbol_reference(symbol)
 420         return func
 421
 422     def _create_source_type(self, source_type):
 423         assert source_type is not None
 424         if source_type.type == CTYPE_VOID:
 425             value = 'void'
 426         elif source_type.type == CTYPE_BASIC_TYPE:
 427             value = source_type.name
 428         elif source_type.type == CTYPE_TYPEDEF:
 429             value = source_type.name
 430         elif source_type.type == CTYPE_ARRAY:
 431             return self._create_source_type(source_type.base_type)
 432         elif source_type.type == CTYPE_POINTER:
 433             value = self._create_source_type(source_type.base_type) + '*'
 434         else:
 435             value = 'gpointer'
 436         return value
 437
 438     def _create_complete_source_type(self, source_type):
 439         assert source_type is not None
 440
 441         const = (source_type.type_qualifier & TYPE_QUALIFIER_CONST)
 442         volatile = (source_type.type_qualifier & TYPE_QUALIFIER_VOLATILE)
 443
 444         if source_type.type == CTYPE_VOID:
 445             return 'void'
 446         elif source_type.type in [CTYPE_BASIC_TYPE,
 447                                   CTYPE_TYPEDEF,
 448                                   CTYPE_STRUCT,
 449                                   CTYPE_UNION,
 450                                   CTYPE_ENUM]:
 451             value = source_type.name
 452             if not value:
 453                 value = 'gpointer'
 454             if const:
 455                 value = 'const ' + value
 456             if volatile:
 457                 value = 'volatile ' + value
 458         elif source_type.type == CTYPE_ARRAY:
 459             return self._create_complete_source_type(source_type.base_type)
 460         elif source_type.type == CTYPE_POINTER:
 461             value = self._create_complete_source_type(source_type.base_type) + '*'
 462             # TODO: handle pointer to function as a special case?
 463             if const:
 464                 value += ' const'
 465             if volatile:
 466                 value += ' volatile'
 467
 468         else:
 469             if const:
 470                 value = 'gconstpointer'
 471             else:
 472                 value = 'gpointer'
 473             if volatile:
 474                 value = 'volatile ' + value
 475             return value
 476
 477         return value
 478
 479     def _create_parameters(self, symbol, base_type):
 480         # warn if we see annotations for unknown parameters
 481         param_names = set(child.ident for child in base_type.child_list)
 482         for i, child in enumerate(base_type.child_list):
 483             yield self._create_parameter(symbol, i, child)
 484
 485     def _synthesize_union_type(self, symbol, parent_symbol):
 486         # Synthesize a named union so that it can be referenced.
 487         parent_ident = parent_symbol.ident
 488         # FIXME: Should split_ctype_namespaces handle the hidden case?
 489         hidden = parent_ident.startswith('_')
 490         if hidden:
 491             parent_ident = parent_ident[1:]
 492         matches = self.split_ctype_namespaces(parent_ident)
 493         (namespace, parent_name) = matches[-1]
 494         assert namespace and parent_name
 495         if hidden:
 496             parent_name = '_' + parent_name
 497         fake_union = ast.Union("%s__%s__union" % (parent_name, symbol.ident))
 498         # _parse_fields accesses <type>.base_type.child_list, so we have to
 499         # pass symbol.base_type even though that refers to the array, not the
 500         # union.
 501         self._parse_fields(symbol.base_type, fake_union)
 502         self._append_new_node(fake_union)
 503         fake_type = ast.Type(
 504             target_giname="%s.%s" % (namespace.name, fake_union.name))
 505         return fake_type
 506
 507     def _create_member(self, symbol, parent_symbol=None):
 508         source_type = symbol.base_type
 509         if (source_type.type == CTYPE_POINTER and
 510             symbol.base_type.base_type.type == CTYPE_FUNCTION):
 511             node = self._create_callback(symbol, member=True)
 512         elif source_type.type == CTYPE_STRUCT and source_type.name is None:
 513             node = self._create_struct(symbol, anonymous=True)
 514         elif source_type.type == CTYPE_UNION and source_type.name is None:
 515             node = self._create_union(symbol, anonymous=True)
 516         else:
 517             # Special handling for fields; we don't have annotations on them
 518             # to apply later, yet.
 519             if source_type.type == CTYPE_ARRAY:
 520                 complete_ctype = self._create_complete_source_type(source_type)
 521                 # If the array contains anonymous unions, like in the GValue
 522                 # struct, we need to handle this specially.  This is necessary
 523                 # to be able to properly calculate the size of the compound
 524                 # type (e.g. GValue) that contains this array, see
 525                 # <https://bugzilla.gnome.org/show_bug.cgi?id=657040>.
 526                 if (source_type.base_type.type == CTYPE_UNION and
 527                     source_type.base_type.name is None):
 528                     synthesized_type = self._synthesize_union_type(symbol, parent_symbol)
 529                     ftype = ast.Array(None, synthesized_type, complete_ctype=complete_ctype)
 530                 else:
 531                     ctype = self._create_source_type(source_type)
 532                     canonical_ctype = self._canonicalize_ctype(ctype)
 533                     if canonical_ctype[-1] == '*':
 534                         derefed_name = canonical_ctype[:-1]
 535                     else:
 536                         derefed_name = canonical_ctype
 537                     if complete_ctype[-1] == '*':
 538                         derefed_complete_ctype = complete_ctype[:-1]
 539                     else:
 540                         derefed_complete_ctype = complete_ctype
 541                     from_ctype = self.create_type_from_ctype_string(ctype,
 542                                                                     complete_ctype=complete_ctype)
 543                     ftype = ast.Array(None, from_ctype,
 544                                       ctype=derefed_name,
 545                                       complete_ctype=derefed_complete_ctype)
 546                 child_list = list(symbol.base_type.child_list)
 547                 ftype.zeroterminated = False
 548                 if child_list:
 549                     ftype.size = child_list[0].const_int
 550             else:
 551                 ftype = self._create_type_from_base(symbol.base_type)
 552             # ast.Fields are assumed to be read-write
 553             # (except for Objects, see also glibtransformer.py)
 554             node = ast.Field(symbol.ident, ftype,
 555                              readable=True, writable=True,
 556                              bits=symbol.const_int)
 557             if symbol.private:
 558                 node.readable = False
 559                 node.writable = False
 560                 node.private = True
 561         return node
 562
 563     def _create_typedef(self, symbol):
 564         ctype = symbol.base_type.type
 565         if (ctype == CTYPE_POINTER and
 566             symbol.base_type.base_type.type == CTYPE_FUNCTION):
 567             node = self._create_typedef_callback(symbol)
 568         elif (ctype == CTYPE_POINTER and
 569             symbol.base_type.base_type.type == CTYPE_STRUCT):
 570             node = self._create_typedef_struct(symbol, disguised=True)
 571         elif ctype == CTYPE_STRUCT:
 572             node = self._create_typedef_struct(symbol)
 573         elif ctype == CTYPE_UNION:
 574             node = self._create_typedef_union(symbol)
 575         elif ctype == CTYPE_ENUM:
 576             return self._create_enum(symbol)
 577         elif ctype in (CTYPE_TYPEDEF,
 578                        CTYPE_POINTER,
 579                        CTYPE_BASIC_TYPE,
 580                        CTYPE_VOID):
 581             try:
 582                 name = self.strip_identifier(symbol.ident)
 583             except TransformerException, e:
 584                 message.warn(e)
 585                 return None
 586             if symbol.base_type.name:
 587                 complete_ctype = self._create_complete_source_type(symbol.base_type)
 588                 target = self.create_type_from_ctype_string(symbol.base_type.name,
 589                                                             complete_ctype=complete_ctype)
 590             else:
 591                 target = ast.TYPE_ANY
 592             if name in ast.type_names:
 593                 return None
 594             return ast.Alias(name, target, ctype=symbol.ident)
 595         else:
 596             raise NotImplementedError(
 597                 "symbol %r of type %s" % (symbol.ident, ctype_name(ctype)))
 598         return node
 599
 600     def _canonicalize_ctype(self, ctype):
 601         # First look up the ctype including any pointers;
 602         # a few type names like 'char*' have their own aliases
 603         # and we need pointer information for those.
 604         firstpass = ast.type_names.get(ctype)
 605
 606         # If we have a particular alias for this, skip deep
 607         # canonicalization to prevent changing
 608         # e.g. char* -> int8*
 609         if firstpass:
 610             return firstpass.target_fundamental
 611
 612         if not ctype.endswith('*'):
 613             return ctype
 614
 615         # We have a pointer type.
 616         # Strip the end pointer, canonicalize our base type
 617         base = ctype[:-1]
 618         canonical_base = self._canonicalize_ctype(base)
 619
 620         # Append the pointer again
 621         canonical = canonical_base + '*'
 622
 623         return canonical
 624
 625     def parse_ctype(self, ctype, is_member=False):
 626         canonical = self._canonicalize_ctype(ctype)
 627
 628         # Remove all pointers - we require standard calling
 629         # conventions.  For example, an 'int' is always passed by
 630         # value (unless it's out or inout).
 631         derefed_typename = canonical.replace('*', '')
 632
 633         # Preserve "pointerness" of struct/union members
 634         if (is_member and canonical.endswith('*') and
 635             derefed_typename in ast.basic_type_names):
 636             return 'gpointer'
 637         else:
 638             return derefed_typename
 639
 640     def _create_type_from_base(self, source_type, is_parameter=False, is_return=False):
 641         ctype = self._create_source_type(source_type)
 642         complete_ctype = self._create_complete_source_type(source_type)
 643         const = ((source_type.type == CTYPE_POINTER) and
 644                  (source_type.base_type.type_qualifier & TYPE_QUALIFIER_CONST))
 645         return self.create_type_from_ctype_string(ctype, is_const=const,
 646                                                   is_parameter=is_parameter, is_return=is_return,
 647                                                   complete_ctype=complete_ctype)
 648
 649     def _create_bare_container_type(self, base, ctype=None,
 650                                     is_const=False, complete_ctype=None):
 651         if base in ('GList', 'GSList', 'GLib.List', 'GLib.SList'):
 652             if base in ('GList', 'GSList'):
 653                 name = 'GLib.' + base[1:]
 654             else:
 655                 name = base
 656             return ast.List(name, ast.TYPE_ANY, ctype=ctype,
 657                         is_const=is_const, complete_ctype=complete_ctype)
 658         elif base in ('GArray', 'GPtrArray', 'GByteArray',
 659                       'GLib.Array', 'GLib.PtrArray', 'GLib.ByteArray',
 660                       'GObject.Array', 'GObject.PtrArray', 'GObject.ByteArray'):
 661             if '.' in base:
 662                 name = 'GLib.' + base.split('.', 1)[1]
 663             else:
 664                 name = 'GLib.' + base[1:]
 665             return ast.Array(name, ast.TYPE_ANY, ctype=ctype,
 666                          is_const=is_const, complete_ctype=complete_ctype)
 667         elif base in ('GHashTable', 'GLib.HashTable', 'GObject.HashTable'):
 668             return ast.Map(ast.TYPE_ANY, ast.TYPE_ANY, ctype=ctype, is_const=is_const,
 669                            complete_ctype=complete_ctype)
 670         return None
 671
 672     def create_type_from_ctype_string(self, ctype, is_const=False,
 673                                       is_parameter=False, is_return=False,
 674                                       complete_ctype=None):
 675         canonical = self._canonicalize_ctype(ctype)
 676         base = canonical.replace('*', '')
 677
 678         # Special default: char ** -> ast.Array, same for GStrv
 679         if (is_return and canonical == 'utf8*') or base == 'GStrv':
 680             bare_utf8 = ast.TYPE_STRING.clone()
 681             bare_utf8.ctype = None
 682             return ast.Array(None, bare_utf8, ctype=ctype,
 683                              is_const=is_const, complete_ctype=complete_ctype)
 684
 685         fundamental = ast.type_names.get(base)
 686         if fundamental is not None:
 687             return ast.Type(target_fundamental=fundamental.target_fundamental,
 688                         ctype=ctype,
 689                         is_const=is_const, complete_ctype=complete_ctype)
 690         container = self._create_bare_container_type(base, ctype=ctype, is_const=is_const,
 691                                                      complete_ctype=complete_ctype)
 692         if container:
 693             return container
 694         return ast.Type(ctype=ctype, is_const=is_const, complete_ctype=complete_ctype)
 695
 696     def _create_parameter(self, parent_symbol, index, symbol):
 697         if symbol.type == CSYMBOL_TYPE_ELLIPSIS:
 698             ptype = ast.Varargs()
 699         else:
 700             ptype = self._create_type_from_base(symbol.base_type, is_parameter=True)
 701
 702         if symbol.ident is None:
 703             if symbol.base_type and symbol.base_type.type != CTYPE_VOID:
 704                 message.warn_symbol(parent_symbol, "missing parameter name; undocumentable")
 705             ident = 'arg%d' % (index, )
 706         else:
 707             ident = symbol.ident
 708
 709         return ast.Parameter(ident, ptype)
 710
 711     def _create_return(self, source_type):
 712         typeval = self._create_type_from_base(source_type, is_return=True)
 713         return ast.Return(typeval)
 714
 715     def _create_const(self, symbol):
 716         if symbol.ident.startswith('_'):
 717             return None
 718
 719         # Don't create constants for non-public things
 720         # http://bugzilla.gnome.org/show_bug.cgi?id=572790
 721         if (symbol.source_filename is None or
 722             not symbol.source_filename.endswith('.h')):
 723             return None
 724         try:
 725             name = self._strip_symbol(symbol)
 726         except TransformerException, e:
 727             message.warn_symbol(symbol, e)
 728             return None
 729         if symbol.const_string is not None:
 730             typeval = ast.TYPE_STRING
 731             value = unicode(symbol.const_string, 'utf-8')
 732         elif symbol.const_int is not None:
 733             if symbol.base_type is not None:
 734                 typeval = self._create_type_from_base(symbol.base_type)
 735             else:
 736                 typeval = ast.TYPE_INT
 737             unaliased = typeval
 738             self._resolve_type_from_ctype(unaliased)
 739             if typeval.target_giname and typeval.ctype:
 740                 target = self.lookup_giname(typeval.target_giname)
 741                 target = self.resolve_aliases(target)
 742                 if isinstance(target, ast.Type):
 743                     unaliased = target
 744             if unaliased == ast.TYPE_UINT64:
 745                 value = str(symbol.const_int % 2**64)
 746             elif unaliased == ast.TYPE_UINT32:
 747                 value = str(symbol.const_int % 2**32)
 748             elif unaliased == ast.TYPE_UINT16:
 749                 value = str(symbol.const_int % 2**16)
 750             elif unaliased == ast.TYPE_UINT8:
 751                 value = str(symbol.const_int % 2**16)
 752             else:
 753                 value = str(symbol.const_int)
 754         elif symbol.const_double is not None:
 755             typeval = ast.TYPE_DOUBLE
 756             value = '%f' % (symbol.const_double, )
 757         else:
 758             raise AssertionError()
 759
 760         const = ast.Constant(name, typeval, value,
 761                              symbol.ident)
 762         const.add_symbol_reference(symbol)
 763         return const
 764
 765     def _create_typedef_struct(self, symbol, disguised=False):
 766         try:
 767             name = self.strip_identifier(symbol.ident)
 768         except TransformerException, e:
 769             message.warn_symbol(symbol, e)
 770             return None
 771         struct = ast.Record(name, symbol.ident, disguised=disguised)
 772         self._parse_fields(symbol, struct)
 773         struct.add_symbol_reference(symbol)
 774         self._typedefs_ns[symbol.ident] = struct
 775         return None
 776
 777     def _create_typedef_union(self, symbol):
 778         try:
 779             name = self.strip_identifier(symbol.ident)
 780         except TransformerException, e:
 781             message.warn(e)
 782             return None
 783         union = ast.Union(name, symbol.ident)
 784         self._parse_fields(symbol, union)
 785         union.add_symbol_reference(symbol)
 786         self._typedefs_ns[symbol.ident] = union
 787         return None
 788
 789     def _create_typedef_callback(self, symbol):
 790         callback = self._create_callback(symbol)
 791         if not callback:
 792             return None
 793         self._typedefs_ns[callback.name] = callback
 794         return callback
 795
 796     def _parse_fields(self, symbol, compound):
 797         for child in symbol.base_type.child_list:
 798             child_node = self._traverse_one(child, parent_symbol=symbol)
 799             if not child_node:
 800                 continue
 801             if isinstance(child_node, ast.Field):
 802                 field = child_node
 803             else:
 804                 field = ast.Field(child.ident, None, True, False,
 805                               anonymous_node=child_node)
 806             compound.fields.append(field)
 807
 808     def _create_compound(self, klass, symbol, anonymous):
 809         if symbol.ident is None:
 810             # the compound is an anonymous member of another union or a struct
 811             assert anonymous
 812             compound = klass(None, None)
 813         else:
 814             compound = self._typedefs_ns.get(symbol.ident, None)
 815
 816         if compound is None:
 817             # This is a bit of a hack; really we should try
 818             # to resolve through the typedefs to find the real
 819             # name
 820             if symbol.ident.startswith('_'):
 821                 compound = self._typedefs_ns.get(symbol.ident[1:], None)
 822             if compound is None:
 823                 if anonymous:
 824                     name = symbol.ident
 825                 else:
 826                     try:
 827                         name = self.strip_identifier(symbol.ident)
 828                     except TransformerException, e:
 829                         message.warn(e)
 830                         return None
 831                 compound = klass(name, symbol.ident)
 832
 833         self._parse_fields(symbol, compound)
 834         compound.add_symbol_reference(symbol)
 835         return compound
 836
 837     def _create_struct(self, symbol, anonymous=False):
 838         return self._create_compound(ast.Record, symbol, anonymous)
 839
 840     def _create_union(self, symbol, anonymous=False):
 841         return self._create_compound(ast.Union, symbol, anonymous)
 842
 843     def _create_callback(self, symbol, member=False):
 844         parameters = list(self._create_parameters(symbol, symbol.base_type.base_type))
 845         retval = self._create_return(symbol.base_type.base_type.base_type)
 846
 847         # Mark the 'user_data' arguments
 848         for i, param in enumerate(parameters):
 849             if (param.type.target_fundamental == 'gpointer' and
 850                 param.argname == 'user_data'):
 851                 param.closure_name = param.argname
 852
 853         if member:
 854             name = symbol.ident
 855         elif symbol.ident.find('_') > 0:
 856             try:
 857                 name = self._strip_symbol(symbol)
 858             except TransformerException, e:
 859                 message.warn_symbol(symbol, e)
 860                 return None
 861         else:
 862             try:
 863                 name = self.strip_identifier(symbol.ident)
 864             except TransformerException, e:
 865                 message.warn(e)
 866                 return None
 867         callback = ast.Callback(name, retval, parameters, False,
 868                                 ctype=symbol.ident)
 869         callback.add_symbol_reference(symbol)
 870
 871         return callback
 872
 873     def create_type_from_user_string(self, typestr):
 874         """Parse a C type string (as might be given from an
 875         annotation) and resolve it.  For compatibility, we can consume
 876 both GI type string (utf8, Foo.Bar) style, as well as C (char *, FooBar) style.
 877
 878 Note that type resolution may not succeed."""
 879         if '.' in typestr:
 880             container = self._create_bare_container_type(typestr)
 881             if container:
 882                 typeval = container
 883             else:
 884                 typeval = self._namespace.type_from_name(typestr)
 885         else:
 886             typeval = self.create_type_from_ctype_string(typestr)
 887
 888         self.resolve_type(typeval)
 889         if typeval.resolved:
 890             # Explicitly clear out the c_type; there isn't one in this case.
 891             typeval.ctype = None
 892         return typeval
 893
 894     def _resolve_type_from_ctype_all_namespaces(self, typeval, pointer_stripped):
 895         # If we can't determine the namespace from the type name,
 896         # fall back to trying all of our includes.  An example of this is mutter,
 897         # which has nominal namespace of "Meta", but a few classes are
 898         # "Mutter".  We don't export that data in introspection currently.
 899         # Basically the library should be fixed, but we'll hack around it here.
 900         for namespace in self._parsed_includes.itervalues():
 901             target = namespace.get_by_ctype(pointer_stripped)
 902             if target:
 903                 typeval.target_giname = '%s.%s' % (namespace.name, target.name)
 904                 return True
 905         return False
 906
 907     def _resolve_type_from_ctype(self, typeval):
 908         assert typeval.ctype is not None
 909         pointer_stripped = typeval.ctype.replace('*', '')
 910         try:
 911             matches = self.split_ctype_namespaces(pointer_stripped)
 912         except ValueError:
 913             return self._resolve_type_from_ctype_all_namespaces(typeval, pointer_stripped)
 914         for namespace, name in matches:
 915             target = namespace.get(name)
 916             if not target:
 917                 target = namespace.get_by_ctype(pointer_stripped)
 918             if target:
 919                 typeval.target_giname = '%s.%s' % (namespace.name, target.name)
 920                 return True
 921         return False
 922
 923     def _resolve_type_from_gtype_name(self, typeval):
 924         assert typeval.gtype_name is not None
 925         for ns in self._iter_namespaces():
 926             node = ns.type_names.get(typeval.gtype_name, None)
 927             if node is not None:
 928                 typeval.target_giname = '%s.%s' % (ns.name, node.name)
 929                 return True
 930         return False
 931
 932     def _resolve_type_internal(self, typeval):
 933         if isinstance(typeval, (ast.Array, ast.List)):
 934             return self.resolve_type(typeval.element_type)
 935         elif isinstance(typeval, ast.Map):
 936             key_resolved = self.resolve_type(typeval.key_type)
 937             value_resolved = self.resolve_type(typeval.value_type)
 938             return key_resolved and value_resolved
 939         elif typeval.resolved:
 940             return True
 941         elif typeval.ctype:
 942             return self._resolve_type_from_ctype(typeval)
 943         elif typeval.gtype_name:
 944             return self._resolve_type_from_gtype_name(typeval)
 945
 946     def resolve_type(self, typeval):
 947         if not self._resolve_type_internal(typeval):
 948             return False
 949
 950         if typeval.target_fundamental or typeval.target_foreign:
 951             return True
 952
 953         assert typeval.target_giname is not None
 954
 955         try:
 956             type_ = self.lookup_giname(typeval.target_giname)
 957         except KeyError:
 958             type_ = None
 959
 960         if type_ is None:
 961             typeval.target_giname = None
 962
 963         return typeval.resolved
 964
 965     def _typepair_to_str(self, item):
 966         nsname, item = item
 967         if nsname is None:
 968             return item.name
 969         return '%s.%s' % (nsname, item.name)
 970
 971     def gtypename_to_giname(self, gtname, names):
 972         resolved = names.type_names.get(gtname)
 973         if resolved:
 974             return self._typepair_to_str(resolved)
 975         resolved = self._names.type_names.get(gtname)
 976         if resolved:
 977             return self._typepair_to_str(resolved)
 978         raise KeyError("Failed to resolve GType name: %r" % (gtname, ))
 979
 980     def ctype_of(self, obj):
 981         if hasattr(obj, 'ctype'):
 982             return obj.ctype
 983         elif hasattr(obj, 'symbol'):
 984             return obj.symbol
 985         else:
 986             return None
 987
 988     def resolve_aliases(self, typenode):
 989         """Removes all aliases from typenode, returns first non-alias
 990         in the typenode alias chain.  Returns typenode argument if it
 991         is not an alias."""
 992         while isinstance(typenode, ast.Alias):
 993             if typenode.target.target_giname is not None:
 994                 typenode = self.lookup_giname(typenode.target.target_giname)
 995             elif typenode.target.target_fundamental is not None:
 996                 typenode = ast.type_names[typenode.target.target_fundamental]
 997             else:
 998                 break
 999         return typenode