giscanner/scannermain.py

   1 #!/usr/bin/env python
   2 # -*- Mode: Python -*-
   3 # GObject-Introspection - a framework for introspecting GObject libraries
   4 # Copyright (C) 2008-2010 Johan Dahlin
   5 # Copyright (C) 2009 Red Hat, Inc.
   6 #
   7 # This program is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU General Public License
   9 # as published by the Free Software Foundation; either version 2
  10 # of the License, or (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  20 # 02110-1301, USA.
  21 #
  22
  23 import errno
  24 import optparse
  25 import os
  26 import shutil
  27 import subprocess
  28 import sys
  29 import tempfile
  30
  31 from giscanner import message
  32 from giscanner.annotationparser import AnnotationParser
  33 from giscanner.ast import Include, Namespace
  34 from giscanner.dumper import compile_introspection_binary
  35 from giscanner.gdumpparser import GDumpParser, IntrospectionBinary
  36 from giscanner.introspectablepass import IntrospectablePass
  37 from giscanner.girparser import GIRParser
  38 from giscanner.girwriter import GIRWriter
  39 from giscanner.maintransformer import MainTransformer
  40 from giscanner.shlibs import resolve_shlibs
  41 from giscanner.sourcescanner import SourceScanner
  42 from giscanner.transformer import Transformer
  43 from . import utils
  44
  45 def _get_option_parser():
  46     parser = optparse.OptionParser('%prog [options] sources')
  47     parser.add_option('', "--quiet",
  48                       action="store_true", dest="quiet",
  49                       default=False,
  50                       help="If passed, do not print details of normal" \
  51                           + " operation")
  52     parser.add_option("", "--format",
  53                       action="store", dest="format",
  54                       default="gir",
  55                       help="format to use, one of gidl, gir")
  56     parser.add_option("-i", "--include",
  57                       action="append", dest="includes", default=[],
  58                       help="Add specified gir file as dependency")
  59     parser.add_option("", "--include-uninstalled",
  60                       action="append", dest="includes_uninstalled", default=[],
  61                       help=("""A file path to a dependency; only use this "
  62                             "when building multiple .gir files inside a "
  63                             "single module."""))
  64     parser.add_option("", "--add-include-path",
  65                       action="append", dest="include_paths", default=[],
  66                       help="include paths for other GIR files")
  67     parser.add_option("", "--program",
  68                       action="store", dest="program", default=None,
  69                       help="program to execute")
  70     parser.add_option("", "--program-arg",
  71                       action="append", dest="program_args", default=[],
  72                       help="extra arguments to program")
  73     parser.add_option("", "--libtool",
  74                       action="store", dest="libtool_path", default=None,
  75                       help="full path to libtool")
  76     parser.add_option("", "--no-libtool",
  77                       action="store_true", dest="nolibtool", default=False,
  78                       help="do not use libtool")
  79     parser.add_option("-l", "--library",
  80                       action="append", dest="libraries", default=[],
  81                       help="libraries of this unit")
  82     parser.add_option("-L", "--library-path",
  83                       action="append", dest="library_paths", default=[],
  84                       help="directories to search for libraries")
  85     parser.add_option("-n", "--namespace",
  86                       action="store", dest="namespace_name",
  87                       help=("name of namespace for this unit, also "
  88                             "used to compute --identifier-prefix and --symbol-prefix"))
  89     parser.add_option("", "--nsversion",
  90                       action="store", dest="namespace_version",
  91                       help="version of namespace for this unit")
  92     parser.add_option("", "--strip-prefix",
  93                       action="store", dest="strip_prefix",
  94                       help="""Option --strip-prefix is deprecated, please see --identifier-prefix
  95 and --symbol-prefix.""")
  96     parser.add_option("", "--identifier-prefix",
  97                       action="append", dest="identifier_prefixes", default=[],
  98                       help="""Remove this prefix from C identifiers (structure typedefs, etc.).
  99 May be specified multiple times.  This is also used as the default for --symbol-prefix if
 100 the latter is not specified.""")
 101     parser.add_option("", "--symbol-prefix",
 102                       action="append", dest="symbol_prefixes", default=[],
 103                       help="Remove this prefix from C symbols (function names)")
 104     parser.add_option("", "--accept-unprefixed",
 105                       action="store_true", dest="accept_unprefixed", default=False,
 106                       help="""If specified, accept symbols and identifiers that do not
 107 match the namespace prefix.""")
 108     parser.add_option("", "--add-init-section",
 109                       action="append", dest="init_sections", default=[],
 110             help="add extra initialization code in the introspection program")
 111     parser.add_option("-o", "--output",
 112                       action="store", dest="output", default="-",
 113                       help="output filename to write to, defaults to - (stdout)")
 114     parser.add_option("", "--pkg",
 115                       action="append", dest="packages", default=[],
 116                       help="pkg-config packages to get cflags from")
 117     parser.add_option("", "--pkg-export",
 118                       action="append", dest="packages_export", default=[],
 119                       help="Associated pkg-config packages for this library")
 120     parser.add_option('', "--warn-all",
 121                       action="store_true", dest="warn_all", default=False,
 122                       help="If true, enable all warnings for introspection")
 123     parser.add_option('', "--warn-error",
 124                       action="store_true", dest="warn_fatal",
 125                       help="Turn warnings into fatal errors")
 126     parser.add_option("-v", "--verbose",
 127                       action="store_true", dest="verbose",
 128                       help="be verbose")
 129     parser.add_option("", "--c-include",
 130                       action="append", dest="c_includes", default=[],
 131                       help="headers which should be included in C programs")
 132
 133     group = optparse.OptionGroup(parser, "Preprocessor options")
 134     group.add_option("-I", help="Pre-processor include file",
 135                      action="append", dest="cpp_includes",
 136                      default=[])
 137     group.add_option("-D", help="Pre-processor define",
 138                      action="append", dest="cpp_defines",
 139                      default=[])
 140     group.add_option("-U", help="Pre-processor undefine",
 141                      action="append", dest="cpp_undefines",
 142                      default=[])
 143     group.add_option("-p", dest="", help="Ignored")
 144     parser.add_option_group(group)
 145
 146     # Private options
 147     parser.add_option('', "--generate-typelib-tests",
 148                       action="store", dest="test_codegen", default=None,
 149                       help=optparse.SUPPRESS_HELP)
 150     parser.add_option('', "--passthrough-gir",
 151                       action="store", dest="passthrough_gir", default=None,
 152                       help=optparse.SUPPRESS_HELP)
 153     parser.add_option('', "--reparse-validate",
 154                       action="store_true", dest="reparse_validate_gir", default=False,
 155                       help=optparse.SUPPRESS_HELP)
 156     parser.add_option("", "--typelib-xml",
 157                       action="store_true", dest="typelib_xml",
 158                       help=optparse.SUPPRESS_HELP)
 159
 160     return parser
 161
 162
 163 def _error(msg):
 164     raise SystemExit('ERROR: %s' % (msg, ))
 165
 166 def passthrough_gir(path, f):
 167     parser = GIRParser()
 168     parser.parse(path)
 169
 170     writer = GIRWriter(parser.get_namespace(),
 171                        parser.get_shared_libraries(),
 172                        parser.get_includes(),
 173                        parser.get_pkgconfig_packages(),
 174                        parser.get_c_includes())
 175     f.write(writer.get_xml())
 176
 177 def test_codegen(optstring):
 178     (namespace, out_h_filename, out_c_filename) = optstring.split(',')
 179     if namespace == 'Everything':
 180         from .testcodegen import EverythingCodeGenerator
 181         gen = EverythingCodeGenerator(out_h_filename, out_c_filename)
 182         gen.write()
 183     else:
 184         _error("Invaild namespace %r" % (namespace, ))
 185     return 0
 186
 187 def process_options(output, allowed_flags):
 188     for option in output.split():
 189         for flag in allowed_flags:
 190             if not option.startswith(flag):
 191                 continue
 192             yield option
 193             break
 194
 195 def process_packages(options, packages):
 196     args = ['pkg-config', '--cflags']
 197     args.extend(packages)
 198     output = subprocess.Popen(args,
 199                               stdout=subprocess.PIPE).communicate()[0]
 200     if output is None:
 201         # the error output should have already appeared on our stderr,
 202         # so we just exit
 203         return 1
 204     # Some pkg-config files on Windows have options we don't understand,
 205     # so we explicitly filter to only the ones we need.
 206     options_whitelist = ['-I', '-D', '-U', '-l', '-L']
 207     filtered_output = list(process_options(output, options_whitelist))
 208     parser = _get_option_parser()
 209     pkg_options, unused = parser.parse_args(filtered_output)
 210     options.cpp_includes.extend(pkg_options.cpp_includes)
 211     options.cpp_defines.extend(pkg_options.cpp_defines)
 212     options.cpp_undefines.extend(pkg_options.cpp_undefines)
 213
 214     args = ['pkg-config', '--libs-only-L']
 215     args.extend(packages)
 216     output = subprocess.Popen(args,
 217                               stdout=subprocess.PIPE).communicate()[0]
 218     if output is None:
 219         return 1
 220     filtered_output = list(process_options(output, options_whitelist))
 221     pkg_options, unused = parser.parse_args(filtered_output)
 222     options.library_paths.extend(pkg_options.library_paths)
 223
 224 def scanner_main(args):
 225     parser = _get_option_parser()
 226     (options, args) = parser.parse_args(args)
 227
 228     if options.passthrough_gir:
 229         passthrough_gir(options.passthrough_gir, sys.stdout)
 230     if options.test_codegen:
 231         return test_codegen(options.test_codegen)
 232
 233     if len(args) <= 1:
 234         _error('Need at least one filename')
 235
 236     if not options.namespace_name:
 237         _error('Namespace name missing')
 238
 239     if options.format == 'gir':
 240         from giscanner.girwriter import GIRWriter as Writer
 241     else:
 242         _error("Unknown format: %s" % (options.format, ))
 243
 244     if not (options.libraries or options.program):
 245         _error("Must specify --program or --library")
 246     libraries = options.libraries
 247
 248     if options.strip_prefix:
 249         _error("Option --strip-prefix is deprecated, please see --identifier-prefix and --symbol-prefix.")
 250
 251     filenames = []
 252     for arg in args:
 253         # We don't support real C++ parsing yet, but we should be able
 254         # to understand C API implemented in C++ files.
 255         if (arg.endswith('.c') or arg.endswith('.cpp') or
 256             arg.endswith('.cc') or arg.endswith('.cxx') or
 257             arg.endswith('.h') or arg.endswith('.hpp') or
 258             arg.endswith('.hxx')):
 259             if not os.path.exists(arg):
 260                 _error('%s: no such a file or directory' % (arg, ))
 261             # Make absolute, because we do comparisons inside scannerparser.c
 262             # against the absolute path that cpp will give us
 263             filenames.append(os.path.abspath(arg))
 264
 265     # We do this dance because the empty list has different semantics from
 266     # None; if the user didn't specify the options, we want to use None so
 267     # the Namespace constructor picks the defaults.
 268     if options.identifier_prefixes:
 269         identifier_prefixes = options.identifier_prefixes
 270     else:
 271         identifier_prefixes = None
 272     if options.symbol_prefixes:
 273         for prefix in options.symbol_prefixes:
 274             # See Transformer._split_c_string_for_namespace_matches() for
 275             # why this check is needed
 276             if prefix.lower() != prefix:
 277                 _error("Values for --symbol-prefix must be entirely lowercase")
 278         symbol_prefixes = options.symbol_prefixes
 279     else:
 280         symbol_prefixes = None
 281
 282     namespace = Namespace(options.namespace_name,
 283                           options.namespace_version,
 284                           identifier_prefixes=identifier_prefixes,
 285                           symbol_prefixes=symbol_prefixes)
 286     logger = message.MessageLogger.get(namespace=namespace)
 287     if options.warn_all:
 288         logger.enable_warnings(True)
 289     transformer = Transformer(namespace,
 290                               accept_unprefixed=options.accept_unprefixed)
 291     transformer.set_include_paths(options.include_paths)
 292     shown_include_warning = False
 293     for include in options.includes:
 294         if os.sep in include:
 295             _error("Invalid include path %r" % (include, ))
 296         try:
 297             include_obj = Include.from_string(include)
 298         except:
 299             _error("Malformed include %r\n" % (include, ))
 300         transformer.register_include(include_obj)
 301     for include_path in options.includes_uninstalled:
 302         transformer.register_include_uninstalled(include_path)
 303
 304     packages = set(options.packages)
 305     packages.update(transformer.get_pkgconfig_packages())
 306     exit_code = process_packages(options, packages)
 307     if exit_code:
 308         return exit_code
 309
 310     # Run the preprocessor, tokenize and construct simple
 311     # objects representing the raw C symbols
 312     ss = SourceScanner()
 313     ss.set_cpp_options(options.cpp_includes,
 314                        options.cpp_defines,
 315                        options.cpp_undefines)
 316     ss.parse_files(filenames)
 317     ss.parse_macros(filenames)
 318
 319     # Transform the C symbols into AST nodes
 320     transformer.parse(ss.get_symbols())
 321
 322     # Transform the C AST nodes into higher level
 323     # GLib/GObject nodes
 324     gdump_parser = GDumpParser(transformer)
 325
 326     # Do enough parsing that we have the get_type() functions to reference
 327     # when creating the introspection binary
 328     gdump_parser.init_parse()
 329
 330     if options.program:
 331         args=[options.program]
 332         args.extend(options.program_args)
 333         binary = IntrospectionBinary(args)
 334     else:
 335         binary = compile_introspection_binary(options,
 336                             gdump_parser.get_get_type_functions())
 337
 338     shlibs = resolve_shlibs(options, binary, libraries)
 339
 340     gdump_parser.set_introspection_binary(binary)
 341     gdump_parser.parse()
 342
 343     ap = AnnotationParser()
 344     blocks = ap.parse(ss.get_comments())
 345
 346     main = MainTransformer(transformer, blocks)
 347     main.transform()
 348
 349     utils.break_on_debug_flag('tree')
 350
 351     final = IntrospectablePass(transformer)
 352     final.validate()
 353
 354     if options.warn_fatal and logger.did_warn():
 355         message.fatal("warnings configured as fatal")
 356         return 1
 357
 358     # Write out AST
 359     if options.packages_export:
 360         exported_packages = options.packages_export
 361     else:
 362         exported_packages = options.packages
 363     writer = Writer(transformer.namespace, shlibs, transformer.get_includes(),
 364                     exported_packages, options.c_includes)
 365     data = writer.get_xml()
 366
 367     if options.output == "-":
 368         output = sys.stdout
 369     elif options.reparse_validate_gir:
 370         main_f, main_f_name = tempfile.mkstemp(suffix='.gir')
 371         main_f = os.fdopen(main_f, 'w')
 372         main_f.write(data)
 373         main_f.close()
 374
 375         temp_f, temp_f_name = tempfile.mkstemp(suffix='.gir')
 376         temp_f = os.fdopen(temp_f, 'w')
 377         passthrough_gir(main_f_name, temp_f)
 378         temp_f.close()
 379         if not utils.files_are_identical(main_f_name, temp_f_name):
 380             _error("Failed to re-parse gir file; scanned=%r passthrough=%r" % (
 381                 main_f_name, temp_f_name))
 382         os.unlink(temp_f_name)
 383         try:
 384             shutil.move(main_f_name, options.output)
 385         except OSError, e:
 386             if e.errno == errno.EPERM:
 387                 os.unlink(main_f_name)
 388                 return 0
 389             raise
 390         return 0
 391     else:
 392         try:
 393             output = open(options.output, "w")
 394         except IOError, e:
 395             _error("opening output for writing: %s" % (e.strerror, ))
 396
 397     try:
 398         output.write(data)
 399     except IOError, e:
 400         _error("while writing output: %s" % (e.strerror, ))
 401
 402     return 0