giscanner/scannermain.py

   1 #!/usr/bin/env python
   2 # -*- Mode: Python -*-
   3 # GObject-Introspection - a framework for introspecting GObject libraries
   4 # Copyright (C) 2008-2010 Johan Dahlin
   5 # Copyright (C) 2009 Red Hat, Inc.
   6 #
   7 # This program is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU General Public License
   9 # as published by the Free Software Foundation; either version 2
  10 # of the License, or (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  20 # 02110-1301, USA.
  21 #
  22
  23 import errno
  24 import optparse
  25 import os
  26 import shutil
  27 import subprocess
  28 import sys
  29 import tempfile
  30
  31 from giscanner import message
  32 from giscanner.annotationparser import AnnotationParser
  33 from giscanner.ast import Include, Namespace
  34 from giscanner.dumper import compile_introspection_binary
  35 from giscanner.gdumpparser import GDumpParser, IntrospectionBinary
  36 from giscanner.introspectablepass import IntrospectablePass
  37 from giscanner.girparser import GIRParser
  38 from giscanner.girwriter import GIRWriter
  39 from giscanner.maintransformer import MainTransformer
  40 from giscanner.shlibs import resolve_shlibs
  41 from giscanner.sourcescanner import SourceScanner
  42 from giscanner.transformer import Transformer
  43 from . import utils
  44
  45 def process_cflags_begin(option, opt, value, parser):
  46     cflags = getattr(parser.values, option.dest)
  47     while len(parser.rargs) > 0 and parser.rargs[0] != '--cflags-end':
  48         cflags.append(parser.rargs.pop(0))
  49
  50 def process_cflags_end(option, opt, value, parser):
  51     pass
  52
  53 def get_preprocessor_option_group(parser):
  54     group = optparse.OptionGroup(parser, "Preprocessor options")
  55     group.add_option("", "--cflags-begin",
  56                      help="Start preprocessor/compiler flags",
  57                      dest="cflags", default=[],
  58                      action="callback", callback=process_cflags_begin)
  59     group.add_option("", "--cflags-end",
  60                      help="End preprocessor/compiler flags",
  61                      action="callback", callback=process_cflags_end)
  62     group.add_option("-I", help="Pre-processor include file",
  63                      action="append", dest="cpp_includes",
  64                      default=[])
  65     group.add_option("-D", help="Pre-processor define",
  66                      action="append", dest="cpp_defines",
  67                      default=[])
  68     group.add_option("-U", help="Pre-processor undefine",
  69                      action="append", dest="cpp_undefines",
  70                      default=[])
  71     group.add_option("-p", dest="", help="Ignored")
  72     return group
  73
  74 def get_windows_option_group(parser):
  75     group = optparse.OptionGroup(parser, "Machine Dependent Options")
  76     group.add_option("-m", help="some machine dependent option",
  77                      action="append", dest='m_option',
  78                      default=[])
  79
  80     return group
  81
  82 def _get_option_parser():
  83     parser = optparse.OptionParser('%prog [options] sources')
  84     parser.add_option('', "--quiet",
  85                       action="store_true", dest="quiet",
  86                       default=False,
  87                       help="If passed, do not print details of normal" \
  88                           + " operation")
  89     parser.add_option("", "--format",
  90                       action="store", dest="format",
  91                       default="gir",
  92                       help="format to use, one of gidl, gir")
  93     parser.add_option("-i", "--include",
  94                       action="append", dest="includes", default=[],
  95                       help="Add specified gir file as dependency")
  96     parser.add_option("", "--include-uninstalled",
  97                       action="append", dest="includes_uninstalled", default=[],
  98                       help=("""A file path to a dependency; only use this "
  99                             "when building multiple .gir files inside a "
 100                             "single module."""))
 101     parser.add_option("", "--add-include-path",
 102                       action="append", dest="include_paths", default=[],
 103                       help="include paths for other GIR files")
 104     parser.add_option("", "--program",
 105                       action="store", dest="program", default=None,
 106                       help="program to execute")
 107     parser.add_option("", "--program-arg",
 108                       action="append", dest="program_args", default=[],
 109                       help="extra arguments to program")
 110     parser.add_option("", "--libtool",
 111                       action="store", dest="libtool_path", default=None,
 112                       help="full path to libtool")
 113     parser.add_option("", "--no-libtool",
 114                       action="store_true", dest="nolibtool", default=False,
 115                       help="do not use libtool")
 116     parser.add_option("", "--external-library",
 117                       action="store_true", dest="external_library", default=False,
 118                       help=("""If true, the library is located on the system,""" +
 119                             """not in the current directory"""))
 120     parser.add_option("-l", "--library",
 121                       action="append", dest="libraries", default=[],
 122                       help="libraries of this unit")
 123     parser.add_option("-L", "--library-path",
 124                       action="append", dest="library_paths", default=[],
 125                       help="directories to search for libraries")
 126     parser.add_option("", "--header-only",
 127                       action="store_true", dest="header_only", default=[],
 128                       help="If specified, just generate a GIR for the given header files")
 129     parser.add_option("-n", "--namespace",
 130                       action="store", dest="namespace_name",
 131                       help=("name of namespace for this unit, also "
 132                             "used to compute --identifier-prefix and --symbol-prefix"))
 133     parser.add_option("", "--nsversion",
 134                       action="store", dest="namespace_version",
 135                       help="version of namespace for this unit")
 136     parser.add_option("", "--strip-prefix",
 137                       action="store", dest="strip_prefix",
 138                       help="""Option --strip-prefix is deprecated, please see --identifier-prefix
 139 and --symbol-prefix.""")
 140     parser.add_option("", "--identifier-prefix",
 141                       action="append", dest="identifier_prefixes", default=[],
 142                       help="""Remove this prefix from C identifiers (structure typedefs, etc.).
 143 May be specified multiple times.  This is also used as the default for --symbol-prefix if
 144 the latter is not specified.""")
 145     parser.add_option("", "--symbol-prefix",
 146                       action="append", dest="symbol_prefixes", default=[],
 147                       help="Remove this prefix from C symbols (function names)")
 148     parser.add_option("", "--accept-unprefixed",
 149                       action="store_true", dest="accept_unprefixed", default=False,
 150                       help="""If specified, accept symbols and identifiers that do not
 151 match the namespace prefix.""")
 152     parser.add_option("", "--add-init-section",
 153                       action="append", dest="init_sections", default=[],
 154             help="add extra initialization code in the introspection program")
 155     parser.add_option("-o", "--output",
 156                       action="store", dest="output", default="-",
 157                       help="output filename to write to, defaults to - (stdout)")
 158     parser.add_option("", "--pkg",
 159                       action="append", dest="packages", default=[],
 160                       help="pkg-config packages to get cflags from")
 161     parser.add_option("", "--pkg-export",
 162                       action="append", dest="packages_export", default=[],
 163                       help="Associated pkg-config packages for this library")
 164     parser.add_option('', "--warn-all",
 165                       action="store_true", dest="warn_all", default=False,
 166                       help="If true, enable all warnings for introspection")
 167     parser.add_option('', "--warn-error",
 168                       action="store_true", dest="warn_fatal",
 169                       help="Turn warnings into fatal errors")
 170     parser.add_option("-v", "--verbose",
 171                       action="store_true", dest="verbose",
 172                       help="be verbose")
 173     parser.add_option("", "--c-include",
 174                       action="append", dest="c_includes", default=[],
 175                       help="headers which should be included in C programs")
 176     parser.add_option("", "--filelist",
 177                       action="store", dest="filelist", default=[],
 178                       help="file containing headers and sources to be scanned")
 179
 180     group = get_preprocessor_option_group(parser)
 181     parser.add_option_group(group)
 182
 183     if os.environ.get('MSYSTEM') == 'MINGW32':
 184         group = get_windows_option_group(parser)
 185         parser.add_option_group(group)
 186
 187     # Private options
 188     parser.add_option('', "--generate-typelib-tests",
 189                       action="store", dest="test_codegen", default=None,
 190                       help=optparse.SUPPRESS_HELP)
 191     parser.add_option('', "--passthrough-gir",
 192                       action="store", dest="passthrough_gir", default=None,
 193                       help=optparse.SUPPRESS_HELP)
 194     parser.add_option('', "--reparse-validate",
 195                       action="store_true", dest="reparse_validate_gir", default=False,
 196                       help=optparse.SUPPRESS_HELP)
 197     parser.add_option("", "--typelib-xml",
 198                       action="store_true", dest="typelib_xml",
 199                       help=optparse.SUPPRESS_HELP)
 200
 201     return parser
 202
 203
 204 def _error(msg):
 205     raise SystemExit('ERROR: %s' % (msg, ))
 206
 207 def passthrough_gir(path, f):
 208     parser = GIRParser()
 209     parser.parse(path)
 210
 211     writer = GIRWriter(parser.get_namespace())
 212     f.write(writer.get_xml())
 213
 214 def test_codegen(optstring):
 215     (namespace, out_h_filename, out_c_filename) = optstring.split(',')
 216     if namespace == 'Everything':
 217         from .testcodegen import EverythingCodeGenerator
 218         gen = EverythingCodeGenerator(out_h_filename, out_c_filename)
 219         gen.write()
 220     else:
 221         _error("Invaild namespace %r" % (namespace, ))
 222     return 0
 223
 224 def process_options(output, allowed_flags):
 225     for option in output.split():
 226         for flag in allowed_flags:
 227             if not option.startswith(flag):
 228                 continue
 229             yield option
 230             break
 231
 232 def process_packages(options, packages):
 233     args = ['pkg-config', '--cflags']
 234     args.extend(packages)
 235     output = subprocess.Popen(args,
 236                               stdout=subprocess.PIPE).communicate()[0]
 237     if output is None:
 238         # the error output should have already appeared on our stderr,
 239         # so we just exit
 240         return 1
 241     # Some pkg-config files on Windows have options we don't understand,
 242     # so we explicitly filter to only the ones we need.
 243     options_whitelist = ['-I', '-D', '-U', '-l', '-L']
 244     filtered_output = list(process_options(output, options_whitelist))
 245     parser = _get_option_parser()
 246     pkg_options, unused = parser.parse_args(filtered_output)
 247     options.cpp_includes.extend(pkg_options.cpp_includes)
 248     options.cpp_defines.extend(pkg_options.cpp_defines)
 249     options.cpp_undefines.extend(pkg_options.cpp_undefines)
 250
 251 def extract_filenames(args):
 252     filenames = []
 253     for arg in args:
 254         # We don't support real C++ parsing yet, but we should be able
 255         # to understand C API implemented in C++ files.
 256         if (arg.endswith('.c') or arg.endswith('.cpp') or
 257             arg.endswith('.cc') or arg.endswith('.cxx') or
 258             arg.endswith('.h') or arg.endswith('.hpp') or
 259             arg.endswith('.hxx')):
 260             if not os.path.exists(arg):
 261                 _error('%s: no such a file or directory' % (arg, ))
 262             # Make absolute, because we do comparisons inside scannerparser.c
 263             # against the absolute path that cpp will give us
 264             filenames.append(os.path.abspath(arg))
 265     return filenames
 266
 267 def extract_filelist(options):
 268     filenames = []
 269     if not os.path.exists(options.filelist):
 270         _error('%s: no such filelist file' % (options.filelist, ))
 271     filelist_file = open(options.filelist, "r")
 272     lines = filelist_file.readlines()
 273     for line in lines:
 274         # We don't support real C++ parsing yet, but we should be able
 275         # to understand C API implemented in C++ files.
 276         filename = line.strip()
 277         if (filename.endswith('.c') or filename.endswith('.cpp') or
 278             filename.endswith('.cc') or filename.endswith('.cxx') or
 279             filename.endswith('.h') or filename.endswith('.hpp') or
 280             filename.endswith('.hxx')):
 281             if not os.path.exists(filename):
 282                 _error('%s: Invalid filelist entry-no such file or directory' % (line, ))
 283             # Make absolute, because we do comparisons inside scannerparser.c
 284             # against the absolute path that cpp will give us
 285             filenames.append(os.path.abspath(filename))
 286     return filenames
 287
 288 def create_namespace(options):
 289     if options.strip_prefix:
 290         print """g-ir-scanner: warning: Option --strip-prefix has been deprecated;
 291 see --identifier-prefix and --symbol-prefix."""
 292         options.identifier_prefixes.append(options.strip_prefix)
 293
 294     # We do this dance because the empty list has different semantics from
 295     # None; if the user didn't specify the options, we want to use None so
 296     # the Namespace constructor picks the defaults.
 297     if options.identifier_prefixes:
 298         identifier_prefixes = options.identifier_prefixes
 299     else:
 300         identifier_prefixes = None
 301     if options.symbol_prefixes:
 302         for prefix in options.symbol_prefixes:
 303             # See Transformer._split_c_string_for_namespace_matches() for
 304             # why this check is needed
 305             if prefix.lower() != prefix:
 306                 _error("Values for --symbol-prefix must be entirely lowercase")
 307         symbol_prefixes = options.symbol_prefixes
 308     else:
 309         symbol_prefixes = None
 310
 311     return Namespace(options.namespace_name,
 312                      options.namespace_version,
 313                      identifier_prefixes=identifier_prefixes,
 314                      symbol_prefixes=symbol_prefixes)
 315
 316 def create_transformer(namespace, options):
 317     transformer = Transformer(namespace,
 318                               accept_unprefixed=options.accept_unprefixed)
 319     transformer.set_include_paths(options.include_paths)
 320     if options.passthrough_gir:
 321         transformer.disable_cache()
 322         transformer.set_passthrough_mode()
 323
 324     for include in options.includes:
 325         if os.sep in include:
 326             _error("Invalid include path %r" % (include, ))
 327         try:
 328             include_obj = Include.from_string(include)
 329         except:
 330             _error("Malformed include %r\n" % (include, ))
 331         transformer.register_include(include_obj)
 332     for include_path in options.includes_uninstalled:
 333         transformer.register_include_uninstalled(include_path)
 334
 335     return transformer
 336
 337 def create_binary(transformer, options, args):
 338     # Transform the C AST nodes into higher level
 339     # GLib/GObject nodes
 340     gdump_parser = GDumpParser(transformer)
 341
 342     # Do enough parsing that we have the get_type() functions to reference
 343     # when creating the introspection binary
 344     gdump_parser.init_parse()
 345
 346     if options.program:
 347         args=[options.program]
 348         args.extend(options.program_args)
 349         binary = IntrospectionBinary(args)
 350     else:
 351         binary = compile_introspection_binary(options,
 352                                               gdump_parser.get_get_type_functions(),
 353                                               gdump_parser.get_error_quark_functions())
 354
 355     shlibs = resolve_shlibs(options, binary, options.libraries)
 356     gdump_parser.set_introspection_binary(binary)
 357     gdump_parser.parse()
 358     return shlibs
 359
 360 def create_source_scanner(options, args):
 361     if hasattr(options, 'filelist') and options.filelist:
 362         filenames = extract_filelist(options)
 363     else:
 364         filenames = extract_filenames(args)
 365
 366     # Run the preprocessor, tokenize and construct simple
 367     # objects representing the raw C symbols
 368     ss = SourceScanner()
 369     ss.set_cpp_options(options.cpp_includes,
 370                        options.cpp_defines,
 371                        options.cpp_undefines,
 372                        cflags=options.cflags)
 373     ss.parse_files(filenames)
 374     ss.parse_macros(filenames)
 375     return ss
 376
 377 def write_output(data, options):
 378     if options.output == "-":
 379         output = sys.stdout
 380     elif options.reparse_validate_gir:
 381         main_f, main_f_name = tempfile.mkstemp(suffix='.gir')
 382         main_f = os.fdopen(main_f, 'w')
 383         main_f.write(data)
 384         main_f.close()
 385
 386         temp_f, temp_f_name = tempfile.mkstemp(suffix='.gir')
 387         temp_f = os.fdopen(temp_f, 'w')
 388         passthrough_gir(main_f_name, temp_f)
 389         temp_f.close()
 390         if not utils.files_are_identical(main_f_name, temp_f_name):
 391             _error("Failed to re-parse gir file; scanned=%r passthrough=%r" % (
 392                 main_f_name, temp_f_name))
 393         os.unlink(temp_f_name)
 394         try:
 395             shutil.move(main_f_name, options.output)
 396         except OSError, e:
 397             if e.errno == errno.EPERM:
 398                 os.unlink(main_f_name)
 399                 return 0
 400             raise
 401         return 0
 402     else:
 403         try:
 404             output = open(options.output, "w")
 405         except IOError, e:
 406             _error("opening output for writing: %s" % (e.strerror, ))
 407
 408     try:
 409         output.write(data)
 410     except IOError, e:
 411         _error("while writing output: %s" % (e.strerror, ))
 412
 413 def scanner_main(args):
 414     parser = _get_option_parser()
 415     (options, args) = parser.parse_args(args)
 416
 417     if options.passthrough_gir:
 418         passthrough_gir(options.passthrough_gir, sys.stdout)
 419     if options.test_codegen:
 420         return test_codegen(options.test_codegen)
 421
 422     if hasattr(options, 'filelist') and not options.filelist:
 423         if len(args) <= 1:
 424             _error('Need at least one filename')
 425
 426     if not options.namespace_name:
 427         _error('Namespace name missing')
 428
 429     if options.format == 'gir':
 430         from giscanner.girwriter import GIRWriter as Writer
 431     else:
 432         _error("Unknown format: %s" % (options.format, ))
 433
 434     if not (options.libraries
 435             or options.program
 436             or options.header_only):
 437         _error("Must specify --program or --library")
 438
 439     namespace = create_namespace(options)
 440     logger = message.MessageLogger.get(namespace=namespace)
 441     if options.warn_all:
 442         logger.enable_warnings(True)
 443     transformer = create_transformer(namespace, options)
 444
 445     packages = set(options.packages)
 446     packages.update(transformer.get_pkgconfig_packages())
 447     if packages:
 448         exit_code = process_packages(options, packages)
 449         if exit_code:
 450             return exit_code
 451
 452     ss = create_source_scanner(options, args)
 453
 454     ap = AnnotationParser()
 455     blocks = ap.parse(ss.get_comments())
 456
 457     # Transform the C symbols into AST nodes
 458     transformer.set_annotations(blocks)
 459     transformer.parse(ss.get_symbols())
 460
 461     if not options.header_only:
 462         shlibs = create_binary(transformer, options, args)
 463     else:
 464         shlibs = []
 465
 466     transformer.namespace.shared_libraries = shlibs
 467
 468     main = MainTransformer(transformer, blocks)
 469     main.transform()
 470
 471     utils.break_on_debug_flag('tree')
 472
 473     final = IntrospectablePass(transformer, blocks)
 474     final.validate()
 475
 476     warning_count = logger.get_warning_count()
 477     if options.warn_fatal and warning_count > 0:
 478         message.fatal("warnings configured as fatal")
 479         return 1
 480     elif warning_count > 0 and options.warn_all is False:
 481         print ("g-ir-scanner: %s: warning: %d warnings suppressed (use --warn-all to see them)"
 482                % (transformer.namespace.name, warning_count, ))
 483
 484     # Write out AST
 485     if options.packages_export:
 486         exported_packages = options.packages_export
 487     else:
 488         exported_packages = options.packages
 489
 490     transformer.namespace.c_includes = options.c_includes
 491     transformer.namespace.exported_packages = exported_packages
 492     writer = Writer(transformer.namespace)
 493     data = writer.get_xml()
 494
 495     write_output(data, options)
 496
 497     return 0