giscanner/scannermain.py

   1 #!/usr/bin/env python
   2 # -*- Mode: Python -*-
   3 # GObject-Introspection - a framework for introspecting GObject libraries
   4 # Copyright (C) 2008-2010 Johan Dahlin
   5 # Copyright (C) 2009 Red Hat, Inc.
   6 #
   7 # This program is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU General Public License
   9 # as published by the Free Software Foundation; either version 2
  10 # of the License, or (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  20 # 02110-1301, USA.
  21 #
  22
  23 import errno
  24 import optparse
  25 import os
  26 import shutil
  27 import subprocess
  28 import sys
  29 import tempfile
  30
  31 from giscanner import message
  32 from giscanner.annotationparser import AnnotationParser
  33 from giscanner.ast import Include, Namespace
  34 from giscanner.dumper import compile_introspection_binary
  35 from giscanner.gdumpparser import GDumpParser, IntrospectionBinary
  36 from giscanner.introspectablepass import IntrospectablePass
  37 from giscanner.girparser import GIRParser
  38 from giscanner.girwriter import GIRWriter
  39 from giscanner.maintransformer import MainTransformer
  40 from giscanner.shlibs import resolve_shlibs
  41 from giscanner.sourcescanner import SourceScanner
  42 from giscanner.transformer import Transformer
  43 from . import utils
  44
  45 def get_preprocessor_option_group(parser):
  46     group = optparse.OptionGroup(parser, "Preprocessor options")
  47     group.add_option("-I", help="Pre-processor include file",
  48                      action="append", dest="cpp_includes",
  49                      default=[])
  50     group.add_option("-D", help="Pre-processor define",
  51                      action="append", dest="cpp_defines",
  52                      default=[])
  53     group.add_option("-U", help="Pre-processor undefine",
  54                      action="append", dest="cpp_undefines",
  55                      default=[])
  56     group.add_option("-p", dest="", help="Ignored")
  57     return group
  58
  59 def _get_option_parser():
  60     parser = optparse.OptionParser('%prog [options] sources')
  61     parser.add_option('', "--quiet",
  62                       action="store_true", dest="quiet",
  63                       default=False,
  64                       help="If passed, do not print details of normal" \
  65                           + " operation")
  66     parser.add_option("", "--format",
  67                       action="store", dest="format",
  68                       default="gir",
  69                       help="format to use, one of gidl, gir")
  70     parser.add_option("-i", "--include",
  71                       action="append", dest="includes", default=[],
  72                       help="Add specified gir file as dependency")
  73     parser.add_option("", "--include-uninstalled",
  74                       action="append", dest="includes_uninstalled", default=[],
  75                       help=("""A file path to a dependency; only use this "
  76                             "when building multiple .gir files inside a "
  77                             "single module."""))
  78     parser.add_option("", "--add-include-path",
  79                       action="append", dest="include_paths", default=[],
  80                       help="include paths for other GIR files")
  81     parser.add_option("", "--program",
  82                       action="store", dest="program", default=None,
  83                       help="program to execute")
  84     parser.add_option("", "--program-arg",
  85                       action="append", dest="program_args", default=[],
  86                       help="extra arguments to program")
  87     parser.add_option("", "--libtool",
  88                       action="store", dest="libtool_path", default=None,
  89                       help="full path to libtool")
  90     parser.add_option("", "--no-libtool",
  91                       action="store_true", dest="nolibtool", default=False,
  92                       help="do not use libtool")
  93     parser.add_option("-l", "--library",
  94                       action="append", dest="libraries", default=[],
  95                       help="libraries of this unit")
  96     parser.add_option("-L", "--library-path",
  97                       action="append", dest="library_paths", default=[],
  98                       help="directories to search for libraries")
  99     parser.add_option("-n", "--namespace",
 100                       action="store", dest="namespace_name",
 101                       help=("name of namespace for this unit, also "
 102                             "used to compute --identifier-prefix and --symbol-prefix"))
 103     parser.add_option("", "--nsversion",
 104                       action="store", dest="namespace_version",
 105                       help="version of namespace for this unit")
 106     parser.add_option("", "--strip-prefix",
 107                       action="store", dest="strip_prefix",
 108                       help="""Option --strip-prefix is deprecated, please see --identifier-prefix
 109 and --symbol-prefix.""")
 110     parser.add_option("", "--identifier-prefix",
 111                       action="append", dest="identifier_prefixes", default=[],
 112                       help="""Remove this prefix from C identifiers (structure typedefs, etc.).
 113 May be specified multiple times.  This is also used as the default for --symbol-prefix if
 114 the latter is not specified.""")
 115     parser.add_option("", "--symbol-prefix",
 116                       action="append", dest="symbol_prefixes", default=[],
 117                       help="Remove this prefix from C symbols (function names)")
 118     parser.add_option("", "--accept-unprefixed",
 119                       action="store_true", dest="accept_unprefixed", default=False,
 120                       help="""If specified, accept symbols and identifiers that do not
 121 match the namespace prefix.""")
 122     parser.add_option("", "--add-init-section",
 123                       action="append", dest="init_sections", default=[],
 124             help="add extra initialization code in the introspection program")
 125     parser.add_option("-o", "--output",
 126                       action="store", dest="output", default="-",
 127                       help="output filename to write to, defaults to - (stdout)")
 128     parser.add_option("", "--pkg",
 129                       action="append", dest="packages", default=[],
 130                       help="pkg-config packages to get cflags from")
 131     parser.add_option("", "--pkg-export",
 132                       action="append", dest="packages_export", default=[],
 133                       help="Associated pkg-config packages for this library")
 134     parser.add_option('', "--warn-all",
 135                       action="store_true", dest="warn_all", default=False,
 136                       help="If true, enable all warnings for introspection")
 137     parser.add_option('', "--warn-error",
 138                       action="store_true", dest="warn_fatal",
 139                       help="Turn warnings into fatal errors")
 140     parser.add_option("-v", "--verbose",
 141                       action="store_true", dest="verbose",
 142                       help="be verbose")
 143     parser.add_option("", "--c-include",
 144                       action="append", dest="c_includes", default=[],
 145                       help="headers which should be included in C programs")
 146
 147     group = get_preprocessor_option_group(parser)
 148     parser.add_option_group(group)
 149
 150     # Private options
 151     parser.add_option('', "--generate-typelib-tests",
 152                       action="store", dest="test_codegen", default=None,
 153                       help=optparse.SUPPRESS_HELP)
 154     parser.add_option('', "--passthrough-gir",
 155                       action="store", dest="passthrough_gir", default=None,
 156                       help=optparse.SUPPRESS_HELP)
 157     parser.add_option('', "--reparse-validate",
 158                       action="store_true", dest="reparse_validate_gir", default=False,
 159                       help=optparse.SUPPRESS_HELP)
 160     parser.add_option("", "--typelib-xml",
 161                       action="store_true", dest="typelib_xml",
 162                       help=optparse.SUPPRESS_HELP)
 163
 164     return parser
 165
 166
 167 def _error(msg):
 168     raise SystemExit('ERROR: %s' % (msg, ))
 169
 170 def passthrough_gir(path, f):
 171     parser = GIRParser()
 172     parser.parse(path)
 173
 174     writer = GIRWriter(parser.get_namespace(),
 175                        parser.get_shared_libraries(),
 176                        parser.get_includes(),
 177                        parser.get_pkgconfig_packages(),
 178                        parser.get_c_includes())
 179     f.write(writer.get_xml())
 180
 181 def test_codegen(optstring):
 182     (namespace, out_h_filename, out_c_filename) = optstring.split(',')
 183     if namespace == 'Everything':
 184         from .testcodegen import EverythingCodeGenerator
 185         gen = EverythingCodeGenerator(out_h_filename, out_c_filename)
 186         gen.write()
 187     else:
 188         _error("Invaild namespace %r" % (namespace, ))
 189     return 0
 190
 191 def process_options(output, allowed_flags):
 192     for option in output.split():
 193         for flag in allowed_flags:
 194             if not option.startswith(flag):
 195                 continue
 196             yield option
 197             break
 198
 199 def process_packages(options, packages):
 200     args = ['pkg-config', '--cflags']
 201     args.extend(packages)
 202     output = subprocess.Popen(args,
 203                               stdout=subprocess.PIPE).communicate()[0]
 204     if output is None:
 205         # the error output should have already appeared on our stderr,
 206         # so we just exit
 207         return 1
 208     # Some pkg-config files on Windows have options we don't understand,
 209     # so we explicitly filter to only the ones we need.
 210     options_whitelist = ['-I', '-D', '-U', '-l', '-L']
 211     filtered_output = list(process_options(output, options_whitelist))
 212     parser = _get_option_parser()
 213     pkg_options, unused = parser.parse_args(filtered_output)
 214     options.cpp_includes.extend(pkg_options.cpp_includes)
 215     options.cpp_defines.extend(pkg_options.cpp_defines)
 216     options.cpp_undefines.extend(pkg_options.cpp_undefines)
 217
 218     args = ['pkg-config', '--libs-only-L']
 219     args.extend(packages)
 220     output = subprocess.Popen(args,
 221                               stdout=subprocess.PIPE).communicate()[0]
 222     if output is None:
 223         return 1
 224     filtered_output = list(process_options(output, options_whitelist))
 225     pkg_options, unused = parser.parse_args(filtered_output)
 226     options.library_paths.extend(pkg_options.library_paths)
 227
 228 def extract_filenames(args):
 229     filenames = []
 230     for arg in args:
 231         # We don't support real C++ parsing yet, but we should be able
 232         # to understand C API implemented in C++ files.
 233         if (arg.endswith('.c') or arg.endswith('.cpp') or
 234             arg.endswith('.cc') or arg.endswith('.cxx') or
 235             arg.endswith('.h') or arg.endswith('.hpp') or
 236             arg.endswith('.hxx')):
 237             if not os.path.exists(arg):
 238                 _error('%s: no such a file or directory' % (arg, ))
 239             # Make absolute, because we do comparisons inside scannerparser.c
 240             # against the absolute path that cpp will give us
 241             filenames.append(os.path.abspath(arg))
 242     return filenames
 243
 244 def create_namespace(options):
 245     if options.strip_prefix:
 246         print """g-ir-scanner: warning: Option --strip-prefix has been deprecated;
 247 see --identifier-prefix and --symbol-prefix."""
 248         options.identifier_prefixes.append(options.strip_prefix)
 249
 250     # We do this dance because the empty list has different semantics from
 251     # None; if the user didn't specify the options, we want to use None so
 252     # the Namespace constructor picks the defaults.
 253     if options.identifier_prefixes:
 254         identifier_prefixes = options.identifier_prefixes
 255     else:
 256         identifier_prefixes = None
 257     if options.symbol_prefixes:
 258         for prefix in options.symbol_prefixes:
 259             # See Transformer._split_c_string_for_namespace_matches() for
 260             # why this check is needed
 261             if prefix.lower() != prefix:
 262                 _error("Values for --symbol-prefix must be entirely lowercase")
 263         symbol_prefixes = options.symbol_prefixes
 264     else:
 265         symbol_prefixes = None
 266
 267     return Namespace(options.namespace_name,
 268                      options.namespace_version,
 269                      identifier_prefixes=identifier_prefixes,
 270                      symbol_prefixes=symbol_prefixes)
 271
 272 def create_transformer(namespace, options):
 273     transformer = Transformer(namespace,
 274                               accept_unprefixed=options.accept_unprefixed)
 275     transformer.set_include_paths(options.include_paths)
 276     if options.passthrough_gir:
 277         transformer.disable_cache()
 278         transformer.set_passthrough_mode()
 279
 280     shown_include_warning = False
 281     for include in options.includes:
 282         if os.sep in include:
 283             _error("Invalid include path %r" % (include, ))
 284         try:
 285             include_obj = Include.from_string(include)
 286         except:
 287             _error("Malformed include %r\n" % (include, ))
 288         transformer.register_include(include_obj)
 289     for include_path in options.includes_uninstalled:
 290         transformer.register_include_uninstalled(include_path)
 291
 292     return transformer
 293
 294 def create_binary(transformer, options, args):
 295     # Transform the C AST nodes into higher level
 296     # GLib/GObject nodes
 297     gdump_parser = GDumpParser(transformer)
 298
 299     # Do enough parsing that we have the get_type() functions to reference
 300     # when creating the introspection binary
 301     gdump_parser.init_parse()
 302
 303     if options.program:
 304         args=[options.program]
 305         args.extend(options.program_args)
 306         binary = IntrospectionBinary(args)
 307     else:
 308         binary = compile_introspection_binary(options,
 309                                               gdump_parser.get_get_type_functions())
 310
 311     shlibs = resolve_shlibs(options, binary, options.libraries)
 312     gdump_parser.set_introspection_binary(binary)
 313     gdump_parser.parse()
 314     return shlibs
 315
 316 def create_source_scanner(options, args):
 317     filenames = extract_filenames(args)
 318
 319     # Run the preprocessor, tokenize and construct simple
 320     # objects representing the raw C symbols
 321     ss = SourceScanner()
 322     ss.set_cpp_options(options.cpp_includes,
 323                        options.cpp_defines,
 324                        options.cpp_undefines)
 325     ss.parse_files(filenames)
 326     ss.parse_macros(filenames)
 327     return ss
 328
 329 def write_output(data, options):
 330     if options.output == "-":
 331         output = sys.stdout
 332     elif options.reparse_validate_gir:
 333         main_f, main_f_name = tempfile.mkstemp(suffix='.gir')
 334         main_f = os.fdopen(main_f, 'w')
 335         main_f.write(data)
 336         main_f.close()
 337
 338         temp_f, temp_f_name = tempfile.mkstemp(suffix='.gir')
 339         temp_f = os.fdopen(temp_f, 'w')
 340         passthrough_gir(main_f_name, temp_f)
 341         temp_f.close()
 342         if not utils.files_are_identical(main_f_name, temp_f_name):
 343             _error("Failed to re-parse gir file; scanned=%r passthrough=%r" % (
 344                 main_f_name, temp_f_name))
 345         os.unlink(temp_f_name)
 346         try:
 347             shutil.move(main_f_name, options.output)
 348         except OSError, e:
 349             if e.errno == errno.EPERM:
 350                 os.unlink(main_f_name)
 351                 return 0
 352             raise
 353         return 0
 354     else:
 355         try:
 356             output = open(options.output, "w")
 357         except IOError, e:
 358             _error("opening output for writing: %s" % (e.strerror, ))
 359
 360     try:
 361         output.write(data)
 362     except IOError, e:
 363         _error("while writing output: %s" % (e.strerror, ))
 364
 365 def scanner_main(args):
 366     parser = _get_option_parser()
 367     (options, args) = parser.parse_args(args)
 368
 369     if options.passthrough_gir:
 370         passthrough_gir(options.passthrough_gir, sys.stdout)
 371     if options.test_codegen:
 372         return test_codegen(options.test_codegen)
 373
 374     if len(args) <= 1:
 375         _error('Need at least one filename')
 376
 377     if not options.namespace_name:
 378         _error('Namespace name missing')
 379
 380     if options.format == 'gir':
 381         from giscanner.girwriter import GIRWriter as Writer
 382     else:
 383         _error("Unknown format: %s" % (options.format, ))
 384
 385     if not (options.libraries or options.program):
 386         _error("Must specify --program or --library")
 387
 388     namespace = create_namespace(options)
 389     logger = message.MessageLogger.get(namespace=namespace)
 390     if options.warn_all:
 391         logger.enable_warnings(True)
 392     transformer = create_transformer(namespace, options)
 393
 394     packages = set(options.packages)
 395     packages.update(transformer.get_pkgconfig_packages())
 396     exit_code = process_packages(options, packages)
 397     if exit_code:
 398         return exit_code
 399
 400     ss = create_source_scanner(options, args)
 401
 402     # Transform the C symbols into AST nodes
 403     transformer.parse(ss.get_symbols())
 404
 405     shlibs = create_binary(transformer, options, args)
 406
 407     ap = AnnotationParser()
 408     blocks = ap.parse(ss.get_comments())
 409
 410     main = MainTransformer(transformer, blocks)
 411     main.transform()
 412
 413     utils.break_on_debug_flag('tree')
 414
 415     final = IntrospectablePass(transformer, blocks)
 416     final.validate()
 417
 418     if options.warn_fatal and logger.did_warn():
 419         message.fatal("warnings configured as fatal")
 420         return 1
 421
 422     # Write out AST
 423     if options.packages_export:
 424         exported_packages = options.packages_export
 425     else:
 426         exported_packages = options.packages
 427
 428     writer = Writer(transformer.namespace, shlibs, transformer.get_includes(),
 429                     exported_packages, options.c_includes)
 430     data = writer.get_xml()
 431
 432     write_output(data, options)
 433
 434     return 0