giscanner/scannermain.py

   1 #!/usr/bin/env python
   2 # -*- Mode: Python -*-
   3 # GObject-Introspection - a framework for introspecting GObject libraries
   4 # Copyright (C) 2008-2010 Johan Dahlin
   5 # Copyright (C) 2009 Red Hat, Inc.
   6 #
   7 # This program is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU General Public License
   9 # as published by the Free Software Foundation; either version 2
  10 # of the License, or (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  20 # 02110-1301, USA.
  21 #
  22
  23 import errno
  24 import optparse
  25 import os
  26 import shutil
  27 import subprocess
  28 import sys
  29 import tempfile
  30
  31 from giscanner import message
  32 from giscanner.annotationparser import GtkDocCommentBlockParser
  33 from giscanner.ast import Include, Namespace
  34 from giscanner.dumper import compile_introspection_binary
  35 from giscanner.gdumpparser import GDumpParser, IntrospectionBinary
  36 from giscanner.introspectablepass import IntrospectablePass
  37 from giscanner.girparser import GIRParser
  38 from giscanner.girwriter import GIRWriter
  39 from giscanner.maintransformer import MainTransformer
  40 from giscanner.shlibs import resolve_shlibs
  41 from giscanner.sourcescanner import SourceScanner, ALL_EXTS
  42 from giscanner.transformer import Transformer
  43 from . import utils
  44
  45
  46 def process_cflags_begin(option, opt, value, parser):
  47     cflags = getattr(parser.values, option.dest)
  48     while len(parser.rargs) > 0 and parser.rargs[0] != '--cflags-end':
  49         arg = parser.rargs.pop(0)
  50         if arg == "-I" and parser.rargs and parser.rargs[0] != '--cflags-end':
  51             # This is a special case where there's a space between -I and the path.
  52             arg += parser.rargs.pop(0)
  53         cflags.append(utils.cflag_real_include_path(arg))
  54
  55
  56 def process_cflags_end(option, opt, value, parser):
  57     pass
  58
  59
  60 def process_cpp_includes(option, opt, value, parser):
  61     cpp_includes = getattr(parser.values, option.dest)
  62     cpp_includes.append(os.path.realpath(value))
  63
  64
  65 def get_preprocessor_option_group(parser):
  66     group = optparse.OptionGroup(parser, "Preprocessor options")
  67     group.add_option("", "--cflags-begin",
  68                      help="Start preprocessor/compiler flags",
  69                      dest="cflags", default=[],
  70                      action="callback", callback=process_cflags_begin)
  71     group.add_option("", "--cflags-end",
  72                      help="End preprocessor/compiler flags",
  73                      action="callback", callback=process_cflags_end)
  74     group.add_option("-I", help="Pre-processor include file",
  75                      dest="cpp_includes", default=[], type="string",
  76                      action="callback", callback=process_cpp_includes)
  77     group.add_option("-D", help="Pre-processor define",
  78                      action="append", dest="cpp_defines",
  79                      default=[])
  80     group.add_option("-U", help="Pre-processor undefine",
  81                      action="append", dest="cpp_undefines",
  82                      default=[])
  83     group.add_option("-p", dest="", help="Ignored")
  84     return group
  85
  86
  87 def get_windows_option_group(parser):
  88     group = optparse.OptionGroup(parser, "Machine Dependent Options")
  89     group.add_option("-m", help="some machine dependent option",
  90                      action="append", dest='m_option',
  91                      default=[])
  92
  93     return group
  94
  95
  96 def _get_option_parser():
  97     parser = optparse.OptionParser('%prog [options] sources')
  98     parser.add_option('', "--quiet",
  99                       action="store_true", dest="quiet",
 100                       default=False,
 101                       help="If passed, do not print details of normal operation")
 102     parser.add_option("", "--format",
 103                       action="store", dest="format",
 104                       default="gir",
 105                       help="format to use, one of gidl, gir")
 106     parser.add_option("-i", "--include",
 107                       action="append", dest="includes", default=[],
 108                       help="Add specified gir file as dependency")
 109     parser.add_option("", "--include-uninstalled",
 110                       action="append", dest="includes_uninstalled", default=[],
 111                       help=("""A file path to a dependency; only use this "
 112                             "when building multiple .gir files inside a "
 113                             "single module."""))
 114     parser.add_option("", "--add-include-path",
 115                       action="append", dest="include_paths", default=[],
 116                       help="include paths for other GIR files")
 117     parser.add_option("", "--program",
 118                       action="store", dest="program", default=None,
 119                       help="program to execute")
 120     parser.add_option("", "--program-arg",
 121                       action="append", dest="program_args", default=[],
 122                       help="extra arguments to program")
 123     parser.add_option("", "--libtool",
 124                       action="store", dest="libtool_path", default=None,
 125                       help="full path to libtool")
 126     parser.add_option("", "--no-libtool",
 127                       action="store_true", dest="nolibtool", default=False,
 128                       help="do not use libtool")
 129     parser.add_option("", "--external-library",
 130                       action="store_true", dest="external_library", default=False,
 131                       help=("""If true, the library is located on the system,""" +
 132                             """not in the current directory"""))
 133     parser.add_option("-l", "--library",
 134                       action="append", dest="libraries", default=[],
 135                       help="libraries of this unit")
 136     parser.add_option("-L", "--library-path",
 137                       action="append", dest="library_paths", default=[],
 138                       help="directories to search for libraries")
 139     parser.add_option("", "--header-only",
 140                       action="store_true", dest="header_only", default=[],
 141                       help="If specified, just generate a GIR for the given header files")
 142     parser.add_option("-n", "--namespace",
 143                       action="store", dest="namespace_name",
 144                       help=("name of namespace for this unit, also "
 145                             "used to compute --identifier-prefix and --symbol-prefix"))
 146     parser.add_option("", "--nsversion",
 147                       action="store", dest="namespace_version",
 148                       help="version of namespace for this unit")
 149     parser.add_option("", "--strip-prefix",
 150                       action="store", dest="strip_prefix",
 151                       help="""Option --strip-prefix is deprecated, please see --identifier-prefix
 152 and --symbol-prefix.""")
 153     parser.add_option("", "--identifier-prefix",
 154                       action="append", dest="identifier_prefixes", default=[],
 155                       help="""Remove this prefix from C identifiers (structure typedefs, etc.).
 156 May be specified multiple times.  This is also used as the default for --symbol-prefix if
 157 the latter is not specified.""")
 158     parser.add_option("", "--symbol-prefix",
 159                       action="append", dest="symbol_prefixes", default=[],
 160                       help="Remove this prefix from C symbols (function names)")
 161     parser.add_option("", "--accept-unprefixed",
 162                       action="store_true", dest="accept_unprefixed", default=False,
 163                       help="""If specified, accept symbols and identifiers that do not
 164 match the namespace prefix.""")
 165     parser.add_option("", "--add-init-section",
 166                       action="append", dest="init_sections", default=[],
 167             help="add extra initialization code in the introspection program")
 168     parser.add_option("-o", "--output",
 169                       action="store", dest="output", default="-",
 170                       help="output filename to write to, defaults to - (stdout)")
 171     parser.add_option("", "--pkg",
 172                       action="append", dest="packages", default=[],
 173                       help="pkg-config packages to get cflags from")
 174     parser.add_option("", "--pkg-export",
 175                       action="append", dest="packages_export", default=[],
 176                       help="Associated pkg-config packages for this library")
 177     parser.add_option('', "--warn-all",
 178                       action="store_true", dest="warn_all", default=False,
 179                       help="If true, enable all warnings for introspection")
 180     parser.add_option('', "--warn-error",
 181                       action="store_true", dest="warn_fatal",
 182                       help="Turn warnings into fatal errors")
 183     parser.add_option("-v", "--verbose",
 184                       action="store_true", dest="verbose",
 185                       help="be verbose")
 186     parser.add_option("", "--c-include",
 187                       action="append", dest="c_includes", default=[],
 188                       help="headers which should be included in C programs")
 189     parser.add_option("", "--filelist",
 190                       action="store", dest="filelist", default=[],
 191                       help="file containing headers and sources to be scanned")
 192
 193     group = get_preprocessor_option_group(parser)
 194     parser.add_option_group(group)
 195
 196     if os.environ.get('MSYSTEM') == 'MINGW32':
 197         group = get_windows_option_group(parser)
 198         parser.add_option_group(group)
 199
 200     # Private options
 201     parser.add_option('', "--generate-typelib-tests",
 202                       action="store", dest="test_codegen", default=None,
 203                       help=optparse.SUPPRESS_HELP)
 204     parser.add_option('', "--passthrough-gir",
 205                       action="store", dest="passthrough_gir", default=None,
 206                       help=optparse.SUPPRESS_HELP)
 207     parser.add_option('', "--reparse-validate",
 208                       action="store_true", dest="reparse_validate_gir", default=False,
 209                       help=optparse.SUPPRESS_HELP)
 210     parser.add_option("", "--typelib-xml",
 211                       action="store_true", dest="typelib_xml",
 212                       help=optparse.SUPPRESS_HELP)
 213
 214     return parser
 215
 216
 217 def _error(msg):
 218     raise SystemExit('ERROR: %s' % (msg, ))
 219
 220
 221 def passthrough_gir(path, f):
 222     parser = GIRParser()
 223     parser.parse(path)
 224
 225     writer = GIRWriter(parser.get_namespace())
 226     f.write(writer.get_xml())
 227
 228
 229 def test_codegen(optstring):
 230     (namespace, out_h_filename, out_c_filename) = optstring.split(',')
 231     if namespace == 'Everything':
 232         from .testcodegen import EverythingCodeGenerator
 233         gen = EverythingCodeGenerator(out_h_filename, out_c_filename)
 234         gen.write()
 235     else:
 236         _error("Invaild namespace %r" % (namespace, ))
 237     return 0
 238
 239
 240 def process_options(output, allowed_flags):
 241     for option in output.split():
 242         for flag in allowed_flags:
 243             if not option.startswith(flag):
 244                 continue
 245             yield option
 246             break
 247
 248
 249 def process_packages(options, packages):
 250     args = ['pkg-config', '--cflags']
 251     args.extend(packages)
 252     output = subprocess.Popen(args,
 253                               stdout=subprocess.PIPE).communicate()[0]
 254     if output is None:
 255         # the error output should have already appeared on our stderr,
 256         # so we just exit
 257         return 1
 258     # Some pkg-config files on Windows have options we don't understand,
 259     # so we explicitly filter to only the ones we need.
 260     options_whitelist = ['-I', '-D', '-U', '-l', '-L']
 261     filtered_output = list(process_options(output, options_whitelist))
 262     parser = _get_option_parser()
 263     pkg_options, unused = parser.parse_args(filtered_output)
 264     options.cpp_includes.extend([os.path.realpath(f) for f in pkg_options.cpp_includes])
 265     options.cpp_defines.extend(pkg_options.cpp_defines)
 266     options.cpp_undefines.extend(pkg_options.cpp_undefines)
 267
 268
 269 def extract_filenames(args):
 270     filenames = []
 271     for arg in args:
 272         # We don't support real C++ parsing yet, but we should be able
 273         # to understand C API implemented in C++ files.
 274         if os.path.splitext(arg)[1] in ALL_EXTS:
 275             if not os.path.exists(arg):
 276                 _error('%s: no such a file or directory' % (arg, ))
 277             # Make absolute, because we do comparisons inside scannerparser.c
 278             # against the absolute path that cpp will give us
 279             filenames.append(arg)
 280     return filenames
 281
 282
 283 def extract_filelist(options):
 284     filenames = []
 285     if not os.path.exists(options.filelist):
 286         _error('%s: no such filelist file' % (options.filelist, ))
 287     filelist_file = open(options.filelist, "r")
 288     lines = filelist_file.readlines()
 289     for line in lines:
 290         # We don't support real C++ parsing yet, but we should be able
 291         # to understand C API implemented in C++ files.
 292         filename = line.strip()
 293         if (filename.endswith('.c') or filename.endswith('.cpp')
 294         or filename.endswith('.cc') or filename.endswith('.cxx')
 295         or filename.endswith('.h') or filename.endswith('.hpp')
 296         or filename.endswith('.hxx')):
 297             if not os.path.exists(filename):
 298                 _error('%s: Invalid filelist entry-no such file or directory' % (line, ))
 299             # Make absolute, because we do comparisons inside scannerparser.c
 300             # against the absolute path that cpp will give us
 301             filenames.append(filename)
 302     return filenames
 303
 304
 305 def create_namespace(options):
 306     if options.strip_prefix:
 307         print """g-ir-scanner: warning: Option --strip-prefix has been deprecated;
 308 see --identifier-prefix and --symbol-prefix."""
 309         options.identifier_prefixes.append(options.strip_prefix)
 310
 311     # We do this dance because the empty list has different semantics from
 312     # None; if the user didn't specify the options, we want to use None so
 313     # the Namespace constructor picks the defaults.
 314     if options.identifier_prefixes:
 315         identifier_prefixes = options.identifier_prefixes
 316     else:
 317         identifier_prefixes = None
 318     if options.symbol_prefixes:
 319         for prefix in options.symbol_prefixes:
 320             # See Transformer._split_c_string_for_namespace_matches() for
 321             # why this check is needed
 322             if prefix.lower() != prefix:
 323                 _error("Values for --symbol-prefix must be entirely lowercase")
 324         symbol_prefixes = options.symbol_prefixes
 325     else:
 326         symbol_prefixes = None
 327
 328     return Namespace(options.namespace_name,
 329                      options.namespace_version,
 330                      identifier_prefixes=identifier_prefixes,
 331                      symbol_prefixes=symbol_prefixes)
 332
 333
 334 def create_transformer(namespace, options):
 335     transformer = Transformer(namespace,
 336                               accept_unprefixed=options.accept_unprefixed)
 337     transformer.set_include_paths(options.include_paths)
 338     if options.passthrough_gir:
 339         transformer.disable_cache()
 340         transformer.set_passthrough_mode()
 341
 342     for include in options.includes:
 343         if os.sep in include:
 344             _error("Invalid include path %r" % (include, ))
 345         try:
 346             include_obj = Include.from_string(include)
 347         except:
 348             _error("Malformed include %r\n" % (include, ))
 349         transformer.register_include(include_obj)
 350     for include_path in options.includes_uninstalled:
 351         transformer.register_include_uninstalled(include_path)
 352
 353     return transformer
 354
 355
 356 def create_binary(transformer, options, args):
 357     # Transform the C AST nodes into higher level
 358     # GLib/GObject nodes
 359     gdump_parser = GDumpParser(transformer)
 360
 361     # Do enough parsing that we have the get_type() functions to reference
 362     # when creating the introspection binary
 363     gdump_parser.init_parse()
 364
 365     if options.program:
 366         args = [options.program]
 367         args.extend(options.program_args)
 368         binary = IntrospectionBinary(args)
 369     else:
 370         binary = compile_introspection_binary(options,
 371                                               gdump_parser.get_get_type_functions(),
 372                                               gdump_parser.get_error_quark_functions())
 373
 374     shlibs = resolve_shlibs(options, binary, options.libraries)
 375     gdump_parser.set_introspection_binary(binary)
 376     gdump_parser.parse()
 377     return shlibs
 378
 379
 380 def create_source_scanner(options, args):
 381     if hasattr(options, 'filelist') and options.filelist:
 382         filenames = extract_filelist(options)
 383     else:
 384         filenames = extract_filenames(args)
 385
 386     # Run the preprocessor, tokenize and construct simple
 387     # objects representing the raw C symbols
 388     ss = SourceScanner()
 389     ss.set_cpp_options(options.cpp_includes,
 390                        options.cpp_defines,
 391                        options.cpp_undefines,
 392                        cflags=options.cflags)
 393     ss.parse_files(filenames)
 394     ss.parse_macros(filenames)
 395     return ss
 396
 397
 398 def write_output(data, options):
 399     if options.output == "-":
 400         output = sys.stdout
 401     elif options.reparse_validate_gir:
 402         main_f, main_f_name = tempfile.mkstemp(suffix='.gir')
 403         main_f = os.fdopen(main_f, 'w')
 404         main_f.write(data)
 405         main_f.close()
 406
 407         temp_f, temp_f_name = tempfile.mkstemp(suffix='.gir')
 408         temp_f = os.fdopen(temp_f, 'w')
 409         passthrough_gir(main_f_name, temp_f)
 410         temp_f.close()
 411         if not utils.files_are_identical(main_f_name, temp_f_name):
 412             _error("Failed to re-parse gir file; scanned=%r passthrough=%r" % (
 413                 main_f_name, temp_f_name))
 414         os.unlink(temp_f_name)
 415         try:
 416             shutil.move(main_f_name, options.output)
 417         except OSError as e:
 418             if e.errno == errno.EPERM:
 419                 os.unlink(main_f_name)
 420                 return 0
 421             raise
 422         return 0
 423     else:
 424         try:
 425             output = open(options.output, "w")
 426         except IOError as e:
 427             _error("opening output for writing: %s" % (e.strerror, ))
 428
 429     try:
 430         output.write(data)
 431     except IOError as e:
 432         _error("while writing output: %s" % (e.strerror, ))
 433
 434
 435 def scanner_main(args):
 436     parser = _get_option_parser()
 437     (options, args) = parser.parse_args(args)
 438
 439     if options.passthrough_gir:
 440         passthrough_gir(options.passthrough_gir, sys.stdout)
 441     if options.test_codegen:
 442         return test_codegen(options.test_codegen)
 443
 444     if hasattr(options, 'filelist') and not options.filelist:
 445         if len(args) <= 1:
 446             _error('Need at least one filename')
 447
 448     if not options.namespace_name:
 449         _error('Namespace name missing')
 450
 451     if options.format == 'gir':
 452         from giscanner.girwriter import GIRWriter as Writer
 453     else:
 454         _error("Unknown format: %s" % (options.format, ))
 455
 456     if not (options.libraries
 457             or options.program
 458             or options.header_only):
 459         _error("Must specify --program or --library")
 460
 461     namespace = create_namespace(options)
 462     logger = message.MessageLogger.get(namespace=namespace)
 463     if options.warn_all:
 464         logger.enable_warnings((message.WARNING, message.ERROR, message.FATAL))
 465
 466     transformer = create_transformer(namespace, options)
 467
 468     packages = set(options.packages)
 469     packages.update(transformer.get_pkgconfig_packages())
 470     if packages:
 471         exit_code = process_packages(options, packages)
 472         if exit_code:
 473             return exit_code
 474
 475     ss = create_source_scanner(options, args)
 476
 477     cbp = GtkDocCommentBlockParser()
 478     blocks = cbp.parse_comment_blocks(ss.get_comments())
 479
 480     # Transform the C symbols into AST nodes
 481     transformer.parse(ss.get_symbols())
 482
 483     if not options.header_only:
 484         shlibs = create_binary(transformer, options, args)
 485     else:
 486         shlibs = []
 487
 488     transformer.namespace.shared_libraries = shlibs
 489
 490     main = MainTransformer(transformer, blocks)
 491     main.transform()
 492
 493     utils.break_on_debug_flag('tree')
 494
 495     final = IntrospectablePass(transformer, blocks)
 496     final.validate()
 497
 498     warning_count = logger.get_warning_count()
 499     if options.warn_fatal and warning_count > 0:
 500         message.fatal("warnings configured as fatal")
 501         return 1
 502     elif warning_count > 0 and options.warn_all is False:
 503         print ("g-ir-scanner: %s: warning: %d warnings suppressed (use --warn-all to see them)"
 504                % (transformer.namespace.name, warning_count, ))
 505
 506     # Write out AST
 507     if options.packages_export:
 508         exported_packages = options.packages_export
 509     else:
 510         exported_packages = options.packages
 511
 512     transformer.namespace.c_includes = options.c_includes
 513     transformer.namespace.exported_packages = exported_packages
 514     writer = Writer(transformer.namespace)
 515     data = writer.get_xml()
 516
 517     write_output(data, options)
 518
 519     return 0