setupinfo.py

   1 import sys
   2 import io
   3 import os
   4 import os.path
   5 from distutils.core import Extension
   6 from distutils.errors import CompileError, DistutilsOptionError
   7 from distutils.command.build_ext import build_ext as _build_ext
   8 from versioninfo import get_base_dir
   9
  10 try:
  11     import Cython.Compiler.Version
  12     CYTHON_INSTALLED = True
  13 except ImportError:
  14     CYTHON_INSTALLED = False
  15
  16 EXT_MODULES = ["lxml.etree", "lxml.objectify"]
  17 COMPILED_MODULES = [
  18     "lxml.builder",
  19     "lxml._elementpath",
  20     "lxml.html.diff",
  21     "lxml.html.clean",
  22     "lxml.sax",
  23 ]
  24 HEADER_FILES = ['etree.h', 'etree_api.h']
  25
  26 if hasattr(sys, 'pypy_version_info') or (
  27         getattr(sys, 'implementation', None) and sys.implementation.name != 'cpython'):
  28     # disable Cython compilation of Python modules in PyPy and other non-CPythons
  29     del COMPILED_MODULES[:]
  30
  31 SOURCE_PATH = "src"
  32 INCLUDE_PACKAGE_PATH = os.path.join(SOURCE_PATH, 'lxml', 'includes')
  33
  34 if sys.version_info[0] >= 3:
  35     _system_encoding = sys.getdefaultencoding()
  36     if _system_encoding is None:
  37         _system_encoding = "iso-8859-1" # :-)
  38     def decode_input(data):
  39         if isinstance(data, str):
  40             return data
  41         return data.decode(_system_encoding)
  42 else:
  43     def decode_input(data):
  44         return data
  45
  46 def env_var(name):
  47     value = os.getenv(name)
  48     if value:
  49         value = decode_input(value)
  50         if sys.platform == 'win32' and ';' in value:
  51             return value.split(';')
  52         else:
  53             return value.split()
  54     else:
  55         return []
  56
  57
  58 def _prefer_reldirs(base_dir, dirs):
  59     return [
  60         os.path.relpath(path) if path.startswith(base_dir) else path
  61         for path in dirs
  62     ]
  63
  64 def ext_modules(static_include_dirs, static_library_dirs,
  65                 static_cflags, static_binaries):
  66     global XML2_CONFIG, XSLT_CONFIG
  67     if OPTION_BUILD_LIBXML2XSLT:
  68         from buildlibxml import build_libxml2xslt, get_prebuilt_libxml2xslt
  69         if sys.platform.startswith('win'):
  70             get_prebuilt_libxml2xslt(
  71                 OPTION_DOWNLOAD_DIR, static_include_dirs, static_library_dirs)
  72         else:
  73             XML2_CONFIG, XSLT_CONFIG = build_libxml2xslt(
  74                 OPTION_DOWNLOAD_DIR, 'build/tmp',
  75                 static_include_dirs, static_library_dirs,
  76                 static_cflags, static_binaries,
  77                 libiconv_version=OPTION_LIBICONV_VERSION,
  78                 libxml2_version=OPTION_LIBXML2_VERSION,
  79                 libxslt_version=OPTION_LIBXSLT_VERSION,
  80                 zlib_version=OPTION_ZLIB_VERSION,
  81                 multicore=OPTION_MULTICORE)
  82
  83     modules = EXT_MODULES + COMPILED_MODULES
  84     if OPTION_WITHOUT_OBJECTIFY:
  85         modules = [entry for entry in modules if 'objectify' not in entry]
  86
  87     module_files = list(os.path.join(SOURCE_PATH, *module.split('.')) for module in modules)
  88     c_files_exist = [os.path.exists(module + '.c') for module in module_files]
  89
  90     use_cython = True
  91     if CYTHON_INSTALLED and (OPTION_WITH_CYTHON or not all(c_files_exist)):
  92         print("Building with Cython %s." % Cython.Compiler.Version.version)
  93         # generate module cleanup code
  94         from Cython.Compiler import Options
  95         Options.generate_cleanup_code = 3
  96         Options.clear_to_none = False
  97     elif not OPTION_WITHOUT_CYTHON and not all(c_files_exist):
  98         for exists, module in zip(c_files_exist, module_files):
  99             if not exists:
 100                 raise RuntimeError(
 101                     "ERROR: Trying to build without Cython, but pre-generated '%s.c' "
 102                     "is not available (pass --without-cython to ignore this error)." % module)
 103     else:
 104         if not all(c_files_exist):
 105             for exists, module in zip(c_files_exist, module_files):
 106                 if not exists:
 107                     print("WARNING: Trying to build without Cython, but pre-generated "
 108                           "'%s.c' is not available." % module)
 109         use_cython = False
 110         print("Building without Cython.")
 111
 112     lib_versions = get_library_versions()
 113     versions_ok = True
 114     if lib_versions[0]:
 115         print("Using build configuration of libxml2 %s and libxslt %s" %
 116               lib_versions)
 117         versions_ok = check_min_version(lib_versions[0], (2, 7, 0), 'libxml2')
 118     else:
 119         print("Using build configuration of libxslt %s" %
 120               lib_versions[1])
 121     versions_ok |= check_min_version(lib_versions[1], (1, 1, 23), 'libxslt')
 122     if not versions_ok:
 123         raise RuntimeError("Dependency missing")
 124
 125     base_dir = get_base_dir()
 126     _include_dirs = _prefer_reldirs(
 127         base_dir, include_dirs(static_include_dirs) + [
 128             SOURCE_PATH,
 129             INCLUDE_PACKAGE_PATH,
 130         ])
 131     _library_dirs = _prefer_reldirs(base_dir, library_dirs(static_library_dirs))
 132     _cflags = cflags(static_cflags)
 133     _ldflags = ['-isysroot', get_xcode_isysroot()] if sys.platform == 'darwin' else None
 134     _define_macros = define_macros()
 135     _libraries = libraries()
 136
 137     if _library_dirs:
 138         message = "Building against libxml2/libxslt in "
 139         if len(_library_dirs) > 1:
 140             print(message + "one of the following directories:")
 141             for dir in _library_dirs:
 142                 print("  " + dir)
 143         else:
 144             print(message + "the following directory: " +
 145                   _library_dirs[0])
 146
 147     if OPTION_AUTO_RPATH:
 148         runtime_library_dirs = _library_dirs
 149     else:
 150         runtime_library_dirs = []
 151
 152     if CYTHON_INSTALLED and OPTION_SHOW_WARNINGS:
 153         from Cython.Compiler import Errors
 154         Errors.LEVEL = 0
 155
 156     cythonize_directives = {
 157         'binding': True,
 158     }
 159     if OPTION_WITH_COVERAGE:
 160         cythonize_directives['linetrace'] = True
 161
 162     result = []
 163     for module, src_file in zip(modules, module_files):
 164         is_py = module in COMPILED_MODULES
 165         main_module_source = src_file + (
 166             '.c' if not use_cython else '.py' if is_py else '.pyx')
 167         result.append(
 168             Extension(
 169                 module,
 170                 sources = [main_module_source],
 171                 depends = find_dependencies(module),
 172                 extra_compile_args = _cflags,
 173                 extra_link_args = None if is_py else _ldflags,
 174                 extra_objects = None if is_py else static_binaries,
 175                 define_macros = _define_macros,
 176                 include_dirs = _include_dirs,
 177                 library_dirs = None if is_py else _library_dirs,
 178                 runtime_library_dirs = None if is_py else runtime_library_dirs,
 179                 libraries = None if is_py else _libraries,
 180             ))
 181     if CYTHON_INSTALLED and OPTION_WITH_CYTHON_GDB:
 182         for ext in result:
 183             ext.cython_gdb = True
 184
 185     if CYTHON_INSTALLED and use_cython:
 186         # build .c files right now and convert Extension() objects
 187         from Cython.Build import cythonize
 188         result = cythonize(result, compiler_directives=cythonize_directives)
 189
 190     # for backwards compatibility reasons, provide "etree[_api].h" also as "lxml.etree[_api].h"
 191     for header_filename in HEADER_FILES:
 192         src_file = os.path.join(SOURCE_PATH, 'lxml', header_filename)
 193         dst_file = os.path.join(SOURCE_PATH, 'lxml', 'lxml.' + header_filename)
 194         if not os.path.exists(src_file):
 195             continue
 196         if os.path.exists(dst_file) and os.path.getmtime(dst_file) >= os.path.getmtime(src_file):
 197             continue
 198
 199         with io.open(src_file, 'r', encoding='iso8859-1') as f:
 200             content = f.read()
 201         for filename in HEADER_FILES:
 202             content = content.replace('"%s"' % filename, '"lxml.%s"' % filename)
 203         with io.open(dst_file, 'w', encoding='iso8859-1') as f:
 204             f.write(content)
 205
 206     return result
 207
 208
 209 def find_dependencies(module):
 210     if not CYTHON_INSTALLED or 'lxml.html' in module:
 211         return []
 212     base_dir = get_base_dir()
 213     package_dir = os.path.join(base_dir, SOURCE_PATH, 'lxml')
 214     includes_dir = os.path.join(base_dir, INCLUDE_PACKAGE_PATH)
 215
 216     pxd_files = [
 217         os.path.join(INCLUDE_PACKAGE_PATH, filename)
 218         for filename in os.listdir(includes_dir)
 219         if filename.endswith('.pxd')
 220     ]
 221
 222     if module == 'lxml.etree':
 223         pxi_files = [
 224             os.path.join(SOURCE_PATH, 'lxml', filename)
 225             for filename in os.listdir(package_dir)
 226             if filename.endswith('.pxi') and 'objectpath' not in filename
 227         ]
 228         pxd_files = [
 229             filename for filename in pxd_files
 230             if 'etreepublic' not in filename
 231         ]
 232     elif module == 'lxml.objectify':
 233         pxi_files = [os.path.join(SOURCE_PATH, 'lxml', 'objectpath.pxi')]
 234     else:
 235         pxi_files = pxd_files = []
 236
 237     return pxd_files + pxi_files
 238
 239
 240 def extra_setup_args():
 241     class CheckLibxml2BuildExt(_build_ext):
 242         """Subclass to check whether libxml2 is really available if the build fails"""
 243         def run(self):
 244             try:
 245                 _build_ext.run(self)  # old-style class in Py2
 246             except CompileError as e:
 247                 print('Compile failed: %s' % e)
 248                 if not seems_to_have_libxml2():
 249                     print_libxml_error()
 250                 raise
 251     result = {'cmdclass': {'build_ext': CheckLibxml2BuildExt}}
 252     return result
 253
 254
 255 def seems_to_have_libxml2():
 256     from distutils import ccompiler
 257     compiler = ccompiler.new_compiler()
 258     return compiler.has_function(
 259         'xmlXPathInit',
 260         include_dirs=include_dirs([]) + ['/usr/include/libxml2'],
 261         includes=['libxml/xpath.h'],
 262         library_dirs=library_dirs([]),
 263         libraries=['xml2'])
 264
 265
 266 def print_libxml_error():
 267     print('*********************************************************************************')
 268     print('Could not find function xmlCheckVersion in library libxml2. Is libxml2 installed?')
 269     if sys.platform in ('darwin',):
 270         print('Perhaps try: xcode-select --install')
 271     print('*********************************************************************************')
 272
 273
 274 def libraries():
 275     standard_libs = []
 276     if 'linux' in sys.platform:
 277         standard_libs.append('rt')
 278     if not OPTION_BUILD_LIBXML2XSLT:
 279         standard_libs.append('z')
 280     standard_libs.append('m')
 281
 282     if sys.platform in ('win32',):
 283         libs = ['libxslt', 'libexslt', 'libxml2', 'iconv']
 284         if OPTION_STATIC:
 285             libs = ['%s_a' % lib for lib in libs]
 286         libs.extend(['zlib', 'WS2_32'])
 287     elif OPTION_STATIC:
 288         libs = standard_libs
 289     else:
 290         libs = ['xslt', 'exslt', 'xml2'] + standard_libs
 291     return libs
 292
 293 def library_dirs(static_library_dirs):
 294     if OPTION_STATIC:
 295         if not static_library_dirs:
 296             static_library_dirs = env_var('LIBRARY')
 297         assert static_library_dirs, "Static build not configured, see doc/build.txt"
 298         return static_library_dirs
 299     # filter them from xslt-config --libs
 300     result = []
 301     possible_library_dirs = flags('libs')
 302     for possible_library_dir in possible_library_dirs:
 303         if possible_library_dir.startswith('-L'):
 304             result.append(possible_library_dir[2:])
 305     return result
 306
 307 def include_dirs(static_include_dirs):
 308     if OPTION_STATIC:
 309         if not static_include_dirs:
 310             static_include_dirs = env_var('INCLUDE')
 311         return static_include_dirs
 312     # filter them from xslt-config --cflags
 313     result = []
 314     possible_include_dirs = flags('cflags')
 315     for possible_include_dir in possible_include_dirs:
 316         if possible_include_dir.startswith('-I'):
 317             result.append(possible_include_dir[2:])
 318     return result
 319
 320 def cflags(static_cflags):
 321     result = []
 322     if not OPTION_SHOW_WARNINGS:
 323         result.append('-w')
 324     if OPTION_DEBUG_GCC:
 325         result.append('-g2')
 326
 327     if OPTION_STATIC:
 328         if not static_cflags:
 329             static_cflags = env_var('CFLAGS')
 330         result.extend(static_cflags)
 331     else:
 332         # anything from xslt-config --cflags that doesn't start with -I
 333         possible_cflags = flags('cflags')
 334         for possible_cflag in possible_cflags:
 335             if not possible_cflag.startswith('-I'):
 336                 result.append(possible_cflag)
 337
 338     if sys.platform in ('darwin',):
 339         for opt in result:
 340             if 'flat_namespace' in opt:
 341                 break
 342         else:
 343             result.append('-flat_namespace')
 344
 345     return result
 346
 347 def define_macros():
 348     macros = []
 349     if OPTION_WITHOUT_ASSERT:
 350         macros.append(('PYREX_WITHOUT_ASSERTIONS', None))
 351     if OPTION_WITHOUT_THREADING:
 352         macros.append(('WITHOUT_THREADING', None))
 353     if OPTION_WITH_REFNANNY:
 354         macros.append(('CYTHON_REFNANNY', None))
 355     if OPTION_WITH_UNICODE_STRINGS:
 356         macros.append(('LXML_UNICODE_STRINGS', '1'))
 357     if OPTION_WITH_COVERAGE:
 358         macros.append(('CYTHON_TRACE_NOGIL', '1'))
 359     # Disable showing C lines in tracebacks, unless explicitly requested.
 360     macros.append(('CYTHON_CLINE_IN_TRACEBACK', '1' if OPTION_WITH_CLINES else '0'))
 361     return macros
 362
 363 _ERROR_PRINTED = False
 364
 365 def run_command(cmd, *args):
 366     if not cmd:
 367         return ''
 368     if args:
 369         cmd = ' '.join((cmd,) + args)
 370     import subprocess
 371     p = subprocess.Popen(cmd, shell=True,
 372                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 373     stdout_data, errors = p.communicate()
 374     global _ERROR_PRINTED
 375     if errors and not _ERROR_PRINTED:
 376         _ERROR_PRINTED = True
 377         print("ERROR: %s" % errors)
 378         print("** make sure the development packages of libxml2 and libxslt are installed **\n")
 379     return decode_input(stdout_data).strip()
 380
 381
 382 def check_min_version(version, min_version, error_name):
 383     if not version:
 384         # this is ok for targets like sdist etc.
 385         return True
 386     version = tuple(map(int, version.split('.')[:3]))
 387     min_version = tuple(min_version)
 388     if version < min_version:
 389         print("Minimum required version of %s is %s, found %s" % (
 390             error_name, '.'.join(map(str, version)), '.'.join(map(str, min_version))))
 391         return False
 392     return True
 393
 394
 395 def get_library_version(config_tool):
 396     is_pkgconfig = "pkg-config" in config_tool
 397     return run_command(config_tool,
 398                        "--modversion" if is_pkgconfig else "--version")
 399
 400
 401 def get_library_versions():
 402     xml2_version = get_library_version(find_xml2_config())
 403     xslt_version = get_library_version(find_xslt_config())
 404     return xml2_version, xslt_version
 405
 406
 407 def flags(option):
 408     xml2_flags = run_command(find_xml2_config(), "--%s" % option)
 409     xslt_flags = run_command(find_xslt_config(), "--%s" % option)
 410
 411     flag_list = xml2_flags.split()
 412     for flag in xslt_flags.split():
 413         if flag not in flag_list:
 414             flag_list.append(flag)
 415     return flag_list
 416
 417
 418 def get_xcode_isysroot():
 419     return run_command('xcrun', '--show-sdk-path')
 420
 421 XSLT_CONFIG = None
 422 XML2_CONFIG = None
 423
 424 def find_xml2_config():
 425     global XML2_CONFIG
 426     if XML2_CONFIG:
 427         return XML2_CONFIG
 428     option = '--with-xml2-config='
 429     for arg in sys.argv:
 430         if arg.startswith(option):
 431             sys.argv.remove(arg)
 432             XML2_CONFIG = arg[len(option):]
 433             return XML2_CONFIG
 434     else:
 435         # default: do nothing, rely only on xslt-config
 436         XML2_CONFIG = os.getenv('XML2_CONFIG', '')
 437     return XML2_CONFIG
 438
 439 def find_xslt_config():
 440     global XSLT_CONFIG
 441     if XSLT_CONFIG:
 442         return XSLT_CONFIG
 443     option = '--with-xslt-config='
 444     for arg in sys.argv:
 445         if arg.startswith(option):
 446             sys.argv.remove(arg)
 447             XSLT_CONFIG = arg[len(option):]
 448             return XSLT_CONFIG
 449     else:
 450         XSLT_CONFIG = os.getenv('XSLT_CONFIG', 'xslt-config')
 451     return XSLT_CONFIG
 452
 453 ## Option handling:
 454
 455 def has_option(name):
 456     try:
 457         sys.argv.remove('--%s' % name)
 458         return True
 459     except ValueError:
 460         pass
 461     # allow passing all cmd line options also as environment variables
 462     env_val = os.getenv(name.upper().replace('-', '_'), 'false').lower()
 463     if env_val == "true":
 464         return True
 465     return False
 466
 467 def option_value(name):
 468     for index, option in enumerate(sys.argv):
 469         if option == '--' + name:
 470             if index+1 >= len(sys.argv):
 471                 raise DistutilsOptionError(
 472                     'The option %s requires a value' % option)
 473             value = sys.argv[index+1]
 474             sys.argv[index:index+2] = []
 475             return value
 476         if option.startswith('--' + name + '='):
 477             value = option[len(name)+3:]
 478             sys.argv[index:index+1] = []
 479             return value
 480     env_val = os.getenv(name.upper().replace('-', '_'))
 481     return env_val
 482
 483 staticbuild = bool(os.environ.get('STATICBUILD', ''))
 484 # pick up any commandline options and/or env variables
 485 OPTION_WITHOUT_OBJECTIFY = has_option('without-objectify')
 486 OPTION_WITH_UNICODE_STRINGS = has_option('with-unicode-strings')
 487 OPTION_WITHOUT_ASSERT = has_option('without-assert')
 488 OPTION_WITHOUT_THREADING = has_option('without-threading')
 489 OPTION_WITHOUT_CYTHON = has_option('without-cython')
 490 OPTION_WITH_CYTHON = has_option('with-cython')
 491 OPTION_WITH_CYTHON_GDB = has_option('cython-gdb')
 492 OPTION_WITH_REFNANNY = has_option('with-refnanny')
 493 OPTION_WITH_COVERAGE = has_option('with-coverage')
 494 OPTION_WITH_CLINES = has_option('with-clines')
 495 if OPTION_WITHOUT_CYTHON:
 496     CYTHON_INSTALLED = False
 497 OPTION_STATIC = staticbuild or has_option('static')
 498 OPTION_DEBUG_GCC = has_option('debug-gcc')
 499 OPTION_SHOW_WARNINGS = has_option('warnings')
 500 OPTION_AUTO_RPATH = has_option('auto-rpath')
 501 OPTION_BUILD_LIBXML2XSLT = staticbuild or has_option('static-deps')
 502 if OPTION_BUILD_LIBXML2XSLT:
 503     OPTION_STATIC = True
 504 OPTION_LIBXML2_VERSION = option_value('libxml2-version')
 505 OPTION_LIBXSLT_VERSION = option_value('libxslt-version')
 506 OPTION_LIBICONV_VERSION = option_value('libiconv-version')
 507 OPTION_ZLIB_VERSION = option_value('zlib-version')
 508 OPTION_MULTICORE = option_value('multicore')
 509 OPTION_DOWNLOAD_DIR = option_value('download-dir')
 510 if OPTION_DOWNLOAD_DIR is None:
 511     OPTION_DOWNLOAD_DIR = 'libs'