setupinfo.py

   1 import sys
   2 import io
   3 import os
   4 import os.path
   5 import subprocess
   6 from distutils.core import Extension
   7 from distutils.errors import CompileError, DistutilsOptionError
   8 from distutils.command.build_ext import build_ext as _build_ext
   9 from versioninfo import get_base_dir
  10
  11 try:
  12     import Cython.Compiler.Version
  13     CYTHON_INSTALLED = True
  14 except ImportError:
  15     CYTHON_INSTALLED = False
  16
  17 EXT_MODULES = ["lxml.etree", "lxml.objectify"]
  18 COMPILED_MODULES = [
  19     "lxml.builder",
  20     "lxml._elementpath",
  21     "lxml.html.diff",
  22     "lxml.html.clean",
  23     "lxml.sax",
  24 ]
  25 HEADER_FILES = ['etree.h', 'etree_api.h']
  26
  27 if hasattr(sys, 'pypy_version_info') or (
  28         getattr(sys, 'implementation', None) and sys.implementation.name != 'cpython'):
  29     # disable Cython compilation of Python modules in PyPy and other non-CPythons
  30     del COMPILED_MODULES[:]
  31
  32 SOURCE_PATH = "src"
  33 INCLUDE_PACKAGE_PATH = os.path.join(SOURCE_PATH, 'lxml', 'includes')
  34
  35 if sys.version_info[0] >= 3:
  36     _system_encoding = sys.getdefaultencoding()
  37     if _system_encoding is None:
  38         _system_encoding = "iso-8859-1" # :-)
  39     def decode_input(data):
  40         if isinstance(data, str):
  41             return data
  42         return data.decode(_system_encoding)
  43 else:
  44     def decode_input(data):
  45         return data
  46
  47 def env_var(name):
  48     value = os.getenv(name)
  49     if value:
  50         value = decode_input(value)
  51         if sys.platform == 'win32' and ';' in value:
  52             return value.split(';')
  53         else:
  54             return value.split()
  55     else:
  56         return []
  57
  58
  59 def _prefer_reldirs(base_dir, dirs):
  60     return [
  61         os.path.relpath(path) if path.startswith(base_dir) else path
  62         for path in dirs
  63     ]
  64
  65 def ext_modules(static_include_dirs, static_library_dirs,
  66                 static_cflags, static_binaries):
  67     global XML2_CONFIG, XSLT_CONFIG
  68     if OPTION_BUILD_LIBXML2XSLT:
  69         from buildlibxml import build_libxml2xslt, get_prebuilt_libxml2xslt
  70         if sys.platform.startswith('win'):
  71             get_prebuilt_libxml2xslt(
  72                 OPTION_DOWNLOAD_DIR, static_include_dirs, static_library_dirs)
  73         else:
  74             XML2_CONFIG, XSLT_CONFIG = build_libxml2xslt(
  75                 OPTION_DOWNLOAD_DIR, 'build/tmp',
  76                 static_include_dirs, static_library_dirs,
  77                 static_cflags, static_binaries,
  78                 libiconv_version=OPTION_LIBICONV_VERSION,
  79                 libxml2_version=OPTION_LIBXML2_VERSION,
  80                 libxslt_version=OPTION_LIBXSLT_VERSION,
  81                 zlib_version=OPTION_ZLIB_VERSION,
  82                 multicore=OPTION_MULTICORE)
  83
  84     modules = EXT_MODULES + COMPILED_MODULES
  85     if OPTION_WITHOUT_OBJECTIFY:
  86         modules = [entry for entry in modules if 'objectify' not in entry]
  87
  88     module_files = list(os.path.join(SOURCE_PATH, *module.split('.')) for module in modules)
  89     c_files_exist = [os.path.exists(module + '.c') for module in module_files]
  90
  91     use_cython = True
  92     if CYTHON_INSTALLED and (OPTION_WITH_CYTHON or not all(c_files_exist)):
  93         print("Building with Cython %s." % Cython.Compiler.Version.version)
  94         # generate module cleanup code
  95         from Cython.Compiler import Options
  96         Options.generate_cleanup_code = 3
  97         Options.clear_to_none = False
  98     elif not OPTION_WITHOUT_CYTHON and not all(c_files_exist):
  99         for exists, module in zip(c_files_exist, module_files):
 100             if not exists:
 101                 raise RuntimeError(
 102                     "ERROR: Trying to build without Cython, but pre-generated '%s.c' "
 103                     "is not available (pass --without-cython to ignore this error)." % module)
 104     else:
 105         if not all(c_files_exist):
 106             for exists, module in zip(c_files_exist, module_files):
 107                 if not exists:
 108                     print("WARNING: Trying to build without Cython, but pre-generated "
 109                           "'%s.c' is not available." % module)
 110         use_cython = False
 111         print("Building without Cython.")
 112
 113     if not check_build_dependencies():
 114         raise RuntimeError("Dependency missing")
 115
 116     base_dir = get_base_dir()
 117     _include_dirs = _prefer_reldirs(
 118         base_dir, include_dirs(static_include_dirs) + [
 119             SOURCE_PATH,
 120             INCLUDE_PACKAGE_PATH,
 121         ])
 122     _library_dirs = _prefer_reldirs(base_dir, library_dirs(static_library_dirs))
 123     _cflags = cflags(static_cflags)
 124     _ldflags = ['-isysroot', get_xcode_isysroot()] if sys.platform == 'darwin' else None
 125     _define_macros = define_macros()
 126     _libraries = libraries()
 127
 128     if _library_dirs:
 129         message = "Building against libxml2/libxslt in "
 130         if len(_library_dirs) > 1:
 131             print(message + "one of the following directories:")
 132             for dir in _library_dirs:
 133                 print("  " + dir)
 134         else:
 135             print(message + "the following directory: " +
 136                   _library_dirs[0])
 137
 138     if OPTION_AUTO_RPATH:
 139         runtime_library_dirs = _library_dirs
 140     else:
 141         runtime_library_dirs = []
 142
 143     if CYTHON_INSTALLED and OPTION_SHOW_WARNINGS:
 144         from Cython.Compiler import Errors
 145         Errors.LEVEL = 0
 146
 147     cythonize_directives = {
 148         'binding': True,
 149     }
 150     if OPTION_WITH_COVERAGE:
 151         cythonize_directives['linetrace'] = True
 152
 153     result = []
 154     for module, src_file in zip(modules, module_files):
 155         is_py = module in COMPILED_MODULES
 156         main_module_source = src_file + (
 157             '.c' if not use_cython else '.py' if is_py else '.pyx')
 158         result.append(
 159             Extension(
 160                 module,
 161                 sources = [main_module_source],
 162                 depends = find_dependencies(module),
 163                 extra_compile_args = _cflags,
 164                 extra_link_args = None if is_py else _ldflags,
 165                 extra_objects = None if is_py else static_binaries,
 166                 define_macros = _define_macros,
 167                 include_dirs = _include_dirs,
 168                 library_dirs = None if is_py else _library_dirs,
 169                 runtime_library_dirs = None if is_py else runtime_library_dirs,
 170                 libraries = None if is_py else _libraries,
 171             ))
 172     if CYTHON_INSTALLED and OPTION_WITH_CYTHON_GDB:
 173         for ext in result:
 174             ext.cython_gdb = True
 175
 176     if CYTHON_INSTALLED and use_cython:
 177         # build .c files right now and convert Extension() objects
 178         from Cython.Build import cythonize
 179         result = cythonize(result, compiler_directives=cythonize_directives)
 180
 181     # for backwards compatibility reasons, provide "etree[_api].h" also as "lxml.etree[_api].h"
 182     for header_filename in HEADER_FILES:
 183         src_file = os.path.join(SOURCE_PATH, 'lxml', header_filename)
 184         dst_file = os.path.join(SOURCE_PATH, 'lxml', 'lxml.' + header_filename)
 185         if not os.path.exists(src_file):
 186             continue
 187         if os.path.exists(dst_file) and os.path.getmtime(dst_file) >= os.path.getmtime(src_file):
 188             continue
 189
 190         with io.open(src_file, 'r', encoding='iso8859-1') as f:
 191             content = f.read()
 192         for filename in HEADER_FILES:
 193             content = content.replace('"%s"' % filename, '"lxml.%s"' % filename)
 194         with io.open(dst_file, 'w', encoding='iso8859-1') as f:
 195             f.write(content)
 196
 197     return result
 198
 199
 200 def find_dependencies(module):
 201     if not CYTHON_INSTALLED or 'lxml.html' in module:
 202         return []
 203     base_dir = get_base_dir()
 204     package_dir = os.path.join(base_dir, SOURCE_PATH, 'lxml')
 205     includes_dir = os.path.join(base_dir, INCLUDE_PACKAGE_PATH)
 206
 207     pxd_files = [
 208         os.path.join(INCLUDE_PACKAGE_PATH, filename)
 209         for filename in os.listdir(includes_dir)
 210         if filename.endswith('.pxd')
 211     ]
 212
 213     if module == 'lxml.etree':
 214         pxi_files = [
 215             os.path.join(SOURCE_PATH, 'lxml', filename)
 216             for filename in os.listdir(package_dir)
 217             if filename.endswith('.pxi') and 'objectpath' not in filename
 218         ]
 219         pxd_files = [
 220             filename for filename in pxd_files
 221             if 'etreepublic' not in filename
 222         ]
 223     elif module == 'lxml.objectify':
 224         pxi_files = [os.path.join(SOURCE_PATH, 'lxml', 'objectpath.pxi')]
 225     else:
 226         pxi_files = pxd_files = []
 227
 228     return pxd_files + pxi_files
 229
 230
 231 def extra_setup_args():
 232     class CheckLibxml2BuildExt(_build_ext):
 233         """Subclass to check whether libxml2 is really available if the build fails"""
 234         def run(self):
 235             try:
 236                 _build_ext.run(self)  # old-style class in Py2
 237             except CompileError as e:
 238                 print('Compile failed: %s' % e)
 239                 if not seems_to_have_libxml2():
 240                     print_libxml_error()
 241                 raise
 242     result = {'cmdclass': {'build_ext': CheckLibxml2BuildExt}}
 243     return result
 244
 245
 246 def seems_to_have_libxml2():
 247     from distutils import ccompiler
 248     compiler = ccompiler.new_compiler()
 249     return compiler.has_function(
 250         'xmlXPathInit',
 251         include_dirs=include_dirs([]) + ['/usr/include/libxml2'],
 252         includes=['libxml/xpath.h'],
 253         library_dirs=library_dirs([]),
 254         libraries=['xml2'])
 255
 256
 257 def print_libxml_error():
 258     print('*********************************************************************************')
 259     print('Could not find function xmlCheckVersion in library libxml2. Is libxml2 installed?')
 260     if sys.platform in ('darwin',):
 261         print('Perhaps try: xcode-select --install')
 262     print('*********************************************************************************')
 263
 264
 265 def libraries():
 266     standard_libs = []
 267     if 'linux' in sys.platform:
 268         standard_libs.append('rt')
 269     if not OPTION_BUILD_LIBXML2XSLT:
 270         standard_libs.append('z')
 271     standard_libs.append('m')
 272
 273     if sys.platform in ('win32',):
 274         libs = ['libxslt', 'libexslt', 'libxml2', 'iconv']
 275         if OPTION_STATIC:
 276             libs = ['%s_a' % lib for lib in libs]
 277         libs.extend(['zlib', 'WS2_32'])
 278     elif OPTION_STATIC:
 279         libs = standard_libs
 280     else:
 281         libs = ['xslt', 'exslt', 'xml2'] + standard_libs
 282     return libs
 283
 284 def library_dirs(static_library_dirs):
 285     if OPTION_STATIC:
 286         if not static_library_dirs:
 287             static_library_dirs = env_var('LIBRARY')
 288         assert static_library_dirs, "Static build not configured, see doc/build.txt"
 289         return static_library_dirs
 290     # filter them from xslt-config --libs
 291     result = []
 292     possible_library_dirs = flags('libs')
 293     for possible_library_dir in possible_library_dirs:
 294         if possible_library_dir.startswith('-L'):
 295             result.append(possible_library_dir[2:])
 296     return result
 297
 298 def include_dirs(static_include_dirs):
 299     if OPTION_STATIC:
 300         if not static_include_dirs:
 301             static_include_dirs = env_var('INCLUDE')
 302         return static_include_dirs
 303     # filter them from xslt-config --cflags
 304     result = []
 305     possible_include_dirs = flags('cflags')
 306     for possible_include_dir in possible_include_dirs:
 307         if possible_include_dir.startswith('-I'):
 308             result.append(possible_include_dir[2:])
 309     return result
 310
 311 def cflags(static_cflags):
 312     result = []
 313     if not OPTION_SHOW_WARNINGS:
 314         result.append('-w')
 315     if OPTION_DEBUG_GCC:
 316         result.append('-g2')
 317
 318     if OPTION_STATIC:
 319         if not static_cflags:
 320             static_cflags = env_var('CFLAGS')
 321         result.extend(static_cflags)
 322     else:
 323         # anything from xslt-config --cflags that doesn't start with -I
 324         possible_cflags = flags('cflags')
 325         for possible_cflag in possible_cflags:
 326             if not possible_cflag.startswith('-I'):
 327                 result.append(possible_cflag)
 328
 329     if sys.platform in ('darwin',):
 330         for opt in result:
 331             if 'flat_namespace' in opt:
 332                 break
 333         else:
 334             result.append('-flat_namespace')
 335
 336     return result
 337
 338 def define_macros():
 339     macros = []
 340     if OPTION_WITHOUT_ASSERT:
 341         macros.append(('PYREX_WITHOUT_ASSERTIONS', None))
 342     if OPTION_WITHOUT_THREADING:
 343         macros.append(('WITHOUT_THREADING', None))
 344     if OPTION_WITH_REFNANNY:
 345         macros.append(('CYTHON_REFNANNY', None))
 346     if OPTION_WITH_UNICODE_STRINGS:
 347         macros.append(('LXML_UNICODE_STRINGS', '1'))
 348     if OPTION_WITH_COVERAGE:
 349         macros.append(('CYTHON_TRACE_NOGIL', '1'))
 350     # Disable showing C lines in tracebacks, unless explicitly requested.
 351     macros.append(('CYTHON_CLINE_IN_TRACEBACK', '1' if OPTION_WITH_CLINES else '0'))
 352     return macros
 353
 354
 355 def run_command(cmd, *args):
 356     if not cmd:
 357         return ''
 358     if args:
 359         cmd = ' '.join((cmd,) + args)
 360
 361     p = subprocess.Popen(cmd, shell=True,
 362                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 363     stdout_data, errors = p.communicate()
 364
 365     if errors:
 366         return ''
 367     return decode_input(stdout_data).strip()
 368
 369
 370 def check_min_version(version, min_version, libname):
 371     if not version:
 372         # this is ok for targets like sdist etc.
 373         return True
 374     lib_version = tuple(map(int, version.split('.')[:3]))
 375     req_version = tuple(map(int, min_version.split('.')[:3]))
 376     if lib_version < req_version:
 377         print("Minimum required version of %s is %s. Your system has version %s." % (
 378             libname, min_version, version))
 379         return False
 380     return True
 381
 382
 383 def get_library_version(prog, libname=None):
 384     if libname:
 385         return run_command(prog, '--modversion %s' % libname)
 386     else:
 387         return run_command(prog, '--version')
 388
 389
 390 PKG_CONFIG = None
 391 XML2_CONFIG = None
 392 XSLT_CONFIG = None
 393
 394 def get_library_versions():
 395     global XML2_CONFIG, XSLT_CONFIG
 396
 397     # Pre-built libraries
 398     if XML2_CONFIG and XSLT_CONFIG:
 399         xml2_version = get_library_version(XML2_CONFIG)
 400         xslt_version = get_library_version(XSLT_CONFIG)
 401         return xml2_version, xslt_version
 402
 403     # Path to xml2-config and xslt-config specified on the command line
 404     if OPTION_WITH_XML2_CONFIG:
 405         xml2_version = get_library_version(OPTION_WITH_XML2_CONFIG)
 406         if xml2_version and OPTION_WITH_XSLT_CONFIG:
 407             xslt_version = get_library_version(OPTION_WITH_XSLT_CONFIG)
 408             if xslt_version:
 409                 XML2_CONFIG = OPTION_WITH_XML2_CONFIG
 410                 XSLT_CONFIG = OPTION_WITH_XSLT_CONFIG
 411                 return xml2_version, xslt_version
 412
 413     # Try pkg-config
 414     global PKG_CONFIG
 415     PKG_CONFIG = os.getenv('PKG_CONFIG', 'pkg-config')
 416     xml2_version = get_library_version(PKG_CONFIG, 'libxml-2.0')
 417     if xml2_version:
 418         xslt_version = get_library_version(PKG_CONFIG, 'libxslt')
 419         if xml2_version and xslt_version:
 420             return xml2_version, xslt_version
 421
 422     # Try xml2-config and xslt-config
 423     XML2_CONFIG = os.getenv('XML2_CONFIG', 'xml2-config')
 424     xml2_version = get_library_version(XML2_CONFIG)
 425     if xml2_version:
 426         XSLT_CONFIG = os.getenv('XSLT_CONFIG', 'xslt-config')
 427         xslt_version = get_library_version(XSLT_CONFIG)
 428         if xml2_version and xslt_version:
 429             return xml2_version, xslt_version
 430
 431     # One or both build dependencies not found. Fail on Linux platforms only.
 432     if sys.platform.startswith('win'):
 433         return '', ''
 434     print("Error: Please make sure the libxml2 and libxslt development packages are installed.")
 435     sys.exit(1)
 436
 437
 438 def check_build_dependencies():
 439     xml2_version, xslt_version = get_library_versions()
 440
 441     xml2_ok = check_min_version(xml2_version, '2.7.0', 'libxml2')
 442     xslt_ok = check_min_version(xslt_version, '1.1.23', 'libxslt')
 443
 444     if xml2_version and xslt_version:
 445         print("Building against libxml2 %s and libxslt %s" % (xml2_version, xslt_version))
 446     else:
 447         print("Building against pre-built libxml2 andl libxslt libraries")
 448
 449     return (xml2_ok and xslt_ok)
 450
 451
 452 def get_flags(prog, option, libname=None):
 453     if libname:
 454         return run_command(prog, '--%s %s' % (option, libname))
 455     else:
 456         return run_command(prog, '--%s' % option)
 457
 458
 459 def flags(option):
 460     if XML2_CONFIG:
 461         xml2_flags = get_flags(XML2_CONFIG, option)
 462         xslt_flags = get_flags(XSLT_CONFIG, option)
 463     else:
 464         xml2_flags = get_flags(PKG_CONFIG, option, 'libxml-2.0')
 465         xslt_flags = get_flags(PKG_CONFIG, option, 'libxslt')
 466
 467     flag_list = xml2_flags.split()
 468     for flag in xslt_flags.split():
 469         if flag not in flag_list:
 470             flag_list.append(flag)
 471     return flag_list
 472
 473
 474 def get_xcode_isysroot():
 475     return run_command('xcrun', '--show-sdk-path')
 476
 477
 478 ## Option handling:
 479
 480 def has_option(name):
 481     try:
 482         sys.argv.remove('--%s' % name)
 483         return True
 484     except ValueError:
 485         pass
 486     # allow passing all cmd line options also as environment variables
 487     env_val = os.getenv(name.upper().replace('-', '_'), 'false').lower()
 488     if env_val == "true":
 489         return True
 490     return False
 491
 492 def option_value(name):
 493     for index, option in enumerate(sys.argv):
 494         if option == '--' + name:
 495             if index+1 >= len(sys.argv):
 496                 raise DistutilsOptionError(
 497                     'The option %s requires a value' % option)
 498             value = sys.argv[index+1]
 499             sys.argv[index:index+2] = []
 500             return value
 501         if option.startswith('--' + name + '='):
 502             value = option[len(name)+3:]
 503             sys.argv[index:index+1] = []
 504             return value
 505     env_val = os.getenv(name.upper().replace('-', '_'))
 506     return env_val
 507
 508 staticbuild = bool(os.environ.get('STATICBUILD', ''))
 509 # pick up any commandline options and/or env variables
 510 OPTION_WITHOUT_OBJECTIFY = has_option('without-objectify')
 511 OPTION_WITH_UNICODE_STRINGS = has_option('with-unicode-strings')
 512 OPTION_WITHOUT_ASSERT = has_option('without-assert')
 513 OPTION_WITHOUT_THREADING = has_option('without-threading')
 514 OPTION_WITHOUT_CYTHON = has_option('without-cython')
 515 OPTION_WITH_CYTHON = has_option('with-cython')
 516 OPTION_WITH_CYTHON_GDB = has_option('cython-gdb')
 517 OPTION_WITH_REFNANNY = has_option('with-refnanny')
 518 OPTION_WITH_COVERAGE = has_option('with-coverage')
 519 OPTION_WITH_CLINES = has_option('with-clines')
 520 if OPTION_WITHOUT_CYTHON:
 521     CYTHON_INSTALLED = False
 522 OPTION_STATIC = staticbuild or has_option('static')
 523 OPTION_DEBUG_GCC = has_option('debug-gcc')
 524 OPTION_SHOW_WARNINGS = has_option('warnings')
 525 OPTION_AUTO_RPATH = has_option('auto-rpath')
 526 OPTION_BUILD_LIBXML2XSLT = staticbuild or has_option('static-deps')
 527 if OPTION_BUILD_LIBXML2XSLT:
 528     OPTION_STATIC = True
 529 OPTION_WITH_XML2_CONFIG = option_value('xml2-config')
 530 OPTION_WITH_XSLT_CONFIG = option_value('xslt-config')
 531 OPTION_LIBXML2_VERSION = option_value('libxml2-version')
 532 OPTION_LIBXSLT_VERSION = option_value('libxslt-version')
 533 OPTION_LIBICONV_VERSION = option_value('libiconv-version')
 534 OPTION_ZLIB_VERSION = option_value('zlib-version')
 535 OPTION_MULTICORE = option_value('multicore')
 536 OPTION_DOWNLOAD_DIR = option_value('download-dir')
 537 if OPTION_DOWNLOAD_DIR is None:
 538     OPTION_DOWNLOAD_DIR = 'libs'