6 from distutils.core import Extension
7 from distutils.errors import CompileError, DistutilsOptionError
8 from distutils.command.build_ext import build_ext as _build_ext
9 from versioninfo import get_base_dir
12 import Cython.Compiler.Version
13 CYTHON_INSTALLED = True
15 CYTHON_INSTALLED = False
17 EXT_MODULES = ["lxml.etree", "lxml.objectify"]
25 HEADER_FILES = ['etree.h', 'etree_api.h']
27 if hasattr(sys, 'pypy_version_info') or (
28 getattr(sys, 'implementation', None) and sys.implementation.name != 'cpython'):
29 # disable Cython compilation of Python modules in PyPy and other non-CPythons
30 del COMPILED_MODULES[:]
33 INCLUDE_PACKAGE_PATH = os.path.join(SOURCE_PATH, 'lxml', 'includes')
35 if sys.version_info[0] >= 3:
36 _system_encoding = sys.getdefaultencoding()
37 if _system_encoding is None:
38 _system_encoding = "iso-8859-1" # :-)
39 def decode_input(data):
40 if isinstance(data, str):
42 return data.decode(_system_encoding)
44 def decode_input(data):
48 value = os.getenv(name)
50 value = decode_input(value)
51 if sys.platform == 'win32' and ';' in value:
52 return value.split(';')
59 def _prefer_reldirs(base_dir, dirs):
61 os.path.relpath(path) if path.startswith(base_dir) else path
65 def ext_modules(static_include_dirs, static_library_dirs,
66 static_cflags, static_binaries):
67 global XML2_CONFIG, XSLT_CONFIG
68 if OPTION_BUILD_LIBXML2XSLT:
69 from buildlibxml import build_libxml2xslt, get_prebuilt_libxml2xslt
70 if sys.platform.startswith('win'):
71 get_prebuilt_libxml2xslt(
72 OPTION_DOWNLOAD_DIR, static_include_dirs, static_library_dirs)
74 XML2_CONFIG, XSLT_CONFIG = build_libxml2xslt(
75 OPTION_DOWNLOAD_DIR, 'build/tmp',
76 static_include_dirs, static_library_dirs,
77 static_cflags, static_binaries,
78 libiconv_version=OPTION_LIBICONV_VERSION,
79 libxml2_version=OPTION_LIBXML2_VERSION,
80 libxslt_version=OPTION_LIBXSLT_VERSION,
81 zlib_version=OPTION_ZLIB_VERSION,
82 multicore=OPTION_MULTICORE)
84 modules = EXT_MODULES + COMPILED_MODULES
85 if OPTION_WITHOUT_OBJECTIFY:
86 modules = [entry for entry in modules if 'objectify' not in entry]
88 module_files = list(os.path.join(SOURCE_PATH, *module.split('.')) for module in modules)
89 c_files_exist = [os.path.exists(module + '.c') for module in module_files]
92 if CYTHON_INSTALLED and (OPTION_WITH_CYTHON or not all(c_files_exist)):
93 print("Building with Cython %s." % Cython.Compiler.Version.version)
94 # generate module cleanup code
95 from Cython.Compiler import Options
96 Options.generate_cleanup_code = 3
97 Options.clear_to_none = False
98 elif not OPTION_WITHOUT_CYTHON and not all(c_files_exist):
99 for exists, module in zip(c_files_exist, module_files):
102 "ERROR: Trying to build without Cython, but pre-generated '%s.c' "
103 "is not available (pass --without-cython to ignore this error)." % module)
105 if not all(c_files_exist):
106 for exists, module in zip(c_files_exist, module_files):
108 print("WARNING: Trying to build without Cython, but pre-generated "
109 "'%s.c' is not available." % module)
111 print("Building without Cython.")
113 if not check_build_dependencies():
114 raise RuntimeError("Dependency missing")
116 base_dir = get_base_dir()
117 _include_dirs = _prefer_reldirs(
118 base_dir, include_dirs(static_include_dirs) + [
120 INCLUDE_PACKAGE_PATH,
122 _library_dirs = _prefer_reldirs(base_dir, library_dirs(static_library_dirs))
123 _cflags = cflags(static_cflags)
124 _ldflags = ['-isysroot', get_xcode_isysroot()] if sys.platform == 'darwin' else None
125 _define_macros = define_macros()
126 _libraries = libraries()
129 message = "Building against libxml2/libxslt in "
130 if len(_library_dirs) > 1:
131 print(message + "one of the following directories:")
132 for dir in _library_dirs:
135 print(message + "the following directory: " +
138 if OPTION_AUTO_RPATH:
139 runtime_library_dirs = _library_dirs
141 runtime_library_dirs = []
143 if CYTHON_INSTALLED and OPTION_SHOW_WARNINGS:
144 from Cython.Compiler import Errors
147 cythonize_directives = {
150 if OPTION_WITH_COVERAGE:
151 cythonize_directives['linetrace'] = True
154 for module, src_file in zip(modules, module_files):
155 is_py = module in COMPILED_MODULES
156 main_module_source = src_file + (
157 '.c' if not use_cython else '.py' if is_py else '.pyx')
161 sources = [main_module_source],
162 depends = find_dependencies(module),
163 extra_compile_args = _cflags,
164 extra_link_args = None if is_py else _ldflags,
165 extra_objects = None if is_py else static_binaries,
166 define_macros = _define_macros,
167 include_dirs = _include_dirs,
168 library_dirs = None if is_py else _library_dirs,
169 runtime_library_dirs = None if is_py else runtime_library_dirs,
170 libraries = None if is_py else _libraries,
172 if CYTHON_INSTALLED and OPTION_WITH_CYTHON_GDB:
174 ext.cython_gdb = True
176 if CYTHON_INSTALLED and use_cython:
177 # build .c files right now and convert Extension() objects
178 from Cython.Build import cythonize
179 result = cythonize(result, compiler_directives=cythonize_directives)
181 # for backwards compatibility reasons, provide "etree[_api].h" also as "lxml.etree[_api].h"
182 for header_filename in HEADER_FILES:
183 src_file = os.path.join(SOURCE_PATH, 'lxml', header_filename)
184 dst_file = os.path.join(SOURCE_PATH, 'lxml', 'lxml.' + header_filename)
185 if not os.path.exists(src_file):
187 if os.path.exists(dst_file) and os.path.getmtime(dst_file) >= os.path.getmtime(src_file):
190 with io.open(src_file, 'r', encoding='iso8859-1') as f:
192 for filename in HEADER_FILES:
193 content = content.replace('"%s"' % filename, '"lxml.%s"' % filename)
194 with io.open(dst_file, 'w', encoding='iso8859-1') as f:
200 def find_dependencies(module):
201 if not CYTHON_INSTALLED or 'lxml.html' in module:
203 base_dir = get_base_dir()
204 package_dir = os.path.join(base_dir, SOURCE_PATH, 'lxml')
205 includes_dir = os.path.join(base_dir, INCLUDE_PACKAGE_PATH)
208 os.path.join(INCLUDE_PACKAGE_PATH, filename)
209 for filename in os.listdir(includes_dir)
210 if filename.endswith('.pxd')
213 if module == 'lxml.etree':
215 os.path.join(SOURCE_PATH, 'lxml', filename)
216 for filename in os.listdir(package_dir)
217 if filename.endswith('.pxi') and 'objectpath' not in filename
220 filename for filename in pxd_files
221 if 'etreepublic' not in filename
223 elif module == 'lxml.objectify':
224 pxi_files = [os.path.join(SOURCE_PATH, 'lxml', 'objectpath.pxi')]
226 pxi_files = pxd_files = []
228 return pxd_files + pxi_files
231 def extra_setup_args():
232 class CheckLibxml2BuildExt(_build_ext):
233 """Subclass to check whether libxml2 is really available if the build fails"""
236 _build_ext.run(self) # old-style class in Py2
237 except CompileError as e:
238 print('Compile failed: %s' % e)
239 if not seems_to_have_libxml2():
242 result = {'cmdclass': {'build_ext': CheckLibxml2BuildExt}}
246 def seems_to_have_libxml2():
247 from distutils import ccompiler
248 compiler = ccompiler.new_compiler()
249 return compiler.has_function(
251 include_dirs=include_dirs([]) + ['/usr/include/libxml2'],
252 includes=['libxml/xpath.h'],
253 library_dirs=library_dirs([]),
257 def print_libxml_error():
258 print('*********************************************************************************')
259 print('Could not find function xmlCheckVersion in library libxml2. Is libxml2 installed?')
260 if sys.platform in ('darwin',):
261 print('Perhaps try: xcode-select --install')
262 print('*********************************************************************************')
267 if 'linux' in sys.platform:
268 standard_libs.append('rt')
269 if not OPTION_BUILD_LIBXML2XSLT:
270 standard_libs.append('z')
271 standard_libs.append('m')
273 if sys.platform in ('win32',):
274 libs = ['libxslt', 'libexslt', 'libxml2', 'iconv']
276 libs = ['%s_a' % lib for lib in libs]
277 libs.extend(['zlib', 'WS2_32'])
281 libs = ['xslt', 'exslt', 'xml2'] + standard_libs
284 def library_dirs(static_library_dirs):
286 if not static_library_dirs:
287 static_library_dirs = env_var('LIBRARY')
288 assert static_library_dirs, "Static build not configured, see doc/build.txt"
289 return static_library_dirs
290 # filter them from xslt-config --libs
292 possible_library_dirs = flags('libs')
293 for possible_library_dir in possible_library_dirs:
294 if possible_library_dir.startswith('-L'):
295 result.append(possible_library_dir[2:])
298 def include_dirs(static_include_dirs):
300 if not static_include_dirs:
301 static_include_dirs = env_var('INCLUDE')
302 return static_include_dirs
303 # filter them from xslt-config --cflags
305 possible_include_dirs = flags('cflags')
306 for possible_include_dir in possible_include_dirs:
307 if possible_include_dir.startswith('-I'):
308 result.append(possible_include_dir[2:])
311 def cflags(static_cflags):
313 if not OPTION_SHOW_WARNINGS:
319 if not static_cflags:
320 static_cflags = env_var('CFLAGS')
321 result.extend(static_cflags)
323 # anything from xslt-config --cflags that doesn't start with -I
324 possible_cflags = flags('cflags')
325 for possible_cflag in possible_cflags:
326 if not possible_cflag.startswith('-I'):
327 result.append(possible_cflag)
329 if sys.platform in ('darwin',):
331 if 'flat_namespace' in opt:
334 result.append('-flat_namespace')
340 if OPTION_WITHOUT_ASSERT:
341 macros.append(('PYREX_WITHOUT_ASSERTIONS', None))
342 if OPTION_WITHOUT_THREADING:
343 macros.append(('WITHOUT_THREADING', None))
344 if OPTION_WITH_REFNANNY:
345 macros.append(('CYTHON_REFNANNY', None))
346 if OPTION_WITH_UNICODE_STRINGS:
347 macros.append(('LXML_UNICODE_STRINGS', '1'))
348 if OPTION_WITH_COVERAGE:
349 macros.append(('CYTHON_TRACE_NOGIL', '1'))
350 # Disable showing C lines in tracebacks, unless explicitly requested.
351 macros.append(('CYTHON_CLINE_IN_TRACEBACK', '1' if OPTION_WITH_CLINES else '0'))
355 def run_command(cmd, *args):
359 cmd = ' '.join((cmd,) + args)
361 p = subprocess.Popen(cmd, shell=True,
362 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
363 stdout_data, errors = p.communicate()
367 return decode_input(stdout_data).strip()
370 def check_min_version(version, min_version, libname):
372 # this is ok for targets like sdist etc.
374 lib_version = tuple(map(int, version.split('.')[:3]))
375 req_version = tuple(map(int, min_version.split('.')[:3]))
376 if lib_version < req_version:
377 print("Minimum required version of %s is %s. Your system has version %s." % (
378 libname, min_version, version))
383 def get_library_version(prog, libname=None):
385 return run_command(prog, '--modversion %s' % libname)
387 return run_command(prog, '--version')
394 def get_library_versions():
395 global XML2_CONFIG, XSLT_CONFIG
397 # Pre-built libraries
398 if XML2_CONFIG and XSLT_CONFIG:
399 xml2_version = get_library_version(XML2_CONFIG)
400 xslt_version = get_library_version(XSLT_CONFIG)
401 return xml2_version, xslt_version
403 # Path to xml2-config and xslt-config specified on the command line
404 if OPTION_WITH_XML2_CONFIG:
405 xml2_version = get_library_version(OPTION_WITH_XML2_CONFIG)
406 if xml2_version and OPTION_WITH_XSLT_CONFIG:
407 xslt_version = get_library_version(OPTION_WITH_XSLT_CONFIG)
409 XML2_CONFIG = OPTION_WITH_XML2_CONFIG
410 XSLT_CONFIG = OPTION_WITH_XSLT_CONFIG
411 return xml2_version, xslt_version
415 PKG_CONFIG = os.getenv('PKG_CONFIG', 'pkg-config')
416 xml2_version = get_library_version(PKG_CONFIG, 'libxml-2.0')
418 xslt_version = get_library_version(PKG_CONFIG, 'libxslt')
419 if xml2_version and xslt_version:
420 return xml2_version, xslt_version
422 # Try xml2-config and xslt-config
423 XML2_CONFIG = os.getenv('XML2_CONFIG', 'xml2-config')
424 xml2_version = get_library_version(XML2_CONFIG)
426 XSLT_CONFIG = os.getenv('XSLT_CONFIG', 'xslt-config')
427 xslt_version = get_library_version(XSLT_CONFIG)
428 if xml2_version and xslt_version:
429 return xml2_version, xslt_version
431 # One or both build dependencies not found. Fail on Linux platforms only.
432 if sys.platform.startswith('win'):
434 print("Error: Please make sure the libxml2 and libxslt development packages are installed.")
438 def check_build_dependencies():
439 xml2_version, xslt_version = get_library_versions()
441 xml2_ok = check_min_version(xml2_version, '2.7.0', 'libxml2')
442 xslt_ok = check_min_version(xslt_version, '1.1.23', 'libxslt')
444 if xml2_version and xslt_version:
445 print("Building against libxml2 %s and libxslt %s" % (xml2_version, xslt_version))
447 print("Building against pre-built libxml2 andl libxslt libraries")
449 return (xml2_ok and xslt_ok)
452 def get_flags(prog, option, libname=None):
454 return run_command(prog, '--%s %s' % (option, libname))
456 return run_command(prog, '--%s' % option)
461 xml2_flags = get_flags(XML2_CONFIG, option)
462 xslt_flags = get_flags(XSLT_CONFIG, option)
464 xml2_flags = get_flags(PKG_CONFIG, option, 'libxml-2.0')
465 xslt_flags = get_flags(PKG_CONFIG, option, 'libxslt')
467 flag_list = xml2_flags.split()
468 for flag in xslt_flags.split():
469 if flag not in flag_list:
470 flag_list.append(flag)
474 def get_xcode_isysroot():
475 return run_command('xcrun', '--show-sdk-path')
480 def has_option(name):
482 sys.argv.remove('--%s' % name)
486 # allow passing all cmd line options also as environment variables
487 env_val = os.getenv(name.upper().replace('-', '_'), 'false').lower()
488 if env_val == "true":
493 def option_value(name, deprecated_for=None):
494 for index, option in enumerate(sys.argv):
495 if option == '--' + name:
496 if index+1 >= len(sys.argv):
497 raise DistutilsOptionError(
498 'The option %s requires a value' % option)
499 value = sys.argv[index+1]
500 sys.argv[index:index+2] = []
502 print_deprecated_option(name, deprecated_for)
504 if option.startswith('--' + name + '='):
505 value = option[len(name)+3:]
506 sys.argv[index:index+1] = []
508 print_deprecated_option(name, deprecated_for)
510 env_name = name.upper().replace('-', '_')
511 env_val = os.getenv(env_name)
512 if env_val and deprecated_for:
513 print_deprecated_option(env_name, deprecated_for.upper().replace('-', '_'))
517 def print_deprecated_option(name, new_name):
518 print("WARN: Option '%s' is deprecated. Use '%s' instead." % (name, new_name))
521 staticbuild = bool(os.environ.get('STATICBUILD', ''))
522 # pick up any commandline options and/or env variables
523 OPTION_WITHOUT_OBJECTIFY = has_option('without-objectify')
524 OPTION_WITH_UNICODE_STRINGS = has_option('with-unicode-strings')
525 OPTION_WITHOUT_ASSERT = has_option('without-assert')
526 OPTION_WITHOUT_THREADING = has_option('without-threading')
527 OPTION_WITHOUT_CYTHON = has_option('without-cython')
528 OPTION_WITH_CYTHON = has_option('with-cython')
529 OPTION_WITH_CYTHON_GDB = has_option('cython-gdb')
530 OPTION_WITH_REFNANNY = has_option('with-refnanny')
531 OPTION_WITH_COVERAGE = has_option('with-coverage')
532 OPTION_WITH_CLINES = has_option('with-clines')
533 if OPTION_WITHOUT_CYTHON:
534 CYTHON_INSTALLED = False
535 OPTION_STATIC = staticbuild or has_option('static')
536 OPTION_DEBUG_GCC = has_option('debug-gcc')
537 OPTION_SHOW_WARNINGS = has_option('warnings')
538 OPTION_AUTO_RPATH = has_option('auto-rpath')
539 OPTION_BUILD_LIBXML2XSLT = staticbuild or has_option('static-deps')
540 if OPTION_BUILD_LIBXML2XSLT:
542 OPTION_WITH_XML2_CONFIG = option_value('with-xml2-config') or option_value('xml2-config', deprecated_for='with-xml2-config')
543 OPTION_WITH_XSLT_CONFIG = option_value('with-xslt-config') or option_value('xslt-config', deprecated_for='with-xslt-config')
544 OPTION_LIBXML2_VERSION = option_value('libxml2-version')
545 OPTION_LIBXSLT_VERSION = option_value('libxslt-version')
546 OPTION_LIBICONV_VERSION = option_value('libiconv-version')
547 OPTION_ZLIB_VERSION = option_value('zlib-version')
548 OPTION_MULTICORE = option_value('multicore')
549 OPTION_DOWNLOAD_DIR = option_value('download-dir')
550 if OPTION_DOWNLOAD_DIR is None:
551 OPTION_DOWNLOAD_DIR = 'libs'