+"""
+Highly experimental script that compiles the CPython standard library using Cython.
+
+Execute the script either in the CPython 'Lib' directory or pass the
+option '--current-python' to compile the standard library of the running
+Python interpreter.
+
+Pass '--parallel' to get a parallel build.
+
+Usage example::
+
+ $ python cystdlib.py --current-python build_ext -i
+"""
+
+import sys
from distutils.core import setup
from Cython.Build import cythonize
from Cython.Compiler import Options
-import sys
-pyver = sys.version_info[0]
-try:
- sys.argv.remove('--current-python')
-except ValueError:
- pass
-else:
- # assume that the stdlib is where the "os" module lives
- import os
- os.chdir(os.path.dirname(os.__file__))
-
-try:
- sys.argv.remove('--parallel')
- import multiprocessing
- parallel_compiles = multiprocessing.cpu_count() * 2
-except (ValueError, ImportError):
- parallel_compiles = None
-
# improve Python compatibility by allowing some broken code
Options.error_on_unknown_names = False
excludes = ['**/test/**/*.py', '**/tests/**/*.py', '**/__init__.py']
+broken = [
+ 'idlelib/MultiCall.py',
+ 'email/utils.py',
+ 'multiprocessing/reduction.py',
+ 'multiprocessing/util.py',
+]
default_directives = dict(auto_cpdef=True)
+special_directives = [
+ (['pkgutil.py'], dict(auto_cpdef=False)),
+]
+
+def build_extensions(includes='**/*.py',
+ excludes=excludes+broken,
+ special_directives=special_directives,
+ parallel=None):
+ if isinstance(includes, str):
+ includes = [includes]
+
+ all_groups = (special_directives or []) + [(includes, {})]
+ extensions = []
+ for modules, directives in all_groups:
+ exclude_now = excludes[:]
+ for other_modules, _ in special_directives:
+ if other_modules != modules:
+ exclude_now.extend(other_modules)
+
+ d = dict(default_directives)
+ d.update(directives)
+
+ extensions.extend(
+ cythonize(modules,
+ exclude=exclude_now,
+ exclude_failures=True,
+ language_level=pyver,
+ compiler_directives=d,
+ nthreads=parallel,
+ ))
+ return extensions
+
+def build(extensions):
+ try:
+ setup(name = 'stuff', ext_modules = extensions)
+ return extensions, True
+ except:
+ import traceback
+ print('error building extensions %s' % (extensions,))
+ traceback.print_exc()
+ return extensions, False
+
+def _build(args):
+ sys_args, ext = args
+ sys.argv[1:] = sys_args
+ return build([ext])
+
+
+if __name__ == '__main__':
+ import sys
+ pyver = sys.version_info[0]
+ try:
+ sys.argv.remove('--current-python')
+ except ValueError:
+ pass
+ else:
+ # assume that the stdlib is where the "os" module lives
+ import os
+ os.chdir(os.path.dirname(os.__file__))
+
+ try:
+ sys.argv.remove('--parallel')
+ import multiprocessing
+ parallel_compiles = multiprocessing.cpu_count() * 2
+ print("Building in %d parallel processes" % parallel_compiles)
+ except (ValueError, ImportError):
+ parallel_compiles = None
-setup(
- name = 'stuff',
- ext_modules = cythonize(["**/*.py"],
- exclude=excludes,
- exclude_failures=True,
- language_level=pyver,
-# compiler_directives=dict(auto_cpdef=True),
- nthreads=parallel_compiles,
- ),
-)
+ extensions = build_extensions(parallel=parallel_compiles)
+ if parallel_compiles:
+ pool = multiprocessing.Pool(parallel_compiles)
+ sys_args = sys.argv[1:]
+ results = pool.map(_build, [ (sys_args, ext) for ext in extensions ])
+ pool.close()
+ pool.join()
+ for ext, result in results:
+ if not result:
+ print("building extension %s failed" % (ext[0],))
+ else:
+ build(extensions)