2 Highly experimental script that compiles the CPython standard library using Cython.
4 Execute the script either in the CPython 'Lib' directory or pass the
5 option '--current-python' to compile the standard library of the running
8 Pass '-j N' to get a parallel build with N processes.
12 $ python cystdlib.py --current-python build_ext -i
17 from distutils.core import setup
18 from Cython.Build import cythonize
19 from Cython.Compiler import Options
21 # improve Python compatibility by allowing some broken code
22 Options.error_on_unknown_names = False
23 Options.error_on_uninitialized = False
25 exclude_patterns = ['**/test/**/*.py', '**/tests/**/*.py', '**/__init__.py']
27 'idlelib/MultiCall.py',
29 'multiprocessing/reduction.py',
30 'multiprocessing/util.py',
31 'threading.py', # interrupt handling
32 'lib2to3/fixes/fix_sys_exc.py',
36 'importlib/_bootstrap',
39 default_directives = dict(
40 auto_cpdef=False, # enable when it's safe, see long list of failures below
42 set_initial_path='SOURCEFILE')
43 default_directives['optimize.inline_defnode_calls'] = True
45 special_directives = [
57 'lib2to3/refactor.py',
60 'multiprocessing/forking.py',
61 'xml/sax/expatreader.py',
64 'xml/etree/ElementTree.py',
74 'multiprocessing/connection.py',
79 ], dict(auto_cpdef=False)),
81 del special_directives[:] # currently unused
83 def build_extensions(includes='**/*.py',
85 special_directives=special_directives,
86 language_level=sys.version_info[0],
88 if isinstance(includes, str):
90 excludes = list(excludes or exclude_patterns) + broken
92 all_groups = (special_directives or []) + [(includes, {})]
94 for modules, directives in all_groups:
95 exclude_now = excludes[:]
96 for other_modules, _ in special_directives:
97 if other_modules != modules:
98 exclude_now.extend(other_modules)
100 d = dict(default_directives)
107 exclude_failures=True,
108 language_level=language_level,
109 compiler_directives=d,
115 def build(extensions):
117 setup(ext_modules=extensions)
121 print('error building extensions %s' % (
122 [ext.name for ext in extensions],))
123 traceback.print_exc()
125 return extensions, result
130 sys.argv[1:] = sys_args
135 from optparse import OptionParser
136 parser = OptionParser('%prog [options] [LIB_DIR (default: ./Lib)]')
138 '--current-python', dest='current_python', action='store_true',
139 help='compile the stdlib of the running Python')
141 '-j', '--jobs', dest='parallel_jobs', metavar='N',
143 help='run builds in N parallel jobs (default: 1)')
145 '-x', '--exclude', dest='excludes', metavar='PATTERN',
146 action="append", help='exclude modules/packages matching PATTERN')
147 options, args = parser.parse_args()
151 parser.error('only one argument expected, got %d' % len(args))
155 if __name__ == '__main__':
156 options, args = parse_args()
157 if options.current_python:
158 # assume that the stdlib is where the "os" module lives
159 os.chdir(os.path.dirname(os.__file__))
164 parallel_jobs = options.parallel_jobs
165 if options.parallel_jobs:
167 import multiprocessing
168 pool = multiprocessing.Pool(parallel_jobs)
169 print("Building in %d parallel processes" % parallel_jobs)
170 except (ImportError, OSError):
171 print("Not building in parallel")
174 extensions = build_extensions(
175 parallel=parallel_jobs,
176 excludes=options.excludes)
177 sys_args = ['build_ext', '-i']
179 results = pool.map(_build, [(sys_args, ext) for ext in extensions])
182 for ext, result in results:
184 print("building extension %s failed" % (ext[0].name,))
186 sys.argv[1:] = sys_args