From 46acdc4c4f0f85c68a506b06442dee4e0dd2c1aa Mon Sep 17 00:00:00 2001 From: DongHun Kwak Date: Thu, 31 Dec 2020 12:08:31 +0900 Subject: [PATCH] Imported Upstream version 0.28.0 --- .travis.yml | 24 +- CHANGES.rst | 158 ++++- Cython/Build/Dependencies.py | 115 ++-- Cython/CodeWriter.py | 2 +- Cython/Compiler/Annotate.py | 23 +- Cython/Compiler/Buffer.py | 8 +- Cython/Compiler/Builtin.py | 7 +- Cython/Compiler/CmdLine.py | 2 + Cython/Compiler/Code.py | 184 +++++- Cython/Compiler/CodeGeneration.py | 2 +- Cython/Compiler/CythonScope.py | 4 + Cython/Compiler/ExprNodes.py | 554 +++++++++------- Cython/Compiler/FusedNode.py | 49 +- Cython/Compiler/Main.py | 35 +- Cython/Compiler/MemoryView.py | 12 +- Cython/Compiler/ModuleNode.py | 321 +++++----- Cython/Compiler/Nodes.py | 693 ++++++++++++++++----- Cython/Compiler/Optimize.py | 496 +++++++++++++-- Cython/Compiler/Options.py | 4 + Cython/Compiler/ParseTreeTransforms.py | 39 +- Cython/Compiler/Parsing.py | 39 +- Cython/Compiler/PyrexTypes.py | 234 ++++--- Cython/Compiler/Pythran.py | 13 +- Cython/Compiler/Scanning.py | 2 +- Cython/Compiler/StringEncoding.py | 8 + Cython/Compiler/Symtab.py | 128 +++- Cython/Compiler/Tests/TestTreeFragment.py | 2 +- Cython/Compiler/Tests/TestTreePath.py | 72 +-- Cython/Compiler/Tests/TestUtilityLoad.py | 20 +- Cython/Compiler/TreeFragment.py | 4 +- Cython/Compiler/TypeInference.py | 29 +- Cython/Compiler/TypeSlots.py | 17 +- Cython/Compiler/UtilNodes.py | 5 +- Cython/Compiler/UtilityCode.py | 10 +- Cython/Compiler/Visitor.py | 15 +- Cython/Coverage.py | 2 + Cython/Debugger/libcython.py | 2 +- Cython/Debugger/libpython.py | 12 +- Cython/Distutils/build_ext.py | 4 +- Cython/Includes/Deprecated/python2.5.pxd | 2 +- Cython/Includes/cpython/__init__.pxd | 4 +- Cython/Includes/cpython/array.pxd | 4 +- Cython/Includes/cpython/object.pxd | 22 +- Cython/Includes/cpython/pythread.pxd | 19 +- Cython/Includes/libc/limits.pxd | 39 +- Cython/Includes/libc/signal.pxd | 102 ++- Cython/Includes/libcpp/deque.pxd | 31 +- Cython/Includes/libcpp/string.pxd | 102 +-- Cython/Includes/libcpp/vector.pxd | 9 +- Cython/Includes/numpy/__init__.pxd | 25 +- Cython/Includes/posix/signal.pxd | 7 + Cython/Includes/posix/time.pxd | 3 - Cython/Parser/Grammar | 2 +- Cython/Shadow.py | 10 +- Cython/StringIOTree.pxd | 17 + Cython/StringIOTree.py | 69 +- Cython/Tests/TestCodeWriter.py | 2 +- Cython/Utility/AsyncGen.c | 37 +- Cython/Utility/Buffer.c | 4 +- Cython/Utility/Builtins.c | 2 +- Cython/Utility/Coroutine.c | 184 +++++- Cython/Utility/CythonFunction.c | 3 +- Cython/Utility/Exceptions.c | 2 +- Cython/Utility/ExtensionTypes.c | 75 +++ Cython/Utility/ImportExport.c | 61 ++ Cython/Utility/MemoryView.pyx | 16 +- Cython/Utility/MemoryView_C.c | 6 +- Cython/Utility/ModuleSetupCode.c | 421 ++++++++----- Cython/Utility/ObjectHandling.c | 389 ++++++++++-- Cython/Utility/Optimize.c | 176 +++++- Cython/Utility/Overflow.c | 2 +- Cython/Utility/Profile.c | 13 +- Cython/Utility/StringTools.c | 26 +- Cython/Utility/TypeConversion.c | 18 +- Demos/freeze/README.txt | 2 +- Tools/cython-mode.el | 2 + Tools/rules.bzl | 2 +- Tools/site_scons/site_tools/pyext.py | 2 +- appveyor.yml | 3 +- docs/src/quickstart/htmlreport.png | Bin 36200 -> 22739 bytes docs/src/quickstart/install.rst | 3 +- docs/src/reference/compilation.rst | 158 ++++- docs/src/reference/extension_types.rst | 26 +- docs/src/reference/language_basics.rst | 205 +++--- docs/src/tutorial/cdef_classes.rst | 2 +- docs/src/tutorial/numpy.rst | 6 + docs/src/tutorial/strings.rst | 18 +- docs/src/userguide/debugging.rst | 6 +- docs/src/userguide/extension_types.rst | 6 +- docs/src/userguide/external_C_code.rst | 42 ++ docs/src/userguide/language_basics.rst | 8 +- docs/src/userguide/memoryviews.rst | 29 +- docs/src/userguide/numpy_pythran.rst | 4 +- docs/src/userguide/numpy_tutorial.rst | 6 - docs/src/userguide/special_methods.rst | 59 +- docs/src/userguide/wrapping_CPlusPlus.rst | 4 +- pyximport/pyximport.py | 6 +- runtests.py | 12 +- setup.py | 25 +- tests/buffers/bufaccess.pyx | 18 +- tests/buffers/mockbuffers.pxi | 17 +- tests/buffers/userbuffer.pyx | 85 +++ tests/bugs.txt | 1 + tests/build/cythonize_rename_ext.srctree | 38 ++ tests/build/setuptools_reimport.srctree | 23 + tests/compile/cnamespec.h | 1 - tests/compile/cnamespec.pyx | 5 +- tests/compile/cpp_class_redefinition.pyx | 2 +- tests/compile/min_async.pyx | 12 + tests/compile/verbatiminclude_cimport.srctree | 36 ++ tests/errors/builtin_type_inheritance.pyx | 6 +- tests/errors/cpp_class_gil_GH1986.pyx | 20 + tests/errors/cpp_no_auto_conversion.pyx | 2 +- tests/errors/e_arrayassign.pyx | 5 +- tests/errors/e_cython_parallel.pyx | 1 + tests/errors/e_directives.pyx | 2 +- tests/errors/e_invalid_num_threads.pyx | 14 +- tests/errors/e_nonlocal_T490.pyx | 2 + tests/errors/subtyping_final_class.pyx | 2 +- tests/memoryview/memoryview.pyx | 48 +- tests/memoryview/memslice.pyx | 195 +++--- tests/memoryview/numpy_memoryview.pyx | 24 +- tests/memoryview/numpy_memoryview_readonly.pyx | 112 ++++ tests/run/annotation_typing.pyx | 27 +- tests/run/arithmetic_analyse_types_helper.h | 2 +- tests/run/asyncio_generators.srctree | 32 +- tests/run/bytearray_coercion.pyx | 25 +- tests/run/bytearraymethods.pyx | 12 + tests/run/cdef_multiple_inheritance.pyx | 45 ++ tests/run/cdef_multiple_inheritance_errors.srctree | 94 +++ tests/run/cdef_multiple_inheritance_nodict.pyx | 48 ++ tests/run/cpdef_pickle.srctree | 69 ++ tests/run/cpp_class_redef.pyx | 10 +- tests/run/cpp_iterators.pyx | 51 +- tests/run/cpp_operators.pyx | 2 +- tests/run/cpp_stl_conversion.pyx | 10 + tests/run/cpp_stl_string.pyx | 41 +- tests/run/cpp_template_functions.pyx | 3 +- tests/run/cpp_templates.pyx | 27 + tests/run/cstringmul.pyx | 36 +- tests/run/cyfunction.pyx | 19 + tests/run/cyfunction_defaults.pyx | 25 + tests/run/dict_getitem.pyx | 68 ++ tests/run/dict_pop.pyx | 34 + tests/run/ext_auto_richcmp.py | 18 + tests/run/extern_include_order.srctree | 56 ++ tests/run/fastcall.pyx | 13 + tests/run/for_from_pyvar_loop_T601.pyx | 3 +- tests/run/for_from_pyvar_loop_T601_extern_def.h | 2 - tests/run/for_in_range_T372.pyx | 7 + tests/run/fstring.pyx | 140 ++++- tests/run/line_profile_test.srctree | 44 ++ tests/run/list.pyx | 149 ++++- tests/run/numpy_subarray.pyx | 10 +- tests/run/numpy_test.pyx | 23 +- tests/run/or.pyx | 15 + tests/run/overflow_check.pxi | 4 +- tests/run/pep526_variable_annotations.py | 59 +- tests/run/pure_py.py | 31 + tests/run/py35_asyncio_async_def.srctree | 58 ++ tests/run/py35_pep492_interop.pyx | 73 ++- tests/run/reduce_pickle.pyx | 3 + tests/run/set.pyx | 26 + tests/run/set_iter.pyx | 99 +++ tests/run/staticmethod.pyx | 8 + tests/run/strmethods.pyx | 20 + tests/run/test_coroutines_pep492.pyx | 25 + tests/run/tss.pyx | 75 +++ tests/run/tuple.pyx | 26 + tests/run/type_inference.pyx | 12 +- tests/run/unicodefunction.pyx | 43 ++ tests/run/verbatiminclude.h | 6 + tests/run/verbatiminclude.pyx | 62 ++ 173 files changed, 6454 insertions(+), 1897 deletions(-) create mode 100644 Cython/StringIOTree.pxd create mode 100644 tests/buffers/userbuffer.pyx create mode 100644 tests/build/cythonize_rename_ext.srctree create mode 100644 tests/build/setuptools_reimport.srctree delete mode 100644 tests/compile/cnamespec.h create mode 100644 tests/compile/min_async.pyx create mode 100644 tests/compile/verbatiminclude_cimport.srctree create mode 100644 tests/errors/cpp_class_gil_GH1986.pyx create mode 100644 tests/memoryview/numpy_memoryview_readonly.pyx create mode 100644 tests/run/cdef_multiple_inheritance.pyx create mode 100644 tests/run/cdef_multiple_inheritance_errors.srctree create mode 100644 tests/run/cdef_multiple_inheritance_nodict.pyx create mode 100644 tests/run/cpdef_pickle.srctree create mode 100644 tests/run/dict_pop.pyx create mode 100644 tests/run/extern_include_order.srctree delete mode 100644 tests/run/for_from_pyvar_loop_T601_extern_def.h create mode 100644 tests/run/py35_asyncio_async_def.srctree create mode 100644 tests/run/set_iter.pyx create mode 100644 tests/run/tss.pyx create mode 100644 tests/run/verbatiminclude.h create mode 100644 tests/run/verbatiminclude.pyx diff --git a/.travis.yml b/.travis.yml index 19a5c9f..13cb868 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,10 +4,17 @@ sudo: false addons: apt: + sources: + - ubuntu-toolchain-r-test packages: - gdb - python-dbg - python3-dbg + - gcc-6 + - g++-6 + # GCC-7 currently takes 5-7 *minutes* to download on travis + #- gcc-7 + #- g++-7 cache: pip: true @@ -32,14 +39,16 @@ env: - USE_CCACHE=1 - CCACHE_SLOPPINESS=pch_defines,time_macros - CCACHE_COMPRESS=1 - - CCACHE_MAXSIZE=100M - - PATH="/usr/lib/ccache:$PATH" + - CCACHE_MAXSIZE=150M + - PATH="/usr/lib/ccache:$HOME/gcc-symlinks:$PATH" matrix: - BACKEND=c - BACKEND=cpp matrix: include: + #- python: 3.7-dev + # env: BACKEND=c PY=3 CC=gcc-7 - os: osx osx_image: xcode6.4 env: BACKEND=c PY=2 @@ -85,6 +94,15 @@ branches: before_install: - | + if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + mkdir "$HOME/gcc-symlinks" + ln -s /usr/bin/gcc-6 $HOME/gcc-symlinks/gcc + ln -s /usr/bin/g++-6 $HOME/gcc-symlinks/g++ + + if [ -n "$CC" ]; then "$CC" --version; else gcc --version; fi + fi + + - | if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then # Install Miniconda curl -s -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda$PY-latest-MacOSX-x86_64.sh; bash miniconda.sh -b -p $HOME/miniconda && rm miniconda.sh; @@ -94,7 +112,7 @@ before_install: install: - python -c 'import sys; print("Python %s" % (sys.version,))' - - if [ -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##2.6*}" ]; then pip install -r test-requirements.txt $( [ -z "${TRAVIS_PYTHON_VERSION##pypy*}" ] || echo " -r test-requirements-cpython.txt" ) ; fi + - if [ -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##2.6*}" ]; then pip install -r test-requirements.txt $( [ -z "${TRAVIS_PYTHON_VERSION##pypy*}" ] || echo " -r test-requirements-cpython.txt" ) $( [ -n "${TRAVIS_PYTHON_VERSION##3.3*}" ] || echo " tornado<5.0" ) ; fi - CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build before_script: ccache -s || true diff --git a/CHANGES.rst b/CHANGES.rst index e4ac35e..e0d275d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,161 @@ Cython Changelog ================ +0.28 (2018-03-13) +================= + +Features added +-------------- + +* Cdef classes can now multiply inherit from ordinary Python classes. + (The primary base must still be a c class, possibly ``object``, and + the other bases must *not* be cdef classes.) + +* Type inference is now supported for Pythran compiled NumPy expressions. + Patch by Nils Braun. (Github issue #1954) + +* The ``const`` modifier can be applied to memoryview declarations to allow + read-only buffers as input. (Github issues #1605, #1869) + +* C code in the docstring of a ``cdef extern`` block is copied verbatimly + into the generated file. + Patch by Jeroen Demeyer. (Github issue #1915) + +* When compiling with gcc, the module init function is now tuned for small + code size instead of whatever compile flags were provided externally. + Cython now also disables some code intensive optimisations in that function + to further reduce the code size. (Github issue #2102) + +* Decorating an async coroutine with ``@cython.iterable_coroutine`` changes its + type at compile time to make it iterable. While this is not strictly in line + with PEP-492, it improves the interoperability with old-style coroutines that + use ``yield from`` instead of ``await``. + +* The IPython magic has preliminary support for JupyterLab. + (Github issue #1775) + +* The new TSS C-API in CPython 3.7 is supported and has been backported. + Patch by Naotoshi Seo. (Github issue #1932) + +* Cython knows the new ``Py_tss_t`` type defined in PEP-539 and automatically + initialises variables declared with that type to ``Py_tss_NEEDS_INIT``, + a value which cannot be used outside of static assignments. + +* The set methods ``.remove()`` and ``.discard()`` are optimised. + Patch by Antoine Pitrou. (Github issue #2042) + +* ``dict.pop()`` is optimised. + Original patch by Antoine Pitrou. (Github issue #2047) + +* Iteration over sets and frozensets is optimised. + (Github issue #2048) + +* Safe integer loops (< range(2^30)) are automatically optimised into C loops. + +* ``alist.extend([a,b,c])`` is optimised into sequential ``list.append()`` calls + for short literal sequences. + +* Calls to builtin methods that are not specifically optimised into C-API calls + now use a cache that avoids repeated lookups of the underlying C function. + (Github issue #2054) + +* Single argument function calls can avoid the argument tuple creation in some cases. + +* Some redundant extension type checks are avoided. + +* Formatting C enum values in f-strings is faster, as well as some other special cases. + +* String formatting with the '%' operator is optimised into f-strings in simple cases. + +* Subscripting (item access) is faster in some cases. + +* Some ``bytearray`` operations have been optimised similar to ``bytes``. + +* Some PEP-484/526 container type declarations are now considered for + loop optimisations. + +* Indexing into memoryview slices with ``view[i][j]`` is now optimised into + ``view[i, j]``. + +* Python compatible ``cython.*`` types can now be mixed with type declarations + in Cython syntax. + +* Name lookups in the module and in classes are faster. + +* Python attribute lookups on extension types without instance dict are faster. + +* Some missing signals were added to ``libc/signal.pxd``. + Patch by Jeroen Demeyer. (Github issue #1914) + +* The warning about repeated extern declarations is now visible by default. + (Github issue #1874) + +* The exception handling of the function types used by CPython's type slot + functions was corrected to match the de-facto standard behaviour, so that + code that uses them directly benefits from automatic and correct exception + propagation. Patch by Jeroen Demeyer. (Github issue #1980) + +* Defining the macro ``CYTHON_NO_PYINIT_EXPORT`` will prevent the module init + function from being exported as symbol, e.g. when linking modules statically + in an embedding setup. Patch by AraHaan. (Github issue #1944) + +Bugs fixed +---------- + +* If a module name is explicitly provided for an ``Extension()`` that is compiled + via ``cythonize()``, it was previously ignored and replaced by the source file + name. It can now be used to override the target module name, e.g. for compiling + prefixed accelerator modules from Python files. (Github issue #2038) + +* The arguments of the ``num_threads`` parameter of parallel sections + were not sufficiently validated and could lead to invalid C code. + (Github issue #1957) + +* Catching exceptions with a non-trivial exception pattern could call into + CPython with a live exception set. This triggered incorrect behaviour + and crashes, especially in CPython 3.7. + +* The signature of the special ``__richcmp__()`` method was corrected to recognise + the type of the first argument as ``self``. It was previously treated as plain + object, but CPython actually guarantees that it always has the correct type. + Note: this can change the semantics of user code that previously relied on + ``self`` being untyped. + +* Some Python 3 exceptions were not recognised as builtins when running Cython + under Python 2. + +* Some async helper functions were not defined in the generated C code when + compiling simple async code. (Github issue #2075) + +* Line tracing did not include generators and coroutines. + (Github issue #1949) + +* C++ declarations for ``unordered_map`` were corrected. + Patch by Michael Schatzow. (Github issue #1484) + +* Iterator declarations in C++ ``deque`` and ``vector`` were corrected. + Patch by Alex Huszagh. (Github issue #1870) + +* The const modifiers in the C++ ``string`` declarations were corrected, together + with the coercion behaviour of string literals into C++ strings. + (Github issue #2132) + +* Some declaration types in ``libc.limits`` were corrected. + Patch by Jeroen Demeyer. (Github issue #2016) + +* ``@cython.final`` was not accepted on Python classes with an ``@cython.cclass`` + decorator. (Github issue #2040) + +* Cython no longer creates useless and incorrect ``PyInstanceMethod`` wrappers for + methods in Python 3. Patch by Jeroen Demeyer. (Github issue #2105) + +* The builtin ``bytearray`` type could not be used as base type of cdef classes. + (Github issue #2106) + +Other changes +------------- + + 0.27.3 (2017-11-03) =================== @@ -419,7 +574,8 @@ Features added * The new METH_FASTCALL calling convention for PyCFunctions is supported in CPython 3.6. See https://bugs.python.org/issue27810 -* Initial support for using Cython modules in Pyston. Patch by Daetalus. +* Initial support for using Cython modules in Pyston. + Patch by Boxiang Sun. * Dynamic Python attributes are allowed on cdef classes if an attribute ``cdef dict __dict__`` is declared in the class. Patch by empyrical. diff --git a/Cython/Build/Dependencies.py b/Cython/Build/Dependencies.py index fa5d199..666307f 100644 --- a/Cython/Build/Dependencies.py +++ b/Cython/Build/Dependencies.py @@ -3,9 +3,17 @@ from __future__ import absolute_import, print_function import cython from .. import __version__ +import os +import shutil +import hashlib +import subprocess import collections -import re, os, sys, time +import re, sys, time from glob import iglob +from io import open as io_open +from os.path import relpath as _relpath +from distutils.extension import Extension +from distutils.util import strtobool try: import gzip @@ -14,34 +22,6 @@ try: except ImportError: gzip_open = open gzip_ext = '' -import shutil -import subprocess -import os - -try: - import hashlib -except ImportError: - import md5 as hashlib - -try: - from io import open as io_open -except ImportError: - from codecs import open as io_open - -try: - from os.path import relpath as _relpath -except ImportError: - # Py<2.6 - def _relpath(path, start=os.path.curdir): - if not path: - raise ValueError("no path specified") - start_list = os.path.abspath(start).split(os.path.sep) - path_list = os.path.abspath(path).split(os.path.sep) - i = len(os.path.commonprefix([start_list, path_list])) - rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:] - if not rel_list: - return os.path.curdir - return os.path.join(*rel_list) try: import pythran @@ -50,9 +30,6 @@ try: except: PythranAvailable = False -from distutils.extension import Extension -from distutils.util import strtobool - from .. import Utils from ..Utils import (cached_function, cached_method, path_exists, safe_makedirs, copy_file_to_dir_if_newer, is_package_dir) @@ -777,11 +754,11 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= cython_sources = [s for s in pattern.sources if os.path.splitext(s)[1] in ('.py', '.pyx')] if cython_sources: - filepattern = cython_sources[0] - if len(cython_sources) > 1: - print("Warning: Multiple cython sources found for extension '%s': %s\n" - "See http://cython.readthedocs.io/en/latest/src/userguide/sharing_declarations.html " - "for sharing declarations among Cython files." % (pattern.name, cython_sources)) + filepattern = cython_sources[0] + if len(cython_sources) > 1: + print("Warning: Multiple cython sources found for extension '%s': %s\n" + "See http://cython.readthedocs.io/en/latest/src/userguide/sharing_declarations.html " + "for sharing declarations among Cython files." % (pattern.name, cython_sources)) else: # ignore non-cython modules module_list.append(pattern) @@ -800,16 +777,16 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= for file in nonempty(sorted(extended_iglob(filepattern)), "'%s' doesn't match any files" % filepattern): if os.path.abspath(file) in to_exclude: continue - pkg = deps.package(file) module_name = deps.fully_qualified_name(file) if '*' in name: if module_name in explicit_modules: continue - elif name != module_name: - print("Warning: Extension name '%s' does not match fully qualified name '%s' of '%s'" % ( - name, module_name, file)) + elif name: module_name = name + if module_name == 'cython': + raise ValueError('cython is a special module, cannot be used as a module name') + if module_name not in seen: try: kwds = deps.distutils_info(file, aliases, base).values @@ -921,22 +898,33 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, deps = create_dependency_tree(ctx, quiet=quiet) build_dir = getattr(options, 'build_dir', None) - modules_by_cfile = {} + def copy_to_build_dir(filepath, root=os.getcwd()): + filepath_abs = os.path.abspath(filepath) + if os.path.isabs(filepath): + filepath = filepath_abs + if filepath_abs.startswith(root): + # distutil extension depends are relative to cwd + mod_dir = join_path(build_dir, + os.path.dirname(_relpath(filepath, root))) + copy_once_if_newer(filepath_abs, mod_dir) + + modules_by_cfile = collections.defaultdict(list) to_compile = [] for m in module_list: if build_dir: - root = os.getcwd() # distutil extension depends are relative to cwd - def copy_to_build_dir(filepath, root=root): - filepath_abs = os.path.abspath(filepath) - if os.path.isabs(filepath): - filepath = filepath_abs - if filepath_abs.startswith(root): - mod_dir = join_path(build_dir, - os.path.dirname(_relpath(filepath, root))) - copy_once_if_newer(filepath_abs, mod_dir) for dep in m.depends: copy_to_build_dir(dep) + cy_sources = [ + source for source in m.sources + if os.path.splitext(source)[1] in ('.pyx', '.py')] + if len(cy_sources) == 1: + # normal "special" case: believe the Extension module name to allow user overrides + full_module_name = m.name + else: + # infer FQMN from source files + full_module_name = None + new_sources = [] for source in m.sources: base, ext = os.path.splitext(source) @@ -981,13 +969,12 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, fingerprint = deps.transitive_fingerprint(source, extra) else: fingerprint = None - to_compile.append((priority, source, c_file, fingerprint, quiet, - options, not exclude_failures, module_metadata.get(m.name))) + to_compile.append(( + priority, source, c_file, fingerprint, quiet, + options, not exclude_failures, module_metadata.get(m.name), + full_module_name)) new_sources.append(c_file) - if c_file not in modules_by_cfile: - modules_by_cfile[c_file] = [m] - else: - modules_by_cfile[c_file].append(m) + modules_by_cfile[c_file].append(m) else: new_sources.append(source) if build_dir: @@ -1103,17 +1090,15 @@ else: # TODO: Share context? Issue: pyx processing leaks into pxd module @record_results -def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_failure=True, embedded_metadata=None, progress=""): - from ..Compiler.Main import compile, default_options +def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, + raise_on_failure=True, embedded_metadata=None, full_module_name=None, + progress=""): + from ..Compiler.Main import compile_single, default_options from ..Compiler.Errors import CompileError, PyrexError if fingerprint: if not os.path.exists(options.cache): - try: - os.mkdir(options.cache) - except: - if not os.path.exists(options.cache): - raise + safe_makedirs(options.cache) # Cython-generated c files are highly compressible. # (E.g. a compression ratio of about 10 for Sage). fingerprint_file = join_path( @@ -1141,7 +1126,7 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_f any_failures = 0 try: - result = compile([pyx_file], options) + result = compile_single(pyx_file, options, full_module_name=full_module_name) if result.num_errors > 0: any_failures = 1 except (EnvironmentError, PyrexError) as e: diff --git a/Cython/CodeWriter.py b/Cython/CodeWriter.py index 5786495..2e4646a 100644 --- a/Cython/CodeWriter.py +++ b/Cython/CodeWriter.py @@ -363,7 +363,7 @@ class CodeWriter(DeclarationWriter): self.dedent() def visit_IfStatNode(self, node): - # The IfClauseNode is handled directly without a seperate match + # The IfClauseNode is handled directly without a separate match # for clariy. self.startline(u"if ") self.visit(node.if_clauses[0].condition) diff --git a/Cython/Compiler/Annotate.py b/Cython/Compiler/Annotate.py index 33f68cd..5feac02 100644 --- a/Cython/Compiler/Annotate.py +++ b/Cython/Compiler/Annotate.py @@ -79,14 +79,6 @@ class AnnotationCCodeWriter(CCodeWriter): css.append(HtmlFormatter().get_style_defs('.cython')) return '\n'.join(css) - _js = """ - function toggleDiv(id) { - theDiv = id.nextElementSibling - if (theDiv.style.display != 'block') theDiv.style.display = 'block'; - else theDiv.style.display = 'none'; - } - """.strip() - _css_template = textwrap.dedent(""" body.cython { font-family: courier; font-size: 12; } @@ -114,6 +106,14 @@ class AnnotationCCodeWriter(CCodeWriter): .cython.code .c_call { color: #0000FF; } """) + # on-click toggle function to show/hide C source code + _onclick_attr = ' onclick="{0}"'.format(( + "(function(s){" + " s.display = s.display === 'block' ? 'none' : 'block'" + "})(this.nextElementSibling.style)" + ).replace(' ', '') # poor dev's JS minification + ) + def save_annotation(self, source_filename, target_filename, coverage_xml=None): with Utils.open_source_file(source_filename) as f: code = f.read() @@ -141,9 +141,6 @@ class AnnotationCCodeWriter(CCodeWriter): -

Generated by Cython {watermark}{more_info}

@@ -151,7 +148,7 @@ class AnnotationCCodeWriter(CCodeWriter): Yellow lines hint at Python interaction.
Click on a line that starts with a "+" to see the C code that Cython generated for it.

- ''').format(css=self._css(), js=self._js, watermark=Version.watermark, + ''').format(css=self._css(), watermark=Version.watermark, filename=os.path.basename(source_filename) if source_filename else '', more_info=coverage_info) ] @@ -253,7 +250,7 @@ class AnnotationCCodeWriter(CCodeWriter): calls['py_macro_api'] + calls['pyx_macro_api']) if c_code: - onclick = " onclick='toggleDiv(this)'" + onclick = self._onclick_attr expandsymbol = '+' else: onclick = '' diff --git a/Cython/Compiler/Buffer.py b/Cython/Compiler/Buffer.py index 3c7c9bb..04385b4 100644 --- a/Cython/Compiler/Buffer.py +++ b/Cython/Compiler/Buffer.py @@ -326,7 +326,7 @@ def put_acquire_arg_buffer(entry, code, pos): code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" % entry.type.dtype.struct_nesting_depth()) code.putln(code.error_goto_if("%s == -1" % getbuffer, pos)) code.putln("}") - # An exception raised in arg parsing cannot be catched, so no + # An exception raised in arg parsing cannot be caught, so no # need to care about the buffer then. put_unpack_buffer_aux_into_scope(entry, code) @@ -370,7 +370,7 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buf_entry, pybuffernd_struct = buffer_aux.buflocal_nd_var.cname flags = get_flags(buffer_aux, buffer_type) - code.putln("{") # Set up necesarry stack for getbuffer + code.putln("{") # Set up necessary stack for getbuffer code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" % buffer_type.dtype.struct_nesting_depth()) getbuffer = get_getbuffer_call(code, "%s", buffer_aux, buffer_type) # fill in object below @@ -617,7 +617,7 @@ class GetAndReleaseBufferUtilityCode(object): def mangle_dtype_name(dtype): - # Use prefixes to seperate user defined types from builtins + # Use prefixes to separate user defined types from builtins # (consider "typedef float unsigned_int") if dtype.is_pyobject: return "object" @@ -636,7 +636,7 @@ def get_type_information_cname(code, dtype, maxdepth=None): and return the name of the type info struct. Structs with two floats of the same size are encoded as complex numbers. - One can seperate between complex numbers declared as struct or with native + One can separate between complex numbers declared as struct or with native encoding by inspecting to see if the fields field of the type is filled in. """ diff --git a/Cython/Compiler/Builtin.py b/Cython/Compiler/Builtin.py index c9ed656..f8ee614 100644 --- a/Cython/Compiler/Builtin.py +++ b/Cython/Compiler/Builtin.py @@ -328,7 +328,10 @@ builtin_types_table = [ ("set", "PySet_Type", [BuiltinMethod("__contains__", "TO", "b", "PySequence_Contains"), BuiltinMethod("clear", "T", "r", "PySet_Clear"), # discard() and remove() have a special treatment for unhashable values -# BuiltinMethod("discard", "TO", "r", "PySet_Discard"), + BuiltinMethod("discard", "TO", "r", "__Pyx_PySet_Discard", + utility_code=UtilityCode.load("py_set_discard", "Optimize.c")), + BuiltinMethod("remove", "TO", "r", "__Pyx_PySet_Remove", + utility_code=UtilityCode.load("py_set_remove", "Optimize.c")), # update is actually variadic (see Github issue #1645) # BuiltinMethod("update", "TO", "r", "__Pyx_PySet_Update", # utility_code=UtilityCode.load_cached("PySet_Update", "Builtins.c")), @@ -388,6 +391,8 @@ def init_builtin_types(): utility = builtin_utility_code.get(name) if name == 'frozenset': objstruct_cname = 'PySetObject' + elif name == 'bytearray': + objstruct_cname = 'PyByteArrayObject' elif name == 'bool': objstruct_cname = None elif name == 'Exception': diff --git a/Cython/Compiler/CmdLine.py b/Cython/Compiler/CmdLine.py index e913810..a587324 100644 --- a/Cython/Compiler/CmdLine.py +++ b/Cython/Compiler/CmdLine.py @@ -154,6 +154,8 @@ def parse_command_line(args): options.capi_reexport_cincludes = True elif option == "--fast-fail": Options.fast_fail = True + elif option == "--cimport-from-pyx": + Options.cimport_from_pyx = True elif option in ('-Werror', '--warning-errors'): Options.warning_errors = True elif option in ('-Wextra', '--warning-extra'): diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py index 92044be..0974ab3 100644 --- a/Cython/Compiler/Code.py +++ b/Cython/Compiler/Code.py @@ -60,10 +60,42 @@ basicsize_builtins_map = { } uncachable_builtins = [ - # builtin names that cannot be cached because they may or may not - # be available at import time + # Global/builtin names that cannot be cached because they may or may not + # be available at import time, for various reasons: + ## - Py3.7+ + 'breakpoint', # might deserve an implementation in Cython + ## - Py3.4+ + '__loader__', + '__spec__', + ## - Py3+ + 'BlockingIOError', + 'BrokenPipeError', + 'ChildProcessError', + 'ConnectionAbortedError', + 'ConnectionError', + 'ConnectionRefusedError', + 'ConnectionResetError', + 'FileExistsError', + 'FileNotFoundError', + 'InterruptedError', + 'IsADirectoryError', + 'ModuleNotFoundError', + 'NotADirectoryError', + 'PermissionError', + 'ProcessLookupError', + 'RecursionError', + 'ResourceWarning', + #'StopAsyncIteration', # backported + 'TimeoutError', + '__build_class__', + 'ascii', # might deserve an implementation in Cython + #'exec', # implemented in Cython + ## - Py2.7+ + 'memoryview', + ## - platform specific 'WindowsError', - '_', # e.g. gettext + ## - others + '_', # e.g. used by gettext ] special_py_methods = set([ @@ -80,6 +112,82 @@ modifier_output_mapper = { is_self_assignment = re.compile(r" *(\w+) = (\1);\s*$").match +class IncludeCode(object): + """ + An include file and/or verbatim C code to be included in the + generated sources. + """ + # attributes: + # + # pieces {order: unicode}: pieces of C code to be generated. + # For the included file, the key "order" is zero. + # For verbatim include code, the "order" is the "order" + # attribute of the original IncludeCode where this piece + # of C code was first added. This is needed to prevent + # duplication if the same include code is found through + # multiple cimports. + # location int: where to put this include in the C sources, one + # of the constants INITIAL, EARLY, LATE + # order int: sorting order (automatically set by increasing counter) + + # Constants for location. If the same include occurs with different + # locations, the earliest one takes precedense. + INITIAL = 0 + EARLY = 1 + LATE = 2 + + counter = 1 # Counter for "order" + + def __init__(self, include=None, verbatim=None, late=True, initial=False): + self.order = self.counter + type(self).counter += 1 + self.pieces = {} + + if include: + if include[0] == '<' and include[-1] == '>': + self.pieces[0] = u'#include {0}'.format(include) + late = False # system include is never late + else: + self.pieces[0] = u'#include "{0}"'.format(include) + + if verbatim: + self.pieces[self.order] = verbatim + + if initial: + self.location = self.INITIAL + elif late: + self.location = self.LATE + else: + self.location = self.EARLY + + def dict_update(self, d, key): + """ + Insert `self` in dict `d` with key `key`. If that key already + exists, update the attributes of the existing value with `self`. + """ + if key in d: + other = d[key] + other.location = min(self.location, other.location) + other.pieces.update(self.pieces) + else: + d[key] = self + + def sortkey(self): + return self.order + + def mainpiece(self): + """ + Return the main piece of C code, corresponding to the include + file. If there was no include file, return None. + """ + return self.pieces.get(0) + + def write(self, code): + # Write values of self.pieces dict, sorted by the keys + for k in sorted(self.pieces): + code.putln(self.pieces[k]) + + def get_utility_dir(): # make this a function and not global variables: # http://trac.cython.org/cython_trac/ticket/475 @@ -332,7 +440,7 @@ class UtilityCode(UtilityCodeBase): hashes/equals by instance proto C prototypes - impl implemenation code + impl implementation code init code to call on module initialization requires utility code dependencies proto_block the place in the resulting file where the prototype should @@ -406,21 +514,22 @@ class UtilityCode(UtilityCodeBase): def inject_string_constants(self, impl, output): """Replace 'PYIDENT("xyz")' by a constant Python identifier cname. """ - if 'PYIDENT(' not in impl: + if 'PYIDENT(' not in impl and 'PYUNICODE(' not in impl: return False, impl replacements = {} def externalise(matchobj): - name = matchobj.group(1) + key = matchobj.groups() try: - cname = replacements[name] + cname = replacements[key] except KeyError: - cname = replacements[name] = output.get_interned_identifier( - StringEncoding.EncodedString(name)).cname + str_type, name = key + cname = replacements[key] = output.get_py_string_const( + StringEncoding.EncodedString(name), identifier=str_type == 'IDENT').cname return cname - impl = re.sub(r'PYIDENT\("([^"]+)"\)', externalise, impl) - assert 'PYIDENT(' not in impl + impl = re.sub(r'PY(IDENT|UNICODE)\("([^"]+)"\)', externalise, impl) + assert 'PYIDENT(' not in impl and 'PYUNICODE(' not in impl return bool(replacements), impl def inject_unbound_methods(self, impl, output): @@ -431,21 +540,18 @@ class UtilityCode(UtilityCodeBase): utility_code = set() def externalise(matchobj): - type_cname, method_name, args = matchobj.groups() - args = [arg.strip() for arg in args[1:].split(',')] - if len(args) == 1: - call = '__Pyx_CallUnboundCMethod0' - utility_code.add("CallUnboundCMethod0") - elif len(args) == 2: - call = '__Pyx_CallUnboundCMethod1' - utility_code.add("CallUnboundCMethod1") - else: - assert False, "CALL_UNBOUND_METHOD() requires 1 or 2 call arguments" - - cname = output.get_cached_unbound_method(type_cname, method_name, len(args)) - return '%s(&%s, %s)' % (call, cname, ', '.join(args)) - - impl = re.sub(r'CALL_UNBOUND_METHOD\(([a-zA-Z_]+),\s*"([^"]+)"((?:,\s*[^),]+)+)\)', externalise, impl) + type_cname, method_name, obj_cname, args = matchobj.groups() + args = [arg.strip() for arg in args[1:].split(',')] if args else [] + assert len(args) < 3, "CALL_UNBOUND_METHOD() does not support %d call arguments" % len(args) + return output.cached_unbound_method_call_code(obj_cname, type_cname, method_name, args) + + impl = re.sub( + r'CALL_UNBOUND_METHOD\(' + r'([a-zA-Z_]+),' # type cname + r'\s*"([^"]+)",' # method name + r'\s*([^),]+)' # object cname + r'((?:,\s*[^),]+)*)' # args* + r'\)', externalise, impl) assert 'CALL_UNBOUND_METHOD(' not in impl for helper in sorted(utility_code): @@ -985,6 +1091,7 @@ class GlobalState(object): 'global_var', 'string_decls', 'decls', + 'late_includes', 'all_the_rest', 'pystring_table', 'cached_builtins', @@ -1239,8 +1346,8 @@ class GlobalState(object): prefix = Naming.const_prefix return "%s%s" % (prefix, name_suffix) - def get_cached_unbound_method(self, type_cname, method_name, args_count): - key = (type_cname, method_name, args_count) + def get_cached_unbound_method(self, type_cname, method_name): + key = (type_cname, method_name) try: cname = self.cached_cmethods[key] except KeyError: @@ -1248,6 +1355,18 @@ class GlobalState(object): 'umethod', '%s_%s' % (type_cname, method_name)) return cname + def cached_unbound_method_call_code(self, obj_cname, type_cname, method_name, arg_cnames): + # admittedly, not the best place to put this method, but it is reused by UtilityCode and ExprNodes ... + utility_code_name = "CallUnboundCMethod%d" % len(arg_cnames) + self.use_utility_code(UtilityCode.load_cached(utility_code_name, "ObjectHandling.c")) + cache_cname = self.get_cached_unbound_method(type_cname, method_name) + args = [obj_cname] + arg_cnames + return "__Pyx_%s(&%s, %s)" % ( + utility_code_name, + cache_cname, + ', '.join(args), + ) + def add_cached_builtin_decl(self, entry): if entry.is_builtin and entry.is_const: if self.should_declare(entry.cname, entry): @@ -1300,7 +1419,7 @@ class GlobalState(object): decl = self.parts['decls'] init = self.parts['init_globals'] cnames = [] - for (type_cname, method_name, _), cname in sorted(self.cached_cmethods.items()): + for (type_cname, method_name), cname in sorted(self.cached_cmethods.items()): cnames.append(cname) method_name_cname = self.get_interned_identifier(StringEncoding.EncodedString(method_name)).cname decl.putln('static __Pyx_CachedCFunction %s = {0, &%s, 0, 0, 0};' % ( @@ -1520,7 +1639,7 @@ class CCodeWriter(object): as well - labels, temps, exc_vars: One must construct a scope in which these can exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for - sanity checking and forward compatabilty). Created insertion points + sanity checking and forward compatibility). Created insertion points looses this scope and cannot access it. - marker: Not copied to insertion point - filename_table, filename_list, input_file_contents: All codewriters @@ -1942,8 +2061,8 @@ class CCodeWriter(object): self.put_xdecref_memoryviewslice(cname, have_gil=have_gil) return - prefix = nanny and '__Pyx' or 'Py' - X = null_check and 'X' or '' + prefix = '__Pyx' if nanny else 'Py' + X = 'X' if null_check else '' if clear: if clear_before_decref: @@ -2293,6 +2412,7 @@ class CCodeWriter(object): self.putln(" #define unlikely(x) __builtin_expect(!!(x), 0)") self.putln("#endif") + class PyrexCodeWriter(object): # f file output file # level int indentation level diff --git a/Cython/Compiler/CodeGeneration.py b/Cython/Compiler/CodeGeneration.py index 6805aa9..e64049c 100644 --- a/Cython/Compiler/CodeGeneration.py +++ b/Cython/Compiler/CodeGeneration.py @@ -12,7 +12,7 @@ class ExtractPxdCode(VisitorTransform): The result is a tuple (StatListNode, ModuleScope), i.e. everything that is needed from the pxd after it is processed. - A purer approach would be to seperately compile the pxd code, + A purer approach would be to separately compile the pxd code, but the result would have to be slightly more sophisticated than pure strings (functions + wanted interned strings + wanted utility code + wanted cached objects) so for now this diff --git a/Cython/Compiler/CythonScope.py b/Cython/Compiler/CythonScope.py index 00b912a..1c25d1a 100644 --- a/Cython/Compiler/CythonScope.py +++ b/Cython/Compiler/CythonScope.py @@ -26,6 +26,10 @@ class CythonScope(ModuleScope): cname='') entry.in_cinclude = True + def is_cpp(self): + # Allow C++ utility code in C++ contexts. + return self.context.cpp + def lookup_type(self, name): # This function should go away when types are all first-level objects. type = parse_basic_type(name) diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index dfe46f8..7e10c6d 100644 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -870,16 +870,19 @@ class ExprNode(Node): elif not src_type.is_error: error(self.pos, "Cannot convert '%s' to memoryviewslice" % (src_type,)) - elif not src.type.conforms_to(dst_type, broadcast=self.is_memview_broadcast, - copying=self.is_memview_copy_assignment): - if src.type.dtype.same_as(dst_type.dtype): - msg = "Memoryview '%s' not conformable to memoryview '%s'." - tup = src.type, dst_type - else: - msg = "Different base types for memoryviews (%s, %s)" - tup = src.type.dtype, dst_type.dtype + else: + if src.type.writable_needed: + dst_type.writable_needed = True + if not src.type.conforms_to(dst_type, broadcast=self.is_memview_broadcast, + copying=self.is_memview_copy_assignment): + if src.type.dtype.same_as(dst_type.dtype): + msg = "Memoryview '%s' not conformable to memoryview '%s'." + tup = src.type, dst_type + else: + msg = "Different base types for memoryviews (%s, %s)" + tup = src.type.dtype, dst_type.dtype - error(self.pos, msg % tup) + error(self.pos, msg % tup) elif dst_type.is_pyobject: if not src.type.is_pyobject: @@ -1081,6 +1084,12 @@ class NoneNode(PyConstNode): def may_be_none(self): return True + def coerce_to(self, dst_type, env): + if not (dst_type.is_pyobject or dst_type.is_memoryviewslice or dst_type.is_error): + # Catch this error early and loudly. + error(self.pos, "Cannot assign None to %s" % dst_type) + return super(NoneNode, self).coerce_to(dst_type, env) + class EllipsisNode(PyConstNode): # '...' in a subscript list. @@ -1433,7 +1442,7 @@ class BytesNode(ConstNode): node.type = Builtin.bytes_type else: self.check_for_coercion_error(dst_type, env, fail=True) - return node + return node elif dst_type in (PyrexTypes.c_char_ptr_type, PyrexTypes.c_const_char_ptr_type): node.type = dst_type return node @@ -1442,8 +1451,10 @@ class BytesNode(ConstNode): else PyrexTypes.c_char_ptr_type) return CastNode(node, dst_type) elif dst_type.assignable_from(PyrexTypes.c_char_ptr_type): - node.type = dst_type - return node + # Exclude the case of passing a C string literal into a non-const C++ string. + if not dst_type.is_cpp_class or dst_type.is_const: + node.type = dst_type + return node # We still need to perform normal coerce_to processing on the # result, because we might be coercing to an extension type, @@ -1863,6 +1874,7 @@ class NameNode(AtomicExprNode): if atype is None: atype = unspecified_type if as_target and env.directives['infer_types'] != False else py_object_type self.entry = env.declare_var(name, atype, self.pos, is_cdef=not as_target) + self.entry.annotation = annotation def analyse_as_module(self, env): # Try to interpret this as a reference to a cimported module. @@ -2073,7 +2085,11 @@ class NameNode(AtomicExprNode): def check_const(self): entry = self.entry - if entry is not None and not (entry.is_const or entry.is_cfunction or entry.is_builtin): + if entry is not None and not ( + entry.is_const or + entry.is_cfunction or + entry.is_builtin or + entry.type.is_const): self.not_const() return False return True @@ -2215,7 +2231,8 @@ class NameNode(AtomicExprNode): setter = 'PyDict_SetItem' namespace = Naming.moddict_cname elif entry.is_pyclass_attr: - setter = 'PyObject_SetItem' + code.globalstate.use_utility_code(UtilityCode.load_cached("SetNameInClass", "ObjectHandling.c")) + setter = '__Pyx_SetNameInClass' else: assert False, repr(entry) code.put_error_if_neg( @@ -2288,7 +2305,11 @@ class NameNode(AtomicExprNode): code.putln('%s = %s;' % (self.result(), result)) else: result = rhs.result_as(self.ctype()) - code.putln('%s = %s;' % (self.result(), result)) + + if is_pythran_expr(self.type): + code.putln('new (&%s) decltype(%s){%s};' % (self.result(), self.result(), result)) + else: + code.putln('%s = %s;' % (self.result(), result)) if debug_disposal_code: print("NameNode.generate_assignment_code:") print("...generating post-assignment code for %s" % rhs) @@ -3294,7 +3315,7 @@ class _IndexingBaseNode(ExprNode): # in most cases, indexing will return a safe reference to an object in a container, # so we consider the result safe if the base object is return self.base.is_ephemeral() or self.base.type in ( - basestring_type, str_type, bytes_type, unicode_type) + basestring_type, str_type, bytes_type, bytearray_type, unicode_type) def check_const_addr(self): return self.base.check_const_addr() and self.index.check_const() @@ -3354,7 +3375,7 @@ class IndexNode(_IndexingBaseNode): return False if isinstance(self.index, SliceNode): # slicing! - if base_type in (bytes_type, str_type, unicode_type, + if base_type in (bytes_type, bytearray_type, str_type, unicode_type, basestring_type, list_type, tuple_type): return False return ExprNode.may_be_none(self) @@ -3454,6 +3475,10 @@ class IndexNode(_IndexingBaseNode): if index_func is not None: return index_func.type.return_type + if is_pythran_expr(base_type) and is_pythran_expr(index_type): + index_with_type = (self.index, index_type) + return PythranExpr(pythran_indexing_type(base_type, [index_with_type])) + # may be slicing or indexing, we don't know if base_type in (unicode_type, str_type): # these types always returns their own type on Python indexing/slicing @@ -3579,7 +3604,7 @@ class IndexNode(_IndexingBaseNode): else: # not using 'uchar' to enable fast and safe error reporting as '-1' self.type = PyrexTypes.c_int_type - elif is_slice and base_type in (bytes_type, str_type, unicode_type, list_type, tuple_type): + elif is_slice and base_type in (bytes_type, bytearray_type, str_type, unicode_type, list_type, tuple_type): self.type = base_type else: item_type = None @@ -3680,23 +3705,33 @@ class IndexNode(_IndexingBaseNode): else: indices = [self.index] - base_type = self.base.type + base = self.base + base_type = base.type replacement_node = None if base_type.is_memoryviewslice: # memoryviewslice indexing or slicing from . import MemoryView + if base.is_memview_slice: + # For memory views, "view[i][j]" is the same as "view[i, j]" => use the latter for speed. + merged_indices = base.merged_indices(indices) + if merged_indices is not None: + base = base.base + base_type = base.type + indices = merged_indices have_slices, indices, newaxes = MemoryView.unellipsify(indices, base_type.ndim) if have_slices: - replacement_node = MemoryViewSliceNode(self.pos, indices=indices, base=self.base) + replacement_node = MemoryViewSliceNode(self.pos, indices=indices, base=base) else: - replacement_node = MemoryViewIndexNode(self.pos, indices=indices, base=self.base) + replacement_node = MemoryViewIndexNode(self.pos, indices=indices, base=base) elif base_type.is_buffer or base_type.is_pythran_expr: if base_type.is_pythran_expr or len(indices) == base_type.ndim: # Buffer indexing is_buffer_access = True indices = [index.analyse_types(env) for index in indices] if base_type.is_pythran_expr: - do_replacement = all(index.type.is_int or index.is_slice or index.type.is_pythran_expr for index in indices) + do_replacement = all( + index.type.is_int or index.is_slice or index.type.is_pythran_expr + for index in indices) if do_replacement: for i,index in enumerate(indices): if index.is_slice: @@ -3706,7 +3741,7 @@ class IndexNode(_IndexingBaseNode): else: do_replacement = all(index.type.is_int for index in indices) if do_replacement: - replacement_node = BufferIndexNode(self.pos, indices=indices, base=self.base) + replacement_node = BufferIndexNode(self.pos, indices=indices, base=base) # On cloning, indices is cloned. Otherwise, unpack index into indices. assert not isinstance(self.index, CloneNode) @@ -3873,6 +3908,8 @@ class IndexNode(_IndexingBaseNode): if not self.is_temp: # all handled in self.calculate_result_code() return + + utility_code = None if self.type.is_pyobject: error_value = 'NULL' if self.index.type.is_int: @@ -3882,32 +3919,38 @@ class IndexNode(_IndexingBaseNode): function = "__Pyx_GetItemInt_Tuple" else: function = "__Pyx_GetItemInt" - code.globalstate.use_utility_code( - TempitaUtilityCode.load_cached("GetItemInt", "ObjectHandling.c")) + utility_code = TempitaUtilityCode.load_cached("GetItemInt", "ObjectHandling.c") else: if self.base.type is dict_type: function = "__Pyx_PyDict_GetItem" - code.globalstate.use_utility_code( - UtilityCode.load_cached("DictGetItem", "ObjectHandling.c")) + utility_code = UtilityCode.load_cached("DictGetItem", "ObjectHandling.c") + elif self.base.type is py_object_type and self.index.type in (str_type, unicode_type): + # obj[str] is probably doing a dict lookup + function = "__Pyx_PyObject_Dict_GetItem" + utility_code = UtilityCode.load_cached("DictGetItem", "ObjectHandling.c") else: - function = "PyObject_GetItem" + function = "__Pyx_PyObject_GetItem" + code.globalstate.use_utility_code( + TempitaUtilityCode.load_cached("GetItemInt", "ObjectHandling.c")) + utility_code = UtilityCode.load_cached("ObjectGetItem", "ObjectHandling.c") elif self.type.is_unicode_char and self.base.type is unicode_type: assert self.index.type.is_int function = "__Pyx_GetItemInt_Unicode" error_value = '(Py_UCS4)-1' - code.globalstate.use_utility_code( - UtilityCode.load_cached("GetItemIntUnicode", "StringTools.c")) + utility_code = UtilityCode.load_cached("GetItemIntUnicode", "StringTools.c") elif self.base.type is bytearray_type: assert self.index.type.is_int assert self.type.is_int function = "__Pyx_GetItemInt_ByteArray" error_value = '-1' - code.globalstate.use_utility_code( - UtilityCode.load_cached("GetItemIntByteArray", "StringTools.c")) + utility_code = UtilityCode.load_cached("GetItemIntByteArray", "StringTools.c") elif not (self.base.type.is_cpp_class and self.exception_check): assert False, "unexpected type %s and base type %s for indexing" % ( self.type, self.base.type) + if utility_code is not None: + code.globalstate.use_utility_code(utility_code) + if self.index.type.is_int: index_code = self.index.result() else: @@ -4104,7 +4147,8 @@ class BufferIndexNode(_IndexingBaseNode): def analyse_buffer_index(self, env, getting): if is_pythran_expr(self.base.type): - self.type = PythranExpr(pythran_indexing_type(self.base.type, self.indices)) + index_with_type_list = [(idx, idx.type) for idx in self.indices] + self.type = PythranExpr(pythran_indexing_type(self.base.type, index_with_type_list)) else: self.base = self.base.coerce_to_simple(env) self.type = self.base.type.dtype @@ -4126,10 +4170,6 @@ class BufferIndexNode(_IndexingBaseNode): def nogil_check(self, env): if self.is_buffer_access or self.is_memview_index: - if env.directives['boundscheck']: - warning(self.pos, "Use boundscheck(False) for faster access", - level=1) - if self.type.is_pyobject: error(self.pos, "Cannot access buffer with object dtype without gil") self.type = error_type @@ -4156,6 +4196,11 @@ class BufferIndexNode(_IndexingBaseNode): """ ndarray[1, 2, 3] and memslice[1, 2, 3] """ + if self.in_nogil_context: + if self.is_buffer_access or self.is_memview_index: + if code.globalstate.directives['boundscheck']: + warning(self.pos, "Use boundscheck(False) for faster access", level=1) + # Assign indices to temps of at least (s)size_t to allow further index calculations. index_temps = [self.get_index_in_temp(code,ivar) for ivar in self.indices] @@ -4189,7 +4234,7 @@ class BufferIndexNode(_IndexingBaseNode): if is_pythran_expr(base_type) and is_pythran_supported_type(rhs.type): obj = code.funcstate.allocate_temp(PythranExpr(pythran_type(self.base.type)), manage_ref=False) # We have got to do this because we have to declare pythran objects - # at the beggining of the functions. + # at the beginning of the functions. # Indeed, Cython uses "goto" statement for error management, and # RAII doesn't work with that kind of construction. # Moreover, the way Pythran expressions are made is that they don't @@ -4258,6 +4303,11 @@ class MemoryViewIndexNode(BufferIndexNode): indices = self.indices have_slices, indices, newaxes = MemoryView.unellipsify(indices, self.base.type.ndim) + if not getting: + self.writable_needed = True + if self.base.is_name or self.base.is_attribute: + self.base.entry.type.writable_needed = True + self.memslice_index = (not newaxes and len(indices) == self.base.type.ndim) axes = [] @@ -4405,6 +4455,37 @@ class MemoryViewSliceNode(MemoryViewIndexNode): else: return MemoryCopySlice(self.pos, self) + def merged_indices(self, indices): + """Return a new list of indices/slices with 'indices' merged into the current ones + according to slicing rules. + Is used to implement "view[i][j]" => "view[i, j]". + Return None if the indices cannot (easily) be merged at compile time. + """ + if not indices: + return None + # NOTE: Need to evaluate "self.original_indices" here as they might differ from "self.indices". + new_indices = self.original_indices[:] + indices = indices[:] + for i, s in enumerate(self.original_indices): + if s.is_slice: + if s.start.is_none and s.stop.is_none and s.step.is_none: + # Full slice found, replace by index. + new_indices[i] = indices[0] + indices.pop(0) + if not indices: + return new_indices + else: + # Found something non-trivial, e.g. a partial slice. + return None + elif not s.type.is_int: + # Not a slice, not an integer index => could be anything... + return None + if indices: + if len(new_indices) + len(indices) > self.base.type.ndim: + return None + new_indices += indices + return new_indices + def is_simple(self): if self.is_ellipsis_noop: # TODO: fix SimpleCallNode.is_simple() @@ -4576,7 +4657,7 @@ class SliceIndexNode(ExprNode): return bytes_type elif base_type.is_pyunicode_ptr: return unicode_type - elif base_type in (bytes_type, str_type, unicode_type, + elif base_type in (bytes_type, bytearray_type, str_type, unicode_type, basestring_type, list_type, tuple_type): return base_type elif base_type.is_ptr or base_type.is_array: @@ -5188,6 +5269,32 @@ class CallNode(ExprNode): return False return ExprNode.may_be_none(self) + def set_py_result_type(self, function, func_type=None): + if func_type is None: + func_type = function.type + if func_type is Builtin.type_type and ( + function.is_name and + function.entry and + function.entry.is_builtin and + function.entry.name in Builtin.types_that_construct_their_instance): + # calling a builtin type that returns a specific object type + if function.entry.name == 'float': + # the following will come true later on in a transform + self.type = PyrexTypes.c_double_type + self.result_ctype = PyrexTypes.c_double_type + else: + self.type = Builtin.builtin_types[function.entry.name] + self.result_ctype = py_object_type + self.may_return_none = False + elif function.is_name and function.type_entry: + # We are calling an extension type constructor. As long as we do not + # support __new__(), the result type is clear + self.type = function.type_entry.type + self.result_ctype = py_object_type + self.may_return_none = False + else: + self.type = py_object_type + def analyse_as_type_constructor(self, env): type = self.function.analyse_as_type(env) if type and type.is_struct_or_union: @@ -5271,6 +5378,11 @@ class SimpleCallNode(CallNode): error(self.args[0].pos, "Unknown type") else: return PyrexTypes.CPtrType(type) + elif attr == 'typeof': + if len(self.args) != 1: + error(self.args.pos, "only one type allowed.") + operand = self.args[0].analyse_types(env) + return operand.type def explicit_args_kwds(self): return self.args, None @@ -5301,37 +5413,18 @@ class SimpleCallNode(CallNode): has_pythran_args &= is_pythran_supported_node_or_none(arg) self.is_numpy_call_with_exprs = bool(has_pythran_args) if self.is_numpy_call_with_exprs: - self.args = None env.add_include_file("pythonic/numpy/%s.hpp" % self.function.attribute) - self.type = PythranExpr(pythran_func_type(self.function.attribute, self.arg_tuple.args)) - self.may_return_none = True - self.is_temp = 1 + return NumPyMethodCallNode.from_node( + self, + function=self.function, + arg_tuple=self.arg_tuple, + type=PythranExpr(pythran_func_type(self.function.attribute, self.arg_tuple.args)), + ) elif func_type.is_pyobject: self.arg_tuple = TupleNode(self.pos, args = self.args) self.arg_tuple = self.arg_tuple.analyse_types(env).coerce_to_pyobject(env) self.args = None - if func_type is Builtin.type_type and function.is_name and \ - function.entry and \ - function.entry.is_builtin and \ - function.entry.name in Builtin.types_that_construct_their_instance: - # calling a builtin type that returns a specific object type - if function.entry.name == 'float': - # the following will come true later on in a transform - self.type = PyrexTypes.c_double_type - self.result_ctype = PyrexTypes.c_double_type - else: - self.type = Builtin.builtin_types[function.entry.name] - self.result_ctype = py_object_type - self.may_return_none = False - elif function.is_name and function.type_entry: - # We are calling an extension type constructor. As - # long as we do not support __new__(), the result type - # is clear - self.type = function.type_entry.type - self.result_ctype = py_object_type - self.may_return_none = False - else: - self.type = py_object_type + self.set_py_result_type(function, func_type) self.is_temp = 1 else: self.args = [ arg.analyse_types(env) for arg in self.args ] @@ -5452,8 +5545,6 @@ class SimpleCallNode(CallNode): for i in range(min(max_nargs, actual_nargs)): formal_arg = func_type.args[i] formal_type = formal_arg.type - if formal_type.is_const: - formal_type = formal_type.const_base_type arg = args[i].coerce_to(formal_type, env) if formal_arg.not_none: # C methods must do the None checks at *call* time @@ -5601,29 +5692,64 @@ class SimpleCallNode(CallNode): return False # skip allocation of unused result temp return True + def generate_evaluation_code(self, code): + function = self.function + if function.is_name or function.is_attribute: + code.globalstate.use_entry_utility_code(function.entry) + + if not function.type.is_pyobject or len(self.arg_tuple.args) > 1 or ( + self.arg_tuple.args and self.arg_tuple.is_literal): + super(SimpleCallNode, self).generate_evaluation_code(code) + return + + # Special case 0-args and try to avoid explicit tuple creation for Python calls with 1 arg. + arg = self.arg_tuple.args[0] if self.arg_tuple.args else None + subexprs = (self.self, self.coerced_self, function, arg) + for subexpr in subexprs: + if subexpr is not None: + subexpr.generate_evaluation_code(code) + + code.mark_pos(self.pos) + assert self.is_temp + self.allocate_temp_result(code) + + if arg is None: + code.globalstate.use_utility_code(UtilityCode.load_cached( + "PyObjectCallNoArg", "ObjectHandling.c")) + code.putln( + "%s = __Pyx_PyObject_CallNoArg(%s); %s" % ( + self.result(), + function.py_result(), + code.error_goto_if_null(self.result(), self.pos))) + else: + code.globalstate.use_utility_code(UtilityCode.load_cached( + "PyObjectCallOneArg", "ObjectHandling.c")) + code.putln( + "%s = __Pyx_PyObject_CallOneArg(%s, %s); %s" % ( + self.result(), + function.py_result(), + arg.py_result(), + code.error_goto_if_null(self.result(), self.pos))) + + code.put_gotref(self.py_result()) + + for subexpr in subexprs: + if subexpr is not None: + subexpr.generate_disposal_code(code) + subexpr.free_temps(code) + def generate_result_code(self, code): func_type = self.function_type() - if self.function.is_name or self.function.is_attribute: - code.globalstate.use_entry_utility_code(self.function.entry) if func_type.is_pyobject: - if func_type is not type_type and not self.arg_tuple.args and self.arg_tuple.is_literal: - code.globalstate.use_utility_code(UtilityCode.load_cached( - "PyObjectCallNoArg", "ObjectHandling.c")) - code.putln( - "%s = __Pyx_PyObject_CallNoArg(%s); %s" % ( - self.result(), - self.function.py_result(), - code.error_goto_if_null(self.result(), self.pos))) - else: - arg_code = self.arg_tuple.py_result() - code.globalstate.use_utility_code(UtilityCode.load_cached( - "PyObjectCall", "ObjectHandling.c")) - code.putln( - "%s = __Pyx_PyObject_Call(%s, %s, NULL); %s" % ( - self.result(), - self.function.py_result(), - arg_code, - code.error_goto_if_null(self.result(), self.pos))) + arg_code = self.arg_tuple.py_result() + code.globalstate.use_utility_code(UtilityCode.load_cached( + "PyObjectCall", "ObjectHandling.c")) + code.putln( + "%s = __Pyx_PyObject_Call(%s, %s, NULL); %s" % ( + self.result(), + self.function.py_result(), + arg_code, + code.error_goto_if_null(self.result(), self.pos))) code.put_gotref(self.py_result()) elif func_type.is_cfunction: if self.has_optional_args: @@ -5687,11 +5813,34 @@ class SimpleCallNode(CallNode): if self.has_optional_args: code.funcstate.release_temp(self.opt_arg_struct) - @classmethod - def from_node(cls, node, **kwargs): - ret = super(SimpleCallNode, cls).from_node(node, **kwargs) - ret.is_numpy_call_with_exprs = node.is_numpy_call_with_exprs - return ret + +class NumPyMethodCallNode(SimpleCallNode): + # Pythran call to a NumPy function or method. + # + # function ExprNode the function/method to call + # arg_tuple TupleNode the arguments as an args tuple + + subexprs = ['function', 'arg_tuple'] + is_temp = True + may_return_none = True + + def generate_evaluation_code(self, code): + code.mark_pos(self.pos) + self.allocate_temp_result(code) + + self.function.generate_evaluation_code(code) + assert self.arg_tuple.mult_factor is None + args = self.arg_tuple.args + for arg in args: + arg.generate_evaluation_code(code) + + code.putln("// function evaluation code for numpy function") + code.putln("__Pyx_call_destructor(%s);" % self.result()) + code.putln("new (&%s) decltype(%s){pythonic::numpy::functor::%s{}(%s)};" % ( + self.result(), + self.result(), + self.function.attribute, + ", ".join(a.pythran_result() for a in args))) class PyMethodCallNode(SimpleCallNode): @@ -5714,16 +5863,6 @@ class PyMethodCallNode(SimpleCallNode): for arg in args: arg.generate_evaluation_code(code) - if self.is_numpy_call_with_exprs: - code.putln("// function evaluation code for numpy function") - code.putln("__Pyx_call_destructor(%s);" % self.result()) - code.putln("new (&%s) decltype(%s){pythonic::numpy::functor::%s{}(%s)};" % ( - self.result(), - self.result(), - self.function.attribute, - ", ".join(a.pythran_result() for a in self.arg_tuple.args))) - return - # make sure function is in temp so that we can replace the reference below if it's a method reuse_function_temp = self.function.is_temp if reuse_function_temp: @@ -6034,6 +6173,37 @@ class PythonCapiCallNode(SimpleCallNode): SimpleCallNode.__init__(self, pos, **kwargs) +class CachedBuiltinMethodCallNode(CallNode): + # Python call to a method of a known Python builtin (only created in transforms) + + subexprs = ['obj', 'args'] + is_temp = True + + def __init__(self, call_node, obj, method_name, args): + super(CachedBuiltinMethodCallNode, self).__init__( + call_node.pos, + obj=obj, method_name=method_name, args=args, + may_return_none=call_node.may_return_none, + type=call_node.type) + + def may_be_none(self): + if self.may_return_none is not None: + return self.may_return_none + return ExprNode.may_be_none(self) + + def generate_result_code(self, code): + type_cname = self.obj.type.cname + obj_cname = self.obj.py_result() + args = [arg.py_result() for arg in self.args] + call_code = code.globalstate.cached_unbound_method_call_code( + obj_cname, type_cname, self.method_name, args) + code.putln("%s = %s; %s" % ( + self.result(), call_code, + code.error_goto_if_null(self.result(), self.pos) + )) + code.put_gotref(self.result()) + + class GeneralCallNode(CallNode): # General Python function call, including keyword, # * and ** arguments. @@ -6092,15 +6262,7 @@ class GeneralCallNode(CallNode): self.positional_args = self.positional_args.analyse_types(env) self.positional_args = \ self.positional_args.coerce_to_pyobject(env) - function = self.function - if function.is_name and function.type_entry: - # We are calling an extension type constructor. As long - # as we do not support __new__(), the result type is clear - self.type = function.type_entry.type - self.result_ctype = py_object_type - self.may_return_none = False - else: - self.type = py_object_type + self.set_py_result_type(self.function) self.is_temp = 1 return self @@ -7365,17 +7527,14 @@ class SequenceNode(ExprNode): code.putln("PyObject* sequence = %s;" % rhs.py_result()) # list/tuple => check size - code.putln("#if !CYTHON_COMPILING_IN_PYPY") - code.putln("Py_ssize_t size = Py_SIZE(sequence);") - code.putln("#else") - code.putln("Py_ssize_t size = PySequence_Size(sequence);") # < 0 => exception - code.putln("#endif") + code.putln("Py_ssize_t size = __Pyx_PySequence_SIZE(sequence);") code.putln("if (unlikely(size != %d)) {" % len(self.args)) code.globalstate.use_utility_code(raise_too_many_values_to_unpack) code.putln("if (size > %d) __Pyx_RaiseTooManyValuesError(%d);" % ( len(self.args), len(self.args))) code.globalstate.use_utility_code(raise_need_more_values_to_unpack) code.putln("else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);") + # < 0 => exception code.putln(code.error_goto(self.pos)) code.putln("}") @@ -7606,10 +7765,10 @@ class TupleNode(SequenceNode): if self.mult_factor or not self.args: return tuple_type arg_types = [arg.infer_type(env) for arg in self.args] - if any(type.is_pyobject or type.is_unspecified or type.is_fused for type in arg_types): + if any(type.is_pyobject or type.is_memoryviewslice or type.is_unspecified or type.is_fused + for type in arg_types): return tuple_type - else: - return env.declare_tuple_type(self.pos, arg_types).type + return env.declare_tuple_type(self.pos, arg_types).type def analyse_types(self, env, skip_children=False): if len(self.args) == 0: @@ -7623,7 +7782,8 @@ class TupleNode(SequenceNode): arg.starred_expr_allowed_here = True self.args[i] = arg.analyse_types(env) if (not self.mult_factor and - not any((arg.is_starred or arg.type.is_pyobject or arg.type.is_fused) for arg in self.args)): + not any((arg.is_starred or arg.type.is_pyobject or arg.type.is_memoryviewslice or arg.type.is_fused) + for arg in self.args)): self.type = env.declare_tuple_type(self.pos, (arg.type for arg in self.args)).type self.is_temp = 1 return self @@ -7706,26 +7866,21 @@ class TupleNode(SequenceNode): if len(self.args) == 0: # result_code is Naming.empty_tuple return - if self.is_partly_literal: - # underlying tuple is const, but factor is not + + if self.is_literal or self.is_partly_literal: tuple_target = code.get_py_const(py_object_type, 'tuple', cleanup_level=2) const_code = code.get_cached_constants_writer() const_code.mark_pos(self.pos) - self.generate_sequence_packing_code(const_code, tuple_target, plain=True) + self.generate_sequence_packing_code(const_code, tuple_target, plain=not self.is_literal) const_code.put_giveref(tuple_target) - code.putln('%s = PyNumber_Multiply(%s, %s); %s' % ( - self.result(), tuple_target, self.mult_factor.py_result(), - code.error_goto_if_null(self.result(), self.pos) + if self.is_literal: + self.result_code = tuple_target + else: + code.putln('%s = PyNumber_Multiply(%s, %s); %s' % ( + self.result(), tuple_target, self.mult_factor.py_result(), + code.error_goto_if_null(self.result(), self.pos) )) - code.put_gotref(self.py_result()) - elif self.is_literal: - # non-empty cached tuple => result is global constant, - # creation code goes into separate code writer - self.result_code = code.get_py_const(py_object_type, 'tuple', cleanup_level=2) - code = code.get_cached_constants_writer() - code.mark_pos(self.pos) - self.generate_sequence_packing_code(code) - code.put_giveref(self.py_result()) + code.put_gotref(self.py_result()) else: self.type.entry.used = True self.generate_sequence_packing_code(code) @@ -8891,66 +9046,6 @@ class ClassCellNode(ExprNode): code.put_incref(self.result(), py_object_type) -class BoundMethodNode(ExprNode): - # Helper class used in the implementation of Python - # class definitions. Constructs an bound method - # object from a class and a function. - # - # function ExprNode Function object - # self_object ExprNode self object - - subexprs = ['function'] - - def analyse_types(self, env): - self.function = self.function.analyse_types(env) - self.type = py_object_type - self.is_temp = 1 - return self - - gil_message = "Constructing a bound method" - - def generate_result_code(self, code): - code.putln( - "%s = __Pyx_PyMethod_New(%s, %s, (PyObject*)%s->ob_type); %s" % ( - self.result(), - self.function.py_result(), - self.self_object.py_result(), - self.self_object.py_result(), - code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) - -class UnboundMethodNode(ExprNode): - # Helper class used in the implementation of Python - # class definitions. Constructs an unbound method - # object from a class and a function. - # - # function ExprNode Function object - - type = py_object_type - is_temp = 1 - - subexprs = ['function'] - - def analyse_types(self, env): - self.function = self.function.analyse_types(env) - return self - - def may_be_none(self): - return False - - gil_message = "Constructing an unbound method" - - def generate_result_code(self, code): - class_cname = code.pyclass_stack[-1].classobj.result() - code.putln( - "%s = __Pyx_PyMethod_New(%s, 0, %s); %s" % ( - self.result(), - self.function.py_result(), - class_cname, - code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) - - class PyCFunctionNode(ExprNode, ModuleNameMixin): # Helper class used in the implementation of Python # functions. Constructs a PyCFunction object @@ -9056,7 +9151,7 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): for arg in nonliteral_other: entry = scope.declare_var(arg.name, arg.type, None, Naming.arg_prefix + arg.name, - allow_pyobject=False) + allow_pyobject=False, allow_memoryview=True) self.defaults.append((arg, entry)) entry = module_scope.declare_struct_or_union( None, 'struct', scope, 1, None, cname=cname) @@ -10426,7 +10521,7 @@ class CythonArrayNode(ExprNode): def allocate_temp_result(self, code): if self.temp_code: - raise RuntimeError("temp allocated mulitple times") + raise RuntimeError("temp allocated multiple times") self.temp_code = code.funcstate.allocate_temp(self.type, True) @@ -10687,6 +10782,10 @@ class TypeofNode(ExprNode): self.literal = literal.coerce_to_pyobject(env) return self + def analyse_as_type(self, env): + self.operand = self.operand.analyse_types(env) + return self.operand.type + def may_be_none(self): return False @@ -11132,7 +11231,7 @@ class AddNode(NumBinopNode): def infer_builtin_types_operation(self, type1, type2): # b'abc' + 'abc' raises an exception in Py3, # so we can safely infer the Py2 type for bytes here - string_types = (bytes_type, str_type, basestring_type, unicode_type) + string_types = (bytes_type, bytearray_type, str_type, basestring_type, unicode_type) if type1 in string_types and type2 in string_types: return string_types[max(string_types.index(type1), string_types.index(type2))] @@ -11191,7 +11290,7 @@ class MulNode(NumBinopNode): def infer_builtin_types_operation(self, type1, type2): # let's assume that whatever builtin type you multiply a string with # will either return a string of the same type or fail with an exception - string_types = (bytes_type, str_type, basestring_type, unicode_type) + string_types = (bytes_type, bytearray_type, str_type, basestring_type, unicode_type) if type1 in string_types and type2.is_builtin_type: return type1 if type2 in string_types and type1.is_builtin_type: @@ -11607,7 +11706,7 @@ class BoolBinopNode(ExprNode): operator=self.operator, operand1=operand1, operand2=operand2) - def generate_bool_evaluation_code(self, code, final_result_temp, and_label, or_label, end_label, fall_through): + def generate_bool_evaluation_code(self, code, final_result_temp, final_result_type, and_label, or_label, end_label, fall_through): code.mark_pos(self.pos) outer_labels = (and_label, or_label) @@ -11616,19 +11715,20 @@ class BoolBinopNode(ExprNode): else: my_label = or_label = code.new_label('next_or') self.operand1.generate_bool_evaluation_code( - code, final_result_temp, and_label, or_label, end_label, my_label) + code, final_result_temp, final_result_type, and_label, or_label, end_label, my_label) and_label, or_label = outer_labels code.put_label(my_label) self.operand2.generate_bool_evaluation_code( - code, final_result_temp, and_label, or_label, end_label, fall_through) + code, final_result_temp, final_result_type, and_label, or_label, end_label, fall_through) def generate_evaluation_code(self, code): self.allocate_temp_result(code) + result_type = PyrexTypes.py_object_type if self.type.is_pyobject else self.type or_label = and_label = None end_label = code.new_label('bool_binop_done') - self.generate_bool_evaluation_code(code, self.result(), and_label, or_label, end_label, end_label) + self.generate_bool_evaluation_code(code, self.result(), result_type, and_label, or_label, end_label, end_label) code.put_label(end_label) gil_message = "Truth-testing Python object" @@ -11713,7 +11813,7 @@ class BoolBinopResultNode(ExprNode): test_result = self.arg.result() return (test_result, self.arg.type.is_pyobject) - def generate_bool_evaluation_code(self, code, final_result_temp, and_label, or_label, end_label, fall_through): + def generate_bool_evaluation_code(self, code, final_result_temp, final_result_type, and_label, or_label, end_label, fall_through): code.mark_pos(self.pos) # x => x @@ -11756,7 +11856,7 @@ class BoolBinopResultNode(ExprNode): code.putln("} else {") self.value.generate_evaluation_code(code) self.value.make_owned_reference(code) - code.putln("%s = %s;" % (final_result_temp, self.value.result())) + code.putln("%s = %s;" % (final_result_temp, self.value.result_as(final_result_type))) self.value.generate_post_assignment_code(code) # disposal: {not (and_label and or_label) [else]} self.arg.generate_disposal_code(code) @@ -12117,6 +12217,11 @@ class CmpNode(object): self.special_bool_cmp_utility_code = UtilityCode.load_cached("PyDictContains", "ObjectHandling.c") self.special_bool_cmp_function = "__Pyx_PyDict_ContainsTF" return True + elif self.operand2.type is Builtin.set_type: + self.operand2 = self.operand2.as_none_safe_node("'NoneType' object is not iterable") + self.special_bool_cmp_utility_code = UtilityCode.load_cached("PySetContains", "ObjectHandling.c") + self.special_bool_cmp_function = "__Pyx_PySet_ContainsTF" + return True elif self.operand2.type is Builtin.unicode_type: self.operand2 = self.operand2.as_none_safe_node("'NoneType' object is not iterable") self.special_bool_cmp_utility_code = UtilityCode.load_cached("PyUnicodeContains", "StringTools.c") @@ -12236,7 +12341,14 @@ class PrimaryCmpNode(ExprNode, CmpNode): is_memslice_nonecheck = False def infer_type(self, env): - # TODO: Actually implement this (after merging with -unstable). + type1 = self.operand1.infer_type(env) + type2 = self.operand2.infer_type(env) + + if is_pythran_expr(type1) or is_pythran_expr(type2): + if is_pythran_supported_type(type1) and is_pythran_supported_type(type2): + return PythranExpr(pythran_binop_type(self.operator, type1, type2)) + + # TODO: implement this for other types. return py_object_type def type_dependencies(self, env): @@ -12662,12 +12774,12 @@ class CoerceToMemViewSliceNode(CoercionNode): def generate_result_code(self, code): self.type.create_from_py_utility_code(self.env) - code.putln("%s = %s(%s);" % (self.result(), - self.type.from_py_function, - self.arg.py_result())) - - error_cond = self.type.error_condition(self.result()) - code.putln(code.error_goto_if(error_cond, self.pos)) + code.putln(self.type.from_py_call_code( + self.arg.py_result(), + self.result(), + self.pos, + code + )) class CastNode(CoercionNode): @@ -12726,6 +12838,15 @@ class PyTypeTestNode(CoercionNode): def nonlocally_immutable(self): return self.arg.nonlocally_immutable() + def reanalyse(self): + if self.type != self.arg.type or not self.arg.is_temp: + return self + if not self.type.typeobj_is_available(): + return self + if self.arg.may_be_none() and self.notnone: + return self.arg.as_none_safe_node("Cannot convert NoneType to %.200s" % self.type.name) + return self.arg + def calculate_constant_result(self): # FIXME pass @@ -13016,6 +13137,7 @@ class CoerceToBooleanNode(CoercionNode): Builtin.set_type: 'PySet_GET_SIZE', Builtin.frozenset_type: 'PySet_GET_SIZE', Builtin.bytes_type: 'PyBytes_GET_SIZE', + Builtin.bytearray_type: 'PyByteArray_GET_SIZE', Builtin.unicode_type: '__Pyx_PyUnicode_IS_TRUE', } @@ -13044,11 +13166,9 @@ class CoerceToBooleanNode(CoercionNode): return test_func = self._special_builtins.get(self.arg.type) if test_func is not None: - code.putln("%s = (%s != Py_None) && (%s(%s) != 0);" % ( - self.result(), - self.arg.py_result(), - test_func, - self.arg.py_result())) + checks = ["(%s != Py_None)" % self.arg.py_result()] if self.arg.may_be_none() else [] + checks.append("(%s(%s) != 0)" % (test_func, self.arg.py_result())) + code.putln("%s = %s;" % (self.result(), '&&'.join(checks))) else: code.putln( "%s = __Pyx_PyObject_IsTrue(%s); %s" % ( diff --git a/Cython/Compiler/FusedNode.py b/Cython/Compiler/FusedNode.py index 3effc15..011290e 100644 --- a/Cython/Compiler/FusedNode.py +++ b/Cython/Compiler/FusedNode.py @@ -225,7 +225,7 @@ class FusedCFuncDefNode(StatListNode): """ Create a new local scope for the copied node and append it to self.nodes. A new local scope is needed because the arguments with the - fused types are aready in the local scope, and we need the specialized + fused types are already in the local scope, and we need the specialized entries created after analyse_declarations on each specialized version of the (CFunc)DefNode. f2s is a dict mapping each fused type to its specialized version @@ -276,7 +276,7 @@ class FusedCFuncDefNode(StatListNode): def _fused_instance_checks(self, normal_types, pyx_code, env): """ - Genereate Cython code for instance checks, matching an object to + Generate Cython code for instance checks, matching an object to specialized types. """ for specialized_type in normal_types: @@ -390,7 +390,7 @@ class FusedCFuncDefNode(StatListNode): coerce_from_py_func=memslice_type.from_py_function, dtype=dtype) decl_code.putln( - "{{memviewslice_cname}} {{coerce_from_py_func}}(object)") + "{{memviewslice_cname}} {{coerce_from_py_func}}(object, int)") pyx_code.context.update( specialized_type_name=specialized_type.specialization_string, @@ -400,7 +400,7 @@ class FusedCFuncDefNode(StatListNode): u""" # try {{dtype}} if itemsize == -1 or itemsize == {{sizeof_dtype}}: - memslice = {{coerce_from_py_func}}(arg) + memslice = {{coerce_from_py_func}}(arg, 0) if memslice.memview: __PYX_XDEC_MEMVIEW(&memslice, 1) # print 'found a match for the buffer through format parsing' @@ -421,10 +421,11 @@ class FusedCFuncDefNode(StatListNode): # The first thing to find a match in this loop breaks out of the loop pyx_code.put_chunk( u""" + """ + (u"arg_is_pythran_compatible = False" if pythran_types else u"") + u""" if ndarray is not None: if isinstance(arg, ndarray): dtype = arg.dtype - arg_is_pythran_compatible = True + """ + (u"arg_is_pythran_compatible = True" if pythran_types else u"") + u""" elif __pyx_memoryview_check(arg): arg_base = arg.base if isinstance(arg_base, ndarray): @@ -438,24 +439,30 @@ class FusedCFuncDefNode(StatListNode): if dtype is not None: itemsize = dtype.itemsize kind = ord(dtype.kind) - # We only support the endianess of the current compiler + dtype_signed = kind == 'i' + """) + pyx_code.indent(2) + if pythran_types: + pyx_code.put_chunk( + u""" + # Pythran only supports the endianness of the current compiler byteorder = dtype.byteorder if byteorder == "<" and not __Pyx_Is_Little_Endian(): arg_is_pythran_compatible = False - if byteorder == ">" and __Pyx_Is_Little_Endian(): + elif byteorder == ">" and __Pyx_Is_Little_Endian(): arg_is_pythran_compatible = False - dtype_signed = kind == 'i' if arg_is_pythran_compatible: cur_stride = itemsize - for dim,stride in zip(reversed(arg.shape),reversed(arg.strides)): - if stride != cur_stride: + shape = arg.shape + strides = arg.strides + for i in range(arg.ndim-1, -1, -1): + if (strides[i]) != cur_stride: arg_is_pythran_compatible = False break - cur_stride *= dim + cur_stride *= shape[i] else: - arg_is_pythran_compatible = not (arg.flags.f_contiguous and arg.ndim > 1) - """) - pyx_code.indent(2) + arg_is_pythran_compatible = not (arg.flags.f_contiguous and (arg.ndim) > 1) + """) pyx_code.named_insertion_point("numpy_dtype_checks") self._buffer_check_numpy_dtype(pyx_code, buffer_types, pythran_types) pyx_code.dedent(2) @@ -464,7 +471,7 @@ class FusedCFuncDefNode(StatListNode): self._buffer_parse_format_string_check( pyx_code, decl_code, specialized_type, env) - def _buffer_declarations(self, pyx_code, decl_code, all_buffer_types): + def _buffer_declarations(self, pyx_code, decl_code, all_buffer_types, pythran_types): """ If we have any buffer specializations, write out some variable declarations and imports. @@ -484,10 +491,14 @@ class FusedCFuncDefNode(StatListNode): cdef Py_ssize_t itemsize cdef bint dtype_signed cdef char kind - cdef bint arg_is_pythran_compatible itemsize = -1 - arg_is_pythran_compatible = False + """) + + if pythran_types: + pyx_code.local_variable_declarations.put_chunk(u""" + cdef bint arg_is_pythran_compatible + cdef Py_ssize_t cur_stride """) pyx_code.imports.put_chunk( @@ -514,7 +525,7 @@ class FusedCFuncDefNode(StatListNode): pyx_code.local_variable_declarations.put_chunk( u""" cdef bint {{dtype_name}}_is_signed - {{dtype_name}}_is_signed = <{{dtype_type}}> -1 < 0 + {{dtype_name}}_is_signed = not (<{{dtype_type}}> -1 > 0) """) def _split_fused_types(self, arg): @@ -670,7 +681,7 @@ class FusedCFuncDefNode(StatListNode): default_idx += 1 if all_buffer_types: - self._buffer_declarations(pyx_code, decl_code, all_buffer_types) + self._buffer_declarations(pyx_code, decl_code, all_buffer_types, pythran_types) env.use_utility_code(Code.UtilityCode.load_cached("Import", "ImportExport.c")) env.use_utility_code(Code.UtilityCode.load_cached("ImportNumPyArray", "ImportExport.c")) diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py index 895f375..6a582ba 100644 --- a/Cython/Compiler/Main.py +++ b/Cython/Compiler/Main.py @@ -18,12 +18,12 @@ try: except ImportError: basestring = str -from . import Errors # Do not import Parsing here, import it when needed, because Parsing imports # Nodes, which globally needs debug command line options initialized to set a # conditional metaclass. These options are processed by CmdLine called from # main() in this file. # import Parsing +from . import Errors from .StringEncoding import EncodedString from .Scanning import PyrexScanner, FileSourceDescriptor from .Errors import PyrexError, CompileError, error, warning @@ -38,6 +38,7 @@ module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_ verbose = 0 + class CompilationData(object): # Bundles the information that is passed from transform to transform. # (For now, this is only) @@ -52,6 +53,7 @@ class CompilationData(object): # result CompilationResult pass + class Context(object): # This class encapsulates the context needed for compiling # one or more Cython implementation files along with their @@ -239,7 +241,7 @@ class Context(object): pxd = self.search_include_directories(qualified_name, ".pxd", pos, sys_path=sys_path) if pxd is None: # XXX Keep this until Includes/Deprecated is removed if (qualified_name.startswith('python') or - qualified_name in ('stdlib', 'stdio', 'stl')): + qualified_name in ('stdlib', 'stdio', 'stl')): standard_include_path = os.path.abspath(os.path.normpath( os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes'))) deprecated_include_path = os.path.join(standard_include_path, 'Deprecated') @@ -356,7 +358,7 @@ class Context(object): from ..Parser import ConcreteSyntaxTree except ImportError: raise RuntimeError( - "Formal grammer can only be used with compiled Cython with an available pgen.") + "Formal grammar can only be used with compiled Cython with an available pgen.") ConcreteSyntaxTree.p_module(source_filename) except UnicodeDecodeError as e: #import traceback @@ -426,6 +428,7 @@ class Context(object): pass result.c_file = None + def get_output_filename(source_filename, cwd, options): if options.cplus: c_suffix = ".cpp" @@ -441,6 +444,7 @@ def get_output_filename(source_filename, cwd, options): else: return suggested_file_name + def create_default_resultobj(compilation_source, options): result = CompilationResult() result.main_source_file = compilation_source.source_desc.filename @@ -451,6 +455,7 @@ def create_default_resultobj(compilation_source, options): result.embedded_metadata = options.embedded_metadata return result + def run_pipeline(source, options, full_module_name=None, context=None): from . import Pipeline @@ -496,15 +501,15 @@ def run_pipeline(source, options, full_module_name=None, context=None): return result -#------------------------------------------------------------------------ +# ------------------------------------------------------------------------ # # Main Python entry points # -#------------------------------------------------------------------------ +# ------------------------------------------------------------------------ class CompilationSource(object): """ - Contains the data necesarry to start up a compilation pipeline for + Contains the data necessary to start up a compilation pipeline for a single compilation unit. """ def __init__(self, source_desc, full_module_name, cwd): @@ -512,6 +517,7 @@ class CompilationSource(object): self.full_module_name = full_module_name self.cwd = cwd + class CompilationOptions(object): """ Options to the Cython compiler: @@ -678,13 +684,14 @@ def compile_multiple(sources, options): processed.add(source) return results + def compile(source, options = None, full_module_name = None, **kwds): """ compile(source [, options], [,