From: JinWang An Date: Mon, 27 Mar 2023 08:02:51 +0000 (+0900) Subject: Imported Upstream version 63.0.0b1 X-Git-Tag: upstream/63.0.0b1^0 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=8f61b9b6f47b8f6c3caf1bbfffab53a04e659d0e;p=platform%2Fupstream%2Fpython-setuptools.git Imported Upstream version 63.0.0b1 --- diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0c090c7..f90cdd5 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 63.0.0 +current_version = 62.6.0 commit = True tag = True diff --git a/.github/workflows/ci-sage.yml b/.github/workflows/ci-sage.yml index 2a91934..425681d 100644 --- a/.github/workflows/ci-sage.yml +++ b/.github/workflows/ci-sage.yml @@ -92,7 +92,7 @@ jobs: fail-fast: false max-parallel: 32 matrix: - tox_system_factor: [ubuntu-trusty-toolchain-gcc_9, ubuntu-xenial-toolchain-gcc_9, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy, ubuntu-kinetic, debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, linuxmint-21, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, fedora-37, centos-7-devtoolset-gcc_11, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.3, opensuse-tumbleweed, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386-devtoolset-gcc_11] + tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy, debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, centos-7, centos-stream-8, centos-stream-9, gentoo-python3.9, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386] tox_packages_factor: [minimal, standard] env: TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }} diff --git a/CHANGES.rst b/CHANGES.rst index eeab4a2..b8ec83c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,26 +1,3 @@ -v63.0.0 -------- - - -Breaking Changes -^^^^^^^^^^^^^^^^ -* #3421: Drop setuptools' support for installing an entrypoint extra requirements at load time: - - the functionality has been broken since v60.8.0. - - the mechanism to do so is deprecated (`fetch_build_eggs`). - - that use case (e.g. a custom command class entrypoint) is covered by making sure the necessary build requirements are declared. - -Documentation changes -^^^^^^^^^^^^^^^^^^^^^ -* #3305: Updated the example pyproject.toml -- by :user:`jacalata` -* #3394: This updates the documentation for the ``file_finders`` hook so that - the logging recommendation aligns with the suggestion to not use - ``distutils`` directly. -* #3397: Fix reference for ``keywords`` to point to the Core Metadata Specification - instead of PEP 314 (the live standard is kept always up-to-date and - consolidates several PEPs together in a single document). -* #3402: Reordered the User Guide's Table of Contents -- by :user:`codeandfire` - - v62.6.0 ------- diff --git a/bootstrap.egg-info/entry_points.txt b/bootstrap.egg-info/entry_points.txt index c00d1d3..a21ca22 100644 --- a/bootstrap.egg-info/entry_points.txt +++ b/bootstrap.egg-info/entry_points.txt @@ -2,6 +2,7 @@ egg_info = setuptools.command.egg_info:egg_info build_py = setuptools.command.build_py:build_py sdist = setuptools.command.sdist:sdist +editable_wheel = setuptools.command.editable_wheel:editable_wheel [distutils.setup_keywords] include_package_data = setuptools.dist:assert_bool diff --git a/changelog.d/3265.change.rst b/changelog.d/3265.change.rst new file mode 100644 index 0000000..ac20398 --- /dev/null +++ b/changelog.d/3265.change.rst @@ -0,0 +1,17 @@ +Added implementation for *editable install* hooks (PEP 660) - **beta** stage. + +- The user will be able select between two distinct behaviors: + + - *lax*, which prioritises the ability of the users of changing the + distributed packages (e.g. adding new files or removing old ones) + + - *strict*, which will try to replicate as much as possible the behavior of + the package as if it would be normally installed by end users. + The *strict* editable installation is not able to detect if files are + added or removed from the project (a new installation is required). + +.. important:: + The *editable* aspect of the *editable install* supported this implementation + is restricted to the Python modules contained in the distributed package. + Changes in binary extensions (e.g. C/C++), entry-point definitions, + dependencies, metadata, datafiles, etc require a new installation. diff --git a/changelog.d/3305.doc.rst b/changelog.d/3305.doc.rst new file mode 100644 index 0000000..39006ff --- /dev/null +++ b/changelog.d/3305.doc.rst @@ -0,0 +1 @@ +Updated the example pyproject.toml -- by :user:`jacalata` diff --git a/changelog.d/3380.change.rst b/changelog.d/3380.change.rst new file mode 100644 index 0000000..9622417 --- /dev/null +++ b/changelog.d/3380.change.rst @@ -0,0 +1,10 @@ +Improved the handling of the ``config_settings`` parameter in both PEP 517 and +PEP 660 interfaces: + +- It is possible now to pass both ``--global-option`` and ``--build-option``. + As discussed in #1928, arbitrary arguments passed via ``--global-option`` + should be placed before the name of the setuptools' internal command, while + ``--build-option`` should come after. + +- Users can pass ``editable-mode=strict`` to select a strict behaviour for the + editable installation. diff --git a/changelog.d/3380.deprecation.rst b/changelog.d/3380.deprecation.rst new file mode 100644 index 0000000..54d3c4c --- /dev/null +++ b/changelog.d/3380.deprecation.rst @@ -0,0 +1,7 @@ +Passing some types of parameters via ``--global-option`` to setuptools PEP 517/PEP 660 backend +is now considered deprecated. The user can pass the same arbitrary parameter +via ``--build-option`` (``--global-option`` is now reserved for flags like +``--verbose`` or ``--quiet``). + +Both ``--build-option`` and ``--global-option`` are supported as a **transitional** effort (a.k.a. "escape hatch"). +In the future a proper list of allowed ``config_settings`` may be created. diff --git a/changelog.d/3392.change.rst b/changelog.d/3392.change.rst new file mode 100644 index 0000000..8ae7fd9 --- /dev/null +++ b/changelog.d/3392.change.rst @@ -0,0 +1,5 @@ +Exposed ``get_output_mapping()`` from ``build_py`` and ``build_ext`` +subcommands. This interface is reserved for the use of ``setuptools`` +Extensions and third part packages are explicitly disallowed to calling it. +However, any implementation overwriting ``build_py`` or ``build_ext`` are +required to honour this interface. diff --git a/changelog.d/3394.doc.rst b/changelog.d/3394.doc.rst new file mode 100644 index 0000000..ea3702b --- /dev/null +++ b/changelog.d/3394.doc.rst @@ -0,0 +1,3 @@ +This updates the documentation for the ``file_finders`` hook so that +the logging recommendation aligns with the suggestion to not use +``distutils`` directly. diff --git a/changelog.d/3397.doc.rst b/changelog.d/3397.doc.rst new file mode 100644 index 0000000..933fc34 --- /dev/null +++ b/changelog.d/3397.doc.rst @@ -0,0 +1,3 @@ +Fix reference for ``keywords`` to point to the Core Metadata Specification +instead of PEP 314 (the live standard is kept always up-to-date and +consolidates several PEPs together in a single document). diff --git a/changelog.d/3402.doc.rst b/changelog.d/3402.doc.rst new file mode 100644 index 0000000..e88ac1f --- /dev/null +++ b/changelog.d/3402.doc.rst @@ -0,0 +1 @@ +Reordered the User Guide's Table of Contents -- by :user:`codeandfire` diff --git a/changelog.d/3412.change.rst b/changelog.d/3412.change.rst new file mode 100644 index 0000000..69f02bc --- /dev/null +++ b/changelog.d/3412.change.rst @@ -0,0 +1,3 @@ +Added ability of collecting source files from custom build sub-commands to +``sdist``. This allows plugins and customization scripts to automatically +add required source files in the source distribution. diff --git a/changelog.d/3414.change.rst b/changelog.d/3414.change.rst new file mode 100644 index 0000000..b29f2c5 --- /dev/null +++ b/changelog.d/3414.change.rst @@ -0,0 +1,4 @@ +Users can *temporarily* specify an environment variable +``SETUPTOOLS_ENABLE_FEATURE=legacy-editable`` as a escape hatch for the +:pep:`660` behavior. This setting is **transitional** and may be removed in the +future. diff --git a/changelog.d/3414.doc.rst b/changelog.d/3414.doc.rst new file mode 100644 index 0000000..b4756da --- /dev/null +++ b/changelog.d/3414.doc.rst @@ -0,0 +1,2 @@ +Updated :doc:`Development Mode ` to reflect on the +implementation of :pep:`660`. diff --git a/docs/userguide/development_mode.rst b/docs/userguide/development_mode.rst index 4130ab7..ee6b5b0 100644 --- a/docs/userguide/development_mode.rst +++ b/docs/userguide/development_mode.rst @@ -1,34 +1,155 @@ -Development Mode -================ - -Under normal circumstances, the ``setuptools`` assume that you are going to -build a distribution of your project, not use it in its "raw" or "unbuilt" -form. However, if you were to use the ``setuptools`` to build a distribution, -you would have to rebuild and reinstall your project every time you made a -change to it during development. - -Another problem that sometimes comes is that you may -need to do development on two related projects at the same time. You may need -to put both projects' packages in the same directory to run them, but need to -keep them separate for revision control purposes. How can you do this? - -Setuptools allows you to deploy your projects for use in a common directory or -staging area, but without copying any files. Thus, you can edit each project's -code in its checkout directory, and only need to run build commands when you -change files that need to be compiled or the provided metadata and setuptools configuration. - -You can perform a ``pip`` installation passing the ``-e/--editable`` -flag (e.g., ``pip install -e .``). It works very similarly to -``pip install .``, except that it doesn't actually install anything. -Instead, it creates a special ``.egg-link`` file in the target directory -(usually ``site-packages``) that links to your project's source code. -It may also update an existing ``easy-install.pth`` file -to include your project's source code, thereby making -it available on ``sys.path`` for all programs using that Python installation. - -You can deploy the same project to multiple staging areas, e.g., if you have -multiple projects on the same machine that are sharing the same project you're -doing development work. +Development Mode (a.k.a. "Editable Installs") +============================================= + +When creating a Python project, developers usually want to implement and test +changes iteratively, before cutting a release and preparing a distribution archive. + +In normal circumstances this can be quite cumbersome and require the developers +to manipulate the ``PATHONPATH`` environment variable or to continuous re-build +and re-install the project. + +To facilitate iterative exploration and experimentation, setuptools allows +users to instruct the Python interpreter and its import machinery to load the +code under development directly from the project folder without having to +copy the files to a different location in the disk. +This means that changes in the Python source code can immediately take place +without requiring a new installation. + +You can enter this "development mode" by performing an :doc:`editable installation +` inside of a :term:`virtual environment`, +using :doc:`pip's ` ``-e/--editable`` flag, as shown bellow: + +.. code-block:: bash + + $ cd your-python-project + $ python -m venv .venv + # Activate your environemt with: + # `source .venv/bin/activate` on Unix/macOS + # or `.venv\Scripts\activate` on Windows + + $ pip install --editable . + + # Now you have access to your package + # as if it was installed in .venv + $ python -c "import your_python_project" + + +An "editable installation" works very similarly to a regular install with +``pip install .``, except that it only installs your package dependencies, +metadata and wrappers for :ref:`console and GUI scripts `. +Under the hood, setuptools will try to create a special :mod:`.pth file ` +in the target directory (usually ``site-packages``) that extends the +``PYTHONPATH`` or install a custom :doc:`import hook `. When you're done with a given development task, you can simply uninstall your package (as you would normally do with ``pip uninstall ``). + +Please note that, by default an editable install will expose at least all the +files that would be available in a regular installation. However, depending on +the file and directory organization in your project, it might also expose +as a side effect files that would not be normally available. +This is allowed so you can iteratively create new Python modules. +Please have a look on the following section if you are looking for a different behaviour. + +.. admonition:: Virtual Environments + + You can think virtual environments as "isolated Python runtime deployments" + that allow users to install different sets of libraries and tools without + messing with the global behaviour of the system. + + They are the safest way of testing new projects and can be created easily + with the :mod:`venv` module from the standard library. + + Please note however that depending on your operating system or distribution, + ``venv`` might not come installed by default with Python. For those cases, + you might need to use the OS package manager to install it. + For example, in Debian/Ubuntu-based systems you can obtain it via: + + .. code-block:: bash + + sudo apt install python3-venv + + Alternatively, you can also try installing :pypi:`virtualenᴠ`. + More information is available on the Python Packaging User Guide on + :doc:`PyPUG:guides/installing-using-pip-and-virtual-environments`. + +.. note:: + .. versionchanged:: v63.0.0 + Editable installation hooks implemented according to :pep:`660`. + Support for :pep:`namespace packages <420>` is still **EXPERIMENTAL**. + + +"Strict" editable installs +-------------------------- + +When thinking about editable installations, users might have the following +expectations: + +1. It should allow developers to add new files (or split/rename existing ones) + and have them automatically exposed. +2. It should behave as close as possible to a regular installation and help + users to detect problems (e.g. new files not being included in the distribution). + +Unfortunately these expectations are in conflict with each other. +To solve this problem ``setuptools`` allows developers to choose a more +*"strict"* mode for the editable installation. This can be done by passing +a special *configuration setting* via :pypi:`pip`, as indicated bellow: + +.. code-block:: bash + + pip install -e . --config-settings editable_mode=strict + +In this mode, new files **won't** be exposed and the editable installs will +try to mimic as much as possible the behavior of a regular install. +Under the hood, ``setuptools`` will create a tree of file links in an auxiliary +directory (``$your_project_dir/build``) and add it to ``PYTHONPATH`` via a +:mod:`.pth file `. (Please be careful to not delete this repository +by mistake otherwise your files may stop being accessible). + + +.. note:: + .. versionadded:: v63.0.0 + *Strict* mode implemented as **EXPERIMENTAL**. + + +Limitations +----------- + +- The *editable* term is used to refer only to Python modules + inside the package directories. Non-Python files, external (data) files, + executable script files, binary extensions, headers and metadata may be + exposed as a *snapshot* of the version they were at the moment of the + installation. +- Adding new dependencies, entry-points or changing your project's metadata + require a fresh "editable" re-installation. +- Console scripts and GUI scripts **MUST** be specified via :doc:`entry-points + ` to work properly. +- *Strict* editable installs require the file system to support + either :wiki:`symbolic ` or :wiki:`hard links `. +- Editable installations may not work with + :doc:`namespaces created with pkgutil or pkg_resouces + `. + Please use :pep:`420`-style implicit namespaces. +- Support for :pep:`420`-style implicit namespace packages for + projects structured using :ref:`flat-layout` is still **experimental**. + If you experience problems, you can try converting your package structure + to the :ref:`src-layout`. + +.. attention:: + Editable installs are **not a perfect replacement for regular installs** + in a test environment. When in doubt, please test your projects as + installed via a regular wheel. There are tools in the Python ecosystem, + like :pypi:`tox` or :pypi:`nox`, that can help you with that + (when used with appropriate configuration). + + +Legacy Behavior +--------------- + +If your project is not compatible with the new "editable installs" or you wish +to use the legacy behavior (that mimics the old and deprecated +``python setup.py develop`` command), you can set an environment variable: + +.. code-block:: + + SETUPTOOLS_USE_FEATURE="legacy-editable" diff --git a/docs/userguide/entry_point.rst b/docs/userguide/entry_point.rst index eff20cf..6ba0028 100644 --- a/docs/userguide/entry_point.rst +++ b/docs/userguide/entry_point.rst @@ -21,6 +21,8 @@ highlighting tool :pypi:`pygments` allows specifying additional styles using the entry point ``pygments.styles``. +.. _console-scripts: + Console Scripts =============== diff --git a/docs/userguide/extension.rst b/docs/userguide/extension.rst index 0008b6c..58c8ec1 100644 --- a/docs/userguide/extension.rst +++ b/docs/userguide/extension.rst @@ -56,8 +56,8 @@ a ``foo`` command, you might add something like this to your project: distutils.commands = foo = mypackage.some_module:foo -(Assuming, of course, that the ``foo`` class in ``mypackage.some_module`` is -a ``setuptools.Command`` subclass.) +Assuming, of course, that the ``foo`` class in ``mypackage.some_module`` is +a ``setuptools.Command`` subclass (documented bellow). Once a project containing such entry points has been activated on ``sys.path``, (e.g. by running ``pip install``) the command(s) will be available to any @@ -72,9 +72,21 @@ Custom commands should try to replicate the same overall behavior as the original classes, and when possible, even inherit from them. You should also consider handling exceptions such as ``CompileError``, -``LinkError``, ``LibError``, among others. These exceptions are available in +``LinkError``, ``LibError``, among others. These exceptions are available in the ``setuptools.errors`` module. +.. autoclass:: setuptools.Command + :members: + + +Supporting sdists and editable installs in ``build`` sub-commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``build`` sub-commands (like ``build_py`` and ``build_ext``) +are encouraged to implement the following protocol: + +.. autoclass:: setuptools.command.build.SubCommand + Adding Arguments ---------------- diff --git a/docs/userguide/index.rst b/docs/userguide/index.rst index d631c5d..e1563c9 100644 --- a/docs/userguide/index.rst +++ b/docs/userguide/index.rst @@ -12,7 +12,7 @@ This document contains information to help Python developers through this process. Please check the :doc:`/userguide/quickstart` for an overview of the workflow. -Also note that ``setuptools`` is what is known in the community as :pep:`build +Also note that ``setuptools`` is what is know in the community as :pep:`build backend <517#terminology-and-goals>`, user facing interfaces are provided by tools such as :pypi:`pip` and :pypi:`build`. To use ``setuptools``, one must explicitly create a ``pyproject.toml`` file as described :doc:`/build_meta`. diff --git a/docs/userguide/package_discovery.rst b/docs/userguide/package_discovery.rst index 2efc62b..93419a2 100644 --- a/docs/userguide/package_discovery.rst +++ b/docs/userguide/package_discovery.rst @@ -156,7 +156,8 @@ all modules and packages meant for distribution are placed inside this directory:: project_root_directory - ├── pyproject.toml # AND/OR setup.cfg, setup.py + ├── pyproject.toml + ├── setup.cfg # or setup.py ├── ... └── src/ └── mypkg/ @@ -189,7 +190,8 @@ flat-layout The package folder(s) are placed directly under the project root:: project_root_directory - ├── pyproject.toml # AND/OR setup.cfg, setup.py + ├── pyproject.toml + ├── setup.cfg # or setup.py ├── ... └── mypkg/ ├── __init__.py @@ -238,7 +240,8 @@ A standalone module is placed directly under the project root, instead of inside a package folder:: project_root_directory - ├── pyproject.toml # AND/OR setup.cfg, setup.py + ├── pyproject.toml + ├── setup.cfg # or setup.py ├── ... └── single_file_lib.py @@ -290,7 +293,7 @@ then returns a list of ``str`` representing the packages it could find. To use it, consider the following directory:: mypkg - ├── pyproject.toml # AND/OR setup.cfg, setup.py + ├── setup.cfg # and/or setup.py, pyproject.toml └── src ├── pkg1 │   └── __init__.py @@ -317,7 +320,7 @@ in ``src`` that start with the name ``pkg`` and not ``additional``: [options.packages.find] where = src include = pkg* - # alternatively: `exclude = additional*` + exclude = additional .. note:: ``pkg`` does not contain an ``__init__.py`` file, therefore @@ -331,7 +334,8 @@ in ``src`` that start with the name ``pkg`` and not ``additional``: # ... packages=find_packages( where='src', - include=['pkg*'], # alternatively: `exclude=['additional*']` + include=['pkg*'], + exclude=['additional'], ), package_dir={"": "src"} # ... @@ -349,7 +353,8 @@ in ``src`` that start with the name ``pkg`` and not ``additional``: [tool.setuptools.packages.find] where = ["src"] - include = ["pkg*"] # alternatively: `exclude = ["additional*"]` + include = ["pkg*"] + exclude = ["additional"] namespaces = false .. note:: @@ -407,7 +412,7 @@ Now, suppose you decide to package the ``foo`` part for distribution and start by creating a project directory organized as follows:: foo - ├── pyproject.toml # AND/OR setup.cfg, setup.py + ├── setup.cfg # and/or setup.py, pyproject.toml └── src └── timmins └── foo @@ -512,7 +517,7 @@ to `PEP 420 `_. It used to be more cumbersome to accomplish the same result. Historically, there were two methods to create namespace packages. One is the ``pkg_resources`` style supported by ``setuptools`` and the other one being ``pkgutils`` style offered by -``pkgutils`` module in Python. Both are now considered *deprecated* despite the +``pkgutils`` module in Python. Both are now considered deprecated despite the fact they still linger in many existing packages. These two differ in many subtle yet significant aspects and you can find out more on `Python packaging user guide `_. @@ -552,7 +557,7 @@ And your directory should look like this .. code-block:: bash foo - ├── pyproject.toml # AND/OR setup.cfg, setup.py + ├── setup.cfg # and/or setup.py, pyproject.toml └── src └── timmins ├── __init__.py @@ -572,7 +577,7 @@ file contains the following: __path__ = __import__('pkgutil').extend_path(__path__, __name__) -The project layout remains the same and ``pyproject.toml/setup.cfg`` remains the same. +The project layout remains the same and ``setup.cfg`` remains the same. ---- diff --git a/pytest.ini b/pytest.ini index 7c86396..aed8b71 100644 --- a/pytest.ini +++ b/pytest.ini @@ -60,3 +60,4 @@ filterwarnings= ignore:Setuptools is replacing distutils ignore:Support for .* in .pyproject.toml. is still .beta. + ignore::setuptools.command.editable_wheel.InformationOnly diff --git a/setup.cfg b/setup.cfg index f22457a..3cc0b55 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = setuptools -version = 63.0.0 +version = 63.0.0b1 author = Python Packaging Authority author_email = distutils-sig@python.org description = Easily download, build, install, upgrade, and uninstall Python packages @@ -113,6 +113,7 @@ distutils.commands = develop = setuptools.command.develop:develop dist_info = setuptools.command.dist_info:dist_info easy_install = setuptools.command.easy_install:easy_install + editable_wheel = setuptools.command.editable_wheel:editable_wheel egg_info = setuptools.command.egg_info:egg_info install = setuptools.command.install:install install_egg_info = setuptools.command.install_egg_info:install_egg_info diff --git a/setuptools/__init__.py b/setuptools/__init__.py index cff0432..6c24cc2 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -94,7 +94,59 @@ _Command = monkey.get_unpatched(distutils.core.Command) class Command(_Command): - __doc__ = _Command.__doc__ + """ + Setuptools internal actions are organized using a *command design pattern*. + This means that each action (or group of closely related actions) executed during + the build should be implemented as a ``Command`` subclass. + + These commands are abstractions and do not necessarily correspond to a command that + can (or should) be executed via a terminal, in a CLI fashion (although historically + they would). + + When creating a new command from scratch, custom defined classes **SHOULD** inherit + from ``setuptools.Command`` and implement a few mandatory methods. + Between these mandatory methods, are listed: + + .. method:: initialize_options(self) + + Set or (reset) all options/attributes/caches used by the command + to their default values. Note that these values may be overwritten during + the build. + + .. method:: finalize_options(self) + + Set final values for all options/attributes used by the command. + Most of the time, each option/attribute/cache should only be set if it does not + have any value yet (e.g. ``if self.attr is None: self.attr = val``). + + .. method:: run(self) + + Execute the actions intended by the command. + (Side effects **SHOULD** only take place when ``run`` is executed, + for example, creating new files or writing to the terminal output). + + A useful analogy for command classes is to think of them as subroutines with local + variables called "options". The options are "declared" in ``initialize_options()`` + and "defined" (given their final values, aka "finalized") in ``finalize_options()``, + both of which must be defined by every command class. The "body" of the subroutine, + (where it does all the work) is the ``run()`` method. + Between ``initialize_options()`` and ``finalize_options()``, ``setuptools`` may set + the values for options/attributes based on user's input (or circumstance), + which means that the implementation should be careful to not overwrite values in + ``finalize_options`` unless necessary. + + Please note that other commands (or other parts of setuptools) may also overwrite + the values of the command's options/attributes multiple times during the build + process. + Therefore it is important to consistently implement ``initialize_options()`` and + ``finalize_options()``. For example, all derived attributes (or attributes that + depend on the value of other attributes) **SHOULD** be recomputed in + ``finalize_options``. + + When overwriting existing commands, custom defined classes **MUST** abide by the + same APIs implemented by the original class. They also **SHOULD** inherit from the + original class. + """ command_consumes_arguments = False @@ -122,6 +174,12 @@ class Command(_Command): currently a string, we split it either on /,\s*/ or /\s+/, so "foo bar baz", "foo,bar,baz", and "foo, bar baz" all become ["foo", "bar", "baz"]. + + .. + TODO: This method seems to be similar to the one in ``distutils.cmd`` + Probably it is just here for backward compatibility with old Python versions? + + :meta private: """ val = getattr(self, option) if val is None: diff --git a/setuptools/_path.py b/setuptools/_path.py index ede9cb0..3767523 100644 --- a/setuptools/_path.py +++ b/setuptools/_path.py @@ -1,7 +1,29 @@ import os +from typing import Union + +_Path = Union[str, os.PathLike] def ensure_directory(path): """Ensure that the parent directory of `path` exists""" dirname = os.path.dirname(path) os.makedirs(dirname, exist_ok=True) + + +def same_path(p1: _Path, p2: _Path) -> bool: + """Differs from os.path.samefile because it does not require paths to exist. + Purely string based (no comparison between i-nodes). + >>> same_path("a/b", "./a/b") + True + >>> same_path("a/b", "a/./b") + True + >>> same_path("a/b", "././a/b") + True + >>> same_path("a/b", "./a/b/c/..") + True + >>> same_path("a/b", "../a/b/c") + False + >>> same_path("a", "a/b") + False + """ + return os.path.normpath(p1) == os.path.normpath(p2) diff --git a/setuptools/_vendor/nspektr-0.3.0.dist-info/INSTALLER b/setuptools/_vendor/nspektr-0.3.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/setuptools/_vendor/nspektr-0.3.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/setuptools/_vendor/nspektr-0.3.0.dist-info/LICENSE b/setuptools/_vendor/nspektr-0.3.0.dist-info/LICENSE new file mode 100644 index 0000000..353924b --- /dev/null +++ b/setuptools/_vendor/nspektr-0.3.0.dist-info/LICENSE @@ -0,0 +1,19 @@ +Copyright Jason R. Coombs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/setuptools/_vendor/nspektr-0.3.0.dist-info/METADATA b/setuptools/_vendor/nspektr-0.3.0.dist-info/METADATA new file mode 100644 index 0000000..aadc374 --- /dev/null +++ b/setuptools/_vendor/nspektr-0.3.0.dist-info/METADATA @@ -0,0 +1,57 @@ +Metadata-Version: 2.1 +Name: nspektr +Version: 0.3.0 +Summary: package inspector +Home-page: https://github.com/jaraco/nspektr +Author: Jason R. Coombs +Author-email: jaraco@jaraco.com +License: UNKNOWN +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Requires-Python: >=3.7 +License-File: LICENSE +Requires-Dist: jaraco.context +Requires-Dist: jaraco.functools +Requires-Dist: more-itertools +Requires-Dist: packaging +Requires-Dist: importlib-metadata (>=3.6) ; python_version < "3.10" +Provides-Extra: docs +Requires-Dist: sphinx ; extra == 'docs' +Requires-Dist: jaraco.packaging (>=9) ; extra == 'docs' +Requires-Dist: rst.linker (>=1.9) ; extra == 'docs' +Provides-Extra: testing +Requires-Dist: pytest (>=6) ; extra == 'testing' +Requires-Dist: pytest-checkdocs (>=2.4) ; extra == 'testing' +Requires-Dist: pytest-flake8 ; extra == 'testing' +Requires-Dist: pytest-cov ; extra == 'testing' +Requires-Dist: pytest-enabler (>=1.0.1) ; extra == 'testing' +Requires-Dist: pytest-black (>=0.3.7) ; (platform_python_implementation != "PyPy") and extra == 'testing' +Requires-Dist: pytest-mypy (>=0.9.1) ; (platform_python_implementation != "PyPy") and extra == 'testing' + +.. image:: https://img.shields.io/pypi/v/nspektr.svg + :target: `PyPI link`_ + +.. image:: https://img.shields.io/pypi/pyversions/nspektr.svg + :target: `PyPI link`_ + +.. _PyPI link: https://pypi.org/project/nspektr + +.. image:: https://github.com/jaraco/nspektr/workflows/tests/badge.svg + :target: https://github.com/jaraco/nspektr/actions?query=workflow%3A%22tests%22 + :alt: tests + +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black + :alt: Code style: Black + +.. .. image:: https://readthedocs.org/projects/skeleton/badge/?version=latest +.. :target: https://skeleton.readthedocs.io/en/latest/?badge=latest + +.. image:: https://img.shields.io/badge/skeleton-2022-informational + :target: https://blog.jaraco.com/skeleton + + diff --git a/setuptools/_vendor/nspektr-0.3.0.dist-info/RECORD b/setuptools/_vendor/nspektr-0.3.0.dist-info/RECORD new file mode 100644 index 0000000..5e5de5e --- /dev/null +++ b/setuptools/_vendor/nspektr-0.3.0.dist-info/RECORD @@ -0,0 +1,11 @@ +nspektr-0.3.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +nspektr-0.3.0.dist-info/LICENSE,sha256=2z8CRrH5J48VhFuZ_sR4uLUG63ZIeZNyL4xuJUKF-vg,1050 +nspektr-0.3.0.dist-info/METADATA,sha256=X0stV4vwFBDBxvzhBl4kAHVdGWPIjEitqAuTJItcQH0,2162 +nspektr-0.3.0.dist-info/RECORD,, +nspektr-0.3.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +nspektr-0.3.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92 +nspektr-0.3.0.dist-info/top_level.txt,sha256=uEA20Ixo04XS3wOIt5-Jk5ZuMkBrtlleFipRr8Y1SjQ,8 +nspektr/__init__.py,sha256=d6-d-ZlGAQQP-MEi_NZMiyn2vLbq8Hw3HxICgm3X0Q8,3949 +nspektr/__pycache__/__init__.cpython-310.pyc,, +nspektr/__pycache__/_compat.cpython-310.pyc,, +nspektr/_compat.py,sha256=2QoozYhuhgow_NMUATmhoM-yppBV3jiZYQgdiP-ww0s,582 diff --git a/setuptools/_vendor/nspektr-0.3.0.dist-info/REQUESTED b/setuptools/_vendor/nspektr-0.3.0.dist-info/REQUESTED new file mode 100644 index 0000000..e69de29 diff --git a/setuptools/_vendor/nspektr-0.3.0.dist-info/WHEEL b/setuptools/_vendor/nspektr-0.3.0.dist-info/WHEEL new file mode 100644 index 0000000..becc9a6 --- /dev/null +++ b/setuptools/_vendor/nspektr-0.3.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.37.1) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/setuptools/_vendor/nspektr-0.3.0.dist-info/top_level.txt b/setuptools/_vendor/nspektr-0.3.0.dist-info/top_level.txt new file mode 100644 index 0000000..b10ef50 --- /dev/null +++ b/setuptools/_vendor/nspektr-0.3.0.dist-info/top_level.txt @@ -0,0 +1 @@ +nspektr diff --git a/setuptools/_vendor/nspektr/__init__.py b/setuptools/_vendor/nspektr/__init__.py new file mode 100644 index 0000000..938bbdb --- /dev/null +++ b/setuptools/_vendor/nspektr/__init__.py @@ -0,0 +1,145 @@ +import itertools +import functools +import contextlib + +from setuptools.extern.packaging.requirements import Requirement +from setuptools.extern.packaging.version import Version +from setuptools.extern.more_itertools import always_iterable +from setuptools.extern.jaraco.context import suppress +from setuptools.extern.jaraco.functools import apply + +from ._compat import metadata, repair_extras + + +def resolve(req: Requirement) -> metadata.Distribution: + """ + Resolve the requirement to its distribution. + + Ignore exception detail for Python 3.9 compatibility. + + >>> resolve(Requirement('pytest<3')) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + importlib.metadata.PackageNotFoundError: No package metadata was found for pytest<3 + """ + dist = metadata.distribution(req.name) + if not req.specifier.contains(Version(dist.version), prereleases=True): + raise metadata.PackageNotFoundError(str(req)) + dist.extras = req.extras # type: ignore + return dist + + +@apply(bool) +@suppress(metadata.PackageNotFoundError) +def is_satisfied(req: Requirement): + return resolve(req) + + +unsatisfied = functools.partial(itertools.filterfalse, is_satisfied) + + +class NullMarker: + @classmethod + def wrap(cls, req: Requirement): + return req.marker or cls() + + def evaluate(self, *args, **kwargs): + return True + + +def find_direct_dependencies(dist, extras=None): + """ + Find direct, declared dependencies for dist. + """ + simple = ( + req + for req in map(Requirement, always_iterable(dist.requires)) + if NullMarker.wrap(req).evaluate(dict(extra=None)) + ) + extra_deps = ( + req + for req in map(Requirement, always_iterable(dist.requires)) + for extra in always_iterable(getattr(dist, 'extras', extras)) + if NullMarker.wrap(req).evaluate(dict(extra=extra)) + ) + return itertools.chain(simple, extra_deps) + + +def traverse(items, visit): + """ + Given an iterable of items, traverse the items. + + For each item, visit is called to return any additional items + to include in the traversal. + """ + while True: + try: + item = next(items) + except StopIteration: + return + yield item + items = itertools.chain(items, visit(item)) + + +def find_req_dependencies(req): + with contextlib.suppress(metadata.PackageNotFoundError): + dist = resolve(req) + yield from find_direct_dependencies(dist) + + +def find_dependencies(dist, extras=None): + """ + Find all reachable dependencies for dist. + + dist is an importlib.metadata.Distribution (or similar). + TODO: create a suitable protocol for type hint. + + >>> deps = find_dependencies(resolve(Requirement('nspektr'))) + >>> all(isinstance(dep, Requirement) for dep in deps) + True + >>> not any('pytest' in str(dep) for dep in deps) + True + >>> test_deps = find_dependencies(resolve(Requirement('nspektr[testing]'))) + >>> any('pytest' in str(dep) for dep in test_deps) + True + """ + + def visit(req, seen=set()): + if req in seen: + return () + seen.add(req) + return find_req_dependencies(req) + + return traverse(find_direct_dependencies(dist, extras), visit) + + +class Unresolved(Exception): + def __iter__(self): + return iter(self.args[0]) + + +def missing(ep): + """ + Generate the unresolved dependencies (if any) of ep. + """ + return unsatisfied(find_dependencies(ep.dist, repair_extras(ep.extras))) + + +def check(ep): + """ + >>> ep, = metadata.entry_points(group='console_scripts', name='pip') + >>> check(ep) + >>> dist = metadata.distribution('nspektr') + + Since 'docs' extras are not installed, requesting them should fail. + + >>> ep = metadata.EntryPoint( + ... group=None, name=None, value='nspektr [docs]')._for(dist) + >>> check(ep) + Traceback (most recent call last): + ... + nspektr.Unresolved: [...] + """ + missed = list(missing(ep)) + if missed: + raise Unresolved(missed) diff --git a/setuptools/_vendor/nspektr/_compat.py b/setuptools/_vendor/nspektr/_compat.py new file mode 100644 index 0000000..3278379 --- /dev/null +++ b/setuptools/_vendor/nspektr/_compat.py @@ -0,0 +1,21 @@ +import contextlib +import sys + + +if sys.version_info >= (3, 10): + import importlib.metadata as metadata +else: + import setuptools.extern.importlib_metadata as metadata # type: ignore # noqa: F401 + + +def repair_extras(extras): + """ + Repair extras that appear as match objects. + + python/importlib_metadata#369 revealed a flaw in the EntryPoint + implementation. This function wraps the extras to ensure + they are proper strings even on older implementations. + """ + with contextlib.suppress(AttributeError): + return list(item.group(0) for item in extras) + return extras diff --git a/setuptools/_vendor/vendored.txt b/setuptools/_vendor/vendored.txt index 84c4006..95de2dc 100644 --- a/setuptools/_vendor/vendored.txt +++ b/setuptools/_vendor/vendored.txt @@ -5,6 +5,7 @@ more_itertools==8.8.0 jaraco.text==3.7.0 importlib_resources==5.4.0 importlib_metadata==4.11.1 +nspektr==0.3.0 # required for importlib_metadata on older Pythons typing_extensions==4.0.1 # required for importlib_resources and _metadata on older Pythons diff --git a/setuptools/build_meta.py b/setuptools/build_meta.py index 5dc65e2..801ec30 100644 --- a/setuptools/build_meta.py +++ b/setuptools/build_meta.py @@ -28,17 +28,22 @@ Again, this is not a formal definition! Just a "taste" of the module. import io import os +import shlex import sys import tokenize import shutil import contextlib import tempfile import warnings +from pathlib import Path +from typing import Dict, Iterator, List, Optional, Union import setuptools import distutils +from ._path import same_path from ._reqs import parse_strings -from .extern.more_itertools import always_iterable +from ._deprecation_warning import SetuptoolsDeprecationWarning +from distutils.util import strtobool __all__ = ['get_requires_for_build_sdist', @@ -46,9 +51,14 @@ __all__ = ['get_requires_for_build_sdist', 'prepare_metadata_for_build_wheel', 'build_wheel', 'build_sdist', + 'get_requires_for_build_editable', + 'build_editable', '__legacy__', 'SetupRequirementsError'] +SETUPTOOLS_ENABLE_FEATURES = os.getenv("SETUPTOOLS_ENABLE_FEATURES", "").lower() +LEGACY_EDITABLE = "legacy-editable" in SETUPTOOLS_ENABLE_FEATURES.replace("_", "-") + class SetupRequirementsError(BaseException): def __init__(self, specifiers): @@ -127,33 +137,189 @@ def suppress_known_deprecation(): yield -class _BuildMetaBackend: +_ConfigSettings = Optional[Dict[str, Union[str, List[str], None]]] +""" +Currently the user can run:: + + pip install -e . --config-settings key=value + python -m build -C--key=value -C key=value + +- pip will pass both key and value as strings and overwriting repeated keys + (pypa/pip#11059). +- build will accumulate values associated with repeated keys in a list. + It will also accept keys with no associated value. + This means that an option passed by build can be ``str | list[str] | None``. +- PEP 517 specifies that ``config_settings`` is an optional dict. +""" - @staticmethod - def _fix_config(config_settings): + +class _ConfigSettingsTranslator: + """Translate ``config_settings`` into distutils-style command arguments. + Only a limited number of options is currently supported. + """ + # See pypa/setuptools#1928 pypa/setuptools#2491 + + def _get_config(self, key: str, config_settings: _ConfigSettings) -> List[str]: """ - Ensure config settings meet certain expectations. - - >>> fc = _BuildMetaBackend._fix_config - >>> fc(None) - {'--global-option': []} - >>> fc({}) - {'--global-option': []} - >>> fc({'--global-option': 'foo'}) - {'--global-option': ['foo']} - >>> fc({'--global-option': ['foo']}) - {'--global-option': ['foo']} + Get the value of a specific key in ``config_settings`` as a list of strings. + + >>> fn = _ConfigSettingsTranslator()._get_config + >>> fn("--global-option", None) + [] + >>> fn("--global-option", {}) + [] + >>> fn("--global-option", {'--global-option': 'foo'}) + ['foo'] + >>> fn("--global-option", {'--global-option': ['foo']}) + ['foo'] + >>> fn("--global-option", {'--global-option': 'foo'}) + ['foo'] + >>> fn("--global-option", {'--global-option': 'foo bar'}) + ['foo', 'bar'] """ - config_settings = config_settings or {} - config_settings['--global-option'] = list(always_iterable( - config_settings.get('--global-option'))) - return config_settings + cfg = config_settings or {} + opts = cfg.get(key) or [] + return shlex.split(opts) if isinstance(opts, str) else opts - def _get_build_requires(self, config_settings, requirements): - config_settings = self._fix_config(config_settings) + def _valid_global_options(self): + """Global options accepted by setuptools (e.g. quiet or verbose).""" + options = (opt[:2] for opt in setuptools.dist.Distribution.global_options) + return {flag for long_and_short in options for flag in long_and_short if flag} + + def _global_args(self, config_settings: _ConfigSettings) -> Iterator[str]: + """ + Let the user specify ``verbose`` or ``quiet`` + escape hatch via + ``--global-option``. + Note: ``-v``, ``-vv``, ``-vvv`` have similar effects in setuptools, + so we just have to cover the basic scenario ``-v``. + + >>> fn = _ConfigSettingsTranslator()._global_args + >>> list(fn(None)) + [] + >>> list(fn({"verbose": "False"})) + ['-q'] + >>> list(fn({"verbose": "1"})) + ['-v'] + >>> list(fn({"--verbose": None})) + ['-v'] + >>> list(fn({"verbose": "true", "--global-option": "-q --no-user-cfg"})) + ['-v', '-q', '--no-user-cfg'] + >>> list(fn({"--quiet": None})) + ['-q'] + """ + cfg = config_settings or {} + falsey = {"false", "no", "0", "off"} + if "verbose" in cfg or "--verbose" in cfg: + level = str(cfg.get("verbose") or cfg.get("--verbose") or "1") + yield ("-q" if level.lower() in falsey else "-v") + if "quiet" in cfg or "--quiet" in cfg: + level = str(cfg.get("quiet") or cfg.get("--quiet") or "1") + yield ("-v" if level.lower() in falsey else "-q") + + valid = self._valid_global_options() + args = self._get_config("--global-option", config_settings) + yield from (arg for arg in args if arg.strip("-") in valid) + + def __dist_info_args(self, config_settings: _ConfigSettings) -> Iterator[str]: + """ + The ``dist_info`` command accepts ``tag-date`` and ``tag-build``. + + .. warning:: + We cannot use this yet as it requires the ``sdist`` and ``bdist_wheel`` + commands run in ``build_sdist`` and ``build_wheel`` to re-use the egg-info + directory created in ``prepare_metadata_for_build_wheel``. + + >>> fn = _ConfigSettingsTranslator()._ConfigSettingsTranslator__dist_info_args + >>> list(fn(None)) + [] + >>> list(fn({"tag-date": "False"})) + ['--no-date'] + >>> list(fn({"tag-date": None})) + ['--no-date'] + >>> list(fn({"tag-date": "true", "tag-build": ".a"})) + ['--tag-date', '--tag-build', '.a'] + """ + cfg = config_settings or {} + if "tag-date" in cfg: + val = strtobool(str(cfg["tag-date"] or "false")) + yield ("--tag-date" if val else "--no-date") + if "tag-build" in cfg: + yield from ["--tag-build", str(cfg["tag-build"])] + + def _editable_args(self, config_settings: _ConfigSettings) -> Iterator[str]: + """ + The ``editable_wheel`` command accepts ``editable-mode=strict``. + + >>> fn = _ConfigSettingsTranslator()._editable_args + >>> list(fn(None)) + [] + >>> list(fn({"editable-mode": "strict"})) + ['--strict'] + >>> list(fn({"editable-mode": "other"})) + Traceback (most recent call last): + ... + ValueError: Invalid value for `editable-mode`: 'other'. Try: 'strict'. + """ + cfg = config_settings or {} + if "editable-mode" not in cfg and "editable_mode" not in cfg: + return + mode = cfg.get("editable-mode") or cfg.get("editable_mode") + if mode != "strict": + msg = f"Invalid value for `editable-mode`: {mode!r}. Try: 'strict'." + raise ValueError(msg) + yield "--strict" + + def _arbitrary_args(self, config_settings: _ConfigSettings) -> Iterator[str]: + """ + Users may expect to pass arbitrary lists of arguments to a command + via "--global-option" (example provided in PEP 517 of a "escape hatch"). + + >>> fn = _ConfigSettingsTranslator()._arbitrary_args + >>> list(fn(None)) + [] + >>> list(fn({})) + [] + >>> list(fn({'--build-option': 'foo'})) + ['foo'] + >>> list(fn({'--build-option': ['foo']})) + ['foo'] + >>> list(fn({'--build-option': 'foo'})) + ['foo'] + >>> list(fn({'--build-option': 'foo bar'})) + ['foo', 'bar'] + >>> warnings.simplefilter('error', SetuptoolsDeprecationWarning) + >>> list(fn({'--global-option': 'foo'})) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + SetuptoolsDeprecationWarning: ...arguments given via `--global-option`... + """ + args = self._get_config("--global-option", config_settings) + global_opts = self._valid_global_options() + bad_args = [] - sys.argv = sys.argv[:1] + ['egg_info'] + \ - config_settings["--global-option"] + for arg in args: + if arg.strip("-") not in global_opts: + bad_args.append(arg) + yield arg + + yield from self._get_config("--build-option", config_settings) + + if bad_args: + msg = f""" + The arguments {bad_args!r} were given via `--global-option`. + Please use `--build-option` instead, + `--global-option` is reserved to flags like `--verbose` or `--quiet`. + """ + warnings.warn(msg, SetuptoolsDeprecationWarning) + + +class _BuildMetaBackend(_ConfigSettingsTranslator): + def _get_build_requires(self, config_settings, requirements): + sys.argv = [ + *sys.argv[:1], + *self._global_args(config_settings), + "egg_info", + *self._arbitrary_args(config_settings), + ] try: with Distribution.patch(): self.run_setup() @@ -174,57 +340,57 @@ class _BuildMetaBackend: exec(compile(code, __file__, 'exec'), locals()) def get_requires_for_build_wheel(self, config_settings=None): - return self._get_build_requires( - config_settings, requirements=['wheel']) + return self._get_build_requires(config_settings, requirements=['wheel']) def get_requires_for_build_sdist(self, config_settings=None): return self._get_build_requires(config_settings, requirements=[]) - def prepare_metadata_for_build_wheel(self, metadata_directory, - config_settings=None): - sys.argv = sys.argv[:1] + [ - 'dist_info', '--egg-base', metadata_directory] - with no_install_setup_requires(): - self.run_setup() - - dist_info_directory = metadata_directory - while True: - dist_infos = [f for f in os.listdir(dist_info_directory) - if f.endswith('.dist-info')] + def _bubble_up_info_directory(self, metadata_directory: str, suffix: str) -> str: + """ + PEP 517 requires that the .dist-info directory be placed in the + metadata_directory. To comply, we MUST copy the directory to the root. - if ( - len(dist_infos) == 0 and - len(_get_immediate_subdirectories(dist_info_directory)) == 1 - ): + Returns the basename of the info directory, e.g. `proj-0.0.0.dist-info`. + """ + candidates = list(Path(metadata_directory).glob(f"**/*{suffix}/")) + assert len(candidates) == 1, f"Exactly one {suffix} should have been produced" + info_dir = candidates[0] - dist_info_directory = os.path.join( - dist_info_directory, os.listdir(dist_info_directory)[0]) - continue + if not same_path(info_dir.parent, metadata_directory): + shutil.move(str(info_dir), metadata_directory) + # PEP 517 allow other files and dirs to exist in metadata_directory - assert len(dist_infos) == 1 - break + return info_dir.name - # PEP 517 requires that the .dist-info directory be placed in the - # metadata_directory. To comply, we MUST copy the directory to the root - if dist_info_directory != metadata_directory: - shutil.move( - os.path.join(dist_info_directory, dist_infos[0]), - metadata_directory) - shutil.rmtree(dist_info_directory, ignore_errors=True) + def prepare_metadata_for_build_wheel(self, metadata_directory, + config_settings=None): + sys.argv = [ + *sys.argv[:1], + *self._global_args(config_settings), + "dist_info", + "--output-dir", metadata_directory, + "--keep-egg-info", + ] + with no_install_setup_requires(): + self.run_setup() - return dist_infos[0] + self._bubble_up_info_directory(metadata_directory, ".egg-info") + return self._bubble_up_info_directory(metadata_directory, ".dist-info") def _build_with_temp_dir(self, setup_command, result_extension, result_directory, config_settings): - config_settings = self._fix_config(config_settings) result_directory = os.path.abspath(result_directory) # Build in a temporary directory, then copy to the target. os.makedirs(result_directory, exist_ok=True) with tempfile.TemporaryDirectory(dir=result_directory) as tmp_dist_dir: - sys.argv = (sys.argv[:1] + setup_command + - ['--dist-dir', tmp_dist_dir] + - config_settings["--global-option"]) + sys.argv = [ + *sys.argv[:1], + *self._global_args(config_settings), + *setup_command, + "--dist-dir", tmp_dist_dir, + *self._arbitrary_args(config_settings), + ] with no_install_setup_requires(): self.run_setup() @@ -249,6 +415,39 @@ class _BuildMetaBackend: '.tar.gz', sdist_directory, config_settings) + def _get_dist_info_dir(self, metadata_directory: Optional[str]) -> Optional[str]: + if not metadata_directory: + return None + dist_info_candidates = list(Path(metadata_directory).glob("*.dist-info")) + assert len(dist_info_candidates) <= 1 + return str(dist_info_candidates[0]) if dist_info_candidates else None + + if not LEGACY_EDITABLE: + + # PEP660 hooks: + # build_editable + # get_requires_for_build_editable + # prepare_metadata_for_build_editable + def build_editable( + self, wheel_directory, config_settings=None, metadata_directory=None + ): + # XXX can or should we hide our editable_wheel command normally? + info_dir = self._get_dist_info_dir(metadata_directory) + opts = ["--dist-info-dir", info_dir] if info_dir else [] + cmd = ["editable_wheel", *opts, *self._editable_args(config_settings)] + return self._build_with_temp_dir( + cmd, ".whl", wheel_directory, config_settings + ) + + def get_requires_for_build_editable(self, config_settings=None): + return self.get_requires_for_build_wheel(config_settings) + + def prepare_metadata_for_build_editable(self, metadata_directory, + config_settings=None): + return self.prepare_metadata_for_build_wheel( + metadata_directory, config_settings + ) + class _BuildMetaLegacyBackend(_BuildMetaBackend): """Compatibility backend for setuptools @@ -299,6 +498,11 @@ prepare_metadata_for_build_wheel = _BACKEND.prepare_metadata_for_build_wheel build_wheel = _BACKEND.build_wheel build_sdist = _BACKEND.build_sdist +if not LEGACY_EDITABLE: + get_requires_for_build_editable = _BACKEND.get_requires_for_build_editable + prepare_metadata_for_build_editable = _BACKEND.prepare_metadata_for_build_editable + build_editable = _BACKEND.build_editable + # The legacy backend __legacy__ = _BuildMetaLegacyBackend() diff --git a/setuptools/command/build.py b/setuptools/command/build.py index 12a4362..1396afd 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -1,8 +1,17 @@ -from distutils.command.build import build as _build +import sys import warnings +from typing import TYPE_CHECKING, List, Dict +from distutils.command.build import build as _build from setuptools import SetuptoolsDeprecationWarning +if sys.version_info >= (3, 8): + from typing import Protocol +elif TYPE_CHECKING: + from typing_extensions import Protocol +else: + from abc import ABC as Protocol + _ORIGINAL_SUBCOMMANDS = {"build_py", "build_clib", "build_ext", "build_scripts"} @@ -22,3 +31,116 @@ class build(_build): warnings.warn(msg, SetuptoolsDeprecationWarning) self.sub_commands = _build.sub_commands super().run() + + +class SubCommand(Protocol): + """In order to support editable installations (see :pep:`660`) all + build subcommands **SHOULD** implement this protocol. They also **MUST** inherit + from ``setuptools.Command``. + + When creating an :pep:`editable wheel <660>`, ``setuptools`` will try to evaluate + custom ``build`` subcommands using the following procedure: + + 1. ``setuptools`` will set the ``editable_mode`` flag will be set to ``True`` + 2. ``setuptools`` will execute the ``run()`` command. + + .. important:: + Subcommands **SHOULD** take advantage of ``editable_mode=True`` to adequate + its behaviour or perform optimisations. + + For example, if a subcommand don't need to generate any extra file and + everything it does is to copy a source file into the build directory, + ``run()`` **SHOULD** simply "early return". + + Similarly, if the subcommand creates files that would be placed alongside + Python files in the final distribution, during an editable install + the command **SHOULD** generate these files "in place" (i.e. write them to + the original source directory, instead of using the build directory). + Note that ``get_output_mapping()`` should reflect that and include mappings + for "in place" builds accordingly. + + 3. ``setuptools`` use any knowledge it can derive from the return values of + ``get_outputs()`` and ``get_output_mapping()`` to create an editable wheel. + When relevant ``setuptools`` **MAY** attempt to use file links based on the value + of ``get_output_mapping()``. Alternatively, ``setuptools`` **MAY** attempt to use + :doc:`import hooks ` to redirect any attempt to import + to the directory with the original source code and other files built in place. + + Please note that custom sub-commands **SHOULD NOT** rely on ``run()`` being + executed (or not) to provide correct return values for ``get_outputs()``, + ``get_output_mapping()`` or ``get_source_files()``. The ``get_*`` methods should + work independently of ``run()``. + """ + + editable_mode: bool = False + """Boolean flag that will be set to ``True`` when setuptools is used for an + editable installation (see :pep:`660`). + Implementations **SHOULD** explicitly set the default value of this attribute to + ``False``. + When subcommands run, they can use this flag to perform optimizations or change + their behaviour accordingly. + """ + + build_lib: str + """String representing the directory where the build artifacts should be stored, + e.g. ``build/lib``. + For example, if a distribution wants to provide a Python module named ``pkg.mod``, + then a corresponding file should be written to ``{build_lib}/package/module.py``. + A way of thinking about this is that the files saved under ``build_lib`` + would be eventually copied to one of the directories in :obj:`site.PREFIXES` + upon installation. + + A command that produces platform-independent files (e.g. compiling text templates + into Python functions), **CAN** initialize ``build_lib`` by copying its value from + the ``build_py`` command. On the other hand, a command that produces + platform-specific files **CAN** initialize ``build_lib`` by copying its value from + the ``build_ext`` command. In general this is done inside the ``finalize_options`` + method with the help of the ``set_undefined_options`` command:: + + def finalize_options(self): + self.set_undefined_options("build_py", ("build_lib", "build_lib")) + ... + """ + + def initialize_options(self): + """(Required by the original :class:`setuptools.Command` interface)""" + + def finalize_options(self): + """(Required by the original :class:`setuptools.Command` interface)""" + + def run(self): + """(Required by the original :class:`setuptools.Command` interface)""" + + def get_source_files(self) -> List[str]: + """ + Return a list of all files that are used by the command to create the expected + outputs. + For example, if your build command transpiles Java files into Python, you should + list here all the Java files. + The primary purpose of this function is to help populating the ``sdist`` + with all the files necessary to build the distribution. + All files should be strings relative to the project root directory. + """ + + def get_outputs(self) -> List[str]: + """ + Return a list of files intended for distribution as they would have been + produced by the build. + These files should be strings in the form of + ``"{build_lib}/destination/file/path"``. + + .. note:: + The return value of ``get_output()`` should include all files used as keys + in ``get_output_mapping()`` plus files that are generated during the build + and don't correspond to any source file already present in the project. + """ + + def get_output_mapping(self) -> Dict[str, str]: + """ + Return a mapping between destination files as they would be produced by the + build (dict keys) into the respective existing (source) files (dict values). + Existing (source) files should be represented as strings relative to the project + root directory. + Destination files should be strings in the form of + ``"{build_lib}/destination/file/path"``. + """ diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index c59eff8..7ad5a87 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -2,14 +2,16 @@ import os import sys import itertools from importlib.machinery import EXTENSION_SUFFIXES +from importlib.util import cache_from_source as _compiled_file_name +from typing import Dict, Iterator, List, Tuple + from distutils.command.build_ext import build_ext as _du_build_ext -from distutils.file_util import copy_file from distutils.ccompiler import new_compiler from distutils.sysconfig import customize_compiler, get_config_var -from distutils.errors import DistutilsError from distutils import log -from setuptools.extension import Library +from setuptools.errors import BaseError +from setuptools.extension import Extension, Library try: # Attempt to use Cython for building extensions, if available @@ -73,6 +75,9 @@ def get_abi3_suffix(): class build_ext(_build_ext): + editable_mode: bool = False + inplace: bool = False + def run(self): """Build extensions in build directory, then copy if --inplace""" old_inplace, self.inplace = self.inplace, 0 @@ -81,27 +86,61 @@ class build_ext(_build_ext): if old_inplace: self.copy_extensions_to_source() + def _get_inplace_equivalent(self, build_py, ext: Extension) -> Tuple[str, str]: + fullname = self.get_ext_fullname(ext.name) + filename = self.get_ext_filename(fullname) + modpath = fullname.split('.') + package = '.'.join(modpath[:-1]) + package_dir = build_py.get_package_dir(package) + inplace_file = os.path.join(package_dir, os.path.basename(filename)) + regular_file = os.path.join(self.build_lib, filename) + return (inplace_file, regular_file) + def copy_extensions_to_source(self): build_py = self.get_finalized_command('build_py') for ext in self.extensions: - fullname = self.get_ext_fullname(ext.name) - filename = self.get_ext_filename(fullname) - modpath = fullname.split('.') - package = '.'.join(modpath[:-1]) - package_dir = build_py.get_package_dir(package) - dest_filename = os.path.join(package_dir, - os.path.basename(filename)) - src_filename = os.path.join(self.build_lib, filename) + inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext) # Always copy, even if source is older than destination, to ensure # that the right extensions for the current Python/platform are # used. - copy_file( - src_filename, dest_filename, verbose=self.verbose, - dry_run=self.dry_run - ) + self.copy_file(regular_file, inplace_file, level=self.verbose) + if ext._needs_stub: - self.write_stub(package_dir or os.curdir, ext, True) + inplace_stub = self._get_equivalent_stub(ext, inplace_file) + self._write_stub_file(inplace_stub, ext, compile=True) + # Always compile stub and remove the original (leave the cache behind) + # (this behaviour was observed in previous iterations of the code) + + def _get_equivalent_stub(self, ext: Extension, output_file: str) -> str: + dir_ = os.path.dirname(output_file) + _, _, name = ext.name.rpartition(".") + return f"{os.path.join(dir_, name)}.py" + + def _get_output_mapping(self) -> Iterator[Tuple[str, str]]: + if not self.inplace: + return + + build_py = self.get_finalized_command('build_py') + opt = self.get_finalized_command('install_lib').optimize or "" + + for ext in self.extensions: + inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext) + yield (regular_file, inplace_file) + + if ext._needs_stub: + # This version of `build_ext` always builds artifacts in another dir, + # when "inplace=True" is given it just copies them back. + # This is done in the `copy_extensions_to_source` function, which + # always compile stub files via `_compile_and_remove_stub`. + # At the end of the process, a `.pyc` stub file is created without the + # corresponding `.py`. + + inplace_stub = self._get_equivalent_stub(ext, inplace_file) + regular_stub = self._get_equivalent_stub(ext, regular_file) + inplace_cache = _compiled_file_name(inplace_stub, optimization=opt) + output_cache = _compiled_file_name(regular_stub, optimization=opt) + yield (output_cache, inplace_cache) def get_ext_filename(self, fullname): so_ext = os.getenv('SETUPTOOLS_EXT_SUFFIX') @@ -131,6 +170,7 @@ class build_ext(_build_ext): self.shlib_compiler = None self.shlibs = [] self.ext_map = {} + self.editable_mode = False def finalize_options(self): _build_ext.finalize_options(self) @@ -161,6 +201,9 @@ class build_ext(_build_ext): if ltd and use_stubs and os.curdir not in ext.runtime_library_dirs: ext.runtime_library_dirs.append(os.curdir) + if self.editable_mode: + self.inplace = True + def setup_shlib_compiler(self): compiler = self.shlib_compiler = new_compiler( compiler=self.compiler, dry_run=self.dry_run, force=self.force @@ -201,8 +244,8 @@ class build_ext(_build_ext): self.compiler = self.shlib_compiler _build_ext.build_extension(self, ext) if ext._needs_stub: - cmd = self.get_finalized_command('build_py').build_lib - self.write_stub(cmd, ext) + build_lib = self.get_finalized_command('build_py').build_lib + self.write_stub(build_lib, ext) finally: self.compiler = _compiler @@ -215,8 +258,15 @@ class build_ext(_build_ext): pkg = '.'.join(ext._full_name.split('.')[:-1] + ['']) return any(pkg + libname in libnames for libname in ext.libraries) - def get_outputs(self): - return _build_ext.get_outputs(self) + self.__get_stubs_outputs() + def get_outputs(self) -> List[str]: + if self.inplace: + return list(self.get_output_mapping().keys()) + return sorted(_build_ext.get_outputs(self) + self.__get_stubs_outputs()) + + def get_output_mapping(self) -> Dict[str, str]: + """See :class:`setuptools.commands.build.SubCommand`""" + mapping = self._get_output_mapping() + return dict(sorted(mapping, key=lambda x: x[0])) def __get_stubs_outputs(self): # assemble the base name for each extension that needs a stub @@ -236,12 +286,13 @@ class build_ext(_build_ext): yield '.pyo' def write_stub(self, output_dir, ext, compile=False): - log.info("writing stub loader for %s to %s", ext._full_name, - output_dir) - stub_file = (os.path.join(output_dir, *ext._full_name.split('.')) + - '.py') + stub_file = os.path.join(output_dir, *ext._full_name.split('.')) + '.py' + self._write_stub_file(stub_file, ext, compile) + + def _write_stub_file(self, stub_file: str, ext: Extension, compile=False): + log.info("writing stub loader for %s to %s", ext._full_name, stub_file) if compile and os.path.exists(stub_file): - raise DistutilsError(stub_file + " already exists! Please delete.") + raise BaseError(stub_file + " already exists! Please delete.") if not self.dry_run: f = open(stub_file, 'w') f.write( @@ -274,16 +325,19 @@ class build_ext(_build_ext): ) f.close() if compile: - from distutils.util import byte_compile + self._compile_and_remove_stub(stub_file) + + def _compile_and_remove_stub(self, stub_file: str): + from distutils.util import byte_compile - byte_compile([stub_file], optimize=0, + byte_compile([stub_file], optimize=0, + force=True, dry_run=self.dry_run) + optimize = self.get_finalized_command('install_lib').optimize + if optimize > 0: + byte_compile([stub_file], optimize=optimize, force=True, dry_run=self.dry_run) - optimize = self.get_finalized_command('install_lib').optimize - if optimize > 0: - byte_compile([stub_file], optimize=optimize, - force=True, dry_run=self.dry_run) - if os.path.exists(stub_file) and not self.dry_run: - os.unlink(stub_file) + if os.path.exists(stub_file) and not self.dry_run: + os.unlink(stub_file) if use_stubs or os.name == 'nt': diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 2fced3d..923a323 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -11,6 +11,8 @@ import itertools import stat import warnings from pathlib import Path +from typing import Dict, Iterator, List, Optional, Tuple + from setuptools._deprecation_warning import SetuptoolsDeprecationWarning from setuptools.extern.more_itertools import unique_everseen @@ -28,6 +30,8 @@ class build_py(orig.build_py): Also, this version of the 'build_py' command allows you to specify both 'py_modules' and 'packages' in the same setup operation. """ + editable_mode: bool = False + existing_egg_info_dir: Optional[str] = None #: Private API, internal use only. def finalize_options(self): orig.build_py.finalize_options(self) @@ -37,9 +41,19 @@ class build_py(orig.build_py): del self.__dict__['data_files'] self.__updated_files = [] + def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1, + link=None, level=1): + # Overwrite base class to allow using links + if link: + infile = str(Path(infile).resolve()) + outfile = str(Path(outfile).resolve()) + return super().copy_file(infile, outfile, preserve_mode, preserve_times, + link, level) + def run(self): """Build modules, packages, and copy data files to build directory""" - if not self.py_modules and not self.packages: + # if self.editable_mode or not (self.py_modules and self.packages): + if not (self.py_modules or self.packages) or self.editable_mode: return if self.py_modules: @@ -112,16 +126,41 @@ class build_py(orig.build_py): ) return self.exclude_data_files(package, src_dir, files) - def build_package_data(self): - """Copy data files into build directory""" + def get_outputs(self, include_bytecode=1) -> List[str]: + """See :class:`setuptools.commands.build.SubCommand`""" + if self.editable_mode: + return list(self.get_output_mapping().keys()) + return super().get_outputs(include_bytecode) + + def get_output_mapping(self) -> Dict[str, str]: + """See :class:`setuptools.commands.build.SubCommand`""" + mapping = itertools.chain( + self._get_package_data_output_mapping(), + self._get_module_mapping(), + ) + return dict(sorted(mapping, key=lambda x: x[0])) + + def _get_module_mapping(self) -> Iterator[Tuple[str, str]]: + """Iterate over all modules producing (dest, src) pairs.""" + for (package, module, module_file) in self.find_all_modules(): + package = package.split('.') + filename = self.get_module_outfile(self.build_lib, package, module) + yield (filename, module_file) + + def _get_package_data_output_mapping(self) -> Iterator[Tuple[str, str]]: + """Iterate over package data producing (dest, src) pairs.""" for package, src_dir, build_dir, filenames in self.data_files: for filename in filenames: target = os.path.join(build_dir, filename) - self.mkpath(os.path.dirname(target)) srcfile = os.path.join(src_dir, filename) - outf, copied = self.copy_file(srcfile, target) - make_writable(target) - srcfile = os.path.abspath(srcfile) + yield (target, srcfile) + + def build_package_data(self): + """Copy data files into build directory""" + for target, srcfile in self._get_package_data_output_mapping(): + self.mkpath(os.path.dirname(target)) + _outf, _copied = self.copy_file(srcfile, target) + make_writable(target) def analyze_manifest(self): self.manifest_files = mf = {} @@ -132,10 +171,19 @@ class build_py(orig.build_py): # Locate package source directory src_dirs[assert_relative(self.get_package_dir(package))] = package - self.run_command('egg_info') + if ( + getattr(self, 'existing_egg_info_dir', None) + and Path(self.existing_egg_info_dir, "SOURCES.txt").exists() + ): + manifest = Path(self.existing_egg_info_dir, "SOURCES.txt") + files = manifest.read_text(encoding="utf-8").splitlines() + else: + self.run_command('egg_info') + ei_cmd = self.get_finalized_command('egg_info') + files = ei_cmd.filelist.files + check = _IncludePackageDataAbuse() - ei_cmd = self.get_finalized_command('egg_info') - for path in ei_cmd.filelist.files: + for path in files: d, f = os.path.split(assert_relative(path)) prev = None oldf = f @@ -189,6 +237,8 @@ class build_py(orig.build_py): def initialize_options(self): self.packages_checked = {} orig.build_py.initialize_options(self) + self.editable_mode = False + self.existing_egg_info_dir = None def get_package_dir(self, package): res = orig.build_py.get_package_dir(self, package) diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index ca540ad..0685c94 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -5,12 +5,17 @@ As defined in the wheel specification import os import re +import shutil +import sys import warnings +from contextlib import contextmanager from inspect import cleandoc +from pathlib import Path from distutils.core import Command from distutils import log from setuptools.extern import packaging +from setuptools._deprecation_warning import SetuptoolsDeprecationWarning class dist_info(Command): @@ -19,28 +24,85 @@ class dist_info(Command): user_options = [ ('egg-base=', 'e', "directory containing .egg-info directories" - " (default: top of the source tree)"), + " (default: top of the source tree)" + " DEPRECATED: use --output-dir."), + ('output-dir=', 'o', "directory inside of which the .dist-info will be" + "created (default: top of the source tree)"), + ('tag-date', 'd', "Add date stamp (e.g. 20050528) to version number"), + ('tag-build=', 'b', "Specify explicit tag to add to version number"), + ('no-date', 'D', "Don't include date stamp [default]"), + ('keep-egg-info', None, "*TRANSITIONAL* will be removed in the future"), ] + boolean_options = ['tag-date', 'keep-egg-info'] + negative_opt = {'no-date': 'tag-date'} + def initialize_options(self): self.egg_base = None + self.output_dir = None + self.name = None + self.dist_info_dir = None + self.tag_date = None + self.tag_build = None + self.keep_egg_info = False def finalize_options(self): - pass + if self.egg_base: + msg = "--egg-base is deprecated for dist_info command. Use --output-dir." + warnings.warn(msg, SetuptoolsDeprecationWarning) + self.output_dir = self.egg_base or self.output_dir + + dist = self.distribution + project_dir = dist.src_root or os.curdir + self.output_dir = Path(self.output_dir or project_dir) + + egg_info = self.reinitialize_command("egg_info") + egg_info.egg_base = str(self.output_dir) + + if self.tag_date: + egg_info.tag_date = self.tag_date + else: + self.tag_date = egg_info.tag_date + + if self.tag_build: + egg_info.tag_build = self.tag_build + else: + self.tag_build = egg_info.tag_build - def run(self): - egg_info = self.get_finalized_command('egg_info') - egg_info.egg_base = self.egg_base egg_info.finalize_options() - egg_info.run() - name = _safe(self.distribution.get_name()) - version = _version(self.distribution.get_version()) - base = self.egg_base or os.curdir - dist_info_dir = os.path.join(base, f"{name}-{version}.dist-info") - log.info("creating '{}'".format(os.path.abspath(dist_info_dir))) + self.egg_info = egg_info + + name = _safe(dist.get_name()) + version = _version(dist.get_version()) + self.name = f"{name}-{version}" + self.dist_info_dir = os.path.join(self.output_dir, f"{self.name}.dist-info") + + @contextmanager + def _maybe_bkp_dir(self, dir_path: str, requires_bkp: bool): + if requires_bkp: + bkp_name = f"{dir_path}.__bkp__" + _rm(bkp_name, ignore_errors=True) + _copy(dir_path, bkp_name, dirs_exist_ok=True, symlinks=True) + try: + yield + finally: + _rm(dir_path, ignore_errors=True) + shutil.move(bkp_name, dir_path) + else: + yield + def run(self): + self.output_dir.mkdir(parents=True, exist_ok=True) + self.egg_info.run() + egg_info_dir = self.egg_info.egg_info + assert os.path.isdir(egg_info_dir), ".egg-info dir should have been created" + + log.info("creating '{}'".format(os.path.abspath(self.dist_info_dir))) bdist_wheel = self.get_finalized_command('bdist_wheel') - bdist_wheel.egg2dist(egg_info.egg_info, dist_info_dir) + + # TODO: if bdist_wheel if merged into setuptools, just add "keep_egg_info" there + with self._maybe_bkp_dir(egg_info_dir, self.keep_egg_info): + bdist_wheel.egg2dist(egg_info_dir, self.dist_info_dir) def _safe(component: str) -> str: @@ -67,3 +129,14 @@ def _version(version: str) -> str: """ warnings.warn(cleandoc(msg)) return _safe(v).strip("_") + + +def _rm(dir_name, **opts): + if os.path.isdir(dir_name): + shutil.rmtree(dir_name, **opts) + + +def _copy(src, dst, **opts): + if sys.version_info < (3, 8): + opts.pop("dirs_exist_ok", None) + shutil.copytree(src, dst, **opts) diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py new file mode 100644 index 0000000..5e205a4 --- /dev/null +++ b/setuptools/command/editable_wheel.py @@ -0,0 +1,737 @@ +""" +Create a wheel that, when installed, will make the source package 'editable' +(add it to the interpreter's path, including metadata) per PEP 660. Replaces +'setup.py develop'. + +.. note:: + One of the mechanisms briefly mentioned in PEP 660 to implement editable installs is + to create a separated directory inside ``build`` and use a .pth file to point to that + directory. In the context of this file such directory is referred as + *auxiliary build directory* or ``auxiliary_dir``. +""" + +import logging +import os +import re +import shutil +import sys +import traceback +import warnings +from contextlib import suppress +from inspect import cleandoc +from itertools import chain +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Tuple, + TypeVar, + Union +) + +from setuptools import Command, errors, namespaces +from setuptools.discovery import find_package_path +from setuptools.dist import Distribution + +if TYPE_CHECKING: + from wheel.wheelfile import WheelFile # noqa + +if sys.version_info >= (3, 8): + from typing import Protocol +elif TYPE_CHECKING: + from typing_extensions import Protocol +else: + from abc import ABC as Protocol + +_Path = Union[str, Path] +_P = TypeVar("_P", bound=_Path) +_logger = logging.getLogger(__name__) + + +_STRICT_WARNING = """ +New or renamed files may not be automatically picked up without a new installation. +""" + +_LAX_WARNING = """ +Options like `package-data`, `include/exclude-package-data` or +`packages.find.exclude/include` may have no effect. +""" + + +class editable_wheel(Command): + """Build 'editable' wheel for development""" + + description = "create a PEP 660 'editable' wheel" + + user_options = [ + ("dist-dir=", "d", "directory to put final built distributions in"), + ("dist-info-dir=", "I", "path to a pre-build .dist-info directory"), + ("strict", None, "perform an strict installation"), + ] + + boolean_options = ["strict"] + + def initialize_options(self): + self.dist_dir = None + self.dist_info_dir = None + self.project_dir = None + self.strict = False + + def finalize_options(self): + dist = self.distribution + self.project_dir = dist.src_root or os.curdir + self.package_dir = dist.package_dir or {} + self.dist_dir = Path(self.dist_dir or os.path.join(self.project_dir, "dist")) + + def run(self): + try: + self.dist_dir.mkdir(exist_ok=True) + self._ensure_dist_info() + + # Add missing dist_info files + bdist_wheel = self.reinitialize_command("bdist_wheel") + bdist_wheel.write_wheelfile(self.dist_info_dir) + + self._create_wheel_file(bdist_wheel) + except Exception as ex: + traceback.print_exc() + msg = """ + Support for editable installs via PEP 660 was recently introduced + in `setuptools`. If you are seeing this error, please report to: + + https://github.com/pypa/setuptools/issues + + Meanwhile you can try the legacy behavior by setting an + environment variable and trying to install again: + + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" + """ + raise errors.InternalError(cleandoc(msg)) from ex + + def _ensure_dist_info(self): + if self.dist_info_dir is None: + dist_info = self.reinitialize_command("dist_info") + dist_info.output_dir = self.dist_dir + dist_info.finalize_options() + dist_info.run() + self.dist_info_dir = dist_info.dist_info_dir + else: + assert str(self.dist_info_dir).endswith(".dist-info") + assert Path(self.dist_info_dir, "METADATA").exists() + + def _install_namespaces(self, installation_dir, pth_prefix): + # XXX: Only required to support the deprecated namespace practice + dist = self.distribution + if not dist.namespace_packages: + return + + src_root = Path(self.project_dir, self.pakcage_dir.get("", ".")).resolve() + installer = _NamespaceInstaller(dist, installation_dir, pth_prefix, src_root) + installer.install_namespaces() + + def _find_egg_info_dir(self) -> Optional[str]: + parent_dir = Path(self.dist_info_dir).parent if self.dist_info_dir else Path() + candidates = map(str, parent_dir.glob("*.egg-info")) + return next(candidates, None) + + def _configure_build( + self, name: str, unpacked_wheel: _Path, build_lib: _Path, tmp_dir: _Path + ): + """Configure commands to behave in the following ways: + + - Build commands can write to ``build_lib`` if they really want to... + (but this folder is expected to be ignored and modules are expected to live + in the project directory...) + - Binary extensions should be built in-place (editable_mode = True) + - Data/header/script files are not part of the "editable" specification + so they are written directly to the unpacked_wheel directory. + """ + # Non-editable files (data, headers, scripts) are written directly to the + # unpacked_wheel + + dist = self.distribution + wheel = str(unpacked_wheel) + build_lib = str(build_lib) + data = str(Path(unpacked_wheel, f"{name}.data", "data")) + headers = str(Path(unpacked_wheel, f"{name}.data", "headers")) + scripts = str(Path(unpacked_wheel, f"{name}.data", "scripts")) + + # egg-info may be generated again to create a manifest (used for package data) + egg_info = dist.reinitialize_command("egg_info", reinit_subcommands=True) + egg_info.egg_base = str(tmp_dir) + egg_info.ignore_egg_info_in_manifest = True + + build = dist.reinitialize_command("build", reinit_subcommands=True) + install = dist.reinitialize_command("install", reinit_subcommands=True) + + build.build_platlib = build.build_purelib = build.build_lib = build_lib + install.install_purelib = install.install_platlib = install.install_lib = wheel + install.install_scripts = build.build_scripts = scripts + install.install_headers = headers + install.install_data = data + + install_scripts = dist.get_command_obj("install_scripts") + install_scripts.no_ep = True + + build.build_temp = str(tmp_dir) + + build_py = dist.get_command_obj("build_py") + build_py.compile = False + build_py.existing_egg_info_dir = self._find_egg_info_dir() + + self._set_editable_mode() + + build.ensure_finalized() + install.ensure_finalized() + + def _set_editable_mode(self): + """Set the ``editable_mode`` flag in the build sub-commands""" + dist = self.distribution + build = dist.get_command_obj("build") + for cmd_name in build.get_sub_commands(): + cmd = dist.get_command_obj(cmd_name) + if hasattr(cmd, "editable_mode"): + cmd.editable_mode = True + + def _collect_build_outputs(self) -> Tuple[List[str], Dict[str, str]]: + files: List[str] = [] + mapping: Dict[str, str] = {} + build = self.get_finalized_command("build") + + for cmd_name in build.get_sub_commands(): + cmd = self.get_finalized_command(cmd_name) + if hasattr(cmd, "get_outputs"): + files.extend(cmd.get_outputs() or []) + if hasattr(cmd, "get_output_mapping"): + mapping.update(cmd.get_output_mapping() or {}) + + return files, mapping + + def _run_build_commands( + self, dist_name: str, unpacked_wheel: _Path, build_lib: _Path, tmp_dir: _Path + ) -> Tuple[List[str], Dict[str, str]]: + self._configure_build(dist_name, unpacked_wheel, build_lib, tmp_dir) + self.run_command("build") + files, mapping = self._collect_build_outputs() + self._run_install("headers") + self._run_install("scripts") + self._run_install("data") + return files, mapping + + def _create_wheel_file(self, bdist_wheel): + from wheel.wheelfile import WheelFile + + dist_info = self.get_finalized_command("dist_info") + dist_name = dist_info.name + tag = "-".join(bdist_wheel.get_tag()) + build_tag = "0.editable" # According to PEP 427 needs to start with digit + archive_name = f"{dist_name}-{build_tag}-{tag}.whl" + wheel_path = Path(self.dist_dir, archive_name) + if wheel_path.exists(): + wheel_path.unlink() + + unpacked_wheel = TemporaryDirectory(suffix=archive_name) + build_lib = TemporaryDirectory(suffix=".build-lib") + build_tmp = TemporaryDirectory(suffix=".build-temp") + + with unpacked_wheel as unpacked, build_lib as lib, build_tmp as tmp: + unpacked_dist_info = Path(unpacked, Path(self.dist_info_dir).name) + shutil.copytree(self.dist_info_dir, unpacked_dist_info) + self._install_namespaces(unpacked, dist_info.name) + files, mapping = self._run_build_commands(dist_name, unpacked, lib, tmp) + strategy = self._select_strategy(dist_name, tag, lib) + with strategy, WheelFile(wheel_path, "w") as wheel_obj: + strategy(wheel_obj, files, mapping) + wheel_obj.write_files(unpacked) + + return wheel_path + + def _run_install(self, category: str): + has_category = getattr(self.distribution, f"has_{category}", None) + if has_category and has_category(): + _logger.info(f"Installing {category} as non editable") + self.run_command(f"install_{category}") + + def _select_strategy( + self, + name: str, + tag: str, + build_lib: _Path, + ) -> "EditableStrategy": + """Decides which strategy to use to implement an editable installation.""" + build_name = f"__editable__.{name}-{tag}" + project_dir = Path(self.project_dir) + + if self.strict or os.getenv("SETUPTOOLS_EDITABLE", None) == "strict": + auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) + return _LinkTree(self.distribution, name, auxiliary_dir, build_lib) + + packages = _find_packages(self.distribution) + has_simple_layout = _simple_layout(packages, self.package_dir, project_dir) + if set(self.package_dir) == {""} and has_simple_layout: + # src-layout(ish) is relatively safe for a simple pth file + src_dir = self.package_dir[""] + return _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) + + # Use a MetaPathFinder to avoid adding accidental top-level packages/modules + return _TopLevelFinder(self.distribution, name) + + +class EditableStrategy(Protocol): + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): + ... + + def __enter__(self): + ... + + def __exit__(self, _exc_type, _exc_value, _traceback): + ... + + +class _StaticPth: + def __init__(self, dist: Distribution, name: str, path_entries: List[Path]): + self.dist = dist + self.name = name + self.path_entries = path_entries + + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): + entries = "\n".join((str(p.resolve()) for p in self.path_entries)) + contents = bytes(f"{entries}\n", "utf-8") + wheel.writestr(f"__editable__.{self.name}.pth", contents) + + def __enter__(self): + msg = f""" + Editable install will be performed using .pth file to extend `sys.path` with: + {self.path_entries!r} + """ + _logger.warning(msg + _LAX_WARNING) + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + ... + + +class _LinkTree(_StaticPth): + """ + Creates a ``.pth`` file that points to a link tree in the ``auxiliary_dir``. + + This strategy will only link files (not dirs), so it can be implemented in + any OS, even if that means using hardlinks instead of symlinks. + + By collocating ``auxiliary_dir`` and the original source code, limitations + with hardlinks should be avoided. + """ + def __init__( + self, dist: Distribution, + name: str, + auxiliary_dir: _Path, + build_lib: _Path, + ): + self.auxiliary_dir = Path(auxiliary_dir) + self.build_lib = Path(build_lib).resolve() + self._file = dist.get_command_obj("build_py").copy_file + super().__init__(dist, name, [self.auxiliary_dir]) + + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): + self._create_links(files, mapping) + super().__call__(wheel, files, mapping) + + def _normalize_output(self, file: str) -> Optional[str]: + # Files relative to build_lib will be normalized to None + with suppress(ValueError): + path = Path(file).resolve().relative_to(self.build_lib) + return str(path).replace(os.sep, '/') + return None + + def _create_file(self, relative_output: str, src_file: str, link=None): + dest = self.auxiliary_dir / relative_output + if not dest.parent.is_dir(): + dest.parent.mkdir(parents=True) + self._file(src_file, dest, link=link) + + def _create_links(self, outputs, output_mapping): + self.auxiliary_dir.mkdir(parents=True, exist_ok=True) + link_type = "sym" if _can_symlink_files(self.auxiliary_dir) else "hard" + mappings = { + self._normalize_output(k): v + for k, v in output_mapping.items() + } + mappings.pop(None, None) # remove files that are not relative to build_lib + + for output in outputs: + relative = self._normalize_output(output) + if relative and relative not in mappings: + self._create_file(relative, output) + + for relative, src in mappings.items(): + self._create_file(relative, src, link=link_type) + + def __enter__(self): + msg = "Strict editable install will be performed using a link tree.\n" + _logger.warning(msg + _STRICT_WARNING) + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + msg = f"""\n + Strict editable installation performed using the auxiliary directory: + {self.auxiliary_dir} + + Please be careful to not remove this directory, otherwise you might not be able + to import/use your package. + """ + warnings.warn(msg, InformationOnly) + + +class _TopLevelFinder: + def __init__(self, dist: Distribution, name: str): + self.dist = dist + self.name = name + + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): + src_root = self.dist.src_root or os.curdir + top_level = chain(_find_packages(self.dist), _find_top_level_modules(self.dist)) + package_dir = self.dist.package_dir or {} + roots = _find_package_roots(top_level, package_dir, src_root) + + namespaces_: Dict[str, List[str]] = dict(chain( + _find_namespaces(self.dist.packages, roots), + ((ns, []) for ns in _find_virtual_namespaces(roots)), + )) + + name = f"__editable__.{self.name}.finder" + finder = _make_identifier(name) + content = bytes(_finder_template(name, roots, namespaces_), "utf-8") + wheel.writestr(f"{finder}.py", content) + + content = bytes(f"import {finder}; {finder}.install()", "utf-8") + wheel.writestr(f"__editable__.{self.name}.pth", content) + + def __enter__(self): + msg = "Editable install will be performed using a meta path finder.\n" + _logger.warning(msg + _LAX_WARNING) + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + ... + + +def _can_symlink_files(base_dir: Path) -> bool: + with TemporaryDirectory(dir=str(base_dir.resolve())) as tmp: + path1, path2 = Path(tmp, "file1.txt"), Path(tmp, "file2.txt") + path1.write_text("file1", encoding="utf-8") + with suppress(AttributeError, NotImplementedError, OSError): + os.symlink(path1, path2) + if path2.is_symlink() and path2.read_text(encoding="utf-8") == "file1": + return True + + try: + os.link(path1, path2) # Ensure hard links can be created + except Exception as ex: + msg = ( + "File system does not seem to support either symlinks or hard links. " + "Strict editable installs require one of them to be supported." + ) + raise LinksNotSupported(msg) from ex + return False + + +def _simple_layout( + packages: Iterable[str], package_dir: Dict[str, str], project_dir: Path +) -> bool: + """Return ``True`` if: + - all packages are contained by the same parent directory, **and** + - all packages become importable if the parent directory is added to ``sys.path``. + + >>> _simple_layout(['a'], {"": "src"}, "/tmp/myproj") + True + >>> _simple_layout(['a', 'a.b'], {"": "src"}, "/tmp/myproj") + True + >>> _simple_layout(['a', 'a.b'], {}, "/tmp/myproj") + True + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"": "src"}, "/tmp/myproj") + True + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "a", "b": "b"}, ".") + True + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "_a", "b": "_b"}, ".") + False + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "_a"}, "/tmp/myproj") + False + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a.a1.a2": "_a2"}, ".") + False + >>> _simple_layout(['a', 'a.b'], {"": "src", "a.b": "_ab"}, "/tmp/myproj") + False + """ + layout = { + pkg: find_package_path(pkg, package_dir, project_dir) + for pkg in packages + } + if not layout: + return False + parent = os.path.commonpath([_parent_path(k, v) for k, v in layout.items()]) + return all( + _normalize_path(Path(parent, *key.split('.'))) == _normalize_path(value) + for key, value in layout.items() + ) + + +def _parent_path(pkg, pkg_path): + """Infer the parent path containing a package, that if added to ``sys.path`` would + allow importing that package. + When ``pkg`` is directly mapped into a directory with a different name, return its + own path. + >>> _parent_path("a", "src/a") + 'src' + >>> _parent_path("b", "src/c") + 'src/c' + """ + parent = pkg_path[:-len(pkg)] if pkg_path.endswith(pkg) else pkg_path + return parent.rstrip("/" + os.sep) + + +def _find_packages(dist: Distribution) -> Iterator[str]: + yield from iter(dist.packages or []) + + py_modules = dist.py_modules or [] + nested_modules = [mod for mod in py_modules if "." in mod] + if dist.ext_package: + yield dist.ext_package + else: + ext_modules = dist.ext_modules or [] + nested_modules += [x.name for x in ext_modules if "." in x.name] + + for module in nested_modules: + package, _, _ = module.rpartition(".") + yield package + + +def _find_top_level_modules(dist: Distribution) -> Iterator[str]: + py_modules = dist.py_modules or [] + yield from (mod for mod in py_modules if "." not in mod) + + if not dist.ext_package: + ext_modules = dist.ext_modules or [] + yield from (x.name for x in ext_modules if "." not in x.name) + + +def _find_package_roots( + packages: Iterable[str], + package_dir: Mapping[str, str], + src_root: _Path, +) -> Dict[str, str]: + pkg_roots: Dict[str, str] = { + pkg: _absolute_root(find_package_path(pkg, package_dir, src_root)) + for pkg in sorted(packages) + } + + return _remove_nested(pkg_roots) + + +def _absolute_root(path: _Path) -> str: + """Works for packages and top-level modules""" + path_ = Path(path) + parent = path_.parent + + if path_.exists(): + return str(path_.resolve()) + else: + return str(parent.resolve() / path_.name) + + +def _find_virtual_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]: + """By carefully designing ``package_dir``, it is possible to implement the logical + structure of PEP 420 in a package without the corresponding directories. + This function will try to find this kind of namespaces. + """ + for pkg in pkg_roots: + if "." not in pkg: + continue + parts = pkg.split(".") + for i in range(len(parts) - 1, 0, -1): + partial_name = ".".join(parts[:i]) + path = Path(find_package_path(partial_name, pkg_roots, "")) + if not path.exists(): + yield partial_name + + +def _find_namespaces( + packages: List[str], pkg_roots: Dict[str, str] +) -> Iterator[Tuple[str, List[str]]]: + for pkg in packages: + path = find_package_path(pkg, pkg_roots, "") + if Path(path).exists() and not Path(path, "__init__.py").exists(): + yield (pkg, [path]) + + +def _remove_nested(pkg_roots: Dict[str, str]) -> Dict[str, str]: + output = dict(pkg_roots.copy()) + + for pkg, path in reversed(list(pkg_roots.items())): + if any( + pkg != other and _is_nested(pkg, path, other, other_path) + for other, other_path in pkg_roots.items() + ): + output.pop(pkg) + + return output + + +def _is_nested(pkg: str, pkg_path: str, parent: str, parent_path: str) -> bool: + """ + Return ``True`` if ``pkg`` is nested inside ``parent`` both logically and in the + file system. + >>> _is_nested("a.b", "path/a/b", "a", "path/a") + True + >>> _is_nested("a.b", "path/a/b", "a", "otherpath/a") + False + >>> _is_nested("a.b", "path/a/b", "c", "path/c") + False + """ + norm_pkg_path = _normalize_path(pkg_path) + rest = pkg.replace(parent, "").strip(".").split(".") + return ( + pkg.startswith(parent) + and norm_pkg_path == _normalize_path(Path(parent_path, *rest)) + ) + + +def _normalize_path(filename: _Path) -> str: + """Normalize a file/dir name for comparison purposes""" + # See pkg_resources.normalize_path + file = os.path.abspath(filename) if sys.platform == 'cygwin' else filename + return os.path.normcase(os.path.realpath(os.path.normpath(file))) + + +def _empty_dir(dir_: _P) -> _P: + """Create a directory ensured to be empty. Existing files may be removed.""" + shutil.rmtree(dir_, ignore_errors=True) + os.makedirs(dir_) + return dir_ + + +def _make_identifier(name: str) -> str: + """Make a string safe to be used as Python identifier. + >>> _make_identifier("12abc") + '_12abc' + >>> _make_identifier("__editable__.myns.pkg-78.9.3_local") + '__editable___myns_pkg_78_9_3_local' + """ + safe = re.sub(r'\W|^(?=\d)', '_', name) + assert safe.isidentifier() + return safe + + +class _NamespaceInstaller(namespaces.Installer): + def __init__(self, distribution, installation_dir, editable_name, src_root): + self.distribution = distribution + self.src_root = src_root + self.installation_dir = installation_dir + self.editable_name = editable_name + self.outputs = [] + + def _get_target(self): + """Installation target.""" + return os.path.join(self.installation_dir, self.editable_name) + + def _get_root(self): + """Where the modules/packages should be loaded from.""" + return repr(str(self.src_root)) + + +_FINDER_TEMPLATE = """\ +import sys +from importlib.machinery import ModuleSpec +from importlib.machinery import all_suffixes as module_suffixes +from importlib.util import spec_from_file_location +from itertools import chain +from pathlib import Path + +MAPPING = {mapping!r} +NAMESPACES = {namespaces!r} +PATH_PLACEHOLDER = {name!r} + ".__path_hook__" + + +class _EditableFinder: # MetaPathFinder + @classmethod + def find_spec(cls, fullname, path=None, target=None): + for pkg, pkg_path in reversed(list(MAPPING.items())): + if fullname.startswith(pkg): + rest = fullname.replace(pkg, "").strip(".").split(".") + return cls._find_spec(fullname, Path(pkg_path, *rest)) + + return None + + @classmethod + def _find_spec(cls, fullname, candidate_path): + init = candidate_path / "__init__.py" + candidates = (candidate_path.with_suffix(x) for x in module_suffixes()) + for candidate in chain([init], candidates): + if candidate.exists(): + return spec_from_file_location(fullname, candidate) + + +class _EditableNamespaceFinder: # PathEntryFinder + @classmethod + def _path_hook(cls, path): + if path == PATH_PLACEHOLDER: + return cls + raise ImportError + + @classmethod + def _paths(cls, fullname): + # Ensure __path__ is not empty for the spec to be considered a namespace. + return NAMESPACES[fullname] or MAPPING.get(fullname) or [PATH_PLACEHOLDER] + + @classmethod + def find_spec(cls, fullname, target=None): + if fullname in NAMESPACES: + spec = ModuleSpec(fullname, None, is_package=True) + spec.submodule_search_locations = cls._paths(fullname) + return spec + return None + + @classmethod + def find_module(cls, fullname): + return None + + +def install(): + if not any(finder == _EditableFinder for finder in sys.meta_path): + sys.meta_path.append(_EditableFinder) + + if not NAMESPACES: + return + + if not any(hook == _EditableNamespaceFinder._path_hook for hook in sys.path_hooks): + # PathEntryFinder is needed to create NamespaceSpec without private APIS + sys.path_hooks.append(_EditableNamespaceFinder._path_hook) + if PATH_PLACEHOLDER not in sys.path: + sys.path.append(PATH_PLACEHOLDER) # Used just to trigger the path hook +""" + + +def _finder_template( + name: str, mapping: Mapping[str, str], namespaces: Dict[str, List[str]] +) -> str: + """Create a string containing the code for the``MetaPathFinder`` and + ``PathEntryFinder``. + """ + mapping = dict(sorted(mapping.items(), key=lambda p: p[0])) + return _FINDER_TEMPLATE.format(name=name, mapping=mapping, namespaces=namespaces) + + +class InformationOnly(UserWarning): + """Currently there is no clear way of displaying messages to the users + that use the setuptools backend directly via ``pip``. + The only thing that might work is a warning, although it is not the + most appropriate tool for the job... + """ + + +class LinksNotSupported(errors.FileError): + """File system does not seem to support either symlinks or hard links.""" diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 42a0178..0c9d45a 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -182,6 +182,7 @@ class egg_info(InfoCommon, Command): self.egg_info = None self.egg_version = None self.broken_egg_info = False + self.ignore_egg_info_in_manifest = False #################################### # allow the 'tag_svn_revision' to be detected and @@ -296,6 +297,7 @@ class egg_info(InfoCommon, Command): self.mkpath(self.egg_info) os.utime(self.egg_info, None) for ep in metadata.entry_points(group='egg_info.writers'): + self.distribution._install_dependencies(ep) writer = ep.load() writer(self, ep.name, os.path.join(self.egg_info, ep.name)) @@ -310,6 +312,7 @@ class egg_info(InfoCommon, Command): """Generate SOURCES.txt manifest file""" manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") mm = manifest_maker(self.distribution) + mm.ignore_egg_info_dir = self.ignore_egg_info_in_manifest mm.manifest = manifest_filename mm.run() self.filelist = mm.filelist @@ -333,6 +336,10 @@ class egg_info(InfoCommon, Command): class FileList(_FileList): # Implementations of the various MANIFEST.in commands + def __init__(self, warn=None, debug_print=None, ignore_egg_info_dir=False): + super().__init__(warn, debug_print) + self.ignore_egg_info_dir = ignore_egg_info_dir + def process_template_line(self, line): # Parse the line: split it up, make sure the right number of words # is there, and return the relevant words. 'action' is always @@ -522,6 +529,10 @@ class FileList(_FileList): return False try: + # ignore egg-info paths + is_egg_info = ".egg-info" in u_path or b".egg-info" in utf8_path + if self.ignore_egg_info_dir and is_egg_info: + return False # accept is either way checks out if os.path.exists(u_path) or os.path.exists(utf8_path): return True @@ -538,12 +549,13 @@ class manifest_maker(sdist): self.prune = 1 self.manifest_only = 1 self.force_manifest = 1 + self.ignore_egg_info_dir = False def finalize_options(self): pass def run(self): - self.filelist = FileList() + self.filelist = FileList(ignore_egg_info_dir=self.ignore_egg_info_dir) if not os.path.exists(self.manifest): self.write_manifest() # it must exist so it'll get in the list self.add_defaults() diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 0ffeacf..4a8cde7 100644 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -4,10 +4,12 @@ import os import sys import io import contextlib +from itertools import chain from .py36compat import sdist_add_defaults from .._importlib import metadata +from .build import _ORIGINAL_SUBCOMMANDS _default_revctrl = list @@ -100,6 +102,10 @@ class sdist(sdist_add_defaults, orig.sdist): if orig_val is not NoValue: setattr(os, 'link', orig_val) + def add_defaults(self): + super().add_defaults() + self._add_defaults_build_sub_commands() + def _add_defaults_optional(self): super()._add_defaults_optional() if os.path.isfile('pyproject.toml'): @@ -112,6 +118,14 @@ class sdist(sdist_add_defaults, orig.sdist): self.filelist.extend(build_py.get_source_files()) self._add_data_files(self._safe_data_files(build_py)) + def _add_defaults_build_sub_commands(self): + build = self.get_finalized_command("build") + missing_cmds = set(build.get_sub_commands()) - _ORIGINAL_SUBCOMMANDS + # ^-- the original built-in sub-commands are already handled by default. + cmds = (self.get_finalized_command(c) for c in missing_cmds) + files = (c.get_source_files() for c in cmds if hasattr(c, "get_source_files")) + self.filelist.extend(chain.from_iterable(files)) + def _safe_data_files(self, build_py): """ Since the ``sdist`` class is also used to compute the MANIFEST diff --git a/setuptools/config/expand.py b/setuptools/config/expand.py index ed75640..384504d 100644 --- a/setuptools/config/expand.py +++ b/setuptools/config/expand.py @@ -45,6 +45,8 @@ from types import ModuleType from distutils.errors import DistutilsOptionError +from .._path import same_path as _same_path + if TYPE_CHECKING: from setuptools.dist import Distribution # noqa from setuptools.discovery import ConfigDiscovery # noqa @@ -328,25 +330,6 @@ def find_packages( return packages -def _same_path(p1: _Path, p2: _Path) -> bool: - """Differs from os.path.samefile because it does not require paths to exist. - Purely string based (no comparison between i-nodes). - >>> _same_path("a/b", "./a/b") - True - >>> _same_path("a/b", "a/./b") - True - >>> _same_path("a/b", "././a/b") - True - >>> _same_path("a/b", "./a/b/c/..") - True - >>> _same_path("a/b", "../a/b/c") - False - >>> _same_path("a", "a/b") - False - """ - return os.path.normpath(p1) == os.path.normpath(p2) - - def _nest_path(parent: _Path, path: _Path) -> str: path = parent if path in {".", ""} else os.path.join(parent, path) return os.path.normpath(path) diff --git a/setuptools/discovery.py b/setuptools/discovery.py index 95c3c7f..6a3d2c9 100644 --- a/setuptools/discovery.py +++ b/setuptools/discovery.py @@ -42,8 +42,18 @@ import os from fnmatch import fnmatchcase from glob import glob from pathlib import Path -from typing import TYPE_CHECKING -from typing import Callable, Dict, Iterator, Iterable, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Callable, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Tuple, + Union +) import _distutils_hack.override # noqa: F401 @@ -435,6 +445,7 @@ class ConfigDiscovery: def _ensure_no_accidental_inclusion(self, detected: List[str], kind: str): if len(detected) > 1: from inspect import cleandoc + from setuptools.errors import PackageDiscoveryError msg = f"""Multiple top-level {kind} discovered in a flat-layout: {detected}. @@ -527,7 +538,7 @@ def remove_stubs(packages: List[str]) -> List[str]: def find_parent_package( - packages: List[str], package_dir: Dict[str, str], root_dir: _Path + packages: List[str], package_dir: Mapping[str, str], root_dir: _Path ) -> Optional[str]: """Find the parent package that is not a namespace.""" packages = sorted(packages, key=len) @@ -550,7 +561,9 @@ def find_parent_package( return None -def find_package_path(name: str, package_dir: Dict[str, str], root_dir: _Path) -> str: +def find_package_path( + name: str, package_dir: Mapping[str, str], root_dir: _Path +) -> str: """Given a package name, return the path where it should be found on disk, considering the ``package_dir`` option. diff --git a/setuptools/dist.py b/setuptools/dist.py index 8242354..c1ad300 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -30,6 +30,7 @@ from distutils.util import rfc822_escape from setuptools.extern import packaging from setuptools.extern import ordered_set from setuptools.extern.more_itertools import unique_everseen, partition +from setuptools.extern import nspektr from ._importlib import metadata @@ -917,8 +918,18 @@ class Distribution(_Distribution): for ep in metadata.entry_points(group='distutils.setup_keywords'): value = getattr(self, ep.name, None) if value is not None: + self._install_dependencies(ep) ep.load()(self, ep.name, value) + def _install_dependencies(self, ep): + """ + Given an entry point, ensure that any declared extras for + its distribution are installed. + """ + for req in nspektr.missing(ep): + # fetch_build_egg expects pkg_resources.Requirement + self.fetch_build_egg(pkg_resources.Requirement(str(req))) + def get_egg_cache_dir(self): egg_cache_dir = os.path.join(os.curdir, '.eggs') if not os.path.exists(egg_cache_dir): @@ -951,6 +962,7 @@ class Distribution(_Distribution): eps = metadata.entry_points(group='distutils.commands', name=command) for ep in eps: + self._install_dependencies(ep) self.cmdclass[command] = cmdclass = ep.load() return cmdclass else: diff --git a/setuptools/extension.py b/setuptools/extension.py index 64baf11..b9a2bad 100644 --- a/setuptools/extension.py +++ b/setuptools/extension.py @@ -113,6 +113,9 @@ class Extension(_Extension): :keyword bool optional: specifies that a build failure in the extension should not abort the build process, but simply not install the failing extension. + + :keyword bool py_limited_api: + opt-in flag for the usage of :doc:`Python's limited API `. """ def __init__(self, name, sources, *args, **kw): diff --git a/setuptools/extern/__init__.py b/setuptools/extern/__init__.py index d3a6dc9..192e55f 100644 --- a/setuptools/extern/__init__.py +++ b/setuptools/extern/__init__.py @@ -71,6 +71,6 @@ class VendorImporter: names = ( 'packaging', 'pyparsing', 'ordered_set', 'more_itertools', 'importlib_metadata', - 'zipp', 'importlib_resources', 'jaraco', 'typing_extensions', 'tomli', + 'zipp', 'importlib_resources', 'jaraco', 'typing_extensions', 'nspektr', 'tomli', ) VendorImporter(__name__, names, 'setuptools._vendor').install() diff --git a/setuptools/tests/contexts.py b/setuptools/tests/contexts.py index 5894882..7ddbc78 100644 --- a/setuptools/tests/contexts.py +++ b/setuptools/tests/contexts.py @@ -123,3 +123,26 @@ def session_locked_tmp_dir(request, tmp_path_factory, name): # ^-- prevent multiple workers to access the directory at once locked_dir.mkdir(exist_ok=True, parents=True) yield locked_dir + + +@contextlib.contextmanager +def save_paths(): + """Make sure ``sys.path``, ``sys.meta_path`` and ``sys.path_hooks`` are preserved""" + prev = sys.path[:], sys.meta_path[:], sys.path_hooks[:] + + try: + yield + finally: + sys.path, sys.meta_path, sys.path_hooks = prev + + +@contextlib.contextmanager +def save_sys_modules(): + """Make sure initial ``sys.modules`` is preserved""" + prev_modules = sys.modules + + try: + sys.modules = sys.modules.copy() + yield + finally: + sys.modules = prev_modules diff --git a/setuptools/tests/namespaces.py b/setuptools/tests/namespaces.py index 245cf8e..34e916f 100644 --- a/setuptools/tests/namespaces.py +++ b/setuptools/tests/namespaces.py @@ -28,6 +28,29 @@ def build_namespace_package(tmpdir, name): return src_dir +def build_pep420_namespace_package(tmpdir, name): + src_dir = tmpdir / name + src_dir.mkdir() + pyproject = src_dir / "pyproject.toml" + namespace, sep, rest = name.rpartition(".") + script = f"""\ + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + + [project] + name = "{name}" + version = "3.14159" + """ + pyproject.write_text(textwrap.dedent(script), encoding='utf-8') + ns_pkg_dir = src_dir / namespace.replace(".", "/") + ns_pkg_dir.mkdir(parents=True) + pkg_mod = ns_pkg_dir / (rest + ".py") + some_functionality = f"name = {rest!r}" + pkg_mod.write_text(some_functionality, encoding='utf-8') + return src_dir + + def make_site_dir(target): """ Add a sitecustomize.py module in target to cause diff --git a/setuptools/tests/test_build_ext.py b/setuptools/tests/test_build_ext.py index 3177a2c..07ebcaf 100644 --- a/setuptools/tests/test_build_ext.py +++ b/setuptools/tests/test_build_ext.py @@ -2,6 +2,7 @@ import os import sys import distutils.command.build_ext as orig from distutils.sysconfig import get_config_var +from importlib.util import cache_from_source as _compiled_file_name from jaraco import path @@ -83,6 +84,97 @@ class TestBuildExt: finally: del os.environ['SETUPTOOLS_EXT_SUFFIX'] + def dist_with_example(self): + files = { + "src": {"mypkg": {"subpkg": {"ext2.c": ""}}}, + "c-extensions": {"ext1": {"main.c": ""}}, + } + + ext1 = Extension("mypkg.ext1", ["c-extensions/ext1/main.c"]) + ext2 = Extension("mypkg.subpkg.ext2", ["src/mypkg/subpkg/ext2.c"]) + ext3 = Extension("ext3", ["c-extension/ext3.c"]) + + path.build(files) + dist = Distribution({ + "script_name": "%test%", + "ext_modules": [ext1, ext2, ext3], + "package_dir": {"": "src"}, + }) + return dist + + def test_get_outputs(self, tmpdir_cwd, monkeypatch): + monkeypatch.setenv('SETUPTOOLS_EXT_SUFFIX', '.mp3') # make test OS-independent + monkeypatch.setattr('setuptools.command.build_ext.use_stubs', False) + dist = self.dist_with_example() + + # Regular build: get_outputs not empty, but get_output_mappings is empty + build_ext = dist.get_command_obj("build_ext") + build_ext.editable_mode = False + build_ext.ensure_finalized() + build_lib = build_ext.build_lib.replace(os.sep, "/") + outputs = [x.replace(os.sep, "/") for x in build_ext.get_outputs()] + assert outputs == [ + f"{build_lib}/ext3.mp3", + f"{build_lib}/mypkg/ext1.mp3", + f"{build_lib}/mypkg/subpkg/ext2.mp3", + ] + assert build_ext.get_output_mapping() == {} + + # Editable build: get_output_mappings should contain everything in get_outputs + dist.reinitialize_command("build_ext") + build_ext.editable_mode = True + build_ext.ensure_finalized() + mapping = { + k.replace(os.sep, "/"): v.replace(os.sep, "/") + for k, v in build_ext.get_output_mapping().items() + } + assert mapping == { + f"{build_lib}/ext3.mp3": "src/ext3.mp3", + f"{build_lib}/mypkg/ext1.mp3": "src/mypkg/ext1.mp3", + f"{build_lib}/mypkg/subpkg/ext2.mp3": "src/mypkg/subpkg/ext2.mp3", + } + + def test_get_output_mapping_with_stub(self, tmpdir_cwd, monkeypatch): + monkeypatch.setenv('SETUPTOOLS_EXT_SUFFIX', '.mp3') # make test OS-independent + monkeypatch.setattr('setuptools.command.build_ext.use_stubs', True) + dist = self.dist_with_example() + + # Editable build should create compiled stubs (.pyc files only, no .py) + build_ext = dist.get_command_obj("build_ext") + build_ext.editable_mode = True + build_ext.ensure_finalized() + for ext in build_ext.extensions: + monkeypatch.setattr(ext, "_needs_stub", True) + + build_lib = build_ext.build_lib.replace(os.sep, "/") + mapping = { + k.replace(os.sep, "/"): v.replace(os.sep, "/") + for k, v in build_ext.get_output_mapping().items() + } + + def C(file): + """Make it possible to do comparisons and tests in a OS-independent way""" + return _compiled_file_name(file).replace(os.sep, "/") + + assert mapping == { + C(f"{build_lib}/ext3.py"): C("src/ext3.py"), + f"{build_lib}/ext3.mp3": "src/ext3.mp3", + C(f"{build_lib}/mypkg/ext1.py"): C("src/mypkg/ext1.py"), + f"{build_lib}/mypkg/ext1.mp3": "src/mypkg/ext1.mp3", + C(f"{build_lib}/mypkg/subpkg/ext2.py"): C("src/mypkg/subpkg/ext2.py"), + f"{build_lib}/mypkg/subpkg/ext2.mp3": "src/mypkg/subpkg/ext2.mp3", + } + + # Ensure only the compiled stubs are present not the raw .py stub + assert f"{build_lib}/mypkg/ext1.py" not in mapping + assert f"{build_lib}/mypkg/subpkg/ext2.py" not in mapping + + # Visualize what the cached stub files look like + example_stub = C(f"{build_lib}/mypkg/ext1.py") + assert example_stub in mapping + assert example_stub.startswith(f"{build_lib}/mypkg/__pycache__/ext1") + assert example_stub.endswith(".pyc") + def test_build_ext_config_handling(tmpdir_cwd): files = { diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index 36940e7..026c8ae 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -5,9 +5,11 @@ import signal import tarfile import importlib import contextlib +import subprocess from concurrent import futures import re from zipfile import ZipFile +from pathlib import Path import pytest from jaraco import path @@ -611,6 +613,71 @@ class TestBuildMetaBackend: with pytest.raises(ImportError, match="^No module named 'hello'$"): build_backend.build_sdist("temp") + _simple_pyproject_example = { + "pyproject.toml": DALS(""" + [project] + name = "proj" + version = "42" + """), + "src": { + "proj": {"__init__.py": ""} + } + } + + def _assert_link_tree(self, parent_dir): + """All files in the directory should be either links or hard links""" + files = list(Path(parent_dir).glob("**/*")) + assert files # Should not be empty + for file in files: + assert file.is_symlink() or os.stat(file).st_nlink > 0 + + @pytest.mark.filterwarnings("ignore::setuptools.SetuptoolsDeprecationWarning") + # Since the backend is running via a process pool, in some operating systems + # we may have problems to make assertions based on warnings/stdout/stderr... + # So the best is to ignore them for the time being. + def test_editable_with_global_option_still_works(self, tmpdir_cwd): + """The usage of --global-option is now discouraged in favour of --build-option. + This is required to make more sense of the provided scape hatch and align with + previous pip behaviour. See pypa/setuptools#1928. + """ + path.build({**self._simple_pyproject_example, '_meta': {}}) + build_backend = self.get_build_backend() + assert not Path("build").exists() + + cfg = {"--global-option": "--strict"} + build_backend.prepare_metadata_for_build_editable("_meta", cfg) + build_backend.build_editable("temp", cfg, "_meta") + + self._assert_link_tree(next(Path("build").glob("__editable__.*"))) + + def test_editable_without_config_settings(self, tmpdir_cwd): + """ + Sanity check to ensure tests with --strict are different from the ones + without --strict. + + --strict should create a local directory with a package tree. + The directory should not get created otherwise. + """ + path.build(self._simple_pyproject_example) + build_backend = self.get_build_backend() + assert not Path("build").exists() + build_backend.build_editable("temp") + assert not Path("build").exists() + + @pytest.mark.parametrize( + "config_settings", [ + {"--build-option": "--strict"}, + {"editable-mode": "strict"}, + ] + ) + def test_editable_with_config_settings(self, tmpdir_cwd, config_settings): + path.build({**self._simple_pyproject_example, '_meta': {}}) + assert not Path("build").exists() + build_backend = self.get_build_backend() + build_backend.prepare_metadata_for_build_editable("_meta", config_settings) + build_backend.build_editable("temp", config_settings, "_meta") + self._assert_link_tree(next(Path("build").glob("__editable__.*"))) + @pytest.mark.parametrize('setup_literal, requirements', [ ("'foo'", ['foo']), ("['foo']", ['foo']), @@ -764,3 +831,27 @@ class TestBuildMetaLegacyBackend(TestBuildMetaBackend): build_backend = self.get_build_backend() build_backend.build_sdist("temp") + + +def test_legacy_editable_install(tmpdir, tmpdir_cwd): + pyproject = """ + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + [project] + name = "myproj" + version = "42" + """ + path.build({"pyproject.toml": DALS(pyproject), "mymod.py": ""}) + + # First: sanity check + cmd = [sys.executable, "-m", "pip", "install", "--no-build-isolation", "-e", "."] + output = str(subprocess.check_output(cmd, cwd=tmpdir), "utf-8").lower() + assert "running setup.py develop for myproj" not in output + assert "created wheel for myproj" in output + + # Then: real test + env = {**os.environ, "SETUPTOOLS_ENABLE_FEATURES": "legacy-editable"} + cmd = [sys.executable, "-m", "pip", "install", "--no-build-isolation", "-e", "."] + output = str(subprocess.check_output(cmd, cwd=tmpdir, env=env), "utf-8").lower() + assert "running setup.py develop for myproj" in output diff --git a/setuptools/tests/test_build_py.py b/setuptools/tests/test_build_py.py index 13fa64d..2b32edb 100644 --- a/setuptools/tests/test_build_py.py +++ b/setuptools/tests/test_build_py.py @@ -1,10 +1,11 @@ import os import stat import shutil +from pathlib import Path +from unittest.mock import Mock import pytest import jaraco.path -from path import Path from setuptools import SetuptoolsDeprecationWarning from setuptools.dist import Distribution @@ -109,67 +110,194 @@ def test_executable_data(tmpdir_cwd): "Script is not executable" -def test_excluded_subpackages(tmp_path): - files = { - "setup.cfg": DALS(""" - [metadata] - name = mypkg - version = 42 +EXAMPLE_WITH_MANIFEST = { + "setup.cfg": DALS(""" + [metadata] + name = mypkg + version = 42 - [options] - include_package_data = True - packages = find: + [options] + include_package_data = True + packages = find: - [options.packages.find] - exclude = *.tests* - """), + [options.packages.find] + exclude = *.tests* + """), + "mypkg": { + "__init__.py": "", + "resource_file.txt": "", + "tests": { + "__init__.py": "", + "test_mypkg.py": "", + "test_file.txt": "", + } + }, + "MANIFEST.in": DALS(""" + global-include *.py *.txt + global-exclude *.py[cod] + prune dist + prune build + prune *.egg-info + """) +} + + +def test_excluded_subpackages(tmpdir_cwd): + jaraco.path.build(EXAMPLE_WITH_MANIFEST) + dist = Distribution({"script_name": "%PEP 517%"}) + dist.parse_config_files() + + build_py = dist.get_command_obj("build_py") + msg = r"Python recognizes 'mypkg\.tests' as an importable package" + with pytest.warns(SetuptoolsDeprecationWarning, match=msg): + # TODO: To fix #3260 we need some transition period to deprecate the + # existing behavior of `include_package_data`. After the transition, we + # should remove the warning and fix the behaviour. + build_py.finalize_options() + build_py.run() + + build_dir = Path(dist.get_command_obj("build_py").build_lib) + assert (build_dir / "mypkg/__init__.py").exists() + assert (build_dir / "mypkg/resource_file.txt").exists() + + # Setuptools is configured to ignore `mypkg.tests`, therefore the following + # files/dirs should not be included in the distribution. + for f in [ + "mypkg/tests/__init__.py", + "mypkg/tests/test_mypkg.py", + "mypkg/tests/test_file.txt", + "mypkg/tests", + ]: + with pytest.raises(AssertionError): + # TODO: Enforce the following assertion once #3260 is fixed + # (remove context manager and the following xfail). + assert not (build_dir / f).exists() + + pytest.xfail("#3260") + + +@pytest.mark.filterwarnings("ignore::setuptools.SetuptoolsDeprecationWarning") +def test_existing_egg_info(tmpdir_cwd, monkeypatch): + """When provided with the ``existing_egg_info_dir`` attribute, build_py should not + attempt to run egg_info again. + """ + # == Pre-condition == + # Generate an egg-info dir + jaraco.path.build(EXAMPLE_WITH_MANIFEST) + dist = Distribution({"script_name": "%PEP 517%"}) + dist.parse_config_files() + assert dist.include_package_data + + egg_info = dist.get_command_obj("egg_info") + dist.run_command("egg_info") + egg_info_dir = next(Path(egg_info.egg_base).glob("*.egg-info")) + assert egg_info_dir.is_dir() + + # == Setup == + build_py = dist.get_command_obj("build_py") + build_py.finalize_options() + egg_info = dist.get_command_obj("egg_info") + egg_info_run = Mock(side_effect=egg_info.run) + monkeypatch.setattr(egg_info, "run", egg_info_run) + + # == Remove caches == + # egg_info is called when build_py looks for data_files, which gets cached. + # We need to ensure it is not cached yet, otherwise it may impact on the tests + build_py.__dict__.pop('data_files', None) + dist.reinitialize_command(egg_info) + + # == Sanity check == + # Ensure that if existing_egg_info is not given, build_py attempts to run egg_info + build_py.existing_egg_info_dir = None + build_py.run() + egg_info_run.assert_called() + + # == Remove caches == + egg_info_run.reset_mock() + build_py.__dict__.pop('data_files', None) + dist.reinitialize_command(egg_info) + + # == Actual test == + # Ensure that if existing_egg_info_dir is given, egg_info doesn't run + build_py.existing_egg_info_dir = egg_info_dir + build_py.run() + egg_info_run.assert_not_called() + assert build_py.data_files + + # Make sure the list of outputs is actually OK + outputs = map(lambda x: x.replace(os.sep, "/"), build_py.get_outputs()) + assert outputs + example = str(Path(build_py.build_lib, "mypkg/__init__.py")).replace(os.sep, "/") + assert example in outputs + + +EXAMPLE_ARBITRARY_MAPPING = { + "pyproject.toml": DALS(""" + [project] + name = "mypkg" + version = "42" + + [tool.setuptools] + packages = ["mypkg", "mypkg.sub1", "mypkg.sub2", "mypkg.sub2.nested"] + + [tool.setuptools.package-dir] + "" = "src" + "mypkg.sub2" = "src/mypkg/_sub2" + "mypkg.sub2.nested" = "other" + """), + "src": { "mypkg": { "__init__.py": "", "resource_file.txt": "", - "tests": { + "sub1": { "__init__.py": "", - "test_mypkg.py": "", - "test_file.txt": "", - } + "mod1.py": "", + }, + "_sub2": { + "mod2.py": "", + }, }, - "MANIFEST.in": DALS(""" - global-include *.py *.txt - global-exclude *.py[cod] - prune dist - prune build - prune *.egg-info - """) - } + }, + "other": { + "__init__.py": "", + "mod3.py": "", + }, + "MANIFEST.in": DALS(""" + global-include *.py *.txt + global-exclude *.py[cod] + """) +} + - with Path(tmp_path): - jaraco.path.build(files) - dist = Distribution({"script_name": "%PEP 517%"}) - dist.parse_config_files() - - build_py = dist.get_command_obj("build_py") - msg = r"Python recognizes 'mypkg\.tests' as an importable package" - with pytest.warns(SetuptoolsDeprecationWarning, match=msg): - # TODO: To fix #3260 we need some transition period to deprecate the - # existing behavior of `include_package_data`. After the transition, we - # should remove the warning and fix the behaviour. - build_py.finalize_options() - build_py.run() - - build_dir = Path(dist.get_command_obj("build_py").build_lib) - assert (build_dir / "mypkg/__init__.py").exists() - assert (build_dir / "mypkg/resource_file.txt").exists() - - # Setuptools is configured to ignore `mypkg.tests`, therefore the following - # files/dirs should not be included in the distribution. - for f in [ - "mypkg/tests/__init__.py", - "mypkg/tests/test_mypkg.py", - "mypkg/tests/test_file.txt", - "mypkg/tests", - ]: - with pytest.raises(AssertionError): - # TODO: Enforce the following assertion once #3260 is fixed - # (remove context manager and the following xfail). - assert not (build_dir / f).exists() - - pytest.xfail("#3260") +def test_get_outputs(tmpdir_cwd): + jaraco.path.build(EXAMPLE_ARBITRARY_MAPPING) + dist = Distribution({"script_name": "%test%"}) + dist.parse_config_files() + + build_py = dist.get_command_obj("build_py") + build_py.editable_mode = True + build_py.ensure_finalized() + build_lib = build_py.build_lib.replace(os.sep, "/") + outputs = [x.replace(os.sep, "/") for x in build_py.get_outputs()] + assert outputs == [ + f"{build_lib}/mypkg/__init__.py", + f"{build_lib}/mypkg/resource_file.txt", + f"{build_lib}/mypkg/sub1/__init__.py", + f"{build_lib}/mypkg/sub1/mod1.py", + f"{build_lib}/mypkg/sub2/mod2.py", + f"{build_lib}/mypkg/sub2/nested/__init__.py", + f"{build_lib}/mypkg/sub2/nested/mod3.py", + ] + mapping = { + k.replace(os.sep, "/"): v.replace(os.sep, "/") + for k, v in build_py.get_output_mapping().items() + } + assert mapping == { + f"{build_lib}/mypkg/__init__.py": "src/mypkg/__init__.py", + f"{build_lib}/mypkg/resource_file.txt": "src/mypkg/resource_file.txt", + f"{build_lib}/mypkg/sub1/__init__.py": "src/mypkg/sub1/__init__.py", + f"{build_lib}/mypkg/sub1/mod1.py": "src/mypkg/sub1/mod1.py", + f"{build_lib}/mypkg/sub2/mod2.py": "src/mypkg/_sub2/mod2.py", + f"{build_lib}/mypkg/sub2/nested/__init__.py": "other/__init__.py", + f"{build_lib}/mypkg/sub2/nested/mod3.py": "other/mod3.py", + } diff --git a/setuptools/tests/test_develop.py b/setuptools/tests/test_develop.py index c52072a..0dd6034 100644 --- a/setuptools/tests/test_develop.py +++ b/setuptools/tests/test_develop.py @@ -5,12 +5,10 @@ import os import sys import subprocess import platform -import pathlib from setuptools.command import test import pytest -import pip_run.launch from setuptools.command.develop import develop from setuptools.dist import Distribution @@ -165,45 +163,3 @@ class TestNamespaces: ] with test.test.paths_on_pythonpath([str(target)]): subprocess.check_call(pkg_resources_imp) - - @pytest.mark.xfail( - platform.python_implementation() == 'PyPy', - reason="Workaround fails on PyPy (why?)", - ) - def test_editable_prefix(self, tmp_path, sample_project): - """ - Editable install to a prefix should be discoverable. - """ - prefix = tmp_path / 'prefix' - - # figure out where pip will likely install the package - site_packages = prefix / next( - pathlib.Path(path).relative_to(sys.prefix) - for path in sys.path - if 'site-packages' in path and path.startswith(sys.prefix) - ) - site_packages.mkdir(parents=True) - - # install workaround - pip_run.launch.inject_sitecustomize(str(site_packages)) - - env = dict(os.environ, PYTHONPATH=str(site_packages)) - cmd = [ - sys.executable, - '-m', - 'pip', - 'install', - '--editable', - str(sample_project), - '--prefix', - str(prefix), - '--no-build-isolation', - ] - subprocess.check_call(cmd, env=env) - - # now run 'sample' with the prefix on the PYTHONPATH - bin = 'Scripts' if platform.system() == 'Windows' else 'bin' - exe = prefix / bin / 'sample' - if sys.version_info < (3, 8) and platform.system() == 'Windows': - exe = str(exe) - subprocess.check_call([exe], env=env) diff --git a/setuptools/tests/test_dist_info.py b/setuptools/tests/test_dist_info.py index 813ef51..350e642 100644 --- a/setuptools/tests/test_dist_info.py +++ b/setuptools/tests/test_dist_info.py @@ -2,6 +2,7 @@ """ import pathlib import re +import shutil import subprocess import sys from functools import partial @@ -91,6 +92,42 @@ class TestDistInfo: dist_info = next(tmp_path.glob("*.dist-info")) assert dist_info.name.startswith("proj-42") + def test_tag_arguments(self, tmp_path): + config = """ + [metadata] + name=proj + version=42 + [egg_info] + tag_date=1 + tag_build=.post + """ + (tmp_path / "setup.cfg").write_text(config, encoding="utf-8") + + print(run_command("dist_info", "--no-date", cwd=tmp_path)) + dist_info = next(tmp_path.glob("*.dist-info")) + assert dist_info.name.startswith("proj-42") + shutil.rmtree(dist_info) + + print(run_command("dist_info", "--tag-build", ".a", cwd=tmp_path)) + dist_info = next(tmp_path.glob("*.dist-info")) + assert dist_info.name.startswith("proj-42a") + + @pytest.mark.parametrize("keep_egg_info", (False, True)) + def test_output_dir(self, tmp_path, keep_egg_info): + config = "[metadata]\nname=proj\nversion=42\n" + (tmp_path / "setup.cfg").write_text(config, encoding="utf-8") + out = (tmp_path / "__out") + out.mkdir() + opts = ["--keep-egg-info"] if keep_egg_info else [] + run_command("dist_info", "--output-dir", out, *opts, cwd=tmp_path) + assert len(list(out.glob("*.dist-info"))) == 1 + assert len(list(tmp_path.glob("*.dist-info"))) == 0 + expected_egg_info = 1 if keep_egg_info else 0 + assert len(list(out.glob("*.egg-info"))) == expected_egg_info + assert len(list(tmp_path.glob("*.egg-info"))) == 0 + assert len(list(out.glob("*.__bkp__"))) == 0 + assert len(list(tmp_path.glob("*.__bkp__"))) == 0 + class TestWheelCompatibility: """Make sure the .dist-info directory produced with the ``dist_info`` command @@ -154,5 +191,5 @@ class TestWheelCompatibility: def run_command(*cmd, **kwargs): opts = {"stderr": subprocess.STDOUT, "text": True, **kwargs} - cmd = [sys.executable, "-c", "__import__('setuptools').setup()", *cmd] + cmd = [sys.executable, "-c", "__import__('setuptools').setup()", *map(str, cmd)] return subprocess.check_output(cmd, **opts) diff --git a/setuptools/tests/test_easy_install.py b/setuptools/tests/test_easy_install.py index d102e58..246d634 100644 --- a/setuptools/tests/test_easy_install.py +++ b/setuptools/tests/test_easy_install.py @@ -846,11 +846,9 @@ class TestSetupRequires: def test_setup_requires_with_transitive_extra_dependency( self, monkeypatch): - ''' - Use case: installing a package with a build dependency on - an already installed `dep[extra]`, which in turn depends - on `extra_dep` (whose is not already installed). - ''' + # Use case: installing a package with a build dependency on + # an already installed `dep[extra]`, which in turn depends + # on `extra_dep` (whose is not already installed). with contexts.save_pkg_resources_state(): with contexts.tempdir() as temp_dir: # Create source distribution for `extra_dep`. @@ -892,75 +890,6 @@ class TestSetupRequires: monkeypatch.setenv(str('PIP_TIMEOUT'), str('0')) run_setup(test_setup_py, [str('--version')]) - def test_setup_requires_with_distutils_command_dep(self, monkeypatch): - ''' - Use case: ensure build requirements' extras - are properly installed and activated. - ''' - with contexts.save_pkg_resources_state(): - with contexts.tempdir() as temp_dir: - # Create source distribution for `extra_dep`. - make_sdist(os.path.join(temp_dir, 'extra_dep-1.0.tar.gz'), [ - ('setup.py', - DALS(""" - import setuptools - setuptools.setup( - name='extra_dep', - version='1.0', - py_modules=['extra_dep'], - ) - """)), - ('setup.cfg', ''), - ('extra_dep.py', ''), - ]) - # Create source tree for `epdep`. - dep_pkg = os.path.join(temp_dir, 'epdep') - os.mkdir(dep_pkg) - path.build({ - 'setup.py': - DALS(""" - import setuptools - setuptools.setup( - name='dep', version='2.0', - py_modules=['epcmd'], - extras_require={'extra': ['extra_dep']}, - entry_points=''' - [distutils.commands] - epcmd = epcmd:epcmd [extra] - ''', - ) - """), - 'setup.cfg': '', - 'epcmd.py': DALS(""" - from distutils.command.build_py import build_py - - import extra_dep - - class epcmd(build_py): - pass - """), - }, prefix=dep_pkg) - # "Install" dep. - run_setup( - os.path.join(dep_pkg, 'setup.py'), [str('dist_info')]) - working_set.add_entry(dep_pkg) - # Create source tree for test package. - test_pkg = os.path.join(temp_dir, 'test_pkg') - test_setup_py = os.path.join(test_pkg, 'setup.py') - os.mkdir(test_pkg) - with open(test_setup_py, 'w') as fp: - fp.write(DALS( - ''' - from setuptools import installer, setup - setup(setup_requires='dep[extra]') - ''')) - # Check... - monkeypatch.setenv(str('PIP_FIND_LINKS'), str(temp_dir)) - monkeypatch.setenv(str('PIP_NO_INDEX'), str('1')) - monkeypatch.setenv(str('PIP_RETRIES'), str('0')) - monkeypatch.setenv(str('PIP_TIMEOUT'), str('0')) - run_setup(test_setup_py, ['epcmd']) - def make_trivial_sdist(dist_path, distname, version): """ diff --git a/setuptools/tests/test_editable_install.py b/setuptools/tests/test_editable_install.py index aac4f5e..a76ab08 100644 --- a/setuptools/tests/test_editable_install.py +++ b/setuptools/tests/test_editable_install.py @@ -1,27 +1,45 @@ +import os +import stat +import sys import subprocess +import platform +from copy import deepcopy +from importlib import import_module +from pathlib import Path from textwrap import dedent +from unittest.mock import Mock +from uuid import uuid4 -import pytest import jaraco.envs -import path +import jaraco.path +import pip_run.launch +import pytest +from path import Path as _Path +from . import contexts, namespaces -@pytest.fixture -def venv(tmp_path, setuptools_wheel): - env = jaraco.envs.VirtualEnv() - vars(env).update( - root=path.Path(tmp_path), # workaround for error on windows - name=".venv", - create_opts=["--no-setuptools"], - req=str(setuptools_wheel), - ) - return env.create() +from setuptools._importlib import resources as importlib_resources +from setuptools.command.editable_wheel import ( + _LinkTree, + _find_virtual_namespaces, + _find_namespaces, + _find_package_roots, + _finder_template, +) +from setuptools.dist import Distribution + + +@pytest.fixture(params=["strict", "lax"]) +def editable_mode(request, monkeypatch): + if request.param == "strict": + monkeypatch.setenv("SETUPTOOLS_EDITABLE", "strict") + yield EXAMPLE = { 'pyproject.toml': dedent("""\ [build-system] - requires = ["setuptools", "wheel"] + requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] @@ -51,6 +69,8 @@ EXAMPLE = { "MANIFEST.in": dedent("""\ global-include *.py *.txt global-exclude *.py[cod] + prune dist + prune build """).strip(), "README.rst": "This is a ``README``", "LICENSE.txt": "---- placeholder MIT license ----", @@ -85,18 +105,17 @@ EXAMPLE = { SETUP_SCRIPT_STUB = "__import__('setuptools').setup()" -MISSING_SETUP_SCRIPT = pytest.param( - None, - marks=pytest.mark.xfail( - reason="Editable install is currently only supported with `setup.py`" - ) -) -@pytest.mark.parametrize("setup_script", [SETUP_SCRIPT_STUB, MISSING_SETUP_SCRIPT]) -def test_editable_with_pyproject(tmp_path, venv, setup_script): +@pytest.mark.parametrize( + "files", + [ + {**EXAMPLE, "setup.py": SETUP_SCRIPT_STUB}, # type: ignore + EXAMPLE, # No setup.py script + ] +) +def test_editable_with_pyproject(tmp_path, venv, files, editable_mode): project = tmp_path / "mypkg" - files = {**EXAMPLE, "setup.py": setup_script} project.mkdir() jaraco.path.build(files, prefix=project) @@ -111,3 +130,549 @@ def test_editable_with_pyproject(tmp_path, venv, setup_script): (project / "src/mypkg/data.txt").write_text("foobar") (project / "src/mypkg/mod.py").write_text("x = 42") assert subprocess.check_output(cmd).strip() == b"3.14159.post0 foobar 42" + + +def test_editable_with_flat_layout(tmp_path, venv, editable_mode): + files = { + "mypkg": { + "pyproject.toml": dedent("""\ + [build-system] + requires = ["setuptools", "wheel"] + build-backend = "setuptools.build_meta" + + [project] + name = "mypkg" + version = "3.14159" + + [tool.setuptools] + packages = ["pkg"] + py-modules = ["mod"] + """), + "pkg": {"__init__.py": "a = 4"}, + "mod.py": "b = 2", + }, + } + jaraco.path.build(files, prefix=tmp_path) + project = tmp_path / "mypkg" + + cmd = [venv.exe(), "-m", "pip", "install", + "--no-build-isolation", # required to force current version of setuptools + "-e", str(project)] + print(str(subprocess.check_output(cmd), "utf-8")) + cmd = [venv.exe(), "-c", "import pkg, mod; print(pkg.a, mod.b)"] + assert subprocess.check_output(cmd).strip() == b"4 2" + + +class TestLegacyNamespaces: + """Ported from test_develop""" + + def test_namespace_package_importable(self, venv, tmp_path, editable_mode): + """ + Installing two packages sharing the same namespace, one installed + naturally using pip or `--single-version-externally-managed` + and the other installed in editable mode should leave the namespace + intact and both packages reachable by import. + """ + pkg_A = namespaces.build_namespace_package(tmp_path, 'myns.pkgA') + pkg_B = namespaces.build_namespace_package(tmp_path, 'myns.pkgB') + # use pip to install to the target directory + opts = ["--no-build-isolation"] # force current version of setuptools + venv.run(["python", "-m", "pip", "install", str(pkg_A), *opts]) + venv.run(["python", "-m", "pip", "install", "-e", str(pkg_B), *opts]) + venv.run(["python", "-c", "import myns.pkgA; import myns.pkgB"]) + # additionally ensure that pkg_resources import works + venv.run(["python", "-c", "import pkg_resources"]) + + +class TestPep420Namespaces: + def test_namespace_package_importable(self, venv, tmp_path, editable_mode): + """ + Installing two packages sharing the same namespace, one installed + normally using pip and the other installed in editable mode + should allow importing both packages. + """ + pkg_A = namespaces.build_pep420_namespace_package(tmp_path, 'myns.n.pkgA') + pkg_B = namespaces.build_pep420_namespace_package(tmp_path, 'myns.n.pkgB') + # use pip to install to the target directory + opts = ["--no-build-isolation"] # force current version of setuptools + venv.run(["python", "-m", "pip", "install", str(pkg_A), *opts]) + venv.run(["python", "-m", "pip", "install", "-e", str(pkg_B), *opts]) + venv.run(["python", "-c", "import myns.n.pkgA; import myns.n.pkgB"]) + + def test_namespace_created_via_package_dir(self, venv, tmp_path, editable_mode): + """Currently users can create a namespace by tweaking `package_dir`""" + files = { + "pkgA": { + "pyproject.toml": dedent("""\ + [build-system] + requires = ["setuptools", "wheel"] + build-backend = "setuptools.build_meta" + + [project] + name = "pkgA" + version = "3.14159" + + [tool.setuptools] + package-dir = {"myns.n.pkgA" = "src"} + """), + "src": {"__init__.py": "a = 1"}, + }, + } + jaraco.path.build(files, prefix=tmp_path) + pkg_A = tmp_path / "pkgA" + pkg_B = namespaces.build_pep420_namespace_package(tmp_path, 'myns.n.pkgB') + pkg_C = namespaces.build_pep420_namespace_package(tmp_path, 'myns.n.pkgC') + + # use pip to install to the target directory + opts = ["--no-build-isolation"] # force current version of setuptools + venv.run(["python", "-m", "pip", "install", str(pkg_A), *opts]) + venv.run(["python", "-m", "pip", "install", "-e", str(pkg_B), *opts]) + venv.run(["python", "-m", "pip", "install", "-e", str(pkg_C), *opts]) + venv.run(["python", "-c", "from myns.n import pkgA, pkgB, pkgC"]) + + +# Moved here from test_develop: +@pytest.mark.xfail( + platform.python_implementation() == 'PyPy', + reason="Workaround fails on PyPy (why?)", +) +@pytest.mark.parametrize("mode", ("strict", "lax")) +def test_editable_with_prefix(tmp_path, sample_project, mode): + """ + Editable install to a prefix should be discoverable. + """ + prefix = tmp_path / 'prefix' + + # figure out where pip will likely install the package + site_packages = prefix / next( + Path(path).relative_to(sys.prefix) + for path in sys.path + if 'site-packages' in path and path.startswith(sys.prefix) + ) + site_packages.mkdir(parents=True) + + # install workaround + pip_run.launch.inject_sitecustomize(str(site_packages)) + + env = dict(os.environ, PYTHONPATH=str(site_packages), SETUPTOOLS_EDITABLE=mode) + cmd = [ + sys.executable, + '-m', + 'pip', + 'install', + '--editable', + str(sample_project), + '--prefix', + str(prefix), + '--no-build-isolation', + ] + subprocess.check_call(cmd, env=env) + + # now run 'sample' with the prefix on the PYTHONPATH + bin = 'Scripts' if platform.system() == 'Windows' else 'bin' + exe = prefix / bin / 'sample' + if sys.version_info < (3, 8) and platform.system() == 'Windows': + exe = str(exe) + subprocess.check_call([exe], env=env) + + +class TestFinderTemplate: + """This test focus in getting a particular implementation detail right. + If at some point in time the implementation is changed for something different, + this test can be modified or even excluded. + """ + def install_finder(self, finder): + loc = {} + exec(finder, loc, loc) + loc["install"]() + + def test_packages(self, tmp_path): + files = { + "src1": { + "pkg1": { + "__init__.py": "", + "subpkg": {"mod1.py": "a = 42"}, + }, + }, + "src2": {"mod2.py": "a = 43"}, + } + jaraco.path.build(files, prefix=tmp_path) + + mapping = { + "pkg1": str(tmp_path / "src1/pkg1"), + "mod2": str(tmp_path / "src2/mod2") + } + template = _finder_template(str(uuid4()), mapping, {}) + + with contexts.save_paths(), contexts.save_sys_modules(): + for mod in ("pkg1", "pkg1.subpkg", "pkg1.subpkg.mod1", "mod2"): + sys.modules.pop(mod, None) + + self.install_finder(template) + mod1 = import_module("pkg1.subpkg.mod1") + mod2 = import_module("mod2") + subpkg = import_module("pkg1.subpkg") + + assert mod1.a == 42 + assert mod2.a == 43 + expected = str((tmp_path / "src1/pkg1/subpkg").resolve()) + assert_path(subpkg, expected) + + def test_namespace(self, tmp_path): + files = {"pkg": {"__init__.py": "a = 13", "text.txt": "abc"}} + jaraco.path.build(files, prefix=tmp_path) + + mapping = {"ns.othername": str(tmp_path / "pkg")} + namespaces = {"ns": []} + + template = _finder_template(str(uuid4()), mapping, namespaces) + with contexts.save_paths(), contexts.save_sys_modules(): + for mod in ("ns", "ns.othername"): + sys.modules.pop(mod, None) + + self.install_finder(template) + pkg = import_module("ns.othername") + text = importlib_resources.files(pkg) / "text.txt" + + expected = str((tmp_path / "pkg").resolve()) + assert_path(pkg, expected) + assert pkg.a == 13 + + # Make sure resources can also be found + assert text.read_text(encoding="utf-8") == "abc" + + def test_combine_namespaces(self, tmp_path): + files = { + "src1": {"ns": {"pkg1": {"__init__.py": "a = 13"}}}, + "src2": {"ns": {"mod2.py": "b = 37"}}, + } + jaraco.path.build(files, prefix=tmp_path) + + mapping = { + "ns.pkgA": str(tmp_path / "src1/ns/pkg1"), + "ns": str(tmp_path / "src2/ns"), + } + namespaces_ = {"ns": [str(tmp_path / "src1"), str(tmp_path / "src2")]} + template = _finder_template(str(uuid4()), mapping, namespaces_) + + with contexts.save_paths(), contexts.save_sys_modules(): + for mod in ("ns", "ns.pkgA", "ns.mod2"): + sys.modules.pop(mod, None) + + self.install_finder(template) + pkgA = import_module("ns.pkgA") + mod2 = import_module("ns.mod2") + + expected = str((tmp_path / "src1/ns/pkg1").resolve()) + assert_path(pkgA, expected) + assert pkgA.a == 13 + assert mod2.b == 37 + + def test_dynamic_path_computation(self, tmp_path): + # Follows the example in PEP 420 + files = { + "project1": {"parent": {"child": {"one.py": "x = 1"}}}, + "project2": {"parent": {"child": {"two.py": "x = 2"}}}, + "project3": {"parent": {"child": {"three.py": "x = 3"}}}, + } + jaraco.path.build(files, prefix=tmp_path) + mapping = {} + namespaces_ = {"parent": [str(tmp_path / "project1/parent")]} + template = _finder_template(str(uuid4()), mapping, namespaces_) + + mods = (f"parent.child.{name}" for name in ("one", "two", "three")) + with contexts.save_paths(), contexts.save_sys_modules(): + for mod in ("parent", "parent.child", "parent.child", *mods): + sys.modules.pop(mod, None) + + self.install_finder(template) + + one = import_module("parent.child.one") + assert one.x == 1 + + with pytest.raises(ImportError): + import_module("parent.child.two") + + sys.path.append(str(tmp_path / "project2")) + two = import_module("parent.child.two") + assert two.x == 2 + + with pytest.raises(ImportError): + import_module("parent.child.three") + + sys.path.append(str(tmp_path / "project3")) + three = import_module("parent.child.three") + assert three.x == 3 + + +def test_pkg_roots(tmp_path): + """This test focus in getting a particular implementation detail right. + If at some point in time the implementation is changed for something different, + this test can be modified or even excluded. + """ + files = { + "a": {"b": {"__init__.py": "ab = 1"}, "__init__.py": "a = 1"}, + "d": {"__init__.py": "d = 1", "e": {"__init__.py": "de = 1"}}, + "f": {"g": {"h": {"__init__.py": "fgh = 1"}}}, + "other": {"__init__.py": "abc = 1"}, + "another": {"__init__.py": "abcxyz = 1"}, + "yet_another": {"__init__.py": "mnopq = 1"}, + } + jaraco.path.build(files, prefix=tmp_path) + package_dir = { + "a.b.c": "other", + "a.b.c.x.y.z": "another", + "m.n.o.p.q": "yet_another" + } + packages = [ + "a", + "a.b", + "a.b.c", + "a.b.c.x.y", + "a.b.c.x.y.z", + "d", + "d.e", + "f", + "f.g", + "f.g.h", + "m.n.o.p.q", + ] + roots = _find_package_roots(packages, package_dir, tmp_path) + assert roots == { + "a": str(tmp_path / "a"), + "a.b.c": str(tmp_path / "other"), + "a.b.c.x.y.z": str(tmp_path / "another"), + "d": str(tmp_path / "d"), + "f": str(tmp_path / "f"), + "m.n.o.p.q": str(tmp_path / "yet_another"), + } + + ns = set(dict(_find_namespaces(packages, roots))) + assert ns == {"f", "f.g"} + + ns = set(_find_virtual_namespaces(roots)) + assert ns == {"a.b.c.x", "a.b.c.x.y", "m", "m.n", "m.n.o", "m.n.o.p"} + + +class TestOverallBehaviour: + PYPROJECT = """\ + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + + [project] + name = "mypkg" + version = "3.14159" + """ + + FLAT_LAYOUT = { + "pyproject.toml": dedent(PYPROJECT), + "MANIFEST.in": EXAMPLE["MANIFEST.in"], + "otherfile.py": "", + "mypkg": { + "__init__.py": "", + "mod1.py": "var = 42", + "subpackage": { + "__init__.py": "", + "mod2.py": "var = 13", + "resource_file.txt": "resource 39", + }, + }, + } + + EXAMPLES = { + "flat-layout": FLAT_LAYOUT, + "src-layout": { + "pyproject.toml": dedent(PYPROJECT), + "MANIFEST.in": EXAMPLE["MANIFEST.in"], + "otherfile.py": "", + "src": {"mypkg": FLAT_LAYOUT["mypkg"]}, + }, + "custom-layout": { + "pyproject.toml": dedent(PYPROJECT) + dedent("""\ + [tool.setuptools] + packages = ["mypkg", "mypkg.subpackage"] + + [tool.setuptools.package-dir] + "mypkg.subpackage" = "other" + """), + "MANIFEST.in": EXAMPLE["MANIFEST.in"], + "otherfile.py": "", + "mypkg": { + "__init__.py": "", + "mod1.py": FLAT_LAYOUT["mypkg"]["mod1.py"], # type: ignore + }, + "other": FLAT_LAYOUT["mypkg"]["subpackage"], # type: ignore + }, + "namespace": { + "pyproject.toml": dedent(PYPROJECT), + "MANIFEST.in": EXAMPLE["MANIFEST.in"], + "otherfile.py": "", + "src": { + "mypkg": { + "mod1.py": FLAT_LAYOUT["mypkg"]["mod1.py"], # type: ignore + "subpackage": FLAT_LAYOUT["mypkg"]["subpackage"], # type: ignore + }, + }, + }, + } + + @pytest.mark.parametrize("layout", EXAMPLES.keys()) + def test_editable_install(self, tmp_path, venv, layout, editable_mode): + project = install_project("mypkg", venv, tmp_path, self.EXAMPLES[layout]) + + # Ensure stray files are not importable + cmd_import_error = """\ + try: + import otherfile + except ImportError as ex: + print(ex) + """ + out = venv.run(["python", "-c", dedent(cmd_import_error)]) + assert b"No module named 'otherfile'" in out + + # Ensure the modules are importable + cmd_get_vars = """\ + import mypkg, mypkg.mod1, mypkg.subpackage.mod2 + print(mypkg.mod1.var, mypkg.subpackage.mod2.var) + """ + out = venv.run(["python", "-c", dedent(cmd_get_vars)]) + assert b"42 13" in out + + # Ensure resources are reachable + cmd_get_resource = """\ + import mypkg.subpackage + from setuptools._importlib import resources as importlib_resources + text = importlib_resources.files(mypkg.subpackage) / "resource_file.txt" + print(text.read_text(encoding="utf-8")) + """ + out = venv.run(["python", "-c", dedent(cmd_get_resource)]) + assert b"resource 39" in out + + # Ensure files are editable + mod1 = next(project.glob("**/mod1.py")) + mod2 = next(project.glob("**/mod2.py")) + resource_file = next(project.glob("**/resource_file.txt")) + + mod1.write_text("var = 17", encoding="utf-8") + mod2.write_text("var = 781", encoding="utf-8") + resource_file.write_text("resource 374", encoding="utf-8") + + out = venv.run(["python", "-c", dedent(cmd_get_vars)]) + assert b"42 13" not in out + assert b"17 781" in out + + out = venv.run(["python", "-c", dedent(cmd_get_resource)]) + assert b"resource 39" not in out + assert b"resource 374" in out + + +class TestLinkTree: + FILES = deepcopy(TestOverallBehaviour.EXAMPLES["src-layout"]) + FILES["pyproject.toml"] += dedent("""\ + [tool.setuptools] + # Temporary workaround: both `include-package-data` and `package-data` configs + # can be removed after #3260 is fixed. + include-package-data = false + package-data = {"*" = ["*.txt"]} + + [tool.setuptools.packages.find] + where = ["src"] + exclude = ["*.subpackage*"] + """) + FILES["src"]["mypkg"]["resource.not_in_manifest"] = "abc" + + def test_generated_tree(self, tmp_path): + jaraco.path.build(self.FILES, prefix=tmp_path) + + with _Path(tmp_path): + name = "mypkg-3.14159" + dist = Distribution({"script_name": "%PEP 517%"}) + dist.parse_config_files() + + wheel = Mock() + aux = tmp_path / ".aux" + build = tmp_path / ".build" + aux.mkdir() + build.mkdir() + + build_py = dist.get_command_obj("build_py") + build_py.editable_mode = True + build_py.build_lib = str(build) + build_py.ensure_finalized() + outputs = build_py.get_outputs() + output_mapping = build_py.get_output_mapping() + + make_tree = _LinkTree(dist, name, aux, build) + make_tree(wheel, outputs, output_mapping) + + mod1 = next(aux.glob("**/mod1.py")) + expected = tmp_path / "src/mypkg/mod1.py" + assert_link_to(mod1, expected) + + assert next(aux.glob("**/subpackage"), None) is None + assert next(aux.glob("**/mod2.py"), None) is None + assert next(aux.glob("**/resource_file.txt"), None) is None + + assert next(aux.glob("**/resource.not_in_manifest"), None) is None + + def test_strict_install(self, tmp_path, venv, monkeypatch): + monkeypatch.setenv("SETUPTOOLS_EDITABLE", "strict") + install_project("mypkg", venv, tmp_path, self.FILES) + + out = venv.run(["python", "-c", "import mypkg.mod1; print(mypkg.mod1.var)"]) + assert b"42" in out + + # Ensure packages excluded from distribution are not importable + cmd_import_error = """\ + try: + from mypkg import subpackage + except ImportError as ex: + print(ex) + """ + out = venv.run(["python", "-c", dedent(cmd_import_error)]) + assert b"cannot import name 'subpackage'" in out + + # Ensure resource files excluded from distribution are not reachable + cmd_get_resource = """\ + import mypkg + from setuptools._importlib import resources as importlib_resources + try: + text = importlib_resources.files(mypkg) / "resource.not_in_manifest" + print(text.read_text(encoding="utf-8")) + except FileNotFoundError as ex: + print(ex) + """ + out = venv.run(["python", "-c", dedent(cmd_get_resource)]) + assert b"No such file or directory" in out + assert b"resource.not_in_manifest" in out + + +def install_project(name, venv, tmp_path, files): + project = tmp_path / name + project.mkdir() + jaraco.path.build(files, prefix=project) + opts = ["--no-build-isolation"] # force current version of setuptools + venv.run(["python", "-m", "pip", "install", "-e", str(project), *opts]) + return project + + +# ---- Assertion Helpers ---- + + +def assert_path(pkg, expected): + # __path__ is not guaranteed to exist, so we have to account for that + if pkg.__path__: + path = next(iter(pkg.__path__), None) + if path: + assert str(Path(path).resolve()) == expected + + +def assert_link_to(file: Path, other: Path): + if file.is_symlink(): + assert str(file.resolve()) == str(other.resolve()) + else: + file_stat = file.stat() + other_stat = other.stat() + assert file_stat[stat.ST_INO] == other_stat[stat.ST_INO] + assert file_stat[stat.ST_DEV] == other_stat[stat.ST_DEV] diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py index 302cff7..4b0d2e1 100644 --- a/setuptools/tests/test_sdist.py +++ b/setuptools/tests/test_sdist.py @@ -10,6 +10,7 @@ from unittest import mock import pytest +from setuptools import Command from setuptools._importlib import metadata from setuptools import SetuptoolsDeprecationWarning from setuptools.command.sdist import sdist @@ -517,6 +518,46 @@ class TestSdistTest: manifest = cmd.filelist.files assert 'pyproject.toml' not in manifest + def test_build_subcommand_source_files(self, tmpdir): + touch(tmpdir / '.myfile~') + + # Sanity check: without custom commands file list should not be affected + dist = Distribution({**SETUP_ATTRS, "script_name": "setup.py"}) + cmd = sdist(dist) + cmd.ensure_finalized() + with quiet(): + cmd.run() + manifest = cmd.filelist.files + assert '.myfile~' not in manifest + + # Test: custom command should be able to augment file list + dist = Distribution({**SETUP_ATTRS, "script_name": "setup.py"}) + build = dist.get_command_obj("build") + build.sub_commands = [*build.sub_commands, ("build_custom", None)] + + class build_custom(Command): + def initialize_options(self): + ... + + def finalize_options(self): + ... + + def run(self): + ... + + def get_source_files(self): + return ['.myfile~'] + + dist.cmdclass.update(build_custom=build_custom) + + cmd = sdist(dist) + cmd.use_defaults = True + cmd.ensure_finalized() + with quiet(): + cmd.run() + manifest = cmd.filelist.files + assert '.myfile~' in manifest + def test_default_revctrl(): """ diff --git a/tools/vendored.py b/tools/vendored.py index 8a122ad..cd15adb 100644 --- a/tools/vendored.py +++ b/tools/vendored.py @@ -89,6 +89,16 @@ def rewrite_more_itertools(pkg_files: Path): more_file.write_text(text) +def rewrite_nspektr(pkg_files: Path, new_root): + for file in pkg_files.glob('*.py'): + text = file.read_text() + text = re.sub(r' (more_itertools)', rf' {new_root}.\1', text) + text = re.sub(r' (jaraco\.\w+)', rf' {new_root}.\1', text) + text = re.sub(r' (packaging)', rf' {new_root}.\1', text) + text = re.sub(r' (importlib_metadata)', rf' {new_root}.\1', text) + file.write_text(text) + + def clean(vendor): """ Remove all files out of the vendor directory except the meta @@ -133,6 +143,7 @@ def update_setuptools(): rewrite_importlib_resources(vendor / 'importlib_resources', 'setuptools.extern') rewrite_importlib_metadata(vendor / 'importlib_metadata', 'setuptools.extern') rewrite_more_itertools(vendor / "more_itertools") + rewrite_nspektr(vendor / "nspektr", 'setuptools.extern') __name__ == '__main__' and update_vendored()