From: DongHun Kwak Date: Wed, 9 Dec 2020 00:35:23 +0000 (+0900) Subject: Imported Upstream version 3.8.4 X-Git-Tag: upstream/3.8.4^0 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=2aed1ddb5f6f43bf84d8f260bcea4d5dfb3f3157;p=platform%2Fupstream%2Fpython3.git Imported Upstream version 3.8.4 --- diff --git a/Doc/Makefile b/Doc/Makefile index 2169f1a3..b8ca1edf 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -143,7 +143,7 @@ clean: venv: $(PYTHON) -m venv $(VENVDIR) $(VENVDIR)/bin/python3 -m pip install -U pip setuptools - $(VENVDIR)/bin/python3 -m pip install -U Sphinx==1.8.2 blurb python-docs-theme + $(VENVDIR)/bin/python3 -m pip install -U Sphinx==2.3.1 blurb python-docs-theme @echo "The venv has been created in the $(VENVDIR) directory" dist: diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 79a8815e..ff4ccb8d 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -196,12 +196,12 @@ PyPreConfig Function to initialize a preconfiguration: - .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) + .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Python Configuration `. - .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) + .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Isolated Configuration `. diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index f41d419b..3bada415 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -98,11 +98,9 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: PyObject* PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) - Convert a sequence of Unicode digits to a Python integer value. The Unicode - string is first encoded to a byte string using :c:func:`PyUnicode_EncodeDecimal` - and then converted using :c:func:`PyLong_FromString`. + Convert a sequence of Unicode digits to a Python integer value. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.10 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyLong_FromUnicodeObject`. @@ -110,9 +108,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base) Convert a sequence of Unicode digits in the string *u* to a Python integer - value. The Unicode string is first encoded to a byte string using - :c:func:`PyUnicode_EncodeDecimal` and then converted using - :c:func:`PyLong_FromString`. + value. .. versionadded:: 3.3 diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 2bf4a0f5..b261efe2 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -700,6 +700,8 @@ Extension modules can continue using them, as they will not be removed in Python :c:func:`PyUnicode_AsWideChar`, :c:func:`PyUnicode_ReadChar` or similar new APIs. + .. deprecated-removed:: 3.3 3.10 + .. c:function:: PyObject* PyUnicode_TransformDecimalToASCII(Py_UNICODE *s, Py_ssize_t size) diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 5f7b3bbc..02379946 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -128,6 +128,7 @@ involved in creating and publishing a project: * `Project structure`_ * `Building and packaging the project`_ * `Uploading the project to the Python Packaging Index`_ +* `The .pypirc file`_ .. _Project structure: \ https://packaging.python.org/tutorials/distributing-packages/ @@ -135,6 +136,8 @@ involved in creating and publishing a project: https://packaging.python.org/tutorials/distributing-packages/#packaging-your-project .. _Uploading the project to the Python Packaging Index: \ https://packaging.python.org/tutorials/distributing-packages/#uploading-your-project-to-pypi +.. _The .pypirc file: \ + https://packaging.python.org/specifications/pypirc/ How do I...? diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 3ef553e8..eee3c3c2 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -17,12 +17,13 @@ What is Python? Python is an interpreted, interactive, object-oriented programming language. It incorporates modules, exceptions, dynamic typing, very high level dynamic data -types, and classes. Python combines remarkable power with very clear syntax. -It has interfaces to many system calls and libraries, as well as to various -window systems, and is extensible in C or C++. It is also usable as an -extension language for applications that need a programmable interface. -Finally, Python is portable: it runs on many Unix variants, on the Mac, and on -Windows 2000 and later. +types, and classes. It supports multiple programming paradigms beyond +object-oriented programming, such as procedural and functional programming. +Python combines remarkable power with very clear syntax. It has interfaces to +many system calls and libraries, as well as to various window systems, and is +extensible in C or C++. It is also usable as an extension language for +applications that need a programmable interface. Finally, Python is portable: +it runs on many Unix variants including Linux and macOS, and on Windows. To find out more, start with :ref:`tutorial-index`. The `Beginner's Guide to Python `_ links to other @@ -295,8 +296,8 @@ How stable is Python? --------------------- Very stable. New, stable releases have been coming out roughly every 6 to 18 -months since 1991, and this seems likely to continue. Currently there are -usually around 18 months between major releases. +months since 1991, and this seems likely to continue. As of version 3.9, +Python will have a major new release every 12 months (:pep:`602`). The developers issue "bugfix" releases of older versions, so the stability of existing releases gradually improves. Bugfix releases, indicated by a third @@ -314,8 +315,8 @@ be maintained after January 1, 2020 ` How many people are using Python? --------------------------------- -There are probably tens of thousands of users, though it's difficult to obtain -an exact count. +There are probably millions of users, though it's difficult to obtain an exact +count. Python is available for free download, so there are no sales figures, and it's available from many different sites and packaged with many Linux distributions, diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 6189cb04..e997d366 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -189,6 +189,10 @@ Glossary A list of bytecode instructions can be found in the documentation for :ref:`the dis module `. + callback + A subroutine function which is passed as an argument to be executed at + some point in the future. + class A template for creating user-defined objects. Class definitions normally contain method definitions which operate on instances of the diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index bc71d85a..b5c2152e 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -319,7 +319,7 @@ inside-out. In Python, you use ``socket.setblocking(0)`` to make it non-blocking. In C, it's more complex, (for one thing, you'll need to choose between the BSD flavor -``O_NONBLOCK`` and the almost indistinguishable Posix flavor ``O_NDELAY``, which +``O_NONBLOCK`` and the almost indistinguishable POSIX flavor ``O_NDELAY``, which is completely different from ``TCP_NODELAY``), but it's the exact same idea. You do this after creating the socket, but before using it. (Actually, if you're nuts, you can switch back and forth.) diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 101e7817..0d302ea4 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -73,7 +73,7 @@ event loop, no other Tasks can run in the same thread. When a Task executes an ``await`` expression, the running Task gets suspended, and the event loop executes the next Task. -To schedule a callback from a different OS thread, the +To schedule a :term:`callback` from another OS thread, the :meth:`loop.call_soon_threadsafe` method should be used. Example:: loop.call_soon_threadsafe(callback, *args) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 9022993e..32bc219c 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -179,8 +179,8 @@ Scheduling callbacks .. method:: loop.call_soon(callback, *args, context=None) - Schedule a *callback* to be called with *args* arguments at - the next iteration of the event loop. + Schedule the *callback* :term:`callback` to be called with + *args* arguments at the next iteration of the event loop. Callbacks are called in the order in which they are registered. Each callback will be called exactly once. diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index ffac9018..816ddcd0 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -996,7 +996,7 @@ loop.subprocess_exec() and SubprocessProtocol An example of a subprocess protocol used to get the output of a subprocess and to wait for the subprocess exit. -The subprocess is created by th :meth:`loop.subprocess_exec` method:: +The subprocess is created by the :meth:`loop.subprocess_exec` method:: import asyncio import sys diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 1d87d2f8..eb1312a9 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -95,14 +95,14 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_shell` for other parameters. -.. important:: - - It is the application's responsibility to ensure that all whitespace and - special characters are quoted appropriately to avoid `shell injection - `_ - vulnerabilities. The :func:`shlex.quote` function can be used to properly - escape whitespace and special shell characters in strings that are going - to be used to construct shell commands. + .. important:: + + It is the application's responsibility to ensure that all whitespace and + special characters are quoted appropriately to avoid `shell injection + `_ + vulnerabilities. The :func:`shlex.quote` function can be used to properly + escape whitespace and special shell characters in strings that are going + to be used to construct shell commands. .. deprecated-removed:: 3.8 3.10 diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 1a23661f..00ce5d4b 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -580,9 +580,9 @@ Waiting Primitives .. function:: as_completed(aws, \*, loop=None, timeout=None) Run :ref:`awaitable objects ` in the *aws* - set concurrently. Return an iterator of :class:`Future` objects. - Each Future object returned represents the earliest result - from the set of the remaining awaitables. + set concurrently. Return an iterator of coroutines. + Each coroutine returned can be awaited to get the earliest next + result from the set of the remaining awaitables. Raises :exc:`asyncio.TimeoutError` if the timeout occurs before all Futures are done. @@ -592,8 +592,8 @@ Waiting Primitives Example:: - for f in as_completed(aws): - earliest_result = await f + for coro in as_completed(aws): + earliest_result = await coro # ... diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 56b75ef0..c3c04db8 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -349,7 +349,7 @@ For simple text calendars this module provides the following functions. .. function:: monthcalendar(year, month) Returns a matrix representing a month's calendar. Each row represents a week; - days outside of the month a represented by zeros. Each week begins with Monday + days outside of the month are represented by zeros. Each week begins with Monday unless set by :func:`setfirstweekday`. diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 6708079f..538e5afc 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -56,8 +56,8 @@ build applications which provide an interactive interpreter prompt. *source* is the source string; *filename* is the optional filename from which source was read, defaulting to ``''``; and *symbol* is the optional - grammar start symbol, which should be either ``'single'`` (the default) or - ``'eval'``. + grammar start symbol, which should be ``'single'`` (the default), ``'eval'`` + or ``'exec'``. Returns a code object (the same as ``compile(source, filename, symbol)``) if the command is complete and valid; ``None`` if the command is incomplete; raises diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index a52d2c62..c66b9d3e 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -43,8 +43,9 @@ To do just the former: :exc:`OverflowError` or :exc:`ValueError` if there is an invalid literal. The *symbol* argument determines whether *source* is compiled as a statement - (``'single'``, the default) or as an :term:`expression` (``'eval'``). Any - other value will cause :exc:`ValueError` to be raised. + (``'single'``, the default), as a sequence of statements (``'exec'``) or + as an :term:`expression` (``'eval'``). Any other value will + cause :exc:`ValueError` to be raised. .. note:: diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 8dcf9451..d4297166 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -1150,6 +1150,8 @@ variants of :func:`functools.lru_cache`:: return value def __setitem__(self, key, value): + if key in self: + self.move_to_end(key) super().__setitem__(key, value) if len(self) > self.maxsize: oldest = next(iter(self)) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 61d39828..7a72c26d 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -167,6 +167,9 @@ The :mod:`csv` module defines the following classes: All other optional or keyword arguments are passed to the underlying :class:`reader` instance. + .. versionchanged:: 3.6 + Returned rows are now of type :class:`OrderedDict`. + .. versionchanged:: 3.8 Returned rows are now of type :class:`dict`. diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 10edcac7..a7fbaaa1 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -23,7 +23,7 @@ using :pep:`526` type annotations. For example this code:: @dataclass class InventoryItem: - '''Class for keeping track of an item in inventory.''' + """Class for keeping track of an item in inventory.""" name: str unit_price: float quantity_on_hand: int = 0 diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 29c6b5a4..b733c440 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -35,7 +35,8 @@ on efficient attribute extraction for output formatting and manipulation. Aware and Naive Objects ----------------------- -Date and time objects may be categorized as "aware" or "naive." +Date and time objects may be categorized as "aware" or "naive" depending on +whether or not they include timezone information. With sufficient knowledge of applicable algorithmic and political time adjustments, such as time zone and daylight saving time information, diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 3dda35fb..8169bd35 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -2130,67 +2130,17 @@ Q. Is the CPython implementation fast for large numbers? A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of the decimal module integrate the high speed `libmpdec `_ library for -arbitrary precision correctly-rounded decimal floating point arithmetic [#]_. +arbitrary precision correctly-rounded decimal floating point arithmetic. ``libmpdec`` uses `Karatsuba multiplication `_ for medium-sized numbers and the `Number Theoretic Transform `_ -for very large numbers. +for very large numbers. However, to realize this performance gain, the +context needs to be set for unrounded calculations. -The context must be adapted for exact arbitrary precision arithmetic. :attr:`Emin` -and :attr:`Emax` should always be set to the maximum values, :attr:`clamp` -should always be 0 (the default). Setting :attr:`prec` requires some care. + >>> c = getcontext() + >>> c.prec = MAX_PREC + >>> c.Emax = MAX_EMAX + >>> c.Emin = MIN_EMIN -The easiest approach for trying out bignum arithmetic is to use the maximum -value for :attr:`prec` as well [#]_:: - - >>> setcontext(Context(prec=MAX_PREC, Emax=MAX_EMAX, Emin=MIN_EMIN)) - >>> x = Decimal(2) ** 256 - >>> x / 128 - Decimal('904625697166532776746648320380374280103671755200316906558262375061821325312') - - -For inexact results, :attr:`MAX_PREC` is far too large on 64-bit platforms and -the available memory will be insufficient:: - - >>> Decimal(1) / 3 - Traceback (most recent call last): - File "", line 1, in - MemoryError - -On systems with overallocation (e.g. Linux), a more sophisticated approach is to -adjust :attr:`prec` to the amount of available RAM. Suppose that you have 8GB of -RAM and expect 10 simultaneous operands using a maximum of 500MB each:: - - >>> import sys - >>> - >>> # Maximum number of digits for a single operand using 500MB in 8-byte words - >>> # with 19 digits per word (4-byte and 9 digits for the 32-bit build): - >>> maxdigits = 19 * ((500 * 1024**2) // 8) - >>> - >>> # Check that this works: - >>> c = Context(prec=maxdigits, Emax=MAX_EMAX, Emin=MIN_EMIN) - >>> c.traps[Inexact] = True - >>> setcontext(c) - >>> - >>> # Fill the available precision with nines: - >>> x = Decimal(0).logical_invert() * 9 - >>> sys.getsizeof(x) - 524288112 - >>> x + 2 - Traceback (most recent call last): - File "", line 1, in - decimal.Inexact: [] - -In general (and especially on systems without overallocation), it is recommended -to estimate even tighter bounds and set the :attr:`Inexact` trap if all calculations -are expected to be exact. - - -.. [#] - .. versionadded:: 3.3 - -.. [#] - .. versionchanged:: 3.9 - This approach now works for all exact results except for non-integer powers. - Also backported to 3.7 and 3.8. +.. versionadded:: 3.3 \ No newline at end of file diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 75380847..2649b7d8 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -273,6 +273,10 @@ overridden:: the next :class:`int` in sequence with the last :class:`int` provided, but the way it does this is an implementation detail and may change. +.. note:: + + The :meth:`_generate_next_value_` method must be defined before any members. + Iteration --------- diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index b1192e7b..75b6fa38 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -147,7 +147,7 @@ Go to Line Clear any selection and update the line and column status. Show Completions - Open a scrollable list allowing selection of keywords and attributes. See + Open a scrollable list allowing selection of existing names. See :ref:`Completions ` in the Editing and navigation section below. Expand Word @@ -469,52 +469,58 @@ are restricted to four spaces due to Tcl/Tk limitations. See also the indent/dedent region commands on the :ref:`Format menu `. - .. _completions: Completions ^^^^^^^^^^^ -Completions are supplied for functions, classes, and attributes of classes, -both built-in and user-defined. Completions are also provided for -filenames. - -The AutoCompleteWindow (ACW) will open after a predefined delay (default is -two seconds) after a '.' or (in a string) an os.sep is typed. If after one -of those characters (plus zero or more other characters) a tab is typed -the ACW will open immediately if a possible continuation is found. - -If there is only one possible completion for the characters entered, a -:kbd:`Tab` will supply that completion without opening the ACW. - -'Show Completions' will force open a completions window, by default the -:kbd:`C-space` will open a completions window. In an empty -string, this will contain the files in the current directory. On a -blank line, it will contain the built-in and user-defined functions and -classes in the current namespaces, plus any modules imported. If some -characters have been entered, the ACW will attempt to be more specific. - -If a string of characters is typed, the ACW selection will jump to the -entry most closely matching those characters. Entering a :kbd:`tab` will -cause the longest non-ambiguous match to be entered in the Editor window or -Shell. Two :kbd:`tab` in a row will supply the current ACW selection, as -will return or a double click. Cursor keys, Page Up/Down, mouse selection, -and the scroll wheel all operate on the ACW. - -"Hidden" attributes can be accessed by typing the beginning of hidden -name after a '.', e.g. '_'. This allows access to modules with -``__all__`` set, or to class-private attributes. - -Completions and the 'Expand Word' facility can save a lot of typing! - -Completions are currently limited to those in the namespaces. Names in -an Editor window which are not via ``__main__`` and :data:`sys.modules` will -not be found. Run the module once with your imports to correct this situation. -Note that IDLE itself places quite a few modules in sys.modules, so -much can be found by default, e.g. the re module. - -If you don't like the ACW popping up unbidden, simply make the delay -longer or disable the extension. +Completions are supplied, when requested and available, for module +names, attributes of classes or functions, or filenames. Each request +method displays a completion box with existing names. (See tab +completions below for an exception.) For any box, change the name +being completed and the item highlighted in the box by +typing and deleting characters; by hitting :kbd:`Up`, :kbd:`Down`, +:kbd:`PageUp`, :kbd:`PageDown`, :kbd:`Home`, and :kbd:`End` keys; +and by a single click within the box. Close the box with :kbd:`Escape`, +:kbd:`Enter`, and double :kbd:`Tab` keys or clicks outside the box. +A double click within the box selects and closes. + +One way to open a box is to type a key character and wait for a +predefined interval. This defaults to 2 seconds; customize it +in the settings dialog. (To prevent auto popups, set the delay to a +large number of milliseconds, such as 100000000.) For imported module +names or class or function attributes, type '.'. +For filenames in the root directory, type :data:`os.sep` or +data:`os.altsep` immediately after an opening quote. (On Windows, +one can specify a drive first.) Move into subdirectories by typing a +directory name and a separator. + +Instead of waiting, or after a box is closed, open a completion box +immediately with Show Completions on the Edit menu. The default hot +key is :kbd:`C-space`. If one types a prefix for the desired name +before opening the box, the first match or near miss is made visible. +The result is the same as if one enters a prefix +after the box is displayed. Show Completions after a quote completes +filenames in the current directory instead of a root directory. + +Hitting :kbd:`Tab` after a prefix usually has the same effect as Show +Completions. (With no prefix, it indents.) However, if there is only +one match to the prefix, that match is immediately added to the editor +text without opening a box. + +Invoking 'Show Completions', or hitting :kbd:`Tab` after a prefix, +outside of a string and without a preceding '.' opens a box with +keywords, builtin names, and available module-level names. + +When editing code in an editor (as oppose to Shell), increase the +available module-level names by running your code +and not restarting the Shell thereafter. This is especially useful +after adding imports at the top of a file. This also increases +possible attribute completions. + +Completion boxes intially exclude names beginning with '_' or, for +modules, not included in '__all__'. The hidden names can be accessed +by typing '_' after '.', either before or after the box is opened. .. _calltips: diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst index f5ad8c72..121a730e 100644 --- a/Doc/library/imp.rst +++ b/Doc/library/imp.rst @@ -8,7 +8,7 @@ **Source code:** :source:`Lib/imp.py` .. deprecated:: 3.4 - The :mod:`imp` package is pending deprecation in favor of :mod:`importlib`. + The :mod:`imp` module is deprecated in favor of :mod:`importlib`. .. index:: statement: import diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 3e5a6738..134afbe6 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -199,9 +199,9 @@ loops that truncate the stream. Return *r* length subsequences of elements from the input *iterable*. - Combinations are emitted in lexicographic sort order. So, if the - input *iterable* is sorted, the combination tuples will be produced - in sorted order. + The combination tuples are emitted in lexicographic ordering according to + the order of the input *iterable*. So, if the input *iterable* is sorted, + the combination tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their value. So if the input elements are unique, there will be no repeat @@ -248,9 +248,9 @@ loops that truncate the stream. Return *r* length subsequences of elements from the input *iterable* allowing individual elements to be repeated more than once. - Combinations are emitted in lexicographic sort order. So, if the - input *iterable* is sorted, the combination tuples will be produced - in sorted order. + The combination tuples are emitted in lexicographic ordering according to + the order of the input *iterable*. So, if the input *iterable* is sorted, + the combination tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their value. So if the input elements are unique, the generated combinations @@ -484,9 +484,9 @@ loops that truncate the stream. of the *iterable* and all possible full-length permutations are generated. - Permutations are emitted in lexicographic sort order. So, if the - input *iterable* is sorted, the permutation tuples will be produced - in sorted order. + The permutation tuples are emitted in lexicographic ordering according to + the order of the input *iterable*. So, if the input *iterable* is sorted, + the combination tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their value. So if the input elements are unique, there will be no repeat diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst index 4548459f..48c1f275 100644 --- a/Doc/library/othergui.rst +++ b/Doc/library/othergui.rst @@ -30,10 +30,11 @@ available for Python: for generating bindings for C++ libraries as Python classes, and is specifically designed for Python. - `PySide `_ - PySide is a newer binding to the Qt toolkit, provided by Nokia. - Compared to PyQt, its licensing scheme is friendlier to non-open source - applications. + `PySide2 `_ + Also known as the Qt for Python project, PySide2 is a newer binding to the + Qt toolkit. It is provided by The Qt Company and aims to provide a + complete port of PySide to Qt 5. Compared to PyQt, its licensing scheme is + friendlier to non-open source applications. `wxPython `_ wxPython is a cross-platform GUI toolkit for Python that is built around @@ -47,7 +48,7 @@ available for Python: an XML-based resource format and more, including an ever growing library of user-contributed modules. -PyGTK, PyQt, and wxPython, all have a modern look and feel and more +PyGTK, PyQt, PySide2, and wxPython, all have a modern look and feel and more widgets than Tkinter. In addition, there are many other GUI toolkits for Python, both cross-platform, and platform-specific. See the `GUI Programming `_ page in the Python Wiki for a diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 5ee7faaa..ca0a507a 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -538,6 +538,7 @@ by the local file. executed in the current environment). .. pdbcommand:: retval + Print the return value for the last return of a function. .. rubric:: Footnotes diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 8f5a2cea..39622aad 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -117,7 +117,7 @@ The module defines the following: .. function:: select(rlist, wlist, xlist[, timeout]) This is a straightforward interface to the Unix :c:func:`select` system call. - The first three arguments are sequences of 'waitable objects': either + The first three arguments are iterables of 'waitable objects': either integers representing file descriptors or objects with a parameterless method named :meth:`~io.IOBase.fileno` returning such an integer: @@ -126,7 +126,7 @@ The module defines the following: * *xlist*: wait for an "exceptional condition" (see the manual page for what your system considers such a condition) - Empty sequences are allowed, but acceptance of three empty sequences is + Empty iterables are allowed, but acceptance of three empty iterables is platform-dependent. (It is known to work on Unix but not on Windows.) The optional *timeout* argument specifies a time-out as a floating point number in seconds. When the *timeout* argument is omitted the function blocks until @@ -141,7 +141,7 @@ The module defines the following: single: socket() (in module socket) single: popen() (in module os) - Among the acceptable object types in the sequences are Python :term:`file + Among the acceptable object types in the iterables are Python :term:`file objects ` (e.g. ``sys.stdin``, or objects returned by :func:`open` or :func:`os.popen`), socket objects returned by :func:`socket.socket`. You may also define a :dfn:`wrapper` class yourself, diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index bd24de72..25b749e5 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -567,12 +567,14 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. available), or "xztar" (if the :mod:`lzma` module is available). *root_dir* is a directory that will be the root directory of the - archive; for example, we typically chdir into *root_dir* before creating the - archive. + archive, all paths in the archive will be relative to it; for example, + we typically chdir into *root_dir* before creating the archive. *base_dir* is the directory where we start archiving from; i.e. *base_dir* will be the common prefix of all files and - directories in the archive. + directories in the archive. *base_dir* must be given relative + to *root_dir*. See :ref:`shutil-archiving-example-with-basedir` for how to + use *base_dir* and *root_dir* together. *root_dir* and *base_dir* both default to the current directory. @@ -724,6 +726,48 @@ The resulting archive contains: -rw-r--r-- tarek/staff 37192 2010-02-06 18:23:10 ./known_hosts +.. _shutil-archiving-example-with-basedir: + +Archiving example with *base_dir* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this example, similar to the `one above `_, +we show how to use :func:`make_archive`, but this time with the usage of +*base_dir*. We now have the following directory structure: + +.. code-block:: shell-session + + $ tree tmp + tmp + └── root + └── structure + ├── content + └── please_add.txt + └── do_not_add.txt + +In the final archive, :file:`please_add.txt` should be included, but +:file:`do_not_add.txt` should not. Therefore we use the following:: + + >>> from shutil import make_archive + >>> import os + >>> archive_name = os.path.expanduser(os.path.join('~', 'myarchive')) + >>> make_archive( + ... archive_name, + ... 'tar', + ... root_dir='tmp/root', + ... base_dir='structure/content', + ... ) + '/Users/tarek/my_archive.tar' + +Listing the files in the resulting archive gives us: + +.. code-block:: shell-session + + $ python -m tarfile -l /Users/tarek/myarchive.tar + structure/content/ + structure/content/please_add.txt + + Querying the size of the output terminal ---------------------------------------- diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 4142e41d..ea0f13c4 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2486,14 +2486,17 @@ provided. - :meth:`~SSLSocket.read` - :meth:`~SSLSocket.write` - :meth:`~SSLSocket.getpeercert` + - :meth:`~SSLSocket.selected_alpn_protocol` - :meth:`~SSLSocket.selected_npn_protocol` - :meth:`~SSLSocket.cipher` - :meth:`~SSLSocket.shared_ciphers` - :meth:`~SSLSocket.compression` - :meth:`~SSLSocket.pending` - :meth:`~SSLSocket.do_handshake` + - :meth:`~SSLSocket.verify_client_post_handshake` - :meth:`~SSLSocket.unwrap` - :meth:`~SSLSocket.get_channel_binding` + - :meth:`~SSLSocket.version` When compared to :class:`SSLSocket`, this object lacks the following features: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 8cc57c30..ccd2f99c 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -434,12 +434,10 @@ Notes: Negative shift counts are illegal and cause a :exc:`ValueError` to be raised. (2) - A left shift by *n* bits is equivalent to multiplication by ``pow(2, n)`` - without overflow check. + A left shift by *n* bits is equivalent to multiplication by ``pow(2, n)``. (3) - A right shift by *n* bits is equivalent to division by ``pow(2, n)`` without - overflow check. + A right shift by *n* bits is equivalent to floor division by ``pow(2, n)``. (4) Performing these calculations with at least one extra sign extension bit in diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index cce7da1c..a93a6c18 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -40,7 +40,7 @@ compatibility with older versions, see the :ref:`call-function-trio` section. .. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\ capture_output=False, shell=False, cwd=None, timeout=None, \ check=False, encoding=None, errors=None, text=None, env=None, \ - universal_newlines=None) + universal_newlines=None, **other_popen_kwargs) Run the command described by *args*. Wait for command to complete, then return a :class:`CompletedProcess` instance. @@ -704,10 +704,11 @@ Instances of the :class:`Popen` class have the following methods: .. method:: Popen.communicate(input=None, timeout=None) Interact with process: Send data to stdin. Read data from stdout and stderr, - until end-of-file is reached. Wait for process to terminate. The optional - *input* argument should be data to be sent to the child process, or - ``None``, if no data should be sent to the child. If streams were opened in - text mode, *input* must be a string. Otherwise, it must be bytes. + until end-of-file is reached. Wait for process to terminate and set the + :attr:`~Popen.returncode` attribute. The optional *input* argument should be + data to be sent to the child process, or ``None``, if no data should be sent + to the child. If streams were opened in text mode, *input* must be a string. + Otherwise, it must be bytes. :meth:`communicate` returns a tuple ``(stdout_data, stderr_data)``. The data will be strings if streams were opened in text mode; otherwise, @@ -755,14 +756,14 @@ Instances of the :class:`Popen` class have the following methods: .. method:: Popen.terminate() - Stop the child. On Posix OSs the method sends SIGTERM to the + Stop the child. On POSIX OSs the method sends SIGTERM to the child. On Windows the Win32 API function :c:func:`TerminateProcess` is called to stop the child. .. method:: Popen.kill() - Kills the child. On Posix OSs the function sends SIGKILL to the child. + Kills the child. On POSIX OSs the function sends SIGKILL to the child. On Windows :meth:`kill` is an alias for :meth:`terminate`. @@ -1049,7 +1050,8 @@ Prior to Python 3.5, these three functions comprised the high level API to subprocess. You can now use :func:`run` in many cases, but lots of existing code calls these functions. -.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, cwd=None, timeout=None) +.. function:: call(args, *, stdin=None, stdout=None, stderr=None, \ + shell=False, cwd=None, timeout=None, **other_popen_kwargs) Run the command described by *args*. Wait for command to complete, then return the :attr:`~Popen.returncode` attribute. @@ -1075,7 +1077,9 @@ calls these functions. .. versionchanged:: 3.3 *timeout* was added. -.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, cwd=None, timeout=None) +.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, \ + shell=False, cwd=None, timeout=None, \ + **other_popen_kwargs) Run command with arguments. Wait for command to complete. If the return code was zero then return, otherwise raise :exc:`CalledProcessError`. The @@ -1106,7 +1110,8 @@ calls these functions. .. function:: check_output(args, *, stdin=None, stderr=None, shell=False, \ cwd=None, encoding=None, errors=None, \ - universal_newlines=None, timeout=None, text=None) + universal_newlines=None, timeout=None, text=None, \ + **other_popen_kwargs) Run command with arguments and return its output. diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index b5a1da80..78a1dfce 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -74,12 +74,12 @@ places. Python currently supports seven schemes: -- *posix_prefix*: scheme for Posix platforms like Linux or Mac OS X. This is +- *posix_prefix*: scheme for POSIX platforms like Linux or Mac OS X. This is the default scheme used when Python or a component is installed. -- *posix_home*: scheme for Posix platforms used when a *home* option is used +- *posix_home*: scheme for POSIX platforms used when a *home* option is used upon installation. This scheme is used when a component is installed through Distutils with a specific home prefix. -- *posix_user*: scheme for Posix platforms used when a component is installed +- *posix_user*: scheme for POSIX platforms used when a component is installed through Distutils and the *user* option is used. This scheme defines paths located under the user home directory. - *nt*: scheme for NT platforms like Windows. diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index c34f2c4a..d60f1c8a 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -784,7 +784,7 @@ How to read a gzip compressed tar archive and display some member information:: import tarfile tar = tarfile.open("sample.tar.gz", "r:gz") for tarinfo in tar: - print(tarinfo.name, "is", tarinfo.size, "bytes in size and is", end="") + print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") if tarinfo.isreg(): print("a regular file.") elif tarinfo.isdir(): diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 7269e181..1467276d 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -672,7 +672,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.ByteString`. This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview`. + and :class:`memoryview` of byte sequences. As a shorthand for this type, :class:`bytes` can be used to annotate arguments of any of the types mentioned above. @@ -1020,9 +1020,9 @@ The module defines the following classes, functions and decorators: ``List[ForwardRef("SomeClass")]``. This class should not be instantiated by a user, but may be used by introspection tools. -.. function:: NewType(typ) +.. function:: NewType(name, tp) - A helper function to indicate a distinct types to a typechecker, + A helper function to indicate a distinct type to a typechecker, see :ref:`distinct`. At runtime it returns a function that returns its argument. Usage:: diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 31a3f41e..59b25183 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -27,7 +27,7 @@ See :pep:`405` for more information about Python virtual environments. .. seealso:: `Python Packaging User Guide: Creating and using virtual environments - `__ + `__ Creating virtual environments diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 2dbe5e33..0d9f21d2 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -163,13 +163,6 @@ Extension types can easily be made to support weak references; see application without adding attributes to those objects. This can be especially useful with objects that override attribute accesses. - .. note:: - - Caution: Because a :class:`WeakKeyDictionary` is built on top of a Python - dictionary, it must not change size when iterating over it. This can be - difficult to ensure for a :class:`WeakKeyDictionary` because actions - performed by the program during iteration may cause items in the - dictionary to vanish "by magic" (as a side effect of garbage collection). :class:`WeakKeyDictionary` objects have an additional method that exposes the internal references directly. The references are not guaranteed to @@ -189,13 +182,6 @@ than needed. Mapping class that references values weakly. Entries in the dictionary will be discarded when no strong reference to the value exists any more. - .. note:: - - Caution: Because a :class:`WeakValueDictionary` is built on top of a Python - dictionary, it must not change size when iterating over it. This can be - difficult to ensure for a :class:`WeakValueDictionary` because actions performed - by the program during iteration may cause items in the dictionary to vanish "by - magic" (as a side effect of garbage collection). :class:`WeakValueDictionary` objects have an additional method that has the same issues as the :meth:`keyrefs` method of :class:`WeakKeyDictionary` diff --git a/Doc/make.bat b/Doc/make.bat index a2f64ffd..d621bd16 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -13,7 +13,7 @@ if not defined SPHINXBUILD ( %PYTHON% -c "import sphinx" > nul 2> nul if errorlevel 1 ( echo Installing sphinx with %PYTHON% - %PYTHON% -m pip install sphinx + %PYTHON% -m pip install sphinx==2.2.0 if errorlevel 1 exit /B ) set SPHINXBUILD=%PYTHON% -c "import sphinx.cmd.build, sys; sys.exit(sphinx.cmd.build.main())" diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 844bd7cc..d0e7fde0 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -325,7 +325,7 @@ of identifiers is based on NFKC. A non-normative HTML file listing all valid identifier characters for Unicode 4.1 can be found at -https://www.dcl.hpi.uni-potsdam.de/home/loewis/table-3131.html. +https://www.unicode.org/Public/13.0.0/ucd/DerivedCoreProperties.txt .. _keywords: diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 2d956124..a128c705 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -311,7 +311,8 @@ class DeprecatedRemoved(Directive): final_argument_whitespace = True option_spec = {} - _label = 'Deprecated since version {deprecated}, will be removed in version {removed}' + _deprecated_label = 'Deprecated since version {deprecated}, will be removed in version {removed}' + _removed_label = 'Deprecated since version {deprecated}, removed in version {removed}' def run(self): node = addnodes.versionmodified() @@ -319,7 +320,15 @@ class DeprecatedRemoved(Directive): node['type'] = 'deprecated-removed' version = (self.arguments[0], self.arguments[1]) node['version'] = version - label = translators['sphinx'].gettext(self._label) + env = self.state.document.settings.env + current_version = tuple(int(e) for e in env.config.version.split('.')) + removed_version = tuple(int(e) for e in self.arguments[1].split('.')) + if current_version < removed_version: + label = self._deprecated_label + else: + label = self._removed_label + + label = translators['sphinx'].gettext(label) text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index e1ef91a8..c51f178c 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -10,7 +10,8 @@ '(?:release/\\d.\\d[\\x\\d\\.]*)']; var all_versions = { - '3.9': 'dev (3.9)', + '3.10': 'dev (3.10)', + '3.9': 'pre (3.9)', '3.8': '3.8', '3.7': '3.7', '3.6': '3.6', diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 8d94137b..68ae3ad1 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -5,3 +5,4 @@ In extensions/pyspecific.py: {% trans %}CPython implementation detail:{% endtrans %} {% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} +{% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %} diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 4730a5fe..7a40be77 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,7 +2,8 @@

{% trans %}Download these documents{% endtrans %}

{% trans %}Docs by version{% endtrans %}

    -
  • {% trans %}Python 3.9 (in development){% endtrans %}
    • +
  • {% trans %}Python 3.10 (in development){% endtrans %}
    • +
  • {% trans %}Python 3.9 (pre-release){% endtrans %}
  • {% trans %}Python 3.8 (stable){% endtrans %}
  • {% trans %}Python 3.7 (stable){% endtrans %}
  • {% trans %}Python 3.6 (security-fixes){% endtrans %}
    • diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 06bdd0d9..250d2a9d 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -114,8 +114,8 @@ accessible. "Directly accessible" here means that an unqualified reference to a name attempts to find the name in the namespace. Although scopes are determined statically, they are used dynamically. At any -time during execution, there are at least three nested scopes whose namespaces -are directly accessible: +time during execution, At any time during execution, there are 3 or 4 nested +scopes whose namespaces are directly accessible: * the innermost scope, which is searched first, contains the local names * the scopes of any enclosing functions, which are searched starting with the diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index f05f5edd..de2c73a3 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -297,11 +297,10 @@ passed using *call by value* (where the *value* is always an object *reference*, not the value of the object). [#]_ When a function calls another function, a new local symbol table is created for that call. -A function definition introduces the function name in the current symbol table. -The value of the function name has a type that is recognized by the interpreter -as a user-defined function. This value can be assigned to another name which -can then also be used as a function. This serves as a general renaming -mechanism:: +A function definition associates the function name with the function object in +the current symbol table. The interpreter recognizes the object pointed to by +that name as a user-defined function. Other names can also point to that same +function object and can also be used to access the function:: >>> fib diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index a404f4be..366a532e 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -172,7 +172,7 @@ Positional and keyword arguments can be arbitrarily combined:: If you have a really long format string that you don't want to split up, it would be nice if you could reference the variables to be formatted by name instead of by position. This can be done by simply passing the dict and using -square brackets ``'[]'`` to access the keys :: +square brackets ``'[]'`` to access the keys. :: >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 8637678} >>> print('Jack: {0[Jack]:d}; Sjoerd: {0[Sjoerd]:d}; ' @@ -257,10 +257,10 @@ left with zeros. It understands about plus and minus signs:: Old string formatting --------------------- -The ``%`` operator can also be used for string formatting. It interprets the -left argument much like a :c:func:`sprintf`\ -style format string to be applied -to the right argument, and returns the string resulting from this formatting -operation. For example:: +The % operator (modulo) can also be used for string formatting. Given ``'string' +% values``, instances of ``%`` in ``string`` are replaced with zero or more +elements of ``values``. This operation is commonly known as string +interpolation. For example:: >>> import math >>> print('The value of pi is approximately %5.3f.' % math.pi) diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index b411fa28..0253625f 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -27,9 +27,8 @@ What you get after installing is a number of things: * A :file:`Python 3.8` folder in your :file:`Applications` folder. In here you find IDLE, the development environment that is a standard part of official - Python distributions; PythonLauncher, which handles double-clicking Python - scripts from the Finder; and the "Build Applet" tool, which allows you to - package Python scripts as standalone applications on your system. + Python distributions; and PythonLauncher, which handles double-clicking Python + scripts from the Finder. * A framework :file:`/Library/Frameworks/Python.framework`, which includes the Python executable and libraries. The installer adds this location to your shell @@ -159,11 +158,6 @@ https://riverbankcomputing.com/software/pyqt/intro. Distributing Python Applications on the Mac =========================================== -The "Build Applet" tool that is placed in the MacPython 3.6 folder is fine for -packaging small Python scripts on your own machine to run as a standard Mac -application. This tool, however, is not robust enough to distribute Python -applications to other users. - The standard tool for deploying standalone Python applications on the Mac is :program:`py2app`. More information on installing and using py2app can be found at http://undefined.org/python/#py2app. diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index 99b2a19a..c81aaf15 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -86,7 +86,7 @@ The command, if run with ``-h``, will show the available options:: PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser See `About Execution Policies - `_ + `_ for more information. The created ``pyvenv.cfg`` file also includes the diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 636f48df..5114a26a 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -108,9 +108,7 @@ approximately 32,000 characters. Your administrator will need to activate the to ``1``. This allows the :func:`open` function, the :mod:`os` module and most other -path functionality to accept and return paths longer than 260 characters when -using strings. (Use of bytes as paths is deprecated on Windows, and this feature -is not available when using bytes.) +path functionality to accept and return paths longer than 260 characters. After changing the above option, no further configuration is required. diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 6b8bd886..880958d3 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -2,8 +2,6 @@ What's New In Python 3.0 **************************** -TEST CHANGE TO BE UNDONE - .. XXX Add trademark info for Apple, Microsoft. :Author: Guido van Rossum diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index f90e7e1a..96d5e31d 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -317,6 +317,9 @@ PyAPI_FUNC(void) _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime); PyAPI_FUNC(void) _PyGILState_Reinit(_PyRuntimeState *runtime); + +PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate); + #ifdef __cplusplus } #endif diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 2f6a68fb..41901a58 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 8 -#define PY_MICRO_VERSION 3 +#define PY_MICRO_VERSION 4 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.8.3" +#define PY_VERSION "3.8.4" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/ast.py b/Lib/ast.py index 0c88bcf4..99a1148a 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -483,6 +483,13 @@ class _ABC(type): return type.__instancecheck__(cls, inst) def _new(cls, *args, **kwargs): + for key in kwargs: + if key not in cls._fields: + # arbitrary keyword arguments are accepted + continue + pos = cls._fields.index(key) + if pos < len(args): + raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") if cls in _const_types: return Constant(*args, **kwargs) return Constant.__new__(cls, *args, **kwargs) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 8c0a5748..1ff8c427 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -101,7 +101,7 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): try: # Register a dummy signal handler to ask Python to write the signal - # number in the wakup file descriptor. _process_self_data() will + # number in the wakeup file descriptor. _process_self_data() will # read signal numbers from this file descriptor to handle signals. signal.signal(sig, _sighandler_noop) diff --git a/Lib/cgi.py b/Lib/cgi.py index c22c71b3..77ab703c 100755 --- a/Lib/cgi.py +++ b/Lib/cgi.py @@ -200,7 +200,10 @@ def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"): ctype = "multipart/form-data; boundary={}".format(boundary) headers = Message() headers.set_type(ctype) - headers['Content-Length'] = pdict['CONTENT-LENGTH'] + try: + headers['Content-Length'] = pdict['CONTENT-LENGTH'] + except KeyError: + pass fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, environ={'REQUEST_METHOD': 'POST'}) return {k: fs.getlist(k) for k in fs} @@ -736,7 +739,8 @@ class FieldStorage: last_line_lfend = True _read = 0 while 1: - if self.limit is not None and _read >= self.limit: + + if self.limit is not None and 0 <= self.limit <= _read: break line = self.fp.readline(1<<16) # bytes self.bytes_read += len(line) diff --git a/Lib/codeop.py b/Lib/codeop.py index 0fa677f6..3c2bb608 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -57,6 +57,7 @@ Compile(): """ import __future__ +import warnings _features = [getattr(__future__, fname) for fname in __future__.all_feature_names] @@ -83,15 +84,18 @@ def _maybe_compile(compiler, source, filename, symbol): except SyntaxError as err: pass - try: - code1 = compiler(source + "\n", filename, symbol) - except SyntaxError as e: - err1 = e + # Suppress warnings after the first compile to avoid duplication. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + try: + code1 = compiler(source + "\n", filename, symbol) + except SyntaxError as e: + err1 = e - try: - code2 = compiler(source + "\n\n", filename, symbol) - except SyntaxError as e: - err2 = e + try: + code2 = compiler(source + "\n\n", filename, symbol) + except SyntaxError as e: + err2 = e try: if code: @@ -112,7 +116,8 @@ def compile_command(source, filename="", symbol="single"): source -- the source string; may contain \n characters filename -- optional filename from which source was read; default "" - symbol -- optional grammar start symbol; "single" (default) or "eval" + symbol -- optional grammar start symbol; "single" (default), "exec" + or "eval" Return value / exceptions raised: diff --git a/Lib/ctypes/test/test_callbacks.py b/Lib/ctypes/test/test_callbacks.py index f622093d..d8e9c5a7 100644 --- a/Lib/ctypes/test/test_callbacks.py +++ b/Lib/ctypes/test/test_callbacks.py @@ -1,5 +1,7 @@ import functools import unittest +from test import support + from ctypes import * from ctypes.test import need_symbol import _ctypes_test @@ -287,7 +289,36 @@ class SampleCallbacksTestCase(unittest.TestCase): self.assertEqual(s.second, check.second) self.assertEqual(s.third, check.third) -################################################################ + def test_callback_too_many_args(self): + def func(*args): + return len(args) + + CTYPES_MAX_ARGCOUNT = 1024 + proto = CFUNCTYPE(c_int, *(c_int,) * CTYPES_MAX_ARGCOUNT) + cb = proto(func) + args1 = (1,) * CTYPES_MAX_ARGCOUNT + self.assertEqual(cb(*args1), CTYPES_MAX_ARGCOUNT) + + args2 = (1,) * (CTYPES_MAX_ARGCOUNT + 1) + with self.assertRaises(ArgumentError): + cb(*args2) + + def test_convert_result_error(self): + def func(): + return ("tuple",) + + proto = CFUNCTYPE(c_int) + ctypes_func = proto(func) + with support.catch_unraisable_exception() as cm: + # don't test the result since it is an uninitialized value + result = ctypes_func() + + self.assertIsInstance(cm.unraisable.exc_value, TypeError) + self.assertEqual(cm.unraisable.err_msg, + "Exception ignored on converting result " + "of ctypes callback function") + self.assertIs(cm.unraisable.object, func) + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_loading.py b/Lib/ctypes/test/test_loading.py index a62044e3..5c48b0db 100644 --- a/Lib/ctypes/test/test_loading.py +++ b/Lib/ctypes/test/test_loading.py @@ -158,11 +158,9 @@ class LoaderTest(unittest.TestCase): # Relative path (but not just filename) should succeed should_pass("WinDLL('./_sqlite3.dll')") - # XXX: This test has started failing on Azure Pipelines CI. See - # bpo-40214 for more information. - if 0: - # Insecure load flags should succeed - should_pass("WinDLL('_sqlite3.dll', winmode=0)") + # Insecure load flags should succeed + # Clear the DLL directory to avoid safe search settings propagating + should_pass("windll.kernel32.SetDllDirectoryW(None); WinDLL('_sqlite3.dll', winmode=0)") # Full path load without DLL_LOAD_DIR shouldn't find dependency should_fail("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + diff --git a/Lib/ctypes/test/test_random_things.py b/Lib/ctypes/test/test_random_things.py index ee5b2128..2988e275 100644 --- a/Lib/ctypes/test/test_random_things.py +++ b/Lib/ctypes/test/test_random_things.py @@ -1,5 +1,9 @@ from ctypes import * -import unittest, sys +import contextlib +from test import support +import unittest +import sys + def callback_func(arg): 42 / arg @@ -34,41 +38,40 @@ class CallbackTracbackTestCase(unittest.TestCase): # created, then a full traceback printed. When SystemExit is # raised in a callback function, the interpreter exits. - def capture_stderr(self, func, *args, **kw): - # helper - call function 'func', and return the captured stderr - import io - old_stderr = sys.stderr - logger = sys.stderr = io.StringIO() - try: - func(*args, **kw) - finally: - sys.stderr = old_stderr - return logger.getvalue() + @contextlib.contextmanager + def expect_unraisable(self, exc_type, exc_msg=None): + with support.catch_unraisable_exception() as cm: + yield + + self.assertIsInstance(cm.unraisable.exc_value, exc_type) + if exc_msg is not None: + self.assertEqual(str(cm.unraisable.exc_value), exc_msg) + self.assertEqual(cm.unraisable.err_msg, + "Exception ignored on calling ctypes " + "callback function") + self.assertIs(cm.unraisable.object, callback_func) def test_ValueError(self): cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 42) - self.assertEqual(out.splitlines()[-1], - "ValueError: 42") + with self.expect_unraisable(ValueError, '42'): + cb(42) def test_IntegerDivisionError(self): cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 0) - self.assertEqual(out.splitlines()[-1][:19], - "ZeroDivisionError: ") + with self.expect_unraisable(ZeroDivisionError): + cb(0) def test_FloatDivisionError(self): cb = CFUNCTYPE(c_int, c_double)(callback_func) - out = self.capture_stderr(cb, 0.0) - self.assertEqual(out.splitlines()[-1][:19], - "ZeroDivisionError: ") + with self.expect_unraisable(ZeroDivisionError): + cb(0.0) def test_TypeErrorDivisionError(self): cb = CFUNCTYPE(c_int, c_char_p)(callback_func) - out = self.capture_stderr(cb, b"spam") - self.assertEqual(out.splitlines()[-1], - "TypeError: " - "unsupported operand type(s) for /: 'int' and 'bytes'") + err_msg = "unsupported operand type(s) for /: 'int' and 'bytes'" + with self.expect_unraisable(TypeError, err_msg): + cb(b"spam") + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_unaligned_structures.py b/Lib/ctypes/test/test_unaligned_structures.py index bcacfc81..ee7fb458 100644 --- a/Lib/ctypes/test/test_unaligned_structures.py +++ b/Lib/ctypes/test/test_unaligned_structures.py @@ -27,7 +27,6 @@ for typ in [c_short, c_int, c_long, c_longlong, class TestStructures(unittest.TestCase): def test_native(self): for typ in structures: -## print typ.value self.assertEqual(typ.value.offset, 1) o = typ() o.value = 4 @@ -35,7 +34,6 @@ class TestStructures(unittest.TestCase): def test_swapped(self): for typ in byteswapped_structures: -## print >> sys.stderr, typ.value self.assertEqual(typ.value.offset, 1) o = typ() o.value = 4 diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 74f79294..10bb33e3 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1092,7 +1092,7 @@ def _asdict_inner(obj, dict_factory): # method, because: # - it does not recurse in to the namedtuple fields and # convert them to dicts (using dict_factory). - # - I don't actually want to return a dict here. The the main + # - I don't actually want to return a dict here. The main # use case here is json.dumps, and it handles converting # namedtuples to lists. Admittedly we're losing some # information here when we produce a json list instead of a diff --git a/Lib/distutils/command/build_py.py b/Lib/distutils/command/build_py.py index cf0ca57c..edc2171c 100644 --- a/Lib/distutils/command/build_py.py +++ b/Lib/distutils/command/build_py.py @@ -5,7 +5,7 @@ Implements the Distutils 'build_py' command.""" import os import importlib.util import sys -from glob import glob +import glob from distutils.core import Command from distutils.errors import * @@ -125,7 +125,7 @@ class build_py (Command): files = [] for pattern in globs: # Each pattern has to be converted to a platform-specific path - filelist = glob(os.path.join(src_dir, convert_path(pattern))) + filelist = glob.glob(os.path.join(glob.escape(src_dir), convert_path(pattern))) # Files that match more than one pattern are only added once files.extend([fn for fn in filelist if fn not in files and os.path.isfile(fn)]) @@ -216,7 +216,7 @@ class build_py (Command): def find_package_modules(self, package, package_dir): self.check_package(package, package_dir) - module_files = glob(os.path.join(package_dir, "*.py")) + module_files = glob.glob(os.path.join(glob.escape(package_dir), "*.py")) modules = [] setup_script = os.path.abspath(self.distribution.script_name) diff --git a/Lib/distutils/tests/__init__.py b/Lib/distutils/tests/__init__.py index 1b939cbd..68037216 100644 --- a/Lib/distutils/tests/__init__.py +++ b/Lib/distutils/tests/__init__.py @@ -15,7 +15,7 @@ by import rather than matching pre-defined names. import os import sys import unittest -from test.support import run_unittest +from test.support import run_unittest, save_restore_warnings_filters here = os.path.dirname(__file__) or os.curdir @@ -26,7 +26,11 @@ def test_suite(): for fn in os.listdir(here): if fn.startswith("test") and fn.endswith(".py"): modname = "distutils.tests." + fn[:-3] - __import__(modname) + # bpo-40055: Save/restore warnings filters to leave them unchanged. + # Importing tests imports docutils which imports pkg_resources + # which adds a warnings filter. + with save_restore_warnings_filters(): + __import__(modname) module = sys.modules[modname] suite.addTest(module.test_suite()) return suite diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 9c55ef7f..51d355fb 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1218,12 +1218,21 @@ def get_bare_quoted_string(value): if value[0] in WSP: token, value = get_fws(value) elif value[:2] == '=?': + valid_ew = False try: token, value = get_encoded_word(value) bare_quoted_string.defects.append(errors.InvalidHeaderDefect( "encoded word inside quoted string")) + valid_ew = True except errors.HeaderParseError: token, value = get_qcontent(value) + # Collapse the whitespace between two encoded words that occur in a + # bare-quoted-string. + if valid_ew and len(bare_quoted_string) > 1: + if (bare_quoted_string[-1].token_type == 'fws' and + bare_quoted_string[-2].token_type == 'encoded-word'): + bare_quoted_string[-1] = EWWhiteSpaceTerminal( + bare_quoted_string[-1], 'fws') else: token, value = get_qcontent(value) bare_quoted_string.append(token) diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py index b904ded9..b91fb0e5 100644 --- a/Lib/email/contentmanager.py +++ b/Lib/email/contentmanager.py @@ -146,13 +146,13 @@ def _encode_text(string, charset, cte, policy): def normal_body(lines): return b'\n'.join(lines) + b'\n' if cte==None: # Use heuristics to decide on the "best" encoding. - try: - return '7bit', normal_body(lines).decode('ascii') - except UnicodeDecodeError: - pass - if (policy.cte_type == '8bit' and - max(len(x) for x in lines) <= policy.max_line_length): - return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') + if max((len(x) for x in lines), default=0) <= policy.max_line_length: + try: + return '7bit', normal_body(lines).decode('ascii') + except UnicodeDecodeError: + pass + if policy.cte_type == '8bit': + return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') sniff = embedded_body(lines[:10]) sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'), policy.max_line_length) diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py index 8d1a2025..d0914fd1 100644 --- a/Lib/email/headerregistry.py +++ b/Lib/email/headerregistry.py @@ -31,6 +31,11 @@ class Address: without any Content Transfer Encoding. """ + + inputs = ''.join(filter(None, (display_name, username, domain, addr_spec))) + if '\r' in inputs or '\n' in inputs: + raise ValueError("invalid arguments; address parts cannot contain CR or LF") + # This clause with its potential 'raise' may only happen when an # application program creates an Address object using an addr_spec # keyword. The email library code itself must always supply username diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 858f620e..07dd029c 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -81,7 +81,7 @@ def formataddr(pair, charset='utf-8'): If the first element of pair is false, then the second element is returned unmodified. - Optional charset if given is the character set that is used to encode + The optional charset is the character set that is used to encode realname in case realname is not ASCII safe. Can be an instance of str or a Charset-like object which has a header_encode method. Default is 'utf-8'. diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 566fb2a0..f3152a55 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -9,13 +9,13 @@ import tempfile __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "41.2.0" +_SETUPTOOLS_VERSION = "47.1.0" -_PIP_VERSION = "19.2.3" +_PIP_VERSION = "20.1.1" _PROJECTS = [ - ("setuptools", _SETUPTOOLS_VERSION), - ("pip", _PIP_VERSION), + ("setuptools", _SETUPTOOLS_VERSION, "py3"), + ("pip", _PIP_VERSION, "py2.py3"), ] @@ -104,8 +104,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False, # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path additional_paths = [] - for project, version in _PROJECTS: - wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) + for project, version, py_tag in _PROJECTS: + wheel_name = "{}-{}-{}-none-any.whl".format(project, version, py_tag) whl = pkgutil.get_data( "ensurepip", "_bundled/{}".format(wheel_name), @@ -116,7 +116,7 @@ def _bootstrap(*, root=None, upgrade=False, user=False, additional_paths.append(os.path.join(tmpdir, wheel_name)) # Construct the arguments to be passed to the pip command - args = ["install", "--no-index", "--find-links", tmpdir] + args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] if root: args += ["--root", root] if upgrade: diff --git a/Lib/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl deleted file mode 100644 index 8118df8a..00000000 Binary files a/Lib/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/pip-20.1.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-20.1.1-py2.py3-none-any.whl new file mode 100644 index 00000000..ea1d0f7c Binary files /dev/null and b/Lib/ensurepip/_bundled/pip-20.1.1-py2.py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl deleted file mode 100644 index 82df6f63..00000000 Binary files a/Lib/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/setuptools-47.1.0-py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-47.1.0-py3-none-any.whl new file mode 100644 index 00000000..f87867ff Binary files /dev/null and b/Lib/ensurepip/_bundled/setuptools-47.1.0-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index 108d389d..14cc00e7 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -60,6 +60,7 @@ class _EnumDict(dict): self._member_names = [] self._last_values = [] self._ignore = [] + self._auto_called = False def __setitem__(self, key, value): """Changes anything not dundered or not a descriptor. @@ -77,6 +78,9 @@ class _EnumDict(dict): ): raise ValueError('_names_ are reserved for future Enum use') if key == '_generate_next_value_': + # check if members already defined as auto() + if self._auto_called: + raise TypeError("_generate_next_value_ must be defined before members") setattr(self, '_generate_next_value', value) elif key == '_ignore_': if isinstance(value, str): @@ -100,6 +104,7 @@ class _EnumDict(dict): # enum overwriting a descriptor? raise TypeError('%r already defined as: %r' % (key, self[key])) if isinstance(value, auto): + self._auto_called = True if value.value == _auto_null: value.value = self._generate_next_value(key, 1, len(self._member_names), self._last_values[:]) value = value.value diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index de7543e3..b43c6d3a 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -1,16 +1,33 @@ -What's New in IDLE 3.8.1 -Released on 2019-12-16? +What's New in IDLE 3.8.4 +Released on 2020-07-03? ====================================== +bpo-37765: Add keywords to module name completion list. Rewrite +Completions section of IDLE doc. + +bpo-41152: The encoding of ``stdin``, ``stdout`` and ``stderr`` in IDLE +is now always UTF-8. + +bpo-41144: Make Open Module open a special module such as os.path. + +bpo-40723: Make test_idle pass when run after import. +Patch by Florian Dahlitz. + + +What's New in IDLE 3.8.3 +Released on 2020-05-13 +====================================== + bpo-38689: IDLE will no longer freeze when inspect.signature fails when fetching a calltip. bpo-27115: For 'Go to Line', use a Query entry box subclass with IDLE standard behavior and improved error checking. -bpo-39885: Since clicking to get an IDLE context menu moves the -cursor, any text selection should be and now is cleared. +bpo-39885: When a context menu is invoked by right-clicking outside +of a selection, clear the selection and move the cursor. Cut and +Copy require that the click be within the selection. bpo-39852: Edit "Go to line" now clears any selection, preventing accidental deletion. It also updates Ln and Col on the status bar. @@ -19,6 +36,11 @@ bpo-39781: Selecting code context lines no longer causes a jump. bpo-39663: Add tests for pyparse find_good_parse_start(). + +What's New in IDLE 3.8.2 +Released on 2020-02-17 +====================================== + bpo-39600: Remove duplicate font names from configuration list. bpo-38792: Close a shell calltip if a :exc:`KeyboardInterrupt` @@ -35,6 +57,11 @@ bpo-32989: Add tests for editor newline_and_indent_event method. Remove unneeded arguments and dead code from pyparse find_good_parse_start method. + +What's New in IDLE 3.8.1 +Released on 2019-12-18 +====================================== + bpo-38943: Fix autocomplete windows not always appearing on some systems. Patch by Johnny Najera. diff --git a/Lib/idlelib/autocomplete.py b/Lib/idlelib/autocomplete.py index c623d45a..e1e9e173 100644 --- a/Lib/idlelib/autocomplete.py +++ b/Lib/idlelib/autocomplete.py @@ -4,6 +4,7 @@ Either on demand or after a user-selected delay after a key character, pop up a list of candidates. """ import __main__ +import keyword import os import string import sys @@ -171,10 +172,13 @@ class AutoComplete: (what, mode), {}) else: if mode == ATTRS: - if what == "": + if what == "": # Main module names. namespace = {**__main__.__builtins__.__dict__, **__main__.__dict__} bigl = eval("dir()", namespace) + kwds = (s for s in keyword.kwlist + if s not in {'True', 'False', 'None'}) + bigl.extend(kwds) bigl.sort() if "__all__" in bigl: smalll = sorted(eval("__all__", namespace)) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index b0f88b54..a178eaf9 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -499,15 +499,23 @@ class EditorWindow(object): rmenu = None def right_menu_event(self, event): - self.text.tag_remove("sel", "1.0", "end") - self.text.mark_set("insert", "@%d,%d" % (event.x, event.y)) + text = self.text + newdex = text.index(f'@{event.x},{event.y}') + try: + in_selection = (text.compare('sel.first', '<=', newdex) and + text.compare(newdex, '<=', 'sel.last')) + except TclError: + in_selection = False + if not in_selection: + text.tag_remove("sel", "1.0", "end") + text.mark_set("insert", newdex) if not self.rmenu: self.make_rmenu() rmenu = self.rmenu self.event = event iswin = sys.platform[:3] == 'win' if iswin: - self.text.config(cursor="arrow") + text.config(cursor="arrow") for item in self.rmenu_specs: try: @@ -520,7 +528,6 @@ class EditorWindow(object): state = getattr(self, verify_state)() rmenu.entryconfigure(label, state=state) - rmenu.tk_popup(event.x_root, event.y_root) if iswin: self.text.config(cursor="ibeam") diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index 424c6b50..81ce5100 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -4,7 +4,7 @@ - IDLE — Python 3.9.0a4 documentation + IDLE — Python 3.10.0a0 documentation @@ -17,7 +17,7 @@ @@ -71,7 +71,7 @@
  • - 3.9.0a4 Documentation » + 3.10.0a0 Documentation »
  • The Python Standard Library »
    • @@ -201,7 +201,7 @@ be undone.

    line visible. A request past the end of the file goes to the end. Clear any selection and update the line and column status.

    -
    Show Completions

    Open a scrollable list allowing selection of keywords and attributes. See +

    Show Completions

    Open a scrollable list allowing selection of existing names. See Completions in the Editing and navigation section below.

    Expand Word

    Expand a prefix you have typed to match a full word in the same window; @@ -465,38 +465,47 @@ are restricted to four spaces due to Tcl/Tk limitations.

    Completions¶

    -

    Completions are supplied for functions, classes, and attributes of classes, -both built-in and user-defined. Completions are also provided for -filenames.

    -

    The AutoCompleteWindow (ACW) will open after a predefined delay (default is -two seconds) after a ‘.’ or (in a string) an os.sep is typed. If after one -of those characters (plus zero or more other characters) a tab is typed -the ACW will open immediately if a possible continuation is found.

    -

    If there is only one possible completion for the characters entered, a -Tab will supply that completion without opening the ACW.

    -

    ‘Show Completions’ will force open a completions window, by default the -C-space will open a completions window. In an empty -string, this will contain the files in the current directory. On a -blank line, it will contain the built-in and user-defined functions and -classes in the current namespaces, plus any modules imported. If some -characters have been entered, the ACW will attempt to be more specific.

    -

    If a string of characters is typed, the ACW selection will jump to the -entry most closely matching those characters. Entering a tab will -cause the longest non-ambiguous match to be entered in the Editor window or -Shell. Two tab in a row will supply the current ACW selection, as -will return or a double click. Cursor keys, Page Up/Down, mouse selection, -and the scroll wheel all operate on the ACW.

    -

    “Hidden” attributes can be accessed by typing the beginning of hidden -name after a ‘.’, e.g. ‘_’. This allows access to modules with -__all__ set, or to class-private attributes.

    -

    Completions and the ‘Expand Word’ facility can save a lot of typing!

    -

    Completions are currently limited to those in the namespaces. Names in -an Editor window which are not via __main__ and sys.modules will -not be found. Run the module once with your imports to correct this situation. -Note that IDLE itself places quite a few modules in sys.modules, so -much can be found by default, e.g. the re module.

    -

    If you don’t like the ACW popping up unbidden, simply make the delay -longer or disable the extension.

    +

    Completions are supplied, when requested and available, for module +names, attributes of classes or functions, or filenames. Each request +method displays a completion box with existing names. (See tab +completions below for an exception.) For any box, change the name +being completed and the item highlighted in the box by +typing and deleting characters; by hitting Up, Down, +PageUp, PageDown, Home, and End keys; +and by a single click within the box. Close the box with Escape, +Enter, and double Tab keys or clicks outside the box. +A double click within the box selects and closes.

    +

    One way to open a box is to type a key character and wait for a +predefined interval. This defaults to 2 seconds; customize it +in the settings dialog. (To prevent auto popups, set the delay to a +large number of milliseconds, such as 100000000.) For imported module +names or class or function attributes, type ‘.’. +For filenames in the root directory, type os.sep or +data:os.altsep immediately after an opening quote. (On Windows, +one can specify a drive first.) Move into subdirectories by typing a +directory name and a separator.

    +

    Instead of waiting, or after a box is closed. open a completion box +immediately with Show Completions on the Edit menu. The default hot +key is C-space. If one types a prefix for the desired name +before opening the box, the first match is displayed. +The result is the same as if one enters a prefix +after the box is displayed. Show Completions after a quote completes +filenames in the current directory instead of a root directory.

    +

    Hitting Tab after a prefix usually has the same effect as Show +Completions. (With no prefix, it indents.) However, if there is only +one match to the prefix, that match is immediately added to the editor +text without opening a box.

    +

    Invoking ‘Show Completions’, or hitting Tab after a prefix, +outside of a string and without a preceding ‘.’ opens a box with +keywords, builtin names, and available module-level names.

    +

    When editing code in an editor (as oppose to Shell), increase the +available module-level names by running your code +and not restarting the Shell thereafter. This is especially useful +after adding imports at the top of a file. This also increases +possible attribute completions.

    +

    Completion boxes intially exclude names beginning with ‘_’ or, for +modules, not included in ‘__all__’. The hidden names can be accessed +by typing ‘_’ after ‘.’, either before or after the box is opened.

    Calltips¶

    @@ -935,7 +944,7 @@ also used for testing.

  • - 3.9.0a4 Documentation » + 3.10.0a0 Documentation »
  • The Python Standard Library »
    • @@ -966,7 +975,7 @@ also used for testing.



    - Last updated on Mar 07, 2020. + Last updated on Jul 08, 2020. Found a bug?
    diff --git a/Lib/idlelib/idle_test/test_autocomplete.py b/Lib/idlelib/idle_test/test_autocomplete.py index 2c478cd5..9c113bd8 100644 --- a/Lib/idlelib/idle_test/test_autocomplete.py +++ b/Lib/idlelib/idle_test/test_autocomplete.py @@ -227,7 +227,7 @@ class AutoCompleteTest(unittest.TestCase): acp = self.autocomplete small, large = acp.fetch_completions( '', ac.ATTRS) - if __main__.__file__ != ac.__file__: + if hasattr(__main__, '__file__') and __main__.__file__ != ac.__file__: self.assertNotIn('AutoComplete', small) # See issue 36405. # Test attributes @@ -240,8 +240,11 @@ class AutoCompleteTest(unittest.TestCase): with patch.dict('__main__.__dict__', {'__all__': ['a', 'b']}): s, b = acp.fetch_completions('', ac.ATTRS) self.assertEqual(s, ['a', 'b']) - self.assertIn('__name__', b) # From __main__.__dict__ - self.assertIn('sum', b) # From __main__.__builtins__.__dict__ + self.assertIn('__name__', b) # From __main__.__dict__. + self.assertIn('sum', b) # From __main__.__builtins__.__dict__. + self.assertIn('nonlocal', b) # From keyword.kwlist. + pos = b.index('False') # Test False not included twice. + self.assertNotEqual(b[pos+1], 'False') # Test attributes with name entity. mock = Mock() diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index 91e8ef89..443dcf02 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -5,6 +5,7 @@ import unittest from collections import namedtuple from test.support import requires from tkinter import Tk +from idlelib.idle_test.mock_idle import Func Editor = editor.EditorWindow @@ -92,6 +93,12 @@ class TestGetLineIndent(unittest.TestCase): ) +def insert(text, string): + text.delete('1.0', 'end') + text.insert('end', string) + text.update() # Force update for colorizer to finish. + + class IndentAndNewlineTest(unittest.TestCase): @classmethod @@ -113,13 +120,6 @@ class IndentAndNewlineTest(unittest.TestCase): cls.root.destroy() del cls.root - def insert(self, text): - t = self.window.text - t.delete('1.0', 'end') - t.insert('end', text) - # Force update for colorizer to finish. - t.update() - def test_indent_and_newline_event(self): eq = self.assertEqual w = self.window @@ -170,13 +170,13 @@ class IndentAndNewlineTest(unittest.TestCase): w.prompt_last_line = '' for test in tests: with self.subTest(label=test.label): - self.insert(test.text) + insert(text, test.text) text.mark_set('insert', test.mark) nl(event=None) eq(get('1.0', 'end'), test.expected) # Selected text. - self.insert(' def f1(self, a, b):\n return a + b') + insert(text, ' def f1(self, a, b):\n return a + b') text.tag_add('sel', '1.17', '1.end') nl(None) # Deletes selected text before adding new line. @@ -184,11 +184,37 @@ class IndentAndNewlineTest(unittest.TestCase): # Preserves the whitespace in shell prompt. w.prompt_last_line = '>>> ' - self.insert('>>> \t\ta =') + insert(text, '>>> \t\ta =') text.mark_set('insert', '1.5') nl(None) eq(get('1.0', 'end'), '>>> \na =\n') +class RMenuTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.window = Editor(root=cls.root) + + @classmethod + def tearDownClass(cls): + cls.window._close() + del cls.window + cls.root.update_idletasks() + for id in cls.root.tk.call('after', 'info'): + cls.root.after_cancel(id) + cls.root.destroy() + del cls.root + + class DummyRMenu: + def tk_popup(x, y): pass + + def test_rclick(self): + pass + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_outwin.py b/Lib/idlelib/idle_test/test_outwin.py index cd099ecd..e347bfca 100644 --- a/Lib/idlelib/idle_test/test_outwin.py +++ b/Lib/idlelib/idle_test/test_outwin.py @@ -58,11 +58,6 @@ class OutputWindowTest(unittest.TestCase): get = self.text.get write = self.window.write - # Test bytes. - b = b'Test bytes.' - eq(write(b), len(b)) - eq(get('1.0', '1.end'), b.decode()) - # No new line - insert stays on same line. delete('1.0', 'end') test_text = 'test text' diff --git a/Lib/idlelib/idle_test/test_query.py b/Lib/idlelib/idle_test/test_query.py index 6d026cb5..e9688626 100644 --- a/Lib/idlelib/idle_test/test_query.py +++ b/Lib/idlelib/idle_test/test_query.py @@ -136,6 +136,9 @@ class ModuleNameTest(unittest.TestCase): dialog = self.Dummy_ModuleName('idlelib') self.assertTrue(dialog.entry_ok().endswith('__init__.py')) self.assertEqual(dialog.entry_error['text'], '') + dialog = self.Dummy_ModuleName('os.path') + self.assertTrue(dialog.entry_ok().endswith('path.py')) + self.assertEqual(dialog.entry_error['text'], '') class GotoTest(unittest.TestCase): diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index 4b2833b8..7641d866 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -1,10 +1,8 @@ -import codecs -from codecs import BOM_UTF8 import os -import re import shlex import sys import tempfile +import tokenize import tkinter.filedialog as tkFileDialog import tkinter.messagebox as tkMessageBox @@ -13,95 +11,12 @@ from tkinter.simpledialog import askstring import idlelib from idlelib.config import idleConf -if idlelib.testing: # Set True by test.test_idle to avoid setlocale. - encoding = 'utf-8' - errors = 'surrogateescape' +encoding = 'utf-8' +if sys.platform == 'win32': + errors = 'surrogatepass' else: - # Try setting the locale, so that we can find out - # what encoding to use - try: - import locale - locale.setlocale(locale.LC_CTYPE, "") - except (ImportError, locale.Error): - pass - - if sys.platform == 'win32': - encoding = 'utf-8' - errors = 'surrogateescape' - else: - try: - # Different things can fail here: the locale module may not be - # loaded, it may not offer nl_langinfo, or CODESET, or the - # resulting codeset may be unknown to Python. We ignore all - # these problems, falling back to ASCII - locale_encoding = locale.nl_langinfo(locale.CODESET) - if locale_encoding: - codecs.lookup(locale_encoding) - except (NameError, AttributeError, LookupError): - # Try getdefaultlocale: it parses environment variables, - # which may give a clue. Unfortunately, getdefaultlocale has - # bugs that can cause ValueError. - try: - locale_encoding = locale.getdefaultlocale()[1] - if locale_encoding: - codecs.lookup(locale_encoding) - except (ValueError, LookupError): - pass - - if locale_encoding: - encoding = locale_encoding.lower() - errors = 'strict' - else: - # POSIX locale or macOS - encoding = 'ascii' - errors = 'surrogateescape' - # Encoding is used in multiple files; locale_encoding nowhere. - # The only use of 'encoding' below is in _decode as initial value - # of deprecated block asking user for encoding. - # Perhaps use elsewhere should be reviewed. - -coding_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) -blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) - -def coding_spec(data): - """Return the encoding declaration according to PEP 263. - - When checking encoded data, only the first two lines should be passed - in to avoid a UnicodeDecodeError if the rest of the data is not unicode. - The first two lines would contain the encoding specification. - - Raise a LookupError if the encoding is declared but unknown. - """ - if isinstance(data, bytes): - # This encoding might be wrong. However, the coding - # spec must be ASCII-only, so any non-ASCII characters - # around here will be ignored. Decoding to Latin-1 should - # never fail (except for memory outage) - lines = data.decode('iso-8859-1') - else: - lines = data - # consider only the first two lines - if '\n' in lines: - lst = lines.split('\n', 2)[:2] - elif '\r' in lines: - lst = lines.split('\r', 2)[:2] - else: - lst = [lines] - for line in lst: - match = coding_re.match(line) - if match is not None: - break - if not blank_re.match(line): - return None - else: - return None - name = match.group(1) - try: - codecs.lookup(name) - except LookupError: - # The standard encoding error does not indicate the encoding - raise LookupError("Unknown encoding: "+name) - return name + errors = 'surrogateescape' + class IOBinding: @@ -118,7 +33,7 @@ class IOBinding: self.save_as) self.__id_savecopy = self.text.bind("<>", self.save_a_copy) - self.fileencoding = None + self.fileencoding = 'utf-8' self.__id_print = self.text.bind("<>", self.print_window) def close(self): @@ -205,34 +120,44 @@ class IOBinding: self.text.focus_set() return "break" - eol = r"(\r\n)|\n|\r" # \r\n (Windows), \n (UNIX), or \r (Mac) - eol_re = re.compile(eol) eol_convention = os.linesep # default def loadfile(self, filename): try: - # open the file in binary mode so that we can handle - # end-of-line convention ourselves. - with open(filename, 'rb') as f: - two_lines = f.readline() + f.readline() - f.seek(0) - bytes = f.read() - except OSError as msg: - tkMessageBox.showerror("I/O Error", str(msg), parent=self.text) + try: + with tokenize.open(filename) as f: + chars = f.read() + fileencoding = f.encoding + eol_convention = f.newlines + converted = False + except (UnicodeDecodeError, SyntaxError): + # Wait for the editor window to appear + self.editwin.text.update() + enc = askstring( + "Specify file encoding", + "The file's encoding is invalid for Python 3.x.\n" + "IDLE will convert it to UTF-8.\n" + "What is the current encoding of the file?", + initialvalue='utf-8', + parent=self.editwin.text) + with open(filename, encoding=enc) as f: + chars = f.read() + fileencoding = f.encoding + eol_convention = f.newlines + converted = True + except OSError as err: + tkMessageBox.showerror("I/O Error", str(err), parent=self.text) return False - chars, converted = self._decode(two_lines, bytes) - if chars is None: + except UnicodeDecodeError: tkMessageBox.showerror("Decoding Error", "File %s\nFailed to Decode" % filename, parent=self.text) return False - # We now convert all end-of-lines to '\n's - firsteol = self.eol_re.search(chars) - if firsteol: - self.eol_convention = firsteol.group(0) - chars = self.eol_re.sub(r"\n", chars) + self.text.delete("1.0", "end") self.set_filename(None) + self.fileencoding = fileencoding + self.eol_convention = eol_convention self.text.insert("1.0", chars) self.reset_undo() self.set_filename(filename) @@ -245,74 +170,6 @@ class IOBinding: self.updaterecentfileslist(filename) return True - def _decode(self, two_lines, bytes): - "Create a Unicode string." - chars = None - # Check presence of a UTF-8 signature first - if bytes.startswith(BOM_UTF8): - try: - chars = bytes[3:].decode("utf-8") - except UnicodeDecodeError: - # has UTF-8 signature, but fails to decode... - return None, False - else: - # Indicates that this file originally had a BOM - self.fileencoding = 'BOM' - return chars, False - # Next look for coding specification - try: - enc = coding_spec(two_lines) - except LookupError as name: - tkMessageBox.showerror( - title="Error loading the file", - message="The encoding '%s' is not known to this Python "\ - "installation. The file may not display correctly" % name, - parent = self.text) - enc = None - except UnicodeDecodeError: - return None, False - if enc: - try: - chars = str(bytes, enc) - self.fileencoding = enc - return chars, False - except UnicodeDecodeError: - pass - # Try ascii: - try: - chars = str(bytes, 'ascii') - self.fileencoding = None - return chars, False - except UnicodeDecodeError: - pass - # Try utf-8: - try: - chars = str(bytes, 'utf-8') - self.fileencoding = 'utf-8' - return chars, False - except UnicodeDecodeError: - pass - # Finally, try the locale's encoding. This is deprecated; - # the user should declare a non-ASCII encoding - try: - # Wait for the editor window to appear - self.editwin.text.update() - enc = askstring( - "Specify file encoding", - "The file's encoding is invalid for Python 3.x.\n" - "IDLE will convert it to UTF-8.\n" - "What is the current encoding of the file?", - initialvalue = encoding, - parent = self.editwin.text) - - if enc: - chars = str(bytes, enc) - self.fileencoding = None - return chars, True - except (UnicodeDecodeError, LookupError): - pass - return None, False # None on failure - def maybesave(self): if self.get_saved(): return "yes" @@ -400,38 +257,30 @@ class IOBinding: # text to us. Don't try to guess further. return chars # Preserve a BOM that might have been present on opening - if self.fileencoding == 'BOM': - return BOM_UTF8 + chars.encode("utf-8") + if self.fileencoding == 'utf-8-sig': + return chars.encode('utf-8-sig') # See whether there is anything non-ASCII in it. # If not, no need to figure out the encoding. try: return chars.encode('ascii') - except UnicodeError: + except UnicodeEncodeError: pass # Check if there is an encoding declared try: - # a string, let coding_spec slice it to the first two lines - enc = coding_spec(chars) - failed = None - except LookupError as msg: - failed = msg - enc = None - else: - if not enc: - # PEP 3120: default source encoding is UTF-8 - enc = 'utf-8' - if enc: - try: - return chars.encode(enc) - except UnicodeError: - failed = "Invalid encoding '%s'" % enc + encoded = chars.encode('ascii', 'replace') + enc, _ = tokenize.detect_encoding(io.BytesIO(encoded).readline) + return chars.encode(enc) + except SyntaxError as err: + failed = str(err) + except UnicodeEncodeError: + failed = "Invalid encoding '%s'" % enc tkMessageBox.showerror( "I/O Error", "%s.\nSaving as UTF-8" % failed, - parent = self.text) + parent=self.text) # Fallback: save as UTF-8, with BOM - ignoring the incorrect # declared encoding - return BOM_UTF8 + chars.encode("utf-8") + return chars.encode('utf-8-sig') def print_window(self, event): confirm = tkMessageBox.askokcancel( diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index 90272b6f..5ab08bba 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -6,7 +6,6 @@ import re from tkinter import messagebox from idlelib.editor import EditorWindow -from idlelib import iomenu file_line_pats = [ @@ -110,8 +109,7 @@ class OutputWindow(EditorWindow): Return: Length of text inserted. """ - if isinstance(s, bytes): - s = s.decode(iomenu.encoding, "replace") + assert isinstance(s, str) self.text.insert(mark, s, tags) self.text.see(mark) self.text.update() diff --git a/Lib/idlelib/query.py b/Lib/idlelib/query.py index 2a88530b..015fc7ad 100644 --- a/Lib/idlelib/query.py +++ b/Lib/idlelib/query.py @@ -19,7 +19,7 @@ Subclass HelpSource gets menu item and path for additions to Help menu. # HelpSource was extracted from configHelpSourceEdit.py (temporarily # config_help.py), with darwin code moved from ok to path_ok. -import importlib +import importlib.util, importlib.abc import os import shlex from sys import executable, platform # Platform is set for one test. @@ -57,7 +57,8 @@ class Query(Toplevel): self.withdraw() # Hide while configuring, especially geometry. self.title(title) self.transient(parent) - self.grab_set() + if not _utest: # Otherwise fail when directly run unittest. + self.grab_set() windowingsystem = self.tk.call('tk', 'windowingsystem') if windowingsystem == 'aqua': @@ -209,17 +210,23 @@ class ModuleName(Query): self.showerror(str(msg)) return None if spec is None: - self.showerror("module not found") + self.showerror("module not found.") return None if not isinstance(spec.loader, importlib.abc.SourceLoader): - self.showerror("not a source-based module") + self.showerror("not a source-based module.") return None try: file_path = spec.loader.get_filename(name) except AttributeError: - self.showerror("loader does not support get_filename", - parent=self) + self.showerror("loader does not support get_filename.") return None + except ImportError: + # Some special modules require this (e.g. os.path) + try: + file_path = spec.loader.get_filename() + except TypeError: + self.showerror("loader failed to get filename.") + return None return file_path @@ -375,7 +382,7 @@ class CustomRun(Query): return cli_args def entry_ok(self): - "Return apparently valid (cli_args, restart) or None" + "Return apparently valid (cli_args, restart) or None." cli_args = self.cli_args_ok() restart = self.restartvar.get() return None if cli_args is None else (cli_args, restart) diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py index 6229be4e..5947268f 100644 --- a/Lib/idlelib/tree.py +++ b/Lib/idlelib/tree.py @@ -38,7 +38,7 @@ def listicons(icondir=ICONDIR): """Utility to display the available icons.""" root = Tk() import glob - list = glob.glob(os.path.join(icondir, "*.gif")) + list = glob.glob(os.path.join(glob.escape(icondir), "*.gif")) list.sort() images = [] row = column = 0 diff --git a/Lib/imghdr.py b/Lib/imghdr.py index 76e8abb2..6e01fd85 100644 --- a/Lib/imghdr.py +++ b/Lib/imghdr.py @@ -152,7 +152,7 @@ def testall(list, recursive, toplevel): if recursive or toplevel: print('recursing down:') import glob - names = glob.glob(os.path.join(filename, '*')) + names = glob.glob(os.path.join(glob.escape(filename), '*')) testall(names, recursive, 0) else: print('*** directory (use -r) ***') diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 873c7644..a3a04f7f 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1370,7 +1370,7 @@ class IPv4Interface(IPv4Address): return False def __hash__(self): - return self._ip ^ self._prefixlen ^ int(self.network.network_address) + return hash((self._ip, self._prefixlen, int(self.network.network_address))) __reduce__ = _IPAddressBase.__reduce__ @@ -2017,7 +2017,7 @@ class IPv6Interface(IPv6Address): return False def __hash__(self): - return self._ip ^ self._prefixlen ^ int(self.network.network_address) + return hash((self._ip, self._prefixlen, int(self.network.network_address))) __reduce__ = _IPAddressBase.__reduce__ diff --git a/Lib/linecache.py b/Lib/linecache.py index 3afcce1f..c87e1807 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -73,10 +73,10 @@ def checkcache(filename=None): try: stat = os.stat(fullname) except OSError: - del cache[filename] + cache.pop(filename, None) continue if size != stat.st_size or mtime != stat.st_mtime: - del cache[filename] + cache.pop(filename, None) def updatecache(filename, module_globals=None): @@ -86,7 +86,7 @@ def updatecache(filename, module_globals=None): if filename in cache: if len(cache[filename]) != 1: - del cache[filename] + cache.pop(filename, None) if not filename or (filename.startswith('<') and filename.endswith('>')): return [] diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index f33b658f..954bb0a7 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -372,7 +372,7 @@ def init(files=None): def read_mime_types(file): try: - f = open(file) + f = open(file, encoding='utf-8') except OSError: return None with f: @@ -447,6 +447,7 @@ def _default_mime_types(): '.dvi' : 'application/x-dvi', '.gtar' : 'application/x-gtar', '.hdf' : 'application/x-hdf', + '.h5' : 'application/x-hdf5', '.latex' : 'application/x-latex', '.mif' : 'application/x-mif', '.cdf' : 'application/x-netcdf', diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 5f8e0f0c..8d0525d5 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -257,10 +257,11 @@ class DefaultContext(BaseContext): if sys.platform == 'win32': return ['spawn'] else: + methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn'] if reduction.HAVE_SEND_HANDLE: - return ['fork', 'spawn', 'forkserver'] - else: - return ['fork', 'spawn'] + methods.append('forkserver') + return methods + # # Context types for fixed start method diff --git a/Lib/pdb.py b/Lib/pdb.py index bf503f1e..08102352 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -79,6 +79,7 @@ import glob import pprint import signal import inspect +import tokenize import traceback import linecache @@ -93,7 +94,7 @@ __all__ = ["run", "pm", "Pdb", "runeval", "runctx", "runcall", "set_trace", def find_function(funcname, filename): cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname)) try: - fp = open(filename) + fp = tokenize.open(filename) except OSError: return None # consumer of this info expects the first line to be 1 @@ -473,7 +474,7 @@ class Pdb(bdb.Bdb, cmd.Cmd): except Exception: ret = [] # Then, try to complete file names as well. - globs = glob.glob(text + '*') + globs = glob.glob(glob.escape(text) + '*') for fn in globs: if os.path.isdir(fn): ret.append(fn + '/') diff --git a/Lib/pickle.py b/Lib/pickle.py index 515cb8a0..af50a9b0 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -339,7 +339,7 @@ def whichmodule(obj, name): return module_name # Protect the iteration by using a list copy of sys.modules against dynamic # modules that trigger imports of other modules upon calls to getattr. - for module_name, module in list(sys.modules.items()): + for module_name, module in sys.modules.copy().items(): if module_name == '__main__' or module is None: continue try: diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 06f0e781..d4473aaa 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Wed May 13 19:29:27 2020 +# Autogenerated by Sphinx on Mon Jul 13 13:47:56 2020 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -4284,7 +4284,8 @@ topics = {'assert': 'The "assert" statement\n' ' the current environment).\n' '\n' 'retval\n' - 'Print the return value for the last return of a function.\n' + '\n' + ' Print the return value for the last return of a function.\n' '\n' '-[ Footnotes ]-\n' '\n' @@ -6037,8 +6038,8 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'A non-normative HTML file listing all valid identifier ' 'characters for\n' - 'Unicode 4.1 can be found at https://www.dcl.hpi.uni-\n' - 'potsdam.de/home/loewis/table-3131.html.\n' + 'Unicode 4.1 can be found at\n' + 'https://www.unicode.org/Public/13.0.0/ucd/DerivedCoreProperties.txt\n' '\n' '\n' 'Keywords\n' diff --git a/Lib/site.py b/Lib/site.py index a065ab0b..9fa21cca 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -444,9 +444,9 @@ def enablerlcompleter(): def write_history(): try: readline.write_history_file(history) - except (FileNotFoundError, PermissionError): - # home directory does not exist or is not writable - # https://bugs.python.org/issue19891 + except OSError: + # bpo-19891, bpo-41193: Home directory does not exist + # or is not writable, or the filesystem is read-only. pass atexit.register(write_history) diff --git a/Lib/sndhdr.py b/Lib/sndhdr.py index 59435313..96595c69 100644 --- a/Lib/sndhdr.py +++ b/Lib/sndhdr.py @@ -241,7 +241,7 @@ def testall(list, recursive, toplevel): if recursive or toplevel: print('recursing down:') import glob - names = glob.glob(os.path.join(filename, '*')) + names = glob.glob(os.path.join(glob.escape(filename), '*')) testall(names, recursive, 0) else: print('*** directory (use -r) ***') diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py index 9501f535..c11c82e1 100644 --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -1,8 +1,7 @@ -#-*- coding: iso-8859-1 -*- # pysqlite2/test/userfunctions.py: tests for user-defined functions and # aggregates. # -# Copyright (C) 2005-2007 Gerhard Häring +# Copyright (C) 2005-2007 Gerhard Häring # # This file is part of pysqlite. # @@ -158,6 +157,7 @@ class FunctionTests(unittest.TestCase): self.con.create_function("isblob", 1, func_isblob) self.con.create_function("islonglong", 1, func_islonglong) self.con.create_function("spam", -1, func) + self.con.execute("create table test(t text)") def tearDown(self): self.con.close() @@ -276,18 +276,36 @@ class FunctionTests(unittest.TestCase): val = cur.fetchone()[0] self.assertEqual(val, 2) + # Regarding deterministic functions: + # + # Between 3.8.3 and 3.15.0, deterministic functions were only used to + # optimize inner loops, so for those versions we can only test if the + # sqlite machinery has factored out a call or not. From 3.15.0 and onward, + # deterministic functions were permitted in WHERE clauses of partial + # indices, which allows testing based on syntax, iso. the query optimizer. + @unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "Requires SQLite 3.8.3 or higher") def CheckFuncNonDeterministic(self): mock = unittest.mock.Mock(return_value=None) - self.con.create_function("deterministic", 0, mock, deterministic=False) - self.con.execute("select deterministic() = deterministic()") - self.assertEqual(mock.call_count, 2) - - @unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "deterministic parameter not supported") + self.con.create_function("nondeterministic", 0, mock, deterministic=False) + if sqlite.sqlite_version_info < (3, 15, 0): + self.con.execute("select nondeterministic() = nondeterministic()") + self.assertEqual(mock.call_count, 2) + else: + with self.assertRaises(sqlite.OperationalError): + self.con.execute("create index t on test(t) where nondeterministic() is not null") + + @unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "Requires SQLite 3.8.3 or higher") def CheckFuncDeterministic(self): mock = unittest.mock.Mock(return_value=None) self.con.create_function("deterministic", 0, mock, deterministic=True) - self.con.execute("select deterministic() = deterministic()") - self.assertEqual(mock.call_count, 1) + if sqlite.sqlite_version_info < (3, 15, 0): + self.con.execute("select deterministic() = deterministic()") + self.assertEqual(mock.call_count, 1) + else: + try: + self.con.execute("create index t on test(t) where deterministic() is not null") + except sqlite.OperationalError: + self.fail("Unexpected failure while creating partial index") @unittest.skipIf(sqlite.sqlite_version_info >= (3, 8, 3), "SQLite < 3.8.3 needed") def CheckFuncDeterministicNotSupported(self): diff --git a/Lib/statistics.py b/Lib/statistics.py index 1e95c0b6..c5c6e47f 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -682,8 +682,10 @@ def _ss(data, c=None): calculated from ``c`` as given. Use the second case with care, as it can lead to garbage results. """ - if c is None: - c = mean(data) + if c is not None: + T, total, count = _sum((x-c)**2 for x in data) + return (T, total) + c = mean(data) T, total, count = _sum((x-c)**2 for x in data) # The following sum should mathematically equal zero, but due to rounding # error may not. diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index ff58481f..8626aa37 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -31,7 +31,7 @@ from test import support # Skip tests if _multiprocessing wasn't built. _multiprocessing = test.support.import_module('_multiprocessing') # Skip tests if sem_open implementation is broken. -test.support.import_module('multiprocessing.synchronize') +support.skip_if_broken_multiprocessing_synchronize() import threading import multiprocessing.connection @@ -4222,7 +4222,7 @@ class _TestImportStar(unittest.TestCase): def get_module_names(self): import glob folder = os.path.dirname(multiprocessing.__file__) - pattern = os.path.join(folder, '*.py') + pattern = os.path.join(glob.escape(folder), '*.py') files = glob.glob(pattern) modules = [os.path.splitext(os.path.split(f)[1])[0] for f in files] modules = ['multiprocessing.' + m for m in modules] @@ -5007,7 +5007,9 @@ class TestStartMethod(unittest.TestCase): self.assertEqual(methods, ['spawn']) else: self.assertTrue(methods == ['fork', 'spawn'] or - methods == ['fork', 'spawn', 'forkserver']) + methods == ['spawn', 'fork'] or + methods == ['fork', 'spawn', 'forkserver'] or + methods == ['spawn', 'fork', 'forkserver']) def test_preload_resources(self): if multiprocessing.get_start_method() != 'forkserver': diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index b90c4b8f..ee6fc933 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -44,28 +44,6 @@ class TestHook: raise self.exc_type("saw event " + event) -class TestFinalizeHook: - """Used in the test_finalize_hooks function to ensure that hooks - are correctly cleaned up, that they are notified about the cleanup, - and are unable to prevent it. - """ - - def __init__(self): - print("Created", id(self), file=sys.stdout, flush=True) - - def __call__(self, event, args): - # Avoid recursion when we call id() below - if event == "builtins.id": - return - - print(event, id(self), file=sys.stdout, flush=True) - - if event == "cpython._PySys_ClearAuditHooks": - raise RuntimeError("Should be ignored") - elif event == "cpython.PyInterpreterState_Clear": - raise RuntimeError("Should be ignored") - - # Simple helpers, since we are not in unittest here def assertEqual(x, y): if x != y: @@ -128,10 +106,6 @@ def test_block_add_hook_baseexception(): pass -def test_finalize_hooks(): - sys.addaudithook(TestFinalizeHook()) - - def test_pickle(): import pickle @@ -350,9 +324,9 @@ def test_socket(): if __name__ == "__main__": - from test.libregrtest.setup import suppress_msvcrt_asserts + from test.support import suppress_msvcrt_asserts - suppress_msvcrt_asserts(False) + suppress_msvcrt_asserts() test = sys.argv[1] globals()[test]() diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 95b4856c..adf31cc9 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -602,7 +602,7 @@ class Regrtest: def cleanup(self): import glob - path = os.path.join(self.tmp_dir, 'test_python_*') + path = os.path.join(glob.escape(self.tmp_dir), 'test_python_*') print("Cleanup %s directory" % self.tmp_dir) for name in glob.glob(path): if os.path.isdir(name): diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 1e0eac3b..2b0bdf99 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -67,7 +67,7 @@ def setup_tests(ns): if ns.threshold is not None: gc.set_threshold(ns.threshold) - suppress_msvcrt_asserts(ns.verbose and ns.verbose >= 2) + support.suppress_msvcrt_asserts(ns.verbose and ns.verbose >= 2) support.use_resources = ns.use_resources @@ -78,31 +78,6 @@ def setup_tests(ns): sys.addaudithook(_test_audit_hook) -def suppress_msvcrt_asserts(verbose): - try: - import msvcrt - except ImportError: - return - - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - return - - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - if verbose: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - else: - msvcrt.CrtSetReportMode(m, 0) - - - def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error handler) to avoid UnicodeEncodeError when printing a traceback""" diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 7c8383f3..9401043d 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -21,19 +21,25 @@ try: except ImportError: _testbuffer = None -try: - import numpy as np -except ImportError: - np = None - from test import support from test.support import ( TestFailed, TESTFN, run_with_locale, no_tracing, _2G, _4G, bigmemtest, reap_threads, forget, + save_restore_warnings_filters ) from pickle import bytes_types + +# bpo-41003: Save/restore warnings filters to leave them unchanged. +# Ignore filters installed by numpy. +try: + with save_restore_warnings_filters(): + import numpy as np +except ImportError: + np = None + + requires_32b = unittest.skipUnless(sys.maxsize < 2**32, "test is only meaningful on 32-bit builds") diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 400eebc5..fb09e062 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -586,25 +586,25 @@ def _requires_unix_version(sysname, min_version): For example, @_requires_unix_version('FreeBSD', (7, 2)) raises SkipTest if the FreeBSD version is less than 7.2. """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kw): - if platform.system() == sysname: - version_txt = platform.release().split('-', 1)[0] - try: - version = tuple(map(int, version_txt.split('.'))) - except ValueError: - pass - else: - if version < min_version: - min_version_txt = '.'.join(map(str, min_version)) - raise unittest.SkipTest( - "%s version %s or higher required, not %s" - % (sysname, min_version_txt, version_txt)) - return func(*args, **kw) - wrapper.min_version = min_version - return wrapper - return decorator + import platform + min_version_txt = '.'.join(map(str, min_version)) + version_txt = platform.release().split('-', 1)[0] + if platform.system() == sysname: + try: + version = tuple(map(int, version_txt.split('.'))) + except ValueError: + skip = False + else: + skip = version < min_version + else: + skip = False + + return unittest.skipIf( + skip, + f"{sysname} version {min_version_txt} or higher required, not " + f"{version_txt}" + ) + def requires_freebsd_version(*min_version): """Decorator raising SkipTest if the OS is FreeBSD and the FreeBSD version is @@ -2625,7 +2625,7 @@ class PythonSymlink: dll, os.path.join(dest_dir, os.path.basename(dll)) )) - for runtime in glob.glob(os.path.join(src_dir, "vcruntime*.dll")): + for runtime in glob.glob(os.path.join(glob.escape(src_dir), "vcruntime*.dll")): self._also_link.append(( runtime, os.path.join(dest_dir, os.path.basename(runtime)) @@ -2825,6 +2825,27 @@ def check__all__(test_case, module, name_of_module=None, extra=(), test_case.assertCountEqual(module.__all__, expected) +def suppress_msvcrt_asserts(verbose=False): + try: + import msvcrt + except ImportError: + return + + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS + | msvcrt.SEM_NOALIGNMENTFAULTEXCEPT + | msvcrt.SEM_NOGPFAULTERRORBOX + | msvcrt.SEM_NOOPENFILEERRORBOX) + + # CrtSetReportMode() is only available in debug build + if hasattr(msvcrt, 'CrtSetReportMode'): + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + if verbose: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + else: + msvcrt.CrtSetReportMode(m, 0) + + class SuppressCrashReport: """Try to prevent a crash report from popping up. @@ -2836,7 +2857,7 @@ class SuppressCrashReport: def __enter__(self): """On Windows, disable Windows Error Reporting dialogs using - SetErrorMode. + SetErrorMode() and CrtSetReportMode(). On UNIX, try to save the previous core file size limit, then set soft limit to 0. @@ -2845,21 +2866,18 @@ class SuppressCrashReport: # see http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621.aspx # GetErrorMode is not available on Windows XP and Windows Server 2003, # but SetErrorMode returns the previous value, so we can use that - import ctypes - self._k32 = ctypes.windll.kernel32 - SEM_NOGPFAULTERRORBOX = 0x02 - self.old_value = self._k32.SetErrorMode(SEM_NOGPFAULTERRORBOX) - self._k32.SetErrorMode(self.old_value | SEM_NOGPFAULTERRORBOX) - - # Suppress assert dialogs in debug builds - # (see http://bugs.python.org/issue23314) try: import msvcrt - msvcrt.CrtSetReportMode - except (AttributeError, ImportError): - # no msvcrt or a release build - pass - else: + except ImportError: + return + + self.old_value = msvcrt.SetErrorMode(msvcrt.SEM_NOGPFAULTERRORBOX) + + msvcrt.SetErrorMode(self.old_value | msvcrt.SEM_NOGPFAULTERRORBOX) + + # bpo-23314: Suppress assert dialogs in debug builds. + # CrtSetReportMode() is only available in debug build. + if hasattr(msvcrt, 'CrtSetReportMode'): self.old_modes = {} for report_type in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, @@ -2905,10 +2923,10 @@ class SuppressCrashReport: return if sys.platform.startswith('win'): - self._k32.SetErrorMode(self.old_value) + import msvcrt + msvcrt.SetErrorMode(self.old_value) if self.old_modes: - import msvcrt for report_type, (old_mode, old_file) in self.old_modes.items(): msvcrt.CrtSetReportMode(report_type, old_mode) msvcrt.CrtSetReportFile(report_type, old_file) @@ -3350,3 +3368,36 @@ class catch_threading_exception: del self.exc_value del self.exc_traceback del self.thread + + +@contextlib.contextmanager +def save_restore_warnings_filters(): + old_filters = warnings.filters[:] + try: + yield + finally: + warnings.filters[:] = old_filters + + +def skip_if_broken_multiprocessing_synchronize(): + """ + Skip tests if the multiprocessing.synchronize module is missing, if there + is no available semaphore implementation, or if creating a lock raises an + OSError (on Linux only). + """ + + # Skip tests if the _multiprocessing extension is missing. + import_module('_multiprocessing') + + # Skip tests if there is no available semaphore implementation: + # multiprocessing.synchronize requires _multiprocessing.SemLock. + synchronize = import_module('multiprocessing.synchronize') + + if sys.platform == "linux": + try: + # bpo-38377: On Linux, creating a semaphore fails with OSError + # if the current user does not have the permission to create + # a file in /dev/shm/ directory. + synchronize.Lock(ctx=None) + except OSError as exc: + raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}") diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 30f8f98a..f14868ad 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -392,6 +392,9 @@ class ShareableTypeTests(unittest.TestCase): self._assert_values(i.to_bytes(2, 'little', signed=True) for i in range(-1, 258)) + def test_strs(self): + self._assert_values(['hello world', '你好世界', '']) + def test_int(self): self._assert_values(itertools.chain(range(-1, 258), [sys.maxsize, -sys.maxsize - 1])) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 8887558c..b921f4a5 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -387,6 +387,15 @@ class AST_Tests(unittest.TestCase): self.assertRaises(TypeError, ast.Num, 1, None, 2) self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) + # Arbitrary keyword arguments are supported + self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') + self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') + + with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): + ast.Num(1, n=2) + with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): + ast.Constant(1, value=2) + self.assertEqual(ast.Num(42).n, 42) self.assertEqual(ast.Num(4.25).n, 4.25) self.assertEqual(ast.Num(4.25j).n, 4.25j) @@ -621,6 +630,19 @@ class AST_Tests(unittest.TestCase): attr_b = tree.body[0].decorator_list[0].value self.assertEqual(attr_b.end_col_offset, 4) + def test_issue40614_feature_version(self): + ast.parse('f"{x=}"', feature_version=(3, 8)) + with self.assertRaises(SyntaxError): + ast.parse('f"{x=}"', feature_version=(3, 7)) + + def test_constant_as_name(self): + for constant in "True", "False", "None": + expr = ast.Expression(ast.Name(constant, ast.Load())) + ast.fix_missing_locations(expr) + with self.assertRaisesRegex(ValueError, f"Name node can't be used with '{constant}' constant"): + compile(expr, "", "eval") + + class ASTHelpers_Test(unittest.TestCase): maxDiff = None diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index aec56da8..85838f17 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1204,7 +1204,7 @@ class EventLoopTestsMixin: ConnectionRefusedError, client.connect, ('127.0.0.1', port)) client.close() - def test_create_datagram_endpoint(self): + def _test_create_datagram_endpoint(self, local_addr, family): class TestMyDatagramProto(MyDatagramProto): def __init__(inner_self): super().__init__(loop=self.loop) @@ -1214,9 +1214,11 @@ class EventLoopTestsMixin: self.transport.sendto(b'resp:'+data, addr) coro = self.loop.create_datagram_endpoint( - TestMyDatagramProto, local_addr=('127.0.0.1', 0)) + TestMyDatagramProto, local_addr=local_addr, family=family) s_transport, server = self.loop.run_until_complete(coro) - host, port = s_transport.get_extra_info('sockname') + sockname = s_transport.get_extra_info('sockname') + host, port = socket.getnameinfo( + sockname, socket.NI_NUMERICHOST|socket.NI_NUMERICSERV) self.assertIsInstance(s_transport, asyncio.Transport) self.assertIsInstance(server, TestMyDatagramProto) @@ -1250,6 +1252,13 @@ class EventLoopTestsMixin: self.assertEqual('CLOSED', client.state) server.transport.close() + def test_create_datagram_endpoint(self): + self._test_create_datagram_endpoint(('127.0.0.1', 0), socket.AF_INET) + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 not supported or enabled') + def test_create_datagram_endpoint_ipv6(self): + self._test_create_datagram_endpoint(('::1', 0), socket.AF_INET6) + def test_create_datagram_endpoint_sock(self): sock = None local_address = ('127.0.0.1', 0) @@ -2635,10 +2644,10 @@ class GetEventLoopTestsMixin: if sys.platform != 'win32': def test_get_event_loop_new_process(self): - # Issue bpo-32126: The multiprocessing module used by + # bpo-32126: The multiprocessing module used by # ProcessPoolExecutor is not functional when the # multiprocessing.synchronize module cannot be imported. - support.import_module('multiprocessing.synchronize') + support.skip_if_broken_multiprocessing_synchronize() async def main(): pool = concurrent.futures.ProcessPoolExecutor() diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index f405c692..f79edbc4 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -51,22 +51,6 @@ class AuditTest(unittest.TestCase): def test_block_add_hook_baseexception(self): self.do_test("test_block_add_hook_baseexception") - def test_finalize_hooks(self): - returncode, events, stderr = self.run_python("test_finalize_hooks") - if stderr: - print(stderr, file=sys.stderr) - if returncode: - self.fail(stderr) - - firstId = events[0][2] - self.assertSequenceEqual( - [ - ("Created", " ", firstId), - ("cpython._PySys_ClearAuditHooks", " ", firstId), - ], - events, - ) - def test_pickle(self): support.import_module("pickle") diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index 6e82cce1..ae168805 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -726,7 +726,7 @@ class StateTestCase(BaseTestCase): ('line', 2, 'tfunc_import'), ('step', ), ('line', 3, 'tfunc_import'), ('quit', ), ] - skip = ('importlib*', 'zipimport', TEST_MODULE) + skip = ('importlib*', 'zipimport', 'encodings.*', TEST_MODULE) with TracerRun(self, skip=skip) as tracer: tracer.runcall(tfunc_import) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 48b0e33a..4a498262 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -370,6 +370,27 @@ class BuiltinTest(unittest.TestCase): rv = ns['f']() self.assertEqual(rv, tuple(expected)) + def test_compile_top_level_await_no_coro(self): + """Make sure top level non-await codes get the correct coroutine flags""" + modes = ('single', 'exec') + code_samples = [ + '''def f():pass\n''', + '''[x for x in l]''', + '''{x for x in l}''', + '''(x for x in l)''', + '''{x:x for x in l}''', + ] + for mode, code_sample in product(modes, code_samples): + source = dedent(code_sample) + co = compile(source, + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + + self.assertNotEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, + msg=f"source={source} mode={mode}") + + def test_compile_top_level_await(self): """Test whether code some top level await can be compiled. diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index eb2f72ee..2591f0a6 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -69,7 +69,7 @@ class BaseTest(unittest.TestCase): # simply use the bigger test data for all tests. test_size = 0 BIG_TEXT = bytearray(128*1024) - for fname in glob.glob(os.path.join(os.path.dirname(__file__), '*.py')): + for fname in glob.glob(os.path.join(glob.escape(os.path.dirname(__file__)), '*.py')): with open(fname, 'rb') as fh: test_size += fh.readinto(memoryview(BIG_TEXT)[test_size:]) if test_size > 128*1024: diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 584c1046..75471270 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -473,6 +473,14 @@ class CAPITest(unittest.TestCase): # Test that subtype_dealloc decref the newly assigned __class__ only once self.assertEqual(new_type_refcnt, sys.getrefcount(_testcapi.HeapCTypeSubclass)) + def test_heaptype_with_setattro(self): + obj = _testcapi.HeapCTypeSetattr() + self.assertEqual(obj.pvalue, 10) + obj.value = 12 + self.assertEqual(obj.pvalue, 12) + del obj.value + self.assertEqual(obj.pvalue, 0) + def test_pynumber_tobase(self): from _testcapi import pynumber_tobase self.assertEqual(pynumber_tobase(123, 2), '0b1111011') @@ -584,6 +592,26 @@ class SubinterpreterTest(unittest.TestCase): self.assertNotEqual(pickle.load(f), id(sys.modules)) self.assertNotEqual(pickle.load(f), id(builtins)) + def test_subinterps_recent_language_features(self): + r, w = os.pipe() + code = """if 1: + import pickle + with open({:d}, "wb") as f: + + def noop(x): return x + + a = (b := f'1{{2}}3') + noop('x') # Py 3.8 (:=) / 3.6 (f'') + + async def foo(arg): return await arg # Py 3.5 + + pickle.dump(dict(a=a, b=b), f) + """.format(w) + + with open(r, "rb") as f: + ret = support.run_in_subinterp(code) + self.assertEqual(ret, 0) + self.assertEqual(pickle.load(f), {'a': '123x', 'b': '123'}) + def test_mutate_exception(self): """ Exceptions saved in global module state get shared between diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py index ab867719..101942de 100644 --- a/Lib/test/test_cgi.py +++ b/Lib/test/test_cgi.py @@ -128,6 +128,20 @@ class CgiTests(unittest.TestCase): 'file': [b'Testing 123.\n'], 'title': ['']} self.assertEqual(result, expected) + def test_parse_multipart_without_content_length(self): + POSTDATA = '''--JfISa01 +Content-Disposition: form-data; name="submit-name" + +just a string + +--JfISa01-- +''' + fp = BytesIO(POSTDATA.encode('latin1')) + env = {'boundary': 'JfISa01'.encode('latin1')} + result = cgi.parse_multipart(fp, env) + expected = {'submit-name': ['just a string\n']} + self.assertEqual(result, expected) + def test_parse_multipart_invalid_encoding(self): BOUNDARY = "JfISa01" POSTDATA = """--JfISa01 diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 98da26fa..1e57ab9d 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -3,12 +3,12 @@ Nick Mathewson """ import unittest -from test.support import is_jython +from test import support from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT import io -if is_jython: +if support.is_jython: import sys def unify_callables(d): @@ -21,7 +21,7 @@ class CodeopTests(unittest.TestCase): def assertValid(self, str, symbol='single'): '''succeed iff str is a valid piece of code''' - if is_jython: + if support.is_jython: code = compile_command(str, "", symbol) self.assertTrue(code) if symbol == "single": @@ -60,7 +60,7 @@ class CodeopTests(unittest.TestCase): av = self.assertValid # special case - if not is_jython: + if not support.is_jython: self.assertEqual(compile_command(""), compile("pass", "", 'single', PyCF_DONT_IMPLY_DEDENT)) @@ -294,6 +294,11 @@ class CodeopTests(unittest.TestCase): self.assertNotEqual(compile_command("a = 1\n", "abc").co_filename, compile("a = 1\n", "def", 'single').co_filename) + def test_warning(self): + # Test that the warning is only returned once. + with support.check_warnings((".*literal", SyntaxWarning)) as w: + compile_command("0 is 0") + self.assertEqual(len(w.warnings), 1) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index ac722981..a5a746eb 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -3,7 +3,7 @@ import test.support # Skip tests if _multiprocessing wasn't built. test.support.import_module('_multiprocessing') # Skip tests if sem_open implementation is broken. -test.support.import_module('multiprocessing.synchronize') +test.support.skip_if_broken_multiprocessing_synchronize() from test.support.script_helper import assert_python_ok diff --git a/Lib/test/test_crashers.py b/Lib/test/test_crashers.py index 58dfd001..31b71202 100644 --- a/Lib/test/test_crashers.py +++ b/Lib/test/test_crashers.py @@ -11,7 +11,7 @@ import test.support from test.support.script_helper import assert_python_failure CRASHER_DIR = os.path.join(os.path.dirname(__file__), "crashers") -CRASHER_FILES = os.path.join(CRASHER_DIR, "*.py") +CRASHER_FILES = os.path.join(glob.escape(CRASHER_DIR), "*.py") infinite_loops = ["infinite_loop_re.py", "nasty_eq_vs_dict.py"] diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index 1db3bef6..571da973 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -33,7 +33,7 @@ def dbm_iterator(): def delete_files(): # we don't know the precise name the underlying database uses # so we use glob to locate all names - for f in glob.glob(_fname + "*"): + for f in glob.glob(glob.escape(_fname) + "*"): test.support.unlink(f) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 0e9cd309..1f37b537 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -5476,41 +5476,6 @@ class CWhitebox(unittest.TestCase): self.assertEqual(Decimal.from_float(cls(101.1)), Decimal.from_float(101.1)) - def test_maxcontext_exact_arith(self): - - # Make sure that exact operations do not raise MemoryError due - # to huge intermediate values when the context precision is very - # large. - - # The following functions fill the available precision and are - # therefore not suitable for large precisions (by design of the - # specification). - MaxContextSkip = ['logical_invert', 'next_minus', 'next_plus', - 'logical_and', 'logical_or', 'logical_xor', - 'next_toward', 'rotate', 'shift'] - - Decimal = C.Decimal - Context = C.Context - localcontext = C.localcontext - - # Here only some functions that are likely candidates for triggering a - # MemoryError are tested. deccheck.py has an exhaustive test. - maxcontext = Context(prec=C.MAX_PREC, Emin=C.MIN_EMIN, Emax=C.MAX_EMAX) - with localcontext(maxcontext): - self.assertEqual(Decimal(0).exp(), 1) - self.assertEqual(Decimal(1).ln(), 0) - self.assertEqual(Decimal(1).log10(), 0) - self.assertEqual(Decimal(10**2).log10(), 2) - self.assertEqual(Decimal(10**223).log10(), 223) - self.assertEqual(Decimal(10**19).logb(), 19) - self.assertEqual(Decimal(4).sqrt(), 2) - self.assertEqual(Decimal("40E9").sqrt(), Decimal('2.0E+5')) - self.assertEqual(divmod(Decimal(10), 3), (3, 1)) - self.assertEqual(Decimal(10) // 3, 3) - self.assertEqual(Decimal(4) / 2, 2) - self.assertEqual(Decimal(400) ** -1, Decimal('0.0025')) - - @requires_docstrings @unittest.skipUnless(C, "test requires C version") class SignatureTest(unittest.TestCase): diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 502b90e7..ad30a051 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -669,7 +669,7 @@ plain ol' Python and is guaranteed to be available. True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests - 12 + 13 >>> for t in real_tests: ... print('{} {}'.format(len(t.examples), t.name)) ... @@ -685,6 +685,7 @@ plain ol' Python and is guaranteed to be available. 2 builtins.int.bit_length 5 builtins.memoryview.hex 1 builtins.oct + 1 builtins.zip Note here that 'bin', 'oct', and 'hex' are functions; 'float.as_integer_ratio', 'float.hex', and 'int.bit_length' are methods; 'float.fromhex' is a classmethod, diff --git a/Lib/test/test_email/test_contentmanager.py b/Lib/test/test_email/test_contentmanager.py index 169058ea..f4f6bb71 100644 --- a/Lib/test/test_email/test_contentmanager.py +++ b/Lib/test/test_email/test_contentmanager.py @@ -303,6 +303,19 @@ class TestRawDataManager(TestEmailBase): self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) self.assertEqual(m.get_content(), content) + def test_set_text_plain_null(self): + m = self._make_message() + content = '' + raw_data_manager.set_content(m, content) + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 7bit + + + """)) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), '\n') + self.assertEqual(m.get_content(), '\n') + def test_set_text_html(self): m = self._make_message() content = "

    Simple message.

    \n" @@ -329,6 +342,21 @@ class TestRawDataManager(TestEmailBase): self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) self.assertEqual(m.get_content(), content) + def test_set_text_plain_long_line_heuristics(self): + m = self._make_message() + content = ("Simple but long message that is over 78 characters" + " long to force transfer encoding.\n") + raw_data_manager.set_content(m, content) + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: quoted-printable + + Simple but long message that is over 78 characters long to = + force transfer encoding. + """)) + self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) + self.assertEqual(m.get_content(), content) + def test_set_text_short_line_minimal_non_ascii_heuristics(self): m = self._make_message() content = "et là il est monté sur moi et il commence à m'éto.\n" diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index 8d89c5dd..7ade9684 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -872,6 +872,25 @@ class TestContentDisposition(TestHeaderBase): {'filename': 'foo'}, [errors.InvalidHeaderDefect]), + 'invalid_parameter_value_with_fws_between_ew': ( + 'attachment; filename="=?UTF-8?Q?Schulbesuchsbest=C3=A4ttigung=2E?=' + ' =?UTF-8?Q?pdf?="', + 'attachment', + {'filename': 'Schulbesuchsbestättigung.pdf'}, + [errors.InvalidHeaderDefect]*3, + ('attachment; filename="Schulbesuchsbestättigung.pdf"'), + ('Content-Disposition: attachment;\n' + ' filename*=utf-8\'\'Schulbesuchsbest%C3%A4ttigung.pdf\n'), + ), + + 'parameter_value_with_fws_between_tokens': ( + 'attachment; filename="File =?utf-8?q?Name?= With Spaces.pdf"', + 'attachment', + {'filename': 'File Name With Spaces.pdf'}, + [errors.InvalidHeaderDefect], + 'attachment; filename="File Name With Spaces.pdf"', + ('Content-Disposition: attachment; filename="File Name With Spaces.pdf"\n'), + ) } @@ -1436,6 +1455,25 @@ class TestAddressAndGroup(TestEmailBase): # with self.assertRaises(ValueError): # Address('foo', 'wők', 'example.com') + def test_crlf_in_constructor_args_raises(self): + cases = ( + dict(display_name='foo\r'), + dict(display_name='foo\n'), + dict(display_name='foo\r\n'), + dict(domain='example.com\r'), + dict(domain='example.com\n'), + dict(domain='example.com\r\n'), + dict(username='wok\r'), + dict(username='wok\n'), + dict(username='wok\r\n'), + dict(addr_spec='wok@example.com\r'), + dict(addr_spec='wok@example.com\n'), + dict(addr_spec='wok@example.com\r\n') + ) + for kwargs in cases: + with self.subTest(kwargs=kwargs), self.assertRaisesRegex(ValueError, "invalid arguments"): + Address(**kwargs) + def test_non_ascii_username_in_addr_spec_raises(self): with self.assertRaises(ValueError): Address('foo', addr_spec='wők@example.com') diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index da79d7af..64a26b9f 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -32,7 +32,7 @@ API_ISOLATED = 3 def debug_build(program): program = os.path.basename(program) name = os.path.splitext(program)[0] - return name.endswith("_d") + return name.casefold().endswith("_d".casefold()) def remove_python_envvars(): @@ -567,7 +567,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG: expected['stdio_errors'] = 'surrogateescape' - if sys.platform == 'win32': + if MS_WINDOWS: default_executable = self.test_exe elif expected['program_name'] is not self.GET_DEFAULT_CONFIG: default_executable = os.path.abspath(expected['program_name']) @@ -601,7 +601,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): pre_config = dict(configs['pre_config']) for key, value in list(expected.items()): if value is self.IGNORE_CONFIG: - del pre_config[key] + pre_config.pop(key, None) del expected[key] self.assertEqual(pre_config, expected) @@ -609,7 +609,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): config = dict(configs['config']) for key, value in list(expected.items()): if value is self.IGNORE_CONFIG: - del config[key] + config.pop(key, None) del expected[key] self.assertEqual(config, expected) @@ -684,6 +684,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.check_pre_config(configs, expected_preconfig) self.check_config(configs, expected_config) self.check_global_config(configs) + return configs def test_init_default_config(self): self.check_all_configs("test_init_initialize_config", api=API_COMPAT) @@ -1035,6 +1036,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): } self.default_program_name(config) env = {'TESTPATH': os.path.pathsep.join(paths)} + self.check_all_configs("test_init_setpath", config, api=API_COMPAT, env=env, ignore_stderr=True) @@ -1092,12 +1094,18 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): # Copy pythonXY.dll (or pythonXY_d.dll) ver = sys.version_info dll = f'python{ver.major}{ver.minor}' + dll3 = f'python{ver.major}' if debug_build(sys.executable): dll += '_d' + dll3 += '_d' dll += '.dll' + dll3 += '.dll' dll = os.path.join(os.path.dirname(self.test_exe), dll) + dll3 = os.path.join(os.path.dirname(self.test_exe), dll3) dll_copy = os.path.join(tmpdir, os.path.basename(dll)) + dll3_copy = os.path.join(tmpdir, os.path.basename(dll3)) shutil.copyfile(dll, dll_copy) + shutil.copyfile(dll3, dll3_copy) # Copy Python program exec_copy = os.path.join(tmpdir, os.path.basename(self.test_exe)) @@ -1225,9 +1233,18 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): config['base_prefix'] = pyvenv_home config['prefix'] = pyvenv_home env = self.copy_paths_by_env(config) - self.check_all_configs("test_init_compat_config", config, - api=API_COMPAT, env=env, - ignore_stderr=True, cwd=tmpdir) + actual = self.check_all_configs("test_init_compat_config", config, + api=API_COMPAT, env=env, + ignore_stderr=True, cwd=tmpdir) + if MS_WINDOWS: + self.assertEqual( + actual['windows']['python3_dll'], + os.path.join( + tmpdir, + os.path.basename(self.EXPECTED_CONFIG['windows']['python3_dll']) + ) + ) + def test_global_pathconfig(self): # Test C API functions getting the path configuration: @@ -1309,7 +1326,7 @@ class AuditingTests(EmbeddingTestsMixin, unittest.TestCase): self.run_embedded_interpreter("test_audit_run_file", timeout=3, returncode=1) def test_audit_run_interactivehook(self): - startup = os.path.join(self.oldcwd, support.TESTFN) + ".py" + startup = os.path.join(self.oldcwd, support.TESTFN) + (support.FS_NONASCII or '') + ".py" with open(startup, "w", encoding="utf-8") as f: print("import sys", file=f) print("sys.__interactivehook__ = lambda: None", file=f) @@ -1321,7 +1338,7 @@ class AuditingTests(EmbeddingTestsMixin, unittest.TestCase): os.unlink(startup) def test_audit_run_startup(self): - startup = os.path.join(self.oldcwd, support.TESTFN) + ".py" + startup = os.path.join(self.oldcwd, support.TESTFN) + (support.FS_NONASCII or '') + ".py" with open(startup, "w", encoding="utf-8") as f: print("pass", file=f) try: diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 89966893..4786d28f 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -40,7 +40,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "setuptools", "pip", ], unittest.mock.ANY, @@ -54,7 +54,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "--root", "/foo/bar/", "setuptools", "pip", ], @@ -66,7 +66,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "--user", "setuptools", "pip", ], unittest.mock.ANY, @@ -77,7 +77,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "--upgrade", "setuptools", "pip", ], unittest.mock.ANY, @@ -88,7 +88,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "-v", "setuptools", "pip", ], unittest.mock.ANY, @@ -99,7 +99,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "-vv", "setuptools", "pip", ], unittest.mock.ANY, @@ -110,7 +110,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "-vvv", "setuptools", "pip", ], unittest.mock.ANY, @@ -260,7 +260,7 @@ class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "setuptools", "pip", ], unittest.mock.ANY, diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 47081cf7..a2a3c567 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -1702,6 +1702,16 @@ class TestEnum(unittest.TestCase): self.assertEqual(Color.blue.value, 2) self.assertEqual(Color.green.value, 3) + def test_auto_order(self): + with self.assertRaises(TypeError): + class Color(Enum): + red = auto() + green = auto() + blue = auto() + def _generate_next_value_(name, start, count, last): + return name + + def test_duplicate_auto(self): class Dupes(Enum): first = primero = auto() diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index ea13533b..57946467 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -265,6 +265,10 @@ class AnnotationsFutureTestCase(unittest.TestCase): eq("dict[str, int]") eq("set[str,]") eq("tuple[str, ...]") + eq("tuple[(str, *types)]") + eq("tuple[xx:yy, (*types,)]") + eq("tuple[str, int, (str, int)]") + eq("tuple[(*int, str, str, (str, int))]") eq("tuple[str, int, float, dict[str, int]]") eq("slice[0]") eq("slice[0:1]") diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index f043c925..d90ca5a5 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -39,7 +39,8 @@ def get_gdb_version(): # 'GNU gdb (GDB) Fedora 7.9.1-17.fc22\n' -> 7.9 # 'GNU gdb 6.1.1 [FreeBSD]\n' -> 6.1 # 'GNU gdb (GDB) Fedora (7.5.1-37.fc18)\n' -> 7.5 - match = re.search(r"^GNU gdb.*?\b(\d+)\.(\d+)", version) + # 'HP gdb 6.7 for HP Itanium (32 or 64 bit) and target HP-UX 11iv2 and 11iv3.\n' -> 6.7 + match = re.search(r"^(?:GNU|HP) gdb.*?\b(\d+)\.(\d+)", version) if match is None: raise Exception("unable to parse GDB version: %r" % version) return (version, int(match.group(1)), int(match.group(2))) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 03cffbe3..06ea86b5 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -911,6 +911,7 @@ class ThreadedNetworkedTestsSSL(ThreadedNetworkedTests): @unittest.skipUnless( support.is_resource_enabled('network'), 'network resource disabled') +@unittest.skip('cyrus.andrew.cmu.edu blocks connections') class RemoteIMAPTest(unittest.TestCase): host = 'cyrus.andrew.cmu.edu' port = 143 @@ -946,6 +947,7 @@ class RemoteIMAPTest(unittest.TestCase): @unittest.skipUnless(ssl, "SSL not available") @unittest.skipUnless( support.is_resource_enabled('network'), 'network resource disabled') +@unittest.skip('cyrus.andrew.cmu.edu blocks connections') class RemoteIMAP_STARTTLSTest(RemoteIMAPTest): def setUp(self): @@ -961,6 +963,7 @@ class RemoteIMAP_STARTTLSTest(RemoteIMAPTest): @unittest.skipUnless(ssl, "SSL not available") +@unittest.skip('cyrus.andrew.cmu.edu blocks connections') class RemoteIMAP_SSLTest(RemoteIMAPTest): port = 993 imap_class = IMAP4_SSL diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index f037821d..52f04916 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -478,7 +478,7 @@ class ImportTests(unittest.TestCase): pyexe = os.path.join(tmp, os.path.basename(sys.executable)) shutil.copy(sys.executable, pyexe) shutil.copy(dllname, tmp) - for f in glob.glob(os.path.join(sys.prefix, "vcruntime*.dll")): + for f in glob.glob(os.path.join(glob.escape(sys.prefix), "vcruntime*.dll")): shutil.copy(f, tmp) shutil.copy(pydname, tmp2) diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py index 695c92a7..d923cec2 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/fixtures.py @@ -1,25 +1,11 @@ -from __future__ import unicode_literals - import os import sys import shutil +import pathlib import tempfile import textwrap import contextlib -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - -try: - import pathlib -except ImportError: - import pathlib2 as pathlib - - -__metaclass__ = type - @contextlib.contextmanager def tempdir(): @@ -58,7 +44,7 @@ def install_finder(finder): class Fixtures: def setUp(self): - self.fixtures = ExitStack() + self.fixtures = contextlib.ExitStack() self.addCleanup(self.fixtures.close) diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index de771117..2eba740e 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2053,6 +2053,18 @@ class IpaddrUnitTest(unittest.TestCase): sixtofouraddr.sixtofour) self.assertFalse(bad_addr.sixtofour) + # issue41004 Hash collisions in IPv4Interface and IPv6Interface + def testV4HashIsNotConstant(self): + ipv4_address1 = ipaddress.IPv4Interface("1.2.3.4") + ipv4_address2 = ipaddress.IPv4Interface("2.3.4.5") + self.assertNotEqual(ipv4_address1.__hash__(), ipv4_address2.__hash__()) + + # issue41004 Hash collisions in IPv4Interface and IPv6Interface + def testV6HashIsNotConstant(self): + ipv6_address1 = ipaddress.IPv6Interface("2001:658:22a:cafe:200:0:0:1") + ipv6_address2 = ipaddress.IPv6Interface("2001:658:22a:cafe:200:0:0:2") + self.assertNotEqual(ipv6_address1.__hash__(), ipv6_address2.__hash__()) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 542b2841..22553d5d 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -62,6 +62,10 @@ class NoIterClass: return i __iter__ = None +class BadIterableClass: + def __iter__(self): + raise ZeroDivisionError + # Main test suite class TestCase(unittest.TestCase): @@ -637,6 +641,7 @@ class TestCase(unittest.TestCase): self.assertRaises(TypeError, lambda: 3 in 12) self.assertRaises(TypeError, lambda: 3 not in map) + self.assertRaises(ZeroDivisionError, lambda: 3 in BadIterableClass()) d = {"one": 1, "two": 2, "three": 3, 1j: 2j} for k in d: @@ -719,6 +724,7 @@ class TestCase(unittest.TestCase): self.assertRaises(TypeError, indexOf, 42, 1) self.assertRaises(TypeError, indexOf, indexOf, indexOf) + self.assertRaises(ZeroDivisionError, indexOf, BadIterableClass(), 1) f = open(TESTFN, "w") try: @@ -1006,6 +1012,7 @@ class TestCase(unittest.TestCase): def test_error_iter(self): for typ in (DefaultIterClass, NoIterClass): self.assertRaises(TypeError, iter, typ()) + self.assertRaises(ZeroDivisionError, iter, BadIterableClass()) def test_main(): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 90bf2a4d..09b273bf 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3621,9 +3621,9 @@ if hasattr(logging.handlers, 'QueueListener'): @patch.object(logging.handlers.QueueListener, 'handle') def test_handle_called_with_mp_queue(self, mock_handle): - # Issue 28668: The multiprocessing (mp) module is not functional + # bpo-28668: The multiprocessing (mp) module is not functional # when the mp.synchronize module cannot be imported. - support.import_module('multiprocessing.synchronize') + support.skip_if_broken_multiprocessing_synchronize() for i in range(self.repeat): log_queue = multiprocessing.Queue() self.setup_and_log(log_queue, '%s_%s' % (self.id(), i)) @@ -3647,9 +3647,9 @@ if hasattr(logging.handlers, 'QueueListener'): indicates that messages were not registered on the queue until _after_ the QueueListener stopped. """ - # Issue 28668: The multiprocessing (mp) module is not functional + # bpo-28668: The multiprocessing (mp) module is not functional # when the mp.synchronize module cannot be imported. - support.import_module('multiprocessing.synchronize') + support.skip_if_broken_multiprocessing_synchronize() for i in range(self.repeat): queue = multiprocessing.Queue() self.setup_and_log(queue, '%s_%s' %(self.id(), i)) diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 0995b1e3..effac97d 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -979,7 +979,7 @@ class _TestMboxMMDF(_TestSingleFile): super().tearDown() self._box.close() self._delete_recursively(self._path) - for lock_remnant in glob.glob(self._path + '.*'): + for lock_remnant in glob.glob(glob.escape(self._path) + '.*'): support.unlink(lock_remnant) def assertMailboxEmpty(self): @@ -1311,7 +1311,7 @@ class TestBabyl(_TestSingleFile, unittest.TestCase): super().tearDown() self._box.close() self._delete_recursively(self._path) - for lock_remnant in glob.glob(self._path + '.*'): + for lock_remnant in glob.glob(glob.escape(self._path) + '.*'): support.unlink(lock_remnant) def test_labels(self): diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 9cac6ce0..683d393f 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -67,6 +67,18 @@ class MimeTypesTestCase(unittest.TestCase): mime_dict = mimetypes.read_mime_types(file) eq(mime_dict[".pyunit"], "x-application/x-unittest") + # bpo-41048: read_mime_types should read the rule file with 'utf-8' encoding. + # Not with locale encoding. _bootlocale has been imported because io.open(...) + # uses it. + with support.temp_dir() as directory: + data = "application/no-mans-land Fran\u00E7ais" + file = pathlib.Path(directory, "sample.mimetype") + file.write_text(data, encoding='utf-8') + import _bootlocale + with support.swap_attr(_bootlocale, 'getpreferredencoding', lambda do_setlocale=True: 'ASCII'): + mime_dict = mimetypes.read_mime_types(file) + eq(mime_dict[".Français"], "application/no-mans-land") + def test_non_standard_types(self): eq = self.assertEqual # First try strict diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py index 23c7e5fb..ca1058b8 100644 --- a/Lib/test/test_modulefinder.py +++ b/Lib/test/test_modulefinder.py @@ -284,10 +284,11 @@ a_cp1252.py # 0xe2 is not allowed in utf8 print('CP1252 test P\xe2t\xe9') import b_utf8 +""" + """\ b_utf8.py # use the default of utf8 print('Unicode test A code point 2090 \u2090 that is not valid in cp1252') -"""] +""".encode('utf-8')] def open_file(path): dirname = os.path.dirname(path) diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py index f9bd0da7..153a8ac0 100644 --- a/Lib/test/test_msilib.py +++ b/Lib/test/test_msilib.py @@ -1,13 +1,13 @@ """ Test suite for the code in msilib """ import os import unittest -from test.support import TESTFN, import_module, unlink +from test.support import TESTFN, FS_NONASCII, import_module, unlink msilib = import_module('msilib') import msilib.schema def init_database(): - path = TESTFN + '.msi' + path = TESTFN + (FS_NONASCII or '') + '.msi' db = msilib.init_database( path, msilib.schema, @@ -42,6 +42,16 @@ class MsiDatabaseTestCase(unittest.TestCase): ) self.addCleanup(unlink, db_path) + def test_view_non_ascii(self): + db, db_path = init_database() + view = db.OpenView("SELECT 'ß-розпад' FROM Property") + view.Execute(None) + record = view.Fetch() + self.assertEqual(record.GetString(1), 'ß-розпад') + view.Close() + db.Close() + self.addCleanup(unlink, db_path) + def test_summaryinfo_getproperty_issue1104(self): db, db_path = init_database() try: diff --git a/Lib/test/test_multiprocessing_main_handling.py b/Lib/test/test_multiprocessing_main_handling.py index b6abfcc7..be1ff10e 100644 --- a/Lib/test/test_multiprocessing_main_handling.py +++ b/Lib/test/test_multiprocessing_main_handling.py @@ -23,7 +23,7 @@ import multiprocessing AVAILABLE_START_METHODS = set(multiprocessing.get_all_start_methods()) # Issue #22332: Skip tests if sem_open implementation is broken. -support.import_module('multiprocessing.synchronize') +support.skip_if_broken_multiprocessing_synchronize() verbose = support.verbose diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index f46d94a2..29f5e427 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -35,6 +35,10 @@ class Seq2(object): def __rmul__(self, other): return other * self.lst +class BadIterable: + def __iter__(self): + raise ZeroDivisionError + class OperatorTestCase: def test_lt(self): @@ -142,6 +146,7 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.countOf) self.assertRaises(TypeError, operator.countOf, None, None) + self.assertRaises(ZeroDivisionError, operator.countOf, BadIterable(), 1) self.assertEqual(operator.countOf([1, 2, 1, 3, 1, 4], 3), 1) self.assertEqual(operator.countOf([1, 2, 1, 3, 1, 4], 5), 0) @@ -176,6 +181,7 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.indexOf) self.assertRaises(TypeError, operator.indexOf, None, None) + self.assertRaises(ZeroDivisionError, operator.indexOf, BadIterable(), 1) self.assertEqual(operator.indexOf([4, 3, 2, 1], 3), 1) self.assertRaises(ValueError, operator.indexOf, [4, 3, 2, 1], 0) @@ -258,6 +264,7 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.contains) self.assertRaises(TypeError, operator.contains, None, None) + self.assertRaises(ZeroDivisionError, operator.contains, BadIterable(), 1) self.assertTrue(operator.contains(range(4), 2)) self.assertFalse(operator.contains(range(4), 5)) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 4c38e919..0e7ae1d8 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -5,6 +5,7 @@ import os import pdb import sys import types +import codecs import unittest import subprocess import textwrap @@ -1226,9 +1227,7 @@ class PdbTestCase(unittest.TestCase): return self._run_pdb(['-m', self.module_name], commands) def _assert_find_function(self, file_content, func_name, expected): - file_content = textwrap.dedent(file_content) - - with open(support.TESTFN, 'w') as f: + with open(support.TESTFN, 'wb') as f: f.write(file_content) expected = None if not expected else ( @@ -1237,22 +1236,49 @@ class PdbTestCase(unittest.TestCase): expected, pdb.find_function(func_name, support.TESTFN)) def test_find_function_empty_file(self): - self._assert_find_function('', 'foo', None) + self._assert_find_function(b'', 'foo', None) def test_find_function_found(self): self._assert_find_function( """\ - def foo(): - pass +def foo(): + pass - def bar(): - pass +def bœr(): + pass - def quux(): - pass - """, - 'bar', - ('bar', 4), +def quux(): + pass +""".encode(), + 'bœr', + ('bœr', 4), + ) + + def test_find_function_found_with_encoding_cookie(self): + self._assert_find_function( + """\ +# coding: iso-8859-15 +def foo(): + pass + +def bœr(): + pass + +def quux(): + pass +""".encode('iso-8859-15'), + 'bœr', + ('bœr', 5), + ) + + def test_find_function_found_with_bom(self): + self._assert_find_function( + codecs.BOM_UTF8 + """\ +def bœr(): + pass +""".encode(), + 'bœr', + ('bœr', 1), ) def test_issue7964(self): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e2fa1977..f89b1af7 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -559,7 +559,7 @@ class CheckActualTests(BaseTestCase): args = ['-Wd', '-E', '-bb', '-m', 'test.regrtest', '--list-tests'] output = self.run_python(args) rough_number_of_tests_found = len(output.splitlines()) - actual_testsuite_glob = os.path.join(os.path.dirname(__file__), + actual_testsuite_glob = os.path.join(glob.escape(os.path.dirname(__file__)), 'test*.py') rough_counted_test_py_files = len(glob.glob(actual_testsuite_glob)) # We're not trying to duplicate test finding logic in here, diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 71f192f9..03bf8d8b 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -29,7 +29,9 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): # test.support.script_helper. env = kw.setdefault('env', dict(os.environ)) env['TERM'] = 'vt100' - return subprocess.Popen(cmd_line, executable=sys.executable, + return subprocess.Popen(cmd_line, + executable=sys.executable, + text=True, stdin=subprocess.PIPE, stdout=stdout, stderr=stderr, **kw) @@ -49,12 +51,11 @@ class TestInteractiveInterpreter(unittest.TestCase): sys.exit(0) """ user_input = dedent(user_input) - user_input = user_input.encode() p = spawn_repl() with SuppressCrashReport(): p.stdin.write(user_input) output = kill_python(p) - self.assertIn(b'After the exception.', output) + self.assertIn('After the exception.', output) # Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr. self.assertIn(p.returncode, (1, 120)) @@ -86,13 +87,26 @@ class TestInteractiveInterpreter(unittest.TestCase):     """ ''' user_input = dedent(user_input) - user_input = user_input.encode() p = spawn_repl() - with SuppressCrashReport(): - p.stdin.write(user_input) + p.stdin.write(user_input) output = kill_python(p) self.assertEqual(p.returncode, 0) + def test_close_stdin(self): + user_input = dedent(''' + import os + print("before close") + os.close(0) + ''') + prepare_repl = dedent(''' + from test.support import suppress_msvcrt_asserts + suppress_msvcrt_asserts() + ''') + process = spawn_repl('-c', prepare_repl) + output = process.communicate(user_input)[0] + self.assertEqual(process.returncode, 0) + self.assertIn('before close', output) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 1bbc6979..9a047fd4 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -526,7 +526,7 @@ class StartupImportTests(unittest.TestCase): # found in sys.path (see site.addpackage()). Skip the test if at least # one .pth file is found. for path in isolated_paths: - pth_files = glob.glob(os.path.join(path, "*.pth")) + pth_files = glob.glob(os.path.join(glob.escape(path), "*.pth")) if pth_files: self.skipTest(f"found {len(pth_files)} .pth files in: {path}") diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index a9a427bc..5b8ad874 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2029,6 +2029,10 @@ class TestVariance(VarianceStdevMixin, NumericTestCase, UnivariateTypeMixin): self.assertEqual(result, exact) self.assertIsInstance(result, Decimal) + def test_center_not_at_mean(self): + data = (1.0, 2.0) + self.assertEqual(self.func(data), 0.5) + self.assertEqual(self.func(data, xbar=2.0), 1.0) class TestPStdev(VarianceStdevMixin, NumericTestCase): # Tests for population standard deviation. @@ -2041,6 +2045,11 @@ class TestPStdev(VarianceStdevMixin, NumericTestCase): expected = math.sqrt(statistics.pvariance(data)) self.assertEqual(self.func(data), expected) + def test_center_not_at_mean(self): + # See issue: 40855 + data = (3, 6, 7, 10) + self.assertEqual(self.func(data), 2.5) + self.assertEqual(self.func(data, mu=0.5), 6.5) class TestStdev(VarianceStdevMixin, NumericTestCase): # Tests for sample standard deviation. @@ -2058,6 +2067,9 @@ class TestStdev(VarianceStdevMixin, NumericTestCase): expected = math.sqrt(statistics.variance(data)) self.assertEqual(self.func(data), expected) + def test_center_not_at_mean(self): + data = (1.0, 2.0) + self.assertEqual(self.func(data, xbar=2.0), 1.0) class TestGeometricMean(unittest.TestCase): diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 454082e6..67e7c559 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -652,6 +652,13 @@ class StructTest(unittest.TestCase): s2 = struct.Struct(s.format.encode()) self.assertEqual(s2.format, s.format) + def test_issue35714(self): + # Embedded null characters should not be allowed in format strings. + for s in '\0', '2\0i', b'\0': + with self.assertRaisesRegex(struct.error, + 'embedded null character'): + struct.calcsize(s) + class UnpackIteratorTest(unittest.TestCase): """ diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 4c900928..6de7aa87 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1605,7 +1605,7 @@ class TestRoundtrip(TestCase): import glob, random fn = support.findfile("tokenize_tests.txt") tempdir = os.path.dirname(fn) or os.curdir - testfiles = glob.glob(os.path.join(tempdir, "test*.py")) + testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py")) # Tokenize is broken on test_pep3131.py because regular expressions are # broken on the obscure unicode identifiers in it. *sigh* diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 4bc21eae..8eacf99c 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,6 +1,6 @@ import os import sys -from test.support import TESTFN, rmtree, unlink, captured_stdout +from test.support import TESTFN, TESTFN_UNICODE, FS_NONASCII, rmtree, unlink, captured_stdout from test.support.script_helper import assert_python_ok, assert_python_failure import textwrap import unittest @@ -429,9 +429,10 @@ class TestCoverageCommandLineOutput(unittest.TestCase): coverfile = 'tmp.cover' def setUp(self): - with open(self.codefile, 'w') as f: + with open(self.codefile, 'w', encoding='iso-8859-15') as f: f.write(textwrap.dedent('''\ - x = 42 + # coding: iso-8859-15 + x = 'spœm' if []: print('unreachable') ''')) @@ -452,9 +453,10 @@ class TestCoverageCommandLineOutput(unittest.TestCase): self.assertEqual(stderr, b'') self.assertFalse(os.path.exists(tracecoverpath)) self.assertTrue(os.path.exists(self.coverfile)) - with open(self.coverfile) as f: + with open(self.coverfile, encoding='iso-8859-15') as f: self.assertEqual(f.read(), - " 1: x = 42\n" + " # coding: iso-8859-15\n" + " 1: x = 'spœm'\n" " 1: if []:\n" " print('unreachable')\n" ) @@ -463,9 +465,10 @@ class TestCoverageCommandLineOutput(unittest.TestCase): argv = '-m trace --count --missing'.split() + [self.codefile] status, stdout, stderr = assert_python_ok(*argv) self.assertTrue(os.path.exists(self.coverfile)) - with open(self.coverfile) as f: + with open(self.coverfile, encoding='iso-8859-15') as f: self.assertEqual(f.read(), textwrap.dedent('''\ - 1: x = 42 + # coding: iso-8859-15 + 1: x = 'spœm' 1: if []: >>>>>> print('unreachable') ''')) @@ -486,14 +489,19 @@ class TestCommandLine(unittest.TestCase): self.assertIn(message, stderr) def test_listfuncs_flag_success(self): - with open(TESTFN, 'w') as fd: - self.addCleanup(unlink, TESTFN) + filename = TESTFN + '.py' + modulename = os.path.basename(TESTFN) + with open(filename, 'w', encoding='utf-8') as fd: + self.addCleanup(unlink, filename) fd.write("a = 1\n") - status, stdout, stderr = assert_python_ok('-m', 'trace', '-l', TESTFN) + status, stdout, stderr = assert_python_ok('-m', 'trace', '-l', filename, + PYTHONIOENCODING='utf-8') self.assertIn(b'functions called:', stdout) + expected = f'filename: {filename}, modulename: {modulename}, funcname: ' + self.assertIn(expected.encode(), stdout) def test_sys_argv_list(self): - with open(TESTFN, 'w') as fd: + with open(TESTFN, 'w', encoding='utf-8') as fd: self.addCleanup(unlink, TESTFN) fd.write("import sys\n") fd.write("print(type(sys.argv))\n") @@ -505,7 +513,8 @@ class TestCommandLine(unittest.TestCase): def test_count_and_summary(self): filename = f'{TESTFN}.py' coverfilename = f'{TESTFN}.cover' - with open(filename, 'w') as fd: + modulename = os.path.basename(TESTFN) + with open(filename, 'w', encoding='utf-8') as fd: self.addCleanup(unlink, filename) self.addCleanup(unlink, coverfilename) fd.write(textwrap.dedent("""\ @@ -522,7 +531,7 @@ class TestCommandLine(unittest.TestCase): stdout = stdout.decode() self.assertEqual(status, 0) self.assertIn('lines cov% module (path)', stdout) - self.assertIn(f'6 100% {TESTFN} ({filename})', stdout) + self.assertIn(f'6 100% {modulename} ({filename})', stdout) def test_run_as_module(self): assert_python_ok('-m', 'trace', '-l', '--module', 'timeit', '-n', '1') diff --git a/Lib/test/test_unicode_file.py b/Lib/test/test_unicode_file.py index b16e4c5b..e8feb42c 100644 --- a/Lib/test/test_unicode_file.py +++ b/Lib/test/test_unicode_file.py @@ -40,7 +40,7 @@ class TestUnicodeFiles(unittest.TestCase): self._do_copyish(filename, filename) # Filename should appear in glob output self.assertTrue( - os.path.abspath(filename)==os.path.abspath(glob.glob(filename)[0])) + os.path.abspath(filename)==os.path.abspath(glob.glob(glob.escape(filename))[0])) # basename should appear in listdir. path, base = os.path.split(os.path.abspath(filename)) file_list = os.listdir(path) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index bc4e95f2..28743f03 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -16,7 +16,8 @@ import sys import tempfile from test.support import (captured_stdout, captured_stderr, requires_zlib, can_symlink, EnvironmentVarGuard, rmtree, - import_module) + import_module, + skip_if_broken_multiprocessing_synchronize) import threading import unittest import venv @@ -324,10 +325,11 @@ class BasicTest(BaseTest): """ Test that the multiprocessing is able to spawn. """ - # Issue bpo-36342: Instanciation of a Pool object imports the + # bpo-36342: Instantiation of a Pool object imports the # multiprocessing.synchronize module. Skip the test if this module # cannot be imported. - import_module('multiprocessing.synchronize') + skip_if_broken_multiprocessing_synchronize() + rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) envpy = os.path.join(os.path.realpath(self.env_dir), @@ -480,7 +482,7 @@ class EnsurePipTest(BaseTest): # executing pip with sudo, you may want sudo's -H flag." # where $HOME is replaced by the HOME environment variable. err = re.sub("^(WARNING: )?The directory .* or its parent directory " - "is not owned by the current user .*$", "", + "is not owned or is not writable by the current user.*$", "", err, flags=re.MULTILINE) self.assertEqual(err.rstrip(), "") # Being fairly specific regarding the expected behaviour for the diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 985adc1c..7b0d06b3 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -1199,13 +1199,13 @@ class EnvironmentVariableTests(BaseTest): @unittest.skipUnless(sys.getfilesystemencoding() != 'ascii', 'requires non-ascii filesystemencoding') def test_nonascii(self): + PYTHONWARNINGS="ignore:DeprecationWarning" + (support.FS_NONASCII or '') rc, stdout, stderr = assert_python_ok("-c", "import sys; sys.stdout.write(str(sys.warnoptions))", PYTHONIOENCODING="utf-8", - PYTHONWARNINGS="ignore:DeprecaciónWarning", + PYTHONWARNINGS=PYTHONWARNINGS, PYTHONDEVMODE="") - self.assertEqual(stdout, - "['ignore:DeprecaciónWarning']".encode('utf-8')) + self.assertEqual(stdout, str([PYTHONWARNINGS]).encode()) class CEnvironmentVariableTests(EnvironmentVariableTests, unittest.TestCase): module = c_warnings diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 28e62dc5..e2e37a17 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -1589,6 +1589,11 @@ class OtherTests(unittest.TestCase): self.assertEqual(zf.filelist[0].filename, "foo.txt") self.assertEqual(zf.filelist[1].filename, "\xf6.txt") + def test_read_after_write_unicode_filenames(self): + with zipfile.ZipFile(TESTFN2, 'w') as zipfp: + zipfp.writestr('приклад', b'sample') + self.assertEqual(zipfp.read('приклад'), b'sample') + def test_exclusive_create_zip_file(self): """Test exclusive creating a new zipfile.""" unlink(TESTFN2) diff --git a/Lib/trace.py b/Lib/trace.py index a4473576..89f17d48 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -287,8 +287,9 @@ class CoverageResults: if self.outfile: # try and store counts and module info into self.outfile try: - pickle.dump((self.counts, self.calledfuncs, self.callers), - open(self.outfile, 'wb'), 1) + with open(self.outfile, 'wb') as f: + pickle.dump((self.counts, self.calledfuncs, self.callers), + f, 1) except OSError as err: print("Can't save counts files because %s" % err, file=sys.stderr) @@ -731,7 +732,7 @@ def main(): sys.argv = [opts.progname, *opts.arguments] sys.path[0] = os.path.dirname(opts.progname) - with open(opts.progname) as fp: + with open(opts.progname, 'rb') as fp: code = compile(fp.read(), opts.progname, 'exec') # try to emulate __main__ namespace as much as possible globs = { diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index e5734b6b..3223c0bf 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -251,7 +251,7 @@ class _AssertWarnsContext(_AssertRaisesBaseContext): def __enter__(self): # The __warningregistry__'s need to be in a pristine state for tests # to work properly. - for v in sys.modules.values(): + for v in list(sys.modules.values()): if getattr(v, '__warningregistry__', None): v.__warningregistry__ = {} self.warnings_manager = warnings.catch_warnings(record=True) diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py index f855c4dc..3dedcbe6 100644 --- a/Lib/unittest/test/test_case.py +++ b/Lib/unittest/test/test_case.py @@ -8,6 +8,7 @@ import logging import warnings import weakref import inspect +import types from copy import deepcopy from test import support @@ -1350,6 +1351,20 @@ test case pass self.assertRaises(TypeError, self.assertWarnsRegex, MyWarn, lambda: True) + def testAssertWarnsModifySysModules(self): + # bpo-29620: handle modified sys.modules during iteration + class Foo(types.ModuleType): + @property + def __warningregistry__(self): + sys.modules['@bar@'] = 'bar' + + sys.modules['@foo@'] = Foo('foo') + try: + self.assertWarns(UserWarning, warnings.warn, 'expected') + finally: + del sys.modules['@foo@'] + del sys.modules['@bar@'] + def testAssertRaisesRegexMismatch(self): def Stub(): raise Exception('Unexpected') diff --git a/Lib/venv/scripts/common/Activate.ps1 b/Lib/venv/scripts/common/Activate.ps1 index 90af274d..2fb3852c 100644 --- a/Lib/venv/scripts/common/Activate.ps1 +++ b/Lib/venv/scripts/common/Activate.ps1 @@ -45,7 +45,7 @@ command: PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser For more information on Execution Policies: -ttps:/go.microsoft.com/fwlink/?LinkID=135170 +https://go.microsoft.com/fwlink/?LinkID=135170 #> Param( diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 9c73bcfb..cea91308 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -413,7 +413,7 @@ class Grail(BaseBrowser): tempdir = os.path.join(tempfile.gettempdir(), ".grail-unix") user = pwd.getpwuid(os.getuid())[0] - filename = os.path.join(tempdir, user + "-*") + filename = os.path.join(glob.escape(tempdir), glob.escape(user) + "-*") maybes = glob.glob(filename) if not maybes: return None @@ -545,12 +545,12 @@ def register_standard_browsers(): register(browser, None, BackgroundBrowser(browser)) else: # Prefer X browsers if present - if os.environ.get("DISPLAY"): + if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"): try: cmd = "xdg-settings get default-web-browser".split() raw_result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) result = raw_result.decode().strip() - except (FileNotFoundError, subprocess.CalledProcessError): + except (FileNotFoundError, subprocess.CalledProcessError, PermissionError) : pass else: global _os_preferred_browser diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 07faacca..73e89666 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -37,7 +37,8 @@ except ImportError: __all__ = ["BadZipFile", "BadZipfile", "error", "ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA", - "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile"] + "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile", + "Path"] class BadZipFile(Exception): pass @@ -1545,7 +1546,7 @@ class ZipFile: # strong encryption raise NotImplementedError("strong encryption (flag bit 6)") - if zinfo.flag_bits & 0x800: + if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & 0x800: # UTF-8 filename fname_str = fname.decode("utf-8") else: diff --git a/Mac/BuildScript/README.rst b/Mac/BuildScript/README.rst index f8b25fa5..94a6bb28 100644 --- a/Mac/BuildScript/README.rst +++ b/Mac/BuildScript/README.rst @@ -7,216 +7,84 @@ framework-based Python out-of-tree, installs it in a funny place with $DESTROOT, massages that installation to remove .pyc files and such, creates an Installer package from the installation plus other files in ``resources`` and ``scripts`` and placed that on a ``.dmg`` disk image. - -For Python 3.4.0, PSF practice is to build two installer variants -for each release. - -1. 32-bit-only, i386 and PPC universal, capable on running on all machines - supported by Mac OS X 10.5 through (at least) 10.9:: - - /path/to/bootstrap/python2.7 build-installer.py \ - --sdk-path=/Developer/SDKs/MacOSX10.5.sdk \ - --universal-archs=32-bit \ - --dep-target=10.5 - - - builds the following third-party libraries - - * NCurses 5.9 (http://bugs.python.org/issue15037) - * SQLite 3.8.11 - * XZ 5.0.5 - - - uses system-supplied versions of third-party libraries - - * readline module links with Apple BSD editline (libedit) - - - requires ActiveState ``Tcl/Tk 8.4`` (currently 8.4.20) to be installed for building - - - recommended build environment: - - * Mac OS X 10.5.8 Intel or PPC - * Xcode 3.1.4 - * ``MacOSX10.5`` SDK - * ``MACOSX_DEPLOYMENT_TARGET=10.5`` - * Apple ``gcc-4.2`` - * bootstrap non-framework Python 2.7 for documentation build with - Sphinx (as of 3.4.1) - - - alternate build environments: - - * Mac OS X 10.6.8 with Xcode 3.2.6 - - need to change ``/System/Library/Frameworks/{Tcl,Tk}.framework/Version/Current`` to ``8.4`` - * Note Xcode 4.* does not support building for PPC so cannot be used for this build - -2. 64-bit / 32-bit, x86_64 and i386 universal, for OS X 10.6 (and later):: +The installer package built on the dmg is a macOS bundle format installer +package. This format is deprecated and is no longer supported by modern +macOS systems; it is usable on macOS 10.6 and earlier systems. +To be usable on newer versions of macOS, the bits in the bundle package +must be assembled in a macOS flat installer package, using current +versions of the pkgbuild and productbuild utilities. To pass macoS +Gatekeeper download quarantine, the final package must be signed +with a valid Apple Developer ID certificate using productsign. +Starting with macOS 10.15 Catalina, Gatekeeper now also requires +that installer packages are submitted to and pass Apple's automated +notarization service using the altool command. To pass notarization, +the binaries included in the package must be built with at least +the macOS 10.9 SDK, mout now be signed with the codesign utility +and executables must opt in to the hardened run time option with +any necessary entitlements. Details of these processes are +available in the on-line Apple Developer Documentation and man pages. + +As of 3.8.0 and 3.7.7, PSF practice is to build one installer variants +for each release. Note that as of this writing, no Pythons support +building on a newer version of macOS that will run on older versions +by setting MACOSX_DEPLOYMENT_TARGET. This is because the various +Python C modules do not yet support runtime testing of macOS +feature availability (for example, by using macOS AvailabilityMacros.h +and weak-linking). To build a Python that is to be used on a +range of macOS releases, always build on the oldest release to be +supported; the necessary shared libraries for that release will +normally also be available on later systems, with the occasional +exception such as the removal of 32-bit libraries in macOS 10.15. + +build-installer requires Apple Developer tools, either from the +Command Line Tools package or from a full Xcode installation. +You should use the most recent version of either for the operating +system version in use. (One notable exception: on macOS 10.6, +Snow Leopard, use Xcode 3, not Xcode 4 which was released later +in the 10.6 support cycle.) + +1. 64-bit, x86_64, for OS X 10.9 (and later):: /path/to/bootstrap/python2.7 build-installer.py \ - --sdk-path=/Developer/SDKs/MacOSX10.6.sdk \ - --universal-archs=intel \ - --dep-target=10.6 + --universal-archs=intel-64 \ + --dep-target=10.9 - builds the following third-party libraries - * NCurses 5.9 (http://bugs.python.org/issue15037) - * SQLite 3.8.11 - * XZ 5.0.5 + * OpenSSL 1.1.1 + * Tcl/Tk 8.6 + * NCurses + * SQLite + * XZ + * libffi - uses system-supplied versions of third-party libraries * readline module links with Apple BSD editline (libedit) - - - requires ActiveState Tcl/Tk 8.5.15.1 (or later) to be installed for building - - - recommended build environment: - - * Mac OS X 10.6.8 (or later) - * Xcode 3.2.6 - * ``MacOSX10.6`` SDK - * ``MACOSX_DEPLOYMENT_TARGET=10.6`` - * Apple ``gcc-4.2`` - * bootstrap non-framework Python 2.7 for documentation build with - Sphinx (as of 3.4.1) - - - alternate build environments: - - * none. Xcode 4.x currently supplies two C compilers. - ``llvm-gcc-4.2.1`` has been found to miscompile Python 3.3.x and - produce a non-functional Python executable. As it appears to be - considered a migration aid by Apple and is not likely to be fixed, - its use should be avoided. The other compiler, ``clang``, has been - undergoing rapid development. While it appears to have become - production-ready in the most recent Xcode 5 releases, the versions - available on the deprecated Xcode 4.x for 10.6 were early releases - and did not receive the level of exposure in production environments - that the Xcode 3 gcc-4.2 compiler has had. - - -* For Python 2.7.x and 3.2.x, the 32-bit-only installer was configured to - support Mac OS X 10.3.9 through (at least) 10.6. Because it is - believed that there are few systems still running OS X 10.3 or 10.4 - and because it has become increasingly difficult to test and - support the differences in these earlier systems, as of Python 3.3.0 the PSF - 32-bit installer no longer supports them. For reference in building such - an installer yourself, the details are:: - - /usr/bin/python build-installer.py \ - --sdk-path=/Developer/SDKs/MacOSX10.4u.sdk \ - --universal-archs=32-bit \ - --dep-target=10.3 - - - builds the following third-party libraries - - * Bzip2 - * NCurses - * GNU Readline (GPL) - * SQLite 3 - * XZ - * Zlib 1.2.3 - * Oracle Sleepycat DB 4.8 (Python 2.x only) - - - requires ActiveState ``Tcl/Tk 8.4`` (currently 8.4.20) to be installed for building + * zlib + * bz2 - recommended build environment: - * Mac OS X 10.5.8 PPC or Intel - * Xcode 3.1.4 (or later) - * ``MacOSX10.4u`` SDK (later SDKs do not support PPC G3 processors) - * ``MACOSX_DEPLOYMENT_TARGET=10.3`` - * Apple ``gcc-4.0`` - * system Python 2.5 for documentation build with Sphinx - - - alternate build environments: - - * Mac OS X 10.6.8 with Xcode 3.2.6 - - need to change ``/System/Library/Frameworks/{Tcl,Tk}.framework/Version/Current`` to ``8.4`` - + * Mac OS X 10.9.5 + * Xcode Command Line Tools 6.2 + * ``MacOSX10.9`` SDK + * ``MACOSX_DEPLOYMENT_TARGET=10.9`` + * Apple ``clang`` General Prerequisites --------------------- -* No Fink (in ``/sw``) or MacPorts (in ``/opt/local``) or other local - libraries or utilities (in ``/usr/local``) as they could +* No Fink (in ``/sw``) or MacPorts (in ``/opt/local``) or Homebrew or + other local libraries or utilities (in ``/usr/local``) as they could interfere with the build. -* The documentation for the release is built using Sphinx - because it is included in the installer. For 2.7.x and 3.x.x up to and - including 3.4.0, the ``Doc/Makefile`` uses ``svn`` to download repos of - ``Sphinx`` and its dependencies. Beginning with 3.4.1, the ``Doc/Makefile`` - assumes there is an externally-provided ``sphinx-build`` and requires at - least Python 2.6 to run. Because of this, it is no longer possible to - build a 3.4.1 or later installer on OS X 10.5 using the Apple-supplied - Python 2.5. - * It is safest to start each variant build with an empty source directory - populated with a fresh copy of the untarred source. + populated with a fresh copy of the untarred source or a source repo. * It is recommended that you remove any existing installed version of the Python being built:: sudo rm -rf /Library/Frameworks/Python.framework/Versions/n.n - -The Recipe ----------- - -Here are the steps you need to follow to build a Python installer: - -* Run ``build-installer.py``. Optionally you can pass a number of arguments - to specify locations of various files. Please see the top of - ``build-installer.py`` for its usage. - - Running this script takes some time, it will not only build Python itself - but also some 3th-party libraries that are needed for extensions. - -* When done the script will tell you where the DMG image is (by default - somewhere in ``/tmp/_py``). - -Building other universal installers -................................... - -It is also possible to build a 4-way universal installer that runs on -OS X 10.5 Leopard or later:: - - /usr/bin/python /build-installer.py \ - --dep-target=10.5 - --universal-archs=all - --sdk-path=/Developer/SDKs/MacOSX10.5.sdk - -This requires that the deployment target is 10.5, and hence -also that you are building on at least OS X 10.5. 4-way includes -``i386``, ``x86_64``, ``ppc``, and ``ppc64`` (G5). ``ppc64`` executable -variants can only be run on G5 machines running 10.5. Note that, -while OS X 10.6 is only supported on Intel-based machines, it is possible -to run ``ppc`` (32-bit) executables unmodified thanks to the Rosetta ppc -emulation in OS X 10.5 and 10.6. The 4-way installer variant must be -built with Xcode 3. It is not regularly built or tested. - -Other ``--universal-archs`` options are ``64-bit`` (``x86_64``, ``ppc64``), -and ``3-way`` (``ppc``, ``i386``, ``x86_64``). None of these options -are regularly exercised; use at your own risk. - - -Testing -------- - -Ideally, the resulting binaries should be installed and the test suite run -on all supported OS X releases and architectures. As a practical matter, -that is generally not possible. At a minimum, variant 1 should be run on -a PPC G4 system with OS X 10.5 and at least one Intel system running OS X -10.9, 10.8, 10.7, 10.6, or 10.5. Variant 2 should be run on 10.9, 10.8, -10.7, and 10.6 systems in both 32-bit and 64-bit modes.:: - - /usr/local/bin/pythonn.n -m test -w -u all,-largefile - /usr/local/bin/pythonn.n-32 -m test -w -u all - -Certain tests will be skipped and some cause the interpreter to fail -which will likely generate ``Python quit unexpectedly`` alert messages -to be generated at several points during a test run. These are normal -during testing and can be ignored. - -It is also recommend to launch IDLE and verify that it is at least -functional. Double-click on the IDLE app icon in ``/Applications/Python n.n``. -It should also be tested from the command line:: - - /usr/local/bin/idlen.n - diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 0ad7298e..4fab4882 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -2,6 +2,20 @@ """ This script is used to build "official" universal installers on macOS. +NEW for 3.9.0 and backports: +- 2.7 end-of-life issues: + - Python 3 installs now update the Current version link + in /Library/Frameworks/Python.framework/Versions +- fully support running under Python 3 as well as 2.7 +- support building on newer macOS systems with SIP +- fully support building on macOS 10.9+ +- support 10.6+ on best effort +- support bypassing docs build by supplying a prebuilt + docs html tarball in the third-party source library, + in the format and filename conventional of those + downloadable from python.org: + python-3.x.y-docs-html.tar.bz2 + NEW for 3.7.0: - support Intel 64-bit-only () and 32-bit-only installer builds - build and use internal Tcl/Tk 8.6 for 10.6+ builds @@ -14,28 +28,7 @@ NEW for 3.7.0: - use generic "gcc" as compiler (CC env var) rather than "gcc-4.2" TODO: -- support SDKROOT and DEVELOPER_DIR xcrun env variables -- test with 10.5 and 10.4 and determine support status - -Please ensure that this script keeps working with Python 2.5, to avoid -bootstrap issues (/usr/bin/python is Python 2.5 on OSX 10.5). Doc builds -use current versions of Sphinx and require a reasonably current python3. -Sphinx and dependencies are installed into a venv using the python3's pip -so will fetch them from PyPI if necessary. Since python3 is now used for -Sphinx, build-installer.py should also be converted to use python3! - -For 3.7.0, when building for a 10.6 or higher deployment target, -build-installer builds and links with its own copy of Tcl/Tk 8.6. -Otherwise, it requires an installed third-party version of -Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets), Tcl/TK 8.5 -(for 10.6 or later), or Tcl/TK 8.6 (for 10.9 or later) -installed in /Library/Frameworks. When installed, -the Python built by this script will attempt to dynamically link first to -Tcl and Tk frameworks in /Library/Frameworks if available otherwise fall -back to the ones in /System/Library/Framework. For the build, we recommend -installing the most recent ActiveTcl 8.6. 8.5, or 8.4 version, depending -on the deployment target. The actual version linked to depends on the -path of /Library/Frameworks/{Tcl,Tk}.framework/Versions/Current. +- test building with SDKROOT and DEVELOPER_DIR xcrun env variables Usage: see USAGE variable in the script. """ @@ -56,14 +49,15 @@ STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR INCLUDE_TIMESTAMP = 1 VERBOSE = 1 -from plistlib import Plist +RUNNING_ON_PYTHON2 = sys.version_info.major == 2 -try: +if RUNNING_ON_PYTHON2: from plistlib import writePlist -except ImportError: - # We're run using python2.3 - def writePlist(plist, path): - plist.write(path) +else: + from plistlib import dump + def writePlist(path, plist): + with open(plist, 'wb') as fp: + dump(path, fp) def shellQuote(value): """ @@ -302,7 +296,7 @@ def library_recipes(): "--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib"%(getVersion(),), ], patchscripts=[ - ("ftp://invisible-island.net/ncurses//5.9/ncurses-5.9-20120616-patch.sh.bz2", + ("ftp://ftp.invisible-island.net/ncurses//5.9/ncurses-5.9-20120616-patch.sh.bz2", "f54bf02a349f96a7c4f0d00922f3a0d4"), ], useLDFlags=False, @@ -313,9 +307,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.31.1", - url="https://sqlite.org/2020/sqlite-autoconf-3310100.tar.gz", - checksum='2d0a553534c521504e3ac3ad3b90f125', + name="SQLite 3.32.2", + url="https://sqlite.org/2020/sqlite-autoconf-3320200.tar.gz", + checksum='eb498918a33159cdf8104997aad29e83', extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' @@ -1066,14 +1060,40 @@ def buildPythonDocs(): curDir = os.getcwd() os.chdir(buildDir) runCommand('make clean') - # Create virtual environment for docs builds with blurb and sphinx - runCommand('make venv') - runCommand('venv/bin/python3 -m pip install -U Sphinx==2.0.1') - runCommand('make html PYTHON=venv/bin/python') + + # Search third-party source directory for a pre-built version of the docs. + # Use the naming convention of the docs.python.org html downloads: + # python-3.9.0b1-docs-html.tar.bz2 + doctarfiles = [ f for f in os.listdir(DEPSRC) + if f.startswith('python-'+getFullVersion()) + if f.endswith('-docs-html.tar.bz2') ] + if doctarfiles: + doctarfile = doctarfiles[0] + if not os.path.exists('build'): + os.mkdir('build') + # if build directory existed, it was emptied by make clean, above + os.chdir('build') + # Extract the first archive found for this version into build + runCommand('tar xjf %s'%shellQuote(os.path.join(DEPSRC, doctarfile))) + # see if tar extracted a directory ending in -docs-html + archivefiles = [ f for f in os.listdir('.') + if f.endswith('-docs-html') + if os.path.isdir(f) ] + if archivefiles: + archivefile = archivefiles[0] + # make it our 'Docs/build/html' directory + print(' -- using pre-built python documentation from %s'%archivefile) + os.rename(archivefile, 'html') + os.chdir(buildDir) + + htmlDir = os.path.join('build', 'html') + if not os.path.exists(htmlDir): + # Create virtual environment for docs builds with blurb and sphinx + runCommand('make venv') + runCommand('venv/bin/python3 -m pip install -U Sphinx==2.3.1') + runCommand('make html PYTHON=venv/bin/python') + os.rename(htmlDir, docdir) os.chdir(curDir) - if not os.path.exists(docdir): - os.mkdir(docdir) - os.rename(os.path.join(buildDir, 'build', 'html'), docdir) def buildPython(): @@ -1099,8 +1119,7 @@ def buildPython(): # Since the extra libs are not in their installed framework location # during the build, augment the library path so that the interpreter # will find them during its extension import sanity checks. - os.environ['DYLD_LIBRARY_PATH'] = os.path.join(WORKDIR, - 'libraries', 'usr', 'local', 'lib') + print("Running configure...") runCommand("%s -C --enable-framework --enable-universalsdk=/ " "--with-universal-archs=%s " @@ -1108,12 +1127,15 @@ def buildPython(): "%s " "%s " "%s " + "%s " "LDFLAGS='-g -L%s/libraries/usr/local/lib' " "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%( shellQuote(os.path.join(SRCDIR, 'configure')), UNIVERSALARCHS, (' ', '--with-computed-gotos ')[PYTHON_3], (' ', '--without-ensurepip ')[PYTHON_3], + (' ', "--with-openssl='%s/libraries/usr/local'"%( + shellQuote(WORKDIR)[1:-1],))[PYTHON_3], (' ', "--with-tcltk-includes='-I%s/libraries/usr/local/include'"%( shellQuote(WORKDIR)[1:-1],))[internalTk()], (' ', "--with-tcltk-libs='-L%s/libraries/usr/local/lib -ltcl8.6 -ltk8.6'"%( @@ -1121,6 +1143,24 @@ def buildPython(): shellQuote(WORKDIR)[1:-1], shellQuote(WORKDIR)[1:-1])) + # As of macOS 10.11 with SYSTEM INTEGRITY PROTECTION, DYLD_* + # environment variables are no longer automatically inherited + # by child processes from their parents. We used to just set + # DYLD_LIBRARY_PATH, pointing to the third-party libs, + # in build-installer.py's process environment and it was + # passed through the make utility into the environment of + # setup.py. Instead, we now append DYLD_LIBRARY_PATH to + # the existing RUNSHARED configuration value when we call + # make for extension module builds. + + runshared_for_make = "".join([ + " RUNSHARED=", + "'", + grepValue("Makefile", "RUNSHARED"), + ' DYLD_LIBRARY_PATH=', + os.path.join(WORKDIR, 'libraries', 'usr', 'local', 'lib'), + "'" ]) + # Look for environment value BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS # and, if defined, append its value to the make command. This allows # us to pass in version control tags, like GITTAG, to a build from a @@ -1135,21 +1175,24 @@ def buildPython(): make_extras = os.getenv("BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS") if make_extras: - make_cmd = "make " + make_extras + make_cmd = "make " + make_extras + runshared_for_make else: - make_cmd = "make" + make_cmd = "make" + runshared_for_make print("Running " + make_cmd) runCommand(make_cmd) - print("Running make install") - runCommand("make install DESTDIR=%s"%( - shellQuote(rootDir))) + make_cmd = "make install DESTDIR=%s %s"%( + shellQuote(rootDir), + runshared_for_make) + print("Running " + make_cmd) + runCommand(make_cmd) - print("Running make frameworkinstallextras") - runCommand("make frameworkinstallextras DESTDIR=%s"%( - shellQuote(rootDir))) + make_cmd = "make frameworkinstallextras DESTDIR=%s %s"%( + shellQuote(rootDir), + runshared_for_make) + print("Running " + make_cmd) + runCommand(make_cmd) - del os.environ['DYLD_LIBRARY_PATH'] print("Copying required shared libraries") if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')): build_lib_dir = os.path.join( @@ -1278,7 +1321,13 @@ def buildPython(): data = fp.read() fp.close() # create build_time_vars dict - exec(data) + if RUNNING_ON_PYTHON2: + exec(data) + else: + g_dict = {} + l_dict = {} + exec(data, g_dict, l_dict) + build_time_vars = l_dict['build_time_vars'] vars = {} for k, v in build_time_vars.items(): if type(v) == type(''): @@ -1309,12 +1358,6 @@ def buildPython(): os.chdir(curdir) - if PYTHON_3: - # Remove the 'Current' link, that way we don't accidentally mess - # with an already installed version of python 2 - os.unlink(os.path.join(rootDir, 'Library', 'Frameworks', - 'Python.framework', 'Versions', 'Current')) - def patchFile(inPath, outPath): data = fileContents(inPath) data = data.replace('$FULL_VERSION', getFullVersion()) @@ -1401,7 +1444,7 @@ def packageFromRecipe(targetDir, recipe): vers = getFullVersion() major, minor = getVersionMajorMinor() - pl = Plist( + pl = dict( CFBundleGetInfoString="Python.%s %s"%(pkgname, vers,), CFBundleIdentifier='org.python.Python.%s'%(pkgname,), CFBundleName='Python.%s'%(pkgname,), @@ -1423,7 +1466,7 @@ def packageFromRecipe(targetDir, recipe): ) writePlist(pl, os.path.join(packageContents, 'Info.plist')) - pl = Plist( + pl = dict( IFPkgDescriptionDescription=readme, IFPkgDescriptionTitle=recipe.get('long_name', "Python.%s"%(pkgname,)), IFPkgDescriptionVersion=vers, @@ -1439,7 +1482,7 @@ def makeMpkgPlist(path): vers = getFullVersion() major, minor = getVersionMajorMinor() - pl = Plist( + pl = dict( CFBundleGetInfoString="Python %s"%(vers,), CFBundleIdentifier='org.python.Python', CFBundleName='Python', @@ -1492,7 +1535,7 @@ def buildInstaller(): os.mkdir(rsrcDir) makeMpkgPlist(os.path.join(pkgroot, 'Info.plist')) - pl = Plist( + pl = dict( IFPkgDescriptionTitle="Python", IFPkgDescriptionVersion=getVersion(), ) diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index 4cb0111d..b1e972ee 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2511 +{\rtf1\ansi\ansicpg1252\cocoartf2513 \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fswiss\fcharset0 Helvetica-Oblique; \f3\fmodern\fcharset0 CourierNewPSMT;} {\colortbl;\red255\green255\blue255;} @@ -53,7 +53,25 @@ Due to new security checks on macOS 10.15 Catalina, when launching IDLE macOS ma macOS 10.15 (Catalina) Gatekeeper Requirements [changed in 3.8.2]\ \f0\b0 \ulnone \ -As of 2020-02-03, Apple has changed how third-party installer packages, like those provided by python.org, are notarized for verification by Gatekeeper and begun enforcing additional requirements such as code signing and use of the hardened runtime. As of 3.8.2, python.org installer packages now meet those additional notarization requirements. The necessary changes in packaging should be transparent to your use of Python but, in the unlikely event that you encounter changes in behavior between 3.8.1 and 3.8.2 in areas like ctypes, importlib, or mmap, please check bugs.python.org for existing reports and, if necessary, open a new issue.\ +As of 2020-02-03, Apple has changed how third-party installer packages, like those provided by python.org, are notarized for verification by Gatekeeper and begun enforcing additional requirements such as code signing and use of the hardened runtime. As of 3.8.2, python.org installer packages now meet those additional notarization requirements. The necessary changes in packaging should be transparent to your use of Python but, in the unlikely event that you encounter changes in behavior between 3.8.1 and newer 3.8.x releases in areas like ctypes, importlib, or mmap, please check bugs.python.org for existing reports and, if necessary, open a new issue.\ + +\f1\b \ul \ +Python 2.7 end-of-life [changed in 3.8.4]\ +\ + +\f0\b0 \ulnone Python 2.7 has now reached end-of-life. As of Python 3.8.4, the +\f3 Python Launcher +\f0 app now has +\f3 python3 +\f0 factory defaults. Also, the +\f3 Current +\f0 link in the +\f3 /Library/Frameworks/Python.framework/Versions +\f0 directory is now updated to point to the Python 3 being installed; previously, only Python 2 installs updated +\f3 Current +\f0 . This change might affect developers using the framework to embed Python in their applications. If another version is desired for embedding, the +\f3 Current +\f0 symlink can be changed manually without affecting 3.8.x behavior.\ \f1\b \ul \ Other changes\ diff --git a/Mac/BuildScript/scripts/postflight.documentation b/Mac/BuildScript/scripts/postflight.documentation index 3cbbc1bf..ec48599c 100755 --- a/Mac/BuildScript/scripts/postflight.documentation +++ b/Mac/BuildScript/scripts/postflight.documentation @@ -12,7 +12,9 @@ SHARE_DOCDIR_TO_FWK="../../.." # make link in /Applications/Python m.n/ for Finder users if [ -d "${APPDIR}" ]; then ln -fhs "${FWK_DOCDIR}/index.html" "${APPDIR}/Python Documentation.html" - open "${APPDIR}" || true # open the applications folder + if [ "${COMMAND_LINE_INSTALL}" != 1 ]; then + open "${APPDIR}" || true # open the applications folder + fi fi # make share/doc link in framework for command line users diff --git a/Mac/PythonLauncher/factorySettings.plist b/Mac/PythonLauncher/factorySettings.plist index 12024213..6f650ae7 100644 --- a/Mac/PythonLauncher/factorySettings.plist +++ b/Mac/PythonLauncher/factorySettings.plist @@ -10,9 +10,9 @@ interpreter_list - /usr/local/bin/pythonw - /usr/bin/pythonw - /sw/bin/pythonw + /usr/local/bin/python3 + /opt/local/bin/python3 + /sw/bin/python3 honourhashbang @@ -35,12 +35,9 @@ interpreter_list - /usr/local/bin/pythonw - /usr/local/bin/python - /usr/bin/pythonw - /usr/bin/python - /sw/bin/pythonw - /sw/bin/python + /usr/local/bin/python3 + /opt/local/bin/python3 + /sw/bin/python3 honourhashbang @@ -63,12 +60,9 @@ interpreter_list - /usr/local/bin/pythonw - /usr/local/bin/python - /usr/bin/pythonw - /usr/bin/python - /sw/bin/pythonw - /sw/bin/python + /usr/local/bin/python3 + /opt/local/bin/python3 + /sw/bin/python3 honourhashbang diff --git a/Mac/Resources/app/Info.plist.in b/Mac/Resources/app/Info.plist.in index 66b5e764..1d624984 100644 --- a/Mac/Resources/app/Info.plist.in +++ b/Mac/Resources/app/Info.plist.in @@ -20,7 +20,7 @@ CFBundleExecutable Python CFBundleGetInfoString - %version%, (c) 2001-2016 Python Software Foundation. + %version%, (c) 2001-2020 Python Software Foundation. CFBundleHelpBookFolder Documentation @@ -55,7 +55,7 @@ NSAppleScriptEnabled NSHumanReadableCopyright - (c) 2001-2016 Python Software Foundation. + (c) 2001-2020 Python Software Foundation. NSHighResolutionCapable diff --git a/Misc/ACKS b/Misc/ACKS index 34a6fc43..a08e917b 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -885,6 +885,7 @@ Vajrasky Kok Guido Kollerie Jacek Kołodziej Jacek Konieczny +Krzysztof Konopko Arkady Koplyarov Peter A. Koren Марк Коренберг @@ -1217,6 +1218,7 @@ Adam Olsen Bryan Olson Grant Olson Koray Oner +Ethan Onstott Piet van Oostrum Tomas Oppelstrup Jason Orendorff @@ -1667,6 +1669,7 @@ Mikhail Terekhov Victor Terrón Pablo Galindo Richard M. Tew +Srinivas Reddy Thatiparthy Tobias Thelen Christian Theune Févry Thibault diff --git a/Misc/NEWS b/Misc/NEWS index 1637f20b..7486b797 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,309 @@ Python News +++++++++++ +What's New in Python 3.8.4 final? +================================= + +*Release date: 2020-07-13* + +Security +-------- + +- bpo-41162: Audit hooks are now cleared later during finalization to avoid + missing events. + +- bpo-29778: Ensure :file:`python3.dll` is loaded from correct locations + when Python is embedded (CVE-2020-15523). + +Core and Builtins +----------------- + +- bpo-41247: Always cache the running loop holder when running + ``asyncio.set_running_loop``. + +- bpo-41252: Fix incorrect refcounting in _ssl.c's + ``_servername_callback()``. + +- bpo-41218: Python 3.8.3 had a regression where compiling with + ast.PyCF_ALLOW_TOP_LEVEL_AWAIT would aggressively mark list comprehension + with CO_COROUTINE. Now only list comprehension making use of async/await + will tagged as so. + +- bpo-41175: Guard against a NULL pointer dereference within bytearrayobject + triggered by the ``bytearray() + bytearray()`` operation. + +- bpo-39960: The "hackcheck" that prevents sneaking around a type's + __setattr__() by calling the superclass method was rewritten to allow C + implemented heap types. + +Library +------- + +- bpo-41235: Fix the error handling in + :meth:`ssl.SSLContext.load_dh_params`. + +- bpo-41193: The ``write_history()`` atexit function of the readline + completer now ignores any :exc:`OSError` to ignore error if the filesystem + is read-only, instead of only ignoring :exc:`FileNotFoundError` and + :exc:`PermissionError`. + +- bpo-41043: Fixed the use of :func:`~glob.glob` in the stdlib: literal part + of the path is now always correctly escaped. + +- bpo-39384: Fixed email.contentmanager to allow set_content() to set a null + string. + +IDLE +---- + +- bpo-37765: Add keywords to module name completion list. Rewrite + Completions section of IDLE doc. + +- bpo-41152: The encoding of ``stdin``, ``stdout`` and ``stderr`` in IDLE is + now always UTF-8. + + +What's New in Python 3.8.4 release candidate 1? +=============================================== + +*Release date: 2020-06-29* + +Security +-------- + +- bpo-41004: The __hash__() methods of ipaddress.IPv4Interface and + ipaddress.IPv6Interface incorrectly generated constant hash values of 32 + and 128 respectively. This resulted in always causing hash collisions. The + fix uses hash() to generate hash values for the tuple of (address, mask + length, network address). + +- bpo-39073: Disallow CR or LF in email.headerregistry.Address arguments to + guard against header injection attacks. + +Core and Builtins +----------------- + +- bpo-41094: Fix decoding errors with audit when open files with non-ASCII + names on non-UTF-8 locale. + +- bpo-41056: Fixes a reference to deallocated stack space during startup + when constructing sys.path involving a relative symlink when code was + supplied via -c. (discovered via Coverity) + +- bpo-35975: Stefan Behnel reported that cf_feature_version is used even + when PyCF_ONLY_AST is not set. This is against the intention and against + the documented behavior, so it's been fixed. + +- bpo-40957: Fix refleak in _Py_fopen_obj() when PySys_Audit() fails + +- bpo-40870: Raise :exc:`ValueError` when validating custom AST's where the + constants ``True``, ``False`` and ``None`` are used within a + :class:`ast.Name` node. + +- bpo-40826: Fix GIL usage in :c:func:`PyOS_Readline`: lock the GIL to set + an exception and pass the Python thread state when checking if there is a + pending signal. + +- bpo-40824: Unexpected errors in calling the ``__iter__`` method are no + longer masked by ``TypeError`` in the :keyword:`in` operator and functions + :func:`~operator.contains`, :func:`~operator.indexOf` and + :func:`~operator.countOf` of the :mod:`operator` module. + +- bpo-40663: Correctly generate annotations where parentheses are omitted + but required (e.g: ``Type[(str, int, *other))]``. + +Library +------- + +- bpo-41138: Fixed the :mod:`trace` module CLI for Python source files with + non-UTF-8 encoding. + +- bpo-31938: Fix default-value signatures of several functions in the + :mod:`select` module - by Anthony Sottile. + +- bpo-41068: Fixed reading files with non-ASCII names from ZIP archive + directly after writing them. + +- bpo-41058: :func:`pdb.find_function` now correctly determines the source + file encoding. + +- bpo-41056: Fix a NULL pointer dereference within the ssl module during a + MemoryError in the keylog callback. (discovered by Coverity) + +- bpo-41048: :func:`mimetypes.read_mime_types` function reads the rule file + using UTF-8 encoding, not the locale encoding. Patch by Srinivas Reddy + Thatiparthy. + +- bpo-40448: :mod:`ensurepip` now disables the use of `pip` cache when + installing the bundled versions of `pip` and `setuptools`. Patch by + Krzysztof Konopko. + +- bpo-40855: The standard deviation and variance functions in the statistics + module were ignoring their mu and xbar arguments. + +- bpo-40807: Stop codeop._maybe_compile, used by code.InteractiveInterpreter + (and IDLE). from from emitting each warning three times. + +- bpo-40834: Fix truncate when sending str object + with_xxsubinterpreters.channel_send. + +- bpo-38488: Update ensurepip to install pip 20.1.1 and setuptools 47.1.0. + +- bpo-40767: :mod:`webbrowser` now properly finds the default browser in + pure Wayland systems by checking the WAYLAND_DISPLAY environment variable. + Patch contributed by Jérémy Attali. + +- bpo-40795: :mod:`ctypes` module: If ctypes fails to convert the result of + a callback or if a ctypes callback function raises an exception, + sys.unraisablehook is now called with an exception set. Previously, the + error was logged into stderr by :c:func:`PyErr_Print`. + +- bpo-30008: Fix :mod:`ssl` code to be compatible with OpenSSL 1.1.x builds + that use ``no-deprecated`` and ``--api=1.1.0``. + +- bpo-40614: :func:`ast.parse` will not parse self documenting expressions + in f-strings when passed ``feature_version`` is less than ``(3, 8)``. + +- bpo-40626: Add h5 file extension as MIME Type application/x-hdf5, as per + HDF Group recommendation for HDF5 formatted data files. Patch contributed + by Mark Schwab. + +- bpo-25872: :mod:`linecache` could crash with a :exc:`KeyError` when + accessed from multiple threads. Fix by Michael Graczyk. + +- bpo-40597: If text content lines are longer than policy.max_line_length, + always use a content-encoding to make sure they are wrapped. + +- bpo-40515: The :mod:`ssl` and :mod:`hashlib` modules now actively check + that OpenSSL is build with thread support. Python 3.7.0 made thread + support mandatory and no longer works safely with a no-thread builds. + +- bpo-13097: ``ctypes`` now raises an ``ArgumentError`` when a callback is + invoked with more than 1024 arguments. + +- bpo-40457: The ssl module now support OpenSSL builds without TLS 1.0 and + 1.1 methods. + +- bpo-39830: Add :class:`zipfile.Path` to ``__all__`` in the :mod:`zipfile` + module. + +- bpo-40025: Raise TypeError when _generate_next_value_ is defined after + members. Patch by Ethan Onstott. + +- bpo-39244: Fixed :class:`multiprocessing.context.get_all_start_methods` to + properly return the default method first on macOS. + +- bpo-39040: Fix parsing of invalid mime headers parameters by collapsing + whitespace between encoded words in a bare-quote-string. + +- bpo-35714: :exc:`struct.error` is now raised if there is a null character + in a :mod:`struct` format string. + +- bpo-36290: AST nodes are now raising :exc:`TypeError` on conflicting + keyword arguments. Patch contributed by Rémi Lapeyre. + +- bpo-29620: :func:`~unittest.TestCase.assertWarns` no longer raises a + ``RuntimeException`` when accessing a module's ``__warningregistry__`` + causes importation of a new module, or when a new module is imported in + another thread. Patch by Kernc. + +- bpo-34226: Fix `cgi.parse_multipart` without content_length. Patch by + Roger Duran + +Tests +----- + +- bpo-41085: Fix integer overflow in the :meth:`array.array.index` method on + 64-bit Windows for index larger than ``2**31``. + +- bpo-38377: On Linux, skip tests using multiprocessing if the current user + cannot create a file in ``/dev/shm/`` directory. Add the + :func:`~test.support.skip_if_broken_multiprocessing_synchronize` function + to the :mod:`test.support` module. + +- bpo-41009: Fix use of ``support.require_{linux|mac|freebsd}_version()`` + decorators as class decorator. + +- bpo-41003: Fix ``test_copyreg`` when ``numpy`` is installed: + ``test.pickletester`` now saves/restores warnings filters when importing + ``numpy``, to ignore filters installed by ``numpy``. + +- bpo-40964: Disable remote :mod:`imaplib` tests, host cyrus.andrew.cmu.edu + is blocking incoming connections. + +- bpo-40055: distutils.tests now saves/restores warnings filters to leave + them unchanged. Importing tests imports docutils which imports + pkg_resources which adds a warnings filter. + +- bpo-34401: Make test_gdb properly run on HP-UX. Patch by Michael Osipov. + +Build +----- + +- bpo-40204: Pin Sphinx version to 2.3.1 in ``Doc/Makefile``. + +- bpo-40653: Move _dirnameW out of HAVE_SYMLINK to fix a potential compiling + issue. + +Windows +------- + +- bpo-41074: Fixed support of non-ASCII names in functions + :func:`msilib.OpenDatabase` and :func:`msilib.init_database` and non-ASCII + SQL in method :meth:`msilib.Database.OpenView`. + +- bpo-40164: Updates Windows OpenSSL to 1.1.1g + +- bpo-39631: Changes the registered MIME type for ``.py`` files on Windows + to ``text/x-python`` instead of ``text/plain``. + +- bpo-40677: Manually define IO_REPARSE_TAG_APPEXECLINK in case some old + Windows SDK doesn't have it. + +- bpo-40650: Include winsock2.h in pytime.c for timeval. + +- bpo-39148: Add IPv6 support to :mod:`asyncio` datagram endpoints in + ProactorEventLoop. Change the raised exception for unknown address + families to ValueError as it's not coming from Windows API. + +macOS +----- + +- bpo-39580: Avoid opening Finder window if running installer from the + command line. Patch contributed by Rick Heil. + +- bpo-41100: Fix configure error when building on macOS 11. Note that the + current Python release was released shortly after the first developer + preview of macOS 11 (Big Sur); there are other known issues with building + and running on the developer preview. Big Sur is expected to be fully + supported in a future bugfix release of Python 3.8.x and with 3.9.0. + +- bpo-41005: fixed an XDG settings issue not allowing macos to open browser + in webbrowser.py + +- bpo-40741: Update macOS installer to use SQLite 3.32.2. + +IDLE +---- + +- bpo-41144: Make Open Module open a special module such as os.path. + +- bpo-39885: Make context menu Cut and Copy work again when right-clicking + within a selection. + +- bpo-40723: Make test_idle pass when run after import. + +Tools/Demos +----------- + +- bpo-40479: Update multissltest helper to test with latest OpenSSL 1.0.2, + 1.1.0, 1.1.1, and 3.0.0-alpha. + +- bpo-40163: Fix multissltest tool. OpenSSL has changed download URL for old + releases. The multissltest tool now tries to download from current and old + download URLs. + + What's New in Python 3.8.3 final? ================================= diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 8e1cd4f5..4ed2af55 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -289,9 +289,6 @@ error: static int set_running_loop(PyObject *loop) { - cached_running_holder = NULL; - cached_running_holder_tsid = 0; - PyObject *ts_dict = PyThreadState_GetDict(); // borrowed if (ts_dict == NULL) { PyErr_SetString( @@ -312,6 +309,12 @@ set_running_loop(PyObject *loop) } Py_DECREF(rl); + cached_running_holder = (PyObject *)rl; + + /* safe to assume state is not NULL as the call to PyThreadState_GetDict() + above already checks if state is NULL */ + cached_running_holder_tsid = PyThreadState_Get()->id; + return 0; } diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index d2d9a658..2a364d6c 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -213,9 +213,6 @@ static void _CallPythonObject(void *mem, pArgs++; } -#define CHECK(what, x) \ -if (x == NULL) _PyTraceback_Add(what, "_ctypes/callbacks.c", __LINE__ - 1), PyErr_Print() - if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { error_object = _ctypes_get_errobj(&space); if (error_object == NULL) @@ -235,7 +232,10 @@ if (x == NULL) _PyTraceback_Add(what, "_ctypes/callbacks.c", __LINE__ - 1), PyEr } result = PyObject_CallObject(callable, arglist); - CHECK("'calling callback function'", result); + if (result == NULL) { + _PyErr_WriteUnraisableMsg("on calling ctypes callback function", + callable); + } #ifdef MS_WIN32 if (flags & FUNCFLAG_USE_LASTERROR) { @@ -251,16 +251,17 @@ if (x == NULL) _PyTraceback_Add(what, "_ctypes/callbacks.c", __LINE__ - 1), PyEr } Py_XDECREF(error_object); - if ((restype != &ffi_type_void) && result) { - PyObject *keep; + if (restype != &ffi_type_void && result) { assert(setfunc); + #ifdef WORDS_BIGENDIAN - /* See the corresponding code in callproc.c, around line 961 */ - if (restype->type != FFI_TYPE_FLOAT && restype->size < sizeof(ffi_arg)) + /* See the corresponding code in _ctypes_callproc(): + in callproc.c, around line 1219. */ + if (restype->type != FFI_TYPE_FLOAT && restype->size < sizeof(ffi_arg)) { mem = (char *)mem + sizeof(ffi_arg) - restype->size; + } #endif - keep = setfunc(mem, result, 0); - CHECK("'converting callback result'", keep); + /* keep is an object we have to keep alive so that the result stays valid. If there is no such object, the setfunc will have returned Py_None. @@ -270,18 +271,32 @@ if (x == NULL) _PyTraceback_Add(what, "_ctypes/callbacks.c", __LINE__ - 1), PyEr be the result. EXCEPT when restype is py_object - Python itself knows how to manage the refcount of these objects. */ - if (keep == NULL) /* Could not convert callback result. */ - PyErr_WriteUnraisable(callable); - else if (keep == Py_None) /* Nothing to keep */ + PyObject *keep = setfunc(mem, result, 0); + + if (keep == NULL) { + /* Could not convert callback result. */ + _PyErr_WriteUnraisableMsg("on converting result " + "of ctypes callback function", + callable); + } + else if (keep == Py_None) { + /* Nothing to keep */ Py_DECREF(keep); + } else if (setfunc != _ctypes_get_fielddesc("O")->setfunc) { if (-1 == PyErr_WarnEx(PyExc_RuntimeWarning, "memory leak in callback function.", 1)) - PyErr_WriteUnraisable(callable); + { + _PyErr_WriteUnraisableMsg("on converting result " + "of ctypes callback function", + callable); + } } } + Py_XDECREF(result); + Done: Py_XDECREF(arglist); PyGILState_Release(state); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 4027bdb6..a9b8675c 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1073,6 +1073,14 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) #define IS_PASS_BY_REF(x) (x > 8 || !POW2(x)) #endif +/* + * bpo-13097: Max number of arguments _ctypes_callproc will accept. + * + * This limit is enforced for the `alloca()` call in `_ctypes_callproc`, + * to avoid allocating a massive buffer on the stack. + */ +#define CTYPES_MAX_ARGCOUNT 1024 + /* * Requirements, must be ensured by the caller: * - argtuple is tuple of arguments @@ -1108,6 +1116,13 @@ PyObject *_ctypes_callproc(PPROC pProc, ++argcount; #endif + if (argcount > CTYPES_MAX_ARGCOUNT) + { + PyErr_Format(PyExc_ArgError, "too many arguments (%zi), maximum is %i", + argcount, CTYPES_MAX_ARGCOUNT); + return NULL; + } + args = (struct argument *)alloca(sizeof(struct argument) * argcount); if (!args) { PyErr_NoMemory(); @@ -1217,7 +1232,9 @@ PyObject *_ctypes_callproc(PPROC pProc, if (rtype->type != FFI_TYPE_FLOAT && rtype->type != FFI_TYPE_STRUCT && rtype->size < sizeof(ffi_arg)) + { resbuf = (char *)resbuf + sizeof(ffi_arg) - rtype->size; + } #endif #ifdef MS_WIN32 @@ -1399,15 +1416,12 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args) if (name != Py_None) { if (PyUnicode_FSConverter(name, &name2) == 0) return NULL; - if (PyBytes_Check(name2)) - name_str = PyBytes_AS_STRING(name2); - else - name_str = PyByteArray_AS_STRING(name2); + name_str = PyBytes_AS_STRING(name2); } else { name_str = NULL; name2 = NULL; } - if (PySys_Audit("ctypes.dlopen", "s", name_str) < 0) { + if (PySys_Audit("ctypes.dlopen", "O", name) < 0) { return NULL; } handle = ctypes_dlopen(name_str, mode); diff --git a/Modules/_decimal/libmpdec/mpdecimal.c b/Modules/_decimal/libmpdec/mpdecimal.c index 0986edb5..bfa8bb34 100644 --- a/Modules/_decimal/libmpdec/mpdecimal.c +++ b/Modules/_decimal/libmpdec/mpdecimal.c @@ -3781,43 +3781,6 @@ mpd_qdiv(mpd_t *q, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status) { _mpd_qdiv(SET_IDEAL_EXP, q, a, b, ctx, status); - - if (*status & MPD_Malloc_error) { - /* Inexact quotients (the usual case) fill the entire context precision, - * which can lead to malloc() failures for very high precisions. Retry - * the operation with a lower precision in case the result is exact. - * - * We need an upper bound for the number of digits of a_coeff / b_coeff - * when the result is exact. If a_coeff' * 1 / b_coeff' is in lowest - * terms, then maxdigits(a_coeff') + maxdigits(1 / b_coeff') is a suitable - * bound. - * - * 1 / b_coeff' is exact iff b_coeff' exclusively has prime factors 2 or 5. - * The largest amount of digits is generated if b_coeff' is a power of 2 or - * a power of 5 and is less than or equal to log5(b_coeff') <= log2(b_coeff'). - * - * We arrive at a total upper bound: - * - * maxdigits(a_coeff') + maxdigits(1 / b_coeff') <= - * a->digits + log2(b_coeff) = - * a->digits + log10(b_coeff) / log10(2) <= - * a->digits + b->digits * 4; - */ - uint32_t workstatus = 0; - mpd_context_t workctx = *ctx; - workctx.prec = a->digits + b->digits * 4; - if (workctx.prec >= ctx->prec) { - return; /* No point in retrying, keep the original error. */ - } - - _mpd_qdiv(SET_IDEAL_EXP, q, a, b, &workctx, &workstatus); - if (workstatus == 0) { /* The result is exact, unrounded, normal etc. */ - *status = 0; - return; - } - - mpd_seterror(q, *status, status); - } } /* Internal function. */ @@ -7739,9 +7702,9 @@ mpd_qinvroot(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, /* END LIBMPDEC_ONLY */ /* Algorithm from decimal.py */ -static void -_mpd_qsqrt(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) +void +mpd_qsqrt(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, + uint32_t *status) { mpd_context_t maxcontext; MPD_NEW_STATIC(c,0,0,0,0); @@ -7873,40 +7836,6 @@ malloc_error: goto out; } -void -mpd_qsqrt(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - _mpd_qsqrt(result, a, ctx, status); - - if (*status & (MPD_Malloc_error|MPD_Division_impossible)) { - /* The above conditions can occur at very high context precisions - * if intermediate values get too large. Retry the operation with - * a lower context precision in case the result is exact. - * - * If the result is exact, an upper bound for the number of digits - * is the number of digits in the input. - * - * NOTE: sqrt(40e9) = 2.0e+5 /\ digits(40e9) = digits(2.0e+5) = 2 - */ - uint32_t workstatus = 0; - mpd_context_t workctx = *ctx; - workctx.prec = a->digits; - - if (workctx.prec >= ctx->prec) { - return; /* No point in repeating this, keep the original error. */ - } - - _mpd_qsqrt(result, a, &workctx, &workstatus); - if (workstatus == 0) { - *status = 0; - return; - } - - mpd_seterror(result, *status, status); - } -} - /******************************************************************************/ /* Base conversions */ diff --git a/Modules/_decimal/tests/deccheck.py b/Modules/_decimal/tests/deccheck.py index 5cd5db57..f907531e 100644 --- a/Modules/_decimal/tests/deccheck.py +++ b/Modules/_decimal/tests/deccheck.py @@ -125,12 +125,6 @@ ContextFunctions = { 'special': ('context.__reduce_ex__', 'context.create_decimal_from_float') } -# Functions that set no context flags but whose result can differ depending -# on prec, Emin and Emax. -MaxContextSkip = ['is_normal', 'is_subnormal', 'logical_invert', 'next_minus', - 'next_plus', 'number_class', 'logical_and', 'logical_or', - 'logical_xor', 'next_toward', 'rotate', 'shift'] - # Functions that require a restricted exponent range for reasonable runtimes. UnaryRestricted = [ '__ceil__', '__floor__', '__int__', '__trunc__', @@ -350,20 +344,6 @@ class TestSet(object): self.pex = RestrictedList() # Python exceptions for P.Decimal self.presults = RestrictedList() # P.Decimal results - # If the above results are exact, unrounded and not clamped, repeat - # the operation with a maxcontext to ensure that huge intermediate - # values do not cause a MemoryError. - self.with_maxcontext = False - self.maxcontext = context.c.copy() - self.maxcontext.prec = C.MAX_PREC - self.maxcontext.Emax = C.MAX_EMAX - self.maxcontext.Emin = C.MIN_EMIN - self.maxcontext.clear_flags() - - self.maxop = RestrictedList() # converted C.Decimal operands - self.maxex = RestrictedList() # Python exceptions for C.Decimal - self.maxresults = RestrictedList() # C.Decimal results - # ====================================================================== # SkipHandler: skip known discrepancies @@ -565,17 +545,13 @@ def function_as_string(t): if t.contextfunc: cargs = t.cop pargs = t.pop - maxargs = t.maxop cfunc = "c_func: %s(" % t.funcname pfunc = "p_func: %s(" % t.funcname - maxfunc = "max_func: %s(" % t.funcname else: cself, cargs = t.cop[0], t.cop[1:] pself, pargs = t.pop[0], t.pop[1:] - maxself, maxargs = t.maxop[0], t.maxop[1:] cfunc = "c_func: %s.%s(" % (repr(cself), t.funcname) pfunc = "p_func: %s.%s(" % (repr(pself), t.funcname) - maxfunc = "max_func: %s.%s(" % (repr(maxself), t.funcname) err = cfunc for arg in cargs: @@ -589,14 +565,6 @@ def function_as_string(t): err = err.rstrip(", ") err += ")" - if t.with_maxcontext: - err += "\n" - err += maxfunc - for arg in maxargs: - err += "%s, " % repr(arg) - err = err.rstrip(", ") - err += ")" - return err def raise_error(t): @@ -609,24 +577,9 @@ def raise_error(t): err = "Error in %s:\n\n" % t.funcname err += "input operands: %s\n\n" % (t.op,) err += function_as_string(t) - - err += "\n\nc_result: %s\np_result: %s\n" % (t.cresults, t.presults) - if t.with_maxcontext: - err += "max_result: %s\n\n" % (t.maxresults) - else: - err += "\n" - - err += "c_exceptions: %s\np_exceptions: %s\n" % (t.cex, t.pex) - if t.with_maxcontext: - err += "max_exceptions: %s\n\n" % t.maxex - else: - err += "\n" - - err += "%s\n" % str(t.context) - if t.with_maxcontext: - err += "%s\n" % str(t.maxcontext) - else: - err += "\n" + err += "\n\nc_result: %s\np_result: %s\n\n" % (t.cresults, t.presults) + err += "c_exceptions: %s\np_exceptions: %s\n\n" % (t.cex, t.pex) + err += "%s\n\n" % str(t.context) raise VerifyError(err) @@ -650,13 +603,6 @@ def raise_error(t): # are printed to stdout. # ====================================================================== -def all_nan(a): - if isinstance(a, C.Decimal): - return a.is_nan() - elif isinstance(a, tuple): - return all(all_nan(v) for v in a) - return False - def convert(t, convstr=True): """ t is the testset. At this stage the testset contains a tuple of operands t.op of various types. For decimal methods the first @@ -671,12 +617,10 @@ def convert(t, convstr=True): for i, op in enumerate(t.op): context.clear_status() - t.maxcontext.clear_flags() if op in RoundModes: t.cop.append(op) t.pop.append(op) - t.maxop.append(op) elif not t.contextfunc and i == 0 or \ convstr and isinstance(op, str): @@ -694,25 +638,11 @@ def convert(t, convstr=True): p = None pex = e.__class__ - try: - C.setcontext(t.maxcontext) - maxop = C.Decimal(op) - maxex = None - except (TypeError, ValueError, OverflowError) as e: - maxop = None - maxex = e.__class__ - finally: - C.setcontext(context.c) - t.cop.append(c) t.cex.append(cex) - t.pop.append(p) t.pex.append(pex) - t.maxop.append(maxop) - t.maxex.append(maxex) - if cex is pex: if str(c) != str(p) or not context.assert_eq_status(): raise_error(t) @@ -722,21 +652,14 @@ def convert(t, convstr=True): else: raise_error(t) - # The exceptions in the maxcontext operation can legitimately - # differ, only test that maxex implies cex: - if maxex is not None and cex is not maxex: - raise_error(t) - elif isinstance(op, Context): t.context = op t.cop.append(op.c) t.pop.append(op.p) - t.maxop.append(t.maxcontext) else: t.cop.append(op) t.pop.append(op) - t.maxop.append(op) return 1 @@ -750,7 +673,6 @@ def callfuncs(t): t.rc and t.rp are the results of the operation. """ context.clear_status() - t.maxcontext.clear_flags() try: if t.contextfunc: @@ -778,35 +700,6 @@ def callfuncs(t): t.rp = None t.pex.append(e.__class__) - # If the above results are exact, unrounded, normal etc., repeat the - # operation with a maxcontext to ensure that huge intermediate values - # do not cause a MemoryError. - if (t.funcname not in MaxContextSkip and - not context.c.flags[C.InvalidOperation] and - not context.c.flags[C.Inexact] and - not context.c.flags[C.Rounded] and - not context.c.flags[C.Subnormal] and - not context.c.flags[C.Clamped] and - not context.clamp and # results are padded to context.prec if context.clamp==1. - not any(isinstance(v, C.Context) for v in t.cop)): # another context is used. - t.with_maxcontext = True - try: - if t.contextfunc: - maxargs = t.maxop - t.rmax = getattr(t.maxcontext, t.funcname)(*maxargs) - else: - maxself = t.maxop[0] - maxargs = t.maxop[1:] - try: - C.setcontext(t.maxcontext) - t.rmax = getattr(maxself, t.funcname)(*maxargs) - finally: - C.setcontext(context.c) - t.maxex.append(None) - except (TypeError, ValueError, OverflowError, MemoryError) as e: - t.rmax = None - t.maxex.append(e.__class__) - def verify(t, stat): """ t is the testset. At this stage the testset contains the following tuples: @@ -821,9 +714,6 @@ def verify(t, stat): """ t.cresults.append(str(t.rc)) t.presults.append(str(t.rp)) - if t.with_maxcontext: - t.maxresults.append(str(t.rmax)) - if isinstance(t.rc, C.Decimal) and isinstance(t.rp, P.Decimal): # General case: both results are Decimals. t.cresults.append(t.rc.to_eng_string()) @@ -835,12 +725,6 @@ def verify(t, stat): t.presults.append(str(t.rp.imag)) t.presults.append(str(t.rp.real)) - if t.with_maxcontext and isinstance(t.rmax, C.Decimal): - t.maxresults.append(t.rmax.to_eng_string()) - t.maxresults.append(t.rmax.as_tuple()) - t.maxresults.append(str(t.rmax.imag)) - t.maxresults.append(str(t.rmax.real)) - nc = t.rc.number_class().lstrip('+-s') stat[nc] += 1 else: @@ -848,9 +732,6 @@ def verify(t, stat): if not isinstance(t.rc, tuple) and not isinstance(t.rp, tuple): if t.rc != t.rp: raise_error(t) - if t.with_maxcontext and not isinstance(t.rmax, tuple): - if t.rmax != t.rc: - raise_error(t) stat[type(t.rc).__name__] += 1 # The return value lists must be equal. @@ -863,20 +744,6 @@ def verify(t, stat): if not t.context.assert_eq_status(): raise_error(t) - if t.with_maxcontext: - # NaN payloads etc. depend on precision and clamp. - if all_nan(t.rc) and all_nan(t.rmax): - return - # The return value lists must be equal. - if t.maxresults != t.cresults: - raise_error(t) - # The Python exception lists (TypeError, etc.) must be equal. - if t.maxex != t.cex: - raise_error(t) - # The context flags must be equal. - if t.maxcontext.flags != t.context.c.flags: - raise_error(t) - # ====================================================================== # Main test loops diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 3e5f9c3e..edadbcb3 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -26,6 +26,10 @@ #include #include "openssl/err.h" +#ifndef OPENSSL_THREADS +# error "OPENSSL_THREADS is not defined, Python requires thread-safe OpenSSL" +#endif + #if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER) /* OpenSSL < 1.1.0 */ #define EVP_MD_CTX_new EVP_MD_CTX_create diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 43b236c2..1944393e 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -75,6 +75,10 @@ static PySocketModule_APIObject PySocketModule; # endif #endif +#ifndef OPENSSL_THREADS +# error "OPENSSL_THREADS is not defined, Python requires thread-safe OpenSSL" +#endif + /* SSL error object */ static PyObject *PySSLErrorObject; static PyObject *PySSLCertVerificationErrorObject; @@ -140,20 +144,29 @@ static void _PySSLFixErrno(void) { # define PY_OPENSSL_1_1_API 1 #endif +/* OpenSSL API compat */ +#ifdef OPENSSL_API_COMPAT +#if OPENSSL_API_COMPAT >= 0x10100000L + +/* OpenSSL API 1.1.0+ does not include version methods */ +#ifndef OPENSSL_NO_TLS1_METHOD +#define OPENSSL_NO_TLS1_METHOD 1 +#endif +#ifndef OPENSSL_NO_TLS1_1_METHOD +#define OPENSSL_NO_TLS1_1_METHOD 1 +#endif +#ifndef OPENSSL_NO_TLS1_2_METHOD +#define OPENSSL_NO_TLS1_2_METHOD 1 +#endif + +#endif /* >= 1.1.0 compcat */ +#endif /* OPENSSL_API_COMPAT */ + /* LibreSSL 2.7.0 provides necessary OpenSSL 1.1.0 APIs */ #if defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER >= 0x2070000fL # define PY_OPENSSL_1_1_API 1 #endif -/* Openssl comes with TLSv1.1 and TLSv1.2 between 1.0.0h and 1.0.1 - http://www.openssl.org/news/changelog.html - */ -#if OPENSSL_VERSION_NUMBER >= 0x10001000L -# define HAVE_TLSv1_2 1 -#else -# define HAVE_TLSv1_2 0 -#endif - /* SNI support (client- and server-side) appeared in OpenSSL 1.0.0 and 0.9.8f * This includes the SSL_set_SSL_CTX() function. */ @@ -208,6 +221,12 @@ static void _PySSLFixErrno(void) { #define TLS_method SSLv23_method #define TLS_client_method SSLv23_client_method #define TLS_server_method SSLv23_server_method +#define ASN1_STRING_get0_data ASN1_STRING_data +#define X509_get0_notBefore X509_get_notBefore +#define X509_get0_notAfter X509_get_notAfter +#define OpenSSL_version_num SSLeay +#define OpenSSL_version SSLeay_version +#define OPENSSL_VERSION SSLEAY_VERSION static int X509_NAME_ENTRY_set(const X509_NAME_ENTRY *ne) { @@ -324,13 +343,9 @@ enum py_ssl_version { PY_SSL_VERSION_SSL2, PY_SSL_VERSION_SSL3=1, PY_SSL_VERSION_TLS, /* SSLv23 */ -#if HAVE_TLSv1_2 PY_SSL_VERSION_TLS1, PY_SSL_VERSION_TLS1_1, PY_SSL_VERSION_TLS1_2, -#else - PY_SSL_VERSION_TLS1, -#endif PY_SSL_VERSION_TLS_CLIENT=0x10, PY_SSL_VERSION_TLS_SERVER, }; @@ -896,7 +911,7 @@ _ssl_configure_hostname(PySSLSocket *self, const char* server_hostname) goto error; } } else { - if (!X509_VERIFY_PARAM_set1_ip(param, ASN1_STRING_data(ip), + if (!X509_VERIFY_PARAM_set1_ip(param, ASN1_STRING_get0_data(ip), ASN1_STRING_length(ip))) { _setSSLError(NULL, 0, __FILE__, __LINE__); goto error; @@ -1372,7 +1387,7 @@ _get_peer_alt_names (X509 *certificate) { goto fail; } PyTuple_SET_ITEM(t, 0, v); - v = PyUnicode_FromStringAndSize((char *)ASN1_STRING_data(as), + v = PyUnicode_FromStringAndSize((char *)ASN1_STRING_get0_data(as), ASN1_STRING_length(as)); if (v == NULL) { Py_DECREF(t); @@ -1668,7 +1683,7 @@ _decode_certificate(X509 *certificate) { ASN1_INTEGER *serialNumber; char buf[2048]; int len, result; - ASN1_TIME *notBefore, *notAfter; + const ASN1_TIME *notBefore, *notAfter; PyObject *pnotBefore, *pnotAfter; retval = PyDict_New(); @@ -1730,7 +1745,7 @@ _decode_certificate(X509 *certificate) { Py_DECREF(sn_obj); (void) BIO_reset(biobuf); - notBefore = X509_get_notBefore(certificate); + notBefore = X509_get0_notBefore(certificate); ASN1_TIME_print(biobuf, notBefore); len = BIO_gets(biobuf, buf, sizeof(buf)-1); if (len < 0) { @@ -1747,7 +1762,7 @@ _decode_certificate(X509 *certificate) { Py_DECREF(pnotBefore); (void) BIO_reset(biobuf); - notAfter = X509_get_notAfter(certificate); + notAfter = X509_get0_notAfter(certificate); ASN1_TIME_print(biobuf, notAfter); len = BIO_gets(biobuf, buf, sizeof(buf)-1); if (len < 0) { @@ -3084,35 +3099,51 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) #endif PySSL_BEGIN_ALLOW_THREADS - if (proto_version == PY_SSL_VERSION_TLS1) + switch(proto_version) { +#if defined(SSL3_VERSION) && !defined(OPENSSL_NO_SSL3) + case PY_SSL_VERSION_SSL3: + ctx = SSL_CTX_new(SSLv3_method()); + break; +#endif +#if (defined(TLS1_VERSION) && \ + !defined(OPENSSL_NO_TLS1) && \ + !defined(OPENSSL_NO_TLS1_METHOD)) + case PY_SSL_VERSION_TLS1: ctx = SSL_CTX_new(TLSv1_method()); -#if HAVE_TLSv1_2 - else if (proto_version == PY_SSL_VERSION_TLS1_1) - ctx = SSL_CTX_new(TLSv1_1_method()); - else if (proto_version == PY_SSL_VERSION_TLS1_2) - ctx = SSL_CTX_new(TLSv1_2_method()); + break; #endif -#ifndef OPENSSL_NO_SSL3 - else if (proto_version == PY_SSL_VERSION_SSL3) - ctx = SSL_CTX_new(SSLv3_method()); +#if (defined(TLS1_1_VERSION) && \ + !defined(OPENSSL_NO_TLS1_1) && \ + !defined(OPENSSL_NO_TLS1_1_METHOD)) + case PY_SSL_VERSION_TLS1_1: + ctx = SSL_CTX_new(TLSv1_1_method()); + break; #endif -#ifndef OPENSSL_NO_SSL2 - else if (proto_version == PY_SSL_VERSION_SSL2) - ctx = SSL_CTX_new(SSLv2_method()); +#if (defined(TLS1_2_VERSION) && \ + !defined(OPENSSL_NO_TLS1_2) && \ + !defined(OPENSSL_NO_TLS1_2_METHOD)) + case PY_SSL_VERSION_TLS1_2: + ctx = SSL_CTX_new(TLSv1_2_method()); + break; #endif - else if (proto_version == PY_SSL_VERSION_TLS) /* SSLv23 */ + case PY_SSL_VERSION_TLS: + /* SSLv23 */ ctx = SSL_CTX_new(TLS_method()); - else if (proto_version == PY_SSL_VERSION_TLS_CLIENT) + break; + case PY_SSL_VERSION_TLS_CLIENT: ctx = SSL_CTX_new(TLS_client_method()); - else if (proto_version == PY_SSL_VERSION_TLS_SERVER) + break; + case PY_SSL_VERSION_TLS_SERVER: ctx = SSL_CTX_new(TLS_server_method()); - else + break; + default: proto_version = -1; + } PySSL_END_ALLOW_THREADS if (proto_version == -1) { PyErr_SetString(PyExc_ValueError, - "invalid protocol version"); + "invalid or unsupported protocol version"); return NULL; } if (ctx == NULL) { @@ -3208,7 +3239,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) conservative and assume it wasn't fixed until release. We do this check at runtime to avoid problems from the dynamic linker. See #25672 for more on this. */ - libver = SSLeay(); + libver = OpenSSL_version_num(); if (!(libver >= 0x10001000UL && libver < 0x1000108fUL) && !(libver >= 0x10000000UL && libver < 0x100000dfUL)) { SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS); @@ -4281,8 +4312,10 @@ _ssl__SSLContext_load_dh_params(PySSLContext *self, PyObject *filepath) } return NULL; } - if (SSL_CTX_set_tmp_dh(self->ctx, dh) == 0) - _setSSLError(NULL, 0, __FILE__, __LINE__); + if (!SSL_CTX_set_tmp_dh(self->ctx, dh)) { + DH_free(dh); + return _setSSLError(NULL, 0, __FILE__, __LINE__); + } DH_free(dh); Py_RETURN_NONE; } @@ -4515,11 +4548,12 @@ _servername_callback(SSL *s, int *al, void *args) * back into a str object, but still as an A-label (bpo-28414) */ servername_str = PyUnicode_FromEncodedObject(servername_bytes, "ascii", NULL); - Py_DECREF(servername_bytes); if (servername_str == NULL) { PyErr_WriteUnraisable(servername_bytes); + Py_DECREF(servername_bytes); goto error; } + Py_DECREF(servername_bytes); result = PyObject_CallFunctionObjArgs( ssl_ctx->set_sni_cb, ssl_socket, servername_str, ssl_ctx, NULL); @@ -5288,7 +5322,11 @@ PySSL_RAND(int len, int pseudo) if (bytes == NULL) return NULL; if (pseudo) { +#ifdef PY_OPENSSL_1_1_API + ok = RAND_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); +#else ok = RAND_pseudo_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); +#endif if (ok == 0 || ok == 1) return Py_BuildValue("NO", bytes, ok == 1 ? Py_True : Py_False); } @@ -6008,7 +6046,7 @@ PyInit__ssl(void) if (!_setup_ssl_threads()) { return NULL; } -#elif OPENSSL_VERSION_1_1 && defined(OPENSSL_THREADS) +#elif OPENSSL_VERSION_1_1 /* OpenSSL 1.1.0 builtin thread support is enabled */ _ssl_locks_count++; #endif @@ -6184,12 +6222,10 @@ PyInit__ssl(void) PY_SSL_VERSION_TLS_SERVER); PyModule_AddIntConstant(m, "PROTOCOL_TLSv1", PY_SSL_VERSION_TLS1); -#if HAVE_TLSv1_2 PyModule_AddIntConstant(m, "PROTOCOL_TLSv1_1", PY_SSL_VERSION_TLS1_1); PyModule_AddIntConstant(m, "PROTOCOL_TLSv1_2", PY_SSL_VERSION_TLS1_2); -#endif /* protocol options */ PyModule_AddIntConstant(m, "OP_ALL", @@ -6197,10 +6233,8 @@ PyInit__ssl(void) PyModule_AddIntConstant(m, "OP_NO_SSLv2", SSL_OP_NO_SSLv2); PyModule_AddIntConstant(m, "OP_NO_SSLv3", SSL_OP_NO_SSLv3); PyModule_AddIntConstant(m, "OP_NO_TLSv1", SSL_OP_NO_TLSv1); -#if HAVE_TLSv1_2 PyModule_AddIntConstant(m, "OP_NO_TLSv1_1", SSL_OP_NO_TLSv1_1); PyModule_AddIntConstant(m, "OP_NO_TLSv1_2", SSL_OP_NO_TLSv1_2); -#endif #ifdef SSL_OP_NO_TLSv1_3 PyModule_AddIntConstant(m, "OP_NO_TLSv1_3", SSL_OP_NO_TLSv1_3); #else @@ -6379,7 +6413,7 @@ PyInit__ssl(void) /* SSLeay() gives us the version of the library linked against, which could be different from the headers version. */ - libver = SSLeay(); + libver = OpenSSL_version_num(); r = PyLong_FromUnsignedLong(libver); if (r == NULL) return NULL; @@ -6389,7 +6423,7 @@ PyInit__ssl(void) r = Py_BuildValue("IIIII", major, minor, fix, patch, status); if (r == NULL || PyModule_AddObject(m, "OPENSSL_VERSION_INFO", r)) return NULL; - r = PyUnicode_FromString(SSLeay_version(SSLEAY_VERSION)); + r = PyUnicode_FromString(OpenSSL_version(OPENSSL_VERSION)); if (r == NULL || PyModule_AddObject(m, "OPENSSL_VERSION", r)) return NULL; diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 858b3d79..b840da2f 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -125,6 +125,12 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) threadstate = PyGILState_Ensure(); + ssl_obj = (PySSLSocket *)SSL_get_app_data(ssl); + assert(PySSLSocket_Check(ssl_obj)); + if (ssl_obj->ctx->keylog_bio == NULL) { + return; + } + /* Allocate a static lock to synchronize writes to keylog file. * The lock is neither released on exit nor on fork(). The lock is * also shared between all SSLContexts although contexts may write to @@ -141,12 +147,6 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) } } - ssl_obj = (PySSLSocket *)SSL_get_app_data(ssl); - assert(PySSLSocket_Check(ssl_obj)); - if (ssl_obj->ctx->keylog_bio == NULL) { - return; - } - PySSL_BEGIN_ALLOW_THREADS PyThread_acquire_lock(lock, 1); res = BIO_printf(ssl_obj->ctx->keylog_bio, "%s\n", line); diff --git a/Modules/_stat.c b/Modules/_stat.c index 6a3020a0..7a799af0 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -40,6 +40,10 @@ typedef unsigned short mode_t; # define FILE_ATTRIBUTE_NO_SCRUB_DATA 0x20000 #endif +#ifndef IO_REPARSE_TAG_APPEXECLINK +# define IO_REPARSE_TAG_APPEXECLINK 0x8000001BL +#endif + #endif /* MS_WINDOWS */ /* From Python's stat.py */ diff --git a/Modules/_struct.c b/Modules/_struct.c index 1c917b75..64a9827e 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1285,6 +1285,10 @@ prepare_s(PyStructObject *self) size_t ncodes; fmt = PyBytes_AS_STRING(self->s_format); + if (strlen(fmt) != (size_t)PyBytes_GET_SIZE(self->s_format)) { + PyErr_SetString(StructError, "embedded null character"); + return -1; + } f = whichtable(&fmt); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index da3579c2..af28af50 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3345,6 +3345,8 @@ run_in_subinterp(PyObject *self, PyObject *args) const char *code; int r; PyThreadState *substate, *mainstate; + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; if (!PyArg_ParseTuple(args, "s:run_in_subinterp", &code)) @@ -3363,7 +3365,7 @@ run_in_subinterp(PyObject *self, PyObject *args) PyErr_SetString(PyExc_RuntimeError, "sub-interpreter creation failed"); return NULL; } - r = PyRun_SimpleString(code); + r = PyRun_SimpleStringFlags(code, &cflags); Py_EndInterpreter(substate); PyThreadState_Swap(mainstate); @@ -6175,6 +6177,79 @@ static PyType_Spec HeapCTypeSubclassWithFinalizer_spec = { HeapCTypeSubclassWithFinalizer_slots }; +PyDoc_STRVAR(heapctypesetattr__doc__, +"A heap type without GC, but with overridden __setattr__.\n\n" +"The 'value' attribute is set to 10 in __init__ and updated via attribute setting."); + +typedef struct { + PyObject_HEAD + long value; +} HeapCTypeSetattrObject; + +static struct PyMemberDef heapctypesetattr_members[] = { + {"pvalue", T_LONG, offsetof(HeapCTypeSetattrObject, value)}, + {NULL} /* Sentinel */ +}; + +static int +heapctypesetattr_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + ((HeapCTypeSetattrObject *)self)->value = 10; + return 0; +} + +static void +heapctypesetattr_dealloc(HeapCTypeSetattrObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_Del(self); + Py_DECREF(tp); +} + +static int +heapctypesetattr_setattro(HeapCTypeSetattrObject *self, PyObject *attr, PyObject *value) +{ + PyObject *svalue = PyUnicode_FromString("value"); + if (svalue == NULL) + return -1; + int eq = PyObject_RichCompareBool(svalue, attr, Py_EQ); + Py_DECREF(svalue); + if (eq < 0) + return -1; + if (!eq) { + return PyObject_GenericSetAttr((PyObject*) self, attr, value); + } + if (value == NULL) { + self->value = 0; + return 0; + } + PyObject *ivalue = PyNumber_Long(value); + if (ivalue == NULL) + return -1; + long v = PyLong_AsLong(ivalue); + Py_DECREF(ivalue); + if (v == -1 && PyErr_Occurred()) + return -1; + self->value = v; + return 0; +} + +static PyType_Slot HeapCTypeSetattr_slots[] = { + {Py_tp_init, heapctypesetattr_init}, + {Py_tp_members, heapctypesetattr_members}, + {Py_tp_setattro, heapctypesetattr_setattro}, + {Py_tp_dealloc, heapctypesetattr_dealloc}, + {Py_tp_doc, (char*)heapctypesetattr__doc__}, + {0, 0}, +}; + +static PyType_Spec HeapCTypeSetattr_spec = { + "_testcapi.HeapCTypeSetattr", + sizeof(HeapCTypeSetattrObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + HeapCTypeSetattr_slots +}; static struct PyModuleDef _testcapimodule = { PyModuleDef_HEAD_INIT, @@ -6334,6 +6409,12 @@ PyInit__testcapi(void) Py_DECREF(subclass_bases); PyModule_AddObject(m, "HeapCTypeSubclass", HeapCTypeSubclass); + PyObject *HeapCTypeSetattr = PyType_FromSpec(&HeapCTypeSetattr_spec); + if (HeapCTypeSetattr == NULL) { + return NULL; + } + PyModule_AddObject(m, "HeapCTypeSetattr", HeapCTypeSetattr); + PyObject *subclass_with_finalizer_bases = PyTuple_Pack(1, HeapCTypeSubclass); if (subclass_with_finalizer_bases == NULL) { return NULL; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 3ea77e69..3a931cae 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -12,10 +12,53 @@ #include "pycore_initconfig.h" +#ifdef MS_WINDOWS +#include + +static int +_add_windows_config(PyObject *configs) +{ + HMODULE hPython3; + wchar_t py3path[MAX_PATH]; + PyObject *dict = PyDict_New(); + PyObject *obj = NULL; + if (!dict) { + return -1; + } + + hPython3 = GetModuleHandleW(PY3_DLLNAME); + if (hPython3 && GetModuleFileNameW(hPython3, py3path, MAX_PATH)) { + obj = PyUnicode_FromWideChar(py3path, -1); + } else { + obj = Py_None; + Py_INCREF(obj); + } + if (obj && + !PyDict_SetItemString(dict, "python3_dll", obj) && + !PyDict_SetItemString(configs, "windows", dict)) { + Py_DECREF(obj); + Py_DECREF(dict); + return 0; + } + Py_DECREF(obj); + Py_DECREF(dict); + return -1; +} +#endif + + static PyObject * get_configs(PyObject *self, PyObject *Py_UNUSED(args)) { - return _Py_GetConfigsAsDict(); + PyObject *dict = _Py_GetConfigsAsDict(); +#ifdef MS_WINDOWS + if (dict) { + if (_add_windows_config(dict) < 0) { + Py_CLEAR(dict); + } + } +#endif + return dict; } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 5289ea0e..abcdd1e8 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -1136,7 +1136,7 @@ array_array_index(arrayobject *self, PyObject *v) cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); Py_DECREF(selfi); if (cmp > 0) { - return PyLong_FromLong((long)i); + return PyLong_FromSsize_t(i); } else if (cmp < 0) return NULL; diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 9015816f..a9e14840 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -8,7 +8,7 @@ PyDoc_STRVAR(select_select__doc__, "\n" "Wait until one or more file descriptors are ready for some kind of I/O.\n" "\n" -"The first three arguments are sequences of file descriptors to be waited for:\n" +"The first three arguments are iterables of file descriptors to be waited for:\n" "rlist -- wait until ready for reading\n" "wlist -- wait until ready for writing\n" "xlist -- wait for an \"exceptional condition\"\n" @@ -65,7 +65,8 @@ exit: #if (defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)) PyDoc_STRVAR(select_poll_register__doc__, -"register($self, fd, eventmask=POLLIN | POLLPRI | POLLOUT, /)\n" +"register($self, fd,\n" +" eventmask=select.POLLIN | select.POLLPRI | select.POLLOUT, /)\n" "--\n" "\n" "Register a file descriptor with the polling object.\n" @@ -226,7 +227,8 @@ exit: #if (defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)) && defined(HAVE_SYS_DEVPOLL_H) PyDoc_STRVAR(select_devpoll_register__doc__, -"register($self, fd, eventmask=POLLIN | POLLPRI | POLLOUT, /)\n" +"register($self, fd,\n" +" eventmask=select.POLLIN | select.POLLPRI | select.POLLOUT, /)\n" "--\n" "\n" "Register a file descriptor with the polling object.\n" @@ -275,7 +277,8 @@ exit: #if (defined(HAVE_POLL) && !defined(HAVE_BROKEN_POLL)) && defined(HAVE_SYS_DEVPOLL_H) PyDoc_STRVAR(select_devpoll_modify__doc__, -"modify($self, fd, eventmask=POLLIN | POLLPRI | POLLOUT, /)\n" +"modify($self, fd,\n" +" eventmask=select.POLLIN | select.POLLPRI | select.POLLOUT, /)\n" "--\n" "\n" "Modify a possible already registered file descriptor.\n" @@ -645,7 +648,8 @@ exit: #if defined(HAVE_EPOLL) PyDoc_STRVAR(select_epoll_register__doc__, -"register($self, /, fd, eventmask=EPOLLIN | EPOLLPRI | EPOLLOUT)\n" +"register($self, /, fd,\n" +" eventmask=select.EPOLLIN | select.EPOLLPRI | select.EPOLLOUT)\n" "--\n" "\n" "Registers a new fd or raises an OSError if the fd is already registered.\n" @@ -1215,4 +1219,4 @@ exit: #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=03041f3d09b04a3d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=86010dde10ca89c6 input=a9049054013a1b77]*/ diff --git a/Modules/main.c b/Modules/main.c index 2a360b58..70be4cfa 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -387,29 +387,70 @@ pymain_run_file(PyConfig *config, PyCompilerFlags *cf) static int pymain_run_startup(PyConfig *config, PyCompilerFlags *cf, int *exitcode) { + int ret; + PyObject *startup_obj = NULL; + if (!config->use_environment) { + return 0; + } +#ifdef MS_WINDOWS + const wchar_t *wstartup = _wgetenv(L"PYTHONSTARTUP"); + if (wstartup == NULL || wstartup[0] == L'\0') { + return 0; + } + PyObject *startup_bytes = NULL; + startup_obj = PyUnicode_FromWideChar(wstartup, wcslen(wstartup)); + if (startup_obj == NULL) { + goto error; + } + startup_bytes = PyUnicode_EncodeFSDefault(startup_obj); + if (startup_bytes == NULL) { + goto error; + } + const char *startup = PyBytes_AS_STRING(startup_bytes); +#else const char *startup = _Py_GetEnv(config->use_environment, "PYTHONSTARTUP"); if (startup == NULL) { return 0; } - if (PySys_Audit("cpython.run_startup", "s", startup) < 0) { - return pymain_err_print(exitcode); + startup_obj = PyUnicode_DecodeFSDefault(startup); + if (startup_obj == NULL) { + goto error; + } +#endif + if (PySys_Audit("cpython.run_startup", "O", startup_obj) < 0) { + goto error; } +#ifdef MS_WINDOWS + FILE *fp = _Py_wfopen(wstartup, L"r"); +#else FILE *fp = _Py_fopen(startup, "r"); +#endif if (fp == NULL) { int save_errno = errno; + PyErr_Clear(); PySys_WriteStderr("Could not open PYTHONSTARTUP\n"); errno = save_errno; - PyErr_SetFromErrnoWithFilename(PyExc_OSError, startup); - - return pymain_err_print(exitcode); + PyErr_SetFromErrnoWithFilenameObjects(PyExc_OSError, startup_obj, NULL); + goto error; } (void) PyRun_SimpleFileExFlags(fp, startup, 0, cf); PyErr_Clear(); fclose(fp); - return 0; + ret = 0; + +done: +#ifdef MS_WINDOWS + Py_XDECREF(startup_bytes); +#endif + Py_XDECREF(startup_obj); + return ret; + +error: + ret = pymain_err_print(exitcode); + goto done; } diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 52ed0bc2..b35f708f 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -670,7 +670,6 @@ make_ipv4_addr(const struct sockaddr_in *addr) return PyUnicode_FromString(buf); } -#ifdef ENABLE_IPV6 /* Convert IPv6 sockaddr to a Python str. */ static PyObject * @@ -683,7 +682,6 @@ make_ipv6_addr(const struct sockaddr_in6 *addr) } return PyUnicode_FromString(buf); } -#endif static PyObject* unparse_address(LPSOCKADDR Address, DWORD Length) @@ -701,7 +699,6 @@ unparse_address(LPSOCKADDR Address, DWORD Length) } return ret; } -#ifdef ENABLE_IPV6 case AF_INET6: { const struct sockaddr_in6 *a = (const struct sockaddr_in6 *)Address; PyObject *addrobj = make_ipv6_addr(a); @@ -716,9 +713,9 @@ unparse_address(LPSOCKADDR Address, DWORD Length) } return ret; } -#endif /* ENABLE_IPV6 */ default: { - return SetFromWindowsErr(ERROR_INVALID_PARAMETER); + PyErr_SetString(PyExc_ValueError, "recvfrom returned unsupported address family"); + return NULL; } } } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index eb0b56ae..726e3723 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8019,8 +8019,6 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd) } #endif /* defined(HAVE_READLINK) || defined(MS_WINDOWS) */ -#ifdef HAVE_SYMLINK - #if defined(MS_WINDOWS) /* Remove the last portion of the path - return 0 on success */ @@ -8043,6 +8041,12 @@ _dirnameW(WCHAR *path) return 0; } +#endif + +#ifdef HAVE_SYMLINK + +#if defined(MS_WINDOWS) + /* Is this path absolute? */ static int _is_absW(const WCHAR *path) diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index ed71d8b0..a71b6422 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -218,7 +218,7 @@ select.select Wait until one or more file descriptors are ready for some kind of I/O. -The first three arguments are sequences of file descriptors to be waited for: +The first three arguments are iterables of file descriptors to be waited for: rlist -- wait until ready for reading wlist -- wait until ready for writing xlist -- wait for an "exceptional condition" @@ -243,7 +243,7 @@ descriptors can be used. static PyObject * select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, PyObject *xlist, PyObject *timeout_obj) -/*[clinic end generated code: output=2b3cfa824f7ae4cf input=177e72184352df25]*/ +/*[clinic end generated code: output=2b3cfa824f7ae4cf input=e467f5d68033de00]*/ { #ifdef SELECT_USES_HEAP pylist *rfd2obj, *wfd2obj, *efd2obj; @@ -299,7 +299,7 @@ select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, } #endif /* SELECT_USES_HEAP */ - /* Convert sequences to fd_sets, and get maximum fd number + /* Convert iterables to fd_sets, and get maximum fd number * propagates the Python exception set in seq2set() */ rfd2obj[0].sentinel = -1; @@ -439,7 +439,7 @@ select.poll.register fd: fildes either an integer, or an object with a fileno() method returning an int - eventmask: unsigned_short(c_default="POLLIN | POLLPRI | POLLOUT") = POLLIN | POLLPRI | POLLOUT + eventmask: unsigned_short(c_default="POLLIN | POLLPRI | POLLOUT") = select.POLLIN | select.POLLPRI | select.POLLOUT an optional bitmask describing the type of events to check for / @@ -448,7 +448,7 @@ Register a file descriptor with the polling object. static PyObject * select_poll_register_impl(pollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=0dc7173c800a4a65 input=f18711d9bb021e25]*/ +/*[clinic end generated code: output=0dc7173c800a4a65 input=34e16cfb28d3c900]*/ { PyObject *key, *value; int err; @@ -817,7 +817,7 @@ select.devpoll.register fd: fildes either an integer, or an object with a fileno() method returning an int - eventmask: unsigned_short(c_default="POLLIN | POLLPRI | POLLOUT") = POLLIN | POLLPRI | POLLOUT + eventmask: unsigned_short(c_default="POLLIN | POLLPRI | POLLOUT") = select.POLLIN | select.POLLPRI | select.POLLOUT an optional bitmask describing the type of events to check for / @@ -827,7 +827,7 @@ Register a file descriptor with the polling object. static PyObject * select_devpoll_register_impl(devpollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=6e07fe8b74abba0c input=5bd7cacc47a8ee46]*/ +/*[clinic end generated code: output=6e07fe8b74abba0c input=22006fabe9567522]*/ { return internal_devpoll_register(self, fd, eventmask, 0); } @@ -838,7 +838,7 @@ select.devpoll.modify fd: fildes either an integer, or an object with a fileno() method returning an int - eventmask: unsigned_short(c_default="POLLIN | POLLPRI | POLLOUT") = POLLIN | POLLPRI | POLLOUT + eventmask: unsigned_short(c_default="POLLIN | POLLPRI | POLLOUT") = select.POLLIN | select.POLLPRI | select.POLLOUT an optional bitmask describing the type of events to check for / @@ -848,7 +848,7 @@ Modify a possible already registered file descriptor. static PyObject * select_devpoll_modify_impl(devpollObject *self, int fd, unsigned short eventmask) -/*[clinic end generated code: output=bc2e6d23aaff98b4 input=48a820fc5967165d]*/ +/*[clinic end generated code: output=bc2e6d23aaff98b4 input=09fa335db7cdc09e]*/ { return internal_devpoll_register(self, fd, eventmask, 1); } @@ -1424,7 +1424,7 @@ select.epoll.register fd: fildes the target file descriptor of the operation - eventmask: unsigned_int(c_default="EPOLLIN | EPOLLPRI | EPOLLOUT", bitwise=True) = EPOLLIN | EPOLLPRI | EPOLLOUT + eventmask: unsigned_int(c_default="EPOLLIN | EPOLLPRI | EPOLLOUT", bitwise=True) = select.EPOLLIN | select.EPOLLPRI | select.EPOLLOUT a bit set composed of the various EPOLL constants Registers a new fd or raises an OSError if the fd is already registered. @@ -1435,7 +1435,7 @@ The epoll interface supports all file descriptors that support poll. static PyObject * select_epoll_register_impl(pyEpoll_Object *self, int fd, unsigned int eventmask) -/*[clinic end generated code: output=318e5e6386520599 input=6cf699c152dd8ca9]*/ +/*[clinic end generated code: output=318e5e6386520599 input=a5071b71edfe3578]*/ { return pyepoll_internal_ctl(self->epfd, EPOLL_CTL_ADD, fd, eventmask); } diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 0c9a2671..119fc355 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -187,14 +187,20 @@ itimer_retval(struct itimerval *iv) #endif static int -is_main(_PyRuntimeState *runtime) +is_main_interp(_PyRuntimeState *runtime, PyInterpreterState *interp) { unsigned long thread = PyThread_get_thread_ident(); - PyInterpreterState *interp = _PyRuntimeState_GetThreadState(runtime)->interp; return (thread == runtime->main_thread && interp == runtime->interpreters.main); } +static int +is_main(_PyRuntimeState *runtime) +{ + PyInterpreterState *interp = _PyRuntimeState_GetThreadState(runtime)->interp; + return is_main_interp(runtime, interp); +} + static PyObject * signal_default_int_handler(PyObject *self, PyObject *args) { @@ -1726,12 +1732,14 @@ PyOS_FiniInterrupts(void) finisignal(); } + +// The caller doesn't have to hold the GIL int -PyOS_InterruptOccurred(void) +_PyOS_InterruptOccurred(PyThreadState *tstate) { if (_Py_atomic_load_relaxed(&Handlers[SIGINT].tripped)) { _PyRuntimeState *runtime = &_PyRuntime; - if (!is_main(runtime)) { + if (!is_main_interp(runtime, tstate->interp)) { return 0; } _Py_atomic_store_relaxed(&Handlers[SIGINT].tripped, 0); @@ -1740,6 +1748,16 @@ PyOS_InterruptOccurred(void) return 0; } + +// The caller must to hold the GIL +int +PyOS_InterruptOccurred(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return _PyOS_InterruptOccurred(tstate); +} + + static void _clear_pending_signals(void) { diff --git a/Objects/abstract.c b/Objects/abstract.c index 4fabfd76..12237d57 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1999,7 +1999,9 @@ _PySequence_IterSearch(PyObject *seq, PyObject *obj, int operation) it = PyObject_GetIter(seq); if (it == NULL) { - type_error("argument of type '%.200s' is not iterable", seq); + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + type_error("argument of type '%.200s' is not iterable", seq); + } return -1; } diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 590b8060..d4d02336 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -276,7 +276,9 @@ PyByteArray_Concat(PyObject *a, PyObject *b) result = (PyByteArrayObject *) \ PyByteArray_FromStringAndSize(NULL, va.len + vb.len); - if (result != NULL) { + // result->ob_bytes is NULL if result is an empty string: + // if va.len + vb.len equals zero. + if (result != NULL && result->ob_bytes != NULL) { memcpy(result->ob_bytes, va.buf, va.len); memcpy(result->ob_bytes + va.len, vb.buf, vb.len); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 5df12177..00128698 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -81,6 +81,9 @@ clear_slotdefs(void); static PyObject * lookup_maybe_method(PyObject *self, _Py_Identifier *attrid, int *unbound); +static int +slot_tp_setattro(PyObject *self, PyObject *name, PyObject *value); + /* * finds the beginning of the docstring's introspection signature. * if present, returns a pointer pointing to the first '('. @@ -5806,22 +5809,38 @@ wrap_delitem(PyObject *self, PyObject *args, void *wrapped) } /* Helper to check for object.__setattr__ or __delattr__ applied to a type. - This is called the Carlo Verre hack after its discoverer. */ + This is called the Carlo Verre hack after its discoverer. See + https://mail.python.org/pipermail/python-dev/2003-April/034535.html + */ static int hackcheck(PyObject *self, setattrofunc func, const char *what) { PyTypeObject *type = Py_TYPE(self); - while (type && type->tp_flags & Py_TPFLAGS_HEAPTYPE) - type = type->tp_base; - /* If type is NULL now, this is a really weird type. - In the spirit of backwards compatibility (?), just shut up. */ - if (type && type->tp_setattro != func) { - PyErr_Format(PyExc_TypeError, - "can't apply this %s to %s object", - what, - type->tp_name); - return 0; + PyObject *mro = type->tp_mro; + if (!mro) { + /* Probably ok not to check the call in this case. */ + return 1; + } + assert(PyTuple_Check(mro)); + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyTypeObject *base = (PyTypeObject*) PyTuple_GET_ITEM(mro, i); + if (base->tp_setattro == func) { + /* 'func' is the earliest non-Python implementation in the MRO. */ + break; + } else if (base->tp_setattro != slot_tp_setattro) { + /* 'base' is not a Python class and overrides 'func'. + Its tp_setattro should be called instead. */ + PyErr_Format(PyExc_TypeError, + "can't apply this %s to %s object", + what, + type->tp_name); + return 0; + } } + /* Either 'func' is not in the mro (which should fail when checking 'self'), + or it's the right slot function to call. */ return 1; } diff --git a/PC/_msi.c b/PC/_msi.c index accbe7a7..5079a524 100644 --- a/PC/_msi.c +++ b/PC/_msi.c @@ -872,14 +872,14 @@ static PyObject* msidb_openview(msiobj *msidb, PyObject *args) { int status; - char *sql; + const wchar_t *sql; MSIHANDLE hView; msiobj *result; - if (!PyArg_ParseTuple(args, "s:OpenView", &sql)) + if (!PyArg_ParseTuple(args, "u:OpenView", &sql)) return NULL; - if ((status = MsiDatabaseOpenView(msidb->h, sql, &hView)) != ERROR_SUCCESS) + if ((status = MsiDatabaseOpenViewW(msidb->h, sql, &hView)) != ERROR_SUCCESS) return msierror(status); result = PyObject_NEW(struct msiobj, &msiview_Type); @@ -998,18 +998,18 @@ static PyTypeObject msidb_Type = { static PyObject* msiopendb(PyObject *obj, PyObject *args) { int status; - char *path; + const wchar_t *path; int persist; MSIHANDLE h; msiobj *result; - if (!PyArg_ParseTuple(args, "si:MSIOpenDatabase", &path, &persist)) + if (!PyArg_ParseTuple(args, "ui:MSIOpenDatabase", &path, &persist)) return NULL; /* We need to validate that persist is a valid MSIDBOPEN_* value. Otherwise, MsiOpenDatabase may treat the value as a pointer, leading to unexpected behavior. */ if (Py_INVALID_PERSIST(persist)) return msierror(ERROR_INVALID_PARAMETER); - status = MsiOpenDatabase(path, (LPCSTR)(SIZE_T)persist, &h); + status = MsiOpenDatabaseW(path, (LPCWSTR)(SIZE_T)persist, &h); if (status != ERROR_SUCCESS) return msierror(status); diff --git a/PC/getpathp.c b/PC/getpathp.c index 3747ffb2..cf20b9f6 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -130,8 +130,6 @@ typedef struct { wchar_t *machine_path; /* from HKEY_LOCAL_MACHINE */ wchar_t *user_path; /* from HKEY_CURRENT_USER */ - wchar_t *dll_path; - const wchar_t *pythonpath_env; } PyCalculatePath; @@ -167,27 +165,37 @@ reduce(wchar_t *dir) static int change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext) { - size_t src_len = wcsnlen_s(src, MAXPATHLEN+1); - size_t i = src_len; - if (i >= MAXPATHLEN+1) { - Py_FatalError("buffer overflow in getpathp.c's reduce()"); - } + if (src && src != dest) { + size_t src_len = wcsnlen_s(src, MAXPATHLEN+1); + size_t i = src_len; + if (i >= MAXPATHLEN+1) { + Py_FatalError("buffer overflow in getpathp.c's reduce()"); + } - while (i > 0 && src[i] != '.' && !is_sep(src[i])) - --i; + while (i > 0 && src[i] != '.' && !is_sep(src[i])) + --i; - if (i == 0) { - dest[0] = '\0'; - return -1; - } + if (i == 0) { + dest[0] = '\0'; + return -1; + } + + if (is_sep(src[i])) { + i = src_len; + } - if (is_sep(src[i])) { - i = src_len; + if (wcsncpy_s(dest, MAXPATHLEN+1, src, i)) { + dest[0] = '\0'; + return -1; + } + } else { + wchar_t *s = wcsrchr(dest, L'.'); + if (s) { + s[0] = '\0'; + } } - if (wcsncpy_s(dest, MAXPATHLEN+1, src, i) || - wcscat_s(dest, MAXPATHLEN+1, ext)) - { + if (wcscat_s(dest, MAXPATHLEN+1, ext)) { dest[0] = '\0'; return -1; } @@ -344,6 +352,19 @@ search_for_prefix(wchar_t *prefix, const wchar_t *argv0_path, const wchar_t *lan } +static int +get_dllpath(wchar_t *dllpath) +{ +#ifdef Py_ENABLE_SHARED + extern HANDLE PyWin_DLLhModule; + if (PyWin_DLLhModule && GetModuleFileNameW(PyWin_DLLhModule, dllpath, MAXPATHLEN)) { + return 0; + } +#endif + return -1; +} + + #ifdef Py_ENABLE_SHARED /* a string loaded from the DLL at startup.*/ @@ -405,7 +426,7 @@ getpythonregpath(HKEY keyBase, int skipcore) goto done; } /* Find out how big our core buffer is, and how many subkeys we have */ - rc = RegQueryInfoKey(newKey, NULL, NULL, NULL, &numKeys, NULL, NULL, + rc = RegQueryInfoKeyW(newKey, NULL, NULL, NULL, &numKeys, NULL, NULL, NULL, NULL, &dataSize, NULL, NULL); if (rc!=ERROR_SUCCESS) { goto done; @@ -516,27 +537,6 @@ done: #endif /* Py_ENABLE_SHARED */ -wchar_t* -_Py_GetDLLPath(void) -{ - wchar_t dll_path[MAXPATHLEN+1]; - memset(dll_path, 0, sizeof(dll_path)); - -#ifdef Py_ENABLE_SHARED - extern HANDLE PyWin_DLLhModule; - if (PyWin_DLLhModule) { - if (!GetModuleFileNameW(PyWin_DLLhModule, dll_path, MAXPATHLEN)) { - dll_path[0] = 0; - } - } -#else - dll_path[0] = 0; -#endif - - return _PyMem_RawWcsdup(dll_path); -} - - static PyStatus get_program_full_path(_PyPathConfig *pathconfig) { @@ -717,19 +717,17 @@ static int get_pth_filename(PyCalculatePath *calculate, wchar_t *filename, const _PyPathConfig *pathconfig) { - if (calculate->dll_path[0]) { - if (!change_ext(filename, calculate->dll_path, L"._pth") && - exists(filename)) - { - return 1; - } + if (get_dllpath(filename) && + !change_ext(filename, filename, L"._pth") && + exists(filename)) + { + return 1; } - if (pathconfig->program_full_path[0]) { - if (!change_ext(filename, pathconfig->program_full_path, L"._pth") && - exists(filename)) - { - return 1; - } + if (pathconfig->program_full_path[0] && + !change_ext(filename, pathconfig->program_full_path, L"._pth") && + exists(filename)) + { + return 1; } return 0; } @@ -1029,9 +1027,12 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig) wchar_t zip_path[MAXPATHLEN+1]; memset(zip_path, 0, sizeof(zip_path)); - change_ext(zip_path, - calculate->dll_path[0] ? calculate->dll_path : pathconfig->program_full_path, - L".zip"); + if (get_dllpath(zip_path) || change_ext(zip_path, zip_path, L".zip")) + { + if (change_ext(zip_path, pathconfig->program_full_path, L".zip")) { + zip_path[0] = L'\0'; + } + } calculate_home_prefix(calculate, argv0_path, zip_path, prefix); @@ -1068,11 +1069,6 @@ calculate_init(PyCalculatePath *calculate, _PyPathConfig *pathconfig, calculate->home = pathconfig->home; calculate->path_env = _wgetenv(L"PATH"); - calculate->dll_path = _Py_GetDLLPath(); - if (calculate->dll_path == NULL) { - return _PyStatus_NO_MEMORY(); - } - calculate->pythonpath_env = config->pythonpath_env; return _PyStatus_OK(); @@ -1084,7 +1080,6 @@ calculate_free(PyCalculatePath *calculate) { PyMem_RawFree(calculate->machine_path); PyMem_RawFree(calculate->user_path); - PyMem_RawFree(calculate->dll_path); } @@ -1094,7 +1089,6 @@ calculate_free(PyCalculatePath *calculate) - PyConfig.pythonpath_env: PYTHONPATH environment variable - _PyPathConfig.home: Py_SetPythonHome() or PYTHONHOME environment variable - - DLL path: _Py_GetDLLPath() - PATH environment variable - __PYVENV_LAUNCHER__ environment variable - GetModuleFileNameW(NULL): fully qualified path of the executable file of @@ -1148,33 +1142,35 @@ int _Py_CheckPython3(void) { wchar_t py3path[MAXPATHLEN+1]; - wchar_t *s; if (python3_checked) { return hPython3 != NULL; } python3_checked = 1; /* If there is a python3.dll next to the python3y.dll, - assume this is a build tree; use that DLL */ - if (_Py_dll_path != NULL) { - wcscpy(py3path, _Py_dll_path); - } - else { - wcscpy(py3path, L""); - } - s = wcsrchr(py3path, L'\\'); - if (!s) { - s = py3path; + use that DLL */ + if (!get_dllpath(py3path)) { + reduce(py3path); + join(py3path, PY3_DLLNAME); + hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + if (hPython3 != NULL) { + return 1; + } } - wcscpy(s, L"\\python3.dll"); - hPython3 = LoadLibraryExW(py3path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); + + /* If we can locate python3.dll in our application dir, + use that DLL */ + hPython3 = LoadLibraryExW(PY3_DLLNAME, NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR); if (hPython3 != NULL) { return 1; } - /* Check sys.prefix\DLLs\python3.dll */ + /* For back-compat, also search {sys.prefix}\DLLs, though + that has not been a normal install layout for a while */ wcscpy(py3path, Py_GetPrefix()); - wcscat(py3path, L"\\DLLs\\python3.dll"); - hPython3 = LoadLibraryExW(py3path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); + if (py3path[0]) { + join(py3path, L"DLLs\\" PY3_DLLNAME); + hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + } return hPython3 != NULL; } diff --git a/PC/python_ver_rc.h b/PC/python_ver_rc.h index f95e755b..d725a9ba 100644 --- a/PC/python_ver_rc.h +++ b/PC/python_ver_rc.h @@ -1,6 +1,7 @@ // Resource script for Python core DLL. // Currently only holds version information. // +#pragma code_page(1252) #include "winver.h" #define PYTHON_COMPANY "Python Software Foundation" diff --git a/PC/winreg.c b/PC/winreg.c index 5dff7dea..caad18e0 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -1451,9 +1451,9 @@ winreg_QueryInfoKey_impl(PyObject *module, HKEY key) if (PySys_Audit("winreg.QueryInfoKey", "n", (Py_ssize_t)key) < 0) { return NULL; } - if ((rc = RegQueryInfoKey(key, NULL, NULL, 0, &nSubKeys, NULL, NULL, - &nValues, NULL, NULL, NULL, &ft)) - != ERROR_SUCCESS) { + if ((rc = RegQueryInfoKeyW(key, NULL, NULL, 0, &nSubKeys, NULL, NULL, + &nValues, NULL, NULL, NULL, &ft)) + != ERROR_SUCCESS) { return PyErr_SetFromWindowsErrWithFunction(rc, "RegQueryInfoKey"); } li.LowPart = ft.dwLowDateTime; diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 88ab3479..ddc0a286 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -53,7 +53,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.6 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.3.0-rc0-r1 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1f +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1g set libraries=%libraries% sqlite-3.31.1.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.9.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.9.0 @@ -77,7 +77,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1f +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1g if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.9.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/liblzma.vcxproj b/PCbuild/liblzma.vcxproj index 6c6872de..8e77062c 100644 --- a/PCbuild/liblzma.vcxproj +++ b/PCbuild/liblzma.vcxproj @@ -91,11 +91,8 @@ - WIN32;HAVE_CONFIG_H;_DEBUG;_LIB;%(PreprocessorDefinitions) - Level3 - ProgramDatabase - Disabled - $(lzmaDir)windows;$(lzmaDir)src/liblzma/common;$(lzmaDir)src/common;$(lzmaDir)src/liblzma/api;$(lzmaDir)src/liblzma/check;$(lzmaDir)src/liblzma/delta;$(lzmaDir)src/liblzma/lz;$(lzmaDir)src/liblzma/lzma;$(lzmaDir)src/liblzma/rangecoder;$(lzmaDir)src/liblzma/simple + WIN32;HAVE_CONFIG_H;_LIB;%(PreprocessorDefinitions) + $(lzmaDir)windows;$(lzmaDir)src/liblzma/common;$(lzmaDir)src/common;$(lzmaDir)src/liblzma/api;$(lzmaDir)src/liblzma/check;$(lzmaDir)src/liblzma/delta;$(lzmaDir)src/liblzma/lz;$(lzmaDir)src/liblzma/lzma;$(lzmaDir)src/liblzma/rangecoder;$(lzmaDir)src/liblzma/simple;%(AdditionalIncludeDirectories) 4028;4113;4133;4244;4267;4996;%(DisableSpecificWarnings) diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index edeb1737..f531b0e9 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -26,11 +26,12 @@ <_PlatformPreprocessorDefinition>_WIN32; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64;_M_X64; <_PydPreprocessorDefinition Condition="$(TargetExt) == '.pyd'">Py_BUILD_CORE_MODULE; + <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)"; $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)PC;$(IntDir);%(AdditionalIncludeDirectories) - WIN32;$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) + WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) MaxSpeed true diff --git a/PCbuild/python.props b/PCbuild/python.props index 2c8555a3..112b9f08 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -62,8 +62,8 @@ $(ExternalsDir)libffi\ $(ExternalsDir)libffi\$(ArchName)\ $(libffiOutDir)include - $(ExternalsDir)openssl-1.1.1f\ - $(ExternalsDir)openssl-bin-1.1.1f\$(ArchName)\ + $(ExternalsDir)openssl-1.1.1g\ + $(ExternalsDir)openssl-bin-1.1.1g\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ $(ExternalsDir)\zlib-1.2.11\ @@ -203,6 +203,8 @@ python$(MajorVersionNumber)$(MinorVersionNumber)$(PyDebugExt) + + python3$(PyDebugExt) .cp$(MajorVersionNumber)$(MinorVersionNumber)-win32 diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 574fcb0e..a708b66d 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -665,8 +665,9 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) } if (fields) { numfields = PySequence_Size(fields); - if (numfields == -1) + if (numfields == -1) { goto cleanup; + } } res = 0; /* if no error occurs, this stays 0 to the end */ @@ -687,15 +688,35 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) } res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); Py_DECREF(name); - if (res < 0) + if (res < 0) { goto cleanup; + } } if (kw) { i = 0; /* needed by PyDict_Next */ while (PyDict_Next(kw, &i, &key, &value)) { + int contains = PySequence_Contains(fields, key); + if (contains == -1) { + res = -1; + goto cleanup; + } else if (contains == 1) { + Py_ssize_t p = PySequence_Index(fields, key); + if (p == -1) { + res = -1; + goto cleanup; + } + if (p < PyTuple_GET_SIZE(args)) { + PyErr_Format(PyExc_TypeError, + "%.400s got multiple values for argument '%U'", + Py_TYPE(self)->tp_name, key); + res = -1; + goto cleanup; + } + } res = PyObject_SetAttr(self, key, value); - if (res < 0) + if (res < 0) { goto cleanup; + } } } cleanup: diff --git a/Parser/myreadline.c b/Parser/myreadline.c index 43e5583b..d7ed357f 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -25,25 +25,36 @@ static PyThread_type_lock _PyOS_ReadlineLock = NULL; int (*PyOS_InputHook)(void) = NULL; /* This function restarts a fgets() after an EINTR error occurred - except if PyOS_InterruptOccurred() returns true. */ + except if _PyOS_InterruptOccurred() returns true. */ static int -my_fgets(char *buf, int len, FILE *fp) +my_fgets(PyThreadState* tstate, char *buf, int len, FILE *fp) { #ifdef MS_WINDOWS - HANDLE hInterruptEvent; + HANDLE handle; + _Py_BEGIN_SUPPRESS_IPH + handle = (HANDLE)_get_osfhandle(fileno(fp)); + _Py_END_SUPPRESS_IPH + + /* bpo-40826: fgets(fp) does crash if fileno(fp) is closed */ + if (handle == INVALID_HANDLE_VALUE) { + return -1; /* EOF */ + } #endif - char *p; - int err; + while (1) { - if (PyOS_InputHook != NULL) + if (PyOS_InputHook != NULL) { (void)(PyOS_InputHook)(); + } + errno = 0; clearerr(fp); - p = fgets(buf, len, fp); - if (p != NULL) + char *p = fgets(buf, len, fp); + if (p != NULL) { return 0; /* No error */ - err = errno; + } + int err = errno; + #ifdef MS_WINDOWS /* Ctrl-C anywhere on the line or Ctrl-Z if the only character on a line will set ERROR_OPERATION_ABORTED. Under normal @@ -59,7 +70,7 @@ my_fgets(char *buf, int len, FILE *fp) through to check for EOF. */ if (GetLastError()==ERROR_OPERATION_ABORTED) { - hInterruptEvent = _PyOS_SigintEvent(); + HANDLE hInterruptEvent = _PyOS_SigintEvent(); switch (WaitForSingleObjectEx(hInterruptEvent, 10, FALSE)) { case WAIT_OBJECT_0: ResetEvent(hInterruptEvent); @@ -69,23 +80,27 @@ my_fgets(char *buf, int len, FILE *fp) } } #endif /* MS_WINDOWS */ + if (feof(fp)) { clearerr(fp); return -1; /* EOF */ } + #ifdef EINTR if (err == EINTR) { - int s; - PyEval_RestoreThread(_PyOS_ReadlineTState); - s = PyErr_CheckSignals(); + PyEval_RestoreThread(tstate); + int s = PyErr_CheckSignals(); PyEval_SaveThread(); - if (s < 0) - return 1; - /* try again */ + + if (s < 0) { + return 1; + } + /* try again */ continue; } #endif - if (PyOS_InterruptOccurred()) { + + if (_PyOS_InterruptOccurred(tstate)) { return 1; /* Interrupt */ } return -2; /* Error */ @@ -99,7 +114,7 @@ my_fgets(char *buf, int len, FILE *fp) extern char _get_console_type(HANDLE handle); char * -_PyOS_WindowsConsoleReadline(HANDLE hStdIn) +_PyOS_WindowsConsoleReadline(PyThreadState *tstate, HANDLE hStdIn) { static wchar_t wbuf_local[1024 * 16]; const DWORD chunk_size = 1024; @@ -134,11 +149,12 @@ _PyOS_WindowsConsoleReadline(HANDLE hStdIn) if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE) == WAIT_OBJECT_0) { ResetEvent(hInterruptEvent); - PyEval_RestoreThread(_PyOS_ReadlineTState); + PyEval_RestoreThread(tstate); s = PyErr_CheckSignals(); PyEval_SaveThread(); - if (s < 0) + if (s < 0) { goto exit; + } } break; } @@ -151,17 +167,22 @@ _PyOS_WindowsConsoleReadline(HANDLE hStdIn) if (wbuf == wbuf_local) { wbuf[total_read] = '\0'; wbuf = (wchar_t*)PyMem_RawMalloc(wbuflen * sizeof(wchar_t)); - if (wbuf) + if (wbuf) { wcscpy_s(wbuf, wbuflen, wbuf_local); + } else { + PyEval_RestoreThread(tstate); PyErr_NoMemory(); + PyEval_SaveThread(); goto exit; } } else { wchar_t *tmp = PyMem_RawRealloc(wbuf, wbuflen * sizeof(wchar_t)); if (tmp == NULL) { + PyEval_RestoreThread(tstate); PyErr_NoMemory(); + PyEval_SaveThread(); goto exit; } wbuf = tmp; @@ -170,33 +191,45 @@ _PyOS_WindowsConsoleReadline(HANDLE hStdIn) if (wbuf[0] == '\x1a') { buf = PyMem_RawMalloc(1); - if (buf) + if (buf) { buf[0] = '\0'; + } else { + PyEval_RestoreThread(tstate); PyErr_NoMemory(); + PyEval_SaveThread(); } goto exit; } - u8len = WideCharToMultiByte(CP_UTF8, 0, wbuf, total_read, NULL, 0, NULL, NULL); + u8len = WideCharToMultiByte(CP_UTF8, 0, + wbuf, total_read, + NULL, 0, + NULL, NULL); buf = PyMem_RawMalloc(u8len + 1); if (buf == NULL) { + PyEval_RestoreThread(tstate); PyErr_NoMemory(); + PyEval_SaveThread(); goto exit; } - u8len = WideCharToMultiByte(CP_UTF8, 0, wbuf, total_read, buf, u8len, NULL, NULL); + + u8len = WideCharToMultiByte(CP_UTF8, 0, + wbuf, total_read, + buf, u8len, + NULL, NULL); buf[u8len] = '\0'; exit: - if (wbuf != wbuf_local) + if (wbuf != wbuf_local) { PyMem_RawFree(wbuf); + } if (err) { - PyEval_RestoreThread(_PyOS_ReadlineTState); + PyEval_RestoreThread(tstate); PyErr_SetFromWindowsErr(err); PyEval_SaveThread(); } - return buf; } @@ -210,6 +243,8 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) { size_t n; char *p, *pr; + PyThreadState *tstate = _PyOS_ReadlineTState; + assert(tstate != NULL); #ifdef MS_WINDOWS if (!Py_LegacyWindowsStdioFlag && sys_stdin == stdin) { @@ -231,7 +266,9 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) if (wlen) { wbuf = PyMem_RawMalloc(wlen * sizeof(wchar_t)); if (wbuf == NULL) { + PyEval_RestoreThread(tstate); PyErr_NoMemory(); + PyEval_SaveThread(); return NULL; } wlen = MultiByteToWideChar(CP_UTF8, 0, prompt, -1, @@ -250,7 +287,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) } } clearerr(sys_stdin); - return _PyOS_WindowsConsoleReadline(hStdIn); + return _PyOS_WindowsConsoleReadline(tstate, hStdIn); } } #endif @@ -258,16 +295,19 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) n = 100; p = (char *)PyMem_RawMalloc(n); if (p == NULL) { + PyEval_RestoreThread(tstate); PyErr_NoMemory(); + PyEval_SaveThread(); return NULL; } fflush(sys_stdout); - if (prompt) + if (prompt) { fprintf(stderr, "%s", prompt); + } fflush(stderr); - switch (my_fgets(p, (int)n, sys_stdin)) { + switch (my_fgets(tstate, p, (int)n, sys_stdin)) { case 0: /* Normal case */ break; case 1: /* Interrupt */ @@ -279,29 +319,40 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) *p = '\0'; break; } + n = strlen(p); while (n > 0 && p[n-1] != '\n') { size_t incr = n+2; if (incr > INT_MAX) { PyMem_RawFree(p); + PyEval_RestoreThread(tstate); PyErr_SetString(PyExc_OverflowError, "input line too long"); + PyEval_SaveThread(); return NULL; } + pr = (char *)PyMem_RawRealloc(p, n + incr); if (pr == NULL) { PyMem_RawFree(p); + PyEval_RestoreThread(tstate); PyErr_NoMemory(); + PyEval_SaveThread(); return NULL; } p = pr; - if (my_fgets(p+n, (int)incr, sys_stdin) != 0) + + if (my_fgets(tstate, p+n, (int)incr, sys_stdin) != 0) { break; + } n += strlen(p+n); } + pr = (char *)PyMem_RawRealloc(p, n+1); if (pr == NULL) { PyMem_RawFree(p); + PyEval_RestoreThread(tstate); PyErr_NoMemory(); + PyEval_SaveThread(); return NULL; } return pr; @@ -324,7 +375,8 @@ PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) char *rv, *res; size_t len; - if (_PyOS_ReadlineTState == _PyThreadState_GET()) { + PyThreadState *tstate = _PyThreadState_GET(); + if (_PyOS_ReadlineTState == tstate) { PyErr_SetString(PyExc_RuntimeError, "can't re-enter readline"); return NULL; @@ -343,7 +395,7 @@ PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) } } - _PyOS_ReadlineTState = _PyThreadState_GET(); + _PyOS_ReadlineTState = tstate; Py_BEGIN_ALLOW_THREADS PyThread_acquire_lock(_PyOS_ReadlineLock, 1); diff --git a/Programs/_testembed.c b/Programs/_testembed.c index b98a38a1..460d70cc 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1106,8 +1106,11 @@ static int test_open_code_hook(void) return result; } +static int _audit_hook_clear_count = 0; + static int _audit_hook(const char *event, PyObject *args, void *userdata) { + assert(args && PyTuple_CheckExact(args)); if (strcmp(event, "_testembed.raise") == 0) { PyErr_SetString(PyExc_RuntimeError, "Intentional error"); return -1; @@ -1116,6 +1119,8 @@ static int _audit_hook(const char *event, PyObject *args, void *userdata) return -1; } return 0; + } else if (strcmp(event, "cpython._PySys_ClearAuditHooks") == 0) { + _audit_hook_clear_count += 1; } return 0; } @@ -1161,6 +1166,9 @@ static int test_audit(void) { int result = _test_audit(42); Py_Finalize(); + if (_audit_hook_clear_count != 1) { + return 0x1000 | _audit_hook_clear_count; + } return result; } diff --git a/Python/Python-ast.c b/Python/Python-ast.c index f73f0358..bcf94569 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -571,8 +571,9 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) } if (fields) { numfields = PySequence_Size(fields); - if (numfields == -1) + if (numfields == -1) { goto cleanup; + } } res = 0; /* if no error occurs, this stays 0 to the end */ @@ -593,15 +594,35 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) } res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); Py_DECREF(name); - if (res < 0) + if (res < 0) { goto cleanup; + } } if (kw) { i = 0; /* needed by PyDict_Next */ while (PyDict_Next(kw, &i, &key, &value)) { + int contains = PySequence_Contains(fields, key); + if (contains == -1) { + res = -1; + goto cleanup; + } else if (contains == 1) { + Py_ssize_t p = PySequence_Index(fields, key); + if (p == -1) { + res = -1; + goto cleanup; + } + if (p < PyTuple_GET_SIZE(args)) { + PyErr_Format(PyExc_TypeError, + "%.400s got multiple values for argument '%U'", + Py_TYPE(self)->tp_name, key); + res = -1; + goto cleanup; + } + } res = PyObject_SetAttr(self, key, value); - if (res < 0) + if (res < 0) { goto cleanup; + } } } cleanup: diff --git a/Python/ast.c b/Python/ast.c index f70d48ba..7c1d24de 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -21,6 +21,25 @@ static int validate_nonempty_seq(asdl_seq *, const char *, const char *); static int validate_stmt(stmt_ty); static int validate_expr(expr_ty, expr_context_ty); +static int +validate_name(PyObject *name) +{ + assert(PyUnicode_Check(name)); + static const char * const forbidden[] = { + "None", + "True", + "False", + NULL + }; + for (int i = 0; forbidden[i] != NULL; i++) { + if (_PyUnicode_EqualToASCIIString(name, forbidden[i])) { + PyErr_Format(PyExc_ValueError, "Name node can't be used with '%s' constant", forbidden[i]); + return 0; + } + } + return 1; +} + static int validate_comprehension(asdl_seq *gens) { @@ -199,6 +218,9 @@ validate_expr(expr_ty exp, expr_context_ty ctx) actual_ctx = exp->v.Starred.ctx; break; case Name_kind: + if (!validate_name(exp->v.Name.id)) { + return 0; + } actual_ctx = exp->v.Name.ctx; break; case List_kind: @@ -786,7 +808,8 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, /* borrowed reference */ c.c_filename = filename; c.c_normalize = NULL; - c.c_feature_version = flags ? flags->cf_feature_version : PY_MINOR_VERSION; + c.c_feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ? + flags->cf_feature_version : PY_MINOR_VERSION; if (TYPE(n) == encoding_decl) n = CHILD(n, 0); @@ -4876,7 +4899,7 @@ fstring_compile_expr(const char *expr_start, const char *expr_end, len = expr_end - expr_start; /* Allocate 3 extra bytes: open paren, close paren, null byte. */ - str = PyMem_RawMalloc(len + 3); + str = PyMem_Malloc(len + 3); if (str == NULL) { PyErr_NoMemory(); return NULL; @@ -4892,7 +4915,7 @@ fstring_compile_expr(const char *expr_start, const char *expr_end, mod_n = PyParser_SimpleParseStringFlagsFilename(str, "", Py_eval_input, 0); if (!mod_n) { - PyMem_RawFree(str); + PyMem_Free(str); return NULL; } /* Reuse str to find the correct column offset. */ @@ -4900,7 +4923,7 @@ fstring_compile_expr(const char *expr_start, const char *expr_end, str[len+1] = '}'; fstring_fix_node_location(n, mod_n, str); mod = PyAST_FromNode(mod_n, &cf, "", c->c_arena); - PyMem_RawFree(str); + PyMem_Free(str); PyNode_Free(mod_n); if (!mod) return NULL; @@ -5202,6 +5225,12 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl, /* Check for =, which puts the text value of the expression in expr_text. */ if (**str == '=') { + if (c->c_feature_version < 8) { + ast_error(c, n, + "f-string: self documenting expressions are " + "only supported in Python 3.8 and greater"); + goto error; + } *str += 1; /* Skip over ASCII whitespace. No need to test for end of string @@ -5410,7 +5439,7 @@ ExprList_Append(ExprList *l, expr_ty exp) Py_ssize_t i; /* We're still using the cached data. Switch to alloc-ing. */ - l->p = PyMem_RawMalloc(sizeof(expr_ty) * new_size); + l->p = PyMem_Malloc(sizeof(expr_ty) * new_size); if (!l->p) return -1; /* Copy the cached data into the new buffer. */ @@ -5418,9 +5447,9 @@ ExprList_Append(ExprList *l, expr_ty exp) l->p[i] = l->data[i]; } else { /* Just realloc. */ - expr_ty *tmp = PyMem_RawRealloc(l->p, sizeof(expr_ty) * new_size); + expr_ty *tmp = PyMem_Realloc(l->p, sizeof(expr_ty) * new_size); if (!tmp) { - PyMem_RawFree(l->p); + PyMem_Free(l->p); l->p = NULL; return -1; } @@ -5448,7 +5477,7 @@ ExprList_Dealloc(ExprList *l) /* Do nothing. */ } else { /* We have dynamically allocated. Free the memory. */ - PyMem_RawFree(l->p); + PyMem_Free(l->p); } l->p = NULL; l->size = -1; diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index 1a7cd236..af9604eb 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -750,6 +750,24 @@ append_ast_ext_slice(_PyUnicodeWriter *writer, slice_ty slice) return 0; } +static int +append_ast_index_slice(_PyUnicodeWriter *writer, slice_ty slice) +{ + int level = PR_TUPLE; + expr_ty value = slice->v.Index.value; + if (value->kind == Tuple_kind) { + for (Py_ssize_t i = 0; i < asdl_seq_LEN(value->v.Tuple.elts); i++) { + expr_ty element = asdl_seq_GET(value->v.Tuple.elts, i); + if (element->kind == Starred_kind) { + ++level; + break; + } + } + } + APPEND_EXPR(value, level); + return 0; +} + static int append_ast_slice(_PyUnicodeWriter *writer, slice_ty slice) { @@ -759,8 +777,7 @@ append_ast_slice(_PyUnicodeWriter *writer, slice_ty slice) case ExtSlice_kind: return append_ast_ext_slice(writer, slice); case Index_kind: - APPEND_EXPR(slice->v.Index.value, PR_TUPLE); - return 0; + return append_ast_index_slice(writer, slice); default: PyErr_SetString(PyExc_SystemError, "unexpected slice kind"); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index fe22bbdd..e42d5f24 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2648,12 +2648,15 @@ static PyMethodDef zip_methods[] = { }; PyDoc_STRVAR(zip_doc, -"zip(*iterables) --> zip object\n\ +"zip(*iterables) --> A zip object yielding tuples until an input is exhausted.\n\ \n\ -Return a zip object whose .__next__() method returns a tuple where\n\ -the i-th element comes from the i-th iterable argument. The .__next__()\n\ -method continues until the shortest iterable in the argument sequence\n\ -is exhausted and then it raises StopIteration."); + >>> list(zip('abcdefg', range(3), range(4)))\n\ + [('a', 0, 0), ('b', 1, 1), ('c', 2, 2)]\n\ +\n\ +The zip object yields n-length tuples, where n is the number of iterables\n\ +passed as positional arguments to zip(). The i-th element in every tuple\n\ +comes from the i-th iterable argument to zip(). This continues until the\n\ +shortest argument is exhausted."); PyTypeObject PyZip_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 43f5264d..eb2b6d08 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -38,8 +38,8 @@ static int win32_urandom_init(int raise) { /* Acquire context */ - if (!CryptAcquireContext(&hCryptProv, NULL, NULL, - PROV_RSA_FULL, CRYPT_VERIFYCONTEXT)) + if (!CryptAcquireContextW(&hCryptProv, NULL, NULL, + PROV_RSA_FULL, CRYPT_VERIFYCONTEXT)) goto error; return 0; diff --git a/Python/compile.c b/Python/compile.c index 913ec992..3259e8a4 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4470,10 +4470,9 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, comprehension_ty outermost; PyObject *qualname = NULL; int is_async_generator = 0; + int top_level_await = IS_TOP_LEVEL_AWAIT(c); + - if (IS_TOP_LEVEL_AWAIT(c)) { - c->u->u_ste->ste_coroutine = 1; - } int is_async_function = c->u->u_ste->ste_coroutine; outermost = (comprehension_ty) asdl_seq_GET(generators, 0); @@ -4485,7 +4484,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, is_async_generator = c->u->u_ste->ste_coroutine; - if (is_async_generator && !is_async_function && type != COMP_GENEXP) { + if (is_async_generator && !is_async_function && type != COMP_GENEXP && !top_level_await) { compiler_error(c, "asynchronous comprehension outside of " "an asynchronous function"); goto error_in_scope; @@ -4524,6 +4523,9 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, qualname = c->u->u_qualname; Py_INCREF(qualname); compiler_exit_scope(c); + if (top_level_await && is_async_generator){ + c->u->u_ste->ste_coroutine = 1; + } if (co == NULL) goto error; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 6deba113..4896c6dc 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -174,9 +174,7 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, char funcname[258], *import_python; const wchar_t *wpathname; -#ifndef _DEBUG _Py_CheckPython3(); -#endif wpathname = _PyUnicode_AsUnicode(pathname); if (wpathname == NULL) diff --git a/Python/fileutils.c b/Python/fileutils.c index e79e732d..b2741167 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1274,7 +1274,12 @@ _Py_open_impl(const char *pathname, int flags, int gil_held) #endif if (gil_held) { - if (PySys_Audit("open", "sOi", pathname, Py_None, flags) < 0) { + PyObject *pathname_obj = PyUnicode_DecodeFSDefault(pathname); + if (pathname_obj == NULL) { + return -1; + } + if (PySys_Audit("open", "OOi", pathname_obj, Py_None, flags) < 0) { + Py_DECREF(pathname_obj); return -1; } @@ -1284,12 +1289,16 @@ _Py_open_impl(const char *pathname, int flags, int gil_held) Py_END_ALLOW_THREADS } while (fd < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); - if (async_err) + if (async_err) { + Py_DECREF(pathname_obj); return -1; + } if (fd < 0) { - PyErr_SetFromErrnoWithFilename(PyExc_OSError, pathname); + PyErr_SetFromErrnoWithFilenameObjects(PyExc_OSError, pathname_obj, NULL); + Py_DECREF(pathname_obj); return -1; } + Py_DECREF(pathname_obj); } else { fd = open(pathname, flags); @@ -1385,9 +1394,15 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode) FILE* _Py_fopen(const char *pathname, const char *mode) { - if (PySys_Audit("open", "ssi", pathname, mode, 0) < 0) { + PyObject *pathname_obj = PyUnicode_DecodeFSDefault(pathname); + if (pathname_obj == NULL) { return NULL; } + if (PySys_Audit("open", "Osi", pathname_obj, mode, 0) < 0) { + Py_DECREF(pathname_obj); + return NULL; + } + Py_DECREF(pathname_obj); FILE *f = fopen(pathname, mode); if (f == NULL) @@ -1461,6 +1476,7 @@ _Py_fopen_obj(PyObject *path, const char *mode) path_bytes = PyBytes_AS_STRING(bytes); if (PySys_Audit("open", "Osi", path, mode, 0) < 0) { + Py_DECREF(bytes); return NULL; } diff --git a/Python/marshal.c b/Python/marshal.c index d3fee323..a9ba7a43 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1396,6 +1396,12 @@ r_object(RFILE *p) if (lnotab == NULL) goto code_error; + if (PySys_Audit("code.__new__", "OOOiiiiii", + code, filename, name, argcount, posonlyargcount, + kwonlyargcount, nlocals, stacksize, flags) < 0) { + goto code_error; + } + v = (PyObject *) PyCode_NewWithPosOnlyArgs( argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 258ff613..60c10449 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -15,9 +15,6 @@ extern "C" { _PyPathConfig _Py_path_config = _PyPathConfig_INIT; -#ifdef MS_WINDOWS -wchar_t *_Py_dll_path = NULL; -#endif static int @@ -105,10 +102,6 @@ _PyPathConfig_ClearGlobal(void) _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); pathconfig_clear(&_Py_path_config); -#ifdef MS_WINDOWS - PyMem_RawFree(_Py_dll_path); - _Py_dll_path = NULL; -#endif PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -145,31 +138,6 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep) } -#ifdef MS_WINDOWS -/* Initialize _Py_dll_path on Windows. Do nothing on other platforms. */ -static PyStatus -_PyPathConfig_InitDLLPath(void) -{ - if (_Py_dll_path != NULL) { - /* Already set: nothing to do */ - return _PyStatus_OK(); - } - - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - _Py_dll_path = _Py_GetDLLPath(); - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - if (_Py_dll_path == NULL) { - return _PyStatus_NO_MEMORY(); - } - return _PyStatus_OK(); -} -#endif - - static PyStatus pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config) { @@ -220,13 +188,6 @@ done: PyStatus _PyConfig_WritePathConfig(const PyConfig *config) { -#ifdef MS_WINDOWS - PyStatus status = _PyPathConfig_InitDLLPath(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } -#endif - return pathconfig_set_from_config(&_Py_path_config, config); } @@ -454,13 +415,6 @@ pathconfig_global_init(void) { PyStatus status; -#ifdef MS_WINDOWS - status = _PyPathConfig_InitDLLPath(); - if (_PyStatus_EXCEPTION(status)) { - Py_ExitStatusException(status); - } -#endif - if (_Py_path_config.module_search_path == NULL) { status = pathconfig_global_read(&_Py_path_config); if (_PyStatus_EXCEPTION(status)) { @@ -679,6 +633,7 @@ _PyPathConfig_ComputeSysPath0(const PyWideStringList *argv, PyObject **path0_p) #ifdef HAVE_READLINK wchar_t link[MAXPATHLEN + 1]; int nr = 0; + wchar_t path0copy[2 * MAXPATHLEN + 1]; if (have_script_arg) { nr = _Py_wreadlink(path0, link, Py_ARRAY_LENGTH(link)); @@ -701,7 +656,6 @@ _PyPathConfig_ComputeSysPath0(const PyWideStringList *argv, PyObject **path0_p) } else { /* Must make a copy, path0copy has room for 2 * MAXPATHLEN */ - wchar_t path0copy[2 * MAXPATHLEN + 1]; wcsncpy(path0copy, path0, MAXPATHLEN); q = wcsrchr(path0copy, SEP); wcsncpy(q+1, link, MAXPATHLEN); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 27cebf33..dc2d13db 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1229,13 +1229,6 @@ Py_FinalizeEx(void) /* nothing */; #endif - /* Clear all loghooks */ - /* We want minimal exposure of this function, so define the extern - * here. The linker should discover the correct function without - * exporting a symbol. */ - extern void _PySys_ClearAuditHooks(void); - _PySys_ClearAuditHooks(); - /* Destroy all modules */ PyImport_Cleanup(); @@ -1306,6 +1299,13 @@ Py_FinalizeEx(void) /* Clear interpreter state and all thread states. */ PyInterpreterState_Clear(interp); + /* Clear all loghooks */ + /* We want minimal exposure of this function, so define the extern + * here. The linker should discover the correct function without + * exporting a symbol. */ + extern void _PySys_ClearAuditHooks(void); + _PySys_ClearAuditHooks(); + /* Now we decref the exception classes. After this point nothing can raise an exception. That's okay, because each Fini() method below has been checked to make sure no exceptions are ever diff --git a/Python/pystate.c b/Python/pystate.c index 3e108556..b1d0f1cb 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1639,7 +1639,7 @@ _str_shared(PyObject *obj, _PyCrossInterpreterData *data) struct _shared_str_data *shared = PyMem_NEW(struct _shared_str_data, 1); shared->kind = PyUnicode_KIND(obj); shared->buffer = PyUnicode_DATA(obj); - shared->len = PyUnicode_GET_LENGTH(obj) - 1; + shared->len = PyUnicode_GET_LENGTH(obj); data->data = (void *)shared; Py_INCREF(obj); data->obj = obj; // Will be "released" (decref'ed) when data released. diff --git a/Python/pythonrun.c b/Python/pythonrun.c index a7da1430..6cdd8ea7 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1337,7 +1337,7 @@ PyParser_ASTFromStringObject(const char *s, PyObject *filename, int start, PyCompilerFlags localflags = _PyCompilerFlags_INIT; perrdetail err; int iflags = PARSER_FLAGS(flags); - if (flags && flags->cf_feature_version < 7) + if (flags && (flags->cf_flags & PyCF_ONLY_AST) && flags->cf_feature_version < 7) iflags |= PyPARSE_ASYNC_HACKS; node *n = PyParser_ParseStringObject(s, filename, diff --git a/Python/pytime.c b/Python/pytime.c index 9ff30069..109d5269 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1,6 +1,6 @@ #include "Python.h" #ifdef MS_WINDOWS -#include +#include /* struct timeval */ #endif #if defined(__APPLE__) diff --git a/README.rst b/README.rst index ae71b671..51a05302 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.8.3 +This is Python version 3.8.4 ============================ .. image:: https://travis-ci.org/python/cpython.svg?branch=3.8 diff --git a/Tools/c-globals/check-c-globals.py b/Tools/c-globals/check-c-globals.py index e68ed927..1371f927 100644 --- a/Tools/c-globals/check-c-globals.py +++ b/Tools/c-globals/check-c-globals.py @@ -37,7 +37,9 @@ IGNORED_VARS = { def find_capi_vars(root): capi_vars = {} for dirname in SOURCE_DIRS: - for filename in glob.glob(os.path.join(ROOT_DIR, dirname, '**/*.[hc]'), + for filename in glob.glob(os.path.join( + glob.escape(os.path.join(ROOT_DIR, dirname)), + '**/*.[hc]'), recursive=True): with open(filename) as file: for name in _find_capi_vars(file): diff --git a/Tools/msi/launcher/launcher_reg.wxs b/Tools/msi/launcher/launcher_reg.wxs index dace97ee..e8d9d24d 100644 --- a/Tools/msi/launcher/launcher_reg.wxs +++ b/Tools/msi/launcher/launcher_reg.wxs @@ -6,14 +6,14 @@ - + - + diff --git a/Tools/ssl/make_ssl_data.py b/Tools/ssl/make_ssl_data.py index d60f3528..c39e38c5 100755 --- a/Tools/ssl/make_ssl_data.py +++ b/Tools/ssl/make_ssl_data.py @@ -39,7 +39,7 @@ if __name__ == "__main__": f = sys.stdout if use_stdout else open(outfile, "w") # mnemonic -> (library code, error prefix, header file) error_libraries = {} - for error_header in glob.glob(os.path.join(openssl_inc, 'include/openssl/*err.h')): + for error_header in glob.glob(os.path.join(glob.escape(openssl_inc), 'include/openssl/*err.h')): base = os.path.basename(error_header) if base in ('buffererr.h', 'objectserr.h', 'storeerr.h'): # Deprecated in 3.0. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 05d6d7de..3818165a 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -30,10 +30,12 @@ import logging import os try: from urllib.request import urlopen + from urllib.error import HTTPError except ImportError: - from urllib2 import urlopen -import subprocess + from urllib2 import urlopen, HTTPError import shutil +import string +import subprocess import sys import tarfile @@ -41,20 +43,21 @@ import tarfile log = logging.getLogger("multissl") OPENSSL_OLD_VERSIONS = [ - "1.0.2", + "1.0.2u", + "1.1.0l", ] OPENSSL_RECENT_VERSIONS = [ - "1.0.2t", - "1.1.0l", - "1.1.1f", + "1.1.1g", + # "3.0.0-alpha2" ] LIBRESSL_OLD_VERSIONS = [ + "2.9.2", ] LIBRESSL_RECENT_VERSIONS = [ - "2.9.2", + "3.1.0", ] # store files in ../multissl @@ -78,7 +81,7 @@ parser.add_argument( parser.add_argument( '--disable-ancient', action='store_true', - help="Don't test OpenSSL < 1.0.2 and LibreSSL < 2.5.3.", + help="Don't test OpenSSL and LibreSSL versions without upstream support", ) parser.add_argument( '--openssl', @@ -143,10 +146,27 @@ parser.add_argument( help="Keep original sources for debugging." ) +OPENSSL_FIPS_CNF = """\ +openssl_conf = openssl_init + +.include {self.install_dir}/ssl/fipsinstall.cnf +# .include {self.install_dir}/ssl/openssl.cnf + +[openssl_init] +providers = provider_sect + +[provider_sect] +fips = fips_sect +default = default_sect + +[default_sect] +activate = 1 +""" + class AbstractBuilder(object): library = None - url_template = None + url_templates = None src_template = None build_template = None install_target = 'install' @@ -185,6 +205,11 @@ class AbstractBuilder(object): def __hash__(self): return hash((self.library, self.version)) + @property + def short_version(self): + """Short version for OpenSSL download URL""" + return None + @property def openssl_cli(self): """openssl CLI binary""" @@ -238,11 +263,23 @@ class AbstractBuilder(object): src_dir = os.path.dirname(self.src_file) if not os.path.isdir(src_dir): os.makedirs(src_dir) - url = self.url_template.format(self.version) - log.info("Downloading from {}".format(url)) - req = urlopen(url) - # KISS, read all, write all - data = req.read() + data = None + for url_template in self.url_templates: + url = url_template.format(v=self.version, s=self.short_version) + log.info("Downloading from {}".format(url)) + try: + req = urlopen(url) + # KISS, read all, write all + data = req.read() + except HTTPError as e: + log.error( + "Download from {} has from failed: {}".format(url, e) + ) + else: + log.info("Successfully downloaded from {}".format(url)) + break + if data is None: + raise ValueError("All download URLs have failed") log.info("Storing {}".format(self.src_file)) with open(self.src_file, "wb") as f: f.write(data) @@ -277,6 +314,7 @@ class AbstractBuilder(object): "shared", "--debug", "--prefix={}".format(self.install_dir) ] + # cmd.extend(["no-deprecated", "--api=1.1.0"]) env = os.environ.copy() # set rpath env["LD_RUN_PATH"] = self.lib_dir @@ -291,9 +329,13 @@ class AbstractBuilder(object): ["make", "-j1", self.install_target], cwd=self.build_dir ) + self._post_install() if not self.args.keep_sources: shutil.rmtree(self.build_dir) + def _post_install(self): + pass + def install(self): log.info(self.openssl_cli) if not self.has_openssl or self.args.force: @@ -359,17 +401,62 @@ class AbstractBuilder(object): class BuildOpenSSL(AbstractBuilder): library = "OpenSSL" - url_template = "https://www.openssl.org/source/openssl-{}.tar.gz" + url_templates = ( + "https://www.openssl.org/source/openssl-{v}.tar.gz", + "https://www.openssl.org/source/old/{s}/openssl-{v}.tar.gz" + ) src_template = "openssl-{}.tar.gz" build_template = "openssl-{}" # only install software, skip docs install_target = 'install_sw' + def _post_install(self): + if self.version.startswith("3.0"): + self._post_install_300() + + def _post_install_300(self): + # create ssl/ subdir with example configs + self._subprocess_call( + ["make", "-j1", "install_ssldirs"], + cwd=self.build_dir + ) + # Install FIPS module + # https://wiki.openssl.org/index.php/OpenSSL_3.0#Completing_the_installation_of_the_FIPS_Module + fipsinstall_cnf = os.path.join( + self.install_dir, "ssl", "fipsinstall.cnf" + ) + openssl_fips_cnf = os.path.join( + self.install_dir, "ssl", "openssl-fips.cnf" + ) + fips_mod = os.path.join(self.lib_dir, "ossl-modules/fips.so") + self._subprocess_call( + [ + self.openssl_cli, "fipsinstall", + "-out", fipsinstall_cnf, + "-module", fips_mod, + "-provider_name", "fips", + "-mac_name", "HMAC", + "-macopt", "digest:SHA256", + "-macopt", "hexkey:00", + "-section_name", "fips_sect" + ] + ) + with open(openssl_fips_cnf, "w") as f: + f.write(OPENSSL_FIPS_CNF.format(self=self)) + @property + def short_version(self): + """Short version for OpenSSL download URL""" + short_version = self.version.rstrip(string.ascii_letters) + if short_version.startswith("0.9"): + short_version = "0.9.x" + return short_version + class BuildLibreSSL(AbstractBuilder): library = "LibreSSL" - url_template = ( - "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-{}.tar.gz") + url_templates = ( + "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-{v}.tar.gz", + ) src_template = "libressl-{}.tar.gz" build_template = "libressl-{}" diff --git a/configure b/configure index 88865616..96dcd0dc 100755 --- a/configure +++ b/configure @@ -3398,7 +3398,7 @@ $as_echo "#define _BSD_SOURCE 1" >>confdefs.h # has no effect, don't bother defining them Darwin/[6789].*) define_xopen_source=no;; - Darwin/1[0-9].*) + Darwin/[12][0-9].*) define_xopen_source=no;; # On AIX 4 and 5.1, mbstate_t is defined only when _XOPEN_SOURCE == 500 but # used in wcsnrtombs() and mbsnrtowcs() even if _XOPEN_SOURCE is not defined diff --git a/configure.ac b/configure.ac index d8de9d49..18a04462 100644 --- a/configure.ac +++ b/configure.ac @@ -498,7 +498,7 @@ case $ac_sys_system/$ac_sys_release in # has no effect, don't bother defining them Darwin/@<:@6789@:>@.*) define_xopen_source=no;; - Darwin/1@<:@0-9@:>@.*) + Darwin/@<:@[12]@:>@@<:@0-9@:>@.*) define_xopen_source=no;; # On AIX 4 and 5.1, mbstate_t is defined only when _XOPEN_SOURCE == 500 but # used in wcsnrtombs() and mbsnrtowcs() even if _XOPEN_SOURCE is not defined diff --git a/setup.py b/setup.py index b168ed40..6340669f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ import os import re import sys import sysconfig -from glob import glob +from glob import glob, escape from distutils import log from distutils.command.build_ext import build_ext @@ -339,7 +339,7 @@ class PyBuildExt(build_ext): # Python header files headers = [sysconfig.get_config_h_filename()] - headers += glob(os.path.join(sysconfig.get_path('include'), "*.h")) + headers += glob(os.path.join(escape(sysconfig.get_path('include')), "*.h")) # The sysconfig variables built by makesetup that list the already # built modules and the disabled modules as configured by the Setup @@ -2256,7 +2256,7 @@ class PyBuildExt(build_ext): self.add(Extension('_sha1', ['sha1module.c'], depends=['hashlib.h'])) - blake2_deps = glob(os.path.join(self.srcdir, + blake2_deps = glob(os.path.join(escape(self.srcdir), 'Modules/_blake2/impl/*')) blake2_deps.append('hashlib.h') @@ -2266,7 +2266,7 @@ class PyBuildExt(build_ext): '_blake2/blake2s_impl.c'], depends=blake2_deps)) - sha3_deps = glob(os.path.join(self.srcdir, + sha3_deps = glob(os.path.join(escape(self.srcdir), 'Modules/_sha3/kcp/*')) sha3_deps.append('hashlib.h') self.add(Extension('_sha3',