From: DongHun Kwak Date: Tue, 25 Jan 2022 23:27:50 +0000 (+0900) Subject: Imported Upstream version 3.9.7 X-Git-Tag: upstream/3.9.7^0 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=1ef4657b14e3ecab7056c9248e7a03fd6ec0f9cd;p=platform%2Fupstream%2Fpython3.git Imported Upstream version 3.9.7 --- diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..81445d2d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*.{py,c,cpp,h,rst,md,yml}] +trim_trailing_whitespace = true +insert_final_newline = true +indent_style = space + +[*.{py,c,cpp,h}] +indent_size = 4 + +[*.yml] +indent_size = 2 diff --git a/Doc/Makefile b/Doc/Makefile index f653d706..57763fc0 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -137,14 +137,22 @@ pydoc-topics: build htmlview: html $(PYTHON) -c "import webbrowser; webbrowser.open('build/html/index.html')" -clean: - -rm -rf build/* $(VENVDIR)/* +clean: clean-venv + -rm -rf build/* + +clean-venv: + rm -rf $(VENVDIR) venv: - $(PYTHON) -m venv $(VENVDIR) - $(VENVDIR)/bin/python3 -m pip install -U pip setuptools - $(VENVDIR)/bin/python3 -m pip install -r requirements.txt - @echo "The venv has been created in the $(VENVDIR) directory" + @if [ -d $(VENVDIR) ] ; then \ + echo "venv already exists."; \ + echo "To recreate it, remove it first with \`make clean-venv'."; \ + else \ + $(PYTHON) -m venv $(VENVDIR); \ + $(VENVDIR)/bin/python3 -m pip install -U pip setuptools; \ + $(VENVDIR)/bin/python3 -m pip install -r requirements.txt; \ + echo "The venv has been created in the $(VENVDIR) directory"; \ + fi dist: rm -rf dist diff --git a/Doc/README.rst b/Doc/README.rst index 380ea4fa..7e8a27b4 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -28,28 +28,31 @@ install the tools into there. Using make ---------- -To get started on UNIX, you can create a virtual environment with the command :: +To get started on UNIX, you can create a virtual environment and build +documentation with the commands:: make venv - -That will install all the tools necessary to build the documentation. Assuming -the virtual environment was created in the ``venv`` directory (the default; -configurable with the VENVDIR variable), you can run the following command to -build the HTML output files:: - make html -By default, if the virtual environment is not created, the Makefile will -look for instances of sphinxbuild and blurb installed on your process PATH -(configurable with the SPHINXBUILD and BLURB variables). +The virtual environment in the ``venv`` directory will contain all the tools +necessary to build the documentation downloaded and installed from PyPI. +If you'd like to create the virtual environment in a different location, +you can specify it using the ``VENVDIR`` variable. + +You can also skip creating the virtual environment altogether, in which case +the Makefile will look for instances of ``sphinxbuild`` and ``blurb`` +installed on your process ``PATH`` (configurable with the ``SPHINXBUILD`` and +``BLURB`` variables). On Windows, we try to emulate the Makefile as closely as possible with a ``make.bat`` file. If you need to specify the Python interpreter to use, -set the PYTHON environment variable instead. +set the PYTHON environment variable. Available make targets are: -* "clean", which removes all build files. +* "clean", which removes all build files and the virtual environment. + +* "clean-venv", which removes the virtual environment directory. * "venv", which creates a virtual environment with all necessary tools installed. diff --git a/Doc/bugs.rst b/Doc/bugs.rst index a17f04d2..192995ab 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -80,7 +80,7 @@ taken on the bug. Article which goes into some detail about how to create a useful bug report. This describes what kind of information is useful and why it is useful. - `Bug Report Writing Guidelines `_ + `Bug Writing Guidelines `_ Information about writing a good bug report. Some of this is specific to the Mozilla project, but describes general good practices. diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 247b6d68..614eb245 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -100,7 +100,7 @@ For convenience, some of these functions will always return a This is the most common way to set the error indicator. The first argument specifies the exception type; it is normally one of the standard exceptions, e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. - The second argument is an error message; it is decoded from ``'utf-8``'. + The second argument is an error message; it is decoded from ``'utf-8'``. .. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 90766dca..f0569ed4 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -221,6 +221,12 @@ or request "multi-phase initialization" by returning the definition struct itsel than 0 and the module state (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* + called before a module is deallocated. For example, when reference + counting is enough to determine that an object is no longer used, + the cyclic garbage collector is not involved and + :c:member:`~PyModuleDef.m_free` is called directly. + .. versionchanged:: 3.9 No longer called before the module state is allocated. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 0a0e03ff..1e30d408 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -436,6 +436,21 @@ Accessing attributes of extension types {NULL} /* Sentinel */ }; + +.. c:function:: PyObject* PyMember_GetOne(const char *obj_addr, struct PyMemberDef *m) + + Get an attribute belonging to the object at address *obj_addr*. The + attribute is described by ``PyMemberDef`` *m*. Returns ``NULL`` + on error. + + +.. c:function:: int PyMember_SetOne(char *obj_addr, struct PyMemberDef *m, PyObject *o) + + Set an attribute belonging to the object at address *obj_addr* to object *o*. + The attribute to set is described by ``PyMemberDef`` *m*. Returns ``0`` + if successful and a negative value on failure. + + .. c:type:: PyGetSetDef Structure to define property-like access for a type. See also description of diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index d7f4bd67..69894cae 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -13,7 +13,7 @@ Type Objects The C structure of the objects used to describe built-in types. -.. c:var:: PyObject* PyType_Type +.. c:var:: PyTypeObject PyType_Type This is the type object for type objects; it is the same object as :class:`type` in the Python layer. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index ddcb8ae3..5a1f5f99 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1294,6 +1294,12 @@ and :c:type:`PyType_Type` effectively act as defaults.) so that *self* knows the contained object can no longer be used. The :c:func:`Py_CLEAR` macro performs the operations in a safe order. + Note that :c:member:`~PyTypeObject.tp_clear` is not *always* called + before an instance is deallocated. For example, when reference counting + is enough to determine that an object is no longer used, the cyclic garbage + collector is not involved and :c:member:`~PyTypeObject.tp_dealloc` is + called directly. + Because the goal of :c:member:`~PyTypeObject.tp_clear` functions is to break reference cycles, it's not necessary to clear contained objects like Python strings or Python integers, which can't participate in reference cycles. On the other hand, it may diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 8b1ed081..8e237046 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -149,7 +149,7 @@ access internal read-only data of Unicode objects: ``PyUnicode_WCHAR_KIND`` is deprecated. -.. c:function:: int PyUnicode_KIND(PyObject *o) +.. c:function:: unsigned int PyUnicode_KIND(PyObject *o) Return one of the PyUnicode kind constants (see above) that indicate how many bytes per character this Unicode object uses to store its data. *o* has to diff --git a/Doc/conf.py b/Doc/conf.py index 079d1771..d2dff7d7 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -86,7 +86,7 @@ templates_path = ['tools/templates'] # Custom sidebar templates, filenames relative to this file. html_sidebars = { - # Defaults taken from http://www.sphinx-doc.org/en/stable/config.html#confval-html_sidebars + # Defaults taken from https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-html_sidebars # Removes the quick search block '**': ['localtoc.html', 'relations.html', 'customsourcelink.html'], 'index': ['indexsidebar.html'], diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 02379946..56a0a2ea 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -131,11 +131,11 @@ involved in creating and publishing a project: * `The .pypirc file`_ .. _Project structure: \ - https://packaging.python.org/tutorials/distributing-packages/ + https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects .. _Building and packaging the project: \ - https://packaging.python.org/tutorials/distributing-packages/#packaging-your-project + https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files .. _Uploading the project to the Python Packaging Index: \ - https://packaging.python.org/tutorials/distributing-packages/#uploading-your-project-to-pypi + https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives .. _The .pypirc file: \ https://packaging.python.org/specifications/pypirc/ diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index a322fa23..145283d0 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -14,17 +14,8 @@ Graphic User Interface FAQ General GUI Questions ===================== -What platform-independent GUI toolkits exist for Python? -======================================================== - -Depending on what platform(s) you are aiming at, there are several. Some -of them haven't been ported to Python 3 yet. At least `Tkinter`_ and `Qt`_ -are known to be Python 3-compatible. - -.. XXX check links - -Tkinter -------- +What GUI toolkits exist for Python? +=================================== Standard builds of Python include an object-oriented interface to the Tcl/Tk widget set, called :ref:`tkinter `. This is probably the easiest to @@ -34,83 +25,12 @@ For more info about Tk, including pointers to the source, see the `Tcl/Tk home page `_. Tcl/Tk is fully portable to the Mac OS X, Windows, and Unix platforms. -wxWidgets ---------- - -wxWidgets (https://www.wxwidgets.org) is a free, portable GUI class -library written in C++ that provides a native look and feel on a -number of platforms, with Windows, Mac OS X, GTK, X11, all listed as -current stable targets. Language bindings are available for a number -of languages including Python, Perl, Ruby, etc. - -`wxPython `_ is the Python binding for -wxwidgets. While it often lags slightly behind the official wxWidgets -releases, it also offers a number of features via pure Python -extensions that are not available in other language bindings. There -is an active wxPython user and developer community. - -Both wxWidgets and wxPython are free, open source, software with -permissive licences that allow their use in commercial products as -well as in freeware or shareware. - - -Qt ---- - -There are bindings available for the Qt toolkit (using either `PyQt -`_ or `PySide -`_) and for KDE (`PyKDE4 `__). -PyQt is currently more mature than PySide, but you must buy a PyQt license from -`Riverbank Computing `_ -if you want to write proprietary applications. PySide is free for all applications. - -Qt 4.5 upwards is licensed under the LGPL license; also, commercial licenses -are available from `The Qt Company `_. - -Gtk+ ----- - -The `GObject introspection bindings `_ -for Python allow you to write GTK+ 3 applications. There is also a -`Python GTK+ 3 Tutorial `_. - -The older PyGtk bindings for the `Gtk+ 2 toolkit `_ have -been implemented by James Henstridge; see . - -Kivy ----- - -`Kivy `_ is a cross-platform GUI library supporting both -desktop operating systems (Windows, macOS, Linux) and mobile devices (Android, -iOS). It is written in Python and Cython, and can use a range of windowing -backends. - -Kivy is free and open source software distributed under the MIT license. - -FLTK ----- - -Python bindings for `the FLTK toolkit `_, a simple yet -powerful and mature cross-platform windowing system, are available from `the -PyFLTK project `_. - -OpenGL ------- - -For OpenGL bindings, see `PyOpenGL `_. - - -What platform-specific GUI toolkits exist for Python? -======================================================== - -By installing the `PyObjc Objective-C bridge -`_, Python programs can use Mac OS X's -Cocoa libraries. - -:ref:`Pythonwin ` by Mark Hammond includes an interface to the -Microsoft Foundation Classes and a Python programming environment -that's written mostly in Python using the MFC classes. - +Depending on what platform(s) you are aiming at, there are also several +alternatives. A `list of cross-platform +`_ and +`platform-specific +`_ GUI +frameworks can be found on the python wiki. Tkinter questions ================= diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index ad51bb6e..04d6592a 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1828,6 +1828,54 @@ For example, here is the implementation of return False +How can a subclass control what data is stored in an immutable instance? +------------------------------------------------------------------------ + +When subclassing an immutable type, override the :meth:`__new__` method +instead of the :meth:`__init__` method. The latter only runs *after* an +instance is created, which is too late to alter data in an immutable +instance. + +All of these immutable classes have a different signature than their +parent class: + +.. testcode:: + + from datetime import date + + class FirstOfMonthDate(date): + "Always choose the first day of the month" + def __new__(cls, year, month, day): + return super().__new__(cls, year, month, 1) + + class NamedInt(int): + "Allow text names for some numbers" + xlat = {'zero': 0, 'one': 1, 'ten': 10} + def __new__(cls, value): + value = cls.xlat.get(value, value) + return super().__new__(cls, value) + + class TitleStr(str): + "Convert str to name suitable for a URL path" + def __new__(cls, s): + s = s.lower().replace(' ', '-') + s = ''.join([c for c in s if c.isalnum() or c == '-']) + return super().__new__(cls, s) + +The classes can be used like this: + +.. doctest:: + + >>> FirstOfMonthDate(2012, 2, 14) + FirstOfMonthDate(2012, 2, 1) + >>> NamedInt('ten') + 10 + >>> NamedInt(20) + 20 + >>> TitleStr('Blog: Why Python Rocks') + 'blog-why-python-rocks' + + Modules ======= diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 4fd01e01..96d33ac1 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -426,12 +426,13 @@ Glossary which describe this functionality. __future__ - A pseudo-module which programmers can use to enable new language features - which are not compatible with the current interpreter. - - By importing the :mod:`__future__` module and evaluating its variables, - you can see when a new feature was first added to the language and when it - becomes the default:: + A :ref:`future statement `, ``from __future__ import ``, + directs the compiler to compile the current module using syntax or + semantics that will become standard in a future release of Python. + The :mod:`__future__` module documents the possible values of + *feature*. By importing this module and evaluating its variables, + you can see when a new feature was first added to the language and + when it will (or did) become the default:: >>> import __future__ >>> __future__.division diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 5777a4c5..20b02c83 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -2979,3 +2979,82 @@ refer to the comments in the code snippet for more detailed information. if __name__=='__main__': main() + + +.. patterns-to-avoid: + +Patterns to avoid +----------------- + +Although the preceding sections have described ways of doing things you might +need to do or deal with, it is worth mentioning some usage patterns which are +*unhelpful*, and which should therefore be avoided in most cases. The following +sections are in no particular order. + + +Opening the same log file multiple times +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +On Windows, you will generally not be able to open the same file multiple times +as this will lead to a "file is in use by another process" error. However, on +POSIX platforms you'll not get any errors if you open the same file multiple +times. This could be done accidentally, for example by: + +* Adding a file handler more than once which references the same file (e.g. by + a copy/paste/forget-to-change error). + +* Opening two files that look different, as they have different names, but are + the same because one is a symbolic link to the other. + +* Forking a process, following which both parent and child have a reference to + the same file. This might be through use of the :mod:`multiprocessing` module, + for example. + +Opening a file multiple times might *appear* to work most of the time, but can +lead to a number of problems in practice: + +* Logging output can be garbled because multiple threads or processes try to + write to the same file. Although logging guards against concurrent use of the + same handler instance by multiple threads, there is no such protection if + concurrent writes are attempted by two different threads using two different + handler instances which happen to point to the same file. + +* An attempt to delete a file (e.g. during file rotation) silently fails, + because there is another reference pointing to it. This can lead to confusion + and wasted debugging time - log entries end up in unexpected places, or are + lost altogether. + +Use the techniques outlined in :ref:`multiple-processes` to circumvent such +issues. + +Using loggers as attributes in a class or passing them as parameters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +While there might be unusual cases where you'll need to do this, in general +there is no point because loggers are singletons. Code can always access a +given logger instance by name using ``logging.getLogger(name)``, so passing +instances around and holding them as instance attributes is pointless. Note +that in other languages such as Java and C#, loggers are often static class +attributes. However, this pattern doesn't make sense in Python, where the +module (and not the class) is the unit of software decomposition. + + +Adding handlers other than :class:`NullHandler` to a logger in a library +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Configuring logging by adding handlers, formatters and filters is the +responsibility of the application developer, not the library developer. If you +are maintaining a library, ensure that you don't add handlers to any of your +loggers other than a :class:`~logging.NullHandler` instance. + + +Creating a lot of loggers +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Loggers are singletons that are never freed during a script execution, and so +creating lots of loggers will use up memory which can't then be freed. Rather +than create a logger per e.g. file processed or network connection made, use +the :ref:`existing mechanisms ` for passing contextual +information into your logs and restrict the loggers created to those describing +areas within your application (generally modules, but occasionally slightly +more fine-grained than that). diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index 1543823c..abcc3428 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -20,8 +20,8 @@ Porting Python 2 Code to Python 3 came into existence, you can read Nick Coghlan's `Python 3 Q & A`_ or Brett Cannon's `Why Python 3 exists`_. - For help with porting, you can email the python-porting_ mailing list with - questions. + + For help with porting, you can view the archived python-porting_ mailing list. The Short Explanation ===================== @@ -446,7 +446,7 @@ to make sure everything functions as expected in both versions of Python. .. _pytype: https://github.com/google/pytype .. _python-future: http://python-future.org/ -.. _python-porting: https://mail.python.org/mailman/listinfo/python-porting +.. _python-porting: https://mail.python.org/pipermail/python-porting/ .. _six: https://pypi.org/project/six .. _tox: https://pypi.org/project/tox .. _trove classifier: https://pypi.org/classifiers diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index aa4713e7..6c01667b 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -853,6 +853,8 @@ is available in ``argparse`` and adds support for boolean actions such as >>> parser.parse_args(['--no-foo']) Namespace(foo=False) +.. versionadded:: 3.9 + The recommended way to create a custom action is to extend :class:`Action`, overriding the ``__call__`` method and optionally the ``__init__`` and ``format_usage`` methods. diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 9328faf5..b8eecbda 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1253,7 +1253,7 @@ Function and class definitions A function definition. * ``name`` is a raw string of the function name. - * ``args`` is a :class:`arguments` node. + * ``args`` is an :class:`arguments` node. * ``body`` is the list of nodes inside the function. * ``decorator_list`` is the list of decorators to be applied, stored outermost first (i.e. the first in the list will be applied last). @@ -1427,7 +1427,7 @@ Function and class definitions * ``bases`` is a list of nodes for explicitly specified base classes. * ``keywords`` is a list of :class:`keyword` nodes, principally for 'metaclass'. Other keywords will be passed to the metaclass, as per `PEP-3115 - `_. + `_. * ``starargs`` and ``kwargs`` are each a single node, as in a function call. starargs will be expanded to join the list of base classes, and kwargs will be passed to the metaclass. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 66eb3fae..4a64475f 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -211,6 +211,10 @@ Scheduling callbacks A thread-safe variant of :meth:`call_soon`. Must be used to schedule callbacks *from another thread*. + Raises :exc:`RuntimeError` if called on a loop that's been closed. + This can happen on a secondary thread when the main application is + shutting down. + See the :ref:`concurrency and multithreading ` section of the documentation. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index e6fa33ac..f7f03810 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -48,11 +48,12 @@ internal error is detected, or when :func:`os._exit` is called. .. function:: unregister(func) - Remove *func* from the list of functions to be run at interpreter - shutdown. After calling :func:`unregister`, *func* is guaranteed not to be - called when the interpreter shuts down, even if it was registered more than - once. :func:`unregister` silently does nothing if *func* was not previously - registered. + Remove *func* from the list of functions to be run at interpreter shutdown. + :func:`unregister` silently does nothing if *func* was not previously + registered. If *func* has been registered more than once, every occurrence + of that function in the :mod:`atexit` call stack will be removed. Equality + comparisons (``==``) are used internally during unregistration, so function + references do not need to have matching identities. .. seealso:: diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 637baf49..b5c3277f 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -322,3 +322,8 @@ Writing and reading a bzip2-compressed file in binary mode: ... content = f.read() >>> content == data # Check equality to original object after round-trip True + +.. testcleanup:: + + import os + os.remove("myfile.bz2") diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 5189a2e8..e00b75fa 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -1149,27 +1149,43 @@ original insertion position is changed and moved to the end:: self.move_to_end(key) An :class:`OrderedDict` would also be useful for implementing -variants of :func:`functools.lru_cache`:: +variants of :func:`functools.lru_cache`: - class LRU(OrderedDict): - 'Limit size, evicting the least recently looked-up key when full' +.. testcode:: - def __init__(self, maxsize=128, /, *args, **kwds): - self.maxsize = maxsize - super().__init__(*args, **kwds) + class LRU: - def __getitem__(self, key): - value = super().__getitem__(key) - self.move_to_end(key) + def __init__(self, func, maxsize=128): + self.func = func + self.maxsize = maxsize + self.cache = OrderedDict() + + def __call__(self, *args): + if args in self.cache: + value = self.cache[args] + self.cache.move_to_end(args) + return value + value = self.func(*args) + if len(self.cache) >= self.maxsize: + self.cache.popitem(False) + self.cache[args] = value return value - def __setitem__(self, key, value): - if key in self: - self.move_to_end(key) - super().__setitem__(key, value) - if len(self) > self.maxsize: - oldest = next(iter(self)) - del self[oldest] +.. doctest:: + :hide: + + >>> def square(x): + ... return x ** 2 + ... + >>> s = LRU(square, maxsize=5) + >>> actual = [(s(x), s(x)) for x in range(20)] + >>> expected = [(x**2, x**2) for x in range(20)] + >>> actual == expected + True + >>> actual = list(s.cache.items()) + >>> expected = [((x,), x**2) for x in range(15, 20)] + >>> actual == expected + True :class:`UserDict` objects @@ -1187,7 +1203,7 @@ attribute. regular dictionary, which is accessible via the :attr:`data` attribute of :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is initialized with its contents; note that a reference to *initialdata* will not - be kept, allowing it be used for other purposes. + be kept, allowing it to be used for other purposes. In addition to supporting the methods and operations of mappings, :class:`UserDict` instances provide the following attribute: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index d57f8ce2..897efc2f 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -350,7 +350,7 @@ The :class:`Future` class encapsulates the asynchronous execution of a callable. If the future is cancelled before completing then :exc:`.CancelledError` will be raised. - If the call raised, this method will raise the same exception. + If the call raised an exception, this method will raise the same exception. .. method:: exception(timeout=None) diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 730d1df9..3de59183 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -46,6 +46,11 @@ can be customized by end users easily. import configparser +.. testcleanup:: + + import os + os.remove("example.ini") + Quick Start ----------- diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index f87ee210..66a0f17a 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -146,7 +146,7 @@ Functions and classes provided: from contextlib import closing from urllib.request import urlopen - with closing(urlopen('http://www.python.org')) as page: + with closing(urlopen('https://www.python.org')) as page: for line in page: print(line) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 7a72c26d..275ee10e 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -94,8 +94,8 @@ The :mod:`csv` module defines the following functions: :class:`Dialect` class or one of the strings returned by the :func:`list_dialects` function. The other optional *fmtparams* keyword arguments can be given to override individual formatting parameters in the current - dialect. For full details about the dialect and formatting parameters, see - section :ref:`csv-fmt-params`. To make it + dialect. For full details about dialects and formatting parameters, see + the :ref:`csv-fmt-params` section. To make it as easy as possible to interface with modules which implement the DB API, the value :const:`None` is written as the empty string. While this isn't a reversible transformation, it makes it easier to dump SQL NULL data values to @@ -117,7 +117,7 @@ The :mod:`csv` module defines the following functions: Associate *dialect* with *name*. *name* must be a string. The dialect can be specified either by passing a sub-class of :class:`Dialect`, or by *fmtparams* keyword arguments, or both, with keyword arguments overriding - parameters of the dialect. For full details about the dialect and formatting + parameters of the dialect. For full details about dialects and formatting parameters, see section :ref:`csv-fmt-params`. @@ -225,9 +225,21 @@ The :mod:`csv` module defines the following classes: .. class:: Dialect - The :class:`Dialect` class is a container class relied on primarily for its - attributes, which are used to define the parameters for a specific - :class:`reader` or :class:`writer` instance. + The :class:`Dialect` class is a container class whose attributes contain + information for how to handle doublequotes, whitespace, delimiters, etc. + Due to the lack of a strict CSV specification, different applications + produce subtly different CSV data. :class:`Dialect` instances define how + :class:`reader` and :class:`writer` instances behave. + + All available :class:`Dialect` names are returned by :func:`list_dialects`, + and they can be registered with specific :class:`reader` and :class:`writer` + classes through their initializer (``__init__``) functions like this:: + + import csv + + with open('students.csv', 'w', newline='') as csvfile: + writer = csv.writer(csvfile, dialect='unix') + ^^^^^^^^^^^^^^ .. class:: excel() @@ -405,8 +417,8 @@ Reader objects (:class:`DictReader` instances and objects returned by the Return the next row of the reader's iterable object as a list (if the object was returned from :func:`reader`) or a dict (if it is a :class:`DictReader` - instance), parsed according to the current dialect. Usually you should call - this as ``next(reader)``. + instance), parsed according to the current :class:`Dialect`. Usually you + should call this as ``next(reader)``. Reader objects have the following public attributes: @@ -446,9 +458,9 @@ read CSV files (assuming they support complex numbers at all). .. method:: csvwriter.writerow(row) - Write the *row* parameter to the writer's file object, formatted according to - the current dialect. Return the return value of the call to the *write* method - of the underlying file object. + Write the *row* parameter to the writer's file object, formatted according + to the current :class:`Dialect`. Return the return value of the call to the + *write* method of the underlying file object. .. versionchanged:: 3.5 Added support of arbitrary iterables. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 08730c4d..6d71398f 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -741,7 +741,7 @@ iterations of the loop. This opcode performs several operations before a with block starts. First, it loads :meth:`~object.__exit__` from the context manager and pushes it onto - the stack for later use by :opcode:`WITH_CLEANUP_START`. Then, + the stack for later use by :opcode:`WITH_EXCEPT_START`. Then, :meth:`~object.__enter__` is called, and a finally block pointing to *delta* is pushed. Finally, the result of calling the ``__enter__()`` method is pushed onto the stack. The next opcode will either ignore it (:opcode:`POP_TOP`), or diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 6bed5c70..85cd193f 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -659,8 +659,10 @@ depending on the system error code. .. exception:: NotADirectoryError - Raised when a directory operation (such as :func:`os.listdir`) is requested - on something which is not a directory. + Raised when a directory operation (such as :func:`os.listdir`) is requested on + something which is not a directory. On most POSIX platforms, it may also be + raised if an operation attempts to open or traverse a non-directory file as if + it were a directory. Corresponds to :c:data:`errno` ``ENOTDIR``. .. exception:: PermissionError diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index 31b9b4af..1d04c3ef 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -22,8 +22,11 @@ The :mod:`filecmp` module defines the following functions: Compare the files named *f1* and *f2*, returning ``True`` if they seem equal, ``False`` otherwise. - If *shallow* is true, files with identical :func:`os.stat` signatures are - taken to be equal. Otherwise, the contents of the files are compared. + If *shallow* is true and the :func:`os.stat` signatures (file type, size, and + modification time) of both files are identical, the files are taken to be + equal. + + Otherwise, the files are treated as different if their sizes or contents differ. Note that no external programs are called from this function, giving it portability and efficiency. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 784bb62e..c5c1c161 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1150,9 +1150,9 @@ are always available. They are listed here in alphabetical order. * ``'replace'`` causes a replacement marker (such as ``'?'``) to be inserted where there is malformed data. - * ``'surrogateescape'`` will represent any incorrect bytes as code - points in the Unicode Private Use Area ranging from U+DC80 to - U+DCFF. These private code points will then be turned back into + * ``'surrogateescape'`` will represent any incorrect bytes as low + surrogate code units ranging from U+DC80 to U+DCFF. + These surrogate code units will then be turned back into the same bytes when the ``surrogateescape`` error handler is used when writing data. This is useful for processing files in an unknown encoding. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 85d4e746..b2901e43 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -199,7 +199,7 @@ The :mod:`functools` module defines the following functions: @lru_cache(maxsize=32) def get_pep(num): 'Retrieve text of a Python Enhancement Proposal' - resource = 'http://www.python.org/dev/peps/pep-%04d/' % num + resource = 'https://www.python.org/dev/peps/pep-%04d/' % num try: with urllib.request.urlopen(resource) as s: return s.read() diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index 0faca218..2bc80da4 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -154,9 +154,10 @@ .. method:: static_order() - Returns an iterable of nodes in a topological order. Using this method - does not require to call :meth:`TopologicalSorter.prepare` or - :meth:`TopologicalSorter.done`. This method is equivalent to:: + Returns an iterator object which will iterate over nodes in a topological + order. When using this method, :meth:`~TopologicalSorter.prepare` and + :meth:`~TopologicalSorter.done` should not be called. This method is + equivalent to:: def static_order(self): self.prepare() @@ -206,4 +207,4 @@ The :mod:`graphlib` module defines the following exception classes: The detected cycle can be accessed via the second element in the :attr:`~CycleError.args` attribute of the exception instance and consists in a list of nodes, such that each node is, in the graph, an immediate predecessor of the next node in the list. In the reported list, - the first and the last node will be the same, to make it clear that it is cyclic. \ No newline at end of file + the first and the last node will be the same, to make it clear that it is cyclic. diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 5a507c12..615ee116 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -80,6 +80,8 @@ library that Python uses on your platform. On most platforms the .. versionadded:: 3.6 :func:`blake2b` and :func:`blake2s` were added. +.. _hashlib-usedforsecurity: + .. versionchanged:: 3.9 All hashlib constructors take a keyword-only argument *usedforsecurity* with default value ``True``. A false value allows the use of insecure and diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 029e9ec5..08b0ddf5 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -197,7 +197,7 @@ provides three different variants: request header it responds back with a ``100 Continue`` followed by ``200 OK`` headers. This method can be overridden to raise an error if the server does not - want the client to continue. For e.g. server can chose to send ``417 + want the client to continue. For e.g. server can choose to send ``417 Expectation Failed`` as a response header and ``return False``. .. versionadded:: 3.2 diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 3c302115..4fa4f1e5 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -518,7 +518,7 @@ and not restarting the Shell thereafter. This is especially useful after adding imports at the top of a file. This also increases possible attribute completions. -Completion boxes intially exclude names beginning with '_' or, for +Completion boxes initially exclude names beginning with '_' or, for modules, not included in '__all__'. The hidden names can be accessed by typing '_' after '.', either before or after the box is opened. diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 9bf34047..1f73017e 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -149,7 +149,7 @@ Distribution files You can also get the full set of files contained within a distribution. The ``files()`` function takes a distribution package name and returns all of the files installed by this distribution. Each file object returned is a -``PackagePath``, a :class:`pathlib.Path` derived object with additional ``dist``, +``PackagePath``, a :class:`pathlib.PurePath` derived object with additional ``dist``, ``size``, and ``hash`` properties as indicated by the metadata. For example:: >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP @@ -173,6 +173,12 @@ Once you have the file, you can also read its contents:: return s.encode('utf-8') return s +You can also use the ``locate`` method to get a the absolute path to the +file:: + + >>> util.locate() # doctest: +SKIP + PosixPath('/home/gustav/example/lib/site-packages/wheel/util.py') + In the case where the metadata file listing files (RECORD or SOURCES.txt) is missing, ``files()`` will return ``None``. The caller may wish to wrap calls to diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 736c43d9..69e31129 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -612,7 +612,7 @@ ABC hierarchy:: .. method:: is_package(fullname) - An abstract method to return a true value if the module is a package, a + An optional method to return a true value if the module is a package, a false value otherwise. :exc:`ImportError` is raised if the :term:`loader` cannot find the module. diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 2cddb417..0fd6e4c0 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -76,3 +76,4 @@ the `Python Package Index `_. unix.rst superseded.rst undoc.rst + security_warnings.rst diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 96e02e83..0ff05cfc 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -315,6 +315,9 @@ I/O Base Classes to control the number of lines read: no more lines will be read if the total size (in bytes/characters) of all lines so far exceeds *hint*. + *hint* values of ``0`` or less, as well as ``None``, are treated as no + hint. + Note that it's already possible to iterate on file objects using ``for line in file: ...`` without calling ``file.readlines()``. diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 1c2263b1..2ab4dd83 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -132,6 +132,11 @@ write code that handles both IP versions correctly. Address objects are The above change was also included in Python 3.9 starting with version 3.9.5. + .. versionchanged:: 3.8.12 + + The above change was also included in Python 3.8 starting with + version 3.8.12. + .. attribute:: version The appropriate version number: ``4`` for IPv4, ``6`` for IPv6. diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 0b5e2fc2..ab448505 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -147,6 +147,8 @@ in :mod:`logging` itself) and defining handlers which are declared either in send it to the socket as a sequence of bytes preceded by a four-byte length string packed in binary using ``struct.pack('>L', n)``. + .. _logging-eval-security: + .. note:: Because portions of the configuration are passed through diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index a7f815e9..6233a2a6 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -893,6 +893,10 @@ interchangeably. :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added to :class:`LoggerAdapter`. These methods delegate to the underlying logger. +.. versionchanged:: 3.6 + Attribute :attr:`manager` and method :meth:`_log` were added, which + delegate to the underlying logger and allow adapters to be nested. + Thread Safety ------------- diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index d65afc20..b38ba54b 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -66,6 +66,8 @@ The module defines these functions: The *version* argument indicates the data format that ``dump`` should use (see below). + .. audit-event:: marshal.dumps value,version marshal.dump + .. function:: load(file) @@ -74,11 +76,18 @@ The module defines these functions: format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. The file must be a readable :term:`binary file`. + .. audit-event:: marshal.load "" marshal.load + .. note:: If an object containing an unsupported type was marshalled with :func:`dump`, :func:`load` will substitute ``None`` for the unmarshallable type. + .. versionchanged:: 3.9.7 + + This call used to raise a ``code.__new__`` audit event for each code object. Now + it raises a single ``marshal.load`` event for the entire load operation. + .. function:: dumps(value[, version]) @@ -89,6 +98,8 @@ The module defines these functions: The *version* argument indicates the data format that ``dumps`` should use (see below). + .. audit-event:: marshal.dumps value,version marshal.dump + .. function:: loads(bytes) @@ -96,6 +107,13 @@ The module defines these functions: :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra bytes in the input are ignored. + .. audit-event:: marshal.loads bytes marshal.load + + .. versionchanged:: 3.9.7 + + This call used to raise a ``code.__new__`` audit event for each code object. Now + it raises a single ``marshal.loads`` event for the entire load operation. + In addition, the following constants are defined: diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index def27bf0..952a5b40 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1187,6 +1187,7 @@ For example: >>> arr2 array('i', [0, 1, 2, 3, 4, 0, 0, 0, 0, 0]) +.. _multiprocessing-recv-pickle-security: .. warning:: diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index d2fe4943..72964920 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -340,6 +340,14 @@ the :mod:`glob` module.) that contains symbolic links. On Windows, it converts forward slashes to backward slashes. To normalize case, use :func:`normcase`. + .. note:: + On POSIX systems, in accordance with `IEEE Std 1003.1 2013 Edition; 4.13 + Pathname Resolution `_, + if a pathname begins with exactly two slashes, the first component + following the leading characters may be interpreted in an implementation-defined + manner, although more than two leading characters shall be treated as a + single character. + .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -450,12 +458,16 @@ the :mod:`glob` module.) On Windows, splits a pathname into drive/UNC sharepoint and relative path. If the path contains a drive letter, drive will contain everything - up to and including the colon. - e.g. ``splitdrive("c:/dir")`` returns ``("c:", "/dir")`` + up to and including the colon:: + + >>> splitdrive("c:/dir") + ("c:", "/dir") If the path contains a UNC path, drive will contain the host name - and share, up to but not including the fourth separator. - e.g. ``splitdrive("//host/computer/dir")`` returns ``("//host/computer", "/dir")`` + and share, up to but not including the fourth separator:: + + >>> splitdrive("//host/computer/dir") + ("//host/computer", "/dir") .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -464,9 +476,24 @@ the :mod:`glob` module.) .. function:: splitext(path) Split the pathname *path* into a pair ``(root, ext)`` such that ``root + ext == - path``, and *ext* is empty or begins with a period and contains at most one - period. Leading periods on the basename are ignored; ``splitext('.cshrc')`` - returns ``('.cshrc', '')``. + path``, and the extension, *ext*, is empty or begins with a period and contains at + most one period. + + If the path contains no extension, *ext* will be ``''``:: + + >>> splitext('bar') + ('bar', '') + + If the path contains an extension, then *ext* will be set to this extension, + including the leading period. Note that previous periods will be ignored:: + + >>> splitext('foo.bar.exe') + ('foo.bar', '.exe') + + Leading periods on the basename are ignored:: + + >>> splitext('.cshrc') + ('.cshrc', '') .. versionchanged:: 3.6 Accepts a :term:`path-like object`. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 2295ffc8..0ad44d1a 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -126,7 +126,7 @@ process and user. .. note:: - On some platforms, including FreeBSD and Mac OS X, setting ``environ`` may + On some platforms, including FreeBSD and macOS, setting ``environ`` may cause memory leaks. Refer to the system documentation for :c:func:`putenv`. @@ -301,7 +301,7 @@ process and user. .. note:: - On Mac OS X, :func:`getgroups` behavior differs somewhat from + On macOS, :func:`getgroups` behavior differs somewhat from other Unix platforms. If the Python interpreter was built with a deployment target of :const:`10.5` or earlier, :func:`getgroups` returns the list of effective group ids associated with the current user process; @@ -448,7 +448,7 @@ process and user. .. note:: - On some platforms, including FreeBSD and Mac OS X, setting ``environ`` may + On some platforms, including FreeBSD and macOS, setting ``environ`` may cause memory leaks. Refer to the system documentation for :c:func:`putenv`. .. audit-event:: os.putenv key,value os.putenv @@ -486,7 +486,7 @@ process and user. .. availability:: Unix. - .. note:: On Mac OS X, the length of *groups* may not exceed the + .. note:: On macOS, the length of *groups* may not exceed the system-defined maximum number of effective group ids, typically 16. See the documentation for :func:`getgroups` for cases where it may not return the same group list set by calling setgroups(). @@ -1280,11 +1280,11 @@ or `the MSDN `_ on Windo On Linux, if *offset* is given as ``None``, the bytes are read from the current position of *in_fd* and the position of *in_fd* is updated. - The second case may be used on Mac OS X and FreeBSD where *headers* and + The second case may be used on macOS and FreeBSD where *headers* and *trailers* are arbitrary sequences of buffers that are written before and after the data from *in_fd* is written. It returns the same as the first case. - On Mac OS X and FreeBSD, a value of ``0`` for *count* specifies to send until + On macOS and FreeBSD, a value of ``0`` for *count* specifies to send until the end of *in_fd* is reached. All platforms support sockets as *out_fd* file descriptor, and some platforms diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst deleted file mode 100644 index 48c1f275..00000000 --- a/Doc/library/othergui.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. _other-gui-packages: - -Other Graphical User Interface Packages -======================================= - -Major cross-platform (Windows, Mac OS X, Unix-like) GUI toolkits are -available for Python: - -.. seealso:: - - `PyGObject `_ - PyGObject provides introspection bindings for C libraries using - `GObject `_. One of - these libraries is the `GTK+ 3 `_ widget set. - GTK+ comes with many more widgets than Tkinter provides. An online - `Python GTK+ 3 Tutorial `_ - is available. - - `PyGTK `_ - PyGTK provides bindings for an older version - of the library, GTK+ 2. It provides an object oriented interface that - is slightly higher level than the C one. There are also bindings to - `GNOME `_. An online `tutorial - `_ is available. - - `PyQt `_ - PyQt is a :program:`sip`\ -wrapped binding to the Qt toolkit. Qt is an - extensive C++ GUI application development framework that is - available for Unix, Windows and Mac OS X. :program:`sip` is a tool - for generating bindings for C++ libraries as Python classes, and - is specifically designed for Python. - - `PySide2 `_ - Also known as the Qt for Python project, PySide2 is a newer binding to the - Qt toolkit. It is provided by The Qt Company and aims to provide a - complete port of PySide to Qt 5. Compared to PyQt, its licensing scheme is - friendlier to non-open source applications. - - `wxPython `_ - wxPython is a cross-platform GUI toolkit for Python that is built around - the popular `wxWidgets `_ (formerly wxWindows) - C++ toolkit. It provides a native look and feel for applications on - Windows, Mac OS X, and Unix systems by using each platform's native - widgets where ever possible, (GTK+ on Unix-like systems). In addition to - an extensive set of widgets, wxPython provides classes for online - documentation and context sensitive help, printing, HTML viewing, - low-level device context drawing, drag and drop, system clipboard access, - an XML-based resource format and more, including an ever growing library - of user-contributed modules. - -PyGTK, PyQt, PySide2, and wxPython, all have a modern look and feel and more -widgets than Tkinter. In addition, there are many other GUI toolkits for -Python, both cross-platform, and platform-specific. See the `GUI Programming -`_ page in the Python Wiki for a -much more complete list, and also for links to documents where the -different GUI toolkits are compared. - diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index b293adf4..49186ce3 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -42,7 +42,7 @@ Cross Platform .. note:: - On Mac OS X (and perhaps other platforms), executable files may be + On macOS (and perhaps other platforms), executable files may be universal files containing multiple architectures. To get at the "64-bitness" of the current interpreter, it is more diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 9abbd8ba..ff7687cc 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -824,10 +824,20 @@ form. .. function:: findall(pattern, string, flags=0) Return all non-overlapping matches of *pattern* in *string*, as a list of - strings. The *string* is scanned left-to-right, and matches are returned in - the order found. If one or more groups are present in the pattern, return a - list of groups; this will be a list of tuples if the pattern has more than - one group. Empty matches are included in the result. + strings or tuples. The *string* is scanned left-to-right, and matches + are returned in the order found. Empty matches are included in the result. + + The result depends on the number of capturing groups in the pattern. + If there are no groups, return a list of strings matching the whole + pattern. If there is exactly one group, return a list of strings + matching that group. If multiple groups are present, return a list + of tuples of strings matching the groups. Non-capturing groups do not + affect the form of the result. + + >>> re.findall(r'\bf[a-z]*', 'which foot or hand fell fastest') + ['foot', 'fell', 'fastest'] + >>> re.findall(r'(\w+)=(\d+)', 'set width=20 and height=10') + [('width', '20'), ('height', '10')] .. versionchanged:: 3.7 Non-empty matches can now start just after a previous empty match. @@ -931,8 +941,8 @@ form. This is useful if you want to match an arbitrary literal string that may have regular expression metacharacters in it. For example:: - >>> print(re.escape('http://www.python.org')) - http://www\.python\.org + >>> print(re.escape('https://www.python.org')) + https://www\.python\.org >>> legal_chars = string.ascii_lowercase + string.digits + "!#$%&'*+-.^_`|~:" >>> print('[%s]+' % re.escape(legal_chars)) diff --git a/Doc/library/security_warnings.rst b/Doc/library/security_warnings.rst new file mode 100644 index 00000000..61fd4e6e --- /dev/null +++ b/Doc/library/security_warnings.rst @@ -0,0 +1,32 @@ +.. _security-warnings: + +.. index:: single: security considerations + +Security Considerations +======================= + +The following modules have specific security considerations: + +* :mod:`cgi`: :ref:`CGI security considerations ` +* :mod:`hashlib`: :ref:`all constructors take a "usedforsecurity" keyword-only + argument disabling known insecure and blocked algorithms + ` +* :mod:`http.server` is not suitable for production use, only implementing + basic security checks +* :mod:`logging`: :ref:`Logging configuration uses eval() + ` +* :mod:`multiprocessing`: :ref:`Connection.recv() uses pickle + ` +* :mod:`pickle`: :ref:`Restricting globals in pickle ` +* :mod:`random` shouldn't be used for security purposes, use :mod:`secrets` + instead +* :mod:`shelve`: :ref:`shelve is based on pickle and thus unsuitable for + dealing with untrusted sources ` +* :mod:`ssl`: :ref:`SSL/TLS security considerations ` +* :mod:`subprocess`: :ref:`Subprocess security considerations + ` +* :mod:`tempfile`: :ref:`mktemp is deprecated due to vulnerability to race + conditions ` +* :mod:`xml`: :ref:`XML vulnerabilities ` +* :mod:`zipfile`: :ref:`maliciously prepared .zip files can cause disk volume + exhaustion ` diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index f08c5817..98ad4082 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -49,13 +49,16 @@ lots of shared sub-objects. The keys are ordinary strings. with shelve.open('spam') as db: db['eggs'] = 'eggs' +.. _shelve-security: + .. warning:: Because the :mod:`shelve` module is backed by :mod:`pickle`, it is insecure to load a shelf from an untrusted source. Like with pickle, loading a shelf can execute arbitrary code. -Shelf objects support all methods supported by dictionaries. This eases the +Shelf objects support most of methods and operations supported by dictionaries +(except copying, constructors and operators ``|`` and ``|=``). This eases the transition from dictionary based scripts to those requiring persistent storage. Two additional methods are supported: diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index d5080da1..11c67074 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -595,6 +595,10 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. .. audit-event:: shutil.make_archive base_name,format,root_dir,base_dir shutil.make_archive + .. note:: + + This function is not thread-safe. + .. versionchanged:: 3.8 The modern pax (POSIX.1-2001) format is now used instead of the legacy GNU format for archives created with ``format="tar"``. diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 26892c99..609fbe86 100755 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -795,8 +795,9 @@ The :mod:`socket` module also offers various network-related services: it is interpreted as the local host. To find the fully qualified name, the hostname returned by :func:`gethostbyaddr` is checked, followed by aliases for the host, if available. The first name which includes a period is selected. In - case no fully qualified domain name is available, the hostname as returned by - :func:`gethostname` is returned. + case no fully qualified domain name is available and *name* was provided, + it is returned unchanged. If *name* was empty or equal to ``'0.0.0.0'``, + the hostname from :func:`gethostname` is returned. .. function:: gethostbyname(hostname) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 4ae4b4dc..c0b6ab07 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -164,7 +164,7 @@ Module functions and constants does not include the type, i. e. if you use something like ``'as "Expiration date [datetime]"'`` in your SQL, then we will parse out everything until the first ``'['`` for the column name and strip - the preceeding space: the column name would simply be "Expiration date". + the preceding space: the column name would simply be "Expiration date". .. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri]) @@ -636,7 +636,7 @@ Cursor Objects This is a nonstandard convenience method for executing multiple SQL statements at once. It issues a ``COMMIT`` statement first, then executes the SQL script it gets as a parameter. This method disregards :attr:`isolation_level`; any - transation control must be added to *sql_script*. + transaction control must be added to *sql_script*. *sql_script* can be an instance of :class:`str`. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index acaf99b7..e0b9285e 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1092,7 +1092,7 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). | | index given by *i* | | | | (same as ``s[i:i] = [x]``) | | +------------------------------+--------------------------------+---------------------+ -| ``s.pop([i])`` | retrieves the item at *i* and | \(2) | +| ``s.pop()`` or ``s.pop(i)`` | retrieves the item at *i* and | \(2) | | | also removes it from *s* | | +------------------------------+--------------------------------+---------------------+ | ``s.remove(x)`` | remove the first item from *s* | \(3) | diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 762da9b1..57a3285f 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -662,7 +662,10 @@ execute, will be re-raised in the parent. The most common exception raised is :exc:`OSError`. This occurs, for example, when trying to execute a non-existent file. Applications should prepare for -:exc:`OSError` exceptions. +:exc:`OSError` exceptions. Note that, when ``shell=True``, :exc:`OSError` +will be raised by the child only if the selected shell itself was not found. +To determine if the shell failed to find the requested application, it is +necessary to check the return code or output from the subprocess. A :exc:`ValueError` will be raised if :class:`Popen` is called with invalid arguments. @@ -680,6 +683,7 @@ Exceptions defined in this module all inherit from :exc:`SubprocessError`. .. versionadded:: 3.3 The :exc:`SubprocessError` base class was added. +.. _subprocess-security: Security Considerations ----------------------- diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index c9306e9b..641abf43 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -138,7 +138,7 @@ identifier. Python currently uses eight paths: If *expand* is set to ``False``, the path will not be expanded using the variables. - If *name* is not found, return ``None``. + If *name* is not found, raise a :exc:`KeyError`. .. function:: get_paths([scheme, [vars, [expand]]]) diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 13088a10..6afb8397 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -102,6 +102,9 @@ Some facts and figures: ``'x:bz2'``, :func:`tarfile.open` accepts the keyword argument *compresslevel* (default ``9``) to specify the compression level of the file. + For modes ``'w:xz'`` and ``'x:xz'``, :func:`tarfile.open` accepts the + keyword argument *preset* to specify the compression level of the file. + For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index f9421da5..915489fa 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -315,6 +315,7 @@ Here are some examples of typical usage of the :mod:`tempfile` module:: >>> # directory and contents have been removed +.. _tempfile-mktemp-deprecated: Deprecated functions and variables ---------------------------------- diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index 16837104..7780e241 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -17,20 +17,29 @@ If you're just wrapping or filling one or two text strings, the convenience functions should be good enough; otherwise, you should use an instance of :class:`TextWrapper` for efficiency. -.. function:: wrap(text, width=70, **kwargs) +.. function:: wrap(text, width=70, *, initial_indent="", \ + subsequent_indent="", expand_tabs=True, \ + replace_whitespace=True, fix_sentence_endings=False, \ + break_long_words=True, drop_whitespace=True, \ + break_on_hyphens=True, tabsize=8, max_lines=None) Wraps the single paragraph in *text* (a string) so every line is at most *width* characters long. Returns a list of output lines, without final newlines. Optional keyword arguments correspond to the instance attributes of - :class:`TextWrapper`, documented below. *width* defaults to ``70``. + :class:`TextWrapper`, documented below. See the :meth:`TextWrapper.wrap` method for additional details on how :func:`wrap` behaves. -.. function:: fill(text, width=70, **kwargs) +.. function:: fill(text, width=70, *, initial_indent="", \ + subsequent_indent="", expand_tabs=True, \ + replace_whitespace=True, fix_sentence_endings=False, \ + break_long_words=True, drop_whitespace=True, \ + break_on_hyphens=True, tabsize=8, \ + max_lines=None) Wraps the single paragraph in *text*, and returns a single string containing the wrapped paragraph. :func:`fill` is shorthand for :: @@ -41,7 +50,9 @@ functions should be good enough; otherwise, you should use an instance of :func:`wrap`. -.. function:: shorten(text, width, **kwargs) +.. function:: shorten(text, width, *, fix_sentence_endings=False, \ + break_long_words=True, break_on_hyphens=True, \ + placeholder=' [...]') Collapse and truncate the given *text* to fit in the given *width*. @@ -65,7 +76,6 @@ functions should be good enough; otherwise, you should use an instance of .. versionadded:: 3.4 - .. function:: dedent(text) Remove any common leading whitespace from every line in *text*. diff --git a/Doc/library/tk.rst b/Doc/library/tk.rst index c6c73f05..0cb8fda4 100644 --- a/Doc/library/tk.rst +++ b/Doc/library/tk.rst @@ -19,16 +19,15 @@ The :mod:`tkinter` package is a thin object-oriented layer on top of Tcl/Tk. To use :mod:`tkinter`, you don't need to write Tcl code, but you will need to consult the Tk documentation, and occasionally the Tcl documentation. :mod:`tkinter` is a set of wrappers that implement the Tk widgets as Python -classes. In addition, the internal module :mod:`_tkinter` provides a threadsafe -mechanism which allows Python and Tcl to interact. +classes. :mod:`tkinter`'s chief virtues are that it is fast, and that it usually comes bundled with Python. Although its standard documentation is weak, good material is available, which includes: references, tutorials, a book and others. :mod:`tkinter` is also famous for having an outdated look and feel, which has been vastly improved in Tk 8.5. Nevertheless, there are many other -GUI libraries that you could be interested in. For more information about -alternatives, see the :ref:`other-gui-packages` section. +GUI libraries that you could be interested in. The Python wiki lists several +alternative `GUI frameworks and tools `_. .. toctree:: @@ -42,7 +41,6 @@ alternatives, see the :ref:`other-gui-packages` section. tkinter.ttk.rst tkinter.tix.rst idle.rst - othergui.rst .. Other sections I have in mind are Tkinter internals diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 7739f2f6..e1b2a87b 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -11,9 +11,8 @@ -------------- The :mod:`tkinter` package ("Tk interface") is the standard Python interface to -the Tk GUI toolkit. Both Tk and :mod:`tkinter` are available on most Unix -platforms, as well as on Windows systems. (Tk itself is not part of Python; it -is maintained at ActiveState.) +the Tcl/Tk GUI toolkit. Both Tk and :mod:`tkinter` are available on most Unix +platforms, including macOS, as well as on Windows systems. Running ``python -m tkinter`` from the command line should open a window demonstrating a simple Tk interface, letting you know that :mod:`tkinter` is @@ -22,69 +21,46 @@ installed, so you can read the Tcl/Tk documentation specific to that version. .. seealso:: - Tkinter documentation: + * `TkDocs `_ + Extensive tutorial on creating user interfaces with Tkinter. Explains key concepts, + and illustrates recommended approaches using the modern API. - `Python Tkinter Resources `_ - The Python Tkinter Topic Guide provides a great deal of information on using Tk - from Python and links to other sources of information on Tk. + * `Tkinter 8.5 reference: a GUI for Python `_ + Reference documentation for Tkinter 8.5 detailing available classes, methods, and options. - `TKDocs `_ - Extensive tutorial plus friendlier widget pages for some of the widgets. + Tcl/Tk Resources: - `Tkinter 8.5 reference: a GUI for Python `_ - On-line reference material. + * `Tk commands `_ + Comprehensive reference to each of the underlying Tcl/Tk commands used by Tkinter. - `Tkinter docs from effbot `_ - Online reference for tkinter supported by effbot.org. + * `Tcl/Tk Home Page `_ + Additional documentation, and links to Tcl/Tk core development. - `Programming Python `_ - Book by Mark Lutz, has excellent coverage of Tkinter. + Books: - `Modern Tkinter for Busy Python Developers `_ - Book by Mark Roseman about building attractive and modern graphical user interfaces with Python and Tkinter. + * `Modern Tkinter for Busy Python Developers `_ + By Mark Roseman. (ISBN 978-1999149567) - `Python and Tkinter Programming `_ - Book by John Grayson (ISBN 1-884777-81-3). + * `Python and Tkinter Programming `_ + By Alan Moore. (ISBN 978-1788835886) - Tcl/Tk documentation: + * `Programming Python `_ + By Mark Lutz; has excellent coverage of Tkinter. (ISBN 978-0596158101) - `Tk commands `_ - Most commands are available as :mod:`tkinter` or :mod:`tkinter.ttk` classes. - Change '8.6' to match the version of your Tcl/Tk installation. - - `Tcl/Tk recent man pages `_ - Recent Tcl/Tk manuals on www.tcl.tk. - - `ActiveState Tcl Home Page `_ - The Tk/Tcl development is largely taking place at ActiveState. - - `Tcl and the Tk Toolkit `_ - Book by John Ousterhout, the inventor of Tcl. - - `Practical Programming in Tcl and Tk `_ - Brent Welch's encyclopedic book. + * `Tcl and the Tk Toolkit (2nd edition) `_ + By John Ousterhout, inventor of Tcl/Tk, and Ken Jones; does not cover Tkinter. (ISBN 978-0321336330) Tkinter Modules --------------- -Most of the time, :mod:`tkinter` is all you really need, but a number of -additional modules are available as well. The Tk interface is located in a -binary module named :mod:`_tkinter`. This module contains the low-level -interface to Tk, and should never be used directly by application programmers. -It is usually a shared library (or DLL), but might in some cases be statically -linked with the Python interpreter. - -In addition to the Tk interface module, :mod:`tkinter` includes a number of -Python modules, :mod:`tkinter.constants` being one of the most important. -Importing :mod:`tkinter` will automatically import :mod:`tkinter.constants`, -so, usually, to use Tkinter all you need is a simple import statement:: - - import tkinter +Support for Tkinter is spread across several modules. Most applications will need the +main :mod:`tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides +the modern themed widget set and API:: -Or, more often:: from tkinter import * + from tkinter import ttk .. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=1) @@ -107,7 +83,10 @@ Or, more often:: subsystem initialized) by calling its :meth:`loadtk` method. -Other modules that provide Tk support include: +The modules that provide Tk support include: + +:mod:`tkinter` + Main Tkinter module. :mod:`tkinter.colorchooser` Dialog to let the user choose a color. @@ -130,9 +109,35 @@ Other modules that provide Tk support include: :mod:`tkinter.simpledialog` Basic dialogs and convenience functions. +:mod:`tkinter.ttk` + Themed widget set introduced in Tk 8.5, providing modern alternatives + for many of the classic widgets in the main :mod:`tkinter` module. + +Additional modules: + +:mod:`_tkinter` + A binary module that contains the low-level interface to Tcl/Tk. + It is automatically imported by the main :mod:`tkinter` module, + and should never be used directly by application programmers. + It is usually a shared library (or DLL), but might in some cases be + statically linked with the Python interpreter. + +:mod:`idlelib` + Python's Integrated Development and Learning Environment (IDLE). Based + on :mod:`tkinter`. + +:mod:`tkinter.constants` + Symbolic constants that can be used in place of strings when passing + various parameters to Tkinter calls. Automatically imported by the + main :mod:`tkinter` module. + :mod:`tkinter.dnd` - Drag-and-drop support for :mod:`tkinter`. This is experimental and should - become deprecated when it is replaced with the Tk DND. + (experimental) Drag-and-drop support for :mod:`tkinter`. This will + become deprecated when it is replaced with the Tk DND. + +:mod:`tkinter.tix` + (deprecated) An older third-party Tcl/Tk package that adds several new + widgets. Better alternatives for most can be found in :mod:`tkinter.ttk`. :mod:`turtle` Turtle graphics in a Tk window. diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 6a9d6191..228cf1e0 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -193,8 +193,8 @@ Methods of TurtleScreen/Screen Window control | :func:`bgcolor` | :func:`bgpic` - | :func:`clear` | :func:`clearscreen` - | :func:`reset` | :func:`resetscreen` + | :func:`clearscreen` + | :func:`resetscreen` | :func:`screensize` | :func:`setworldcoordinates` @@ -1069,7 +1069,6 @@ More drawing control ~~~~~~~~~~~~~~~~~~~~ .. function:: reset() - :noindex: Delete the turtle's drawings from the screen, re-center the turtle and set variables to the default values. @@ -1091,7 +1090,6 @@ More drawing control .. function:: clear() - :noindex: Delete the turtle's drawings from the screen. Do not move turtle. State and position of the turtle as well as drawings of other turtles are not affected. @@ -1627,11 +1625,7 @@ Window control .. function:: clear() - clearscreen() - - Delete all drawings and all turtles from the TurtleScreen. Reset the now - empty TurtleScreen to its initial state: white background, no background - image, no event bindings and tracing on. + :noindex: .. note:: This TurtleScreen method is available as a global function only under the @@ -1639,10 +1633,15 @@ Window control derived from the Turtle method ``clear``. -.. function:: reset() - resetscreen() +.. function:: clearscreen() - Reset all Turtles on the Screen to their initial state. + Delete all drawings and all turtles from the TurtleScreen. Reset the now + empty TurtleScreen to its initial state: white background, no background + image, no event bindings and tracing on. + + +.. function:: reset() + :noindex: .. note:: This TurtleScreen method is available as a global function only under the @@ -1650,6 +1649,11 @@ Window control derived from the Turtle method ``reset``. +.. function:: resetscreen() + + Reset all Turtles on the Screen to their initial state. + + .. function:: screensize(canvwidth=None, canvheight=None, bg=None) :param canvwidth: positive integer, new width of canvas in pixels diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index b3e71705..3c74b617 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -2207,7 +2207,7 @@ In this example we monkey patch ``method`` to return ``sentinel.some_object``: >>> real.method.return_value = sentinel.some_object >>> result = real.method() >>> assert result is sentinel.some_object - >>> sentinel.some_object + >>> result sentinel.some_object diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index b7bfb655..5980ff5b 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -39,7 +39,7 @@ parameters: ``-n`` opens the URL in a new browser window, if possible; ``-t`` opens the URL in a new browser page ("tab"). The options are, naturally, mutually exclusive. Usage example:: - python -m webbrowser -t "http://www.python.org" + python -m webbrowser -t "https://www.python.org" The following exception is defined: @@ -176,7 +176,7 @@ Notes: Here are some simple examples:: - url = 'http://docs.python.org/' + url = 'https://docs.python.org/' # Open URL in a new tab, if a browser window is already open. webbrowser.open_new_tab(url) diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index bf72c465..d3a5f872 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -145,7 +145,7 @@ module documentation. This section lists the differences between the API and For the :class:`Document` node, an additional keyword argument *encoding* can be used to specify the encoding field of the XML header. - Silimarly, explicitly stating the *standalone* argument causes the + Similarly, explicitly stating the *standalone* argument causes the standalone document declarations to be added to the prologue of the XML document. If the value is set to `True`, `standalone="yes"` is added, diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index 1981cab7..e3b35162 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -60,22 +60,26 @@ circumvent firewalls. The following table gives an overview of the known attacks and whether the various modules are vulnerable to them. -========================= ============== =============== ============== ============== ============== -kind sax etree minidom pulldom xmlrpc -========================= ============== =============== ============== ============== ============== -billion laughs **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -quadratic blowup **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -external entity expansion Safe (4) Safe (1) Safe (2) Safe (4) Safe (3) -`DTD`_ retrieval Safe (4) Safe Safe Safe (4) Safe -decompression bomb Safe Safe Safe Safe **Vulnerable** -========================= ============== =============== ============== ============== ============== - -1. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a +========================= ================== ================== ================== ================== ================== +kind sax etree minidom pulldom xmlrpc +========================= ================== ================== ================== ================== ================== +billion laughs **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) +quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) +external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) +`DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe +decompression bomb Safe Safe Safe Safe **Vulnerable** +========================= ================== ================== ================== ================== ================== + +1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and + "quadratic blowup" vulnerabilities. Items still listed as vulnerable due to + potential reliance on system-provided libraries. Check + :data:`pyexpat.EXPAT_VERSION`. +2. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a :exc:`ParserError` when an entity occurs. -2. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns +3. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns the unexpanded entity verbatim. -3. :mod:`xmlrpclib` doesn't expand external entities and omits them. -4. Since Python 3.7.1, external general entities are no longer processed by +4. :mod:`xmlrpclib` doesn't expand external entities and omits them. +5. Since Python 3.7.1, external general entities are no longer processed by default. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 7126d8bd..3d1e8d8c 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -78,7 +78,6 @@ The module defines the following items: of the last modification to the file; the fields are described in section :ref:`zipinfo-objects`. - .. function:: is_zipfile(filename) Returns ``True`` if *filename* is a valid ZIP file based on its magic number, @@ -406,6 +405,11 @@ ZipFile Objects If ``arcname`` (or ``filename``, if ``arcname`` is not given) contains a null byte, the name of the file in the archive will be truncated at the null byte. + .. note:: + + A leading slash in the filename may lead to the archive being impossible to + open in some zip programs on Windows systems. + .. versionchanged:: 3.6 Calling :meth:`write` on a ZipFile created with mode ``'r'`` or a closed ZipFile will raise a :exc:`ValueError`. Previously, @@ -844,6 +848,8 @@ Exceeding limitations on different file systems can cause decompression failed. Such as allowable characters in the directory entries, length of the file name, length of the pathname, size of a single file, and number of files, etc. +.. _zipfile-resources-limitations: + Resources limitations ~~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 51fa750d..5333de91 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1138,6 +1138,7 @@ Raising ``0.0`` to a negative power results in a :exc:`ZeroDivisionError`. Raising a negative number to a fractional power results in a :class:`complex` number. (In earlier versions it raised a :exc:`ValueError`.) +This operation can be customized using the special :meth:`__pow__` method. .. _unary: @@ -1159,14 +1160,16 @@ All unary arithmetic and bitwise operations have the same priority: single: operator; - (minus) single: - (minus); unary operator -The unary ``-`` (minus) operator yields the negation of its numeric argument. +The unary ``-`` (minus) operator yields the negation of its numeric argument; the +operation can be overridden with the :meth:`__neg__` special method. .. index:: single: plus single: operator; + (plus) single: + (plus); unary operator -The unary ``+`` (plus) operator yields its numeric argument unchanged. +The unary ``+`` (plus) operator yields its numeric argument unchanged; the +operation can be overridden with the :meth:`__pos__` special method. .. index:: single: inversion @@ -1174,7 +1177,10 @@ The unary ``+`` (plus) operator yields its numeric argument unchanged. The unary ``~`` (invert) operator yields the bitwise inversion of its integer argument. The bitwise inversion of ``x`` is defined as ``-(x+1)``. It only -applies to integral numbers. +applies to integral numbers or to custom objects that override the +:meth:`__invert__` special method. + + .. index:: exception: TypeError @@ -1210,6 +1216,9 @@ the other must be a sequence. In the former case, the numbers are converted to a common type and then multiplied together. In the latter case, sequence repetition is performed; a negative repetition factor yields an empty sequence. +This operation can be customized using the special :meth:`__mul__` and +:meth:`__rmul__` methods. + .. index:: single: matrix multiplication operator: @ (at) @@ -1232,6 +1241,9 @@ integer; the result is that of mathematical division with the 'floor' function applied to the result. Division by zero raises the :exc:`ZeroDivisionError` exception. +This operation can be customized using the special :meth:`__truediv__` and +:meth:`__floordiv__` methods. + .. index:: single: modulo operator: % (percent) @@ -1255,6 +1267,8 @@ also overloaded by string objects to perform old-style string formatting (also known as interpolation). The syntax for string formatting is described in the Python Library Reference, section :ref:`old-string-formatting`. +The *modulo* operation can be customized using the special :meth:`__mod__` method. + The floor division operator, the modulo operator, and the :func:`divmod` function are not defined for complex numbers. Instead, convert to a floating point number using the :func:`abs` function if appropriate. @@ -1269,6 +1283,9 @@ must either both be numbers or both be sequences of the same type. In the former case, the numbers are converted to a common type and then added together. In the latter case, the sequences are concatenated. +This operation can be customized using the special :meth:`__add__` and +:meth:`__radd__` methods. + .. index:: single: subtraction single: operator; - (minus) @@ -1277,6 +1294,8 @@ In the latter case, the sequences are concatenated. The ``-`` (subtraction) operator yields the difference of its arguments. The numeric arguments are first converted to a common type. +This operation can be customized using the special :meth:`__sub__` method. + .. _shifting: @@ -1296,6 +1315,9 @@ The shifting operations have lower priority than the arithmetic operations: These operators accept integers as arguments. They shift the first argument to the left or right by the number of bits given by the second argument. +This operation can be customized using the special :meth:`__lshift__` and +:meth:`__rshift__` methods. + .. index:: exception: ValueError A right shift by *n* bits is defined as floor division by ``pow(2,n)``. A left @@ -1321,7 +1343,8 @@ Each of the three bitwise operations has a different priority level: operator: & (ampersand) The ``&`` operator yields the bitwise AND of its arguments, which must be -integers. +integers or one of them must be a custom object overriding :meth:`__and__` or +:meth:`__rand__` special methods. .. index:: pair: bitwise; xor @@ -1329,7 +1352,8 @@ integers. operator: ^ (caret) The ``^`` operator yields the bitwise XOR (exclusive OR) of its arguments, which -must be integers. +must be integers or one of them must be a custom object overriding :meth:`__xor__` or +:meth:`__rxor__` special methods. .. index:: pair: bitwise; or @@ -1337,7 +1361,8 @@ must be integers. operator: | (vertical bar) The ``|`` operator yields the bitwise (inclusive) OR of its arguments, which -must be integers. +must be integers or one of them must be a custom object overriding :meth:`__or__` or +:meth:`__ror__` special methods. .. _comparisons: @@ -1365,7 +1390,9 @@ in mathematics: comp_operator: "<" | ">" | "==" | ">=" | "<=" | "!=" : | "is" ["not"] | ["not"] "in" -Comparisons yield boolean values: ``True`` or ``False``. +Comparisons yield boolean values: ``True`` or ``False``. Custom +:dfn:`rich comparison methods` may return non-boolean values. In this case +Python will call :func:`bool` on such value in boolean contexts. .. index:: pair: chaining; comparisons diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 152162ab..a96746b6 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -5,7 +5,7 @@ {% block body %}

{{ docstitle|e }}

- {% trans %}Welcome! This is the documentation for Python {{ release }}.{% endtrans %} + {% trans %}Welcome! This is the official documentation for Python {{ release }}.{% endtrans %}

{% trans %}Parts of the documentation:{% endtrans %}

diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 129ab21c..08202f03 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -479,7 +479,7 @@ Here's an example that fails due to this restriction:: >>> function(0, a=0) Traceback (most recent call last): File "", line 1, in - TypeError: function() got multiple values for keyword argument 'a' + TypeError: function() got multiple values for argument 'a' When a final formal parameter of the form ``**name`` is present, it receives a dictionary (see :ref:`typesmapping`) containing all keyword arguments except for @@ -615,7 +615,7 @@ parameters as there is a ``/`` in the function definition:: >>> pos_only_arg(arg=1) Traceback (most recent call last): File "", line 1, in - TypeError: pos_only_arg() got an unexpected keyword argument 'arg' + TypeError: pos_only_arg() got some positional-only arguments passed as keyword arguments: 'arg' The third function ``kwd_only_args`` only allows keyword arguments as indicated by a ``*`` in the function definition:: @@ -645,7 +645,7 @@ definition:: >>> combined_example(pos_only=1, standard=2, kwd_only=3) Traceback (most recent call last): File "", line 1, in - TypeError: combined_example() got an unexpected keyword argument 'pos_only' + TypeError: combined_example() got some positional-only arguments passed as keyword arguments: 'pos_only' Finally, consider this function definition which has a potential collision between the positional argument ``name`` and ``**kwds`` which has ``name`` as a key:: diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 516c925b..4e2ed1d1 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -53,7 +53,7 @@ programs, however, and result in error messages as shown here:: >>> '2' + 2 Traceback (most recent call last): File "", line 1, in - TypeError: Can't convert 'int' object to str implicitly + TypeError: can only concatenate str (not "int") to str The last line of the error message indicates what happened. Exceptions come in different types, and the type is printed as part of the message: the types in diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 4613cf76..8763626e 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -269,14 +269,6 @@ to obtain individual characters, *slicing* allows you to obtain substring:: >>> word[2:5] # characters from position 2 (included) to 5 (excluded) 'tho' -Note how the start is always included, and the end always excluded. This -makes sure that ``s[:i] + s[i:]`` is always equal to ``s``:: - - >>> word[:2] + word[2:] - 'Python' - >>> word[:4] + word[4:] - 'Python' - Slice indices have useful defaults; an omitted first index defaults to zero, an omitted second index defaults to the size of the string being sliced. :: @@ -287,6 +279,14 @@ omitted second index defaults to the size of the string being sliced. :: >>> word[-2:] # characters from the second-last (included) to the end 'on' +Note how the start is always included, and the end always excluded. This +makes sure that ``s[:i] + s[i:]`` is always equal to ``s``:: + + >>> word[:2] + word[2:] + 'Python' + >>> word[:4] + word[4:] + 'Python' + One way to remember how slices work is to think of the indices as pointing *between* characters, with the left edge of the first character numbered 0. Then the right edge of the last character of a string of *n* characters has diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index dbc3875a..1b3bb152 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -2247,3 +2247,16 @@ separator key, with ``&`` as the default. This change also affects functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +Notable changes in Python 3.8.12 +================================ + +Starting with Python 3.8.12 the :mod:`ipaddress` module no longer accepts +any leading zeros in IPv4 address strings. Leading zeros are ambiguous and +interpreted as octal notation by some libraries. For example the legacy +function :func:`socket.inet_aton` treats leading zeros as octal notation. +glibc implementation of modern :func:`~socket.inet_pton` does not accept +any leading zeros. + +(Originally contributed by Christian Heimes in :issue:`36384`, and backported +to 3.8 by Achraf Merzouki.) diff --git a/Grammar/python.gram b/Grammar/python.gram index 544b4f79..38405b72 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -692,5 +692,5 @@ invalid_group: | '(' a=starred_expression ')' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "can't use starred expression here") } invalid_import_from_targets: - | import_from_as_names ',' { + | import_from_as_names ',' NEWLINE { RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") } diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 3d925e22..9bcb5f5e 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -42,7 +42,7 @@ PyAPI_FUNC(int) _PyMem_SetDefaultAllocator( fills newly allocated memory with CLEANBYTE (0xCD) and newly freed memory with DEADBYTE (0xDD). Detect also "untouchable bytes" marked with FORBIDDENBYTE (0xFD). */ -static inline int _PyMem_IsPtrFreed(void *ptr) +static inline int _PyMem_IsPtrFreed(const void *ptr) { uintptr_t value = (uintptr_t)ptr; #if SIZEOF_VOID_P == 8 diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 19e2aaea..af883f23 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 9 -#define PY_MICRO_VERSION 6 +#define PY_MICRO_VERSION 7 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.9.6" +#define PY_VERSION "3.9.7" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index b6ecf8ea..acfaff80 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -642,6 +642,7 @@ class Set(Collection): hx = hash(x) h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 h &= MASK + h ^= (h >> 11) ^ (h >> 25) h = h * 69069 + 907133923 h &= MASK if h > MAX: diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index b267780f..2a276843 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -51,10 +51,14 @@ class WeakSet: self.update(data) def _commit_removals(self): - l = self._pending_removals + pop = self._pending_removals.pop discard = self.data.discard - while l: - discard(l.pop()) + while True: + try: + item = pop() + except IndexError: + return + discard(item) def __iter__(self): with _IterationGuard(self): diff --git a/Lib/argparse.py b/Lib/argparse.py index 2fb1da59..f45a2ed4 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -727,6 +727,8 @@ def _get_action_name(argument): return argument.metavar elif argument.dest not in (None, SUPPRESS): return argument.dest + elif argument.choices: + return '{' + ','.join(argument.choices) + '}' else: return None @@ -873,7 +875,7 @@ class BooleanOptionalAction(Action): _option_strings.append(option_string) if help is not None and default is not None: - help += f" (default: {default})" + help += " (default: %(default)s)" super().__init__( option_strings=_option_strings, diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index d6262ae7..d7e0575e 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -635,16 +635,17 @@ def __sleep0(): async def sleep(delay, result=None, *, loop=None): """Coroutine that completes after a given time (in seconds).""" + if loop is not None: + warnings.warn("The loop argument is deprecated since Python 3.8, " + "and scheduled for removal in Python 3.10.", + DeprecationWarning, stacklevel=2) + if delay <= 0: await __sleep0() return result if loop is None: loop = events.get_running_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) future = loop.create_future() h = loop.call_later(delay, @@ -750,13 +751,14 @@ def gather(*coros_or_futures, loop=None, return_exceptions=False): after catching an exception (raised by one of the awaitables) from gather won't cancel any other awaitables. """ + if loop is not None: + warnings.warn("The loop argument is deprecated since Python 3.8, " + "and scheduled for removal in Python 3.10.", + DeprecationWarning, stacklevel=2) + if not coros_or_futures: if loop is None: loop = events.get_event_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) outer = loop.create_future() outer.set_result([]) return outer diff --git a/Lib/asyncio/threads.py b/Lib/asyncio/threads.py index 34b7513a..db048a82 100644 --- a/Lib/asyncio/threads.py +++ b/Lib/asyncio/threads.py @@ -13,7 +13,7 @@ async def to_thread(func, /, *args, **kwargs): """Asynchronously run function *func* in a separate thread. Any *args and **kwargs supplied for this function are directly passed - to *func*. Also, the current :class:`contextvars.Context` is propogated, + to *func*. Also, the current :class:`contextvars.Context` is propagated, allowing context variables from the main thread to be accessed in the separate thread. diff --git a/Lib/compileall.py b/Lib/compileall.py index fe7f450c..39f4bb39 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -219,8 +219,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not force: try: mtime = int(os.stat(fullname).st_mtime) - expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER, - 0, mtime) + expect = struct.pack('<4sLL', importlib.util.MAGIC_NUMBER, + 0, mtime & 0xFFFF_FFFF) for cfile in opt_cfiles.values(): with open(cfile, 'rb') as chandle: actual = chandle.read(12) @@ -252,9 +252,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, else: print('*** ', end='') # escape non-printable characters in msg - msg = err.msg.encode(sys.stdout.encoding, - errors='backslashreplace') - msg = msg.decode(sys.stdout.encoding) + encoding = sys.stdout.encoding or sys.getdefaultencoding() + msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding) print(msg) except (SyntaxError, UnicodeError, OSError) as e: success = False diff --git a/Lib/contextlib.py b/Lib/contextlib.py index ff92d9f9..f60f0c27 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -97,18 +97,20 @@ class _GeneratorContextManagerBase: # for the class instead. # See http://bugs.python.org/issue19404 for more details. - -class _GeneratorContextManager(_GeneratorContextManagerBase, - AbstractContextManager, - ContextDecorator): - """Helper for @contextmanager decorator.""" - def _recreate_cm(self): - # _GCM instances are one-shot context managers, so the + # _GCMB instances are one-shot context managers, so the # CM must be recreated each time a decorated function is # called return self.__class__(self.func, self.args, self.kwds) + +class _GeneratorContextManager( + _GeneratorContextManagerBase, + AbstractContextManager, + ContextDecorator, +): + """Helper for @contextmanager decorator.""" + def __enter__(self): # do not keep args and kwds alive unnecessarily # they are only needed for recreation, which is not possible anymore @@ -118,8 +120,8 @@ class _GeneratorContextManager(_GeneratorContextManagerBase, except StopIteration: raise RuntimeError("generator didn't yield") from None - def __exit__(self, type, value, traceback): - if type is None: + def __exit__(self, typ, value, traceback): + if typ is None: try: next(self.gen) except StopIteration: @@ -130,9 +132,9 @@ class _GeneratorContextManager(_GeneratorContextManagerBase, if value is None: # Need to force instantiation so we can reliably # tell if we get the same exception back - value = type() + value = typ() try: - self.gen.throw(type, value, traceback) + self.gen.throw(typ, value, traceback) except StopIteration as exc: # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration @@ -142,35 +144,39 @@ class _GeneratorContextManager(_GeneratorContextManagerBase, # Don't re-raise the passed in exception. (issue27122) if exc is value: return False - # Likewise, avoid suppressing if a StopIteration exception + # Avoid suppressing if a StopIteration exception # was passed to throw() and later wrapped into a RuntimeError - # (see PEP 479). - if type is StopIteration and exc.__cause__ is value: + # (see PEP 479 for sync generators; async generators also + # have this behavior). But do this only if the exception wrapped + # by the RuntimeError is actually Stop(Async)Iteration (see + # issue29692). + if ( + isinstance(value, StopIteration) + and exc.__cause__ is value + ): return False raise - except: + except BaseException as exc: # only re-raise if it's *not* the exception that was # passed to throw(), because __exit__() must not raise # an exception unless __exit__() itself failed. But throw() # has to raise the exception to signal propagation, so this # fixes the impedance mismatch between the throw() protocol # and the __exit__() protocol. - # - # This cannot use 'except BaseException as exc' (as in the - # async implementation) to maintain compatibility with - # Python 2, where old-style class exceptions are not caught - # by 'except BaseException'. - if sys.exc_info()[1] is value: - return False - raise + if exc is not value: + raise + return False raise RuntimeError("generator didn't stop after throw()") class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, AbstractAsyncContextManager): - """Helper for @asynccontextmanager.""" + """Helper for @asynccontextmanager decorator.""" async def __aenter__(self): + # do not keep args and kwds alive unnecessarily + # they are only needed for recreation, which is not possible anymore + del self.args, self.kwds, self.func try: return await self.gen.__anext__() except StopAsyncIteration: @@ -181,35 +187,48 @@ class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, try: await self.gen.__anext__() except StopAsyncIteration: - return + return False else: raise RuntimeError("generator didn't stop") else: if value is None: + # Need to force instantiation so we can reliably + # tell if we get the same exception back value = typ() - # See _GeneratorContextManager.__exit__ for comments on subtleties - # in this implementation try: await self.gen.athrow(typ, value, traceback) - raise RuntimeError("generator didn't stop after athrow()") except StopAsyncIteration as exc: + # Suppress StopIteration *unless* it's the same exception that + # was passed to throw(). This prevents a StopIteration + # raised inside the "with" statement from being suppressed. return exc is not value except RuntimeError as exc: + # Don't re-raise the passed in exception. (issue27122) if exc is value: return False - # Avoid suppressing if a StopIteration exception - # was passed to throw() and later wrapped into a RuntimeError + # Avoid suppressing if a Stop(Async)Iteration exception + # was passed to athrow() and later wrapped into a RuntimeError # (see PEP 479 for sync generators; async generators also # have this behavior). But do this only if the exception wrapped # by the RuntimeError is actully Stop(Async)Iteration (see # issue29692). - if isinstance(value, (StopIteration, StopAsyncIteration)): - if exc.__cause__ is value: - return False + if ( + isinstance(value, (StopIteration, StopAsyncIteration)) + and exc.__cause__ is value + ): + return False raise except BaseException as exc: + # only re-raise if it's *not* the exception that was + # passed to throw(), because __exit__() must not raise + # an exception unless __exit__() itself failed. But throw() + # has to raise the exception to signal propagation, so this + # fixes the impedance mismatch between the throw() protocol + # and the __exit__() protocol. if exc is not value: raise + return False + raise RuntimeError("generator didn't stop after athrow()") def contextmanager(func): diff --git a/Lib/ctypes/test/test_functions.py b/Lib/ctypes/test/test_functions.py index 75628924..d3c6536f 100644 --- a/Lib/ctypes/test/test_functions.py +++ b/Lib/ctypes/test/test_functions.py @@ -389,7 +389,7 @@ class FunctionTestCase(unittest.TestCase): (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) def test_sf1651235(self): - # see http://www.python.org/sf/1651235 + # see https://www.python.org/sf/1651235 proto = CFUNCTYPE(c_int, RECT, POINT) def callback(*args): diff --git a/Lib/ctypes/test/test_loading.py b/Lib/ctypes/test/test_loading.py index ba655bce..2f9eb9d3 100644 --- a/Lib/ctypes/test/test_loading.py +++ b/Lib/ctypes/test/test_loading.py @@ -91,7 +91,7 @@ class LoaderTest(unittest.TestCase): # NOT fit into a 32-bit integer. FreeLibrary must be able # to accept this address. - # These are tests for http://www.python.org/sf/1703286 + # These are tests for https://www.python.org/sf/1703286 handle = LoadLibrary("advapi32") FreeLibrary(handle) diff --git a/Lib/distutils/README b/Lib/distutils/README index 23f48850..73bd2518 100644 --- a/Lib/distutils/README +++ b/Lib/distutils/README @@ -2,10 +2,10 @@ This directory contains the Distutils package. There's a full documentation available at: - http://docs.python.org/distutils/ + https://docs.python.org/distutils/ The Distutils-SIG web page is also a good starting point: - http://www.python.org/sigs/distutils-sig/ + https://www.python.org/sigs/distutils-sig/ $Id$ diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 41ff6f8c..178329fb 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -67,6 +67,8 @@ def _parsedate_tz(data): if not data: return data = data.split() + if not data: # This happens for whitespace-only input. + return None # The FWS after the comma after the day-of-week is optional, so search and # adjust for this. if data[0].endswith(',') or data[0].lower() in _daynames: diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py index b91fb0e5..3cf62dc8 100644 --- a/Lib/email/contentmanager.py +++ b/Lib/email/contentmanager.py @@ -238,9 +238,7 @@ def set_bytes_content(msg, data, maintype, subtype, cte='base64', data = binascii.b2a_qp(data, istext=False, header=False, quotetabs=True) data = data.decode('ascii') elif cte == '7bit': - # Make sure it really is only ASCII. The early warning here seems - # worth the overhead...if you care write your own content manager :). - data.encode('ascii') + data = data.decode('ascii') elif cte in ('8bit', 'binary'): data = data.decode('ascii', 'surrogateescape') msg.set_payload(data) diff --git a/Lib/email/message.py b/Lib/email/message.py index db30d9a1..6752ce0f 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -982,7 +982,7 @@ class MIMEPart(Message): if subtype in preferencelist: yield (preferencelist.index(subtype), part) return - if maintype != 'multipart': + if maintype != 'multipart' or not self.is_multipart(): return if subtype != 'related': for subpart in part.iter_parts(): @@ -1087,7 +1087,7 @@ class MIMEPart(Message): Return an empty iterator for a non-multipart. """ - if self.get_content_maintype() == 'multipart': + if self.is_multipart(): yield from self.get_payload() def get_content(self, *args, content_manager=None, **kw): diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 1a7719db..48d30160 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -109,7 +109,7 @@ def formataddr(pair, charset='utf-8'): def getaddresses(fieldvalues): """Return a list of (REALNAME, EMAIL) for each fieldvalue.""" - all = COMMASPACE.join(fieldvalues) + all = COMMASPACE.join(str(v) for v in fieldvalues) a = _AddressList(all) return a.addresslist diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 14a39037..7b03970c 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -13,9 +13,9 @@ from . import _bundled __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "56.0.0" +_SETUPTOOLS_VERSION = "57.4.0" -_PIP_VERSION = "21.1.3" +_PIP_VERSION = "21.2.3" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION, "py3"), diff --git a/Lib/ensurepip/_bundled/pip-21.1.3-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-21.1.3-py3-none-any.whl deleted file mode 100644 index d96a40a9..00000000 Binary files a/Lib/ensurepip/_bundled/pip-21.1.3-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/pip-21.2.3-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-21.2.3-py3-none-any.whl new file mode 100644 index 00000000..d417df63 Binary files /dev/null and b/Lib/ensurepip/_bundled/pip-21.2.3-py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-56.0.0-py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-56.0.0-py3-none-any.whl deleted file mode 100644 index 264ef10e..00000000 Binary files a/Lib/ensurepip/_bundled/setuptools-56.0.0-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/setuptools-57.4.0-py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-57.4.0-py3-none-any.whl new file mode 100644 index 00000000..af8f8ba2 Binary files /dev/null and b/Lib/ensurepip/_bundled/setuptools-57.4.0-py3-none-any.whl differ diff --git a/Lib/filecmp.py b/Lib/filecmp.py index 7a4da6be..1893f909 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -36,8 +36,9 @@ def cmp(f1, f2, shallow=True): f2 -- Second file name - shallow -- Just check stat signature (do not read the files). - defaults to True. + shallow -- treat files as identical if their stat signatures (type, size, + mtime) are identical. Otherwise, files are considered different + if their sizes or contents differ. [default: True] Return value: diff --git a/Lib/functools.py b/Lib/functools.py index 5cab497d..97744a86 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -88,84 +88,84 @@ def wraps(wrapped, def _gt_from_lt(self, other, NotImplemented=NotImplemented): 'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).' - op_result = self.__lt__(other) + op_result = type(self).__lt__(self, other) if op_result is NotImplemented: return op_result return not op_result and self != other def _le_from_lt(self, other, NotImplemented=NotImplemented): 'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).' - op_result = self.__lt__(other) + op_result = type(self).__lt__(self, other) if op_result is NotImplemented: return op_result return op_result or self == other def _ge_from_lt(self, other, NotImplemented=NotImplemented): 'Return a >= b. Computed by @total_ordering from (not a < b).' - op_result = self.__lt__(other) + op_result = type(self).__lt__(self, other) if op_result is NotImplemented: return op_result return not op_result def _ge_from_le(self, other, NotImplemented=NotImplemented): 'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).' - op_result = self.__le__(other) + op_result = type(self).__le__(self, other) if op_result is NotImplemented: return op_result return not op_result or self == other def _lt_from_le(self, other, NotImplemented=NotImplemented): 'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).' - op_result = self.__le__(other) + op_result = type(self).__le__(self, other) if op_result is NotImplemented: return op_result return op_result and self != other def _gt_from_le(self, other, NotImplemented=NotImplemented): 'Return a > b. Computed by @total_ordering from (not a <= b).' - op_result = self.__le__(other) + op_result = type(self).__le__(self, other) if op_result is NotImplemented: return op_result return not op_result def _lt_from_gt(self, other, NotImplemented=NotImplemented): 'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).' - op_result = self.__gt__(other) + op_result = type(self).__gt__(self, other) if op_result is NotImplemented: return op_result return not op_result and self != other def _ge_from_gt(self, other, NotImplemented=NotImplemented): 'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).' - op_result = self.__gt__(other) + op_result = type(self).__gt__(self, other) if op_result is NotImplemented: return op_result return op_result or self == other def _le_from_gt(self, other, NotImplemented=NotImplemented): 'Return a <= b. Computed by @total_ordering from (not a > b).' - op_result = self.__gt__(other) + op_result = type(self).__gt__(self, other) if op_result is NotImplemented: return op_result return not op_result def _le_from_ge(self, other, NotImplemented=NotImplemented): 'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).' - op_result = self.__ge__(other) + op_result = type(self).__ge__(self, other) if op_result is NotImplemented: return op_result return not op_result or self == other def _gt_from_ge(self, other, NotImplemented=NotImplemented): 'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).' - op_result = self.__ge__(other) + op_result = type(self).__ge__(self, other) if op_result is NotImplemented: return op_result return op_result and self != other def _lt_from_ge(self, other, NotImplemented=NotImplemented): 'Return a < b. Computed by @total_ordering from (not a >= b).' - op_result = self.__ge__(other) + op_result = type(self).__ge__(self, other) if op_result is NotImplemented: return op_result return not op_result @@ -491,7 +491,7 @@ def lru_cache(maxsize=128, typed=False): with f.cache_info(). Clear the cache and statistics with f.cache_clear(). Access the underlying function with f.__wrapped__. - See: http://en.wikipedia.org/wiki/Cache_replacement_policies#Least_recently_used_(LRU) + See: https://en.wikipedia.org/wiki/Cache_replacement_policies#Least_recently_used_(LRU) """ @@ -659,7 +659,7 @@ def cache(user_function, /): def _c3_merge(sequences): """Merges MROs in *sequences* to a single MRO using the C3 algorithm. - Adapted from http://www.python.org/download/releases/2.3/mro/. + Adapted from https://www.python.org/download/releases/2.3/mro/. """ result = [] diff --git a/Lib/http/client.py b/Lib/http/client.py index 97529250..0fd9021b 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -105,6 +105,9 @@ globals().update(http.HTTPStatus.__members__) # Mapping status codes to official W3C names responses = {v: v.phrase for v in http.HTTPStatus.__members__.values()} +# maximal amount of data to read at one time in _safe_read +MAXAMOUNT = 1048576 + # maximal line length when calling readline(). _MAXLINE = 65536 _MAXHEADERS = 100 @@ -604,24 +607,43 @@ class HTTPResponse(io.BufferedIOBase): raise IncompleteRead(bytes(b[0:total_bytes])) def _safe_read(self, amt): - """Read the number of bytes requested. + """Read the number of bytes requested, compensating for partial reads. + + Normally, we have a blocking socket, but a read() can be interrupted + by a signal (resulting in a partial read). + + Note that we cannot distinguish between EOF and an interrupt when zero + bytes have been read. IncompleteRead() will be raised in this + situation. This function should be used when bytes "should" be present for reading. If the bytes are truly not available (due to EOF), then the IncompleteRead exception can be used to detect the problem. """ - data = self.fp.read(amt) - if len(data) < amt: - raise IncompleteRead(data, amt-len(data)) - return data + s = [] + while amt > 0: + chunk = self.fp.read(min(amt, MAXAMOUNT)) + if not chunk: + raise IncompleteRead(b''.join(s), amt) + s.append(chunk) + amt -= len(chunk) + return b"".join(s) def _safe_readinto(self, b): """Same as _safe_read, but for reading into a buffer.""" - amt = len(b) - n = self.fp.readinto(b) - if n < amt: - raise IncompleteRead(bytes(b[:n]), amt-n) - return n + total_bytes = 0 + mvb = memoryview(b) + while total_bytes < len(b): + if MAXAMOUNT < len(mvb): + temp_mvb = mvb[0:MAXAMOUNT] + n = self.fp.readinto(temp_mvb) + else: + n = self.fp.readinto(mvb) + if not n: + raise IncompleteRead(bytes(mvb[0:total_bytes]), len(b)) + mvb = mvb[n:] + total_bytes += n + return total_bytes def read1(self, n=-1): """Read with at most one underlying system call. If at least one diff --git a/Lib/idlelib/macosx.py b/Lib/idlelib/macosx.py index eeaab59a..470de5d8 100644 --- a/Lib/idlelib/macosx.py +++ b/Lib/idlelib/macosx.py @@ -83,7 +83,7 @@ def tkVersionWarning(root): return False return ("WARNING: The version of Tcl/Tk ({0}) in use may" " be unstable.\n" - "Visit http://www.python.org/download/mac/tcltk/" + "Visit https://www.python.org/download/mac/tcltk/" " for current information.".format(patchlevel)) else: return False diff --git a/Lib/inspect.py b/Lib/inspect.py index 18bed902..9d1d46dc 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2250,17 +2250,18 @@ def _signature_from_callable(obj, *, callable objects. """ + _get_signature_of = functools.partial(_signature_from_callable, + follow_wrapper_chains=follow_wrapper_chains, + skip_bound_arg=skip_bound_arg, + sigcls=sigcls) + if not callable(obj): raise TypeError('{!r} is not a callable object'.format(obj)) if isinstance(obj, types.MethodType): # In this case we skip the first parameter of the underlying # function (usually `self` or `cls`). - sig = _signature_from_callable( - obj.__func__, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + sig = _get_signature_of(obj.__func__) if skip_bound_arg: return _signature_bound_method(sig) @@ -2274,11 +2275,7 @@ def _signature_from_callable(obj, *, # If the unwrapped object is a *method*, we might want to # skip its first parameter (self). # See test_signature_wrapped_bound_method for details. - return _signature_from_callable( - obj, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + return _get_signature_of(obj) try: sig = obj.__signature__ @@ -2305,11 +2302,7 @@ def _signature_from_callable(obj, *, # (usually `self`, or `cls`) will not be passed # automatically (as for boundmethods) - wrapped_sig = _signature_from_callable( - partialmethod.func, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + wrapped_sig = _get_signature_of(partialmethod.func) sig = _signature_get_partial(wrapped_sig, partialmethod, (None,)) first_wrapped_param = tuple(wrapped_sig.parameters.values())[0] @@ -2335,11 +2328,7 @@ def _signature_from_callable(obj, *, skip_bound_arg=skip_bound_arg) if isinstance(obj, functools.partial): - wrapped_sig = _signature_from_callable( - obj.func, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + wrapped_sig = _get_signature_of(obj.func) return _signature_get_partial(wrapped_sig, obj) sig = None @@ -2350,29 +2339,25 @@ def _signature_from_callable(obj, *, # in its metaclass call = _signature_get_user_defined_method(type(obj), '__call__') if call is not None: - sig = _signature_from_callable( - call, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + sig = _get_signature_of(call) else: - # Now we check if the 'obj' class has a '__new__' method + factory_method = None new = _signature_get_user_defined_method(obj, '__new__') - if new is not None: - sig = _signature_from_callable( - new, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - else: - # Finally, we should have at least __init__ implemented - init = _signature_get_user_defined_method(obj, '__init__') - if init is not None: - sig = _signature_from_callable( - init, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + init = _signature_get_user_defined_method(obj, '__init__') + # Now we check if the 'obj' class has an own '__new__' method + if '__new__' in obj.__dict__: + factory_method = new + # or an own '__init__' method + elif '__init__' in obj.__dict__: + factory_method = init + # If not, we take inherited '__new__' or '__init__', if present + elif new is not None: + factory_method = new + elif init is not None: + factory_method = init + + if factory_method is not None: + sig = _get_signature_of(factory_method) if sig is None: # At this point we know, that `obj` is a class, with no user- @@ -2418,11 +2403,7 @@ def _signature_from_callable(obj, *, call = _signature_get_user_defined_method(type(obj), '__call__') if call is not None: try: - sig = _signature_from_callable( - call, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + sig = _get_signature_of(call) except ValueError as ex: msg = 'no signature found for {!r}'.format(obj) raise ValueError(msg) from ex diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 0e2685d4..099dfa77 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -512,13 +512,14 @@ def generate_tokens(readline): stashed = tok continue - if token == 'def': + if token in ('def', 'for'): if (stashed and stashed[0] == NAME and stashed[1] == 'async'): - async_def = True - async_def_indent = indents[-1] + if token == 'def': + async_def = True + async_def_indent = indents[-1] yield (ASYNC, stashed[1], stashed[2], stashed[3], diff --git a/Lib/lib2to3/tests/data/py2_test_grammar.py b/Lib/lib2to3/tests/data/py2_test_grammar.py index b5a4137d..86631617 100644 --- a/Lib/lib2to3/tests/data/py2_test_grammar.py +++ b/Lib/lib2to3/tests/data/py2_test_grammar.py @@ -735,7 +735,7 @@ hello world s = a[-5:] s = a[:-1] s = a[-4:-3] - # A rough test of SF bug 1333982. http://python.org/sf/1333982 + # A rough test of SF bug 1333982. https://python.org/sf/1333982 # The testing here is fairly incomplete. # Test cases should include: commas with 1 and 2 colons d = {} diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index d0622320..e1eee524 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -714,7 +714,7 @@ class GrammarTests(unittest.TestCase): s = a[-5:] s = a[:-1] s = a[-4:-3] - # A rough test of SF bug 1333982. http://python.org/sf/1333982 + # A rough test of SF bug 1333982. https://python.org/sf/1333982 # The testing here is fairly incomplete. # Test cases should include: commas with 1 and 2 colons d = {} diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index d5db66b9..034b5039 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -196,20 +196,27 @@ class TestAsyncAwait(GrammarTest): self.validate("""await = 1""") self.validate("""def async(): pass""") - def test_async_with(self): + def test_async_for(self): self.validate("""async def foo(): async for a in b: pass""") - self.invalid_syntax("""def foo(): - async for a in b: pass""") - - def test_async_for(self): + def test_async_with(self): self.validate("""async def foo(): async with a: pass""") self.invalid_syntax("""def foo(): async with a: pass""") + def test_async_generator(self): + self.validate( + """async def foo(): + return (i * 2 async for i in arange(42))""" + ) + self.validate( + """def foo(): + return (i * 2 async for i in arange(42))""" + ) + class TestRaiseChanges(GrammarTest): def test_2x_style_1(self): diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 867ef4eb..74b67602 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -356,7 +356,8 @@ class TimedRotatingFileHandler(BaseRotatingHandler): dirName, baseName = os.path.split(self.baseFilename) fileNames = os.listdir(dirName) result = [] - prefix = baseName + "." + # See bpo-44753: Don't use the extension when computing the prefix. + prefix = os.path.splitext(baseName)[0] + "." plen = len(prefix) for fileName in fileNames: if fileName[:plen] == prefix: diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 0eb16c66..dfa566c6 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -8,8 +8,7 @@ # Licensed to PSF under a Contributor Agreement. # -__all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token', - 'SharedMemoryManager' ] +__all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ] # # Imports @@ -35,9 +34,11 @@ from . import util from . import get_context try: from . import shared_memory - HAS_SHMEM = True except ImportError: HAS_SHMEM = False +else: + HAS_SHMEM = True + __all__.append('SharedMemoryManager') # # Register some things for pickling @@ -959,7 +960,7 @@ def MakeProxyType(name, exposed, _cache={}): def AutoProxy(token, serializer, manager=None, authkey=None, - exposed=None, incref=True): + exposed=None, incref=True, manager_owned=False): ''' Return an auto-proxy for `token` ''' @@ -979,7 +980,7 @@ def AutoProxy(token, serializer, manager=None, authkey=None, ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed) proxy = ProxyType(token, serializer, manager=manager, authkey=authkey, - incref=incref) + incref=incref, manager_owned=manager_owned) proxy._isauto = True return proxy diff --git a/Lib/operator.py b/Lib/operator.py index fb58851f..241fdbb6 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -155,10 +155,10 @@ def contains(a, b): return b in a def countOf(a, b): - "Return the number of times b occurs in a." + "Return the number of items in a which are, or which equal, b." count = 0 for i in a: - if i == b: + if i is b or i == b: count += 1 return count @@ -173,7 +173,7 @@ def getitem(a, b): def indexOf(a, b): "Return the first index of b in a." for i, j in enumerate(a): - if j == b: + if j is b or j == b: return i else: raise ValueError('sequence.index(x): x not in sequence') diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 60e2855b..7aeda14a 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -132,16 +132,25 @@ class _WindowsFlavour(_Flavour): ext_namespace_prefix = '\\\\?\\' reserved_names = ( - {'CON', 'PRN', 'AUX', 'NUL'} | - {'COM%d' % i for i in range(1, 10)} | - {'LPT%d' % i for i in range(1, 10)} + {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | + {'COM%s' % c for c in '123456789\xb9\xb2\xb3'} | + {'LPT%s' % c for c in '123456789\xb9\xb2\xb3'} ) # Interesting findings about extended paths: - # - '\\?\c:\a', '//?/c:\a' and '//?/c:/a' are all supported - # but '\\?\c:/a' is not - # - extended paths are always absolute; "relative" extended paths will - # fail. + # * '\\?\c:\a' is an extended path, which bypasses normal Windows API + # path processing. Thus relative paths are not resolved and slash is not + # translated to backslash. It has the native NT path limit of 32767 + # characters, but a bit less after resolving device symbolic links, + # such as '\??\C:' => '\Device\HarddiskVolume2'. + # * '\\?\c:/a' looks for a device named 'C:/a' because slash is a + # regular name character in the object namespace. + # * '\\?\c:\foo/bar' is invalid because '/' is illegal in NT filesystems. + # The only path separator at the filesystem level is backslash. + # * '//?/c:\a' and '//?/c:/a' are effectively equivalent to '\\.\c:\a' and + # thus limited to MAX_PATH. + # * Prior to Windows 8, ANSI API bytes paths are limited to MAX_PATH, + # even with the '\\?\' prefix. def splitroot(self, part, sep=sep): first = part[0:1] @@ -231,15 +240,16 @@ class _WindowsFlavour(_Flavour): def is_reserved(self, parts): # NOTE: the rules for reserved names seem somewhat complicated - # (e.g. r"..\NUL" is reserved but not r"foo\NUL"). - # We err on the side of caution and return True for paths which are - # not considered reserved by Windows. + # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not + # exist). We err on the side of caution and return True for paths + # which are not considered reserved by Windows. if not parts: return False if parts[0].startswith('\\\\'): # UNC paths are never reserved return False - return parts[-1].partition('.')[0].upper() in self.reserved_names + name = parts[-1].partition('.')[0].partition(':')[0].rstrip(' ') + return name.upper() in self.reserved_names def make_uri(self, path): # Under Windows, file URIs use the UTF-8 encoding. diff --git a/Lib/pdb.py b/Lib/pdb.py index ff40f7b2..94321115 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1026,7 +1026,11 @@ class Pdb(bdb.Bdb, cmd.Cmd): if arg: import shlex argv0 = sys.argv[0:1] - sys.argv = shlex.split(arg) + try: + sys.argv = shlex.split(arg) + except ValueError as e: + self.error('Cannot run %s: %s' % (arg, e)) + return sys.argv[:0] = argv0 # this is caught in the main debugger loop raise Restart @@ -1690,6 +1694,14 @@ def main(): print('Error:', mainpyfile, 'does not exist') sys.exit(1) + if run_as_module: + import runpy + try: + runpy._get_module_details(mainpyfile) + except Exception: + traceback.print_exc() + sys.exit(1) + sys.argv[:] = args # Hide "pdb.py" and pdb options from argument list if not run_as_module: diff --git a/Lib/platform.py b/Lib/platform.py index 0bce4381..d6412e16 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -280,6 +280,7 @@ def _syscmd_ver(system='', release='', version='', for cmd in ('ver', 'command /c ver', 'cmd /c ver'): try: info = subprocess.check_output(cmd, + stdin=subprocess.DEVNULL, stderr=subprocess.DEVNULL, text=True, shell=True) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index ecb4e5a8..af2814bd 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -349,6 +349,7 @@ def normpath(path): initial_slashes = path.startswith(sep) # POSIX allows one or two initial slashes, but treats three or more # as single slash. + # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13) if (initial_slashes and path.startswith(sep*2) and not path.startswith(sep*3)): initial_slashes = 2 diff --git a/Lib/pydoc.py b/Lib/pydoc.py index ffa4b62c..4f9d227f 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -694,7 +694,7 @@ class HTMLDoc(Doc): url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) results.append('%s' % (url, escape(all))) elif pep: - url = 'http://www.python.org/dev/peps/pep-%04d/' % int(pep) + url = 'https://www.python.org/dev/peps/pep-%04d/' % int(pep) results.append('%s' % (url, escape(all))) elif selfdot: # Create a link for methods like 'self.method(...)' @@ -1617,13 +1617,14 @@ def pipepager(text, cmd): def tempfilepager(text, cmd): """Page through text by invoking a program on a temporary file.""" import tempfile - filename = tempfile.mktemp() - with open(filename, 'w', errors='backslashreplace') as file: - file.write(text) - try: + with tempfile.TemporaryDirectory() as tempdir: + filename = os.path.join(tempdir, 'pydoc.out') + with open(filename, 'w', errors='backslashreplace', + encoding=os.device_encoding(0) if + sys.platform == 'win32' else None + ) as file: + file.write(text) os.system(cmd + ' "' + filename + '"') - finally: - os.unlink(filename) def _escape_stdout(text): # Escape non-encodable characters to avoid encoding errors later diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 8b854497..bc694349 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Jun 28 10:13:28 2021 +# Autogenerated by Sphinx on Mon Aug 30 20:40:44 2021 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -1259,6 +1259,10 @@ topics = {'assert': 'The "assert" statement\n' 'In the latter case, sequence repetition is performed; a negative\n' 'repetition factor yields an empty sequence.\n' '\n' + 'This operation can be customized using the special "__mul__()" ' + 'and\n' + '"__rmul__()" methods.\n' + '\n' 'The "@" (at) operator is intended to be used for matrix\n' 'multiplication. No builtin Python types implement this operator.\n' '\n' @@ -1274,6 +1278,10 @@ topics = {'assert': 'The "assert" statement\n' 'result. Division by zero raises the "ZeroDivisionError" ' 'exception.\n' '\n' + 'This operation can be customized using the special "__truediv__()" ' + 'and\n' + '"__floordiv__()" methods.\n' + '\n' 'The "%" (modulo) operator yields the remainder from the division ' 'of\n' 'the first argument by the second. The numeric arguments are ' @@ -1305,6 +1313,10 @@ topics = {'assert': 'The "assert" statement\n' 'string formatting is described in the Python Library Reference,\n' 'section printf-style String Formatting.\n' '\n' + 'The *modulo* operation can be customized using the special ' + '"__mod__()"\n' + 'method.\n' + '\n' 'The floor division operator, the modulo operator, and the ' '"divmod()"\n' 'function are not defined for complex numbers. Instead, convert to ' @@ -1319,9 +1331,16 @@ topics = {'assert': 'The "assert" statement\n' 'and then added together. In the latter case, the sequences are\n' 'concatenated.\n' '\n' + 'This operation can be customized using the special "__add__()" ' + 'and\n' + '"__radd__()" methods.\n' + '\n' 'The "-" (subtraction) operator yields the difference of its ' 'arguments.\n' - 'The numeric arguments are first converted to a common type.\n', + 'The numeric arguments are first converted to a common type.\n' + '\n' + 'This operation can be customized using the special "__sub__()" ' + 'method.\n', 'bitwise': 'Binary bitwise operations\n' '*************************\n' '\n' @@ -1334,14 +1353,18 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'The "&" operator yields the bitwise AND of its arguments, which ' 'must\n' - 'be integers.\n' + 'be integers or one of them must be a custom object overriding\n' + '"__and__()" or "__rand__()" special methods.\n' '\n' 'The "^" operator yields the bitwise XOR (exclusive OR) of its\n' - 'arguments, which must be integers.\n' + 'arguments, which must be integers or one of them must be a ' + 'custom\n' + 'object overriding "__xor__()" or "__rxor__()" special methods.\n' '\n' 'The "|" operator yields the bitwise (inclusive) OR of its ' 'arguments,\n' - 'which must be integers.\n', + 'which must be integers or one of them must be a custom object\n' + 'overriding "__or__()" or "__ror__()" special methods.\n', 'bltin-code-objects': 'Code Objects\n' '************\n' '\n' @@ -1787,7 +1810,11 @@ topics = {'assert': 'The "assert" statement\n' ' comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n' ' | "is" ["not"] | ["not"] "in"\n' '\n' - 'Comparisons yield boolean values: "True" or "False".\n' + 'Comparisons yield boolean values: "True" or "False". Custom ' + '*rich\n' + 'comparison methods* may return non-boolean values. In this ' + 'case Python\n' + 'will call "bool()" on such value in boolean contexts.\n' '\n' 'Comparisons can be chained arbitrarily, e.g., "x < y <= z" ' 'is\n' @@ -7472,7 +7499,10 @@ topics = {'assert': 'The "assert" statement\n' '"ZeroDivisionError".\n' 'Raising a negative number to a fractional power results in a ' '"complex"\n' - 'number. (In earlier versions it raised a "ValueError".)\n', + 'number. (In earlier versions it raised a "ValueError".)\n' + '\n' + 'This operation can be customized using the special "__pow__()" ' + 'method.\n', 'raise': 'The "raise" statement\n' '*********************\n' '\n' @@ -7872,6 +7902,10 @@ topics = {'assert': 'The "assert" statement\n' 'the\n' 'second argument.\n' '\n' + 'This operation can be customized using the special ' + '"__lshift__()" and\n' + '"__rshift__()" methods.\n' + '\n' 'A right shift by *n* bits is defined as floor division by ' '"pow(2,n)".\n' 'A left shift by *n* bits is defined as multiplication with ' @@ -10092,7 +10126,7 @@ topics = {'assert': 'The "assert" statement\n' '*start* and\n' ' *end* are interpreted as in slice notation.\n' '\n' - "str.encode(encoding='utf-8', errors='strict')\n" + 'str.encode(encoding="utf-8", errors="strict")\n' '\n' ' Return an encoded version of the string as a bytes ' 'object. Default\n' @@ -10598,7 +10632,7 @@ topics = {'assert': 'The "assert" statement\n' 'followed by\n' ' the string itself.\n' '\n' - 'str.rsplit(sep=None, maxsplit=- 1)\n' + 'str.rsplit(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -10639,7 +10673,7 @@ topics = {'assert': 'The "assert" statement\n' " >>> 'Monty Python'.removesuffix(' Python')\n" " 'Monty'\n" '\n' - 'str.split(sep=None, maxsplit=- 1)\n' + 'str.split(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -11611,7 +11645,7 @@ topics = {'assert': 'The "assert" statement\n' ' points. All the code points in the range "U+0000 - ' 'U+10FFFF"\n' ' can be represented in a string. Python doesn’t have a ' - '*char*\n' + '"char"\n' ' type; instead, every code point in the string is ' 'represented\n' ' as a string object with length "1". The built-in ' @@ -13419,7 +13453,7 @@ topics = {'assert': 'The "assert" statement\n' '| | "s[i:i] = ' '[x]") | |\n' '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.pop([i])" | retrieves the item at *i* ' + '| "s.pop()" or "s.pop(i)" | retrieves the item at *i* ' 'and | (2) |\n' '| | also removes it from ' '*s* | |\n' @@ -13882,7 +13916,7 @@ topics = {'assert': 'The "assert" statement\n' '| | "s[i:i] = ' '[x]") | |\n' '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.pop([i])" | retrieves the item at ' + '| "s.pop()" or "s.pop(i)" | retrieves the item at ' '*i* and | (2) |\n' '| | also removes it from ' '*s* | |\n' @@ -13947,15 +13981,21 @@ topics = {'assert': 'The "assert" statement\n' ' u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n' '\n' 'The unary "-" (minus) operator yields the negation of its numeric\n' - 'argument.\n' + 'argument; the operation can be overridden with the "__neg__()" ' + 'special\n' + 'method.\n' '\n' 'The unary "+" (plus) operator yields its numeric argument ' - 'unchanged.\n' + 'unchanged;\n' + 'the operation can be overridden with the "__pos__()" special ' + 'method.\n' '\n' 'The unary "~" (invert) operator yields the bitwise inversion of ' 'its\n' 'integer argument. The bitwise inversion of "x" is defined as\n' - '"-(x+1)". It only applies to integral numbers.\n' + '"-(x+1)". It only applies to integral numbers or to custom ' + 'objects\n' + 'that override the "__invert__()" special method.\n' '\n' 'In all three cases, if the argument does not have the proper type, ' 'a\n' diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index bca4a7bc..923f5c05 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -169,13 +169,20 @@ class Completer: if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): match = "%s.%s" % (expr, word) - try: - val = getattr(thisobject, word) - except Exception: - pass # Include even if attribute not set + if isinstance(getattr(type(thisobject), word, None), + property): + # bpo-44752: thisobject.word is a method decorated by + # `@property`. What follows applies a postfix if + # thisobject.word is callable, but know we know that + # this is not callable (because it is a property). + # Also, getattr(thisobject, word) will evaluate the + # property method, which is not desirable. + matches.append(match) + continue + if (value := getattr(thisobject, word, None)) is not None: + matches.append(self._callable_postfix(value, match)) else: - match = self._callable_postfix(val, match) - matches.append(match) + matches.append(match) if matches or not noprefix: break if noprefix == '_': diff --git a/Lib/shutil.py b/Lib/shutil.py index 5cb796e8..08384cf9 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -261,28 +261,36 @@ def copyfile(src, dst, *, follow_symlinks=True): if not follow_symlinks and _islink(src): os.symlink(os.readlink(src), dst) else: - with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst: - # macOS - if _HAS_FCOPYFILE: - try: - _fastcopy_fcopyfile(fsrc, fdst, posix._COPYFILE_DATA) - return dst - except _GiveupOnFastCopy: - pass - # Linux - elif _USE_CP_SENDFILE: - try: - _fastcopy_sendfile(fsrc, fdst) + try: + with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst: + # macOS + if _HAS_FCOPYFILE: + try: + _fastcopy_fcopyfile(fsrc, fdst, posix._COPYFILE_DATA) + return dst + except _GiveupOnFastCopy: + pass + # Linux + elif _USE_CP_SENDFILE: + try: + _fastcopy_sendfile(fsrc, fdst) + return dst + except _GiveupOnFastCopy: + pass + # Windows, see: + # https://github.com/python/cpython/pull/7160#discussion_r195405230 + elif _WINDOWS and file_size > 0: + _copyfileobj_readinto(fsrc, fdst, min(file_size, COPY_BUFSIZE)) return dst - except _GiveupOnFastCopy: - pass - # Windows, see: - # https://github.com/python/cpython/pull/7160#discussion_r195405230 - elif _WINDOWS and file_size > 0: - _copyfileobj_readinto(fsrc, fdst, min(file_size, COPY_BUFSIZE)) - return dst - - copyfileobj(fsrc, fdst) + + copyfileobj(fsrc, fdst) + + # Issue 43219, raise a less confusing exception + except IsADirectoryError as e: + if os.path.exists(dst): + raise + else: + raise FileNotFoundError(f'Directory does not exist: {dst}') from e return dst diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 4a5ca669..b1fd45a0 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -367,10 +367,15 @@ class SMTP: def putcmd(self, cmd, args=""): """Send a command to the server.""" if args == "": - str = '%s%s' % (cmd, CRLF) + s = cmd else: - str = '%s %s%s' % (cmd, args, CRLF) - self.send(str) + s = f'{cmd} {args}' + if '\r' in s or '\n' in s: + s = s.replace('\n', '\\n').replace('\r', '\\r') + raise ValueError( + f'command and arguments contain prohibited newline characters: {s}' + ) + self.send(f'{s}{CRLF}') def getreply(self): """Get a reply from the server. diff --git a/Lib/socket.py b/Lib/socket.py index cafa573a..46fc49ca 100755 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -781,8 +781,9 @@ def getfqdn(name=''): An empty argument is interpreted as meaning the local host. First the hostname returned by gethostbyaddr() is checked, then - possibly existing aliases. In case no FQDN is available, hostname - from gethostname() is returned. + possibly existing aliases. In case no FQDN is available and `name` + was given, it is returned unchanged. If `name` was empty or '0.0.0.0', + hostname from gethostname() is returned. """ name = name.strip() if not name or name == '0.0.0.0': diff --git a/Lib/sqlite3/test/backup.py b/Lib/sqlite3/test/backup.py index ad1da972..6a772c64 100644 --- a/Lib/sqlite3/test/backup.py +++ b/Lib/sqlite3/test/backup.py @@ -150,7 +150,8 @@ class BackupTests(unittest.TestCase): self.cx.backup(bck, name='non-existing') self.assertIn( str(cm.exception), - ['SQL logic error', 'SQL logic error or missing database'] + ['SQL logic error', 'SQL logic error or missing database', + 'unknown database non-existing'] ) self.cx.execute("ATTACH DATABASE ':memory:' AS attached_db") diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index ad9c9f02..d00b60ca 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -21,11 +21,13 @@ # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. +import subprocess import threading import unittest import sqlite3 as sqlite +import sys -from test.support import TESTFN, unlink +from test.support import SHORT_TIMEOUT, TESTFN, unlink class ModuleTests(unittest.TestCase): @@ -192,6 +194,27 @@ class ConnectionTests(unittest.TestCase): sqlite.connect(':memory:', check_same_thread=False) self.assertEqual(str(cm.exception), 'shared connections not available') + +class UninitialisedConnectionTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.Connection.__new__(sqlite.Connection) + + def test_uninit_operations(self): + funcs = ( + lambda: self.cx.isolation_level, + lambda: self.cx.total_changes, + lambda: self.cx.in_transaction, + lambda: self.cx.iterdump(), + lambda: self.cx.cursor(), + lambda: self.cx.close(), + ) + for func in funcs: + with self.subTest(func=func): + self.assertRaisesRegex(sqlite.ProgrammingError, + "Base Connection.__init__ not called", + func) + + class CursorTests(unittest.TestCase): def setUp(self): self.cx = sqlite.connect(":memory:") @@ -372,6 +395,9 @@ class CursorTests(unittest.TestCase): def __init__(self): self.value = 5 + def __iter__(self): + return self + def __next__(self): if self.value == 10: raise StopIteration @@ -933,6 +959,77 @@ class SqliteOnConflictTests(unittest.TestCase): self.assertEqual(self.cu.fetchall(), [('Very different data!', 'foo')]) +class MultiprocessTests(unittest.TestCase): + CONNECTION_TIMEOUT = SHORT_TIMEOUT / 1000. # Defaults to 30 ms + + def tearDown(self): + unlink(TESTFN) + + def test_ctx_mgr_rollback_if_commit_failed(self): + # bpo-27334: ctx manager does not rollback if commit fails + SCRIPT = f"""if 1: + import sqlite3 + def wait(): + print("started") + assert "database is locked" in input() + + cx = sqlite3.connect("{TESTFN}", timeout={self.CONNECTION_TIMEOUT}) + cx.create_function("wait", 0, wait) + with cx: + cx.execute("create table t(t)") + try: + # execute two transactions; both will try to lock the db + cx.executescript(''' + -- start a transaction and wait for parent + begin transaction; + select * from t; + select wait(); + rollback; + + -- start a new transaction; would fail if parent holds lock + begin transaction; + select * from t; + rollback; + ''') + finally: + cx.close() + """ + + # spawn child process + proc = subprocess.Popen( + [sys.executable, "-c", SCRIPT], + encoding="utf-8", + bufsize=0, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + self.addCleanup(proc.communicate) + + # wait for child process to start + self.assertEqual("started", proc.stdout.readline().strip()) + + cx = sqlite.connect(TESTFN, timeout=self.CONNECTION_TIMEOUT) + try: # context manager should correctly release the db lock + with cx: + cx.execute("insert into t values('test')") + except sqlite.OperationalError as exc: + proc.stdin.write(str(exc)) + else: + proc.stdin.write("no error") + finally: + cx.close() + + # terminate child process + self.assertIsNone(proc.returncode) + try: + proc.communicate(input="end", timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + proc.communicate() + raise + self.assertEqual(proc.returncode, 0) + + def suite(): module_suite = unittest.makeSuite(ModuleTests, "Check") connection_suite = unittest.makeSuite(ConnectionTests, "Check") @@ -943,10 +1040,12 @@ def suite(): closed_con_suite = unittest.makeSuite(ClosedConTests, "Check") closed_cur_suite = unittest.makeSuite(ClosedCurTests, "Check") on_conflict_suite = unittest.makeSuite(SqliteOnConflictTests, "Check") + uninit_con_suite = unittest.makeSuite(UninitialisedConnectionTests) + multiproc_con_suite = unittest.makeSuite(MultiprocessTests) return unittest.TestSuite(( module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, ext_suite, closed_con_suite, closed_cur_suite, - on_conflict_suite, + on_conflict_suite, uninit_con_suite, multiproc_con_suite, )) def test(): diff --git a/Lib/sqlite3/test/regression.py b/Lib/sqlite3/test/regression.py index 6aa86d57..6ba33564 100644 --- a/Lib/sqlite3/test/regression.py +++ b/Lib/sqlite3/test/regression.py @@ -127,11 +127,11 @@ class RegressionTests(unittest.TestCase): con = sqlite.connect(":memory:",detect_types=sqlite.PARSE_DECLTYPES) con.execute("create table foo(bar timestamp)") con.execute("insert into foo(bar) values (?)", (datetime.datetime.now(),)) - con.execute(SELECT) + con.execute(SELECT).close() con.execute("drop table foo") con.execute("create table foo(bar integer)") con.execute("insert into foo(bar) values (5)") - con.execute(SELECT) + con.execute(SELECT).close() def CheckBindMutatingList(self): # Issue41662: Crash when mutate a list of parameters during iteration. diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py index c11c82e1..1bceefe8 100644 --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -27,6 +27,8 @@ import sqlite3 as sqlite def func_returntext(): return "foo" +def func_returntextwithnull(): + return "1\x002" def func_returnunicode(): return "bar" def func_returnint(): @@ -137,11 +139,21 @@ class AggrSum: def finalize(self): return self.val +class AggrText: + def __init__(self): + self.txt = "" + def step(self, txt): + self.txt = self.txt + txt + def finalize(self): + return self.txt + + class FunctionTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") self.con.create_function("returntext", 0, func_returntext) + self.con.create_function("returntextwithnull", 0, func_returntextwithnull) self.con.create_function("returnunicode", 0, func_returnunicode) self.con.create_function("returnint", 0, func_returnint) self.con.create_function("returnfloat", 0, func_returnfloat) @@ -185,6 +197,12 @@ class FunctionTests(unittest.TestCase): self.assertEqual(type(val), str) self.assertEqual(val, "foo") + def CheckFuncReturnTextWithNullChar(self): + cur = self.con.cursor() + res = cur.execute("select returntextwithnull()").fetchone()[0] + self.assertEqual(type(res), str) + self.assertEqual(res, "1\x002") + def CheckFuncReturnUnicode(self): cur = self.con.cursor() cur.execute("select returnunicode()") @@ -236,9 +254,11 @@ class FunctionTests(unittest.TestCase): def CheckParamString(self): cur = self.con.cursor() - cur.execute("select isstring(?)", ("foo",)) - val = cur.fetchone()[0] - self.assertEqual(val, 1) + for text in ["foo", str()]: + with self.subTest(text=text): + cur.execute("select isstring(?)", (text,)) + val = cur.fetchone()[0] + self.assertEqual(val, 1) def CheckParamInt(self): cur = self.con.cursor() @@ -341,6 +361,7 @@ class AggregateTests(unittest.TestCase): self.con.create_aggregate("checkType", 2, AggrCheckType) self.con.create_aggregate("checkTypes", -1, AggrCheckTypes) self.con.create_aggregate("mysum", 1, AggrSum) + self.con.create_aggregate("aggtxt", 1, AggrText) def tearDown(self): #self.cur.close() @@ -387,9 +408,9 @@ class AggregateTests(unittest.TestCase): def CheckAggrCheckParamStr(self): cur = self.con.cursor() - cur.execute("select checkType('str', ?)", ("foo",)) + cur.execute("select checkTypes('str', ?, ?)", ("foo", str())) val = cur.fetchone()[0] - self.assertEqual(val, 1) + self.assertEqual(val, 2) def CheckAggrCheckParamInt(self): cur = self.con.cursor() @@ -429,6 +450,15 @@ class AggregateTests(unittest.TestCase): val = cur.fetchone()[0] self.assertEqual(val, 60) + def CheckAggrText(self): + cur = self.con.cursor() + for txt in ["foo", "1\x002"]: + with self.subTest(txt=txt): + cur.execute("select aggtxt(?) from test", (txt,)) + val = cur.fetchone()[0] + self.assertEqual(val, txt) + + class AuthorizerTests(unittest.TestCase): @staticmethod def authorizer_cb(action, arg1, arg2, dbname, source): diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index e47905c8..3072e586 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -607,6 +607,7 @@ class _TestProcess(BaseTestCase): del c p.start() p.join() + gc.collect() # For PyPy or other GCs. self.assertIs(wr(), None) self.assertEqual(q.get(), 5) close_queue(q) @@ -2282,6 +2283,16 @@ class _TestContainers(BaseTestCase): self.assertIsInstance(outer[0], list) # Not a ListProxy self.assertEqual(outer[-1][-1]['feed'], 3) + def test_nested_queue(self): + a = self.list() # Test queue inside list + a.append(self.Queue()) + a[0].put(123) + self.assertEqual(a[0].get(), 123) + b = self.dict() # Test queue inside dict + b[0] = self.Queue() + b[0].put(456) + self.assertEqual(b[0].get(), 456) + def test_namespace(self): n = self.Namespace() n.name = 'Bob' @@ -2653,6 +2664,7 @@ class _TestPool(BaseTestCase): self.pool.map(identity, objs) del objs + gc.collect() # For PyPy or other GCs. time.sleep(DELTA) # let threaded cleanup code run self.assertEqual(set(wr() for wr in refs), {None}) # With a process pool, copies of the objects are returned, check @@ -4155,6 +4167,7 @@ class _TestFinalize(BaseTestCase): util._finalizer_registry.clear() def tearDown(self): + gc.collect() # For PyPy or other GCs. self.assertFalse(util._finalizer_registry) util._finalizer_registry.update(self.registry_backup) @@ -4166,12 +4179,14 @@ class _TestFinalize(BaseTestCase): a = Foo() util.Finalize(a, conn.send, args=('a',)) del a # triggers callback for a + gc.collect() # For PyPy or other GCs. b = Foo() close_b = util.Finalize(b, conn.send, args=('b',)) close_b() # triggers callback for b close_b() # does nothing because callback has already been called del b # does nothing because callback has already been called + gc.collect() # For PyPy or other GCs. c = Foo() util.Finalize(c, conn.send, args=('c',)) diff --git a/Lib/test/_typed_dict_helper.py b/Lib/test/_typed_dict_helper.py new file mode 100644 index 00000000..d333db19 --- /dev/null +++ b/Lib/test/_typed_dict_helper.py @@ -0,0 +1,18 @@ +"""Used to test `get_type_hints()` on a cross-module inherited `TypedDict` class + +This script uses future annotations to postpone a type that won't be available +on the module inheriting from to `Foo`. The subclass in the other module should +look something like this: + + class Bar(_typed_dict_helper.Foo, total=False): + b: int +""" + +from __future__ import annotations + +from typing import Optional, TypedDict + +OptionalIntType = Optional[int] + +class Foo(TypedDict): + a: OptionalIntType diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 8e66594e..95216bcc 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -6,6 +6,7 @@ module with arguments identifying each test. """ import contextlib +import os import sys @@ -106,6 +107,32 @@ def test_block_add_hook_baseexception(): pass +def test_marshal(): + import marshal + o = ("a", "b", "c", 1, 2, 3) + payload = marshal.dumps(o) + + with TestHook() as hook: + assertEqual(o, marshal.loads(marshal.dumps(o))) + + try: + with open("test-marshal.bin", "wb") as f: + marshal.dump(o, f) + with open("test-marshal.bin", "rb") as f: + assertEqual(o, marshal.load(f)) + finally: + os.unlink("test-marshal.bin") + + actual = [(a[0], a[1]) for e, a in hook.seen if e == "marshal.dumps"] + assertSequenceEqual(actual, [(o, marshal.version)] * 2) + + actual = [a[0] for e, a in hook.seen if e == "marshal.loads"] + assertSequenceEqual(actual, [payload]) + + actual = [e for e, a in hook.seen if e == "marshal.load"] + assertSequenceEqual(actual, ["marshal.load"]) + + def test_pickle(): import pickle diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index c0bb051b..6f0b30af 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -139,6 +139,39 @@ ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', # default (see bpo-30822). RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') + +class Namespace(argparse.Namespace): + def __init__(self, **kwargs) -> None: + self.testdir = None + self.verbose = 0 + self.quiet = False + self.exclude = False + self.single = False + self.randomize = False + self.fromfile = None + self.findleaks = 1 + self.fail_env_changed = False + self.use_resources = None + self.trace = False + self.coverdir = 'coverage' + self.runleaks = False + self.huntrleaks = False + self.verbose2 = False + self.verbose3 = False + self.print_slow = False + self.random_seed = None + self.use_mp = None + self.forever = False + self.header = False + self.failfast = False + self.match_tests = None + self.ignore_tests = None + self.pgo = False + self.pgo_extended = False + + super().__init__(**kwargs) + + class _ArgParser(argparse.ArgumentParser): def error(self, message): @@ -319,13 +352,7 @@ def resources_list(string): def _parse_args(args, **kwargs): # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=1, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None, ignore_tests=None, - pgo=False) + ns = Namespace() for k, v in kwargs.items(): if not hasattr(ns, k): raise TypeError('%r is an invalid keyword argument ' diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e20bf9a3..6daf68b6 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -12,10 +12,10 @@ import time import unittest from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( - findtests, runtest, get_abs_module, - STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, - INTERRUPTED, CHILD_ERROR, TEST_DID_NOT_RUN, TIMEOUT, - PROGRESS_MIN_TIME, format_test_result, is_failed) + findtests, runtest, get_abs_module, is_failed, + STDTESTS, NOTTESTS, PROGRESS_MIN_TIME, + Passed, Failed, EnvChanged, Skipped, ResourceDenied, Interrupted, + ChildError, DidNotRun) from test.libregrtest.setup import setup_tests from test.libregrtest.pgo import setup_pgo_tests from test.libregrtest.utils import removepy, count, format_duration, printlist @@ -99,34 +99,32 @@ class Regrtest: | set(self.run_no_tests)) def accumulate_result(self, result, rerun=False): - test_name = result.test_name - ok = result.result + test_name = result.name - if ok not in (CHILD_ERROR, INTERRUPTED) and not rerun: - self.test_times.append((result.test_time, test_name)) + if not isinstance(result, (ChildError, Interrupted)) and not rerun: + self.test_times.append((result.duration_sec, test_name)) - if ok == PASSED: + if isinstance(result, Passed): self.good.append(test_name) - elif ok in (FAILED, CHILD_ERROR): - if not rerun: - self.bad.append(test_name) - elif ok == ENV_CHANGED: - self.environment_changed.append(test_name) - elif ok == SKIPPED: - self.skipped.append(test_name) - elif ok == RESOURCE_DENIED: + elif isinstance(result, ResourceDenied): self.skipped.append(test_name) self.resource_denieds.append(test_name) - elif ok == TEST_DID_NOT_RUN: + elif isinstance(result, Skipped): + self.skipped.append(test_name) + elif isinstance(result, EnvChanged): + self.environment_changed.append(test_name) + elif isinstance(result, Failed): + if not rerun: + self.bad.append(test_name) + self.rerun.append(result) + elif isinstance(result, DidNotRun): self.run_no_tests.append(test_name) - elif ok == INTERRUPTED: + elif isinstance(result, Interrupted): self.interrupted = True - elif ok == TIMEOUT: - self.bad.append(test_name) else: - raise ValueError("invalid test result: %r" % ok) + raise ValueError("invalid test result: %r" % result) - if rerun and ok not in {FAILED, CHILD_ERROR, INTERRUPTED}: + if rerun and not isinstance(result, (Failed, Interrupted)): self.bad.remove(test_name) xml_data = result.xml_data @@ -311,15 +309,31 @@ class Regrtest: self.log() self.log("Re-running failed tests in verbose mode") - self.rerun = self.bad[:] - for test_name in self.rerun: - self.log(f"Re-running {test_name} in verbose mode") + rerun_list = self.rerun[:] + self.rerun = [] + for result in rerun_list: + test_name = result.name + errors = result.errors or [] + failures = result.failures or [] + error_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in errors] + failure_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in failures] self.ns.verbose = True + orig_match_tests = self.ns.match_tests + if errors or failures: + if self.ns.match_tests is None: + self.ns.match_tests = [] + self.ns.match_tests.extend(error_names) + self.ns.match_tests.extend(failure_names) + matching = "matching: " + ", ".join(self.ns.match_tests) + self.log(f"Re-running {test_name} in verbose mode ({matching})") + else: + self.log(f"Re-running {test_name} in verbose mode") result = runtest(self.ns, test_name) + self.ns.match_tests = orig_match_tests self.accumulate_result(result, rerun=True) - if result.result == INTERRUPTED: + if isinstance(result, Interrupted): break if self.bad: @@ -380,7 +394,7 @@ class Regrtest: if self.rerun: print() print("%s:" % count(len(self.rerun), "re-run test")) - printlist(self.rerun) + printlist(r.name for r in self.rerun) if self.run_no_tests: print() @@ -420,14 +434,14 @@ class Regrtest: result = runtest(self.ns, test_name) self.accumulate_result(result) - if result.result == INTERRUPTED: + if isinstance(result, Interrupted): break - previous_test = format_test_result(result) + previous_test = str(result) test_time = time.monotonic() - start_time if test_time >= PROGRESS_MIN_TIME: previous_test = "%s in %s" % (previous_test, format_duration(test_time)) - elif result.result == PASSED: + elif isinstance(result, Passed): # be quiet: say nothing if the test passed shortly previous_test = None diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 9338b280..b70641c0 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -1,4 +1,5 @@ -import collections +from __future__ import annotations + import faulthandler import functools import gc @@ -12,32 +13,109 @@ import unittest from test import support from test.libregrtest.refleak import dash_R, clear_caches +from test.libregrtest.cmdline import Namespace from test.libregrtest.save_env import saved_test_environment from test.libregrtest.utils import format_duration, print_warning -# Test result constants. -PASSED = 1 -FAILED = 0 -ENV_CHANGED = -1 -SKIPPED = -2 -RESOURCE_DENIED = -3 -INTERRUPTED = -4 -CHILD_ERROR = -5 # error in a child process -TEST_DID_NOT_RUN = -6 -TIMEOUT = -7 - -_FORMAT_TEST_RESULT = { - PASSED: '%s passed', - FAILED: '%s failed', - ENV_CHANGED: '%s failed (env changed)', - SKIPPED: '%s skipped', - RESOURCE_DENIED: '%s skipped (resource denied)', - INTERRUPTED: '%s interrupted', - CHILD_ERROR: '%s crashed', - TEST_DID_NOT_RUN: '%s run no tests', - TIMEOUT: '%s timed out', -} +class TestResult: + def __init__( + self, + name: str, + duration_sec: float = 0.0, + xml_data: list[str] | None = None, + ) -> None: + self.name = name + self.duration_sec = duration_sec + self.xml_data = xml_data + + def __str__(self) -> str: + return f"{self.name} finished" + + +class Passed(TestResult): + def __str__(self) -> str: + return f"{self.name} passed" + + +class Failed(TestResult): + def __init__( + self, + name: str, + duration_sec: float = 0.0, + xml_data: list[str] | None = None, + errors: list[tuple[str, str]] | None = None, + failures: list[tuple[str, str]] | None = None, + ) -> None: + super().__init__(name, duration_sec=duration_sec, xml_data=xml_data) + self.errors = errors + self.failures = failures + + def __str__(self) -> str: + if self.errors and self.failures: + le = len(self.errors) + lf = len(self.failures) + error_s = "error" + ("s" if le > 1 else "") + failure_s = "failure" + ("s" if lf > 1 else "") + return f"{self.name} failed ({le} {error_s}, {lf} {failure_s})" + + if self.errors: + le = len(self.errors) + error_s = "error" + ("s" if le > 1 else "") + return f"{self.name} failed ({le} {error_s})" + + if self.failures: + lf = len(self.failures) + failure_s = "failure" + ("s" if lf > 1 else "") + return f"{self.name} failed ({lf} {failure_s})" + + return f"{self.name} failed" + + +class UncaughtException(Failed): + def __str__(self) -> str: + return f"{self.name} failed (uncaught exception)" + + +class EnvChanged(Failed): + def __str__(self) -> str: + return f"{self.name} failed (env changed)" + + +class RefLeak(Failed): + def __str__(self) -> str: + return f"{self.name} failed (reference leak)" + + +class Skipped(TestResult): + def __str__(self) -> str: + return f"{self.name} skipped" + + +class ResourceDenied(Skipped): + def __str__(self) -> str: + return f"{self.name} skipped (resource denied)" + + +class Interrupted(TestResult): + def __str__(self) -> str: + return f"{self.name} interrupted" + + +class ChildError(Failed): + def __str__(self) -> str: + return f"{self.name} crashed" + + +class DidNotRun(TestResult): + def __str__(self) -> str: + return f"{self.name} ran no tests" + + +class Timeout(Failed): + def __str__(self) -> str: + return f"{self.name} timed out ({format_duration(self.duration_sec)})" + # Minimum duration of a test to display its duration or to mention that # the test is running in background @@ -66,21 +144,10 @@ NOTTESTS = set() FOUND_GARBAGE = [] -def is_failed(result, ns): - ok = result.result - if ok in (PASSED, RESOURCE_DENIED, SKIPPED, TEST_DID_NOT_RUN): - return False - if ok == ENV_CHANGED: +def is_failed(result: TestResult, ns: Namespace) -> bool: + if isinstance(result, EnvChanged): return ns.fail_env_changed - return True - - -def format_test_result(result): - fmt = _FORMAT_TEST_RESULT.get(result.result, "%s") - text = fmt % result.test_name - if result.result == TIMEOUT: - text = '%s (%s)' % (text, format_duration(result.test_time)) - return text + return isinstance(result, Failed) def findtestdir(path=None): @@ -100,7 +167,7 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): return stdtests + sorted(tests) -def get_abs_module(ns, test_name): +def get_abs_module(ns: Namespace, test_name: str) -> str: if test_name.startswith('test.') or ns.testdir: return test_name else: @@ -108,10 +175,7 @@ def get_abs_module(ns, test_name): return 'test.' + test_name -TestResult = collections.namedtuple('TestResult', - 'test_name result test_time xml_data') - -def _runtest(ns, test_name): +def _runtest(ns: Namespace, test_name: str) -> TestResult: # Handle faulthandler timeout, capture stdout+stderr, XML serialization # and measure time. @@ -139,7 +203,7 @@ def _runtest(ns, test_name): sys.stderr = stream result = _runtest_inner(ns, test_name, display_failure=False) - if result != PASSED: + if not isinstance(result, Passed): output = stream.getvalue() orig_stderr.write(output) orig_stderr.flush() @@ -155,36 +219,26 @@ def _runtest(ns, test_name): if xml_list: import xml.etree.ElementTree as ET - xml_data = [ET.tostring(x).decode('us-ascii') for x in xml_list] - else: - xml_data = None + result.xml_data = [ + ET.tostring(x).decode('us-ascii') + for x in xml_list + ] - test_time = time.perf_counter() - start_time - - return TestResult(test_name, result, test_time, xml_data) + result.duration_sec = time.perf_counter() - start_time + return result finally: if use_timeout: faulthandler.cancel_dump_traceback_later() support.junit_xml_list = None -def runtest(ns, test_name): +def runtest(ns: Namespace, test_name: str) -> TestResult: """Run a single test. ns -- regrtest namespace of options test_name -- the name of the test - Returns the tuple (result, test_time, xml_data), where result is one - of the constants: - - INTERRUPTED KeyboardInterrupt - RESOURCE_DENIED test skipped because resource denied - SKIPPED test skipped for some other reason - ENV_CHANGED test failed because it changed the execution environment - FAILED test failed - PASSED test passed - EMPTY_TEST_SUITE test ran no subtests. - TIMEOUT test timed out. + Returns a TestResult sub-class depending on the kind of result received. If ns.xmlpath is not None, xml_data is a list containing each generated testsuite element. @@ -196,7 +250,7 @@ def runtest(ns, test_name): msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", file=sys.stderr, flush=True) - return TestResult(test_name, FAILED, 0.0, None) + return Failed(test_name) def _test_module(the_module): @@ -209,7 +263,7 @@ def _test_module(the_module): support.run_unittest(tests) -def _runtest_inner2(ns, test_name): +def _runtest_inner2(ns: Namespace, test_name: str) -> bool: # Load the test function, run the test function, handle huntrleaks # and findleaks to detect leaks @@ -253,7 +307,9 @@ def _runtest_inner2(ns, test_name): return refleak -def _runtest_inner(ns, test_name, display_failure=True): +def _runtest_inner( + ns: Namespace, test_name: str, display_failure: bool = True +) -> TestResult: # Detect environment changes, handle exceptions. # Reset the environment_altered flag to detect if a test altered @@ -271,37 +327,43 @@ def _runtest_inner(ns, test_name, display_failure=True): except support.ResourceDenied as msg: if not ns.quiet and not ns.pgo: print(f"{test_name} skipped -- {msg}", flush=True) - return RESOURCE_DENIED + return ResourceDenied(test_name) except unittest.SkipTest as msg: if not ns.quiet and not ns.pgo: print(f"{test_name} skipped -- {msg}", flush=True) - return SKIPPED + return Skipped(test_name) + except support.TestFailedWithDetails as exc: + msg = f"test {test_name} failed" + if display_failure: + msg = f"{msg} -- {exc}" + print(msg, file=sys.stderr, flush=True) + return Failed(test_name, errors=exc.errors, failures=exc.failures) except support.TestFailed as exc: msg = f"test {test_name} failed" if display_failure: msg = f"{msg} -- {exc}" print(msg, file=sys.stderr, flush=True) - return FAILED + return Failed(test_name) except support.TestDidNotRun: - return TEST_DID_NOT_RUN + return DidNotRun(test_name) except KeyboardInterrupt: print() - return INTERRUPTED + return Interrupted(test_name) except: if not ns.pgo: msg = traceback.format_exc() print(f"test {test_name} crashed -- {msg}", file=sys.stderr, flush=True) - return FAILED + return UncaughtException(test_name) if refleak: - return FAILED + return RefLeak(test_name) if environment.changed: - return ENV_CHANGED - return PASSED + return EnvChanged(test_name) + return Passed(test_name) -def cleanup_test_droppings(test_name, verbose): +def cleanup_test_droppings(test_name: str, verbose: int) -> None: # First kill any dangling references to open files etc. # This can also issue some ResourceWarnings which would otherwise get # triggered during the following test run, and possibly produce failures. diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 7a18e454..b9404d53 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,4 +1,5 @@ -import collections +from __future__ import annotations + import faulthandler import json import os @@ -9,12 +10,14 @@ import sys import threading import time import traceback -import types +from typing import NamedTuple, NoReturn, Literal, Any, TYPE_CHECKING + from test import support +from test.libregrtest.cmdline import Namespace +from test.libregrtest.main import Regrtest from test.libregrtest.runtest import ( - runtest, INTERRUPTED, CHILD_ERROR, PROGRESS_MIN_TIME, - format_test_result, TestResult, is_failed, TIMEOUT) + runtest, is_failed, TestResult, Interrupted, Timeout, ChildError, PROGRESS_MIN_TIME) from test.libregrtest.setup import setup_tests from test.libregrtest.utils import format_duration, print_warning @@ -35,21 +38,21 @@ JOIN_TIMEOUT = 30.0 # seconds USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) -def must_stop(result, ns): - if result.result == INTERRUPTED: +def must_stop(result: TestResult, ns: Namespace) -> bool: + if isinstance(result, Interrupted): return True if ns.failfast and is_failed(result, ns): return True return False -def parse_worker_args(worker_args): +def parse_worker_args(worker_args) -> tuple[Namespace, str]: ns_dict, test_name = json.loads(worker_args) - ns = types.SimpleNamespace(**ns_dict) + ns = Namespace(**ns_dict) return (ns, test_name) -def run_test_in_subprocess(testname, ns): +def run_test_in_subprocess(testname: str, ns: Namespace) -> subprocess.Popen: ns_dict = vars(ns) worker_args = (ns_dict, testname) worker_args = json.dumps(worker_args) @@ -74,15 +77,15 @@ def run_test_in_subprocess(testname, ns): **kw) -def run_tests_worker(ns, test_name): +def run_tests_worker(ns: Namespace, test_name: str) -> NoReturn: setup_tests(ns) result = runtest(ns, test_name) print() # Force a newline (just in case) - # Serialize TestResult as list in JSON - print(json.dumps(list(result)), flush=True) + # Serialize TestResult as dict in JSON + print(json.dumps(result, cls=EncodeTestResult), flush=True) sys.exit(0) @@ -109,15 +112,24 @@ class MultiprocessIterator: self.tests_iter = None -MultiprocessResult = collections.namedtuple('MultiprocessResult', - 'result stdout stderr error_msg') +class MultiprocessResult(NamedTuple): + result: TestResult + stdout: str + stderr: str + error_msg: str + + +if TYPE_CHECKING: + ExcStr = str + QueueOutput = tuple[Literal[False], MultiprocessResult] | tuple[Literal[True], ExcStr] + class ExitThread(Exception): pass class TestWorkerProcess(threading.Thread): - def __init__(self, worker_id, runner): + def __init__(self, worker_id: int, runner: "MultiprocessTestRunner") -> None: super().__init__() self.worker_id = worker_id self.pending = runner.pending @@ -131,7 +143,7 @@ class TestWorkerProcess(threading.Thread): self._killed = False self._stopped = False - def __repr__(self): + def __repr__(self) -> str: info = [f'TestWorkerProcess #{self.worker_id}'] if self.is_alive(): info.append("running") @@ -147,7 +159,7 @@ class TestWorkerProcess(threading.Thread): f'time={format_duration(dt)}')) return '<%s>' % ' '.join(info) - def _kill(self): + def _kill(self) -> None: popen = self._popen if popen is None: return @@ -175,18 +187,22 @@ class TestWorkerProcess(threading.Thread): except OSError as exc: print_warning(f"Failed to kill {what}: {exc!r}") - def stop(self): + def stop(self) -> None: # Method called from a different thread to stop this thread self._stopped = True self._kill() - def mp_result_error(self, test_name, error_type, stdout='', stderr='', - err_msg=None): - test_time = time.monotonic() - self.start_time - result = TestResult(test_name, error_type, test_time, None) - return MultiprocessResult(result, stdout, stderr, err_msg) - - def _run_process(self, test_name): + def mp_result_error( + self, + test_result: TestResult, + stdout: str = '', + stderr: str = '', + err_msg=None + ) -> MultiprocessResult: + test_result.duration_sec = time.monotonic() - self.start_time + return MultiprocessResult(test_result, stdout, stderr, err_msg) + + def _run_process(self, test_name: str) -> tuple[int, str, str]: self.start_time = time.monotonic() self.current_test_name = test_name @@ -245,11 +261,11 @@ class TestWorkerProcess(threading.Thread): self._popen = None self.current_test_name = None - def _runtest(self, test_name): + def _runtest(self, test_name: str) -> MultiprocessResult: retcode, stdout, stderr = self._run_process(test_name) if retcode is None: - return self.mp_result_error(test_name, TIMEOUT, stdout, stderr) + return self.mp_result_error(Timeout(test_name), stdout, stderr) err_msg = None if retcode != 0: @@ -262,18 +278,17 @@ class TestWorkerProcess(threading.Thread): else: try: # deserialize run_tests_worker() output - result = json.loads(result) - result = TestResult(*result) + result = json.loads(result, object_hook=decode_test_result) except Exception as exc: err_msg = "Failed to parse worker JSON: %s" % exc if err_msg is not None: - return self.mp_result_error(test_name, CHILD_ERROR, + return self.mp_result_error(ChildError(test_name), stdout, stderr, err_msg) return MultiprocessResult(result, stdout, stderr, err_msg) - def run(self): + def run(self) -> None: while not self._stopped: try: try: @@ -292,7 +307,7 @@ class TestWorkerProcess(threading.Thread): self.output.put((True, traceback.format_exc())) break - def _wait_completed(self): + def _wait_completed(self) -> None: popen = self._popen # stdout and stderr must be closed to ensure that communicate() @@ -307,7 +322,7 @@ class TestWorkerProcess(threading.Thread): f"(timeout={format_duration(JOIN_TIMEOUT)}): " f"{exc!r}") - def wait_stopped(self, start_time): + def wait_stopped(self, start_time: float) -> None: # bpo-38207: MultiprocessTestRunner.stop_workers() called self.stop() # which killed the process. Sometimes, killing the process from the # main thread does not interrupt popen.communicate() in @@ -331,7 +346,7 @@ class TestWorkerProcess(threading.Thread): break -def get_running(workers): +def get_running(workers: list[TestWorkerProcess]) -> list[TestWorkerProcess]: running = [] for worker in workers: current_test_name = worker.current_test_name @@ -345,11 +360,11 @@ def get_running(workers): class MultiprocessTestRunner: - def __init__(self, regrtest): + def __init__(self, regrtest: Regrtest) -> None: self.regrtest = regrtest self.log = self.regrtest.log self.ns = regrtest.ns - self.output = queue.Queue() + self.output: queue.Queue[QueueOutput] = queue.Queue() self.pending = MultiprocessIterator(self.regrtest.tests) if self.ns.timeout is not None: # Rely on faulthandler to kill a worker process. This timouet is @@ -361,7 +376,7 @@ class MultiprocessTestRunner: self.worker_timeout = None self.workers = None - def start_workers(self): + def start_workers(self) -> None: self.workers = [TestWorkerProcess(index, self) for index in range(1, self.ns.use_mp + 1)] msg = f"Run tests in parallel using {len(self.workers)} child processes" @@ -373,14 +388,14 @@ class MultiprocessTestRunner: for worker in self.workers: worker.start() - def stop_workers(self): + def stop_workers(self) -> None: start_time = time.monotonic() for worker in self.workers: worker.stop() for worker in self.workers: worker.wait_stopped(start_time) - def _get_result(self): + def _get_result(self) -> QueueOutput | None: if not any(worker.is_alive() for worker in self.workers): # all worker threads are done: consume pending results try: @@ -406,21 +421,22 @@ class MultiprocessTestRunner: if running and not self.ns.pgo: self.log('running: %s' % ', '.join(running)) - def display_result(self, mp_result): + def display_result(self, mp_result: MultiprocessResult) -> None: result = mp_result.result - text = format_test_result(result) + text = str(result) if mp_result.error_msg is not None: # CHILD_ERROR text += ' (%s)' % mp_result.error_msg - elif (result.test_time >= PROGRESS_MIN_TIME and not self.ns.pgo): - text += ' (%s)' % format_duration(result.test_time) + elif (result.duration_sec >= PROGRESS_MIN_TIME and not self.ns.pgo): + text += ' (%s)' % format_duration(result.duration_sec) running = get_running(self.workers) if running and not self.ns.pgo: text += ' -- running: %s' % ', '.join(running) self.regrtest.display_progress(self.test_index, text) - def _process_result(self, item): + def _process_result(self, item: QueueOutput) -> bool: + """Returns True if test runner must stop.""" if item[0]: # Thread got an exception format_exc = item[1] @@ -442,7 +458,7 @@ class MultiprocessTestRunner: return False - def run_tests(self): + def run_tests(self) -> None: self.start_workers() self.test_index = 0 @@ -468,5 +484,41 @@ class MultiprocessTestRunner: self.stop_workers() -def run_tests_multiprocess(regrtest): +def run_tests_multiprocess(regrtest: Regrtest) -> None: MultiprocessTestRunner(regrtest).run_tests() + + +class EncodeTestResult(json.JSONEncoder): + """Encode a TestResult (sub)class object into a JSON dict.""" + + def default(self, o: Any) -> dict[str, Any]: + if isinstance(o, TestResult): + result = vars(o) + result["__test_result__"] = o.__class__.__name__ + return result + + return super().default(o) + + +def decode_test_result(d: dict[str, Any]) -> TestResult | dict[str, Any]: + """Decode a TestResult (sub)class object from a JSON dict.""" + + if "__test_result__" not in d: + return d + + cls_name = d.pop("__test_result__") + for cls in get_all_test_result_classes(): + if cls.__name__ == cls_name: + return cls(**d) + + +def get_all_test_result_classes() -> set[type[TestResult]]: + prev_count = 0 + classes = {TestResult} + while len(classes) > prev_count: + prev_count = len(classes) + to_add = [] + for cls in classes: + to_add.extend(cls.__subclasses__()) + classes.update(to_add) + return classes diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index b3975254..4b378a7d 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -3,6 +3,7 @@ Various tests for synchronization primitives. """ import os +import gc import sys import time from _thread import start_new_thread, TIMEOUT_MAX @@ -220,6 +221,7 @@ class BaseLockTests(BaseTestCase): lock = self.locktype() ref = weakref.ref(lock) del lock + gc.collect() # For PyPy or other GCs. self.assertIsNone(ref()) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 229e1ac0..7e5a29b1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -121,6 +121,17 @@ class Error(Exception): class TestFailed(Error): """Test failed.""" +class TestFailedWithDetails(TestFailed): + """Test failed.""" + def __init__(self, msg, errors, failures): + self.msg = msg + self.errors = errors + self.failures = failures + super().__init__(msg, errors, failures) + + def __str__(self): + return self.msg + class TestDidNotRun(Error): """Test did not run any subtests.""" @@ -975,6 +986,25 @@ TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR) # TEST_DATA_DIR is used as a target download location for remote resources TEST_DATA_DIR = os.path.join(TEST_HOME_DIR, "data") + +def darwin_malloc_err_warning(test_name): + """Assure user that loud errors generated by macOS libc's malloc are + expected.""" + if sys.platform != 'darwin': + return + + import shutil + msg = ' NOTICE ' + detail = (f'{test_name} may generate "malloc can\'t allocate region"\n' + 'warnings on macOS systems. This behavior is known. Do not\n' + 'report a bug unless tests are also failing. See bpo-40928.') + + padding, _ = shutil.get_terminal_size() + print(msg.center(padding, '-')) + print(detail) + print('-' * padding) + + def findfile(filename, subdir=None): """Try to find a file on sys.path or in the test directory. If it is not found the argument passed to the function is returned (this does not @@ -1808,7 +1838,9 @@ def _run_suite(suite): else: err = "multiple errors occurred" if not verbose: err += "; run in verbose mode for details" - raise TestFailed(err) + errors = [(str(tc), exc_str) for tc, exc_str in result.errors] + failures = [(str(tc), exc_str) for tc, exc_str in result.failures] + raise TestFailedWithDetails(err, errors, failures) # By default, don't filter tests @@ -3209,3 +3241,32 @@ def infinite_recursion(max_depth=75): yield finally: sys.setrecursionlimit(original_depth) + +def ignore_deprecations_from(module: str, *, like: str) -> object: + token = object() + warnings.filterwarnings( + "ignore", + category=DeprecationWarning, + module=module, + message=like + fr"(?#support{id(token)})", + ) + return token + +def clear_ignored_deprecations(*tokens: object) -> None: + if not tokens: + raise ValueError("Provide token or tokens returned by ignore_deprecations_from") + + new_filters = [] + endswith = tuple(rf"(?#support{id(token)})" for token in tokens) + for action, message, category, module, lineno in warnings.filters: + if action == "ignore" and category is DeprecationWarning: + if isinstance(message, re.Pattern): + msg = message.pattern + else: + msg = message or "" + if msg.endswith(endswith): + continue + new_filters.append((action, message, category, module, lineno)) + if warnings.filters != new_filters: + warnings.filters[:] = new_filters + warnings._filters_mutated() diff --git a/Lib/test/support/warnings_helper.py b/Lib/test/support/warnings_helper.py new file mode 100644 index 00000000..a024fbe5 --- /dev/null +++ b/Lib/test/support/warnings_helper.py @@ -0,0 +1,199 @@ +import contextlib +import functools +import re +import sys +import warnings + + +def check_syntax_warning(testcase, statement, errtext='', + *, lineno=1, offset=None): + # Test also that a warning is emitted only once. + from test.support import check_syntax_error + with warnings.catch_warnings(record=True) as warns: + warnings.simplefilter('always', SyntaxWarning) + compile(statement, '', 'exec') + testcase.assertEqual(len(warns), 1, warns) + + warn, = warns + testcase.assertTrue(issubclass(warn.category, SyntaxWarning), + warn.category) + if errtext: + testcase.assertRegex(str(warn.message), errtext) + testcase.assertEqual(warn.filename, '') + testcase.assertIsNotNone(warn.lineno) + if lineno is not None: + testcase.assertEqual(warn.lineno, lineno) + + # SyntaxWarning should be converted to SyntaxError when raised, + # since the latter contains more information and provides better + # error report. + with warnings.catch_warnings(record=True) as warns: + warnings.simplefilter('error', SyntaxWarning) + check_syntax_error(testcase, statement, errtext, + lineno=lineno, offset=offset) + # No warnings are leaked when a SyntaxError is raised. + testcase.assertEqual(warns, []) + + +def ignore_warnings(*, category): + """Decorator to suppress deprecation warnings. + + Use of context managers to hide warnings make diffs + more noisy and tools like 'git blame' less useful. + """ + def decorator(test): + @functools.wraps(test) + def wrapper(self, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', category=category) + return test(self, *args, **kwargs) + return wrapper + return decorator + + +class WarningsRecorder(object): + """Convenience wrapper for the warnings list returned on + entry to the warnings.catch_warnings() context manager. + """ + def __init__(self, warnings_list): + self._warnings = warnings_list + self._last = 0 + + def __getattr__(self, attr): + if len(self._warnings) > self._last: + return getattr(self._warnings[-1], attr) + elif attr in warnings.WarningMessage._WARNING_DETAILS: + return None + raise AttributeError("%r has no attribute %r" % (self, attr)) + + @property + def warnings(self): + return self._warnings[self._last:] + + def reset(self): + self._last = len(self._warnings) + + +@contextlib.contextmanager +def check_warnings(*filters, **kwargs): + """Context manager to silence warnings. + + Accept 2-tuples as positional arguments: + ("message regexp", WarningCategory) + + Optional argument: + - if 'quiet' is True, it does not fail if a filter catches nothing + (default True without argument, + default False if some filters are defined) + + Without argument, it defaults to: + check_warnings(("", Warning), quiet=True) + """ + quiet = kwargs.get('quiet') + if not filters: + filters = (("", Warning),) + # Preserve backward compatibility + if quiet is None: + quiet = True + return _filterwarnings(filters, quiet) + + +@contextlib.contextmanager +def check_no_warnings(testcase, message='', category=Warning, force_gc=False): + """Context manager to check that no warnings are emitted. + + This context manager enables a given warning within its scope + and checks that no warnings are emitted even with that warning + enabled. + + If force_gc is True, a garbage collection is attempted before checking + for warnings. This may help to catch warnings emitted when objects + are deleted, such as ResourceWarning. + + Other keyword arguments are passed to warnings.filterwarnings(). + """ + from test.support import gc_collect + with warnings.catch_warnings(record=True) as warns: + warnings.filterwarnings('always', + message=message, + category=category) + yield + if force_gc: + gc_collect() + testcase.assertEqual(warns, []) + + +@contextlib.contextmanager +def check_no_resource_warning(testcase): + """Context manager to check that no ResourceWarning is emitted. + + Usage: + + with check_no_resource_warning(self): + f = open(...) + ... + del f + + You must remove the object which may emit ResourceWarning before + the end of the context manager. + """ + with check_no_warnings(testcase, category=ResourceWarning, force_gc=True): + yield + + +def _filterwarnings(filters, quiet=False): + """Catch the warnings, then check if all the expected + warnings have been raised and re-raise unexpected warnings. + If 'quiet' is True, only re-raise the unexpected warnings. + """ + # Clear the warning registry of the calling module + # in order to re-raise the warnings. + frame = sys._getframe(2) + registry = frame.f_globals.get('__warningregistry__') + if registry: + registry.clear() + with warnings.catch_warnings(record=True) as w: + # Set filter "always" to record all warnings. Because + # test_warnings swap the module, we need to look up in + # the sys.modules dictionary. + sys.modules['warnings'].simplefilter("always") + yield WarningsRecorder(w) + # Filter the recorded warnings + reraise = list(w) + missing = [] + for msg, cat in filters: + seen = False + for w in reraise[:]: + warning = w.message + # Filter out the matching messages + if (re.match(msg, str(warning), re.I) and + issubclass(warning.__class__, cat)): + seen = True + reraise.remove(w) + if not seen and not quiet: + # This filter caught nothing + missing.append((msg, cat.__name__)) + if reraise: + raise AssertionError("unhandled warning %s" % reraise[0]) + if missing: + raise AssertionError("filter (%r, %s) did not catch any warning" % + missing[0]) + + +@contextlib.contextmanager +def save_restore_warnings_filters(): + old_filters = warnings.filters[:] + try: + yield + finally: + warnings.filters[:] = old_filters + + +def _warn_about_deprecation(): + warnings.warn( + "This is used in test_support test to ensure" + " support.ignore_deprecations_from() works as expected." + " You should not be seeing this.", + DeprecationWarning, + stacklevel=0, + ) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 7aec021f..cbfa8818 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -38,6 +38,20 @@ def _run_output(interp, request, shared=None): return rpipe.read() +def _wait_for_interp_to_run(interp, timeout=None): + # bpo-37224: Running this test file in multiprocesses will fail randomly. + # The failure reason is that the thread can't acquire the cpu to + # run subinterpreter eariler than the main thread in multiprocess. + if timeout is None: + timeout = support.SHORT_TIMEOUT + start_time = time.monotonic() + deadline = start_time + timeout + while not interpreters.is_running(interp): + if time.monotonic() > deadline: + raise RuntimeError('interp is not running') + time.sleep(0.010) + + @contextlib.contextmanager def _running(interp): r, w = os.pipe() @@ -50,6 +64,7 @@ def _running(interp): t = threading.Thread(target=run) t.start() + _wait_for_interp_to_run(interp) yield diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 22cae626..5d8bc142 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2057,6 +2057,30 @@ class TestAddSubparsers(TestCase): ret = parser.parse_args(()) self.assertIsNone(ret.command) + def test_required_subparsers_no_destination_error(self): + parser = ErrorRaisingArgumentParser() + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(()) + self.assertRegex( + excinfo.exception.stderr, + 'error: the following arguments are required: {foo,bar}\n$' + ) + + def test_wrong_argument_subparsers_no_destination_error(self): + parser = ErrorRaisingArgumentParser() + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertRegex( + excinfo.exception.stderr, + r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from 'foo', 'bar'\)\n$" + ) + def test_optional_subparsers(self): parser = ErrorRaisingArgumentParser() subparsers = parser.add_subparsers(dest='command', required=False) @@ -4235,6 +4259,9 @@ class TestHelpArgumentDefaults(HelpTestCase): argument_signatures = [ Sig('--foo', help='foo help - oh and by the way, %(default)s'), Sig('--bar', action='store_true', help='bar help'), + Sig('--taz', action=argparse.BooleanOptionalAction, + help='Whether to taz it', default=True), + Sig('--quux', help="Set the quux", default=42), Sig('spam', help='spam help'), Sig('badger', nargs='?', default='wooden', help='badger help'), ] @@ -4243,25 +4270,29 @@ class TestHelpArgumentDefaults(HelpTestCase): [Sig('--baz', type=int, default=42, help='baz help')]), ] usage = '''\ - usage: PROG [-h] [--foo FOO] [--bar] [--baz BAZ] spam [badger] + usage: PROG [-h] [--foo FOO] [--bar] [--taz | --no-taz] [--quux QUUX] + [--baz BAZ] + spam [badger] ''' help = usage + '''\ description positional arguments: - spam spam help - badger badger help (default: wooden) + spam spam help + badger badger help (default: wooden) optional arguments: - -h, --help show this help message and exit - --foo FOO foo help - oh and by the way, None - --bar bar help (default: False) + -h, --help show this help message and exit + --foo FOO foo help - oh and by the way, None + --bar bar help (default: False) + --taz, --no-taz Whether to taz it (default: True) + --quux QUUX Set the quux (default: 42) title: description - --baz BAZ baz help (default: 42) + --baz BAZ baz help (default: 42) ''' version = '' diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py old mode 100644 new mode 100755 index f731b704..05d638a1 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1006,6 +1006,7 @@ class BaseTest: p = weakref.proxy(s) self.assertEqual(p.tobytes(), s.tobytes()) s = None + support.gc_collect() # For PyPy or other GCs. self.assertRaises(ReferenceError, len, p) @unittest.skipUnless(hasattr(sys, 'getrefcount'), diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 62bf8774..8d2ad223 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -3,6 +3,7 @@ import types import unittest from test.support import import_module +from test.support import gc_collect asyncio = import_module("asyncio") @@ -659,6 +660,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): await g.__anext__() await g.__anext__() del g + gc_collect() # For PyPy or other GCs. await asyncio.sleep(0.1) diff --git a/Lib/test/test_asyncio/__init__.py b/Lib/test/test_asyncio/__init__.py index c77c7a81..4a7a8689 100644 --- a/Lib/test/test_asyncio/__init__.py +++ b/Lib/test/test_asyncio/__init__.py @@ -1,8 +1,38 @@ import os -from test.support import load_package_tests, import_module +from test import support +import unittest # Skip tests if we don't have concurrent.futures. -import_module('concurrent.futures') +support.import_module('concurrent.futures') -def load_tests(*args): - return load_package_tests(os.path.dirname(__file__), *args) + +def load_tests(loader, _, pattern): + pkg_dir = os.path.dirname(__file__) + suite = AsyncioTestSuite() + return support.load_package_tests(pkg_dir, loader, suite, pattern) + + +class AsyncioTestSuite(unittest.TestSuite): + """A custom test suite that also runs setup/teardown for the whole package. + + Normally unittest only runs setUpModule() and tearDownModule() within each + test module part of the test suite. Copying those functions to each file + would be tedious, let's run this once and for all. + """ + def run(self, result, debug=False): + ignore = support.ignore_deprecations_from + tokens = { + ignore("asyncio.base_events", like=r".*loop argument.*"), + ignore("asyncio.unix_events", like=r".*loop argument.*"), + ignore("asyncio.futures", like=r".*loop argument.*"), + ignore("asyncio.runners", like=r".*loop argument.*"), + ignore("asyncio.subprocess", like=r".*loop argument.*"), + ignore("asyncio.tasks", like=r".*loop argument.*"), + ignore("test.test_asyncio.test_events", like=r".*loop argument.*"), + ignore("test.test_asyncio.test_queues", like=r".*loop argument.*"), + ignore("test.test_asyncio.test_tasks", like=r".*loop argument.*"), + } + try: + super().run(result, debug=debug) + finally: + support.clear_ignored_deprecations(*tokens) diff --git a/Lib/test/test_asyncio/functional.py b/Lib/test/test_asyncio/functional.py index 5cd06593..70bc4089 100644 --- a/Lib/test/test_asyncio/functional.py +++ b/Lib/test/test_asyncio/functional.py @@ -29,10 +29,6 @@ class FunctionalTestCaseMixin: self.loop.set_exception_handler(self.loop_exception_handler) self.__unhandled_exceptions = [] - # Disable `_get_running_loop`. - self._old_get_running_loop = asyncio.events._get_running_loop - asyncio.events._get_running_loop = lambda: None - def tearDown(self): try: self.loop.close() @@ -43,7 +39,6 @@ class FunctionalTestCaseMixin: self.fail('unexpected calls to loop.call_exception_handler()') finally: - asyncio.events._get_running_loop = self._old_get_running_loop asyncio.set_event_loop(None) self.loop = None diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index fa6c49d8..7114c2f0 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1968,10 +1968,11 @@ class SubprocessTestsMixin: functools.partial(MySubprocessProtocol, self.loop), 'exit 7', stdin=None, stdout=None, stderr=None, start_new_session=True) - _, proto = yield self.loop.run_until_complete(connect) + transp, proto = self.loop.run_until_complete(connect) self.assertIsInstance(proto, MySubprocessProtocol) self.loop.run_until_complete(proto.completed) self.assertEqual(7, proto.returncode) + transp.close() def test_subprocess_exec_invalid_args(self): async def connect(**kwds): diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index ec00896c..c16bfc08 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -876,6 +876,8 @@ class PyFutureInheritanceTests(BaseFutureInheritanceTests, return futures._PyFuture +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') class CFutureInheritanceTests(BaseFutureInheritanceTests, test_utils.TestCase): def _get_future_cls(self): diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 948820c8..bc01a8bb 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -278,6 +278,7 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin): # No garbage is left if SSL is closed uncleanly client_context = weakref.ref(client_context) + support.gc_collect() self.assertIsNone(client_context()) def test_create_connection_memory_leak(self): @@ -341,6 +342,7 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin): # No garbage is left for SSL client from loop.create_connection, even # if user stores the SSLTransport in corresponding protocol instance client_context = weakref.ref(client_context) + support.gc_collect() self.assertIsNone(client_context()) def test_start_tls_client_buf_proto_1(self): @@ -640,6 +642,7 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin): # The 10s handshake timeout should be cancelled to free related # objects without really waiting for 10s client_sslctx = weakref.ref(client_sslctx) + support.gc_collect() self.assertIsNone(client_sslctx()) def test_create_connection_ssl_slow_handshake(self): diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 01f62b7f..d2d78033 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2611,6 +2611,7 @@ class BaseTaskTests: self.new_task(self.loop, gen) finally: gen.close() + gc.collect() # For PyPy or other GCs. self.assertTrue(m_log.error.called) message = m_log.error.call_args[0][0] @@ -3196,15 +3197,18 @@ class GenericTaskTests(test_utils.TestCase): def test_future_subclass(self): self.assertTrue(issubclass(asyncio.Task, asyncio.Future)) + @support.cpython_only def test_asyncio_module_compiled(self): # Because of circular imports it's easy to make _asyncio # module non-importable. This is a simple test that will # fail on systems where C modules were successfully compiled - # (hence the test for _functools), but _asyncio somehow didn't. + # (hence the test for _functools etc), but _asyncio somehow didn't. try: import _functools + import _json + import _pickle except ImportError: - pass + self.skipTest('C modules are not available') else: try: import _asyncio diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index a9ac6fee..387a3122 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -51,6 +51,11 @@ class AuditTest(unittest.TestCase): def test_block_add_hook_baseexception(self): self.do_test("test_block_add_hook_baseexception") + def test_marshal(self): + support.import_module("marshal") + + self.do_test("test_marshal") + def test_pickle(self): support.import_module("pickle") diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index b21c3acf..0cbe8396 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -537,6 +537,18 @@ class CAPITest(unittest.TestCase): self.assertRaises(TypeError, pynumber_tobase, '123', 10) self.assertRaises(SystemError, pynumber_tobase, 123, 0) + def test_pyobject_repr_from_null(self): + s = _testcapi.pyobject_repr_from_null() + self.assertEqual(s, '') + + def test_pyobject_str_from_null(self): + s = _testcapi.pyobject_str_from_null() + self.assertEqual(s, '') + + def test_pyobject_bytes_from_null(self): + s = _testcapi.pyobject_bytes_from_null() + self.assertEqual(s, b'') + class TestPendingCalls(unittest.TestCase): diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index ac3dde74..c696ede8 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -135,7 +135,7 @@ try: except ImportError: ctypes = None from test.support import (run_doctest, run_unittest, cpython_only, - check_impl_detail) + check_impl_detail, gc_collect) def consts(t): @@ -213,7 +213,7 @@ class CodeTest(unittest.TestCase): CodeType = type(co) # test code constructor - return CodeType(co.co_argcount, + CodeType(co.co_argcount, co.co_posonlyargcount, co.co_kwonlyargcount, co.co_nlocals, @@ -337,6 +337,7 @@ class CodeWeakRefTest(unittest.TestCase): coderef = weakref.ref(f.__code__, callback) self.assertTrue(bool(coderef())) del f + gc_collect() # For PyPy or other GCs. self.assertFalse(bool(coderef())) self.assertTrue(self.called) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 6a819358..4ae9748e 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1791,6 +1791,18 @@ class TestCollectionABCs(ABCTestCase): self.assertTrue(f1 != l1) self.assertTrue(f1 != l2) + def test_Set_hash_matches_frozenset(self): + sets = [ + {}, {1}, {None}, {-1}, {0.0}, {"abc"}, {1, 2, 3}, + {10**100, 10**101}, {"a", "b", "ab", ""}, {False, True}, + {object(), object(), object()}, {float("nan")}, {frozenset()}, + {*range(1000)}, {*range(1000)} - {100, 200, 300}, + {*range(sys.maxsize - 10, sys.maxsize + 10)}, + ] + for s in sets: + fs = frozenset(s) + self.assertEqual(hash(fs), Set._hash(fs), msg=s) + def test_Mapping(self): for sample in [dict]: self.assertIsInstance(sample(), Mapping) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 3bbc6817..a03735d3 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -75,9 +75,28 @@ class CompileallTestsBase: with open(self.bc_path, 'rb') as file: data = file.read(12) mtime = int(os.stat(self.source_path).st_mtime) - compare = struct.pack('<4sll', importlib.util.MAGIC_NUMBER, 0, mtime) + compare = struct.pack('<4sLL', importlib.util.MAGIC_NUMBER, 0, + mtime & 0xFFFF_FFFF) return data, compare + def test_year_2038_mtime_compilation(self): + # Test to make sure we can handle mtimes larger than what a 32-bit + # signed number can hold as part of bpo-34990 + try: + os.utime(self.source_path, (2**32 - 1, 2**32 - 1)) + except (OverflowError, OSError): + self.skipTest("filesystem doesn't support timestamps near 2**32") + self.assertTrue(compileall.compile_file(self.source_path)) + + def test_larger_than_32_bit_times(self): + # This is similar to the test above but we skip it if the OS doesn't + # support modification times larger than 32-bits. + try: + os.utime(self.source_path, (2**35, 2**35)) + except (OverflowError, OSError): + self.skipTest("filesystem doesn't support large timestamps") + self.assertTrue(compileall.compile_file(self.source_path)) + def recreation_check(self, metadata): """Check that compileall recreates bytecode when the new metadata is used.""" @@ -96,7 +115,7 @@ class CompileallTestsBase: def test_mtime(self): # Test a change in mtime leads to a new .pyc. - self.recreation_check(struct.pack('<4sll', importlib.util.MAGIC_NUMBER, + self.recreation_check(struct.pack('<4sLL', importlib.util.MAGIC_NUMBER, 0, 1)) def test_magic_number(self): @@ -164,6 +183,14 @@ class CompileallTestsBase: compileall.compile_file(data_file) self.assertFalse(os.path.exists(os.path.join(data_dir, '__pycache__'))) + + def test_compile_file_encoding_fallback(self): + # Bug 44666 reported that compile_file failed when sys.stdout.encoding is None + self.add_bad_source_file() + with contextlib.redirect_stdout(io.StringIO()): + self.assertFalse(compileall.compile_file(self.bad_source_path)) + + def test_optimize(self): # make sure compiling with different optimization settings than the # interpreter's creates the correct file names diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index dee5c7fa..0db9be51 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -1,4 +1,5 @@ import unittest +import sys from test import support from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) @@ -206,6 +207,54 @@ class ComplexTest(unittest.TestCase): b = 5.1+2.3j self.assertRaises(ValueError, pow, a, b, 0) + # Check some boundary conditions; some of these used to invoke + # undefined behaviour (https://bugs.python.org/issue44698). We're + # not actually checking the results of these operations, just making + # sure they don't crash (for example when using clang's + # UndefinedBehaviourSanitizer). + values = (sys.maxsize, sys.maxsize+1, sys.maxsize-1, + -sys.maxsize, -sys.maxsize+1, -sys.maxsize+1) + for real in values: + for imag in values: + with self.subTest(real=real, imag=imag): + c = complex(real, imag) + try: + c ** real + except OverflowError: + pass + try: + c ** c + except OverflowError: + pass + + def test_pow_with_small_integer_exponents(self): + # Check that small integer exponents are handled identically + # regardless of their type. + values = [ + complex(5.0, 12.0), + complex(5.0e100, 12.0e100), + complex(-4.0, INF), + complex(INF, 0.0), + ] + exponents = [-19, -5, -3, -2, -1, 0, 1, 2, 3, 5, 19] + for value in values: + for exponent in exponents: + with self.subTest(value=value, exponent=exponent): + try: + int_pow = value**exponent + except OverflowError: + int_pow = "overflow" + try: + float_pow = value**float(exponent) + except OverflowError: + float_pow = "overflow" + try: + complex_pow = value**complex(exponent) + except OverflowError: + complex_pow = "overflow" + self.assertEqual(str(float_pow), str(int_pow)) + self.assertEqual(str(complex_pow), str(int_pow)) + def test_boolcontext(self): for i in range(100): self.assertTrue(complex(random() + 1e-6, random() + 1e-6)) diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 4fe4bf79..ad64f4bd 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -448,6 +448,7 @@ class ThreadPoolShutdownTest(ThreadPoolMixin, ExecutorShutdownTest, BaseTestCase executor.map(abs, range(-5, 5)) threads = executor._threads del executor + support.gc_collect() # For PyPy or other GCs. for t in threads: self.assertRegex(t.name, r'^SpecialPool_[0-4]$') @@ -458,6 +459,7 @@ class ThreadPoolShutdownTest(ThreadPoolMixin, ExecutorShutdownTest, BaseTestCase executor.map(abs, range(-5, 5)) threads = executor._threads del executor + support.gc_collect() # For PyPy or other GCs. for t in threads: # Ensure that our default name is reasonably sane and unique when @@ -520,6 +522,7 @@ class ProcessPoolShutdownTest(ExecutorShutdownTest): call_queue = executor._call_queue executor_manager_thread = executor._executor_manager_thread del executor + support.gc_collect() # For PyPy or other GCs. # Make sure that all the executor resources were properly cleaned by # the shutdown process @@ -744,6 +747,7 @@ class AsCompletedTests: futures_list.remove(future) wr = weakref.ref(future) del future + support.gc_collect() # For PyPy or other GCs. self.assertIsNone(wr()) futures_list[0].set_result("test") @@ -751,6 +755,7 @@ class AsCompletedTests: futures_list.remove(future) wr = weakref.ref(future) del future + support.gc_collect() # For PyPy or other GCs. self.assertIsNone(wr()) if futures_list: futures_list[0].set_result("test") @@ -850,6 +855,7 @@ class ExecutorTest: for obj in self.executor.map(make_dummy_object, range(10)): wr = weakref.ref(obj) del obj + support.gc_collect() # For PyPy or other GCs. self.assertIsNone(wr()) diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 024f9126..ad071bb2 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -125,19 +125,22 @@ class ContextManagerTestCase(unittest.TestCase): self.assertEqual(state, [1, 42, 999]) def test_contextmanager_except_stopiter(self): - stop_exc = StopIteration('spam') @contextmanager def woohoo(): yield - try: - with self.assertWarnsRegex(DeprecationWarning, - "StopIteration"): - with woohoo(): - raise stop_exc - except Exception as ex: - self.assertIs(ex, stop_exc) - else: - self.fail('StopIteration was suppressed') + + class StopIterationSubclass(StopIteration): + pass + + for stop_exc in (StopIteration('spam'), StopIterationSubclass('spam')): + with self.subTest(type=type(stop_exc)): + try: + with woohoo(): + raise stop_exc + except Exception as ex: + self.assertIs(ex, stop_exc) + else: + self.fail(f'{stop_exc} was suppressed') def test_contextmanager_except_pep479(self): code = """\ diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 43fb7fce..9d6854c7 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -207,7 +207,18 @@ class AsyncContextManagerTestCase(unittest.TestCase): async def woohoo(): yield - for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')): + class StopIterationSubclass(StopIteration): + pass + + class StopAsyncIterationSubclass(StopAsyncIteration): + pass + + for stop_exc in ( + StopIteration('spam'), + StopAsyncIteration('ham'), + StopIterationSubclass('spam'), + StopAsyncIterationSubclass('spam') + ): with self.subTest(type=type(stop_exc)): try: async with woohoo(): diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 35f72fb2..4db2f39d 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -7,6 +7,7 @@ import abc from operator import le, lt, ge, gt, eq, ne import unittest +from test import support order_comparisons = le, lt, ge, gt equality_comparisons = eq, ne @@ -816,6 +817,7 @@ class TestCopy(unittest.TestCase): self.assertEqual(v[c], d) self.assertEqual(len(v), 2) del c, d + support.gc_collect() # For PyPy or other GCs. self.assertEqual(len(v), 1) x, y = C(), C() # The underlying containers are decoupled @@ -845,6 +847,7 @@ class TestCopy(unittest.TestCase): self.assertEqual(v[a].i, b.i) self.assertEqual(v[c].i, d.i) del c + support.gc_collect() # For PyPy or other GCs. self.assertEqual(len(v), 1) def test_deepcopy_weakvaluedict(self): @@ -868,6 +871,7 @@ class TestCopy(unittest.TestCase): self.assertIs(t, d) del x, y, z, t del d + support.gc_collect() # For PyPy or other GCs. self.assertEqual(len(v), 1) def test_deepcopy_bound_method(self): diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 2dab17dc..89fec4b2 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -442,9 +442,15 @@ class TestDialectRegistry(unittest.TestCase): class testUni(csv.excel): delimiter = "\u039B" + class unspecified(): + # A class to pass as dialect but with no dialect attributes. + pass + csv.register_dialect('testC', testC) try: self.compare_dialect_123("1,2,3\r\n") + self.compare_dialect_123("1,2,3\r\n", dialect=None) + self.compare_dialect_123("1,2,3\r\n", dialect=unspecified) self.compare_dialect_123("1\t2\t3\r\n", testA) self.compare_dialect_123("1:2:3\r\n", dialect=testB()) self.compare_dialect_123("1|2|3\r\n", dialect='testC') diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index fd62f37d..d37c53da 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -35,12 +35,17 @@ import locale from test.support import (run_unittest, run_doctest, is_resource_enabled, requires_IEEE_754, requires_docstrings) from test.support import (import_fresh_module, TestFailed, - run_with_locale, cpython_only) + run_with_locale, cpython_only, + darwin_malloc_err_warning) import random import inspect import threading +if sys.platform == 'darwin': + darwin_malloc_err_warning('test_decimal') + + C = import_fresh_module('decimal', fresh=['_decimal']) P = import_fresh_module('decimal', blocked=['_decimal']) orig_sys_decimal = sys.modules['decimal'] diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 67c0ace3..f913c76f 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -902,6 +902,7 @@ class TestSubclass(unittest.TestCase): p = weakref.proxy(d) self.assertEqual(str(p), str(d)) d = None + support.gc_collect() # For PyPy or other GCs. self.assertRaises(ReferenceError, str, p) def test_strange_subclass(self): diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py index 7d3e30cb..77252502 100644 --- a/Lib/test/test_docxmlrpc.py +++ b/Lib/test/test_docxmlrpc.py @@ -133,7 +133,7 @@ class DocXMLRPCHTTPGETServer(unittest.TestCase): self.assertIn( (b'
add(x, y)
' b'Add two instances together. This ' - b'follows ' + b'follows ' b'PEP008, but has nothing
\nto do ' b'with ' b'RFC1952. Case should matter: pEp008 ' diff --git a/Lib/test/test_email/test_contentmanager.py b/Lib/test/test_email/test_contentmanager.py index f4f6bb71..694cef4b 100644 --- a/Lib/test/test_email/test_contentmanager.py +++ b/Lib/test/test_email/test_contentmanager.py @@ -776,6 +776,18 @@ class TestRawDataManager(TestEmailBase): foo """).encode('ascii')) + def test_set_content_bytes_cte_7bit(self): + m = self._make_message() + m.set_content(b'ASCII-only message.\n', + maintype='application', subtype='octet-stream', cte='7bit') + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: application/octet-stream + Content-Transfer-Encoding: 7bit + MIME-Version: 1.0 + + ASCII-only message. + """)) + content_object_params = { 'text_plain': ('content', ()), 'text_html': ('content', ('html',)), diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index ab68cdd8..217a5b87 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -3002,6 +3002,8 @@ class TestMiscellaneous(TestEmailBase): def test_parsedate_returns_None_for_invalid_strings(self): self.assertIsNone(utils.parsedate('')) self.assertIsNone(utils.parsedate_tz('')) + self.assertIsNone(utils.parsedate(' ')) + self.assertIsNone(utils.parsedate_tz(' ')) self.assertIsNone(utils.parsedate('0')) self.assertIsNone(utils.parsedate_tz('0')) self.assertIsNone(utils.parsedate('A Complete Waste of Time')) @@ -3262,6 +3264,11 @@ Foo addrs = utils.getaddresses(['User ((nested comment)) ']) eq(addrs[0][1], 'foo@bar.com') + def test_getaddresses_header_obj(self): + """Test the handling of a Header object.""" + addrs = utils.getaddresses([Header('Al Person ')]) + self.assertEqual(addrs[0][1], 'aperson@dom.ain') + def test_make_msgid_collisions(self): # Test make_msgid uniqueness, even with multiple threads class MsgidsThread(Thread): diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index 7aaf780c..920a3d6a 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -487,10 +487,14 @@ class TestEmailMessageBase: self.assertEqual(list(m.iter_attachments()), attachments) def message_as_iter_parts(self, body_parts, attachments, parts, msg): + def _is_multipart_msg(msg): + return 'Content-Type: multipart' in msg + m = self._str_msg(msg) allparts = list(m.walk()) parts = [allparts[n] for n in parts] - self.assertEqual(list(m.iter_parts()), parts) + iter_parts = list(m.iter_parts()) if _is_multipart_msg(msg) else [] + self.assertEqual(iter_parts, parts) class _TestContentManager: def get_content(self, msg, *args, **kw): @@ -923,6 +927,34 @@ class TestEmailMessage(TestEmailMessageBase, TestEmailBase): b'123456789-123456789\n 123456789 Hello ' b'=?utf-8?q?W=C3=B6rld!?= 123456789 123456789\n\n') + def test_get_body_malformed(self): + """test for bpo-42892""" + msg = textwrap.dedent("""\ + Message-ID: <674392CA.4347091@email.au> + Date: Wed, 08 Nov 2017 08:50:22 +0700 + From: Foo Bar + MIME-Version: 1.0 + To: email@email.com + Subject: Python Email + Content-Type: multipart/mixed; + boundary="------------879045806563892972123996" + X-Global-filter:Messagescannedforspamandviruses:passedalltests + + This is a multi-part message in MIME format. + --------------879045806563892972123996 + Content-Type: text/plain; charset=ISO-8859-1; format=flowed + Content-Transfer-Encoding: 7bit + + Your message is ready to be sent with the following file or link + attachments: + XU89 - 08.11.2017 + """) + m = self._str_msg(msg) + # In bpo-42892, this would raise + # AttributeError: 'str' object has no attribute 'is_attachment' + m.get_body() + + class TestMIMEPart(TestEmailMessageBase, TestEmailBase): # Doing the full test run here may seem a bit redundant, since the two # classes are almost identical. But what if they drift apart? So we do diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 5e73044a..12bf2121 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -1934,6 +1934,7 @@ class TestEnum(unittest.TestCase): raise Exception('Exception not raised.') def test_missing_exceptions_reset(self): + import gc import weakref # class TestEnum(enum.Enum): @@ -1960,8 +1961,9 @@ class TestEnum(unittest.TestCase): class_2_ref = weakref.ref(Class2()) # # The exception raised by Enum creates a reference loop and thus - # Class2 instances will stick around until the next gargage collection + # Class2 instances will stick around until the next garbage collection # cycle, unlike Class1. + gc.collect() # For PyPy or other GCs. self.assertIs(class_1_ref(), None) self.assertIs(class_2_ref(), None) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 0f7b7f84..70e438d2 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -235,7 +235,8 @@ class ExceptionTests(unittest.TestCase): """, 9, 20) check("pass\npass\npass\n(1+)\npass\npass\npass", 4, 4) check("(1+)", 1, 4) - check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 0, -1) + check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 0, + 0 if support.use_old_parser() else -1) # Errors thrown by symtable.c check('x = [(yield i) for i in range(3)]', 1, 5) @@ -626,6 +627,7 @@ class ExceptionTests(unittest.TestCase): except MyException as e: pass obj = None + gc_collect() # For PyPy or other GCs. obj = wr() self.assertIsNone(obj) @@ -637,6 +639,7 @@ class ExceptionTests(unittest.TestCase): except MyException: pass obj = None + gc_collect() # For PyPy or other GCs. obj = wr() self.assertIsNone(obj) @@ -648,6 +651,7 @@ class ExceptionTests(unittest.TestCase): except: pass obj = None + gc_collect() # For PyPy or other GCs. obj = wr() self.assertIsNone(obj) @@ -660,6 +664,7 @@ class ExceptionTests(unittest.TestCase): except: break obj = None + gc_collect() # For PyPy or other GCs. obj = wr() self.assertIsNone(obj) @@ -678,6 +683,7 @@ class ExceptionTests(unittest.TestCase): # must clear the latter manually for our test to succeed. e.__context__ = None obj = None + gc_collect() # For PyPy or other GCs. obj = wr() # guarantee no ref cycles on CPython (don't gc_collect) if check_impl_detail(cpython=False): @@ -868,6 +874,7 @@ class ExceptionTests(unittest.TestCase): next(g) testfunc(g) g = obj = None + gc_collect() # For PyPy or other GCs. obj = wr() self.assertIsNone(obj) @@ -921,8 +928,151 @@ class ExceptionTests(unittest.TestCase): raise Exception(MyObject()) except: pass + gc_collect() # For PyPy or other GCs. self.assertEqual(e, (None, None, None)) + def test_raise_does_not_create_context_chain_cycle(self): + class A(Exception): + pass + class B(Exception): + pass + class C(Exception): + pass + + # Create a context chain: + # C -> B -> A + # Then raise A in context of C. + try: + try: + raise A + except A as a_: + a = a_ + try: + raise B + except B as b_: + b = b_ + try: + raise C + except C as c_: + c = c_ + self.assertIsInstance(a, A) + self.assertIsInstance(b, B) + self.assertIsInstance(c, C) + self.assertIsNone(a.__context__) + self.assertIs(b.__context__, a) + self.assertIs(c.__context__, b) + raise a + except A as e: + exc = e + + # Expect A -> C -> B, without cycle + self.assertIs(exc, a) + self.assertIs(a.__context__, c) + self.assertIs(c.__context__, b) + self.assertIsNone(b.__context__) + + def test_no_hang_on_context_chain_cycle1(self): + # See issue 25782. Cycle in context chain. + + def cycle(): + try: + raise ValueError(1) + except ValueError as ex: + ex.__context__ = ex + raise TypeError(2) + + try: + cycle() + except Exception as e: + exc = e + + self.assertIsInstance(exc, TypeError) + self.assertIsInstance(exc.__context__, ValueError) + self.assertIs(exc.__context__.__context__, exc.__context__) + + def test_no_hang_on_context_chain_cycle2(self): + # See issue 25782. Cycle at head of context chain. + + class A(Exception): + pass + class B(Exception): + pass + class C(Exception): + pass + + # Context cycle: + # +-----------+ + # V | + # C --> B --> A + with self.assertRaises(C) as cm: + try: + raise A() + except A as _a: + a = _a + try: + raise B() + except B as _b: + b = _b + try: + raise C() + except C as _c: + c = _c + a.__context__ = c + raise c + + self.assertIs(cm.exception, c) + # Verify the expected context chain cycle + self.assertIs(c.__context__, b) + self.assertIs(b.__context__, a) + self.assertIs(a.__context__, c) + + def test_no_hang_on_context_chain_cycle3(self): + # See issue 25782. Longer context chain with cycle. + + class A(Exception): + pass + class B(Exception): + pass + class C(Exception): + pass + class D(Exception): + pass + class E(Exception): + pass + + # Context cycle: + # +-----------+ + # V | + # E --> D --> C --> B --> A + with self.assertRaises(E) as cm: + try: + raise A() + except A as _a: + a = _a + try: + raise B() + except B as _b: + b = _b + try: + raise C() + except C as _c: + c = _c + a.__context__ = c + try: + raise D() + except D as _d: + d = _d + e = E() + raise e + + self.assertIs(cm.exception, e) + # Verify the expected context chain cycle + self.assertIs(e.__context__, d) + self.assertIs(d.__context__, c) + self.assertIs(c.__context__, b) + self.assertIs(b.__context__, a) + self.assertIs(a.__context__, c) + def test_unicode_change_attributes(self): # See issue 7309. This was a crasher. @@ -997,6 +1147,21 @@ class ExceptionTests(unittest.TestCase): self.assertIsInstance(v, RecursionError, type(v)) self.assertIn("maximum recursion depth exceeded", str(v)) + + @cpython_only + def test_trashcan_recursion(self): + # See bpo-33930 + + def foo(): + o = object() + for x in range(1_000_000): + # Create a big chain of method objects that will trigger + # a deep chain of calls when they need to be destructed. + o = o.__dir__ + + foo() + support.gc_collect() + @cpython_only def test_recursion_normalizing_exception(self): # Issue #22898. @@ -1177,6 +1342,7 @@ class ExceptionTests(unittest.TestCase): self.assertNotEqual(wr(), None) else: self.fail("MemoryError not raised") + gc_collect() # For PyPy or other GCs. self.assertEqual(wr(), None) @no_tracing @@ -1197,6 +1363,7 @@ class ExceptionTests(unittest.TestCase): self.assertNotEqual(wr(), None) else: self.fail("RecursionError not raised") + gc_collect() # For PyPy or other GCs. self.assertEqual(wr(), None) def test_errno_ENOTDIR(self): @@ -1217,6 +1384,7 @@ class ExceptionTests(unittest.TestCase): with support.catch_unraisable_exception() as cm: del obj + gc_collect() # For PyPy or other GCs. self.assertEqual(cm.unraisable.object, BrokenDel.__del__) self.assertIsNotNone(cm.unraisable.exc_traceback) diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index cd642e7a..155914e5 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -7,7 +7,7 @@ from weakref import proxy import io import _pyio as pyio -from test.support import TESTFN +from test.support import TESTFN, gc_collect from test import support from collections import UserList @@ -29,6 +29,7 @@ class AutoFileTests: self.assertEqual(self.f.tell(), p.tell()) self.f.close() self.f = None + gc_collect() # For PyPy or other GCs. self.assertRaises(ReferenceError, getattr, p, 'tell') def testAttributes(self): @@ -153,6 +154,22 @@ class OtherFileTests: f.close() self.fail('%r is an invalid file mode' % mode) + def testStdin(self): + if sys.platform == 'osf1V5': + # This causes the interpreter to exit on OSF1 v5.1. + self.skipTest( + ' sys.stdin.seek(-1) may crash the interpreter on OSF1.' + ' Test manually.') + + if not sys.stdin.isatty(): + # Issue 14853: stdin becomes seekable when redirected to a file + self.skipTest('stdin must be a TTY in this test') + + with self.assertRaises((IOError, ValueError)): + sys.stdin.seek(-1) + with self.assertRaises((IOError, ValueError)): + sys.stdin.truncate() + def testBadModeArgument(self): # verify that we get a sensible error message for bad mode argument bad_mode = "qwerty" diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 26e4500a..77c7f69a 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -10,7 +10,7 @@ from weakref import proxy from functools import wraps from test.support import (TESTFN, TESTFN_UNICODE, check_warnings, run_unittest, - make_bad_fd, cpython_only, swap_attr) + make_bad_fd, cpython_only, swap_attr, gc_collect) from collections import UserList import _io # C implementation of io @@ -35,6 +35,7 @@ class AutoFileTests: self.assertEqual(self.f.tell(), p.tell()) self.f.close() self.f = None + gc_collect() # For PyPy or other GCs. self.assertRaises(ReferenceError, getattr, p, 'tell') def testSeekTell(self): diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 4f66b41e..fcb87491 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -1446,6 +1446,20 @@ class HexFloatTestCase(unittest.TestCase): self.identical(fromHex('0X1.0000000000001fp0'), 1.0+2*EPS) self.identical(fromHex('0x1.00000000000020p0'), 1.0+2*EPS) + # Regression test for a corner-case bug reported in b.p.o. 44954 + self.identical(fromHex('0x.8p-1074'), 0.0) + self.identical(fromHex('0x.80p-1074'), 0.0) + self.identical(fromHex('0x.81p-1074'), TINY) + self.identical(fromHex('0x8p-1078'), 0.0) + self.identical(fromHex('0x8.0p-1078'), 0.0) + self.identical(fromHex('0x8.1p-1078'), TINY) + self.identical(fromHex('0x80p-1082'), 0.0) + self.identical(fromHex('0x81p-1082'), TINY) + self.identical(fromHex('.8p-1074'), 0.0) + self.identical(fromHex('8p-1078'), 0.0) + self.identical(fromHex('-.8p-1074'), -0.0) + self.identical(fromHex('+8p-1078'), 0.0) + def test_roundtrip(self): def roundtrip(x): return fromHex(toHex(x)) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 05e7102d..518ebdf1 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -211,12 +211,8 @@ f'{a * f"-{x()}-"}'""" self.assertEqual(call.lineno, 3) self.assertEqual(call.col_offset, 11) + @unittest.skipIf(use_old_parser(), "The old parser gets the offsets incorrectly for fstrings") def test_ast_line_numbers_duplicate_expression(self): - """Duplicate expression - - NOTE: this is currently broken, always sets location of the first - expression. - """ expr = """ a = 10 f'{a * x()} {a * x()} {a * x()}' @@ -266,9 +262,9 @@ f'{a * x()} {a * x()} {a * x()}' self.assertEqual(binop.lineno, 3) self.assertEqual(binop.left.lineno, 3) self.assertEqual(binop.right.lineno, 3) - self.assertEqual(binop.col_offset, 3) # FIXME: this is wrong - self.assertEqual(binop.left.col_offset, 3) # FIXME: this is wrong - self.assertEqual(binop.right.col_offset, 7) # FIXME: this is wrong + self.assertEqual(binop.col_offset, 13) + self.assertEqual(binop.left.col_offset, 13) + self.assertEqual(binop.right.col_offset, 17) # check the third binop location binop = t.body[1].value.values[4].value self.assertEqual(type(binop), ast.BinOp) @@ -278,9 +274,33 @@ f'{a * x()} {a * x()} {a * x()}' self.assertEqual(binop.lineno, 3) self.assertEqual(binop.left.lineno, 3) self.assertEqual(binop.right.lineno, 3) - self.assertEqual(binop.col_offset, 3) # FIXME: this is wrong - self.assertEqual(binop.left.col_offset, 3) # FIXME: this is wrong - self.assertEqual(binop.right.col_offset, 7) # FIXME: this is wrong + self.assertEqual(binop.col_offset, 23) + self.assertEqual(binop.left.col_offset, 23) + self.assertEqual(binop.right.col_offset, 27) + + @unittest.skipIf(use_old_parser(), "The old parser gets the offsets incorrectly for fstrings") + def test_ast_numbers_fstring_with_formatting(self): + + t = ast.parse('f"Here is that pesky {xxx:.3f} again"') + self.assertEqual(len(t.body), 1) + self.assertEqual(t.body[0].lineno, 1) + + self.assertEqual(type(t.body[0]), ast.Expr) + self.assertEqual(type(t.body[0].value), ast.JoinedStr) + self.assertEqual(len(t.body[0].value.values), 3) + + self.assertEqual(type(t.body[0].value.values[0]), ast.Constant) + self.assertEqual(type(t.body[0].value.values[1]), ast.FormattedValue) + self.assertEqual(type(t.body[0].value.values[2]), ast.Constant) + + _, expr, _ = t.body[0].value.values + + name = expr.value + self.assertEqual(type(name), ast.Name) + self.assertEqual(name.lineno, 1) + self.assertEqual(name.end_lineno, 1) + self.assertEqual(name.col_offset, 22) + self.assertEqual(name.end_col_offset, 25) def test_ast_line_numbers_multiline_fstring(self): # See bpo-30465 for details. diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 11e8aa35..987020ea 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -164,6 +164,7 @@ class TestPartial: p = proxy(f) self.assertEqual(f.func, p.func) f = None + support.gc_collect() # For PyPy or other GCs. self.assertRaises(ReferenceError, getattr, p, 'func') def test_with_bound_and_unbound_methods(self): @@ -1153,6 +1154,34 @@ class TestTotalOrdering(unittest.TestCase): method_copy = pickle.loads(pickle.dumps(method, proto)) self.assertIs(method_copy, method) + + def test_total_ordering_for_metaclasses_issue_44605(self): + + @functools.total_ordering + class SortableMeta(type): + def __new__(cls, name, bases, ns): + return super().__new__(cls, name, bases, ns) + + def __lt__(self, other): + if not isinstance(other, SortableMeta): + pass + return self.__name__ < other.__name__ + + def __eq__(self, other): + if not isinstance(other, SortableMeta): + pass + return self.__name__ == other.__name__ + + class B(metaclass=SortableMeta): + pass + + class A(metaclass=SortableMeta): + pass + + self.assertTrue(A < B) + self.assertFalse(A > B) + + @functools.total_ordering class Orderable_LT: def __init__(self, value): diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 38c9cb71..59cff20e 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1360,6 +1360,34 @@ class GCTogglingTests(unittest.TestCase): # empty __dict__. self.assertEqual(x, None) + +class PythonFinalizationTests(unittest.TestCase): + def test_ast_fini(self): + # bpo-44184: Regression test for subtype_dealloc() when deallocating + # an AST instance also destroy its AST type: subtype_dealloc() must + # not access the type memory after deallocating the instance, since + # the type memory can be freed as well. The test is also related to + # _PyAST_Fini() which clears references to AST types. + code = textwrap.dedent(""" + import ast + import codecs + + # Small AST tree to keep their AST types alive + tree = ast.parse("def f(x, y): return 2*x-y") + x = [tree] + x.append(x) + + # Put the cycle somewhere to survive until the last GC collection. + # Codec search functions are only cleared at the end of + # interpreter_clear(). + def search_func(encoding): + return None + search_func.a = x + codecs.register(search_func) + """) + assert_python_ok("-c", code) + + def test_main(): enabled = gc.isenabled() gc.disable() @@ -1369,7 +1397,11 @@ def test_main(): try: gc.collect() # Delete 2nd generation garbage - run_unittest(GCTests, GCTogglingTests, GCCallbackTests) + run_unittest( + GCTests, + GCCallbackTests, + GCTogglingTests, + PythonFinalizationTests) finally: gc.set_debug(debug) # test gc.enable() even if GC is disabled by default diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 3bf15228..f919f9c2 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -270,6 +270,32 @@ class ExceptionTest(unittest.TestCase): self.assertEqual(next(g), "done") self.assertEqual(sys.exc_info(), (None, None, None)) + def test_except_throw_bad_exception(self): + class E(Exception): + def __new__(cls, *args, **kwargs): + return cls + + def boring_generator(): + yield + + gen = boring_generator() + + err_msg = 'should have returned an instance of BaseException' + + with self.assertRaisesRegex(TypeError, err_msg): + gen.throw(E) + + self.assertRaises(StopIteration, next, gen) + + def generator(): + with self.assertRaisesRegex(TypeError, err_msg): + yield + + gen = generator() + next(gen) + with self.assertRaises(StopIteration): + gen.throw(E) + def test_stopiteration_error(self): # See also PEP 479. @@ -1940,6 +1966,8 @@ True """ coroutine_tests = """\ +>>> from test.support import gc_collect + Sending a value into a started generator: >>> def f(): @@ -2163,7 +2191,7 @@ And finalization: >>> g = f() >>> next(g) ->>> del g +>>> del g; gc_collect() # For PyPy or other GCs. exiting @@ -2178,7 +2206,7 @@ GeneratorExit is not caught by except Exception: >>> g = f() >>> next(g) ->>> del g +>>> del g; gc_collect() # For PyPy or other GCs. finally diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index b33fedaa..70e562f4 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -591,9 +591,18 @@ cgi_file6 = """\ #!%s import os -print("Content-type: text/plain") +print("X-ambv: was here") +print("Content-type: text/html") print() -print(repr(os.environ)) +print("
")
+for k, v in os.environ.items():
+    try:
+        k.encode('ascii')
+        v.encode('ascii')
+    except UnicodeEncodeError:
+        continue  # see: BPO-44647
+    print(f"{k}={v}")
+print("
") """ @@ -848,8 +857,8 @@ class CGIHTTPServerTestCase(BaseTestCase): with self.subTest(headers): res = self.request('/cgi-bin/file6.py', 'GET', headers=headers) self.assertEqual(http.HTTPStatus.OK, res.status) - expected = f"'HTTP_ACCEPT': {expected!r}" - self.assertIn(expected.encode('ascii'), res.read()) + expected = f"HTTP_ACCEPT={expected}".encode('ascii') + self.assertIn(expected, res.read()) class SocketlessRequestHandler(SimpleHTTPRequestHandler): diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index d9e26a30..dc7f5c24 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2958,6 +2958,47 @@ class TestSignatureObject(unittest.TestCase): ('bar', 2, ..., "keyword_only")), ...)) + def test_signature_on_subclass(self): + class A: + def __new__(cls, a=1, *args, **kwargs): + return object.__new__(cls) + class B(A): + def __init__(self, b): + pass + class C(A): + def __new__(cls, a=1, b=2, *args, **kwargs): + return object.__new__(cls) + class D(A): + pass + + self.assertEqual(self.signature(B), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C), + ((('a', 1, ..., 'positional_or_keyword'), + ('b', 2, ..., 'positional_or_keyword'), + ('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + self.assertEqual(self.signature(D), + ((('a', 1, ..., 'positional_or_keyword'), + ('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + def test_signature_on_generic_subclass(self): + from typing import Generic, TypeVar + + T = TypeVar('T') + + class A(Generic[T]): + def __init__(self, *, a: int) -> None: + pass + + self.assertEqual(self.signature(A), + ((('a', ..., int, 'keyword_only'),), + None)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_class_without_init(self): diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 1f0210f8..8622939c 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4310,6 +4310,31 @@ class SignalsTest(unittest.TestCase): """Check that a partial write, when it gets interrupted, properly invokes the signal handler, and bubbles up the exception raised in the latter.""" + + # XXX This test has three flaws that appear when objects are + # XXX not reference counted. + + # - if wio.write() happens to trigger a garbage collection, + # the signal exception may be raised when some __del__ + # method is running; it will not reach the assertRaises() + # call. + + # - more subtle, if the wio object is not destroyed at once + # and survives this function, the next opened file is likely + # to have the same fileno (since the file descriptor was + # actively closed). When wio.__del__ is finally called, it + # will close the other's test file... To trigger this with + # CPython, try adding "global wio" in this function. + + # - This happens only for streams created by the _pyio module, + # because a wio.close() that fails still consider that the + # file needs to be closed again. You can try adding an + # "assert wio.closed" at the end of the function. + + # Fortunately, a little gc.collect() seems to be enough to + # work around all these issues. + support.gc_collect() # For PyPy or other GCs. + read_results = [] def _read(): s = os.read(r, 1) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 71012642..802f7b8f 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1423,6 +1423,7 @@ class TestBasicOps(unittest.TestCase): p = weakref.proxy(a) self.assertEqual(getattr(p, '__class__'), type(b)) del a + support.gc_collect() # For PyPy or other GCs. self.assertRaises(ReferenceError, getattr, p, '__class__') ans = list('abc') diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 29f5e427..82a1ddc2 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -149,6 +149,11 @@ class OperatorTestCase: self.assertRaises(ZeroDivisionError, operator.countOf, BadIterable(), 1) self.assertEqual(operator.countOf([1, 2, 1, 3, 1, 4], 3), 1) self.assertEqual(operator.countOf([1, 2, 1, 3, 1, 4], 5), 0) + # is but not == + nan = float("nan") + self.assertEqual(operator.countOf([nan, nan, 21], nan), 2) + # == but not is + self.assertEqual(operator.countOf([{}, 1, {}, 2], {}), 2) def test_delitem(self): operator = self.module @@ -184,6 +189,9 @@ class OperatorTestCase: self.assertRaises(ZeroDivisionError, operator.indexOf, BadIterable(), 1) self.assertEqual(operator.indexOf([4, 3, 2, 1], 3), 1) self.assertRaises(ValueError, operator.indexOf, [4, 3, 2, 1], 0) + nan = float("nan") + self.assertEqual(operator.indexOf([nan, nan, 21], nan), 0) + self.assertEqual(operator.indexOf([{}, 1, {}, 2], {}), 0) def test_invert(self): operator = self.module diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 3da35710..0314938e 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1248,19 +1248,35 @@ class PureWindowsPathTest(_BasePurePathTest, unittest.TestCase): self.assertIs(False, P('').is_reserved()) self.assertIs(False, P('/').is_reserved()) self.assertIs(False, P('/foo/bar').is_reserved()) + # UNC paths are never reserved. + self.assertIs(False, P('//my/share/nul/con/aux').is_reserved()) + # Case-insenstive DOS-device names are reserved. + self.assertIs(True, P('nul').is_reserved()) + self.assertIs(True, P('aux').is_reserved()) + self.assertIs(True, P('prn').is_reserved()) self.assertIs(True, P('con').is_reserved()) - self.assertIs(True, P('NUL').is_reserved()) + self.assertIs(True, P('conin$').is_reserved()) + self.assertIs(True, P('conout$').is_reserved()) + # COM/LPT + 1-9 or + superscript 1-3 are reserved. + self.assertIs(True, P('COM1').is_reserved()) + self.assertIs(True, P('LPT9').is_reserved()) + self.assertIs(True, P('com\xb9').is_reserved()) + self.assertIs(True, P('com\xb2').is_reserved()) + self.assertIs(True, P('lpt\xb3').is_reserved()) + # DOS-device name mataching ignores characters after a dot or + # a colon and also ignores trailing spaces. self.assertIs(True, P('NUL.txt').is_reserved()) - self.assertIs(True, P('com1').is_reserved()) - self.assertIs(True, P('com9.bar').is_reserved()) + self.assertIs(True, P('PRN ').is_reserved()) + self.assertIs(True, P('AUX .txt').is_reserved()) + self.assertIs(True, P('COM1:bar').is_reserved()) + self.assertIs(True, P('LPT9 :bar').is_reserved()) + # DOS-device names are only matched at the beginning + # of a path component. self.assertIs(False, P('bar.com9').is_reserved()) - self.assertIs(True, P('lpt1').is_reserved()) - self.assertIs(True, P('lpt9.bar').is_reserved()) self.assertIs(False, P('bar.lpt9').is_reserved()) - # Only the last component matters. + # Only the last path component matters. + self.assertIs(True, P('c:/baz/con/NUL').is_reserved()) self.assertIs(False, P('c:/NUL/con/baz').is_reserved()) - # UNC paths are never reserved. - self.assertIs(False, P('//my/share/nul/con/aux').is_reserved()) class PurePathTest(_BasePurePathTest, unittest.TestCase): cls = pathlib.PurePath diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 1c007a2b..0653a64b 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1632,6 +1632,20 @@ def bœr(): self.assertIn("ImportError: No module named t_main.__main__", stdout.splitlines()) + def test_package_without_a_main(self): + pkg_name = 't_pkg' + module_name = 't_main' + support.rmtree(pkg_name) + modpath = pkg_name + '/' + module_name + os.makedirs(modpath) + with open(modpath + '/__init__.py', 'w') as f: + pass + self.addCleanup(support.rmtree, pkg_name) + stdout, stderr = self._run_pdb(['-m', modpath.replace('/', '.')], "") + self.assertIn( + "'t_pkg.t_main' is a package and cannot be directly executed", + stdout) + def test_blocks_at_first_code_line(self): script = """ #This is a comment, on line 2 @@ -1736,6 +1750,21 @@ def bœr(): '(Pdb) ', ]) + def test_issue34266(self): + '''do_run handles exceptions from parsing its arg''' + def check(bad_arg, msg): + commands = "\n".join([ + f'run {bad_arg}', + 'q', + ]) + stdout, _ = self.run_pdb_script('pass', commands + '\n') + self.assertEqual(stdout.splitlines()[1:], [ + '-> pass', + f'(Pdb) *** Cannot run {bad_arg}: {msg}', + '(Pdb) ', + ]) + check('\\', 'No escaped character') + check('"', 'No closing quotation') def test_issue42384(self): '''When running `python foo.py` sys.path[0] is an absolute path. `python -m pdb foo.py` should behave the same''' diff --git a/Lib/test/test_peg_parser.py b/Lib/test/test_peg_parser.py index f4aaef84..b7bd8639 100644 --- a/Lib/test/test_peg_parser.py +++ b/Lib/test/test_peg_parser.py @@ -231,11 +231,6 @@ TEST_CASES = [ ('f-string_doublestarred', "f'{ {**x} }'"), ('f-string_escape_brace', "f'{{Escape'"), ('f-string_escape_closing_brace', "f'Escape}}'"), - ('f-string_repr', "f'{a!r}'"), - ('f-string_str', "f'{a!s}'"), - ('f-string_ascii', "f'{a!a}'"), - ('f-string_debug', "f'{a=}'"), - ('f-string_padding', "f'{a:03d}'"), ('f-string_multiline', """ f''' diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 2307b133..70672dc1 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -6,6 +6,7 @@ import io import collections import struct import sys +import warnings import weakref import unittest @@ -366,7 +367,10 @@ def getmodule(module): return sys.modules[module] except KeyError: try: - __import__(module) + with warnings.catch_warnings(): + action = 'always' if support.verbose else 'ignore' + warnings.simplefilter(action, DeprecationWarning) + __import__(module) except AttributeError as exc: if support.verbose: print("Can't import module %r: %s" % (module, exc)) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index d88e28a9..dd7f0b11 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -7,6 +7,7 @@ import time import unittest import weakref from test import support +from test.support import gc_collect py_queue = support.import_fresh_module('queue', blocked=['_queue']) c_queue = support.import_fresh_module('queue', fresh=['_queue']) @@ -588,6 +589,7 @@ class BaseSimpleQueueTest: q.put(C()) for i in range(N): wr = weakref.ref(q.get()) + gc_collect() # For PyPy or other GCs. self.assertIsNone(wr()) diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index 57da0e15..8dc62a93 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -438,6 +438,7 @@ class TestContext(unittest.TestCase): f() def test_3611(self): + import gc # A re-raised exception in a __del__ caused the __context__ # to be cleared class C: @@ -451,9 +452,11 @@ class TestContext(unittest.TestCase): x = C() try: try: - x.x + f.x except AttributeError: + # make x.__del__ trigger del x + gc.collect() # For PyPy or other GCs. raise TypeError except Exception as e: self.assertNotEqual(e.__context__, None) diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 30fa129b..16b0fc06 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -374,26 +374,41 @@ class RangeTest(unittest.TestCase): list(r)) def test_iterator_pickling(self): - testcases = [(13,), (0, 11), (-22, 10), (20, 3, -1), - (13, 21, 3), (-2, 2, 2), (2**65, 2**65+2)] + testcases = [(13,), (0, 11), (-22, 10), (20, 3, -1), (13, 21, 3), + (-2, 2, 2), (2**31-3, 2**31-1), (2**33, 2**33+2), + (2**63-3, 2**63-1), (2**65, 2**65+2)] for proto in range(pickle.HIGHEST_PROTOCOL + 1): for t in testcases: - it = itorg = iter(range(*t)) - data = list(range(*t)) - - d = pickle.dumps(it, proto) - it = pickle.loads(d) - self.assertEqual(type(itorg), type(it)) - self.assertEqual(list(it), data) - - it = pickle.loads(d) - try: - next(it) - except StopIteration: - continue + with self.subTest(proto=proto, t=t): + it = itorg = iter(range(*t)) + data = list(range(*t)) + + d = pickle.dumps(it, proto) + it = pickle.loads(d) + self.assertEqual(type(itorg), type(it)) + self.assertEqual(list(it), data) + + it = pickle.loads(d) + try: + next(it) + except StopIteration: + continue + d = pickle.dumps(it, proto) + it = pickle.loads(d) + self.assertEqual(list(it), data[1:]) + + def test_iterator_pickling_overflowing_index(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + it = iter(range(2**32 + 2)) + _, _, idx = it.__reduce__() + self.assertEqual(idx, 0) + it.__setstate__(2**32 + 1) # undocumented way to set r->index + _, _, idx = it.__reduce__() + self.assertEqual(idx, 2**32 + 1) d = pickle.dumps(it, proto) it = pickle.loads(d) - self.assertEqual(list(it), data[1:]) + self.assertEqual(next(it), 2**32 + 1) def test_exhausted_iterator_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 67ee9b7f..7898abb7 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -147,11 +147,15 @@ print("History length:", readline.get_current_history_length()) def test_auto_history_enabled(self): output = run_pty(self.auto_history_script.format(True)) - self.assertIn(b"History length: 1\r\n", output) + # bpo-44949: Sometimes, the newline character is not written at the + # end, so don't expect it in the output. + self.assertIn(b"History length: 1", output) def test_auto_history_disabled(self): output = run_pty(self.auto_history_script.format(False)) - self.assertIn(b"History length: 0\r\n", output) + # bpo-44949: Sometimes, the newline character is not written at the + # end, so don't expect it in the output. + self.assertIn(b"History length: 0", output) def test_nonascii(self): loc = locale.setlocale(locale.LC_CTYPE, None) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a77638b1..b35468fa 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -15,6 +15,7 @@ import sys import sysconfig import tempfile import textwrap +import time import unittest from test import libregrtest from test import support @@ -413,7 +414,7 @@ class BaseTestCase(unittest.TestCase): def check_executed_tests(self, output, tests, skipped=(), failed=(), env_changed=(), omitted=(), - rerun=(), no_test_ran=(), + rerun={}, no_test_ran=(), randomize=False, interrupted=False, fail_env_changed=False): if isinstance(tests, str): @@ -426,8 +427,6 @@ class BaseTestCase(unittest.TestCase): env_changed = [env_changed] if isinstance(omitted, str): omitted = [omitted] - if isinstance(rerun, str): - rerun = [rerun] if isinstance(no_test_ran, str): no_test_ran = [no_test_ran] @@ -465,12 +464,12 @@ class BaseTestCase(unittest.TestCase): self.check_line(output, regex) if rerun: - regex = list_regex('%s re-run test%s', rerun) + regex = list_regex('%s re-run test%s', rerun.keys()) self.check_line(output, regex) regex = LOG_PREFIX + r"Re-running failed tests in verbose mode" self.check_line(output, regex) - for test_name in rerun: - regex = LOG_PREFIX + f"Re-running {test_name} in verbose mode" + for name, match in rerun.items(): + regex = LOG_PREFIX + f"Re-running {name} in verbose mode \\(matching: {match}\\)" self.check_line(output, regex) if no_test_ran: @@ -548,11 +547,10 @@ class BaseTestCase(unittest.TestCase): class CheckActualTests(BaseTestCase): - """ - Check that regrtest appears to find the expected set of tests. - """ - def test_finds_expected_number_of_tests(self): + """ + Check that regrtest appears to find the expected set of tests. + """ args = ['-Wd', '-E', '-bb', '-m', 'test.regrtest', '--list-tests'] output = self.run_python(args) rough_number_of_tests_found = len(output.splitlines()) @@ -1080,15 +1078,18 @@ class ArgsTestCase(BaseTestCase): import unittest class Tests(unittest.TestCase): - def test_bug(self): - # test always fail + def test_succeed(self): + return + + def test_fail_always(self): + # test that always fails self.fail("bug") """) testname = self.create_test(code=code) output = self.run_tests("-w", testname, exitcode=2) self.check_executed_tests(output, [testname], - failed=testname, rerun=testname) + failed=testname, rerun={testname: "test_fail_always"}) def test_rerun_success(self): # FAILURE then SUCCESS @@ -1097,7 +1098,8 @@ class ArgsTestCase(BaseTestCase): import unittest class Tests(unittest.TestCase): - failed = False + def test_succeed(self): + return def test_fail_once(self): if not hasattr(builtins, '_test_failed'): @@ -1108,7 +1110,7 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests("-w", testname, exitcode=0) self.check_executed_tests(output, [testname], - rerun=testname) + rerun={testname: "test_fail_once"}) def test_no_tests_ran(self): code = textwrap.dedent(""" diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 0dc1080c..423b3a29 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -81,17 +81,41 @@ class TestRlcompleter(unittest.TestCase): if x.startswith('s')]) def test_excessive_getattr(self): - # Ensure getattr() is invoked no more than once per attribute + """Ensure getattr() is invoked no more than once per attribute""" + + # note the special case for @property methods below; that is why + # we use __dir__ and __getattr__ in class Foo to create a "magic" + # class attribute 'bar'. This forces `getattr` to call __getattr__ + # (which is doesn't necessarily do). class Foo: calls = 0 + bar = '' + def __getattribute__(self, name): + if name == 'bar': + self.calls += 1 + return None + return super().__getattribute__(name) + + f = Foo() + completer = rlcompleter.Completer(dict(f=f)) + self.assertEqual(completer.complete('f.b', 0), 'f.bar') + self.assertEqual(f.calls, 1) + + def test_property_method_not_called(self): + class Foo: + _bar = 0 + property_called = False + @property def bar(self): - self.calls += 1 - return None + self.property_called = True + return self._bar + f = Foo() completer = rlcompleter.Completer(dict(f=f)) self.assertEqual(completer.complete('f.b', 0), 'f.bar') - self.assertEqual(f.calls, 1) + self.assertFalse(f.property_called) + def test_uncreated_attr(self): # Attributes like properties and slots should be completed even when diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 4239b264..59d59127 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -2,6 +2,7 @@ import unittest import weakref from test.support import check_syntax_error, cpython_only +from test.support import gc_collect class ScopeTests(unittest.TestCase): @@ -422,6 +423,7 @@ class ScopeTests(unittest.TestCase): for i in range(100): f1() + gc_collect() # For PyPy or other GCs. self.assertEqual(Foo.count, 0) def testClassAndGlobal(self): @@ -754,6 +756,7 @@ class ScopeTests(unittest.TestCase): tester.dig() ref = weakref.ref(tester) del tester + gc_collect() # For PyPy or other GCs. self.assertIsNone(ref()) diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index faf25a79..ca2f4e28 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -606,6 +606,7 @@ class TestSet(TestJointOps, unittest.TestCase): p = weakref.proxy(s) self.assertEqual(str(p), str(s)) s = None + support.gc_collect() # For PyPy or other GCs. self.assertRaises(ReferenceError, str, p) def test_rich_compare(self): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index e378033a..768d4b46 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -36,6 +36,7 @@ TESTFN2 = TESTFN + "2" TESTFN_SRC = TESTFN + "_SRC" TESTFN_DST = TESTFN + "_DST" MACOS = sys.platform.startswith("darwin") +SOLARIS = sys.platform.startswith("sunos") AIX = sys.platform[:3] == 'aix' try: import grp @@ -1243,6 +1244,15 @@ class TestCopy(BaseTest, unittest.TestCase): # Make sure file is not corrupted. self.assertEqual(read_file(src_file), 'foo') + @unittest.skipIf(MACOS or SOLARIS or _winapi, 'On MACOS, Solaris and Windows the errors are not confusing (though different)') + def test_copyfile_nonexistent_dir(self): + # Issue 43219 + src_dir = self.mkdtemp() + src_file = os.path.join(src_dir, 'foo') + dst = os.path.join(src_dir, 'does_not_exist/') + write_file(src_file, 'foo') + self.assertRaises(FileNotFoundError, shutil.copyfile, src_file, dst) + class TestArchives(BaseTest, unittest.TestCase): diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index c8c4d3ce..52e9d6be 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -332,6 +332,16 @@ class DebuggingServerTests(unittest.TestCase): self.assertEqual(smtp.getreply(), expected) smtp.quit() + def test_issue43124_putcmd_escapes_newline(self): + # see: https://bugs.python.org/issue43124 + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + with self.assertRaises(ValueError) as exc: + smtp.putcmd('helo\nX-INJECTED') + self.assertIn("prohibited newline characters", str(exc.exception)) + smtp.quit() + def testVRFY(self): smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=support.LOOPBACK_TIMEOUT) @@ -413,6 +423,51 @@ class DebuggingServerTests(unittest.TestCase): mexpect = '%s%s\n%s' % (MSG_BEGIN, m, MSG_END) self.assertEqual(self.output.getvalue(), mexpect) + def test_issue43124_escape_localhostname(self): + # see: https://bugs.python.org/issue43124 + # connect and send mail + m = 'wazzuuup\nlinetwo' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='hi\nX-INJECTED', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + with self.assertRaises(ValueError) as exc: + smtp.sendmail("hi@me.com", "you@me.com", m) + self.assertIn( + "prohibited newline characters: ehlo hi\\nX-INJECTED", + str(exc.exception), + ) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + debugout = smtpd.DEBUGSTREAM.getvalue() + self.assertNotIn("X-INJECTED", debugout) + + def test_issue43124_escape_options(self): + # see: https://bugs.python.org/issue43124 + # connect and send mail + m = 'wazzuuup\nlinetwo' + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + + self.addCleanup(smtp.close) + smtp.sendmail("hi@me.com", "you@me.com", m) + with self.assertRaises(ValueError) as exc: + smtp.mail("hi@me.com", ["X-OPTION\nX-INJECTED-1", "X-OPTION2\nX-INJECTED-2"]) + msg = str(exc.exception) + self.assertIn("prohibited newline characters", msg) + self.assertIn("X-OPTION\\nX-INJECTED-1 X-OPTION2\\nX-INJECTED-2", msg) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + debugout = smtpd.DEBUGSTREAM.getvalue() + self.assertNotIn("X-OPTION", debugout) + self.assertNotIn("X-OPTION2", debugout) + self.assertNotIn("X-INJECTED-1", debugout) + self.assertNotIn("X-INJECTED-2", debugout) + def testSendNullSender(self): m = 'A test message' smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 595cae6b..e0f80ca5 100755 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -868,6 +868,7 @@ class GeneralModuleTests(unittest.TestCase): p = proxy(s) self.assertEqual(p.fileno(), s.fileno()) s = None + support.gc_collect() # For PyPy or other GCs. try: p.fileno() except ReferenceError: diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index ed048133..f93fbe92 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2969,6 +2969,7 @@ class POSIXProcessTestCase(BaseTestCase): pid = p.pid with support.check_warnings(('', ResourceWarning)): p = None + support.gc_collect() # For PyPy or other GCs. os.kill(pid, signal.SIGKILL) if mswindows: diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index b5a16f9c..60a77411 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -11,6 +11,8 @@ import tempfile import textwrap import time import unittest +import warnings + from test import support from test.support import script_helper from test.support import socket_helper @@ -19,6 +21,33 @@ TESTFN = support.TESTFN class TestSupport(unittest.TestCase): + @classmethod + def setUpClass(cls): + orig_filter_len = len(warnings.filters) + cls._warnings_helper_token = support.ignore_deprecations_from( + "test.test_support", like=".*used in test_support.*" + ) + cls._test_support_token = support.ignore_deprecations_from( + "test.test_support", like=".*You should NOT be seeing this.*" + ) + assert len(warnings.filters) == orig_filter_len + 2 + + @classmethod + def tearDownClass(cls): + orig_filter_len = len(warnings.filters) + support.clear_ignored_deprecations( + cls._warnings_helper_token, + cls._test_support_token, + ) + assert len(warnings.filters) == orig_filter_len - 2 + + def test_ignored_deprecations_are_silent(self): + """Test support.ignore_deprecations_from() silences warnings""" + with warnings.catch_warnings(record=True) as warning_objs: + _warn_about_deprecation() + warnings.warn("You should NOT be seeing this.", DeprecationWarning) + messages = [str(w.message) for w in warning_objs] + self.assertEqual(len(messages), 0, messages) def test_import_module(self): support.import_module("ftplib") @@ -676,6 +705,17 @@ class TestSupport(unittest.TestCase): # SuppressCrashReport +def _warn_about_deprecation(): + # In 3.10+ this lives in test.support.warnings_helper + warnings.warn( + "This is used in test_support test to ensure" + " support.ignore_deprecations_from() works as expected." + " You should not be seeing this.", + DeprecationWarning, + stacklevel=0, + ) + + def test_main(): tests = [TestSupport] support.run_unittest(*tests) diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 74d97735..eaa94ea9 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -429,9 +429,6 @@ SyntaxError: expression cannot contain assignment, perhaps you meant "=="? >>> f((x)=2) Traceback (most recent call last): SyntaxError: expression cannot contain assignment, perhaps you meant "=="? ->>> f(True=2) -Traceback (most recent call last): -SyntaxError: expression cannot contain assignment, perhaps you meant "=="? >>> f(__debug__=1) Traceback (most recent call last): SyntaxError: cannot assign to __debug__ @@ -534,38 +531,6 @@ isn't, there should be a syntax error. ... SyntaxError: 'break' outside loop -This raises a SyntaxError, it used to raise a SystemError. -Context for this change can be found on issue #27514 - -In 2.5 there was a missing exception and an assert was triggered in a debug -build. The number of blocks must be greater than CO_MAXBLOCKS. SF #1565514 - - >>> while 1: - ... while 2: - ... while 3: - ... while 4: - ... while 5: - ... while 6: - ... while 8: - ... while 9: - ... while 10: - ... while 11: - ... while 12: - ... while 13: - ... while 14: - ... while 15: - ... while 16: - ... while 17: - ... while 18: - ... while 19: - ... while 20: - ... while 21: - ... while 22: - ... break - Traceback (most recent call last): - ... - SyntaxError: too many statically nested blocks - Misuse of the nonlocal and global statement can lead to a few unique syntax errors. >>> def f(): @@ -730,6 +695,13 @@ SyntaxError: trailing comma not allowed without surrounding parentheses Traceback (most recent call last): SyntaxError: trailing comma not allowed without surrounding parentheses +# Check that we dont raise the "trailing comma" error if there is more +# input to the left of the valid part that we parsed. + +>>> from t import x,y, and 3 +Traceback (most recent call last): +SyntaxError: invalid syntax + >>> (): int Traceback (most recent call last): SyntaxError: only single target (not tuple) can be annotated @@ -981,7 +953,7 @@ def func2(): def test_invalid_line_continuation_error_position(self): self._check_error(r"a = 3 \ 4", "unexpected character after line continuation character", - lineno=1, offset=9) + lineno=1, offset=(10 if support.use_old_parser() else 9)) def test_invalid_line_continuation_left_recursive(self): # Check bpo-42218: SyntaxErrors following left-recursive rules @@ -991,6 +963,42 @@ def func2(): self._check_error("A.\u03bc\\\n", "unexpected EOF while parsing") + @support.cpython_only + def test_syntax_error_on_deeply_nested_blocks(self): + # This raises a SyntaxError, it used to raise a SystemError. Context + # for this change can be found on issue #27514 + + # In 2.5 there was a missing exception and an assert was triggered in a + # debug build. The number of blocks must be greater than CO_MAXBLOCKS. + # SF #1565514 + + source = """ +while 1: + while 2: + while 3: + while 4: + while 5: + while 6: + while 8: + while 9: + while 10: + while 11: + while 12: + while 13: + while 14: + while 15: + while 16: + while 17: + while 18: + while 19: + while 20: + while 21: + while 22: + break +""" + self._check_error(source, "too many statically nested blocks") + + def test_main(): support.run_unittest(SyntaxTestCase) from test import test_syntax diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 482e918a..7519309c 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -602,6 +602,50 @@ class TraceTestCase(unittest.TestCase): self.compare_events(doit_async.__code__.co_firstlineno, tracer.events, events) + def test_21_async_for_else(self): + + async def async_gen(): + yield -2 + + async def async_test(): + global a + a = 2 + async for i in async_gen(): + a = 4 + else: + a = 6 + + def run(tracer): + x = async_test() + try: + sys.settrace(tracer) + x.send(None) + finally: + sys.settrace(None) + + tracer = self.make_tracer() + events = [ + (0, 'call'), + (2, 'line'), + (3, 'line'), + (-3, 'call'), + (-2, 'line'), + (-2, 'return'), + (3, 'exception'), + (4, 'line'), + (3, 'line'), + (-2, 'call'), + (-2, 'return'), + (3, 'exception'), + (6, 'line'), + (6, 'return')] + try: + run(tracer.trace) + except Exception: + pass + self.compare_events(async_test.__code__.co_firstlineno, + tracer.events, events) + class SkipLineEventsTraceTestCase(TraceTestCase): """Repeat the trace tests, but with per-line events skipped""" diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 29cde91b..6279309e 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1705,15 +1705,30 @@ class CreateTest(WriteTestBase, unittest.TestCase): class GzipCreateTest(GzipTest, CreateTest): - pass + + def test_create_with_compresslevel(self): + with tarfile.open(tmpname, self.mode, compresslevel=1) as tobj: + tobj.add(self.file_path) + with tarfile.open(tmpname, 'r:gz', compresslevel=1) as tobj: + pass class Bz2CreateTest(Bz2Test, CreateTest): - pass + + def test_create_with_compresslevel(self): + with tarfile.open(tmpname, self.mode, compresslevel=1) as tobj: + tobj.add(self.file_path) + with tarfile.open(tmpname, 'r:bz2', compresslevel=1) as tobj: + pass class LzmaCreateTest(LzmaTest, CreateTest): - pass + + # Unlike gz and bz2, xz uses the preset keyword instead of compresslevel. + # It does not allow for preset to be specified when reading. + def test_create_with_preset(self): + with tarfile.open(tmpname, self.mode, preset=1) as tobj: + tobj.add(self.file_path) class CreateWithXModeTest(CreateTest): diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 0bbb39b1..fa974d18 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -41,8 +41,14 @@ def get_tk_patchlevel(): class TkinterTest(unittest.TestCase): def testFlattenLen(self): - # flatten() + # Object without length. self.assertRaises(TypeError, _tkinter._flatten, True) + # Object with length, but not sequence. + self.assertRaises(TypeError, _tkinter._flatten, {}) + # Sequence or set, but not tuple or list. + # (issue44608: there were leaks in the following cases) + self.assertRaises(TypeError, _tkinter._flatten, 'string') + self.assertRaises(TypeError, _tkinter._flatten, {'set'}) class TclTest(unittest.TestCase): diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index fcc706ed..7dbbf9b7 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -428,6 +428,7 @@ class TestMkstempInner(TestBadTempdir, BaseTestCase): self.do_create(dir=dir).write(b"blat") self.do_create(dir=pathlib.Path(dir)).write(b"blat") finally: + support.gc_collect() # For PyPy or other GCs. os.rmdir(dir) def test_file_mode(self): @@ -821,6 +822,8 @@ class TestMktemp(BaseTestCase): extant = list(range(TEST_FILES)) for i in extant: extant[i] = self.do_create(pre="aa") + del extant + support.gc_collect() # For PyPy or other GCs. ## def test_warning(self): ## # mktemp issues a warning when used diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index 77e46f2c..3f8a539f 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -131,6 +131,7 @@ class ThreadRunningTests(BasicThreadTest): del task while not done: time.sleep(POLL_SLEEP) + support.gc_collect() # For PyPy or other GCs. self.assertEqual(thread._count(), orig) def test_unraisable_exception(self): diff --git a/Lib/test/test_threading_local.py b/Lib/test/test_threading_local.py index 2fd14ae2..5ddb8c41 100644 --- a/Lib/test/test_threading_local.py +++ b/Lib/test/test_threading_local.py @@ -36,7 +36,7 @@ class BaseLocalTest: t.join() del t - gc.collect() + support.gc_collect() # For PyPy or other GCs. self.assertEqual(len(weaklist), n) # XXX _threading_local keeps the local of the last stopped thread alive. @@ -45,7 +45,7 @@ class BaseLocalTest: # Assignment to the same thread local frees it sometimes (!) local.someothervar = None - gc.collect() + support.gc_collect() # For PyPy or other GCs. deadlist = [weak for weak in weaklist if weak() is None] self.assertIn(len(deadlist), (n-1, n), (n, len(deadlist))) @@ -88,7 +88,7 @@ class BaseLocalTest: # 2) GC the cycle (triggers threadmodule.c::local_clear # before local_dealloc) del cycle - gc.collect() + support.gc_collect() # For PyPy or other GCs. e1.set() e2.wait() @@ -189,7 +189,7 @@ class BaseLocalTest: x.local.x = x wr = weakref.ref(x) del x - gc.collect() + support.gc_collect() # For PyPy or other GCs. self.assertIsNone(wr()) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 6de7aa87..e05a07fa 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1457,6 +1457,16 @@ class TestTokenize(TestCase): # See http://bugs.python.org/issue16152 self.assertExactTypeEqual('@ ', token.AT) + def test_comment_at_the_end_of_the_source_without_newline(self): + # See http://bugs.python.org/issue44667 + source = 'b = 1\n\n#test' + expected_tokens = [token.NAME, token.EQUAL, token.NUMBER, token.NEWLINE, token.NL, token.COMMENT] + + tokens = list(tokenize(BytesIO(source.encode('utf-8')).readline)) + self.assertEqual(tok_name[tokens[0].exact_type], tok_name[ENCODING]) + for i in range(6): + self.assertEqual(tok_name[tokens[i + 1].exact_type], tok_name[expected_tokens[i]]) + self.assertEqual(tok_name[tokens[-1].exact_type], tok_name[token.ENDMARKER]) class UntokenizeTest(TestCase): diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 5bb3a58b..987fcb95 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -92,6 +92,43 @@ class TracebackCases(unittest.TestCase): lst = traceback.format_exception_only(e.__class__, e) self.assertEqual(lst, ['KeyboardInterrupt\n']) + def test_traceback_context_recursionerror(self): + # Test that for long traceback chains traceback does not itself + # raise a recursion error while printing (Issue43048) + + # Calling f() creates a stack-overflowing __context__ chain. + def f(): + try: + raise ValueError('hello') + except ValueError: + f() + + try: + f() + except RecursionError: + exc_info = sys.exc_info() + + traceback.format_exception(exc_info[0], exc_info[1], exc_info[2]) + + def test_traceback_cause_recursionerror(self): + # Same as test_traceback_context_recursionerror, but with + # a __cause__ chain. + + def f(): + e = None + try: + f() + except Exception as exc: + e = exc + raise Exception from e + + try: + f() + except Exception: + exc_info = sys.exc_info() + + traceback.format_exception(exc_info[0], exc_info[1], exc_info[2]) + def test_format_exception_only_bad__str__(self): class X(Exception): def __str__(self): diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index 5d5f0570..d8ad1ce2 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -232,17 +232,9 @@ class TestVec2D(VectorComparisonMixin, unittest.TestCase): self.assertVectorsAlmostEqual(-vec, expected) def test_distance(self): - vec = Vec2D(6, 8) - expected = 10 - self.assertEqual(abs(vec), expected) - - vec = Vec2D(0, 0) - expected = 0 - self.assertEqual(abs(vec), expected) - - vec = Vec2D(2.5, 6) - expected = 6.5 - self.assertEqual(abs(vec), expected) + self.assertEqual(abs(Vec2D(6, 8)), 10) + self.assertEqual(abs(Vec2D(0, 0)), 0) + self.assertAlmostEqual(abs(Vec2D(2.5, 6)), 6.5) def test_rotate(self): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 4bdb2a0f..e05ac453 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -29,6 +29,7 @@ import weakref import types from test import mod_generics_cache +from test import _typed_dict_helper class BaseTestCase(TestCase): @@ -744,6 +745,9 @@ class ProtocolTests(BaseTestCase): class C(P): pass self.assertIsInstance(C(), C) + with self.assertRaises(TypeError): + C(42) + T = TypeVar('T') class PG(Protocol[T]): pass @@ -758,6 +762,8 @@ class ProtocolTests(BaseTestCase): class CG(PG[T]): pass self.assertIsInstance(CG[int](), CG) + with self.assertRaises(TypeError): + CG[int](42) def test_cannot_instantiate_abstract(self): @runtime_checkable @@ -1193,6 +1199,37 @@ class ProtocolTests(BaseTestCase): self.assertEqual(C[int]().test, 'OK') + class B: + def __init__(self): + self.test = 'OK' + + class D1(B, P[T]): + pass + + self.assertEqual(D1[int]().test, 'OK') + + class D2(P[T], B): + pass + + self.assertEqual(D2[int]().test, 'OK') + + def test_new_called(self): + T = TypeVar('T') + + class P(Protocol[T]): pass + + class C(P[T]): + def __new__(cls, *args): + self = super().__new__(cls, *args) + self.test = 'OK' + return self + + self.assertEqual(C[int]().test, 'OK') + with self.assertRaises(TypeError): + C[int](42) + with self.assertRaises(TypeError): + C[int](a=42) + def test_protocols_bad_subscripts(self): T = TypeVar('T') S = TypeVar('S') @@ -2808,6 +2845,9 @@ class Point2D(TypedDict): x: int y: int +class Bar(_typed_dict_helper.Foo, total=False): + b: int + class LabelPoint2D(Point2D, Label): ... class Options(TypedDict, total=False): @@ -3944,6 +3984,12 @@ class TypedDictTests(BaseTestCase): 'voice': str, } + def test_get_type_hints(self): + self.assertEqual( + get_type_hints(Bar), + {'a': typing.Optional[int], 'b': int} + ) + class IOTests(BaseTestCase): diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 56a42f05..5a3e59c3 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -12,6 +12,7 @@ import random from test import support from test.support import script_helper, ALWAYS_EQ +from test.support import gc_collect # Used in ReferencesTestCase.test_ref_created_during_del() . ref_from_del = None @@ -135,6 +136,7 @@ class ReferencesTestCase(TestBase): ref1 = weakref.ref(o, self.callback) ref2 = weakref.ref(o, self.callback) del o + gc_collect() # For PyPy or other GCs. self.assertIsNone(ref1(), "expected reference to be invalidated") self.assertIsNone(ref2(), "expected reference to be invalidated") self.assertEqual(self.cbcalled, 2, @@ -168,13 +170,16 @@ class ReferencesTestCase(TestBase): ref1 = weakref.proxy(o, self.callback) ref2 = weakref.proxy(o, self.callback) del o + gc_collect() # For PyPy or other GCs. def check(proxy): proxy.bar self.assertRaises(ReferenceError, check, ref1) self.assertRaises(ReferenceError, check, ref2) - self.assertRaises(ReferenceError, bool, weakref.proxy(C())) + ref3 = weakref.proxy(C()) + gc_collect() # For PyPy or other GCs. + self.assertRaises(ReferenceError, bool, ref3) self.assertEqual(self.cbcalled, 2) def check_basic_ref(self, factory): @@ -191,6 +196,7 @@ class ReferencesTestCase(TestBase): o = factory() ref = weakref.ref(o, self.callback) del o + gc_collect() # For PyPy or other GCs. self.assertEqual(self.cbcalled, 1, "callback did not properly set 'cbcalled'") self.assertIsNone(ref(), @@ -215,6 +221,7 @@ class ReferencesTestCase(TestBase): self.assertEqual(weakref.getweakrefcount(o), 2, "wrong weak ref count for object") del proxy + gc_collect() # For PyPy or other GCs. self.assertEqual(weakref.getweakrefcount(o), 1, "wrong weak ref count for object after deleting proxy") @@ -411,6 +418,36 @@ class ReferencesTestCase(TestBase): # can be killed in the middle of the call "blech" in p + def test_proxy_next(self): + arr = [4, 5, 6] + def iterator_func(): + yield from arr + it = iterator_func() + + class IteratesWeakly: + def __iter__(self): + return weakref.proxy(it) + + weak_it = IteratesWeakly() + + # Calls proxy.__next__ + self.assertEqual(list(weak_it), [4, 5, 6]) + + def test_proxy_bad_next(self): + # bpo-44720: PyIter_Next() shouldn't be called if the reference + # isn't an iterator. + + not_an_iterator = lambda: 0 + + class A: + def __iter__(self): + return weakref.proxy(not_an_iterator) + a = A() + + msg = "Weakref proxy referenced a non-iterator" + with self.assertRaisesRegex(TypeError, msg): + list(a) + def test_proxy_reversed(self): class MyObj: def __len__(self): @@ -422,14 +459,20 @@ class ReferencesTestCase(TestBase): self.assertEqual("".join(reversed(weakref.proxy(obj))), "cba") def test_proxy_hash(self): - cool_hash = 299_792_458 - class MyObj: def __hash__(self): - return cool_hash + return 42 + + obj = MyObj() + with self.assertRaises(TypeError): + hash(weakref.proxy(obj)) + + class MyObj: + __hash__ = None obj = MyObj() - self.assertEqual(hash(weakref.proxy(obj)), cool_hash) + with self.assertRaises(TypeError): + hash(weakref.proxy(obj)) def test_getweakrefcount(self): o = C() @@ -444,6 +487,7 @@ class ReferencesTestCase(TestBase): "got wrong number of weak reference objects") del ref1, ref2, proxy1, proxy2 + gc_collect() # For PyPy or other GCs. self.assertEqual(weakref.getweakrefcount(o), 0, "weak reference objects not unlinked from" " referent when discarded.") @@ -457,6 +501,7 @@ class ReferencesTestCase(TestBase): ref1 = weakref.ref(o, self.callback) ref2 = weakref.ref(o, self.callback) del ref1 + gc_collect() # For PyPy or other GCs. self.assertEqual(weakref.getweakrefs(o), [ref2], "list of refs does not match") @@ -464,10 +509,12 @@ class ReferencesTestCase(TestBase): ref1 = weakref.ref(o, self.callback) ref2 = weakref.ref(o, self.callback) del ref2 + gc_collect() # For PyPy or other GCs. self.assertEqual(weakref.getweakrefs(o), [ref1], "list of refs does not match") del ref1 + gc_collect() # For PyPy or other GCs. self.assertEqual(weakref.getweakrefs(o), [], "list of refs not cleared") @@ -953,6 +1000,7 @@ class SubclassableWeakrefTestCase(TestBase): self.assertTrue(mr.called) self.assertEqual(mr.value, 24) del o + gc_collect() # For PyPy or other GCs. self.assertIsNone(mr()) self.assertTrue(mr.called) @@ -1255,15 +1303,18 @@ class MappingTestCase(TestBase): del items1, items2 self.assertEqual(len(dict), self.COUNT) del objects[0] + gc_collect() # For PyPy or other GCs. self.assertEqual(len(dict), self.COUNT - 1, "deleting object did not cause dictionary update") del objects, o + gc_collect() # For PyPy or other GCs. self.assertEqual(len(dict), 0, "deleting the values did not clear the dictionary") # regression on SF bug #447152: dict = weakref.WeakValueDictionary() self.assertRaises(KeyError, dict.__getitem__, 1) dict[2] = C() + gc_collect() # For PyPy or other GCs. self.assertRaises(KeyError, dict.__getitem__, 2) def test_weak_keys(self): @@ -1284,9 +1335,11 @@ class MappingTestCase(TestBase): del items1, items2 self.assertEqual(len(dict), self.COUNT) del objects[0] + gc_collect() # For PyPy or other GCs. self.assertEqual(len(dict), (self.COUNT - 1), "deleting object did not cause dictionary update") del objects, o + gc_collect() # For PyPy or other GCs. self.assertEqual(len(dict), 0, "deleting the keys did not clear the dictionary") o = Object(42) @@ -1785,6 +1838,7 @@ class MappingTestCase(TestBase): for o in objs: count += 1 del d[o] + gc_collect() # For PyPy or other GCs. self.assertEqual(len(d), 0) self.assertEqual(count, 2) @@ -2093,6 +2147,7 @@ class FinalizeTestCase(unittest.TestCase): libreftest = """ Doctest for examples in the library reference: weakref.rst +>>> from test.support import gc_collect >>> import weakref >>> class Dict(dict): ... pass @@ -2112,6 +2167,7 @@ True >>> o is o2 True >>> del o, o2 +>>> gc_collect() # For PyPy or other GCs. >>> print(r()) None @@ -2164,6 +2220,7 @@ True >>> id2obj(a_id) is a True >>> del a +>>> gc_collect() # For PyPy or other GCs. >>> try: ... id2obj(a_id) ... except KeyError: diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 49a9b5c3..9b31d5fc 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -5,6 +5,7 @@ from collections import UserString as ustr from collections.abc import Set, MutableSet import gc import contextlib +from test import support class Foo: @@ -48,6 +49,7 @@ class TestWeakSet(unittest.TestCase): self.assertEqual(len(self.s), len(self.d)) self.assertEqual(len(self.fs), 1) del self.obj + support.gc_collect() # For PyPy or other GCs. self.assertEqual(len(self.fs), 0) def test_contains(self): @@ -57,6 +59,7 @@ class TestWeakSet(unittest.TestCase): self.assertNotIn(1, self.s) self.assertIn(self.obj, self.fs) del self.obj + support.gc_collect() # For PyPy or other GCs. self.assertNotIn(ustr('F'), self.fs) def test_union(self): @@ -215,6 +218,7 @@ class TestWeakSet(unittest.TestCase): self.assertEqual(self.s, dup) self.assertRaises(TypeError, self.s.add, []) self.fs.add(Foo()) + support.gc_collect() # For PyPy or other GCs. self.assertTrue(len(self.fs) == 1) self.fs.add(self.obj) self.assertTrue(len(self.fs) == 1) @@ -406,6 +410,7 @@ class TestWeakSet(unittest.TestCase): n1 = len(s) del it gc.collect() + gc.collect() # For PyPy or other GCs. n2 = len(s) # one item may be kept alive inside the iterator self.assertIn(n1, (0, 1)) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index bfc0d054..3a6d43b8 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2456,6 +2456,7 @@ class BasicElementTest(ElementTestCase, unittest.TestCase): wref = weakref.ref(e, wref_cb) self.assertEqual(wref().tag, 'e') del e + gc_collect() # For PyPy or other GCs. self.assertEqual(flag, True) self.assertEqual(wref(), None) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 56028607..80c98b4b 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -34,14 +34,9 @@ raise_src = 'def do_raise(): raise TypeError\n' def make_pyc(co, mtime, size): data = marshal.dumps(co) - if type(mtime) is type(0.0): - # Mac mtimes need a bit of special casing - if mtime < 0x7fffffff: - mtime = int(mtime) - else: - mtime = int(-0x100000000 + int(mtime)) pyc = (importlib.util.MAGIC_NUMBER + - struct.pack(" frame -> outcome -> outcome.errors + # outcome.expectedFailure -> frame -> outcome -> outcome.expectedFailure + outcome.errors.clear() + outcome.expectedFailure = None + + # clear the outcome, no more needed + self._outcome = None + finally: result.stopTest(self) - if orig_result is None: - stopTestRun = getattr(result, 'stopTestRun', None) - if stopTestRun is not None: - stopTestRun() - - # explicitly break reference cycles: - # outcome.errors -> frame -> outcome -> outcome.errors - # outcome.expectedFailure -> frame -> outcome -> outcome.expectedFailure - outcome.errors.clear() - outcome.expectedFailure = None - - # clear the outcome, no more needed - self._outcome = None + if stopTestRun is not None: + stopTestRun() def doCleanups(self): """Execute all cleanup functions. Normally called for you after diff --git a/Lib/unittest/suite.py b/Lib/unittest/suite.py index 41993f9c..6f45b6fe 100644 --- a/Lib/unittest/suite.py +++ b/Lib/unittest/suite.py @@ -149,6 +149,7 @@ class TestSuite(BaseTestSuite): if getattr(currentClass, "__unittest_skip__", False): return + failed = False try: currentClass._classSetupFailed = False except TypeError: @@ -157,27 +158,32 @@ class TestSuite(BaseTestSuite): pass setUpClass = getattr(currentClass, 'setUpClass', None) + doClassCleanups = getattr(currentClass, 'doClassCleanups', None) if setUpClass is not None: _call_if_exists(result, '_setupStdout') try: - setUpClass() - except Exception as e: - if isinstance(result, _DebugResult): - raise - currentClass._classSetupFailed = True - className = util.strclass(currentClass) - self._createClassOrModuleLevelException(result, e, - 'setUpClass', - className) + try: + setUpClass() + except Exception as e: + if isinstance(result, _DebugResult): + raise + failed = True + try: + currentClass._classSetupFailed = True + except TypeError: + pass + className = util.strclass(currentClass) + self._createClassOrModuleLevelException(result, e, + 'setUpClass', + className) + if failed and doClassCleanups is not None: + doClassCleanups() + for exc_info in currentClass.tearDown_exceptions: + self._createClassOrModuleLevelException( + result, exc_info[1], 'setUpClass', className, + info=exc_info) finally: _call_if_exists(result, '_restoreStdout') - if currentClass._classSetupFailed is True: - currentClass.doClassCleanups() - if len(currentClass.tearDown_exceptions) > 0: - for exc in currentClass.tearDown_exceptions: - self._createClassOrModuleLevelException( - result, exc[1], 'setUpClass', className, - info=exc) def _get_previous_module(self, result): previousModule = None @@ -205,20 +211,22 @@ class TestSuite(BaseTestSuite): if setUpModule is not None: _call_if_exists(result, '_setupStdout') try: - setUpModule() - except Exception as e: try: - case.doModuleCleanups() - except Exception as exc: - self._createClassOrModuleLevelException(result, exc, + setUpModule() + except Exception as e: + if isinstance(result, _DebugResult): + raise + result._moduleSetUpFailed = True + self._createClassOrModuleLevelException(result, e, 'setUpModule', currentModule) - if isinstance(result, _DebugResult): - raise - result._moduleSetUpFailed = True - self._createClassOrModuleLevelException(result, e, - 'setUpModule', - currentModule) + if result._moduleSetUpFailed: + try: + case.doModuleCleanups() + except Exception as e: + self._createClassOrModuleLevelException(result, e, + 'setUpModule', + currentModule) finally: _call_if_exists(result, '_restoreStdout') @@ -251,30 +259,33 @@ class TestSuite(BaseTestSuite): except KeyError: return - tearDownModule = getattr(module, 'tearDownModule', None) - if tearDownModule is not None: - _call_if_exists(result, '_setupStdout') + _call_if_exists(result, '_setupStdout') + try: + tearDownModule = getattr(module, 'tearDownModule', None) + if tearDownModule is not None: + try: + tearDownModule() + except Exception as e: + if isinstance(result, _DebugResult): + raise + self._createClassOrModuleLevelException(result, e, + 'tearDownModule', + previousModule) try: - tearDownModule() + case.doModuleCleanups() except Exception as e: if isinstance(result, _DebugResult): raise self._createClassOrModuleLevelException(result, e, 'tearDownModule', previousModule) - finally: - _call_if_exists(result, '_restoreStdout') - try: - case.doModuleCleanups() - except Exception as e: - self._createClassOrModuleLevelException(result, e, - 'tearDownModule', - previousModule) + finally: + _call_if_exists(result, '_restoreStdout') def _tearDownPreviousClass(self, test, result): previousClass = getattr(result, '_previousTestClass', None) currentClass = test.__class__ - if currentClass == previousClass: + if currentClass == previousClass or previousClass is None: return if getattr(previousClass, '_classSetupFailed', False): return @@ -284,27 +295,34 @@ class TestSuite(BaseTestSuite): return tearDownClass = getattr(previousClass, 'tearDownClass', None) - if tearDownClass is not None: - _call_if_exists(result, '_setupStdout') - try: - tearDownClass() - except Exception as e: - if isinstance(result, _DebugResult): - raise - className = util.strclass(previousClass) - self._createClassOrModuleLevelException(result, e, - 'tearDownClass', - className) - finally: - _call_if_exists(result, '_restoreStdout') - previousClass.doClassCleanups() - if len(previousClass.tearDown_exceptions) > 0: - for exc in previousClass.tearDown_exceptions: - className = util.strclass(previousClass) - self._createClassOrModuleLevelException(result, exc[1], - 'tearDownClass', - className, - info=exc) + doClassCleanups = getattr(previousClass, 'doClassCleanups', None) + if tearDownClass is None and doClassCleanups is None: + return + + _call_if_exists(result, '_setupStdout') + try: + if tearDownClass is not None: + try: + tearDownClass() + except Exception as e: + if isinstance(result, _DebugResult): + raise + className = util.strclass(previousClass) + self._createClassOrModuleLevelException(result, e, + 'tearDownClass', + className) + if doClassCleanups is not None: + doClassCleanups() + for exc_info in previousClass.tearDown_exceptions: + if isinstance(result, _DebugResult): + raise exc_info[1] + className = util.strclass(previousClass) + self._createClassOrModuleLevelException(result, exc_info[1], + 'tearDownClass', + className, + info=exc_info) + finally: + _call_if_exists(result, '_restoreStdout') class _ErrorHolder(object): diff --git a/Lib/unittest/test/test_assertions.py b/Lib/unittest/test/test_assertions.py index f5e64d68..a0db3423 100644 --- a/Lib/unittest/test/test_assertions.py +++ b/Lib/unittest/test/test_assertions.py @@ -2,6 +2,7 @@ import datetime import warnings import weakref import unittest +from test.support import gc_collect from itertools import product @@ -124,8 +125,10 @@ class Test_Assertions(unittest.TestCase): self.foo() Foo("test_functional").run() + gc_collect() # For PyPy or other GCs. self.assertIsNone(wr()) Foo("test_with").run() + gc_collect() # For PyPy or other GCs. self.assertIsNone(wr()) def testAssertNotRegex(self): diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py index f855c4dc..590dd749 100644 --- a/Lib/unittest/test/test_case.py +++ b/Lib/unittest/test/test_case.py @@ -18,7 +18,7 @@ from unittest.test.support import ( TestEquality, TestHashing, LoggingResult, LegacyLoggingResult, ResultWithNoStartTestRunStopTestRun ) -from test.support import captured_stderr +from test.support import captured_stderr, gc_collect log_foo = logging.getLogger('foo') @@ -1845,6 +1845,7 @@ test case for method_name in ('test1', 'test2'): testcase = TestCase(method_name) testcase.run() + gc_collect() # For PyPy or other GCs. self.assertEqual(MyException.ninstance, 0) diff --git a/Lib/unittest/test/test_result.py b/Lib/unittest/test/test_result.py index 0ffb87b4..0f7f3b7e 100644 --- a/Lib/unittest/test/test_result.py +++ b/Lib/unittest/test/test_result.py @@ -3,9 +3,11 @@ import sys import textwrap from test import support +from test.support import captured_stdout import traceback import unittest +from unittest.util import strclass class MockTraceback(object): @@ -22,6 +24,16 @@ def restore_traceback(): unittest.result.traceback = traceback +def bad_cleanup1(): + print('do cleanup1') + raise TypeError('bad cleanup1') + + +def bad_cleanup2(): + print('do cleanup2') + raise ValueError('bad cleanup2') + + class Test_TestResult(unittest.TestCase): # Note: there are not separate tests for TestResult.wasSuccessful(), # TestResult.errors, TestResult.failures, TestResult.testsRun or @@ -633,36 +645,320 @@ class TestOutputBuffering(unittest.TestCase): self.assertEqual(result._original_stderr.getvalue(), expectedErrMessage) self.assertMultiLineEqual(message, expectedFullMessage) + def testBufferSetUp(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def setUp(self): + print('set up') + 1/0 + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\nset up\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 1) + description = f'test_foo ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(str(test_case), description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferTearDown(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def tearDown(self): + print('tear down') + 1/0 + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\ntear down\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 1) + description = f'test_foo ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(str(test_case), description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferDoCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def setUp(self): + print('set up') + self.addCleanup(bad_cleanup1) + self.addCleanup(bad_cleanup2) + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\nset up\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 2) + description = f'test_foo ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(str(test_case), description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(str(test_case), description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferSetUp_DoCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def setUp(self): + print('set up') + self.addCleanup(bad_cleanup1) + self.addCleanup(bad_cleanup2) + 1/0 + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\nset up\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 3) + description = f'test_foo ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(str(test_case), description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(str(test_case), description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertEqual(str(test_case), description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferTearDown_DoCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def setUp(self): + print('set up') + self.addCleanup(bad_cleanup1) + self.addCleanup(bad_cleanup2) + def tearDown(self): + print('tear down') + 1/0 + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\nset up\ntear down\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 3) + description = f'test_foo ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(str(test_case), description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(str(test_case), description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertEqual(str(test_case), description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + def testBufferSetupClass(self): - result = unittest.TestResult() + with captured_stdout() as stdout: + result = unittest.TestResult() result.buffer = True class Foo(unittest.TestCase): @classmethod def setUpClass(cls): + print('set up class') 1/0 def test_foo(self): pass suite = unittest.TestSuite([Foo('test_foo')]) suite(result) + expected_out = '\nStdout:\nset up class\n' + self.assertEqual(stdout.getvalue(), expected_out) self.assertEqual(len(result.errors), 1) + description = f'setUpClass ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertIn(expected_out, formatted_exc) def testBufferTearDownClass(self): - result = unittest.TestResult() + with captured_stdout() as stdout: + result = unittest.TestResult() result.buffer = True class Foo(unittest.TestCase): @classmethod def tearDownClass(cls): + print('tear down class') 1/0 def test_foo(self): pass suite = unittest.TestSuite([Foo('test_foo')]) suite(result) + expected_out = '\nStdout:\ntear down class\n' + self.assertEqual(stdout.getvalue(), expected_out) self.assertEqual(len(result.errors), 1) + description = f'tearDownClass ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferDoClassCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + @classmethod + def setUpClass(cls): + print('set up class') + cls.addClassCleanup(bad_cleanup1) + cls.addClassCleanup(bad_cleanup2) + @classmethod + def tearDownClass(cls): + print('tear down class') + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\ntear down class\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 2) + description = f'tearDownClass ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferSetupClass_DoClassCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + @classmethod + def setUpClass(cls): + print('set up class') + cls.addClassCleanup(bad_cleanup1) + cls.addClassCleanup(bad_cleanup2) + 1/0 + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\nset up class\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 3) + description = f'setUpClass ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn('\nStdout:\nset up class\n', formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(test_case.description, description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferTearDownClass_DoClassCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + @classmethod + def setUpClass(cls): + print('set up class') + cls.addClassCleanup(bad_cleanup1) + cls.addClassCleanup(bad_cleanup2) + @classmethod + def tearDownClass(cls): + print('tear down class') + 1/0 + def test_foo(self): + pass + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\ntear down class\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 3) + description = f'tearDownClass ({strclass(Foo)})' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn('\nStdout:\ntear down class\n', formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(test_case.description, description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) def testBufferSetUpModule(self): - result = unittest.TestResult() + with captured_stdout() as stdout: + result = unittest.TestResult() result.buffer = True class Foo(unittest.TestCase): @@ -671,6 +967,7 @@ class TestOutputBuffering(unittest.TestCase): class Module(object): @staticmethod def setUpModule(): + print('set up module') 1/0 Foo.__module__ = 'Module' @@ -678,10 +975,18 @@ class TestOutputBuffering(unittest.TestCase): self.addCleanup(sys.modules.pop, 'Module') suite = unittest.TestSuite([Foo('test_foo')]) suite(result) + expected_out = '\nStdout:\nset up module\n' + self.assertEqual(stdout.getvalue(), expected_out) self.assertEqual(len(result.errors), 1) + description = 'setUpModule (Module)' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertIn(expected_out, formatted_exc) def testBufferTearDownModule(self): - result = unittest.TestResult() + with captured_stdout() as stdout: + result = unittest.TestResult() result.buffer = True class Foo(unittest.TestCase): @@ -690,6 +995,7 @@ class TestOutputBuffering(unittest.TestCase): class Module(object): @staticmethod def tearDownModule(): + print('tear down module') 1/0 Foo.__module__ = 'Module' @@ -697,7 +1003,124 @@ class TestOutputBuffering(unittest.TestCase): self.addCleanup(sys.modules.pop, 'Module') suite = unittest.TestSuite([Foo('test_foo')]) suite(result) + expected_out = '\nStdout:\ntear down module\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 1) + description = 'tearDownModule (Module)' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferDoModuleCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def test_foo(self): + pass + class Module(object): + @staticmethod + def setUpModule(): + print('set up module') + unittest.addModuleCleanup(bad_cleanup1) + unittest.addModuleCleanup(bad_cleanup2) + + Foo.__module__ = 'Module' + sys.modules['Module'] = Module + self.addCleanup(sys.modules.pop, 'Module') + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) self.assertEqual(len(result.errors), 1) + description = 'tearDownModule (Module)' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferSetUpModule_DoModuleCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def test_foo(self): + pass + class Module(object): + @staticmethod + def setUpModule(): + print('set up module') + unittest.addModuleCleanup(bad_cleanup1) + unittest.addModuleCleanup(bad_cleanup2) + 1/0 + + Foo.__module__ = 'Module' + sys.modules['Module'] = Module + self.addCleanup(sys.modules.pop, 'Module') + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\nset up module\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 2) + description = 'setUpModule (Module)' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn('\nStdout:\nset up module\n', formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertIn(expected_out, formatted_exc) + self.assertEqual(test_case.description, description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + + def testBufferTearDownModule_DoModuleCleanups(self): + with captured_stdout() as stdout: + result = unittest.TestResult() + result.buffer = True + + class Foo(unittest.TestCase): + def test_foo(self): + pass + class Module(object): + @staticmethod + def setUpModule(): + print('set up module') + unittest.addModuleCleanup(bad_cleanup1) + unittest.addModuleCleanup(bad_cleanup2) + @staticmethod + def tearDownModule(): + print('tear down module') + 1/0 + + Foo.__module__ = 'Module' + sys.modules['Module'] = Module + self.addCleanup(sys.modules.pop, 'Module') + suite = unittest.TestSuite([Foo('test_foo')]) + suite(result) + expected_out = '\nStdout:\ntear down module\ndo cleanup2\ndo cleanup1\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(len(result.errors), 2) + description = 'tearDownModule (Module)' + test_case, formatted_exc = result.errors[0] + self.assertEqual(test_case.description, description) + self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn('\nStdout:\ntear down module\n', formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(test_case.description, description) + self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('TypeError', formatted_exc) + self.assertIn(expected_out, formatted_exc) if __name__ == '__main__': diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py index dd9a1b6d..453e6c3d 100644 --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -222,14 +222,42 @@ class TestClassCleanup(unittest.TestCase): self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) - def test_debug_executes_classCleanUp(self): + def test_run_class_cleanUp_without_tearDownClass(self): ordering = [] + blowUp = True class TestableTest(unittest.TestCase): @classmethod def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) + if blowUp: + raise Exception() + def testNothing(self): + ordering.append('test') + @classmethod + @property + def tearDownClass(cls): + raise AttributeError + + runTests(TestableTest) + self.assertEqual(ordering, ['setUpClass', 'cleanup_good']) + + ordering = [] + blowUp = False + runTests(TestableTest) + self.assertEqual(ordering, + ['setUpClass', 'test', 'cleanup_good']) + + def test_debug_executes_classCleanUp(self): + ordering = [] + blowUp = False + + class TestableTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('setUpClass') + cls.addClassCleanup(cleanup, ordering, blowUp=blowUp) def testNothing(self): ordering.append('test') @classmethod @@ -241,6 +269,48 @@ class TestClassCleanup(unittest.TestCase): self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) + ordering = [] + blowUp = True + suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) + with self.assertRaises(Exception) as cm: + suite.debug() + self.assertEqual(str(cm.exception), 'CleanUpExc') + self.assertEqual(ordering, + ['setUpClass', 'test', 'tearDownClass', 'cleanup_exc']) + + def test_debug_executes_classCleanUp_when_teardown_exception(self): + ordering = [] + blowUp = False + + class TestableTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('setUpClass') + cls.addClassCleanup(cleanup, ordering, blowUp=blowUp) + def testNothing(self): + ordering.append('test') + @classmethod + def tearDownClass(cls): + raise Exception('TearDownClassExc') + + suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) + with self.assertRaises(Exception) as cm: + suite.debug() + self.assertEqual(str(cm.exception), 'TearDownClassExc') + self.assertEqual(ordering, ['setUpClass', 'test']) + self.assertTrue(TestableTest._class_cleanups) + TestableTest._class_cleanups.clear() + + ordering = [] + blowUp = True + suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) + with self.assertRaises(Exception) as cm: + suite.debug() + self.assertEqual(str(cm.exception), 'TearDownClassExc') + self.assertEqual(ordering, ['setUpClass', 'test']) + self.assertTrue(TestableTest._class_cleanups) + TestableTest._class_cleanups.clear() + def test_doClassCleanups_with_errors_addClassCleanUp(self): class TestableTest(unittest.TestCase): def testNothing(self): @@ -332,6 +402,7 @@ class TestClassCleanup(unittest.TestCase): self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'tearDownClass', 'cleanup_exc']) + ordering = [] class_blow_up = True method_blow_up = False @@ -355,6 +426,26 @@ class TestClassCleanup(unittest.TestCase): ['setUpClass', 'setUp', 'tearDownClass', 'cleanup_exc']) + def test_with_errors_in_tearDownClass(self): + ordering = [] + class TestableTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('setUpClass') + cls.addClassCleanup(cleanup, ordering) + def testNothing(self): + ordering.append('test') + @classmethod + def tearDownClass(cls): + ordering.append('tearDownClass') + raise Exception('TearDownExc') + + result = runTests(TestableTest) + self.assertEqual(result.errors[0][1].splitlines()[-1], + 'Exception: TearDownExc') + self.assertEqual(ordering, + ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) + class TestModuleCleanUp(unittest.TestCase): def test_add_and_do_ModuleCleanup(self): @@ -532,13 +623,69 @@ class TestModuleCleanUp(unittest.TestCase): 'tearDownModule2', 'cleanup_good']) self.assertEqual(unittest.case._module_cleanups, []) - def test_debug_module_executes_cleanUp(self): + def test_run_module_cleanUp_without_teardown(self): ordering = [] class Module(object): @staticmethod def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) + + class TestableTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('setUpClass') + def testNothing(self): + ordering.append('test') + @classmethod + def tearDownClass(cls): + ordering.append('tearDownClass') + + TestableTest.__module__ = 'Module' + sys.modules['Module'] = Module + runTests(TestableTest) + self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', + 'tearDownClass', 'cleanup_good']) + self.assertEqual(unittest.case._module_cleanups, []) + + def test_run_module_cleanUp_when_teardown_exception(self): + ordering = [] + class Module(object): + @staticmethod + def setUpModule(): + ordering.append('setUpModule') + unittest.addModuleCleanup(cleanup, ordering) + @staticmethod + def tearDownModule(): + raise Exception('CleanUpExc') + + class TestableTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('setUpClass') + def testNothing(self): + ordering.append('test') + @classmethod + def tearDownClass(cls): + ordering.append('tearDownClass') + + TestableTest.__module__ = 'Module' + sys.modules['Module'] = Module + result = runTests(TestableTest) + self.assertEqual(result.errors[0][1].splitlines()[-1], + 'Exception: CleanUpExc') + self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', + 'tearDownClass', 'cleanup_good']) + self.assertEqual(unittest.case._module_cleanups, []) + + def test_debug_module_executes_cleanUp(self): + ordering = [] + blowUp = False + class Module(object): + @staticmethod + def setUpModule(): + ordering.append('setUpModule') + unittest.addModuleCleanup(cleanup, ordering, blowUp=blowUp) @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -562,6 +709,60 @@ class TestModuleCleanUp(unittest.TestCase): 'tearDownModule', 'cleanup_good']) self.assertEqual(unittest.case._module_cleanups, []) + ordering = [] + blowUp = True + suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) + with self.assertRaises(Exception) as cm: + suite.debug() + self.assertEqual(str(cm.exception), 'CleanUpExc') + self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', + 'tearDownClass', 'tearDownModule', 'cleanup_exc']) + self.assertEqual(unittest.case._module_cleanups, []) + + def test_debug_module_cleanUp_when_teardown_exception(self): + ordering = [] + blowUp = False + class Module(object): + @staticmethod + def setUpModule(): + ordering.append('setUpModule') + unittest.addModuleCleanup(cleanup, ordering, blowUp=blowUp) + @staticmethod + def tearDownModule(): + raise Exception('TearDownModuleExc') + + class TestableTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + ordering.append('setUpClass') + def testNothing(self): + ordering.append('test') + @classmethod + def tearDownClass(cls): + ordering.append('tearDownClass') + + TestableTest.__module__ = 'Module' + sys.modules['Module'] = Module + suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) + with self.assertRaises(Exception) as cm: + suite.debug() + self.assertEqual(str(cm.exception), 'TearDownModuleExc') + self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', + 'tearDownClass']) + self.assertTrue(unittest.case._module_cleanups) + unittest.case._module_cleanups.clear() + + ordering = [] + blowUp = True + suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) + with self.assertRaises(Exception) as cm: + suite.debug() + self.assertEqual(str(cm.exception), 'TearDownModuleExc') + self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', + 'tearDownClass']) + self.assertTrue(unittest.case._module_cleanups) + unittest.case._module_cleanups.clear() + def test_addClassCleanup_arg_errors(self): cleanups = [] def cleanup(*args, **kwargs): @@ -717,9 +918,9 @@ class TestModuleCleanUp(unittest.TestCase): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') - self.assertEqual(result.errors[1][1].splitlines()[-1], 'Exception: ModuleExc') + self.assertEqual(result.errors[1][1].splitlines()[-1], + 'Exception: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'cleanup_exc']) ordering = [] diff --git a/Lib/unittest/test/test_skipping.py b/Lib/unittest/test/test_skipping.py index 1c178a95..79b27e6d 100644 --- a/Lib/unittest/test/test_skipping.py +++ b/Lib/unittest/test/test_skipping.py @@ -7,30 +7,50 @@ class Test_TestSkipping(unittest.TestCase): def test_skipping(self): class Foo(unittest.TestCase): + def defaultTestResult(self): + return LoggingResult(events) def test_skip_me(self): self.skipTest("skip") events = [] result = LoggingResult(events) test = Foo("test_skip_me") - test.run(result) + self.assertIs(test.run(result), result) self.assertEqual(events, ['startTest', 'addSkip', 'stopTest']) self.assertEqual(result.skipped, [(test, "skip")]) + events = [] + result = test.run() + self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip', + 'stopTest', 'stopTestRun']) + self.assertEqual(result.skipped, [(test, "skip")]) + self.assertEqual(result.testsRun, 1) + # Try letting setUp skip the test now. class Foo(unittest.TestCase): + def defaultTestResult(self): + return LoggingResult(events) def setUp(self): self.skipTest("testing") def test_nothing(self): pass events = [] result = LoggingResult(events) test = Foo("test_nothing") - test.run(result) + self.assertIs(test.run(result), result) self.assertEqual(events, ['startTest', 'addSkip', 'stopTest']) self.assertEqual(result.skipped, [(test, "testing")]) self.assertEqual(result.testsRun, 1) + events = [] + result = test.run() + self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip', + 'stopTest', 'stopTestRun']) + self.assertEqual(result.skipped, [(test, "testing")]) + self.assertEqual(result.testsRun, 1) + def test_skipping_subtests(self): class Foo(unittest.TestCase): + def defaultTestResult(self): + return LoggingResult(events) def test_skip_me(self): with self.subTest(a=1): with self.subTest(b=2): @@ -40,7 +60,7 @@ class Test_TestSkipping(unittest.TestCase): events = [] result = LoggingResult(events) test = Foo("test_skip_me") - test.run(result) + self.assertIs(test.run(result), result) self.assertEqual(events, ['startTest', 'addSkip', 'addSkip', 'addSkip', 'stopTest']) self.assertEqual(len(result.skipped), 3) @@ -54,11 +74,22 @@ class Test_TestSkipping(unittest.TestCase): self.assertIsNot(subtest, test) self.assertEqual(result.skipped[2], (test, "skip 3")) + events = [] + result = test.run() + self.assertEqual(events, + ['startTestRun', 'startTest', 'addSkip', 'addSkip', + 'addSkip', 'stopTest', 'stopTestRun']) + self.assertEqual([msg for subtest, msg in result.skipped], + ['skip 1', 'skip 2', 'skip 3']) + def test_skipping_decorators(self): op_table = ((unittest.skipUnless, False, True), (unittest.skipIf, True, False)) for deco, do_skip, dont_skip in op_table: class Foo(unittest.TestCase): + def defaultTestResult(self): + return LoggingResult(events) + @deco(do_skip, "testing") def test_skip(self): pass @@ -66,10 +97,11 @@ class Test_TestSkipping(unittest.TestCase): def test_dont_skip(self): pass test_do_skip = Foo("test_skip") test_dont_skip = Foo("test_dont_skip") + suite = unittest.TestSuite([test_do_skip, test_dont_skip]) events = [] result = LoggingResult(events) - suite.run(result) + self.assertIs(suite.run(result), result) self.assertEqual(len(result.skipped), 1) expected = ['startTest', 'addSkip', 'stopTest', 'startTest', 'addSuccess', 'stopTest'] @@ -78,16 +110,39 @@ class Test_TestSkipping(unittest.TestCase): self.assertEqual(result.skipped, [(test_do_skip, "testing")]) self.assertTrue(result.wasSuccessful()) + events = [] + result = test_do_skip.run() + self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip', + 'stopTest', 'stopTestRun']) + self.assertEqual(result.skipped, [(test_do_skip, "testing")]) + + events = [] + result = test_dont_skip.run() + self.assertEqual(events, ['startTestRun', 'startTest', 'addSuccess', + 'stopTest', 'stopTestRun']) + self.assertEqual(result.skipped, []) + def test_skip_class(self): @unittest.skip("testing") class Foo(unittest.TestCase): + def defaultTestResult(self): + return LoggingResult(events) def test_1(self): record.append(1) + events = [] record = [] - result = unittest.TestResult() + result = LoggingResult(events) test = Foo("test_1") suite = unittest.TestSuite([test]) - suite.run(result) + self.assertIs(suite.run(result), result) + self.assertEqual(events, ['startTest', 'addSkip', 'stopTest']) + self.assertEqual(result.skipped, [(test, "testing")]) + self.assertEqual(record, []) + + events = [] + result = test.run() + self.assertEqual(events, ['startTestRun', 'startTest', 'addSkip', + 'stopTest', 'stopTestRun']) self.assertEqual(result.skipped, [(test, "testing")]) self.assertEqual(record, []) @@ -102,7 +157,7 @@ class Test_TestSkipping(unittest.TestCase): result = unittest.TestResult() test = Foo("test_1") suite = unittest.TestSuite([test]) - suite.run(result) + self.assertIs(suite.run(result), result) self.assertEqual(result.skipped, [(test, "testing")]) self.assertEqual(record, []) @@ -114,7 +169,7 @@ class Test_TestSkipping(unittest.TestCase): events = [] result = LoggingResult(events) test = Foo("test_die") - test.run(result) + self.assertIs(test.run(result), result) self.assertEqual(events, ['startTest', 'addExpectedFailure', 'stopTest']) self.assertEqual(result.expectedFailures[0][0], test) @@ -129,7 +184,7 @@ class Test_TestSkipping(unittest.TestCase): events = [] result = LoggingResult(events) test = Foo("test_1") - test.run(result) + self.assertIs(test.run(result), result) self.assertEqual(events, ['startTest', 'addExpectedFailure', 'stopTest']) self.assertEqual(result.expectedFailures[0][0], test) @@ -147,7 +202,7 @@ class Test_TestSkipping(unittest.TestCase): events = [] result = LoggingResult(events) test = Bar("test_1") - test.run(result) + self.assertIs(test.run(result), result) self.assertEqual(events, ['startTest', 'addExpectedFailure', 'stopTest']) self.assertEqual(result.expectedFailures[0][0], test) @@ -170,7 +225,7 @@ class Test_TestSkipping(unittest.TestCase): events = [] result = LoggingResult(events) test = Foo("test_die") - test.run(result) + self.assertIs(test.run(result), result) self.assertEqual(events, ['startTest', 'addSubTestSuccess', 'addExpectedFailure', 'stopTest']) @@ -186,7 +241,7 @@ class Test_TestSkipping(unittest.TestCase): events = [] result = LoggingResult(events) test = Foo("test_die") - test.run(result) + self.assertIs(test.run(result), result) self.assertEqual(events, ['startTest', 'addUnexpectedSuccess', 'stopTest']) self.assertFalse(result.failures) @@ -208,7 +263,7 @@ class Test_TestSkipping(unittest.TestCase): events = [] result = LoggingResult(events) test = Foo("test_die") - test.run(result) + self.assertIs(test.run(result), result) self.assertEqual(events, ['startTest', 'addSubTestSuccess', 'addSubTestSuccess', @@ -232,7 +287,7 @@ class Test_TestSkipping(unittest.TestCase): result = unittest.TestResult() test = Foo("test_1") suite = unittest.TestSuite([test]) - suite.run(result) + self.assertIs(suite.run(result), result) self.assertEqual(result.skipped, [(test, "testing")]) self.assertFalse(Foo.wasSetUp) self.assertFalse(Foo.wasTornDown) @@ -252,7 +307,7 @@ class Test_TestSkipping(unittest.TestCase): result = unittest.TestResult() test = Foo("test_1") suite = unittest.TestSuite([test]) - suite.run(result) + self.assertIs(suite.run(result), result) self.assertEqual(result.skipped, [(test, "testing")]) def test_skip_without_reason(self): @@ -264,7 +319,7 @@ class Test_TestSkipping(unittest.TestCase): result = unittest.TestResult() test = Foo("test_1") suite = unittest.TestSuite([test]) - suite.run(result) + self.assertIs(suite.run(result), result) self.assertEqual(result.skipped, [(test, "")]) if __name__ == "__main__": diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index af3fb9ca..bbdc2254 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -64,7 +64,7 @@ opener = urllib.request.build_opener(proxy_support, authinfo, # install it urllib.request.install_opener(opener) -f = urllib.request.urlopen('http://www.python.org/') +f = urllib.request.urlopen('https://www.python.org/') """ # XXX issues: diff --git a/Lib/weakref.py b/Lib/weakref.py index 5fa851dd..994ea8aa 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -2,7 +2,7 @@ This module is an implementation of PEP 205: -http://www.python.org/dev/peps/pep-0205/ +https://www.python.org/dev/peps/pep-0205/ """ # Naming convention: Variables named "wr" are weak reference objects; @@ -119,14 +119,17 @@ class WeakValueDictionary(_collections_abc.MutableMapping): self.data = {} self.update(other, **kw) - def _commit_removals(self): - l = self._pending_removals + def _commit_removals(self, _atomic_removal=_remove_dead_weakref): + pop = self._pending_removals.pop d = self.data # We shouldn't encounter any KeyError, because this method should # always be called *before* mutating the dict. - while l: - key = l.pop() - _remove_dead_weakref(d, key) + while True: + try: + key = pop() + except IndexError: + return + _atomic_removal(d, key) def __getitem__(self, key): if self._pending_removals: @@ -370,7 +373,10 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): if self._iterating: self._pending_removals.append(k) else: - del self.data[k] + try: + del self.data[k] + except KeyError: + pass self._remove = remove # A list of dead weakrefs (keys to be removed) self._pending_removals = [] @@ -384,11 +390,16 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): # because a dead weakref never compares equal to a live weakref, # even if they happened to refer to equal objects. # However, it means keys may already have been removed. - l = self._pending_removals + pop = self._pending_removals.pop d = self.data - while l: + while True: + try: + key = pop() + except IndexError: + return + try: - del d[l.pop()] + del d[key] except KeyError: pass diff --git a/Lib/xml/etree/ElementInclude.py b/Lib/xml/etree/ElementInclude.py index 53030627..40a9b222 100644 --- a/Lib/xml/etree/ElementInclude.py +++ b/Lib/xml/etree/ElementInclude.py @@ -42,7 +42,7 @@ # -------------------------------------------------------------------- # Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/psf/license for licensing details. +# See https://www.python.org/psf/license for licensing details. ## # Limited XInclude support for the ElementTree package. diff --git a/Lib/xml/etree/ElementPath.py b/Lib/xml/etree/ElementPath.py index d318e65d..880ea7bd 100644 --- a/Lib/xml/etree/ElementPath.py +++ b/Lib/xml/etree/ElementPath.py @@ -48,7 +48,7 @@ # -------------------------------------------------------------------- # Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/psf/license for licensing details. +# See https://www.python.org/psf/license for licensing details. ## # Implementation module for XPath support. There's usually no reason diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index ac82ed80..fde303c8 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -35,7 +35,7 @@ #--------------------------------------------------------------------- # Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/psf/license for licensing details. +# See https://www.python.org/psf/license for licensing details. # # ElementTree # Copyright (c) 1999-2008 by Fredrik Lundh. All rights reserved. @@ -1283,7 +1283,7 @@ class XMLPullParser: def __init__(self, events=None, *, _parser=None): # The _parser argument is for internal use only and must not be relied # upon in user code. It will be removed in a future release. - # See http://bugs.python.org/issue17741 for more details. + # See https://bugs.python.org/issue17741 for more details. self._events_queue = collections.deque() self._parser = _parser or XMLParser(target=TreeBuilder()) diff --git a/Lib/xml/etree/__init__.py b/Lib/xml/etree/__init__.py index 27fd8f6d..e2ec5342 100644 --- a/Lib/xml/etree/__init__.py +++ b/Lib/xml/etree/__init__.py @@ -30,4 +30,4 @@ # -------------------------------------------------------------------- # Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/psf/license for licensing details. +# See https://www.python.org/psf/license for licensing details. diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 287e3243..69a260f5 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -750,7 +750,7 @@ class ServerHTMLDoc(pydoc.HTMLDoc): url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) results.append('%s' % (url, escape(all))) elif pep: - url = 'http://www.python.org/dev/peps/pep-%04d/' % int(pep) + url = 'https://www.python.org/dev/peps/pep-%04d/' % int(pep) results.append('%s' % (url, escape(all))) elif text[end:end+1] == '(': results.append(self.namelink(name, methods, funcs, classes)) diff --git a/Mac/BuildScript/0001-Darwin-platform-allows-to-build-on-releases-before-Y.patch b/Mac/BuildScript/0001-Darwin-platform-allows-to-build-on-releases-before-Y.patch new file mode 100644 index 00000000..51ccdc22 --- /dev/null +++ b/Mac/BuildScript/0001-Darwin-platform-allows-to-build-on-releases-before-Y.patch @@ -0,0 +1,59 @@ +From cef404f1e7a598166cbc2fd2e0048f7e2d752ad5 Mon Sep 17 00:00:00 2001 +From: David Carlier +Date: Tue, 24 Aug 2021 22:40:14 +0100 +Subject: [PATCH] Darwin platform allows to build on releases before + Yosemite/ios 8. + +issue #16407 #16408 +--- + crypto/rand/rand_unix.c | 5 +---- + include/crypto/rand.h | 10 ++++++++++ + 2 files changed, 11 insertions(+), 4 deletions(-) + +diff --git a/crypto/rand/rand_unix.c b/crypto/rand/rand_unix.c +index 43f1069d15..0f4525106a 100644 +--- a/crypto/rand/rand_unix.c ++++ b/crypto/rand/rand_unix.c +@@ -34,9 +34,6 @@ + #if defined(__OpenBSD__) + # include + #endif +-#if defined(__APPLE__) +-# include +-#endif + + #if defined(OPENSSL_SYS_UNIX) || defined(__DJGPP__) + # include +@@ -381,7 +378,7 @@ static ssize_t syscall_random(void *buf, size_t buflen) + if (errno != ENOSYS) + return -1; + } +-# elif defined(__APPLE__) ++# elif defined(OPENSSL_APPLE_CRYPTO_RANDOM) + if (CCRandomGenerateBytes(buf, buflen) == kCCSuccess) + return (ssize_t)buflen; + +diff --git a/include/crypto/rand.h b/include/crypto/rand.h +index 5350d3a931..674f840fd1 100644 +--- a/include/crypto/rand.h ++++ b/include/crypto/rand.h +@@ -20,6 +20,16 @@ + + # include + ++# if defined(__APPLE__) && !defined(OPENSSL_NO_APPLE_CRYPTO_RANDOM) ++# include ++# if (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101000) || \ ++ (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED >= 80000) ++# define OPENSSL_APPLE_CRYPTO_RANDOM 1 ++# include ++# include ++# endif ++# endif ++ + /* forward declaration */ + typedef struct rand_pool_st RAND_POOL; + +-- +2.33.0 + diff --git a/Mac/BuildScript/README.rst b/Mac/BuildScript/README.rst index 94a6bb28..4f74e7dc 100644 --- a/Mac/BuildScript/README.rst +++ b/Mac/BuildScript/README.rst @@ -19,33 +19,87 @@ Starting with macOS 10.15 Catalina, Gatekeeper now also requires that installer packages are submitted to and pass Apple's automated notarization service using the altool command. To pass notarization, the binaries included in the package must be built with at least -the macOS 10.9 SDK, mout now be signed with the codesign utility +the macOS 10.9 SDK, must now be signed with the codesign utility, and executables must opt in to the hardened run time option with any necessary entitlements. Details of these processes are available in the on-line Apple Developer Documentation and man pages. -As of 3.8.0 and 3.7.7, PSF practice is to build one installer variants -for each release. Note that as of this writing, no Pythons support -building on a newer version of macOS that will run on older versions +A goal of PSF-provided (python.org) Python binaries for macOS is to +support a wide-range of operating system releases with one set of +binaries. Currently, the oldest release supported by python.org +binaries is macOS 10.9; it is still possible to build Python and +Python installers on older versions of macOS but we not regularly +test on those systems nor provide binaries for them. + +Prior to Python 3.9.1, no Python releases supported building on a +newer version of macOS that will run on older versions by setting MACOSX_DEPLOYMENT_TARGET. This is because the various -Python C modules do not yet support runtime testing of macOS +Python C modules did not yet support runtime testing of macOS feature availability (for example, by using macOS AvailabilityMacros.h -and weak-linking). To build a Python that is to be used on a -range of macOS releases, always build on the oldest release to be -supported; the necessary shared libraries for that release will -normally also be available on later systems, with the occasional -exception such as the removal of 32-bit libraries in macOS 10.15. - -build-installer requires Apple Developer tools, either from the +and weak-linking). To build a Python that is to be used on a +range of macOS releases, it was necessary to always build on the +oldest release to be supported; the necessary shared libraries for +that release will normally also be available on later systems, +with the occasional exception such as the removal of 32-bit +libraries in macOS 10.15. For 3.9.x and recent earlier systems, +PSF practice was to provide a "macOS 64-bit Intel installer" variant +that was built on 10.9 that would run on macOS 10.9 and later. + +Starting with 3.9.1, Python fully supports macOS "weaklinking", +meaning it is now possible to build a Python on a current macOS version +with a deployment target of an earlier macOS system. For 3.9.1 and +later systems, we provide a "macOS 64-bit universal2 installer" +variant, currently build on macOS 11 Big Sur with fat binaries +natively supporting both Apple Silicon (arm64) and Intel-64 +(x86_64) Macs running macOS 10.9 or later. + +The legacy "macOS 64-bit Intel installer" variant is expected to +be retired prior to the end of 3.9.x support. + +build-installer.py requires Apple Developer tools, either from the Command Line Tools package or from a full Xcode installation. You should use the most recent version of either for the operating system version in use. (One notable exception: on macOS 10.6, Snow Leopard, use Xcode 3, not Xcode 4 which was released later -in the 10.6 support cycle.) +in the 10.6 support cycle.) build-installer.py also must be run +with recent versions of Python 3.x or 2.7. On older systems, +due to changes in TLS practices, it may be easier to manually +download and cache third-party source distributions used by +build-installer.py rather than have it attempt to automatically +download them. + +1. universal2, arm64 and x86_64, for OS X 10.9 (and later):: + + /path/to/bootstrap/python3 build-installer.py \ + --universal-archs=universal2 \ + --dep-target=10.9 + + - builds the following third-party libraries + + * OpenSSL 1.1.1 + * Tcl/Tk 8.6 + * NCurses + * SQLite + * XZ + * libffi + + - uses system-supplied versions of third-party libraries + + * readline module links with Apple BSD editline (libedit) + * zlib + * bz2 + + - recommended build environment: + + * Mac OS X 11 or later + * Xcode Command Line Tools 12.5 or later + * current default macOS SDK + * ``MACOSX_DEPLOYMENT_TARGET=10.9`` + * Apple ``clang`` -1. 64-bit, x86_64, for OS X 10.9 (and later):: +2. legacy Intel 64-bit, x86_64, for OS X 10.9 (and later):: - /path/to/bootstrap/python2.7 build-installer.py \ + /path/to/bootstrap/python3 build-installer.py \ --universal-archs=intel-64 \ --dep-target=10.9 diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index b07def17..cbb6cab9 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -2,6 +2,10 @@ """ This script is used to build "official" universal installers on macOS. +NEW for 3.10 and backports: +- support universal2 variant with arm64 and x86_64 archs +- enable clang optimizations when building on 10.15+ + NEW for 3.9.0 and backports: - 2.7 end-of-life issues: - Python 3 installs now update the Current version link @@ -234,17 +238,16 @@ THIRD_PARTY_LIBS = [] def library_recipes(): result = [] - LT_10_5 = bool(getDeptargetTuple() < (10, 5)) - # Since Apple removed the header files for the deprecated system # OpenSSL as of the Xcode 7 release (for OS X 10.10+), we do not # have much choice but to build our own copy here, too. result.extend([ dict( - name="OpenSSL 1.1.1k", - url="https://www.openssl.org/source/openssl-1.1.1k.tar.gz", - checksum='c4e7d95f782b08116afa27b30393dd27', + name="OpenSSL 1.1.1l", + url="https://www.openssl.org/source/openssl-1.1.1l.tar.gz", + checksum='ac0d4387f3ba0ad741b0580dd45f6ff3', + patches=['0001-Darwin-platform-allows-to-build-on-releases-before-Y.patch'], buildrecipe=build_universal_openssl, configure=None, install=None, @@ -364,7 +367,7 @@ def library_recipes(): '-DSQLITE_ENABLE_JSON1 ' '-DSQLITE_ENABLE_RTREE ' '-DSQLITE_TCL=0 ' - '%s' % ('','-DSQLITE_WITHOUT_ZONEMALLOC ')[LT_10_5]), + ), configure_pre=[ '--enable-threadsafe', '--enable-shared=no', @@ -375,47 +378,6 @@ def library_recipes(): ), ]) - if getDeptargetTuple() < (10, 5): - result.extend([ - dict( - name="Bzip2 1.0.6", - url="http://bzip.org/1.0.6/bzip2-1.0.6.tar.gz", - checksum='00b516f4704d4a7cb50a1d97e6e8e15b', - configure=None, - install='make install CC=%s CXX=%s, PREFIX=%s/usr/local/ CFLAGS="-arch %s"'%( - CC, CXX, - shellQuote(os.path.join(WORKDIR, 'libraries')), - ' -arch '.join(ARCHLIST), - ), - ), - dict( - name="ZLib 1.2.3", - url="http://www.gzip.org/zlib/zlib-1.2.3.tar.gz", - checksum='debc62758716a169df9f62e6ab2bc634', - configure=None, - install='make install CC=%s CXX=%s, prefix=%s/usr/local/ CFLAGS="-arch %s"'%( - CC, CXX, - shellQuote(os.path.join(WORKDIR, 'libraries')), - ' -arch '.join(ARCHLIST), - ), - ), - dict( - # Note that GNU readline is GPL'd software - name="GNU Readline 6.1.2", - url="http://ftp.gnu.org/pub/gnu/readline/readline-6.1.tar.gz" , - checksum='fc2f7e714fe792db1ce6ddc4c9fb4ef3', - patchlevel='0', - patches=[ - # The readline maintainers don't do actual micro releases, but - # just ship a set of patches. - ('http://ftp.gnu.org/pub/gnu/readline/readline-6.1-patches/readline61-001', - 'c642f2e84d820884b0bf9fd176bc6c3f'), - ('http://ftp.gnu.org/pub/gnu/readline/readline-6.1-patches/readline61-002', - '1a76781a1ea734e831588285db7ec9b1'), - ] - ), - ]) - if not PYTHON_3: result.extend([ dict( diff --git a/Mac/BuildScript/resources/License.rtf b/Mac/BuildScript/resources/License.rtf index 1dfdc1ed..f08b6509 100644 --- a/Mac/BuildScript/resources/License.rtf +++ b/Mac/BuildScript/resources/License.rtf @@ -16,7 +16,7 @@ Python was created in the early 1990s by Guido van Rossum at Stichting Mathemati \ In 1995, Guido continued his work on Python at the Corporation for National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) in Reston, Virginia where he released several versions of the software.\ \ -In May 2000, Guido and the Python core development team moved to BeOpen.com to form the BeOpen PythonLabs team. In October of the same year, the PythonLabs team moved to Digital Creations (now Zope Corporation, see http://www.zope.org). In 2001, the Python Software Foundation (PSF, see http://www.python.org/psf/) was formed, a non-profit organization created specifically to own Python-related Intellectual Property. Zope Corporation is a sponsoring member of the PSF.\ +In May 2000, Guido and the Python core development team moved to BeOpen.com to form the BeOpen PythonLabs team. In October of the same year, the PythonLabs team moved to Digital Creations (now Zope Corporation, see http://www.zope.org). In 2001, the Python Software Foundation (PSF, see https://www.python.org/psf/) was formed, a non-profit organization created specifically to own Python-related Intellectual Property. Zope Corporation is a sponsoring member of the PSF.\ \ All Python releases are Open Source (see http://www.opensource.org for the Open Source Definition). Historically, most, but not all, Python releases have also been GPL-compatible; the table below summarizes the various releases.\ \ diff --git a/Mac/README.rst b/Mac/README.rst index aa019dfe..1670fe57 100644 --- a/Mac/README.rst +++ b/Mac/README.rst @@ -226,7 +226,7 @@ distribution, this is installed by default on macOS 10.4 or later. Be aware, though, that the Cocoa-based AquaTk's supplied starting with macOS 10.6 have proven to be unstable. If possible, you should consider installing a newer version before building on macOS 10.6 or later, such as -the ActiveTcl 8.6. See http://www.python.org/download/mac/tcltk/. If you +the ActiveTcl 8.6. See https://www.python.org/download/mac/tcltk/. If you are building with an SDK, ensure that the newer Tcl and Tk frameworks are seen in the SDK's ``Library/Frameworks`` directory; you may need to manually create symlinks to their installed location, ``/Library/Frameworks``. @@ -293,7 +293,7 @@ GUI programs. As of 3.4.0, the ``pythonwx.x`` aliases are no longer installed. How do I create a binary distribution? ====================================== -Download and unpack the source release from http://www.python.org/download/. +Download and unpack the source release from https://www.python.org/download/. Go to the directory ``Mac/BuildScript``. There you will find a script ``build-installer.py`` that does all the work. This will download and build a number of 3rd-party libaries, configures and builds a framework Python, @@ -334,9 +334,9 @@ The configure script sometimes emits warnings like the one below:: configure: WARNING: libintl.h: section "Present But Cannot Be Compiled" configure: WARNING: libintl.h: proceeding with the preprocessor's result configure: WARNING: libintl.h: in the future, the compiler will take precedence - configure: WARNING: ## -------------------------------------- ## - configure: WARNING: ## Report this to http://bugs.python.org/ ## - configure: WARNING: ## -------------------------------------- ## + configure: WARNING: ## --------------------------------------- ## + configure: WARNING: ## Report this to https://bugs.python.org/ ## + configure: WARNING: ## --------------------------------------- ## This almost always means you are trying to build a universal binary for Python and have libraries in ``/usr/local`` that don't contain the required @@ -399,8 +399,8 @@ The basic implementation pattern is: Resources ========= - * http://www.python.org/download/mac/ + * https://www.python.org/downloads/macos/ - * http://www.python.org/community/sigs/current/pythonmac-sig/ + * https://www.python.org/community/sigs/current/pythonmac-sig/ * https://devguide.python.org/ diff --git a/Makefile.pre.in b/Makefile.pre.in index 2b685710..6ce7a614 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -762,7 +762,7 @@ regen-abidump: all @$(UPDATE_FILE) $(srcdir)/Doc/data/python$(LDVERSION).abi $(srcdir)/Doc/data/python$(LDVERSION).abi.new check-abidump: all - abidiff "libpython$(LDVERSION).so" $(srcdir)/Doc/data/python$(LDVERSION).abi --drop-private-types --no-architecture --no-added-syms + abidiff $(srcdir)/Doc/data/python$(LDVERSION).abi "libpython$(LDVERSION).so" --drop-private-types --no-architecture --no-added-syms ############################################################################ # Regenerate all generated files diff --git a/Misc/ACKS b/Misc/ACKS index f01cc5d9..59ae1462 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -781,6 +781,7 @@ Fredrik Håård Florian Höch Oleg Höfling Robert Hölzl +Stefan Hölzl Catalin Iacob Mihai Ibanescu Ali Ikinci @@ -1278,6 +1279,7 @@ Peter Otten Michael Otteneder Richard Oudkerk Russel Owen +Noah Oxer Joonas Paalasmaa Martin Packman Shriphani Palakodety diff --git a/Misc/NEWS b/Misc/NEWS index 486ba433..c2b4555e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,358 @@ Python News +++++++++++ +What's New in Python 3.9.7 final? +================================= + +*Release date: 2021-08-30* + +Security +-------- + +- bpo-42278: Replaced usage of :func:`tempfile.mktemp` with + :class:`~tempfile.TemporaryDirectory` to avoid a potential race condition. + +- bpo-41180: Add auditing events to the :mod:`marshal` module, and stop + raising ``code.__init__`` events for every unmarshalled code object. + Directly instantiated code objects will continue to raise an event, and + audit event handlers should inspect or collect the raw marshal data. This + reduces a significant performance overhead when loading from ``.pyc`` + files. + +- bpo-44394: Update the vendored copy of libexpat to 2.4.1 (from 2.2.8) to + get the fix for the CVE-2013-0340 "Billion Laughs" vulnerability. This + copy is most used on Windows and macOS. + +- bpo-43124: Made the internal ``putcmd`` function in :mod:`smtplib` + sanitize input for presence of ``\r`` and ``\n`` characters to avoid + (unlikely) command injection. + +Core and Builtins +----------------- + +- bpo-45018: Fixed pickling of range iterators that iterated for over 2**32 + times. + +- bpo-44962: Fix a race in WeakKeyDictionary, WeakValueDictionary and + WeakSet when two threads attempt to commit the last pending removal. This + fixes asyncio.create_task and fixes a data loss in asyncio.run where + shutdown_asyncgens is not run + +- bpo-44954: Fixed a corner case bug where the result of + ``float.fromhex('0x.8p-1074')`` was rounded the wrong way. + +- bpo-44947: Refine the syntax error for trailing commas in import + statements. Patch by Pablo Galindo. + +- bpo-44698: Restore behaviour of complex exponentiation with integer-valued + exponent of type :class:`float` or :class:`complex`. + +- bpo-44885: Correct the ast locations of f-strings with format specs and + repeated expressions. Patch by Pablo Galindo + +- bpo-44872: Use new trashcan macros (Py_TRASHCAN_BEGIN/END) in + frameobject.c instead of the old ones (Py_TRASHCAN_SAFE_BEGIN/END). + +- bpo-33930: Fix segmentation fault with deep recursion when cleaning method + objects. Patch by Augusto Goulart and Pablo Galindo. + +- bpo-25782: Fix bug where ``PyErr_SetObject`` hangs when the current + exception has a cycle in its context chain. + +- bpo-44856: Fix reference leaks in the error paths of ``update_bases()`` + and ``__build_class__``. Patch by Pablo Galindo. + +- bpo-44698: Fix undefined behaviour in complex object exponentiation. + +- bpo-44562: Remove uses of :c:func:`PyObject_GC_Del` in error path when + initializing :class:`types.GenericAlias`. + +- bpo-44523: Remove the pass-through for :func:`hash` of + :class:`weakref.proxy` objects to prevent unintended consequences when the + original referred object dies while the proxy is part of a hashable + object. Patch by Pablo Galindo. + +- bpo-44472: Fix ltrace functionality when exceptions are raised. Patch by + Pablo Galindo + +- bpo-44184: Fix a crash at Python exit when a deallocator function removes + the last strong reference to a heap type. Patch by Victor Stinner. + +- bpo-39091: Fix crash when using passing a non-exception to a generator's + ``throw()`` method. Patch by Noah Oxer + +Library +------- + +- bpo-41620: :meth:`~unittest.TestCase.run` now always return a + :class:`~unittest.TestResult` instance. Previously it returned ``None`` if + the test class or method was decorated with a skipping decorator. + +- bpo-43913: Fix bugs in cleaning up classes and modules in :mod:`unittest`: + + * Functions registered with :func:`~unittest.addModuleCleanup` were not called unless the user defines ``tearDownModule()`` in their test module. + * Functions registered with :meth:`~unittest.TestCase.addClassCleanup` were not called if ``tearDownClass`` is set to ``None``. + * Buffering in :class:`~unittest.TestResult` did not work with functions registered with ``addClassCleanup()`` and ``addModuleCleanup()``. + * Errors in functions registered with ``addClassCleanup()`` and ``addModuleCleanup()`` were not handled correctly in buffered and debug modes. + * Errors in ``setUpModule()`` and functions registered with ``addModuleCleanup()`` were reported in wrong order. + * And several lesser bugs. + +- bpo-45001: Made email date parsing more robust against malformed input, + namely a whitespace-only ``Date:`` header. Patch by Wouter Bolsterlee. + +- bpo-44449: Fix a crash in the signal handler of the :mod:`faulthandler` + module: no longer modify the reference count of frame objects. Patch by + Victor Stinner. + +- bpo-44955: Method :meth:`~unittest.TestResult.stopTestRun` is now always + called in pair with method :meth:`~unittest.TestResult.startTestRun` for + :class:`~unittest.TestResult` objects implicitly created in + :meth:`~unittest.TestCase.run`. Previously it was not called for test + methods and classes decorated with a skipping decorator. + +- bpo-38956: :class:`argparse.BooleanOptionalAction`'s default value is no + longer printed twice when used with + :class:`argparse.ArgumentDefaultsHelpFormatter`. + +- bpo-44581: Upgrade bundled pip to 21.2.3 and setuptools to 57.4.0 + +- bpo-44849: Fix the :func:`os.set_inheritable` function on FreeBSD 14 for + file descriptor opened with the :data:`~os.O_PATH` flag: ignore the + :data:`~errno.EBADF` error on ``ioctl()``, fallback on the ``fcntl()`` + implementation. Patch by Victor Stinner. + +- bpo-44605: The @functools.total_ordering() decorator now works with + metaclasses. + +- bpo-44822: :mod:`sqlite3` user-defined functions and aggregators returning + :class:`strings ` with embedded NUL characters are no longer + truncated. Patch by Erlend E. Aasland. + +- bpo-44815: Always show ``loop=`` arg deprecations in + :func:`asyncio.gather` and :func:`asyncio.sleep` + +- bpo-44806: Non-protocol subclasses of :class:`typing.Protocol` ignore now + the ``__init__`` method inherited from protocol base classes. + +- bpo-44667: The :func:`tokenize.tokenize` doesn't incorrectly generate a + ``NEWLINE`` token if the source doesn't end with a new line character but + the last line is a comment, as the function is already generating a ``NL`` + token. Patch by Pablo Galindo + +- bpo-42853: Fix ``http.client.HTTPSConnection`` fails to download >2GiB + data. + +- bpo-44752: :mod:`rcompleter` does not call :func:`getattr` on + :class:`property` objects to avoid the side-effect of evaluating the + corresponding method. + +- bpo-44720: ``weakref.proxy`` objects referencing non-iterators now raise + ``TypeError`` rather than dereferencing the null ``tp_iternext`` slot and + crashing. + +- bpo-44704: The implementation of ``collections.abc.Set._hash()`` now + matches that of ``frozenset.__hash__()``. + +- bpo-44666: Fixed issue in :func:`compileall.compile_file` when + ``sys.stdout`` is redirected. Patch by Stefan Hölzl. + +- bpo-40897: Give priority to using the current class constructor in + :func:`inspect.signature`. Patch by Weipeng Hong. + +- bpo-44608: Fix memory leak in :func:`_tkinter._flatten` if it is called + with a sequence or set, but not list or tuple. + +- bpo-41928: Update :func:`shutil.copyfile` to raise + :exc:`FileNotFoundError` instead of confusing :exc:`IsADirectoryError` + when a path ending with a :const:`os.path.sep` does not exist; + :func:`shutil.copy` and :func:`shutil.copy2` are also affected. + +- bpo-44566: handle StopIteration subclass raised from + @contextlib.contextmanager generator + +- bpo-44558: Make the implementation consistency of + :func:`~operator.indexOf` between C and Python versions. Patch by Dong-hee + Na. + +- bpo-41249: Fixes ``TypedDict`` to work with ``typing.get_type_hints()`` + and postponed evaluation of annotations across modules. + +- bpo-44461: Fix bug with :mod:`pdb`'s handling of import error due to a + package which does not have a ``__main__`` module + +- bpo-42892: Fixed an exception thrown while parsing a malformed multipart + email by :class:`email.message.EmailMessage`. + +- bpo-27827: :meth:`pathlib.PureWindowsPath.is_reserved` now identifies a + greater range of reserved filenames, including those with trailing spaces + or colons. + +- bpo-34266: Handle exceptions from parsing the arg of :mod:`pdb`'s + run/restart command. + +- bpo-27334: The :mod:`sqlite3` context manager now performs a rollback + (thus releasing the database lock) if commit failed. Patch by Luca Citi + and Erlend E. Aasland. + +- bpo-43853: Improved string handling for :mod:`sqlite3` user-defined + functions and aggregates: + + * It is now possible to pass strings with embedded null characters to UDFs + * Conversion failures now correctly raise :exc:`MemoryError` + + Patch by Erlend E. Aasland. + +- bpo-43048: Handle `RecursionError` in + :class:`~traceback.TracebackException`'s constructor, so that long + exceptions chains are truncated instead of causing traceback formatting to + fail. + +- bpo-41402: Fix :meth:`email.message.EmailMessage.set_content` when called + with binary data and ``7bit`` content transfer encoding. + +- bpo-32695: The *compresslevel* and *preset* keyword arguments of + :func:`tarfile.open` are now both documented and tested. + +- bpo-34990: Fixed a Y2k38 bug in the compileall module where it would fail + to compile files with a modification time after the year 2038. + +- bpo-38840: Fix ``test___all__`` on platforms lacking a shared memory + implementation. + +- bpo-30256: Pass multiprocessing BaseProxy argument ``manager_owned`` + through AutoProxy. + +- bpo-27513: :func:`email.utils.getaddresses` now accepts + :class:`email.header.Header` objects along with string values. Patch by + Zackery Spytz. + +- bpo-33349: lib2to3 now recognizes async generators everywhere. + +- bpo-29298: Fix ``TypeError`` when required subparsers without ``dest`` do + not receive arguments. Patch by Anthony Sottile. + +Documentation +------------- + +- bpo-44903: Removed the othergui.rst file, any references to it, and the + list of GUI frameworks in the FAQ. In their place I've added links to the + Python Wiki `page on GUI frameworks + `. + +- bpo-44756: Reverted automated virtual environment creation on ``make + html`` when building documentation. It turned out to be disruptive for + downstream distributors. + +- bpo-44693: Update the definition of __future__ in the glossary by + replacing the confusing word "pseudo-module" with a more accurate + description. + +- bpo-35183: Add typical examples to os.path.splitext docs + +- bpo-30511: Clarify that :func:`shutil.make_archive` is not thread-safe due + to reliance on changing the current working directory. + +- bpo-44561: Update of three expired hyperlinks in + Doc/distributing/index.rst: "Project structure", "Building and packaging + the project", and "Uploading the project to the Python Packaging Index". + +- bpo-42958: Updated the docstring and docs of :func:`filecmp.cmp` to be + more accurate and less confusing especially in respect to *shallow* arg. + +- bpo-44558: Match the docstring and python implementation of + :func:`~operator.countOf` to the behavior of its c implementation. + +- bpo-44544: List all kwargs for :func:`textwrap.wrap`, + :func:`textwrap.fill`, and :func:`textwrap.shorten`. Now, there are nav + links to attributes of :class:`TextWrap`, which makes navigation much + easier while minimizing duplication in the documentation. + +- bpo-38062: Clarify that atexit uses equality comparisons internally. + +- bpo-43066: Added a warning to :mod:`zipfile` docs: filename arg with a + leading slash may cause archive to be un-openable on Windows systems. + +- bpo-27752: Documentation of csv.Dialect is more descriptive. + +- bpo-44453: Fix documentation for the return type of + :func:`sysconfig.get_path`. + +- bpo-39498: Add a "Security Considerations" index which links to standard + library modules that have explicitly documented security considerations. + +- bpo-33479: Remove the unqualified claim that tkinter is threadsafe. It has + not been true for several years and likely never was. An explanation of + what is true may be added later, after more discussion, and possibly after + patching _tkinter.c, + +Tests +----- + +- bpo-25130: Add calls of :func:`gc.collect` in tests to support PyPy. + +- bpo-45011: Made tests relying on the :mod:`_asyncio` C extension module + optional to allow running on alternative Python implementations. Patch by + Serhiy Storchaka. + +- bpo-44949: Fix auto history tests of test_readline: sometimes, the newline + character is not written at the end, so don't expect it in the output. + +- bpo-44852: Add ability to wholesale silence DeprecationWarnings while + running the regression test suite. + +- bpo-40928: Notify users running test_decimal regression tests on macOS of + potential harmless "malloc can't allocate region" messages spewed by + test_decimal. + +- bpo-44734: Fixed floating point precision issue in turtle tests. + +- bpo-44708: Regression tests, when run with -w, are now re-running only the + affected test methods instead of re-running the entire test file. + +- bpo-30256: Add test for nested queues when using ``multiprocessing`` + shared objects ``AutoProxy[Queue]`` inside ``ListProxy`` and ``DictProxy`` + +Build +----- + +- bpo-44535: Enable building using a Visual Studio 2022 install on Windows. + +- bpo-43298: Improved error message when building without a Windows SDK + installed. + +Windows +------- + +- bpo-45007: Update to OpenSSL 1.1.1l in Windows build + +- bpo-44572: Avoid consuming standard input in the :mod:`platform` module + +- bpo-40263: This is a follow-on bug from + https://bugs.python.org/issue26903. Once that is applied we run into an + off-by-one assertion problem. The assert was not correct. + +macOS +----- + +- bpo-45007: Update macOS installer builds to use OpenSSL 1.1.1l. + +- bpo-44689: :meth:`ctypes.util.find_library` now works correctly on macOS + 11 Big Sur even if Python is built on an older version of macOS. + Previously, when built on older macOS systems, ``find_library`` was not + able to find macOS system libraries when running on Big Sur due to + changes in how system libraries are stored. + +Tools/Demos +----------- + +- bpo-44756: In the Makefile for documentation (:file:`Doc/Makefile`), the + ``build`` rule is dependent on the ``venv`` rule. Therefore, ``html``, + ``latex``, and other build-dependent rules are also now dependent on + ``venv``. The ``venv`` rule only performs an action if ``$(VENVDIR)`` does + not exist. :file:`Doc/README.rst` was updated; most users now only need to + type ``make html``. + + What's New in Python 3.9.6 final? ================================= diff --git a/Modules/_csv.c b/Modules/_csv.c index 59109b01..7e06db46 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -399,9 +399,14 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_XINCREF(skipinitialspace); Py_XINCREF(strict); if (dialect != NULL) { -#define DIALECT_GETATTR(v, n) \ - if (v == NULL) \ - v = PyObject_GetAttrString(dialect, n) +#define DIALECT_GETATTR(v, n) \ + do { \ + if (v == NULL) { \ + v = PyObject_GetAttrString(dialect, n); \ + if (v == NULL) \ + PyErr_Clear(); \ + } \ + } while (0) DIALECT_GETATTR(delimiter, "delimiter"); DIALECT_GETATTR(doublequote, "doublequote"); DIALECT_GETATTR(escapechar, "escapechar"); @@ -410,7 +415,6 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) DIALECT_GETATTR(quoting, "quoting"); DIALECT_GETATTR(skipinitialspace, "skipinitialspace"); DIALECT_GETATTR(strict, "strict"); - PyErr_Clear(); } /* check types and convert to C values */ diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 18984d15..b0f1e0bd 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1449,14 +1449,37 @@ copy_com_pointer(PyObject *self, PyObject *args) return r; } #else - +#ifdef __APPLE__ #ifdef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH +#define HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH_RUNTIME \ + __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) +#else +// Support the deprecated case of compiling on an older macOS version +static void *libsystem_b_handle; +static bool (*_dyld_shared_cache_contains_path)(const char *path); + +__attribute__((constructor)) void load_dyld_shared_cache_contains_path(void) { + libsystem_b_handle = dlopen("/usr/lib/libSystem.B.dylib", RTLD_LAZY); + if (libsystem_b_handle != NULL) { + _dyld_shared_cache_contains_path = dlsym(libsystem_b_handle, "_dyld_shared_cache_contains_path"); + } +} + +__attribute__((destructor)) void unload_dyld_shared_cache_contains_path(void) { + if (libsystem_b_handle != NULL) { + dlclose(libsystem_b_handle); + } +} +#define HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH_RUNTIME \ + _dyld_shared_cache_contains_path != NULL +#endif + static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) { PyObject *name, *name2; char *name_str; - if (__builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *)) { + if (HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH_RUNTIME) { int r; if (!PyArg_ParseTuple(args, "O", &name)) @@ -1999,7 +2022,7 @@ PyMethodDef _ctypes_module_methods[] = { {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, #endif -#ifdef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH +#ifdef __APPLE__ {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, #endif {"alignment", align_func, METH_O, alignment_doc}, diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index c84f8382..a32cc4b8 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -35,7 +35,7 @@ A number of SysV or ncurses functions don't have wrappers yet; if you need a given function, add it and send a patch. See - http://www.python.org/dev/patches/ for instructions on how to submit + https://www.python.org/dev/patches/ for instructions on how to submit patches to Python. Here's a list of currently unsupported functions: diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 2c92a8ae..67a00a34 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1,6 +1,6 @@ /*-------------------------------------------------------------------- * Licensed to PSF under a Contributor Agreement. - * See http://www.python.org/psf/license for licensing details. + * See https://www.python.org/psf/license for licensing details. * * _elementtree - C accelerator for xml.etree.ElementTree * Copyright (c) 1999-2009 by Secret Labs AB. All rights reserved. diff --git a/Modules/_operator.c b/Modules/_operator.c index 6f8f68f4..0136e380 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -494,12 +494,12 @@ _operator_indexOf_impl(PyObject *module, PyObject *a, PyObject *b) /*[clinic input] _operator.countOf = _operator.indexOf -Return the number of times b occurs in a. +Return the number of items in a which are, or which equal, b. [clinic start generated code]*/ static Py_ssize_t _operator_countOf_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=9e1623197daf3382 input=0c3a2656add252db]*/ +/*[clinic end generated code: output=9e1623197daf3382 input=93ea57f170f3f0bb]*/ { return PySequence_Count(a, b); } diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index df4e9466..0949e8d4 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -102,8 +102,6 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject database = PyBytes_AsString(database_obj); - self->initialized = 1; - self->begin_statement = NULL; Py_CLEAR(self->statement_cache); @@ -149,7 +147,7 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject Py_INCREF(isolation_level); } Py_CLEAR(self->isolation_level); - if (pysqlite_connection_set_isolation_level(self, isolation_level, NULL) < 0) { + if (pysqlite_connection_set_isolation_level(self, isolation_level, NULL) != 0) { Py_DECREF(isolation_level); return -1; } @@ -208,6 +206,8 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject self->ProgrammingError = pysqlite_ProgrammingError; self->NotSupportedError = pysqlite_NotSupportedError; + self->initialized = 1; + return 0; } @@ -339,6 +339,12 @@ PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) return NULL; } + if (!self->initialized) { + PyErr_SetString(pysqlite_ProgrammingError, + "Base Connection.__init__ not called."); + return NULL; + } + pysqlite_do_all_statements(self, ACTION_FINALIZE, 1); if (self->db) { @@ -509,10 +515,17 @@ _pysqlite_set_result(sqlite3_context* context, PyObject* py_val) } else if (PyFloat_Check(py_val)) { sqlite3_result_double(context, PyFloat_AsDouble(py_val)); } else if (PyUnicode_Check(py_val)) { - const char *str = PyUnicode_AsUTF8(py_val); - if (str == NULL) + Py_ssize_t sz; + const char *str = PyUnicode_AsUTF8AndSize(py_val, &sz); + if (str == NULL) { + return -1; + } + if (sz > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "string is longer than INT_MAX bytes"); return -1; - sqlite3_result_text(context, str, -1, SQLITE_TRANSIENT); + } + sqlite3_result_text(context, str, (int)sz, SQLITE_TRANSIENT); } else if (PyObject_CheckBuffer(py_val)) { Py_buffer view; if (PyObject_GetBuffer(py_val, &view, PyBUF_SIMPLE) != 0) { @@ -540,7 +553,6 @@ PyObject* _pysqlite_build_py_params(sqlite3_context *context, int argc, sqlite3_ int i; sqlite3_value* cur_value; PyObject* cur_py_value; - const char* val_str; Py_ssize_t buflen; args = PyTuple_New(argc); @@ -557,16 +569,19 @@ PyObject* _pysqlite_build_py_params(sqlite3_context *context, int argc, sqlite3_ case SQLITE_FLOAT: cur_py_value = PyFloat_FromDouble(sqlite3_value_double(cur_value)); break; - case SQLITE_TEXT: - val_str = (const char*)sqlite3_value_text(cur_value); - cur_py_value = PyUnicode_FromString(val_str); - /* TODO: have a way to show errors here */ - if (!cur_py_value) { - PyErr_Clear(); - Py_INCREF(Py_None); - cur_py_value = Py_None; + case SQLITE_TEXT: { + sqlite3 *db = sqlite3_context_db_handle(context); + const char *text = (const char *)sqlite3_value_text(cur_value); + + if (text == NULL && sqlite3_errcode(db) == SQLITE_NOMEM) { + PyErr_NoMemory(); + goto error; } + + Py_ssize_t size = sqlite3_value_bytes(cur_value); + cur_py_value = PyUnicode_FromStringAndSize(text, size); break; + } case SQLITE_BLOB: buflen = sqlite3_value_bytes(cur_value); cur_py_value = PyBytes_FromStringAndSize( @@ -579,8 +594,7 @@ PyObject* _pysqlite_build_py_params(sqlite3_context *context, int argc, sqlite3_ } if (!cur_py_value) { - Py_DECREF(args); - return NULL; + goto error; } PyTuple_SetItem(args, i, cur_py_value); @@ -588,6 +602,10 @@ PyObject* _pysqlite_build_py_params(sqlite3_context *context, int argc, sqlite3_ } return args; + +error: + Py_DECREF(args); + return NULL; } void _pysqlite_func_callback(sqlite3_context* context, int argc, sqlite3_value** argv) @@ -1143,6 +1161,9 @@ int pysqlite_check_thread(pysqlite_Connection* self) static PyObject* pysqlite_connection_get_isolation_level(pysqlite_Connection* self, void* unused) { + if (!pysqlite_check_connection(self)) { + return NULL; + } Py_INCREF(self->isolation_level); return self->isolation_level; } @@ -1175,11 +1196,17 @@ pysqlite_connection_set_isolation_level(pysqlite_Connection* self, PyObject* iso return -1; } if (isolation_level == Py_None) { - PyObject *res = pysqlite_connection_commit(self, NULL); - if (!res) { - return -1; + /* We might get called during connection init, so we cannot use + * pysqlite_connection_commit() here. */ + if (self->db && !sqlite3_get_autocommit(self->db)) { + int rc; + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_exec(self->db, "COMMIT", NULL, NULL, NULL); + Py_END_ALLOW_THREADS + if (rc != SQLITE_OK) { + return _pysqlite_seterror(self->db, NULL); + } } - Py_DECREF(res); self->begin_statement = NULL; } else { @@ -1744,22 +1771,36 @@ pysqlite_connection_enter(pysqlite_Connection* self, PyObject* args) static PyObject * pysqlite_connection_exit(pysqlite_Connection* self, PyObject* args) { - PyObject* exc_type, *exc_value, *exc_tb; - const char* method_name; - PyObject* result; - + PyObject *exc_type, *exc_value, *exc_tb; if (!PyArg_ParseTuple(args, "OOO", &exc_type, &exc_value, &exc_tb)) { return NULL; } + int commit = 0; + PyObject *result; if (exc_type == Py_None && exc_value == Py_None && exc_tb == Py_None) { - method_name = "commit"; - } else { - method_name = "rollback"; + commit = 1; + result = pysqlite_connection_commit(self, NULL); } - - result = PyObject_CallMethod((PyObject*)self, method_name, NULL); - if (!result) { + else { + result = pysqlite_connection_rollback(self, NULL); + } + + if (result == NULL) { + if (commit) { + /* Commit failed; try to rollback in order to unlock the database. + * If rollback also fails, chain the exceptions. */ + PyObject *exc, *val, *tb; + PyErr_Fetch(&exc, &val, &tb); + result = pysqlite_connection_rollback(self, NULL); + if (result == NULL) { + _PyErr_ChainExceptions(exc, val, tb); + } + else { + Py_DECREF(result); + PyErr_Restore(exc, val, tb); + } + } return NULL; } Py_DECREF(result); diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 97e314b2..0498c153 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -4229,7 +4229,7 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, goto error; } - /* validata cadata type and load cadata */ + /* validate cadata type and load cadata */ if (cadata) { if (PyUnicode_Check(cadata)) { PyObject *cadata_ascii = PyUnicode_AsASCIIString(cadata); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 53a6752d..f6558cae 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2215,16 +2215,22 @@ test_long_numbits(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } -/* Example passing NULLs to PyObject_Str(NULL). */ +static PyObject * +pyobject_repr_from_null(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyObject_Repr(NULL); +} static PyObject * -test_null_strings(PyObject *self, PyObject *Py_UNUSED(ignored)) +pyobject_str_from_null(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *o1 = PyObject_Str(NULL), *o2 = PyObject_Str(NULL); - PyObject *tuple = PyTuple_Pack(2, o1, o2); - Py_XDECREF(o1); - Py_XDECREF(o2); - return tuple; + return PyObject_Str(NULL); +} + +static PyObject * +pyobject_bytes_from_null(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyObject_Bytes(NULL); } static PyObject * @@ -5371,7 +5377,9 @@ static PyMethodDef TestMethods[] = { {"test_k_code", test_k_code, METH_NOARGS}, {"test_empty_argparse", test_empty_argparse, METH_NOARGS}, {"parse_tuple_and_keywords", parse_tuple_and_keywords, METH_VARARGS}, - {"test_null_strings", test_null_strings, METH_NOARGS}, + {"pyobject_repr_from_null", pyobject_repr_from_null, METH_NOARGS}, + {"pyobject_str_from_null", pyobject_str_from_null, METH_NOARGS}, + {"pyobject_bytes_from_null", pyobject_bytes_from_null, METH_NOARGS}, {"test_string_from_format", (PyCFunction)test_string_from_format, METH_NOARGS}, {"test_with_docstring", test_with_docstring, METH_NOARGS, PyDoc_STR("This is a pretty normal docstring.")}, diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index b7958ea6..19853ce4 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -3197,8 +3197,10 @@ _tkinter__flatten(PyObject *module, PyObject *item) context.size = 0; - if (!_flatten1(&context, item,0)) + if (!_flatten1(&context, item, 0)) { + Py_XDECREF(context.tuple); return NULL; + } if (_PyTuple_Resize(&context.tuple, context.size)) return NULL; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index e1672c47..1e0e4ec8 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -32,7 +32,7 @@ */ /* Licensed to PSF under a Contributor Agreement. */ -/* See http://www.python.org/2.4/license for licensing details. */ +/* See https://www.python.org/2.4/license for licensing details. */ #include "Python.h" #include "structmember.h" // PyMemberDef @@ -1713,7 +1713,7 @@ _winapi_WaitForMultipleObjects_impl(PyObject *module, PyObject *handle_seq, nhandles = PySequence_Length(handle_seq); if (nhandles == -1) return NULL; - if (nhandles < 0 || nhandles >= MAXIMUM_WAIT_OBJECTS - 1) { + if (nhandles < 0 || nhandles > MAXIMUM_WAIT_OBJECTS - 1) { PyErr_Format(PyExc_ValueError, "need at most %zd handles, got a sequence of length %zd", MAXIMUM_WAIT_OBJECTS - 1, nhandles); diff --git a/Modules/clinic/_operator.c.h b/Modules/clinic/_operator.c.h index f9e353d8..0e39f3b7 100644 --- a/Modules/clinic/_operator.c.h +++ b/Modules/clinic/_operator.c.h @@ -957,7 +957,7 @@ PyDoc_STRVAR(_operator_countOf__doc__, "countOf($module, a, b, /)\n" "--\n" "\n" -"Return the number of times b occurs in a."); +"Return the number of items in a which are, or which equal, b."); #define _OPERATOR_COUNTOF_METHODDEF \ {"countOf", (PyCFunction)(void(*)(void))_operator_countOf, METH_FASTCALL, _operator_countOf__doc__}, @@ -1491,4 +1491,4 @@ _operator__compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t na exit: return return_value; } -/*[clinic end generated code: output=e7ed71a8c475a901 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ed2bc172e42320d8 input=a9049054013a1b77]*/ diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index 8d288f0f..3c0142e7 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,5 +1,5 @@ Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper -Copyright (c) 2001-2017 Expat maintainers +Copyright (c) 2001-2019 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/ascii.h b/Modules/expat/ascii.h index c3587e57..1f594d2e 100644 --- a/Modules/expat/ascii.h +++ b/Modules/expat/ascii.h @@ -6,8 +6,11 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 1999-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000 Clark Cooper + Copyright (c) 2002 Fred L. Drake, Jr. + Copyright (c) 2007 Karl Waclawek + Copyright (c) 2017 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/asciitab.h b/Modules/expat/asciitab.h index 63b1d1b4..af766fb2 100644 --- a/Modules/expat/asciitab.h +++ b/Modules/expat/asciitab.h @@ -7,7 +7,9 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2002 Fred L. Drake, Jr. + Copyright (c) 2017 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 6c8eb1fd..b7d6d354 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -7,7 +7,14 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2000-2005 Fred L. Drake, Jr. + Copyright (c) 2001-2002 Greg Stein + Copyright (c) 2002-2016 Karl Waclawek + Copyright (c) 2016-2021 Sebastian Pipping + Copyright (c) 2016 Cristian Rodríguez + Copyright (c) 2016 Thomas Beutlich + Copyright (c) 2017 Rhodri James Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -115,7 +122,11 @@ enum XML_Error { XML_ERROR_RESERVED_PREFIX_XMLNS, XML_ERROR_RESERVED_NAMESPACE_URI, /* Added in 2.2.1. */ - XML_ERROR_INVALID_ARGUMENT + XML_ERROR_INVALID_ARGUMENT, + /* Added in 2.3.0. */ + XML_ERROR_NO_BUFFER, + /* Added in 2.4.0. */ + XML_ERROR_AMPLIFICATION_LIMIT_BREACH }; enum XML_Content_Type { @@ -318,7 +329,7 @@ typedef void(XMLCALL *XML_EndDoctypeDeclHandler)(void *userData); For internal entities (), value will be non-NULL and systemId, publicID, and notationName will be NULL. - The value string is NOT nul-terminated; the length is provided in + The value string is NOT null-terminated; the length is provided in the value_length argument. Since it is legal to have zero-length values, do not use this argument to test for internal entities. @@ -513,7 +524,7 @@ typedef struct { Otherwise it must return XML_STATUS_ERROR. If info does not describe a suitable encoding, then the parser will - return an XML_UNKNOWN_ENCODING error. + return an XML_ERROR_UNKNOWN_ENCODING error. */ typedef int(XMLCALL *XML_UnknownEncodingHandler)(void *encodingHandlerData, const XML_Char *name, @@ -707,7 +718,7 @@ XML_GetBase(XML_Parser parser); /* Returns the number of the attribute/value pairs passed in last call to the XML_StartElementHandler that were specified in the start-tag rather than defaulted. Each attribute/value pair counts as 2; thus - this correspondds to an index into the atts array passed to the + this corresponds to an index into the atts array passed to the XML_StartElementHandler. Returns -1 if parser == NULL. */ XMLPARSEAPI(int) @@ -716,7 +727,7 @@ XML_GetSpecifiedAttributeCount(XML_Parser parser); /* Returns the index of the ID attribute passed in the last call to XML_StartElementHandler, or -1 if there is no ID attribute or parser == NULL. Each attribute/value pair counts as 2; thus this - correspondds to an index into the atts array passed to the + corresponds to an index into the atts array passed to the XML_StartElementHandler. */ XMLPARSEAPI(int) @@ -997,7 +1008,10 @@ enum XML_FeatureEnum { XML_FEATURE_SIZEOF_XML_LCHAR, XML_FEATURE_NS, XML_FEATURE_LARGE_SIZE, - XML_FEATURE_ATTR_INFO + XML_FEATURE_ATTR_INFO, + /* Added in Expat 2.4.0. */ + XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT /* Additional features must be added to the end of this enum. */ }; @@ -1010,12 +1024,24 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); +#ifdef XML_DTD +/* Added in Expat 2.4.0. */ +XMLPARSEAPI(XML_Bool) +XML_SetBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor); + +/* Added in Expat 2.4.0. */ +XMLPARSEAPI(XML_Bool) +XML_SetBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); +#endif + /* Expat follows the semantic versioning convention. See http://semver.org. */ #define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 2 -#define XML_MICRO_VERSION 8 +#define XML_MINOR_VERSION 4 +#define XML_MICRO_VERSION 1 #ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index f2b75dda..12c560e1 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -7,7 +7,14 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2000-2004 Fred L. Drake, Jr. + Copyright (c) 2001-2002 Greg Stein + Copyright (c) 2002-2006 Karl Waclawek + Copyright (c) 2016 Cristian Rodríguez + Copyright (c) 2016-2019 Sebastian Pipping + Copyright (c) 2017 Rhodri James + Copyright (c) 2018 Yury Gribov Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/iasciitab.h b/Modules/expat/iasciitab.h index ea97cfcf..5d8646f2 100644 --- a/Modules/expat/iasciitab.h +++ b/Modules/expat/iasciitab.h @@ -7,7 +7,9 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2002 Fred L. Drake, Jr. + Copyright (c) 2017 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 60913dab..444eba0f 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -25,8 +25,12 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2002-2003 Fred L. Drake, Jr. + Copyright (c) 2002-2006 Karl Waclawek + Copyright (c) 2003 Greg Stein + Copyright (c) 2016-2021 Sebastian Pipping + Copyright (c) 2018 Yury Gribov + Copyright (c) 2019 David Loffredo Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -101,22 +105,58 @@ # endif #endif +#include // ULONG_MAX + +#if defined(_WIN32) && ! defined(__USE_MINGW_ANSI_STDIO) +# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" +# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u" +# else +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# endif +#else +# define EXPAT_FMT_ULL(midpart) "%" midpart "llu" +# if ! defined(ULONG_MAX) +# error Compiler did not define ULONG_MAX for us +# elif ULONG_MAX == 18446744073709551615u // 2^64-1 +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "lu" +# else +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# endif +#endif + #ifndef UNUSED_P # define UNUSED_P(p) (void)p #endif +/* NOTE BEGIN If you ever patch these defaults to greater values + for non-attack XML payload in your environment, + please file a bug report with libexpat. Thank you! +*/ +#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \ + 100.0f +#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ + 8388608 // 8 MiB, 2^23 +/* NOTE END */ + +#include "expat.h" // so we can use type XML_Parser below + #ifdef __cplusplus extern "C" { #endif -#ifdef XML_ENABLE_VISIBILITY -# if XML_ENABLE_VISIBILITY -__attribute__((visibility("default"))) -# endif +void _INTERNAL_trim_to_complete_utf8_characters(const char *from, + const char **fromLimRef); + +#if defined(XML_DTD) +unsigned long long testingAccountingGetCountBytesDirect(XML_Parser parser); +unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser); +const char *unsignedCharToPrintable(unsigned char c); #endif -void -_INTERNAL_trim_to_complete_utf8_characters(const char *from, - const char **fromLimRef); #ifdef __cplusplus } diff --git a/Modules/expat/latin1tab.h b/Modules/expat/latin1tab.h index 6f916041..b681d278 100644 --- a/Modules/expat/latin1tab.h +++ b/Modules/expat/latin1tab.h @@ -7,7 +7,9 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2002 Fred L. Drake, Jr. + Copyright (c) 2017 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/nametab.h b/Modules/expat/nametab.h index 3681df34..63485446 100644 --- a/Modules/expat/nametab.h +++ b/Modules/expat/nametab.h @@ -6,8 +6,8 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2017 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index bfee65a3..e5406d7e 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -11,6 +11,9 @@ * -------------------------------------------------------------------------- * HISTORY: * + * 2020-10-03 (Sebastian Pipping) + * - Drop support for Visual Studio 9.0/2008 and earlier + * * 2019-08-03 (Sebastian Pipping) * - Mark part of sip24_valid as to be excluded from clang-format * - Re-format code using clang-format 9 @@ -96,15 +99,7 @@ #define SIPHASH_H #include /* size_t */ - -#if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER < 1600) -/* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */ -typedef unsigned __int8 uint8_t; -typedef unsigned __int32 uint32_t; -typedef unsigned __int64 uint64_t; -#else -# include /* uint64_t uint32_t uint8_t */ -#endif +#include /* uint64_t uint32_t uint8_t */ /* * Workaround to not require a C++11 compiler for using ULL suffix diff --git a/Modules/expat/utf8tab.h b/Modules/expat/utf8tab.h index a22986ac..88efcf91 100644 --- a/Modules/expat/utf8tab.h +++ b/Modules/expat/utf8tab.h @@ -7,7 +7,9 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2002 Fred L. Drake, Jr. + Copyright (c) 2017 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index 562a4a82..2ecd61b5 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -6,8 +6,10 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2002 Greg Stein + Copyright (c) 2005 Karl Waclawek + Copyright (c) 2017-2021 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -40,17 +42,4 @@ #include #include -#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */ -# include -#else /* !defined(HAVE_EXPAT_CONFIG_H) */ - -# define XML_NS 1 -# define XML_DTD 1 -# define XML_CONTEXT_BYTES 1024 - -/* we will assume all Windows platforms are little endian */ -# define BYTEORDER 1234 - -#endif /* !defined(HAVE_EXPAT_CONFIG_H) */ - #endif /* ndef WINCONFIG_H */ diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index e740f0e1..5ba56eae 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* f2d0ab6d1d4422a08cf1cf3bbdfba96b49dea42fb5ff4615e03a2a25c306e769 (2.2.8+) +/* 8539b9040d9d901366a62560a064af7cb99811335784b363abc039c5b0ebc416 (2.4.1+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -7,7 +7,31 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2000-2006 Fred L. Drake, Jr. + Copyright (c) 2001-2002 Greg Stein + Copyright (c) 2002-2016 Karl Waclawek + Copyright (c) 2005-2009 Steven Solie + Copyright (c) 2016 Eric Rahm + Copyright (c) 2016-2021 Sebastian Pipping + Copyright (c) 2016 Gaurav + Copyright (c) 2016 Thomas Beutlich + Copyright (c) 2016 Gustavo Grieco + Copyright (c) 2016 Pascal Cuoq + Copyright (c) 2016 Ed Schouten + Copyright (c) 2017-2018 Rhodri James + Copyright (c) 2017 Václav Slavík + Copyright (c) 2017 Viktor Szakats + Copyright (c) 2017 Chanho Park + Copyright (c) 2017 Rolf Eike Beer + Copyright (c) 2017 Hans Wennborg + Copyright (c) 2018 Anton Maklakov + Copyright (c) 2018 Benjamin Peterson + Copyright (c) 2018 Marco Maggi + Copyright (c) 2018 Mariusz Zaborski + Copyright (c) 2019 David Loffredo + Copyright (c) 2019-2020 Ben Wagner + Copyright (c) 2019 Vadim Zeitlin Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -36,7 +60,9 @@ #ifdef _WIN32 /* force stdlib to define rand_s() */ -# define _CRT_RAND_S +# if ! defined(_CRT_RAND_S) +# define _CRT_RAND_S +# endif #endif #include @@ -45,6 +71,8 @@ #include /* UINT_MAX */ #include /* fprintf */ #include /* getenv, rand_s */ +#include /* uintptr_t */ +#include /* isnan */ #ifdef _WIN32 # define getpid GetCurrentProcessId @@ -60,9 +88,9 @@ #ifdef _WIN32 # include "winconfig.h" -#elif defined(HAVE_EXPAT_CONFIG_H) -# include -#endif /* ndef _WIN32 */ +#endif + +#include #include "ascii.h" #include "expat.h" @@ -97,14 +125,14 @@ enabled. For end user security, that is probably not what you want. \ \ Your options include: \ - * Linux + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ - * Linux + glibc <2.25 (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ + * Linux >=3.17 + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ + * Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ * BSD / macOS >=10.7 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ - * BSD / macOS <10.7 (arc4random): HAVE_ARC4RANDOM, \ + * BSD / macOS (including <10.7) (arc4random): HAVE_ARC4RANDOM, \ * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ - * Linux / BSD / macOS (/dev/urandom): XML_DEV_URANDOM \ - * Windows (rand_s): _WIN32. \ + * Linux (including <3.17) / BSD / macOS (including <10.7) (/dev/urandom): XML_DEV_URANDOM, \ + * Windows >=Vista (rand_s): _WIN32. \ \ If insist on not using any of these, bypass this error by defining \ XML_POOR_ENTROPY; you have been warned. \ @@ -119,9 +147,7 @@ # define XmlGetInternalEncoding XmlGetUtf16InternalEncoding # define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS # define XmlEncode XmlUtf16Encode -/* Using pointer subtraction to convert to integer type. */ -# define MUST_CONVERT(enc, s) \ - (! (enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1)) +# define MUST_CONVERT(enc, s) (! (enc)->isUtf16 || (((uintptr_t)(s)) & 1)) typedef unsigned short ICHAR; #else # define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX @@ -371,6 +397,31 @@ typedef struct open_internal_entity { XML_Bool betweenDecl; /* WFC: PE Between Declarations */ } OPEN_INTERNAL_ENTITY; +enum XML_Account { + XML_ACCOUNT_DIRECT, /* bytes directly passed to the Expat parser */ + XML_ACCOUNT_ENTITY_EXPANSION, /* intermediate bytes produced during entity + expansion */ + XML_ACCOUNT_NONE /* i.e. do not account, was accounted already */ +}; + +#ifdef XML_DTD +typedef unsigned long long XmlBigCount; +typedef struct accounting { + XmlBigCount countBytesDirect; + XmlBigCount countBytesIndirect; + int debugLevel; + float maximumAmplificationFactor; // >=1.0 + unsigned long long activationThresholdBytes; +} ACCOUNTING; + +typedef struct entity_stats { + unsigned int countEverOpened; + unsigned int currentDepth; + unsigned int maximumDepthSeen; + int debugLevel; +} ENTITY_STATS; +#endif /* XML_DTD */ + typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start, const char *end, const char **endPtr); @@ -401,16 +452,18 @@ static enum XML_Error initializeEncoding(XML_Parser parser); static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, int tok, const char *next, const char **nextPtr, - XML_Bool haveMore, XML_Bool allowClosingDoctype); + XML_Bool haveMore, XML_Bool allowClosingDoctype, + enum XML_Account account); static enum XML_Error processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl); static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *start, const char *end, const char **endPtr, - XML_Bool haveMore); + XML_Bool haveMore, enum XML_Account account); static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *, const char **startPtr, const char *end, - const char **nextPtr, XML_Bool haveMore); + const char **nextPtr, XML_Bool haveMore, + enum XML_Account account); #ifdef XML_DTD static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr, const char *end, @@ -420,7 +473,8 @@ static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *, static void freeBindings(XML_Parser parser, BINDING *bindings); static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *, const char *s, TAG_NAME *tagNamePtr, - BINDING **bindingsPtr); + BINDING **bindingsPtr, + enum XML_Account account); static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, const XML_Char *uri, BINDING **bindingsPtr); @@ -429,15 +483,18 @@ static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata, XML_Parser parser); static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata, const char *, - const char *, STRING_POOL *); + const char *, STRING_POOL *, + enum XML_Account account); static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata, const char *, - const char *, STRING_POOL *); + const char *, STRING_POOL *, + enum XML_Account account); static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *); static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, - const char *start, const char *end); + const char *start, const char *end, + enum XML_Account account); static int reportProcessingInstruction(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); static int reportComment(XML_Parser parser, const ENCODING *enc, @@ -501,6 +558,34 @@ static XML_Parser parserCreate(const XML_Char *encodingName, static void parserInit(XML_Parser parser, const XML_Char *encodingName); +#ifdef XML_DTD +static float accountingGetCurrentAmplification(XML_Parser rootParser); +static void accountingReportStats(XML_Parser originParser, const char *epilog); +static void accountingOnAbort(XML_Parser originParser); +static void accountingReportDiff(XML_Parser rootParser, + unsigned int levelsAwayFromRootParser, + const char *before, const char *after, + ptrdiff_t bytesMore, int source_line, + enum XML_Account account); +static XML_Bool accountingDiffTolerated(XML_Parser originParser, int tok, + const char *before, const char *after, + int source_line, + enum XML_Account account); + +static void entityTrackingReportStats(XML_Parser parser, ENTITY *entity, + const char *action, int sourceLine); +static void entityTrackingOnOpen(XML_Parser parser, ENTITY *entity, + int sourceLine); +static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity, + int sourceLine); + +static XML_Parser getRootParserOf(XML_Parser parser, + unsigned int *outLevelDiff); +#endif /* XML_DTD */ + +static unsigned long getDebugLevel(const char *variableName, + unsigned long defaultDebugLevel); + #define poolStart(pool) ((pool)->start) #define poolEnd(pool) ((pool)->ptr) #define poolLength(pool) ((pool)->ptr - (pool)->start) @@ -614,6 +699,10 @@ struct XML_ParserStruct { enum XML_ParamEntityParsing m_paramEntityParsing; #endif unsigned long m_hash_secret_salt; +#ifdef XML_DTD + ACCOUNTING m_accounting; + ENTITY_STATS m_entity_stats; +#endif }; #define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s))) @@ -734,6 +823,15 @@ writeRandomBytes_arc4random(void *target, size_t count) { #ifdef _WIN32 +/* Provide declaration of rand_s() for MinGW-32 (not 64, which has it), + as it didn't declare it in its header prior to version 5.3.0 of its + runtime package (mingwrt, containing stdlib.h). The upstream fix + was introduced at https://osdn.net/projects/mingw/ticket/39658 . */ +# if defined(__MINGW32__) && defined(__MINGW32_VERSION) \ + && __MINGW32_VERSION < 5003000L && ! defined(__MINGW64_VERSION_MAJOR) +__declspec(dllimport) int rand_s(unsigned int *); +# endif + /* Obtain entropy on Windows using the rand_s() function which * generates cryptographically secure random numbers. Internally it * uses RtlGenRandom API which is present in Windows XP and later. @@ -789,9 +887,8 @@ gather_time_entropy(void) { static unsigned long ENTROPY_DEBUG(const char *label, unsigned long entropy) { - const char *const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG"); - if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) { - fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", label, + if (getDebugLevel("EXPAT_ENTROPY_DEBUG", 0) >= 1u) { + fprintf(stderr, "expat: Entropy: %s --> 0x%0*lx (%lu bytes)\n", label, (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy)); } return entropy; @@ -1053,6 +1150,18 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER; #endif parser->m_hash_secret_salt = 0; + +#ifdef XML_DTD + memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); + parser->m_accounting.debugLevel = getDebugLevel("EXPAT_ACCOUNTING_DEBUG", 0u); + parser->m_accounting.maximumAmplificationFactor + = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT; + parser->m_accounting.activationThresholdBytes + = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT; + + memset(&parser->m_entity_stats, 0, sizeof(ENTITY_STATS)); + parser->m_entity_stats.debugLevel = getDebugLevel("EXPAT_ENTITY_DEBUG", 0u); +#endif } /* moves list of bindings to m_freeBindingList */ @@ -1399,6 +1508,7 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) { parser->m_useForeignDTD = useDTD; return XML_ERROR_NONE; #else + UNUSED_P(useDTD); return XML_ERROR_FEATURE_REQUIRES_XML_DTD; #endif } @@ -1780,7 +1890,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { int nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ - if (len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) { + if ((XML_Size)len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -1872,6 +1982,12 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_errorCode = XML_ERROR_FINISHED; return XML_STATUS_ERROR; case XML_INITIALIZED: + /* Has someone called XML_GetBuffer successfully before? */ + if (! parser->m_bufferPtr) { + parser->m_errorCode = XML_ERROR_NO_BUFFER; + return XML_STATUS_ERROR; + } + if (parser->m_parentParser == NULL && ! startParsing(parser)) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; @@ -2155,7 +2271,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { (void)offset; (void)size; #endif /* defined XML_CONTEXT_BYTES */ - return (char *)0; + return (const char *)0; } XML_Size XMLCALL @@ -2316,6 +2432,14 @@ XML_ErrorString(enum XML_Error code) { /* Added in 2.2.5. */ case XML_ERROR_INVALID_ARGUMENT: /* Constant added in 2.2.1, already */ return XML_L("invalid argument"); + /* Added in 2.3.0. */ + case XML_ERROR_NO_BUFFER: + return XML_L( + "a successful prior call to function XML_GetBuffer is required"); + /* Added in 2.4.0. */ + case XML_ERROR_AMPLIFICATION_LIMIT_BREACH: + return XML_L( + "limit on input amplification factor (from DTD and entities) breached"); } return NULL; } @@ -2352,41 +2476,75 @@ XML_ExpatVersionInfo(void) { const XML_Feature *XMLCALL XML_GetFeatureList(void) { - static const XML_Feature features[] - = {{XML_FEATURE_SIZEOF_XML_CHAR, XML_L("sizeof(XML_Char)"), - sizeof(XML_Char)}, - {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"), - sizeof(XML_LChar)}, + static const XML_Feature features[] = { + {XML_FEATURE_SIZEOF_XML_CHAR, XML_L("sizeof(XML_Char)"), + sizeof(XML_Char)}, + {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"), + sizeof(XML_LChar)}, #ifdef XML_UNICODE - {XML_FEATURE_UNICODE, XML_L("XML_UNICODE"), 0}, + {XML_FEATURE_UNICODE, XML_L("XML_UNICODE"), 0}, #endif #ifdef XML_UNICODE_WCHAR_T - {XML_FEATURE_UNICODE_WCHAR_T, XML_L("XML_UNICODE_WCHAR_T"), 0}, + {XML_FEATURE_UNICODE_WCHAR_T, XML_L("XML_UNICODE_WCHAR_T"), 0}, #endif #ifdef XML_DTD - {XML_FEATURE_DTD, XML_L("XML_DTD"), 0}, + {XML_FEATURE_DTD, XML_L("XML_DTD"), 0}, #endif #ifdef XML_CONTEXT_BYTES - {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), - XML_CONTEXT_BYTES}, + {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), + XML_CONTEXT_BYTES}, #endif #ifdef XML_MIN_SIZE - {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0}, + {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0}, #endif #ifdef XML_NS - {XML_FEATURE_NS, XML_L("XML_NS"), 0}, + {XML_FEATURE_NS, XML_L("XML_NS"), 0}, #endif #ifdef XML_LARGE_SIZE - {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0}, + {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0}, #endif #ifdef XML_ATTR_INFO - {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, + {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, +#endif +#ifdef XML_DTD + /* Added in Expat 2.4.0. */ + {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_L("XML_BLAP_MAX_AMP"), + (long int) + EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT}, + {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, + XML_L("XML_BLAP_ACT_THRES"), + EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT}, #endif - {XML_FEATURE_END, NULL, 0}}; + {XML_FEATURE_END, NULL, 0}}; return features; } +#ifdef XML_DTD +XML_Bool XMLCALL +XML_SetBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor) { + if ((parser == NULL) || (parser->m_parentParser != NULL) + || isnan(maximumAmplificationFactor) + || (maximumAmplificationFactor < 1.0f)) { + return XML_FALSE; + } + parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; + return XML_TRUE; +} + +XML_Bool XMLCALL +XML_SetBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if ((parser == NULL) || (parser->m_parentParser != NULL)) { + return XML_FALSE; + } + parser->m_accounting.activationThresholdBytes = activationThresholdBytes; + return XML_TRUE; +} +#endif /* XML_DTD */ + /* Initially tag->rawName always points into the parse buffer; for those TAG instances opened while the current parse buffer was processed, and not yet closed, we need to store tag->rawName in a more @@ -2439,9 +2597,9 @@ storeRawNames(XML_Parser parser) { static enum XML_Error PTRCALL contentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { - enum XML_Error result - = doContent(parser, 0, parser->m_encoding, start, end, endPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer); + enum XML_Error result = doContent( + parser, 0, parser->m_encoding, start, end, endPtr, + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_ACCOUNT_DIRECT); if (result == XML_ERROR_NONE) { if (! storeRawNames(parser)) return XML_ERROR_NO_MEMORY; @@ -2466,6 +2624,14 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, int tok = XmlContentTok(parser->m_encoding, start, end, &next); switch (tok) { case XML_TOK_BOM: +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, start, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif /* XML_DTD */ + /* If we are at the end of the buffer, this would cause the next stage, i.e. externalEntityInitProcessor3, to pass control directly to doContent (by detecting XML_TOK_NONE) without processing any xml text @@ -2503,6 +2669,10 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, const char *next = start; /* XmlContentTok doesn't always set the last arg */ parser->m_eventPtr = start; tok = XmlContentTok(parser->m_encoding, start, end, &next); + /* Note: These bytes are accounted later in: + - processXmlDecl + - externalEntityContentProcessor + */ parser->m_eventEndPtr = next; switch (tok) { @@ -2544,7 +2714,8 @@ externalEntityContentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = doContent(parser, 1, parser->m_encoding, start, end, endPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, + XML_ACCOUNT_ENTITY_EXPANSION); if (result == XML_ERROR_NONE) { if (! storeRawNames(parser)) return XML_ERROR_NO_MEMORY; @@ -2555,7 +2726,7 @@ externalEntityContentProcessor(XML_Parser parser, const char *start, static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *s, const char *end, const char **nextPtr, - XML_Bool haveMore) { + XML_Bool haveMore, enum XML_Account account) { /* save one level of indirection */ DTD *const dtd = parser->m_dtd; @@ -2573,6 +2744,17 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, for (;;) { const char *next = s; /* XmlContentTok doesn't always set the last arg */ int tok = XmlContentTok(enc, s, end, &next); +#ifdef XML_DTD + const char *accountAfter + = ((tok == XML_TOK_TRAILING_RSQB) || (tok == XML_TOK_TRAILING_CR)) + ? (haveMore ? s /* i.e. 0 bytes */ : end) + : next; + if (! accountingDiffTolerated(parser, tok, s, accountAfter, __LINE__, + account)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif *eventEndPP = next; switch (tok) { case XML_TOK_TRAILING_CR: @@ -2628,6 +2810,14 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { +#ifdef XML_DTD + /* NOTE: We are replacing 4-6 characters original input for 1 character + * so there is no amplification and hence recording without + * protection. */ + accountingDiffTolerated(parser, tok, (char *)&ch, + ((char *)&ch) + sizeof(XML_Char), __LINE__, + XML_ACCOUNT_ENTITY_EXPANSION); +#endif /* XML_DTD */ if (parser->m_characterDataHandler) parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1); else if (parser->m_defaultHandler) @@ -2746,7 +2936,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } tag->name.str = (XML_Char *)tag->buf; *toPtr = XML_T('\0'); - result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings)); + result + = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings), account); if (result) return result; if (parser->m_startElementHandler) @@ -2770,7 +2961,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (! name.str) return XML_ERROR_NO_MEMORY; poolFinish(&parser->m_tempPool); - result = storeAtts(parser, enc, s, &name, &bindings); + result = storeAtts(parser, enc, s, &name, &bindings, + XML_ACCOUNT_NONE /* token spans whole start tag */); if (result != XML_ERROR_NONE) { freeBindings(parser, bindings); return result; @@ -2905,7 +3097,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, /* END disabled code */ else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); - result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore); + result + = doCdataSection(parser, enc, &next, end, nextPtr, haveMore, account); if (result != XML_ERROR_NONE) return result; else if (! next) { @@ -3034,7 +3227,8 @@ freeBindings(XML_Parser parser, BINDING *bindings) { */ static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, - TAG_NAME *tagNamePtr, BINDING **bindingsPtr) { + TAG_NAME *tagNamePtr, BINDING **bindingsPtr, + enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ ELEMENT_TYPE *elementType; int nDefaultAtts; @@ -3144,7 +3338,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* normalize the attribute value */ result = storeAttributeValue( parser, enc, isCdata, parser->m_atts[i].valuePtr, - parser->m_atts[i].valueEnd, &parser->m_tempPool); + parser->m_atts[i].valueEnd, &parser->m_tempPool, account); if (result) return result; appAtts[attIndex] = poolStart(&parser->m_tempPool); @@ -3533,9 +3727,9 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, static enum XML_Error PTRCALL cdataSectionProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { - enum XML_Error result - = doCdataSection(parser, parser->m_encoding, &start, end, endPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer); + enum XML_Error result = doCdataSection( + parser, parser->m_encoding, &start, end, endPtr, + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_ACCOUNT_DIRECT); if (result != XML_ERROR_NONE) return result; if (start) { @@ -3555,7 +3749,8 @@ cdataSectionProcessor(XML_Parser parser, const char *start, const char *end, */ static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, - const char *end, const char **nextPtr, XML_Bool haveMore) { + const char *end, const char **nextPtr, XML_Bool haveMore, + enum XML_Account account) { const char *s = *startPtr; const char **eventPP; const char **eventEndPP; @@ -3571,8 +3766,16 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *startPtr = NULL; for (;;) { - const char *next; + const char *next = s; /* in case of XML_TOK_NONE or XML_TOK_PARTIAL */ int tok = XmlCdataSectionTok(enc, s, end, &next); +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#else + UNUSED_P(account); +#endif *eventEndPP = next; switch (tok) { case XML_TOK_CDATA_SECT_CLOSE: @@ -3689,7 +3892,7 @@ ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end, static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore) { - const char *next; + const char *next = *startPtr; /* in case of XML_TOK_NONE or XML_TOK_PARTIAL */ int tok; const char *s = *startPtr; const char **eventPP; @@ -3717,6 +3920,13 @@ doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventPP = s; *startPtr = NULL; tok = XmlIgnoreSectionTok(enc, s, end, &next); +# ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +# endif *eventEndPP = next; switch (tok) { case XML_TOK_IGNORE_SECT: @@ -3801,6 +4011,15 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, const char *versionend; const XML_Char *storedversion = NULL; int standalone = -1; + +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, XML_TOK_XML_DECL, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif + if (! (parser->m_ns ? XmlParseXmlDeclNS : XmlParseXmlDecl)( isGeneralTextEntity, parser->m_encoding, s, next, &parser->m_eventPtr, &version, &versionend, &encodingName, &newEncoding, &standalone)) { @@ -3950,6 +4169,10 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, for (;;) { tok = XmlPrologTok(parser->m_encoding, start, end, &next); + /* Note: Except for XML_TOK_BOM below, these bytes are accounted later in: + - storeEntityValue + - processXmlDecl + */ parser->m_eventEndPtr = next; if (tok <= 0) { if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) { @@ -3968,7 +4191,8 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, break; } /* found end of entity value - can store it now */ - return storeEntityValue(parser, parser->m_encoding, s, end); + return storeEntityValue(parser, parser->m_encoding, s, end, + XML_ACCOUNT_DIRECT); } else if (tok == XML_TOK_XML_DECL) { enum XML_Error result; result = processXmlDecl(parser, 0, start, next); @@ -3995,6 +4219,14 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, */ else if (tok == XML_TOK_BOM && next == end && ! parser->m_parsingStatus.finalBuffer) { +# ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +# endif + *nextPtr = next; return XML_ERROR_NONE; } @@ -4037,16 +4269,24 @@ externalParEntProcessor(XML_Parser parser, const char *s, const char *end, } /* This would cause the next stage, i.e. doProlog to be passed XML_TOK_BOM. However, when parsing an external subset, doProlog will not accept a BOM - as valid, and report a syntax error, so we have to skip the BOM + as valid, and report a syntax error, so we have to skip the BOM, and + account for the BOM bytes. */ else if (tok == XML_TOK_BOM) { + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } + s = next; tok = XmlPrologTok(parser->m_encoding, s, end, &next); } parser->m_processor = prologProcessor; return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, + XML_ACCOUNT_DIRECT); } static enum XML_Error PTRCALL @@ -4059,6 +4299,9 @@ entityValueProcessor(XML_Parser parser, const char *s, const char *end, for (;;) { tok = XmlPrologTok(enc, start, end, &next); + /* Note: These bytes are accounted later in: + - storeEntityValue + */ if (tok <= 0) { if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) { *nextPtr = s; @@ -4076,7 +4319,7 @@ entityValueProcessor(XML_Parser parser, const char *s, const char *end, break; } /* found end of entity value - can store it now */ - return storeEntityValue(parser, enc, s, end); + return storeEntityValue(parser, enc, s, end, XML_ACCOUNT_DIRECT); } start = next; } @@ -4090,13 +4333,14 @@ prologProcessor(XML_Parser parser, const char *s, const char *end, const char *next = s; int tok = XmlPrologTok(parser->m_encoding, s, end, &next); return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, + XML_ACCOUNT_DIRECT); } static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, int tok, const char *next, const char **nextPtr, XML_Bool haveMore, - XML_Bool allowClosingDoctype) { + XML_Bool allowClosingDoctype, enum XML_Account account) { #ifdef XML_DTD static const XML_Char externalSubsetName[] = {ASCII_HASH, '\0'}; #endif /* XML_DTD */ @@ -4123,6 +4367,10 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, static const XML_Char enumValueSep[] = {ASCII_PIPE, '\0'}; static const XML_Char enumValueStart[] = {ASCII_LPAREN, '\0'}; +#ifndef XML_DTD + UNUSED_P(account); +#endif + /* save one level of indirection */ DTD *const dtd = parser->m_dtd; @@ -4187,6 +4435,19 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } } role = XmlTokenRole(&parser->m_prologState, tok, s, next, enc); +#ifdef XML_DTD + switch (role) { + case XML_ROLE_INSTANCE_START: // bytes accounted in contentProcessor + case XML_ROLE_XML_DECL: // bytes accounted in processXmlDecl + case XML_ROLE_TEXT_DECL: // bytes accounted in processXmlDecl + break; + default: + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } + } +#endif switch (role) { case XML_ROLE_XML_DECL: { enum XML_Error result = processXmlDecl(parser, 0, s, next); @@ -4462,7 +4723,8 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, const XML_Char *attVal; enum XML_Error result = storeAttributeValue( parser, enc, parser->m_declAttributeIsCdata, - s + enc->minBytesPerChar, next - enc->minBytesPerChar, &dtd->pool); + s + enc->minBytesPerChar, next - enc->minBytesPerChar, &dtd->pool, + XML_ACCOUNT_NONE); if (result) return result; attVal = poolStart(&dtd->pool); @@ -4495,8 +4757,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; case XML_ROLE_ENTITY_VALUE: if (dtd->keepProcessing) { - enum XML_Error result = storeEntityValue( - parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar); + enum XML_Error result + = storeEntityValue(parser, enc, s + enc->minBytesPerChar, + next - enc->minBytesPerChar, XML_ACCOUNT_NONE); if (parser->m_declEntity) { parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool); parser->m_declEntity->textLen @@ -4886,12 +5149,15 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; + entityTrackingOnOpen(parser, entity, __LINE__); if (! parser->m_externalEntityRefHandler( parser->m_externalEntityRefHandlerArg, 0, entity->base, entity->systemId, entity->publicId)) { + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; } + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; handleDefault = XML_FALSE; if (! dtd->paramEntityRead) { @@ -5089,6 +5355,13 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, for (;;) { const char *next = NULL; int tok = XmlPrologTok(parser->m_encoding, s, end, &next); +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif parser->m_eventEndPtr = next; switch (tok) { /* report partial linebreak - it might be the last token */ @@ -5162,6 +5435,9 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { return XML_ERROR_NO_MEMORY; } entity->open = XML_TRUE; +#ifdef XML_DTD + entityTrackingOnOpen(parser, entity, __LINE__); +#endif entity->processed = 0; openEntity->next = parser->m_openInternalEntities; parser->m_openInternalEntities = openEntity; @@ -5170,8 +5446,8 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { openEntity->betweenDecl = betweenDecl; openEntity->internalEventPtr = NULL; openEntity->internalEventEndPtr = NULL; - textStart = (char *)entity->textPtr; - textEnd = (char *)(entity->textPtr + entity->textLen); + textStart = (const char *)entity->textPtr; + textEnd = (const char *)(entity->textPtr + entity->textLen); /* Set a safe default value in case 'next' does not get set */ next = textStart; @@ -5180,17 +5456,22 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, - tok, next, &next, XML_FALSE, XML_FALSE); + tok, next, &next, XML_FALSE, XML_FALSE, + XML_ACCOUNT_ENTITY_EXPANSION); } else #endif /* XML_DTD */ result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, - textStart, textEnd, &next, XML_FALSE); + textStart, textEnd, &next, XML_FALSE, + XML_ACCOUNT_ENTITY_EXPANSION); if (result == XML_ERROR_NONE) { if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { entity->processed = (int)(next - textStart); parser->m_processor = internalEntityProcessor; } else { +#ifdef XML_DTD + entityTrackingOnClose(parser, entity, __LINE__); +#endif /* XML_DTD */ entity->open = XML_FALSE; parser->m_openInternalEntities = openEntity->next; /* put openEntity back in list of free instances */ @@ -5213,8 +5494,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return XML_ERROR_UNEXPECTED_STATE; entity = openEntity->entity; - textStart = ((char *)entity->textPtr) + entity->processed; - textEnd = (char *)(entity->textPtr + entity->textLen); + textStart = ((const char *)entity->textPtr) + entity->processed; + textEnd = (const char *)(entity->textPtr + entity->textLen); /* Set a safe default value in case 'next' does not get set */ next = textStart; @@ -5223,20 +5504,24 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, - tok, next, &next, XML_FALSE, XML_TRUE); + tok, next, &next, XML_FALSE, XML_TRUE, + XML_ACCOUNT_ENTITY_EXPANSION); } else #endif /* XML_DTD */ result = doContent(parser, openEntity->startTagLevel, parser->m_internalEncoding, textStart, textEnd, &next, - XML_FALSE); + XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); if (result != XML_ERROR_NONE) return result; else if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { - entity->processed = (int)(next - (char *)entity->textPtr); + entity->processed = (int)(next - (const char *)entity->textPtr); return result; } else { +#ifdef XML_DTD + entityTrackingOnClose(parser, entity, __LINE__); +#endif entity->open = XML_FALSE; parser->m_openInternalEntities = openEntity->next; /* put openEntity back in list of free instances */ @@ -5250,7 +5535,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, parser->m_processor = prologProcessor; tok = XmlPrologTok(parser->m_encoding, s, end, &next); return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, + XML_ACCOUNT_DIRECT); } else #endif /* XML_DTD */ { @@ -5258,7 +5544,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, /* see externalEntityContentProcessor vs contentProcessor */ return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding, s, end, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, + XML_ACCOUNT_DIRECT); } } @@ -5273,9 +5560,10 @@ errorProcessor(XML_Parser parser, const char *s, const char *end, static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, - const char *ptr, const char *end, STRING_POOL *pool) { + const char *ptr, const char *end, STRING_POOL *pool, + enum XML_Account account) { enum XML_Error result - = appendAttributeValue(parser, enc, isCdata, ptr, end, pool); + = appendAttributeValue(parser, enc, isCdata, ptr, end, pool, account); if (result) return result; if (! isCdata && poolLength(pool) && poolLastChar(pool) == 0x20) @@ -5287,11 +5575,23 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, - const char *ptr, const char *end, STRING_POOL *pool) { + const char *ptr, const char *end, STRING_POOL *pool, + enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ +#ifndef XML_DTD + UNUSED_P(account); +#endif + for (;;) { - const char *next; + const char *next + = ptr; /* XmlAttributeValueTok doesn't always set the last arg */ int tok = XmlAttributeValueTok(enc, ptr, end, &next); +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, ptr, next, __LINE__, account)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif switch (tok) { case XML_TOK_NONE: return XML_ERROR_NONE; @@ -5351,6 +5651,14 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { +#ifdef XML_DTD + /* NOTE: We are replacing 4-6 characters original input for 1 character + * so there is no amplification and hence recording without + * protection. */ + accountingDiffTolerated(parser, tok, (char *)&ch, + ((char *)&ch) + sizeof(XML_Char), __LINE__, + XML_ACCOUNT_ENTITY_EXPANSION); +#endif /* XML_DTD */ if (! poolAppendChar(pool, ch)) return XML_ERROR_NO_MEMORY; break; @@ -5428,9 +5736,16 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, enum XML_Error result; const XML_Char *textEnd = entity->textPtr + entity->textLen; entity->open = XML_TRUE; +#ifdef XML_DTD + entityTrackingOnOpen(parser, entity, __LINE__); +#endif result = appendAttributeValue(parser, parser->m_internalEncoding, - isCdata, (char *)entity->textPtr, - (char *)textEnd, pool); + isCdata, (const char *)entity->textPtr, + (const char *)textEnd, pool, + XML_ACCOUNT_ENTITY_EXPANSION); +#ifdef XML_DTD + entityTrackingOnClose(parser, entity, __LINE__); +#endif entity->open = XML_FALSE; if (result) return result; @@ -5460,13 +5775,16 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, - const char *entityTextPtr, const char *entityTextEnd) { + const char *entityTextPtr, const char *entityTextEnd, + enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ STRING_POOL *pool = &(dtd->entityValuePool); enum XML_Error result = XML_ERROR_NONE; #ifdef XML_DTD int oldInEntityValue = parser->m_prologState.inEntityValue; parser->m_prologState.inEntityValue = 1; +#else + UNUSED_P(account); #endif /* XML_DTD */ /* never return Null for the value argument in EntityDeclHandler, since this would indicate an external entity; therefore we @@ -5477,8 +5795,19 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, } for (;;) { - const char *next; + const char *next + = entityTextPtr; /* XmlEntityValueTok doesn't always set the last arg */ int tok = XmlEntityValueTok(enc, entityTextPtr, entityTextEnd, &next); + +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, entityTextPtr, next, __LINE__, + account)) { + accountingOnAbort(parser); + result = XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + goto endEntityValue; + } +#endif + switch (tok) { case XML_TOK_PARAM_ENTITY_REF: #ifdef XML_DTD @@ -5514,13 +5843,16 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; + entityTrackingOnOpen(parser, entity, __LINE__); if (! parser->m_externalEntityRefHandler( parser->m_externalEntityRefHandlerArg, 0, entity->base, entity->systemId, entity->publicId)) { + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; goto endEntityValue; } + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; if (! dtd->paramEntityRead) dtd->keepProcessing = dtd->standalone; @@ -5528,9 +5860,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, dtd->keepProcessing = dtd->standalone; } else { entity->open = XML_TRUE; + entityTrackingOnOpen(parser, entity, __LINE__); result = storeEntityValue( - parser, parser->m_internalEncoding, (char *)entity->textPtr, - (char *)(entity->textPtr + entity->textLen)); + parser, parser->m_internalEncoding, (const char *)entity->textPtr, + (const char *)(entity->textPtr + entity->textLen), + XML_ACCOUNT_ENTITY_EXPANSION); + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; if (result) goto endEntityValue; @@ -6485,7 +6820,7 @@ hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) { static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) { iter->p = table->v; - iter->end = iter->p + table->size; + iter->end = iter->p ? iter->p + table->size : NULL; } static NAMED *FASTCALL @@ -6891,3 +7226,755 @@ copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { memcpy(result, s, charsRequired * sizeof(XML_Char)); return result; } + +#ifdef XML_DTD + +static float +accountingGetCurrentAmplification(XML_Parser rootParser) { + const XmlBigCount countBytesOutput + = rootParser->m_accounting.countBytesDirect + + rootParser->m_accounting.countBytesIndirect; + const float amplificationFactor + = rootParser->m_accounting.countBytesDirect + ? (countBytesOutput + / (float)(rootParser->m_accounting.countBytesDirect)) + : 1.0f; + assert(! rootParser->m_parentParser); + return amplificationFactor; +} + +static void +accountingReportStats(XML_Parser originParser, const char *epilog) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + if (rootParser->m_accounting.debugLevel < 1) { + return; + } + + const float amplificationFactor + = accountingGetCurrentAmplification(rootParser); + fprintf(stderr, + "expat: Accounting(%p): Direct " EXPAT_FMT_ULL( + "10") ", indirect " EXPAT_FMT_ULL("10") ", amplification %8.2f%s", + (void *)rootParser, rootParser->m_accounting.countBytesDirect, + rootParser->m_accounting.countBytesIndirect, + (double)amplificationFactor, epilog); +} + +static void +accountingOnAbort(XML_Parser originParser) { + accountingReportStats(originParser, " ABORTING\n"); +} + +static void +accountingReportDiff(XML_Parser rootParser, + unsigned int levelsAwayFromRootParser, const char *before, + const char *after, ptrdiff_t bytesMore, int source_line, + enum XML_Account account) { + assert(! rootParser->m_parentParser); + + fprintf(stderr, + " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%d, xmlparse.c:%d) %*s\"", + bytesMore, (account == XML_ACCOUNT_DIRECT) ? "DIR" : "EXP", + levelsAwayFromRootParser, source_line, 10, ""); + + const char ellipis[] = "[..]"; + const size_t ellipsisLength = sizeof(ellipis) /* because compile-time */ - 1; + const unsigned int contextLength = 10; + + /* Note: Performance is of no concern here */ + const char *walker = before; + if ((rootParser->m_accounting.debugLevel >= 3) + || (after - before) + <= (ptrdiff_t)(contextLength + ellipsisLength + contextLength)) { + for (; walker < after; walker++) { + fprintf(stderr, "%s", unsignedCharToPrintable(walker[0])); + } + } else { + for (; walker < before + contextLength; walker++) { + fprintf(stderr, "%s", unsignedCharToPrintable(walker[0])); + } + fprintf(stderr, ellipis); + walker = after - contextLength; + for (; walker < after; walker++) { + fprintf(stderr, "%s", unsignedCharToPrintable(walker[0])); + } + } + fprintf(stderr, "\"\n"); +} + +static XML_Bool +accountingDiffTolerated(XML_Parser originParser, int tok, const char *before, + const char *after, int source_line, + enum XML_Account account) { + /* Note: We need to check the token type *first* to be sure that + * we can even access variable , safely. + * E.g. for XML_TOK_NONE may hold an invalid pointer. */ + switch (tok) { + case XML_TOK_INVALID: + case XML_TOK_PARTIAL: + case XML_TOK_PARTIAL_CHAR: + case XML_TOK_NONE: + return XML_TRUE; + } + + if (account == XML_ACCOUNT_NONE) + return XML_TRUE; /* because these bytes have been accounted for, already */ + + unsigned int levelsAwayFromRootParser; + const XML_Parser rootParser + = getRootParserOf(originParser, &levelsAwayFromRootParser); + assert(! rootParser->m_parentParser); + + const int isDirect + = (account == XML_ACCOUNT_DIRECT) && (originParser == rootParser); + const ptrdiff_t bytesMore = after - before; + + XmlBigCount *const additionTarget + = isDirect ? &rootParser->m_accounting.countBytesDirect + : &rootParser->m_accounting.countBytesIndirect; + + /* Detect and avoid integer overflow */ + if (*additionTarget > (XmlBigCount)(-1) - (XmlBigCount)bytesMore) + return XML_FALSE; + *additionTarget += bytesMore; + + const XmlBigCount countBytesOutput + = rootParser->m_accounting.countBytesDirect + + rootParser->m_accounting.countBytesIndirect; + const float amplificationFactor + = accountingGetCurrentAmplification(rootParser); + const XML_Bool tolerated + = (countBytesOutput < rootParser->m_accounting.activationThresholdBytes) + || (amplificationFactor + <= rootParser->m_accounting.maximumAmplificationFactor); + + if (rootParser->m_accounting.debugLevel >= 2) { + accountingReportStats(rootParser, ""); + accountingReportDiff(rootParser, levelsAwayFromRootParser, before, after, + bytesMore, source_line, account); + } + + return tolerated; +} + +unsigned long long +testingAccountingGetCountBytesDirect(XML_Parser parser) { + if (! parser) + return 0; + return parser->m_accounting.countBytesDirect; +} + +unsigned long long +testingAccountingGetCountBytesIndirect(XML_Parser parser) { + if (! parser) + return 0; + return parser->m_accounting.countBytesIndirect; +} + +static void +entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, + const char *action, int sourceLine) { + assert(! rootParser->m_parentParser); + if (rootParser->m_entity_stats.debugLevel < 1) + return; + +# if defined(XML_UNICODE) + const char *const entityName = "[..]"; +# else + const char *const entityName = entity->name; +# endif + + fprintf( + stderr, + "expat: Entities(%p): Count %9d, depth %2d/%2d %*s%s%s; %s length %d (xmlparse.c:%d)\n", + (void *)rootParser, rootParser->m_entity_stats.countEverOpened, + rootParser->m_entity_stats.currentDepth, + rootParser->m_entity_stats.maximumDepthSeen, + (rootParser->m_entity_stats.currentDepth - 1) * 2, "", + entity->is_param ? "%" : "&", entityName, action, entity->textLen, + sourceLine); +} + +static void +entityTrackingOnOpen(XML_Parser originParser, ENTITY *entity, int sourceLine) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + rootParser->m_entity_stats.countEverOpened++; + rootParser->m_entity_stats.currentDepth++; + if (rootParser->m_entity_stats.currentDepth + > rootParser->m_entity_stats.maximumDepthSeen) { + rootParser->m_entity_stats.maximumDepthSeen++; + } + + entityTrackingReportStats(rootParser, entity, "OPEN ", sourceLine); +} + +static void +entityTrackingOnClose(XML_Parser originParser, ENTITY *entity, int sourceLine) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + entityTrackingReportStats(rootParser, entity, "CLOSE", sourceLine); + rootParser->m_entity_stats.currentDepth--; +} + +static XML_Parser +getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) { + XML_Parser rootParser = parser; + unsigned int stepsTakenUpwards = 0; + while (rootParser->m_parentParser) { + rootParser = rootParser->m_parentParser; + stepsTakenUpwards++; + } + assert(! rootParser->m_parentParser); + if (outLevelDiff != NULL) { + *outLevelDiff = stepsTakenUpwards; + } + return rootParser; +} + +const char * +unsignedCharToPrintable(unsigned char c) { + switch (c) { + case 0: + return "\\0"; + case 1: + return "\\x1"; + case 2: + return "\\x2"; + case 3: + return "\\x3"; + case 4: + return "\\x4"; + case 5: + return "\\x5"; + case 6: + return "\\x6"; + case 7: + return "\\x7"; + case 8: + return "\\x8"; + case 9: + return "\\t"; + case 10: + return "\\n"; + case 11: + return "\\xB"; + case 12: + return "\\xC"; + case 13: + return "\\r"; + case 14: + return "\\xE"; + case 15: + return "\\xF"; + case 16: + return "\\x10"; + case 17: + return "\\x11"; + case 18: + return "\\x12"; + case 19: + return "\\x13"; + case 20: + return "\\x14"; + case 21: + return "\\x15"; + case 22: + return "\\x16"; + case 23: + return "\\x17"; + case 24: + return "\\x18"; + case 25: + return "\\x19"; + case 26: + return "\\x1A"; + case 27: + return "\\x1B"; + case 28: + return "\\x1C"; + case 29: + return "\\x1D"; + case 30: + return "\\x1E"; + case 31: + return "\\x1F"; + case 32: + return " "; + case 33: + return "!"; + case 34: + return "\\\""; + case 35: + return "#"; + case 36: + return "$"; + case 37: + return "%"; + case 38: + return "&"; + case 39: + return "'"; + case 40: + return "("; + case 41: + return ")"; + case 42: + return "*"; + case 43: + return "+"; + case 44: + return ","; + case 45: + return "-"; + case 46: + return "."; + case 47: + return "/"; + case 48: + return "0"; + case 49: + return "1"; + case 50: + return "2"; + case 51: + return "3"; + case 52: + return "4"; + case 53: + return "5"; + case 54: + return "6"; + case 55: + return "7"; + case 56: + return "8"; + case 57: + return "9"; + case 58: + return ":"; + case 59: + return ";"; + case 60: + return "<"; + case 61: + return "="; + case 62: + return ">"; + case 63: + return "?"; + case 64: + return "@"; + case 65: + return "A"; + case 66: + return "B"; + case 67: + return "C"; + case 68: + return "D"; + case 69: + return "E"; + case 70: + return "F"; + case 71: + return "G"; + case 72: + return "H"; + case 73: + return "I"; + case 74: + return "J"; + case 75: + return "K"; + case 76: + return "L"; + case 77: + return "M"; + case 78: + return "N"; + case 79: + return "O"; + case 80: + return "P"; + case 81: + return "Q"; + case 82: + return "R"; + case 83: + return "S"; + case 84: + return "T"; + case 85: + return "U"; + case 86: + return "V"; + case 87: + return "W"; + case 88: + return "X"; + case 89: + return "Y"; + case 90: + return "Z"; + case 91: + return "["; + case 92: + return "\\\\"; + case 93: + return "]"; + case 94: + return "^"; + case 95: + return "_"; + case 96: + return "`"; + case 97: + return "a"; + case 98: + return "b"; + case 99: + return "c"; + case 100: + return "d"; + case 101: + return "e"; + case 102: + return "f"; + case 103: + return "g"; + case 104: + return "h"; + case 105: + return "i"; + case 106: + return "j"; + case 107: + return "k"; + case 108: + return "l"; + case 109: + return "m"; + case 110: + return "n"; + case 111: + return "o"; + case 112: + return "p"; + case 113: + return "q"; + case 114: + return "r"; + case 115: + return "s"; + case 116: + return "t"; + case 117: + return "u"; + case 118: + return "v"; + case 119: + return "w"; + case 120: + return "x"; + case 121: + return "y"; + case 122: + return "z"; + case 123: + return "{"; + case 124: + return "|"; + case 125: + return "}"; + case 126: + return "~"; + case 127: + return "\\x7F"; + case 128: + return "\\x80"; + case 129: + return "\\x81"; + case 130: + return "\\x82"; + case 131: + return "\\x83"; + case 132: + return "\\x84"; + case 133: + return "\\x85"; + case 134: + return "\\x86"; + case 135: + return "\\x87"; + case 136: + return "\\x88"; + case 137: + return "\\x89"; + case 138: + return "\\x8A"; + case 139: + return "\\x8B"; + case 140: + return "\\x8C"; + case 141: + return "\\x8D"; + case 142: + return "\\x8E"; + case 143: + return "\\x8F"; + case 144: + return "\\x90"; + case 145: + return "\\x91"; + case 146: + return "\\x92"; + case 147: + return "\\x93"; + case 148: + return "\\x94"; + case 149: + return "\\x95"; + case 150: + return "\\x96"; + case 151: + return "\\x97"; + case 152: + return "\\x98"; + case 153: + return "\\x99"; + case 154: + return "\\x9A"; + case 155: + return "\\x9B"; + case 156: + return "\\x9C"; + case 157: + return "\\x9D"; + case 158: + return "\\x9E"; + case 159: + return "\\x9F"; + case 160: + return "\\xA0"; + case 161: + return "\\xA1"; + case 162: + return "\\xA2"; + case 163: + return "\\xA3"; + case 164: + return "\\xA4"; + case 165: + return "\\xA5"; + case 166: + return "\\xA6"; + case 167: + return "\\xA7"; + case 168: + return "\\xA8"; + case 169: + return "\\xA9"; + case 170: + return "\\xAA"; + case 171: + return "\\xAB"; + case 172: + return "\\xAC"; + case 173: + return "\\xAD"; + case 174: + return "\\xAE"; + case 175: + return "\\xAF"; + case 176: + return "\\xB0"; + case 177: + return "\\xB1"; + case 178: + return "\\xB2"; + case 179: + return "\\xB3"; + case 180: + return "\\xB4"; + case 181: + return "\\xB5"; + case 182: + return "\\xB6"; + case 183: + return "\\xB7"; + case 184: + return "\\xB8"; + case 185: + return "\\xB9"; + case 186: + return "\\xBA"; + case 187: + return "\\xBB"; + case 188: + return "\\xBC"; + case 189: + return "\\xBD"; + case 190: + return "\\xBE"; + case 191: + return "\\xBF"; + case 192: + return "\\xC0"; + case 193: + return "\\xC1"; + case 194: + return "\\xC2"; + case 195: + return "\\xC3"; + case 196: + return "\\xC4"; + case 197: + return "\\xC5"; + case 198: + return "\\xC6"; + case 199: + return "\\xC7"; + case 200: + return "\\xC8"; + case 201: + return "\\xC9"; + case 202: + return "\\xCA"; + case 203: + return "\\xCB"; + case 204: + return "\\xCC"; + case 205: + return "\\xCD"; + case 206: + return "\\xCE"; + case 207: + return "\\xCF"; + case 208: + return "\\xD0"; + case 209: + return "\\xD1"; + case 210: + return "\\xD2"; + case 211: + return "\\xD3"; + case 212: + return "\\xD4"; + case 213: + return "\\xD5"; + case 214: + return "\\xD6"; + case 215: + return "\\xD7"; + case 216: + return "\\xD8"; + case 217: + return "\\xD9"; + case 218: + return "\\xDA"; + case 219: + return "\\xDB"; + case 220: + return "\\xDC"; + case 221: + return "\\xDD"; + case 222: + return "\\xDE"; + case 223: + return "\\xDF"; + case 224: + return "\\xE0"; + case 225: + return "\\xE1"; + case 226: + return "\\xE2"; + case 227: + return "\\xE3"; + case 228: + return "\\xE4"; + case 229: + return "\\xE5"; + case 230: + return "\\xE6"; + case 231: + return "\\xE7"; + case 232: + return "\\xE8"; + case 233: + return "\\xE9"; + case 234: + return "\\xEA"; + case 235: + return "\\xEB"; + case 236: + return "\\xEC"; + case 237: + return "\\xED"; + case 238: + return "\\xEE"; + case 239: + return "\\xEF"; + case 240: + return "\\xF0"; + case 241: + return "\\xF1"; + case 242: + return "\\xF2"; + case 243: + return "\\xF3"; + case 244: + return "\\xF4"; + case 245: + return "\\xF5"; + case 246: + return "\\xF6"; + case 247: + return "\\xF7"; + case 248: + return "\\xF8"; + case 249: + return "\\xF9"; + case 250: + return "\\xFA"; + case 251: + return "\\xFB"; + case 252: + return "\\xFC"; + case 253: + return "\\xFD"; + case 254: + return "\\xFE"; + case 255: + return "\\xFF"; + default: + assert(0); /* never gets here */ + return "dead code"; + } + assert(0); /* never gets here */ +} + +#endif /* XML_DTD */ + +static unsigned long +getDebugLevel(const char *variableName, unsigned long defaultDebugLevel) { + const char *const valueOrNull = getenv(variableName); + if (valueOrNull == NULL) { + return defaultDebugLevel; + } + const char *const value = valueOrNull; + + errno = 0; + char *afterValue = (char *)value; + unsigned long debugLevel = strtoul(value, &afterValue, 10); + if ((errno != 0) || (afterValue[0] != '\0')) { + errno = 0; + return defaultDebugLevel; + } + + return debugLevel; +} diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 4d3e3e86..08173b0f 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -7,7 +7,14 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2002 Greg Stein + Copyright (c) 2002-2006 Karl Waclawek + Copyright (c) 2002-2003 Fred L. Drake, Jr. + Copyright (c) 2005-2009 Steven Solie + Copyright (c) 2016-2021 Sebastian Pipping + Copyright (c) 2017 Rhodri James + Copyright (c) 2019 David Loffredo Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -34,11 +41,9 @@ #ifdef _WIN32 # include "winconfig.h" -#else -# ifdef HAVE_EXPAT_CONFIG_H -# include -# endif -#endif /* ndef _WIN32 */ +#endif + +#include #include "expat_external.h" #include "internal.h" @@ -1220,6 +1225,8 @@ common(PROLOG_STATE *state, int tok) { #ifdef XML_DTD if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF) return XML_ROLE_INNER_PARAM_ENTITY_REF; +#else + UNUSED_P(tok); #endif state->handler = error; return XML_ROLE_ERROR; diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h index 036aba64..d6e1fa15 100644 --- a/Modules/expat/xmlrole.h +++ b/Modules/expat/xmlrole.h @@ -7,7 +7,10 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2002 Karl Waclawek + Copyright (c) 2002 Fred L. Drake, Jr. + Copyright (c) 2017 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index 54cfedb8..f2b6b406 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -7,7 +7,19 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2001-2003 Fred L. Drake, Jr. + Copyright (c) 2002 Greg Stein + Copyright (c) 2002-2016 Karl Waclawek + Copyright (c) 2005-2009 Steven Solie + Copyright (c) 2016-2021 Sebastian Pipping + Copyright (c) 2016 Pascal Cuoq + Copyright (c) 2016 Don Lewis + Copyright (c) 2017 Rhodri James + Copyright (c) 2017 Alexander Bluhm + Copyright (c) 2017 Benbuck Nason + Copyright (c) 2017 José Gutiérrez de la Concha + Copyright (c) 2019 David Loffredo Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -30,26 +42,16 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#ifdef _WIN32 -# include "winconfig.h" -#else -# ifdef HAVE_EXPAT_CONFIG_H -# include -# endif -#endif /* ndef _WIN32 */ - #include #include /* memcpy */ +#include -#if defined(_MSC_VER) && (_MSC_VER <= 1700) -/* for vs2012/11.0/1700 and earlier Visual Studio compilers */ -# define bool int -# define false 0 -# define true 1 -#else -# include +#ifdef _WIN32 +# include "winconfig.h" #endif +#include + #include "expat_external.h" #include "internal.h" #include "xmltok.h" @@ -269,8 +271,14 @@ sb_byteToAscii(const ENCODING *enc, const char *p) { #define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p)) #define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p)) -#define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) +#ifdef XML_MIN_SIZE +# define IS_INVALID_CHAR(enc, p, n) \ + (AS_NORMAL_ENCODING(enc)->isInvalid##n \ + && AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) +#else +# define IS_INVALID_CHAR(enc, p, n) \ + (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) +#endif #ifdef XML_MIN_SIZE # define IS_NAME_CHAR_MINBPC(enc, p) \ @@ -589,13 +597,13 @@ static const struct normal_encoding ascii_encoding static int PTRFASTCALL unicode_byte_type(char hi, char lo) { switch ((unsigned char)hi) { - /* 0xD800–0xDBFF first 16-bit code unit or high surrogate (W1) */ + /* 0xD800-0xDBFF first 16-bit code unit or high surrogate (W1) */ case 0xD8: case 0xD9: case 0xDA: case 0xDB: return BT_LEAD4; - /* 0xDC00–0xDFFF second 16-bit code unit or low surrogate (W2) */ + /* 0xDC00-0xDFFF second 16-bit code unit or low surrogate (W2) */ case 0xDC: case 0xDD: case 0xDE: diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index 2adbf530..6f630c2f 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -7,7 +7,11 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2002 Fred L. Drake, Jr. + Copyright (c) 2002-2005 Karl Waclawek + Copyright (c) 2016-2017 Sebastian Pipping + Copyright (c) 2017 Rhodri James Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index c209221c..0430591b 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -1,4 +1,4 @@ -/* This file is included! +/* This file is included (from xmltok.c, 1-3 times depending on XML_MIN_SIZE)! __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -7,7 +7,15 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2002 Fred L. Drake, Jr. + Copyright (c) 2002-2016 Karl Waclawek + Copyright (c) 2016-2021 Sebastian Pipping + Copyright (c) 2017 Rhodri James + Copyright (c) 2018 Benjamin Peterson + Copyright (c) 2018 Anton Maklakov + Copyright (c) 2019 David Loffredo + Copyright (c) 2020 Boris Kolpackov Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -32,7 +40,7 @@ #ifdef XML_TOK_IMPL_C -# ifndef IS_INVALID_CHAR +# ifndef IS_INVALID_CHAR // i.e. for UTF-16 and XML_MIN_SIZE not defined # define IS_INVALID_CHAR(enc, ptr, n) (0) # endif @@ -1768,13 +1776,14 @@ PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end, # define LEAD_CASE(n) \ case BT_LEAD##n: \ ptr += n; \ + pos->columnNumber++; \ break; LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4) # undef LEAD_CASE case BT_LF: - pos->columnNumber = (XML_Size)-1; + pos->columnNumber = 0; pos->lineNumber++; ptr += MINBPC(enc); break; @@ -1783,13 +1792,13 @@ PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end, ptr += MINBPC(enc); if (HAS_CHAR(enc, ptr, end) && BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); - pos->columnNumber = (XML_Size)-1; + pos->columnNumber = 0; break; default: ptr += MINBPC(enc); + pos->columnNumber++; break; } - pos->columnNumber++; } } diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h index e925dbc7..c518aada 100644 --- a/Modules/expat/xmltok_impl.h +++ b/Modules/expat/xmltok_impl.h @@ -7,7 +7,8 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2017-2019 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c index e1b46a7c..5fd83922 100644 --- a/Modules/expat/xmltok_ns.c +++ b/Modules/expat/xmltok_ns.c @@ -7,7 +7,11 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper + Copyright (c) 2002 Greg Stein + Copyright (c) 2002 Fred L. Drake, Jr. + Copyright (c) 2002-2006 Karl Waclawek + Copyright (c) 2017 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -89,7 +93,7 @@ NS(XmlInitEncoding)(INIT_ENCODING *p, const ENCODING **encPtr, static const ENCODING * NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end) { # define ENCODING_MAX 128 - char buf[ENCODING_MAX] = {0}; + char buf[ENCODING_MAX]; char *p = buf; int i; XmlUtf8Convert(enc, &ptr, end, &p, p + ENCODING_MAX - 1); diff --git a/Modules/gc_weakref.txt b/Modules/gc_weakref.txt index 2f18402c..b5b9f7b2 100644 --- a/Modules/gc_weakref.txt +++ b/Modules/gc_weakref.txt @@ -47,7 +47,7 @@ soon as we execute Python code, threads other than the gc thread can run too, and they can do ordinary things with weakrefs that end up resurrecting CT while gc is running. - http://www.python.org/sf/1055820 + https://www.python.org/sf/1055820 shows how innocent it can be, and also how nasty. Variants of the three focussed test cases attached to that bug report are now part of Python's diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 95ef8d79..bc5382af 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -729,7 +729,7 @@ static PyObject * tee_fromiterable(PyObject *iterable) { teeobject *to; - PyObject *it = NULL; + PyObject *it; it = PyObject_GetIter(iterable); if (it == NULL) @@ -739,21 +739,22 @@ tee_fromiterable(PyObject *iterable) goto done; } - to = PyObject_GC_New(teeobject, &tee_type); - if (to == NULL) - goto done; - to->dataobj = (teedataobject *)teedataobject_newinternal(it); - if (!to->dataobj) { - PyObject_GC_Del(to); + PyObject *dataobj = teedataobject_newinternal(it); + if (!dataobj) { to = NULL; goto done; } - + to = PyObject_GC_New(teeobject, &tee_type); + if (to == NULL) { + Py_DECREF(dataobj); + goto done; + } + to->dataobj = (teedataobject *)dataobj; to->index = 0; to->weakreflist = NULL; PyObject_GC_Track(to); done: - Py_XDECREF(it); + Py_DECREF(it); return (PyObject *)to; } diff --git a/Objects/classobject.c b/Objects/classobject.c index fd9f8757..af73be3d 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -462,7 +462,7 @@ instancemethod_traverse(PyObject *self, visitproc visit, void *arg) { static PyObject * instancemethod_call(PyObject *self, PyObject *arg, PyObject *kw) { - return PyObject_Call(PyMethod_GET_FUNCTION(self), arg, kw); + return PyObject_Call(PyInstanceMethod_GET_FUNCTION(self), arg, kw); } static PyObject * diff --git a/Objects/complexobject.c b/Objects/complexobject.c index a4903778..e09cc15f 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -169,14 +169,7 @@ c_powu(Py_complex x, long n) static Py_complex c_powi(Py_complex x, long n) { - Py_complex cn; - - if (n > 100 || n < -100) { - cn.real = (double) n; - cn.imag = 0.; - return _Py_c_pow(x,cn); - } - else if (n > 0) + if (n > 0) return c_powu(x,n); else return _Py_c_quot(c_1, c_powu(x,-n)); @@ -529,8 +522,6 @@ static PyObject * complex_pow(PyObject *v, PyObject *w, PyObject *z) { Py_complex p; - Py_complex exponent; - long int_exponent; Py_complex a, b; TO_COMPLEX(v, a); TO_COMPLEX(w, b); @@ -540,12 +531,14 @@ complex_pow(PyObject *v, PyObject *w, PyObject *z) return NULL; } errno = 0; - exponent = b; - int_exponent = (long)exponent.real; - if (exponent.imag == 0. && exponent.real == int_exponent) - p = c_powi(a, int_exponent); - else - p = _Py_c_pow(a, exponent); + // Check whether the exponent has a small integer value, and if so use + // a faster and more accurate algorithm. + if (b.imag == 0.0 && b.real == floor(b.real) && fabs(b.real) <= 100.0) { + p = c_powi(a, (long)b.real); + } + else { + p = _Py_c_pow(a, b); + } Py_ADJUST_ERANGE2(p.real, p.imag); if (errno == EDOM) { diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 9f501409..6adc90ab 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1443,8 +1443,8 @@ float_fromhex(PyTypeObject *type, PyObject *string) bits lsb, lsb-2, lsb-3, lsb-4, ... is 1. */ if ((digit & half_eps) != 0) { round_up = 0; - if ((digit & (3*half_eps-1)) != 0 || - (half_eps == 8 && (HEX_DIGIT(key_digit+1) & 1) != 0)) + if ((digit & (3*half_eps-1)) != 0 || (half_eps == 8 && + key_digit+1 < ndigits && (HEX_DIGIT(key_digit+1) & 1) != 0)) round_up = 1; else for (i = key_digit-1; i >= 0; i--) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 50846dec..4ae17bcf 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -575,7 +575,7 @@ frame_dealloc(PyFrameObject *f) if (_PyObject_GC_IS_TRACKED(f)) _PyObject_GC_UNTRACK(f); - Py_TRASHCAN_SAFE_BEGIN(f) + Py_TRASHCAN_BEGIN(f, frame_dealloc); /* Kill all local variables */ valuestack = f->f_valuestack; for (p = f->f_localsplus; p < valuestack; p++) @@ -609,7 +609,7 @@ frame_dealloc(PyFrameObject *f) } Py_DECREF(co); - Py_TRASHCAN_SAFE_END(f) + Py_TRASHCAN_END; } static inline Py_ssize_t diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 945d2059..69ad8a6d 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -602,7 +602,7 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } if (!setup_ga(self, origin, arguments)) { - type->tp_free((PyObject *)self); + Py_DECREF(self); return NULL; } return (PyObject *)self; @@ -640,14 +640,14 @@ PyTypeObject Py_GenericAliasType = { PyObject * Py_GenericAlias(PyObject *origin, PyObject *args) { - gaobject *alias = PyObject_GC_New(gaobject, &Py_GenericAliasType); + gaobject *alias = (gaobject*) PyType_GenericAlloc( + (PyTypeObject *)&Py_GenericAliasType, 0); if (alias == NULL) { return NULL; } if (!setup_ga(alias, origin, args)) { - PyObject_GC_Del((PyObject *)alias); + Py_DECREF(alias); return NULL; } - _PyObject_GC_TRACK(alias); return (PyObject *)alias; } diff --git a/Objects/listobject.c b/Objects/listobject.c index f9eb3706..6c94ba57 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -165,9 +165,10 @@ PyList_New(Py_ssize_t size) static PyObject * list_new_prealloc(Py_ssize_t size) { + assert(size > 0); PyListObject *op = (PyListObject *) PyList_New(0); - if (size == 0 || op == NULL) { - return (PyObject *) op; + if (op == NULL) { + return NULL; } assert(op->ob_item == NULL); op->ob_item = PyMem_New(PyObject *, size); @@ -446,6 +447,9 @@ list_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) PyObject **src, **dest; Py_ssize_t i, len; len = ihigh - ilow; + if (len <= 0) { + return PyList_New(0); + } np = (PyListObject *) list_new_prealloc(len); if (np == NULL) return NULL; @@ -500,6 +504,9 @@ list_concat(PyListObject *a, PyObject *bb) if (Py_SIZE(a) > PY_SSIZE_T_MAX - Py_SIZE(b)) return PyErr_NoMemory(); size = Py_SIZE(a) + Py_SIZE(b); + if (size == 0) { + return PyList_New(0); + } np = (PyListObject *) list_new_prealloc(size); if (np == NULL) { return NULL; diff --git a/Objects/methodobject.c b/Objects/methodobject.c index 7b430416..2df63cfd 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -160,7 +160,10 @@ PyCMethod_GetClass(PyObject *op) static void meth_dealloc(PyCFunctionObject *m) { - _PyObject_GC_UNTRACK(m); + // The Py_TRASHCAN mechanism requires that we be able to + // call PyObject_GC_UnTrack twice on an object. + PyObject_GC_UnTrack(m); + Py_TRASHCAN_BEGIN(m, meth_dealloc); if (m->m_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*) m); } @@ -170,6 +173,7 @@ meth_dealloc(PyCFunctionObject *m) Py_XDECREF(m->m_self); Py_XDECREF(m->m_module); PyObject_GC_Del(m); + Py_TRASHCAN_END; } static PyObject * diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 751dbb98..dd027936 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -793,7 +793,7 @@ rangeiter_reduce(rangeiterobject *r, PyObject *Py_UNUSED(ignored)) if (range == NULL) goto err; /* return the result */ - return Py_BuildValue("N(N)i", _PyEval_GetBuiltinId(&PyId_iter), + return Py_BuildValue("N(N)l", _PyEval_GetBuiltinId(&PyId_iter), range, r->index); err: Py_XDECREF(start); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index acbe3fa3..f201515d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1231,14 +1231,22 @@ subtype_dealloc(PyObject *self) /* Extract the type again; tp_del may have changed it */ type = Py_TYPE(self); + // Don't read type memory after calling basedealloc() since basedealloc() + // can deallocate the type and free its memory. + int type_needs_decref = (type->tp_flags & Py_TPFLAGS_HEAPTYPE + && !(base->tp_flags & Py_TPFLAGS_HEAPTYPE)); + /* Call the base tp_dealloc() */ assert(basedealloc); basedealloc(self); - /* Only decref if the base type is not already a heap allocated type. - Otherwise, basedealloc should have decref'd it already */ - if (type->tp_flags & Py_TPFLAGS_HEAPTYPE && !(base->tp_flags & Py_TPFLAGS_HEAPTYPE)) + /* Can't reference self beyond this point. It's possible tp_del switched + our type from a HEAPTYPE to a non-HEAPTYPE, so be careful about + reference counting. Only decref if the base type is not already a heap + allocated type. Otherwise, basedealloc should have decref'd it already */ + if (type_needs_decref) { Py_DECREF(type); + } /* Done */ return; @@ -1333,6 +1341,12 @@ subtype_dealloc(PyObject *self) if (_PyType_IS_GC(base)) { _PyObject_GC_TRACK(self); } + + // Don't read type memory after calling basedealloc() since basedealloc() + // can deallocate the type and free its memory. + int type_needs_decref = (type->tp_flags & Py_TPFLAGS_HEAPTYPE + && !(base->tp_flags & Py_TPFLAGS_HEAPTYPE)); + assert(basedealloc); basedealloc(self); @@ -1340,8 +1354,9 @@ subtype_dealloc(PyObject *self) our type from a HEAPTYPE to a non-HEAPTYPE, so be careful about reference counting. Only decref if the base type is not already a heap allocated type. Otherwise, basedealloc should have decref'd it already */ - if (type->tp_flags & Py_TPFLAGS_HEAPTYPE && !(base->tp_flags & Py_TPFLAGS_HEAPTYPE)) - Py_DECREF(type); + if (type_needs_decref) { + Py_DECREF(type); + } endlabel: Py_TRASHCAN_END diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 313e8aba..bb56c7db 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -657,6 +657,12 @@ proxy_iternext(PyWeakReference *proxy) return NULL; PyObject *obj = PyWeakref_GET_OBJECT(proxy); + if (!PyIter_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "Weakref proxy referenced a non-iterator '%.200s' object", + Py_TYPE(obj)->tp_name); + return NULL; + } Py_INCREF(obj); PyObject* res = PyIter_Next(obj); Py_DECREF(obj); @@ -732,21 +738,6 @@ static PyMappingMethods proxy_as_mapping = { }; -static Py_hash_t -proxy_hash(PyObject *self) -{ - PyWeakReference *proxy = (PyWeakReference *)self; - if (!proxy_checkref(proxy)) { - return -1; - } - PyObject *obj = PyWeakref_GET_OBJECT(proxy); - Py_INCREF(obj); - Py_hash_t res = PyObject_Hash(obj); - Py_DECREF(obj); - return res; -} - - PyTypeObject _PyWeakref_ProxyType = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -763,7 +754,8 @@ _PyWeakref_ProxyType = { &proxy_as_number, /* tp_as_number */ &proxy_as_sequence, /* tp_as_sequence */ &proxy_as_mapping, /* tp_as_mapping */ - proxy_hash, /* tp_hash */ +// Notice that tp_hash is intentionally omitted as proxies are "mutable" (when the reference dies). + 0, /* tp_hash */ 0, /* tp_call */ proxy_str, /* tp_str */ proxy_getattr, /* tp_getattro */ diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 1aa0cebd..c3a77480 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -52,8 +52,8 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.6 -if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1k +if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.3.0 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1l set libraries=%libraries% sqlite-3.35.5.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.9.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.9.0 @@ -76,8 +76,8 @@ for %%e in (%libraries%) do ( echo.Fetching external binaries... set binaries= -if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1k-1 +if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.3.0 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1l if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.9.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/prepare_libffi.bat b/PCbuild/prepare_libffi.bat index 8380c640..002ac6a4 100644 --- a/PCbuild/prepare_libffi.bat +++ b/PCbuild/prepare_libffi.bat @@ -38,6 +38,7 @@ set BUILD_ARM32= set BUILD_ARM64= set BUILD_PDB= set BUILD_NOOPT= +set COPY_LICENSE= set INSTALL_CYGWIN= :CheckOpts @@ -49,6 +50,7 @@ if /I "%1"=="-arm32" (set BUILD_ARM32=1) & shift & goto :CheckOpts if /I "%1"=="-arm64" (set BUILD_ARM64=1) & shift & goto :CheckOpts if /I "%1"=="-pdb" (set BUILD_PDB=-g) & shift & goto :CheckOpts if /I "%1"=="-noopt" (set BUILD_NOOPT=CFLAGS='-Od -warn all') & shift & goto :CheckOpts +if /I "%1"=="-license" (set COPY_LICENSE=1) & shift & goto :CheckOpts if /I "%1"=="-?" goto :Usage if /I "%1"=="--install-cygwin" (set INSTALL_CYGWIN=1) & shift & goto :CheckOpts goto :Usage @@ -60,6 +62,7 @@ if NOT DEFINED BUILD_X64 if NOT DEFINED BUILD_X86 if NOT DEFINED BUILD_ARM32 if set BUILD_X86=1 set BUILD_ARM32=1 set BUILD_ARM64=1 + set COPY_LICENSE=1 ) if "%INSTALL_CYGWIN%"=="1" call :InstallCygwin @@ -98,9 +101,14 @@ if not exist Makefile.in ( ) if "%BUILD_X64%"=="1" call :BuildOne x64 x86_64-w64-cygwin x86_64-w64-cygwin +if errorlevel 1 exit /B %ERRORLEVEL% if "%BUILD_X86%"=="1" call :BuildOne x86 i686-pc-cygwin i686-pc-cygwin +if errorlevel 1 exit /B %ERRORLEVEL% if "%BUILD_ARM32%"=="1" call :BuildOne x86_arm i686-pc-cygwin arm-w32-cygwin +if errorlevel 1 exit /B %ERRORLEVEL% if "%BUILD_ARM64%"=="1" call :BuildOne x86_arm64 i686-pc-cygwin aarch64-w64-cygwin +if errorlevel 1 exit /B %ERRORLEVEL% +if "%COPY_LICENSE%"=="1" copy /y "%LIBFFI_SOURCE%\LICENSE" "%LIBFFI_OUT%\LICENSE" popd endlocal @@ -179,11 +187,11 @@ if "%LIBFFI_TEST%" EQU "1" ( echo copying files to %_LIBFFI_OUT% if not exist %_LIBFFI_OUT%\include (md %_LIBFFI_OUT%\include) -copy %ARTIFACTS%\.libs\libffi-7.dll %_LIBFFI_OUT% -copy %ARTIFACTS%\.libs\libffi-7.lib %_LIBFFI_OUT% -copy %ARTIFACTS%\.libs\libffi-7.pdb %_LIBFFI_OUT% -copy %ARTIFACTS%\fficonfig.h %_LIBFFI_OUT%\include -copy %ARTIFACTS%\include\*.h %_LIBFFI_OUT%\include +copy %ARTIFACTS%\.libs\libffi-*.dll %_LIBFFI_OUT% || exit /B 1 +copy %ARTIFACTS%\.libs\libffi-*.lib %_LIBFFI_OUT% || exit /B 1 +copy %ARTIFACTS%\.libs\libffi-*.pdb %_LIBFFI_OUT% +copy %ARTIFACTS%\fficonfig.h %_LIBFFI_OUT%\include || exit /B 1 +copy %ARTIFACTS%\include\*.h %_LIBFFI_OUT%\include || exit /B 1 endlocal exit /b diff --git a/PCbuild/python.props b/PCbuild/python.props index 054e1c54..733e64bc 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -11,6 +11,7 @@ We set BasePlatformToolset for ICC's benefit, it's otherwise ignored. --> + v142 v142 v141 v140 @@ -59,11 +60,11 @@ $(ExternalsDir)sqlite-3.35.5.0\ $(ExternalsDir)bzip2-1.0.6\ $(ExternalsDir)xz-5.2.2\ - $(ExternalsDir)libffi\ - $(ExternalsDir)libffi\$(ArchName)\ + $(ExternalsDir)libffi-3.3.0\ + $(ExternalsDir)libffi-3.3.0\$(ArchName)\ $(libffiOutDir)include - $(ExternalsDir)openssl-1.1.1k\ - $(ExternalsDir)openssl-bin-1.1.1k-1\$(ArchName)\ + $(ExternalsDir)openssl-1.1.1l\ + $(ExternalsDir)openssl-bin-1.1.1l\$(ArchName)\ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ $(ExternalsDir)\zlib-1.2.11\ @@ -108,9 +109,18 @@ 10.0.10586.0 - $(_RegistryVersion) + $(_RegistryVersion) - + + + + <_Message>Failed to locate a Windows SDK installation. + <_Message>$(_Message) If the build fails, please use the Visual Studio Installer to install the Windows SDK. + <_Message>$(_Message) (Ignore the version number specified in the error message and select the latest.) + + + + $(DefaultWindowsSDKVersion) @@ -175,7 +185,7 @@ 11 12 - + $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber) $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)$(ReleaseLevelName) diff --git a/Parser/pegen/parse.c b/Parser/pegen/parse.c index 78e68d16..dc388ce3 100644 --- a/Parser/pegen/parse.c +++ b/Parser/pegen/parse.c @@ -15375,7 +15375,7 @@ invalid_group_rule(Parser *p) return _res; } -// invalid_import_from_targets: import_from_as_names ',' +// invalid_import_from_targets: import_from_as_names ',' NEWLINE static void * invalid_import_from_targets_rule(Parser *p) { @@ -15386,21 +15386,24 @@ invalid_import_from_targets_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // import_from_as_names ',' + { // import_from_as_names ',' NEWLINE if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> invalid_import_from_targets[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "import_from_as_names ','")); + D(fprintf(stderr, "%*c> invalid_import_from_targets[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "import_from_as_names ',' NEWLINE")); Token * _literal; asdl_seq* import_from_as_names_var; + Token * newline_var; if ( (import_from_as_names_var = import_from_as_names_rule(p)) // import_from_as_names && (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ invalid_import_from_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "import_from_as_names ','")); + D(fprintf(stderr, "%*c+ invalid_import_from_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "import_from_as_names ',' NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "trailing comma not allowed without surrounding parentheses" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -15411,7 +15414,7 @@ invalid_import_from_targets_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_import_from_targets[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "import_from_as_names ','")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "import_from_as_names ',' NEWLINE")); } _res = NULL; done: diff --git a/Parser/pegen/parse_string.c b/Parser/pegen/parse_string.c index c852e5b8..41fdc2d8 100644 --- a/Parser/pegen/parse_string.c +++ b/Parser/pegen/parse_string.c @@ -284,49 +284,48 @@ _PyPegen_parsestr(Parser *p, int *bytesmode, int *rawmode, PyObject **result, /* Fix locations for the given node and its children. `parent` is the enclosing node. + `expr_start` is the starting position of the expression (pointing to the open brace). `n` is the node which locations are going to be fixed relative to parent. `expr_str` is the child node's string representation, including braces. */ static bool -fstring_find_expr_location(Token *parent, char *expr_str, int *p_lines, int *p_cols) +fstring_find_expr_location(Token *parent, const char* expr_start, char *expr_str, int *p_lines, int *p_cols) { *p_lines = 0; *p_cols = 0; + assert(expr_start != NULL && *expr_start == '{'); if (parent && parent->bytes) { char *parent_str = PyBytes_AsString(parent->bytes); if (!parent_str) { return false; } - char *substr = strstr(parent_str, expr_str); - if (substr) { - // The following is needed, in order to correctly shift the column - // offset, in the case that (disregarding any whitespace) a newline - // immediately follows the opening curly brace of the fstring expression. - bool newline_after_brace = 1; - char *start = substr + 1; - while (start && *start != '}' && *start != '\n') { - if (*start != ' ' && *start != '\t' && *start != '\f') { - newline_after_brace = 0; - break; - } - start++; + // The following is needed, in order to correctly shift the column + // offset, in the case that (disregarding any whitespace) a newline + // immediately follows the opening curly brace of the fstring expression. + bool newline_after_brace = 1; + const char *start = expr_start + 1; + while (start && *start != '}' && *start != '\n') { + if (*start != ' ' && *start != '\t' && *start != '\f') { + newline_after_brace = 0; + break; } + start++; + } - // Account for the characters from the last newline character to our - // left until the beginning of substr. - if (!newline_after_brace) { - start = substr; - while (start > parent_str && *start != '\n') { - start--; - } - *p_cols += (int)(substr - start); + // Account for the characters from the last newline character to our + // left until the beginning of expr_start. + if (!newline_after_brace) { + start = expr_start; + while (start > parent_str && *start != '\n') { + start--; } - /* adjust the start based on the number of newlines encountered - before the f-string expression */ - for (char* p = parent_str; p < substr; p++) { - if (*p == '\n') { - (*p_lines)++; - } + *p_cols += (int)(expr_start - start); + } + /* adjust the start based on the number of newlines encountered + before the f-string expression */ + for (const char *p = parent_str; p < expr_start; p++) { + if (*p == '\n') { + (*p_lines)++; } } } @@ -370,7 +369,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end, len = expr_end - expr_start; /* Allocate 3 extra bytes: open paren, close paren, null byte. */ - str = PyMem_Malloc(len + 3); + str = PyMem_Calloc(len + 3, sizeof(char)); if (str == NULL) { PyErr_NoMemory(); return NULL; @@ -378,18 +377,11 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end, // The call to fstring_find_expr_location is responsible for finding the column offset // the generated AST nodes need to be shifted to the right, which is equal to the number - // of the f-string characters before the expression starts. In order to correctly compute - // this offset, strstr gets called in fstring_find_expr_location which only succeeds - // if curly braces appear before and after the f-string expression (exactly like they do - // in the f-string itself), hence the following lines. - str[0] = '{'; + // of the f-string characters before the expression starts. memcpy(str+1, expr_start, len); - str[len+1] = '}'; - str[len+2] = 0; - int lines, cols; - if (!fstring_find_expr_location(t, str, &lines, &cols)) { - PyMem_FREE(str); + if (!fstring_find_expr_location(t, expr_start-1, str+1, &lines, &cols)) { + PyMem_Free(str); return NULL; } diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index f3c1d9b2..1a57db97 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -548,7 +548,7 @@ decoding_fgets(char *s, int size, struct tok_state *tok) "Non-UTF-8 code starting with '\\x%.2x' " "in file %U on line %i, " "but no encoding declared; " - "see http://python.org/dev/peps/pep-0263/ for details", + "see https://python.org/dev/peps/pep-0263/ for details", badchar, tok->filename, tok->lineno + 1); return error_ret(tok); } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 614012df..21c70f9a 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -71,6 +71,7 @@ update_bases(PyObject *bases, PyObject *const *args, Py_ssize_t nargs) /* If this is a first successful replacement, create new_bases list and copy previously encountered bases. */ if (!(new_bases = PyList_New(i))) { + Py_DECREF(new_base); goto error; } for (j = 0; j < i; j++) { @@ -81,6 +82,7 @@ update_bases(PyObject *bases, PyObject *const *args, Py_ssize_t nargs) } j = PyList_GET_SIZE(new_bases); if (PyList_SetSlice(new_bases, j, j, new_base) < 0) { + Py_DECREF(new_base); goto error; } Py_DECREF(new_base); @@ -102,8 +104,9 @@ static PyObject * builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *orig_bases; - PyObject *cls = NULL, *cell = NULL; + PyObject *func, *name, *winner, *prep; + PyObject *cls = NULL, *cell = NULL, *ns = NULL, *meta = NULL, *orig_bases = NULL; + PyObject *mkw = NULL, *bases = NULL; int isclass = 0; /* initialize to prevent gcc warning */ if (nargs < 2) { @@ -140,26 +143,20 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, else { mkw = _PyStack_AsDict(args + nargs, kwnames); if (mkw == NULL) { - Py_DECREF(bases); - return NULL; + goto error; } meta = _PyDict_GetItemIdWithError(mkw, &PyId_metaclass); if (meta != NULL) { Py_INCREF(meta); if (_PyDict_DelItemId(mkw, &PyId_metaclass) < 0) { - Py_DECREF(meta); - Py_DECREF(mkw); - Py_DECREF(bases); - return NULL; + goto error; } /* metaclass is explicitly given, check if it's indeed a class */ isclass = PyType_Check(meta); } else if (PyErr_Occurred()) { - Py_DECREF(mkw); - Py_DECREF(bases); - return NULL; + goto error; } } if (meta == NULL) { @@ -182,10 +179,7 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, winner = (PyObject *)_PyType_CalculateMetaclass((PyTypeObject *)meta, bases); if (winner == NULL) { - Py_DECREF(meta); - Py_XDECREF(mkw); - Py_DECREF(bases); - return NULL; + goto error; } if (winner != meta) { Py_DECREF(meta); @@ -207,10 +201,7 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py_DECREF(prep); } if (ns == NULL) { - Py_DECREF(meta); - Py_XDECREF(mkw); - Py_DECREF(bases); - return NULL; + goto error; } if (!PyMapping_Check(ns)) { PyErr_Format(PyExc_TypeError, @@ -251,13 +242,13 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, } error: Py_XDECREF(cell); - Py_DECREF(ns); - Py_DECREF(meta); + Py_XDECREF(ns); + Py_XDECREF(meta); Py_XDECREF(mkw); - Py_DECREF(bases); if (bases != orig_bases) { Py_DECREF(orig_bases); } + Py_DECREF(bases); return cls; } diff --git a/Python/ceval.c b/Python/ceval.c index 91e879e8..86a5d81b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4598,11 +4598,14 @@ static int prtrace(PyThreadState *tstate, PyObject *v, const char *str) { printf("%s ", str); + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); if (PyObject_Print(v, stdout, 0) != 0) { /* Don't know what else to do */ _PyErr_Clear(tstate); } printf("\n"); + PyErr_Restore(type, value, traceback); return 1; } #endif diff --git a/Python/compile.c b/Python/compile.c index 722d52db..f426050c 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2816,6 +2816,12 @@ compiler_async_for(struct compiler *c, stmt_ty s) /* Except block for __anext__ */ compiler_use_next_block(c, except); + + /* We don't want to trace the END_ASYNC_FOR, so make sure + * that it has the same lineno as the following instruction. */ + if (asdl_seq_LEN(s->v.For.orelse)) { + SET_LOC(c, (stmt_ty)asdl_seq_GET(s->v.For.orelse, 0)); + } ADDOP(c, END_ASYNC_FOR); /* `else` block */ diff --git a/Python/errors.c b/Python/errors.c index 87af39d5..8a2ba8f9 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -85,17 +85,29 @@ _PyErr_GetTopmostException(PyThreadState *tstate) } static PyObject* -_PyErr_CreateException(PyObject *exception, PyObject *value) +_PyErr_CreateException(PyObject *exception_type, PyObject *value) { + PyObject *exc; + if (value == NULL || value == Py_None) { - return _PyObject_CallNoArg(exception); + exc = _PyObject_CallNoArg(exception_type); } else if (PyTuple_Check(value)) { - return PyObject_Call(exception, value, NULL); + exc = PyObject_Call(exception_type, value, NULL); } else { - return PyObject_CallOneArg(exception, value); + exc = PyObject_CallOneArg(exception_type, value); + } + + if (exc != NULL && !PyExceptionInstance_Check(exc)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %s", + exception_type, Py_TYPE(exc)->tp_name); + Py_CLEAR(exc); } + + return exc; } void @@ -136,12 +148,16 @@ _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value) value = fixed_value; } - /* Avoid reference cycles through the context chain. + /* Avoid creating new reference cycles through the + context chain, while taking care not to hang on + pre-existing ones. This is O(chain length) but context chains are usually very short. Sensitive readers may try to inline the call to PyException_GetContext. */ if (exc_value != value) { PyObject *o = exc_value, *context; + PyObject *slow_o = o; /* Floyd's cycle detection algo */ + int slow_update_toggle = 0; while ((context = PyException_GetContext(o))) { Py_DECREF(context); if (context == value) { @@ -149,6 +165,16 @@ _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value) break; } o = context; + if (o == slow_o) { + /* pre-existing cycle - all exceptions on the + path were visited and checked. */ + break; + } + if (slow_update_toggle) { + slow_o = PyException_GetContext(slow_o); + Py_DECREF(slow_o); + } + slow_update_toggle = !slow_update_toggle; } PyException_SetContext(value, exc_value); } diff --git a/Python/fileutils.c b/Python/fileutils.c index 45ea2043..11c659d5 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1314,10 +1314,11 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) return 0; } -#ifdef __linux__ +#ifdef O_PATH if (errno == EBADF) { - // On Linux, ioctl(FIOCLEX) will fail with EBADF for O_PATH file descriptors - // Fall through to the fcntl() path + // bpo-44849: On Linux and FreeBSD, ioctl(FIOCLEX) fails with EBADF + // on O_PATH file descriptors. Fall through to the fcntl() + // implementation. } else #endif diff --git a/Python/marshal.c b/Python/marshal.c index c4538bd3..baafa3ec 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -596,14 +596,18 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) { char buf[BUFSIZ]; WFILE wf; + if (PySys_Audit("marshal.dumps", "Oi", x, version) < 0) { + return; /* caller must check PyErr_Occurred() */ + } memset(&wf, 0, sizeof(wf)); wf.fp = fp; wf.ptr = wf.buf = buf; wf.end = wf.ptr + sizeof(buf); wf.error = WFERR_OK; wf.version = version; - if (w_init_refs(&wf, version)) - return; /* caller mush check PyErr_Occurred() */ + if (w_init_refs(&wf, version)) { + return; /* caller must check PyErr_Occurred() */ + } w_object(x, &wf); w_clear_refs(&wf); w_flush(&wf); @@ -1371,12 +1375,6 @@ r_object(RFILE *p) if (lnotab == NULL) goto code_error; - if (PySys_Audit("code.__new__", "OOOiiiiii", - code, filename, name, argcount, posonlyargcount, - kwonlyargcount, nlocals, stacksize, flags) < 0) { - goto code_error; - } - v = (PyObject *) PyCode_NewWithPosOnlyArgs( argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, @@ -1435,6 +1433,15 @@ read_object(RFILE *p) fprintf(stderr, "XXX readobject called with exception set\n"); return NULL; } + if (p->ptr && p->end) { + if (PySys_Audit("marshal.loads", "y#", p->ptr, (Py_ssize_t)(p->end - p->ptr)) < 0) { + return NULL; + } + } else if (p->fp || p->readable) { + if (PySys_Audit("marshal.load", NULL) < 0) { + return NULL; + } + } v = r_object(p); if (v == NULL && !PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object"); @@ -1531,7 +1538,7 @@ PyMarshal_ReadObjectFromFile(FILE *fp) rf.refs = PyList_New(0); if (rf.refs == NULL) return NULL; - result = r_object(&rf); + result = read_object(&rf); Py_DECREF(rf.refs); if (rf.buf != NULL) PyMem_FREE(rf.buf); @@ -1552,7 +1559,7 @@ PyMarshal_ReadObjectFromString(const char *str, Py_ssize_t len) rf.refs = PyList_New(0); if (rf.refs == NULL) return NULL; - result = r_object(&rf); + result = read_object(&rf); Py_DECREF(rf.refs); if (rf.buf != NULL) PyMem_FREE(rf.buf); @@ -1564,6 +1571,9 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) { WFILE wf; + if (PySys_Audit("marshal.dumps", "Oi", x, version) < 0) { + return NULL; + } memset(&wf, 0, sizeof(wf)); wf.str = PyBytes_FromStringAndSize((char *)NULL, 50); if (wf.str == NULL) diff --git a/Python/traceback.c b/Python/traceback.c index a9a9dd92..83f3074f 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -799,7 +799,10 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) PUTS(fd, "Stack (most recent call first):\n"); } - frame = PyThreadState_GetFrame(tstate); + // Use a borrowed reference. Avoid Py_INCREF/Py_DECREF, since this function + // can be called in a signal handler by the faulthandler module which must + // not modify Python objects. + frame = tstate->frame; if (frame == NULL) { PUTS(fd, "\n"); return; @@ -808,17 +811,14 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) depth = 0; while (1) { if (MAX_FRAME_DEPTH <= depth) { - Py_DECREF(frame); PUTS(fd, " ...\n"); break; } if (!PyFrame_Check(frame)) { - Py_DECREF(frame); break; } dump_frame(fd, frame); - PyFrameObject *back = PyFrame_GetBack(frame); - Py_DECREF(frame); + PyFrameObject *back = frame->f_back; if (back == NULL) { break; diff --git a/README.rst b/README.rst index cb5bbd27..a419856d 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.9.6 +This is Python version 3.9.7 ============================ .. image:: https://travis-ci.org/python/cpython.svg?branch=3.9 @@ -233,12 +233,13 @@ Proposals for enhancement ------------------------- If you have a proposal to change Python, you may want to send an email to the -comp.lang.python or `python-ideas`_ mailing lists for initial feedback. A +`comp.lang.python`_ or `python-ideas`_ mailing lists for initial feedback. A Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. All current PEPs, as well as guidelines for submitting a new PEP, are listed at `python.org/dev/peps/ `_. .. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ +.. _comp.lang.python: https://mail.python.org/mailman/listinfo/python-list Release Schedule diff --git a/Tools/freeze/README b/Tools/freeze/README index 5bc5b049..098107c5 100644 --- a/Tools/freeze/README +++ b/Tools/freeze/README @@ -293,4 +293,4 @@ module ...: Additional Python modules (referenced by pathname) ---Guido van Rossum (home page: http://www.python.org/~guido/) +--Guido van Rossum (home page: https://www.python.org/~guido/) diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index b1d281d7..b29b0d8e 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -52,8 +52,8 @@ Python of course has no preprocessor so this doesn't work so well. Thus, pygettext searches only for _() by default, but see the -k/--keyword flag below for how to augment this. - [1] http://www.python.org/workshops/1997-10/proceedings/loewis.html - [2] http://www.gnu.org/software/gettext/gettext.html + [1] https://www.python.org/workshops/1997-10/proceedings/loewis.html + [2] https://www.gnu.org/software/gettext/gettext.html NOTE: pygettext attempts to be option and feature compatible with GNU xgettext where ever possible. However some options are still missing or are diff --git a/Tools/msi/README.txt b/Tools/msi/README.txt index 82635f3d..c85ee456 100644 --- a/Tools/msi/README.txt +++ b/Tools/msi/README.txt @@ -167,7 +167,7 @@ The following properties may be passed when building these projects. by providing a unique URI for this property. It does not need to be an active internet address. Defaults to $(ComputerName). - Official releases use http://www.python.org/(architecture name) + Official releases use https://www.python.org/(architecture name) /p:DownloadUrlBase=(any URI) Specifies the base of a URL where missing parts of the installer layout diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index b7baf21e..1c8670eb 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -12,7 +12,7 @@ rem rem The following substitutions will be applied to the release URI: rem Variable Description Example rem {arch} architecture amd64, win32 -set RELEASE_URI=http://www.python.org/{arch} +set RELEASE_URI=https://www.python.org/{arch} rem This is the URL that will be used to download installation files. rem The files available from the default URL *will* conflict with your diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs index ddd6870f..e2f87188 100644 --- a/Tools/msi/bundle/bundle.wxs +++ b/Tools/msi/bundle/bundle.wxs @@ -7,7 +7,7 @@ Version="$(var.Version)" IconSourceFile="..\..\..\PC\icons\setup.ico" Manufacturer="!(loc.Manufacturer)" - AboutUrl="http://www.python.org/" + AboutUrl="https://www.python.org/" Compressed="no" dep:ProviderKey="CPython-$(var.MajorVersionNumber).$(var.MinorVersionNumber)$(var.PyArchExt)$(var.PyTestExt)"> diff --git a/Tools/msi/common_en-US.wxl_template b/Tools/msi/common_en-US.wxl_template index c95c271c..adb8f40b 100644 --- a/Tools/msi/common_en-US.wxl_template +++ b/Tools/msi/common_en-US.wxl_template @@ -14,5 +14,5 @@ A newer version of !(loc.ProductName) is already installed. An incorrect version of a prerequisite package is installed. Please uninstall any other versions of !(loc.ProductName) and try installing this again. The TARGETDIR variable must be provided when invoking this installer. - http://www.python.org/ + https://www.python.org/ diff --git a/Tools/msi/exe/exe_en-US.wxl_template b/Tools/msi/exe/exe_en-US.wxl_template index 1f9e2903..4cd9e3e6 100644 --- a/Tools/msi/exe/exe_en-US.wxl_template +++ b/Tools/msi/exe/exe_en-US.wxl_template @@ -4,5 +4,5 @@ executable Python {{ShortVersion}} ({{Bitness}}) Launches the !(loc.ProductName) interpreter. - http://www.python.org/ + https://www.python.org/ diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props index f88765f2..4fe915cb 100644 --- a/Tools/msi/msi.props +++ b/Tools/msi/msi.props @@ -28,7 +28,7 @@ that intend to bundle Python should rebuild these modules with their own URI to avoid conflicting with the official releases. - The official releases use "http://www.python.org/$(ArchName)" + The official releases use "https://www.python.org/$(ArchName)" This is not the same as the DownloadUrl property used in the bundle projects. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 1904f3bf..6baec7bc 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -49,8 +49,8 @@ OPENSSL_OLD_VERSIONS = [ ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1k", - # "3.0.0-alpha14" + "1.1.1l", + "3.0.0-beta1" ] LIBRESSL_OLD_VERSIONS = [ @@ -147,23 +147,6 @@ parser.add_argument( help="Keep original sources for debugging." ) -OPENSSL_FIPS_CNF = """\ -openssl_conf = openssl_init - -.include {self.install_dir}/ssl/fipsinstall.cnf -# .include {self.install_dir}/ssl/openssl.cnf - -[openssl_init] -providers = provider_sect - -[provider_sect] -fips = fips_sect -default = default_sect - -[default_sect] -activate = 1 -""" - class AbstractBuilder(object): library = None @@ -306,12 +289,12 @@ class AbstractBuilder(object): log.info("Unpacking files to {}".format(self.build_dir)) tf.extractall(self.build_dir, members) - def _build_src(self): + def _build_src(self, config_args=()): """Now build openssl""" log.info("Running build in {}".format(self.build_dir)) cwd = self.build_dir cmd = [ - "./config", + "./config", *config_args, "shared", "--debug", "--prefix={}".format(self.install_dir) ] @@ -415,35 +398,19 @@ class BuildOpenSSL(AbstractBuilder): if self.version.startswith("3.0"): self._post_install_300() + def _build_src(self, config_args=()): + if self.version.startswith("3.0"): + config_args += ("enable-fips",) + super()._build_src(config_args) + def _post_install_300(self): # create ssl/ subdir with example configs - self._subprocess_call( - ["make", "-j1", "install_ssldirs"], - cwd=self.build_dir - ) # Install FIPS module - # https://wiki.openssl.org/index.php/OpenSSL_3.0#Completing_the_installation_of_the_FIPS_Module - fipsinstall_cnf = os.path.join( - self.install_dir, "ssl", "fipsinstall.cnf" - ) - openssl_fips_cnf = os.path.join( - self.install_dir, "ssl", "openssl-fips.cnf" - ) - fips_mod = os.path.join(self.lib_dir, "ossl-modules/fips.so") self._subprocess_call( - [ - self.openssl_cli, "fipsinstall", - "-out", fipsinstall_cnf, - "-module", fips_mod, - # "-provider_name", "fips", - # "-mac_name", "HMAC", - # "-macopt", "digest:SHA256", - # "-macopt", "hexkey:00", - # "-section_name", "fips_sect" - ] + ["make", "-j1", "install_ssldirs", "install_fips"], + cwd=self.build_dir ) - with open(openssl_fips_cnf, "w") as f: - f.write(OPENSSL_FIPS_CNF.format(self=self)) + @property def short_version(self): """Short version for OpenSSL download URL""" diff --git a/setup.py b/setup.py index 04eb6b29..54d4b942 100644 --- a/setup.py +++ b/setup.py @@ -540,6 +540,9 @@ class PyBuildExt(build_ext): "APIs, https://github.com/libressl-portable/portable/issues/381") print() + if os.environ.get("PYTHONSTRICTEXTENSIONBUILD") and (self.failed or self.failed_on_import): + raise RuntimeError("Failed to build some stdlib modules") + def build_extension(self, ext): if ext.name == '_ctypes': @@ -950,7 +953,7 @@ class PyBuildExt(build_ext): # Python PEP-3118 (buffer protocol) test module self.add(Extension('_testbuffer', ['_testbuffer.c'])) - # Test loading multiple modules from one compiled file (http://bugs.python.org/issue16421) + # Test loading multiple modules from one compiled file (https://bugs.python.org/issue16421) self.add(Extension('_testimportmultiple', ['_testimportmultiple.c'])) # Test multi-phase extension module init (PEP 489) @@ -1150,7 +1153,7 @@ class PyBuildExt(build_ext): # similar functionality (but slower of course) implemented in Python. # Sleepycat^WOracle Berkeley DB interface. - # http://www.oracle.com/database/berkeley-db/db/index.html + # https://www.oracle.com/database/technologies/related/berkeleydb.html # # This requires the Sleepycat^WOracle DB code. The supported versions # are set below. Visit the URL above to download @@ -1192,7 +1195,7 @@ class PyBuildExt(build_ext): '/usr/include/db3', '/usr/local/include/db3', '/opt/sfw/include/db3', - # Fink defaults (http://fink.sourceforge.net/) + # Fink defaults (https://www.finkproject.org/) '/sw/include/db4', '/sw/include/db3', ] @@ -1204,7 +1207,7 @@ class PyBuildExt(build_ext): db_inc_paths.append('/usr/local/include/db4%d' % x) db_inc_paths.append('/pkg/db-4.%d/include' % x) db_inc_paths.append('/opt/db-4.%d/include' % x) - # MacPorts default (http://www.macports.org/) + # MacPorts default (https://www.macports.org/) db_inc_paths.append('/opt/local/include/db4%d' % x) # 3.x minor number specific paths for x in gen_db_minor_ver_nums(3): @@ -1521,6 +1524,8 @@ class PyBuildExt(build_ext): # if --enable-loadable-sqlite-extensions configure option is used. if '--enable-loadable-sqlite-extensions' not in sysconfig.get_config_var("CONFIG_ARGS"): sqlite_defines.append(("SQLITE_OMIT_LOAD_EXTENSION", "1")) + elif MACOS and sqlite_incdir == os.path.join(MACOS_SDK_ROOT, "usr/include"): + raise DistutilsError("System version of SQLite does not support loadable extensions") if MACOS: # In every directory on the search path search for a dynamic @@ -2287,12 +2292,12 @@ class PyBuildExt(build_ext): # Workarounds for toolchain bugs: if sysconfig.get_config_var('HAVE_IPA_PURE_CONST_BUG'): # Some versions of gcc miscompile inline asm: - # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=46491 - # http://gcc.gnu.org/ml/gcc/2010-11/msg00366.html + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=46491 + # https://gcc.gnu.org/ml/gcc/2010-11/msg00366.html extra_compile_args.append('-fno-ipa-pure-const') if sysconfig.get_config_var('HAVE_GLIBC_MEMMOVE_BUG'): # _FORTIFY_SOURCE wrappers for memmove and bcopy are incorrect: - # http://sourceware.org/ml/libc-alpha/2010-12/msg00009.html + # https://sourceware.org/ml/libc-alpha/2010-12/msg00009.html undef_macros.append('_FORTIFY_SOURCE') # Uncomment for extra functionality: @@ -2567,7 +2572,7 @@ def main(): setup(# PyPI Metadata (PEP 301) name = "Python", version = sys.version.split()[0], - url = "http://www.python.org/%d.%d" % sys.version_info[:2], + url = "https://www.python.org/%d.%d" % sys.version_info[:2], maintainer = "Guido van Rossum and the Python community", maintainer_email = "python-dev@python.org", description = "A high-level object-oriented programming language",