From: Hyunjee Kim Date: Wed, 20 Mar 2019 01:52:04 +0000 (+0900) Subject: Imported Upstream version 3.6.4 X-Git-Tag: upstream/3.7.3~8 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f1e333db2af3f5811f50a1d6490414935401a151;p=platform%2Fupstream%2Fpython3.git Imported Upstream version 3.6.4 --- diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 3d851b7c..8c2de969 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -473,7 +473,7 @@ Buffer-related functions (*order* is ``'A'``). Return ``0`` otherwise. -.. c:function:: void PyBuffer_FillContiguousStrides(int ndim, Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t itemsize, char order) +.. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order) Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if *order* is ``'C'`` or Fortran-style if *order* is ``'F'``) array of the diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 39542bd1..305e9903 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -188,7 +188,7 @@ not be *NULL*, and the type is not checked: .. versionadded:: 3.3 -.. c:function:: int PyDateTime_DELTA_GET_MICROSECOND(PyDateTime_Delta *o) +.. c:function:: int PyDateTime_DELTA_GET_MICROSECONDS(PyDateTime_Delta *o) Return the number of microseconds, as an int from 0 through 999999. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 873fb2ac..73bec7c9 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -150,7 +150,7 @@ The default raw memory block allocator uses the following functions: Frees the memory block pointed to by *p*, which must have been returned by a previous call to :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or - :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_Free(p)`` has been + :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_RawFree(p)`` has been called before, undefined behavior occurs. If *p* is *NULL*, no operation is performed. @@ -263,6 +263,69 @@ versions and is therefore deprecated in extension modules. * ``PyMem_DEL(ptr)`` +Object allocators +================= + +The following function sets, modeled after the ANSI C standard, but specifying +behavior when requesting zero bytes, are available for allocating and releasing +memory from the Python heap. + +By default, these functions use :ref:`pymalloc memory allocator `. + +.. warning:: + + The :term:`GIL ` must be held when using these + functions. + +.. c:function:: void* PyObject_Malloc(size_t n) + + Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the + allocated memory, or *NULL* if the request fails. + + Requesting zero bytes returns a distinct non-*NULL* pointer if possible, as + if ``PyObject_Malloc(1)`` had been called instead. The memory will not have + been initialized in any way. + + +.. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) + + Allocates *nelem* elements each whose size in bytes is *elsize* and returns + a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the + request fails. The memory is initialized to zeros. + + Requesting zero elements or elements of size zero bytes returns a distinct + non-*NULL* pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called + instead. + + .. versionadded:: 3.5 + + +.. c:function:: void* PyObject_Realloc(void *p, size_t n) + + Resizes the memory block pointed to by *p* to *n* bytes. The contents will be + unchanged to the minimum of the old and the new sizes. + + If *p* is *NULL*, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* + is equal to zero, the memory block is resized but is not freed, and the + returned pointer is non-*NULL*. + + Unless *p* is *NULL*, it must have been returned by a previous call to + :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or :c:func:`PyObject_Calloc`. + + If the request fails, :c:func:`PyObject_Realloc` returns *NULL* and *p* remains + a valid pointer to the previous memory area. + + +.. c:function:: void PyObject_Free(void *p) + + Frees the memory block pointed to by *p*, which must have been returned by a + previous call to :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or + :c:func:`PyObject_Calloc`. Otherwise, or if ``PyObject_Free(p)`` has been called + before, undefined behavior occurs. + + If *p* is *NULL*, no operation is performed. + + Customize Memory Allocators =========================== diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 6ab59429..3897fdd8 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -141,7 +141,8 @@ the same library that the Python runtime is using. Read and execute statements from a file associated with an interactive device until EOF is reached. The user will be prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF. + (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF or a negative + number upon failure. .. c:var:: int (*PyOS_InputHook)(void) diff --git a/Doc/conf.py b/Doc/conf.py index d4ee50de..43826ec0 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -37,7 +37,8 @@ highlight_language = 'python3' needs_sphinx = '1.2' # Ignore any .rst files in the venv/ directory. -exclude_patterns = ['venv/*', 'README.rst'] +venvdir = os.getenv('VENVDIR', 'venv') +exclude_patterns = [venvdir+'/*', 'README.rst'] # Options for HTML output @@ -88,11 +89,10 @@ html_split_index = True # Options for LaTeX output # ------------------------ +latex_engine = 'xelatex' + # Get LaTeX to handle Unicode correctly latex_elements = { - 'inputenc': r'\usepackage[utf8x]{inputenc}', - 'utf8extra': '', - 'fontenc': r'\usepackage[T1,T2A]{fontenc}', } # Additional stuff for the LaTeX preamble. diff --git a/Doc/docutils.conf b/Doc/docutils.conf new file mode 100644 index 00000000..bda4f5dc --- /dev/null +++ b/Doc/docutils.conf @@ -0,0 +1,2 @@ +[restructuredtext parser] +smartquotes-locales: ja: ""'' diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 93584627..a517e130 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -40,7 +40,7 @@ A Simple Example Let's create an extension module called ``spam`` (the favorite food of Monty Python fans...) and let's say we want to create a Python interface to the C -library function :c:func:`system`. [#]_ This function takes a null-terminated +library function :c:func:`system` [#]_. This function takes a null-terminated character string as argument and returns an integer. We want this function to be callable from Python as follows:: @@ -917,7 +917,7 @@ It is also possible to :dfn:`borrow` [#]_ a reference to an object. The borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must not hold on to the object longer than the owner from which it was borrowed. Using a borrowed reference after the owner has disposed of it risks using freed -memory and should be avoided completely. [#]_ +memory and should be avoided completely [#]_. The advantage of borrowing over owning a reference is that you don't need to take care of disposing of the reference on all possible paths through the code @@ -1088,7 +1088,7 @@ checking. The C function calling mechanism guarantees that the argument list passed to C functions (``args`` in the examples) is never *NULL* --- in fact it guarantees -that it is always a tuple. [#]_ +that it is always a tuple [#]_. It is a severe error to ever let a *NULL* pointer "escape" to the Python user. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index abd5da9d..d28d224c 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -666,7 +666,7 @@ Fortunately, Python's cyclic-garbage collector will eventually figure out that the list is garbage and free it. In the second version of the :class:`Noddy` example, we allowed any kind of -object to be stored in the :attr:`first` or :attr:`last` attributes. [#]_ This +object to be stored in the :attr:`first` or :attr:`last` attributes [#]_. This means that :class:`Noddy` objects can participate in cycles:: >>> import noddy2 diff --git a/Doc/glossary.rst b/Doc/glossary.rst index dba9186d..b947520b 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -535,7 +535,10 @@ Glossary iterables include all sequence types (such as :class:`list`, :class:`str`, and :class:`tuple`) and some non-sequence types like :class:`dict`, :term:`file objects `, and objects of any classes you define - with an :meth:`__iter__` or :meth:`__getitem__` method. Iterables can be + with an :meth:`__iter__` method or with a :meth:`__getitem__` method + that implements :term:`Sequence` semantics. + + Iterables can be used in a :keyword:`for` loop and in many other places where a sequence is needed (:func:`zip`, :func:`map`, ...). When an iterable object is passed as an argument to the built-in function :func:`iter`, it returns an diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index b3493758..5e85a9aa 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -180,7 +180,7 @@ descriptor is useful for monitoring just a few chosen attributes:: The protocol is simple and offers exciting possibilities. Several use cases are so common that they have been packaged into individual function calls. -Properties, bound and unbound methods, static methods, and class methods are all +Properties, bound methods, static methods, and class methods are all based on the descriptor protocol. @@ -266,22 +266,23 @@ Python's object oriented features are built upon a function based environment. Using non-data descriptors, the two are merged seamlessly. Class dictionaries store methods as functions. In a class definition, methods -are written using :keyword:`def` and :keyword:`lambda`, the usual tools for -creating functions. The only difference from regular functions is that the +are written using :keyword:`def` or :keyword:`lambda`, the usual tools for +creating functions. Methods only differ from regular functions in that the first argument is reserved for the object instance. By Python convention, the instance reference is called *self* but may be called *this* or any other variable name. To support method calls, functions include the :meth:`__get__` method for binding methods during attribute access. This means that all functions are -non-data descriptors which return bound or unbound methods depending whether -they are invoked from an object or a class. In pure python, it works like -this:: +non-data descriptors which return bound methods when they are invoked from an +object. In pure python, it works like this:: class Function(object): . . . def __get__(self, obj, objtype=None): "Simulate func_descr_get() in Objects/funcobject.c" + if obj is None: + return self return types.MethodType(self, obj) Running the interpreter shows how the function descriptor works in practice:: @@ -291,25 +292,34 @@ Running the interpreter shows how the function descriptor works in practice:: ... return x ... >>> d = D() - >>> D.__dict__['f'] # Stored internally as a function - - >>> D.f # Get from a class becomes an unbound method - - >>> d.f # Get from an instance becomes a bound method + + # Access through the class dictionary does not invoke __get__. + # It just returns the underlying function object. + >>> D.__dict__['f'] + + + # Dotted access from a class calls __get__() which just returns + # the underlying function unchanged. + >>> D.f + + + # The function has a __qualname__ attribute to support introspection + >>> D.f.__qualname__ + 'D.f' + + # Dotted access from an instance calls __get__() which returns the + # function wrapped in a bound method object + >>> d.f > -The output suggests that bound and unbound methods are two different types. -While they could have been implemented that way, the actual C implementation of -:c:type:`PyMethod_Type` in :source:`Objects/classobject.c` is a single object -with two different representations depending on whether the :attr:`im_self` -field is set or is *NULL* (the C equivalent of ``None``). - -Likewise, the effects of calling a method object depend on the :attr:`im_self` -field. If set (meaning bound), the original function (stored in the -:attr:`im_func` field) is called as expected with the first argument set to the -instance. If unbound, all of the arguments are passed unchanged to the original -function. The actual C implementation of :func:`instancemethod_call()` is only -slightly more complex in that it includes some type checking. + # Internally, the bound method stores the underlying function, + # the bound instance, and the class of the bound instance. + >>> d.f.__func__ + + >>> d.f.__self__ + <__main__.D object at 0x1012e1f98> + >>> d.f.__class__ + Static Methods and Class Methods diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index d9b7c909..eef63478 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -153,8 +153,8 @@ These sequences can be included inside a character class. For example, ``','`` or ``'.'``. The final metacharacter in this section is ``.``. It matches anything except a -newline character, and there's an alternate mode (``re.DOTALL``) where it will -match even a newline. ``'.'`` is often used where you want to match "any +newline character, and there's an alternate mode (:const:`re.DOTALL`) where it will +match even a newline. ``.`` is often used where you want to match "any character". @@ -168,14 +168,11 @@ wouldn't be much of an advance. Another capability is that you can specify that portions of the RE must be repeated a certain number of times. The first metacharacter for repeating things that we'll look at is ``*``. ``*`` -doesn't match the literal character ``*``; instead, it specifies that the +doesn't match the literal character ``'*'``; instead, it specifies that the previous character can be matched zero or more times, instead of exactly once. -For example, ``ca*t`` will match ``ct`` (0 ``a`` characters), ``cat`` (1 ``a``), -``caaat`` (3 ``a`` characters), and so forth. The RE engine has various -internal limitations stemming from the size of C's ``int`` type that will -prevent it from matching over 2 billion ``a`` characters; patterns -are usually not written to match that much data. +For example, ``ca*t`` will match ``'ct'`` (0 ``'a'`` characters), ``'cat'`` (1 ``'a'``), +``'caaat'`` (3 ``'a'`` characters), and so forth. Repetitions such as ``*`` are :dfn:`greedy`; when repeating a RE, the matching engine will try to repeat it as many times as possible. If later portions of the @@ -185,7 +182,7 @@ fewer repetitions. A step-by-step example will make this more obvious. Let's consider the expression ``a[bcd]*b``. This matches the letter ``'a'``, zero or more letters from the class ``[bcd]``, and finally ends with a ``'b'``. Now imagine matching -this RE against the string ``abcbd``. +this RE against the string ``'abcbd'``. +------+-----------+---------------------------------+ | Step | Matched | Explanation | @@ -218,7 +215,7 @@ this RE against the string ``abcbd``. | | | it succeeds. | +------+-----------+---------------------------------+ -The end of the RE has now been reached, and it has matched ``abcb``. This +The end of the RE has now been reached, and it has matched ``'abcb'``. This demonstrates how the matching engine goes as far as it can at first, and if no match is found it will then progressively back up and retry the rest of the RE again and again. It will back up until it has tried zero matches for @@ -229,24 +226,23 @@ Another repeating metacharacter is ``+``, which matches one or more times. Pay careful attention to the difference between ``*`` and ``+``; ``*`` matches *zero* or more times, so whatever's being repeated may not be present at all, while ``+`` requires at least *one* occurrence. To use a similar example, -``ca+t`` will match ``cat`` (1 ``a``), ``caaat`` (3 ``a``'s), but won't match -``ct``. +``ca+t`` will match ``'cat'`` (1 ``'a'``), ``'caaat'`` (3 ``'a'``\ s), but won't +match ``'ct'``. There are two more repeating qualifiers. The question mark character, ``?``, matches either once or zero times; you can think of it as marking something as -being optional. For example, ``home-?brew`` matches either ``homebrew`` or -``home-brew``. +being optional. For example, ``home-?brew`` matches either ``'homebrew'`` or +``'home-brew'``. The most complicated repeated qualifier is ``{m,n}``, where *m* and *n* are decimal integers. This qualifier means there must be at least *m* repetitions, -and at most *n*. For example, ``a/{1,3}b`` will match ``a/b``, ``a//b``, and -``a///b``. It won't match ``ab``, which has no slashes, or ``a////b``, which +and at most *n*. For example, ``a/{1,3}b`` will match ``'a/b'``, ``'a//b'``, and +``'a///b'``. It won't match ``'ab'``, which has no slashes, or ``'a////b'``, which has four. You can omit either *m* or *n*; in that case, a reasonable value is assumed for the missing value. Omitting *m* is interpreted as a lower limit of 0, while -omitting *n* results in an upper bound of infinity --- actually, the upper bound -is the 2-billion limit mentioned earlier, but that might as well be infinity. +omitting *n* results in an upper bound of infinity. Readers of a reductionist bent may notice that the three other qualifiers can all be expressed using this notation. ``{0,}`` is the same as ``*``, ``{1,}`` @@ -366,7 +362,7 @@ for a complete listing. | | returns them as an :term:`iterator`. | +------------------+-----------------------------------------------+ -:meth:`~re.regex.match` and :meth:`~re.regex.search` return ``None`` if no match can be found. If +:meth:`~re.pattern.match` and :meth:`~re.pattern.search` return ``None`` if no match can be found. If they're successful, a :ref:`match object ` instance is returned, containing information about the match: where it starts and ends, the substring it matched, and more. @@ -388,24 +384,24 @@ Python interpreter, import the :mod:`re` module, and compile a RE:: Now, you can try matching various strings against the RE ``[a-z]+``. An empty string shouldn't match at all, since ``+`` means 'one or more repetitions'. -:meth:`match` should return ``None`` in this case, which will cause the +:meth:`~re.pattern.match` should return ``None`` in this case, which will cause the interpreter to print no output. You can explicitly print the result of -:meth:`match` to make this clear. :: +:meth:`!match` to make this clear. :: >>> p.match("") >>> print(p.match("")) None Now, let's try it on a string that it should match, such as ``tempo``. In this -case, :meth:`match` will return a :ref:`match object `, so you +case, :meth:`~re.pattern.match` will return a :ref:`match object `, so you should store the result in a variable for later use. :: >>> m = p.match('tempo') - >>> m #doctest: +ELLIPSIS + >>> m <_sre.SRE_Match object; span=(0, 5), match='tempo'> Now you can query the :ref:`match object ` for information -about the matching string. :ref:`match object ` instances +about the matching string. Match object instances also have several methods and attributes; the most important ones are: +------------------+--------------------------------------------+ @@ -432,15 +428,15 @@ Trying these methods will soon clarify their meaning:: :meth:`~re.match.group` returns the substring that was matched by the RE. :meth:`~re.match.start` and :meth:`~re.match.end` return the starting and ending index of the match. :meth:`~re.match.span` -returns both start and end indexes in a single tuple. Since the :meth:`match` -method only checks if the RE matches at the start of a string, :meth:`start` -will always be zero. However, the :meth:`search` method of patterns +returns both start and end indexes in a single tuple. Since the :meth:`~re.pattern.match` +method only checks if the RE matches at the start of a string, :meth:`!start` +will always be zero. However, the :meth:`~re.pattern.search` method of patterns scans through the string, so the match may not start at zero in that case. :: >>> print(p.match('::: message')) None - >>> m = p.search('::: message'); print(m) #doctest: +ELLIPSIS + >>> m = p.search('::: message'); print(m) <_sre.SRE_Match object; span=(4, 11), match='message'> >>> m.group() 'message' @@ -459,14 +455,14 @@ In actual programs, the most common style is to store the print('No match') Two pattern methods return all of the matches for a pattern. -:meth:`~re.regex.findall` returns a list of matching strings:: +:meth:`~re.pattern.findall` returns a list of matching strings:: >>> p = re.compile('\d+') >>> p.findall('12 drummers drumming, 11 pipers piping, 10 lords a-leaping') ['12', '11', '10'] -:meth:`findall` has to create the entire list before it can be returned as the -result. The :meth:`~re.regex.finditer` method returns a sequence of +:meth:`~re.pattern.findall` has to create the entire list before it can be returned as the +result. The :meth:`~re.pattern.finditer` method returns a sequence of :ref:`match object ` instances as an :term:`iterator`:: >>> iterator = p.finditer('12 drummers drumming, 11 ... 10 ...') @@ -529,14 +525,14 @@ of each one. | | characters with the respective property. | +---------------------------------+--------------------------------------------+ | :const:`DOTALL`, :const:`S` | Make ``.`` match any character, including | -| | newlines | +| | newlines. | +---------------------------------+--------------------------------------------+ -| :const:`IGNORECASE`, :const:`I` | Do case-insensitive matches | +| :const:`IGNORECASE`, :const:`I` | Do case-insensitive matches. | +---------------------------------+--------------------------------------------+ -| :const:`LOCALE`, :const:`L` | Do a locale-aware match | +| :const:`LOCALE`, :const:`L` | Do a locale-aware match. | +---------------------------------+--------------------------------------------+ | :const:`MULTILINE`, :const:`M` | Multi-line matching, affecting ``^`` and | -| | ``$`` | +| | ``$``. | +---------------------------------+--------------------------------------------+ | :const:`VERBOSE`, :const:`X` | Enable verbose REs, which can be organized | | (for 'extended') | more cleanly and understandably. | @@ -549,27 +545,41 @@ of each one. Perform case-insensitive matching; character class and literal strings will match letters by ignoring case. For example, ``[A-Z]`` will match lowercase - letters, too, and ``Spam`` will match ``Spam``, ``spam``, or ``spAM``. This - lowercasing doesn't take the current locale into account; it will if you also - set the :const:`LOCALE` flag. + letters, too. Full Unicode matching also works unless the :const:`ASCII` + flag is used to disable non-ASCII matches. When the Unicode patterns + ``[a-z]`` or ``[A-Z]`` are used in combination with the :const:`IGNORECASE` + flag, they will match the 52 ASCII letters and 4 additional non-ASCII + letters: 'İ' (U+0130, Latin capital letter I with dot above), 'ı' (U+0131, + Latin small letter dotless i), 'ſ' (U+017F, Latin small letter long s) and + 'K' (U+212A, Kelvin sign). ``Spam`` will match ``'Spam'``, ``'spam'``, + ``'spAM'``, or ``'ſpam'`` (the latter is matched only in Unicode mode). + This lowercasing doesn't take the current locale into account; + it will if you also set the :const:`LOCALE` flag. .. data:: L LOCALE :noindex: - Make ``\w``, ``\W``, ``\b``, and ``\B``, dependent on the current locale - instead of the Unicode database. - - Locales are a feature of the C library intended to help in writing programs that - take account of language differences. For example, if you're processing French - text, you'd want to be able to write ``\w+`` to match words, but ``\w`` only - matches the character class ``[A-Za-z]``; it won't match ``'é'`` or ``'ç'``. If - your system is configured properly and a French locale is selected, certain C - functions will tell the program that ``'é'`` should also be considered a letter. + Make ``\w``, ``\W``, ``\b``, ``\B`` and case-insensitive matching dependent + on the current locale instead of the Unicode database. + + Locales are a feature of the C library intended to help in writing programs + that take account of language differences. For example, if you're + processing encoded French text, you'd want to be able to write ``\w+`` to + match words, but ``\w`` only matches the character class ``[A-Za-z]`` in + bytes patterns; it won't match bytes corresponding to ``é`` or ``ç``. + If your system is configured properly and a French locale is selected, + certain C functions will tell the program that the byte corresponding to + ``é`` should also be considered a letter. Setting the :const:`LOCALE` flag when compiling a regular expression will cause the resulting compiled object to use these C functions for ``\w``; this is slower, but also enables ``\w+`` to match French words as you'd expect. + The use of this flag is discouraged in Python 3 as the locale mechanism + is very unreliable, it only handles one "culture" at a time, and it only + works with 8-bit locales. Unicode matching is already enabled by default + in Python 3 for Unicode (str) patterns, and it is able to handle different + locales/languages. .. data:: M @@ -667,11 +677,11 @@ zero-width assertions should never be repeated, because if they match once at a given location, they can obviously be matched an infinite number of times. ``|`` - Alternation, or the "or" operator. If A and B are regular expressions, - ``A|B`` will match any string that matches either ``A`` or ``B``. ``|`` has very + Alternation, or the "or" operator. If *A* and *B* are regular expressions, + ``A|B`` will match any string that matches either *A* or *B*. ``|`` has very low precedence in order to make it work reasonably when you're alternating - multi-character strings. ``Crow|Servo`` will match either ``Crow`` or ``Servo``, - not ``Cro``, a ``'w'`` or an ``'S'``, and ``ervo``. + multi-character strings. ``Crow|Servo`` will match either ``'Crow'`` or ``'Servo'``, + not ``'Cro'``, a ``'w'`` or an ``'S'``, and ``'ervo'``. To match a literal ``'|'``, use ``\|``, or enclose it inside a character class, as in ``[|]``. @@ -689,8 +699,7 @@ given location, they can obviously be matched an infinite number of times. >>> print(re.search('^From', 'Reciting From Memory')) None - .. To match a literal \character{\^}, use \regexp{\e\^} or enclose it - .. inside a character class, as in \regexp{[{\e}\^]}. + To match a literal ``'^'``, use ``\^``. ``$`` Matches at the end of a line, which is defined as either the end of the string, @@ -725,7 +734,7 @@ given location, they can obviously be matched an infinite number of times. match when it's contained inside another word. :: >>> p = re.compile(r'\bclass\b') - >>> print(p.search('no class at all')) #doctest: +ELLIPSIS + >>> print(p.search('no class at all')) <_sre.SRE_Match object; span=(3, 8), match='class'> >>> print(p.search('the declassified algorithm')) None @@ -743,7 +752,7 @@ given location, they can obviously be matched an infinite number of times. >>> p = re.compile('\bclass\b') >>> print(p.search('no class at all')) None - >>> print(p.search('\b' + 'class' + '\b')) #doctest: +ELLIPSIS + >>> print(p.search('\b' + 'class' + '\b')) <_sre.SRE_Match object; span=(0, 7), match='\x08class\x08'> Second, inside a character class, where there's no use for this assertion, @@ -786,7 +795,8 @@ of a group with a repeating qualifier, such as ``*``, ``+``, ``?``, or Groups indicated with ``'('``, ``')'`` also capture the starting and ending index of the text that they match; this can be retrieved by passing an argument -to :meth:`group`, :meth:`start`, :meth:`end`, and :meth:`span`. Groups are +to :meth:`~re.match.group`, :meth:`~re.match.start`, :meth:`~re.match.end`, and +:meth:`~re.match.span`. Groups are numbered starting with 0. Group 0 is always present; it's the whole RE, so :ref:`match object ` methods all have group 0 as their default argument. Later we'll see how to express groups that don't capture the span @@ -812,13 +822,13 @@ from left to right. :: >>> m.group(2) 'b' -:meth:`group` can be passed multiple group numbers at a time, in which case it +:meth:`~re.match.group` can be passed multiple group numbers at a time, in which case it will return a tuple containing the corresponding values for those groups. :: >>> m.group(2,1,2) ('b', 'abc', 'b') -The :meth:`groups` method returns a tuple containing the strings for all the +The :meth:`~re.match.groups` method returns a tuple containing the strings for all the subgroups, from 1 up to however many there are. :: >>> m.groups() @@ -834,7 +844,7 @@ backreferences in a RE. For example, the following RE detects doubled words in a string. :: - >>> p = re.compile(r'(\b\w+)\s+\1') + >>> p = re.compile(r'\b(\w+)\s+\1\b') >>> p.search('Paris in the the spring').group() 'the the' @@ -933,9 +943,9 @@ number of the group. There's naturally a variant that uses the group name instead of the number. This is another Python extension: ``(?P=name)`` indicates that the contents of the group called *name* should again be matched at the current point. The regular expression for finding doubled words, -``(\b\w+)\s+\1`` can also be written as ``(?P\b\w+)\s+(?P=word)``:: +``\b(\w+)\s+\1\b`` can also be written as ``\b(?P\w+)\s+(?P=word)\b``:: - >>> p = re.compile(r'(?P\b\w+)\s+(?P=word)') + >>> p = re.compile(r'\b(?P\w+)\s+(?P=word)\b') >>> p.search('Paris in the the spring').group() 'the the' @@ -1034,7 +1044,7 @@ using the following pattern methods: | ``sub()`` | Find all substrings where the RE matches, and | | | replace them with a different string | +------------------+-----------------------------------------------+ -| ``subn()`` | Does the same thing as :meth:`sub`, but | +| ``subn()`` | Does the same thing as :meth:`!sub`, but | | | returns the new string and the number of | | | replacements | +------------------+-----------------------------------------------+ @@ -1043,10 +1053,10 @@ using the following pattern methods: Splitting Strings ----------------- -The :meth:`split` method of a pattern splits a string apart +The :meth:`~re.pattern.split` method of a pattern splits a string apart wherever the RE matches, returning a list of the pieces. It's similar to the -:meth:`split` method of strings but provides much more generality in the -delimiters that you can split by; string :meth:`split` only supports splitting by +:meth:`~str.split` method of strings but provides much more generality in the +delimiters that you can split by; string :meth:`!split` only supports splitting by whitespace or by a fixed string. As you'd expect, there's a module-level :func:`re.split` function, too. @@ -1098,7 +1108,7 @@ Search and Replace ------------------ Another common task is to find all the matches for a pattern, and replace them -with a different string. The :meth:`sub` method takes a replacement value, +with a different string. The :meth:`~re.pattern.sub` method takes a replacement value, which can be either a string or a function, and the string to be processed. .. method:: .sub(replacement, string[, count=0]) @@ -1112,7 +1122,7 @@ which can be either a string or a function, and the string to be processed. replaced; *count* must be a non-negative integer. The default value of 0 means to replace all occurrences. -Here's a simple example of using the :meth:`sub` method. It replaces colour +Here's a simple example of using the :meth:`~re.pattern.sub` method. It replaces colour names with the word ``colour``:: >>> p = re.compile('(blue|white|red)') @@ -1121,7 +1131,7 @@ names with the word ``colour``:: >>> p.sub('colour', 'blue socks and red shoes', count=1) 'colour socks and red shoes' -The :meth:`subn` method does the same work, but returns a 2-tuple containing the +The :meth:`~re.pattern.subn` method does the same work, but returns a 2-tuple containing the new string value and the number of replacements that were performed:: >>> p = re.compile('(blue|white|red)') @@ -1206,24 +1216,24 @@ Use String Methods Sometimes using the :mod:`re` module is a mistake. If you're matching a fixed string, or a single character class, and you're not using any :mod:`re` features -such as the :const:`IGNORECASE` flag, then the full power of regular expressions +such as the :const:`~re.IGNORECASE` flag, then the full power of regular expressions may not be required. Strings have several methods for performing operations with fixed strings and they're usually much faster, because the implementation is a single small C loop that's been optimized for the purpose, instead of the large, more generalized regular expression engine. One example might be replacing a single fixed string with another one; for -example, you might replace ``word`` with ``deed``. ``re.sub()`` seems like the -function to use for this, but consider the :meth:`replace` method. Note that -:func:`replace` will also replace ``word`` inside words, turning ``swordfish`` +example, you might replace ``word`` with ``deed``. :func:`re.sub` seems like the +function to use for this, but consider the :meth:`~str.replace` method. Note that +:meth:`!replace` will also replace ``word`` inside words, turning ``swordfish`` into ``sdeedfish``, but the naive RE ``word`` would have done that, too. (To avoid performing the substitution on parts of words, the pattern would have to be ``\bword\b``, in order to require that ``word`` have a word boundary on -either side. This takes the job beyond :meth:`replace`'s abilities.) +either side. This takes the job beyond :meth:`!replace`'s abilities.) Another common task is deleting every occurrence of a single character from a string or replacing it with another single character. You might do this with -something like ``re.sub('\n', ' ', S)``, but :meth:`translate` is capable of +something like ``re.sub('\n', ' ', S)``, but :meth:`~str.translate` is capable of doing both tasks and will be faster than any regular expression operation can be. @@ -1234,18 +1244,18 @@ can be solved with a faster and simpler string method. match() versus search() ----------------------- -The :func:`match` function only checks if the RE matches at the beginning of the -string while :func:`search` will scan forward through the string for a match. -It's important to keep this distinction in mind. Remember, :func:`match` will +The :func:`~re.match` function only checks if the RE matches at the beginning of the +string while :func:`~re.search` will scan forward through the string for a match. +It's important to keep this distinction in mind. Remember, :func:`!match` will only report a successful match which will start at 0; if the match wouldn't -start at zero, :func:`match` will *not* report it. :: +start at zero, :func:`!match` will *not* report it. :: >>> print(re.match('super', 'superstition').span()) (0, 5) >>> print(re.match('super', 'insuperable')) None -On the other hand, :func:`search` will scan forward through the string, +On the other hand, :func:`~re.search` will scan forward through the string, reporting the first match it finds. :: >>> print(re.search('super', 'superstition').span()) @@ -1284,12 +1294,12 @@ doesn't work because of the greedy nature of ``.*``. :: >>> print(re.match('<.*>', s).group()) Title -The RE matches the ``'<'`` in ````, and the ``.*`` consumes the rest of +The RE matches the ``'<'`` in ``''``, and the ``.*`` consumes the rest of the string. There's still more left in the RE, though, and the ``>`` can't match at the end of the string, so the regular expression engine has to backtrack character by character until it finds a match for the ``>``. The -final match extends from the ``'<'`` in ```` to the ``'>'`` in -````, which isn't what you want. +final match extends from the ``'<'`` in ``''`` to the ``'>'`` in +``''``, which isn't what you want. In this case, the solution is to use the non-greedy qualifiers ``*?``, ``+?``, ``??``, or ``{m,n}?``, which match as *little* text as possible. In the above @@ -1315,7 +1325,7 @@ notation, but they're not terribly readable. REs of moderate complexity can become lengthy collections of backslashes, parentheses, and metacharacters, making them difficult to read and understand. -For such REs, specifying the ``re.VERBOSE`` flag when compiling the regular +For such REs, specifying the :const:`re.VERBOSE` flag when compiling the regular expression can be helpful, because it allows you to format the regular expression more clearly. @@ -1354,5 +1364,5 @@ Friedl's Mastering Regular Expressions, published by O'Reilly. Unfortunately, it exclusively concentrates on Perl and Java's flavours of regular expressions, and doesn't contain any Python material at all, so it won't be useful as a reference for programming in Python. (The first edition covered Python's -now-removed :mod:`regex` module, which won't help you much.) Consider checking +now-removed :mod:`!regex` module, which won't help you much.) Consider checking it out from your library. diff --git a/Doc/includes/email-alternative.py b/Doc/includes/email-alternative.py index 2e142b1e..df7ca6f3 100644 --- a/Doc/includes/email-alternative.py +++ b/Doc/includes/email-alternative.py @@ -32,7 +32,7 @@ msg.add_alternative("""\

Salut!

Cela ressemble à un excellent - recipie déjeuner.

diff --git a/Doc/includes/sqlite3/load_extension.py b/Doc/includes/sqlite3/load_extension.py index 015aa0de..b997c706 100644 --- a/Doc/includes/sqlite3/load_extension.py +++ b/Doc/includes/sqlite3/load_extension.py @@ -11,7 +11,7 @@ con.execute("select load_extension('./fts3.so')") # alternatively you can load the extension using an API call: # con.load_extension("./fts3.so") -# disable extension laoding again +# disable extension loading again con.enable_load_extension(False) # example from SQLite wiki diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index 4c9a528d..1ab05a62 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -288,7 +288,8 @@ and off individually. They are described here in more detail. Fixes duplicate types in the second argument of :func:`isinstance`. For example, ``isinstance(x, (int, int))`` is converted to ``isinstance(x, - (int))``. + int)`` and ``isinstance(x, (int, float, int))`` is converted to + ``isinstance(x, (int, float))``. .. 2to3fixer:: itertools_imports diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 6001db32..9522dd62 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -278,7 +278,7 @@ The :mod:`abc` module also provides the following decorators: :func:`abstractmethod`, making this decorator redundant. -.. decorator:: abstractproperty(fget=None, fset=None, fdel=None, doc=None) +.. decorator:: abstractproperty A subclass of the built-in :func:`property`, indicating an abstract property. diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 1838eb95..e2ad3eb0 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -209,9 +209,9 @@ The fix is to call the :func:`ensure_future` function or the Detect exceptions never consumed -------------------------------- -Python usually calls :func:`sys.displayhook` on unhandled exceptions. If +Python usually calls :func:`sys.excepthook` on unhandled exceptions. If :meth:`Future.set_exception` is called, but the exception is never consumed, -:func:`sys.displayhook` is not called. Instead, :ref:`a log is emitted +:func:`sys.excepthook` is not called. Instead, :ref:`a log is emitted ` when the future is deleted by the garbage collector, with the traceback where the exception was raised. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 83bbb70b..27c170e3 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -500,6 +500,9 @@ Creating listening connections This method is a :ref:`coroutine `. When completed, the coroutine returns a ``(transport, protocol)`` pair. + .. versionadded:: 3.5.3 + + Watch file descriptors ---------------------- diff --git a/Doc/library/asyncio-eventloops.rst b/Doc/library/asyncio-eventloops.rst index d74fcb1e..7970e903 100644 --- a/Doc/library/asyncio-eventloops.rst +++ b/Doc/library/asyncio-eventloops.rst @@ -148,10 +148,9 @@ process based on the calling context. A policy is an object implementing the :class:`AbstractEventLoopPolicy` interface. For most users of :mod:`asyncio`, policies never have to be dealt with -explicitly, since the default global policy is sufficient. +explicitly, since the default global policy is sufficient (see below). -The default policy defines context as the current thread, and manages an event -loop per thread that interacts with :mod:`asyncio`. The module-level functions +The module-level functions :func:`get_event_loop` and :func:`set_event_loop` provide convenient access to event loops managed by the default policy. @@ -189,6 +188,13 @@ An event loop policy must implement the following interface: context, :meth:`set_event_loop` must be called explicitly. +The default policy defines context as the current thread, and manages an event +loop per thread that interacts with :mod:`asyncio`. If the current thread +doesn't already have an event loop associated with it, the default policy's +:meth:`~AbstractEventLoopPolicy.get_event_loop` method creates one when +called from the main thread, but raises :exc:`RuntimeError` otherwise. + + Access to the global loop policy -------------------------------- @@ -200,3 +206,24 @@ Access to the global loop policy Set the current event loop policy. If *policy* is ``None``, the default policy is restored. + + +Customizing the event loop policy +--------------------------------- + +To implement a new event loop policy, it is recommended you subclass the +concrete default event loop policy :class:`DefaultEventLoopPolicy` +and override the methods for which you want to change behavior, for example:: + + class MyEventLoopPolicy(asyncio.DefaultEventLoopPolicy): + + def get_event_loop(self): + """Get the event loop. + + This may be None or an instance of EventLoop. + """ + loop = super().get_event_loop() + # Do something with loop ... + return loop + + asyncio.set_event_loop_policy(MyEventLoopPolicy()) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 5298c110..cc8fffb0 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -216,7 +216,7 @@ Future raise an exception when the future isn't done yet. - Callbacks registered with :meth:`add_done_callback` are always called - via the event loop's :meth:`~AbstractEventLoop.call_soon_threadsafe`. + via the event loop's :meth:`~AbstractEventLoop.call_soon`. - This class is not compatible with the :func:`~concurrent.futures.wait` and :func:`~concurrent.futures.as_completed` functions in the diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 1d84d458..5c60b604 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -21,7 +21,7 @@ program is killed by a signal not handled by Python, when a Python fatal internal error is detected, or when :func:`os._exit` is called. -.. function:: register(func, *args, **kargs) +.. function:: register(func, *args, **kwargs) Register *func* as a function to be executed at termination. Any optional arguments that are to be passed to *func* must be passed as arguments to diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index b9c1868d..6e249ecf 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -802,7 +802,7 @@ The design is such that one can use the factory functions returned by the :func:`lookup` function to construct the instance. -.. class:: StreamReaderWriter(stream, Reader, Writer, errors) +.. class:: StreamReaderWriter(stream, Reader, Writer, errors='strict') Creates a :class:`StreamReaderWriter` instance. *stream* must be a file-like object. *Reader* and *Writer* must be factory functions or classes providing the @@ -826,7 +826,7 @@ The design is such that one can use the factory functions returned by the :func:`lookup` function to construct the instance. -.. class:: StreamRecoder(stream, encode, decode, Reader, Writer, errors) +.. class:: StreamRecoder(stream, encode, decode, Reader, Writer, errors='strict') Creates a :class:`StreamRecoder` instance which implements a two-way conversion: *encode* and *decode* work on the frontend — the data visible to diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 58b03b9b..60154532 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -107,7 +107,12 @@ ABC Inherits from Abstract Methods Mixin .. class:: Iterable ABC for classes that provide the :meth:`__iter__` method. - See also the definition of :term:`iterable`. + + Checking ``isinstance(obj, Iterable)`` detects classes that are registered + as :class:`Iterable` or that have an :meth:`__iter__` method, but it does + not detect classes that iterate with the :meth:`__getitem__` method. + The only reliable way to determine whether an object is :term:`iterable` + is to call ``iter(obj)``. .. class:: Collection diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 43714f74..e1290d4c 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -172,7 +172,7 @@ The :mod:`csv` module defines the following classes: A short usage example:: >>> import csv - >>> with open('names.csv') as csvfile: + >>> with open('names.csv', newline='') as csvfile: ... reader = csv.DictReader(csvfile) ... for row in reader: ... print(row['first_name'], row['last_name']) @@ -211,7 +211,7 @@ The :mod:`csv` module defines the following classes: import csv - with open('names.csv', 'w') as csvfile: + with open('names.csv', 'w', newline='') as csvfile: fieldnames = ['first_name', 'last_name'] writer = csv.DictWriter(csvfile, fieldnames=fieldnames) @@ -270,7 +270,7 @@ The :mod:`csv` module defines the following classes: An example for :class:`Sniffer` use:: - with open('example.csv') as csvfile: + with open('example.csv', newline='') as csvfile: dialect = csv.Sniffer().sniff(csvfile.read(1024)) csvfile.seek(0) reader = csv.reader(csvfile, dialect) diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index b6ac2517..04b2a260 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -176,14 +176,12 @@ C library: Checks for a non-ASCII character (ordinal values 0x80 and above). -These functions accept either integers or strings; when the argument is a +These functions accept either integers or single-character strings; when the argument is a string, it is first converted using the built-in function :func:`ord`. -Note that all these functions check ordinal bit values derived from the first +Note that all these functions check ordinal bit values derived from the character of the string you pass in; they do not actually know anything about -the host machine's character encoding. For functions that know about the -character encoding (and handle internationalization properly) see the -:mod:`string` module. +the host machine's character encoding. The following two functions take either a single-character string or integer byte value; they return a value of the same type. diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index c99c37a5..d770c03c 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -74,7 +74,7 @@ Panel objects have the following methods: .. method:: Panel.hidden() - Returns true if the panel is hidden (not visible), false otherwise. + Returns ``True`` if the panel is hidden (not visible), ``False`` otherwise. .. method:: Panel.hide() diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 668d2826..14e32d78 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -19,6 +19,14 @@ for Windows, DOS, and possibly other systems as well. This extension module is designed to match the API of ncurses, an open-source curses library hosted on Linux and the BSD variants of Unix. +.. note:: + + Whenever the documentation mentions a *character* it can be specified + as an integer, a one-character Unicode string or a one-byte byte string. + + Whenever the documentation mentions a *character string* it can be specified + as a Unicode string or a byte string. + .. note:: Since version 5.4, the ncurses library decides how to interpret non-ASCII data @@ -104,8 +112,8 @@ The module :mod:`curses` defines the following functions: .. function:: color_content(color_number) Return the intensity of the red, green, and blue (RGB) components in the color - *color_number*, which must be between ``0`` and :const:`COLORS`. A 3-tuple is - returned, containing the R,G,B values for the given color, which will be between + *color_number*, which must be between ``0`` and :const:`COLORS`. Return a 3-tuple, + containing the R,G,B values for the given color, which will be between ``0`` (no component) and ``1000`` (maximum amount of component). @@ -119,9 +127,9 @@ The module :mod:`curses` defines the following functions: .. function:: curs_set(visibility) - Set the cursor state. *visibility* can be set to 0, 1, or 2, for invisible, - normal, or very visible. If the terminal supports the visibility requested, the - previous cursor state is returned; otherwise, an exception is raised. On many + Set the cursor state. *visibility* can be set to ``0``, ``1``, or ``2``, for invisible, + normal, or very visible. If the terminal supports the visibility requested, return the + previous cursor state; otherwise raise an exception. On many terminals, the "visible" mode is an underline cursor and the "very visible" mode is a block cursor. @@ -154,12 +162,12 @@ The module :mod:`curses` defines the following functions: representing the desired next state. The :func:`doupdate` ground updates the physical screen to match the virtual screen. - The virtual screen may be updated by a :meth:`noutrefresh` call after write - operations such as :meth:`addstr` have been performed on a window. The normal - :meth:`refresh` call is simply :meth:`noutrefresh` followed by :func:`doupdate`; + The virtual screen may be updated by a :meth:`~window.noutrefresh` call after write + operations such as :meth:`~window.addstr` have been performed on a window. The normal + :meth:`~window.refresh` call is simply :meth:`!noutrefresh` followed by :func:`!doupdate`; if you have to update multiple windows, you can speed performance and perhaps - reduce screen flicker by issuing :meth:`noutrefresh` calls on all windows, - followed by a single :func:`doupdate`. + reduce screen flicker by issuing :meth:`!noutrefresh` calls on all windows, + followed by a single :func:`!doupdate`. .. function:: echo() @@ -175,7 +183,7 @@ The module :mod:`curses` defines the following functions: .. function:: erasechar() - Return the user's current erase character. Under Unix operating systems this + Return the user's current erase character as a one-byte bytes object. Under Unix operating systems this is a property of the controlling tty of the curses program, and is not set by the curses library itself. @@ -183,9 +191,9 @@ The module :mod:`curses` defines the following functions: .. function:: filter() The :func:`.filter` routine, if used, must be called before :func:`initscr` is - called. The effect is that, during those calls, :envvar:`LINES` is set to 1; the - capabilities clear, cup, cud, cud1, cuu1, cuu, vpa are disabled; and the home - string is set to the value of cr. The effect is that the cursor is confined to + called. The effect is that, during those calls, :envvar:`LINES` is set to ``1``; the + capabilities ``clear``, ``cup``, ``cud``, ``cud1``, ``cuu1``, ``cuu``, ``vpa`` are disabled; and the ``home`` + string is set to the value of ``cr``. The effect is that the cursor is confined to the current line, and so are screen updates. This may be used for enabling character-at-a-time line editing without touching the rest of the screen. @@ -205,7 +213,7 @@ The module :mod:`curses` defines the following functions: .. function:: getmouse() - After :meth:`getch` returns :const:`KEY_MOUSE` to signal a mouse event, this + After :meth:`~window.getch` returns :const:`KEY_MOUSE` to signal a mouse event, this method should be call to retrieve the queued mouse event, represented as a 5-tuple ``(id, x, y, z, bstate)``. *id* is an ID value used to distinguish multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is @@ -219,8 +227,8 @@ The module :mod:`curses` defines the following functions: .. function:: getsyx() - Return the current coordinates of the virtual screen cursor in y and x. If - leaveok is currently true, then -1,-1 is returned. + Return the current coordinates of the virtual screen cursor as a tuple + ``(y, x)``. If :meth:`leaveok ` is currently ``True``, then return ``(-1, -1)``. .. function:: getwin(file) @@ -260,7 +268,7 @@ The module :mod:`curses` defines the following functions: Used for half-delay mode, which is similar to cbreak mode in that characters typed by the user are immediately available to the program. However, after - blocking for *tenths* tenths of seconds, an exception is raised if nothing has + blocking for *tenths* tenths of seconds, raise an exception if nothing has been typed. The value of *tenths* must be a number between ``1`` and ``255``. Use :func:`nocbreak` to leave half-delay mode. @@ -273,7 +281,7 @@ The module :mod:`curses` defines the following functions: :const:`COLORS`. Each of *r*, *g*, *b*, must be a value between ``0`` and ``1000``. When :func:`init_color` is used, all occurrences of that color on the screen immediately change to the new definition. This function is a no-op on - most terminals; it is active only if :func:`can_change_color` returns ``1``. + most terminals; it is active only if :func:`can_change_color` returns ``True``. .. function:: init_pair(pair_number, fg, bg) @@ -313,32 +321,32 @@ The module :mod:`curses` defines the following functions: .. function:: keyname(k) - Return the name of the key numbered *k*. The name of a key generating printable + Return the name of the key numbered *k* as a bytes object. The name of a key generating printable ASCII character is the key's character. The name of a control-key combination - is a two-character string consisting of a caret followed by the corresponding + is a two-byte bytes object consisting of a caret (``b'^'``) followed by the corresponding printable ASCII character. The name of an alt-key combination (128--255) is a - string consisting of the prefix 'M-' followed by the name of the corresponding + bytes object consisting of the prefix ``b'M-'`` followed by the name of the corresponding ASCII character. .. function:: killchar() - Return the user's current line kill character. Under Unix operating systems + Return the user's current line kill character as a one-byte bytes object. Under Unix operating systems this is a property of the controlling tty of the curses program, and is not set by the curses library itself. .. function:: longname() - Return a string containing the terminfo long name field describing the current + Return a bytes object containing the terminfo long name field describing the current terminal. The maximum length of a verbose description is 128 characters. It is defined only after the call to :func:`initscr`. -.. function:: meta(yes) +.. function:: meta(flag) - If *yes* is 1, allow 8-bit characters to be input. If *yes* is 0, allow only - 7-bit chars. + If *flag* is ``True``, allow 8-bit characters to be input. If + *flag* is ``False``, allow only 7-bit chars. .. function:: mouseinterval(interval) @@ -352,7 +360,7 @@ The module :mod:`curses` defines the following functions: Set the mouse events to be reported, and return a tuple ``(availmask, oldmask)``. *availmask* indicates which of the specified mouse events can be - reported; on complete failure it returns 0. *oldmask* is the previous value of + reported; on complete failure it returns ``0``. *oldmask* is the previous value of the given window's mouse event mask. If this function is never called, no mouse events are ever reported. @@ -365,13 +373,13 @@ The module :mod:`curses` defines the following functions: .. function:: newpad(nlines, ncols) Create and return a pointer to a new pad data structure with the given number - of lines and columns. A pad is returned as a window object. + of lines and columns. Return a pad as a window object. A pad is like a window, except that it is not restricted by the screen size, and is not necessarily associated with a particular part of the screen. Pads can be used when a large window is needed, and only a part of the window will be on the screen at one time. Automatic refreshes of pads (such as from scrolling or - echoing of input) do not occur. The :meth:`refresh` and :meth:`noutrefresh` + echoing of input) do not occur. The :meth:`~window.refresh` and :meth:`~window.noutrefresh` methods of a pad require 6 arguments to specify the part of the pad to be displayed and the location on the screen to be used for the display. The arguments are *pminrow*, *pmincol*, *sminrow*, *smincol*, *smaxrow*, *smaxcol*; the *p* @@ -419,9 +427,9 @@ The module :mod:`curses` defines the following functions: .. function:: noqiflush() - When the :func:`noqiflush` routine is used, normal flush of input and output queues - associated with the INTR, QUIT and SUSP characters will not be done. You may - want to call :func:`noqiflush` in a signal handler if you want output to + When the :func:`!noqiflush` routine is used, normal flush of input and output queues + associated with the ``INTR``, ``QUIT`` and ``SUSP`` characters will not be done. You may + want to call :func:`!noqiflush` in a signal handler if you want output to continue as though the interrupt had not occurred, after the handler exits. @@ -442,14 +450,14 @@ The module :mod:`curses` defines the following functions: :func:`color_pair` is the counterpart to this function. -.. function:: putp(string) +.. function:: putp(str) Equivalent to ``tputs(str, 1, putchar)``; emit the value of a specified terminfo capability for the current terminal. Note that the output of :func:`putp` always goes to standard output. -.. function:: qiflush( [flag] ) +.. function:: qiflush([flag]) If *flag* is ``False``, the effect is the same as calling :func:`noqiflush`. If *flag* is ``True``, or no argument is provided, the queues will be flushed when @@ -486,7 +494,7 @@ The module :mod:`curses` defines the following functions: Backend function used by :func:`resizeterm`, performing most of the work; when resizing the windows, :func:`resize_term` blank-fills the areas that are extended. The calling application should fill in these areas with - appropriate data. The :func:`resize_term` function attempts to resize all + appropriate data. The :func:`!resize_term` function attempts to resize all windows. However, due to the calling convention of pads, it is not possible to resize these without additional interaction with the application. @@ -506,16 +514,17 @@ The module :mod:`curses` defines the following functions: .. function:: setsyx(y, x) - Set the virtual screen cursor to *y*, *x*. If *y* and *x* are both -1, then - leaveok is set. + Set the virtual screen cursor to *y*, *x*. If *y* and *x* are both ``-1``, then + :meth:`leaveok ` is set ``True``. -.. function:: setupterm([termstr, fd]) +.. function:: setupterm(term=None, fd=-1) - Initialize the terminal. *termstr* is a string giving the terminal name; if - omitted, the value of the :envvar:`TERM` environment variable will be used. *fd* is the + Initialize the terminal. *term* is a string giving + the terminal name, or ``None``; if omitted or ``None``, the value of the + :envvar:`TERM` environment variable will be used. *fd* is the file descriptor to which any initialization sequences will be sent; if not - supplied, the file descriptor for ``sys.stdout`` will be used. + supplied or ``-1``, the file descriptor for ``sys.stdout`` will be used. .. function:: start_color() @@ -540,13 +549,14 @@ The module :mod:`curses` defines the following functions: .. function:: termname() - Return the value of the environment variable :envvar:`TERM`, truncated to 14 characters. + Return the value of the environment variable :envvar:`TERM`, as a bytes object, + truncated to 14 characters. .. function:: tigetflag(capname) Return the value of the Boolean capability corresponding to the terminfo - capability name *capname*. The value ``-1`` is returned if *capname* is not a + capability name *capname* as an integer. Return the value ``-1`` if *capname* is not a Boolean capability, or ``0`` if it is canceled or absent from the terminal description. @@ -554,7 +564,7 @@ The module :mod:`curses` defines the following functions: .. function:: tigetnum(capname) Return the value of the numeric capability corresponding to the terminfo - capability name *capname*. The value ``-2`` is returned if *capname* is not a + capability name *capname* as an integer. Return the value ``-2`` if *capname* is not a numeric capability, or ``-1`` if it is canceled or absent from the terminal description. @@ -562,13 +572,14 @@ The module :mod:`curses` defines the following functions: .. function:: tigetstr(capname) Return the value of the string capability corresponding to the terminfo - capability name *capname*. ``None`` is returned if *capname* is not a string - capability, or is canceled or absent from the terminal description. + capability name *capname* as a bytes object. Return ``None`` if *capname* + is not a terminfo "string capability", or is canceled or absent from the + terminal description. .. function:: tparm(str[, ...]) - Instantiate the string *str* with the supplied parameters, where *str* should + Instantiate the bytes object *str* with the supplied parameters, where *str* should be a parameterized string obtained from the terminfo database. E.g. ``tparm(tigetstr("cup"), 5, 3)`` could result in ``b'\033[6;4H'``, the exact result depending on terminal type. @@ -588,18 +599,18 @@ The module :mod:`curses` defines the following functions: .. function:: unctrl(ch) - Return a string which is a printable representation of the character *ch*. - Control characters are displayed as a caret followed by the character, for - example as ``^C``. Printing characters are left as they are. + Return a bytes object which is a printable representation of the character *ch*. + Control characters are represented as a caret followed by the character, for + example as ``b'^C'``. Printing characters are left as they are. .. function:: ungetch(ch) - Push *ch* so the next :meth:`getch` will return it. + Push *ch* so the next :meth:`~window.getch` will return it. .. note:: - Only one *ch* can be pushed before :meth:`getch` is called. + Only one *ch* can be pushed before :meth:`!getch` is called. .. function:: update_lines_cols() @@ -611,11 +622,11 @@ The module :mod:`curses` defines the following functions: .. function:: unget_wch(ch) - Push *ch* so the next :meth:`get_wch` will return it. + Push *ch* so the next :meth:`~window.get_wch` will return it. .. note:: - Only one *ch* can be pushed before :meth:`get_wch` is called. + Only one *ch* can be pushed before :meth:`!get_wch` is called. .. versionadded:: 3.3 @@ -640,7 +651,7 @@ The module :mod:`curses` defines the following functions: Allow use of default values for colors on terminals supporting this feature. Use this to support transparency in your application. The default color is assigned - to the color number -1. After calling this function, ``init_pair(x, + to the color number ``-1``. After calling this function, ``init_pair(x, curses.COLOR_RED, -1)`` initializes, for instance, color pair *x* to a red foreground color on the default background. @@ -652,7 +663,7 @@ The module :mod:`curses` defines the following functions: this function will restore the terminal to a sane state before re-raising the exception and generating a traceback. The callable object *func* is then passed the main window 'stdscr' as its first argument, followed by any other arguments - passed to :func:`wrapper`. Before calling *func*, :func:`wrapper` turns on + passed to :func:`!wrapper`. Before calling *func*, :func:`!wrapper` turns on cbreak mode, turns off echo, enables the terminal keypad, and initializes colors if the terminal has color support. On exit (whether normally or by exception) it restores cooked mode, turns on echo, and disables the terminal keypad. @@ -670,13 +681,6 @@ the following methods and attributes: .. method:: window.addch(ch[, attr]) window.addch(y, x, ch[, attr]) - .. note:: - - A *character* means a C character (an ASCII code), rather than a Python - character (a string of length 1). (This note is true whenever the - documentation mentions a character.) The built-in :func:`ord` is handy for - conveying strings to codes. - Paint character *ch* at ``(y, x)`` with attributes *attr*, overwriting any character previously painter at that location. By default, the character position and attributes are the current settings for the window object. @@ -685,15 +689,16 @@ the following methods and attributes: .. method:: window.addnstr(str, n[, attr]) window.addnstr(y, x, str, n[, attr]) - Paint at most *n* characters of the string *str* at ``(y, x)`` with attributes + Paint at most *n* characters of the character string *str* at + ``(y, x)`` with attributes *attr*, overwriting anything previously on the display. .. method:: window.addstr(str[, attr]) window.addstr(y, x, str[, attr]) - Paint the string *str* at ``(y, x)`` with attributes *attr*, overwriting - anything previously on the display. + Paint the character string *str* at ``(y, x)`` with attributes + *attr*, overwriting anything previously on the display. .. method:: window.attroff(attr) @@ -710,8 +715,8 @@ the following methods and attributes: .. method:: window.attrset(attr) - Set the "background" set of attributes to *attr*. This set is initially 0 (no - attributes). + Set the "background" set of attributes to *attr*. This set is initially + ``0`` (no attributes). .. method:: window.bkgd(ch[, attr]) @@ -741,8 +746,7 @@ the following methods and attributes: Draw a border around the edges of the window. Each parameter specifies the character to use for a specific part of the border; see the table below for more - details. The characters can be specified as integers or as one-character - strings. + details. .. note:: @@ -783,11 +787,11 @@ the following methods and attributes: window.chgat(y, x, num, attr) Set the attributes of *num* characters at the current cursor position, or at - position ``(y, x)`` if supplied. If no value of *num* is given or *num* = -1, - the attribute will be set on all the characters to the end of the line. This - function does not move the cursor. The changed line will be touched using the - :meth:`touchline` method so that the contents will be redisplayed by the next - window refresh. + position ``(y, x)`` if supplied. If *num* is not given or is ``-1``, + the attribute will be set on all the characters to the end of the line. This + function moves cursor to position ``(y, x)`` if supplied. The changed line + will be touched using the :meth:`touchline` method so that the contents will + be redisplayed by the next window refresh. .. method:: window.clear() @@ -796,9 +800,9 @@ the following methods and attributes: call to :meth:`refresh`. -.. method:: window.clearok(yes) +.. method:: window.clearok(flag) - If *yes* is 1, the next call to :meth:`refresh` will clear the window + If *flag* is ``True``, the next call to :meth:`refresh` will clear the window completely. @@ -880,15 +884,16 @@ the following methods and attributes: .. method:: window.getch([y, x]) Get a character. Note that the integer returned does *not* have to be in ASCII - range: function keys, keypad keys and so on return numbers higher than 256. In - no-delay mode, -1 is returned if there is no input, else :func:`getch` waits - until a key is pressed. + range: function keys, keypad keys and so on are represented by numbers higher + than 255. In no-delay mode, return ``-1`` if there is no input, otherwise + wait until a key is pressed. .. method:: window.get_wch([y, x]) Get a wide character. Return a character for most keys, or an integer for function keys, keypad keys, and other special keys. + In no-delay mode, raise an exception if there is no input. .. versionadded:: 3.3 @@ -897,7 +902,7 @@ the following methods and attributes: Get a character, returning a string instead of an integer, as :meth:`getch` does. Function keys, keypad keys and other special keys return a multibyte - string containing the key name. In no-delay mode, an exception is raised if + string containing the key name. In no-delay mode, raise an exception if there is no input. @@ -909,13 +914,16 @@ the following methods and attributes: .. method:: window.getparyx() Return the beginning coordinates of this window relative to its parent window - into two integer variables y and x. Return ``-1, -1`` if this window has no + as a tuple ``(y, x)``. Return ``(-1, -1)`` if this window has no parent. -.. method:: window.getstr([y, x]) +.. method:: window.getstr() + window.getstr(n) + window.getstr(y, x) + window.getstr(y, x, n) - Read a string from the user, with primitive line editing capacity. + Read a bytes object from the user, with primitive line editing capacity. .. method:: window.getyx() @@ -939,9 +947,9 @@ the following methods and attributes: insert/delete is enabled by default. -.. method:: window.idlok(yes) +.. method:: window.idlok(flag) - If called with *yes* equal to 1, :mod:`curses` will try and use hardware line + If *flag* is ``True``, :mod:`curses` will try and use hardware line editing facilities. Otherwise, line insertion/deletion are disabled. @@ -1003,7 +1011,7 @@ the following methods and attributes: .. method:: window.instr([n]) window.instr(y, x[, n]) - Return a string of characters, extracted from the window starting at the + Return a bytes object of characters, extracted from the window starting at the current cursor position, or at *y*, *x* if specified. Attributes are stripped from the characters. If *n* is specified, :meth:`instr` returns a string at most *n* characters long (exclusive of the trailing NUL). @@ -1022,20 +1030,20 @@ the following methods and attributes: :meth:`refresh`; otherwise return ``False``. -.. method:: window.keypad(yes) +.. method:: window.keypad(flag) - If *yes* is 1, escape sequences generated by some keys (keypad, function keys) - will be interpreted by :mod:`curses`. If *yes* is 0, escape sequences will be + If *flag* is ``True``, escape sequences generated by some keys (keypad, function keys) + will be interpreted by :mod:`curses`. If *flag* is ``False``, escape sequences will be left as is in the input stream. -.. method:: window.leaveok(yes) +.. method:: window.leaveok(flag) - If *yes* is 1, cursor is left where it is on update, instead of being at "cursor + If *flag* is ``True``, cursor is left where it is on update, instead of being at "cursor position." This reduces cursor movement where possible. If possible the cursor will be made invisible. - If *yes* is 0, cursor will always be at "cursor position" after an update. + If *flag* is ``False``, cursor will always be at "cursor position" after an update. .. method:: window.move(new_y, new_x) @@ -1055,16 +1063,16 @@ the following methods and attributes: Move the window so its upper-left corner is at ``(new_y, new_x)``. -.. method:: window.nodelay(yes) +.. method:: window.nodelay(flag) - If *yes* is ``1``, :meth:`getch` will be non-blocking. + If *flag* is ``True``, :meth:`getch` will be non-blocking. -.. method:: window.notimeout(yes) +.. method:: window.notimeout(flag) - If *yes* is ``1``, escape sequences will not be timed out. + If *flag* is ``True``, escape sequences will not be timed out. - If *yes* is ``0``, after a few milliseconds, an escape sequence will not be + If *flag* is ``False``, after a few milliseconds, an escape sequence will not be interpreted, and will be left in the input stream as is. @@ -1153,8 +1161,8 @@ the following methods and attributes: Control what happens when the cursor of a window is moved off the edge of the window or scrolling region, either as a result of a newline action on the bottom - line, or typing the last character of the last line. If *flag* is false, the - cursor is left on the bottom line. If *flag* is true, the window is scrolled up + line, or typing the last character of the last line. If *flag* is ``False``, the + cursor is left on the bottom line. If *flag* is ``True``, the window is scrolled up one line. Note that in order to get the physical scrolling effect on the terminal, it is also necessary to call :meth:`idlok`. @@ -1202,7 +1210,7 @@ the following methods and attributes: .. method:: window.syncok(flag) - If called with *flag* set to ``True``, then :meth:`syncup` is called automatically + If *flag* is ``True``, then :meth:`syncup` is called automatically whenever there is a change in the window. @@ -1216,9 +1224,9 @@ the following methods and attributes: Set blocking or non-blocking read behavior for the window. If *delay* is negative, blocking read is used (which will wait indefinitely for input). If - *delay* is zero, then non-blocking read is used, and -1 will be returned by - :meth:`getch` if no input is waiting. If *delay* is positive, then - :meth:`getch` will block for *delay* milliseconds, and return -1 if there is + *delay* is zero, then non-blocking read is used, and :meth:`getch` will + return ``-1`` if no input is waiting. If *delay* is positive, then + :meth:`getch` will block for *delay* milliseconds, and return ``-1`` if there is still no input at the end of that time. @@ -1226,7 +1234,7 @@ the following methods and attributes: Pretend *count* lines have been changed, starting with line *start*. If *changed* is supplied, it specifies whether the affected lines are marked as - having been changed (*changed*\ =1) or unchanged (*changed*\ =0). + having been changed (*changed*\ ``=True``) or unchanged (*changed*\ ``=False``). .. method:: window.touchwin() @@ -1268,7 +1276,7 @@ The :mod:`curses` module defines the following data members: .. data:: version - A string representing the current version of the module. Also available as + A bytes object representing the current version of the module. Also available as :const:`__version__`. Some constants are available to specify character cell attributes. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index c7957820..c68b15a9 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1053,7 +1053,7 @@ All of the following opcodes use their arguments. Pops all function arguments, and the function itself off the stack, and pushes the return value. Note that this opcode pops at most three items from the stack. Var-positional and var-keyword arguments are packed - by :opcode:`BUILD_MAP_UNPACK_WITH_CALL` and + by :opcode:`BUILD_TUPLE_UNPACK_WITH_CALL` and :opcode:`BUILD_MAP_UNPACK_WITH_CALL`. .. versionadded:: 3.6 diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index c797f633..652339e5 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -78,6 +78,9 @@ options: Providing both of the script selection options will trigger an exception. +.. versionchanged:: 3.6.3 + The exit status is non-zero if the command fails. + Module API ---------- diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index bd4c94fc..7a7a84d1 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -182,9 +182,9 @@ are always available. They are listed here in alphabetical order. base 16). :exc:`ValueError` will be raised if *i* is outside that range. -.. function:: classmethod(function) +.. decorator:: classmethod - Return a class method for *function*. + Transform a method into a class method. A class method receives the class as implicit first argument, just like an instance method receives the instance. To declare a class method, use this @@ -1384,9 +1384,9 @@ are always available. They are listed here in alphabetical order. For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. -.. function:: staticmethod(function) +.. decorator:: staticmethod - Return a static method for *function*. + Transform a method into a static method. A static method does not receive an implicit first argument. To declare a static method, use this idiom:: @@ -1405,12 +1405,21 @@ are always available. They are listed here in alphabetical order. :func:`classmethod` for a variant that is useful for creating alternate class constructors. + Like all decorators, it is also possible to call ``staticmethod`` as + a regular function and do something with its result. This is needed + in some cases where you need a reference to a function from a class + body and you want to avoid the automatic transformation to instance + method. For these cases, use this idiom: + + class C: + builtin_open = staticmethod(open) + For more information on static methods, consult the documentation on the standard type hierarchy in :ref:`types`. - .. index:: - single: string; str() (built-in function) +.. index:: + single: string; str() (built-in function) .. _func-str: .. class:: str(object='') diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 9a8defee..28062c11 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -264,9 +264,9 @@ The :mod:`functools` module defines the following functions: return value -.. decorator:: singledispatch(default) +.. decorator:: singledispatch - Transforms a function into a :term:`single-dispatch ` :term:`generic function`. To define a generic function, decorate it with the ``@singledispatch`` diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 053d9d38..407853c2 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -134,7 +134,7 @@ Class-based API The class-based API of the :mod:`gettext` module gives you more flexibility and greater convenience than the GNU :program:`gettext` API. It is the recommended -way of localizing your Python applications and modules. :mod:`gettext` defines +way of localizing your Python applications and modules. :mod:`!gettext` defines a "translations" class which implements the parsing of GNU :file:`.mo` format files, and has methods for returning strings. Instances of this "translations" class can also install themselves in the built-in namespace as the function @@ -219,7 +219,7 @@ Translation classes are what actually implement the translation of original source file message strings to translated message strings. The base class used by all translation classes is :class:`NullTranslations`; this provides the basic interface you can use to write your own specialized translation classes. Here -are the methods of :class:`NullTranslations`: +are the methods of :class:`!NullTranslations`: .. class:: NullTranslations(fp=None) @@ -247,13 +247,13 @@ are the methods of :class:`NullTranslations`: .. method:: gettext(message) - If a fallback has been set, forward :meth:`.gettext` to the fallback. + If a fallback has been set, forward :meth:`!gettext` to the fallback. Otherwise, return *message*. Overridden in derived classes. .. method:: ngettext(singular, plural, n) - If a fallback has been set, forward :meth:`ngettext` to the fallback. + If a fallback has been set, forward :meth:`!ngettext` to the fallback. Otherwise, return *singular* if *n* is 1; return *plural* otherwise. Overridden in derived classes. @@ -261,7 +261,7 @@ are the methods of :class:`NullTranslations`: .. method:: lgettext(message) .. method:: lngettext(singular, plural, n) - Equivalent to :meth:`.gettext` and :meth:`ngettext`, but the translation + Equivalent to :meth:`.gettext` and :meth:`.ngettext`, but the translation is returned as a byte string encoded in the preferred system encoding if no encoding was explicitly set with :meth:`set_output_charset`. Overridden in derived classes. diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 725dce6f..3a27a5b5 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -510,15 +510,19 @@ to users and later verify them to make sure they weren't tampered with:: ... h.update(cookie) ... return h.hexdigest().encode('utf-8') >>> - >>> cookie = b'user:vatrogasac' + >>> def verify(cookie, sig): + ... good_sig = sign(cookie) + ... return compare_digest(good_sig, sig) + >>> + >>> cookie = b'user-alice' >>> sig = sign(cookie) >>> print("{0},{1}".format(cookie.decode('utf-8'), sig)) - user:vatrogasac,349cf904533767ed2d755279a8df84d0 - >>> compare_digest(cookie, sig) + user-alice,b'43b3c982cf697e0c5ab22172d1ca7421' + >>> verify(cookie, sig) True - >>> compare_digest(b'user:policajac', sig) + >>> verify(b'user-bob', sig) False - >>> compare_digest(cookie, b'0102030405060708090a0b0c0d0e0f00') + >>> verify(cookie, b'0102030405060708090a0b0c0d0e0f00') False Even though there's a native keyed hashing mode, BLAKE2 can, of course, be used diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 6406e306..d194362f 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1045,7 +1045,15 @@ find and load modules. .. class:: ModuleSpec(name, loader, *, origin=None, loader_state=None, is_package=None) - A specification for a module's import-system-related state. + A specification for a module's import-system-related state. This is + typically exposed as the module's ``__spec__`` attribute. In the + descriptions below, the names in parentheses give the corresponding + attribute available directly on the module object. + E.g. ``module.__spec__.origin == module.__file__``. Note however that + while the *values* are usually equivalent, they can differ since there is + no synchronization between the two objects. Thus it is possible to update + the module's ``__path__`` at runtime, and this will not be automatically + reflected in ``__spec__.submodule_search_locations``. .. versionadded:: 3.4 diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index b0d0a8c8..594af39f 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -53,7 +53,7 @@ Iterator Arguments Results :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` :func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` :func:`filterfalse` pred, seq elements of seq where pred(elem) is false ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` -:func:`groupby` iterable[, keyfunc] sub-iterators grouped by value of keyfunc(v) +:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` :func:`takewhile` pred, seq seq[0], seq[1], until pred fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` @@ -753,15 +753,16 @@ which incur interpreter overhead. def roundrobin(*iterables): "roundrobin('ABC', 'D', 'EF') --> A D E B F C" # Recipe credited to George Sakkis - pending = len(iterables) + num_active = len(iterables) nexts = cycle(iter(it).__next__ for it in iterables) - while pending: + while num_active: try: for next in nexts: yield next() except StopIteration: - pending -= 1 - nexts = cycle(islice(nexts, pending)) + # Remove the iterator we just exhausted from the cycle. + num_active -= 1 + nexts = cycle(islice(nexts, num_active)) def partition(pred, iterable): 'Use a predicate to partition entries into false entries and true entries' diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 5aaf4a39..b04442bc 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -542,17 +542,23 @@ library. Access to message catalogs -------------------------- +.. function:: gettext(msg) +.. function:: dgettext(domain, msg) +.. function:: dcgettext(domain, msg, category) +.. function:: textdomain(domain) +.. function:: bindtextdomain(domain, dir) + The locale module exposes the C library's gettext interface on systems that -provide this interface. It consists of the functions :func:`gettext`, -:func:`dgettext`, :func:`dcgettext`, :func:`textdomain`, :func:`bindtextdomain`, -and :func:`bind_textdomain_codeset`. These are similar to the same functions in +provide this interface. It consists of the functions :func:`!gettext`, +:func:`!dgettext`, :func:`!dcgettext`, :func:`!textdomain`, :func:`!bindtextdomain`, +and :func:`!bind_textdomain_codeset`. These are similar to the same functions in the :mod:`gettext` module, but use the C library's binary format for message catalogs, and the C library's search algorithms for locating message catalogs. Python applications should normally find no need to invoke these functions, and should use :mod:`gettext` instead. A known exception to this rule are applications that link with additional C libraries which internally invoke -:c:func:`gettext` or :func:`dcgettext`. For these applications, it may be +:c:func:`gettext` or :c:func:`dcgettext`. For these applications, it may be necessary to bind the text domain, so that the libraries can properly locate their message catalogs. diff --git a/Doc/library/msilib.rst b/Doc/library/msilib.rst index 0a420329..85443174 100644 --- a/Doc/library/msilib.rst +++ b/Doc/library/msilib.rst @@ -124,9 +124,9 @@ structures. .. seealso:: - `FCICreateFile `_ - `UuidCreate `_ - `UuidToString `_ + `FCICreate `_ + `UuidCreate `_ + `UuidToString `_ .. _database-objects: @@ -155,9 +155,9 @@ Database Objects .. seealso:: - `MSIDatabaseOpenView `_ - `MSIDatabaseCommit `_ - `MSIGetSummaryInformation `_ + `MSIDatabaseOpenView `_ + `MSIDatabaseCommit `_ + `MSIGetSummaryInformation `_ .. _view-objects: @@ -203,11 +203,11 @@ View Objects .. seealso:: - `MsiViewExecute `_ - `MSIViewGetColumnInfo `_ - `MsiViewFetch `_ - `MsiViewModify `_ - `MsiViewClose `_ + `MsiViewExecute `_ + `MSIViewGetColumnInfo `_ + `MsiViewFetch `_ + `MsiViewModify `_ + `MsiViewClose `_ .. _summary-objects: @@ -247,10 +247,10 @@ Summary Information Objects .. seealso:: - `MsiSummaryInfoGetProperty `_ - `MsiSummaryInfoGetPropertyCount `_ - `MsiSummaryInfoSetProperty `_ - `MsiSummaryInfoPersist `_ + `MsiSummaryInfoGetProperty `_ + `MsiSummaryInfoGetPropertyCount `_ + `MsiSummaryInfoSetProperty `_ + `MsiSummaryInfoPersist `_ .. _record-objects: @@ -301,18 +301,18 @@ Record Objects .. seealso:: - `MsiRecordGetFieldCount `_ - `MsiRecordSetString `_ - `MsiRecordSetStream `_ - `MsiRecordSetInteger `_ - `MsiRecordClear `_ + `MsiRecordGetFieldCount `_ + `MsiRecordSetString `_ + `MsiRecordSetStream `_ + `MsiRecordSetInteger `_ + `MsiRecordClearData `_ .. _msi-errors: Errors ------ -All wrappers around MSI functions raise :exc:`MsiError`; the string inside the +All wrappers around MSI functions raise :exc:`MSIError`; the string inside the exception will contain more detail. @@ -392,15 +392,15 @@ Directory Objects .. method:: remove_pyc() - Remove ``.pyc``/``.pyo`` files on uninstall. + Remove ``.pyc`` files on uninstall. .. seealso:: - `Directory Table `_ - `File Table `_ - `Component Table `_ - `FeatureComponents Table `_ + `Directory Table `_ + `File Table `_ + `Component Table `_ + `FeatureComponents Table `_ .. _features: @@ -425,7 +425,7 @@ Features .. seealso:: - `Feature Table `_ + `Feature Table `_ .. _msi-gui: @@ -520,13 +520,13 @@ for installing Python packages. .. seealso:: - `Dialog Table `_ - `Control Table `_ - `Control Types `_ - `ControlCondition Table `_ - `ControlEvent Table `_ - `EventMapping Table `_ - `RadioButton Table `_ + `Dialog Table `_ + `Control Table `_ + `Control Types `_ + `ControlCondition Table `_ + `ControlEvent Table `_ + `EventMapping Table `_ + `RadioButton Table `_ .. _msi-tables: diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 2f770b63..9521f77c 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1821,8 +1821,8 @@ Running the following commands creates a server for a single shared queue which remote clients can access:: >>> from multiprocessing.managers import BaseManager - >>> import queue - >>> queue = queue.Queue() + >>> from queue import Queue + >>> queue = Queue() >>> class QueueManager(BaseManager): pass >>> QueueManager.register('get_queue', callable=lambda:queue) >>> m = QueueManager(address=('', 50000), authkey=b'abracadabra') @@ -2165,7 +2165,7 @@ with the :class:`Pool` class. .. versionadded:: 3.3 - .. method:: starmap_async(func, iterable[, chunksize[, callback[, error_back]]]) + .. method:: starmap_async(func, iterable[, chunksize[, callback[, error_callback]]]) A combination of :meth:`starmap` and :meth:`map_async` that iterates over *iterable* of iterables and calls *func* with the iterables unpacked. @@ -2280,7 +2280,7 @@ multiple connections at the same time. If a welcome message is not received, then :exc:`~multiprocessing.AuthenticationError` is raised. -.. function:: Client(address[, family[, authenticate[, authkey]]]) +.. function:: Client(address[, family[, authkey]]) Attempt to set up a connection to the listener which is using address *address*, returning a :class:`~multiprocessing.Connection`. @@ -2289,14 +2289,13 @@ multiple connections at the same time. generally be omitted since it can usually be inferred from the format of *address*. (See :ref:`multiprocessing-address-formats`) - If *authenticate* is ``True`` or *authkey* is a byte string then digest - authentication is used. The key used for authentication will be either - *authkey* or ``current_process().authkey`` if *authkey* is ``None``. - If authentication fails then - :exc:`~multiprocessing.AuthenticationError` is raised. See - :ref:`multiprocessing-auth-keys`. + If *authkey* is given and not None, it should be a byte string and will be + used as the secret key for an HMAC-based authentication challenge. No + authentication is done if *authkey* is None. + :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. + See :ref:`multiprocessing-auth-keys`. -.. class:: Listener([address[, family[, backlog[, authenticate[, authkey]]]]]) +.. class:: Listener([address[, family[, backlog[, authkey]]]]) A wrapper for a bound socket or Windows named pipe which is 'listening' for connections. @@ -2325,17 +2324,10 @@ multiple connections at the same time. to the :meth:`~socket.socket.listen` method of the socket once it has been bound. - If *authenticate* is ``True`` (``False`` by default) or *authkey* is not - ``None`` then digest authentication is used. - - If *authkey* is a byte string then it will be used as the - authentication key; otherwise it must be ``None``. - - If *authkey* is ``None`` and *authenticate* is ``True`` then - ``current_process().authkey`` is used as the authentication key. If - *authkey* is ``None`` and *authenticate* is ``False`` then no - authentication is done. If authentication fails then - :exc:`~multiprocessing.AuthenticationError` is raised. + If *authkey* is given and not None, it should be a byte string and will be + used as the secret key for an HMAC-based authentication challenge. No + authentication is done if *authkey* is None. + :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. See :ref:`multiprocessing-auth-keys`. .. method:: accept() diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 8121b480..60bf23a5 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -17,9 +17,10 @@ The :mod:`operator` module exports a set of efficient functions corresponding to the intrinsic operators of Python. For example, ``operator.add(x, y)`` is -equivalent to the expression ``x+y``. The function names are those used for -special class methods; variants without leading and trailing ``__`` are also -provided for convenience. +equivalent to the expression ``x+y``. Many function names are those used for +special methods, without the double underscores. For backward compatibility, +many of these have a variant with the double underscores kept. The variants +without the double underscores are preferred for clarity. The functions fall into categories that perform object comparisons, logical operations, mathematical operations and sequence operations. diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 56fc4ed2..fae8945f 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -14,8 +14,9 @@ This module provides regular expression matching operations similar to those found in Perl. -Both patterns and strings to be searched can be Unicode strings as well as -8-bit strings. However, Unicode strings and 8-bit strings cannot be mixed: +Both patterns and strings to be searched can be Unicode strings (:class:`str`) +as well as 8-bit strings (:class:`bytes`). +However, Unicode strings and 8-bit strings cannot be mixed: that is, you cannot match a Unicode string with a byte pattern or vice-versa; similarly, when asking for a substitution, the replacement string must be of the same type as both the pattern and the search string. @@ -81,9 +82,7 @@ strings to be matched ``'in single quotes'``.) Some characters, like ``'|'`` or ``'('``, are special. Special characters either stand for classes of ordinary characters, or affect -how the regular expressions around them are interpreted. Regular -expression pattern strings may not contain null bytes, but can specify -the null byte using a ``\number`` notation such as ``'\x00'``. +how the regular expressions around them are interpreted. Repetition qualifiers (``*``, ``+``, ``?``, ``{m,n}``, etc) cannot be directly nested. This avoids ambiguity with the non-greedy modifier suffix @@ -94,16 +93,16 @@ the expression ``(?:a{6})*`` matches any multiple of six ``'a'`` characters. The special characters are: -``'.'`` +``.`` (Dot.) In the default mode, this matches any character except a newline. If the :const:`DOTALL` flag has been specified, this matches any character including a newline. -``'^'`` +``^`` (Caret.) Matches the start of the string, and in :const:`MULTILINE` mode also matches immediately after each newline. -``'$'`` +``$`` Matches the end of the string or just before the newline at the end of the string, and in :const:`MULTILINE` mode also matches before a newline. ``foo`` matches both 'foo' and 'foobar', while the regular expression ``foo$`` matches @@ -112,28 +111,28 @@ The special characters are: a single ``$`` in ``'foo\n'`` will find two (empty) matches: one just before the newline, and one at the end of the string. -``'*'`` +``*`` Causes the resulting RE to match 0 or more repetitions of the preceding RE, as many repetitions as are possible. ``ab*`` will match 'a', 'ab', or 'a' followed by any number of 'b's. -``'+'`` +``+`` Causes the resulting RE to match 1 or more repetitions of the preceding RE. ``ab+`` will match 'a' followed by any non-zero number of 'b's; it will not match just 'a'. -``'?'`` +``?`` Causes the resulting RE to match 0 or 1 repetitions of the preceding RE. ``ab?`` will match either 'a' or 'ab'. ``*?``, ``+?``, ``??`` The ``'*'``, ``'+'``, and ``'?'`` qualifiers are all :dfn:`greedy`; they match as much text as possible. Sometimes this behaviour isn't desired; if the RE - ``<.*>`` is matched against `` b ``, it will match the entire - string, and not just ````. Adding ``?`` after the qualifier makes it + ``<.*>`` is matched against ``' b '``, it will match the entire + string, and not just ``''``. Adding ``?`` after the qualifier makes it perform the match in :dfn:`non-greedy` or :dfn:`minimal` fashion; as *few* characters as possible will be matched. Using the RE ``<.*?>`` will match - only ````. + only ``''``. ``{m}`` Specifies that exactly *m* copies of the previous RE should be matched; fewer @@ -145,8 +144,8 @@ The special characters are: RE, attempting to match as many repetitions as possible. For example, ``a{3,5}`` will match from 3 to 5 ``'a'`` characters. Omitting *m* specifies a lower bound of zero, and omitting *n* specifies an infinite upper bound. As an - example, ``a{4,}b`` will match ``aaaab`` or a thousand ``'a'`` characters - followed by a ``b``, but not ``aaab``. The comma may not be omitted or the + example, ``a{4,}b`` will match ``'aaaab'`` or a thousand ``'a'`` characters + followed by a ``'b'``, but not ``'aaab'``. The comma may not be omitted or the modifier would be confused with the previously described form. ``{m,n}?`` @@ -156,7 +155,7 @@ The special characters are: 6-character string ``'aaaaaa'``, ``a{3,5}`` will match 5 ``'a'`` characters, while ``a{3,5}?`` will only match 3 characters. -``'\'`` +``\`` Either escapes special characters (permitting you to match characters like ``'*'``, ``'?'``, and so forth), or signals a special sequence; special sequences are discussed below. @@ -179,8 +178,8 @@ The special characters are: them by a ``'-'``, for example ``[a-z]`` will match any lowercase ASCII letter, ``[0-5][0-9]`` will match all the two-digits numbers from ``00`` to ``59``, and ``[0-9A-Fa-f]`` will match any hexadecimal digit. If ``-`` is escaped (e.g. - ``[a\-z]``) or if it's placed as the first or last character (e.g. ``[a-]``), - it will match a literal ``'-'``. + ``[a\-z]``) or if it's placed as the first or last character + (e.g. ``[-a]`` or ``[a-]``), it will match a literal ``'-'``. * Special characters lose their special meaning inside sets. For example, ``[(+*)]`` will match any of the literal characters ``'('``, ``'+'``, @@ -201,13 +200,13 @@ The special characters are: place it at the beginning of the set. For example, both ``[()[\]{}]`` and ``[]()[{}]`` will both match a parenthesis. -``'|'`` - ``A|B``, where A and B can be arbitrary REs, creates a regular expression that - will match either A or B. An arbitrary number of REs can be separated by the +``|`` + ``A|B``, where *A* and *B* can be arbitrary REs, creates a regular expression that + will match either *A* or *B*. An arbitrary number of REs can be separated by the ``'|'`` in this way. This can be used inside groups (see below) as well. As the target string is scanned, REs separated by ``'|'`` are tried from left to right. When one pattern completely matches, that branch is accepted. This means - that once ``A`` matches, ``B`` will not be tested further, even if it would + that once *A* matches, *B* will not be tested further, even if it would produce a longer overall match. In other words, the ``'|'`` operator is never greedy. To match a literal ``'|'``, use ``\|``, or enclose it inside a character class, as in ``[|]``. @@ -217,7 +216,7 @@ The special characters are: start and end of a group; the contents of a group can be retrieved after a match has been performed, and can be matched later in the string with the ``\number`` special sequence, described below. To match the literals ``'('`` or ``')'``, - use ``\(`` or ``\)``, or enclose them inside a character class: ``[(] [)]``. + use ``\(`` or ``\)``, or enclose them inside a character class: ``[(]``, ``[)]``. ``(?...)`` This is an extension notation (a ``'?'`` following a ``'('`` is not meaningful @@ -232,10 +231,11 @@ The special characters are: letters set the corresponding flags: :const:`re.A` (ASCII-only matching), :const:`re.I` (ignore case), :const:`re.L` (locale dependent), :const:`re.M` (multi-line), :const:`re.S` (dot matches all), - and :const:`re.X` (verbose), for the entire regular expression. (The - flags are described in :ref:`contents-of-module-re`.) This - is useful if you wish to include the flags as part of the regular - expression, instead of passing a *flag* argument to the + :const:`re.U` (Unicode matching), and :const:`re.X` (verbose), + for the entire regular expression. + (The flags are described in :ref:`contents-of-module-re`.) + This is useful if you wish to include the flags as part of the + regular expression, instead of passing a *flag* argument to the :func:`re.compile` function. Flags should be used first in the expression string. @@ -272,10 +272,10 @@ The special characters are: | in the same pattern itself | * ``(?P=quote)`` (as shown) | | | * ``\1`` | +---------------------------------------+----------------------------------+ - | when processing match object ``m`` | * ``m.group('quote')`` | + | when processing match object *m* | * ``m.group('quote')`` | | | * ``m.end('quote')`` (etc.) | +---------------------------------------+----------------------------------+ - | in a string passed to the ``repl`` | * ``\g`` | + | in a string passed to the *repl* | * ``\g`` | | argument of ``re.sub()`` | * ``\g<1>`` | | | * ``\1`` | +---------------------------------------+----------------------------------+ @@ -289,18 +289,18 @@ The special characters are: ``(?=...)`` Matches if ``...`` matches next, but doesn't consume any of the string. This is - called a lookahead assertion. For example, ``Isaac (?=Asimov)`` will match + called a :dfn:`lookahead assertion`. For example, ``Isaac (?=Asimov)`` will match ``'Isaac '`` only if it's followed by ``'Asimov'``. ``(?!...)`` - Matches if ``...`` doesn't match next. This is a negative lookahead assertion. + Matches if ``...`` doesn't match next. This is a :dfn:`negative lookahead assertion`. For example, ``Isaac (?!Asimov)`` will match ``'Isaac '`` only if it's *not* followed by ``'Asimov'``. ``(?<=...)`` Matches if the current position in the string is preceded by a match for ``...`` that ends at the current position. This is called a :dfn:`positive lookbehind - assertion`. ``(?<=abc)def`` will find a match in ``abcdef``, since the + assertion`. ``(?<=abc)def`` will find a match in ``'abcdef'``, since the lookbehind will back up 3 characters and check if the contained pattern matches. The contained pattern must only match strings of some fixed length, meaning that ``abc`` or ``a|b`` are allowed, but ``a*`` and ``a{3,4}`` are not. Note that @@ -358,26 +358,26 @@ character ``'$'``. ``\b`` Matches the empty string, but only at the beginning or end of a word. - A word is defined as a sequence of Unicode alphanumeric or underscore - characters, so the end of a word is indicated by whitespace or a - non-alphanumeric, non-underscore Unicode character. Note that formally, + A word is defined as a sequence of word characters. Note that formally, ``\b`` is defined as the boundary between a ``\w`` and a ``\W`` character (or vice versa), or between ``\w`` and the beginning/end of the string. This means that ``r'\bfoo\b'`` matches ``'foo'``, ``'foo.'``, ``'(foo)'``, ``'bar foo baz'`` but not ``'foobar'`` or ``'foo3'``. - By default Unicode alphanumerics are the ones used, but this can be changed - by using the :const:`ASCII` flag. Inside a character range, ``\b`` - represents the backspace character, for compatibility with Python's string - literals. + By default Unicode alphanumerics are the ones used in Unicode patterns, but + this can be changed by using the :const:`ASCII` flag. Word boundaries are + determined by the current locale if the :const:`LOCALE` flag is used. + Inside a character range, ``\b`` represents the backspace character, for + compatibility with Python's string literals. ``\B`` Matches the empty string, but only when it is *not* at the beginning or end of a word. This means that ``r'py\B'`` matches ``'python'``, ``'py3'``, ``'py2'``, but not ``'py'``, ``'py.'``, or ``'py!'``. - ``\B`` is just the opposite of ``\b``, so word characters are - Unicode alphanumerics or the underscore, although this can be changed - by using the :const:`ASCII` flag. + ``\B`` is just the opposite of ``\b``, so word characters in Unicode + patterns are Unicode alphanumerics or the underscore, although this can + be changed by using the :const:`ASCII` flag. Word boundaries are + determined by the current locale if the :const:`LOCALE` flag is used. ``\d`` For Unicode (str) patterns: @@ -387,11 +387,12 @@ character ``'$'``. used only ``[0-9]`` is matched (but the flag affects the entire regular expression, so in such cases using an explicit ``[0-9]`` may be a better choice). + For 8-bit (bytes) patterns: Matches any decimal digit; this is equivalent to ``[0-9]``. ``\D`` - Matches any character which is not a Unicode decimal digit. This is + Matches any character which is not a decimal digit. This is the opposite of ``\d``. If the :const:`ASCII` flag is used this becomes the equivalent of ``[^0-9]`` (but the flag affects the entire regular expression, so in such cases using an explicit ``[^0-9]`` may @@ -412,7 +413,7 @@ character ``'$'``. this is equivalent to ``[ \t\n\r\f\v]``. ``\S`` - Matches any character which is not a Unicode whitespace character. This is + Matches any character which is not a whitespace character. This is the opposite of ``\s``. If the :const:`ASCII` flag is used this becomes the equivalent of ``[^ \t\n\r\f\v]`` (but the flag affects the entire regular expression, so in such cases using an explicit ``[^ \t\n\r\f\v]`` may @@ -426,16 +427,21 @@ character ``'$'``. ``[a-zA-Z0-9_]`` is matched (but the flag affects the entire regular expression, so in such cases using an explicit ``[a-zA-Z0-9_]`` may be a better choice). + For 8-bit (bytes) patterns: Matches characters considered alphanumeric in the ASCII character set; - this is equivalent to ``[a-zA-Z0-9_]``. + this is equivalent to ``[a-zA-Z0-9_]``. If the :const:`LOCALE` flag is + used, matches characters considered alphanumeric in the current locale + and the underscore. ``\W`` - Matches any character which is not a Unicode word character. This is + Matches any character which is not a word character. This is the opposite of ``\w``. If the :const:`ASCII` flag is used this becomes the equivalent of ``[^a-zA-Z0-9_]`` (but the flag affects the entire regular expression, so in such cases using an explicit - ``[^a-zA-Z0-9_]`` may be a better choice). + ``[^a-zA-Z0-9_]`` may be a better choice). If the :const:`LOCALE` flag is + used, matches characters considered alphanumeric in the current locale + and the underscore. ``\Z`` Matches only at the end of the string. @@ -451,7 +457,7 @@ accepted by the regular expression parser:: only inside character classes.) ``'\u'`` and ``'\U'`` escape sequences are only recognized in Unicode -patterns. In bytes patterns they are not treated specially. +patterns. In bytes patterns they are errors. Octal escapes are included in a limited form. If the first digit is a 0, or if there are three octal digits, it is considered an octal escape. Otherwise, it is @@ -526,6 +532,7 @@ form. Make ``\w``, ``\W``, ``\b``, ``\B``, ``\d``, ``\D``, ``\s`` and ``\S`` perform ASCII-only matching instead of full Unicode matching. This is only meaningful for Unicode patterns, and is ignored for byte patterns. + Corresponds to the inline flag ``(?a)``. Note that for backward compatibility, the :const:`re.U` flag still exists (as well as its synonym :const:`re.UNICODE` and its embedded @@ -537,26 +544,40 @@ form. .. data:: DEBUG Display debug information about compiled expression. + No corresponding inline flag. .. data:: I IGNORECASE Perform case-insensitive matching; expressions like ``[A-Z]`` will also - match lowercase letters. The current locale does not change the effect of - this flag. Full Unicode matching (such as ``Ü`` matching ``ü``) also - works unless the :const:`re.ASCII` flag is also used to disable non-ASCII - matches. - + match lowercase letters. Full Unicode matching (such as ``Ü`` matching + ``ü``) also works unless the :const:`re.ASCII` flag is used to disable + non-ASCII matches. The current locale does not change the effect of this + flag unless the :const:`re.LOCALE` flag is also used. + Corresponds to the inline flag ``(?i)``. + + Note that when the Unicode patterns ``[a-z]`` or ``[A-Z]`` are used in + combination with the :const:`IGNORECASE` flag, they will match the 52 ASCII + letters and 4 additional non-ASCII letters: 'İ' (U+0130, Latin capital + letter I with dot above), 'ı' (U+0131, Latin small letter dotless i), + 'ſ' (U+017F, Latin small letter long s) and 'K' (U+212A, Kelvin sign). + If the :const:`ASCII` flag is used, only letters 'a' to 'z' + and 'A' to 'Z' are matched (but the flag affects the entire regular + expression, so in such cases using an explicit ``(?-i:[a-zA-Z])`` may be + a better choice). .. data:: L LOCALE - Make ``\w``, ``\W``, ``\b``, ``\B``, ``\s`` and ``\S`` dependent on the - current locale. The use of this flag is discouraged as the locale mechanism - is very unreliable, and it only handles one "culture" at a time anyway; - you should use Unicode matching instead, which is the default in Python 3 - for Unicode (str) patterns. This flag can be used only with bytes patterns. + Make ``\w``, ``\W``, ``\b``, ``\B`` and case-insensitive matching + dependent on the current locale. This flag can be used only with bytes + patterns. The use of this flag is discouraged as the locale mechanism + is very unreliable, it only handles one "culture" at a time, and it only + works with 8-bit locales. Unicode matching is already enabled by default + in Python 3 for Unicode (str) patterns, and it is able to handle different + locales/languages. + Corresponds to the inline flag ``(?L)``. .. versionchanged:: 3.6 :const:`re.LOCALE` can be used only with bytes patterns and is @@ -572,6 +593,7 @@ form. end of each line (immediately preceding each newline). By default, ``'^'`` matches only at the beginning of the string, and ``'$'`` only at the end of the string and immediately before the newline (if any) at the end of the string. + Corresponds to the inline flag ``(?m)``. .. data:: S @@ -579,6 +601,7 @@ form. Make the ``'.'`` special character match any character at all, including a newline; without this flag, ``'.'`` will match anything *except* a newline. + Corresponds to the inline flag ``(?s)``. .. data:: X @@ -587,7 +610,8 @@ form. This flag allows you to write regular expressions that look nicer and are more readable by allowing you to visually separate logical sections of the pattern and add comments. Whitespace within the pattern is ignored, except - when in a character class or when preceded by an unescaped backslash. + when in a character class, or when preceded by an unescaped backslash, + or within tokens like ``*?``, ``(?:`` or ``(?P<...>``. When a line contains a ``#`` that is not in a character class and is not preceded by an unescaped backslash, all characters from the leftmost such ``#`` through the end of the line are ignored. @@ -600,7 +624,7 @@ form. \d * # some fractional digits""", re.X) b = re.compile(r"\d+\.\d*") - + Corresponds to the inline flag ``(?x)``. .. function:: search(pattern, string, flags=0) @@ -644,20 +668,20 @@ form. splits occur, and the remainder of the string is returned as the final element of the list. :: - >>> re.split('\W+', 'Words, words, words.') + >>> re.split(r'\W+', 'Words, words, words.') ['Words', 'words', 'words', ''] - >>> re.split('(\W+)', 'Words, words, words.') + >>> re.split(r'(\W+)', 'Words, words, words.') ['Words', ', ', 'words', ', ', 'words', '.', ''] - >>> re.split('\W+', 'Words, words, words.', 1) + >>> re.split(r'\W+', 'Words, words, words.', 1) ['Words', 'words, words.'] >>> re.split('[a-f]+', '0a3B9', flags=re.IGNORECASE) ['0', '3', '9'] If there are capturing groups in the separator and it matches at the start of the string, the result will start with an empty string. The same holds for - the end of the string: + the end of the string:: - >>> re.split('(\W+)', '...words, words...') + >>> re.split(r'(\W+)', '...words, words...') ['', '...', 'words', ', ', 'words', '...', ''] That way, separator components are always found at the same relative @@ -666,7 +690,7 @@ form. .. note:: :func:`split` doesn't currently split a string on an empty pattern match. - For example: + For example:: >>> re.split('x*', 'axbc') ['a', 'bc'] @@ -723,7 +747,7 @@ form. converted to a single newline character, ``\r`` is converted to a carriage return, and so forth. Unknown escapes such as ``\&`` are left alone. Backreferences, such as ``\6``, are replaced with the substring matched by group 6 in the pattern. - For example: + For example:: >>> re.sub(r'def\s+([a-zA-Z_][a-zA-Z_0-9]*)\s*\(\s*\):', ... r'static PyObject*\npy_\1(void)\n{', @@ -731,8 +755,8 @@ form. 'static PyObject*\npy_myfunc(void)\n{' If *repl* is a function, it is called for every non-overlapping occurrence of - *pattern*. The function takes a single match object argument, and returns the - replacement string. For example: + *pattern*. The function takes a single :ref:`match object ` + argument, and returns the replacement string. For example:: >>> def dashrepl(matchobj): ... if matchobj.group(0) == '-': return ' ' @@ -742,7 +766,7 @@ form. >>> re.sub(r'\sAND\s', ' & ', 'Baked Beans And Spam', flags=re.IGNORECASE) 'Baked Beans & Spam' - The pattern may be a string or an RE object. + The pattern may be a string or a :ref:`pattern object `. The optional argument *count* is the maximum number of pattern occurrences to be replaced; *count* must be a non-negative integer. If omitted or zero, all @@ -804,6 +828,14 @@ form. >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) \/|\-|\+|\*\*|\* + This functions must not be used for the replacement string in :func:`sub` + and :func:`subn`, only backslashes should be escaped. For example:: + + >>> digits_re = r'\d+' + >>> sample = '/usr/sbin/sendmail - 0 errors, 12 warnings' + >>> print(re.sub(digits_re, digits_re.replace('\\', r'\\'), sample)) + /usr/sbin/sendmail - \d+ errors, \d+ warnings + .. versionchanged:: 3.3 The ``'_'`` character is no longer escaped. @@ -871,12 +903,12 @@ attributes: from *pos* to ``endpos - 1`` will be searched for a match. If *endpos* is less than *pos*, no match will be found; otherwise, if *rx* is a compiled regular expression object, ``rx.search(string, 0, 50)`` is equivalent to - ``rx.search(string[:50], 0)``. + ``rx.search(string[:50], 0)``. :: - >>> pattern = re.compile("d") - >>> pattern.search("dog") # Match at index 0 - <_sre.SRE_Match object; span=(0, 1), match='d'> - >>> pattern.search("dog", 1) # No match; search doesn't include the "d" + >>> pattern = re.compile("d") + >>> pattern.search("dog") # Match at index 0 + <_sre.SRE_Match object; span=(0, 1), match='d'> + >>> pattern.search("dog", 1) # No match; search doesn't include the "d" .. method:: regex.match(string[, pos[, endpos]]) @@ -887,12 +919,12 @@ attributes: different from a zero-length match. The optional *pos* and *endpos* parameters have the same meaning as for the - :meth:`~regex.search` method. + :meth:`~regex.search` method. :: - >>> pattern = re.compile("o") - >>> pattern.match("dog") # No match as "o" is not at the start of "dog". - >>> pattern.match("dog", 1) # Match as "o" is the 2nd character of "dog". - <_sre.SRE_Match object; span=(1, 2), match='o'> + >>> pattern = re.compile("o") + >>> pattern.match("dog") # No match as "o" is not at the start of "dog". + >>> pattern.match("dog", 1) # Match as "o" is the 2nd character of "dog". + <_sre.SRE_Match object; span=(1, 2), match='o'> If you want to locate a match anywhere in *string*, use :meth:`~regex.search` instead (see also :ref:`search-vs-match`). @@ -905,13 +937,13 @@ attributes: match the pattern; note that this is different from a zero-length match. The optional *pos* and *endpos* parameters have the same meaning as for the - :meth:`~regex.search` method. + :meth:`~regex.search` method. :: - >>> pattern = re.compile("o[gh]") - >>> pattern.fullmatch("dog") # No match as "o" is not at the start of "dog". - >>> pattern.fullmatch("ogre") # No match as not the full string matches. - >>> pattern.fullmatch("doggie", 1, 3) # Matches within given limits. - <_sre.SRE_Match object; span=(1, 3), match='og'> + >>> pattern = re.compile("o[gh]") + >>> pattern.fullmatch("dog") # No match as "o" is not at the start of "dog". + >>> pattern.fullmatch("ogre") # No match as not the full string matches. + >>> pattern.fullmatch("doggie", 1, 3) # Matches within given limits. + <_sre.SRE_Match object; span=(1, 3), match='og'> .. versionadded:: 3.4 @@ -925,14 +957,14 @@ attributes: Similar to the :func:`findall` function, using the compiled pattern, but also accepts optional *pos* and *endpos* parameters that limit the search - region like for :meth:`match`. + region like for :meth:`search`. .. method:: regex.finditer(string[, pos[, endpos]]) Similar to the :func:`finditer` function, using the compiled pattern, but also accepts optional *pos* and *endpos* parameters that limit the search - region like for :meth:`match`. + region like for :meth:`search`. .. method:: regex.sub(repl, string, count=0) @@ -1010,7 +1042,7 @@ Match objects support the following methods and attributes: pattern, an :exc:`IndexError` exception is raised. If a group is contained in a part of the pattern that did not match, the corresponding result is ``None``. If a group is contained in a part of the pattern that matched multiple times, - the last match is returned. + the last match is returned. :: >>> m = re.match(r"(\w+) (\w+)", "Isaac Newton, physicist") >>> m.group(0) # The entire match @@ -1027,7 +1059,7 @@ Match objects support the following methods and attributes: string argument is not used as a group name in the pattern, an :exc:`IndexError` exception is raised. - A moderately complicated example: + A moderately complicated example:: >>> m = re.match(r"(?P\w+) (?P\w+)", "Malcolm Reynolds") >>> m.group('first_name') @@ -1035,14 +1067,14 @@ Match objects support the following methods and attributes: >>> m.group('last_name') 'Reynolds' - Named groups can also be referred to by their index: + Named groups can also be referred to by their index:: >>> m.group(1) 'Malcolm' >>> m.group(2) 'Reynolds' - If a group matches multiple times, only the last match is accessible: + If a group matches multiple times, only the last match is accessible:: >>> m = re.match(r"(..)+", "a1b2c3") # Matches 3 times. >>> m.group(1) # Returns only the last match. @@ -1052,7 +1084,7 @@ Match objects support the following methods and attributes: .. method:: match.__getitem__(g) This is identical to ``m.group(g)``. This allows easier access to - an individual group from a match: + an individual group from a match:: >>> m = re.match(r"(\w+) (\w+)", "Isaac Newton, physicist") >>> m[0] # The entire match @@ -1071,7 +1103,7 @@ Match objects support the following methods and attributes: many groups are in the pattern. The *default* argument is used for groups that did not participate in the match; it defaults to ``None``. - For example: + For example:: >>> m = re.match(r"(\d+)\.(\d+)", "24.1632") >>> m.groups() @@ -1079,7 +1111,7 @@ Match objects support the following methods and attributes: If we make the decimal place and everything after it optional, not all groups might participate in the match. These groups will default to ``None`` unless - the *default* argument is given: + the *default* argument is given:: >>> m = re.match(r"(\d+)\.?(\d+)?", "24") >>> m.groups() # Second group defaults to None. @@ -1092,7 +1124,7 @@ Match objects support the following methods and attributes: Return a dictionary containing all the *named* subgroups of the match, keyed by the subgroup name. The *default* argument is used for groups that did not - participate in the match; it defaults to ``None``. For example: + participate in the match; it defaults to ``None``. For example:: >>> m = re.match(r"(?P\w+) (?P\w+)", "Malcolm Reynolds") >>> m.groupdict() @@ -1115,7 +1147,7 @@ Match objects support the following methods and attributes: ``m.start(0)`` is 1, ``m.end(0)`` is 2, ``m.start(1)`` and ``m.end(1)`` are both 2, and ``m.start(2)`` raises an :exc:`IndexError` exception. - An example that will remove *remove_this* from email addresses: + An example that will remove *remove_this* from email addresses:: >>> email = "tony@tiremove_thisger.net" >>> m = re.search("remove_this", email) @@ -1161,7 +1193,7 @@ Match objects support the following methods and attributes: .. attribute:: match.re - The regular expression object whose :meth:`~regex.match` or + The :ref:`regular expression object ` whose :meth:`~regex.match` or :meth:`~regex.search` method produced this match instance. @@ -1194,7 +1226,7 @@ a 5-character string with each character representing a card, "a" for ace, "k" for king, "q" for queen, "j" for jack, "t" for 10, and "2" through "9" representing the card with that value. -To see if a given string is a valid hand, one could do the following: +To see if a given string is a valid hand, one could do the following:: >>> valid = re.compile(r"^[a2-9tjqk]{5}$") >>> displaymatch(valid.match("akt5q")) # Valid. @@ -1205,7 +1237,7 @@ To see if a given string is a valid hand, one could do the following: "" That last hand, ``"727ak"``, contained a pair, or two of the same valued cards. -To match this with a regular expression, one could use backreferences as such: +To match this with a regular expression, one could use backreferences as such:: >>> pair = re.compile(r".*(.).*\1") >>> displaymatch(pair.match("717ak")) # Pair of 7s. @@ -1307,7 +1339,7 @@ restrict the match at the beginning of the string:: Note however that in :const:`MULTILINE` mode :func:`match` only matches at the beginning of the string, whereas using :func:`search` with a regular expression -beginning with ``'^'`` will match at the beginning of each line. +beginning with ``'^'`` will match at the beginning of each line. :: >>> re.match('X', 'A\nB\nX', re.MULTILINE) # No match >>> re.search('^X', 'A\nB\nX', re.MULTILINE) # Match @@ -1323,7 +1355,7 @@ easily read and modified by Python as demonstrated in the following example that creates a phonebook. First, here is the input. Normally it may come from a file, here we are using -triple-quoted string syntax: +triple-quoted string syntax:: >>> text = """Ross McFluff: 834.345.1254 155 Elm Street ... @@ -1398,7 +1430,7 @@ Finding all Adverbs :func:`findall` matches *all* occurrences of a pattern, not just the first one as :func:`search` does. For example, if one was a writer and wanted to find all of the adverbs in some text, he or she might use :func:`findall` in -the following manner: +the following manner:: >>> text = "He was carefully disguised but captured quickly by police." >>> re.findall(r"\w+ly", text) @@ -1412,7 +1444,7 @@ If one wants more information about all matches of a pattern than the matched text, :func:`finditer` is useful as it provides :ref:`match objects ` instead of strings. Continuing with the previous example, if one was a writer who wanted to find all of the adverbs *and their positions* in -some text, he or she would use :func:`finditer` in the following manner: +some text, he or she would use :func:`finditer` in the following manner:: >>> text = "He was carefully disguised but captured quickly by police." >>> for m in re.finditer(r"\w+ly", text): @@ -1427,7 +1459,7 @@ Raw String Notation Raw string notation (``r"text"``) keeps regular expressions sane. Without it, every backslash (``'\'``) in a regular expression would have to be prefixed with another one to escape it. For example, the two following lines of code are -functionally identical: +functionally identical:: >>> re.match(r"\W(.)\1\W", " ff ") <_sre.SRE_Match object; span=(0, 4), match=' ff '> @@ -1437,7 +1469,7 @@ functionally identical: When one wants to match a literal backslash, it must be escaped in the regular expression. With raw string notation, this means ``r"\\"``. Without raw string notation, one must use ``"\\\\"``, making the following lines of code -functionally identical: +functionally identical:: >>> re.match(r"\\", r"\\") <_sre.SRE_Match object; span=(0, 1), match='\\'> diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 54c54b46..837816e9 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -312,13 +312,13 @@ sessions, by only appending the new history. :: try: readline.read_history_file(histfile) - h_len = readline.get_history_length() + h_len = readline.get_current_history_length() except FileNotFoundError: open(histfile, 'wb').close() h_len = 0 def save(prev_h_len, histfile): - new_h_len = readline.get_history_length() + new_h_len = readline.get_current_history_length() readline.set_history_length(1000) readline.append_history_file(new_h_len - prev_h_len, histfile) atexit.register(save, h_len, histfile) diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 4d4a6161..6094a7b8 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -69,7 +69,7 @@ Scheduler Objects Schedule a new event. The *time* argument should be a numeric type compatible with the return value of the *timefunc* function passed to the constructor. Events scheduled for the same *time* will be executed in the order of their - *priority*. + *priority*. A lower number represents a higher priority. Executing the event means executing ``action(*argument, **kwargs)``. *argument* is a sequence holding the positional arguments for *action*. diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 218a31c8..e12c8c97 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -289,7 +289,7 @@ Server Objects .. XXX should the default implementations of these be documented, or should it be assumed that the user will look at socketserver.py? - .. method:: finish_request() + .. method:: finish_request(request, client_address) Actually processes the request by instantiating :attr:`RequestHandlerClass` and calling its :meth:`~BaseRequestHandler.handle` method. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 9fef7d7f..ef0c0bf6 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -421,6 +421,10 @@ Connection Objects If you want to clear any previously installed progress handler, call the method with :const:`None` for *handler*. + Returning a non-zero value from the handler function will terminate the + currently executing query and cause it to raise an :exc:`OperationalError` + exception. + .. method:: set_trace_callback(trace_callback) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index d7e04672..495cd590 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -218,7 +218,7 @@ instead. The *ciphers* parameter sets the available ciphers for this SSL object. It should be a string in the `OpenSSL cipher list format - `_. + `_. The parameter ``do_handshake_on_connect`` specifies whether to do the SSL handshake automatically after doing a :meth:`socket.connect`, or whether the @@ -1445,7 +1445,7 @@ to speed up repeated connections from the same clients. Set the available ciphers for sockets created with this context. It should be a string in the `OpenSSL cipher list format - `_. + `_. If no cipher can be selected (because compile-time options or other configuration forbids use of all the specified ciphers), an :class:`SSLError` will be raised. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index b8c4d593..75e97b9d 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2564,8 +2564,9 @@ arbitrary binary data. bytearray.partition(sep) Split the sequence at the first occurrence of *sep*, and return a 3-tuple - containing the part before the separator, the separator, and the part - after the separator. If the separator is not found, return a 3-tuple + containing the part before the separator, the separator itself or its + bytearray copy, and the part after the separator. + If the separator is not found, return a 3-tuple containing a copy of the original sequence, followed by two empty bytes or bytearray objects. @@ -2620,8 +2621,9 @@ arbitrary binary data. bytearray.rpartition(sep) Split the sequence at the last occurrence of *sep*, and return a 3-tuple - containing the part before the separator, the separator, and the part - after the separator. If the separator is not found, return a 3-tuple + containing the part before the separator, the separator itself or its + bytearray copy, and the part after the separator. + If the separator is not found, return a 3-tuple containing a copy of the original sequence, followed by two empty bytes or bytearray objects. diff --git a/Doc/library/string.rst b/Doc/library/string.rst index a0977b64..7a9fcc38 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -746,8 +746,18 @@ to parse template strings. To do this, you can override these class attributes: * *idpattern* -- This is the regular expression describing the pattern for non-braced placeholders (the braces will be added automatically as - appropriate). The default value is the regular expression - ``[_a-z][_a-z0-9]*``. + appropriate). The default value is the regular expression + ``(?-i:[_a-zA-Z][_a-zA-Z0-9]*)``. + + .. note:: + + Since default *flags* is ``re.IGNORECASE``, pattern ``[a-z]`` can match + with some non-ASCII characters. That's why we use local ``-i`` flag here. + + While *flags* is kept to ``re.IGNORECASE`` for backward compatibility, + you can override it to ``0`` or ``re.IGNORECASE | re.ASCII`` when + subclassing. + * *flags* -- The regular expression flags that will be applied when compiling the regular expression used for recognizing substitutions. The default value diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 27f3e825..ea7e664f 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -584,7 +584,7 @@ Instances of the :class:`Popen` class have the following methods: .. method:: Popen.poll() Check if child process has terminated. Set and return - :attr:`~Popen.returncode` attribute. + :attr:`~Popen.returncode` attribute. Otherwise, returns ``None``. .. method:: Popen.wait(timeout=None) diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index ad6a9f7c..7693ecd0 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -12,7 +12,7 @@ -------------- This module provides an interface to the POSIX calls for tty I/O control. For a -complete description of these calls, see :manpage:`termios(2)` Unix manual +complete description of these calls, see :manpage:`termios(3)` Unix manual page. It is only available for those Unix versions that support POSIX *termios* style tty I/O control configured during installation. diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 9d4ff7ad..1a3f8f9c 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -440,7 +440,7 @@ The :mod:`test.support` module defines the following functions: otherwise. -.. decorator:: skip_unless_symlink() +.. decorator:: skip_unless_symlink A decorator for running tests that require support for symbolic links. diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 021e29e7..c3757546 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -291,10 +291,10 @@ since it is impossible to detect the termination of alien threads. .. attribute:: ident The 'thread identifier' of this thread or ``None`` if the thread has not - been started. This is a nonzero integer. See the - :func:`_thread.get_ident()` function. Thread identifiers may be recycled - when a thread exits and another thread is created. The identifier is - available even after the thread has exited. + been started. This is a nonzero integer. See the :func:`get_ident` + function. Thread identifiers may be recycled when a thread exits and + another thread is created. The identifier is available even after the + thread has exited. .. method:: is_alive() diff --git a/Doc/library/time.rst b/Doc/library/time.rst index d2e2ac44..0624099f 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -118,14 +118,10 @@ An explanation of some terminology and conventions is in order. +-------------------------+-------------------------+-------------------------+ -The module defines the following functions and data items: - -.. data:: altzone - - The offset of the local DST timezone, in seconds west of UTC, if one is defined. - This is negative if the local DST timezone is east of UTC (as in Western Europe, - including the UK). Only use this if ``daylight`` is nonzero. +.. _time-functions: +Functions +--------- .. function:: asctime([t]) @@ -165,7 +161,8 @@ The module defines the following functions and data items: .. function:: clock_getres(clk_id) - Return the resolution (precision) of the specified clock *clk_id*. + Return the resolution (precision) of the specified clock *clk_id*. Refer to + :ref:`time-clock-id-constants` for a list of accepted values for *clk_id*. Availability: Unix. @@ -174,7 +171,8 @@ The module defines the following functions and data items: .. function:: clock_gettime(clk_id) - Return the time of the specified clock *clk_id*. + Return the time of the specified clock *clk_id*. Refer to + :ref:`time-clock-id-constants` for a list of accepted values for *clk_id*. Availability: Unix. @@ -183,66 +181,8 @@ The module defines the following functions and data items: .. function:: clock_settime(clk_id, time) - Set the time of the specified clock *clk_id*. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_HIGHRES - - The Solaris OS has a CLOCK_HIGHRES timer that attempts to use an optimal - hardware source, and may give close to nanosecond resolution. CLOCK_HIGHRES - is the nonadjustable, high-resolution clock. - - Availability: Solaris. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_MONOTONIC - - Clock that cannot be set and represents monotonic time since some unspecified - starting point. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_MONOTONIC_RAW - - Similar to :data:`CLOCK_MONOTONIC`, but provides access to a raw - hardware-based time that is not subject to NTP adjustments. - - Availability: Linux 2.6.28 or later. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_PROCESS_CPUTIME_ID - - High-resolution per-process timer from the CPU. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_REALTIME - - System-wide real-time clock. Setting this clock requires appropriate - privileges. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_THREAD_CPUTIME_ID - - Thread-specific CPU-time clock. + Set the time of the specified clock *clk_id*. Currently, + :data:`CLOCK_REALTIME` is the only accepted value for *clk_id*. Availability: Unix. @@ -257,11 +197,6 @@ The module defines the following functions and data items: ``asctime(localtime(secs))``. Locale information is not used by :func:`ctime`. -.. data:: daylight - - Nonzero if a DST timezone is defined. - - .. function:: get_clock_info(name) Get information on the specified clock as a namespace object. @@ -279,7 +214,7 @@ The module defines the following functions and data items: - *adjustable*: ``True`` if the clock can be changed automatically (e.g. by a NTP daemon) or manually by the system administrator, ``False`` otherwise - *implementation*: The name of the underlying C function used to get - the clock value + the clock value. Refer to :ref:`time-clock-id-constants` for possible values. - *monotonic*: ``True`` if the clock cannot go backward, ``False`` otherwise - *resolution*: The resolution of the clock in seconds (:class:`float`) @@ -607,18 +542,6 @@ The module defines the following functions and data items: :class:`struct_time` object is returned, from which the components of the calendar date may be accessed as attributes. -.. data:: timezone - - The offset of the local (non-DST) timezone, in seconds west of UTC (negative in - most of Western Europe, positive in the US, zero in the UK). - - -.. data:: tzname - - A tuple of two strings: the first is the name of the local non-DST timezone, the - second is the name of the local DST timezone. If no DST timezone is defined, - the second string should not be used. - .. function:: tzset() @@ -703,6 +626,111 @@ The module defines the following functions and data items: ('EET', 'EEST') +.. _time-clock-id-constants: + +Clock ID Constants +------------------ + +These constants are used as parameters for :func:`clock_getres` and +:func:`clock_gettime`. + +.. data:: CLOCK_HIGHRES + + The Solaris OS has a ``CLOCK_HIGHRES`` timer that attempts to use an optimal + hardware source, and may give close to nanosecond resolution. + ``CLOCK_HIGHRES`` is the nonadjustable, high-resolution clock. + + Availability: Solaris. + + .. versionadded:: 3.3 + + +.. data:: CLOCK_MONOTONIC + + Clock that cannot be set and represents monotonic time since some unspecified + starting point. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. data:: CLOCK_MONOTONIC_RAW + + Similar to :data:`CLOCK_MONOTONIC`, but provides access to a raw + hardware-based time that is not subject to NTP adjustments. + + Availability: Linux 2.6.28 or later. + + .. versionadded:: 3.3 + + +.. data:: CLOCK_PROCESS_CPUTIME_ID + + High-resolution per-process timer from the CPU. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. data:: CLOCK_THREAD_CPUTIME_ID + + Thread-specific CPU-time clock. + + Availability: Unix. + + .. versionadded:: 3.3 + + +The following constant is the only parameter that can be sent to +:func:`clock_settime`. + +.. data:: CLOCK_REALTIME + + System-wide real-time clock. Setting this clock requires appropriate + privileges. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. _time-timezone-constants: + +Timezone Constants +------------------- + +.. data:: altzone + + The offset of the local DST timezone, in seconds west of UTC, if one is defined. + This is negative if the local DST timezone is east of UTC (as in Western Europe, + including the UK). Only use this if ``daylight`` is nonzero. See note below. + +.. data:: daylight + + Nonzero if a DST timezone is defined. See note below. + +.. data:: timezone + + The offset of the local (non-DST) timezone, in seconds west of UTC (negative in + most of Western Europe, positive in the US, zero in the UK). See note below. + +.. data:: tzname + + A tuple of two strings: the first is the name of the local non-DST timezone, the + second is the name of the local DST timezone. If no DST timezone is defined, + the second string should not be used. See note below. + +.. note:: + + For the above Timezone constants (:data:`altzone`, :data:`daylight`, :data:`timezone`, + and :data:`tzname`), the value is determined by the timezone rules in effect + at module load time or the last time :func:`tzset` is called and may be incorrect + for times in the past. It is recommended to use the :attr:`tm_gmtoff` and + :attr:`tm_zone` results from :func:`localtime` to obtain timezone information. + + .. seealso:: Module :mod:`datetime` diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 3e1faed8..f51add2b 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -154,7 +154,7 @@ background material, while the second half can be taken to the keyboard as a handy reference. When trying to answer questions of the form "how do I do blah", it is often best -to find out how to do"blah" in straight Tk, and then convert this back into the +to find out how to do "blah" in straight Tk, and then convert this back into the corresponding :mod:`tkinter` call. Python programmers can often guess at the correct Python command by looking at the Tk documentation. This means that in order to use Tkinter, you will have to know a little bit about Tk. This document diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 3dad182c..927f7f3c 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -1099,26 +1099,42 @@ ttk.Treeview If *selop* is not specified, returns selected items. Otherwise, it will act according to the following selection methods. + .. deprecated-removed:: 3.6 3.8 + Using ``selection()`` for changing the selection state is deprecated. + Use the following selection methods instead. - .. method:: selection_set(items) + + .. method:: selection_set(*items) *items* becomes the new selection. + .. versionchanged:: 3.6 + *items* can be passed as separate arguments, not just as a single tuple. + - .. method:: selection_add(items) + .. method:: selection_add(*items) Add *items* to the selection. + .. versionchanged:: 3.6 + *items* can be passed as separate arguments, not just as a single tuple. + - .. method:: selection_remove(items) + .. method:: selection_remove(*items) Remove *items* from the selection. + .. versionchanged:: 3.6 + *items* can be passed as separate arguments, not just as a single tuple. - .. method:: selection_toggle(items) + + .. method:: selection_toggle(*items) Toggle the selection state of each item in *items*. + .. versionchanged:: 3.6 + *items* can be passed as separate arguments, not just as a single tuple. + .. method:: set(item, column=None, value=None) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 1e48fecd..9c4777ac 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -111,8 +111,7 @@ More precisely, the expression ``some_value is Derived(some_value)`` is always true at runtime. This also means that it is not possible to create a subtype of ``Derived`` -since it is an identity function at runtime, not an actual type. Similarly, it -is not possible to create another :func:`NewType` based on a ``Derived`` type:: +since it is an identity function at runtime, not an actual type:: from typing import NewType @@ -121,9 +120,16 @@ is not possible to create another :func:`NewType` based on a ``Derived`` type:: # Fails at runtime and does not typecheck class AdminUserId(UserId): pass - # Also does not typecheck +However, it is possible to create a :func:`NewType` based on a 'derived' ``NewType``:: + + from typing import NewType + + UserId = NewType('UserId', int) + ProUserId = NewType('ProUserId', UserId) +and typechecking for ``ProUserId`` will work as expected. + See :pep:`484` for more details. .. note:: @@ -140,6 +146,8 @@ See :pep:`484` for more details. ``Derived`` is expected. This is useful when you want to prevent logic errors with minimal runtime cost. +.. versionadded:: 3.5.2 + Callable -------- @@ -488,6 +496,8 @@ The module defines the following classes, functions and decorators: ``Type[Any]`` is equivalent to ``Type`` which in turn is equivalent to ``type``, which is the root of Python's metaclass hierarchy. + .. versionadded:: 3.5.2 + .. class:: Iterable(Generic[T_co]) A generic version of :class:`collections.abc.Iterable`. @@ -668,6 +678,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.defaultdict`. + .. versionadded:: 3.5.2 + .. class:: Counter(collections.Counter, Dict[T, int]) A generic version of :class:`collections.Counter`. @@ -756,13 +768,15 @@ The module defines the following classes, functions and decorators: def add_unicode_checkmark(text: Text) -> Text: return text + u' \u2713' + .. versionadded:: 3.5.2 + .. class:: io Wrapper namespace for I/O stream types. - This defines the generic type ``IO[AnyStr]`` and aliases ``TextIO`` - and ``BinaryIO`` for respectively ``IO[str]`` and ``IO[bytes]``. - These represent the types of I/O streams such as returned by + This defines the generic type ``IO[AnyStr]`` and subclasses ``TextIO`` + and ``BinaryIO``, deriving from ``IO[str]`` and ``IO[bytes]``, + respectively. These represent the types of I/O streams such as returned by :func:`open`. These types are also accessible directly as ``typing.IO``, @@ -841,6 +855,8 @@ The module defines the following classes, functions and decorators: UserId = NewType('UserId', int) first_user = UserId(1) + .. versionadded:: 3.5.2 + .. function:: cast(typ, val) Cast a value to a type. @@ -891,17 +907,17 @@ The module defines the following classes, functions and decorators: See :pep:`484` for details and comparison with other typing semantics. -.. decorator:: no_type_check(arg) +.. decorator:: no_type_check Decorator to indicate that annotations are not type hints. - The argument must be a class or function; if it is a class, it + This works as class or function :term:`decorator`. With a class, it applies recursively to all methods defined in that class (but not to methods defined in its superclasses or subclasses). This mutates the function(s) in place. -.. decorator:: no_type_check_decorator(decorator) +.. decorator:: no_type_check_decorator Decorator to give another decorator the :func:`no_type_check` effect. @@ -1048,3 +1064,5 @@ The module defines the following classes, functions and decorators: "forward reference", to hide the ``expensive_mod`` reference from the interpreter runtime. Type annotations for local variables are not evaluated, so the second annotation does not need to be enclosed in quotes. + + .. versionadded:: 3.5.2 diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 2099bd1e..e52f1400 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -402,10 +402,14 @@ you can do it yourself:: def suite(): suite = unittest.TestSuite() - suite.addTest(WidgetTestCase('test_default_size')) - suite.addTest(WidgetTestCase('test_resize')) + suite.addTest(WidgetTestCase('test_default_widget_size')) + suite.addTest(WidgetTestCase('test_widget_resize')) return suite + if __name__ == '__main__': + runner = unittest.TextTestRunner() + runner.run(suite()) + You can place the definitions of test cases and test suites in the same modules as the code they are to test (such as :file:`widget.py`), but there are several advantages to placing the test code in a separate module, such as diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 7d31932f..e3b90e67 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -69,10 +69,10 @@ structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. .. method:: request_rate(useragent) Returns the contents of the ``Request-rate`` parameter from - ``robots.txt`` in the form of a :func:`~collections.namedtuple` - ``(requests, seconds)``. If there is no such parameter or it doesn't - apply to the *useragent* specified or the ``robots.txt`` entry for this - parameter has invalid syntax, return ``None``. + ``robots.txt`` as a :term:`named tuple` ``RequestRate(requests, seconds)``. + If there is no such parameter or it doesn't apply to the *useragent* + specified or the ``robots.txt`` entry for this parameter has invalid + syntax, return ``None``. .. versionadded:: 3.6 diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 4b425a48..dca93624 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -773,7 +773,6 @@ Is semantically equivalent to:: mgr = (EXPR) aexit = type(mgr).__aexit__ aenter = type(mgr).__aenter__(mgr) - exc = True VAR = await aenter try: diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index d92be975..ff890a81 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1692,8 +1692,8 @@ precedence and have a left-to-right chaining feature as described in the | ``+``, ``-`` | Addition and subtraction | +-----------------------------------------------+-------------------------------------+ | ``*``, ``@``, ``/``, ``//``, ``%`` | Multiplication, matrix | -| | multiplication division, | -| | remainder [#]_ | +| | multiplication, division, floor | +| | division, remainder [#]_ | +-----------------------------------------------+-------------------------------------+ | ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT | +-----------------------------------------------+-------------------------------------+ diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 8cf16cad..7fbf8ed9 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -519,8 +519,9 @@ and the loader that executes it. Most importantly, it allows the import machinery to perform the boilerplate operations of loading, whereas without a module spec the loader had that responsibility. -See :class:`~importlib.machinery.ModuleSpec` for more specifics on what -information a module's spec may hold. +The module's spec is exposed as the ``__spec__`` attribute on a module object. +See :class:`~importlib.machinery.ModuleSpec` for details on the contents of +the module spec. .. versionadded:: 3.4 diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index c2fd2b74..4a5abf62 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -446,9 +446,6 @@ instance of the :class:`bytes` type instead of the :class:`str` type. They may only contain ASCII characters; bytes with a numeric value of 128 or greater must be expressed with escapes. -As of Python 3.3 it is possible again to prefix string literals with a -``u`` prefix to simplify maintenance of dual 2.x and 3.x codebases. - Both string and bytes literals may optionally be prefixed with a letter ``'r'`` or ``'R'``; such strings are called :dfn:`raw strings` and treat backslashes as literal characters. As a result, in string literals, ``'\U'`` and ``'\u'`` @@ -799,10 +796,6 @@ Some examples of floating point literals:: 3.14 10. .001 1e100 3.14e-10 0e0 3.14_15_93 -Note that numeric literals do not include a sign; a phrase like ``-1`` is -actually an expression composed of the unary operator ``-`` and the literal -``1``. - .. versionchanged:: 3.6 Underscores are now allowed for grouping purposes in literals. diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 8786d73f..8d173838 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -931,7 +931,7 @@ annotation. .. impl-detail:: - The current implementation does not enforce some of these restriction, but + The current implementation does not enforce some of these restrictions, but programs should not abuse this freedom, as future implementations may enforce them or silently change the meaning of the program. diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index bd31faca..c450f5ea 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -13,8 +13,6 @@ '3.7': 'dev (3.7)', '3.6': '3.6', '3.5': '3.5', - '3.4': '3.4', - '3.3': '3.3', '2.7': '2.7', }; diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 6cd5a328..482ffc0b 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -130,9 +130,6 @@ library/exceptions,,:err,err.object[err.start:err.end] library/functions,,:step,a[start:stop:step] library/functions,,:stop,"a[start:stop, i]" library/functions,,:stop,a[start:stop:step] -library/hashlib,,:vatrogasac,>>> cookie = b'user:vatrogasac' -library/hashlib,,:vatrogasac,"user:vatrogasac,349cf904533767ed2d755279a8df84d0" -library/hashlib,,:policajac,">>> compare_digest(b'user:policajac', sig)" library/hashlib,,:LEAF,"h00 = blake2b(buf[0:LEAF_SIZE], fanout=FANOUT, depth=DEPTH," library/http.client,,:port,host:port library/http.cookies,,`,!#$%&'*+-.^_`|~: diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 413c0a76..c73c4064 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,9 +2,9 @@

{% trans %}Download these documents{% endtrans %}

{% trans %}Docs for other versions{% endtrans %}

diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 8956aa5a..6415ae66 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -212,6 +212,13 @@ to each other are automatically concatenated. :: >>> 'Py' 'thon' 'Python' +This feature is particularly useful when you want to break long strings:: + + >>> text = ('Put several strings within parentheses ' + ... 'to have them joined together.') + >>> text + 'Put several strings within parentheses to have them joined together.' + This only works with two literals though, not with variables or expressions:: >>> prefix = 'Py' @@ -227,13 +234,6 @@ If you want to concatenate variables or a variable and a literal, use ``+``:: >>> prefix + 'thon' 'Python' -This feature is particularly useful when you want to break long strings:: - - >>> text = ('Put several strings within parentheses ' - ... 'to have them joined together.') - >>> text - 'Put several strings within parentheses to have them joined together.' - Strings can be *indexed* (subscripted), with the first character having index 0. There is no separate character type; a character is simply a string of size one:: diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 40a06b9a..9ffb7149 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -387,8 +387,6 @@ Miscellaneous options Skip the first line of the source, allowing use of non-Unix forms of ``#!cmd``. This is intended for a DOS specific hack only. - .. note:: The line numbers in error messages will be off by one. - .. cmdoption:: -X diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index 53f431b5..4292592b 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -33,7 +33,7 @@ On Windows, invoke the ``venv`` command as follows:: Alternatively, if you configured the ``PATH`` and ``PATHEXT`` variables for your :ref:`Python installation `:: - c:\>python -m venv myenv c:\path\to\myenv + c:\>python -m venv c:\path\to\myenv The command, if run with ``-h``, will show the available options:: diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 1301f4ca..847b5014 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -1852,6 +1852,11 @@ Build and C API Changes * The :c:func:`PyUnicode_FSConverter` and :c:func:`PyUnicode_FSDecoder` functions will now accept :term:`path-like objects `. +* The ``PyExc_RecursionErrorInst`` singleton that was part of the public API + has been removed as its members being never cleared may cause a segfault + during finalization of the interpreter. Contributed by Xavier de Gaye in + :issue:`22898` and :issue:`30697`. + Other Improvements ================== diff --git a/Include/patchlevel.h b/Include/patchlevel.h index f06b6549..e909bb24 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 6 -#define PY_MICRO_VERSION 3 +#define PY_MICRO_VERSION 4 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.6.3" +#define PY_VERSION "3.6.4" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/py_curses.h b/Include/py_curses.h index 3c216970..b09dde48 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -7,14 +7,9 @@ ** On Mac OS X 10.2 [n]curses.h and stdlib.h use different guards ** against multiple definition of wchar_t. */ -#ifdef _BSD_WCHAR_T_DEFINED_ +#ifdef _BSD_WCHAR_T_DEFINED_ #define _WCHAR_T #endif - -/* the following define is necessary for OS X 10.6; without it, the - Apple-supplied ncurses.h sets NCURSES_OPAQUE to 1, and then Python - can't get at the WINDOW flags field. */ -#define NCURSES_OPAQUE 0 #endif /* __APPLE__ */ #ifdef __FreeBSD__ @@ -22,7 +17,7 @@ ** On FreeBSD, [n]curses.h and stdlib.h/wchar.h use different guards ** against multiple definition of wchar_t and wint_t. */ -#ifdef _XOPEN_SOURCE_EXTENDED +#ifdef _XOPEN_SOURCE_EXTENDED #ifndef __FreeBSD_version #include #endif @@ -44,22 +39,28 @@ #endif #endif +#if !defined(HAVE_CURSES_IS_PAD) && defined(WINDOW_HAS_FLAGS) +/* The following definition is necessary for ncurses 5.7; without it, + some of [n]curses.h set NCURSES_OPAQUE to 1, and then Python + can't get at the WINDOW flags field. */ +#define NCURSES_OPAQUE 0 +#endif + #ifdef HAVE_NCURSES_H #include #else #include -#ifdef HAVE_TERM_H -/* for tigetstr, which is not declared in SysV curses */ -#include -#endif #endif #ifdef HAVE_NCURSES_H /* configure was checking , but we will - use , which has all these features. */ -#ifndef WINDOW_HAS_FLAGS + use , which has some or all these features. */ +#if !defined(WINDOW_HAS_FLAGS) && !(NCURSES_OPAQUE+0) #define WINDOW_HAS_FLAGS 1 #endif +#if !defined(HAVE_CURSES_IS_PAD) && NCURSES_VERSION_PATCH+0 >= 20090906 +#define HAVE_CURSES_IS_PAD 1 +#endif #ifndef MVWDELCH_IS_EXPRESSION #define MVWDELCH_IS_EXPRESSION 1 #endif @@ -74,12 +75,12 @@ extern "C" { /* Type declarations */ typedef struct { - PyObject_HEAD - WINDOW *win; - char *encoding; + PyObject_HEAD + WINDOW *win; + char *encoding; } PyCursesWindowObject; -#define PyCursesWindow_Check(v) (Py_TYPE(v) == &PyCursesWindow_Type) +#define PyCursesWindow_Check(v) (Py_TYPE(v) == &PyCursesWindow_Type) #define PyCurses_CAPSULE_NAME "_curses._C_API" diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 8c1dbc50..c28c1373 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -219,8 +219,6 @@ PyAPI_DATA(PyObject *) PyExc_IOError; PyAPI_DATA(PyObject *) PyExc_WindowsError; #endif -PyAPI_DATA(PyObject *) PyExc_RecursionErrorInst; - /* Predefined warning categories */ PyAPI_DATA(PyObject *) PyExc_Warning; PyAPI_DATA(PyObject *) PyExc_UserWarning; diff --git a/Include/pyhash.h b/Include/pyhash.h index a814af67..9cfd071e 100644 --- a/Include/pyhash.h +++ b/Include/pyhash.h @@ -16,7 +16,7 @@ PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); #define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ /* Parameters used for the numeric hash implementation. See notes for - _Py_HashDouble in Objects/object.c. Numeric hashes are based on + _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on reduction modulo the prime 2**_PyHASH_BITS - 1. */ #if SIZEOF_VOID_P >= 8 diff --git a/Include/pytime.h b/Include/pytime.h index 87ac7fcb..158c460a 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -29,9 +29,20 @@ typedef enum { _PyTime_ROUND_CEILING=1, /* Round to nearest with ties going to nearest even integer. For example, used to round from a Python float. */ - _PyTime_ROUND_HALF_EVEN + _PyTime_ROUND_HALF_EVEN=2, + /* Round away from zero + For example, used for timeout. _PyTime_ROUND_CEILING rounds + -1e-9 to 0 milliseconds which causes bpo-31786 issue. + _PyTime_ROUND_UP rounds -1e-9 to -1 millisecond which keeps + the timeout sign as expected. select.poll(timeout) must block + for negative values." */ + _PyTime_ROUND_UP=3, + /* _PyTime_ROUND_TIMEOUT (an alias for _PyTime_ROUND_UP) should be + used for timeouts. */ + _PyTime_ROUND_TIMEOUT = _PyTime_ROUND_UP } _PyTime_round_t; + /* Convert a time_t to a PyLong. */ PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( time_t sec); diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index a4967b85..8cc655c7 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -1222,6 +1222,11 @@ class BaseEventLoop(events.AbstractEventLoop): handler is set, and can be called by a custom exception handler that wants to defer to the default behavior. + This default handler logs the error message and other + context-dependent information. In debug mode, a truncated + stack trace is also appended showing where the given object + (e.g. a handle or future or task) was created, if any. + The context parameter has the same meaning as in `call_exception_handler()`. """ diff --git a/Lib/asyncio/constants.py b/Lib/asyncio/constants.py index f9e12328..e74209e1 100644 --- a/Lib/asyncio/constants.py +++ b/Lib/asyncio/constants.py @@ -5,3 +5,8 @@ LOG_THRESHOLD_FOR_CONNLOST_WRITES = 5 # Seconds to wait before retrying accept(). ACCEPT_RETRY_DELAY = 1 + +# Number of stack entries to capture in debug mode. +# The large the number, the slower the operation in debug mode +# (see extract_stack() in events.py) +DEBUG_STACK_DEPTH = 10 diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index b2adaadf..520a309f 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -10,6 +10,7 @@ import traceback import types from . import compat +from . import constants from . import events from . import base_futures from .log import logger @@ -91,7 +92,7 @@ class CoroWrapper: assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen self.gen = gen self.func = func # Used to unwrap @coroutine decorator - self._source_traceback = traceback.extract_stack(sys._getframe(1)) + self._source_traceback = events.extract_stack(sys._getframe(1)) self.__name__ = getattr(gen, '__name__', None) self.__qualname__ = getattr(gen, '__qualname__', None) @@ -183,8 +184,9 @@ class CoroWrapper: tb = getattr(self, '_source_traceback', ()) if tb: tb = ''.join(traceback.format_list(tb)) - msg += ('\nCoroutine object created at ' - '(most recent call last):\n') + msg += (f'\nCoroutine object created at ' + f'(most recent call last, truncated to ' + f'{constants.DEBUG_STACK_DEPTH} last lines):\n') msg += tb.rstrip() logger.error(msg) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index d41f3f5b..05dc8969 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -19,7 +19,8 @@ import sys import threading import traceback -from asyncio import compat +from . import compat +from . import constants def _get_function_source(func): @@ -77,6 +78,23 @@ def _format_callback_source(func, args): return func_repr +def extract_stack(f=None, limit=None): + """Replacement for traceback.extract_stack() that only does the + necessary work for asyncio debug mode. + """ + if f is None: + f = sys._getframe().f_back + if limit is None: + # Limit the amount of work to a reasonable amount, as extract_stack() + # can be called for each coroutine and future in debug mode. + limit = constants.DEBUG_STACK_DEPTH + stack = traceback.StackSummary.extract(traceback.walk_stack(f), + limit=limit, + lookup_lines=False) + stack.reverse() + return stack + + class Handle: """Object returned by callback registration methods.""" @@ -90,7 +108,7 @@ class Handle: self._cancelled = False self._repr = None if self._loop.get_debug(): - self._source_traceback = traceback.extract_stack(sys._getframe(1)) + self._source_traceback = extract_stack(sys._getframe(1)) else: self._source_traceback = None diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index cff9590e..9ff14303 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -112,11 +112,13 @@ class Future: Differences: + - This class is not thread-safe. + - result() and exception() do not take a timeout argument and raise an exception when the future isn't done yet. - Callbacks registered with add_done_callback() are always called - via the event loop's call_soon_threadsafe(). + via the event loop's call_soon(). - This class is not compatible with the wait() and as_completed() methods in the concurrent.futures package. @@ -141,8 +143,7 @@ class Future: # `yield Future()` (incorrect). _asyncio_future_blocking = False - _log_traceback = False # Used for Python 3.4 and later - _tb_logger = None # Used for Python 3.3 only + _log_traceback = False def __init__(self, *, loop=None): """Initialize the future. @@ -157,7 +158,7 @@ class Future: self._loop = loop self._callbacks = [] if self._loop.get_debug(): - self._source_traceback = traceback.extract_stack(sys._getframe(1)) + self._source_traceback = events.extract_stack(sys._getframe(1)) _repr_info = base_futures._future_repr_info @@ -238,9 +239,6 @@ class Future: if self._state != _FINISHED: raise InvalidStateError('Result is not ready.') self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None if self._exception is not None: raise self._exception return self._result @@ -258,9 +256,6 @@ class Future: if self._state != _FINISHED: raise InvalidStateError('Exception is not set.') self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None return self._exception def add_done_callback(self, fn): diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 2d38972c..1c66d67b 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -167,6 +167,12 @@ class Queue: yield from getter except: getter.cancel() # Just in case getter is not done yet. + + try: + self._getters.remove(getter) + except ValueError: + pass + if not self.empty() and not getter.cancelled(): # We were woken up by put_nowait(), but can't take # the call. Wake up the next in line. diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 9dbe550b..942b627a 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -363,25 +363,25 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") fut = self.create_future() - self._sock_recv(fut, False, sock, n) + self._sock_recv(fut, None, sock, n) return fut - def _sock_recv(self, fut, registered, sock, n): + def _sock_recv(self, fut, registered_fd, sock, n): # _sock_recv() can add itself as an I/O callback if the operation can't # be done immediately. Don't use it directly, call sock_recv(). - fd = sock.fileno() - if registered: + if registered_fd is not None: # Remove the callback early. It should be rare that the # selector says the fd is ready but the call still returns # EAGAIN, and I am willing to take a hit in that case in # order to simplify the common case. - self.remove_reader(fd) + self.remove_reader(registered_fd) if fut.cancelled(): return try: data = sock.recv(n) except (BlockingIOError, InterruptedError): - self.add_reader(fd, self._sock_recv, fut, True, sock, n) + fd = sock.fileno() + self.add_reader(fd, self._sock_recv, fut, fd, sock, n) except Exception as exc: fut.set_exception(exc) else: @@ -402,16 +402,14 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): raise ValueError("the socket must be non-blocking") fut = self.create_future() if data: - self._sock_sendall(fut, False, sock, data) + self._sock_sendall(fut, None, sock, data) else: fut.set_result(None) return fut - def _sock_sendall(self, fut, registered, sock, data): - fd = sock.fileno() - - if registered: - self.remove_writer(fd) + def _sock_sendall(self, fut, registered_fd, sock, data): + if registered_fd is not None: + self.remove_writer(registered_fd) if fut.cancelled(): return @@ -428,7 +426,8 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): else: if n: data = data[n:] - self.add_writer(fd, self._sock_sendall, fut, True, sock, data) + fd = sock.fileno() + self.add_writer(fd, self._sock_sendall, fut, fd, sock, data) @coroutine def sock_connect(self, sock, address): diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 7948c4c3..f4d8a484 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -294,11 +294,10 @@ class _SSLPipe(object): class _SSLProtocolTransport(transports._FlowControlMixin, transports.Transport): - def __init__(self, loop, ssl_protocol, app_protocol): + def __init__(self, loop, ssl_protocol): self._loop = loop # SSLProtocol instance self._ssl_protocol = ssl_protocol - self._app_protocol = app_protocol self._closed = False def get_extra_info(self, name, default=None): @@ -306,10 +305,10 @@ class _SSLProtocolTransport(transports._FlowControlMixin, return self._ssl_protocol._get_extra_info(name, default) def set_protocol(self, protocol): - self._app_protocol = protocol + self._ssl_protocol._app_protocol = protocol def get_protocol(self): - return self._app_protocol + return self._ssl_protocol._app_protocol def is_closing(self): return self._closed @@ -436,8 +435,7 @@ class SSLProtocol(protocols.Protocol): self._waiter = waiter self._loop = loop self._app_protocol = app_protocol - self._app_transport = _SSLProtocolTransport(self._loop, - self, self._app_protocol) + self._app_transport = _SSLProtocolTransport(self._loop, self) # _SSLPipe instance (None until the connection is made) self._sslpipe = None self._session_established = False diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index a82cc79a..4089b0e0 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -35,6 +35,9 @@ class IncompleteReadError(EOFError): self.partial = partial self.expected = expected + def __reduce__(self): + return type(self), (self.partial, self.expected) + class LimitOverrunError(Exception): """Reached the buffer limit while looking for a separator. @@ -46,6 +49,9 @@ class LimitOverrunError(Exception): super().__init__(message) self.consumed = consumed + def __reduce__(self): + return type(self), (self.args[0], self.consumed) + @coroutine def open_connection(host=None, port=None, *, diff --git a/Lib/codecs.py b/Lib/codecs.py index 39ec8454..fd6c6f59 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -479,15 +479,17 @@ class StreamReader(Codec): self.charbuffer = self._empty_charbuffer.join(self.linebuffer) self.linebuffer = None + if chars < 0: + # For compatibility with other read() methods that take a + # single argument + chars = size + # read until we get the required number of characters (if available) while True: # can the request be satisfied from the character buffer? if chars >= 0: if len(self.charbuffer) >= chars: break - elif size >= 0: - if len(self.charbuffer) >= size: - break # we need more data if size < 0: newdata = self.stream.read() diff --git a/Lib/ctypes/test/test_anon.py b/Lib/ctypes/test/test_anon.py index d892b598..d378392e 100644 --- a/Lib/ctypes/test/test_anon.py +++ b/Lib/ctypes/test/test_anon.py @@ -1,4 +1,5 @@ import unittest +import test.support from ctypes import * class AnonTest(unittest.TestCase): @@ -35,6 +36,18 @@ class AnonTest(unittest.TestCase): {"_fields_": [], "_anonymous_": ["x"]})) + @test.support.cpython_only + def test_issue31490(self): + # There shouldn't be an assertion failure in case the class has an + # attribute whose name is specified in _anonymous_ but not in _fields_. + + # AttributeError: 'x' is specified in _anonymous_ but not in _fields_ + with self.assertRaises(AttributeError): + class Name(Structure): + _fields_ = [] + _anonymous_ = ["x"] + x = 42 + def test_nested(self): class ANON_S(Structure): _fields_ = [("a", c_int)] diff --git a/Lib/ctypes/test/test_frombuffer.py b/Lib/ctypes/test/test_frombuffer.py index 7ab38f1b..55c24435 100644 --- a/Lib/ctypes/test/test_frombuffer.py +++ b/Lib/ctypes/test/test_frombuffer.py @@ -121,12 +121,21 @@ class Test(unittest.TestCase): (c_int * 1).from_buffer_copy(a, 16 * sizeof(c_int)) def test_abstract(self): + from ctypes import _Pointer, _SimpleCData, _CFuncPtr + self.assertRaises(TypeError, Array.from_buffer, bytearray(10)) self.assertRaises(TypeError, Structure.from_buffer, bytearray(10)) self.assertRaises(TypeError, Union.from_buffer, bytearray(10)) + self.assertRaises(TypeError, _CFuncPtr.from_buffer, bytearray(10)) + self.assertRaises(TypeError, _Pointer.from_buffer, bytearray(10)) + self.assertRaises(TypeError, _SimpleCData.from_buffer, bytearray(10)) + self.assertRaises(TypeError, Array.from_buffer_copy, b"123") self.assertRaises(TypeError, Structure.from_buffer_copy, b"123") self.assertRaises(TypeError, Union.from_buffer_copy, b"123") + self.assertRaises(TypeError, _CFuncPtr.from_buffer_copy, b"123") + self.assertRaises(TypeError, _Pointer.from_buffer_copy, b"123") + self.assertRaises(TypeError, _SimpleCData.from_buffer_copy, b"123") if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_funcptr.py b/Lib/ctypes/test/test_funcptr.py index f34734b1..e0b9b54e 100644 --- a/Lib/ctypes/test/test_funcptr.py +++ b/Lib/ctypes/test/test_funcptr.py @@ -123,5 +123,10 @@ class CFuncPtrTestCase(unittest.TestCase): self.assertEqual(strtok(None, b"\n"), b"c") self.assertEqual(strtok(None, b"\n"), None) + def test_abstract(self): + from ctypes import _CFuncPtr + + self.assertRaises(TypeError, _CFuncPtr, 13, "name", 42, "iid") + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py index 363f5861..e4c25fd8 100644 --- a/Lib/ctypes/test/test_parameters.py +++ b/Lib/ctypes/test/test_parameters.py @@ -1,5 +1,6 @@ import unittest from ctypes.test import need_symbol +import test.support class SimpleTypesTestCase(unittest.TestCase): @@ -169,6 +170,36 @@ class SimpleTypesTestCase(unittest.TestCase): # ArgumentError: argument 1: ValueError: 99 self.assertRaises(ArgumentError, func, 99) + def test_abstract(self): + from ctypes import (Array, Structure, Union, _Pointer, + _SimpleCData, _CFuncPtr) + + self.assertRaises(TypeError, Array.from_param, 42) + self.assertRaises(TypeError, Structure.from_param, 42) + self.assertRaises(TypeError, Union.from_param, 42) + self.assertRaises(TypeError, _CFuncPtr.from_param, 42) + self.assertRaises(TypeError, _Pointer.from_param, 42) + self.assertRaises(TypeError, _SimpleCData.from_param, 42) + + @test.support.cpython_only + def test_issue31311(self): + # __setstate__ should neither raise a SystemError nor crash in case + # of a bad __dict__. + from ctypes import Structure + + class BadStruct(Structure): + @property + def __dict__(self): + pass + with self.assertRaises(TypeError): + BadStruct().__setstate__({}, b'foo') + + class WorseStruct(Structure): + @property + def __dict__(self): + 1/0 + with self.assertRaises(ZeroDivisionError): + WorseStruct().__setstate__({}, b'foo') ################################################################ diff --git a/Lib/ctypes/test/test_pointers.py b/Lib/ctypes/test/test_pointers.py index 751f85fd..e9751587 100644 --- a/Lib/ctypes/test/test_pointers.py +++ b/Lib/ctypes/test/test_pointers.py @@ -213,6 +213,11 @@ class PointersTestCase(unittest.TestCase): from ctypes import _pointer_type_cache del _pointer_type_cache[id(P)] + def test_abstract(self): + from ctypes import _Pointer + + self.assertRaises(TypeError, _Pointer.set_type, 42) + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_struct_fields.py b/Lib/ctypes/test/test_struct_fields.py index 22eb3b0c..8045cc82 100644 --- a/Lib/ctypes/test/test_struct_fields.py +++ b/Lib/ctypes/test/test_struct_fields.py @@ -46,5 +46,29 @@ class StructFieldsTestCase(unittest.TestCase): Y._fields_ = [] self.assertRaises(AttributeError, setattr, X, "_fields_", []) + # __set__ and __get__ should raise a TypeError in case their self + # argument is not a ctype instance. + def test___set__(self): + class MyCStruct(Structure): + _fields_ = (("field", c_int),) + self.assertRaises(TypeError, + MyCStruct.field.__set__, 'wrong type self', 42) + + class MyCUnion(Union): + _fields_ = (("field", c_int),) + self.assertRaises(TypeError, + MyCUnion.field.__set__, 'wrong type self', 42) + + def test___get__(self): + class MyCStruct(Structure): + _fields_ = (("field", c_int),) + self.assertRaises(TypeError, + MyCStruct.field.__get__, 'wrong type self', 42) + + class MyCUnion(Union): + _fields_ = (("field", c_int),) + self.assertRaises(TypeError, + MyCUnion.field.__get__, 'wrong type self', 42) + if __name__ == "__main__": unittest.main() diff --git a/Lib/datetime.py b/Lib/datetime.py index b95536fb..150664ea 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -827,7 +827,7 @@ class date: month = self._month if day is None: day = self._day - return date(year, month, day) + return type(self)(year, month, day) # Comparisons of date objects with other. @@ -1315,7 +1315,7 @@ class time: tzinfo = self.tzinfo if fold is None: fold = self._fold - return time(hour, minute, second, microsecond, tzinfo, fold=fold) + return type(self)(hour, minute, second, microsecond, tzinfo, fold=fold) # Pickle support. @@ -1596,7 +1596,7 @@ class datetime(date): tzinfo = self.tzinfo if fold is None: fold = self.fold - return datetime(year, month, day, hour, minute, second, + return type(self)(year, month, day, hour, minute, second, microsecond, tzinfo, fold=fold) def _local_timezone(self): diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 9b9697f7..3ebbbe53 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -96,90 +96,6 @@ EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') def quote_string(value): return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' -# -# Accumulator for header folding -# - -class _Folded: - - def __init__(self, maxlen, policy): - self.maxlen = maxlen - self.policy = policy - self.lastlen = 0 - self.stickyspace = None - self.firstline = True - self.done = [] - self.current = [] - - def newline(self): - self.done.extend(self.current) - self.done.append(self.policy.linesep) - self.current.clear() - self.lastlen = 0 - - def finalize(self): - if self.current: - self.newline() - - def __str__(self): - return ''.join(self.done) - - def append(self, stoken): - self.current.append(stoken) - - def append_if_fits(self, token, stoken=None): - if stoken is None: - stoken = str(token) - l = len(stoken) - if self.stickyspace is not None: - stickyspace_len = len(self.stickyspace) - if self.lastlen + stickyspace_len + l <= self.maxlen: - self.current.append(self.stickyspace) - self.lastlen += stickyspace_len - self.current.append(stoken) - self.lastlen += l - self.stickyspace = None - self.firstline = False - return True - if token.has_fws: - ws = token.pop_leading_fws() - if ws is not None: - self.stickyspace += str(ws) - stickyspace_len += len(ws) - token._fold(self) - return True - if stickyspace_len and l + 1 <= self.maxlen: - margin = self.maxlen - l - if 0 < margin < stickyspace_len: - trim = stickyspace_len - margin - self.current.append(self.stickyspace[:trim]) - self.stickyspace = self.stickyspace[trim:] - stickyspace_len = trim - self.newline() - self.current.append(self.stickyspace) - self.current.append(stoken) - self.lastlen = l + stickyspace_len - self.stickyspace = None - self.firstline = False - return True - if not self.firstline: - self.newline() - self.current.append(self.stickyspace) - self.current.append(stoken) - self.stickyspace = None - self.firstline = False - return True - if self.lastlen + l <= self.maxlen: - self.current.append(stoken) - self.lastlen += l - return True - if l < self.maxlen: - self.newline() - self.current.append(stoken) - self.lastlen = l - return True - return False - # # TokenList and its subclasses # @@ -187,6 +103,8 @@ class _Folded: class TokenList(list): token_type = None + syntactic_break = True + ew_combine_allowed = True def __init__(self, *args, **kw): super().__init__(*args, **kw) @@ -207,84 +125,13 @@ class TokenList(list): def all_defects(self): return sum((x.all_defects for x in self), self.defects) - # - # Folding API - # - # parts(): - # - # return a list of objects that constitute the "higher level syntactic - # objects" specified by the RFC as the best places to fold a header line. - # The returned objects must include leading folding white space, even if - # this means mutating the underlying parse tree of the object. Each object - # is only responsible for returning *its* parts, and should not drill down - # to any lower level except as required to meet the leading folding white - # space constraint. - # - # _fold(folded): - # - # folded: the result accumulator. This is an instance of _Folded. - # (XXX: I haven't finished factoring this out yet, the folding code - # pretty much uses this as a state object.) When the folded.current - # contains as much text as will fit, the _fold method should call - # folded.newline. - # folded.lastlen: the current length of the test stored in folded.current. - # folded.maxlen: The maximum number of characters that may appear on a - # folded line. Differs from the policy setting in that "no limit" is - # represented by +inf, which means it can be used in the trivially - # logical fashion in comparisons. - # - # Currently no subclasses implement parts, and I think this will remain - # true. A subclass only needs to implement _fold when the generic version - # isn't sufficient. _fold will need to be implemented primarily when it is - # possible for encoded words to appear in the specialized token-list, since - # there is no generic algorithm that can know where exactly the encoded - # words are allowed. A _fold implementation is responsible for filling - # lines in the same general way that the top level _fold does. It may, and - # should, call the _fold method of sub-objects in a similar fashion to that - # of the top level _fold. - # - # XXX: I'm hoping it will be possible to factor the existing code further - # to reduce redundancy and make the logic clearer. - - @property - def parts(self): - klass = self.__class__ - this = [] - for token in self: - if token.startswith_fws(): - if this: - yield this[0] if len(this)==1 else klass(this) - this.clear() - end_ws = token.pop_trailing_ws() - this.append(token) - if end_ws: - yield klass(this) - this = [end_ws] - if this: - yield this[0] if len(this)==1 else klass(this) - def startswith_fws(self): return self[0].startswith_fws() - def pop_leading_fws(self): - if self[0].token_type == 'fws': - return self.pop(0) - return self[0].pop_leading_fws() - - def pop_trailing_ws(self): - if self[-1].token_type == 'cfws': - return self.pop(-1) - return self[-1].pop_trailing_ws() - @property - def has_fws(self): - for part in self: - if part.has_fws: - return True - return False - - def has_leading_comment(self): - return self[0].has_leading_comment() + def as_ew_allowed(self): + """True if all top level tokens of this part may be RFC2047 encoded.""" + return all(part.as_ew_allowed for part in self) @property def comments(self): @@ -294,69 +141,13 @@ class TokenList(list): return comments def fold(self, *, policy): - # max_line_length 0/None means no limit, ie: infinitely long. - maxlen = policy.max_line_length or float("+inf") - folded = _Folded(maxlen, policy) - self._fold(folded) - folded.finalize() - return str(folded) - - def as_encoded_word(self, charset): - # This works only for things returned by 'parts', which include - # the leading fws, if any, that should be used. - res = [] - ws = self.pop_leading_fws() - if ws: - res.append(ws) - trailer = self.pop(-1) if self[-1].token_type=='fws' else '' - res.append(_ew.encode(str(self), charset)) - res.append(trailer) - return ''.join(res) - - def cte_encode(self, charset, policy): - res = [] - for part in self: - res.append(part.cte_encode(charset, policy)) - return ''.join(res) - - def _fold(self, folded): - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - tlen = len(tstr) - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - # XXX: this should be a policy setting when utf8 is False. - charset = 'utf-8' - tstr = part.cte_encode(charset, folded.policy) - tlen = len(tstr) - if folded.append_if_fits(part, tstr): - continue - # Peel off the leading whitespace if any and make it sticky, to - # avoid infinite recursion. - ws = part.pop_leading_fws() - if ws is not None: - folded.stickyspace = str(ws) - if folded.append_if_fits(part): - continue - if part.has_fws: - part._fold(folded) - continue - # There are no fold points in this one; it is too long for a single - # line and can't be split...we just have to put it on its own line. - folded.append(tstr) - folded.newline() + return _refold_parse_tree(self, policy=policy) def pprint(self, indent=''): - print('\n'.join(self._pp(indent=''))) + print(self.ppstr(indent=indent)) def ppstr(self, indent=''): - return '\n'.join(self._pp(indent='')) + return '\n'.join(self._pp(indent=indent)) def _pp(self, indent=''): yield '{}{}/{}('.format( @@ -391,173 +182,11 @@ class UnstructuredTokenList(TokenList): token_type = 'unstructured' - def _fold(self, folded): - last_ew = None - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - is_ew = False - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - charset = 'utf-8' - if last_ew is not None: - # We've already done an EW, combine this one with it - # if there's room. - chunk = get_unstructured( - ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset) - oldlastlen = sum(len(x) for x in folded.current[:last_ew]) - schunk = str(chunk) - lchunk = len(schunk) - if oldlastlen + lchunk <= folded.maxlen: - del folded.current[last_ew:] - folded.append(schunk) - folded.lastlen = oldlastlen + lchunk - continue - tstr = part.as_encoded_word(charset) - is_ew = True - if folded.append_if_fits(part, tstr): - if is_ew: - last_ew = len(folded.current) - 1 - continue - if is_ew or last_ew: - # It's too big to fit on the line, but since we've - # got encoded words we can use encoded word folding. - part._fold_as_ew(folded) - continue - # Peel off the leading whitespace if any and make it sticky, to - # avoid infinite recursion. - ws = part.pop_leading_fws() - if ws is not None: - folded.stickyspace = str(ws) - if folded.append_if_fits(part): - continue - if part.has_fws: - part._fold(folded) - continue - # It can't be split...we just have to put it on its own line. - folded.append(tstr) - folded.newline() - last_ew = None - - def cte_encode(self, charset, policy): - res = [] - last_ew = None - for part in self: - spart = str(part) - try: - spart.encode('us-ascii') - res.append(spart) - except UnicodeEncodeError: - if last_ew is None: - res.append(part.cte_encode(charset, policy)) - last_ew = len(res) - else: - tl = get_unstructured(''.join(res[last_ew:] + [spart])) - res.append(tl.as_encoded_word(charset)) - return ''.join(res) - class Phrase(TokenList): token_type = 'phrase' - def _fold(self, folded): - # As with Unstructured, we can have pure ASCII with or without - # surrogateescape encoded bytes, or we could have unicode. But this - # case is more complicated, since we have to deal with the various - # sub-token types and how they can be composed in the face of - # unicode-that-needs-CTE-encoding, and the fact that if a token a - # comment that becomes a barrier across which we can't compose encoded - # words. - last_ew = None - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - tlen = len(tstr) - has_ew = False - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - charset = 'utf-8' - if last_ew is not None and not part.has_leading_comment(): - # We've already done an EW, let's see if we can combine - # this one with it. The last_ew logic ensures that all we - # have at this point is atoms, no comments or quoted - # strings. So we can treat the text between the last - # encoded word and the content of this token as - # unstructured text, and things will work correctly. But - # we have to strip off any trailing comment on this token - # first, and if it is a quoted string we have to pull out - # the content (we're encoding it, so it no longer needs to - # be quoted). - if part[-1].token_type == 'cfws' and part.comments: - remainder = part.pop(-1) - else: - remainder = '' - for i, token in enumerate(part): - if token.token_type == 'bare-quoted-string': - part[i] = UnstructuredTokenList(token[:]) - chunk = get_unstructured( - ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset) - schunk = str(chunk) - lchunk = len(schunk) - if last_ew + lchunk <= folded.maxlen: - del folded.current[last_ew:] - folded.append(schunk) - folded.lastlen = sum(len(x) for x in folded.current) - continue - tstr = part.as_encoded_word(charset) - tlen = len(tstr) - has_ew = True - if folded.append_if_fits(part, tstr): - if has_ew and not part.comments: - last_ew = len(folded.current) - 1 - elif part.comments or part.token_type == 'quoted-string': - # If a comment is involved we can't combine EWs. And if a - # quoted string is involved, it's not worth the effort to - # try to combine them. - last_ew = None - continue - part._fold(folded) - - def cte_encode(self, charset, policy): - res = [] - last_ew = None - is_ew = False - for part in self: - spart = str(part) - try: - spart.encode('us-ascii') - res.append(spart) - except UnicodeEncodeError: - is_ew = True - if last_ew is None: - if not part.comments: - last_ew = len(res) - res.append(part.cte_encode(charset, policy)) - elif not part.has_leading_comment(): - if part[-1].token_type == 'cfws' and part.comments: - remainder = part.pop(-1) - else: - remainder = '' - for i, token in enumerate(part): - if token.token_type == 'bare-quoted-string': - part[i] = UnstructuredTokenList(token[:]) - tl = get_unstructured(''.join(res[last_ew:] + [spart])) - res[last_ew:] = [tl.as_encoded_word(charset)] - if part.comments or (not is_ew and part.token_type == 'quoted-string'): - last_ew = None - return ''.join(res) - class Word(TokenList): token_type = 'word' @@ -567,9 +196,6 @@ class CFWSList(WhiteSpaceTokenList): token_type = 'cfws' - def has_leading_comment(self): - return bool(self.comments) - class Atom(TokenList): @@ -579,6 +205,7 @@ class Atom(TokenList): class Token(TokenList): token_type = 'token' + encode_as_ew = False class EncodedWord(TokenList): @@ -588,13 +215,6 @@ class EncodedWord(TokenList): charset = None lang = None - @property - def encoded(self): - if self.cte is not None: - return self.cte - _ew.encode(str(self), self.charset) - - class QuotedString(TokenList): @@ -865,6 +485,7 @@ class InvalidMailbox(TokenList): class Domain(TokenList): token_type = 'domain' + as_ew_allowed = False @property def domain(self): @@ -879,11 +500,13 @@ class DotAtom(TokenList): class DotAtomText(TokenList): token_type = 'dot-atom-text' + as_ew_allowed = True class AddrSpec(TokenList): token_type = 'addr-spec' + as_ew_allowed = False @property def local_part(self): @@ -916,11 +539,13 @@ class AddrSpec(TokenList): class ObsLocalPart(TokenList): token_type = 'obs-local-part' + as_ew_allowed = False class DisplayName(Phrase): token_type = 'display-name' + ew_combine_allowed = False @property def display_name(self): @@ -960,6 +585,7 @@ class DisplayName(Phrase): class LocalPart(TokenList): token_type = 'local-part' + as_ew_allowed = False @property def value(self): @@ -995,6 +621,7 @@ class LocalPart(TokenList): class DomainLiteral(TokenList): token_type = 'domain-literal' + as_ew_allowed = False @property def domain(self): @@ -1081,6 +708,7 @@ class Value(TokenList): class MimeParameters(TokenList): token_type = 'mime-parameters' + syntactic_break = False @property def params(self): @@ -1165,6 +793,10 @@ class MimeParameters(TokenList): class ParameterizedHeaderValue(TokenList): + # Set this false so that the value doesn't wind up on a new line even + # if it and the parameters would fit there but not on the first line. + syntactic_break = False + @property def params(self): for token in reversed(self): @@ -1172,18 +804,11 @@ class ParameterizedHeaderValue(TokenList): return token.params return {} - @property - def parts(self): - if self and self[-1].token_type == 'mime-parameters': - # We don't want to start a new line if all of the params don't fit - # after the value, so unwrap the parameter list. - return TokenList(self[:-1] + self[-1]) - return TokenList(self).parts - class ContentType(ParameterizedHeaderValue): token_type = 'content-type' + as_ew_allowed = False maintype = 'text' subtype = 'plain' @@ -1191,40 +816,27 @@ class ContentType(ParameterizedHeaderValue): class ContentDisposition(ParameterizedHeaderValue): token_type = 'content-disposition' + as_ew_allowed = False content_disposition = None class ContentTransferEncoding(TokenList): token_type = 'content-transfer-encoding' + as_ew_allowed = False cte = '7bit' class HeaderLabel(TokenList): token_type = 'header-label' + as_ew_allowed = False class Header(TokenList): token_type = 'header' - def _fold(self, folded): - folded.append(str(self.pop(0))) - folded.lastlen = len(folded.current[0]) - # The first line of the header is different from all others: we don't - # want to start a new object on a new line if it has any fold points in - # it that would allow part of it to be on the first header line. - # Further, if the first fold point would fit on the new line, we want - # to do that, but if it doesn't we want to put it on the first line. - # Folded supports this via the stickyspace attribute. If this - # attribute is not None, it does the special handling. - folded.stickyspace = str(self.pop(0)) if self[0].token_type == 'cfws' else '' - rest = self.pop(0) - if self: - raise ValueError("Malformed Header token list") - rest._fold(folded) - # # Terminal classes and instances @@ -1232,6 +844,10 @@ class Header(TokenList): class Terminal(str): + as_ew_allowed = True + ew_combine_allowed = True + syntactic_break = True + def __new__(cls, value, token_type): self = super().__new__(cls, value) self.token_type = token_type @@ -1241,6 +857,9 @@ class Terminal(str): def __repr__(self): return "{}({})".format(self.__class__.__name__, super().__repr__()) + def pprint(self): + print(self.__class__.__name__ + '/' + self.token_type) + @property def all_defects(self): return list(self.defects) @@ -1254,29 +873,14 @@ class Terminal(str): '' if not self.defects else ' {}'.format(self.defects), )] - def cte_encode(self, charset, policy): - value = str(self) - try: - value.encode('us-ascii') - return value - except UnicodeEncodeError: - return _ew.encode(value, charset) - def pop_trailing_ws(self): # This terminates the recursion. return None - def pop_leading_fws(self): - # This terminates the recursion. - return None - @property def comments(self): return [] - def has_leading_comment(self): - return False - def __getnewargs__(self): return(str(self), self.token_type) @@ -1290,8 +894,6 @@ class WhiteSpaceTerminal(Terminal): def startswith_fws(self): return True - has_fws = True - class ValueTerminal(Terminal): @@ -1302,11 +904,6 @@ class ValueTerminal(Terminal): def startswith_fws(self): return False - has_fws = False - - def as_encoded_word(self, charset): - return _ew.encode(str(self), charset) - class EWWhiteSpaceTerminal(WhiteSpaceTerminal): @@ -1314,15 +911,9 @@ class EWWhiteSpaceTerminal(WhiteSpaceTerminal): def value(self): return '' - @property - def encoded(self): - return self[:] - def __str__(self): return '' - has_fws = True - # XXX these need to become classes and used as instances so # that a program can't change them in a parse tree and screw @@ -2752,7 +2343,7 @@ def get_parameter(value): if value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {!r}".format(value)) - appendto.append(ValueTerminal("'", 'RFC2231 delimiter')) + appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) value = value[1:] if value and value[0] != "'": token, value = get_attrtext(value) @@ -2761,7 +2352,7 @@ def get_parameter(value): if not value or value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {}".format(value)) - appendto.append(ValueTerminal("'", 'RFC2231 delimiter')) + appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) value = value[1:] if remainder is not None: # Treat the rest of value as bare quoted string content. @@ -2966,3 +2557,255 @@ def parse_content_transfer_encoding_header(value): token, value = get_phrase(value) cte_header.append(token) return cte_header + + +# +# Header folding +# +# Header folding is complex, with lots of rules and corner cases. The +# following code does its best to obey the rules and handle the corner +# cases, but you can be sure there are few bugs:) +# +# This folder generally canonicalizes as it goes, preferring the stringified +# version of each token. The tokens contain information that supports the +# folder, including which tokens can be encoded in which ways. +# +# Folded text is accumulated in a simple list of strings ('lines'), each +# one of which should be less than policy.max_line_length ('maxlen'). +# + +def _steal_trailing_WSP_if_exists(lines): + wsp = '' + if lines and lines[-1] and lines[-1][-1] in WSP: + wsp = lines[-1][-1] + lines[-1] = lines[-1][:-1] + return wsp + +def _refold_parse_tree(parse_tree, *, policy): + """Return string of contents of parse_tree folded according to RFC rules. + + """ + # max_line_length 0/None means no limit, ie: infinitely long. + maxlen = policy.max_line_length or float("+inf") + encoding = 'utf-8' if policy.utf8 else 'us-ascii' + lines = [''] + last_ew = None + wrap_as_ew_blocked = 0 + want_encoding = False + end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') + parts = list(parse_tree) + while parts: + part = parts.pop(0) + if part is end_ew_not_allowed: + wrap_as_ew_blocked -= 1 + continue + tstr = str(part) + try: + tstr.encode(encoding) + charset = encoding + except UnicodeEncodeError: + if any(isinstance(x, errors.UndecodableBytesDefect) + for x in part.all_defects): + charset = 'unknown-8bit' + else: + # If policy.utf8 is false this should really be taken from a + # 'charset' property on the policy. + charset = 'utf-8' + want_encoding = True + if part.token_type == 'mime-parameters': + # Mime parameter folding (using RFC2231) is extra special. + _fold_mime_parameters(part, lines, maxlen, encoding) + continue + if want_encoding and not wrap_as_ew_blocked: + if not part.as_ew_allowed: + want_encoding = False + last_ew = None + if part.syntactic_break: + encoded_part = part.fold(policy=policy)[:-1] # strip nl + if policy.linesep not in encoded_part: + # It fits on a single line + if len(encoded_part) > maxlen - len(lines[-1]): + # But not on this one, so start a new one. + newline = _steal_trailing_WSP_if_exists(lines) + # XXX what if encoded_part has no leading FWS? + lines.append(newline) + lines[-1] += encoded_part + continue + # Either this is not a major syntactic break, so we don't + # want it on a line by itself even if it fits, or it + # doesn't fit on a line by itself. Either way, fall through + # to unpacking the subparts and wrapping them. + if not hasattr(part, 'encode'): + # It's not a Terminal, do each piece individually. + parts = list(part) + parts + else: + # It's a terminal, wrap it as an encoded word, possibly + # combining it with previously encoded words if allowed. + last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew, + part.ew_combine_allowed, charset) + want_encoding = False + continue + if len(tstr) <= maxlen - len(lines[-1]): + lines[-1] += tstr + continue + # This part is too long to fit. The RFC wants us to break at + # "major syntactic breaks", so unless we don't consider this + # to be one, check if it will fit on the next line by itself. + if (part.syntactic_break and + len(tstr) + 1 <= maxlen): + newline = _steal_trailing_WSP_if_exists(lines) + if newline or part.startswith_fws(): + lines.append(newline + tstr) + continue + if not hasattr(part, 'encode'): + # It's not a terminal, try folding the subparts. + newparts = list(part) + if not part.as_ew_allowed: + wrap_as_ew_blocked += 1 + newparts.append(end_ew_not_allowed) + parts = newparts + parts + continue + if part.as_ew_allowed and not wrap_as_ew_blocked: + # It doesn't need CTE encoding, but encode it anyway so we can + # wrap it. + parts.insert(0, part) + want_encoding = True + continue + # We can't figure out how to wrap, it, so give up. + newline = _steal_trailing_WSP_if_exists(lines) + if newline or part.startswith_fws(): + lines.append(newline + tstr) + else: + # We can't fold it onto the next line either... + lines[-1] += tstr + return policy.linesep.join(lines) + policy.linesep + +def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): + """Fold string to_encode into lines as encoded word, combining if allowed. + Return the new value for last_ew, or None if ew_combine_allowed is False. + + If there is already an encoded word in the last line of lines (indicated by + a non-None value for last_ew) and ew_combine_allowed is true, decode the + existing ew, combine it with to_encode, and re-encode. Otherwise, encode + to_encode. In either case, split to_encode as necessary so that the + encoded segments fit within maxlen. + + """ + if last_ew is not None and ew_combine_allowed: + to_encode = str( + get_unstructured(lines[-1][last_ew:] + to_encode)) + lines[-1] = lines[-1][:last_ew] + if to_encode[0] in WSP: + # We're joining this to non-encoded text, so don't encode + # the leading blank. + leading_wsp = to_encode[0] + to_encode = to_encode[1:] + if (len(lines[-1]) == maxlen): + lines.append(_steal_trailing_WSP_if_exists(lines)) + lines[-1] += leading_wsp + trailing_wsp = '' + if to_encode[-1] in WSP: + # Likewise for the trailing space. + trailing_wsp = to_encode[-1] + to_encode = to_encode[:-1] + new_last_ew = len(lines[-1]) if last_ew is None else last_ew + while to_encode: + remaining_space = maxlen - len(lines[-1]) + # The RFC2047 chrome takes up 7 characters plus the length + # of the charset name. + encode_as = 'utf-8' if charset == 'us-ascii' else charset + text_space = remaining_space - len(encode_as) - 7 + if text_space <= 0: + lines.append(' ') + # XXX We'll get an infinite loop here if maxlen is <= 7 + continue + first_part = to_encode[:text_space] + ew = _ew.encode(first_part, charset=encode_as) + excess = len(ew) - remaining_space + if excess > 0: + # encode always chooses the shortest encoding, so this + # is guaranteed to fit at this point. + first_part = first_part[:-excess] + ew = _ew.encode(first_part) + lines[-1] += ew + to_encode = to_encode[len(first_part):] + if to_encode: + lines.append(' ') + new_last_ew = len(lines[-1]) + lines[-1] += trailing_wsp + return new_last_ew if ew_combine_allowed else None + +def _fold_mime_parameters(part, lines, maxlen, encoding): + """Fold TokenList 'part' into the 'lines' list as mime parameters. + + Using the decoded list of parameters and values, format them according to + the RFC rules, including using RFC2231 encoding if the value cannot be + expressed in 'encoding' and/or the paramter+value is too long to fit within + 'maxlen'. + + """ + # Special case for RFC2231 encoding: start from decoded values and use + # RFC2231 encoding iff needed. + # + # Note that the 1 and 2s being added to the length calculations are + # accounting for the possibly-needed spaces and semicolons we'll be adding. + # + for name, value in part.params: + # XXX What if this ';' puts us over maxlen the first time through the + # loop? We should split the header value onto a newline in that case, + # but to do that we need to recognize the need earlier or reparse the + # header, so I'm going to ignore that bug for now. It'll only put us + # one character over. + if not lines[-1].rstrip().endswith(';'): + lines[-1] += ';' + charset = encoding + error_handler = 'strict' + try: + value.encode(encoding) + encoding_required = False + except UnicodeEncodeError: + encoding_required = True + if utils._has_surrogates(value): + charset = 'unknown-8bit' + error_handler = 'surrogateescape' + else: + charset = 'utf-8' + if encoding_required: + encoded_value = urllib.parse.quote( + value, safe='', errors=error_handler) + tstr = "{}*={}''{}".format(name, charset, encoded_value) + else: + tstr = '{}={}'.format(name, quote_string(value)) + if len(lines[-1]) + len(tstr) + 1 < maxlen: + lines[-1] = lines[-1] + ' ' + tstr + continue + elif len(tstr) + 2 <= maxlen: + lines.append(' ' + tstr) + continue + # We need multiple sections. We are allowed to mix encoded and + # non-encoded sections, but we aren't going to. We'll encode them all. + section = 0 + extra_chrome = charset + "''" + while value: + chrome_len = len(name) + len(str(section)) + 3 + len(extra_chrome) + if maxlen <= chrome_len + 3: + # We need room for the leading blank, the trailing semicolon, + # and at least one character of the value. If we don't + # have that, we'd be stuck, so in that case fall back to + # the RFC standard width. + maxlen = 78 + splitpoint = maxchars = maxlen - chrome_len - 2 + while True: + partial = value[:splitpoint] + encoded_value = urllib.parse.quote( + partial, safe='', errors=error_handler) + if len(encoded_value) <= maxchars: + break + splitpoint -= 1 + lines.append(" {}*{}*={}{}".format( + name, section, extra_chrome, encoded_value)) + extra_chrome = '' + section += 1 + value = value[splitpoint:] + if value: + lines[-1] += ';' diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py index 0fc2231e..f5be87f4 100644 --- a/Lib/email/headerregistry.py +++ b/Lib/email/headerregistry.py @@ -245,13 +245,16 @@ class BaseHeader(str): the header name and the ': ' separator. """ - # At some point we need to only put fws here if it was in the source. + # At some point we need to put fws here iif it was in the source. header = parser.Header([ parser.HeaderLabel([ parser.ValueTerminal(self.name, 'header-name'), parser.ValueTerminal(':', 'header-sep')]), - parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]), - self._parse_tree]) + ]) + if self._parse_tree: + header.append( + parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')])) + header.append(self._parse_tree) return header.fold(policy=policy) diff --git a/Lib/email/utils.py b/Lib/email/utils.py index a759d233..39c22406 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -215,6 +215,12 @@ def parsedate_to_datetime(data): def parseaddr(addr): + """ + Parse addr into its constituent realname and email address parts. + + Return a tuple of realname and email address, unless the parse fails, in + which case return a 2-tuple of ('', ''). + """ addrs = _AddressList(addr).addresslist if not addrs: return '', '' diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 9f5d1510..d69e09fa 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -25,7 +25,7 @@ def _run_pip(args, additional_paths=None): # Install the bundled software import pip - pip.main(args) + return pip.main(args) def version(): @@ -53,6 +53,21 @@ def bootstrap(*, root=None, upgrade=False, user=False, Bootstrap pip into the current Python installation (or the given root directory). + Note that calling this function will alter both sys.path and os.environ. + """ + # Discard the return value + _bootstrap(root=root, upgrade=upgrade, user=user, + altinstall=altinstall, default_pip=default_pip, + verbosity=verbosity) + + +def _bootstrap(*, root=None, upgrade=False, user=False, + altinstall=False, default_pip=False, + verbosity=0): + """ + Bootstrap pip into the current Python installation (or the given root + directory). Returns pip command status code. + Note that calling this function will alter both sys.path and os.environ. """ if altinstall and default_pip: @@ -99,7 +114,7 @@ def bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -126,7 +141,7 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + return _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) def _main(argv=None): @@ -180,7 +195,7 @@ def _main(argv=None): args = parser.parse_args(argv) - bootstrap( + return _bootstrap( root=args.root, upgrade=args.upgrade, user=args.user, diff --git a/Lib/ensurepip/__main__.py b/Lib/ensurepip/__main__.py index 77527d7a..03eef0dd 100644 --- a/Lib/ensurepip/__main__.py +++ b/Lib/ensurepip/__main__.py @@ -1,4 +1,5 @@ import ensurepip +import sys if __name__ == "__main__": - ensurepip._main() + sys.exit(ensurepip._main()) diff --git a/Lib/ensurepip/_uninstall.py b/Lib/ensurepip/_uninstall.py index 750365ec..b2579043 100644 --- a/Lib/ensurepip/_uninstall.py +++ b/Lib/ensurepip/_uninstall.py @@ -2,6 +2,7 @@ import argparse import ensurepip +import sys def _main(argv=None): @@ -23,8 +24,8 @@ def _main(argv=None): args = parser.parse_args(argv) - ensurepip._uninstall_helper(verbosity=args.verbosity) + return ensurepip._uninstall_helper(verbosity=args.verbosity) if __name__ == "__main__": - _main() + sys.exit(_main()) diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index f03e9cde..6e1f0cc9 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,61 @@ Released on 2017-09-25? ======================== +bpo-32164: Delete unused file idlelib/tabbedpages.py. +Use of TabbedPageSet in configdialog was replaced by ttk.Notebook. + +bpo-32100: Fix old and new bugs in pathbrowser; improve tests. +Patch mostly by Cheryl Sabella. + +bpo-31860: The font sample in the settings dialog is now editable. +Edits persist while IDLE remains open. +Patch by Serhiy Storchake and Terry Jan Reedy. + +bpo-31858: Restrict shell prompt manipulaton to the shell. +Editor and output windows only see an empty last prompt line. This +simplifies the code and fixes a minor bug when newline is inserted. +Sys.ps1, if present, is read on Shell start-up, but is not set or changed. +Patch by Terry Jan Reedy. + +bpo-28603: Fix a TypeError that caused a shell restart when printing +a traceback that includes an exception that is unhashable. +Patch by Zane Bitter. + +bpo-13802: Use non-Latin characters in the Font settings sample. +Even if one selects a font that defines a limited subset of the unicode +Basic Multilingual Plane, tcl/tk will use other fonts that define a +character. The expanded example give users of non-Latin characters +a better idea of what they might see in the shell and editors. + +To make room for the expanded sample, frames on the Font tab are +re-arranged. The Font/Tabs help explains a bit about the additions. +Patch by Terry Jan Reedy + +bpo-31460: Simplify the API of IDLE's Module Browser. +Passing a widget instead of an flist with a root widget opens the +option of creating a browser frame that is only part of a window. +Passing a full file name instead of pieces assumed to come from a +.py file opens the possibility of browsing python files that do not +end in .py. + +bpo-31649: Make _htest and _utest parameters keyword-only. +These are used to adjust code for human and unit tests. + +bpo-31459: Rename module browser from Class Browser to Module Browser. +The original module-level class and method browser became a module +browser, with the addition of module-level functions, years ago. +Nested classes and functions were added yesterday. For back- +compatibility, the virtual event <>, which +appears on the Keys tab of the Settings dialog, is not changed. +Patch by Cheryl Sabella. + +bpo-1612262: Module browser now shows nested classes and functions. +Original patches for code and tests by Guilherme Polo and +Cheryl Sabella, respectively. Revisions by Terry Jan Reedy. + +bpo-31500: Tk's default fonts now are scaled on HiDPI displays. +This affects all dialogs. Patch by Serhiy Storchaka. + bpo-31493: Fix code context update and font update timers. Canceling timers prevents a warning message when test_idle completes. @@ -42,7 +97,7 @@ extension.cfg. All take effect as soon as one clicks Apply or Ok. '<>'. Any (global) customizations made before 3.6.3 will not affect their keyset-specific customization after 3.6.3. and vice versa. - Inital patch by Charles Wohlganger, revised by Terry Jan Reedy. + Initial patch by Charles Wohlganger, revised by Terry Jan Reedy. bpo-31051: Rearrange condigdialog General tab. Sort non-Help options into Window (Shell+Editor) and Editor (only). diff --git a/Lib/idlelib/_pyclbr.py b/Lib/idlelib/_pyclbr.py new file mode 100644 index 00000000..e85566db --- /dev/null +++ b/Lib/idlelib/_pyclbr.py @@ -0,0 +1,402 @@ +# A private copy of 3.7.0a1 pyclbr for use by idlelib.browser +"""Parse a Python module and describe its classes and functions. + +Parse enough of a Python file to recognize imports and class and +function definitions, and to find out the superclasses of a class. + +The interface consists of a single function: + readmodule_ex(module, path=None) +where module is the name of a Python module, and path is an optional +list of directories where the module is to be searched. If present, +path is prepended to the system search path sys.path. The return value +is a dictionary. The keys of the dictionary are the names of the +classes and functions defined in the module (including classes that are +defined via the from XXX import YYY construct). The values are +instances of classes Class and Function. One special key/value pair is +present for packages: the key '__path__' has a list as its value which +contains the package search path. + +Classes and Functions have a common superclass: _Object. Every instance +has the following attributes: + module -- name of the module; + name -- name of the object; + file -- file in which the object is defined; + lineno -- line in the file where the object's definition starts; + parent -- parent of this object, if any; + children -- nested objects contained in this object. +The 'children' attribute is a dictionary mapping names to objects. + +Instances of Function describe functions with the attributes from _Object. + +Instances of Class describe classes with the attributes from _Object, +plus the following: + super -- list of super classes (Class instances if possible); + methods -- mapping of method names to beginning line numbers. +If the name of a super class is not recognized, the corresponding +entry in the list of super classes is not a class instance but a +string giving the name of the super class. Since import statements +are recognized and imported modules are scanned as well, this +shouldn't happen often. +""" + +import io +import sys +import importlib.util +import tokenize +from token import NAME, DEDENT, OP + +__all__ = ["readmodule", "readmodule_ex", "Class", "Function"] + +_modules = {} # Initialize cache of modules we've seen. + + +class _Object: + "Informaton about Python class or function." + def __init__(self, module, name, file, lineno, parent): + self.module = module + self.name = name + self.file = file + self.lineno = lineno + self.parent = parent + self.children = {} + + def _addchild(self, name, obj): + self.children[name] = obj + + +class Function(_Object): + "Information about a Python function, including methods." + def __init__(self, module, name, file, lineno, parent=None): + _Object.__init__(self, module, name, file, lineno, parent) + + +class Class(_Object): + "Information about a Python class." + def __init__(self, module, name, super, file, lineno, parent=None): + _Object.__init__(self, module, name, file, lineno, parent) + self.super = [] if super is None else super + self.methods = {} + + def _addmethod(self, name, lineno): + self.methods[name] = lineno + + +def _nest_function(ob, func_name, lineno): + "Return a Function after nesting within ob." + newfunc = Function(ob.module, func_name, ob.file, lineno, ob) + ob._addchild(func_name, newfunc) + if isinstance(ob, Class): + ob._addmethod(func_name, lineno) + return newfunc + +def _nest_class(ob, class_name, lineno, super=None): + "Return a Class after nesting within ob." + newclass = Class(ob.module, class_name, super, ob.file, lineno, ob) + ob._addchild(class_name, newclass) + return newclass + +def readmodule(module, path=None): + """Return Class objects for the top-level classes in module. + + This is the original interface, before Functions were added. + """ + + res = {} + for key, value in _readmodule(module, path or []).items(): + if isinstance(value, Class): + res[key] = value + return res + +def readmodule_ex(module, path=None): + """Return a dictionary with all functions and classes in module. + + Search for module in PATH + sys.path. + If possible, include imported superclasses. + Do this by reading source, without importing (and executing) it. + """ + return _readmodule(module, path or []) + +def _readmodule(module, path, inpackage=None): + """Do the hard work for readmodule[_ex]. + + If inpackage is given, it must be the dotted name of the package in + which we are searching for a submodule, and then PATH must be the + package search path; otherwise, we are searching for a top-level + module, and path is combined with sys.path. + """ + # Compute the full module name (prepending inpackage if set). + if inpackage is not None: + fullmodule = "%s.%s" % (inpackage, module) + else: + fullmodule = module + + # Check in the cache. + if fullmodule in _modules: + return _modules[fullmodule] + + # Initialize the dict for this module's contents. + tree = {} + + # Check if it is a built-in module; we don't do much for these. + if module in sys.builtin_module_names and inpackage is None: + _modules[module] = tree + return tree + + # Check for a dotted module name. + i = module.rfind('.') + if i >= 0: + package = module[:i] + submodule = module[i+1:] + parent = _readmodule(package, path, inpackage) + if inpackage is not None: + package = "%s.%s" % (inpackage, package) + if not '__path__' in parent: + raise ImportError('No package named {}'.format(package)) + return _readmodule(submodule, parent['__path__'], package) + + # Search the path for the module. + f = None + if inpackage is not None: + search_path = path + else: + search_path = path + sys.path + spec = importlib.util._find_spec_from_path(fullmodule, search_path) + _modules[fullmodule] = tree + # Is module a package? + if spec.submodule_search_locations is not None: + tree['__path__'] = spec.submodule_search_locations + try: + source = spec.loader.get_source(fullmodule) + if source is None: + return tree + except (AttributeError, ImportError): + # If module is not Python source, we cannot do anything. + return tree + + fname = spec.loader.get_filename(fullmodule) + return _create_tree(fullmodule, path, fname, source, tree, inpackage) + + +def _create_tree(fullmodule, path, fname, source, tree, inpackage): + """Return the tree for a particular module. + + fullmodule (full module name), inpackage+module, becomes o.module. + path is passed to recursive calls of _readmodule. + fname becomes o.file. + source is tokenized. Imports cause recursive calls to _readmodule. + tree is {} or {'__path__': }. + inpackage, None or string, is passed to recursive calls of _readmodule. + + The effect of recursive calls is mutation of global _modules. + """ + f = io.StringIO(source) + + stack = [] # Initialize stack of (class, indent) pairs. + + g = tokenize.generate_tokens(f.readline) + try: + for tokentype, token, start, _end, _line in g: + if tokentype == DEDENT: + lineno, thisindent = start + # Close previous nested classes and defs. + while stack and stack[-1][1] >= thisindent: + del stack[-1] + elif token == 'def': + lineno, thisindent = start + # Close previous nested classes and defs. + while stack and stack[-1][1] >= thisindent: + del stack[-1] + tokentype, func_name, start = next(g)[0:3] + if tokentype != NAME: + continue # Skip def with syntax error. + cur_func = None + if stack: + cur_obj = stack[-1][0] + cur_func = _nest_function(cur_obj, func_name, lineno) + else: + # It is just a function. + cur_func = Function(fullmodule, func_name, fname, lineno) + tree[func_name] = cur_func + stack.append((cur_func, thisindent)) + elif token == 'class': + lineno, thisindent = start + # Close previous nested classes and defs. + while stack and stack[-1][1] >= thisindent: + del stack[-1] + tokentype, class_name, start = next(g)[0:3] + if tokentype != NAME: + continue # Skip class with syntax error. + # Parse what follows the class name. + tokentype, token, start = next(g)[0:3] + inherit = None + if token == '(': + names = [] # Initialize list of superclasses. + level = 1 + super = [] # Tokens making up current superclass. + while True: + tokentype, token, start = next(g)[0:3] + if token in (')', ',') and level == 1: + n = "".join(super) + if n in tree: + # We know this super class. + n = tree[n] + else: + c = n.split('.') + if len(c) > 1: + # Super class form is module.class: + # look in module for class. + m = c[-2] + c = c[-1] + if m in _modules: + d = _modules[m] + if c in d: + n = d[c] + names.append(n) + super = [] + if token == '(': + level += 1 + elif token == ')': + level -= 1 + if level == 0: + break + elif token == ',' and level == 1: + pass + # Only use NAME and OP (== dot) tokens for type name. + elif tokentype in (NAME, OP) and level == 1: + super.append(token) + # Expressions in the base list are not supported. + inherit = names + if stack: + cur_obj = stack[-1][0] + cur_class = _nest_class( + cur_obj, class_name, lineno, inherit) + else: + cur_class = Class(fullmodule, class_name, inherit, + fname, lineno) + tree[class_name] = cur_class + stack.append((cur_class, thisindent)) + elif token == 'import' and start[1] == 0: + modules = _getnamelist(g) + for mod, _mod2 in modules: + try: + # Recursively read the imported module. + if inpackage is None: + _readmodule(mod, path) + else: + try: + _readmodule(mod, path, inpackage) + except ImportError: + _readmodule(mod, []) + except: + # If we can't find or parse the imported module, + # too bad -- don't die here. + pass + elif token == 'from' and start[1] == 0: + mod, token = _getname(g) + if not mod or token != "import": + continue + names = _getnamelist(g) + try: + # Recursively read the imported module. + d = _readmodule(mod, path, inpackage) + except: + # If we can't find or parse the imported module, + # too bad -- don't die here. + continue + # Add any classes that were defined in the imported module + # to our name space if they were mentioned in the list. + for n, n2 in names: + if n in d: + tree[n2 or n] = d[n] + elif n == '*': + # Don't add names that start with _. + for n in d: + if n[0] != '_': + tree[n] = d[n] + except StopIteration: + pass + + f.close() + return tree + + +def _getnamelist(g): + """Return list of (dotted-name, as-name or None) tuples for token source g. + + An as-name is the name that follows 'as' in an as clause. + """ + names = [] + while True: + name, token = _getname(g) + if not name: + break + if token == 'as': + name2, token = _getname(g) + else: + name2 = None + names.append((name, name2)) + while token != "," and "\n" not in token: + token = next(g)[1] + if token != ",": + break + return names + + +def _getname(g): + "Return (dotted-name or None, next-token) tuple for token source g." + parts = [] + tokentype, token = next(g)[0:2] + if tokentype != NAME and token != '*': + return (None, token) + parts.append(token) + while True: + tokentype, token = next(g)[0:2] + if token != '.': + break + tokentype, token = next(g)[0:2] + if tokentype != NAME: + break + parts.append(token) + return (".".join(parts), token) + + +def _main(): + "Print module output (default this file) for quick visual check." + import os + try: + mod = sys.argv[1] + except: + mod = __file__ + if os.path.exists(mod): + path = [os.path.dirname(mod)] + mod = os.path.basename(mod) + if mod.lower().endswith(".py"): + mod = mod[:-3] + else: + path = [] + tree = readmodule_ex(mod, path) + lineno_key = lambda a: getattr(a, 'lineno', 0) + objs = sorted(tree.values(), key=lineno_key, reverse=True) + indent_level = 2 + while objs: + obj = objs.pop() + if isinstance(obj, list): + # Value is a __path__ key. + continue + if not hasattr(obj, 'indent'): + obj.indent = 0 + + if isinstance(obj, _Object): + new_objs = sorted(obj.children.values(), + key=lineno_key, reverse=True) + for ob in new_objs: + ob.indent = obj.indent + indent_level + objs.extend(new_objs) + if isinstance(obj, Class): + print("{}class {} {} {}" + .format(' ' * obj.indent, obj.name, obj.super, obj.lineno)) + elif isinstance(obj, Function): + print("{}def {} {}".format(' ' * obj.indent, obj.name, obj.lineno)) + +if __name__ == "__main__": + _main() diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 4cf4744f..571dd8fe 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -1,17 +1,16 @@ -"""Class browser. +"""Module browser. XXX TO DO: - reparse when source changed (maybe just a button would be OK?) (or recheck on window popup) - add popup menu with more options (e.g. doc strings, base classes, imports) -- show function argument list? (have to do pattern matching on source) -- should the classes and methods lists also be in the module's menu bar? - add base classes to class browser tree +- finish removing limitation to x.py files (ModuleBrowserTreeItem) """ import os -import pyclbr +from idlelib import _pyclbr as pyclbr import sys from idlelib.config import idleConf @@ -19,23 +18,57 @@ from idlelib import pyshell from idlelib.tree import TreeNode, TreeItem, ScrolledCanvas from idlelib.windows import ListedToplevel + file_open = None # Method...Item and Class...Item use this. # Normally pyshell.flist.open, but there is no pyshell.flist for htest. -class ClassBrowser: + +def transform_children(child_dict, modname=None): + """Transform a child dictionary to an ordered sequence of objects. + + The dictionary maps names to pyclbr information objects. + Filter out imported objects. + Augment class names with bases. + Sort objects by line number. + + The current tree only calls this once per child_dic as it saves + TreeItems once created. A future tree and tests might violate this, + so a check prevents multiple in-place augmentations. + """ + obs = [] # Use list since values should already be sorted. + for key, obj in child_dict.items(): + if modname is None or obj.module == modname: + if hasattr(obj, 'super') and obj.super and obj.name == key: + # If obj.name != key, it has already been suffixed. + supers = [] + for sup in obj.super: + if type(sup) is type(''): + sname = sup + else: + sname = sup.name + if sup.module != obj.module: + sname = f'{sup.module}.{sname}' + supers.append(sname) + obj.name += '({})'.format(', '.join(supers)) + obs.append(obj) + return sorted(obs, key=lambda o: o.lineno) + + +class ModuleBrowser: """Browse module classes and functions in IDLE. """ + # This class is also the base class for pathbrowser.PathBrowser. + # Init and close are inherited, other methods are overridden. + # PathBrowser.__init__ does not call __init__ below. - def __init__(self, flist, name, path, _htest=False): - # XXX This API should change, if the file doesn't end in ".py" - # XXX the code here is bogus! + def __init__(self, master, path, *, _htest=False, _utest=False): """Create a window for browsing a module's structure. Args: - flist: filelist.FileList instance used as the root for the window. - name: Python module to parse. - path: Module search path. - _htest - bool, change box when location running htest. + master: parent for widgets. + path: full path of file to browse. + _htest - bool; change box location when running htest. + -utest - bool; suppress contents when running unittest. Global variables: file_open: Function used for opening a file. @@ -46,56 +79,63 @@ class ClassBrowser: creating ModuleBrowserTreeItem as the rootnode for the tree and subsequently in the children. """ - global file_open - if not _htest: - file_open = pyshell.flist.open - self.name = name - self.file = os.path.join(path[0], self.name + ".py") + self.master = master + self.path = path self._htest = _htest - self.init(flist) + self._utest = _utest + self.init() def close(self, event=None): "Dismiss the window and the tree nodes." self.top.destroy() self.node.destroy() - def init(self, flist): + def init(self): "Create browser tkinter widgets, including the tree." - self.flist = flist - # reset pyclbr + global file_open + root = self.master + flist = (pyshell.flist if not (self._htest or self._utest) + else pyshell.PyShellFileList(root)) + file_open = flist.open pyclbr._modules.clear() + # create top - self.top = top = ListedToplevel(flist.root) + self.top = top = ListedToplevel(root) top.protocol("WM_DELETE_WINDOW", self.close) top.bind("", self.close) if self._htest: # place dialog below parent if running htest top.geometry("+%d+%d" % - (flist.root.winfo_rootx(), flist.root.winfo_rooty() + 200)) + (root.winfo_rootx(), root.winfo_rooty() + 200)) self.settitle() top.focus_set() + # create scrolled canvas theme = idleConf.CurrentTheme() background = idleConf.GetHighlight(theme, 'normal')['background'] - sc = ScrolledCanvas(top, bg=background, highlightthickness=0, takefocus=1) + sc = ScrolledCanvas(top, bg=background, highlightthickness=0, + takefocus=1) sc.frame.pack(expand=1, fill="both") item = self.rootnode() self.node = node = TreeNode(sc.canvas, None, item) - node.update() - node.expand() + if not self._utest: + node.update() + node.expand() def settitle(self): "Set the window title." - self.top.wm_title("Class Browser - " + self.name) - self.top.wm_iconname("Class Browser") + self.top.wm_title("Module Browser - " + os.path.basename(self.path)) + self.top.wm_iconname("Module Browser") def rootnode(self): "Return a ModuleBrowserTreeItem as the root of the tree." - return ModuleBrowserTreeItem(self.file) + return ModuleBrowserTreeItem(self.path) + class ModuleBrowserTreeItem(TreeItem): """Browser tree for Python module. Uses TreeItem as the basis for the structure of the tree. + Used by both browsers. """ def __init__(self, file): @@ -115,16 +155,8 @@ class ModuleBrowserTreeItem(TreeItem): return "python" def GetSubList(self): - """Return the list of ClassBrowserTreeItem items. - - Each item returned from listclasses is the first level of - classes/functions within the module. - """ - sublist = [] - for name in self.listclasses(): - item = ClassBrowserTreeItem(name, self.classes, self.file) - sublist.append(item) - return sublist + "Return ChildBrowserTreeItems for children." + return [ChildBrowserTreeItem(obj) for obj in self.listchildren()] def OnDoubleClick(self): "Open a module in an editor window when double clicked." @@ -132,89 +164,44 @@ class ModuleBrowserTreeItem(TreeItem): return if not os.path.exists(self.file): return - pyshell.flist.open(self.file) + file_open(self.file) def IsExpandable(self): "Return True if Python (.py) file." return os.path.normcase(self.file[-3:]) == ".py" - def listclasses(self): - """Return list of classes and functions in the module. - - The dictionary output from pyclbr is re-written as a - list of tuples in the form (lineno, name) and - then sorted so that the classes and functions are - processed in line number order. The returned list only - contains the name and not the line number. An instance - variable self.classes contains the pyclbr dictionary values, - which are instances of Class and Function. - """ - dir, file = os.path.split(self.file) - name, ext = os.path.splitext(file) + def listchildren(self): + "Return sequenced classes and functions in the module." + dir, base = os.path.split(self.file) + name, ext = os.path.splitext(base) if os.path.normcase(ext) != ".py": return [] try: - dict = pyclbr.readmodule_ex(name, [dir] + sys.path) + tree = pyclbr.readmodule_ex(name, [dir] + sys.path) except ImportError: return [] - items = [] - self.classes = {} - for key, cl in dict.items(): - if cl.module == name: - s = key - if hasattr(cl, 'super') and cl.super: - supers = [] - for sup in cl.super: - if type(sup) is type(''): - sname = sup - else: - sname = sup.name - if sup.module != cl.module: - sname = "%s.%s" % (sup.module, sname) - supers.append(sname) - s = s + "(%s)" % ", ".join(supers) - items.append((cl.lineno, s)) - self.classes[s] = cl - items.sort() - list = [] - for item, s in items: - list.append(s) - return list - -class ClassBrowserTreeItem(TreeItem): - """Browser tree for classes within a module. + return transform_children(tree, name) + + +class ChildBrowserTreeItem(TreeItem): + """Browser tree for child nodes within the module. Uses TreeItem as the basis for the structure of the tree. """ - def __init__(self, name, classes, file): - """Create a TreeItem for the class/function. - - Args: - name: Name of the class/function. - classes: Dictonary of Class/Function instances from pyclbr. - file: Full path and module name. - - Instance variables: - self.cl: Class/Function instance for the class/function name. - self.isfunction: True if self.cl is a Function. - """ - self.name = name - # XXX - Does classes need to be an instance variable? - self.classes = classes - self.file = file - try: - self.cl = self.classes[self.name] - except (IndexError, KeyError): - self.cl = None - self.isfunction = isinstance(self.cl, pyclbr.Function) + def __init__(self, obj): + "Create a TreeItem for a pyclbr class/function object." + self.obj = obj + self.name = obj.name + self.isfunction = isinstance(obj, pyclbr.Function) def GetText(self): "Return the name of the function/class to display." + name = self.name if self.isfunction: - return "def " + self.name + "(...)" + return "def " + name + "(...)" else: - return "class " + self.name + return "class " + name def GetIconName(self): "Return the name of the icon to display." @@ -224,102 +211,38 @@ class ClassBrowserTreeItem(TreeItem): return "folder" def IsExpandable(self): - "Return True if this class has methods." - if self.cl: - try: - return not not self.cl.methods - except AttributeError: - return False - return None + "Return True if self.obj has nested objects." + return self.obj.children != {} def GetSubList(self): - """Return Class methods as a list of MethodBrowserTreeItem items. - - Each item is a method within the class. - """ - if not self.cl: - return [] - sublist = [] - for name in self.listmethods(): - item = MethodBrowserTreeItem(name, self.cl, self.file) - sublist.append(item) - return sublist + "Return ChildBrowserTreeItems for children." + return [ChildBrowserTreeItem(obj) + for obj in transform_children(self.obj.children)] def OnDoubleClick(self): - "Open module with file_open and position to lineno, if it exists." - if not os.path.exists(self.file): - return - edit = file_open(self.file) - if hasattr(self.cl, 'lineno'): - lineno = self.cl.lineno - edit.gotoline(lineno) - - def listmethods(self): - "Return list of methods within a class sorted by lineno." - if not self.cl: - return [] - items = [] - for name, lineno in self.cl.methods.items(): - items.append((lineno, name)) - items.sort() - list = [] - for item, name in items: - list.append(name) - return list - -class MethodBrowserTreeItem(TreeItem): - """Browser tree for methods within a class. - - Uses TreeItem as the basis for the structure of the tree. - """ - - def __init__(self, name, cl, file): - """Create a TreeItem for the methods. - - Args: - name: Name of the class/function. - cl: pyclbr.Class instance for name. - file: Full path and module name. - """ - self.name = name - self.cl = cl - self.file = file - - def GetText(self): - "Return the method name to display." - return "def " + self.name + "(...)" - - def GetIconName(self): - "Return the name of the icon to display." - return "python" - - def IsExpandable(self): - "Return False as there are no tree items after methods." - return False + "Open module with file_open and position to lineno." + try: + edit = file_open(self.obj.file) + edit.gotoline(self.obj.lineno) + except (OSError, AttributeError): + pass - def OnDoubleClick(self): - "Open module with file_open and position at the method start." - if not os.path.exists(self.file): - return - edit = file_open(self.file) - edit.gotoline(self.cl.methods[self.name]) -def _class_browser(parent): # htest # - try: +def _module_browser(parent): # htest # + if len(sys.argv) > 1: # If pass file on command line. + file = sys.argv[1] + else: file = __file__ - except NameError: - file = sys.argv[0] - if sys.argv[1:]: - file = sys.argv[1] - else: - file = sys.argv[0] - dir, file = os.path.split(file) - name = os.path.splitext(file)[0] - flist = pyshell.PyShellFileList(parent) - global file_open - file_open = flist.open - ClassBrowser(flist, name, [dir], _htest=True) + # Add nested objects for htest. + class Nested_in_func(TreeNode): + def nested_in_class(): pass + def closure(): + class Nested_in_closure: pass + ModuleBrowser(parent, file, _htest=True) if __name__ == "__main__": + if len(sys.argv) == 1: # If pass file on command line, unittest fails. + from unittest import main + main('idlelib.idle_test.test_browser', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(_class_browser) + run(_module_browser) diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index 5556b767..3a865f86 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -14,7 +14,7 @@ class GetKeysDialog(Toplevel): keyerror_title = 'Key Sequence Error' def __init__(self, parent, title, action, currentKeySequences, - _htest=False, _utest=False): + *, _htest=False, _utest=False): """ action - string, the name of the virtual event these keys will be mapped to diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 0f530c66..4e8394be 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -11,7 +11,7 @@ Refer to comments in EditorWindow autoindent code for details. """ from tkinter import (Toplevel, Listbox, Text, Scale, Canvas, StringVar, BooleanVar, IntVar, TRUE, FALSE, - TOP, BOTTOM, RIGHT, LEFT, SOLID, GROOVE, NORMAL, DISABLED, + TOP, BOTTOM, RIGHT, LEFT, SOLID, GROOVE, NONE, BOTH, X, Y, W, E, EW, NS, NSEW, NW, HORIZONTAL, VERTICAL, ANCHOR, ACTIVE, END) from tkinter.ttk import (Button, Checkbutton, Entry, Frame, Label, LabelFrame, @@ -25,7 +25,6 @@ from idlelib.config_key import GetKeysDialog from idlelib.dynoption import DynOptionMenu from idlelib import macosx from idlelib.query import SectionName, HelpSource -from idlelib.tabbedpages import TabbedPageSet from idlelib.textview import view_text from idlelib.autocomplete import AutoComplete from idlelib.codecontext import CodeContext @@ -41,7 +40,7 @@ class ConfigDialog(Toplevel): """Config dialog for IDLE. """ - def __init__(self, parent, title='', _htest=False, _utest=False): + def __init__(self, parent, title='', *, _htest=False, _utest=False): """Show the tabbed dialog for user configuration. Args: @@ -105,7 +104,7 @@ class ConfigDialog(Toplevel): load_configs: Load pages except for extensions. activate_config_changes: Tell editors to reload. """ - self.note = note = Notebook(self, width=450, height=450) + self.note = note = Notebook(self) self.highpage = HighPage(note) self.fontpage = FontPage(note, self.highpage) self.keyspage = KeysPage(note) @@ -189,6 +188,11 @@ class ConfigDialog(Toplevel): """ self.destroy() + def destroy(self): + global font_sample_text + font_sample_text = self.fontpage.font_sample.get('1.0', 'end') + super().destroy() + def help(self): """Create textview for config dialog help. @@ -429,6 +433,35 @@ class ConfigDialog(Toplevel): # def other_methods(): # # Define tab-specific behavior. +font_sample_text = ( + '\n' + 'AaBbCcDdEeFfGgHhIiJj\n1234567890#:+=(){}[]\n' + '\u00a2\u00a3\u00a5\u00a7\u00a9\u00ab\u00ae\u00b6\u00bd\u011e' + '\u00c0\u00c1\u00c2\u00c3\u00c4\u00c5\u00c7\u00d0\u00d8\u00df\n' + '\n\n' + '\u0250\u0255\u0258\u025e\u025f\u0264\u026b\u026e\u0270\u0277' + '\u027b\u0281\u0283\u0286\u028e\u029e\u02a2\u02ab\u02ad\u02af\n' + '\u0391\u03b1\u0392\u03b2\u0393\u03b3\u0394\u03b4\u0395\u03b5' + '\u0396\u03b6\u0397\u03b7\u0398\u03b8\u0399\u03b9\u039a\u03ba\n' + '\u0411\u0431\u0414\u0434\u0416\u0436\u041f\u043f\u0424\u0444' + '\u0427\u0447\u042a\u044a\u042d\u044d\u0460\u0464\u046c\u04dc\n' + '\n\n' + '\u05d0\u05d1\u05d2\u05d3\u05d4\u05d5\u05d6\u05d7\u05d8\u05d9' + '\u05da\u05db\u05dc\u05dd\u05de\u05df\u05e0\u05e1\u05e2\u05e3\n' + '\u0627\u0628\u062c\u062f\u0647\u0648\u0632\u062d\u0637\u064a' + '\u0660\u0661\u0662\u0663\u0664\u0665\u0666\u0667\u0668\u0669\n' + '\n\n' + '\u0966\u0967\u0968\u0969\u096a\u096b\u096c\u096d\u096e\u096f' + '\u0905\u0906\u0907\u0908\u0909\u090a\u090f\u0910\u0913\u0914\n' + '\u0be6\u0be7\u0be8\u0be9\u0bea\u0beb\u0bec\u0bed\u0bee\u0bef' + '\u0b85\u0b87\u0b89\u0b8e\n' + '\n\n' + '\u3007\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\n' + '\u6c49\u5b57\u6f22\u5b57\u4eba\u6728\u706b\u571f\u91d1\u6c34\n' + '\uac00\ub0d0\ub354\ub824\ubaa8\ubd64\uc218\uc720\uc988\uce58\n' + '\u3042\u3044\u3046\u3048\u304a\u30a2\u30a4\u30a6\u30a8\u30aa\n' + ) + class FontPage(Frame): @@ -479,8 +512,8 @@ class FontPage(Frame): font_size_title: Label (*)sizelist: DynOptionMenu - font_size (*)bold_toggle: Checkbutton - font_bold - frame_font_sample: Frame - (*)font_sample: Label + frame_sample: LabelFrame + (*)font_sample: Label frame_indent: LabelFrame indent_title: Label (*)indent_scale: Scale - space_num @@ -490,10 +523,12 @@ class FontPage(Frame): self.font_bold = tracers.add(BooleanVar(self), self.var_changed_font) self.space_num = tracers.add(IntVar(self), ('main', 'Indent', 'num-spaces')) - # Create widgets: - # body and body section frames. + # Define frames and widgets. frame_font = LabelFrame( - self, borderwidth=2, relief=GROOVE, text=' Base Editor Font ') + self, borderwidth=2, relief=GROOVE, text=' Shell/Editor Font ') + frame_sample = LabelFrame( + self, borderwidth=2, relief=GROOVE, + text=' Font Sample (Editable) ') frame_indent = LabelFrame( self, borderwidth=2, relief=GROOVE, text=' Indentation Width ') # frame_font. @@ -501,7 +536,7 @@ class FontPage(Frame): frame_font_param = Frame(frame_font) font_name_title = Label( frame_font_name, justify=LEFT, text='Font Face :') - self.fontlist = Listbox(frame_font_name, height=5, + self.fontlist = Listbox(frame_font_name, height=15, takefocus=True, exportselection=FALSE) self.fontlist.bind('', self.on_fontlist_select) self.fontlist.bind('', self.on_fontlist_select) @@ -514,11 +549,9 @@ class FontPage(Frame): self.bold_toggle = Checkbutton( frame_font_param, variable=self.font_bold, onvalue=1, offvalue=0, text='Bold') - frame_font_sample = Frame(frame_font, relief=SOLID, borderwidth=1) - temp_font = tkFont.Font(self, ('courier', 10, 'normal')) - self.font_sample = Label( - frame_font_sample, justify=LEFT, font=temp_font, - text='AaBbCcDdEe\nFfGgHhIiJj\n1234567890\n#:+=(){}[]') + # frame_sample. + self.font_sample = Text(frame_sample, width=20, height=20) + self.font_sample.insert(END, font_sample_text) # frame_indent. indent_title = Label( frame_indent, justify=LEFT, @@ -527,10 +560,12 @@ class FontPage(Frame): frame_indent, variable=self.space_num, orient='horizontal', tickinterval=2, from_=2, to=16) - # Pack widgets: - # body. - frame_font.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) - frame_indent.pack(side=LEFT, padx=5, pady=5, fill=Y) + # Grid and pack widgets: + self.columnconfigure(1, weight=1) + frame_font.grid(row=0, column=0, padx=5, pady=5) + frame_sample.grid(row=0, column=1, rowspan=2, padx=5, pady=5, + sticky='nsew') + frame_indent.grid(row=1, column=0, padx=5, pady=5, sticky='ew') # frame_font. frame_font_name.pack(side=TOP, padx=5, pady=5, fill=X) frame_font_param.pack(side=TOP, padx=5, pady=5, fill=X) @@ -540,10 +575,9 @@ class FontPage(Frame): font_size_title.pack(side=LEFT, anchor=W) self.sizelist.pack(side=LEFT, anchor=W) self.bold_toggle.pack(side=LEFT, anchor=W, padx=20) - frame_font_sample.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + # frame_sample. self.font_sample.pack(expand=TRUE, fill=BOTH) # frame_indent. - frame_indent.pack(side=TOP, fill=X) indent_title.pack(side=TOP, anchor=W, padx=5) self.indent_scale.pack(side=TOP, padx=5, fill=X) @@ -1408,7 +1442,7 @@ class KeysPage(Frame): self.bindingslist['xscrollcommand'] = scroll_target_x.set self.button_new_keys = Button( frame_custom, text='Get New Keys for Selection', - command=self.get_new_keys, state=DISABLED) + command=self.get_new_keys, state='disabled') # frame_key_sets. frames = [Frame(frame_key_sets, padding=2, borderwidth=0) for i in range(2)] @@ -2108,10 +2142,26 @@ When you click either the Apply or Ok buttons, settings in this dialog that are different from IDLE's default are saved in a .idlerc directory in your home directory. Except as noted, these changes apply to all versions of IDLE installed on this -machine. Some do not take affect until IDLE is restarted. -[Cancel] only cancels changes made since the last save. +machine. [Cancel] only cancels changes made since the last save. ''' help_pages = { + 'Fonts/Tabs':''' +Font sample: This shows what a selection of Basic Multilingual Plane +unicode characters look like for the current font selection. If the +selected font does not define a character, Tk attempts to find another +font that does. Substitute glyphs depend on what is available on a +particular system and will not necessarily have the same size as the +font selected. Line contains 20 characters up to Devanagari, 14 for +Tamil, and 10 for East Asia. + +Hebrew and Arabic letters should display right to left, starting with +alef, \u05d0 and \u0627. Arabic digits display left to right. The +Devanagari and Tamil lines start with digits. The East Asian lines +are Chinese digits, Chinese Hanzi, Korean Hangul, and Japanese +Hiragana and Katakana. + +You can edit the font sample. Changes remain until IDLE is closed. +''', 'Highlights': ''' Highlighting: The IDLE Dark color theme is new in October 2015. It can only diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 855d3750..b51c45c9 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -99,10 +99,6 @@ class EditorWindow(object): self.flist = flist root = root or flist.root self.root = root - try: - sys.ps1 - except AttributeError: - sys.ps1 = '>>> ' self.menubar = Menu(root) self.top = top = windows.ListedToplevel(root, menu=self.menubar) if flist: @@ -116,6 +112,8 @@ class EditorWindow(object): self.top.instance_dict = {} self.recent_files_path = os.path.join( idleConf.userdir, 'recent-files.lst') + + self.prompt_last_line = '' # Override in PyShell self.text_frame = text_frame = Frame(top) self.vbar = vbar = Scrollbar(text_frame, name='vbar') self.width = idleConf.GetOption('main', 'EditorWindow', @@ -190,7 +188,7 @@ class EditorWindow(object): flist.dict[key] = self text.bind("<>", self.new_callback) text.bind("<>", self.flist.close_all_callback) - text.bind("<>", self.open_class_browser) + text.bind("<>", self.open_module_browser) text.bind("<>", self.open_path_browser) text.bind("<>", self.open_turtle_demo) @@ -632,10 +630,10 @@ class EditorWindow(object): def open_module(self): """Get module name from user and open it. - Return module path or None for calls by open_class_browser + Return module path or None for calls by open_module_browser when latter is not invoked in named editor window. """ - # XXX This, open_class_browser, and open_path_browser + # XXX This, open_module_browser, and open_path_browser # would fit better in iomenu.IOBinding. try: name = self.text.get("sel.first", "sel.last").strip() @@ -657,22 +655,20 @@ class EditorWindow(object): self.open_module() return "break" - def open_class_browser(self, event=None): + def open_module_browser(self, event=None): filename = self.io.filename if not (self.__class__.__name__ == 'PyShellEditorWindow' and filename): filename = self.open_module() if filename is None: return "break" - head, tail = os.path.split(filename) - base, ext = os.path.splitext(tail) from idlelib import browser - browser.ClassBrowser(self.flist, base, [head]) + browser.ModuleBrowser(self.root, filename) return "break" def open_path_browser(self, event=None): from idlelib import pathbrowser - pathbrowser.PathBrowser(self.flist) + pathbrowser.PathBrowser(self.root) return "break" def open_turtle_demo(self, event = None): @@ -1215,13 +1211,9 @@ class EditorWindow(object): assert have > 0 want = ((have - 1) // self.indentwidth) * self.indentwidth # Debug prompt is multilined.... - if self.context_use_ps1: - last_line_of_prompt = sys.ps1.split('\n')[-1] - else: - last_line_of_prompt = '' ncharsdeleted = 0 while 1: - if chars == last_line_of_prompt: + if chars == self.prompt_last_line: # '' unless PyShell break chars = chars[:-1] ncharsdeleted = ncharsdeleted + 1 @@ -1290,8 +1282,7 @@ class EditorWindow(object): indent = line[:i] # strip whitespace before insert point unless it's in the prompt i = 0 - last_line_of_prompt = sys.ps1.split('\n')[-1] - while line and line[-1] in " \t" and line != last_line_of_prompt: + while line and line[-1] in " \t" and line != self.prompt_last_line: line = line[:-1] i = i+1 if i: diff --git a/Lib/idlelib/filelist.py b/Lib/idlelib/filelist.py index f46ad7cd..5e1a3dcd 100644 --- a/Lib/idlelib/filelist.py +++ b/Lib/idlelib/filelist.py @@ -113,8 +113,10 @@ class FileList: def _test(): from idlelib.editor import fixwordbreaks + from idlelib.run import fix_scaling import sys root = Tk() + fix_scaling(root) fixwordbreaks(root) root.withdraw() flist = FileList(root) diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index 679ac78d..77b4b189 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -23,7 +23,7 @@ class AboutDialog(Toplevel): """Modal about dialog for idle """ - def __init__(self, parent, title=None, _htest=False, _utest=False): + def __init__(self, parent, title=None, *, _htest=False, _utest=False): """Create popup, do not return until tk widget destroyed. parent - parent of this dialog diff --git a/Lib/idlelib/idle_test/README.txt b/Lib/idlelib/idle_test/README.txt index c580fb9e..5f3678fc 100644 --- a/Lib/idlelib/idle_test/README.txt +++ b/Lib/idlelib/idle_test/README.txt @@ -144,9 +144,18 @@ green. Idle tests must not disturb the environment in a way that makes other tests fail (issue 18081). To run an individual Testcase or test method, extend the dotted name -given to unittest on the command line. +given to unittest on the command line or use the test -m option. The +latter allows use of other regrtest options. When using the latter, +all components of the pattern must be present, but any can be replaced +by '*'. python -m unittest -v idlelib.idle_test.test_xyz.Test_case.test_meth +python -m test -m idlelib.idle_test.text_xyz.Test_case.test_meth test_idle + +The test suite can be run in an IDLE user process from Shell. +>>> import test.autotest # Issue 25588, 2017/10/13, 3.6.4, 3.7.0a2. +There are currently failures not usually present, and this does not +work when run from the editor. 4. Human-mediated Tests diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index e483bbc9..442f55e2 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -86,7 +86,7 @@ _calltip_window_spec = { "Typing ') should hide the calltip.\n" } -_class_browser_spec = { +_module_browser_spec = { 'file': 'browser', 'kwds': {}, 'msg': "Inspect names of module, class(with superclass if " diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py new file mode 100644 index 00000000..05eb4718 --- /dev/null +++ b/Lib/idlelib/idle_test/test_browser.py @@ -0,0 +1,253 @@ +""" Test idlelib.browser. + +Coverage: 88% +(Higher, because should exclude 3 lines that .coveragerc won't exclude.) +""" + +from collections import deque +import os.path +from idlelib import _pyclbr as pyclbr +from tkinter import Tk + +from test.support import requires +import unittest +from unittest import mock +from idlelib.idle_test.mock_idle import Func + +from idlelib import browser +from idlelib import filelist +from idlelib.tree import TreeNode + + +class ModuleBrowserTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.mb = browser.ModuleBrowser(cls.root, __file__, _utest=True) + + @classmethod + def tearDownClass(cls): + cls.mb.close() + cls.root.update_idletasks() + cls.root.destroy() + del cls.root, cls.mb + + def test_init(self): + mb = self.mb + eq = self.assertEqual + eq(mb.path, __file__) + eq(pyclbr._modules, {}) + self.assertIsInstance(mb.node, TreeNode) + self.assertIsNotNone(browser.file_open) + + def test_settitle(self): + mb = self.mb + self.assertIn(os.path.basename(__file__), mb.top.title()) + self.assertEqual(mb.top.iconname(), 'Module Browser') + + def test_rootnode(self): + mb = self.mb + rn = mb.rootnode() + self.assertIsInstance(rn, browser.ModuleBrowserTreeItem) + + def test_close(self): + mb = self.mb + mb.top.destroy = Func() + mb.node.destroy = Func() + mb.close() + self.assertTrue(mb.top.destroy.called) + self.assertTrue(mb.node.destroy.called) + del mb.top.destroy, mb.node.destroy + + +# Nested tree same as in test_pyclbr.py except for supers on C0. C1. +mb = pyclbr +module, fname = 'test', 'test.py' +f0 = mb.Function(module, 'f0', fname, 1) +f1 = mb._nest_function(f0, 'f1', 2) +f2 = mb._nest_function(f1, 'f2', 3) +c1 = mb._nest_class(f0, 'c1', 5) +C0 = mb.Class(module, 'C0', ['base'], fname, 6) +F1 = mb._nest_function(C0, 'F1', 8) +C1 = mb._nest_class(C0, 'C1', 11, ['']) +C2 = mb._nest_class(C1, 'C2', 12) +F3 = mb._nest_function(C2, 'F3', 14) +mock_pyclbr_tree = {'f0': f0, 'C0': C0} + +# Adjust C0.name, C1.name so tests do not depend on order. +browser.transform_children(mock_pyclbr_tree, 'test') # C0(base) +browser.transform_children(C0.children) # C1() + +# The class below checks that the calls above are correct +# and that duplicate calls have no effect. + + +class TransformChildrenTest(unittest.TestCase): + + def test_transform_module_children(self): + eq = self.assertEqual + transform = browser.transform_children + # Parameter matches tree module. + tcl = list(transform(mock_pyclbr_tree, 'test')) + eq(tcl, [f0, C0]) + eq(tcl[0].name, 'f0') + eq(tcl[1].name, 'C0(base)') + # Check that second call does not change suffix. + tcl = list(transform(mock_pyclbr_tree, 'test')) + eq(tcl[1].name, 'C0(base)') + # Nothing to traverse if parameter name isn't same as tree module. + tcl = list(transform(mock_pyclbr_tree, 'different name')) + eq(tcl, []) + + def test_transform_node_children(self): + eq = self.assertEqual + transform = browser.transform_children + # Class with two children, one name altered. + tcl = list(transform(C0.children)) + eq(tcl, [F1, C1]) + eq(tcl[0].name, 'F1') + eq(tcl[1].name, 'C1()') + tcl = list(transform(C0.children)) + eq(tcl[1].name, 'C1()') + # Function with two children. + eq(list(transform(f0.children)), [f1, c1]) + + +class ModuleBrowserTreeItemTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.mbt = browser.ModuleBrowserTreeItem(fname) + + def test_init(self): + self.assertEqual(self.mbt.file, fname) + + def test_gettext(self): + self.assertEqual(self.mbt.GetText(), fname) + + def test_geticonname(self): + self.assertEqual(self.mbt.GetIconName(), 'python') + + def test_isexpandable(self): + self.assertTrue(self.mbt.IsExpandable()) + + def test_listchildren(self): + save_rex = browser.pyclbr.readmodule_ex + save_tc = browser.transform_children + browser.pyclbr.readmodule_ex = Func(result=mock_pyclbr_tree) + browser.transform_children = Func(result=[f0, C0]) + try: + self.assertEqual(self.mbt.listchildren(), [f0, C0]) + finally: + browser.pyclbr.readmodule_ex = save_rex + browser.transform_children = save_tc + + def test_getsublist(self): + mbt = self.mbt + mbt.listchildren = Func(result=[f0, C0]) + sub0, sub1 = mbt.GetSubList() + del mbt.listchildren + self.assertIsInstance(sub0, browser.ChildBrowserTreeItem) + self.assertIsInstance(sub1, browser.ChildBrowserTreeItem) + self.assertEqual(sub0.name, 'f0') + self.assertEqual(sub1.name, 'C0(base)') + + @mock.patch('idlelib.browser.file_open') + def test_ondoubleclick(self, fopen): + mbt = self.mbt + + with mock.patch('os.path.exists', return_value=False): + mbt.OnDoubleClick() + fopen.assert_not_called() + + with mock.patch('os.path.exists', return_value=True): + mbt.OnDoubleClick() + fopen.assert_called() + fopen.called_with(fname) + + +class ChildBrowserTreeItemTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + CBT = browser.ChildBrowserTreeItem + cls.cbt_f1 = CBT(f1) + cls.cbt_C1 = CBT(C1) + cls.cbt_F1 = CBT(F1) + + @classmethod + def tearDownClass(cls): + del cls.cbt_C1, cls.cbt_f1, cls.cbt_F1 + + def test_init(self): + eq = self.assertEqual + eq(self.cbt_C1.name, 'C1()') + self.assertFalse(self.cbt_C1.isfunction) + eq(self.cbt_f1.name, 'f1') + self.assertTrue(self.cbt_f1.isfunction) + + def test_gettext(self): + self.assertEqual(self.cbt_C1.GetText(), 'class C1()') + self.assertEqual(self.cbt_f1.GetText(), 'def f1(...)') + + def test_geticonname(self): + self.assertEqual(self.cbt_C1.GetIconName(), 'folder') + self.assertEqual(self.cbt_f1.GetIconName(), 'python') + + def test_isexpandable(self): + self.assertTrue(self.cbt_C1.IsExpandable()) + self.assertTrue(self.cbt_f1.IsExpandable()) + self.assertFalse(self.cbt_F1.IsExpandable()) + + def test_getsublist(self): + eq = self.assertEqual + CBT = browser.ChildBrowserTreeItem + + f1sublist = self.cbt_f1.GetSubList() + self.assertIsInstance(f1sublist[0], CBT) + eq(len(f1sublist), 1) + eq(f1sublist[0].name, 'f2') + + eq(self.cbt_F1.GetSubList(), []) + + @mock.patch('idlelib.browser.file_open') + def test_ondoubleclick(self, fopen): + goto = fopen.return_value.gotoline = mock.Mock() + self.cbt_F1.OnDoubleClick() + fopen.assert_called() + goto.assert_called() + goto.assert_called_with(self.cbt_F1.obj.lineno) + # Failure test would have to raise OSError or AttributeError. + + +class NestedChildrenTest(unittest.TestCase): + "Test that all the nodes in a nested tree are added to the BrowserTree." + + def test_nested(self): + queue = deque() + actual_names = [] + # The tree items are processed in breadth first order. + # Verify that processing each sublist hits every node and + # in the right order. + expected_names = ['f0', 'C0(base)', + 'f1', 'c1', 'F1', 'C1()', + 'f2', 'C2', + 'F3'] + CBT = browser.ChildBrowserTreeItem + queue.extend((CBT(f0), CBT(C0))) + while queue: + cb = queue.popleft() + sublist = cb.GetSubList() + queue.extend(sublist) + self.assertIn(cb.name, cb.GetText()) + self.assertIn(cb.GetIconName(), ('python', 'folder')) + self.assertIs(cb.IsExpandable(), sublist != []) + actual_names.append(cb.name) + self.assertEqual(actual_names, expected_names) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index dc7f69c2..982dc0b7 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -44,10 +44,9 @@ def tearDownModule(): tracers.detach() tracers.clear() changes.clear() - del dialog root.update_idletasks() root.destroy() - del root + root = dialog = None class FontPageTest(unittest.TestCase): @@ -192,6 +191,7 @@ class FontPageTest(unittest.TestCase): def test_set_samples(self): d = self.page del d.set_samples # Unmask method for test + orig_samples = d.font_sample, d.highlight_sample d.font_sample, d.highlight_sample = {}, {} d.font_name.set('test') d.font_size.set('5') @@ -202,7 +202,7 @@ class FontPageTest(unittest.TestCase): d.set_samples() self.assertTrue(d.font_sample == d.highlight_sample == expected) - del d.font_sample, d.highlight_sample + d.font_sample, d.highlight_sample = orig_samples d.set_samples = Func() # Re-mask for other tests. diff --git a/Lib/idlelib/idle_test/test_pathbrowser.py b/Lib/idlelib/idle_test/test_pathbrowser.py index 813cbcc6..74b716a3 100644 --- a/Lib/idlelib/idle_test/test_pathbrowser.py +++ b/Lib/idlelib/idle_test/test_pathbrowser.py @@ -1,11 +1,68 @@ +""" Test idlelib.pathbrowser. +""" + + +import os.path +import pyclbr # for _modules +import sys # for sys.path +from tkinter import Tk + +from test.support import requires import unittest -import os -import sys -import idlelib +from idlelib.idle_test.mock_idle import Func + +import idlelib # for __file__ +from idlelib import browser from idlelib import pathbrowser +from idlelib.tree import TreeNode + class PathBrowserTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.pb = pathbrowser.PathBrowser(cls.root, _utest=True) + + @classmethod + def tearDownClass(cls): + cls.pb.close() + cls.root.update_idletasks() + cls.root.destroy() + del cls.root, cls.pb + + def test_init(self): + pb = self.pb + eq = self.assertEqual + eq(pb.master, self.root) + eq(pyclbr._modules, {}) + self.assertIsInstance(pb.node, TreeNode) + self.assertIsNotNone(browser.file_open) + + def test_settitle(self): + pb = self.pb + self.assertEqual(pb.top.title(), 'Path Browser') + self.assertEqual(pb.top.iconname(), 'Path Browser') + + def test_rootnode(self): + pb = self.pb + rn = pb.rootnode() + self.assertIsInstance(rn, pathbrowser.PathBrowserTreeItem) + + def test_close(self): + pb = self.pb + pb.top.destroy = Func() + pb.node.destroy = Func() + pb.close() + self.assertTrue(pb.top.destroy.called) + self.assertTrue(pb.node.destroy.called) + del pb.top.destroy, pb.node.destroy + + +class DirBrowserTreeItemTest(unittest.TestCase): + def test_DirBrowserTreeItem(self): # Issue16226 - make sure that getting a sublist works d = pathbrowser.DirBrowserTreeItem('') @@ -16,6 +73,9 @@ class PathBrowserTest(unittest.TestCase): self.assertEqual(d.ispackagedir(dir), True) self.assertEqual(d.ispackagedir(dir + '/Icons'), False) + +class PathBrowserTreeItemTest(unittest.TestCase): + def test_PathBrowserTreeItem(self): p = pathbrowser.PathBrowserTreeItem() self.assertEqual(p.GetText(), 'sys.path') @@ -23,5 +83,6 @@ class PathBrowserTest(unittest.TestCase): self.assertEqual(len(sub), len(sys.path)) self.assertEqual(type(sub[0]), pathbrowser.DirBrowserTreeItem) + if __name__ == '__main__': unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_replace.py b/Lib/idlelib/idle_test/test_replace.py index 2ecbd341..df76dec3 100644 --- a/Lib/idlelib/idle_test/test_replace.py +++ b/Lib/idlelib/idle_test/test_replace.py @@ -74,14 +74,14 @@ class ReplaceDialogTest(unittest.TestCase): replace() equal(text.get('1.8', '1.12'), 'asdf') - # dont "match word" case + # don't "match word" case text.mark_set('insert', '1.0') pv.set('is') rv.set('hello') replace() equal(text.get('1.2', '1.7'), 'hello') - # dont "match case" case + # don't "match case" case pv.set('string') rv.set('world') replace() diff --git a/Lib/idlelib/idle_test/test_run.py b/Lib/idlelib/idle_test/test_run.py new file mode 100644 index 00000000..d7e627d2 --- /dev/null +++ b/Lib/idlelib/idle_test/test_run.py @@ -0,0 +1,35 @@ +import unittest +from unittest import mock + +from test.support import captured_stderr +import idlelib.run as idlerun + + +class RunTest(unittest.TestCase): + def test_print_exception_unhashable(self): + class UnhashableException(Exception): + def __eq__(self, other): + return True + + ex1 = UnhashableException('ex1') + ex2 = UnhashableException('ex2') + try: + raise ex2 from ex1 + except UnhashableException: + try: + raise ex1 + except UnhashableException: + with captured_stderr() as output: + with mock.patch.object(idlerun, + 'cleanup_traceback') as ct: + ct.side_effect = lambda t, e: t + idlerun.print_exception() + + tb = output.getvalue().strip().splitlines() + self.assertEqual(11, len(tb)) + self.assertIn('UnhashableException: ex2', tb[3]) + self.assertIn('UnhashableException: ex1', tb[10]) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index 3414c7b3..f9b6907b 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -487,11 +487,11 @@ class IOBinding: opendialog = None savedialog = None - filetypes = [ + filetypes = ( ("Python files", "*.py *.pyw", "TEXT"), ("Text files", "*.txt", "TEXT"), ("All files", "*"), - ] + ) defaultextension = '.py' if sys.platform == 'darwin' else '' diff --git a/Lib/idlelib/mainmenu.py b/Lib/idlelib/mainmenu.py index d1dcb83d..143570d6 100644 --- a/Lib/idlelib/mainmenu.py +++ b/Lib/idlelib/mainmenu.py @@ -25,7 +25,7 @@ menudefs = [ ('_New File', '<>'), ('_Open...', '<>'), ('Open _Module...', '<>'), - ('Class _Browser', '<>'), + ('Module _Browser', '<>'), ('_Path Browser', '<>'), None, ('_Save', '<>'), diff --git a/Lib/idlelib/paragraph.py b/Lib/idlelib/paragraph.py index cf8dfdb6..1270115a 100644 --- a/Lib/idlelib/paragraph.py +++ b/Lib/idlelib/paragraph.py @@ -158,7 +158,7 @@ def reformat_comment(data, limit, comment_header): newdata = reformat_paragraph(data, format_width) # re-split and re-insert the comment header. newdata = newdata.split("\n") - # If the block ends in a \n, we dont want the comment prefix + # If the block ends in a \n, we don't want the comment prefix # inserted after it. (Im not sure it makes sense to reformat a # comment block that is not made of complete lines, but whatever!) # Can't think of a clean solution, so we hack away diff --git a/Lib/idlelib/pathbrowser.py b/Lib/idlelib/pathbrowser.py index 6c19508d..6de242d0 100644 --- a/Lib/idlelib/pathbrowser.py +++ b/Lib/idlelib/pathbrowser.py @@ -2,19 +2,20 @@ import importlib.machinery import os import sys -from idlelib.browser import ClassBrowser, ModuleBrowserTreeItem -from idlelib.pyshell import PyShellFileList +from idlelib.browser import ModuleBrowser, ModuleBrowserTreeItem from idlelib.tree import TreeItem -class PathBrowser(ClassBrowser): +class PathBrowser(ModuleBrowser): - def __init__(self, flist, _htest=False): + def __init__(self, master, *, _htest=False, _utest=False): """ _htest - bool, change box location when running htest """ + self.master = master self._htest = _htest - self.init(flist) + self._utest = _utest + self.init() def settitle(self): "Set window titles." @@ -99,8 +100,7 @@ class DirBrowserTreeItem(TreeItem): def _path_browser(parent): # htest # - flist = PyShellFileList(parent) - PathBrowser(flist, _htest=True) + PathBrowser(parent, _htest=True) parent.mainloop() if __name__ == "__main__": diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 47df7443..8b07d52c 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -12,6 +12,8 @@ import tkinter.messagebox as tkMessageBox if TkVersion < 8.5: root = Tk() # otherwise create root in main root.withdraw() + from idlelib.run import fix_scaling + fix_scaling(root) tkMessageBox.showerror("Idle Cannot Start", "Idle requires tcl/tk 8.5+, not %s." % TkVersion, parent=root) @@ -855,15 +857,17 @@ class PyShell(OutputWindow): fixwordbreaks(root) root.withdraw() flist = PyShellFileList(root) - # + OutputWindow.__init__(self, flist, None, None) - # -## self.config(usetabs=1, indentwidth=8, context_use_ps1=1) + self.usetabs = True # indentwidth must be 8 when using tabs. See note in EditorWindow: self.indentwidth = 8 - self.context_use_ps1 = True - # + + self.sys_ps1 = sys.ps1 if hasattr(sys, 'ps1') else '>>> ' + self.prompt_last_line = self.sys_ps1.split('\n')[-1] + self.prompt = self.sys_ps1 # Changes when debug active + text = self.text text.configure(wrap="char") text.bind("<>", self.enter_callback) @@ -876,7 +880,7 @@ class PyShell(OutputWindow): if use_subprocess: text.bind("<>", self.view_restart_mark) text.bind("<>", self.restart_shell) - # + self.save_stdout = sys.stdout self.save_stderr = sys.stderr self.save_stdin = sys.stdin @@ -949,7 +953,7 @@ class PyShell(OutputWindow): debugger_r.close_remote_debugger(self.interp.rpcclt) self.resetoutput() self.console.write("[DEBUG OFF]\n") - sys.ps1 = ">>> " + self.prompt = self.sys_ps1 self.showprompt() self.set_debugger_indicator() @@ -961,7 +965,7 @@ class PyShell(OutputWindow): dbg_gui = debugger.Debugger(self) self.interp.setdebugger(dbg_gui) dbg_gui.load_breakpoints() - sys.ps1 = "[DEBUG ON]\n>>> " + self.prompt = "[DEBUG ON]\n" + self.sys_ps1 self.showprompt() self.set_debugger_indicator() @@ -1246,11 +1250,7 @@ class PyShell(OutputWindow): def showprompt(self): self.resetoutput() - try: - s = str(sys.ps1) - except: - s = "" - self.console.write(s) + self.console.write(self.prompt) self.text.mark_set("insert", "end-1c") self.set_line_and_column() self.io.reset_undo() @@ -1457,6 +1457,8 @@ def main(): NoDefaultRoot() root = Tk(className="Idle") root.withdraw() + from idlelib.run import fix_scaling + fix_scaling(root) # set application icon icondir = os.path.join(os.path.dirname(__file__), 'Icons') diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 9f6604bb..176fe3db 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -134,13 +134,17 @@ def main(del_exitfunc=False): # exiting but got an extra KBI? Try again! continue try: - seq, request = rpc.request_queue.get(block=True, timeout=0.05) + request = rpc.request_queue.get(block=True, timeout=0.05) except queue.Empty: + request = None + # Issue 32207: calling handle_tk_events here adds spurious + # queue.Empty traceback to event handling exceptions. + if request: + seq, (method, args, kwargs) = request + ret = method(*args, **kwargs) + rpc.response_queue.put((seq, ret)) + else: handle_tk_events() - continue - method, args, kwargs = request - ret = method(*args, **kwargs) - rpc.response_queue.put((seq, ret)) except KeyboardInterrupt: if quitting: exit_now = True @@ -184,6 +188,7 @@ def show_socket_error(err, address): import tkinter from tkinter.messagebox import showerror root = tkinter.Tk() + fix_scaling(root) root.withdraw() msg = f"IDLE's subprocess can't connect to {address[0]}:{address[1]}.\n"\ f"Fatal OSError #{err.errno}: {err.strerror}.\n"\ @@ -202,16 +207,16 @@ def print_exception(): seen = set() def print_exc(typ, exc, tb): - seen.add(exc) + seen.add(id(exc)) context = exc.__context__ cause = exc.__cause__ - if cause is not None and cause not in seen: + if cause is not None and id(cause) not in seen: print_exc(type(cause), cause, cause.__traceback__) print("\nThe above exception was the direct cause " "of the following exception:\n", file=efile) elif (context is not None and not exc.__suppress_context__ and - context not in seen): + id(context) not in seen): print_exc(type(context), context, context.__traceback__) print("\nDuring handling of the above exception, " "another exception occurred:\n", file=efile) @@ -277,6 +282,18 @@ def exit(): sys.exit(0) +def fix_scaling(root): + """Scale fonts on HiDPI displays.""" + import tkinter.font + scaling = float(root.tk.call('tk', 'scaling')) + if scaling > 1.4: + for name in tkinter.font.names(root): + font = tkinter.font.Font(root=root, name=name, exists=True) + size = int(font['size']) + if size < 0: + font['size'] = round(-0.75*size) + + class MyRPCServer(rpc.RPCServer): def handle_error(self, request, client_address): diff --git a/Lib/idlelib/tabbedpages.py b/Lib/idlelib/tabbedpages.py deleted file mode 100644 index 4186fa20..00000000 --- a/Lib/idlelib/tabbedpages.py +++ /dev/null @@ -1,498 +0,0 @@ -"""An implementation of tabbed pages using only standard Tkinter. - -Originally developed for use in IDLE. Based on tabpage.py. - -Classes exported: -TabbedPageSet -- A Tkinter implementation of a tabbed-page widget. -TabSet -- A widget containing tabs (buttons) in one or more rows. - -""" -from tkinter import * - -class InvalidNameError(Exception): pass -class AlreadyExistsError(Exception): pass - - -class TabSet(Frame): - """A widget containing tabs (buttons) in one or more rows. - - Only one tab may be selected at a time. - - """ - def __init__(self, page_set, select_command, - tabs=None, n_rows=1, max_tabs_per_row=5, - expand_tabs=False, **kw): - """Constructor arguments: - - select_command -- A callable which will be called when a tab is - selected. It is called with the name of the selected tab as an - argument. - - tabs -- A list of strings, the names of the tabs. Should be specified in - the desired tab order. The first tab will be the default and first - active tab. If tabs is None or empty, the TabSet will be initialized - empty. - - n_rows -- Number of rows of tabs to be shown. If n_rows <= 0 or is - None, then the number of rows will be decided by TabSet. See - _arrange_tabs() for details. - - max_tabs_per_row -- Used for deciding how many rows of tabs are needed, - when the number of rows is not constant. See _arrange_tabs() for - details. - - """ - Frame.__init__(self, page_set, **kw) - self.select_command = select_command - self.n_rows = n_rows - self.max_tabs_per_row = max_tabs_per_row - self.expand_tabs = expand_tabs - self.page_set = page_set - - self._tabs = {} - self._tab2row = {} - if tabs: - self._tab_names = list(tabs) - else: - self._tab_names = [] - self._selected_tab = None - self._tab_rows = [] - - self.padding_frame = Frame(self, height=2, - borderwidth=0, relief=FLAT, - background=self.cget('background')) - self.padding_frame.pack(side=TOP, fill=X, expand=False) - - self._arrange_tabs() - - def add_tab(self, tab_name): - """Add a new tab with the name given in tab_name.""" - if not tab_name: - raise InvalidNameError("Invalid Tab name: '%s'" % tab_name) - if tab_name in self._tab_names: - raise AlreadyExistsError("Tab named '%s' already exists" %tab_name) - - self._tab_names.append(tab_name) - self._arrange_tabs() - - def remove_tab(self, tab_name): - """Remove the tab named """ - if not tab_name in self._tab_names: - raise KeyError("No such Tab: '%s" % tab_name) - - self._tab_names.remove(tab_name) - self._arrange_tabs() - - def set_selected_tab(self, tab_name): - """Show the tab named as the selected one""" - if tab_name == self._selected_tab: - return - if tab_name is not None and tab_name not in self._tabs: - raise KeyError("No such Tab: '%s" % tab_name) - - # deselect the current selected tab - if self._selected_tab is not None: - self._tabs[self._selected_tab].set_normal() - self._selected_tab = None - - if tab_name is not None: - # activate the tab named tab_name - self._selected_tab = tab_name - tab = self._tabs[tab_name] - tab.set_selected() - # move the tab row with the selected tab to the bottom - tab_row = self._tab2row[tab] - tab_row.pack_forget() - tab_row.pack(side=TOP, fill=X, expand=0) - - def _add_tab_row(self, tab_names, expand_tabs): - if not tab_names: - return - - tab_row = Frame(self) - tab_row.pack(side=TOP, fill=X, expand=0) - self._tab_rows.append(tab_row) - - for tab_name in tab_names: - tab = TabSet.TabButton(tab_name, self.select_command, - tab_row, self) - if expand_tabs: - tab.pack(side=LEFT, fill=X, expand=True) - else: - tab.pack(side=LEFT) - self._tabs[tab_name] = tab - self._tab2row[tab] = tab_row - - # tab is the last one created in the above loop - tab.is_last_in_row = True - - def _reset_tab_rows(self): - while self._tab_rows: - tab_row = self._tab_rows.pop() - tab_row.destroy() - self._tab2row = {} - - def _arrange_tabs(self): - """ - Arrange the tabs in rows, in the order in which they were added. - - If n_rows >= 1, this will be the number of rows used. Otherwise the - number of rows will be calculated according to the number of tabs and - max_tabs_per_row. In this case, the number of rows may change when - adding/removing tabs. - - """ - # remove all tabs and rows - while self._tabs: - self._tabs.popitem()[1].destroy() - self._reset_tab_rows() - - if not self._tab_names: - return - - if self.n_rows is not None and self.n_rows > 0: - n_rows = self.n_rows - else: - # calculate the required number of rows - n_rows = (len(self._tab_names) - 1) // self.max_tabs_per_row + 1 - - # not expanding the tabs with more than one row is very ugly - expand_tabs = self.expand_tabs or n_rows > 1 - i = 0 # index in self._tab_names - for row_index in range(n_rows): - # calculate required number of tabs in this row - n_tabs = (len(self._tab_names) - i - 1) // (n_rows - row_index) + 1 - tab_names = self._tab_names[i:i + n_tabs] - i += n_tabs - self._add_tab_row(tab_names, expand_tabs) - - # re-select selected tab so it is properly displayed - selected = self._selected_tab - self.set_selected_tab(None) - if selected in self._tab_names: - self.set_selected_tab(selected) - - class TabButton(Frame): - """A simple tab-like widget.""" - - bw = 2 # borderwidth - - def __init__(self, name, select_command, tab_row, tab_set): - """Constructor arguments: - - name -- The tab's name, which will appear in its button. - - select_command -- The command to be called upon selection of the - tab. It is called with the tab's name as an argument. - - """ - Frame.__init__(self, tab_row, borderwidth=self.bw, relief=RAISED) - - self.name = name - self.select_command = select_command - self.tab_set = tab_set - self.is_last_in_row = False - - self.button = Radiobutton( - self, text=name, command=self._select_event, - padx=5, pady=1, takefocus=FALSE, indicatoron=FALSE, - highlightthickness=0, selectcolor='', borderwidth=0) - self.button.pack(side=LEFT, fill=X, expand=True) - - self._init_masks() - self.set_normal() - - def _select_event(self, *args): - """Event handler for tab selection. - - With TabbedPageSet, this calls TabbedPageSet.change_page, so that - selecting a tab changes the page. - - Note that this does -not- call set_selected -- it will be called by - TabSet.set_selected_tab, which should be called when whatever the - tabs are related to changes. - - """ - self.select_command(self.name) - return - - def set_selected(self): - """Assume selected look""" - self._place_masks(selected=True) - - def set_normal(self): - """Assume normal look""" - self._place_masks(selected=False) - - def _init_masks(self): - page_set = self.tab_set.page_set - background = page_set.pages_frame.cget('background') - # mask replaces the middle of the border with the background color - self.mask = Frame(page_set, borderwidth=0, relief=FLAT, - background=background) - # mskl replaces the bottom-left corner of the border with a normal - # left border - self.mskl = Frame(page_set, borderwidth=0, relief=FLAT, - background=background) - self.mskl.ml = Frame(self.mskl, borderwidth=self.bw, - relief=RAISED) - self.mskl.ml.place(x=0, y=-self.bw, - width=2*self.bw, height=self.bw*4) - # mskr replaces the bottom-right corner of the border with a normal - # right border - self.mskr = Frame(page_set, borderwidth=0, relief=FLAT, - background=background) - self.mskr.mr = Frame(self.mskr, borderwidth=self.bw, - relief=RAISED) - - def _place_masks(self, selected=False): - height = self.bw - if selected: - height += self.bw - - self.mask.place(in_=self, - relx=0.0, x=0, - rely=1.0, y=0, - relwidth=1.0, width=0, - relheight=0.0, height=height) - - self.mskl.place(in_=self, - relx=0.0, x=-self.bw, - rely=1.0, y=0, - relwidth=0.0, width=self.bw, - relheight=0.0, height=height) - - page_set = self.tab_set.page_set - if selected and ((not self.is_last_in_row) or - (self.winfo_rootx() + self.winfo_width() < - page_set.winfo_rootx() + page_set.winfo_width()) - ): - # for a selected tab, if its rightmost edge isn't on the - # rightmost edge of the page set, the right mask should be one - # borderwidth shorter (vertically) - height -= self.bw - - self.mskr.place(in_=self, - relx=1.0, x=0, - rely=1.0, y=0, - relwidth=0.0, width=self.bw, - relheight=0.0, height=height) - - self.mskr.mr.place(x=-self.bw, y=-self.bw, - width=2*self.bw, height=height + self.bw*2) - - # finally, lower the tab set so that all of the frames we just - # placed hide it - self.tab_set.lower() - - -class TabbedPageSet(Frame): - """A Tkinter tabbed-pane widget. - - Constains set of 'pages' (or 'panes') with tabs above for selecting which - page is displayed. Only one page will be displayed at a time. - - Pages may be accessed through the 'pages' attribute, which is a dictionary - of pages, using the name given as the key. A page is an instance of a - subclass of Tk's Frame widget. - - The page widgets will be created (and destroyed when required) by the - TabbedPageSet. Do not call the page's pack/place/grid/destroy methods. - - Pages may be added or removed at any time using the add_page() and - remove_page() methods. - - """ - - class Page(object): - """Abstract base class for TabbedPageSet's pages. - - Subclasses must override the _show() and _hide() methods. - - """ - uses_grid = False - - def __init__(self, page_set): - self.frame = Frame(page_set, borderwidth=2, relief=RAISED) - - def _show(self): - raise NotImplementedError - - def _hide(self): - raise NotImplementedError - - class PageRemove(Page): - """Page class using the grid placement manager's "remove" mechanism.""" - uses_grid = True - - def _show(self): - self.frame.grid(row=0, column=0, sticky=NSEW) - - def _hide(self): - self.frame.grid_remove() - - class PageLift(Page): - """Page class using the grid placement manager's "lift" mechanism.""" - uses_grid = True - - def __init__(self, page_set): - super(TabbedPageSet.PageLift, self).__init__(page_set) - self.frame.grid(row=0, column=0, sticky=NSEW) - self.frame.lower() - - def _show(self): - self.frame.lift() - - def _hide(self): - self.frame.lower() - - class PagePackForget(Page): - """Page class using the pack placement manager's "forget" mechanism.""" - def _show(self): - self.frame.pack(fill=BOTH, expand=True) - - def _hide(self): - self.frame.pack_forget() - - def __init__(self, parent, page_names=None, page_class=PageLift, - n_rows=1, max_tabs_per_row=5, expand_tabs=False, - **kw): - """Constructor arguments: - - page_names -- A list of strings, each will be the dictionary key to a - page's widget, and the name displayed on the page's tab. Should be - specified in the desired page order. The first page will be the default - and first active page. If page_names is None or empty, the - TabbedPageSet will be initialized empty. - - n_rows, max_tabs_per_row -- Parameters for the TabSet which will - manage the tabs. See TabSet's docs for details. - - page_class -- Pages can be shown/hidden using three mechanisms: - - * PageLift - All pages will be rendered one on top of the other. When - a page is selected, it will be brought to the top, thus hiding all - other pages. Using this method, the TabbedPageSet will not be resized - when pages are switched. (It may still be resized when pages are - added/removed.) - - * PageRemove - When a page is selected, the currently showing page is - hidden, and the new page shown in its place. Using this method, the - TabbedPageSet may resize when pages are changed. - - * PagePackForget - This mechanism uses the pack placement manager. - When a page is shown it is packed, and when it is hidden it is - unpacked (i.e. pack_forget). This mechanism may also cause the - TabbedPageSet to resize when the page is changed. - - """ - Frame.__init__(self, parent, **kw) - - self.page_class = page_class - self.pages = {} - self._pages_order = [] - self._current_page = None - self._default_page = None - - self.columnconfigure(0, weight=1) - self.rowconfigure(1, weight=1) - - self.pages_frame = Frame(self) - self.pages_frame.grid(row=1, column=0, sticky=NSEW) - if self.page_class.uses_grid: - self.pages_frame.columnconfigure(0, weight=1) - self.pages_frame.rowconfigure(0, weight=1) - - # the order of the following commands is important - self._tab_set = TabSet(self, self.change_page, n_rows=n_rows, - max_tabs_per_row=max_tabs_per_row, - expand_tabs=expand_tabs) - if page_names: - for name in page_names: - self.add_page(name) - self._tab_set.grid(row=0, column=0, sticky=NSEW) - - self.change_page(self._default_page) - - def add_page(self, page_name): - """Add a new page with the name given in page_name.""" - if not page_name: - raise InvalidNameError("Invalid TabPage name: '%s'" % page_name) - if page_name in self.pages: - raise AlreadyExistsError( - "TabPage named '%s' already exists" % page_name) - - self.pages[page_name] = self.page_class(self.pages_frame) - self._pages_order.append(page_name) - self._tab_set.add_tab(page_name) - - if len(self.pages) == 1: # adding first page - self._default_page = page_name - self.change_page(page_name) - - def remove_page(self, page_name): - """Destroy the page whose name is given in page_name.""" - if not page_name in self.pages: - raise KeyError("No such TabPage: '%s" % page_name) - - self._pages_order.remove(page_name) - - # handle removing last remaining, default, or currently shown page - if len(self._pages_order) > 0: - if page_name == self._default_page: - # set a new default page - self._default_page = self._pages_order[0] - else: - self._default_page = None - - if page_name == self._current_page: - self.change_page(self._default_page) - - self._tab_set.remove_tab(page_name) - page = self.pages.pop(page_name) - page.frame.destroy() - - def change_page(self, page_name): - """Show the page whose name is given in page_name.""" - if self._current_page == page_name: - return - if page_name is not None and page_name not in self.pages: - raise KeyError("No such TabPage: '%s'" % page_name) - - if self._current_page is not None: - self.pages[self._current_page]._hide() - self._current_page = None - - if page_name is not None: - self._current_page = page_name - self.pages[page_name]._show() - - self._tab_set.set_selected_tab(page_name) - - -def _tabbed_pages(parent): # htest # - top=Toplevel(parent) - x, y = map(int, parent.geometry().split('+')[1:]) - top.geometry("+%d+%d" % (x, y + 175)) - top.title("Test tabbed pages") - tabPage=TabbedPageSet(top, page_names=['Foobar','Baz'], n_rows=0, - expand_tabs=False, - ) - tabPage.pack(side=TOP, expand=TRUE, fill=BOTH) - Label(tabPage.pages['Foobar'].frame, text='Foo', pady=20).pack() - Label(tabPage.pages['Foobar'].frame, text='Bar', pady=20).pack() - Label(tabPage.pages['Baz'].frame, text='Baz').pack() - entryPgName=Entry(top) - buttonAdd=Button(top, text='Add Page', - command=lambda:tabPage.add_page(entryPgName.get())) - buttonRemove=Button(top, text='Remove Page', - command=lambda:tabPage.remove_page(entryPgName.get())) - labelPgName=Label(top, text='name of page to add/remove:') - buttonAdd.pack(padx=5, pady=5) - buttonRemove.pack(padx=5, pady=5) - labelPgName.pack(padx=5) - entryPgName.pack(padx=5) - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_tabbed_pages) diff --git a/Lib/idlelib/textview.py b/Lib/idlelib/textview.py index de4b190f..e3b55065 100644 --- a/Lib/idlelib/textview.py +++ b/Lib/idlelib/textview.py @@ -57,7 +57,7 @@ class ViewWindow(Toplevel): "A simple text viewer dialog for IDLE." def __init__(self, parent, title, text, modal=True, - _htest=False, _utest=False): + *, _htest=False, _utest=False): """Show the given text in a scrollable window with a 'close' button. If modal is left True, users cannot interact with other windows diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 4cf8aec1..e2343dd4 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -994,7 +994,7 @@ def _gcd_import(name, package=None, level=0): return _find_and_load(name, _gcd_import) -def _handle_fromlist(module, fromlist, import_): +def _handle_fromlist(module, fromlist, import_, *, recursive=False): """Figure out what __import__ should return. The import_ parameter is a callable which takes the name of module to @@ -1005,13 +1005,19 @@ def _handle_fromlist(module, fromlist, import_): # The hell that is fromlist ... # If a package was imported, try to import stuff from fromlist. if hasattr(module, '__path__'): - if '*' in fromlist: - fromlist = list(fromlist) - fromlist.remove('*') - if hasattr(module, '__all__'): - fromlist.extend(module.__all__) for x in fromlist: - if not hasattr(module, x): + if not isinstance(x, str): + if recursive: + where = module.__name__ + '.__all__' + else: + where = "``from list''" + raise TypeError(f"Item in {where} must be str, " + f"not {type(x).__name__}") + elif x == '*': + if not recursive and hasattr(module, '__all__'): + _handle_fromlist(module, module.__all__, import_, + recursive=True) + elif not hasattr(module, x): from_name = '{}.{}'.format(module.__name__, x) try: _call_with_frames_removed(import_, from_name) @@ -1019,7 +1025,8 @@ def _handle_fromlist(module, fromlist, import_): # Backwards-compatibility dictates we ignore failed # imports triggered by fromlist for modules that don't # exist. - if exc.name == from_name: + if (exc.name == from_name and + sys.modules.get(from_name, _NEEDS_LOADING) is not None): continue raise return module diff --git a/Lib/inspect.py b/Lib/inspect.py index 3317f58f..e9c2dbd5 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -505,13 +505,16 @@ def unwrap(func, *, stop=None): def _is_wrapper(f): return hasattr(f, '__wrapped__') and not stop(f) f = func # remember the original func for error reporting - memo = {id(f)} # Memoise by id to tolerate non-hashable objects + # Memoise by id to tolerate non-hashable objects, but store objects to + # ensure they aren't destroyed, which would allow their IDs to be reused. + memo = {id(f): f} + recursion_limit = sys.getrecursionlimit() while _is_wrapper(func): func = func.__wrapped__ id_func = id(func) - if id_func in memo: + if (id_func in memo) or (len(memo) >= recursion_limit): raise ValueError('wrapper loop when unwrapping {!r}'.format(f)) - memo.add(id_func) + memo[id_func] = func return func # -------------------------------------------------- source code extraction diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 09655367..3617dbb6 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1669,7 +1669,7 @@ class LoggerAdapter(object): """ if self.isEnabledFor(level): msg, kwargs = self.process(msg, kwargs) - self.logger._log(level, msg, args, **kwargs) + self.logger.log(level, msg, *args, **kwargs) def isEnabledFor(self, level): """ @@ -1715,9 +1715,13 @@ class LoggerAdapter(object): return self.logger.manager @manager.setter - def set_manager(self, value): + def manager(self, value): self.logger.manager = value + @property + def name(self): + return self.logger.name + def __repr__(self): logger = self.logger level = getLevelName(logger.getEffectiveLevel()) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 2f934b33..11ebcf12 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -356,10 +356,10 @@ class TimedRotatingFileHandler(BaseRotatingHandler): suffix = fileName[plen:] if self.extMatch.match(suffix): result.append(os.path.join(dirName, fileName)) - result.sort() if len(result) < self.backupCount: result = [] else: + result.sort() result = result[:len(result) - self.backupCount] return result @@ -1183,7 +1183,9 @@ class HTTPHandler(logging.Handler): i = host.find(":") if i >= 0: host = host[:i] - h.putheader("Host", host) + # See issue #30904: putrequest call above already adds this header + # on Python 3.x. + # h.putheader("Host", host) if self.method == "POST": h.putheader("Content-type", "application/x-www-form-urlencoded") diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index d5ce6257..fd198959 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -33,6 +33,7 @@ class ForkServer(object): def __init__(self): self._forkserver_address = None self._forkserver_alive_fd = None + self._forkserver_pid = None self._inherited_fds = None self._lock = threading.Lock() self._preload_modules = ['__main__'] @@ -89,8 +90,17 @@ class ForkServer(object): ''' with self._lock: semaphore_tracker.ensure_running() - if self._forkserver_alive_fd is not None: - return + if self._forkserver_pid is not None: + # forkserver was launched before, is it still running? + pid, status = os.waitpid(self._forkserver_pid, os.WNOHANG) + if not pid: + # still alive + return + # dead, launch it again + os.close(self._forkserver_alive_fd) + self._forkserver_address = None + self._forkserver_alive_fd = None + self._forkserver_pid = None cmd = ('from multiprocessing.forkserver import main; ' + 'main(%d, %d, %r, **%r)') @@ -127,6 +137,7 @@ class ForkServer(object): os.close(alive_r) self._forkserver_address = address self._forkserver_alive_fd = alive_w + self._forkserver_pid = pid # # @@ -149,11 +160,11 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): util._close_stdin() - # ignoring SIGCHLD means no need to reap zombie processes; - # letting SIGINT through avoids KeyboardInterrupt tracebacks handlers = { + # no need to reap zombie processes; signal.SIGCHLD: signal.SIG_IGN, - signal.SIGINT: signal.SIG_DFL, + # protect the process from ^C + signal.SIGINT: signal.SIG_IGN, } old_handlers = {sig: signal.signal(sig, val) for (sig, val) in handlers.items()} diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index d2ebd7cf..5d0fa569 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -14,8 +14,14 @@ class Popen(object): method = 'fork' def __init__(self, process_obj): - sys.stdout.flush() - sys.stderr.flush() + try: + sys.stdout.flush() + except (AttributeError, ValueError): + pass + try: + sys.stderr.flush() + except (AttributeError, ValueError): + pass self.returncode = None self._launch(process_obj) diff --git a/Lib/multiprocessing/semaphore_tracker.py b/Lib/multiprocessing/semaphore_tracker.py index de7738ee..3b50a46d 100644 --- a/Lib/multiprocessing/semaphore_tracker.py +++ b/Lib/multiprocessing/semaphore_tracker.py @@ -29,6 +29,7 @@ class SemaphoreTracker(object): def __init__(self): self._lock = threading.Lock() self._fd = None + self._pid = None def getfd(self): self.ensure_running() @@ -40,8 +41,20 @@ class SemaphoreTracker(object): This can be run from any process. Usually a child process will use the semaphore created by its parent.''' with self._lock: - if self._fd is not None: - return + if self._pid is not None: + # semaphore tracker was launched before, is it still running? + pid, status = os.waitpid(self._pid, os.WNOHANG) + if not pid: + # => still alive + return + # => dead, launch it again + os.close(self._fd) + self._fd = None + self._pid = None + + warnings.warn('semaphore_tracker: process died unexpectedly, ' + 'relaunching. Some semaphores might leak.') + fds_to_pass = [] try: fds_to_pass.append(sys.stderr.fileno()) @@ -55,12 +68,13 @@ class SemaphoreTracker(object): exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd % r] - util.spawnv_passfds(exe, args, fds_to_pass) + pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(w) raise else: self._fd = w + self._pid = pid finally: os.close(r) diff --git a/Lib/netrc.py b/Lib/netrc.py index bbb3d23b..1d90f5d9 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -127,15 +127,15 @@ class netrc: rep = "" for host in self.hosts.keys(): attrs = self.hosts[host] - rep = rep + "machine "+ host + "\n\tlogin " + repr(attrs[0]) + "\n" + rep += f"machine {host}\n\tlogin {attrs[0]}\n" if attrs[1]: - rep = rep + "account " + repr(attrs[1]) - rep = rep + "\tpassword " + repr(attrs[2]) + "\n" + rep += f"\taccount {attrs[1]}\n" + rep += f"\tpassword {attrs[2]}\n" for macro in self.macros.keys(): - rep = rep + "macdef " + macro + "\n" + rep += f"macdef {macro}\n" for line in self.macros[macro]: - rep = rep + line - rep = rep + "\n" + rep += line + rep += "\n" return rep if __name__ == '__main__': diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 09be5fd3..a9186437 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -590,6 +590,8 @@ class InvalidFileException (ValueError): _BINARY_FORMAT = {1: 'B', 2: 'H', 4: 'L', 8: 'Q'} +_undefined = object() + class _BinaryPlistParser: """ Read or write a binary plist file, following the description of the binary @@ -620,9 +622,11 @@ class _BinaryPlistParser: ) = struct.unpack('>6xBBQQQ', trailer) self._fp.seek(offset_table_offset) self._object_offsets = self._read_ints(num_objects, offset_size) - return self._read_object(self._object_offsets[top_object]) + self._objects = [_undefined] * num_objects + return self._read_object(top_object) - except (OSError, IndexError, struct.error): + except (OSError, IndexError, struct.error, OverflowError, + UnicodeDecodeError): raise InvalidFileException() def _get_size(self, tokenL): @@ -640,68 +644,76 @@ class _BinaryPlistParser: if size in _BINARY_FORMAT: return struct.unpack('>' + _BINARY_FORMAT[size] * n, data) else: + if not size or len(data) != size * n: + raise InvalidFileException() return tuple(int.from_bytes(data[i: i + size], 'big') for i in range(0, size * n, size)) def _read_refs(self, n): return self._read_ints(n, self._ref_size) - def _read_object(self, offset): + def _read_object(self, ref): """ - read the object at offset. + read the object by reference. May recursively read sub-objects (content of an array/dict/set) """ + result = self._objects[ref] + if result is not _undefined: + return result + + offset = self._object_offsets[ref] self._fp.seek(offset) token = self._fp.read(1)[0] tokenH, tokenL = token & 0xF0, token & 0x0F if token == 0x00: - return None + result = None elif token == 0x08: - return False + result = False elif token == 0x09: - return True + result = True # The referenced source code also mentions URL (0x0c, 0x0d) and # UUID (0x0e), but neither can be generated using the Cocoa libraries. elif token == 0x0f: - return b'' + result = b'' elif tokenH == 0x10: # int - return int.from_bytes(self._fp.read(1 << tokenL), - 'big', signed=tokenL >= 3) + result = int.from_bytes(self._fp.read(1 << tokenL), + 'big', signed=tokenL >= 3) elif token == 0x22: # real - return struct.unpack('>f', self._fp.read(4))[0] + result = struct.unpack('>f', self._fp.read(4))[0] elif token == 0x23: # real - return struct.unpack('>d', self._fp.read(8))[0] + result = struct.unpack('>d', self._fp.read(8))[0] elif token == 0x33: # date f = struct.unpack('>d', self._fp.read(8))[0] # timestamp 0 of binary plists corresponds to 1/1/2001 # (year of Mac OS X 10.0), instead of 1/1/1970. - return datetime.datetime(2001, 1, 1) + datetime.timedelta(seconds=f) + result = (datetime.datetime(2001, 1, 1) + + datetime.timedelta(seconds=f)) elif tokenH == 0x40: # data s = self._get_size(tokenL) if self._use_builtin_types: - return self._fp.read(s) + result = self._fp.read(s) else: - return Data(self._fp.read(s)) + result = Data(self._fp.read(s)) elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) result = self._fp.read(s).decode('ascii') - return result + result = result elif tokenH == 0x60: # unicode string s = self._get_size(tokenL) - return self._fp.read(s * 2).decode('utf-16be') + result = self._fp.read(s * 2).decode('utf-16be') # tokenH == 0x80 is documented as 'UID' and appears to be used for # keyed-archiving, not in plists. @@ -709,8 +721,9 @@ class _BinaryPlistParser: elif tokenH == 0xA0: # array s = self._get_size(tokenL) obj_refs = self._read_refs(s) - return [self._read_object(self._object_offsets[x]) - for x in obj_refs] + result = [] + self._objects[ref] = result + result.extend(self._read_object(x) for x in obj_refs) # tokenH == 0xB0 is documented as 'ordset', but is not actually # implemented in the Apple reference code. @@ -723,12 +736,15 @@ class _BinaryPlistParser: key_refs = self._read_refs(s) obj_refs = self._read_refs(s) result = self._dict_type() + self._objects[ref] = result for k, o in zip(key_refs, obj_refs): - result[self._read_object(self._object_offsets[k]) - ] = self._read_object(self._object_offsets[o]) - return result + result[self._read_object(k)] = self._read_object(o) + + else: + raise InvalidFileException() - raise InvalidFileException() + self._objects[ref] = result + return result def _count_to_size(count): if count < 1 << 8: @@ -743,6 +759,8 @@ def _count_to_size(count): else: return 8 +_scalars = (str, int, float, datetime.datetime, bytes) + class _BinaryPlistWriter (object): def __init__(self, fp, sort_keys, skipkeys): self._fp = fp @@ -798,8 +816,7 @@ class _BinaryPlistWriter (object): # First check if the object is in the object table, not used for # containers to ensure that two subcontainers with the same contents # will be serialized as distinct values. - if isinstance(value, ( - str, int, float, datetime.datetime, bytes, bytearray)): + if isinstance(value, _scalars): if (type(value), value) in self._objtable: return @@ -807,15 +824,17 @@ class _BinaryPlistWriter (object): if (type(value.data), value.data) in self._objtable: return + elif id(value) in self._objidtable: + return + # Add to objectreference map refnum = len(self._objlist) self._objlist.append(value) - try: - if isinstance(value, Data): - self._objtable[(type(value.data), value.data)] = refnum - else: - self._objtable[(type(value), value)] = refnum - except TypeError: + if isinstance(value, _scalars): + self._objtable[(type(value), value)] = refnum + elif isinstance(value, Data): + self._objtable[(type(value.data), value.data)] = refnum + else: self._objidtable[id(value)] = refnum # And finally recurse into containers @@ -842,12 +861,11 @@ class _BinaryPlistWriter (object): self._flatten(o) def _getrefnum(self, value): - try: - if isinstance(value, Data): - return self._objtable[(type(value.data), value.data)] - else: - return self._objtable[(type(value), value)] - except TypeError: + if isinstance(value, _scalars): + return self._objtable[(type(value), value)] + elif isinstance(value, Data): + return self._objtable[(type(value.data), value.data)] + else: return self._objidtable[id(value)] def _write_size(self, token, size): diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index deb540d5..8dc41a2c 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Sep 18 23:00:11 2017 +# Autogenerated by Sphinx on Tue Dec 5 03:11:02 2017 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -2674,7 +2674,6 @@ topics = {'assert': 'The "assert" statement\n' ' mgr = (EXPR)\n' ' aexit = type(mgr).__aexit__\n' ' aenter = type(mgr).__aenter__(mgr)\n' - ' exc = True\n' '\n' ' VAR = await aenter\n' ' try:\n' @@ -4503,12 +4502,6 @@ topics = {'assert': 'The "assert" statement\n' ' 3.14 10. .001 1e100 3.14e-10 0e0 ' '3.14_15_93\n' '\n' - 'Note that numeric literals do not include a sign; a phrase like ' - '"-1"\n' - 'is actually an expression composed of the unary operator "-" and ' - 'the\n' - 'literal "1".\n' - '\n' 'Changed in version 3.6: Underscores are now allowed for ' 'grouping\n' 'purposes in literals.\n', @@ -5497,7 +5490,7 @@ topics = {'assert': 'The "assert" statement\n' '\n' '**CPython implementation detail:** The current implementation does ' 'not\n' - 'enforce some of these restriction, but programs should not abuse ' + 'enforce some of these restrictions, but programs should not abuse ' 'this\n' 'freedom, as future implementations may enforce them or silently ' 'change\n' @@ -6744,7 +6737,9 @@ topics = {'assert': 'The "assert" statement\n' 'Addition and subtraction |\n' '+-------------------------------------------------+---------------------------------------+\n' '| "*", "@", "/", "//", "%" | ' - 'Multiplication, matrix multiplication |\n' + 'Multiplication, matrix |\n' + '| | ' + 'multiplication, division, floor |\n' '| | ' 'division, remainder [5] |\n' '+-------------------------------------------------+---------------------------------------+\n' @@ -10102,11 +10097,6 @@ topics = {'assert': 'The "assert" statement\n' 'or\n' 'greater must be expressed with escapes.\n' '\n' - 'As of Python 3.3 it is possible again to prefix string literals ' - 'with a\n' - '"u" prefix to simplify maintenance of dual 2.x and 3.x ' - 'codebases.\n' - '\n' 'Both string and bytes literals may optionally be prefixed with a\n' 'letter "\'r\'" or "\'R\'"; such strings are called *raw strings* ' 'and treat\n' diff --git a/Lib/random.py b/Lib/random.py index ad1c9167..0152e5ea 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -109,9 +109,10 @@ class Random(_random.Random): """ if version == 1 and isinstance(a, (str, bytes)): + a = a.decode('latin-1') if isinstance(a, bytes) else a x = ord(a[0]) << 7 if a else 0 - for c in a: - x = ((1000003 * x) ^ ord(c)) & 0xFFFFFFFFFFFFFFFF + for c in map(ord, a): + x = ((1000003 * x) ^ c) & 0xFFFFFFFFFFFFFFFF x ^= len(a) a = -2 if x == -1 else x diff --git a/Lib/sqlite3/test/regression.py b/Lib/sqlite3/test/regression.py index 7dd00505..34cd2335 100644 --- a/Lib/sqlite3/test/regression.py +++ b/Lib/sqlite3/test/regression.py @@ -24,6 +24,8 @@ import datetime import unittest import sqlite3 as sqlite +import weakref +from test import support class RegressionTests(unittest.TestCase): def setUp(self): @@ -188,6 +190,9 @@ class RegressionTests(unittest.TestCase): cur = Cursor(con) with self.assertRaises(sqlite.ProgrammingError): cur.execute("select 4+5").fetchall() + with self.assertRaisesRegex(sqlite.ProgrammingError, + r'^Base Cursor\.__init__ not called\.$'): + cur.close() def CheckStrSubclass(self): """ @@ -376,6 +381,22 @@ class RegressionTests(unittest.TestCase): counter += 1 self.assertEqual(counter, 3, "should have returned exactly three rows") + def CheckBpo31770(self): + """ + The interpreter shouldn't crash in case Cursor.__init__() is called + more than once. + """ + def callback(*args): + pass + con = sqlite.connect(":memory:") + cur = sqlite.Cursor(con) + ref = weakref.ref(cur, callback) + cur.__init__(con) + del cur + # The interpreter shouldn't crash when ref is collected. + del ref + support.gc_collect() + def suite(): regression_suite = unittest.makeSuite(RegressionTests, "Check") diff --git a/Lib/string.py b/Lib/string.py index c9020076..670c1951 100644 --- a/Lib/string.py +++ b/Lib/string.py @@ -78,7 +78,11 @@ class Template(metaclass=_TemplateMetaclass): """A string class for supporting $-substitutions.""" delimiter = '$' - idpattern = r'[_a-z][_a-z0-9]*' + # r'[a-z]' matches to non-ASCII letters when used with IGNORECASE, + # but without ASCII flag. We can't add re.ASCII to flags because of + # backward compatibility. So we use local -i flag and [a-zA-Z] pattern. + # See https://bugs.python.org/issue31672 + idpattern = r'(?-i:[_a-zA-Z][_a-zA-Z0-9]*)' flags = _re.IGNORECASE def __init__(self, template): diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index db30e6b9..f01c0041 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4,6 +4,7 @@ import unittest import queue as pyqueue +import contextlib import time import io import itertools @@ -425,6 +426,75 @@ class _TestProcess(BaseTestCase): self.assertEqual(q.get(), 5) close_queue(q) + @classmethod + def _test_error_on_stdio_flush(self, evt): + evt.set() + + def test_error_on_stdio_flush(self): + streams = [io.StringIO(), None] + streams[0].close() + for stream_name in ('stdout', 'stderr'): + for stream in streams: + old_stream = getattr(sys, stream_name) + setattr(sys, stream_name, stream) + try: + evt = self.Event() + proc = self.Process(target=self._test_error_on_stdio_flush, + args=(evt,)) + proc.start() + proc.join() + self.assertTrue(evt.is_set()) + finally: + setattr(sys, stream_name, old_stream) + + @classmethod + def _sleep_and_set_event(self, evt, delay=0.0): + time.sleep(delay) + evt.set() + + def check_forkserver_death(self, signum): + # bpo-31308: if the forkserver process has died, we should still + # be able to create and run new Process instances (the forkserver + # is implicitly restarted). + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + sm = multiprocessing.get_start_method() + if sm != 'forkserver': + # The fork method by design inherits all fds from the parent, + # trying to go against it is a lost battle + self.skipTest('test not appropriate for {}'.format(sm)) + + from multiprocessing.forkserver import _forkserver + _forkserver.ensure_running() + + evt = self.Event() + proc = self.Process(target=self._sleep_and_set_event, args=(evt, 1.0)) + proc.start() + + pid = _forkserver._forkserver_pid + os.kill(pid, signum) + time.sleep(1.0) # give it time to die + + evt2 = self.Event() + proc2 = self.Process(target=self._sleep_and_set_event, args=(evt2,)) + proc2.start() + proc2.join() + self.assertTrue(evt2.is_set()) + self.assertEqual(proc2.exitcode, 0) + + proc.join() + self.assertTrue(evt.is_set()) + self.assertIn(proc.exitcode, (0, 255)) + + def test_forkserver_sigint(self): + # Catchable signal + self.check_forkserver_death(signal.SIGINT) + + def test_forkserver_sigkill(self): + # Uncatchable signal + if os.name != 'nt': + self.check_forkserver_death(signal.SIGKILL) + # # @@ -4056,14 +4126,14 @@ class TestStartMethod(unittest.TestCase): self.fail("failed spawning forkserver or grandchild") -# -# Check that killing process does not leak named semaphores -# - @unittest.skipIf(sys.platform == "win32", "test semantics don't make sense on Windows") class TestSemaphoreTracker(unittest.TestCase): + def test_semaphore_tracker(self): + # + # Check that killing process does not leak named semaphores + # import subprocess cmd = '''if 1: import multiprocessing as mp, time, os @@ -4097,6 +4167,40 @@ class TestSemaphoreTracker(unittest.TestCase): self.assertRegex(err, expected) self.assertRegex(err, r'semaphore_tracker: %r: \[Errno' % name1) + def check_semaphore_tracker_death(self, signum, should_die): + # bpo-31310: if the semaphore tracker process has died, it should + # be restarted implicitly. + from multiprocessing.semaphore_tracker import _semaphore_tracker + _semaphore_tracker.ensure_running() + pid = _semaphore_tracker._pid + os.kill(pid, signum) + time.sleep(1.0) # give it time to die + + ctx = multiprocessing.get_context("spawn") + with contextlib.ExitStack() as stack: + if should_die: + stack.enter_context(self.assertWarnsRegex( + UserWarning, + "semaphore_tracker: process died")) + sem = ctx.Semaphore() + sem.acquire() + sem.release() + wr = weakref.ref(sem) + # ensure `sem` gets collected, which triggers communication with + # the semaphore tracker + del sem + gc.collect() + self.assertIsNone(wr()) + + def test_semaphore_tracker_sigint(self): + # Catchable signal (ignored by semaphore tracker) + self.check_semaphore_tracker_death(signal.SIGINT, False) + + def test_semaphore_tracker_sigkill(self): + # Uncatchable signal. + self.check_semaphore_tracker_death(signal.SIGKILL, True) + + class TestSimpleQueue(unittest.TestCase): @classmethod diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index b25e6c17..f23a5305 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -846,6 +846,46 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): self.assertRaises(TypeError, divmod, t, 10) + def test_issue31293(self): + # The interpreter shouldn't crash in case a timedelta is divided or + # multiplied by a float with a bad as_integer_ratio() method. + def get_bad_float(bad_ratio): + class BadFloat(float): + def as_integer_ratio(self): + return bad_ratio + return BadFloat() + + with self.assertRaises(TypeError): + timedelta() / get_bad_float(1 << 1000) + with self.assertRaises(TypeError): + timedelta() * get_bad_float(1 << 1000) + + for bad_ratio in [(), (42, ), (1, 2, 3)]: + with self.assertRaises(ValueError): + timedelta() / get_bad_float(bad_ratio) + with self.assertRaises(ValueError): + timedelta() * get_bad_float(bad_ratio) + + def test_issue31752(self): + # The interpreter shouldn't crash because divmod() returns negative + # remainder. + class BadInt(int): + def __mul__(self, other): + return Prod() + + class Prod: + def __radd__(self, other): + return Sum() + + class Sum(int): + def __divmod__(self, other): + # negative remainder + return (0, -1) + + timedelta(microseconds=BadInt(1)) + timedelta(hours=BadInt(1)) + timedelta(weeks=BadInt(1)) + ############################################################################# # date tests @@ -1460,6 +1500,13 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase): base = cls(2000, 2, 29) self.assertRaises(ValueError, base.replace, year=2001) + def test_subclass_replace(self): + class DateSubclass(self.theclass): + pass + + dt = DateSubclass(2012, 1, 1) + self.assertIs(type(dt.replace(year=2013)), DateSubclass) + def test_subclass_date(self): class C(self.theclass): @@ -2559,6 +2606,13 @@ class TestTime(HarmlessMixedComparison, unittest.TestCase): self.assertRaises(ValueError, base.replace, second=100) self.assertRaises(ValueError, base.replace, microsecond=1000000) + def test_subclass_replace(self): + class TimeSubclass(self.theclass): + pass + + ctime = TimeSubclass(12, 30) + self.assertIs(type(ctime.replace(hour=10)), TimeSubclass) + def test_subclass_time(self): class C(self.theclass): diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 1dbe88ef..bc308fe1 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -20,7 +20,6 @@ import time import unittest from test import support -android_not_root = support.android_not_root @contextlib.contextmanager def kill_on_error(proc): @@ -312,14 +311,16 @@ class SocketEINTRTest(EINTRBaseTest): # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=203162 @support.requires_freebsd_version(10, 3) @unittest.skipUnless(hasattr(os, 'mkfifo'), 'needs mkfifo()') - @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def _test_open(self, do_open_close_reader, do_open_close_writer): filename = support.TESTFN # Use a fifo: until the child opens it for reading, the parent will # block when trying to open it for writing. support.unlink(filename) - os.mkfifo(filename) + try: + os.mkfifo(filename) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) self.addCleanup(support.unlink, filename) code = '\n'.join(( diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index d44f69dd..15e51525 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -256,12 +256,12 @@ class Regrtest: if isinstance(test, unittest.TestSuite): self._list_cases(test) elif isinstance(test, unittest.TestCase): - if support._match_test(test): + if support.match_test(test): print(test.id()) def list_cases(self): support.verbose = False - support.match_tests = self.ns.match_tests + support.set_match_tests(self.ns.match_tests) for test in self.selected: abstest = get_abs_module(self.ns, test) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index d716b83e..aefec125 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -102,7 +102,7 @@ def runtest(ns, test): if use_timeout: faulthandler.dump_traceback_later(ns.timeout, exit=True) try: - support.match_tests = ns.match_tests + support.set_match_tests(ns.match_tests) if ns.failfast: support.failfast = True if output_on_failure: diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 1d24531f..bf899a9e 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -14,17 +14,26 @@ from test.libregrtest.refleak import warm_caches def setup_tests(ns): - # Display the Python traceback on fatal errors (e.g. segfault) - faulthandler.enable(all_threads=True) - - # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] - if hasattr(signal, 'SIGALRM'): - signals.append(signal.SIGALRM) - if hasattr(signal, 'SIGUSR1'): - signals.append(signal.SIGUSR1) - for signum in signals: - faulthandler.register(signum, chain=True) + try: + stderr_fd = sys.__stderr__.fileno() + except (ValueError, AttributeError): + # Catch ValueError to catch io.UnsupportedOperation on TextIOBase + # and ValueError on a closed stream. + # + # Catch AttributeError for stderr being None. + stderr_fd = None + else: + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True, file=stderr_fd) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True, file=stderr_fd) replace_stdout() support.record_original_stdout(sys.stdout) @@ -109,7 +118,17 @@ def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error handler) to avoid UnicodeEncodeError when printing a traceback""" stdout = sys.stdout - sys.stdout = open(stdout.fileno(), 'w', + try: + fd = stdout.fileno() + except ValueError: + # On IDLE, sys.stdout has no file descriptor and is not a TextIOWrapper + # object. Leaving sys.stdout unchanged. + # + # Catch ValueError to catch io.UnsupportedOperation on TextIOBase + # and ValueError on a closed stream. + return + + sys.stdout = open(fd, 'w', encoding=stdout.encoding, errors="backslashreplace", closefd=False, diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 5c833610..a1c0bd72 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2534,7 +2534,7 @@ class AbstractPickleModuleTests(unittest.TestCase): f = open(TESTFN, "wb") try: f.close() - self.assertRaises(ValueError, pickle.dump, 123, f) + self.assertRaises(ValueError, self.dump, 123, f) finally: os.remove(TESTFN) @@ -2543,16 +2543,16 @@ class AbstractPickleModuleTests(unittest.TestCase): f = open(TESTFN, "wb") try: f.close() - self.assertRaises(ValueError, pickle.dump, 123, f) + self.assertRaises(ValueError, self.dump, 123, f) finally: os.remove(TESTFN) def test_load_from_and_dump_to_file(self): stream = io.BytesIO() data = [123, {}, 124] - pickle.dump(data, stream) + self.dump(data, stream) stream.seek(0) - unpickled = pickle.load(stream) + unpickled = self.load(stream) self.assertEqual(unpickled, data) def test_highest_protocol(self): @@ -2562,20 +2562,20 @@ class AbstractPickleModuleTests(unittest.TestCase): def test_callapi(self): f = io.BytesIO() # With and without keyword arguments - pickle.dump(123, f, -1) - pickle.dump(123, file=f, protocol=-1) - pickle.dumps(123, -1) - pickle.dumps(123, protocol=-1) - pickle.Pickler(f, -1) - pickle.Pickler(f, protocol=-1) + self.dump(123, f, -1) + self.dump(123, file=f, protocol=-1) + self.dumps(123, -1) + self.dumps(123, protocol=-1) + self.Pickler(f, -1) + self.Pickler(f, protocol=-1) def test_bad_init(self): # Test issue3664 (pickle can segfault from a badly initialized Pickler). # Override initialization without calling __init__() of the superclass. - class BadPickler(pickle.Pickler): + class BadPickler(self.Pickler): def __init__(self): pass - class BadUnpickler(pickle.Unpickler): + class BadUnpickler(self.Unpickler): def __init__(self): pass self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 15cce34e..c238ef7b 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -2,6 +2,7 @@ Collect various informations about Python to help debugging test failures. """ from __future__ import print_function +import errno import re import sys import traceback @@ -114,6 +115,14 @@ def collect_sys(info_add): encoding = '%s/%s' % (encoding, errors) info_add('sys.%s.encoding' % name, encoding) + # Were we compiled --with-pydebug or with #define Py_DEBUG? + Py_DEBUG = hasattr(sys, 'gettotalrefcount') + if Py_DEBUG: + text = 'Yes (sys.gettotalrefcount() present)' + else: + text = 'No (sys.gettotalrefcount() missing)' + info_add('Py_DEBUG', text) + def collect_platform(info_add): import platform @@ -223,11 +232,17 @@ def collect_os(info_add): if hasattr(os, 'getrandom'): # PEP 524: Check if system urandom is initialized try: - os.getrandom(1, os.GRND_NONBLOCK) - state = 'ready (initialized)' - except BlockingIOError as exc: - state = 'not seeded yet (%s)' % exc - info_add('os.getrandom', state) + try: + os.getrandom(1, os.GRND_NONBLOCK) + state = 'ready (initialized)' + except BlockingIOError as exc: + state = 'not seeded yet (%s)' % exc + info_add('os.getrandom', state) + except OSError as exc: + # Python was compiled on a more recent Linux version + # than the current Linux kernel: ignore OSError(ENOSYS) + if exc.errno != errno.ENOSYS: + raise def collect_readline(info_add): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a5b8c46f..2930ab24 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -93,7 +93,7 @@ __all__ = [ "check__all__", "requires_android_level", "requires_multiprocessing_queue", # sys "is_jython", "is_android", "check_impl_detail", "unix_shell", - "setswitchinterval", "android_not_root", + "setswitchinterval", # network "HOST", "IPV6_ENABLED", "find_unused_port", "bind_port", "open_urlresource", "bind_unix_socket", @@ -281,7 +281,6 @@ max_memuse = 0 # Disable bigmem tests (they will still be run with # small sizes, to make sure they work.) real_max_memuse = 0 failfast = False -match_tests = None # _original_stdout is meant to hold stdout at the time regrtest began. # This may be "the real" stdout, or IDLE's emulation of stdout, or whatever. @@ -779,7 +778,6 @@ is_jython = sys.platform.startswith('java') _ANDROID_API_LEVEL = sysconfig.get_config_var('ANDROID_API_LEVEL') is_android = (_ANDROID_API_LEVEL is not None and _ANDROID_API_LEVEL > 0) -android_not_root = (is_android and os.geteuid() != 0) if sys.platform != 'win32': unix_shell = '/system/bin/sh' if is_android else '/bin/sh' @@ -1898,21 +1896,67 @@ def _run_suite(suite): raise TestFailed(err) -def _match_test(test): - global match_tests +# By default, don't filter tests +_match_test_func = None +_match_test_patterns = None - if match_tests is None: + +def match_test(test): + # Function used by support.run_unittest() and regrtest --list-cases + if _match_test_func is None: return True - test_id = test.id() + else: + return _match_test_func(test.id()) - for match_test in match_tests: - if fnmatch.fnmatchcase(test_id, match_test): - return True - for name in test_id.split("."): - if fnmatch.fnmatchcase(name, match_test): +def _is_full_match_test(pattern): + # If a pattern contains at least one dot, it's considered + # as a full test identifier. + # Example: 'test.test_os.FileTests.test_access'. + # + # Reject patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, reject 'test_access*'. + return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) + + +def set_match_tests(patterns): + global _match_test_func, _match_test_patterns + + if patterns == _match_test_patterns: + # No change: no need to recompile patterns. + return + + if not patterns: + func = None + # set_match_tests(None) behaves as set_match_tests(()) + patterns = () + elif all(map(_is_full_match_test, patterns)): + # Simple case: all patterns are full test identifier. + # The test.bisect utility only uses such full test identifiers. + func = set(patterns).__contains__ + else: + regex = '|'.join(map(fnmatch.translate, patterns)) + # The search *is* case sensitive on purpose: + # don't use flags=re.IGNORECASE + regex_match = re.compile(regex).match + + def match_test_regex(test_id): + if regex_match(test_id): + # The regex matchs the whole identifier like + # 'test.test_os.FileTests.test_access' return True - return False + else: + # Try to match parts of the test identifier. + # For example, split 'test.test_os.FileTests.test_access' + # into: 'test', 'test_os', 'FileTests' and 'test_access'. + return any(map(regex_match, test_id.split("."))) + + func = match_test_regex + + # Create a copy since patterns can be mutable and so modified later + _match_test_patterns = tuple(patterns) + _match_test_func = func + def run_unittest(*classes): @@ -1929,7 +1973,7 @@ def run_unittest(*classes): suite.addTest(cls) else: suite.addTest(unittest.makeSuite(cls)) - _filter_suite(suite, _match_test) + _filter_suite(suite, match_test) _run_suite(suite) #======================================================================= @@ -2621,3 +2665,42 @@ def disable_faulthandler(): finally: if is_enabled: faulthandler.enable(file=fd, all_threads=True) + + +class SaveSignals: + """ + Save an restore signal handlers. + + This class is only able to save/restore signal handlers registered + by the Python signal module: see bpo-13285 for "external" signal + handlers. + """ + + def __init__(self): + import signal + self.signal = signal + self.signals = list(range(1, signal.NSIG)) + # SIGKILL and SIGSTOP signals cannot be ignored nor catched + for signame in ('SIGKILL', 'SIGSTOP'): + try: + signum = getattr(signal, signame) + except AttributeError: + continue + self.signals.remove(signum) + self.handlers = {} + + def save(self): + for signum in self.signals: + handler = self.signal.getsignal(signum) + if handler is None: + # getsignal() returns None if a signal handler was not + # registered by the Python signal module, + # and the handler is not SIG_DFL nor SIG_IGN. + # + # Ignore the signal: we cannot restore the handler. + continue + self.handlers[signum] = handler + + def restore(self): + for signum, handler in self.handlers.items(): + self.signal.signal(signum, handler) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 8c62408b..e68d0de2 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -421,6 +421,16 @@ class AST_Tests(unittest.TestCase): compile(empty_yield_from, "", "exec") self.assertIn("field value is required", str(cm.exception)) + @support.cpython_only + def test_issue31592(self): + # There shouldn't be an assertion failure in case of a bad + # unicodedata.normalize(). + import unicodedata + def bad_normalize(*args): + return None + with support.swap_attr(unicodedata, 'normalize', bad_normalize): + self.assertRaises(TypeError, ast.parse, '\u03D5') + class ASTHelpers_Test(unittest.TestCase): diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index fe5a6dbf..2137cde6 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -295,6 +295,23 @@ class QueueGetTests(_QueueTestBase): loop=self.loop), ) + def test_cancelled_getters_not_being_held_in_self_getters(self): + def a_generator(): + yield 0.1 + yield 0.2 + + self.loop = self.new_test_loop(a_generator) + @asyncio.coroutine + def consumer(queue): + try: + item = yield from asyncio.wait_for(queue.get(), 0.1, loop=self.loop) + except asyncio.TimeoutError: + pass + + queue = asyncio.Queue(loop=self.loop, maxsize=5) + self.loop.run_until_complete(self.loop.create_task(consumer(queue))) + self.assertEqual(len(queue._getters), 0) + class QueuePutTests(_QueueTestBase): diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index c50b3e49..3477573e 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -182,7 +182,28 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): f = self.loop.sock_recv(sock, 1024) self.assertIsInstance(f, asyncio.Future) - self.loop._sock_recv.assert_called_with(f, False, sock, 1024) + self.loop._sock_recv.assert_called_with(f, None, sock, 1024) + + def test_sock_recv_reconnection(self): + sock = mock.Mock() + sock.fileno.return_value = 10 + sock.recv.side_effect = BlockingIOError + sock.gettimeout.return_value = 0.0 + + self.loop.add_reader = mock.Mock() + self.loop.remove_reader = mock.Mock() + fut = self.loop.sock_recv(sock, 1024) + callback = self.loop.add_reader.call_args[0][1] + params = self.loop.add_reader.call_args[0][2:] + + # emulate the old socket has closed, but the new one has + # the same fileno, so callback is called with old (closed) socket + sock.fileno.return_value = -1 + sock.recv.side_effect = OSError(9) + callback(*params) + + self.assertIsInstance(fut.exception(), OSError) + self.assertEqual((10,), self.loop.remove_reader.call_args[0]) def test__sock_recv_canceled_fut(self): sock = mock.Mock() @@ -190,7 +211,7 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): f = asyncio.Future(loop=self.loop) f.cancel() - self.loop._sock_recv(f, False, sock, 1024) + self.loop._sock_recv(f, None, sock, 1024) self.assertFalse(sock.recv.called) def test__sock_recv_unregister(self): @@ -201,7 +222,7 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): f.cancel() self.loop.remove_reader = mock.Mock() - self.loop._sock_recv(f, True, sock, 1024) + self.loop._sock_recv(f, 10, sock, 1024) self.assertEqual((10,), self.loop.remove_reader.call_args[0]) def test__sock_recv_tryagain(self): @@ -211,8 +232,8 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock.recv.side_effect = BlockingIOError self.loop.add_reader = mock.Mock() - self.loop._sock_recv(f, False, sock, 1024) - self.assertEqual((10, self.loop._sock_recv, f, True, sock, 1024), + self.loop._sock_recv(f, None, sock, 1024) + self.assertEqual((10, self.loop._sock_recv, f, 10, sock, 1024), self.loop.add_reader.call_args[0]) def test__sock_recv_exception(self): @@ -221,7 +242,7 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock.fileno.return_value = 10 err = sock.recv.side_effect = OSError() - self.loop._sock_recv(f, False, sock, 1024) + self.loop._sock_recv(f, None, sock, 1024) self.assertIs(err, f.exception()) def test_sock_sendall(self): @@ -231,7 +252,7 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): f = self.loop.sock_sendall(sock, b'data') self.assertIsInstance(f, asyncio.Future) self.assertEqual( - (f, False, sock, b'data'), + (f, None, sock, b'data'), self.loop._sock_sendall.call_args[0]) def test_sock_sendall_nodata(self): @@ -244,13 +265,34 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): self.assertIsNone(f.result()) self.assertFalse(self.loop._sock_sendall.called) + def test_sock_sendall_reconnection(self): + sock = mock.Mock() + sock.fileno.return_value = 10 + sock.send.side_effect = BlockingIOError + sock.gettimeout.return_value = 0.0 + + self.loop.add_writer = mock.Mock() + self.loop.remove_writer = mock.Mock() + fut = self.loop.sock_sendall(sock, b'data') + callback = self.loop.add_writer.call_args[0][1] + params = self.loop.add_writer.call_args[0][2:] + + # emulate the old socket has closed, but the new one has + # the same fileno, so callback is called with old (closed) socket + sock.fileno.return_value = -1 + sock.send.side_effect = OSError(9) + callback(*params) + + self.assertIsInstance(fut.exception(), OSError) + self.assertEqual((10,), self.loop.remove_writer.call_args[0]) + def test__sock_sendall_canceled_fut(self): sock = mock.Mock() f = asyncio.Future(loop=self.loop) f.cancel() - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertFalse(sock.send.called) def test__sock_sendall_unregister(self): @@ -261,7 +303,7 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): f.cancel() self.loop.remove_writer = mock.Mock() - self.loop._sock_sendall(f, True, sock, b'data') + self.loop._sock_sendall(f, 10, sock, b'data') self.assertEqual((10,), self.loop.remove_writer.call_args[0]) def test__sock_sendall_tryagain(self): @@ -271,9 +313,9 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock.send.side_effect = BlockingIOError self.loop.add_writer = mock.Mock() - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertEqual( - (10, self.loop._sock_sendall, f, True, sock, b'data'), + (10, self.loop._sock_sendall, f, 10, sock, b'data'), self.loop.add_writer.call_args[0]) def test__sock_sendall_interrupted(self): @@ -283,9 +325,9 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock.send.side_effect = InterruptedError self.loop.add_writer = mock.Mock() - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertEqual( - (10, self.loop._sock_sendall, f, True, sock, b'data'), + (10, self.loop._sock_sendall, f, 10, sock, b'data'), self.loop.add_writer.call_args[0]) def test__sock_sendall_exception(self): @@ -294,7 +336,7 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock.fileno.return_value = 10 err = sock.send.side_effect = OSError() - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertIs(f.exception(), err) def test__sock_sendall(self): @@ -304,7 +346,7 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock.fileno.return_value = 10 sock.send.return_value = 4 - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertTrue(f.done()) self.assertIsNone(f.result()) @@ -316,10 +358,10 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock.send.return_value = 2 self.loop.add_writer = mock.Mock() - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertFalse(f.done()) self.assertEqual( - (10, self.loop._sock_sendall, f, True, sock, b'ta'), + (10, self.loop._sock_sendall, f, 10, sock, b'ta'), self.loop.add_writer.call_args[0]) def test__sock_sendall_none(self): @@ -330,10 +372,10 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock.send.return_value = 0 self.loop.add_writer = mock.Mock() - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertFalse(f.done()) self.assertEqual( - (10, self.loop._sock_sendall, f, True, sock, b'data'), + (10, self.loop._sock_sendall, f, 10, sock, b'data'), self.loop.add_writer.call_args[0]) def test_sock_connect_timeout(self): diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index bcd236ea..f573ae8f 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -121,6 +121,14 @@ class SslProtoHandshakeTests(test_utils.TestCase): ssl_proto.connection_lost(None) self.assertIsNone(ssl_proto._get_extra_info('socket')) + def test_set_new_app_protocol(self): + waiter = asyncio.Future(loop=self.loop) + ssl_proto = self.ssl_protocol(waiter) + new_app_proto = asyncio.Protocol() + ssl_proto._app_transport.set_protocol(new_app_proto) + self.assertIs(ssl_proto._app_transport.get_protocol(), new_app_proto) + self.assertIs(ssl_proto._app_protocol, new_app_proto) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index b47433a4..6d16d200 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -3,6 +3,7 @@ import gc import os import queue +import pickle import socket import sys import threading @@ -845,6 +846,23 @@ os.close(fd) stream._transport.__repr__.return_value = "" self.assertEqual(">", repr(stream)) + def test_IncompleteReadError_pickleable(self): + e = asyncio.IncompleteReadError(b'abc', 10) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_protocol=proto): + e2 = pickle.loads(pickle.dumps(e, protocol=proto)) + self.assertEqual(str(e), str(e2)) + self.assertEqual(e.partial, e2.partial) + self.assertEqual(e.expected, e2.expected) + + def test_LimitOverrunError_pickleable(self): + e = asyncio.LimitOverrunError('message', 10) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_protocol=proto): + e2 = pickle.loads(pickle.dumps(e, protocol=proto)) + self.assertEqual(str(e), str(e2)) + self.assertEqual(e.consumed, e2.consumed) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 243faf6b..42da1fa1 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1918,7 +1918,7 @@ class BaseTaskTests: regex = (r'^ ' r'was never yielded from\n' - r'Coroutine object created at \(most recent call last\):\n' + r'Coroutine object created at \(most recent call last, truncated to \d+ last lines\):\n' r'.*\n' r' File "%s", line %s, in test_coroutine_never_yielded\n' r' coro_noop\(\)$' diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index b83f2f10..f302da41 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -17,7 +17,7 @@ from test import support from itertools import permutations, product from random import randrange, sample, choice import warnings -import sys, array, io +import sys, array, io, os from decimal import Decimal from fractions import Fraction @@ -37,7 +37,8 @@ except ImportError: ctypes = None try: - with warnings.catch_warnings(): + with support.EnvironmentVarGuard() as os.environ, \ + warnings.catch_warnings(): from numpy import ndarray as numpy_array except ImportError: numpy_array = None diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index cd82fa64..6fcc26a0 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -540,8 +540,16 @@ class BaseBytesTest: self.assertEqual(b.replace(b'i', b'a'), b'massassappa') self.assertEqual(b.replace(b'ss', b'x'), b'mixixippi') + def test_replace_int_error(self): + self.assertRaises(TypeError, self.type2test(b'a b').replace, 32, b'') + def test_split_string_error(self): self.assertRaises(TypeError, self.type2test(b'a b').split, ' ') + self.assertRaises(TypeError, self.type2test(b'a b').rsplit, ' ') + + def test_split_int_error(self): + self.assertRaises(TypeError, self.type2test(b'a b').split, 32) + self.assertRaises(TypeError, self.type2test(b'a b').rsplit, 32) def test_split_unicodewhitespace(self): for b in (b'a\x1Cb', b'a\x1Db', b'a\x1Eb', b'a\x1Fb'): @@ -550,9 +558,6 @@ class BaseBytesTest: b = self.type2test(b"\x09\x0A\x0B\x0C\x0D\x1C\x1D\x1E\x1F") self.assertEqual(b.split(), [b'\x1c\x1d\x1e\x1f']) - def test_rsplit_string_error(self): - self.assertRaises(TypeError, self.type2test(b'a b').rsplit, ' ') - def test_rsplit_unicodewhitespace(self): b = self.type2test(b"\x09\x0A\x0B\x0C\x0D\x1C\x1D\x1E\x1F") self.assertEqual(b.rsplit(), [b'\x1c\x1d\x1e\x1f']) @@ -568,6 +573,14 @@ class BaseBytesTest: self.assertEqual(b.rpartition(b'i'), (b'mississipp', b'i', b'')) self.assertEqual(b.rpartition(b'w'), (b'', b'', b'mississippi')) + def test_partition_string_error(self): + self.assertRaises(TypeError, self.type2test(b'a b').partition, ' ') + self.assertRaises(TypeError, self.type2test(b'a b').rpartition, ' ') + + def test_partition_int_error(self): + self.assertRaises(TypeError, self.type2test(b'a b').partition, 32) + self.assertRaises(TypeError, self.type2test(b'a b').rpartition, 32) + def test_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): for b in b"", b"a", b"abc", b"\xffab\x80", b"\0\0\377\0\0": @@ -600,9 +613,14 @@ class BaseBytesTest: self.assertEqual(self.type2test(b'abc').rstrip(memoryview(b'ac')), b'ab') def test_strip_string_error(self): - self.assertRaises(TypeError, self.type2test(b'abc').strip, 'b') - self.assertRaises(TypeError, self.type2test(b'abc').lstrip, 'b') - self.assertRaises(TypeError, self.type2test(b'abc').rstrip, 'b') + self.assertRaises(TypeError, self.type2test(b'abc').strip, 'ac') + self.assertRaises(TypeError, self.type2test(b'abc').lstrip, 'ac') + self.assertRaises(TypeError, self.type2test(b'abc').rstrip, 'ac') + + def test_strip_int_error(self): + self.assertRaises(TypeError, self.type2test(b' abc ').strip, 32) + self.assertRaises(TypeError, self.type2test(b' abc ').lstrip, 32) + self.assertRaises(TypeError, self.type2test(b' abc ').rstrip, 32) def test_center(self): # Fill character can be either bytes or bytearray (issue 12380) @@ -625,6 +643,11 @@ class BaseBytesTest: self.assertEqual(b.rjust(7, fill_type(b'-')), self.type2test(b'----abc')) + def test_xjust_int_error(self): + self.assertRaises(TypeError, self.type2test(b'abc').center, 7, 32) + self.assertRaises(TypeError, self.type2test(b'abc').ljust, 7, 32) + self.assertRaises(TypeError, self.type2test(b'abc').rjust, 7, 32) + def test_ord(self): b = self.type2test(b'\0A\x7f\x80\xff') self.assertEqual([ord(b[i:i+1]) for i in range(len(b))], diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 216851c2..6e4286ed 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -13,7 +13,7 @@ import time import unittest from test import support from test.support import MISSING_C_DOCSTRINGS -from test.support.script_helper import assert_python_failure +from test.support.script_helper import assert_python_failure, assert_python_ok try: import _posixsubprocess except ImportError: @@ -241,6 +241,38 @@ class CAPITest(unittest.TestCase): def test_buildvalue_N(self): _testcapi.test_buildvalue_N() + def test_set_nomemory(self): + code = """if 1: + import _testcapi + + class C(): pass + + # The first loop tests both functions and that remove_mem_hooks() + # can be called twice in a row. The second loop checks a call to + # set_nomemory() after a call to remove_mem_hooks(). The third + # loop checks the start and stop arguments of set_nomemory(). + for outer_cnt in range(1, 4): + start = 10 * outer_cnt + for j in range(100): + if j == 0: + if outer_cnt != 3: + _testcapi.set_nomemory(start) + else: + _testcapi.set_nomemory(start, start + 1) + try: + C() + except MemoryError as e: + if outer_cnt != 3: + _testcapi.remove_mem_hooks() + print('MemoryError', outer_cnt, j) + _testcapi.remove_mem_hooks() + break + """ + rc, out, err = assert_python_ok('-c', code) + self.assertIn(b'MemoryError 1 10', out) + self.assertIn(b'MemoryError 2 20', out) + self.assertIn(b'MemoryError 3 30', out) + @unittest.skipUnless(threading, 'Threading required for this test.') class TestPendingCalls(unittest.TestCase): @@ -369,23 +401,30 @@ class EmbeddingTests(unittest.TestCase): def tearDown(self): os.chdir(self.oldcwd) - def run_embedded_interpreter(self, *args): + def run_embedded_interpreter(self, *args, env=None): """Runs a test in the embedded interpreter""" cmd = [self.test_exe] cmd.extend(args) + if env is not None and sys.platform == 'win32': + # Windows requires at least the SYSTEMROOT environment variable to + # start Python. + env = env.copy() + env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) + universal_newlines=True, + env=env) (out, err) = p.communicate() self.assertEqual(p.returncode, 0, "bad returncode %d, stderr is %r" % (p.returncode, err)) return out, err - def test_subinterps(self): + def test_repeated_init_and_subinterpreters(self): # This is just a "don't crash" test - out, err = self.run_embedded_interpreter() + out, err = self.run_embedded_interpreter('repeated_init_and_subinterpreters') if support.verbose: print() print(out) @@ -403,13 +442,14 @@ class EmbeddingTests(unittest.TestCase): def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams - out, err = self.run_embedded_interpreter("forced_io_encoding") + env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") + out, err = self.run_embedded_interpreter("forced_io_encoding", env=env) if support.verbose: print() print(out) print(err) - expected_errors = sys.__stdout__.errors - expected_stdin_encoding = sys.__stdin__.encoding + expected_stream_encoding = "utf-8" + expected_errors = "surrogateescape" expected_pipe_encoding = self._get_default_pipe_encoding() expected_output = '\n'.join([ "--- Use defaults ---", @@ -437,13 +477,33 @@ class EmbeddingTests(unittest.TestCase): "stdout: latin-1:replace", "stderr: latin-1:backslashreplace"]) expected_output = expected_output.format( - in_encoding=expected_stdin_encoding, - out_encoding=expected_pipe_encoding, + in_encoding=expected_stream_encoding, + out_encoding=expected_stream_encoding, errors=expected_errors) # This is useful if we ever trip over odd platform behaviour self.maxDiff = None self.assertEqual(out.strip(), expected_output) + def test_pre_initialization_api(self): + """ + Checks the few parts of the C-API that work before the runtine + is initialized (via Py_Initialize()). + """ + env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) + out, err = self.run_embedded_interpreter("pre_initialization_api", env=env) + self.assertEqual(out, '') + self.assertEqual(err, '') + + def test_bpo20891(self): + """ + bpo-20891: Calling PyGILState_Ensure in a non-Python thread before + calling PyEval_InitThreads() must not crash. PyGILState_Ensure() must + call PyEval_InitThreads() for us in this case. + """ + out, err = self.run_embedded_interpreter("bpo20891") + self.assertEqual(out, '') + self.assertEqual(err, '') + class SkipitemTest(unittest.TestCase): diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 68537482..55faf4c4 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -102,6 +102,7 @@ consts: ('None',) """ +import inspect import sys import threading import unittest @@ -130,6 +131,10 @@ def dump(co): print("%s: %s" % (attr, getattr(co, "co_" + attr))) print("consts:", tuple(consts(co.co_consts))) +# Needed for test_closure_injection below +# Defined at global scope to avoid implicitly closing over __class__ +def external_getitem(self, i): + return f"Foreign getitem: {super().__getitem__(i)}" class CodeTest(unittest.TestCase): @@ -141,6 +146,46 @@ class CodeTest(unittest.TestCase): self.assertEqual(co.co_name, "funcname") self.assertEqual(co.co_firstlineno, 15) + @cpython_only + def test_closure_injection(self): + # From https://bugs.python.org/issue32176 + from types import FunctionType, CodeType + + def create_closure(__class__): + return (lambda: __class__).__closure__ + + def new_code(c): + '''A new code object with a __class__ cell added to freevars''' + return CodeType( + c.co_argcount, c.co_kwonlyargcount, c.co_nlocals, + c.co_stacksize, c.co_flags, c.co_code, c.co_consts, c.co_names, + c.co_varnames, c.co_filename, c.co_name, c.co_firstlineno, + c.co_lnotab, c.co_freevars + ('__class__',), c.co_cellvars) + + def add_foreign_method(cls, name, f): + code = new_code(f.__code__) + assert not f.__closure__ + closure = create_closure(cls) + defaults = f.__defaults__ + setattr(cls, name, FunctionType(code, globals(), name, defaults, closure)) + + class List(list): + pass + + add_foreign_method(List, "__getitem__", external_getitem) + + # Ensure the closure injection actually worked + function = List.__getitem__ + class_ref = function.__closure__[0].cell_contents + self.assertIs(class_ref, List) + + # Ensure the code correctly indicates it accesses a free variable + self.assertFalse(function.__code__.co_flags & inspect.CO_NOFREE, + hex(function.__code__.co_flags)) + + # Ensure the zero-arg super() call in the injected method works + obj = List([1, 2, 3]) + self.assertEqual(obj[0], "Foreign getitem: 1") def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) @@ -231,7 +276,7 @@ if check_impl_detail(cpython=True) and ctypes is not None: SetExtra.restype = ctypes.c_int GetExtra = py._PyCode_GetExtra - GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, + GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.POINTER(ctypes.c_voidp)) GetExtra.restype = ctypes.c_int diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 1a8f6990..24db0ace 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -28,16 +28,24 @@ class TestInteractiveConsole(unittest.TestCase): self.sysmod = stack.enter_context(prepatch) if sys.excepthook is sys.__excepthook__: self.sysmod.excepthook = self.sysmod.__excepthook__ + del self.sysmod.ps1 + del self.sysmod.ps2 def test_ps1(self): self.infunc.side_effect = EOFError('Finished') self.console.interact() self.assertEqual(self.sysmod.ps1, '>>> ') + self.sysmod.ps1 = 'custom1> ' + self.console.interact() + self.assertEqual(self.sysmod.ps1, 'custom1> ') def test_ps2(self): self.infunc.side_effect = EOFError('Finished') self.console.interact() self.assertEqual(self.sysmod.ps2, '... ') + self.sysmod.ps1 = 'custom2> ' + self.console.interact() + self.assertEqual(self.sysmod.ps1, 'custom2> ') def test_console_stderr(self): self.infunc.side_effect = ["'antioch'", "", EOFError('Finished')] diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 1e63ed8d..eb21a391 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -196,19 +196,33 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual(f.read(), ''.join(lines[1:])) self.assertEqual(f.read(), '') + # Issue #32110: Test readline() followed by read(n) + f = getreader() + self.assertEqual(f.readline(), lines[0]) + self.assertEqual(f.read(1), lines[1][0]) + self.assertEqual(f.read(0), '') + self.assertEqual(f.read(100), data[len(lines[0]) + 1:][:100]) + # Issue #16636: Test readline() followed by readlines() f = getreader() self.assertEqual(f.readline(), lines[0]) self.assertEqual(f.readlines(), lines[1:]) self.assertEqual(f.read(), '') - # Test read() followed by read() + # Test read(n) followed by read() f = getreader() self.assertEqual(f.read(size=40, chars=5), data[:5]) self.assertEqual(f.read(), data[5:]) self.assertEqual(f.read(), '') - # Issue #12446: Test read() followed by readlines() + # Issue #32110: Test read(n) followed by read(n) + f = getreader() + self.assertEqual(f.read(size=40, chars=5), data[:5]) + self.assertEqual(f.read(1), data[5]) + self.assertEqual(f.read(0), '') + self.assertEqual(f.read(100), data[6:106]) + + # Issue #12446: Test read(n) followed by readlines() f = getreader() self.assertEqual(f.read(size=40, chars=5), data[:5]) self.assertEqual(f.readlines(), [lines[0][5:]] + lines[1:]) @@ -1203,6 +1217,8 @@ class EscapeDecodeTest(unittest.TestCase): check(br"\8", b"\\8") with self.assertWarns(DeprecationWarning): check(br"\9", b"\\9") + with self.assertWarns(DeprecationWarning): + check(b"\\\xfa", b"\\\xfa") def test_errors(self): decode = codecs.escape_decode @@ -2474,6 +2490,8 @@ class UnicodeEscapeTest(unittest.TestCase): check(br"\8", "\\8") with self.assertWarns(DeprecationWarning): check(br"\9", "\\9") + with self.assertWarns(DeprecationWarning): + check(b"\\\xfa", "\\\xfa") def test_decode_errors(self): decode = codecs.unicode_escape_decode diff --git a/Lib/test/test_crypt.py b/Lib/test/test_crypt.py index e4f58979..44a3ad44 100644 --- a/Lib/test/test_crypt.py +++ b/Lib/test/test_crypt.py @@ -1,33 +1,45 @@ +import sys from test import support import unittest crypt = support.import_module('crypt') +if sys.platform.startswith('openbsd'): + raise unittest.SkipTest('The only supported method on OpenBSD is Blowfish') + class CryptTestCase(unittest.TestCase): def test_crypt(self): - c = crypt.crypt('mypassword', 'ab') - if support.verbose: - print('Test encryption: ', c) + cr = crypt.crypt('mypassword') + cr2 = crypt.crypt('mypassword', cr) + self.assertEqual(cr2, cr) + cr = crypt.crypt('mypassword', 'ab') + if cr is not None: + cr2 = crypt.crypt('mypassword', cr) + self.assertEqual(cr2, cr) def test_salt(self): self.assertEqual(len(crypt._saltchars), 64) for method in crypt.methods: salt = crypt.mksalt(method) - self.assertEqual(len(salt), - method.salt_chars + (3 if method.ident else 0)) + self.assertIn(len(salt) - method.salt_chars, {0, 1, 3, 4, 6, 7}) + if method.ident: + self.assertIn(method.ident, salt[:len(salt)-method.salt_chars]) def test_saltedcrypt(self): for method in crypt.methods: - pw = crypt.crypt('assword', method) - self.assertEqual(len(pw), method.total_size) - pw = crypt.crypt('assword', crypt.mksalt(method)) - self.assertEqual(len(pw), method.total_size) + cr = crypt.crypt('assword', method) + self.assertEqual(len(cr), method.total_size) + cr2 = crypt.crypt('assword', cr) + self.assertEqual(cr2, cr) + cr = crypt.crypt('assword', crypt.mksalt(method)) + self.assertEqual(len(cr), method.total_size) def test_methods(self): # Guarantee that METHOD_CRYPT is the last method in crypt.methods. self.assertTrue(len(crypt.methods) >= 1) self.assertEqual(crypt.METHOD_CRYPT, crypt.methods[-1]) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 0d0b160d..3b442fe6 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -15,7 +15,7 @@ import sys import tempfile import unittest -from test.support import requires, import_module, verbose +from test.support import requires, import_module, verbose, SaveSignals # Optionally test curses module. This currently requires that the # 'curses' resource be given on the regrtest command line using the -u @@ -25,9 +25,12 @@ requires('curses') # If either of these don't exist, skip the tests. curses = import_module('curses') -import_module('curses.panel') import_module('curses.ascii') import_module('curses.textpad') +try: + import curses.panel +except ImportError: + pass def requires_curses_func(name): return unittest.skipUnless(hasattr(curses, name), @@ -63,6 +66,8 @@ class TestCurses(unittest.TestCase): del cls.tmp def setUp(self): + self.save_signals = SaveSignals() + self.save_signals.save() if verbose: # just to make the test output a little more readable print() @@ -72,6 +77,7 @@ class TestCurses(unittest.TestCase): def tearDown(self): curses.resetty() curses.endwin() + self.save_signals.restore() def test_window_funcs(self): "Test the methods of windows" @@ -86,7 +92,7 @@ class TestCurses(unittest.TestCase): with self.subTest(meth=meth.__qualname__, args=args): meth(*args) - for meth in [stdscr.box, stdscr.clear, stdscr.clrtobot, + for meth in [stdscr.clear, stdscr.clrtobot, stdscr.clrtoeol, stdscr.cursyncup, stdscr.delch, stdscr.deleteln, stdscr.erase, stdscr.getbegyx, stdscr.getbkgd, stdscr.getkey, stdscr.getmaxyx, @@ -120,6 +126,13 @@ class TestCurses(unittest.TestCase): win.border(65, 66, 67, 68, 69, [], 71, 72) + win.box(65, 67) + win.box('!', '_') + win.box(b':', b'~') + self.assertRaises(TypeError, win.box, 65, 66, 67) + self.assertRaises(TypeError, win.box, 65) + win.box() + stdscr.clearok(1) win4 = stdscr.derwin(2,2) @@ -135,7 +148,9 @@ class TestCurses(unittest.TestCase): stdscr.idcok(1) stdscr.idlok(1) - stdscr.immedok(1) + if hasattr(stdscr, 'immedok'): + stdscr.immedok(1) + stdscr.immedok(0) stdscr.insch('c') stdscr.insdelln(1) stdscr.insnstr('abc', 3) @@ -169,25 +184,27 @@ class TestCurses(unittest.TestCase): stdscr.setscrreg(10,15) win3 = stdscr.subwin(10,10) win3 = stdscr.subwin(10,10, 5,5) - stdscr.syncok(1) + if hasattr(stdscr, 'syncok') and not sys.platform.startswith("sunos"): + stdscr.syncok(1) stdscr.timeout(5) stdscr.touchline(5,5) stdscr.touchline(5,5,0) stdscr.vline('a', 3) stdscr.vline('a', 3, curses.A_STANDOUT) - stdscr.chgat(5, 2, 3, curses.A_BLINK) - stdscr.chgat(3, curses.A_BOLD) - stdscr.chgat(5, 8, curses.A_UNDERLINE) - stdscr.chgat(curses.A_BLINK) + if hasattr(stdscr, 'chgat'): + stdscr.chgat(5, 2, 3, curses.A_BLINK) + stdscr.chgat(3, curses.A_BOLD) + stdscr.chgat(5, 8, curses.A_UNDERLINE) + stdscr.chgat(curses.A_BLINK) stdscr.refresh() stdscr.vline(1,1, 'a', 3) stdscr.vline(1,1, 'a', 3, curses.A_STANDOUT) - if hasattr(curses, 'resize'): - stdscr.resize() - if hasattr(curses, 'enclose'): - stdscr.enclose() + if hasattr(stdscr, 'resize'): + stdscr.resize(25, 80) + if hasattr(stdscr, 'enclose'): + stdscr.enclose(10, 10) self.assertRaises(ValueError, stdscr.getstr, -400) self.assertRaises(ValueError, stdscr.getstr, 2, 3, -400) @@ -208,15 +225,19 @@ class TestCurses(unittest.TestCase): "Test module-level functions" for func in [curses.baudrate, curses.beep, curses.can_change_color, curses.cbreak, curses.def_prog_mode, curses.doupdate, - curses.filter, curses.flash, curses.flushinp, + curses.flash, curses.flushinp, curses.has_colors, curses.has_ic, curses.has_il, curses.isendwin, curses.killchar, curses.longname, curses.nocbreak, curses.noecho, curses.nonl, curses.noqiflush, curses.noraw, curses.reset_prog_mode, curses.termattrs, - curses.termname, curses.erasechar, curses.getsyx]: + curses.termname, curses.erasechar]: with self.subTest(func=func.__qualname__): func() + if hasattr(curses, 'filter'): + curses.filter() + if hasattr(curses, 'getsyx'): + curses.getsyx() # Functions that actually need arguments if curses.tigetstr("cnorm"): @@ -240,15 +261,18 @@ class TestCurses(unittest.TestCase): curses.putp(b'abc') curses.qiflush() curses.raw() ; curses.raw(1) - curses.setsyx(5,5) + if hasattr(curses, 'setsyx'): + curses.setsyx(5,5) curses.tigetflag('hc') curses.tigetnum('co') curses.tigetstr('cr') curses.tparm(b'cr') - curses.typeahead(sys.__stdin__.fileno()) + if hasattr(curses, 'typeahead'): + curses.typeahead(sys.__stdin__.fileno()) curses.unctrl('a') curses.ungetch('a') - curses.use_env(1) + if hasattr(curses, 'use_env'): + curses.use_env(1) # Functions only available on a few platforms def test_colors_funcs(self): @@ -282,6 +306,7 @@ class TestCurses(unittest.TestCase): curses.ungetmouse(0, 0, 0, 0, curses.BUTTON1_PRESSED) m = curses.getmouse() + @requires_curses_func('panel') def test_userptr_without_set(self): w = curses.newwin(10, 10) p = curses.panel.new_panel(w) @@ -290,6 +315,7 @@ class TestCurses(unittest.TestCase): msg='userptr should fail since not set'): p.userptr() + @requires_curses_func('panel') def test_userptr_memory_leak(self): w = curses.newwin(10, 10) p = curses.panel.new_panel(w) @@ -302,16 +328,20 @@ class TestCurses(unittest.TestCase): self.assertEqual(sys.getrefcount(obj), nrefs, "set_userptr leaked references") + @requires_curses_func('panel') def test_userptr_segfault(self): - panel = curses.panel.new_panel(self.stdscr) + w = curses.newwin(10, 10) + panel = curses.panel.new_panel(w) class A: def __del__(self): panel.set_userptr(None) panel.set_userptr(A()) panel.set_userptr(None) + @requires_curses_func('panel') def test_new_curses_panel(self): - panel = curses.panel.new_panel(self.stdscr) + w = curses.newwin(10, 10) + panel = curses.panel.new_panel(w) self.assertRaises(TypeError, type(panel)) @requires_curses_func('is_term_resized') @@ -338,6 +368,9 @@ class TestCurses(unittest.TestCase): self.stdscr.getkey() @requires_curses_func('unget_wch') + # XXX Remove the decorator when ncurses on OpenBSD be updated + @unittest.skipIf(sys.platform.startswith("openbsd"), + "OpenBSD's curses (v.5.7) has bugs") def test_unget_wch(self): stdscr = self.stdscr encoding = stdscr.encoding @@ -404,6 +437,8 @@ class TestCurses(unittest.TestCase): def test_issue13051(self): stdscr = self.stdscr + if not hasattr(stdscr, 'resize'): + raise unittest.SkipTest('requires curses.window.resize') box = curses.textpad.Textbox(stdscr, insert_mode=True) lines, cols = stdscr.getmaxyx() stdscr.resize(lines-2, cols-2) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index e0ec87d2..1667617b 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -14,18 +14,7 @@ class TestTokens(TestEmailBase): self.assertEqual(x, ' \t') self.assertEqual(str(x), '') self.assertEqual(x.value, '') - self.assertEqual(x.encoded, ' \t') - - # UnstructuredTokenList - - def test_undecodable_bytes_error_preserved(self): - badstr = b"le pouf c\xaflebre".decode('ascii', 'surrogateescape') - unst = parser.get_unstructured(badstr) - self.assertDefectsEqual(unst.all_defects, [errors.UndecodableBytesDefect]) - parts = list(unst.parts) - self.assertDefectsEqual(parts[0].all_defects, []) - self.assertDefectsEqual(parts[1].all_defects, []) - self.assertDefectsEqual(parts[2].all_defects, [errors.UndecodableBytesDefect]) + self.assertEqual(x.token_type, 'fws') class TestParserMixin: @@ -139,7 +128,6 @@ class TestParser(TestParserMixin, TestEmailBase): 'first second', [], '') - self.assertEqual(ew.encoded, '=?us-ascii*jive?q?first_second?=') self.assertEqual(ew.charset, 'us-ascii') self.assertEqual(ew.lang, 'jive') @@ -150,7 +138,6 @@ class TestParser(TestParserMixin, TestEmailBase): 'first second', [], '') - self.assertEqual(ew.encoded, '=?us-ascii?q?first_second?=') self.assertEqual(ew.charset, 'us-ascii') self.assertEqual(ew.lang, '') @@ -2700,28 +2687,37 @@ class TestFolding(TestEmailBase): # and with unicode tokens in the comments. Spaces inside the quotes # currently don't do the right thing. - def test_initial_whitespace_splitting(self): + def test_split_at_whitespace_after_header_before_long_token(self): body = parser.get_unstructured(' ' + 'x'*77) header = parser.Header([ parser.HeaderLabel([parser.ValueTerminal('test:', 'atext')]), parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]), body]) self._test(header, 'test: \n ' + 'x'*77 + '\n') - def test_whitespace_splitting(self): + def test_split_at_whitespace_before_long_token(self): self._test(parser.get_unstructured('xxx ' + 'y'*77), 'xxx \n ' + 'y'*77 + '\n') + def test_overlong_encodeable_is_wrapped(self): + first_token_with_whitespace = 'xxx ' + chrome_leader = '=?utf-8?q?' + len_chrome = len(chrome_leader) + 2 + len_non_y = len_chrome + len(first_token_with_whitespace) + self._test(parser.get_unstructured(first_token_with_whitespace + + 'y'*80), + first_token_with_whitespace + chrome_leader + + 'y'*(78-len_non_y) + '?=\n' + + ' ' + chrome_leader + 'y'*(80-(78-len_non_y)) + '?=\n') + def test_long_filename_attachment(self): - folded = self.policy.fold('Content-Disposition', 'attachment; filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TES.txt"') - self.assertEqual( - 'Content-Disposition: attachment;\n filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TES.txt"\n', - folded - ) - folded = self.policy.fold('Content-Disposition', 'attachment; filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_T.txt"') - self.assertEqual( - 'Content-Disposition: attachment;\n filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_T.txt"\n', - folded - ) + self._test(parser.parse_content_disposition_header( + 'attachment; filename="TEST_TEST_TEST_TEST' + '_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TES.txt"'), + "attachment;\n" + " filename*0*=us-ascii''TEST_TEST_TEST_TEST_TEST_TEST" + "_TEST_TEST_TEST_TEST_TEST;\n" + " filename*1*=_TEST_TES.txt\n", + ) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index c4f18290..c1aeaefa 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -27,7 +27,6 @@ class TestGeneratorBase: None """), - # From is wrapped because wrapped it fits in 40. 40: textwrap.dedent("""\ To: whom_it_may_concern@example.com From: @@ -40,11 +39,11 @@ class TestGeneratorBase: None """), - # Neither to nor from fit even if put on a new line, - # so we leave them sticking out on the first line. 20: textwrap.dedent("""\ - To: whom_it_may_concern@example.com - From: nobody_you_want_to_know@example.com + To: + whom_it_may_concern@example.com + From: + nobody_you_want_to_know@example.com Subject: We the willing led by the unknowing are doing @@ -169,6 +168,53 @@ class TestGeneratorBase: g.flatten(msg) self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[0])) + def test_rfc2231_wrapping(self): + # This is pretty much just to make sure we don't have an infinite + # loop; I don't expect anyone to hit this in the field. + msg = self.msgmaker(self.typ(textwrap.dedent("""\ + To: nobody + Content-Disposition: attachment; + filename="afilenamelongenoghtowraphere" + + None + """))) + expected = textwrap.dedent("""\ + To: nobody + Content-Disposition: attachment; + filename*0*=us-ascii''afilename; + filename*1*=longenoghtowraphere + + None + """) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(max_line_length=33)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + def test_rfc2231_wrapping_switches_to_default_len_if_too_narrow(self): + # This is just to make sure we don't have an infinite loop; I don't + # expect anyone to hit this in the field, so I'm not bothering to make + # the result optimal (the encoding isn't needed). + msg = self.msgmaker(self.typ(textwrap.dedent("""\ + To: nobody + Content-Disposition: attachment; + filename="afilenamelongenoghtowraphere" + + None + """))) + expected = textwrap.dedent("""\ + To: nobody + Content-Disposition: + attachment; + filename*0*=us-ascii''afilenamelongenoghtowraphere + + None + """) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(max_line_length=20)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + class TestGenerator(TestGeneratorBase, TestEmailBase): diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index af836dc9..30ce0ba5 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -229,14 +229,14 @@ class TestContentTypeHeader(TestHeaderBase): defects = args[1] if l>1 else [] decoded = args[2] if l>2 and args[2] is not DITTO else source header = 'Content-Type:' + ' ' if source else '' - folded = args[3] if l>3 else header + source + '\n' + folded = args[3] if l>3 else header + decoded + '\n' h = self.make_header('Content-Type', source) self.assertEqual(h.content_type, content_type) self.assertEqual(h.maintype, maintype) self.assertEqual(h.subtype, subtype) self.assertEqual(h.params, parmdict) with self.assertRaises(TypeError): - h.params['abc'] = 'xyz' # params is read-only. + h.params['abc'] = 'xyz' # make sure params is read-only. self.assertDefectsEqual(h.defects, defects) self.assertEqual(h, decoded) self.assertEqual(h.fold(policy=policy.default), folded) @@ -373,9 +373,10 @@ class TestContentTypeHeader(TestHeaderBase): 'text/plain; Charset="utf-8"'), # Since this is pretty much the ur-mimeheader, we'll put all the tests - # that exercise the parameter parsing and formatting here. - # - # XXX: question: is minimal quoting preferred? + # that exercise the parameter parsing and formatting here. Note that + # when we refold we may canonicalize, so things like whitespace, + # quoting, and rfc2231 encoding may change from what was in the input + # header. 'unquoted_param_value': ( 'text/plain; title=foo', @@ -384,7 +385,8 @@ class TestContentTypeHeader(TestHeaderBase): 'plain', {'title': 'foo'}, [], - 'text/plain; title="foo"'), + 'text/plain; title="foo"', + ), 'param_value_with_tspecials': ( 'text/plain; title="(bar)foo blue"', @@ -415,7 +417,8 @@ class TestContentTypeHeader(TestHeaderBase): 'mixed', {'boundary': 'CPIMSSMTPC06p5f3tG'}, [], - 'Multipart/mixed; boundary="CPIMSSMTPC06p5f3tG"'), + 'Multipart/mixed; boundary="CPIMSSMTPC06p5f3tG"', + ), 'spaces_around_semis': ( ('image/jpeg; name="wibble.JPG" ; x-mac-type="4A504547" ; ' @@ -429,14 +432,31 @@ class TestContentTypeHeader(TestHeaderBase): [], ('image/jpeg; name="wibble.JPG"; x-mac-type="4A504547"; ' 'x-mac-creator="474B4F4E"'), - # XXX: it could be that we will eventually prefer to fold starting - # from the decoded value, in which case these spaces and similar - # spaces in other tests will be wrong. - ('Content-Type: image/jpeg; name="wibble.JPG" ; ' - 'x-mac-type="4A504547" ;\n' + ('Content-Type: image/jpeg; name="wibble.JPG";' + ' x-mac-type="4A504547";\n' ' x-mac-creator="474B4F4E"\n'), ), + 'lots_of_mime_params': ( + ('image/jpeg; name="wibble.JPG"; x-mac-type="4A504547"; ' + 'x-mac-creator="474B4F4E"; x-extrastuff="make it longer"'), + 'image/jpeg', + 'image', + 'jpeg', + {'name': 'wibble.JPG', + 'x-mac-type': '4A504547', + 'x-mac-creator': '474B4F4E', + 'x-extrastuff': 'make it longer'}, + [], + ('image/jpeg; name="wibble.JPG"; x-mac-type="4A504547"; ' + 'x-mac-creator="474B4F4E"; x-extrastuff="make it longer"'), + # In this case the whole of the MimeParameters does *not* fit + # one one line, so we break at a lower syntactic level. + ('Content-Type: image/jpeg; name="wibble.JPG";' + ' x-mac-type="4A504547";\n' + ' x-mac-creator="474B4F4E"; x-extrastuff="make it longer"\n'), + ), + 'semis_inside_quotes': ( 'image/jpeg; name="Jim&&Jill"', 'image/jpeg', @@ -460,19 +480,25 @@ class TestContentTypeHeader(TestHeaderBase): [], r'image/jpeg; name="Jim \"Bob\" Jill"'), - # XXX: This test works except for the refolding of the header. I'll - # deal with that bug when I deal with the other folding bugs. - #'non_ascii_in_params': ( - # ('foo\xa7/bar; b\xa7r=two; ' - # 'baz=thr\xa7e'.encode('latin-1').decode('us-ascii', - # 'surrogateescape')), - # 'foo\uFFFD/bar', - # 'foo\uFFFD', - # 'bar', - # {'b\uFFFDr': 'two', 'baz': 'thr\uFFFDe'}, - # [errors.UndecodableBytesDefect]*3, - # 'foo�/bar; b�r="two"; baz="thr�e"', - # ), + 'non_ascii_in_params': ( + ('foo\xa7/bar; b\xa7r=two; ' + 'baz=thr\xa7e'.encode('latin-1').decode('us-ascii', + 'surrogateescape')), + 'foo\uFFFD/bar', + 'foo\uFFFD', + 'bar', + {'b\uFFFDr': 'two', 'baz': 'thr\uFFFDe'}, + [errors.UndecodableBytesDefect]*3, + 'foo�/bar; b�r="two"; baz="thr�e"', + # XXX Two bugs here: the mime type is not allowed to be an encoded + # word, and we shouldn't be emitting surrogates in the parameter + # names. But I don't know what the behavior should be here, so I'm + # punting for now. In practice this is unlikely to be encountered + # since headers with binary in them only come from a binary source + # and are almost certain to be re-emitted without refolding. + 'Content-Type: =?unknown-8bit?q?foo=A7?=/bar; b\udca7r="two";\n' + " baz*=unknown-8bit''thr%A7e\n", + ), # RFC 2231 parameter tests. @@ -494,19 +520,20 @@ class TestContentTypeHeader(TestHeaderBase): [], r'image/jpeg; bar="baz\"foobar\"baz"'), - # XXX: This test works except for the refolding of the header. I'll - # deal with that bug when I deal with the other folding bugs. - #'non_ascii_rfc2231_value': ( - # ('text/plain; charset=us-ascii; ' - # "title*=us-ascii'en'This%20is%20" - # 'not%20f\xa7n').encode('latin-1').decode('us-ascii', - # 'surrogateescape'), - # 'text/plain', - # 'text', - # 'plain', - # {'charset': 'us-ascii', 'title': 'This is not f\uFFFDn'}, - # [errors.UndecodableBytesDefect], - # 'text/plain; charset="us-ascii"; title="This is not f�n"'), + 'non_ascii_rfc2231_value': ( + ('text/plain; charset=us-ascii; ' + "title*=us-ascii'en'This%20is%20" + 'not%20f\xa7n').encode('latin-1').decode('us-ascii', + 'surrogateescape'), + 'text/plain', + 'text', + 'plain', + {'charset': 'us-ascii', 'title': 'This is not f\uFFFDn'}, + [errors.UndecodableBytesDefect], + 'text/plain; charset="us-ascii"; title="This is not f�n"', + 'Content-Type: text/plain; charset="us-ascii";\n' + " title*=unknown-8bit''This%20is%20not%20f%A7n\n", + ), 'rfc2231_encoded_charset': ( 'text/plain; charset*=ansi-x3.4-1968\'\'us-ascii', @@ -529,8 +556,6 @@ class TestContentTypeHeader(TestHeaderBase): {'name': 'This is ***fun*** is it not.pdf'}, [], 'text/plain; name="This is ***fun*** is it not.pdf"', - ('Content-Type: text/plain;\tname*0*=\'\'This%20is%20;\n' - '\tname*1*=%2A%2A%2Afun%2A%2A%2A%20;\tname*2="is it not.pdf"\n'), ), # Make sure we also handle it if there are spurious double quotes. @@ -545,9 +570,6 @@ class TestContentTypeHeader(TestHeaderBase): {'name': 'This is even more ***fun*** is it not.pdf'}, [errors.InvalidHeaderDefect]*2, 'text/plain; name="This is even more ***fun*** is it not.pdf"', - ('Content-Type: text/plain;\t' - 'name*0*="us-ascii\'\'This%20is%20even%20more%20";\n' - '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";\tname*2="is it not.pdf"\n'), ), 'rfc2231_single_quote_inside_double_quotes': ( @@ -562,9 +584,8 @@ class TestContentTypeHeader(TestHeaderBase): [errors.InvalidHeaderDefect]*2, ('text/plain; charset="us-ascii"; ' 'title="This is really ***fun*** isn\'t it!"'), - ('Content-Type: text/plain; charset=us-ascii;\n' - '\ttitle*0*="us-ascii\'en\'This%20is%20really%20";\n' - '\ttitle*1*="%2A%2A%2Afun%2A%2A%2A%20";\ttitle*2="isn\'t it!"\n'), + ('Content-Type: text/plain; charset="us-ascii";\n' + ' title="This is really ***fun*** isn\'t it!"\n'), ), 'rfc2231_single_quote_in_value_with_charset_and_lang': ( @@ -576,9 +597,6 @@ class TestContentTypeHeader(TestHeaderBase): {'name': "Frank's Document"}, [errors.InvalidHeaderDefect]*2, 'application/x-foo; name="Frank\'s Document"', - ('Content-Type: application/x-foo;\t' - 'name*0*="us-ascii\'en-us\'Frank\'s";\n' - ' name*1*=" Document"\n'), ), 'rfc2231_single_quote_in_non_encoded_value': ( @@ -590,9 +608,6 @@ class TestContentTypeHeader(TestHeaderBase): {'name': "us-ascii'en-us'Frank's Document"}, [], 'application/x-foo; name="us-ascii\'en-us\'Frank\'s Document"', - ('Content-Type: application/x-foo;\t' - 'name*0="us-ascii\'en-us\'Frank\'s";\n' - ' name*1=" Document"\n'), ), 'rfc2231_no_language_or_charset': ( @@ -615,12 +630,8 @@ class TestContentTypeHeader(TestHeaderBase): {'name': 'This is even more ***fun*** is it.pdf'}, [errors.InvalidHeaderDefect]*2, 'text/plain; name="This is even more ***fun*** is it.pdf"', - ('Content-Type: text/plain;\t' - 'name*0*="\'\'This%20is%20even%20more%20";\n' - '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";\tname*2="is it.pdf"\n'), ), - # XXX: see below...the first name line here should be *0 not *0*. 'rfc2231_partly_encoded': ( ("text/plain;" '\tname*0*="\'\'This%20is%20even%20more%20";' @@ -632,9 +643,6 @@ class TestContentTypeHeader(TestHeaderBase): {'name': 'This is even more ***fun*** is it.pdf'}, [errors.InvalidHeaderDefect]*2, 'text/plain; name="This is even more ***fun*** is it.pdf"', - ('Content-Type: text/plain;\t' - 'name*0*="\'\'This%20is%20even%20more%20";\n' - '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";\tname*2="is it.pdf"\n'), ), 'rfc2231_partly_encoded_2': ( @@ -647,10 +655,11 @@ class TestContentTypeHeader(TestHeaderBase): 'plain', {'name': 'This is even more %2A%2A%2Afun%2A%2A%2A%20is it.pdf'}, [errors.InvalidHeaderDefect], - 'text/plain; name="This is even more %2A%2A%2Afun%2A%2A%2A%20is it.pdf"', - ('Content-Type: text/plain;\t' - 'name*0*="\'\'This%20is%20even%20more%20";\n' - '\tname*1="%2A%2A%2Afun%2A%2A%2A%20";\tname*2="is it.pdf"\n'), + ('text/plain;' + ' name="This is even more %2A%2A%2Afun%2A%2A%2A%20is it.pdf"'), + ('Content-Type: text/plain;\n' + ' name="This is even more %2A%2A%2Afun%2A%2A%2A%20is' + ' it.pdf"\n'), ), 'rfc2231_unknown_charset_treated_as_ascii': ( @@ -669,9 +678,12 @@ class TestContentTypeHeader(TestHeaderBase): 'plain', {'charset': 'utf-8\uFFFD\uFFFD\uFFFD'}, [errors.UndecodableBytesDefect], - 'text/plain; charset="utf-8\uFFFD\uFFFD\uFFFD"'), + 'text/plain; charset="utf-8\uFFFD\uFFFD\uFFFD"', + "Content-Type: text/plain;" + " charset*=unknown-8bit''utf-8%F1%F2%F3\n", + ), - 'rfc2231_utf_8_in_supposedly_ascii_charset_parameter_value': ( + 'rfc2231_utf8_in_supposedly_ascii_charset_parameter_value': ( "text/plain; charset*=ascii''utf-8%E2%80%9D", 'text/plain', 'text', @@ -679,9 +691,11 @@ class TestContentTypeHeader(TestHeaderBase): {'charset': 'utf-8”'}, [errors.UndecodableBytesDefect], 'text/plain; charset="utf-8”"', + # XXX Should folding change the charset to utf8? Currently it just + # reproduces the original, which is arguably fine. + "Content-Type: text/plain;" + " charset*=unknown-8bit''utf-8%E2%80%9D\n", ), - # XXX: if the above were *re*folded, it would get tagged as utf-8 - # instead of ascii in the param, since it now contains non-ASCII. 'rfc2231_encoded_then_unencoded_segments': ( ('application/x-foo;' @@ -694,9 +708,6 @@ class TestContentTypeHeader(TestHeaderBase): {'name': 'My Document For You'}, [errors.InvalidHeaderDefect], 'application/x-foo; name="My Document For You"', - ('Content-Type: application/x-foo;\t' - 'name*0*="us-ascii\'en-us\'My";\n' - '\tname*1=" Document";\tname*2=" For You"\n'), ), # My reading of the RFC is that this is an invalid header. The RFC @@ -713,11 +724,6 @@ class TestContentTypeHeader(TestHeaderBase): {'name': 'My Document For You'}, [errors.InvalidHeaderDefect]*3, 'application/x-foo; name="My Document For You"', - ("Content-Type: application/x-foo;\tname*0=us-ascii'en-us'My;\t" - # XXX: the newline is in the wrong place, come back and fix - # this when the rest of tests pass. - 'name*1*=" Document"\n;' - '\tname*2*=" For You"\n'), ), # XXX: I would say this one should default to ascii/en for the @@ -730,8 +736,7 @@ class TestContentTypeHeader(TestHeaderBase): # charset'lang'value pattern exactly *and* there is at least one # encoded segment. Implementing that algorithm will require some # refactoring, so I haven't done it (yet). - - 'rfc2231_qouted_unencoded_then_encoded_segments': ( + 'rfc2231_quoted_unencoded_then_encoded_segments': ( ('application/x-foo;' '\tname*0="us-ascii\'en-us\'My";' '\tname*1*=" Document";' @@ -742,9 +747,25 @@ class TestContentTypeHeader(TestHeaderBase): {'name': "us-ascii'en-us'My Document For You"}, [errors.InvalidHeaderDefect]*2, 'application/x-foo; name="us-ascii\'en-us\'My Document For You"', - ('Content-Type: application/x-foo;\t' - 'name*0="us-ascii\'en-us\'My";\n' - '\tname*1*=" Document";\tname*2*=" For You"\n'), + ), + + # Make sure our folding algorithm produces multiple sections correctly. + # We could mix encoded and non-encoded segments, but we don't, we just + # make them all encoded. It might be worth fixing that, since the + # sections can get used for wrapping ascii text. + 'rfc2231_folded_segments_correctly_formatted': ( + ('application/x-foo;' + '\tname="' + "with spaces"*8 + '"'), + 'application/x-foo', + 'application', + 'x-foo', + {'name': "with spaces"*8}, + [], + 'application/x-foo; name="' + "with spaces"*8 + '"', + "Content-Type: application/x-foo;\n" + " name*0*=us-ascii''with%20spaceswith%20spaceswith%20spaceswith" + "%20spaceswith;\n" + " name*1*=%20spaceswith%20spaceswith%20spaceswith%20spaces\n" ), } @@ -827,8 +848,8 @@ class TestContentDisposition(TestHeaderBase): [], ('attachment; filename="genome.jpeg"; ' 'modification-date="Wed, 12 Feb 1997 16:29:51 -0500"'), - ('Content-Disposition: attachment; filename=genome.jpeg;\n' - ' modification-date="Wed, 12 Feb 1997 16:29:51 -0500";\n'), + ('Content-Disposition: attachment; filename="genome.jpeg";\n' + ' modification-date="Wed, 12 Feb 1997 16:29:51 -0500"\n'), ), 'no_value': ( @@ -873,7 +894,7 @@ class TestMIMEVersionHeader(TestHeaderBase): if source: source = ' ' + source self.assertEqual(h.fold(policy=policy.default), - 'MIME-Version:' + source + '\n') + 'MIME-Version:' + source + '\n') version_string_params = { @@ -1546,15 +1567,39 @@ class TestFolding(TestHeaderBase): 'singlewordthatwontfit') self.assertEqual( h.fold(policy=policy.default.clone(max_line_length=20)), - 'Subject: thisisaverylonglineconsistingofasinglewordthatwontfit\n') + 'Subject: \n' + ' =?utf-8?q?thisisa?=\n' + ' =?utf-8?q?verylon?=\n' + ' =?utf-8?q?glineco?=\n' + ' =?utf-8?q?nsistin?=\n' + ' =?utf-8?q?gofasin?=\n' + ' =?utf-8?q?gleword?=\n' + ' =?utf-8?q?thatwon?=\n' + ' =?utf-8?q?tfit?=\n' + ) def test_fold_unstructured_with_two_overlong_words(self): h = self.make_header('Subject', 'thisisaverylonglineconsistingofa' 'singlewordthatwontfit plusanotherverylongwordthatwontfit') self.assertEqual( h.fold(policy=policy.default.clone(max_line_length=20)), - 'Subject: thisisaverylonglineconsistingofasinglewordthatwontfit\n' - ' plusanotherverylongwordthatwontfit\n') + 'Subject: \n' + ' =?utf-8?q?thisisa?=\n' + ' =?utf-8?q?verylon?=\n' + ' =?utf-8?q?glineco?=\n' + ' =?utf-8?q?nsistin?=\n' + ' =?utf-8?q?gofasin?=\n' + ' =?utf-8?q?gleword?=\n' + ' =?utf-8?q?thatwon?=\n' + ' =?utf-8?q?tfit_pl?=\n' + ' =?utf-8?q?usanoth?=\n' + ' =?utf-8?q?erveryl?=\n' + ' =?utf-8?q?ongword?=\n' + ' =?utf-8?q?thatwon?=\n' + ' =?utf-8?q?tfit?=\n' + ) + + # XXX Need test for when max_line_length is less than the chrome size. def test_fold_unstructured_with_slightly_long_word(self): h = self.make_header('Subject', 'thislongwordislessthanmaxlinelen') @@ -1590,6 +1635,18 @@ class TestFolding(TestHeaderBase): self.assertEqual(h.fold(policy=policy.default), 'Date: Sat, 02 Feb 2002 17:00:06 -0800\n') + def test_fold_overlong_words_using_RFC2047(self): + h = self.make_header( + 'X-Report-Abuse', + '') + self.assertEqual( + h.fold(policy=policy.default), + 'X-Report-Abuse: =?utf-8?q?=3Chttps=3A//www=2Emailitapp=2E' + 'com/report=5F?=\n' + ' =?utf-8?q?abuse=2Ephp=3Fmid=3Dxxx-xxx-xxxx' + 'xxxxxxxxxxxxxxxxxxxx=3D=3D-xxx-?=\n' + ' =?utf-8?q?xx-xx=3E?=\n') if __name__ == '__main__': diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 9b04c18b..89966893 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -20,6 +20,7 @@ class EnsurepipMixin: def setUp(self): run_pip_patch = unittest.mock.patch("ensurepip._run_pip") self.run_pip = run_pip_patch.start() + self.run_pip.return_value = 0 self.addCleanup(run_pip_patch.stop) # Avoid side effects on the actual os module @@ -255,7 +256,7 @@ class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase): self.assertFalse(self.run_pip.called) def test_basic_bootstrapping(self): - ensurepip._main([]) + exit_code = ensurepip._main([]) self.run_pip.assert_called_once_with( [ @@ -267,6 +268,13 @@ class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase): additional_paths = self.run_pip.call_args[0][1] self.assertEqual(len(additional_paths), 2) + self.assertEqual(exit_code, 0) + + def test_bootstrapping_error_code(self): + self.run_pip.return_value = 2 + exit_code = ensurepip._main([]) + self.assertEqual(exit_code, 2) + class TestUninstallationMainFunction(EnsurepipMixin, unittest.TestCase): @@ -280,7 +288,7 @@ class TestUninstallationMainFunction(EnsurepipMixin, unittest.TestCase): def test_basic_uninstall(self): with fake_pip(): - ensurepip._uninstall._main([]) + exit_code = ensurepip._uninstall._main([]) self.run_pip.assert_called_once_with( [ @@ -289,6 +297,13 @@ class TestUninstallationMainFunction(EnsurepipMixin, unittest.TestCase): ] ) + self.assertEqual(exit_code, 0) + + def test_uninstall_error_code(self): + with fake_pip(): + self.run_pip.return_value = 2 + exit_code = ensurepip._uninstall._main([]) + self.assertEqual(exit_code, 2) if __name__ == "__main__": diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 53851cb6..f3bf223a 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -10,8 +10,8 @@ import errno from test.support import (TESTFN, captured_stderr, check_impl_detail, check_warnings, cpython_only, gc_collect, run_unittest, - no_tracing, unlink, import_module) - + no_tracing, unlink, import_module, script_helper, + SuppressCrashReport) class NaiveException(Exception): def __init__(self, x): self.x = x @@ -936,6 +936,105 @@ class ExceptionTests(unittest.TestCase): self.assertTrue(isinstance(v, RecursionError), type(v)) self.assertIn("maximum recursion depth exceeded", str(v)) + @cpython_only + def test_recursion_normalizing_exception(self): + # Issue #22898. + # Test that a RecursionError is raised when tstate->recursion_depth is + # equal to recursion_limit in PyErr_NormalizeException() and check + # that a ResourceWarning is printed. + # Prior to #22898, the recursivity of PyErr_NormalizeException() was + # controled by tstate->recursion_depth and a PyExc_RecursionErrorInst + # singleton was being used in that case, that held traceback data and + # locals indefinitely and would cause a segfault in _PyExc_Fini() upon + # finalization of these locals. + code = """if 1: + import sys + from _testcapi import get_recursion_depth + + class MyException(Exception): pass + + def setrecursionlimit(depth): + while 1: + try: + sys.setrecursionlimit(depth) + return depth + except RecursionError: + # sys.setrecursionlimit() raises a RecursionError if + # the new recursion limit is too low (issue #25274). + depth += 1 + + def recurse(cnt): + cnt -= 1 + if cnt: + recurse(cnt) + else: + generator.throw(MyException) + + def gen(): + f = open(%a, mode='rb', buffering=0) + yield + + generator = gen() + next(generator) + recursionlimit = sys.getrecursionlimit() + depth = get_recursion_depth() + try: + # Upon the last recursive invocation of recurse(), + # tstate->recursion_depth is equal to (recursion_limit - 1) + # and is equal to recursion_limit when _gen_throw() calls + # PyErr_NormalizeException(). + recurse(setrecursionlimit(depth + 2) - depth - 1) + finally: + sys.setrecursionlimit(recursionlimit) + print('Done.') + """ % __file__ + rc, out, err = script_helper.assert_python_failure("-Wd", "-c", code) + # Check that the program does not fail with SIGABRT. + self.assertEqual(rc, 1) + self.assertIn(b'RecursionError', err) + self.assertIn(b'ResourceWarning', err) + self.assertIn(b'Done.', out) + + @cpython_only + def test_recursion_normalizing_infinite_exception(self): + # Issue #30697. Test that a RecursionError is raised when + # PyErr_NormalizeException() maximum recursion depth has been + # exceeded. + code = """if 1: + import _testcapi + try: + raise _testcapi.RecursingInfinitelyError + finally: + print('Done.') + """ + rc, out, err = script_helper.assert_python_failure("-c", code) + self.assertEqual(rc, 1) + self.assertIn(b'RecursionError: maximum recursion depth exceeded ' + b'while normalizing an exception', err) + self.assertIn(b'Done.', out) + + @cpython_only + def test_recursion_normalizing_with_no_memory(self): + # Issue #30697. Test that in the abort that occurs when there is no + # memory left and the size of the Python frames stack is greater than + # the size of the list of preallocated MemoryError instances, the + # Fatal Python error message mentions MemoryError. + code = """if 1: + import _testcapi + class C(): pass + def recurse(cnt): + cnt -= 1 + if cnt: + recurse(cnt) + else: + _testcapi.set_nomemory(0) + C() + recurse(16) + """ + with SuppressCrashReport(): + rc, out, err = script_helper.assert_python_failure("-c", code) + self.assertIn(b'Fatal Python error: Cannot recover from ' + b'MemoryErrors while normalizing exceptions.', err) @cpython_only def test_MemoryError(self): @@ -1104,6 +1203,23 @@ class ExceptionTests(unittest.TestCase): self.assertIn("test message", report) self.assertTrue(report.endswith("\n")) + @cpython_only + def test_memory_error_in_PyErr_PrintEx(self): + code = """if 1: + import _testcapi + class C(): pass + _testcapi.set_nomemory(0, %d) + C() + """ + + # Issue #30817: Abort in PyErr_PrintEx() when no memory. + # Span a large range of tests as the CPython code always evolves with + # changes that add or remove memory allocations. + for i in range(1, 20): + rc, out, err = script_helper.assert_python_failure("-c", code % i) + self.assertIn(rc, (1, 120)) + self.assertIn(b'MemoryError', err) + class ImportErrorTests(unittest.TestCase): diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 20fe1e26..b8d8ea3c 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -755,6 +755,22 @@ class FaultHandlerTests(unittest.TestCase): 3, name) + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') + def test_ignore_exception(self): + for exc_code in ( + 0xE06D7363, # MSC exception ("Emsc") + 0xE0434352, # COM Callable Runtime exception ("ECCR") + ): + code = f""" + import faulthandler + faulthandler.enable() + faulthandler._raise_exception({exc_code}) + """ + code = dedent(code) + output, exitcode = self.get_output(code) + self.assertEqual(output, []) + self.assertEqual(exitcode, exc_code) + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') def test_raise_nonfatal_exception(self): # These exceptions are not strictly errors. Letting diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index f698e13f..01e11da0 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -8,7 +8,6 @@ import sys import unittest import warnings from test import support -android_not_root = support.android_not_root def create_file(filename, data=b'foo'): @@ -213,9 +212,11 @@ class GenericTest: def test_samefile_on_symlink(self): self._test_samefile_on_link_func(os.symlink) - @unittest.skipIf(android_not_root, "hard links not allowed, non root user") def test_samefile_on_link(self): - self._test_samefile_on_link_func(os.link) + try: + self._test_samefile_on_link_func(os.link) + except PermissionError as e: + self.skipTest('os.link(): %s' % e) def test_samestat(self): test_fn1 = support.TESTFN @@ -253,9 +254,11 @@ class GenericTest: def test_samestat_on_symlink(self): self._test_samestat_on_link_func(os.symlink) - @unittest.skipIf(android_not_root, "hard links not allowed, non root user") def test_samestat_on_link(self): - self._test_samestat_on_link_func(os.link) + try: + self._test_samestat_on_link_func(os.link) + except PermissionError as e: + self.skipTest('os.link(): %s' % e) def test_sameopenfile(self): filename = support.TESTFN diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index b457bd3f..295d4d4a 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -431,6 +431,30 @@ class TestGzip(BaseTest): with gzip.GzipFile(fileobj=f, mode="w") as g: pass + def test_fileobj_mode(self): + gzip.GzipFile(self.filename, "wb").close() + with open(self.filename, "r+b") as f: + with gzip.GzipFile(fileobj=f, mode='r') as g: + self.assertEqual(g.mode, gzip.READ) + with gzip.GzipFile(fileobj=f, mode='w') as g: + self.assertEqual(g.mode, gzip.WRITE) + with gzip.GzipFile(fileobj=f, mode='a') as g: + self.assertEqual(g.mode, gzip.WRITE) + with gzip.GzipFile(fileobj=f, mode='x') as g: + self.assertEqual(g.mode, gzip.WRITE) + with self.assertRaises(ValueError): + gzip.GzipFile(fileobj=f, mode='z') + for mode in "rb", "r+b": + with open(self.filename, mode) as f: + with gzip.GzipFile(fileobj=f) as g: + self.assertEqual(g.mode, gzip.READ) + for mode in "wb", "ab", "xb": + if "x" in mode: + support.unlink(self.filename) + with open(self.filename, mode) as f: + with gzip.GzipFile(fileobj=f) as g: + self.assertEqual(g.mode, gzip.WRITE) + def test_bytes_filename(self): str_filename = self.filename try: diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 647719ea..98174889 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -618,6 +618,24 @@ class HashLibTestCase(unittest.TestCase): "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1"+ "7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923") + @requires_blake2 + def test_case_blake2b_all_parameters(self): + # This checks that all the parameters work in general, and also that + # parameter byte order doesn't get confused on big endian platforms. + self.check('blake2b', b"foo", + "920568b0c5873b2f0ab67bedb6cf1b2b", + digest_size=16, + key=b"bar", + salt=b"baz", + person=b"bing", + fanout=2, + depth=3, + leaf_size=4, + node_offset=5, + node_depth=6, + inner_size=7, + last_node=True) + @requires_blake2 def test_blake2b_vectors(self): for msg, key, md in read_vectors('blake2b'): @@ -643,6 +661,24 @@ class HashLibTestCase(unittest.TestCase): self.check('blake2s', b"abc", "508c5e8c327c14e2e1a72ba34eeb452f37458b209ed63a294d999b4c86675982") + @requires_blake2 + def test_case_blake2s_all_parameters(self): + # This checks that all the parameters work in general, and also that + # parameter byte order doesn't get confused on big endian platforms. + self.check('blake2s', b"foo", + "bf2a8f7fe3c555012a6f8046e646bc75", + digest_size=16, + key=b"bar", + salt=b"baz", + person=b"bing", + fanout=2, + depth=3, + leaf_size=4, + node_offset=5, + node_depth=6, + inner_size=7, + last_node=True) + @requires_blake2 def test_blake2s_vectors(self): for msg, key, md in read_vectors('blake2s'): diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 22b3bf52..66e937e0 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -370,7 +370,8 @@ class SimpleHTTPServerTestCase(BaseTestCase): reader.close() return body - @support.requires_mac_ver(10, 5) + @unittest.skipIf(sys.platform == 'darwin', + 'undecodable name cannot always be decoded on macOS') @unittest.skipIf(sys.platform == 'win32', 'undecodable name cannot be decoded on win32') @unittest.skipUnless(support.TESTFN_UNDECODABLE, diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 6f35f494..1c7605c7 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -315,8 +315,24 @@ class ImportTests(unittest.TestCase): loader.get_data(imp.__file__) # Will need to create a newly opened file def test_load_source(self): - with self.assertRaisesRegex(ValueError, 'embedded null'): - imp.load_source(__name__, __file__ + "\0") + # Create a temporary module since load_source(name) modifies + # sys.modules[name] attributes like __loader___ + modname = f"tmp{__name__}" + mod = type(sys.modules[__name__])(modname) + with support.swap_item(sys.modules, modname, mod): + with self.assertRaisesRegex(ValueError, 'embedded null'): + imp.load_source(modname, __file__ + "\0") + + @support.cpython_only + def test_issue31315(self): + # There shouldn't be an assertion failure in imp.create_dynamic(), + # when spec.name is not a string. + create_dynamic = support.get_attribute(imp, 'create_dynamic') + class BadSpec: + name = None + origin = 'foo' + with self.assertRaises(TypeError): + create_dynamic(BadSpec()) class ReloadTests(unittest.TestCase): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index e8111214..b73a96f7 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -353,6 +353,18 @@ class ImportTests(unittest.TestCase): with self.assertRaises(ImportError): from test_from_import_AttributeError import does_not_exist + @cpython_only + def test_issue31492(self): + # There shouldn't be an assertion failure in case of failing to import + # from a module with a bad __name__ attribute, or in case of failing + # to access an attribute of such a module. + with swap_attr(os, '__name__', None): + with self.assertRaises(ImportError): + from os import does_not_exist + + with self.assertRaises(AttributeError): + os.does_not_exist + def test_concurrency(self): sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'data')) try: diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py index a7bf2749..0cd9de4d 100644 --- a/Lib/test/test_importlib/import_/test_api.py +++ b/Lib/test/test_importlib/import_/test_api.py @@ -82,6 +82,20 @@ class APITest: self.__import__(PKG_NAME, fromlist=[SUBMOD_NAME.rpartition('.')[-1]]) + def test_blocked_fromlist(self): + # If fromlist entry is None, let a ModuleNotFoundError propagate. + # issue31642 + mod = types.ModuleType(PKG_NAME) + mod.__path__ = [] + with util.import_state(meta_path=[self.bad_finder_loader]): + with util.uncache(PKG_NAME, SUBMOD_NAME): + sys.modules[PKG_NAME] = mod + sys.modules[SUBMOD_NAME] = None + with self.assertRaises(ModuleNotFoundError) as cm: + self.__import__(PKG_NAME, + fromlist=[SUBMOD_NAME.rpartition('.')[-1]]) + self.assertEqual(cm.exception.name, SUBMOD_NAME) + class OldAPITests(APITest): bad_finder_loader = BadLoaderFinder diff --git a/Lib/test/test_importlib/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py index 14640032..018c1721 100644 --- a/Lib/test/test_importlib/import_/test_fromlist.py +++ b/Lib/test/test_importlib/import_/test_fromlist.py @@ -1,5 +1,6 @@ """Test that the semantics relating to the 'fromlist' argument are correct.""" from .. import util +import warnings import unittest @@ -73,6 +74,13 @@ class HandlingFromlist: self.assertTrue(hasattr(module, 'module')) self.assertEqual(module.module.__name__, 'pkg.module') + def test_nonexistent_from_package(self): + with util.mock_modules('pkg.__init__') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('pkg', fromlist=['non_existent']) + self.assertEqual(module.__name__, 'pkg') + self.assertFalse(hasattr(module, 'non_existent')) + def test_module_from_package_triggers_ModuleNotFoundError(self): # If a submodule causes an ModuleNotFoundError because it tries # to import a module which doesn't exist, that should let the @@ -122,6 +130,41 @@ class HandlingFromlist: self.assertEqual(module.module1.__name__, 'pkg.module1') self.assertEqual(module.module2.__name__, 'pkg.module2') + def test_nonexistent_in_all(self): + with util.mock_modules('pkg.__init__') as importer: + with util.import_state(meta_path=[importer]): + importer['pkg'].__all__ = ['non_existent'] + module = self.__import__('pkg', fromlist=['*']) + self.assertEqual(module.__name__, 'pkg') + self.assertFalse(hasattr(module, 'non_existent')) + + def test_star_in_all(self): + with util.mock_modules('pkg.__init__') as importer: + with util.import_state(meta_path=[importer]): + importer['pkg'].__all__ = ['*'] + module = self.__import__('pkg', fromlist=['*']) + self.assertEqual(module.__name__, 'pkg') + self.assertFalse(hasattr(module, '*')) + + def test_invalid_type(self): + with util.mock_modules('pkg.__init__') as importer: + with util.import_state(meta_path=[importer]), \ + warnings.catch_warnings(): + warnings.simplefilter('error', BytesWarning) + with self.assertRaisesRegex(TypeError, r'\bfrom\b'): + self.__import__('pkg', fromlist=[b'attr']) + with self.assertRaisesRegex(TypeError, r'\bfrom\b'): + self.__import__('pkg', fromlist=iter([b'attr'])) + + def test_invalid_type_in_all(self): + with util.mock_modules('pkg.__init__') as importer: + with util.import_state(meta_path=[importer]), \ + warnings.catch_warnings(): + warnings.simplefilter('error', BytesWarning) + importer['pkg'].__all__ = [b'attr'] + with self.assertRaisesRegex(TypeError, r'\bpkg\.__all__\b'): + self.__import__('pkg', fromlist=['*']) + (Frozen_FromList, Source_FromList diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index facf040d..bdad3864 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2021,7 +2021,7 @@ class TestSignatureObject(unittest.TestCase): ((('args', ..., ..., 'var_positional'),), ...)) self.assertEqual(self.signature(A.f3), ((('args', ..., ..., 'var_positional'),), ...)) - self.assertEqual(self.signature(A.f4), + self.assertEqual(self.signature(A.f4), ((('args', ..., ..., 'var_positional'), ('kwargs', ..., ..., 'var_keyword')), ...)) @cpython_only @@ -3557,6 +3557,19 @@ class TestSignatureDefinitions(unittest.TestCase): self.assertIsNone(obj.__text_signature__) +class NTimesUnwrappable: + def __init__(self, n): + self.n = n + self._next = None + + @property + def __wrapped__(self): + if self.n <= 0: + raise Exception("Unwrapped too many times") + if self._next is None: + self._next = NTimesUnwrappable(self.n - 1) + return self._next + class TestUnwrap(unittest.TestCase): def test_unwrap_one(self): @@ -3612,6 +3625,11 @@ class TestUnwrap(unittest.TestCase): __wrapped__ = func self.assertIsNone(inspect.unwrap(C())) + def test_recursion_limit(self): + obj = NTimesUnwrappable(sys.getrecursionlimit() + 1) + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(obj) + class TestMain(unittest.TestCase): def test_only_source(self): module = importlib.import_module('unittest') diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 14bbd619..a36076e0 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -506,5 +506,13 @@ class IntTestCases(unittest.TestCase): check('123\ud800') check('123\ud800', 10) + def test_issue31619(self): + self.assertEqual(int('1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1_0_1', 2), + 0b1010101010101010101010101010101) + self.assertEqual(int('1_2_3_4_5_6_7_0_1_2_3', 8), 0o12345670123) + self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789) + self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index c431f0dc..a978134e 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1984,6 +1984,30 @@ class RegressionTests(unittest.TestCase): with self.assertRaises(StopIteration): next(it) + def test_issue30347_1(self): + def f(n): + if n == 5: + list(b) + return n != 6 + for (k, b) in groupby(range(10), f): + list(b) # shouldn't crash + + def test_issue30347_2(self): + class K: + def __init__(self, v): + pass + def __eq__(self, other): + nonlocal i + i += 1 + if i == 1: + next(g, None) + return True + i = 0 + g = next(groupby(range(10), K))[1] + for j in range(2): + next(g, None) # shouldn't crash + + class SubclassWithKwargsTest(unittest.TestCase): def test_keywords_in_subclass(self): # count is not subclassable... diff --git a/Lib/test/test_json/test_speedups.py b/Lib/test/test_json/test_speedups.py index 56f18820..5dad6920 100644 --- a/Lib/test/test_json/test_speedups.py +++ b/Lib/test/test_json/test_speedups.py @@ -36,6 +36,27 @@ class TestEncode(CTest): b"\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75", None) + def test_bad_str_encoder(self): + # Issue #31505: There shouldn't be an assertion failure in case + # c_make_encoder() receives a bad encoder() argument. + def bad_encoder1(*args): + return None + enc = self.json.encoder.c_make_encoder(None, lambda obj: str(obj), + bad_encoder1, None, ': ', ', ', + False, False, False) + with self.assertRaises(TypeError): + enc('spam', 4) + with self.assertRaises(TypeError): + enc({'spam': 42}, 4) + + def bad_encoder2(*args): + 1/0 + enc = self.json.encoder.c_make_encoder(None, lambda obj: str(obj), + bad_encoder2, None, ': ', ', ', + False, False, False) + with self.assertRaises(ZeroDivisionError): + enc('spam', 4) + def test_bad_bool_args(self): def test(name): self.json.encoder.JSONEncoder(**{name: BadBool()}).encode({'a': 1}) diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py index 9f498868..1099c759 100644 --- a/Lib/test/test_kqueue.py +++ b/Lib/test/test_kqueue.py @@ -208,6 +208,30 @@ class TestKQueue(unittest.TestCase): b.close() kq.close() + def test_issue30058(self): + # changelist must be an iterable + kq = select.kqueue() + a, b = socket.socketpair() + ev = select.kevent(a, select.KQ_FILTER_READ, select.KQ_EV_ADD | select.KQ_EV_ENABLE) + + kq.control([ev], 0) + # not a list + kq.control((ev,), 0) + # __len__ is not consistent with __iter__ + class BadList: + def __len__(self): + return 0 + def __iter__(self): + for i in range(100): + yield ev + kq.control(BadList(), 0) + # doesn't have __len__ + kq.control(iter([ev]), 0) + + a.close() + b.close() + kq.close() + def test_close(self): open_file = open(__file__, "rb") self.addCleanup(open_file.close) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 25046c3c..b325f769 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3881,7 +3881,6 @@ class BasicConfigTest(unittest.TestCase): class LoggerAdapterTest(unittest.TestCase): - def setUp(self): super(LoggerAdapterTest, self).setUp() old_handler_list = logging._handlerList[:] @@ -3956,15 +3955,37 @@ class LoggerAdapterTest(unittest.TestCase): self.assertFalse(self.adapter.hasHandlers()) def test_nested(self): + class Adapter(logging.LoggerAdapter): + prefix = 'Adapter' + + def process(self, msg, kwargs): + return f"{self.prefix} {msg}", kwargs + msg = 'Adapters can be nested, yo.' - adapter_adapter = logging.LoggerAdapter(logger=self.adapter, extra=None) + adapter = Adapter(logger=self.logger, extra=None) + adapter_adapter = Adapter(logger=adapter, extra=None) + adapter_adapter.prefix = 'AdapterAdapter' + self.assertEqual(repr(adapter), repr(adapter_adapter)) adapter_adapter.log(logging.CRITICAL, msg, self.recording) - self.assertEqual(len(self.recording.records), 1) record = self.recording.records[0] self.assertEqual(record.levelno, logging.CRITICAL) - self.assertEqual(record.msg, msg) + self.assertEqual(record.msg, f"Adapter AdapterAdapter {msg}") self.assertEqual(record.args, (self.recording,)) + orig_manager = adapter_adapter.manager + self.assertIs(adapter.manager, orig_manager) + self.assertIs(self.logger.manager, orig_manager) + temp_manager = object() + try: + adapter_adapter.manager = temp_manager + self.assertIs(adapter_adapter.manager, temp_manager) + self.assertIs(adapter.manager, temp_manager) + self.assertIs(self.logger.manager, temp_manager) + finally: + adapter_adapter.manager = orig_manager + self.assertIs(adapter_adapter.manager, orig_manager) + self.assertIs(adapter.manager, orig_manager) + self.assertIs(self.logger.manager, orig_manager) class LoggerTest(BaseTest): diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 2ba94433..3807b95b 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -746,7 +746,7 @@ class TestMaildir(TestMailbox, unittest.TestCase): hostname = hostname.replace(':', r'\072') pid = os.getpid() pattern = re.compile(r"(?P