From e49176c60cd79cd948045ab640ecb057aec32783 Mon Sep 17 00:00:00 2001 From: DongHun Kwak Date: Tue, 8 Dec 2020 13:40:39 +0900 Subject: [PATCH] Imported Upstream version 3.7.4 --- .azure-pipelines/ci.yml | 4 +- .azure-pipelines/posix-steps.yml | 4 + .azure-pipelines/pr.yml | 4 +- Doc/README.rst | 9 + Doc/c-api/datetime.rst | 16 + Doc/c-api/exceptions.rst | 14 +- Doc/c-api/init.rst | 18 + Doc/c-api/intro.rst | 6 +- Doc/c-api/long.rst | 6 +- Doc/c-api/memory.rst | 11 +- Doc/c-api/object.rst | 19 +- Doc/c-api/structures.rst | 63 +- Doc/c-api/typeobj.rst | 4 +- Doc/c-api/unicode.rst | 24 +- Doc/c-api/veryhigh.rst | 4 + Doc/conf.py | 2 + Doc/data/refcounts.dat | 17 + Doc/distributing/index.rst | 12 +- Doc/distutils/index.rst | 1 - Doc/distutils/packageindex.rst | 249 +------ Doc/distutils/setupscript.rst | 7 +- Doc/distutils/uploading.rst | 3 +- Doc/extending/embedding.rst | 1 + Doc/extending/extending.rst | 11 +- Doc/extending/newtypes_tutorial.rst | 1 + Doc/faq/extending.rst | 2 + Doc/faq/general.rst | 12 +- Doc/faq/gui.rst | 2 +- Doc/faq/programming.rst | 3 + Doc/faq/windows.rst | 2 + Doc/glossary.rst | 30 +- Doc/howto/descriptor.rst | 4 +- Doc/howto/unicode.rst | 15 +- Doc/includes/custom.c | 1 + Doc/includes/custom2.c | 1 + Doc/includes/custom3.c | 1 + Doc/includes/custom4.c | 1 + Doc/includes/run-func.c | 1 + Doc/includes/sqlite3/adapter_datetime.py | 2 + Doc/includes/sqlite3/adapter_point_1.py | 2 + Doc/includes/sqlite3/adapter_point_2.py | 2 + Doc/includes/sqlite3/connect_db_1.py | 3 - Doc/includes/sqlite3/connect_db_2.py | 3 - Doc/includes/sqlite3/countcursors.py | 2 + Doc/includes/sqlite3/ctx_manager.py | 4 + Doc/includes/sqlite3/execsql_fetchonerow.py | 2 + Doc/includes/sqlite3/execsql_printall_1.py | 2 + Doc/includes/sqlite3/execute_1.py | 2 + Doc/includes/sqlite3/execute_3.py | 12 - Doc/includes/sqlite3/executemany_1.py | 2 + Doc/includes/sqlite3/executemany_2.py | 2 + Doc/includes/sqlite3/executescript.py | 1 + Doc/includes/sqlite3/insert_more_people.py | 2 + Doc/includes/sqlite3/load_extension.py | 2 + Doc/includes/sqlite3/md5func.py | 2 + Doc/includes/sqlite3/mysumaggr.py | 2 + Doc/includes/sqlite3/parse_colnames.py | 2 + Doc/includes/sqlite3/pysqlite_datetime.py | 2 + Doc/includes/sqlite3/row_factory.py | 2 + Doc/includes/sqlite3/rowclass.py | 2 + Doc/includes/sqlite3/shortcut_methods.py | 4 + Doc/includes/sqlite3/simple_tableprinter.py | 2 + Doc/includes/sqlite3/text_factory.py | 2 + Doc/includes/sublist.c | 1 + Doc/library/_thread.rst | 8 +- Doc/library/asyncio-eventloop.rst | 10 +- Doc/library/asyncio-protocol.rst | 2 +- Doc/library/asyncio-subprocess.rst | 2 +- Doc/library/asyncio-sync.rst | 9 +- Doc/library/asyncio-task.rst | 2 +- Doc/library/bz2.rst | 84 ++- Doc/library/cmath.rst | 13 +- Doc/library/codecs.rst | 2 +- Doc/library/collections.rst | 23 +- Doc/library/concurrent.futures.rst | 18 +- Doc/library/configparser.rst | 6 + Doc/library/ctypes.rst | 4 +- Doc/library/datetime.rst | 158 +++-- Doc/library/dis.rst | 18 +- Doc/library/doctest.rst | 18 +- Doc/library/email.compat32-message.rst | 1 + Doc/library/email.encoders.rst | 5 + Doc/library/email.generator.rst | 2 +- Doc/library/exceptions.rst | 9 +- Doc/library/fileinput.rst | 5 +- Doc/library/functions.rst | 25 +- Doc/library/gettext.rst | 46 +- Doc/library/http.client.rst | 5 + Doc/library/http.cookies.rst | 16 +- Doc/library/idle.rst | 66 +- Doc/library/io.rst | 14 +- Doc/library/math.rst | 6 +- Doc/library/operator.rst | 6 +- Doc/library/os.path.rst | 8 +- Doc/library/pathlib.rst | 2 +- Doc/library/pty.rst | 29 +- Doc/library/pyclbr.rst | 2 +- Doc/library/queue.rst | 5 + Doc/library/random.rst | 5 + Doc/library/re.rst | 1 + Doc/library/readline.rst | 2 +- Doc/library/sched.rst | 3 +- Doc/library/signal.rst | 3 +- Doc/library/socket.rst | 2 +- Doc/library/sqlite3.rst | 6 +- Doc/library/stdtypes.rst | 4 +- Doc/library/subprocess.rst | 27 +- Doc/library/sys.rst | 10 +- Doc/library/threading.rst | 2 +- Doc/library/time.rst | 2 +- Doc/library/timeit.rst | 6 +- Doc/library/tkinter.rst | 6 +- Doc/library/turtle.rst | 4 +- Doc/library/typing.rst | 42 +- Doc/library/unittest.mock.rst | 2 +- Doc/library/unittest.rst | 28 +- Doc/library/urllib.parse.rst | 41 +- Doc/library/urllib.request.rst | 59 +- Doc/library/venv.rst | 7 +- Doc/library/warnings.rst | 10 +- Doc/library/weakref.rst | 24 +- Doc/library/xml.dom.minidom.rst | 25 +- Doc/license.rst | 2 +- Doc/reference/datamodel.rst | 32 +- Doc/reference/executionmodel.rst | 2 +- Doc/reference/expressions.rst | 33 +- Doc/reference/import.rst | 40 ++ Doc/reference/lexical_analysis.rst | 2 +- Doc/reference/simple_stmts.rst | 3 +- Doc/tools/extensions/suspicious.py | 10 +- Doc/tools/static/switchers.js | 4 +- Doc/tools/templates/download.html | 3 +- Doc/tools/templates/indexsidebar.html | 3 +- Doc/tools/templates/layout.html | 11 + Doc/tutorial/controlflow.rst | 18 +- Doc/tutorial/interpreter.rst | 10 +- Doc/tutorial/introduction.rst | 2 +- Doc/tutorial/modules.rst | 8 +- Doc/tutorial/whatnow.rst | 8 +- Doc/using/mac.rst | 6 +- Doc/using/windows.rst | 43 +- Doc/whatsnew/2.6.rst | 2 +- Doc/whatsnew/3.2.rst | 16 +- Doc/whatsnew/3.7.rst | 5 +- Include/abstract.h | 2 +- Include/internal/mem.h | 24 + Include/internal/pystate.h | 3 + Include/objimpl.h | 6 +- Include/patchlevel.h | 4 +- Include/pydtrace.h | 2 +- Include/pylifecycle.h | 4 +- Include/pymem.h | 2 - Lib/_dummy_thread.py | 32 +- Lib/_pyio.py | 26 +- Lib/asyncio/base_events.py | 20 +- Lib/asyncio/proactor_events.py | 2 +- Lib/asyncio/selector_events.py | 19 +- Lib/asyncio/sslproto.py | 6 +- Lib/asyncio/tasks.py | 51 +- Lib/asyncio/unix_events.py | 20 +- Lib/bdb.py | 14 +- Lib/cProfile.py | 14 +- Lib/codecs.py | 8 +- Lib/collections/__init__.py | 22 +- Lib/concurrent/futures/_base.py | 16 +- Lib/concurrent/futures/process.py | 30 +- Lib/concurrent/futures/thread.py | 16 +- Lib/contextlib.py | 34 +- Lib/ctypes/__init__.py | 10 +- Lib/ctypes/test/test_arrays.py | 32 + Lib/ctypes/test/test_buffers.py | 9 + Lib/curses/__init__.py | 10 +- Lib/distutils/command/build.py | 2 +- Lib/distutils/command/check.py | 3 +- Lib/distutils/spawn.py | 39 +- Lib/distutils/sysconfig.py | 13 +- Lib/distutils/tests/includetest.rst | 1 + Lib/distutils/tests/test_check.py | 16 +- Lib/distutils/tests/test_spawn.py | 47 +- Lib/distutils/tests/test_sysconfig.py | 94 ++- Lib/email/_header_value_parser.py | 50 +- Lib/email/feedparser.py | 2 +- Lib/email/header.py | 2 +- Lib/email/policy.py | 3 +- Lib/fileinput.py | 6 +- Lib/functools.py | 15 +- Lib/http/client.py | 26 +- Lib/idlelib/NEWS.txt | 53 +- Lib/idlelib/autocomplete.py | 29 +- Lib/idlelib/browser.py | 7 +- Lib/idlelib/calltip.py | 15 +- Lib/idlelib/colorizer.py | 2 +- Lib/idlelib/config-keys.def | 5 + Lib/idlelib/config.py | 46 +- Lib/idlelib/configdialog.py | 6 +- Lib/idlelib/delegator.py | 2 +- Lib/idlelib/editor.py | 10 +- Lib/idlelib/grep.py | 94 ++- Lib/idlelib/help.html | 80 ++- Lib/idlelib/help.py | 2 +- Lib/idlelib/hyperparser.py | 2 +- Lib/idlelib/idle_test/htest.py | 11 +- Lib/idlelib/idle_test/mock_tk.py | 2 +- Lib/idlelib/idle_test/test_autocomplete.py | 149 +++- Lib/idlelib/idle_test/test_autoexpand.py | 4 +- Lib/idlelib/idle_test/test_browser.py | 28 +- Lib/idlelib/idle_test/test_calltip.py | 120 ++-- Lib/idlelib/idle_test/test_config.py | 6 +- Lib/idlelib/idle_test/test_configdialog.py | 45 +- Lib/idlelib/idle_test/test_grep.py | 92 ++- Lib/idlelib/idle_test/test_hyperparser.py | 6 +- Lib/idlelib/idle_test/test_parenmatch.py | 2 +- Lib/idlelib/idle_test/test_pyparse.py | 2 +- Lib/idlelib/idle_test/test_query.py | 105 ++- Lib/idlelib/idle_test/test_searchbase.py | 16 +- Lib/idlelib/iomenu.py | 2 + Lib/idlelib/mainmenu.py | 1 + Lib/idlelib/pyshell.py | 18 +- Lib/idlelib/query.py | 57 +- Lib/idlelib/replace.py | 111 ++- Lib/idlelib/run.py | 53 +- Lib/idlelib/runscript.py | 46 +- Lib/idlelib/search.py | 70 +- Lib/idlelib/searchbase.py | 6 +- Lib/idlelib/squeezer.py | 2 +- Lib/idlelib/undo.py | 2 +- Lib/idlelib/zoomheight.py | 133 +++- Lib/inspect.py | 6 +- Lib/ipaddress.py | 4 + Lib/lib2to3/pgen2/tokenize.py | 2 +- Lib/logging/__init__.py | 67 +- Lib/logging/handlers.py | 5 + Lib/msilib/__init__.py | 2 +- Lib/multiprocessing/managers.py | 27 +- Lib/multiprocessing/resource_sharer.py | 2 +- Lib/pathlib.py | 15 +- Lib/pdb.py | 6 +- Lib/pickle.py | 3 + Lib/platform.py | 26 +- Lib/posixpath.py | 2 +- Lib/profile.py | 14 +- Lib/pstats.py | 2 +- Lib/pydoc_data/topics.py | 125 ++-- Lib/shutil.py | 21 +- Lib/site.py | 7 - Lib/ssl.py | 66 +- Lib/subprocess.py | 4 +- Lib/test/_test_multiprocessing.py | 2 +- Lib/test/bisect_cmd.py | 6 +- Lib/test/datetimetester.py | 37 +- Lib/test/libregrtest/cmdline.py | 17 +- Lib/test/libregrtest/main.py | 281 ++++---- Lib/test/libregrtest/refleak.py | 78 +- Lib/test/libregrtest/runtest.py | 289 +++++--- Lib/test/libregrtest/runtest_mp.py | 442 +++++++----- Lib/test/libregrtest/save_env.py | 4 +- Lib/test/libregrtest/setup.py | 22 - Lib/test/libregrtest/utils.py | 7 +- Lib/test/libregrtest/win_utils.py | 105 +++ Lib/test/pickletester.py | 21 +- Lib/test/pythoninfo.py | 53 +- Lib/test/selfsigned_pythontestdotnet.pem | 46 +- Lib/test/seq_tests.py | 1 + Lib/test/string_tests.py | 2 +- Lib/test/support/__init__.py | 18 +- Lib/test/test_array.py | 4 +- Lib/test/test_asyncio/test_base_events.py | 39 + Lib/test/test_asyncio/test_events.py | 3 +- Lib/test/test_asyncio/test_selector_events.py | 28 +- Lib/test/test_asyncio/test_sslproto.py | 111 ++- Lib/test/test_asyncio/test_subprocess.py | 17 +- Lib/test/test_asyncio/test_tasks.py | 11 +- Lib/test/test_asyncio/test_unix_events.py | 6 +- Lib/test/test_bytes.py | 5 +- Lib/test/test_call.py | 17 + Lib/test/test_capi.py | 31 +- Lib/test/test_cmd_line_script.py | 17 + Lib/test/test_codecs.py | 89 +++ Lib/test/test_collections.py | 36 +- Lib/test/test_concurrent_futures.py | 32 + Lib/test/test_contextlib.py | 10 + Lib/test/test_contextlib_async.py | 9 + Lib/test/test_dataclasses.py | 2 +- Lib/test/test_descr.py | 25 +- Lib/test/test_dummy_thread.py | 21 + .../test_email/test__header_value_parser.py | 24 +- Lib/test/test_email/test_email.py | 12 + Lib/test/test_email/test_generator.py | 22 + Lib/test/test_email/test_headerregistry.py | 11 +- Lib/test/test_email/test_message.py | 131 ++++ Lib/test/test_email/test_policy.py | 21 + Lib/test/test_functools.py | 24 + Lib/test/test_genericclass.py | 2 +- Lib/test/test_gzip.py | 2 +- Lib/test/test_httplib.py | 49 +- Lib/test/test_imaplib.py | 18 +- Lib/test/test_importlib/test_lazy.py | 2 +- Lib/test/test_inspect.py | 18 + Lib/test/test_io.py | 34 +- Lib/test/test_ipaddress.py | 16 + Lib/test/test_logging.py | 80 ++- Lib/test/test_mmap.py | 4 +- Lib/test/test_msilib.py | 9 + Lib/test/test_nntplib.py | 37 +- Lib/test/test_os.py | 9 + Lib/test/test_parser.py | 20 + Lib/test/test_pathlib.py | 8 +- Lib/test/test_pdb.py | 31 +- Lib/test/test_peepholer.py | 7 + Lib/test/test_platform.py | 24 +- Lib/test/test_posix.py | 10 + Lib/test/test_random.py | 2 +- Lib/test/test_re.py | 28 +- Lib/test/test_regrtest.py | 92 ++- Lib/test/test_robotparser.py | 28 +- Lib/test/test_set.py | 6 + Lib/test/test_shutil.py | 76 ++ Lib/test/test_site.py | 14 +- Lib/test/test_socket.py | 9 +- Lib/test/test_sort.py | 5 + Lib/test/test_ssl.py | 61 +- Lib/test/test_sys.py | 9 +- Lib/test/test_sysconfig.py | 17 +- Lib/test/test_threading.py | 114 ++- Lib/test/test_time.py | 5 +- Lib/test/test_tools/test_i18n.py | 2 +- Lib/test/test_tools/test_lll.py | 39 + Lib/test/test_trace.py | 11 + Lib/test/test_typing.py | 28 +- Lib/test/test_urllib.py | 71 ++ Lib/test/test_urlparse.py | 17 +- Lib/test/test_userlist.py | 6 + Lib/test/test_venv.py | 42 +- Lib/test/test_weakref.py | 27 + Lib/test/test_wsgiref.py | 43 ++ Lib/test/test_xmlrpc.py | 7 +- Lib/test/test_zipfile.py | 37 + Lib/threading.py | 49 +- Lib/tokenize.py | 2 +- Lib/trace.py | 14 +- Lib/turtle.py | 2 +- Lib/typing.py | 3 + Lib/unittest/case.py | 15 +- Lib/unittest/mock.py | 57 +- Lib/unittest/test/testmock/testhelpers.py | 40 +- Lib/unittest/test/testmock/testmock.py | 77 ++ Lib/unittest/test/testmock/testpatch.py | 20 + Lib/unittest/test/testmock/testwith.py | 22 +- Lib/urllib/parse.py | 44 +- Lib/urllib/request.py | 2 +- Lib/urllib/robotparser.py | 8 +- Lib/venv/__init__.py | 79 ++- Lib/venv/scripts/nt/activate.bat | 2 +- Lib/weakref.py | 22 +- Lib/wsgiref/handlers.py | 15 +- Lib/zipfile.py | 89 +-- Mac/BuildScript/build-installer.py | 24 +- Mac/BuildScript/resources/ReadMe.rtf | 131 ++-- Mac/BuildScript/resources/Welcome.rtf | 8 +- Makefile.pre.in | 2 + Misc/ACKS | 10 +- Misc/NEWS | 665 ++++++++++++++++++ Misc/python-config.in | 2 - Misc/python-config.sh.in | 7 +- Modules/Setup.dist | 2 +- Modules/_ctypes/_ctypes.c | 18 +- Modules/_ctypes/callbacks.c | 10 +- Modules/_ctypes/callproc.c | 6 +- Modules/_ctypes/malloc_closure.c | 2 +- Modules/_elementtree.c | 6 +- Modules/_functoolsmodule.c | 7 +- Modules/_io/_iomodule.c | 2 +- Modules/_io/bytesio.c | 9 +- Modules/_io/clinic/iobase.c.h | 8 +- Modules/_io/iobase.c | 7 +- Modules/_io/winconsoleio.c | 2 +- Modules/_localemodule.c | 2 +- Modules/_lzmamodule.c | 4 +- Modules/_multiprocessing/semaphore.c | 2 +- Modules/_pickle.c | 8 +- Modules/_sqlite/connection.c | 24 +- Modules/_ssl.c | 156 +++- Modules/_testcapimodule.c | 83 +++ Modules/_uuidmodule.c | 9 +- Modules/_xxtestfuzz/README.rst | 10 + .../dictionaries/fuzz_json_loads.dict | 40 ++ .../fuzz_json_loads_corpus/empty_array.json | 1 + .../fuzz_json_loads_corpus/empty_object.json | 1 + .../fuzz_json_loads_corpus/pass1.json | 58 ++ .../fuzz_json_loads_corpus/pass2.json | 1 + .../fuzz_json_loads_corpus/pass3.json | 6 + .../fuzz_json_loads_corpus/simple_array.json | 1 + Modules/_xxtestfuzz/fuzz_tests.txt | 1 + Modules/_xxtestfuzz/fuzzer.c | 59 ++ Modules/arraymodule.c | 6 +- Modules/binascii.c | 2 +- Modules/cmathmodule.c | 4 +- Modules/expat/expat.h | 2 +- Modules/expat/expat_external.h | 14 +- Modules/expat/internal.h | 5 + Modules/expat/winconfig.h | 4 - Modules/expat/xmlparse.c | 29 +- Modules/expat/xmltok.c | 3 - Modules/faulthandler.c | 24 +- Modules/itertoolsmodule.c | 1 + Modules/main.c | 56 +- Modules/mathmodule.c | 4 +- Modules/mmapmodule.c | 6 +- Modules/parsermodule.c | 39 +- Modules/posixmodule.c | 32 +- Modules/readline.c | 6 +- Modules/signalmodule.c | 13 +- Modules/socketmodule.c | 4 +- Modules/timemodule.c | 14 + Objects/bytearrayobject.c | 3 +- Objects/bytesobject.c | 5 +- Objects/call.c | 19 +- Objects/descrobject.c | 10 +- Objects/listobject.c | 45 +- Objects/longobject.c | 4 +- Objects/object.c | 28 +- Objects/obmalloc.c | 33 +- Objects/odictobject.c | 4 +- Objects/rangeobject.c | 2 +- Objects/setobject.c | 8 - Objects/stringlib/codecs.h | 6 +- Objects/structseq.c | 16 +- Objects/tupleobject.c | 3 +- Objects/typeobject.c | 47 +- Objects/unicodeobject.c | 31 +- PC/getpathp.c | 12 +- PC/launcher.c | 6 +- PC/layout/support/appxmanifest.py | 13 +- PC/layout/support/options.py | 10 +- PC/python_uwp.cpp | 248 +++---- PC/winreg.c | 4 +- PCbuild/get_externals.bat | 16 +- PCbuild/openssl.props | 3 +- PCbuild/openssl.vcxproj | 61 +- PCbuild/prepare_ssl.bat | 6 +- PCbuild/python.props | 6 +- PCbuild/readme.txt | 4 +- Parser/tokenizer.c | 9 +- Python/ast.c | 2 +- Python/bltinmodule.c | 8 +- Python/ceval.c | 6 +- Python/compile.c | 8 +- Python/dtoa.c | 79 +-- Python/dynload_win.c | 2 +- Python/errors.c | 137 ++-- Python/fileutils.c | 17 +- Python/getargs.c | 12 +- Python/hamt.c | 2 +- Python/import.c | 19 +- Python/pathconfig.c | 13 +- Python/peephole.c | 68 +- Python/pyarena.c | 2 +- Python/pylifecycle.c | 44 +- Python/pystate.c | 16 +- Python/sysmodule.c | 23 +- README.rst | 2 +- Tools/demo/rpython.py | 2 +- Tools/freeze/freeze.py | 5 +- Tools/msi/lib/lib_files.wxs | 15 + Tools/msi/make_cat.ps1 | 5 +- Tools/msi/msi.props | 5 +- Tools/msi/sdktools.psm1 | 6 +- Tools/nuget/make_pkg.proj | 2 +- Tools/scripts/lll.py | 7 +- Tools/ssl/multissltests.py | 6 +- configure | 16 +- configure.ac | 2 +- setup.py | 63 +- 473 files changed, 8158 insertions(+), 3156 deletions(-) delete mode 100644 Doc/includes/sqlite3/connect_db_1.py delete mode 100644 Doc/includes/sqlite3/connect_db_2.py delete mode 100644 Doc/includes/sqlite3/execute_3.py create mode 100644 Lib/distutils/tests/includetest.rst create mode 100644 Lib/test/libregrtest/win_utils.py create mode 100644 Lib/test/test_tools/test_lll.py create mode 100644 Modules/_xxtestfuzz/dictionaries/fuzz_json_loads.dict create mode 100644 Modules/_xxtestfuzz/fuzz_json_loads_corpus/empty_array.json create mode 100644 Modules/_xxtestfuzz/fuzz_json_loads_corpus/empty_object.json create mode 100644 Modules/_xxtestfuzz/fuzz_json_loads_corpus/pass1.json create mode 100644 Modules/_xxtestfuzz/fuzz_json_loads_corpus/pass2.json create mode 100644 Modules/_xxtestfuzz/fuzz_json_loads_corpus/pass3.json create mode 100644 Modules/_xxtestfuzz/fuzz_json_loads_corpus/simple_array.json diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index 15a83dd0..fcfac85e 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -59,7 +59,7 @@ jobs: variables: testRunTitle: '$(build.sourceBranchName)-linux' testRunPlatform: linux - openssl_version: 1.1.0j + openssl_version: 1.1.1c steps: - template: ./posix-steps.yml @@ -116,7 +116,7 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.0j + openssl_version: 1.1.1c steps: - template: ./posix-steps.yml diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml index a4160e5a..15e3f0b9 100644 --- a/.azure-pipelines/posix-steps.yml +++ b/.azure-pipelines/posix-steps.yml @@ -10,6 +10,10 @@ steps: clean: true fetchDepth: 5 +# Work around a known issue affecting Ubuntu VMs on Pipelines +- script: sudo setfacl -Rb /home/vsts + displayName: 'Workaround ACL issue' + - script: ${{ parameters.sudo_dependencies }} ./.azure-pipelines/posix-deps-${{ parameters.dependencies }}.sh $(openssl_version) displayName: 'Install dependencies' diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 0bd7921b..2486f88a 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -59,7 +59,7 @@ jobs: variables: testRunTitle: '$(system.pullRequest.TargetBranch)-linux' testRunPlatform: linux - openssl_version: 1.1.0j + openssl_version: 1.1.1c steps: - template: ./posix-steps.yml @@ -116,7 +116,7 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.0j + openssl_version: 1.1.1c steps: - template: ./posix-steps.yml diff --git a/Doc/README.rst b/Doc/README.rst index d7bcc5ba..9fc39834 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -113,6 +113,15 @@ Then, from the ``Doc`` directory, run :: where ```` is one of html, text, latex, or htmlhelp (for explanations see the make targets above). +Deprecation header +================== + +You can define the ``outdated`` variable in ``html_context`` to show a +red banner on each page redirecting to the "latest" version. + +The link points to the same page on ``/3/``, sadly for the moment the +language is lost during the process. + Contributing ============ diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 78724619..77b1b216 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -98,12 +98,28 @@ Macros to create objects: minute, second and microsecond. +.. c:function:: PyObject* PyDateTime_FromDateAndTimeAndFold(int year, int month, int day, int hour, int minute, int second, int usecond, int fold) + + Return a :class:`datetime.datetime` object with the specified year, month, day, hour, + minute, second, microsecond and fold. + + .. versionadded:: 3.6 + + .. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond) Return a :class:`datetime.time` object with the specified hour, minute, second and microsecond. +.. c:function:: PyObject* PyTime_FromTimeAndFold(int hour, int minute, int second, int usecond, int fold) + + Return a :class:`datetime.time` object with the specified hour, minute, second, + microsecond and fold. + + .. versionadded:: 3.6 + + .. c:function:: PyObject* PyDelta_FromDSU(int days, int seconds, int useconds) Return a :class:`datetime.timedelta` object representing the given number diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index cd06096e..13e00b54 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -81,6 +81,8 @@ Printing and clearing in which the unraisable exception occurred. If possible, the repr of *obj* will be printed in the warning message. + An exception must be set when calling this function. + Raising exceptions ================== @@ -514,13 +516,13 @@ Signal Handling single: SIGINT single: KeyboardInterrupt (built-in exception) - This function simulates the effect of a :const:`SIGINT` signal arriving --- the - next time :c:func:`PyErr_CheckSignals` is called, :exc:`KeyboardInterrupt` will - be raised. It may be called without holding the interpreter lock. - - .. % XXX This was described as obsolete, but is used in - .. % _thread.interrupt_main() (used from IDLE), so it's still needed. + Simulate the effect of a :const:`SIGINT` signal arriving. The next time + :c:func:`PyErr_CheckSignals` is called, the Python signal handler for + :const:`SIGINT` will be called. + If :const:`SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. .. c:function:: int PySignal_SetWakeupFd(int fd) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 50998be5..93fcfe62 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -37,6 +37,7 @@ The following functions can be safely called before Python is initialized: * Informative functions: + * :c:func:`Py_IsInitialized` * :c:func:`PyMem_GetAllocator` * :c:func:`PyObject_GetArenaAllocator` * :c:func:`Py_GetBuildInfo` @@ -855,6 +856,12 @@ code, or when embedding the Python interpreter: *NULL*. If the lock has been created, the current thread must not have acquired it, otherwise deadlock ensues. + .. note:: + Calling this function from a thread when the runtime is finalizing + will terminate the thread, even if the thread was not created by Python. + You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to + check if the interpreter is in process of being finalized before calling + this function to avoid unwanted termination. .. c:function:: PyThreadState* PyThreadState_Get() @@ -902,6 +909,12 @@ with sub-interpreters: When the function returns, the current thread will hold the GIL and be able to call arbitrary Python code. Failure is a fatal error. + .. note:: + Calling this function from a thread when the runtime is finalizing + will terminate the thread, even if the thread was not created by Python. + You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to + check if the interpreter is in process of being finalized before calling + this function to avoid unwanted termination. .. c:function:: void PyGILState_Release(PyGILState_STATE) @@ -1382,6 +1395,11 @@ These functions are only intended to be used by advanced debugging tools. Return the interpreter state object at the head of the list of all such objects. +.. c:function:: PyInterpreterState* PyInterpreterState_Main() + + Return the main interpreter state object. + + .. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) Return the next interpreter state object after *interp* from the list of all diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 15006100..330871bc 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -48,7 +48,8 @@ Include Files All function, type and macro definitions needed to use the Python/C API are included in your code by the following line:: - #include "Python.h" + #define PY_SSIZE_T_CLEAN + #include This implies inclusion of the following standard headers: ````, ````, ````, ````, ```` and ```` @@ -60,6 +61,9 @@ This implies inclusion of the following standard headers: ````, headers on some systems, you *must* include :file:`Python.h` before any standard headers are included. + It is recommended to always define ``PY_SSIZE_T_CLEAN`` before including + ``Python.h``. See :ref:`arg-parsing` for a description of this macro. + All user visible names defined by Python.h (except those defined by the included standard headers) have one of the prefixes ``Py`` or ``_Py``. Names beginning with ``_Py`` are for internal use by the Python implementation and should not be diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 5b1f386f..71144f16 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -259,7 +259,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. If the value of *obj* is out of range for an :c:type:`unsigned long`, return the reduction of that value modulo ``ULONG_MAX + 1``. - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``(unsigned long)-1`` on error. Use :c:func:`PyErr_Occurred` to + disambiguate. .. c:function:: unsigned long long PyLong_AsUnsignedLongLongMask(PyObject *obj) @@ -271,7 +272,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. If the value of *obj* is out of range for an :c:type:`unsigned long long`, return the reduction of that value modulo ``PY_ULLONG_MAX + 1``. - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``(unsigned long long)-1`` on error. Use :c:func:`PyErr_Occurred` + to disambiguate. .. c:function:: double PyLong_AsDouble(PyObject *pylong) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index b79b7e49..9b42900b 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -440,8 +440,9 @@ Customize Memory Allocators Setup hooks to detect bugs in the Python memory allocator functions. - Newly allocated memory is filled with the byte ``0xCB``, freed memory is - filled with the byte ``0xDB``. + Newly allocated memory is filled with the byte ``0xCD`` (``CLEANBYTE``), + freed memory is filled with the byte ``0xDD`` (``DEADBYTE``). Memory blocks + are surrounded by "forbidden bytes" (``FORBIDDENBYTE``: byte ``0xFD``). Runtime checks: @@ -471,6 +472,12 @@ Customize Memory Allocators if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and :c:data:`PYMEM_DOMAIN_MEM` domains are called. + .. versionchanged:: 3.7.3 + Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and + ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD`` + and ``0xFD`` to use the same values than Windows CRT debug ``malloc()`` + and ``free()``. + .. _pymalloc: diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a64ff2e6..c09d97a6 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -261,7 +261,8 @@ Object Protocol *args* must not be *NULL*, use an empty tuple if no arguments are needed. If no named arguments are needed, *kwargs* can be *NULL*. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``callable(*args, **kwargs)``. @@ -272,7 +273,8 @@ Object Protocol Call a callable Python object *callable*, with arguments given by the tuple *args*. If no arguments are needed, then *args* can be *NULL*. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``callable(*args)``. @@ -283,7 +285,8 @@ Object Protocol The C arguments are described using a :c:func:`Py_BuildValue` style format string. The format can be *NULL*, indicating that no arguments are provided. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``callable(*args)``. @@ -302,7 +305,8 @@ Object Protocol The format can be *NULL*, indicating that no arguments are provided. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``obj.name(arg1, arg2, ...)``. @@ -320,7 +324,8 @@ Object Protocol :c:type:`PyObject\*` arguments. The arguments are provided as a variable number of parameters followed by *NULL*. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + *NULL* on failure. This is the equivalent of the Python expression: ``callable(arg1, arg2, ...)``. @@ -331,7 +336,9 @@ Object Protocol Calls a method of the Python object *obj*, where the name of the method is given as a Python string object in *name*. It is called with a variable number of :c:type:`PyObject\*` arguments. The arguments are provided as a variable number - of parameters followed by *NULL*. Returns the result of the call on success, or + of parameters followed by *NULL*. + + Return the result of the call on success, or raise an exception and return *NULL* on failure. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index da45da1d..274beeef 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -114,10 +114,20 @@ the definition of all other Python objects. .. c:type:: PyCFunctionWithKeywords - Type of the functions used to implement Python callables in C that take - keyword arguments: they take three :c:type:`PyObject\*` parameters and return - one such value. See :c:type:`PyCFunction` above for the meaning of the return - value. + Type of the functions used to implement Python callables in C + with signature :const:`METH_VARARGS | METH_KEYWORDS`. + + +.. c:type:: _PyCFunctionFast + + Type of the functions used to implement Python callables in C + with signature :const:`METH_FASTCALL`. + + +.. c:type:: _PyCFunctionFastWithKeywords + + Type of the functions used to implement Python callables in C + with signature :const:`METH_FASTCALL | METH_KEYWORDS`. .. c:type:: PyMethodDef @@ -149,10 +159,11 @@ specific C type of the *self* object. The :attr:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding -convention. Of the calling convention flags, only :const:`METH_VARARGS` and -:const:`METH_KEYWORDS` can be combined. Any of the calling convention flags -can be combined with a binding flag. +convention. +There are four basic calling conventions for positional arguments +and two of them can be combined with :const:`METH_KEYWORDS` to support +also keyword arguments. So there are a total of 6 calling conventions: .. data:: METH_VARARGS @@ -164,13 +175,41 @@ can be combined with a binding flag. using :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_UnpackTuple`. -.. data:: METH_KEYWORDS +.. data:: METH_VARARGS | METH_KEYWORDS Methods with these flags must be of type :c:type:`PyCFunctionWithKeywords`. - The function expects three parameters: *self*, *args*, and a dictionary of - all the keyword arguments. The flag must be combined with - :const:`METH_VARARGS`, and the parameters are typically processed using - :c:func:`PyArg_ParseTupleAndKeywords`. + The function expects three parameters: *self*, *args*, *kwargs* where + *kwargs* is a dictionary of all the keyword arguments or possibly *NULL* + if there are no keyword arguments. The parameters are typically processed + using :c:func:`PyArg_ParseTupleAndKeywords`. + + +.. data:: METH_FASTCALL + + Fast calling convention supporting only positional arguments. + The methods have the type :c:type:`_PyCFunctionFast`. + The first parameter is *self*, the second parameter is a C array + of :c:type:`PyObject\*` values indicating the arguments and the third + parameter is the number of arguments (the length of the array). + + This is not part of the :ref:`limited API `. + + .. versionadded:: 3.7 + + +.. data:: METH_FASTCALL | METH_KEYWORDS + + Extension of :const:`METH_FASTCALL` supporting also keyword arguments, + with methods of type :c:type:`_PyCFunctionFastWithKeywords`. + Keyword arguments are passed the same way as in the vectorcall protocol: + there is an additional fourth :c:type:`PyObject\*` parameter + which is a tuple representing the names of the keyword arguments + or possibly *NULL* if there are no keywords. The values of the keyword + arguments are stored in the *args* array, after the positional arguments. + + This is not part of the :ref:`limited API `. + + .. versionadded:: 3.7 .. data:: METH_NOARGS diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 6cbcc273..532508ee 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1256,7 +1256,7 @@ Sequence Object Structures signature. It should modify its first operand, and return it. This slot may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceConcat` will fall back to :c:func:`PySequence_Concat`. It is also used by the - augmented assignment ``+=``, after trying numeric inplace addition + augmented assignment ``+=``, after trying numeric in-place addition via the :c:member:`~PyNumberMethods.nb_inplace_add` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat @@ -1265,7 +1265,7 @@ Sequence Object Structures signature. It should modify its first operand, and return it. This slot may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceRepeat` will fall back to :c:func:`PySequence_Repeat`. It is also used by the - augmented assignment ``*=``, after trying numeric inplace multiplication + augmented assignment ``*=``, after trying numeric in-place multiplication via the :c:member:`~PyNumberMethods.nb_inplace_multiply` slot. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 39c067d4..1d724a32 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -22,14 +22,14 @@ in the Unicode object. The :c:type:`Py_UNICODE*` representation is deprecated and inefficient; it should be avoided in performance- or memory-sensitive situations. -Due to the transition between the old APIs and the new APIs, unicode objects +Due to the transition between the old APIs and the new APIs, Unicode objects can internally be in two states depending on how they were created: -* "canonical" unicode objects are all objects created by a non-deprecated - unicode API. They use the most efficient representation allowed by the +* "canonical" Unicode objects are all objects created by a non-deprecated + Unicode API. They use the most efficient representation allowed by the implementation. -* "legacy" unicode objects have been created through one of the deprecated +* "legacy" Unicode objects have been created through one of the deprecated APIs (typically :c:func:`PyUnicode_FromUnicode`) and only bear the :c:type:`Py_UNICODE*` representation; you will have to call :c:func:`PyUnicode_READY` on them before calling any other API. @@ -152,7 +152,7 @@ access internal read-only data of Unicode objects: .. c:function:: void* PyUnicode_DATA(PyObject *o) - Return a void pointer to the raw unicode buffer. *o* has to be a Unicode + Return a void pointer to the raw Unicode buffer. *o* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 @@ -430,7 +430,7 @@ APIs: .. c:function:: PyObject* PyUnicode_FromFormat(const char *format, ...) Take a C :c:func:`printf`\ -style *format* string and a variable number of - arguments, calculate the size of the resulting Python unicode string and return + arguments, calculate the size of the resulting Python Unicode string and return a string with the values formatted into it. The variable arguments must be C types and must correspond exactly to the format characters in the *format* ASCII-encoded string. The following format characters are allowed: @@ -504,9 +504,9 @@ APIs: | :attr:`%A` | PyObject\* | The result of calling | | | | :func:`ascii`. | +-------------------+---------------------+--------------------------------+ - | :attr:`%U` | PyObject\* | A unicode object. | + | :attr:`%U` | PyObject\* | A Unicode object. | +-------------------+---------------------+--------------------------------+ - | :attr:`%V` | PyObject\*, | A unicode object (which may be | + | :attr:`%V` | PyObject\*, | A Unicode object (which may be | | | const char\* | *NULL*) and a null-terminated | | | | C character array as a second | | | | parameter (which will be used, | @@ -1670,7 +1670,7 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *uni, const char *string) - Compare a unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less + Compare a Unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less than, equal, and greater than, respectively. It is best to pass only ASCII-encoded strings, but the function interprets the input string as ISO-8859-1 if it contains non-ASCII characters. @@ -1680,7 +1680,7 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) - Rich compare two unicode strings and return one of the following: + Rich compare two Unicode strings and return one of the following: * ``NULL`` in case an exception was raised * :const:`Py_True` or :const:`Py_False` for successful comparisons @@ -1708,7 +1708,7 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: void PyUnicode_InternInPlace(PyObject **string) Intern the argument *\*string* in place. The argument must be the address of a - pointer variable pointing to a Python unicode string object. If there is an + pointer variable pointing to a Python Unicode string object. If there is an existing interned string that is the same as *\*string*, it sets *\*string* to it (decrementing the reference count of the old string object and incrementing the reference count of the interned string object), otherwise it leaves @@ -1721,6 +1721,6 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: PyObject* PyUnicode_InternFromString(const char *v) A combination of :c:func:`PyUnicode_FromString` and - :c:func:`PyUnicode_InternInPlace`, returning either a new unicode string + :c:func:`PyUnicode_InternInPlace`, returning either a new Unicode string object that has been interned, or a new ("owned") reference to an earlier interned string object with the same value. diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index c891f632..317093e9 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -109,6 +109,10 @@ the same library that the Python runtime is using. (:func:`sys.getfilesystemencoding`). If *closeit* is true, the file is closed before PyRun_SimpleFileExFlags returns. + .. note:: + On Windows, *fp* should be opened as binary mode (e.g. ``fopen(filename, "rb")``. + Otherwise, Python may not handle script file with LF line ending correctly. + .. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) diff --git a/Doc/conf.py b/Doc/conf.py index eab3c39c..da8b9d7b 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -46,6 +46,8 @@ smartquotes_excludes = { 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'], } +# Avoid a warning with Sphinx >= 2.0 +master_doc = 'contents' # Options for HTML output # ----------------------- diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 35527c17..213ddcb6 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -413,6 +413,16 @@ PyDateTime_FromDateAndTime:int:minute:: PyDateTime_FromDateAndTime:int:second:: PyDateTime_FromDateAndTime:int:usecond:: +PyDateTime_FromDateAndTimeAndFold:PyObject*::+1: +PyDateTime_FromDateAndTimeAndFold:int:year:: +PyDateTime_FromDateAndTimeAndFold:int:month:: +PyDateTime_FromDateAndTimeAndFold:int:day:: +PyDateTime_FromDateAndTimeAndFold:int:hour:: +PyDateTime_FromDateAndTimeAndFold:int:minute:: +PyDateTime_FromDateAndTimeAndFold:int:second:: +PyDateTime_FromDateAndTimeAndFold:int:usecond:: +PyDateTime_FromDateAndTimeAndFold:int:fold:: + PyDateTime_FromTimestamp:PyObject*::+1: PyDateTime_FromTimestamp:PyObject*:args:0: @@ -2210,6 +2220,13 @@ PyTime_FromTime:int:minute:: PyTime_FromTime:int:second:: PyTime_FromTime:int:usecond:: +PyTime_FromTimeAndFold:PyObject*::+1: +PyTime_FromTimeAndFold:int:hour:: +PyTime_FromTimeAndFold:int:minute:: +PyTime_FromTimeAndFold:int:second:: +PyTime_FromTimeAndFold:int:usecond:: +PyTime_FromTimeAndFold:int:fold:: + PyTraceMalloc_Track:int::: PyTraceMalloc_Track:unsigned int:domain:: PyTraceMalloc_Track:uintptr_t:ptr:: diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 5dd14b1f..2e46c7ac 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -113,11 +113,17 @@ recommended tools`_. .. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations -Reading the guide -================= +.. index:: + single: Python Package Index (PyPI) + single: PyPI; (see Python Package Index (PyPI)) + +.. _publishing-python-packages: + +Reading the Python Packaging User Guide +======================================= The Python Packaging User Guide covers the various key steps and elements -involved in creating a project: +involved in creating and publishing a project: * `Project structure`_ * `Building and packaging the project`_ diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst index d6f7640f..aaf4536d 100644 --- a/Doc/distutils/index.rst +++ b/Doc/distutils/index.rst @@ -34,7 +34,6 @@ very little overhead for build/release/install mechanics. configfile.rst sourcedist.rst builtdist.rst - packageindex.rst examples.rst extending.rst commandref.rst diff --git a/Doc/distutils/packageindex.rst b/Doc/distutils/packageindex.rst index 50cb74f2..ccb9a598 100644 --- a/Doc/distutils/packageindex.rst +++ b/Doc/distutils/packageindex.rst @@ -1,6 +1,4 @@ -.. index:: - single: Python Package Index (PyPI) - single: PyPI; (see Python Package Index (PyPI)) +:orphan: .. _package-index: @@ -8,246 +6,11 @@ The Python Package Index (PyPI) ******************************* -The `Python Package Index (PyPI)`_ stores :ref:`meta-data ` -describing distributions packaged with distutils, as well as package data like -distribution files if a package author wishes. - -Distutils provides the :command:`register` and :command:`upload` commands for -pushing meta-data and distribution files to PyPI, respectively. See -:ref:`package-commands` for information on these commands. - - -PyPI overview -============= - -PyPI lets you submit any number of versions of your distribution to the index. -If you alter the meta-data for a particular version, you can submit it again -and the index will be updated. - -PyPI holds a record for each (name, version) combination submitted. The first -user to submit information for a given name is designated the Owner of that -name. Changes can be submitted through the :command:`register` command or -through the web interface. Owners can designate other users as Owners or -Maintainers. Maintainers can edit the package information, but not designate -new Owners or Maintainers. - -By default PyPI displays only the newest version of a given package. The web -interface lets one change this default behavior and manually select which -versions to display and hide. - -For each version, PyPI displays a home page. The home page is created from -the ``long_description`` which can be submitted via the :command:`register` -command. See :ref:`package-display` for more information. - - -.. _package-commands: - -Distutils commands -================== - -Distutils exposes two commands for submitting package data to PyPI: the -:ref:`register ` command for submitting meta-data to PyPI -and the :ref:`upload ` command for submitting distribution -files. Both commands read configuration data from a special file called a -:ref:`.pypirc file `. - - -.. _package-register: - -The ``register`` command ------------------------- - -The distutils command :command:`register` is used to submit your distribution's -meta-data to an index server. It is invoked as follows:: - - python setup.py register - -Distutils will respond with the following prompt:: - - running register - We need to know who you are, so please choose either: - 1. use your existing login, - 2. register as a new user, - 3. have the server generate a new password for you (and email it to you), or - 4. quit - Your selection [default 1]: - -Note: if your username and password are saved locally, you will not see this -menu. Also, refer to :ref:`pypirc` for how to store your credentials in a -:file:`.pypirc` file. - -If you have not registered with PyPI, then you will need to do so now. You -should choose option 2, and enter your details as required. Soon after -submitting your details, you will receive an email which will be used to confirm -your registration. - -Once you are registered, you may choose option 1 from the menu. You will be -prompted for your PyPI username and password, and :command:`register` will then -submit your meta-data to the index. - -See :ref:`package-cmdoptions` for options to the :command:`register` command. - - -.. _package-upload: - -The ``upload`` command ----------------------- - -The distutils command :command:`upload` pushes the distribution files to PyPI. - -The command is invoked immediately after building one or more distribution -files. For example, the command :: - - python setup.py sdist bdist_wininst upload - -will cause the source distribution and the Windows installer to be uploaded to -PyPI. Note that these will be uploaded even if they are built using an earlier -invocation of :file:`setup.py`, but that only distributions named on the command -line for the invocation including the :command:`upload` command are uploaded. - -If a :command:`register` command was previously called in the same command, -and if the password was entered in the prompt, :command:`upload` will reuse the -entered password. This is useful if you do not want to store a password in -clear text in a :file:`.pypirc` file. - -You can use the ``--sign`` option to tell :command:`upload` to sign each -uploaded file using GPG (GNU Privacy Guard). The :program:`gpg` program must -be available for execution on the system :envvar:`PATH`. You can also specify -which key to use for signing using the ``--identity=name`` option. - -See :ref:`package-cmdoptions` for additional options to the :command:`upload` -command. - - -.. _package-cmdoptions: - -Additional command options --------------------------- - -This section describes options common to both the :command:`register` and -:command:`upload` commands. - -The ``--repository`` or ``-r`` option lets you specify a PyPI server -different from the default. For example:: - - python setup.py sdist bdist_wininst upload -r https://example.com/pypi - -For convenience, a name can be used in place of the URL when the -:file:`.pypirc` file is configured to do so. For example:: - - python setup.py register -r other - -See :ref:`pypirc` for more information on defining alternate servers. - -The ``--show-response`` option displays the full response text from the PyPI -server, which is useful when debugging problems with registering and uploading. - - -.. index:: - single: .pypirc file - single: Python Package Index (PyPI); .pypirc file - -.. _pypirc: - -The ``.pypirc`` file --------------------- - -The :command:`register` and :command:`upload` commands both check for the -existence of a :file:`.pypirc` file at the location :file:`$HOME/.pypirc`. -If this file exists, the command uses the username, password, and repository -URL configured in the file. The format of a :file:`.pypirc` file is as -follows: - -.. code-block:: ini - - [distutils] - index-servers = - pypi - - [pypi] - repository: - username: - password: - -The *distutils* section defines an *index-servers* variable that lists the -name of all sections describing a repository. - -Each section describing a repository defines three variables: - -- *repository*, that defines the url of the PyPI server. Defaults to - ``https://upload.pypi.org/legacy/``. -- *username*, which is the registered username on the PyPI server. -- *password*, that will be used to authenticate. If omitted the user - will be prompt to type it when needed. - -If you want to define another server a new section can be created and -listed in the *index-servers* variable: - -.. code-block:: ini - - [distutils] - index-servers = - pypi - other - - [pypi] - repository: - username: - password: - - [other] - repository: https://example.com/pypi - username: - password: - -This allows the :command:`register` and :command:`upload` commands to be -called with the ``--repository`` option as described in -:ref:`package-cmdoptions`. - -Specifically, you might want to add the `PyPI Test Repository -`_ to your ``.pypirc`` to facilitate -testing before doing your first upload to ``PyPI`` itself. - - -.. _package-display: - -PyPI package display -==================== - -The ``long_description`` field plays a special role at PyPI. It is used by -the server to display a home page for the registered package. - -If you use the `reStructuredText `_ -syntax for this field, PyPI will parse it and display an HTML output for -the package home page. - -The ``long_description`` field can be attached to a text file located -in the package:: - - from distutils.core import setup - - with open('README.txt') as file: - long_description = file.read() - - setup(name='Distutils', - long_description=long_description) - -In that case, :file:`README.txt` is a regular reStructuredText text file located -in the root of the package besides :file:`setup.py`. - -To prevent registering broken reStructuredText content, you can use the -:program:`rst2html` program that is provided by the :mod:`docutils` package and -check the ``long_description`` from the command line: - -.. code-block:: shell-session - - $ python setup.py --long-description | rst2html.py > output.html - -:mod:`docutils` will display a warning if there's something wrong with your -syntax. Because PyPI applies additional checks (e.g. by passing ``--no-raw`` -to ``rst2html.py`` in the command above), being able to run the command above -without warnings does not guarantee that PyPI will convert the content -successfully. +The `Python Package Index (PyPI)`_ stores metadata describing distributions +packaged with distutils and other publishing tools, as well the distribution +archives themselves. +References to up to date PyPI documentation can be found at +:ref:`publishing-python-packages`. .. _Python Package Index (PyPI): https://pypi.org diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst index 54ed1aeb..1f99f62f 100644 --- a/Doc/distutils/setupscript.rst +++ b/Doc/distutils/setupscript.rst @@ -523,7 +523,7 @@ following way:: setup(..., data_files=[('bitmaps', ['bm/b1.gif', 'bm/b2.gif']), - ('config', ['cfg/data.cfg']), + ('config', ['cfg/data.cfg'])], ) Each (*directory*, *files*) pair in the sequence specifies the installation @@ -612,9 +612,8 @@ Notes: provided, distutils lists it as the author in :file:`PKG-INFO`. (4) - The ``long_description`` field is used by PyPI when you are - :ref:`registering ` a package, to - :ref:`build its home page `. + The ``long_description`` field is used by PyPI when you publish a package, + to build its project page. (5) The ``license`` field is a text indicating the license covering the diff --git a/Doc/distutils/uploading.rst b/Doc/distutils/uploading.rst index 4bce6997..4c391cab 100644 --- a/Doc/distutils/uploading.rst +++ b/Doc/distutils/uploading.rst @@ -4,4 +4,5 @@ Uploading Packages to the Package Index *************************************** -The contents of this page have moved to the section :ref:`package-index`. +References to up to date PyPI documentation can be found at +:ref:`publishing-python-packages`. diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 7e4fc19d..13d83b72 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -53,6 +53,7 @@ interface. This interface is intended to execute a Python script without needing to interact with the application directly. This can for example be used to perform some operation on a file. :: + #define PY_SSIZE_T_CLEAN #include int diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index b788a557..afed3aab 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -55,8 +55,9 @@ called ``spam``, the C file containing its implementation is called :file:`spammodule.c`; if the module name is very long, like ``spammify``, the module name can be just :file:`spammify.c`.) -The first line of our file can be:: +The first two lines of our file can be:: + #define PY_SSIZE_T_CLEAN #include which pulls in the Python API (you can add a comment describing the purpose of @@ -68,6 +69,9 @@ the module and a copyright notice if you like). headers on some systems, you *must* include :file:`Python.h` before any standard headers are included. + It is recommended to always define ``PY_SSIZE_T_CLEAN`` before including + ``Python.h``. See :ref:`parsetuple` for a description of this macro. + All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or ``PY``, except those defined in standard header files. For convenience, and since they are used extensively by the Python interpreter, ``"Python.h"`` @@ -729,7 +733,8 @@ it returns false and raises an appropriate exception. Here is an example module which uses keywords, based on an example by Geoff Philbrick (philbrick@hks.com):: - #include "Python.h" + #define PY_SSIZE_T_CLEAN /* Make "s#" use Py_ssize_t rather than int. */ + #include static PyObject * keywdarg_parrot(PyObject *self, PyObject *args, PyObject *keywds) @@ -1228,7 +1233,7 @@ The function :c:func:`spam_system` is modified in a trivial way:: In the beginning of the module, right after the line :: - #include "Python.h" + #include two more lines must be added:: diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index ac48637b..07c2ef71 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -92,6 +92,7 @@ The second bit is the definition of the type object. :: .tp_doc = "Custom objects", .tp_basicsize = sizeof(CustomObject), .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, .tp_new = PyType_GenericNew, }; diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 74e1af6e..2ad27658 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -280,6 +280,7 @@ solution then is to call :c:func:`PyParser_ParseString` and test for ``e.error`` equal to ``E_EOF``, which means the input is incomplete. Here's a sample code fragment, untested, inspired by code from Alex Farber:: + #define PY_SSIZE_T_CLEAN #include #include #include @@ -318,6 +319,7 @@ complete example using the GNU readline library (you may want to ignore #include #include + #define PY_SSIZE_T_CLEAN #include #include #include diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 7ee340d7..3ef553e8 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -306,14 +306,10 @@ guaranteed that interfaces will remain the same throughout a series of bugfix releases. The latest stable releases can always be found on the `Python download page -`_. There are two production-ready version -of Python: 2.x and 3.x, but the recommended one at this times is Python 3.x. -Although Python 2.x is still widely used, `it will not be -maintained after January 1, 2020 `_. -Python 2.x was known for having more third-party libraries available, however, -by the time of this writing, most of the widely used libraries support Python 3.x, -and some are even dropping the Python 2.x support. - +`_. There are two production-ready versions +of Python: 2.x and 3.x. The recommended version is 3.x, which is supported by +most widely used libraries. Although 2.x is still widely used, `it will not +be maintained after January 1, 2020 `_. How many people are using Python? --------------------------------- diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 4f9979bf..781da467 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -104,7 +104,7 @@ What platform-specific GUI toolkits exist for Python? ======================================================== By installing the `PyObjc Objective-C bridge -`_, Python programs can use Mac OS X's +`_, Python programs can use Mac OS X's Cocoa libraries. :ref:`Pythonwin ` by Mark Hammond includes an interface to the diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 31614189..f14e8cc8 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -16,6 +16,9 @@ Is there a source code level debugger with breakpoints, single-stepping, etc.? Yes. +Several debuggers for Python are described below, and the built-in function +:func:`breakpoint` allows you to drop into any of them. + The pdb module is a simple but adequate console-mode debugger for Python. It is part of the standard Python library, and is :mod:`documented in the Library Reference Manual `. You can also write your own debugger by using the code diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index a46d4c17..a92ec1b5 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -15,6 +15,8 @@ Python on Windows FAQ .. XXX need review for Python 3. XXX need review for Windows Vista/Seven? +.. _faq-run-program-under-windows: + How do I run a Python program under Windows? -------------------------------------------- diff --git a/Doc/glossary.rst b/Doc/glossary.rst index fb8ff2a7..df6f6b64 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -13,10 +13,10 @@ Glossary examples which can be executed interactively in the interpreter. ``...`` - The default Python prompt of the interactive shell when entering code for - an indented code block, when within a pair of matching left and right - delimiters (parentheses, square brackets, curly braces or triple quotes), - or after specifying a decorator. + The default Python prompt of the interactive shell when entering the + code for an indented code block, when within a pair of matching left and + right delimiters (parentheses, square brackets, curly braces or triple + quotes), or after specifying a decorator. 2to3 A tool that tries to convert Python 2.x code to Python 3.x code by @@ -221,6 +221,15 @@ Glossary statement by defining :meth:`__enter__` and :meth:`__exit__` methods. See :pep:`343`. + context variable + A variable which can have different values depending on its context. + This is similar to Thread-Local Storage in which each execution + thread may have a different value for a variable. However, with context + variables, there may be several contexts in one execution thread and the + main usage for context variables is to keep track of variables in + concurrent asynchronous tasks. + See :mod:`contextvars`. + contiguous .. index:: C-contiguous, Fortran contiguous @@ -499,8 +508,10 @@ Glossary Hashability makes an object usable as a dictionary key and a set member, because these data structures use the hash value internally. - All of Python's immutable built-in objects are hashable; mutable - containers (such as lists or dictionaries) are not. Objects which are + Most of Python's immutable built-in objects are hashable; mutable + containers (such as lists or dictionaries) are not; immutable + containers (such as tuples and frozensets) are only hashable if + their elements are hashable. Objects which are instances of user-defined classes are hashable by default. They all compare unequal (except with themselves), and their hash value is derived from their :func:`id`. @@ -659,6 +670,11 @@ Glossary :term:`finder`. See :pep:`302` for details and :class:`importlib.abc.Loader` for an :term:`abstract base class`. + magic method + .. index:: pair: magic; method + + An informal synonym for :term:`special method`. + mapping A container object that supports arbitrary key lookups and implements the methods specified in the :class:`~collections.abc.Mapping` or @@ -1000,6 +1016,8 @@ Glossary (subscript) notation uses :class:`slice` objects internally. special method + .. index:: pair: special; method + A method that is called implicitly by Python to execute a certain operation on a type, such as addition. Such methods have names starting and ending with double underscores. Special methods are documented in diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 51520b72..3d1da5ac 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -372,9 +372,9 @@ calls are unexciting:: ... print(x) ... f = staticmethod(f) ... - >>> print(E.f(3)) + >>> E.f(3) 3 - >>> print(E().f(3)) + >>> E().f(3) 3 Using the non-data descriptor protocol, a pure Python version of diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 5339bf45..d4b1b2a7 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -135,17 +135,22 @@ used than UTF-8.) UTF-8 uses the following rules: UTF-8 has several convenient properties: 1. It can handle any Unicode code point. -2. A Unicode string is turned into a sequence of bytes containing no embedded zero - bytes. This avoids byte-ordering issues, and means UTF-8 strings can be - processed by C functions such as ``strcpy()`` and sent through protocols that - can't handle zero bytes. +2. A Unicode string is turned into a sequence of bytes that contains embedded + zero bytes only where they represent the null character (U+0000). This means + that UTF-8 strings can be processed by C functions such as ``strcpy()`` and sent + through protocols that can't handle zero bytes for anything other than + end-of-string markers. 3. A string of ASCII text is also valid UTF-8 text. 4. UTF-8 is fairly compact; the majority of commonly used characters can be represented with one or two bytes. 5. If bytes are corrupted or lost, it's possible to determine the start of the next UTF-8-encoded code point and resynchronize. It's also unlikely that random 8-bit data will look like valid UTF-8. - +6. UTF-8 is a byte oriented encoding. The encoding specifies that each + character is represented by a specific sequence of one or more bytes. This + avoids the byte-ordering issues that can occur with integer and word oriented + encodings, like UTF-16 and UTF-32, where the sequence of bytes varies depending + on the hardware on which the string was encoded. References diff --git a/Doc/includes/custom.c b/Doc/includes/custom.c index fb2c7b2a..13d16f54 100644 --- a/Doc/includes/custom.c +++ b/Doc/includes/custom.c @@ -1,3 +1,4 @@ +#define PY_SSIZE_T_CLEAN #include typedef struct { diff --git a/Doc/includes/custom2.c b/Doc/includes/custom2.c index 51ab4b80..6477a19d 100644 --- a/Doc/includes/custom2.c +++ b/Doc/includes/custom2.c @@ -1,3 +1,4 @@ +#define PY_SSIZE_T_CLEAN #include #include "structmember.h" diff --git a/Doc/includes/custom3.c b/Doc/includes/custom3.c index 09e87355..213d0864 100644 --- a/Doc/includes/custom3.c +++ b/Doc/includes/custom3.c @@ -1,3 +1,4 @@ +#define PY_SSIZE_T_CLEAN #include #include "structmember.h" diff --git a/Doc/includes/custom4.c b/Doc/includes/custom4.c index 0994d8fd..b0b2906d 100644 --- a/Doc/includes/custom4.c +++ b/Doc/includes/custom4.c @@ -1,3 +1,4 @@ +#define PY_SSIZE_T_CLEAN #include #include "structmember.h" diff --git a/Doc/includes/run-func.c b/Doc/includes/run-func.c index 9caf1fdb..392f86d6 100644 --- a/Doc/includes/run-func.c +++ b/Doc/includes/run-func.c @@ -1,3 +1,4 @@ +#define PY_SSIZE_T_CLEAN #include int diff --git a/Doc/includes/sqlite3/adapter_datetime.py b/Doc/includes/sqlite3/adapter_datetime.py index be333951..d5221d80 100644 --- a/Doc/includes/sqlite3/adapter_datetime.py +++ b/Doc/includes/sqlite3/adapter_datetime.py @@ -13,3 +13,5 @@ cur = con.cursor() now = datetime.datetime.now() cur.execute("select ?", (now,)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/adapter_point_1.py b/Doc/includes/sqlite3/adapter_point_1.py index 6b1af841..77daf8f1 100644 --- a/Doc/includes/sqlite3/adapter_point_1.py +++ b/Doc/includes/sqlite3/adapter_point_1.py @@ -14,3 +14,5 @@ cur = con.cursor() p = Point(4.0, -3.2) cur.execute("select ?", (p,)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/adapter_point_2.py b/Doc/includes/sqlite3/adapter_point_2.py index d670700f..cb863316 100644 --- a/Doc/includes/sqlite3/adapter_point_2.py +++ b/Doc/includes/sqlite3/adapter_point_2.py @@ -15,3 +15,5 @@ cur = con.cursor() p = Point(4.0, -3.2) cur.execute("select ?", (p,)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/connect_db_1.py b/Doc/includes/sqlite3/connect_db_1.py deleted file mode 100644 index 1b975232..00000000 --- a/Doc/includes/sqlite3/connect_db_1.py +++ /dev/null @@ -1,3 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") diff --git a/Doc/includes/sqlite3/connect_db_2.py b/Doc/includes/sqlite3/connect_db_2.py deleted file mode 100644 index f9728b36..00000000 --- a/Doc/includes/sqlite3/connect_db_2.py +++ /dev/null @@ -1,3 +0,0 @@ -import sqlite3 - -con = sqlite3.connect(":memory:") diff --git a/Doc/includes/sqlite3/countcursors.py b/Doc/includes/sqlite3/countcursors.py index ef3e70a2..112f4770 100644 --- a/Doc/includes/sqlite3/countcursors.py +++ b/Doc/includes/sqlite3/countcursors.py @@ -13,3 +13,5 @@ con = sqlite3.connect(":memory:", factory=CountCursorsConnection) cur1 = con.cursor() cur2 = con.cursor() print(con.numcursors) + +con.close() diff --git a/Doc/includes/sqlite3/ctx_manager.py b/Doc/includes/sqlite3/ctx_manager.py index 7af4ad1e..6db77d45 100644 --- a/Doc/includes/sqlite3/ctx_manager.py +++ b/Doc/includes/sqlite3/ctx_manager.py @@ -14,3 +14,7 @@ try: con.execute("insert into person(firstname) values (?)", ("Joe",)) except sqlite3.IntegrityError: print("couldn't add Joe twice") + +# Connection object used as context manager only commits or rollbacks transactions, +# so the connection object should be closed manually +con.close() diff --git a/Doc/includes/sqlite3/execsql_fetchonerow.py b/Doc/includes/sqlite3/execsql_fetchonerow.py index 078873bf..115bcb50 100644 --- a/Doc/includes/sqlite3/execsql_fetchonerow.py +++ b/Doc/includes/sqlite3/execsql_fetchonerow.py @@ -15,3 +15,5 @@ for (name_last, age) in cur: cur.execute(SELECT) for row in cur: print('%s is %d years old.' % (row[0], row[1])) + +con.close() diff --git a/Doc/includes/sqlite3/execsql_printall_1.py b/Doc/includes/sqlite3/execsql_printall_1.py index a4ce5c52..19306e6e 100644 --- a/Doc/includes/sqlite3/execsql_printall_1.py +++ b/Doc/includes/sqlite3/execsql_printall_1.py @@ -11,3 +11,5 @@ cur.execute("select * from people order by age") # Retrieve all rows as a sequence and print that sequence: print(cur.fetchall()) + +con.close() diff --git a/Doc/includes/sqlite3/execute_1.py b/Doc/includes/sqlite3/execute_1.py index f864a898..3466b126 100644 --- a/Doc/includes/sqlite3/execute_1.py +++ b/Doc/includes/sqlite3/execute_1.py @@ -14,3 +14,5 @@ cur.execute("insert into people values (?, ?)", (who, age)) cur.execute("select * from people where name_last=:who and age=:age", {"who": who, "age": age}) print(cur.fetchone()) + +con.close() diff --git a/Doc/includes/sqlite3/execute_3.py b/Doc/includes/sqlite3/execute_3.py deleted file mode 100644 index 0353683f..00000000 --- a/Doc/includes/sqlite3/execute_3.py +++ /dev/null @@ -1,12 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") - -cur = con.cursor() - -who = "Yeltsin" -age = 72 - -cur.execute("select name_last, age from people where name_last=:who and age=:age", - locals()) -print(cur.fetchone()) diff --git a/Doc/includes/sqlite3/executemany_1.py b/Doc/includes/sqlite3/executemany_1.py index efae1063..edf6f8b7 100644 --- a/Doc/includes/sqlite3/executemany_1.py +++ b/Doc/includes/sqlite3/executemany_1.py @@ -22,3 +22,5 @@ cur.executemany("insert into characters(c) values (?)", theIter) cur.execute("select c from characters") print(cur.fetchall()) + +con.close() diff --git a/Doc/includes/sqlite3/executemany_2.py b/Doc/includes/sqlite3/executemany_2.py index 527358eb..02a594c8 100644 --- a/Doc/includes/sqlite3/executemany_2.py +++ b/Doc/includes/sqlite3/executemany_2.py @@ -13,3 +13,5 @@ cur.executemany("insert into characters(c) values (?)", char_generator()) cur.execute("select c from characters") print(cur.fetchall()) + +con.close() diff --git a/Doc/includes/sqlite3/executescript.py b/Doc/includes/sqlite3/executescript.py index 7e535817..aea8943f 100644 --- a/Doc/includes/sqlite3/executescript.py +++ b/Doc/includes/sqlite3/executescript.py @@ -22,3 +22,4 @@ cur.executescript(""" 1987 ); """) +con.close() diff --git a/Doc/includes/sqlite3/insert_more_people.py b/Doc/includes/sqlite3/insert_more_people.py index edbc79e7..10cf9372 100644 --- a/Doc/includes/sqlite3/insert_more_people.py +++ b/Doc/includes/sqlite3/insert_more_people.py @@ -14,3 +14,5 @@ for person in newPeople: # The changes will not be saved unless the transaction is committed explicitly: con.commit() + +con.close() diff --git a/Doc/includes/sqlite3/load_extension.py b/Doc/includes/sqlite3/load_extension.py index b997c706..624cfe26 100644 --- a/Doc/includes/sqlite3/load_extension.py +++ b/Doc/includes/sqlite3/load_extension.py @@ -24,3 +24,5 @@ con.executescript(""" """) for row in con.execute("select rowid, name, ingredients from recipe where name match 'pie'"): print(row) + +con.close() diff --git a/Doc/includes/sqlite3/md5func.py b/Doc/includes/sqlite3/md5func.py index 0056b2d6..16dc348b 100644 --- a/Doc/includes/sqlite3/md5func.py +++ b/Doc/includes/sqlite3/md5func.py @@ -9,3 +9,5 @@ con.create_function("md5", 1, md5sum) cur = con.cursor() cur.execute("select md5(?)", (b"foo",)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/mysumaggr.py b/Doc/includes/sqlite3/mysumaggr.py index d2dfd2c0..11f96395 100644 --- a/Doc/includes/sqlite3/mysumaggr.py +++ b/Doc/includes/sqlite3/mysumaggr.py @@ -18,3 +18,5 @@ cur.execute("insert into test(i) values (1)") cur.execute("insert into test(i) values (2)") cur.execute("select mysum(i) from test") print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/parse_colnames.py b/Doc/includes/sqlite3/parse_colnames.py index cc68c764..5f01dbfe 100644 --- a/Doc/includes/sqlite3/parse_colnames.py +++ b/Doc/includes/sqlite3/parse_colnames.py @@ -6,3 +6,5 @@ cur = con.cursor() cur.execute('select ? as "x [timestamp]"', (datetime.datetime.now(),)) dt = cur.fetchone()[0] print(dt, type(dt)) + +con.close() diff --git a/Doc/includes/sqlite3/pysqlite_datetime.py b/Doc/includes/sqlite3/pysqlite_datetime.py index 68d49358..5d843f90 100644 --- a/Doc/includes/sqlite3/pysqlite_datetime.py +++ b/Doc/includes/sqlite3/pysqlite_datetime.py @@ -18,3 +18,5 @@ cur.execute('select current_date as "d [date]", current_timestamp as "ts [timest row = cur.fetchone() print("current_date", row[0], type(row[0])) print("current_timestamp", row[1], type(row[1])) + +con.close() diff --git a/Doc/includes/sqlite3/row_factory.py b/Doc/includes/sqlite3/row_factory.py index e436ffc6..9de6e7b1 100644 --- a/Doc/includes/sqlite3/row_factory.py +++ b/Doc/includes/sqlite3/row_factory.py @@ -11,3 +11,5 @@ con.row_factory = dict_factory cur = con.cursor() cur.execute("select 1 as a") print(cur.fetchone()["a"]) + +con.close() diff --git a/Doc/includes/sqlite3/rowclass.py b/Doc/includes/sqlite3/rowclass.py index 92b5ad60..fc602870 100644 --- a/Doc/includes/sqlite3/rowclass.py +++ b/Doc/includes/sqlite3/rowclass.py @@ -10,3 +10,5 @@ for row in cur: assert row["name"] == row["nAmE"] assert row[1] == row["age"] assert row[1] == row["AgE"] + +con.close() diff --git a/Doc/includes/sqlite3/shortcut_methods.py b/Doc/includes/sqlite3/shortcut_methods.py index 71600d4f..98a39411 100644 --- a/Doc/includes/sqlite3/shortcut_methods.py +++ b/Doc/includes/sqlite3/shortcut_methods.py @@ -18,3 +18,7 @@ for row in con.execute("select firstname, lastname from person"): print(row) print("I just deleted", con.execute("delete from person").rowcount, "rows") + +# close is not a shortcut method and it's not called automatically, +# so the connection object should be closed manually +con.close() diff --git a/Doc/includes/sqlite3/simple_tableprinter.py b/Doc/includes/sqlite3/simple_tableprinter.py index 231d8726..148a1707 100644 --- a/Doc/includes/sqlite3/simple_tableprinter.py +++ b/Doc/includes/sqlite3/simple_tableprinter.py @@ -24,3 +24,5 @@ for row in cur: print(fieldValue.ljust(FIELD_MAX_WIDTH), end=' ') print() # Finish the row with a newline. + +con.close() diff --git a/Doc/includes/sqlite3/text_factory.py b/Doc/includes/sqlite3/text_factory.py index 5f96cdb5..a857a155 100644 --- a/Doc/includes/sqlite3/text_factory.py +++ b/Doc/includes/sqlite3/text_factory.py @@ -25,3 +25,5 @@ con.text_factory = lambda x: x.decode("utf-8") + "foo" cur.execute("select ?", ("bar",)) row = cur.fetchone() assert row[0] == "barfoo" + +con.close() diff --git a/Doc/includes/sublist.c b/Doc/includes/sublist.c index 376dddfa..76ff9394 100644 --- a/Doc/includes/sublist.c +++ b/Doc/includes/sublist.c @@ -1,3 +1,4 @@ +#define PY_SSIZE_T_CLEAN #include typedef struct { diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index acffabf2..d1c28f47 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -53,8 +53,12 @@ This module defines the following constants and functions: .. function:: interrupt_main() - Raise a :exc:`KeyboardInterrupt` exception in the main thread. A subthread can - use this function to interrupt the main thread. + Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main + thread. A thread can use this function to interrupt the main thread. + + If :data:`signal.SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. .. function:: exit() diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index d59cf055..7e1d571e 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -281,9 +281,9 @@ clocks to track time. the event loop's internal monotonic clock. .. note:: - - Timeouts (relative *delay* or absolute *when*) should not - exceed one day. + .. versionchanged:: 3.8 + In Python 3.7 and earlier timeouts (relative *delay* or absolute *when*) + should not exceed one day. This has been fixed in Python 3.8. .. seealso:: @@ -1605,7 +1605,7 @@ using the :meth:`loop.add_signal_handler` method:: import os import signal - def ask_exit(signame): + def ask_exit(signame, loop): print("got signal %s: exit" % signame) loop.stop() @@ -1615,7 +1615,7 @@ using the :meth:`loop.add_signal_handler` method:: for signame in {'SIGINT', 'SIGTERM'}: loop.add_signal_handler( getattr(signal, signame), - functools.partial(ask_exit, signame)) + functools.partial(ask_exit, signame, loop)) await asyncio.sleep(3600) diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index f54dece0..d8d7947c 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -73,7 +73,7 @@ Transports are classes provided by :mod:`asyncio` in order to abstract various kinds of communication channels. Transport objects are always instantiated by an -ref:`asyncio event loop `. +:ref:`asyncio event loop `. asyncio implements transports for TCP, UDP, SSL, and subprocess pipes. The methods available on a transport depend on the transport's kind. diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 0bcf6617..af7e36ed 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -56,7 +56,7 @@ See also the `Examples`_ subsection. Creating Subprocesses ===================== -.. coroutinefunction:: create_subprocess_exec(\*args, stdin=None, \ +.. coroutinefunction:: create_subprocess_exec(program, \*args, stdin=None, \ stdout=None, stderr=None, loop=None, \ limit=None, \*\*kwds) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 18b56297..993bd13f 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -17,7 +17,7 @@ those of the :mod:`threading` module with two important caveats: argument; use the :func:`asyncio.wait_for` function to perform operations with timeouts. -asyncio has the following basic sychronization primitives: +asyncio has the following basic synchronization primitives: * :class:`Lock` * :class:`Event` @@ -66,6 +66,13 @@ Lock This method waits until the lock is *unlocked*, sets it to *locked* and returns ``True``. + When more than one coroutine is blocked in :meth:`acquire` + waiting for the lock to be unlocked, only one coroutine + eventually proceeds. + + Acquiring a lock is *fair*: the coroutine that proceeds will be + the first coroutine that started waiting on the lock. + .. method:: release() Release the lock. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 9e685b17..29ccafee 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -40,7 +40,7 @@ be executed:: >>> main() -To actually run a coroutine asyncio provides three main mechanisms: +To actually run a coroutine, asyncio provides three main mechanisms: * The :func:`asyncio.run` function to run the top-level entry point "main()" function (see the above example.) diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index d5f62251..b247de6d 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -83,7 +83,7 @@ All of the classes in this module may safely be accessed from multiple threads. The *buffering* argument is ignored. Its use is deprecated. - If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be a number between + If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be an integer between ``1`` and ``9`` specifying the level of compression: ``1`` produces the least compression, and ``9`` (default) produces the most compression. @@ -144,7 +144,7 @@ Incremental (de)compression incrementally. For one-shot compression, use the :func:`compress` function instead. - *compresslevel*, if given, must be a number between ``1`` and ``9``. The + *compresslevel*, if given, must be an integer between ``1`` and ``9``. The default is ``9``. .. method:: compress(data) @@ -230,9 +230,9 @@ One-shot (de)compression .. function:: compress(data, compresslevel=9) - Compress *data*. + Compress *data*, a :term:`bytes-like object `. - *compresslevel*, if given, must be a number between ``1`` and ``9``. The + *compresslevel*, if given, must be an integer between ``1`` and ``9``. The default is ``9``. For incremental compression, use a :class:`BZ2Compressor` instead. @@ -240,7 +240,7 @@ One-shot (de)compression .. function:: decompress(data) - Decompress *data*. + Decompress *data*, a :term:`bytes-like object `. If *data* is the concatenation of multiple compressed streams, decompress all of the streams. @@ -250,3 +250,77 @@ One-shot (de)compression .. versionchanged:: 3.3 Support for multi-stream inputs was added. +.. _bz2-usage-examples: + +Examples of usage +----------------- + +Below are some examples of typical usage of the :mod:`bz2` module. + +Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression: + + >>> import bz2 + + >>> data = b"""\ + ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue + ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, + ... sit amet cursus ante. In interdum laoreet mi, sit amet ultrices purus + ... pulvinar a. Nam gravida euismod magna, non varius justo tincidunt feugiat. + ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo + ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum + ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" + + >>> c = bz2.compress(data) + >>> len(data) / len(c) # Data compression ratio + 1.513595166163142 + + >>> d = bz2.decompress(c) + >>> data == d # Check equality to original object after round-trip + True + +Using :class:`BZ2Compressor` for incremental compression: + + >>> import bz2 + + >>> def gen_data(chunks=10, chunksize=1000): + ... """Yield incremental blocks of chunksize bytes.""" + ... for _ in range(chunks): + ... yield b"z" * chunksize + ... + >>> comp = bz2.BZ2Compressor() + >>> out = b"" + >>> for chunk in gen_data(): + ... # Provide data to the compressor object + ... out = out + comp.compress(chunk) + ... + >>> # Finish the compression process. Call this once you have + >>> # finished providing data to the compressor. + >>> out = out + comp.flush() + +The example above uses a very "nonrandom" stream of data +(a stream of `b"z"` chunks). Random data tends to compress poorly, +while ordered, repetitive data usually yields a high compression ratio. + +Writing and reading a bzip2-compressed file in binary mode: + + >>> import bz2 + + >>> data = b"""\ + ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue + ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, + ... sit amet cursus ante. In interdum laoreet mi, sit amet ultrices purus + ... pulvinar a. Nam gravida euismod magna, non varius justo tincidunt feugiat. + ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo + ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum + ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" + + >>> with bz2.open("myfile.bz2", "wb") as f: + ... # Write compressed data to file + ... unused = f.write(data) + + >>> with bz2.open("myfile.bz2", "rb") as f: + ... # Decompress data from file + ... content = f.read() + + >>> content == data # Check equality to original object after round-trip + True diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 9d81730f..28cd96b0 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -6,13 +6,12 @@ -------------- -This module is always available. It provides access to mathematical functions -for complex numbers. The functions in this module accept integers, -floating-point numbers or complex numbers as arguments. They will also accept -any Python object that has either a :meth:`__complex__` or a :meth:`__float__` -method: these methods are used to convert the object to a complex or -floating-point number, respectively, and the function is then applied to the -result of the conversion. +This module provides access to mathematical functions for complex numbers. The +functions in this module accept integers, floating-point numbers or complex +numbers as arguments. They will also accept any Python object that has either a +:meth:`__complex__` or a :meth:`__float__` method: these methods are used to +convert the object to a complex or floating-point number, respectively, and +the function is then applied to the result of the conversion. .. note:: diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index ef788bf2..ff4f4933 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -638,7 +638,7 @@ define in order to be compatible with the Python codec registry. .. method:: setstate(state) - Set the state of the encoder to *state*. *state* must be a decoder state + Set the state of the decoder to *state*. *state* must be a decoder state returned by :meth:`getstate`. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 0413f469..9f47e897 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -941,14 +941,14 @@ field names, the method and attribute names start with an underscore. >>> Pixel(11, 22, 128, 255, 0) Pixel(x=11, y=22, red=128, green=255, blue=0) -.. attribute:: somenamedtuple._fields_defaults +.. attribute:: somenamedtuple._field_defaults Dictionary mapping field names to default values. .. doctest:: >>> Account = namedtuple('Account', ['type', 'balance'], defaults=[0]) - >>> Account._fields_defaults + >>> Account._field_defaults {'balance': 0} >>> Account('premium') Account(type='premium', balance=0) @@ -959,7 +959,7 @@ function: >>> getattr(p, 'x') 11 -To convert a dictionary to a named tuple, use the double-star-operator +To convert a dictionary to a named tuple, use the ``**`` operator (as described in :ref:`tut-unpacking-arguments`): >>> d = {'x': 11, 'y': 22} @@ -1016,17 +1016,20 @@ customize a prototype instance: .. seealso:: - * `Recipe for named tuple abstract base class with a metaclass mix-in - `_ - by Jan Kaliszewski. Besides providing an :term:`abstract base class` for - named tuples, it also supports an alternate :term:`metaclass`-based - constructor that is convenient for use cases where named tuples are being - subclassed. + * See :class:`typing.NamedTuple` for a way to add type hints for named + tuples. It also provides an elegant notation using the :keyword:`class` + keyword:: + + class Component(NamedTuple): + part_number: int + weight: float + description: Optional[str] = None * See :meth:`types.SimpleNamespace` for a mutable namespace based on an underlying dictionary instead of a tuple. - * See :meth:`typing.NamedTuple` for a way to add type hints for named tuples. + * The :mod:`dataclasses` module provides a decorator and functions for + automatically adding generated special methods to user-defined classes. :class:`OrderedDict` objects diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index a5749154..9295df84 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -216,6 +216,10 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. given, it will default to the number of processors on the machine. If *max_workers* is lower or equal to ``0``, then a :exc:`ValueError` will be raised. + On Windows, *max_workers* must be equal or lower than ``61``. If it is not + then :exc:`ValueError` will be raised. If *max_workers* is ``None``, then + the default chosen will be at most ``61``, even if more processors are + available. *mp_context* can be a multiprocessing context or None. It will be used to launch the workers. If *mp_context* is ``None`` or not given, the default multiprocessing context is used. @@ -289,9 +293,10 @@ The :class:`Future` class encapsulates the asynchronous execution of a callable. .. method:: cancel() - Attempt to cancel the call. If the call is currently being executed and - cannot be cancelled then the method will return ``False``, otherwise the - call will be cancelled and the method will return ``True``. + Attempt to cancel the call. If the call is currently being executed or + finished running and cannot be cancelled then the method will return + ``False``, otherwise the call will be cancelled and the method will + return ``True``. .. method:: cancelled() @@ -397,8 +402,9 @@ Module Functions Wait for the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* to complete. Returns a named 2-tuple of sets. The first set, named ``done``, contains the futures that - completed (finished or were cancelled) before the wait completed. The second - set, named ``not_done``, contains uncompleted futures. + completed (finished or cancelled futures) before the wait completed. The + second set, named ``not_done``, contains the futures that did not complete + (pending or running futures). *timeout* can be used to control the maximum number of seconds to wait before returning. *timeout* can be an int or float. If *timeout* is not specified @@ -429,7 +435,7 @@ Module Functions Returns an iterator over the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* that yields futures as - they complete (finished or were cancelled). Any futures given by *fs* that + they complete (finished or cancelled futures). Any futures given by *fs* that are duplicated will be returned once. Any futures that completed before :func:`as_completed` is called will be yielded first. The returned iterator raises a :exc:`concurrent.futures.TimeoutError` if :meth:`~iterator.__next__` diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 95cc3520..68b663ff 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -723,6 +723,12 @@ be overridden by subclasses or by attribute assignment. >>> list(custom['Section2'].keys()) ['AnotherKey'] + .. note:: + The optionxform function transforms option names to a canonical form. + This should be an idempotent function: if the name is already in + canonical form, it should be returned unchanged. + + .. attribute:: ConfigParser.SECTCRE A compiled regular expression used to parse section headers. The default diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 500aad88..60a2ec1c 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2365,7 +2365,7 @@ other data types containing pointer type fields. and so on). Later assignments to the :attr:`_fields_` class variable will raise an AttributeError. - It is possible to defined sub-subclasses of structure types, they inherit + It is possible to define sub-subclasses of structure types, they inherit the fields of the base class plus the :attr:`_fields_` defined in the sub-subclass, if any. @@ -2413,7 +2413,7 @@ other data types containing pointer type fields. td.lptdesc = POINTER(some_type) td.u.lptdesc = POINTER(some_type) - It is possible to defined sub-subclasses of structures, they inherit the + It is possible to define sub-subclasses of structures, they inherit the fields of the base class. If the subclass definition has a separate :attr:`_fields_` variable, the fields specified in this are appended to the fields of the base class. diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 121f73bb..270b2386 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -254,8 +254,9 @@ Supported operations: | | rounded to the nearest multiple of | | | timedelta.resolution using round-half-to-even.| +--------------------------------+-----------------------------------------------+ -| ``f = t2 / t3`` | Division (3) of *t2* by *t3*. Returns a | -| | :class:`float` object. | +| ``f = t2 / t3`` | Division (3) of overall duration *t2* by | +| | interval unit *t3*. Returns a :class:`float` | +| | object. | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 / f or t1 = t2 / i`` | Delta divided by a float or an int. The result| | | is rounded to the nearest multiple of | @@ -351,7 +352,8 @@ Instance methods: .. method:: timedelta.total_seconds() Return the total number of seconds contained in the duration. Equivalent to - ``td / timedelta(seconds=1)``. + ``td / timedelta(seconds=1)``. For interval units other than seconds, use the + division form directly (e.g. ``td / timedelta(microseconds=1)``). Note that for very large time intervals (greater than 270 years on most platforms) this method will lose microsecond accuracy. @@ -1346,56 +1348,64 @@ Examples of working with datetime objects: Using datetime with tzinfo: - >>> from datetime import timedelta, datetime, tzinfo - >>> class GMT1(tzinfo): + >>> from datetime import timedelta, datetime, tzinfo, timezone + >>> class KabulTz(tzinfo): + ... # Kabul used +4 until 1945, when they moved to +4:30 + ... UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc) ... def utcoffset(self, dt): - ... return timedelta(hours=1) + self.dst(dt) - ... def dst(self, dt): - ... # DST starts last Sunday in March - ... d = datetime(dt.year, 4, 1) # ends last Sunday in October - ... self.dston = d - timedelta(days=d.weekday() + 1) - ... d = datetime(dt.year, 11, 1) - ... self.dstoff = d - timedelta(days=d.weekday() + 1) - ... if self.dston <= dt.replace(tzinfo=None) < self.dstoff: - ... return timedelta(hours=1) + ... if dt.year < 1945: + ... return timedelta(hours=4) + ... elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30): + ... # If dt falls in the imaginary range, use fold to decide how + ... # to resolve. See PEP495 + ... return timedelta(hours=4, minutes=(30 if dt.fold else 0)) ... else: - ... return timedelta(0) - ... def tzname(self,dt): - ... return "GMT +1" + ... return timedelta(hours=4, minutes=30) + ... + ... def fromutc(self, dt): + ... # A custom implementation is required for fromutc as + ... # the input to this function is a datetime with utc values + ... # but with a tzinfo set to self + ... # See datetime.astimezone or fromtimestamp + ... + ... # Follow same validations as in datetime.tzinfo + ... if not isinstance(dt, datetime): + ... raise TypeError("fromutc() requires a datetime argument") + ... if dt.tzinfo is not self: + ... raise ValueError("dt.tzinfo is not self") + ... + ... if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE: + ... return dt + timedelta(hours=4, minutes=30) + ... else: + ... return dt + timedelta(hours=4) ... - >>> class GMT2(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=2) + self.dst(dt) ... def dst(self, dt): - ... d = datetime(dt.year, 4, 1) - ... self.dston = d - timedelta(days=d.weekday() + 1) - ... d = datetime(dt.year, 11, 1) - ... self.dstoff = d - timedelta(days=d.weekday() + 1) - ... if self.dston <= dt.replace(tzinfo=None) < self.dstoff: - ... return timedelta(hours=1) + ... return timedelta(0) + ... + ... def tzname(self, dt): + ... if dt >= self.UTC_MOVE_DATE: + ... return "+04:30" ... else: - ... return timedelta(0) - ... def tzname(self,dt): - ... return "GMT +2" + ... return "+04" ... - >>> gmt1 = GMT1() - >>> # Daylight Saving Time - >>> dt1 = datetime(2006, 11, 21, 16, 30, tzinfo=gmt1) - >>> dt1.dst() - datetime.timedelta(0) - >>> dt1.utcoffset() - datetime.timedelta(seconds=3600) - >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=gmt1) - >>> dt2.dst() - datetime.timedelta(seconds=3600) - >>> dt2.utcoffset() - datetime.timedelta(seconds=7200) + ... def __repr__(self): + ... return f"{self.__class__.__name__}()" + ... + >>> tz1 = KabulTz() + >>> # Datetime before the change + >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1) + >>> print(dt1.utcoffset()) + 4:00:00 + >>> # Datetime after the change + >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1) + >>> print(dt2.utcoffset()) + 4:30:00 >>> # Convert datetime to another time zone - >>> dt3 = dt2.astimezone(GMT2()) - >>> dt3 # doctest: +ELLIPSIS - datetime.datetime(2006, 6, 14, 14, 0, tzinfo=) - >>> dt2 # doctest: +ELLIPSIS - datetime.datetime(2006, 6, 14, 13, 0, tzinfo=) + >>> dt3 = dt2.astimezone(timezone.utc) + >>> dt3 + datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc) + >>> dt2 + datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz()) >>> dt2.utctimetuple() == dt3.utctimetuple() True @@ -1637,26 +1647,27 @@ Instance methods: Example: >>> from datetime import time, tzinfo, timedelta - >>> class GMT1(tzinfo): + >>> class TZ1(tzinfo): ... def utcoffset(self, dt): ... return timedelta(hours=1) ... def dst(self, dt): ... return timedelta(0) ... def tzname(self,dt): - ... return "Europe/Prague" + ... return "+01:00" + ... def __repr__(self): + ... return f"{self.__class__.__name__}()" ... - >>> t = time(12, 10, 30, tzinfo=GMT1()) - >>> t # doctest: +ELLIPSIS - datetime.time(12, 10, 30, tzinfo=) - >>> gmt = GMT1() + >>> t = time(12, 10, 30, tzinfo=TZ1()) + >>> t + datetime.time(12, 10, 30, tzinfo=TZ1()) >>> t.isoformat() '12:10:30+01:00' >>> t.dst() datetime.timedelta(0) >>> t.tzname() - 'Europe/Prague' + '+01:00' >>> t.strftime("%H:%M:%S %Z") - '12:10:30 Europe/Prague' + '12:10:30 +01:00' >>> 'The {} is {:%H:%M}.'.format("time", t) 'The time is 12:10.' @@ -2029,6 +2040,9 @@ For :class:`date` objects, the format codes for hours, minutes, seconds, and microseconds should not be used, as :class:`date` objects have no such values. If they're used anyway, ``0`` is substituted for them. +For the :meth:`datetime.strptime` class method, the default value is ``1900-01-01T00:00:00.000``: +any components not specified in the format string will be pulled from the default value. [#]_ + The full set of format codes supported varies across platforms, because Python calls the platform C library's :func:`strftime` function, and platform variations are common. To see the full set of format codes supported on your @@ -2062,7 +2076,7 @@ format codes. | | where 0 is Sunday and 6 is | | | | | Saturday. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%d`` | Day of the month as a | 01, 02, ..., 31 | | +| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | @@ -2075,29 +2089,29 @@ format codes. | | || Januar, Februar, ..., | | | | | Dezember (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | | +| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%y`` | Year without century as a | 00, 01, ..., 99 | | +| ``%y`` | Year without century as a | 00, 01, ..., 99 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | | | number. | 2014, ..., 9998, 9999 | | +-----------+--------------------------------+------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | | +| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | | +| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | | | AM or PM. || am, pm (de_DE) | \(3) | +-----------+--------------------------------+------------------------+-------+ -| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | | +| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4) | -| | decimal number. | | | +| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4), | +| | decimal number. | | \(9) | +-----------+--------------------------------+------------------------+-------+ | ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | | | number, zero-padded on the | 999999 | | @@ -2111,19 +2125,19 @@ format codes. | ``%Z`` | Time zone name (empty string | (empty), UTC, EST, CST | | | | if the object is naive). | | | +-----------+--------------------------------+------------------------+-------+ -| ``%j`` | Day of the year as a | 001, 002, ..., 366 | | +| ``%j`` | Day of the year as a | 001, 002, ..., 366 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Sunday as the first day of | | | +| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| | (Sunday as the first day of | | \(9) | | | the week) as a zero padded | | | | | decimal number. All days in a | | | | | new year preceding the first | | | | | Sunday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Monday as the first day of | | | +| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| | (Monday as the first day of | | \(9) | | | the week) as a decimal number. | | | | | All days in a new year | | | | | preceding the first Monday | | | @@ -2163,8 +2177,8 @@ incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. | ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | | | number where 1 is Monday. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8) | -| | number with Monday as | | | +| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8), | +| | number with Monday as | | \(9) | | | the first day of the week. | | | | | Week 01 is the week containing | | | | | Jan 4. | | | @@ -2260,6 +2274,12 @@ Notes: :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not interchangeable. +(9) + When used with the :meth:`strptime` method, the leading zero is optional + for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%J``, ``%U``, + ``%W``, and ``%V``. Format ``%y`` does require a leading zero. + .. rubric:: Footnotes .. [#] If, that is, we ignore the effects of Relativity +.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since ``1900`` is not a leap year. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 6bbef384..5e6f002d 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1059,9 +1059,13 @@ All of the following opcodes use their arguments. .. opcode:: RAISE_VARARGS (argc) - Raises an exception. *argc* indicates the number of arguments to the raise - statement, ranging from 0 to 3. The handler will find the traceback as TOS2, - the parameter as TOS1, and the exception as TOS. + Raises an exception using one of the 3 forms of the ``raise`` statement, + depending on the value of *argc*: + + * 0: ``raise`` (re-raise previous exception) + * 1: ``raise TOS`` (raise exception instance or type at ``TOS``) + * 2: ``raise TOS1 from TOS`` (raise exception instance or type at ``TOS1`` + with ``__cause__`` set to ``TOS``) .. opcode:: CALL_FUNCTION (argc) @@ -1161,10 +1165,10 @@ All of the following opcodes use their arguments. .. opcode:: EXTENDED_ARG (ext) - Prefixes any opcode which has an argument too big to fit into the default two - bytes. *ext* holds two additional bytes which, taken together with the - subsequent opcode's argument, comprise a four-byte argument, *ext* being the - two most-significant bytes. + Prefixes any opcode which has an argument too big to fit into the default one + byte. *ext* holds an additional byte which act as higher bits in the argument. + For each opcode, at most three prefixal ``EXTENDED_ARG`` are allowed, forming + an argument from two-byte to four-byte. .. opcode:: FORMAT_VALUE (flags) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index a138e687..e7c0033e 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -771,23 +771,27 @@ Warnings :mod:`doctest` is serious about requiring exact matches in expected output. If even a single character doesn't match, the test fails. This will probably surprise you a few times, as you learn exactly what Python does and doesn't -guarantee about output. For example, when printing a dict, Python doesn't -guarantee that the key-value pairs will be printed in any particular order, so a -test like :: +guarantee about output. For example, when printing a set, Python doesn't +guarantee that the element is printed in any particular order, so a test like :: >>> foo() - {"Hermione": "hippogryph", "Harry": "broomstick"} + {"Hermione", "Harry"} is vulnerable! One workaround is to do :: - >>> foo() == {"Hermione": "hippogryph", "Harry": "broomstick"} + >>> foo() == {"Hermione", "Harry"} True instead. Another is to do :: - >>> d = sorted(foo().items()) + >>> d = sorted(foo()) >>> d - [('Harry', 'broomstick'), ('Hermione', 'hippogryph')] + ['Harry', 'Hermione'] + +.. note:: + + Before Python 3.6, when printing a dict, Python did not guarantee that + the key-value pairs was printed in any particular order. There are others, but you get the idea. diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index f02b35b9..09ea64a5 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -6,6 +6,7 @@ .. module:: email.message :synopsis: The base class representing email messages in a fashion backward compatible with Python 3.2 + :noindex: The :class:`Message` class is very similar to the diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index e24ac7b9..debd1c85 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -12,6 +12,11 @@ This module is part of the legacy (``Compat32``) email API. In the new API the functionality is provided by the *cte* parameter of the :meth:`~email.message.EmailMessage.set_content` method. +This module is deprecated in Python 3. The functions provided here +should not be called explicitly since the :class:`~email.mime.text.MIMEText` +class sets the content type and CTE header using the *_subtype* and *_charset* +values passed during the instaniation of that class. + The remaining text in this section is the original documentation of the module. When creating :class:`~email.message.Message` objects from scratch, you often diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index 2575a513..c825aa1a 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -184,7 +184,7 @@ to be using :class:`BytesGenerator`, and not :class:`Generator`. (This is required because strings cannot represent non-ASCII bytes.) Convert any bytes with the high bit set as needed using an ASCII-compatible :mailheader:`Content-Transfer-Encoding`. That is, - transform parts with non-ASCII :mailheader:`Cotnent-Transfer-Encoding` + transform parts with non-ASCII :mailheader:`Content-Transfer-Encoding` (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatible :mailheader:`Content-Transfer-Encoding`, and encode RFC-invalid non-ASCII bytes in headers using the MIME ``unknown-8bit`` character set, thus diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 57ed2914..52a505e0 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -691,8 +691,13 @@ The following exceptions are used as warning categories; see the .. exception:: PendingDeprecationWarning - Base class for warnings about features which will be deprecated in the - future. + Base class for warnings about features which are obsolete and + expected to be deprecated in the future, but are not deprecated + at the moment. + + This class is rarely used as emitting a warning about a possible + upcoming deprecation is unusual, and :exc:`DeprecationWarning` + is preferred for already active deprecations. .. exception:: SyntaxWarning diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 1fc11ffc..bf81749f 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -23,8 +23,9 @@ The typical use is:: This iterates over the lines of all files listed in ``sys.argv[1:]``, defaulting to ``sys.stdin`` if the list is empty. If a filename is ``'-'``, it is also -replaced by ``sys.stdin``. To specify an alternative list of filenames, pass it -as the first argument to :func:`.input`. A single file name is also allowed. +replaced by ``sys.stdin`` and the optional arguments *mode* and *openhook* +are ignored. To specify an alternative list of filenames, pass it as the +first argument to :func:`.input`. A single file name is also allowed. All files are opened in text mode by default, but you can override this by specifying the *mode* parameter in the call to :func:`.input` or diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 9326b8d0..11147525 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -210,19 +210,18 @@ are always available. They are listed here in alphabetical order. @classmethod def f(cls, arg1, arg2, ...): ... - The ``@classmethod`` form is a function :term:`decorator` -- see the description - of function definitions in :ref:`function` for details. + The ``@classmethod`` form is a function :term:`decorator` -- see + :ref:`function` for details. - It can be called either on the class (such as ``C.f()``) or on an instance (such + A class method can be called either on the class (such as ``C.f()``) or on an instance (such as ``C().f()``). The instance is ignored except for its class. If a class method is called for a derived class, the derived class object is passed as the implied first argument. Class methods are different than C++ or Java static methods. If you want those, - see :func:`staticmethod` in this section. + see :func:`staticmethod`. - For more information on class methods, consult the documentation on the standard - type hierarchy in :ref:`types`. + For more information on class methods, see :ref:`types`. .. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) @@ -843,7 +842,8 @@ are always available. They are listed here in alphabetical order. Update and return a dictionary representing the current local symbol table. Free variables are returned by :func:`locals` when it is called in function - blocks, but not in class blocks. + blocks, but not in class blocks. Note that at the module level, :func:`locals` + and :func:`globals` are the same dictionary. .. note:: The contents of this dictionary should not be modified; changes may not @@ -1445,11 +1445,11 @@ are always available. They are listed here in alphabetical order. @staticmethod def f(arg1, arg2, ...): ... - The ``@staticmethod`` form is a function :term:`decorator` -- see the - description of function definitions in :ref:`function` for details. + The ``@staticmethod`` form is a function :term:`decorator` -- see + :ref:`function` for details. - It can be called either on the class (such as ``C.f()``) or on an instance (such - as ``C().f()``). The instance is ignored except for its class. + A static method can be called either on the class (such as ``C.f()``) or on an instance (such + as ``C().f()``). Static methods in Python are similar to those found in Java or C++. Also see :func:`classmethod` for a variant that is useful for creating alternate class @@ -1464,8 +1464,7 @@ are always available. They are listed here in alphabetical order. class C: builtin_open = staticmethod(open) - For more information on static methods, consult the documentation on the - standard type hierarchy in :ref:`types`. + For more information on static methods, see :ref:`types`. .. index:: diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index c98c5e7d..94ed340a 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -13,7 +13,7 @@ The :mod:`gettext` module provides internationalization (I18N) and localization (L10N) services for your Python modules and applications. It supports both the -GNU ``gettext`` message catalog API and a higher level, class-based API that may +GNU :program:`gettext` message catalog API and a higher level, class-based API that may be more appropriate for Python files. The interface described below allows you to write your module and application messages in one natural language, and provide a catalog of translated messages for running under different natural @@ -38,7 +38,7 @@ class-based API instead. Bind the *domain* to the locale directory *localedir*. More concretely, :mod:`gettext` will look for binary :file:`.mo` files for the given domain using - the path (on Unix): :file:`localedir/language/LC_MESSAGES/domain.mo`, where + the path (on Unix): :file:`{localedir}/{language}/LC_MESSAGES/{domain}.mo`, where *languages* is searched for in the environment variables :envvar:`LANGUAGE`, :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and :envvar:`LANG` respectively. @@ -136,17 +136,16 @@ Class-based API The class-based API of the :mod:`gettext` module gives you more flexibility and greater convenience than the GNU :program:`gettext` API. It is the recommended way of localizing your Python applications and modules. :mod:`!gettext` defines -a "translations" class which implements the parsing of GNU :file:`.mo` format -files, and has methods for returning strings. Instances of this "translations" -class can also install themselves in the built-in namespace as the function -:func:`_`. +a :class:`GNUTranslations` class which implements the parsing of GNU :file:`.mo` format +files, and has methods for returning strings. Instances of this class can also +install themselves in the built-in namespace as the function :func:`_`. .. function:: find(domain, localedir=None, languages=None, all=False) This function implements the standard :file:`.mo` file search algorithm. It takes a *domain*, identical to what :func:`textdomain` takes. Optional - *localedir* is as in :func:`bindtextdomain` Optional *languages* is a list of + *localedir* is as in :func:`bindtextdomain`. Optional *languages* is a list of strings, where each string is a language code. If *localedir* is not given, then the default system locale directory is used. @@ -170,10 +169,10 @@ class can also install themselves in the built-in namespace as the function .. function:: translation(domain, localedir=None, languages=None, class_=None, fallback=False, codeset=None) - Return a :class:`Translations` instance based on the *domain*, *localedir*, + Return a :class:`*Translations` instance based on the *domain*, *localedir*, and *languages*, which are first passed to :func:`find` to get a list of the associated :file:`.mo` file paths. Instances with identical :file:`.mo` file - names are cached. The actual class instantiated is either *class_* if + names are cached. The actual class instantiated is *class_* if provided, otherwise :class:`GNUTranslations`. The class's constructor must take a single :term:`file object` argument. If provided, *codeset* will change the charset used to encode translated strings in the @@ -233,7 +232,7 @@ are the methods of :class:`!NullTranslations`: .. method:: _parse(fp) - No-op'd in the base class, this method takes file object *fp*, and reads + No-op in the base class, this method takes file object *fp*, and reads the data from the file, initializing its message catalog. If you have an unsupported message catalog file format, you should override this method to parse your format. @@ -275,7 +274,8 @@ are the methods of :class:`!NullTranslations`: .. method:: info() - Return the "protected" :attr:`_info` variable. + Return the "protected" :attr:`_info` variable, a dictionary containing + the metadata found in the message catalog file. .. method:: charset() @@ -326,15 +326,15 @@ The :mod:`gettext` module provides one additional class derived from :meth:`_parse` to enable reading GNU :program:`gettext` format :file:`.mo` files in both big-endian and little-endian format. -:class:`GNUTranslations` parses optional meta-data out of the translation -catalog. It is convention with GNU :program:`gettext` to include meta-data as -the translation for the empty string. This meta-data is in :rfc:`822`\ -style +:class:`GNUTranslations` parses optional metadata out of the translation +catalog. It is convention with GNU :program:`gettext` to include metadata as +the translation for the empty string. This metadata is in :rfc:`822`\ -style ``key: value`` pairs, and should contain the ``Project-Id-Version`` key. If the key ``Content-Type`` is found, then the ``charset`` property is used to initialize the "protected" :attr:`_charset` instance variable, defaulting to ``None`` if not found. If the charset encoding is specified, then all message ids and message strings read from the catalog are converted to Unicode using -this encoding, else ASCII encoding is assumed. +this encoding, else ASCII is assumed. Since message ids are read as Unicode strings too, all :meth:`*gettext` methods will assume message ids as Unicode strings, not byte strings. @@ -436,7 +436,7 @@ take the following steps: #. run a suite of tools over your marked files to generate raw messages catalogs -#. create language specific translations of the message catalogs +#. create language-specific translations of the message catalogs #. use the :mod:`gettext` module so that message strings are properly translated @@ -446,9 +446,8 @@ it in ``_('...')`` --- that is, a call to the function :func:`_`. For example:: filename = 'mylog.txt' message = _('writing a log message') - fp = open(filename, 'w') - fp.write(message) - fp.close() + with open(filename, 'w') as fp: + fp.write(message) In this example, the string ``'writing a log message'`` is marked as a candidate for translation, while the strings ``'mylog.txt'`` and ``'w'`` are not. @@ -499,7 +498,7 @@ Localizing your module ^^^^^^^^^^^^^^^^^^^^^^ If you are localizing your module, you must take care not to make global -changes, e.g. to the built-in namespace. You should not use the GNU ``gettext`` +changes, e.g. to the built-in namespace. You should not use the GNU :program:`gettext` API but instead the class-based API. Let's say your module is called "spam" and the module's various natural language @@ -653,8 +652,9 @@ implementations, and valuable experience to the creation of this module: .. [#] The default locale directory is system dependent; for example, on RedHat Linux it is :file:`/usr/share/locale`, but on Solaris it is :file:`/usr/lib/locale`. The :mod:`gettext` module does not try to support these system dependent - defaults; instead its default is :file:`sys.prefix/share/locale`. For this - reason, it is always best to call :func:`bindtextdomain` with an explicit - absolute path at the start of your application. + defaults; instead its default is :file:`{sys.prefix}/share/locale` (see + :data:`sys.prefix`). For this reason, it is always best to call + :func:`bindtextdomain` with an explicit absolute path at the start of your + application. .. [#] See the footnote for :func:`bindtextdomain` above. diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 3408c103..3ebeb10a 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -94,6 +94,11 @@ The module provides the following classes: :func:`ssl._create_unverified_context` can be passed to the *context* parameter. + .. versionchanged:: 3.7.4 + This class now enables TLS 1.3 + :attr:`ssl.SSLContext.post_handshake_auth` for the default *context* or + when *cert_file* is passed with a custom *context*. + .. deprecated:: 3.6 *key_file* and *cert_file* are deprecated in favor of *context*. diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index fb8317ad..01bedfd2 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -55,8 +55,9 @@ in Cookie name (as :attr:`~Morsel.key`). .. class:: SimpleCookie([input]) This class derives from :class:`BaseCookie` and overrides :meth:`value_decode` - and :meth:`value_encode` to be the identity and :func:`str` respectively. - + and :meth:`value_encode`. SimpleCookie supports strings as cookie values. + When setting the value, SimpleCookie calls the builtin :func:`str()` to convert + the value to a string. Values received from HTTP are kept as strings. .. seealso:: @@ -76,15 +77,16 @@ Cookie Objects .. method:: BaseCookie.value_decode(val) - Return a decoded value from a string representation. Return value can be any - type. This method does nothing in :class:`BaseCookie` --- it exists so it can be - overridden. + Return a tuple ``(real_value, coded_value)`` from a string representation. + ``real_value`` can be any type. This method does no decoding in + :class:`BaseCookie` --- it exists so it can be overridden. .. method:: BaseCookie.value_encode(val) - Return an encoded value. *val* can be any type, but return value must be a - string. This method does nothing in :class:`BaseCookie` --- it exists so it can + Return a tuple ``(real_value, coded_value)``. *val* can be any type, but + ``coded_value`` will always be converted to a string. + This method does no encoding in :class:`BaseCookie` --- it exists so it can be overridden. In general, it should be the case that :meth:`value_encode` and diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 8ef9e2ee..fb886a71 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -50,7 +50,7 @@ default title and context menu. On macOS, there is one application menu. It dynamically changes according to the window currently selected. It has an IDLE menu, and some entries -described below are moved around to conform to Apple guidlines. +described below are moved around to conform to Apple guidelines. File menu (Shell and Editor) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -207,9 +207,13 @@ Strip trailing whitespace Run menu (Editor window only) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. _python-shell: + Python Shell Open or wake up the Python Shell window. +.. _check-module: + Check Module Check the syntax of the module currently open in the Editor window. If the module has not been saved IDLE will either prompt the user to save or @@ -217,8 +221,10 @@ Check Module there is a syntax error, the approximate location is indicated in the Editor window. +.. _run-module: + Run Module - Do Check Module (above). If no error, restart the shell to clean the + Do :ref:`Check Module `. If no error, restart the shell to clean the environment, then execute the module. Output is displayed in the Shell window. Note that output requires use of ``print`` or ``write``. When execution is complete, the Shell retains focus and displays a prompt. @@ -226,6 +232,14 @@ Run Module This is similar to executing a file with ``python -i file`` at a command line. +.. _run-custom: + +Run... Customized + Same as :ref:`Run Module `, but run the module with customized + settings. *Command Line Arguments* extend :data:`sys.argv` as if passed + on a command line. The module can be run in the Shell without restarting. + + Shell menu (Shell window only) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -281,16 +295,19 @@ Configure IDLE menu. For more, see :ref:`Setting preferences ` under Help and preferences. -Zoom/Restore Height - Toggles the window between normal size and maximum height. The initial size - defaults to 40 lines by 80 chars unless changed on the General tab of the - Configure IDLE dialog. - Show/Hide Code Context (Editor Window only) Open a pane at the top of the edit window which shows the block context of the code which has scrolled above the top of the window. See :ref:`Code Context ` in the Editing and Navigation section below. +Zoom/Restore Height + Toggles the window between normal size and maximum height. The initial size + defaults to 40 lines by 80 chars unless changed on the General tab of the + Configure IDLE dialog. The maximum height for a screen is determined by + momentarily maximizing a window the first time one is zoomed on the screen. + Changing screen settings may invalidate the saved height. This toogle has + no effect when a window is maximized. + Window menu (Shell and Editor) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -356,8 +373,8 @@ Shell and Output windows also have the following. Go to file/line Same as in Debug menu. -The Shell window also has an output squeezing facility explained in the -the *Python Shell window* subsection below. +The Shell window also has an output squeezing facility explained in the *Python +Shell window* subsection below. Squeeze If the cursor is over an output line, squeeze all the output between @@ -700,6 +717,9 @@ If ``sys`` is reset by user code, such as with ``importlib.reload(sys)``, IDLE's changes are lost and input from the keyboard and output to the screen will not work correctly. +When user code raises SystemExit either directly or by calling sys.exit, IDLE +returns to a Shell prompt instead of exiting. + User output in Shell ^^^^^^^^^^^^^^^^^^^^ @@ -716,21 +736,17 @@ In contrast, some system text windows only keep the last n lines of output. A Windows console, for instance, keeps a user-settable 1 to 9999 lines, with 300 the default. -A Tk Text widget, and hence IDLE's Shell, displays characters (codepoints) -in the the BMP (Basic Multilingual Plane) subset of Unicode. -Which characters are displayed with a proper glyph and which with a -replacement box depends on the operating system and installed fonts. -Tab characters cause the following text to begin after -the next tab stop. (They occur every 8 'characters'). -Newline characters cause following text to appear on a new line. -Other control characters are ignored or displayed as a space, box, or -something else, depending on the operating system and font. -(Moving the text cursor through such output with arrow keys may exhibit -some surprising spacing behavior.) - -.. code-block:: none - - >>> s = 'a\tb\a<\x02><\r>\bc\nd' +A Tk Text widget, and hence IDLE's Shell, displays characters (codepoints) in +the BMP (Basic Multilingual Plane) subset of Unicode. Which characters are +displayed with a proper glyph and which with a replacement box depends on the +operating system and installed fonts. Tab characters cause the following text +to begin after the next tab stop. (They occur every 8 'characters'). Newline +characters cause following text to appear on a new line. Other control +characters are ignored or displayed as a space, box, or something else, +depending on the operating system and font. (Moving the text cursor through +such output with arrow keys may exhibit some surprising spacing behavior.) :: + + >>> s = 'a\tb\a<\x02><\r>\bc\nd' # Enter 22 chars. >>> len(s) 14 >>> s # Display repr(s) @@ -773,7 +789,7 @@ facilitate development of tkinter programs. Enter ``import tkinter as tk; root = tk.Tk()`` in standard Python and nothing appears. Enter the same in IDLE and a tk window appears. In standard Python, one must also enter ``root.update()`` to see the window. IDLE does the equivalent in the -background, about 20 times a second, which is about every 50 milleseconds. +background, about 20 times a second, which is about every 50 milliseconds. Next enter ``b = tk.Button(root, text='button'); b.pack()``. Again, nothing visibly changes in standard Python until one enters ``root.update()``. diff --git a/Doc/library/io.rst b/Doc/library/io.rst index e623a041..9738c5c2 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -226,7 +226,7 @@ I/O Base Classes implementations represent a file that cannot be read, written or seeked. - Even though :class:`IOBase` does not declare :meth:`read`, :meth:`readinto`, + Even though :class:`IOBase` does not declare :meth:`read` or :meth:`write` because their signatures will vary, implementations and clients should consider those methods part of the interface. Also, implementations may raise a :exc:`ValueError` (or :exc:`UnsupportedOperation`) @@ -234,9 +234,7 @@ I/O Base Classes The basic type used for binary data read from or written to a file is :class:`bytes`. Other :term:`bytes-like objects ` are - accepted as method arguments too. In some cases, such as - :meth:`~RawIOBase.readinto`, a writable object such as :class:`bytearray` - is required. Text I/O classes work with :class:`str` data. + accepted as method arguments too. Text I/O classes work with :class:`str` data. Note that calling any method (even inquiries) on a closed stream is undefined. Implementations may raise :exc:`ValueError` in this case. @@ -405,7 +403,8 @@ I/O Base Classes Read bytes into a pre-allocated, writable :term:`bytes-like object` *b*, and return the - number of bytes read. If the object is in non-blocking mode and no bytes + number of bytes read. For example, *b* might be a :class:`bytearray`. + If the object is in non-blocking mode and no bytes are available, ``None`` is returned. .. method:: write(b) @@ -495,6 +494,7 @@ I/O Base Classes Read bytes into a pre-allocated, writable :term:`bytes-like object` *b* and return the number of bytes read. + For example, *b* might be a :class:`bytearray`. Like :meth:`read`, multiple reads may be issued to the underlying raw stream, unless the latter is interactive. @@ -757,8 +757,7 @@ Text I/O .. class:: TextIOBase Base class for text streams. This class provides a character and line based - interface to stream I/O. There is no :meth:`readinto` method because - Python's character strings are immutable. It inherits :class:`IOBase`. + interface to stream I/O. It inherits :class:`IOBase`. There is no public constructor. :class:`TextIOBase` provides or overrides these data attributes and @@ -1048,4 +1047,3 @@ The above implicitly extends to text files, since the :func:`open()` function will wrap a buffered object inside a :class:`TextIOWrapper`. This includes standard streams and therefore affects the built-in function :func:`print()` as well. - diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 33aec573..fd96d7a9 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -10,8 +10,8 @@ -------------- -This module is always available. It provides access to the mathematical -functions defined by the C standard. +This module provides access to the mathematical functions defined by the C +standard. These functions cannot be used with complex numbers; use the functions of the same name from the :mod:`cmath` module if you require support for complex @@ -50,7 +50,7 @@ Number-theoretic and representation functions .. function:: factorial(x) - Return *x* factorial. Raises :exc:`ValueError` if *x* is not integral or + Return *x* factorial as an integer. Raises :exc:`ValueError` if *x* is not integral or is negative. diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index e4d6d05a..11152f6a 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -438,8 +438,8 @@ Python syntax and the functions in the :mod:`operator` module. | Ordering | ``a > b`` | ``gt(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ -Inplace Operators ------------------ +In-place Operators +------------------ Many operations have an "in-place" version. Listed below are functions providing a more primitive access to in-place operators than the usual syntax @@ -462,7 +462,7 @@ value is computed, but not assigned back to the input variable: >>> a 'hello' -For mutable targets such as lists and dictionaries, the inplace method +For mutable targets such as lists and dictionaries, the in-place method will perform the update, so no subsequent assignment is necessary: >>> s = ['h', 'e', 'l', 'l', 'o'] diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index d78ab068..ef0e35d9 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -81,7 +81,7 @@ the :mod:`glob` module.) .. function:: commonpath(paths) Return the longest common sub-path of each pathname in the sequence - *paths*. Raise ValueError if *paths* contains both absolute and relative + *paths*. Raise :exc:`ValueError` if *paths* contains both absolute and relative pathnames, or if *paths* is empty. Unlike :func:`commonprefix`, this returns a valid path. @@ -315,9 +315,9 @@ the :mod:`glob` module.) .. function:: normcase(path) - Normalize the case of a pathname. On Unix and Mac OS X, this returns the - path unchanged; on case-insensitive filesystems, it converts the path to - lowercase. On Windows, it also converts forward slashes to backward slashes. + Normalize the case of a pathname. On Windows, convert all characters in the + pathname to lowercase, and also convert forward slashes to backward slashes. + On other operating systems, return the path unchanged. Raise a :exc:`TypeError` if the type of *path* is not ``str`` or ``bytes`` (directly or indirectly through the :class:`os.PathLike` interface). diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 7d4b41b1..5b2df3fe 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -966,7 +966,7 @@ call fails (for example because the path doesn't exist): is raised. .. versionadded:: 3.6 - The *strict* argument. + The *strict* argument (pre-3.6 behavior is strict). .. method:: Path.rglob(pattern) diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 0ab76606..12268437 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -43,11 +43,32 @@ The :mod:`pty` module defines the following functions: Spawn a process, and connect its controlling terminal with the current process's standard io. This is often used to baffle programs which insist on - reading from the controlling terminal. + reading from the controlling terminal. It is expected that the process + spawned behind the pty will eventually terminate, and when it does *spawn* + will return. + + The functions *master_read* and *stdin_read* are passed a file descriptor + which they should read from, and they should always return a byte string. In + order to force spawn to return before the child process exits an + :exc:`OSError` should be thrown. + + The default implementation for both functions will read and return up to 1024 + bytes each time the function is called. The *master_read* callback is passed + the pseudoterminal’s master file descriptor to read output from the child + process, and *stdin_read* is passed file descriptor 0, to read from the + parent process's standard input. + + Returning an empty byte string from either callback is interpreted as an + end-of-file (EOF) condition, and that callback will not be called after + that. If *stdin_read* signals EOF the controlling terminal can no longer + communicate with the parent process OR the child process. Unless the child + process will quit without any input, *spawn* will then loop forever. If + *master_read* signals EOF the same behavior results (on linux at least). + + If both callbacks signal EOF then *spawn* will probably never return, unless + *select* throws an error on your platform when passed three empty lists. This + is a bug, documented in `issue 26228 `_. - The functions *master_read* and *stdin_read* should be functions which read from - a file descriptor. The defaults try to read 1024 bytes each time they are - called. .. versionchanged:: 3.4 :func:`spawn` now returns the status value from :func:`os.waitpid` diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index a70c8df6..b80a2fae 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -44,7 +44,7 @@ modules. .. versionadded:: 3.7 Descriptors for nested definitions. They are accessed through the - new children attibute. Each has a new parent attribute. + new children attribute. Each has a new parent attribute. The descriptors returned by these functions are instances of Function and Class classes. Users are not expected to create instances diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 7335a64b..0a42da1d 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -152,6 +152,11 @@ provide the public methods described below. Otherwise (*block* is false), return an item if one is immediately available, else raise the :exc:`Empty` exception (*timeout* is ignored in that case). + Prior to 3.0 on POSIX systems, and for all versions on Windows, if + *block* is true and *timeout* is ``None``, this operation goes into + an uninterruptible wait on an underlying lock. This means that no exceptions + can occur, and in particular a SIGINT will not trigger a :exc:`KeyboardInterrupt`. + .. method:: Queue.get_nowait() diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 7d051e18..42979ffb 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -310,6 +310,11 @@ be found in any statistics text. Alternative Generator --------------------- +.. class:: Random([seed]) + + Class that implements the default pseudo-random number generator used by the + :mod:`random` module. + .. class:: SystemRandom([seed]) Class that uses the :func:`os.urandom` function for generating random numbers diff --git a/Doc/library/re.rst b/Doc/library/re.rst index dc3f428b..2e6c7f71 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -902,6 +902,7 @@ form. Unknown escapes in *repl* consisting of ``'\'`` and an ASCII letter now are errors. + .. versionchanged:: 3.7 Empty matches for the pattern are replaced when adjacent to a previous non-empty match. diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 16c28cf7..eae0a6df 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -19,7 +19,7 @@ function. Readline keybindings may be configured via an initialization file, typically ``.inputrc`` in your home directory. See `Readline Init File -`_ +`_ in the GNU Readline manual for information about the format and allowable constructs of that file, and the capabilities of the Readline library in general. diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 6094a7b8..03753af4 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -20,8 +20,7 @@ scheduler: The :class:`scheduler` class defines a generic interface to scheduling events. It needs two functions to actually deal with the "outside world" --- *timefunc* should be callable without arguments, and return a number (the "time", in any - units whatsoever). If time.monotonic is not available, the *timefunc* default - is time.time instead. The *delayfunc* function should be callable with one + units whatsoever). The *delayfunc* function should be callable with one argument, compatible with the output of *timefunc*, and should delay that many time units. *delayfunc* will also be called with the argument ``0`` after each event is run to allow other threads an opportunity to run in multi-threaded diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 148a59c7..b6716eea 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -16,7 +16,8 @@ The :func:`signal.signal` function allows defining custom handlers to be executed when a signal is received. A small number of default handlers are installed: :const:`SIGPIPE` is ignored (so write errors on pipes and sockets can be reported as ordinary Python exceptions) and :const:`SIGINT` is -translated into a :exc:`KeyboardInterrupt` exception. +translated into a :exc:`KeyboardInterrupt` exception if the parent process +has not changed it. A handler for a particular signal, once set, remains installed until it is explicitly reset (Python emulates the BSD style interface regardless of the diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 37b15774..15c65b97 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -373,7 +373,7 @@ Constants Enables CAN FD support in a CAN_RAW socket. This is disabled by default. This allows your application to send both CAN and CAN FD frames; however, - you one must accept both CAN and CAN FD frames when reading from the socket. + you must accept both CAN and CAN FD frames when reading from the socket. This constant is documented in the Linux documentation. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 569bfced..e20b4b3d 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -530,6 +530,7 @@ Connection Objects with open('dump.sql', 'w') as f: for line in con.iterdump(): f.write('%s\n' % line) + con.close() .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250) @@ -566,8 +567,11 @@ Connection Objects print(f'Copied {total-remaining} of {total} pages...') con = sqlite3.connect('existing_db.db') - with sqlite3.connect('backup.db') as bck: + bck = sqlite3.connect('backup.db') + with bck: con.backup(bck, pages=1, progress=progress) + bck.close() + con.close() Example 2, copy an existing database into a transient copy:: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c21cb0d9..3b74331e 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2677,8 +2677,8 @@ arbitrary binary data. containing the part before the separator, the separator itself or its bytearray copy, and the part after the separator. If the separator is not found, return a 3-tuple - containing a copy of the original sequence, followed by two empty bytes or - bytearray objects. + containing two empty bytes or bytearray objects, followed by a copy of the + original sequence. The separator to search for may be any :term:`bytes-like object`. diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 18be75c1..f9ace662 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -55,7 +55,9 @@ compatibility with older versions, see the :ref:`call-function-trio` section. If *capture_output* is true, stdout and stderr will be captured. When used, the internal :class:`Popen` object is automatically created with ``stdout=PIPE`` and ``stderr=PIPE``. The *stdout* and *stderr* arguments may - not be used as well. + not be supplied at the same time as *capture_output*. If you wish to capture + and combine both streams into one, use ``stdout=PIPE`` and ``stderr=STDOUT`` + instead of *capture_output*. The *timeout* argument is passed to :meth:`Popen.communicate`. If the timeout expires, the child process will be killed and waited for. The @@ -1002,14 +1004,14 @@ calls these functions. Run the command described by *args*. Wait for command to complete, then return the :attr:`~Popen.returncode` attribute. - This is equivalent to:: + Code needing to capture stdout or stderr should use :func:`run` instead: run(...).returncode - (except that the *input* and *check* parameters are not supported) + To suppress stdout or stderr, supply a value of :data:`DEVNULL`. - The arguments shown above are merely the most - common ones. The full function signature is largely the + The arguments shown above are merely some common ones. + The full function signature is the same as that of the :class:`Popen` constructor - this function passes all supplied arguments other than *timeout* directly through to that interface. @@ -1030,14 +1032,14 @@ calls these functions. :exc:`CalledProcessError` object will have the return code in the :attr:`~CalledProcessError.returncode` attribute. - This is equivalent to:: + Code needing to capture stdout or stderr should use :func:`run` instead: run(..., check=True) - (except that the *input* parameter is not supported) + To suppress stdout or stderr, supply a value of :data:`DEVNULL`. - The arguments shown above are merely the most - common ones. The full function signature is largely the + The arguments shown above are merely some common ones. + The full function signature is the same as that of the :class:`Popen` constructor - this function passes all supplied arguments other than *timeout* directly through to that interface. @@ -1067,7 +1069,7 @@ calls these functions. run(..., check=True, stdout=PIPE).stdout - The arguments shown above are merely the most common ones. + The arguments shown above are merely some common ones. The full function signature is largely the same as that of :func:`run` - most arguments are passed directly through to that interface. However, explicitly passing ``input=None`` to inherit the parent's @@ -1077,8 +1079,9 @@ calls these functions. encoding of the output data may depend on the command being invoked, so the decoding to text will often need to be handled at the application level. - This behaviour may be overridden by setting *universal_newlines* to - ``True`` as described above in :ref:`frequently-used-arguments`. + This behaviour may be overridden by setting *text*, *encoding*, *errors*, + or *universal_newlines* to ``True`` as described in + :ref:`frequently-used-arguments` and :func:`run`. To also capture standard error in the result, use ``stderr=subprocess.STDOUT``:: diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index ace0e280..9a8c2ca0 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -30,6 +30,12 @@ always available. To loop over the standard input, or the list of files given on the command line, see the :mod:`fileinput` module. + .. note:: + On Unix, command line arguments are passed by bytes from OS. Python decodes + them with filesystem encoding and "surrogateescape" error handler. + When you need original bytes, you can get it by + ``[os.fsencode(arg) for arg in sys.argv]``. + .. data:: base_exec_prefix @@ -1165,8 +1171,8 @@ always available. Set the system's trace function, which allows you to implement a Python source code debugger in Python. The function is thread-specific; for a - debugger to support multiple threads, it must be registered using - :func:`settrace` for each thread being debugged. + debugger to support multiple threads, it must register a trace function using + :func:`settrace` for each thread being debugged or use :func:`threading.settrace`. Trace functions should have three arguments: *frame*, *event*, and *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index d7dbcb10..c58a6ad7 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -250,7 +250,7 @@ since it is impossible to detect the termination of alien threads. You may override this method in a subclass. The standard :meth:`run` method invokes the callable object passed to the object's constructor as - the *target* argument, if any, with sequential and keyword arguments taken + the *target* argument, if any, with positional and keyword arguments taken from the *args* and *kwargs* arguments, respectively. .. method:: join(timeout=None) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 8c6813bb..4230c19f 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -153,7 +153,7 @@ Functions :c:func:`QueryPerformanceCounter`. The resolution is typically better than one microsecond. - .. deprecated:: 3.3 + .. deprecated-removed:: 3.3 3.8 The behaviour of this function depends on the platform: use :func:`perf_counter` or :func:`process_time` instead, depending on your requirements, to have a well defined behaviour. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 8ca37034..ef7a4e40 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -44,8 +44,12 @@ This can be achieved from the :ref:`python-interface` with:: >>> timeit.timeit('"-".join(map(str, range(100)))', number=10000) 0.23702679807320237 +A callable can also be passed from the :ref:`python-interface`:: -Note however that :mod:`timeit` will automatically determine the number of + >>> timeit.timeit(lambda: "-".join(map(str, range(100))), number=10000) + 0.19665591977536678 + +Note however that :func:`.timeit` will automatically determine the number of repetitions only when the command-line interface is used. In the :ref:`timeit-examples` section you can find more advanced examples. diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 60cf892e..e1fc051d 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -31,7 +31,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. `TKDocs `_ Extensive tutorial plus friendlier widget pages for some of the widgets. - `Tkinter reference: a GUI for Python `_ + `Tkinter 8.5 reference: a GUI for Python `_ On-line reference material. `Tkinter docs from effbot `_ @@ -41,7 +41,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. Book by Mark Lutz, has excellent coverage of Tkinter. `Modern Tkinter for Busy Python Developers `_ - Book by Mark Rozerman about building attractive and modern graphical user interfaces with Python and Tkinter. + Book by Mark Roseman about building attractive and modern graphical user interfaces with Python and Tkinter. `Python and Tkinter Programming `_ Book by John Grayson (ISBN 1-884777-81-3). @@ -55,7 +55,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. `Tcl/Tk recent man pages `_ Recent Tcl/Tk manuals on www.tcl.tk. - `ActiveState Tcl Home Page `_ + `ActiveState Tcl Home Page `_ The Tk/Tcl development is largely taking place at ActiveState. `Tcl and the Tk Toolkit `_ diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 175010b8..5d7f0608 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -20,8 +20,8 @@ Introduction ============ Turtle graphics is a popular way for introducing programming to kids. It was -part of the original Logo programming language developed by Wally Feurzig and -Seymour Papert in 1966. +part of the original Logo programming language developed by Wally Feurzeig, +Seymour Papert and Cynthia Solomon in 1967. Imagine a robotic turtle starting at (0, 0) in the x-y plane. After an ``import turtle``, give it the command ``turtle.forward(15)``, and it moves (on-screen!) 15 pixels in the diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index e7560622..12f4c03f 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -555,7 +555,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.Collection` - .. versionadded:: 3.6 + .. versionadded:: 3.6.0 .. class:: AbstractSet(Sized, Collection[T_co]) @@ -599,6 +599,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.deque`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: List(list, MutableSequence[T]) @@ -648,6 +649,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.Awaitable`. + .. versionadded:: 3.5.2 + .. class:: Coroutine(Awaitable[V_co], Generic[T_co T_contra, V_co]) A generic version of :class:`collections.abc.Coroutine`. @@ -661,25 +664,33 @@ The module defines the following classes, functions and decorators: async def bar() -> None: x = await c # type: int + .. versionadded:: 3.5.3 + .. class:: AsyncIterable(Generic[T_co]) A generic version of :class:`collections.abc.AsyncIterable`. + .. versionadded:: 3.5.2 + .. class:: AsyncIterator(AsyncIterable[T_co]) A generic version of :class:`collections.abc.AsyncIterator`. + .. versionadded:: 3.5.2 + .. class:: ContextManager(Generic[T_co]) A generic version of :class:`contextlib.AbstractContextManager`. - .. versionadded:: 3.6 + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.0 .. class:: AsyncContextManager(Generic[T_co]) A generic version of :class:`contextlib.AbstractAsyncContextManager`. - .. versionadded:: 3.6 + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 .. class:: Dict(dict, MutableMapping[KT, VT]) @@ -708,12 +719,14 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.Counter`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) A generic version of :class:`collections.ChainMap`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -778,7 +791,7 @@ The module defines the following classes, functions and decorators: yield start start = await increment(start) - .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 .. class:: Text @@ -815,7 +828,7 @@ The module defines the following classes, functions and decorators: .. class:: NamedTuple - Typed version of namedtuple. + Typed version of :func:`collections.namedtuple`. Usage:: @@ -941,6 +954,24 @@ The module defines the following classes, functions and decorators: This wraps the decorator with something that wraps the decorated function in :func:`no_type_check`. +.. decorator:: type_check_only + + Decorator to mark a class or function to be unavailable at runtime. + + This decorator is itself not available at runtime. It is mainly + intended to mark classes that are defined in type stub files if + an implementation returns an instance of a private class:: + + @type_check_only + class Response: # private or not available at runtime + code: int + def get_header(self, name: str) -> str: ... + + def fetch_response() -> Response: ... + + Note that returning instances of private classes is not recommended. + It is usually preferable to make such classes public. + .. data:: Any Special type indicating an unconstrained type. @@ -959,6 +990,7 @@ The module defines the following classes, functions and decorators: raise RuntimeError('no way') .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 .. data:: Union diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 6daf0fec..b76ae712 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -1418,7 +1418,7 @@ passed by keyword *after* any of the standard arguments created by :func:`patch` >>> test_function() If :func:`patch.multiple` is used as a context manager, the value returned by the -context manger is a dictionary where created mocks are keyed by name: +context manager is a dictionary where created mocks are keyed by name:: >>> with patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) as values: ... assert 'other' in repr(values['other']) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 3e9e38c0..bbe14299 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -510,7 +510,8 @@ that is broken and will fail, but shouldn't be counted as a failure on a :class:`TestResult`. Skipping a test is simply a matter of using the :func:`skip` :term:`decorator` -or one of its conditional variants. +or one of its conditional variants, calling :meth:`TestCase.skipTest` within a +:meth:`~TestCase.setUp` or test method, or raising :exc:`SkipTest` directly. Basic skipping looks like this:: @@ -531,16 +532,23 @@ Basic skipping looks like this:: # windows specific testing code pass + def test_maybe_skipped(self): + if not external_resource_available(): + self.skipTest("external resource not available") + # test code that depends on the external resource + pass + This is the output of running the example above in verbose mode:: test_format (__main__.MyTestCase) ... skipped 'not supported in this library version' test_nothing (__main__.MyTestCase) ... skipped 'demonstrating skipping' + test_maybe_skipped (__main__.MyTestCase) ... skipped 'external resource not available' test_windows_support (__main__.MyTestCase) ... skipped 'requires Windows' ---------------------------------------------------------------------- - Ran 3 tests in 0.005s + Ran 4 tests in 0.005s - OK (skipped=3) + OK (skipped=4) Classes can be skipped just like methods:: @@ -568,7 +576,7 @@ the test unless the passed object has a certain attribute:: return lambda func: func return unittest.skip("{!r} doesn't have {!r}".format(obj, attr)) -The following decorators implement test skipping and expected failures: +The following decorators and exception implement test skipping and expected failures: .. decorator:: skip(reason) @@ -1000,7 +1008,7 @@ Test cases int('XYZ') .. versionadded:: 3.1 - under the name ``assertRaisesRegexp``. + Added under the name ``assertRaisesRegexp``. .. versionchanged:: 3.2 Renamed to :meth:`assertRaisesRegex`. @@ -1145,7 +1153,7 @@ Test cases +---------------------------------------+--------------------------------+--------------+ | :meth:`assertCountEqual(a, b) | *a* and *b* have the same | 3.2 | | ` | elements in the same number, | | - | | regardless of their order | | + | | regardless of their order. | | +---------------------------------------+--------------------------------+--------------+ @@ -1193,7 +1201,7 @@ Test cases expression suitable for use by :func:`re.search`. .. versionadded:: 3.1 - under the name ``assertRegexpMatches``. + Added under the name ``assertRegexpMatches``. .. versionchanged:: 3.2 The method ``assertRegexpMatches()`` has been renamed to :meth:`.assertRegex`. @@ -1483,14 +1491,14 @@ along with their deprecated aliases: ============================== ====================== ======================= .. deprecated:: 3.1 - the fail* aliases listed in the second column. + The fail* aliases listed in the second column have been deprecated. .. deprecated:: 3.2 - the assert* aliases listed in the third column. + The assert* aliases listed in the third column have been deprecated. .. deprecated:: 3.2 ``assertRegexpMatches`` and ``assertRaisesRegexp`` have been renamed to :meth:`.assertRegex` and :meth:`.assertRaisesRegex`. .. deprecated:: 3.5 - the ``assertNotRegexpMatches`` name in favor of :meth:`.assertNotRegex`. + The ``assertNotRegexpMatches`` name is deprecated in favor of :meth:`.assertNotRegex`. .. _testsuite-objects: diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index b565e1ed..ddc3ee23 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -39,8 +39,9 @@ or on combining URL components into a URL string. .. function:: urlparse(urlstring, scheme='', allow_fragments=True) - Parse a URL into six components, returning a 6-tuple. This corresponds to the - general structure of a URL: ``scheme://netloc/path;parameters?query#fragment``. + Parse a URL into six components, returning a 6-item :term:`named tuple`. This + corresponds to the general structure of a URL: + ``scheme://netloc/path;parameters?query#fragment``. Each tuple item is a string, possibly empty. The components are not broken up in smaller parts (for example, the network location is a single string), and % escapes are not expanded. The delimiters as shown above are not part of the @@ -88,8 +89,8 @@ or on combining URL components into a URL string. or query component, and :attr:`fragment` is set to the empty string in the return value. - The return value is actually an instance of a subclass of :class:`tuple`. This - class has the following additional read-only convenience attributes: + The return value is a :term:`named tuple`, which means that its items can + be accessed by index or as named attributes, which are: +------------------+-------+--------------------------+----------------------+ | Attribute | Index | Value | Value if not present | @@ -129,6 +130,24 @@ or on combining URL components into a URL string. ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is decomposed before parsing, no error will be raised. + As is the case with all named tuples, the subclass has a few additional methods + and attributes that are particularly useful. One such method is :meth:`_replace`. + The :meth:`_replace` method will return a new ParseResult object replacing specified + fields with new values. + + .. doctest:: + :options: +NORMALIZE_WHITESPACE + + >>> from urllib.parse import urlparse + >>> u = urlparse('//www.cwi.nl:80/%7Eguido/Python.html') + >>> u + ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + params='', query='', fragment='') + >>> u._replace(scheme='http') + ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + params='', query='', fragment='') + + .. versionchanged:: 3.2 Added IPv6 URL parsing capabilities. @@ -230,11 +249,13 @@ or on combining URL components into a URL string. This should generally be used instead of :func:`urlparse` if the more recent URL syntax allowing parameters to be applied to each segment of the *path* portion of the URL (see :rfc:`2396`) is wanted. A separate function is needed to - separate the path segments and parameters. This function returns a 5-tuple: - (addressing scheme, network location, path, query, fragment identifier). + separate the path segments and parameters. This function returns a 5-item + :term:`named tuple`:: + + (addressing scheme, network location, path, query, fragment identifier). - The return value is actually an instance of a subclass of :class:`tuple`. This - class has the following additional read-only convenience attributes: + The return value is a :term:`named tuple`, its items can be accessed by index + or as named attributes: +------------------+-------+-------------------------+----------------------+ | Attribute | Index | Value | Value if not present | @@ -330,8 +351,8 @@ or on combining URL components into a URL string. string. If there is no fragment identifier in *url*, return *url* unmodified and an empty string. - The return value is actually an instance of a subclass of :class:`tuple`. This - class has the following additional read-only convenience attributes: + The return value is a :term:`named tuple`, its items can be accessed by index + or as named attributes: +------------------+-------+-------------------------+----------------------+ | Attribute | Index | Value | Value if not present | diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 982f57f1..1bc81e05 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -608,23 +608,39 @@ OpenerDirector Objects *handler* should be an instance of :class:`BaseHandler`. The following methods are searched, and added to the possible chains (note that HTTP errors are a - special case). + special case). Note that, in the following, *protocol* should be replaced + with the actual protocol to handle, for example :meth:`http_response` would + be the HTTP protocol response handler. Also *type* should be replaced with + the actual HTTP code, for example :meth:`http_error_404` would handle HTTP + 404 errors. - * :meth:`protocol_open` --- signal that the handler knows how to open *protocol* + * :meth:`_open` --- signal that the handler knows how to open *protocol* URLs. - * :meth:`http_error_type` --- signal that the handler knows how to handle HTTP + See |protocol_open|_ for more information. + + * :meth:`http_error_\` --- signal that the handler knows how to handle HTTP errors with HTTP error code *type*. - * :meth:`protocol_error` --- signal that the handler knows how to handle errors + See |http_error_nnn|_ for more information. + + * :meth:`_error` --- signal that the handler knows how to handle errors from (non-\ ``http``) *protocol*. - * :meth:`protocol_request` --- signal that the handler knows how to pre-process + * :meth:`_request` --- signal that the handler knows how to pre-process *protocol* requests. - * :meth:`protocol_response` --- signal that the handler knows how to + See |protocol_request|_ for more information. + + * :meth:`_response` --- signal that the handler knows how to post-process *protocol* responses. + See |protocol_response|_ for more information. + +.. |protocol_open| replace:: :meth:`BaseHandler._open` +.. |http_error_nnn| replace:: :meth:`BaseHandler.http_error_\` +.. |protocol_request| replace:: :meth:`BaseHandler._request` +.. |protocol_response| replace:: :meth:`BaseHandler._response` .. method:: OpenerDirector.open(url, data=None[, timeout]) @@ -643,7 +659,7 @@ OpenerDirector Objects Handle an error of the given protocol. This will call the registered error handlers for the given protocol with the given arguments (which are protocol specific). The HTTP protocol is a special case which uses the HTTP response - code to determine the specific error handler; refer to the :meth:`http_error_\*` + code to determine the specific error handler; refer to the :meth:`http_error_\` methods of the handler classes. Return values and exceptions raised are the same as those of :func:`urlopen`. @@ -653,17 +669,17 @@ OpenerDirector objects open URLs in three stages: The order in which these methods are called within each stage is determined by sorting the handler instances. -#. Every handler with a method named like :meth:`protocol_request` has that +#. Every handler with a method named like :meth:`_request` has that method called to pre-process the request. -#. Handlers with a method named like :meth:`protocol_open` are called to handle +#. Handlers with a method named like :meth:`_open` are called to handle the request. This stage ends when a handler either returns a non-\ :const:`None` value (ie. a response), or raises an exception (usually :exc:`~urllib.error.URLError`). Exceptions are allowed to propagate. In fact, the above algorithm is first tried for methods named :meth:`default_open`. If all such methods return :const:`None`, the algorithm - is repeated for methods named like :meth:`protocol_open`. If all such methods + is repeated for methods named like :meth:`_open`. If all such methods return :const:`None`, the algorithm is repeated for methods named :meth:`unknown_open`. @@ -671,7 +687,7 @@ sorting the handler instances. :class:`OpenerDirector` instance's :meth:`~OpenerDirector.open` and :meth:`~OpenerDirector.error` methods. -#. Every handler with a method named like :meth:`protocol_response` has that +#. Every handler with a method named like :meth:`_response` has that method called to post-process the response. @@ -700,7 +716,7 @@ The following attribute and methods should only be used by classes derived from .. note:: The convention has been adopted that subclasses defining - :meth:`protocol_request` or :meth:`protocol_response` methods are named + :meth:`_request` or :meth:`_response` methods are named :class:`\*Processor`; all others are named :class:`\*Handler`. @@ -725,7 +741,8 @@ The following attribute and methods should only be used by classes derived from This method will be called before any protocol-specific open method. -.. method:: BaseHandler.protocol_open(req) +.. _protocol_open: +.. method:: BaseHandler._open(req) :noindex: This method is *not* defined in :class:`BaseHandler`, but subclasses should @@ -762,7 +779,8 @@ The following attribute and methods should only be used by classes derived from :func:`urlopen`. -.. method:: BaseHandler.http_error_nnn(req, fp, code, msg, hdrs) +.. _http_error_nnn: +.. method:: BaseHandler.http_error_(req, fp, code, msg, hdrs) *nnn* should be a three-digit HTTP error code. This method is also not defined in :class:`BaseHandler`, but will be called, if it exists, on an instance of a @@ -774,7 +792,8 @@ The following attribute and methods should only be used by classes derived from :meth:`http_error_default`. -.. method:: BaseHandler.protocol_request(req) +.. _protocol_request: +.. method:: BaseHandler._request(req) :noindex: This method is *not* defined in :class:`BaseHandler`, but subclasses should @@ -785,7 +804,8 @@ The following attribute and methods should only be used by classes derived from :class:`Request` object. -.. method:: BaseHandler.protocol_response(req, response) +.. _protocol_response: +.. method:: BaseHandler._response(req, response) :noindex: This method is *not* defined in :class:`BaseHandler`, but subclasses should @@ -873,10 +893,10 @@ ProxyHandler Objects -------------------- -.. method:: ProxyHandler.protocol_open(request) +.. method:: ProxyHandler._open(request) :noindex: - The :class:`ProxyHandler` will have a method :meth:`protocol_open` for every + The :class:`ProxyHandler` will have a method :meth:`_open` for every *protocol* which has a proxy in the *proxies* dictionary given in the constructor. The method will modify requests to go through the proxy, by calling ``request.set_proxy()``, and call the next handler in the chain to @@ -1132,7 +1152,7 @@ HTTPErrorProcessor Objects For 200 error codes, the response object is returned immediately. For non-200 error codes, this simply passes the job on to the - :meth:`protocol_error_code` handler methods, via :meth:`OpenerDirector.error`. + :meth:`http_error_\` handler methods, via :meth:`OpenerDirector.error`. Eventually, :class:`HTTPDefaultErrorHandler` will raise an :exc:`~urllib.error.HTTPError` if no other handler handles the error. @@ -1415,6 +1435,7 @@ some point in the future. The *data* argument has the same meaning as the *data* argument of :func:`urlopen`. + This method always quotes *fullurl* using :func:`~urllib.parse.quote`. .. method:: open_unknown(fullurl, data=None) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index fefc522b..d61df484 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -239,14 +239,19 @@ creation according to their needs, the :class:`EnvBuilder` class. There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ - symlinks=False, with_pip=False) + symlinks=False, with_pip=False, prompt=None) Create an :class:`EnvBuilder` with the given keyword arguments, and call its :meth:`~EnvBuilder.create` method with the *env_dir* argument. + .. versionadded:: 3.3 + .. versionchanged:: 3.4 Added the ``with_pip`` parameter + .. versionchanged:: 3.6 + Added the ``prompt`` parameter + An example of extending ``EnvBuilder`` -------------------------------------- diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index b04bd79e..d121f320 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -109,11 +109,11 @@ The following warnings category classes are currently defined: +----------------------------------+-----------------------------------------------+ .. versionchanged:: 3.7 - Previously :exc:`DeprecationWarning` and :exc:`FutureWarning` were - distinguished based on whether a feature was being removed entirely or - changing its behaviour. They are now distinguished based on their - intended audience and the way they're handled by the default warnings - filters. + Previously :exc:`DeprecationWarning` and :exc:`FutureWarning` were + distinguished based on whether a feature was being removed entirely or + changing its behaviour. They are now distinguished based on their + intended audience and the way they're handled by the default warnings + filters. .. _warning-filter: diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 40bb06ad..8d8a0b5d 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -65,8 +65,8 @@ exposed by the :mod:`weakref` module for the benefit of advanced uses. Not all objects can be weakly referenced; those objects which can include class instances, functions written in Python (but not in C), instance methods, sets, -frozensets, some :term:`file objects `, :term:`generator`\s, type -objects, sockets, arrays, deques, regular expression pattern objects, and code +frozensets, some :term:`file objects `, :term:`generators `, +type objects, sockets, arrays, deques, regular expression pattern objects, and code objects. .. versionchanged:: 3.2 @@ -80,9 +80,10 @@ support weak references but can add support through subclassing:: obj = Dict(red=1, green=2, blue=3) # this object is weak referenceable -Other built-in types such as :class:`tuple` and :class:`int` do not support weak -references even when subclassed (This is an implementation detail and may be -different across various Python implementations.). +.. impl-detail:: + + Other built-in types such as :class:`tuple` and :class:`int` do not support weak + references even when subclassed. Extension types can easily be made to support weak references; see :ref:`weakref-support`. @@ -489,11 +490,14 @@ Unless you set the :attr:`~finalize.atexit` attribute to :const:`False`, a finalizer will be called when the program exits if it is still alive. For instance - >>> obj = Object() - >>> weakref.finalize(obj, print, "obj dead or exiting") #doctest:+ELLIPSIS - - >>> exit() #doctest:+SKIP - obj dead or exiting +.. doctest:: + :options: +SKIP + + >>> obj = Object() + >>> weakref.finalize(obj, print, "obj dead or exiting") + + >>> exit() + obj dead or exiting Comparing finalizers with :meth:`__del__` methods diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index a37caf62..c68b0371 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -134,11 +134,12 @@ module documentation. This section lists the differences between the API and .. method:: Node.writexml(writer, indent="", addindent="", newl="") - Write XML to the writer object. The writer should have a :meth:`write` method - which matches that of the file object interface. The *indent* parameter is the - indentation of the current node. The *addindent* parameter is the incremental - indentation to use for subnodes of the current one. The *newl* parameter - specifies the string to use to terminate newlines. + Write XML to the writer object. The writer receives texts but not bytes as input, + it should have a :meth:`write` method which matches that of the file object + interface. The *indent* parameter is the indentation of the current node. + The *addindent* parameter is the incremental indentation to use for subnodes + of the current one. The *newl* parameter specifies the string to use to + terminate newlines. For the :class:`Document` node, an additional keyword argument *encoding* can be used to specify the encoding field of the XML header. @@ -227,22 +228,8 @@ The following interfaces have no implementation in :mod:`xml.dom.minidom`: * :class:`DOMTimeStamp` -* :class:`DocumentType` - -* :class:`DOMImplementation` - -* :class:`CharacterData` - -* :class:`CDATASection` - -* :class:`Notation` - -* :class:`Entity` - * :class:`EntityReference` -* :class:`DocumentFragment` - Most of these reflect information in the XML document that is not of general utility to most DOM users. diff --git a/Doc/license.rst b/Doc/license.rst index a315b6f8..ac0e2b30 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -561,7 +561,7 @@ SipHash24 --------- The file :file:`Python/pyhash.c` contains Marek Majkowski' implementation of -Dan Bernstein's SipHash24 algorithm. The contains the following note:: +Dan Bernstein's SipHash24 algorithm. It contains the following note:: Copyright (c) 2013 Marek Majkowski diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 22c9b415..962ec7e6 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -475,13 +475,13 @@ Callable types | :attr:`__doc__` | The function's documentation | Writable | | | string, or ``None`` if | | | | unavailable; not inherited by | | - | | subclasses | | + | | subclasses. | | +-------------------------+-------------------------------+-----------+ - | :attr:`~definition.\ | The function's name | Writable | + | :attr:`~definition.\ | The function's name. | Writable | | __name__` | | | +-------------------------+-------------------------------+-----------+ | :attr:`~definition.\ | The function's | Writable | - | __qualname__` | :term:`qualified name` | | + | __qualname__` | :term:`qualified name`. | | | | | | | | .. versionadded:: 3.3 | | +-------------------------+-------------------------------+-----------+ @@ -493,7 +493,7 @@ Callable types | | argument values for those | | | | arguments that have defaults, | | | | or ``None`` if no arguments | | - | | have a default value | | + | | have a default value. | | +-------------------------+-------------------------------+-----------+ | :attr:`__code__` | The code object representing | Writable | | | the compiled function body. | | @@ -1315,9 +1315,9 @@ Basic customization Called by the :func:`format` built-in function, and by extension, evaluation of :ref:`formatted string literals ` and the :meth:`str.format` method, to produce a "formatted" - string representation of an object. The ``format_spec`` argument is + string representation of an object. The *format_spec* argument is a string that contains a description of the formatting options desired. - The interpretation of the ``format_spec`` argument is up to the type + The interpretation of the *format_spec* argument is up to the type implementing :meth:`__format__`, however most classes will either delegate formatting to one of the built-in types, or use a similar formatting option syntax. @@ -1861,11 +1861,11 @@ passed through to all metaclass operations described below. When a class definition is executed, the following steps occur: -* MRO entries are resolved -* the appropriate metaclass is determined -* the class namespace is prepared -* the class body is executed -* the class object is created +* MRO entries are resolved; +* the appropriate metaclass is determined; +* the class namespace is prepared; +* the class body is executed; +* the class object is created. Resolving MRO entries @@ -1889,11 +1889,11 @@ Determining the appropriate metaclass The appropriate metaclass for a class definition is determined as follows: -* if no bases and no explicit metaclass are given, then :func:`type` is used +* if no bases and no explicit metaclass are given, then :func:`type` is used; * if an explicit metaclass is given and it is *not* an instance of - :func:`type`, then it is used directly as the metaclass + :func:`type`, then it is used directly as the metaclass; * if an instance of :func:`type` is given as the explicit metaclass, or - bases are defined, then the most derived metaclass is used + bases are defined, then the most derived metaclass is used. The most derived metaclass is selected from the explicitly specified metaclass (if any) and the metaclasses (i.e. ``type(cls)``) of all specified @@ -1981,7 +1981,7 @@ invoked after creating the class object: * first, ``type.__new__`` collects all of the descriptors in the class namespace that define a :meth:`~object.__set_name__` method; * second, all of these ``__set_name__`` methods are called with the class - being defined and the assigned name of that particular descriptor; and + being defined and the assigned name of that particular descriptor; * finally, the :meth:`~object.__init_subclass__` hook is called on the immediate parent of the new class in its method resolution order. @@ -2053,7 +2053,7 @@ Emulating generic types ----------------------- One can implement the generic class syntax as specified by :pep:`484` -(for example ``List[int]``) by defining a special method +(for example ``List[int]``) by defining a special method: .. classmethod:: object.__class_getitem__(cls, key) diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index ba7130d6..49cb86b5 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -243,7 +243,7 @@ re-entering the offending piece of code from the top). When an exception is not handled at all, the interpreter terminates execution of the program, or returns to its interactive main loop. In either case, it prints -a stack backtrace, except when the exception is :exc:`SystemExit`. +a stack traceback, except when the exception is :exc:`SystemExit`. Exceptions are identified by class instances. The :keyword:`except` clause is selected depending on the class of the instance: it must reference the class of diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index d7da7ad4..990600ec 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -144,7 +144,7 @@ the single expression that makes up the expression list. .. index:: pair: empty; tuple An empty pair of parentheses yields an empty tuple object. Since tuples are -immutable, the rules for literals apply (i.e., two occurrences of the empty +immutable, the same rules as for literals apply (i.e., two occurrences of the empty tuple may or may not yield the same object). .. index:: @@ -484,8 +484,8 @@ will raise :exc:`AttributeError` or :exc:`TypeError`, while When the underlying iterator is complete, the :attr:`~StopIteration.value` attribute of the raised :exc:`StopIteration` instance becomes the value of the yield expression. It can be either set explicitly when raising -:exc:`StopIteration`, or automatically when the sub-iterator is a generator -(by returning a value from the sub-generator). +:exc:`StopIteration`, or automatically when the subiterator is a generator +(by returning a value from the subgenerator). .. versionchanged:: 3.3 Added ``yield from `` to delegate control flow to a subiterator. @@ -504,7 +504,7 @@ on the right hand side of an assignment statement. :pep:`380` - Syntax for Delegating to a Subgenerator The proposal to introduce the :token:`yield_from` syntax, making delegation - to sub-generators easy. + to subgenerators easy. :pep:`525` - Asynchronous Generators The proposal that expanded on :pep:`492` by adding generator capabilities to @@ -613,7 +613,7 @@ Asynchronous generator functions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The presence of a yield expression in a function or method defined using -:keyword:`async def` further defines the function as a +:keyword:`async def` further defines the function as an :term:`asynchronous generator` function. When an asynchronous generator function is called, it returns an @@ -678,13 +678,13 @@ which are used to control the execution of a generator function. Returns an awaitable which when run starts to execute the asynchronous generator or resumes it at the last executed yield expression. When an - asynchronous generator function is resumed with a :meth:`~agen.__anext__` + asynchronous generator function is resumed with an :meth:`~agen.__anext__` method, the current yield expression always evaluates to :const:`None` in the returned awaitable, which when run will continue to the next yield expression. The value of the :token:`expression_list` of the yield expression is the value of the :exc:`StopIteration` exception raised by the completing coroutine. If the asynchronous generator exits without - yielding another value, the awaitable instead raises an + yielding another value, the awaitable instead raises a :exc:`StopAsyncIteration` exception, signalling that the asynchronous iteration has completed. @@ -712,7 +712,7 @@ which are used to control the execution of a generator function. where the asynchronous generator was paused, and returns the next value yielded by the generator function as the value of the raised :exc:`StopIteration` exception. If the asynchronous generator exits - without yielding another value, an :exc:`StopAsyncIteration` exception is + without yielding another value, a :exc:`StopAsyncIteration` exception is raised by the awaitable. If the generator function does not catch the passed-in exception, or raises a different exception, then when the awaitable is run that exception @@ -1568,14 +1568,15 @@ y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and ``False`` otherwise. For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z`` with ``x == z`` is -produced while iterating over ``y``. If an exception is raised during the -iteration, it is as if :keyword:`in` raised that exception. +:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z``, for which the +expression ``x is z or x == z`` is true, is produced while iterating over ``y``. +If an exception is raised during the iteration, it is as if :keyword:`in` raised +that exception. Lastly, the old-style iteration protocol is tried: if a class defines :meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative -integer index *i* such that ``x == y[i]``, and all lower integer indices do not -raise :exc:`IndexError` exception. (If any other exception is raised, it is as +integer index *i* such that ``x is y[i] or x == y[i]``, and no lower integer index +raises the :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). .. index:: @@ -1584,7 +1585,7 @@ if :keyword:`in` raised that exception). pair: membership; test object: sequence -The operator :keyword:`not in` is defined to have the inverse true value of +The operator :keyword:`not in` is defined to have the inverse truth value of :keyword:`in`. .. index:: @@ -1599,8 +1600,8 @@ The operator :keyword:`not in` is defined to have the inverse true value of Identity comparisons -------------------- -The operators :keyword:`is` and :keyword:`is not` test for object identity: ``x -is y`` is true if and only if *x* and *y* are the same object. Object identity +The operators :keyword:`is` and :keyword:`is not` test for an object's identity: ``x +is y`` is true if and only if *x* and *y* are the same object. An Object's identity is determined using the :meth:`id` function. ``x is not y`` yields the inverse truth value. [#]_ diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 9a0ab39d..88290c88 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -921,6 +921,46 @@ it is sufficient to raise :exc:`ModuleNotFoundError` directly from ``None``. The latter indicates that the meta path search should continue, while raising an exception terminates it immediately. +.. _relativeimports: + +Package Relative Imports +======================== + +Relative imports use leading dots. A single leading dot indicates a relative +import, starting with the current package. Two or more leading dots indicate a +relative import to the parent(s) of the current package, one level per dot +after the first. For example, given the following package layout:: + + package/ + __init__.py + subpackage1/ + __init__.py + moduleX.py + moduleY.py + subpackage2/ + __init__.py + moduleZ.py + moduleA.py + +In either ``subpackage1/moduleX.py`` or ``subpackage1/__init__.py``, +the following are valid relative imports:: + + from .moduleY import spam + from .moduleY import spam as ham + from . import moduleY + from ..subpackage1 import moduleY + from ..subpackage2.moduleZ import eggs + from ..moduleA import foo + +Absolute imports may use either the ``import <>`` or ``from <> import <>`` +syntax, but relative imports may only use the second form; the reason +for this is that:: + + import XXX.YYY.ZZZ + +should expose ``XXX.YYY.ZZZ`` as a usable expression, but .moduleY is +not a valid expression. + Special considerations for __main__ =================================== diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index b3c3215a..7b90f3b3 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -70,7 +70,7 @@ Comments A comment starts with a hash character (``#``) that is not part of a string literal, and ends at the end of the physical line. A comment signifies the end of the logical line unless the implicit line joining rules are invoked. Comments -are ignored by the syntax; they are not tokens. +are ignored by the syntax. .. _encodings: diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 07072d95..1779832a 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -829,7 +829,8 @@ exists. Two dots means up one package level. Three dots is up two levels, etc. So if you execute ``from . import mod`` from a module in the ``pkg`` package then you will end up importing ``pkg.mod``. If you execute ``from ..subpkg2 import mod`` from within ``pkg.subpkg1`` you will import ``pkg.subpkg2.mod``. -The specification for relative imports is contained within :pep:`328`. +The specification for relative imports is contained in +the :ref:`relativeimports` section. :func:`importlib.import_module` is provided to support applications that determine dynamically the modules to be loaded. diff --git a/Doc/tools/extensions/suspicious.py b/Doc/tools/extensions/suspicious.py index 0a70e57d..8d80f675 100644 --- a/Doc/tools/extensions/suspicious.py +++ b/Doc/tools/extensions/suspicious.py @@ -48,6 +48,7 @@ import sys from docutils import nodes from sphinx.builders import Builder +import sphinx.util detect_all = re.compile(r''' ::(?=[^=])| # two :: (but NOT ::=) @@ -85,6 +86,7 @@ class CheckSuspiciousMarkupBuilder(Builder): Checks for possibly invalid markup that may leak into the output. """ name = 'suspicious' + logger = sphinx.util.logging.getLogger("CheckSuspiciousMarkupBuilder") def init(self): # create output file @@ -116,7 +118,7 @@ class CheckSuspiciousMarkupBuilder(Builder): self.warn('Found %s/%s unused rules:' % (len(unused_rules), len(self.rules))) for rule in unused_rules: - self.info(repr(rule)) + self.logger.info(repr(rule)) return def check_issue(self, line, lineno, issue): @@ -146,7 +148,7 @@ class CheckSuspiciousMarkupBuilder(Builder): return False def report_issue(self, text, lineno, issue): - if not self.any_issue: self.info() + if not self.any_issue: self.logger.info() self.any_issue = True self.write_log_entry(lineno, issue, text) if py3: @@ -181,7 +183,7 @@ class CheckSuspiciousMarkupBuilder(Builder): A csv file, with exactly the same format as suspicious.csv Fields: document name (normalized), line number, issue, surrounding text """ - self.info("loading ignore rules... ", nonl=1) + self.logger.info("loading ignore rules... ", nonl=1) self.rules = rules = [] try: if py3: @@ -206,7 +208,7 @@ class CheckSuspiciousMarkupBuilder(Builder): rule = Rule(docname, lineno, issue, text) rules.append(rule) f.close() - self.info('done, %d rules loaded' % len(self.rules)) + self.logger.info('done, %d rules loaded' % len(self.rules)) def get_lineno(node): diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index 20dad93d..fa298a76 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -10,7 +10,8 @@ '(?:release/\\d.\\d[\\x\\d\\.]*)']; var all_versions = { - '3.8': 'dev (3.8)', + '3.9': 'dev (3.9)', + '3.8': 'pre (3.8)', '3.7': '3.7', '3.6': '3.6', '3.5': '3.5', @@ -22,6 +23,7 @@ 'fr': 'French', 'ja': 'Japanese', 'ko': 'Korean', + 'zh-cn': 'Simplified Chinese', }; function build_version_select(current_version, current_release) { diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 1a99b18b..d9364d6c 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -12,8 +12,7 @@ {% if last_updated %}

Last updated on: {{ last_updated }}.

{% endif %}

To download an archive containing all the documents for this version of -Python in one of various formats, follow one of links in this table. The numbers -in the table are the size of the download files in megabytes.

+Python in one of various formats, follow one of links in this table.

diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 1181f717..0adaf038 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,7 +2,8 @@

{% trans %}Download these documents{% endtrans %}

{% trans %}Docs by version{% endtrans %}

    -
  • {% trans %}Python 3.8 (in development){% endtrans %}
    • +
  • {% trans %}Python 3.9 (in development){% endtrans %}
    • +
  • {% trans %}Python 3.8 (pre-release){% endtrans %}
  • {% trans %}Python 3.7 (stable){% endtrans %}
  • {% trans %}Python 3.6 (security-fixes){% endtrans %}
  • {% trans %}Python 3.5 (security-fixes){% endtrans %}
    • diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 8cf903de..32e178b3 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -1,4 +1,15 @@ {% extends "!layout.html" %} + +{% block header %} +{%- if outdated %} +
    + {% trans %}This document is for an old version of Python that is no longer supported. + You should upgrade, and read the {% endtrans %} + {% trans %} Python documentation for the current stable release{% endtrans %}. +
    +{%- endif %} +{% endblock %} + {% block rootrellink %}
    • diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 90573453..c3ce1205 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -279,9 +279,11 @@ variables of the function. More precisely, all variable assignments in a function store the value in the local symbol table; whereas variable references first look in the local symbol table, then in the local symbol tables of enclosing functions, then in the global symbol table, and finally in the table -of built-in names. Thus, global variables cannot be directly assigned a value -within a function (unless named in a :keyword:`global` statement), although they -may be referenced. +of built-in names. Thus, global variables and variables of enclosing functions +cannot be directly assigned a value within a function (unless, for global +variables, named in a :keyword:`global` statement, or, for variables of enclosing +functions, named in a :keyword:`nonlocal` statement), although they may be +referenced. The actual parameters (arguments) to a function call are introduced in the local symbol table of the called function when it is called; thus, arguments are @@ -482,9 +484,9 @@ When a final formal parameter of the form ``**name`` is present, it receives a dictionary (see :ref:`typesmapping`) containing all keyword arguments except for those corresponding to a formal parameter. This may be combined with a formal parameter of the form ``*name`` (described in the next subsection) which -receives a tuple containing the positional arguments beyond the formal parameter -list. (``*name`` must occur before ``**name``.) For example, if we define a -function like this:: +receives a :ref:`tuple ` containing the positional +arguments beyond the formal parameter list. (``*name`` must occur +before ``**name``.) For example, if we define a function like this:: def cheeseshop(kind, *arguments, **keywords): print("-- Do you have any", kind, "?") @@ -560,7 +562,7 @@ The reverse situation occurs when the arguments are already in a list or tuple but need to be unpacked for a function call requiring separate positional arguments. For instance, the built-in :func:`range` function expects separate *start* and *stop* arguments. If they are not available separately, write the -function call with the ``*``\ -operator to unpack the arguments out of a list +function call with the ``*`` operator to unpack the arguments out of a list or tuple:: >>> list(range(3, 6)) # normal call with separate arguments @@ -573,7 +575,7 @@ or tuple:: single: **; in function calls In the same fashion, dictionaries can deliver keyword arguments with the -``**``\ -operator:: +``**`` operator:: >>> def parrot(voltage, state='a stiff', action='voom'): ... print("-- This parrot wouldn't", action, end=' ') diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 6812f648..647dd72b 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -23,12 +23,10 @@ is an installation option, other places are possible; check with your local Python guru or system administrator. (E.g., :file:`/usr/local/python` is a popular alternative location.) -On Windows machines, the Python installation is usually placed in -:file:`C:\\Python37`, though you can change this when you're running the -installer. To add this directory to your path, you can type the following -command into the command prompt in a DOS box:: - - set path=%path%;C:\python37 +On Windows machines where you have installed from the :ref:`Microsoft Store +`, the :file:`python3.7` command will be available. If you have +the :ref:`py.exe launcher ` installed, you can use the :file:`py` +command. See :ref:`setting-envvars` for other ways to launch Python. Typing an end-of-file character (:kbd:`Control-D` on Unix, :kbd:`Control-Z` on Windows) at the primary prompt causes the interpreter to exit with a zero exit diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 3e0c9955..a4dbd635 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -383,7 +383,7 @@ items of different types, but usually the items all have the same type. :: >>> squares [1, 4, 9, 16, 25] -Like strings (and all other built-in :term:`sequence` type), lists can be +Like strings (and all other built-in :term:`sequence` types), lists can be indexed and sliced:: >>> squares[0] # indexing returns the item diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index accc3064..d0a68faa 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -425,9 +425,9 @@ your package (expressed in terms of a hierarchical filesystem): When importing the package, Python searches through the directories on ``sys.path`` looking for the package subdirectory. -The :file:`__init__.py` files are required to make Python treat the directories -as containing packages; this is done to prevent directories with a common name, -such as ``string``, from unintentionally hiding valid modules that occur later +The :file:`__init__.py` files are required to make Python treat directories +containing the file as packages. This prevents directories with a common name, +such as ``string``, unintentionally hiding valid modules that occur later on the module search path. In the simplest case, :file:`__init__.py` can just be an empty file, but it can also execute initialization code for the package or set the ``__all__`` variable, described later. @@ -523,7 +523,7 @@ Although certain modules are designed to export only names that follow certain patterns when you use ``import *``, it is still considered bad practice in production code. -Remember, there is nothing wrong with using ``from Package import +Remember, there is nothing wrong with using ``from package import specific_submodule``! In fact, this is the recommended notation unless the importing module needs to use submodules with the same name from different packages. diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst index d876d074..32082013 100644 --- a/Doc/tutorial/whatnow.rst +++ b/Doc/tutorial/whatnow.rst @@ -39,7 +39,7 @@ More Python resources: * https://docs.python.org: Fast access to Python's documentation. * https://pypi.org: The Python Package Index, previously also nicknamed - the Cheese Shop, is an index of user-created Python modules that are available + the Cheese Shop [#]_, is an index of user-created Python modules that are available for download. Once you begin releasing code, you can register it here so that others can find it. @@ -68,3 +68,9 @@ Before posting, be sure to check the list of :ref:`Frequently Asked Questions ` (also called the FAQ). The FAQ answers many of the questions that come up again and again, and may already contain the solution for your problem. + +.. rubric:: Footnotes + +.. [#] "Cheese Shop" is a Monty Python's sketch: a customer enters a cheese shop, + but whatever cheese he asks for, the clerk says it's missing. + diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index a386728e..e685993b 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -25,7 +25,7 @@ there. What you get after installing is a number of things: -* A :file:`MacPython 3.6` folder in your :file:`Applications` folder. In here +* A :file:`Python 3.7` folder in your :file:`Applications` folder. In here you find IDLE, the development environment that is a standard part of official Python distributions; PythonLauncher, which handles double-clicking Python scripts from the Finder; and the "Build Applet" tool, which allows you to @@ -93,7 +93,7 @@ aware of: programs that talk to the Aqua window manager (in other words, anything that has a GUI) need to be run in a special way. Use :program:`pythonw` instead of :program:`python` to start such scripts. -With Python 3.6, you can use either :program:`python` or :program:`pythonw`. +With Python 3.7, you can use either :program:`python` or :program:`pythonw`. Configuration @@ -141,7 +141,7 @@ There are several options for building GUI applications on the Mac with Python. *PyObjC* is a Python binding to Apple's Objective-C/Cocoa framework, which is the foundation of most modern Mac development. Information on PyObjC is -available from https://pythonhosted.org/pyobjc/. +available from https://pypi.org/project/pyobjc/. The standard Python GUI toolkit is :mod:`tkinter`, based on the cross-platform Tk toolkit (https://www.tcl.tk). An Aqua-native version of Tk is bundled with OS diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 8654bc2b..bcc618ca 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -154,7 +154,9 @@ of available options is shown below. | DefaultJustForMeTargetDir | The default install directory for | :file:`%LocalAppData%\\\ | | | just-for-me installs | Programs\\PythonXY` or | | | | :file:`%LocalAppData%\\\ | -| | | Programs\\PythonXY-32` | +| | | Programs\\PythonXY-32` or| +| | | :file:`%LocalAppData%\\\ | +| | | Programs\\PythonXY-64` | +---------------------------+--------------------------------------+--------------------------+ | DefaultCustomTargetDir | The default custom install directory | (empty) | | | displayed in the UI | | @@ -762,9 +764,16 @@ on Windows which you hope will be useful on Unix, you should use one of the shebang lines starting with ``/usr``. Any of the above virtual commands can be suffixed with an explicit version -(either just the major version, or the major and minor version) - for example -``/usr/bin/python2.7`` - which will cause that specific version to be located -and used. +(either just the major version, or the major and minor version). +Furthermore the 32-bit version can be requested by adding "-32" after the +minor version. I.e. ``/usr/bin/python2.7-32`` will request usage of the +32-bit python 2.7. + +.. versionadded:: 3.7 + + Beginning with python launcher 3.7 it is possible to request 64-bit version + by the "-64" suffix. Furthermore it is possible to specify a major and + architecture without minor (i.e. ``/usr/bin/python3-64``). The ``/usr/bin/env`` form of shebang line has one further special property. Before looking for installed Python interpreters, this form will search the @@ -806,17 +815,18 @@ Customizing default Python versions In some cases, a version qualifier can be included in a command to dictate which version of Python will be used by the command. A version qualifier starts with a major version number and can optionally be followed by a period -('.') and a minor version specifier. If the minor qualifier is specified, it -may optionally be followed by "-32" to indicate the 32-bit implementation of -that version be used. +('.') and a minor version specifier. Furthermore it is possible to specifiy +if a 32 or 64 bit implementation shall be requested by adding "-32" or "-64". For example, a shebang line of ``#!python`` has no version qualifier, while ``#!python3`` has a version qualifier which specifies only a major version. -If no version qualifiers are found in a command, the environment variable -``PY_PYTHON`` can be set to specify the default version qualifier - the default -value is "2". Note this value could specify just a major version (e.g. "2") or -a major.minor qualifier (e.g. "2.6"), or even major.minor-32. +If no version qualifiers are found in a command, the environment +variable :envvar:`PY_PYTHON` can be set to specify the default version +qualifier. If it is not set, the default is "3". The variable can +specify any value that may be passed on the command line, such as "3", +"3.7", "3.7-32" or "3.7-64". (Note that the "-64" option is only +available with the launcher included with Python 3.7 or newer.) If no minor version qualifiers are found, the environment variable ``PY_PYTHON{major}`` (where ``{major}`` is the current major version qualifier @@ -834,8 +844,8 @@ of the specified version if available. This is so the behavior of the launcher can be predicted knowing only what versions are installed on the PC and without regard to the order in which they were installed (i.e., without knowing whether a 32 or 64-bit version of Python and corresponding launcher was -installed last). As noted above, an optional "-32" suffix can be used on a -version specifier to change this behaviour. +installed last). As noted above, an optional "-32" or "-64" suffix can be +used on a version specifier to change this behaviour. Examples: @@ -1033,7 +1043,9 @@ The `PyWin32 `_ module by Mark Hammond is a collection of modules for advanced Windows-specific support. This includes utilities for: -* `Component Object Model `_ (COM) +* `Component Object Model + `_ + (COM) * Win32 API calls * Registry * Event log @@ -1099,8 +1111,7 @@ For extension modules, consult :ref:`building-on-windows`. MinGW gcc under Windows" or "Installing Python extension with distutils and without Microsoft Visual C++" by Sébastien Sauvage, 2003 - `MingW -- Python extensions `_ - by Trent Apted et al, 2007 + `MingW -- Python extensions `_ Other Platforms diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 512b8edb..b6174a19 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -227,7 +227,7 @@ the Python community. Sphinx is a standalone package that can be used for writing, and almost two dozen other projects -(`listed on the Sphinx web site `__) +(`listed on the Sphinx web site `__) have adopted Sphinx as their documentation tool. .. seealso:: diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 2d740006..ca3eda05 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -50,7 +50,9 @@ This article explains the new features in Python 3.2 as compared to 3.1. It focuses on a few highlights and gives a few examples. For full details, see the -`Misc/NEWS `_ file. +`Misc/NEWS +`_ +file. .. seealso:: @@ -969,10 +971,10 @@ sites do not finish before midnight, the barrier times-out and the ballots are sealed and deposited in a queue for later handling. See `Barrier Synchronization Patterns -`_ for -more examples of how barriers can be used in parallel computing. Also, there is +`_ +for more examples of how barriers can be used in parallel computing. Also, there is a simple but thorough explanation of barriers in `The Little Book of Semaphores -`_, *section 3.6*. +`_, *section 3.6*. (Contributed by Kristján Valur Jónsson with an API review by Jeffrey Yasskin in :issue:`8777`.) @@ -2512,9 +2514,9 @@ repository. This distributed version control system should make it easier for members of the community to create and share external changesets. See :pep:`385` for details. -To learn to use the new version control system, see the `tutorial by Joel -Spolsky `_ or the `Guide to Mercurial Workflows -`_. +To learn to use the new version control system, see the `Quick Start +`_ or the `Guide to +Mercurial Workflows `_. Build and C API Changes diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index e8572ec0..e78af225 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -1307,7 +1307,7 @@ including failing the host name check now raises :exc:`~ssl.SSLCertVerificationError` and aborts the handshake with a proper TLS Alert message. The new exception contains additional information. Host name validation can be customized with -:attr:`SSLContext.host_flags `. +:attr:`SSLContext.hostname_checks_common_name `. (Contributed by Christian Heimes in :issue:`31399`.) .. note:: @@ -1320,8 +1320,7 @@ The ``ssl`` module no longer sends IP addresses in SNI TLS extension. (Contributed by Christian Heimes in :issue:`32185`.) :func:`~ssl.match_hostname` no longer supports partial wildcards like -``www*.example.org``. :attr:`SSLContext.host_flags ` -has partial wildcard matching disabled by default. +``www*.example.org``. (Contributed by Mandeep Singh in :issue:`23033` and Christian Heimes in :issue:`31399`.) diff --git a/Include/abstract.h b/Include/abstract.h index 4088f75f..3fe5a006 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -220,7 +220,7 @@ PyAPI_FUNC(int) _PyObject_HasFastCall(PyObject *callable); If nargs is equal to zero, args can be NULL. kwargs can be NULL. nargs must be greater or equal to zero. - Return the result on success. Raise an exception on return NULL on + Return the result on success. Raise an exception and return NULL on error. */ PyAPI_FUNC(PyObject *) _PyObject_FastCallDict( PyObject *callable, diff --git a/Include/internal/mem.h b/Include/internal/mem.h index a731e30e..5896e4a0 100644 --- a/Include/internal/mem.h +++ b/Include/internal/mem.h @@ -145,6 +145,30 @@ PyAPI_FUNC(void) _PyGC_Initialize(struct _gc_runtime_state *); #define _PyGC_generation0 _PyRuntime.gc.generation0 +/* Heuristic checking if a pointer value is newly allocated + (uninitialized) or newly freed. The pointer is not dereferenced, only the + pointer value is checked. + + The heuristic relies on the debug hooks on Python memory allocators which + fills newly allocated memory with CLEANBYTE (0xCD) and newly freed memory + with DEADBYTE (0xDD). Detect also "untouchable bytes" marked + with FORBIDDENBYTE (0xFD). */ +static inline int _PyMem_IsPtrFreed(void *ptr) +{ + uintptr_t value = (uintptr_t)ptr; +#if SIZEOF_VOID_P == 8 + return (value == (uintptr_t)0xCDCDCDCDCDCDCDCD + || value == (uintptr_t)0xDDDDDDDDDDDDDDDD + || value == (uintptr_t)0xFDFDFDFDFDFDFDFD); +#elif SIZEOF_VOID_P == 4 + return (value == (uintptr_t)0xCDCDCDCD + || value == (uintptr_t)0xDDDDDDDD + || value == (uintptr_t)0xFDFDFDFD); +#else +# error "unknown pointer size" +#endif +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 721d34fa..5891339b 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -118,6 +118,9 @@ PyAPI_FUNC(void) _PyRuntimeState_Fini(_PyRuntimeState *); Return NULL on success, or return an error message on failure. */ PyAPI_FUNC(_PyInitError) _PyRuntime_Initialize(void); +PyAPI_FUNC(void) _PyRuntime_Finalize(void); + + #define _Py_CURRENTLY_FINALIZING(tstate) \ (_PyRuntime.finalizing == tstate) diff --git a/Include/objimpl.h b/Include/objimpl.h index 057bb50c..0436ba78 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -255,7 +255,11 @@ typedef union _gc_head { union _gc_head *gc_prev; Py_ssize_t gc_refs; } gc; - double dummy; /* force worst-case alignment */ + long double dummy; /* force worst-case alignment */ + // malloc returns memory block aligned for any built-in types and + // long double is the largest standard C type. + // On amd64 linux, long double requires 16 byte alignment. + // See bpo-27987 for more discussion. } PyGC_Head; extern PyGC_Head *_PyGC_generation0; diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 61ed4c48..a388239c 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 7 -#define PY_MICRO_VERSION 3 +#define PY_MICRO_VERSION 4 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.7.3" +#define PY_VERSION "3.7.4" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/pydtrace.h b/Include/pydtrace.h index 037961d4..7a042781 100644 --- a/Include/pydtrace.h +++ b/Include/pydtrace.h @@ -29,7 +29,7 @@ static inline void PyDTrace_LINE(const char *arg0, const char *arg1, int arg2) { static inline void PyDTrace_FUNCTION_ENTRY(const char *arg0, const char *arg1, int arg2) {} static inline void PyDTrace_FUNCTION_RETURN(const char *arg0, const char *arg1, int arg2) {} static inline void PyDTrace_GC_START(int arg0) {} -static inline void PyDTrace_GC_DONE(int arg0) {} +static inline void PyDTrace_GC_DONE(Py_ssize_t arg0) {} static inline void PyDTrace_INSTANCE_NEW_START(int arg0) {} static inline void PyDTrace_INSTANCE_NEW_DONE(int arg0) {} static inline void PyDTrace_INSTANCE_DELETE_START(int arg0) {} diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 05f17dc7..5d9f049d 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -139,7 +139,9 @@ PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetPath(void); #ifdef Py_BUILD_CORE PyAPI_FUNC(_PyInitError) _PyPathConfig_Init(const _PyCoreConfig *core_config); -PyAPI_FUNC(PyObject*) _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv); +PyAPI_FUNC(int) _PyPathConfig_ComputeArgv0( + int argc, wchar_t **argv, + PyObject **argv0_p); PyAPI_FUNC(int) _Py_FindEnvConfigValue( FILE *env_file, const wchar_t *key, diff --git a/Include/pymem.h b/Include/pymem.h index ef6e0bb5..458a6489 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -55,8 +55,6 @@ PyAPI_FUNC(int) PyTraceMalloc_Untrack( PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback( unsigned int domain, uintptr_t ptr); - -PyAPI_FUNC(int) _PyMem_IsFreed(void *ptr, size_t size); #endif /* !defined(Py_LIMITED_API) */ diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py index a2cae54b..2e46a076 100644 --- a/Lib/_dummy_thread.py +++ b/Lib/_dummy_thread.py @@ -14,7 +14,7 @@ Suggested usage is:: # Exports only things specified by thread documentation; # skipping obsolete synonyms allocate(), start_new(), exit_thread(). __all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock', - 'interrupt_main', 'LockType'] + 'interrupt_main', 'LockType', 'RLock'] # A dummy value TIMEOUT_MAX = 2**31 @@ -148,6 +148,36 @@ class LockType(object): hex(id(self)) ) + +class RLock(LockType): + """Dummy implementation of threading._RLock. + + Re-entrant lock can be aquired multiple times and needs to be released + just as many times. This dummy implemention does not check wheter the + current thread actually owns the lock, but does accounting on the call + counts. + """ + def __init__(self): + super().__init__() + self._levels = 0 + + def acquire(self, waitflag=None, timeout=-1): + """Aquire the lock, can be called multiple times in succession. + """ + locked = super().acquire(waitflag, timeout) + if locked: + self._levels += 1 + return locked + + def release(self): + """Release needs to be called once for every call to acquire(). + """ + if self._levels == 0: + raise error + if self._levels == 1: + super().release() + self._levels -= 1 + # Used to signal that interrupt_main was called in a "thread" _interrupt = False # True when not executing in a "thread" diff --git a/Lib/_pyio.py b/Lib/_pyio.py index f0d4f4ed..e81cc512 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -287,16 +287,15 @@ class IOBase(metaclass=abc.ABCMeta): derived classes can override selectively; the default implementations represent a file that cannot be read, written or seeked. - Even though IOBase does not declare read, readinto, or write because + Even though IOBase does not declare read or write because their signatures will vary, implementations and clients should consider those methods part of the interface. Also, implementations may raise UnsupportedOperation when operations they do not support are called. The basic type used for binary data read from or written to a file is - bytes. Other bytes-like objects are accepted as method arguments too. In - some cases (such as readinto), a writable object is required. Text I/O - classes work with str data. + bytes. Other bytes-like objects are accepted as method arguments too. + Text I/O classes work with str data. Note that calling any method (even inquiries) on a closed stream is undefined. Implementations may raise OSError in this case. @@ -547,6 +546,11 @@ class IOBase(metaclass=abc.ABCMeta): return lines def writelines(self, lines): + """Write a list of lines to the stream. + + Line separators are not added, so it is usual for each of the lines + provided to have a line separator at the end. + """ self._checkClosed() for line in lines: self.write(line) @@ -835,6 +839,10 @@ class BytesIO(BufferedIOBase): """Buffered I/O implementation using an in-memory bytes buffer.""" + # Initialize _buffer as soon as possible since it's used by __del__() + # which calls close() + _buffer = None + def __init__(self, initial_bytes=None): buf = bytearray() if initial_bytes is not None: @@ -862,7 +870,8 @@ class BytesIO(BufferedIOBase): return memoryview(self._buffer) def close(self): - self._buffer.clear() + if self._buffer is not None: + self._buffer.clear() super().close() def read(self, size=-1): @@ -1759,8 +1768,7 @@ class TextIOBase(IOBase): """Base class for text I/O. This class provides a character and line based interface to stream - I/O. There is no readinto method because Python's character strings - are immutable. There is no public constructor. + I/O. There is no public constructor. """ def read(self, size=-1): @@ -1933,6 +1941,10 @@ class TextIOWrapper(TextIOBase): _CHUNK_SIZE = 2048 + # Initialize _buffer as soon as possible since it's used by __del__() + # which calls close() + _buffer = None + # The write_through argument has no effect here since this # implementation always writes through. The argument is present only # so that the signature can match the signature of the C version. diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 65e0529e..52134372 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -56,11 +56,6 @@ _MIN_SCHEDULED_TIMER_HANDLES = 100 # before cleanup of cancelled handles is performed. _MIN_CANCELLED_TIMER_HANDLES_FRACTION = 0.5 -# Exceptions which must not call the exception handler in fatal error -# methods (_fatal_error()) -_FATAL_ERROR_IGNORE = (BrokenPipeError, - ConnectionResetError, ConnectionAbortedError) - _HAS_IPv6 = hasattr(socket, 'AF_INET6') # Maximum timeout passed to select to avoid OS limitations @@ -96,7 +91,7 @@ def _set_reuseport(sock): 'SO_REUSEPORT defined but not implemented.') -def _ipaddr_info(host, port, family, type, proto): +def _ipaddr_info(host, port, family, type, proto, flowinfo=0, scopeid=0): # Try to skip getaddrinfo if "host" is already an IP. Users might have # handled name resolution in their own code and pass in resolved IPs. if not hasattr(socket, 'inet_pton'): @@ -145,7 +140,7 @@ def _ipaddr_info(host, port, family, type, proto): socket.inet_pton(af, host) # The host has already been resolved. if _HAS_IPv6 and af == socket.AF_INET6: - return af, type, proto, '', (host, port, 0, 0) + return af, type, proto, '', (host, port, flowinfo, scopeid) else: return af, type, proto, '', (host, port) except OSError: @@ -832,7 +827,7 @@ class BaseEventLoop(events.AbstractEventLoop): read = await self.run_in_executor(None, file.readinto, view) if not read: break # EOF - await self.sock_sendall(sock, view) + await self.sock_sendall(sock, view[:read]) total_sent += read return total_sent finally: @@ -1083,11 +1078,11 @@ class BaseEventLoop(events.AbstractEventLoop): if blocksize <= 0: return total_sent view = memoryview(buf)[:blocksize] - read = file.readinto(view) + read = await self.run_in_executor(None, file.readinto, view) if not read: return total_sent # EOF await proto.drain() - transp.write(view) + transp.write(view[:read]) total_sent += read finally: if total_sent > 0 and hasattr(file, 'seek'): @@ -1229,7 +1224,8 @@ class BaseEventLoop(events.AbstractEventLoop): if local_addr: sock.bind(local_address) if remote_addr: - await self.sock_connect(sock, remote_address) + if not allow_broadcast: + await self.sock_connect(sock, remote_address) r_addr = remote_address except OSError as exc: if sock is not None: @@ -1270,7 +1266,7 @@ class BaseEventLoop(events.AbstractEventLoop): family=0, type=socket.SOCK_STREAM, proto=0, flags=0, loop): host, port = address[:2] - info = _ipaddr_info(host, port, family, type, proto) + info = _ipaddr_info(host, port, family, type, proto, *address[2:]) if info is not None: # "host" is already a resolved IP. return [info] diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index a638cced..0296e0f7 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -96,7 +96,7 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin, def _fatal_error(self, exc, message='Fatal error on pipe transport'): try: - if isinstance(exc, base_events._FATAL_ERROR_IGNORE): + if isinstance(exc, OSError): if self._loop.get_debug(): logger.debug("%r: %s", self, message, exc_info=True) else: diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index d2861d34..23bd8ad8 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -578,7 +578,10 @@ class _SelectorTransport(transports._FlowControlMixin, def __init__(self, loop, sock, protocol, extra=None, server=None): super().__init__(extra, loop) self._extra['socket'] = sock - self._extra['sockname'] = sock.getsockname() + try: + self._extra['sockname'] = sock.getsockname() + except OSError: + self._extra['sockname'] = None if 'peername' not in self._extra: try: self._extra['peername'] = sock.getpeername() @@ -657,7 +660,7 @@ class _SelectorTransport(transports._FlowControlMixin, def _fatal_error(self, exc, message='Fatal error on transport'): # Should be called from exception handler only. - if isinstance(exc, base_events._FATAL_ERROR_IGNORE): + if isinstance(exc, OSError): if self._loop.get_debug(): logger.debug("%r: %s", self, message, exc_info=True) else: @@ -968,9 +971,11 @@ class _SelectorDatagramTransport(_SelectorTransport): if not data: return - if self._address and addr not in (None, self._address): - raise ValueError( - f'Invalid address: must be None or {self._address}') + if self._address: + if addr not in (None, self._address): + raise ValueError( + f'Invalid address: must be None or {self._address}') + addr = self._address if self._conn_lost and self._address: if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: @@ -981,7 +986,7 @@ class _SelectorDatagramTransport(_SelectorTransport): if not self._buffer: # Attempt to send it right away first. try: - if self._address: + if self._extra['peername']: self._sock.send(data) else: self._sock.sendto(data, addr) @@ -1004,7 +1009,7 @@ class _SelectorDatagramTransport(_SelectorTransport): while self._buffer: data, addr = self._buffer.popleft() try: - if self._address: + if self._extra['peername']: self._sock.send(data) else: self._sock.sendto(data, addr) diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 12fdb0d1..02d29738 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -499,7 +499,11 @@ class SSLProtocol(protocols.Protocol): self._app_transport._closed = True self._transport = None self._app_transport = None + if getattr(self, '_handshake_timeout_handle', None): + self._handshake_timeout_handle.cancel() self._wakeup_waiter(exc) + self._app_protocol = None + self._sslpipe = None def pause_writing(self): """Called when the low-level transport's buffer goes over @@ -696,7 +700,7 @@ class SSLProtocol(protocols.Protocol): self._fatal_error(exc, 'Fatal error on SSL transport') def _fatal_error(self, exc, message='Fatal error on transport'): - if isinstance(exc, base_events._FATAL_ERROR_IGNORE): + if isinstance(exc, OSError): if self._loop.get_debug(): logger.debug("%r: %s", self, message, exc_info=True) else: diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 416c346b..c82b95e3 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -35,9 +35,22 @@ def all_tasks(loop=None): """Return a set of all tasks for the loop.""" if loop is None: loop = events.get_running_loop() - # NB: set(_all_tasks) is required to protect - # from https://bugs.python.org/issue34970 bug - return {t for t in list(_all_tasks) + # Looping over a WeakSet (_all_tasks) isn't safe as it can be updated from another + # thread while we do so. Therefore we cast it to list prior to filtering. The list + # cast itself requires iteration, so we repeat it several times ignoring + # RuntimeErrors (which are not very likely to occur). See issues 34970 and 36607 for + # details. + i = 0 + while True: + try: + tasks = list(_all_tasks) + except RuntimeError: + i += 1 + if i >= 1000: + raise + else: + break + return {t for t in tasks if futures._get_loop(t) is loop and not t.done()} @@ -47,9 +60,22 @@ def _all_tasks_compat(loop=None): # method. if loop is None: loop = events.get_event_loop() - # NB: set(_all_tasks) is required to protect - # from https://bugs.python.org/issue34970 bug - return {t for t in list(_all_tasks) if futures._get_loop(t) is loop} + # Looping over a WeakSet (_all_tasks) isn't safe as it can be updated from another + # thread while we do so. Therefore we cast it to list prior to filtering. The list + # cast itself requires iteration, so we repeat it several times ignoring + # RuntimeErrors (which are not very likely to occur). See issues 34970 and 36607 for + # details. + i = 0 + while True: + try: + tasks = list(_all_tasks) + except RuntimeError: + i += 1 + if i >= 1000: + raise + else: + break + return {t for t in tasks if futures._get_loop(t) is loop} class Task(futures._PyFuture): # Inherit Python Task implementation @@ -457,10 +483,11 @@ async def _wait(fs, timeout, return_when, loop): finally: if timeout_handle is not None: timeout_handle.cancel() + for f in fs: + f.remove_done_callback(_on_completion) done, pending = set(), set() for f in fs: - f.remove_done_callback(_on_completion) if f.done(): done.add(f) else: @@ -773,7 +800,7 @@ def shield(arg, *, loop=None): loop = futures._get_loop(inner) outer = loop.create_future() - def _done_callback(inner): + def _inner_done_callback(inner): if outer.cancelled(): if not inner.cancelled(): # Mark inner's result as retrieved. @@ -789,7 +816,13 @@ def shield(arg, *, loop=None): else: outer.set_result(inner.result()) - inner.add_done_callback(_done_callback) + + def _outer_done_callback(outer): + if not inner.done(): + inner.remove_done_callback(_inner_done_callback) + + inner.add_done_callback(_inner_done_callback) + outer.add_done_callback(_outer_done_callback) return outer diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 639300f9..a0fc996d 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -717,7 +717,7 @@ class _UnixWritePipeTransport(transports._FlowControlMixin, def _fatal_error(self, exc, message='Fatal error on pipe transport'): # should be called by exception handler only - if isinstance(exc, base_events._FATAL_ERROR_IGNORE): + if isinstance(exc, OSError): if self._loop.get_debug(): logger.debug("%r: %s", self, message, exc_info=True) else: @@ -758,12 +758,18 @@ class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): # other end). Notably this is needed on AIX, and works # just fine on other platforms. stdin, stdin_w = socket.socketpair() - self._proc = subprocess.Popen( - args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr, - universal_newlines=False, bufsize=bufsize, **kwargs) - if stdin_w is not None: - stdin.close() - self._proc.stdin = open(stdin_w.detach(), 'wb', buffering=bufsize) + try: + self._proc = subprocess.Popen( + args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr, + universal_newlines=False, bufsize=bufsize, **kwargs) + if stdin_w is not None: + stdin.close() + self._proc.stdin = open(stdin_w.detach(), 'wb', buffering=bufsize) + stdin_w = None + finally: + if stdin_w is not None: + stdin.close() + stdin_w.close() class AbstractChildWatcher: diff --git a/Lib/bdb.py b/Lib/bdb.py index 25c6260c..caf20773 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -616,11 +616,23 @@ class Bdb: # This method is more useful to debug a single function call. - def runcall(self, func, *args, **kwds): + def runcall(*args, **kwds): """Debug a single function call. Return the result of the function call. """ + if len(args) >= 2: + self, func, *args = args + elif not args: + raise TypeError("descriptor 'runcall' of 'Bdb' object " + "needs an argument") + elif 'func' in kwds: + func = kwds.pop('func') + self, *args = args + else: + raise TypeError('runcall expected at least 1 positional argument, ' + 'got %d' % (len(args)-1)) + self.reset() sys.settrace(self.trace_dispatch) res = None diff --git a/Lib/cProfile.py b/Lib/cProfile.py index f6e423b3..9485d3d5 100755 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -103,7 +103,19 @@ class Profile(_lsprof.Profiler): return self # This method is more useful to profile a single function call. - def runcall(self, func, *args, **kw): + def runcall(*args, **kw): + if len(args) >= 2: + self, func, *args = args + elif not args: + raise TypeError("descriptor 'runcall' of 'Profile' object " + "needs an argument") + elif 'func' in kw: + func = kw.pop('func') + self, *args = args + else: + raise TypeError('runcall expected at least 1 positional argument, ' + 'got %d' % (len(args)-1)) + self.enable() try: return func(*args, **kw) diff --git a/Lib/codecs.py b/Lib/codecs.py index a70ed20f..cfca5d38 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -838,7 +838,7 @@ class StreamRecoder: def writelines(self, list): - data = ''.join(list) + data = b''.join(list) data, bytesdecoded = self.decode(data, self.errors) return self.writer.write(data) @@ -847,6 +847,12 @@ class StreamRecoder: self.reader.reset() self.writer.reset() + def seek(self, offset, whence=0): + # Seeks must be propagated to both the readers and writers + # as they might need to reset their internal buffers. + self.reader.seek(offset, whence) + self.writer.seek(offset, whence) + def __getattr__(self, name, getattr=getattr): diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 9a753db7..64bbee8f 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -443,6 +443,8 @@ def namedtuple(typename, field_names, *, rename=False, defaults=None, module=Non '__doc__': f'{typename}({arg_list})', '__slots__': (), '_fields': field_names, + '_field_defaults': field_defaults, + # alternate spelling for backward compatiblity '_fields_defaults': field_defaults, '__new__': __new__, '_make': _make, @@ -1034,6 +1036,13 @@ class UserDict(_collections_abc.MutableMapping): # Now, add the methods in dicts but not in MutableMapping def __repr__(self): return repr(self.data) + def __copy__(self): + inst = self.__class__.__new__(self.__class__) + inst.__dict__.update(self.__dict__) + # Create a copy and avoid triggering descriptors + inst.__dict__["data"] = self.__dict__["data"].copy() + return inst + def copy(self): if self.__class__ is UserDict: return UserDict(self.data.copy()) @@ -1046,6 +1055,7 @@ class UserDict(_collections_abc.MutableMapping): self.data = data c.update(self) return c + @classmethod def fromkeys(cls, iterable, value=None): d = cls() @@ -1081,7 +1091,11 @@ class UserList(_collections_abc.MutableSequence): return other.data if isinstance(other, UserList) else other def __contains__(self, item): return item in self.data def __len__(self): return len(self.data) - def __getitem__(self, i): return self.data[i] + def __getitem__(self, i): + if isinstance(i, slice): + return self.__class__(self.data[i]) + else: + return self.data[i] def __setitem__(self, i, item): self.data[i] = item def __delitem__(self, i): del self.data[i] def __add__(self, other): @@ -1110,6 +1124,12 @@ class UserList(_collections_abc.MutableSequence): def __imul__(self, n): self.data *= n return self + def __copy__(self): + inst = self.__class__.__new__(self.__class__) + inst.__dict__.update(self.__dict__) + # Create a copy and avoid triggering descriptors + inst.__dict__["data"] = self.__dict__["data"][:] + return inst def append(self, item): self.data.append(item) def insert(self, i, item): self.data.insert(i, item) def pop(self, i=-1): return self.data.pop(i) diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index 61c81bb7..46d30a50 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -400,7 +400,10 @@ class Future(object): if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]: self._done_callbacks.append(fn) return - fn(self) + try: + fn(self) + except Exception: + LOGGER.exception('exception calling callback for %r', self) def result(self, timeout=None): """Return the result of the call that the future represents. @@ -536,7 +539,7 @@ class Future(object): class Executor(object): """This is an abstract base class for concrete asynchronous executors.""" - def submit(self, fn, *args, **kwargs): + def submit(*args, **kwargs): """Submits a callable to be executed with the given arguments. Schedules the callable to be executed as fn(*args, **kwargs) and returns @@ -545,6 +548,15 @@ class Executor(object): Returns: A Future representing the given call. """ + if len(args) >= 2: + pass + elif not args: + raise TypeError("descriptor 'submit' of 'Executor' object " + "needs an argument") + elif 'fn' not in kwargs: + raise TypeError('submit expected at least 1 positional argument, ' + 'got %d' % (len(args)-1)) + raise NotImplementedError() def map(self, fn, *iterables, timeout=None, chunksize=1): diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index ce7d642b..9106552c 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -57,6 +57,7 @@ import threading import weakref from functools import partial import itertools +import sys import traceback # Workers are created as daemon threads and processes. This is done to allow the @@ -109,6 +110,12 @@ def _python_exit(): EXTRA_QUEUED_CALLS = 1 +# On Windows, WaitForMultipleObjects is used to wait for processes to finish. +# It can wait on, at most, 63 objects. There is an overhead of two objects: +# - the result queue reader +# - the thread wakeup reader +_MAX_WINDOWS_WORKERS = 63 - 2 + # Hack to embed stringification of remote traceback in local traceback class _RemoteTraceback(Exception): @@ -497,16 +504,23 @@ class ProcessPoolExecutor(_base.Executor): worker processes will be created as the machine has processors. mp_context: A multiprocessing context to launch the workers. This object should provide SimpleQueue, Queue and Process. - initializer: An callable used to initialize worker processes. + initializer: A callable used to initialize worker processes. initargs: A tuple of arguments to pass to the initializer. """ _check_system_limits() if max_workers is None: self._max_workers = os.cpu_count() or 1 + if sys.platform == 'win32': + self._max_workers = min(_MAX_WINDOWS_WORKERS, + self._max_workers) else: if max_workers <= 0: raise ValueError("max_workers must be greater than 0") + elif (sys.platform == 'win32' and + max_workers > _MAX_WINDOWS_WORKERS): + raise ValueError( + f"max_workers must be <= {_MAX_WINDOWS_WORKERS}") self._max_workers = max_workers @@ -593,7 +607,19 @@ class ProcessPoolExecutor(_base.Executor): p.start() self._processes[p.pid] = p - def submit(self, fn, *args, **kwargs): + def submit(*args, **kwargs): + if len(args) >= 2: + self, fn, *args = args + elif not args: + raise TypeError("descriptor 'submit' of 'ProcessPoolExecutor' object " + "needs an argument") + elif 'fn' in kwargs: + fn = kwargs.pop('fn') + self, *args = args + else: + raise TypeError('submit expected at least 1 positional argument, ' + 'got %d' % (len(args)-1)) + with self._shutdown_lock: if self._broken: raise BrokenProcessPool(self._broken) diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py index 78359711..9e3fb8b6 100644 --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -118,7 +118,7 @@ class ThreadPoolExecutor(_base.Executor): max_workers: The maximum number of threads that can be used to execute the given calls. thread_name_prefix: An optional name prefix to give our threads. - initializer: An callable used to initialize worker threads. + initializer: A callable used to initialize worker threads. initargs: A tuple of arguments to pass to the initializer. """ if max_workers is None: @@ -142,7 +142,19 @@ class ThreadPoolExecutor(_base.Executor): self._initializer = initializer self._initargs = initargs - def submit(self, fn, *args, **kwargs): + def submit(*args, **kwargs): + if len(args) >= 2: + self, fn, *args = args + elif not args: + raise TypeError("descriptor 'submit' of 'ThreadPoolExecutor' object " + "needs an argument") + elif 'fn' in kwargs: + fn = kwargs.pop('fn') + self, *args = args + else: + raise TypeError('submit expected at least 1 positional argument, ' + 'got %d' % (len(args)-1)) + with self._shutdown_lock: if self._broken: raise BrokenThreadPool(self._broken) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 1ff8cdf1..2d745ea3 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -378,7 +378,8 @@ class _BaseExitStack: return _exit_wrapper @staticmethod - def _create_cb_wrapper(callback, *args, **kwds): + def _create_cb_wrapper(*args, **kwds): + callback, *args = args def _exit_wrapper(exc_type, exc, tb): callback(*args, **kwds) return _exit_wrapper @@ -427,11 +428,23 @@ class _BaseExitStack: self._push_cm_exit(cm, _exit) return result - def callback(self, callback, *args, **kwds): + def callback(*args, **kwds): """Registers an arbitrary callback and arguments. Cannot suppress exceptions. """ + if len(args) >= 2: + self, callback, *args = args + elif not args: + raise TypeError("descriptor 'callback' of '_BaseExitStack' object " + "needs an argument") + elif 'callback' in kwds: + callback = kwds.pop('callback') + self, *args = args + else: + raise TypeError('callback expected at least 1 positional argument, ' + 'got %d' % (len(args)-1)) + _exit_wrapper = self._create_cb_wrapper(callback, *args, **kwds) # We changed the signature, so using @wraps is not appropriate, but @@ -540,7 +553,8 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager): return _exit_wrapper @staticmethod - def _create_async_cb_wrapper(callback, *args, **kwds): + def _create_async_cb_wrapper(*args, **kwds): + callback, *args = args async def _exit_wrapper(exc_type, exc, tb): await callback(*args, **kwds) return _exit_wrapper @@ -575,11 +589,23 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager): self._push_async_cm_exit(exit, exit_method) return exit # Allow use as a decorator - def push_async_callback(self, callback, *args, **kwds): + def push_async_callback(*args, **kwds): """Registers an arbitrary coroutine function and arguments. Cannot suppress exceptions. """ + if len(args) >= 2: + self, callback, *args = args + elif not args: + raise TypeError("descriptor 'push_async_callback' of " + "'AsyncExitStack' object needs an argument") + elif 'callback' in kwds: + callback = kwds.pop('callback') + self, *args = args + else: + raise TypeError('push_async_callback expected at least 1 ' + 'positional argument, got %d' % (len(args)-1)) + _exit_wrapper = self._create_async_cb_wrapper(callback, *args, **kwds) # We changed the signature, so using @wraps is not appropriate, but diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 61467739..dae408a8 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -279,7 +279,15 @@ def create_unicode_buffer(init, size=None): """ if isinstance(init, str): if size is None: - size = len(init)+1 + if sizeof(c_wchar) == 2: + # UTF-16 requires a surrogate pair (2 wchar_t) for non-BMP + # characters (outside [U+0000; U+FFFF] range). +1 for trailing + # NUL character. + size = sum(2 if ord(c) > 0xFFFF else 1 for c in init) + 1 + else: + # 32-bit wchar_t (1 wchar_t per Unicode character). +1 for + # trailing NUL character. + size = len(init) + 1 buftype = c_wchar * size buf = buftype() buf.value = init diff --git a/Lib/ctypes/test/test_arrays.py b/Lib/ctypes/test/test_arrays.py index 6e562cfd..63a00b9e 100644 --- a/Lib/ctypes/test/test_arrays.py +++ b/Lib/ctypes/test/test_arrays.py @@ -69,6 +69,17 @@ class ArrayTestCase(unittest.TestCase): from operator import delitem self.assertRaises(TypeError, delitem, ca, 0) + def test_step_overflow(self): + a = (c_int * 5)() + a[3::sys.maxsize] = (1,) + self.assertListEqual(a[3::sys.maxsize], [1]) + a = (c_char * 5)() + a[3::sys.maxsize] = b"A" + self.assertEqual(a[3::sys.maxsize], b"A") + a = (c_wchar * 5)() + a[3::sys.maxsize] = u"X" + self.assertEqual(a[3::sys.maxsize], u"X") + def test_numeric_arrays(self): alen = 5 @@ -183,6 +194,27 @@ class ArrayTestCase(unittest.TestCase): _type_ = c_int _length_ = 1.87 + def test_empty_element_struct(self): + class EmptyStruct(Structure): + _fields_ = [] + + obj = (EmptyStruct * 2)() # bpo37188: Floating point exception + self.assertEqual(sizeof(obj), 0) + + def test_empty_element_array(self): + class EmptyArray(Array): + _type_ = c_int + _length_ = 0 + + obj = (EmptyArray * 2)() # bpo37188: Floating point exception + self.assertEqual(sizeof(obj), 0) + + def test_bpo36504_signed_int_overflow(self): + # The overflow check in PyCArrayType_new() could cause signed integer + # overflow. + with self.assertRaises(OverflowError): + c_char * sys.maxsize * 2 + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @bigmemtest(size=_2G, memuse=1, dry_run=False) def test_large_array(self, size): diff --git a/Lib/ctypes/test/test_buffers.py b/Lib/ctypes/test/test_buffers.py index 166faaf4..15782be7 100644 --- a/Lib/ctypes/test/test_buffers.py +++ b/Lib/ctypes/test/test_buffers.py @@ -60,5 +60,14 @@ class StringBufferTestCase(unittest.TestCase): self.assertEqual(b[::2], "ac") self.assertEqual(b[::5], "a") + @need_symbol('c_wchar') + def test_create_unicode_buffer_non_bmp(self): + expected = 5 if sizeof(c_wchar) == 2 else 3 + for s in '\U00010000\U00100000', '\U00010000\U0010ffff': + b = create_unicode_buffer(s) + self.assertEqual(len(b), expected) + self.assertEqual(b[-1], '\0') + + if __name__ == "__main__": unittest.main() diff --git a/Lib/curses/__init__.py b/Lib/curses/__init__.py index 47378741..b3373af7 100644 --- a/Lib/curses/__init__.py +++ b/Lib/curses/__init__.py @@ -60,7 +60,7 @@ except NameError: # raises an exception, wrapper() will restore the terminal to a sane state so # you can read the resulting traceback. -def wrapper(func, *args, **kwds): +def wrapper(*args, **kwds): """Wrapper function that initializes curses and calls another function, restoring normal keyboard/screen behavior on error. The callable object 'func' is then passed the main window 'stdscr' @@ -68,6 +68,14 @@ def wrapper(func, *args, **kwds): wrapper(). """ + if args: + func, *args = args + elif 'func' in kwds: + func = kwds.pop('func') + else: + raise TypeError('wrapper expected at least 1 positional argument, ' + 'got %d' % len(args)) + try: # Initialize curses stdscr = initscr() diff --git a/Lib/distutils/command/build.py b/Lib/distutils/command/build.py index c6f52e61..a86df0bc 100644 --- a/Lib/distutils/command/build.py +++ b/Lib/distutils/command/build.py @@ -116,7 +116,7 @@ class build(Command): self.build_scripts = os.path.join(self.build_base, 'scripts-%d.%d' % sys.version_info[:2]) - if self.executable is None: + if self.executable is None and sys.executable: self.executable = os.path.normpath(sys.executable) if isinstance(self.parallel, str): diff --git a/Lib/distutils/command/check.py b/Lib/distutils/command/check.py index 7ebe707c..04c2f964 100644 --- a/Lib/distutils/command/check.py +++ b/Lib/distutils/command/check.py @@ -120,7 +120,8 @@ class check(Command): def _check_rst_data(self, data): """Returns warnings when the provided data doesn't compile.""" - source_path = StringIO() + # the include and csv_table directives need this to be a path + source_path = self.distribution.script_name or 'setup.py' parser = Parser() settings = frontend.OptionParser(components=(Parser,)).get_default_values() settings.tab_width = 4 diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py index 53876880..d3a12c28 100644 --- a/Lib/distutils/spawn.py +++ b/Lib/distutils/spawn.py @@ -172,21 +172,32 @@ def find_executable(executable, path=None): A string listing directories separated by 'os.pathsep'; defaults to os.environ['PATH']. Returns the complete filename or None if not found. """ - if path is None: - path = os.environ.get('PATH', os.defpath) - - paths = path.split(os.pathsep) - base, ext = os.path.splitext(executable) - + _, ext = os.path.splitext(executable) if (sys.platform == 'win32') and (ext != '.exe'): executable = executable + '.exe' - if not os.path.isfile(executable): - for p in paths: - f = os.path.join(p, executable) - if os.path.isfile(f): - # the file exists, we have a shot at spawn working - return f - return None - else: + if os.path.isfile(executable): return executable + + if path is None: + path = os.environ.get('PATH', None) + if path is None: + try: + path = os.confstr("CS_PATH") + except (AttributeError, ValueError): + # os.confstr() or CS_PATH is not available + path = os.defpath + # bpo-35755: Don't use os.defpath if the PATH environment variable is + # set to an empty string + + # PATH='' doesn't match, whereas PATH=':' looks in the current directory + if not path: + return None + + paths = path.split(os.pathsep) + for p in paths: + f = os.path.join(p, executable) + if os.path.isfile(f): + # the file exists, we have a shot at spawn working + return f + return None diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 83160f8d..0a034ee0 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -28,7 +28,12 @@ BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) if "_PYTHON_PROJECT_BASE" in os.environ: project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"]) else: - project_base = os.path.dirname(os.path.abspath(sys.executable)) + if sys.executable: + project_base = os.path.dirname(os.path.abspath(sys.executable)) + else: + # sys.executable can be empty if argv[0] has been changed and Python is + # unable to retrieve the real program name + project_base = os.getcwd() # python_build: (Boolean) if true, we're either building Python or @@ -183,8 +188,8 @@ def customize_compiler(compiler): _osx_support.customize_compiler(_config_vars) _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' - (cc, cxx, opt, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \ - get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS', + (cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \ + get_config_vars('CC', 'CXX', 'CFLAGS', 'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS') if 'CC' in os.environ: @@ -207,7 +212,7 @@ def customize_compiler(compiler): if 'LDFLAGS' in os.environ: ldshared = ldshared + ' ' + os.environ['LDFLAGS'] if 'CFLAGS' in os.environ: - cflags = opt + ' ' + os.environ['CFLAGS'] + cflags = cflags + ' ' + os.environ['CFLAGS'] ldshared = ldshared + ' ' + os.environ['CFLAGS'] if 'CPPFLAGS' in os.environ: cpp = cpp + ' ' + os.environ['CPPFLAGS'] diff --git a/Lib/distutils/tests/includetest.rst b/Lib/distutils/tests/includetest.rst new file mode 100644 index 00000000..d7b4ae38 --- /dev/null +++ b/Lib/distutils/tests/includetest.rst @@ -0,0 +1 @@ +This should be included. diff --git a/Lib/distutils/tests/test_check.py b/Lib/distutils/tests/test_check.py index 3d22868e..e534aca1 100644 --- a/Lib/distutils/tests/test_check.py +++ b/Lib/distutils/tests/test_check.py @@ -1,4 +1,5 @@ """Tests for distutils.command.check.""" +import os import textwrap import unittest from test.support import run_unittest @@ -13,13 +14,19 @@ except ImportError: pygments = None +HERE = os.path.dirname(__file__) + + class CheckTestCase(support.LoggingSilencer, support.TempdirManager, unittest.TestCase): - def _run(self, metadata=None, **options): + def _run(self, metadata=None, cwd=None, **options): if metadata is None: metadata = {} + if cwd is not None: + old_dir = os.getcwd() + os.chdir(cwd) pkg_info, dist = self.create_dist(**metadata) cmd = check(dist) cmd.initialize_options() @@ -27,6 +34,8 @@ class CheckTestCase(support.LoggingSilencer, setattr(cmd, name, value) cmd.ensure_finalized() cmd.run() + if cwd is not None: + os.chdir(old_dir) return cmd def test_check_metadata(self): @@ -99,6 +108,11 @@ class CheckTestCase(support.LoggingSilencer, cmd = self._run(metadata, strict=1, restructuredtext=1) self.assertEqual(cmd._warnings, 0) + # check that includes work to test #31292 + metadata['long_description'] = 'title\n=====\n\n.. include:: includetest.rst' + cmd = self._run(metadata, cwd=HERE, strict=1, restructuredtext=1) + self.assertEqual(cmd._warnings, 0) + @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils") def test_check_restructuredtext_with_syntax_highlight(self): # Don't fail if there is a `code` or `code-block` directive diff --git a/Lib/distutils/tests/test_spawn.py b/Lib/distutils/tests/test_spawn.py index 0d455385..f9ae69ef 100644 --- a/Lib/distutils/tests/test_spawn.py +++ b/Lib/distutils/tests/test_spawn.py @@ -87,11 +87,52 @@ class SpawnTestCase(support.TempdirManager, rv = find_executable(dont_exist_program , path=tmp_dir) self.assertIsNone(rv) - # test os.defpath: missing PATH environment variable + # PATH='': no match, except in the current directory with test_support.EnvironmentVarGuard() as env: - with mock.patch('distutils.spawn.os.defpath', tmp_dir): - env.pop('PATH') + env['PATH'] = '' + with unittest.mock.patch('distutils.spawn.os.confstr', + return_value=tmp_dir, create=True), \ + unittest.mock.patch('distutils.spawn.os.defpath', + tmp_dir): + rv = find_executable(program) + self.assertIsNone(rv) + + # look in current directory + with test_support.change_cwd(tmp_dir): + rv = find_executable(program) + self.assertEqual(rv, program) + + # PATH=':': explicitly looks in the current directory + with test_support.EnvironmentVarGuard() as env: + env['PATH'] = os.pathsep + with unittest.mock.patch('distutils.spawn.os.confstr', + return_value='', create=True), \ + unittest.mock.patch('distutils.spawn.os.defpath', ''): + rv = find_executable(program) + self.assertIsNone(rv) + + # look in current directory + with test_support.change_cwd(tmp_dir): + rv = find_executable(program) + self.assertEqual(rv, program) + + # missing PATH: test os.confstr("CS_PATH") and os.defpath + with test_support.EnvironmentVarGuard() as env: + env.pop('PATH', None) + + # without confstr + with unittest.mock.patch('distutils.spawn.os.confstr', + side_effect=ValueError, + create=True), \ + unittest.mock.patch('distutils.spawn.os.defpath', + tmp_dir): + rv = find_executable(program) + self.assertEqual(rv, filename) + # with confstr + with unittest.mock.patch('distutils.spawn.os.confstr', + return_value=tmp_dir, create=True), \ + unittest.mock.patch('distutils.spawn.os.defpath', ''): rv = find_executable(program) self.assertEqual(rv, filename) diff --git a/Lib/distutils/tests/test_sysconfig.py b/Lib/distutils/tests/test_sysconfig.py index fe4a2994..236755d0 100644 --- a/Lib/distutils/tests/test_sysconfig.py +++ b/Lib/distutils/tests/test_sysconfig.py @@ -1,4 +1,5 @@ """Tests for distutils.sysconfig.""" +import contextlib import os import shutil import subprocess @@ -9,7 +10,7 @@ import unittest from distutils import sysconfig from distutils.ccompiler import get_default_compiler from distutils.tests import support -from test.support import TESTFN, run_unittest, check_warnings +from test.support import TESTFN, run_unittest, check_warnings, swap_item class SysconfigTestCase(support.EnvironGuard, unittest.TestCase): def setUp(self): @@ -74,12 +75,7 @@ class SysconfigTestCase(support.EnvironGuard, unittest.TestCase): os.chdir(cwd) self.assertEqual(srcdir, srcdir2) - @unittest.skipUnless(get_default_compiler() == 'unix', - 'not testing if default compiler is not unix') - def test_customize_compiler(self): - os.environ['AR'] = 'my_ar' - os.environ['ARFLAGS'] = '-arflags' - + def customize_compiler(self): # make sure AR gets caught class compiler: compiler_type = 'unix' @@ -87,9 +83,89 @@ class SysconfigTestCase(support.EnvironGuard, unittest.TestCase): def set_executables(self, **kw): self.exes = kw + sysconfig_vars = { + 'AR': 'sc_ar', + 'CC': 'sc_cc', + 'CXX': 'sc_cxx', + 'ARFLAGS': '--sc-arflags', + 'CFLAGS': '--sc-cflags', + 'CCSHARED': '--sc-ccshared', + 'LDSHARED': 'sc_ldshared', + 'SHLIB_SUFFIX': 'sc_shutil_suffix', + + # On macOS, disable _osx_support.customize_compiler() + 'CUSTOMIZED_OSX_COMPILER': 'True', + } + comp = compiler() - sysconfig.customize_compiler(comp) - self.assertEqual(comp.exes['archiver'], 'my_ar -arflags') + with contextlib.ExitStack() as cm: + for key, value in sysconfig_vars.items(): + cm.enter_context(swap_item(sysconfig._config_vars, key, value)) + sysconfig.customize_compiler(comp) + + return comp + + @unittest.skipUnless(get_default_compiler() == 'unix', + 'not testing if default compiler is not unix') + def test_customize_compiler(self): + # Make sure that sysconfig._config_vars is initialized + sysconfig.get_config_vars() + + os.environ['AR'] = 'env_ar' + os.environ['CC'] = 'env_cc' + os.environ['CPP'] = 'env_cpp' + os.environ['CXX'] = 'env_cxx --env-cxx-flags' + os.environ['LDSHARED'] = 'env_ldshared' + os.environ['LDFLAGS'] = '--env-ldflags' + os.environ['ARFLAGS'] = '--env-arflags' + os.environ['CFLAGS'] = '--env-cflags' + os.environ['CPPFLAGS'] = '--env-cppflags' + + comp = self.customize_compiler() + self.assertEqual(comp.exes['archiver'], + 'env_ar --env-arflags') + self.assertEqual(comp.exes['preprocessor'], + 'env_cpp --env-cppflags') + self.assertEqual(comp.exes['compiler'], + 'env_cc --sc-cflags --env-cflags --env-cppflags') + self.assertEqual(comp.exes['compiler_so'], + ('env_cc --sc-cflags ' + '--env-cflags ''--env-cppflags --sc-ccshared')) + self.assertEqual(comp.exes['compiler_cxx'], + 'env_cxx --env-cxx-flags') + self.assertEqual(comp.exes['linker_exe'], + 'env_cc') + self.assertEqual(comp.exes['linker_so'], + ('env_ldshared --env-ldflags --env-cflags' + ' --env-cppflags')) + self.assertEqual(comp.shared_lib_extension, 'sc_shutil_suffix') + + del os.environ['AR'] + del os.environ['CC'] + del os.environ['CPP'] + del os.environ['CXX'] + del os.environ['LDSHARED'] + del os.environ['LDFLAGS'] + del os.environ['ARFLAGS'] + del os.environ['CFLAGS'] + del os.environ['CPPFLAGS'] + + comp = self.customize_compiler() + self.assertEqual(comp.exes['archiver'], + 'sc_ar --sc-arflags') + self.assertEqual(comp.exes['preprocessor'], + 'sc_cc -E') + self.assertEqual(comp.exes['compiler'], + 'sc_cc --sc-cflags') + self.assertEqual(comp.exes['compiler_so'], + 'sc_cc --sc-cflags --sc-ccshared') + self.assertEqual(comp.exes['compiler_cxx'], + 'sc_cxx') + self.assertEqual(comp.exes['linker_exe'], + 'sc_cc') + self.assertEqual(comp.exes['linker_so'], + 'sc_ldshared') + self.assertEqual(comp.shared_lib_extension, 'sc_shutil_suffix') def test_parse_makefile_base(self): self.makefile = TESTFN diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 416da1a8..fc00b4a0 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -68,6 +68,7 @@ XXX: provide complete list of token types. """ import re +import sys import urllib # For urllib.parse.unquote from string import hexdigits from collections import OrderedDict @@ -96,6 +97,18 @@ EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') def quote_string(value): return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' +# Match a RFC 2047 word, looks like =?utf-8?q?someword?= +rfc2047_matcher = re.compile(r''' + =\? # literal =? + [^?]* # charset + \? # literal ? + [qQbB] # literal 'q' or 'b', case insensitive + \? # literal ? + .*? # encoded word + \?= # literal ?= +''', re.VERBOSE | re.MULTILINE) + + # # TokenList and its subclasses # @@ -1049,6 +1062,10 @@ def get_encoded_word(value): _validate_xtext(vtext) ew.append(vtext) text = ''.join(remainder) + # Encoded words should be followed by a WS + if value and value[0] not in WSP: + ew.defects.append(errors.InvalidHeaderDefect( + "missing trailing whitespace after encoded-word")) return ew, value def get_unstructured(value): @@ -1101,6 +1118,11 @@ def get_unstructured(value): unstructured.append(token) continue tok, *remainder = _wsp_splitter(value, 1) + # Split in the middle of an atom if there is a rfc2047 encoded word + # which does not have WSP on both sides. The defect will be registered + # the next time through the loop. + if rfc2047_matcher.search(tok): + tok, *remainder = value.partition('=?') vtext = ValueTerminal(tok, 'vtext') _validate_xtext(vtext) unstructured.append(vtext) @@ -2591,7 +2613,7 @@ def _refold_parse_tree(parse_tree, *, policy): """ # max_line_length 0/None means no limit, ie: infinitely long. - maxlen = policy.max_line_length or float("+inf") + maxlen = policy.max_line_length or sys.maxsize encoding = 'utf-8' if policy.utf8 else 'us-ascii' lines = [''] last_ew = None @@ -2626,7 +2648,7 @@ def _refold_parse_tree(parse_tree, *, policy): want_encoding = False last_ew = None if part.syntactic_break: - encoded_part = part.fold(policy=policy)[:-1] # strip nl + encoded_part = part.fold(policy=policy)[:-len(policy.linesep)] if policy.linesep not in encoded_part: # It fits on a single line if len(encoded_part) > maxlen - len(lines[-1]): @@ -2661,6 +2683,7 @@ def _refold_parse_tree(parse_tree, *, policy): newline = _steal_trailing_WSP_if_exists(lines) if newline or part.startswith_fws(): lines.append(newline + tstr) + last_ew = None continue if not hasattr(part, 'encode'): # It's not a terminal, try folding the subparts. @@ -2724,16 +2747,19 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): lines.append(' ') # XXX We'll get an infinite loop here if maxlen is <= 7 continue - first_part = to_encode[:text_space] - ew = _ew.encode(first_part, charset=encode_as) - excess = len(ew) - remaining_space - if excess > 0: - # encode always chooses the shortest encoding, so this - # is guaranteed to fit at this point. - first_part = first_part[:-excess] - ew = _ew.encode(first_part) - lines[-1] += ew - to_encode = to_encode[len(first_part):] + + to_encode_word = to_encode[:text_space] + encoded_word = _ew.encode(to_encode_word, charset=encode_as) + excess = len(encoded_word) - remaining_space + while excess > 0: + # Since the chunk to encode is guaranteed to fit into less than 100 characters, + # shrinking it by one at a time shouldn't take long. + to_encode_word = to_encode_word[:-1] + encoded_word = _ew.encode(to_encode_word, charset=encode_as) + excess = len(encoded_word) - remaining_space + lines[-1] += encoded_word + to_encode = to_encode[len(to_encode_word):] + if to_encode: lines.append(' ') new_last_ew = len(lines[-1]) diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 7c07ca86..97d3f514 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -320,7 +320,7 @@ class FeedParser: self._cur.set_payload(EMPTYSTRING.join(lines)) return # Make sure a valid content type was specified per RFC 2045:6.4. - if (self._cur.get('content-transfer-encoding', '8bit').lower() + if (str(self._cur.get('content-transfer-encoding', '8bit')).lower() not in ('7bit', '8bit', 'binary')): defect = errors.InvalidMultipartContentTransferEncodingDefect() self.policy.handle_defect(self._cur, defect) diff --git a/Lib/email/header.py b/Lib/email/header.py index 7b30a039..4ab0032b 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -431,7 +431,7 @@ class _ValueFormatter: if end_of_line != (' ', ''): self._current_line.push(*end_of_line) if len(self._current_line) > 0: - if self._current_line.is_onlyws(): + if self._current_line.is_onlyws() and self._lines: self._lines[-1] += str(self._current_line) else: self._lines.append(str(self._current_line)) diff --git a/Lib/email/policy.py b/Lib/email/policy.py index 5131311a..611deb50 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -3,6 +3,7 @@ code that adds all the email6 features. """ import re +import sys from email._policybase import Policy, Compat32, compat32, _extend_docstrings from email.utils import _has_surrogates from email.headerregistry import HeaderRegistry as HeaderRegistry @@ -203,7 +204,7 @@ class EmailPolicy(Policy): def _fold(self, name, value, refold_binary=False): if hasattr(value, 'name'): return value.fold(policy=self) - maxlen = self.max_line_length if self.max_line_length else float('inf') + maxlen = self.max_line_length if self.max_line_length else sys.maxsize lines = value.splitlines() refold = (self.refold_source == 'all' or self.refold_source == 'long' and diff --git a/Lib/fileinput.py b/Lib/fileinput.py index c6fc9a19..c8b8b236 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -8,9 +8,9 @@ Typical use is: This iterates over the lines of all files listed in sys.argv[1:], defaulting to sys.stdin if the list is empty. If a filename is '-' it -is also replaced by sys.stdin. To specify an alternative list of -filenames, pass it as the argument to input(). A single file name is -also allowed. +is also replaced by sys.stdin and the optional arguments mode and +openhook are ignored. To specify an alternative list of filenames, +pass it as the argument to input(). A single file name is also allowed. Functions filename(), lineno() return the filename and cumulative line number of the line that has just been read; filelineno() returns its diff --git a/Lib/functools.py b/Lib/functools.py index b734899b..1daa1d17 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -323,7 +323,20 @@ class partialmethod(object): callables as instance methods. """ - def __init__(self, func, *args, **keywords): + def __init__(*args, **keywords): + if len(args) >= 2: + self, func, *args = args + elif not args: + raise TypeError("descriptor '__init__' of partialmethod " + "needs an argument") + elif 'func' in keywords: + func = keywords.pop('func') + self, *args = args + else: + raise TypeError("type 'partialmethod' takes at least one argument, " + "got %d" % (len(args)-1)) + args = tuple(args) + if not callable(func) and not hasattr(func, "__get__"): raise TypeError("{!r} is not callable or a descriptor" .format(func)) diff --git a/Lib/http/client.py b/Lib/http/client.py index 5aa178d7..dd23edcd 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -140,6 +140,16 @@ _MAXHEADERS = 100 _is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch _is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search +# These characters are not allowed within HTTP URL paths. +# See https://tools.ietf.org/html/rfc3986#section-3.3 and the +# https://tools.ietf.org/html/rfc3986#appendix-A pchar definition. +# Prevents CVE-2019-9740. Includes control characters such as \r\n. +# We don't restrict chars above \x7f as putrequest() limits us to ASCII. +_contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]') +# Arguably only these _should_ allowed: +# _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$") +# We are more lenient for assumed real world compatibility purposes. + # We always set the Content-Length header for these methods because some # servers will otherwise respond with a 411 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} @@ -320,8 +330,8 @@ class HTTPResponse(io.BufferedIOBase): self.headers = self.msg = parse_headers(self.fp) if self.debuglevel > 0: - for hdr in self.headers: - print("header:", hdr + ":", self.headers.get(hdr)) + for hdr, val in self.headers.items(): + print("header:", hdr + ":", val) # are we using the chunked-style of transfer encoding? tr_enc = self.headers.get("transfer-encoding") @@ -1101,6 +1111,11 @@ class HTTPConnection: self._method = method if not url: url = '/' + # Prevent CVE-2019-9740. + match = _contains_disallowed_url_pchar_re.search(url) + if match: + raise InvalidURL(f"URL can't contain control characters. {url!r} " + f"(found at least {match.group()!r})") request = '%s %s %s' % (method, url, self._http_vsn_str) # Non-ASCII characters should have been eliminated earlier @@ -1366,6 +1381,9 @@ else: self.cert_file = cert_file if context is None: context = ssl._create_default_https_context() + # enable PHA for TLS 1.3 connections if available + if context.post_handshake_auth is not None: + context.post_handshake_auth = True will_verify = context.verify_mode != ssl.CERT_NONE if check_hostname is None: check_hostname = context.check_hostname @@ -1374,6 +1392,10 @@ else: "either CERT_OPTIONAL or CERT_REQUIRED") if key_file or cert_file: context.load_cert_chain(cert_file, key_file) + # cert and key file means the user wants to authenticate. + # enable TLS 1.3 PHA implicitly even for custom contexts. + if context.post_handshake_auth is not None: + context.post_handshake_auth = True self._context = context if check_hostname is not None: self._context.check_hostname = check_hostname diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index c343538a..42227b60 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -1,8 +1,53 @@ -What's New in IDLE 3.7.3 -Released on 2019-??-?? +What's New in IDLE 3.7.4 +Released on 2019-06-24? ====================================== +bpo-37321: Both subprocess connection error messages now refer to +the 'Startup failure' section of the IDLE doc. + +bpo-37039: Adjust "Zoom Height" to individual screens by momemtarily +maximizing the window on first use with a particular screen. Changing +screen settings may invalidate the saved height. While a window is +maximized, "Zoom Height" has no effect. + +bpo-35763: Make calltip reminder about '/' meaning positional-only less +obtrusive by only adding it when there is room on the first line. + +bpo-35610: Replace now redundant editor.context_use_ps1 with +.prompt_last_line. This finishes change started in bpo-31858. + +bpo-32411: Stop sorting dict created with desired line order. + +bpo-37038: Make idlelib.run runnable; add test clause. + +bpo-36958: Print any argument other than None or int passed to +SystemExit or sys.exit(). + +bpo-36807: When saving a file, call file.flush() and os.fsync() +so bits are flushed to e.g. a USB drive. + +bpo-36429: Fix starting IDLE with pyshell. +Add idlelib.pyshell alias at top; remove pyshell alias at bottom. +Remove obsolete __name__=='__main__' command. + +bpo-30348: Increase test coverage of idlelib.autocomplete by 30%. +Patch by Louie Lu. + +bpo-23205: Add tests and refactor grep's findfiles. + +bpo-36405: Use dict unpacking in idlelib. + +bpo-36396: Remove fgBg param of idlelib.config.GetHighlight(). +This param was only used twice and changed the return type. + +bpo-23216: IDLE: Add docstrings to search modules. + + +What's New in IDLE 3.7.3 +Released on 2019-03-25 +====================================== + bpo-36176: Fix IDLE autocomplete & calltip popup colors. Prevent conflicts with Linux dark themes (and slightly darken calltip background). @@ -58,7 +103,7 @@ Add some internal references within the IDLE doc. What's New in IDLE 3.7.2 -Released on 2018-12-21? +Released on 2018-12-24 ====================================== bpo-34864: When starting IDLE on MacOS, warn if the system setting @@ -101,7 +146,7 @@ The main change is the elimination of chapter-section numbers. What's New in IDLE 3.7.1 -Released on 2018-07-31? +Released on 2018-10-20 ====================================== bpo-1529353: Output over N lines (50 by default) is squeezed down to a button. diff --git a/Lib/idlelib/autocomplete.py b/Lib/idlelib/autocomplete.py index 9caf50d5..e20b757d 100644 --- a/Lib/idlelib/autocomplete.py +++ b/Lib/idlelib/autocomplete.py @@ -3,6 +3,7 @@ Either on demand or after a user-selected delay after a key character, pop up a list of candidates. """ +import __main__ import os import string import sys @@ -14,7 +15,6 @@ COMPLETE_ATTRIBUTES, COMPLETE_FILES = range(1, 2+1) from idlelib import autocomplete_w from idlelib.config import idleConf from idlelib.hyperparser import HyperParser -import __main__ # This string includes all chars that may be in an identifier. # TODO Update this here and elsewhere. @@ -104,9 +104,14 @@ class AutoComplete: def open_completions(self, evalfuncs, complete, userWantsWin, mode=None): """Find the completions and create the AutoCompleteWindow. Return True if successful (no syntax error or so found). - if complete is True, then if there's nothing to complete and no + If complete is True, then if there's nothing to complete and no start of completion, won't open completions and return False. If mode is given, will open a completion list only in this mode. + + Action Function Eval Complete WantWin Mode + ^space force_open_completions True, False, True no + . or / try_open_completions False, False, False yes + tab autocomplete False, True, True no """ # Cancel another delayed call, if it exists. if self._delayed_completion_id is not None: @@ -117,11 +122,11 @@ class AutoComplete: curline = self.text.get("insert linestart", "insert") i = j = len(curline) if hp.is_in_string() and (not mode or mode==COMPLETE_FILES): - # Find the beginning of the string - # fetch_completions will look at the file system to determine whether the - # string value constitutes an actual file name - # XXX could consider raw strings here and unescape the string value if it's - # not raw. + # Find the beginning of the string. + # fetch_completions will look at the file system to determine + # whether the string value constitutes an actual file name + # XXX could consider raw strings here and unescape the string + # value if it's not raw. self._remove_autocomplete_window() mode = COMPLETE_FILES # Find last separator or string start @@ -182,8 +187,8 @@ class AutoComplete: else: if mode == COMPLETE_ATTRIBUTES: if what == "": - namespace = __main__.__dict__.copy() - namespace.update(__main__.__builtins__.__dict__) + namespace = {**__main__.__builtins__.__dict__, + **__main__.__dict__} bigl = eval("dir()", namespace) bigl.sort() if "__all__" in bigl: @@ -218,10 +223,8 @@ class AutoComplete: return smalll, bigl def get_entity(self, name): - """Lookup name in a namespace spanning sys.modules and __main.dict__""" - namespace = sys.modules.copy() - namespace.update(__main__.__dict__) - return eval(name, namespace) + "Lookup name in a namespace spanning sys.modules and __main.dict__." + return eval(name, {**sys.modules, **__main__.__dict__}) AutoComplete.reload() diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 234883fe..e5b0bc53 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -29,9 +29,10 @@ def transform_children(child_dict, modname=None): The dictionary maps names to pyclbr information objects. Filter out imported objects. Augment class names with bases. - Sort objects by line number. + The insertion order of the dictonary is assumed to have been in line + number order, so sorting is not necessary. - The current tree only calls this once per child_dic as it saves + The current tree only calls this once per child_dict as it saves TreeItems once created. A future tree and tests might violate this, so a check prevents multiple in-place augmentations. """ @@ -51,7 +52,7 @@ def transform_children(child_dict, modname=None): supers.append(sname) obj.name += '({})'.format(', '.join(supers)) obs.append(obj) - return sorted(obs, key=lambda o: o.lineno) + return obs class ModuleBrowser: diff --git a/Lib/idlelib/calltip.py b/Lib/idlelib/calltip.py index 2a9a131e..a3dda267 100644 --- a/Lib/idlelib/calltip.py +++ b/Lib/idlelib/calltip.py @@ -4,6 +4,7 @@ Call Tips are floating windows which display function, class, and method parameter and docstring information when you type an opening parenthesis, and which disappear when you type a closing parenthesis. """ +import __main__ import inspect import re import sys @@ -12,7 +13,6 @@ import types from idlelib import calltip_w from idlelib.hyperparser import HyperParser -import __main__ class Calltip: @@ -103,10 +103,9 @@ def get_entity(expression): in a namespace spanning sys.modules and __main.dict__. """ if expression: - namespace = sys.modules.copy() - namespace.update(__main__.__dict__) + namespace = {**sys.modules, **__main__.__dict__} try: - return eval(expression, namespace) + return eval(expression, namespace) # Only protect user code. except BaseException: # An uncaught exception closes idle, and eval can raise any # exception, especially if user classes are involved. @@ -119,7 +118,7 @@ _INDENT = ' '*4 # for wrapped signatures _first_param = re.compile(r'(?<=\()\w*\,?\s*') _default_callable_argspec = "See source or doc" _invalid_method = "invalid method signature" -_argument_positional = "\n['/' marks preceding arguments as positional-only]\n" +_argument_positional = " # '/' marks preceding args as positional-only." def get_argspec(ob): '''Return a string describing the signature of a callable object, or ''. @@ -145,11 +144,11 @@ def get_argspec(ob): if msg.startswith(_invalid_method): return _invalid_method - if '/' in argspec: - """Using AC's positional argument should add the explain""" + if '/' in argspec and len(argspec) < _MAX_COLS - len(_argument_positional): + # Add explanation TODO remove after 3.7, before 3.9. argspec += _argument_positional if isinstance(fob, type) and argspec == '()': - """fob with no argument, use default callable argspec""" + # If fob has no argument, use default callable argspec. argspec = _default_callable_argspec lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT) diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index facbef8b..db1266fe 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -40,7 +40,7 @@ def color_config(text): # Not automatic because ColorDelegator does not know 'text'. theme = idleConf.CurrentTheme() normal_colors = idleConf.GetHighlight(theme, 'normal') - cursor_color = idleConf.GetHighlight(theme, 'cursor', fgBg='fg') + cursor_color = idleConf.GetHighlight(theme, 'cursor')['foreground'] select_colors = idleConf.GetHighlight(theme, 'hilite') text.config( foreground=normal_colors['foreground'], diff --git a/Lib/idlelib/config-keys.def b/Lib/idlelib/config-keys.def index fd235194..f71269b5 100644 --- a/Lib/idlelib/config-keys.def +++ b/Lib/idlelib/config-keys.def @@ -63,6 +63,7 @@ force-open-calltip= format-paragraph= flash-paren= run-module= +run-custom= check-module= zoom-height= @@ -122,6 +123,7 @@ force-open-calltip= format-paragraph= flash-paren= run-module= +run-custom= check-module= zoom-height= @@ -181,6 +183,7 @@ force-open-calltip= format-paragraph= flash-paren= run-module= +run-custom= check-module= zoom-height= @@ -240,6 +243,7 @@ force-open-calltip= format-paragraph= flash-paren= run-module= +run-custom= check-module= zoom-height= @@ -300,5 +304,6 @@ force-open-calltip= format-paragraph= flash-paren= run-module= +run-custom= check-module= zoom-height= diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 79d988f9..2233dacd 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -12,7 +12,7 @@ database keys are tuples (config-type, section, item). As implemented, there are separate dicts for default and user values. Each has config-type keys 'main', 'extensions', 'highlight', and 'keys'. The value for each key is a ConfigParser instance that maps section and item -to values. For 'main' and 'extenstons', user values override +to values. For 'main' and 'extensions', user values override default values. For 'highlight' and 'keys', user sections augment the default sections (and must, therefore, have distinct names). @@ -34,7 +34,6 @@ import idlelib class InvalidConfigType(Exception): pass class InvalidConfigSet(Exception): pass -class InvalidFgBg(Exception): pass class InvalidTheme(Exception): pass class IdleConfParser(ConfigParser): @@ -283,34 +282,20 @@ class IdleConf: raise InvalidConfigSet('Invalid configSet specified') return cfgParser.sections() - def GetHighlight(self, theme, element, fgBg=None): - """Return individual theme element highlight color(s). + def GetHighlight(self, theme, element): + """Return dict of theme element highlight colors. - fgBg - string ('fg' or 'bg') or None. - If None, return a dictionary containing fg and bg colors with - keys 'foreground' and 'background'. Otherwise, only return - fg or bg color, as specified. Colors are intended to be - appropriate for passing to Tkinter in, e.g., a tag_config call). + The keys are 'foreground' and 'background'. The values are + tkinter color strings for configuring backgrounds and tags. """ - if self.defaultCfg['highlight'].has_section(theme): - themeDict = self.GetThemeDict('default', theme) - else: - themeDict = self.GetThemeDict('user', theme) - fore = themeDict[element + '-foreground'] - if element == 'cursor': # There is no config value for cursor bg - back = themeDict['normal-background'] - else: - back = themeDict[element + '-background'] - highlight = {"foreground": fore, "background": back} - if not fgBg: # Return dict of both colors - return highlight - else: # Return specified color only - if fgBg == 'fg': - return highlight["foreground"] - if fgBg == 'bg': - return highlight["background"] - else: - raise InvalidFgBg('Invalid fgBg specified') + cfg = ('default' if self.defaultCfg['highlight'].has_section(theme) + else 'user') + theme_dict = self.GetThemeDict(cfg, theme) + fore = theme_dict[element + '-foreground'] + if element == 'cursor': + element = 'normal' + back = theme_dict[element + '-background'] + return {"foreground": fore, "background": back} def GetThemeDict(self, type, themeName): """Return {option:value} dict for elements in themeName. @@ -606,7 +591,9 @@ class IdleConf: former_extension_events = { # Those with user-configurable keys. '<>', '<>', '<>', '<>', '<>', - '<>', '<>', '<>'} + '<>', '<>', '<>', + '<>', + } def GetCoreKeys(self, keySetName=None): """Return dict of core virtual-key keybindings for keySetName. @@ -673,6 +660,7 @@ class IdleConf: '<>': [''], '<>': [''], '<>': [''], + '<>': [''], '<>': [''], '<>': [''], } diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 5fdaf82d..807ff604 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -199,7 +199,7 @@ class ConfigDialog(Toplevel): def help(self): """Create textview for config dialog help. - Attrbutes accessed: + Attributes accessed: note Methods: @@ -1252,7 +1252,7 @@ class HighPage(Frame): colors = idleConf.GetHighlight(theme, element) if element == 'cursor': # Cursor sample needs special painting. colors['background'] = idleConf.GetHighlight( - theme, 'normal', fgBg='bg') + theme, 'normal')['background'] # Handle any unsaved changes to this theme. if theme in changes['highlight']: theme_dict = changes['highlight'][theme] @@ -2225,7 +2225,7 @@ key set, with a different name. 'General': ''' General: -AutoComplete: Popupwait is milleseconds to wait after key char, without +AutoComplete: Popupwait is milliseconds to wait after key char, without cursor movement, before popping up completion box. Key char is '.' after identifier or a '/' (or '\\' on Windows) within a string. diff --git a/Lib/idlelib/delegator.py b/Lib/idlelib/delegator.py index dc2a1aae..55c95da8 100644 --- a/Lib/idlelib/delegator.py +++ b/Lib/idlelib/delegator.py @@ -14,7 +14,7 @@ class Delegator: def resetcache(self): "Removes added attributes while leaving original attributes." - # Function is really about resetting delagator dict + # Function is really about resetting delegator dict # to original state. Cache is just a means for key in self.__cache: try: diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 83260329..606de71a 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -228,10 +228,6 @@ class EditorWindow(object): self.indentwidth = self.tabwidth self.set_notabs_indentwidth() - # If context_use_ps1 is true, parsing searches back for a ps1 line; - # else searches for a popular (if, def, ...) Python stmt. - self.context_use_ps1 = False - # When searching backwards for a reliable place to begin parsing, # first start num_context_lines[0] lines back, then # num_context_lines[1] lines back if that didn't work, and so on. @@ -308,6 +304,7 @@ class EditorWindow(object): scriptbinding = ScriptBinding(self) text.bind("<>", scriptbinding.check_module_event) text.bind("<>", scriptbinding.run_module_event) + text.bind("<>", scriptbinding.run_custom_event) text.bind("<>", self.Rstrip(self).do_rstrip) ctip = self.Calltip(self) text.bind("<>", ctip.try_open_calltip_event) @@ -319,7 +316,7 @@ class EditorWindow(object): self.CodeContext(self).toggle_code_context_event) def _filename_to_unicode(self, filename): - """Return filename as BMP unicode so diplayable in Tk.""" + """Return filename as BMP unicode so displayable in Tk.""" # Decode bytes to unicode. if isinstance(filename, bytes): try: @@ -1337,14 +1334,13 @@ class EditorWindow(object): # open/close first need to find the last stmt lno = index2line(text.index('insert')) y = pyparse.Parser(self.indentwidth, self.tabwidth) - if not self.context_use_ps1: + if not self.prompt_last_line: for context in self.num_context_lines: startat = max(lno - context, 1) startatindex = repr(startat) + ".0" rawtext = text.get(startatindex, "insert") y.set_code(rawtext) bod = y.find_good_parse_start( - self.context_use_ps1, self._build_char_in_string_func(startatindex)) if bod is not None or startat == 1: break diff --git a/Lib/idlelib/grep.py b/Lib/idlelib/grep.py index 873233ec..12513594 100644 --- a/Lib/idlelib/grep.py +++ b/Lib/idlelib/grep.py @@ -14,11 +14,16 @@ from idlelib.searchbase import SearchDialogBase from idlelib import searchengine # Importing OutputWindow here fails due to import loop -# EditorWindow -> GrepDialop -> OutputWindow -> EditorWindow +# EditorWindow -> GrepDialog -> OutputWindow -> EditorWindow def grep(text, io=None, flist=None): - """Create or find singleton GrepDialog instance. + """Open the Find in Files dialog. + + Module-level function to access the singleton GrepDialog + instance and open the dialog. If text is selected, it is + used as the search phrase; otherwise, the previous entry + is used. Args: text: Text widget that contains the selected text for @@ -26,7 +31,6 @@ def grep(text, io=None, flist=None): io: iomenu.IOBinding instance with default path to search. flist: filelist.FileList instance for OutputWindow parent. """ - root = text._root() engine = searchengine.get(root) if not hasattr(engine, "_grepdialog"): @@ -36,6 +40,27 @@ def grep(text, io=None, flist=None): dialog.open(text, searchphrase, io) +def walk_error(msg): + "Handle os.walk error." + print(msg) + + +def findfiles(folder, pattern, recursive): + """Generate file names in dir that match pattern. + + Args: + folder: Root directory to search. + pattern: File pattern to match. + recursive: True to include subdirectories. + """ + for dirpath, _, filenames in os.walk(folder, onerror=walk_error): + yield from (os.path.join(dirpath, name) + for name in filenames + if fnmatch.fnmatch(name, pattern)) + if not recursive: + break + + class GrepDialog(SearchDialogBase): "Dialog for searching multiple files." @@ -50,17 +75,29 @@ class GrepDialog(SearchDialogBase): searchengine instance to prepare the search. Attributes: - globvar: Value of Text Entry widget for path to search. - recvar: Boolean value of Checkbutton widget - for traversing through subdirectories. + flist: filelist.Filelist instance for OutputWindow parent. + globvar: String value of Entry widget for path to search. + globent: Entry widget for globvar. Created in + create_entries(). + recvar: Boolean value of Checkbutton widget for + traversing through subdirectories. """ - SearchDialogBase.__init__(self, root, engine) + super().__init__(root, engine) self.flist = flist self.globvar = StringVar(root) self.recvar = BooleanVar(root) def open(self, text, searchphrase, io=None): - "Make dialog visible on top of others and ready to use." + """Make dialog visible on top of others and ready to use. + + Extend the SearchDialogBase open() to set the initial value + for globvar. + + Args: + text: Multicall object containing the text information. + searchphrase: String phrase to search. + io: iomenu.IOBinding instance containing file path. + """ SearchDialogBase.open(self, text, searchphrase) if io: path = io.filename or "" @@ -85,9 +122,9 @@ class GrepDialog(SearchDialogBase): btn.pack(side="top", fill="both") def create_command_buttons(self): - "Create base command buttons and add button for search." + "Create base command buttons and add button for Search Files." SearchDialogBase.create_command_buttons(self) - self.make_button("Search Files", self.default_command, 1) + self.make_button("Search Files", self.default_command, isdef=True) def default_command(self, event=None): """Grep for search pattern in file path. The default command is bound @@ -119,16 +156,21 @@ class GrepDialog(SearchDialogBase): search each line for the matching pattern. If the pattern is found, write the file and line information to stdout (which is an OutputWindow). + + Args: + prog: The compiled, cooked search pattern. + path: String containing the search path. """ - dir, base = os.path.split(path) - list = self.findfiles(dir, base, self.recvar.get()) - list.sort() + folder, filepat = os.path.split(path) + if not folder: + folder = os.curdir + filelist = sorted(findfiles(folder, filepat, self.recvar.get())) self.close() pat = self.engine.getpat() print(f"Searching {pat!r} in {path} ...") hits = 0 try: - for fn in list: + for fn in filelist: try: with open(fn, errors='replace') as f: for lineno, line in enumerate(f, 1): @@ -146,30 +188,6 @@ class GrepDialog(SearchDialogBase): # so in OW.write, OW.text.insert fails. pass - def findfiles(self, dir, base, rec): - """Return list of files in the dir that match the base pattern. - - If rec is True, recursively iterate through subdirectories. - """ - try: - names = os.listdir(dir or os.curdir) - except OSError as msg: - print(msg) - return [] - list = [] - subdirs = [] - for name in names: - fn = os.path.join(dir, name) - if os.path.isdir(fn): - subdirs.append(fn) - else: - if fnmatch.fnmatch(name, base): - list.append(fn) - if rec: - for subdir in subdirs: - list.extend(self.findfiles(subdir, base, rec)) - return list - def _grep_dialog(parent): # htest # from tkinter import Toplevel, Text, SEL, END diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index b654ab7f..91803fd0 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -6,7 +6,7 @@ - IDLE — Python 3.8.0a1 documentation + IDLE — Python 3.9.0a0 documentation @@ -19,7 +19,7 @@ @@ -50,6 +50,7 @@ +
FormatPacked as .zipPacked as .tar.bz2