From 9307c5d28bd33ce69e90e047c96f09cac4d16ce2 Mon Sep 17 00:00:00 2001 From: DongHun Kwak Date: Mon, 16 Oct 2017 20:04:11 +0900 Subject: [PATCH] Imported Upstream version 3.6.2 Change-Id: Ib672f0d36e295bb49ce443b7a82b92ae1a5774b1 Signed-off-by: DongHun Kwak --- .github/PULL_REQUEST_TEMPLATE.md | 9 - .gitignore | 96 -- Doc/Makefile | 9 +- Doc/c-api/exceptions.rst | 222 ++- Doc/c-api/long.rst | 13 +- Doc/c-api/marshal.rst | 8 +- Doc/c-api/memory.rst | 2 +- Doc/c-api/module.rst | 2 +- Doc/c-api/slice.rst | 11 + Doc/c-api/structures.rst | 4 +- Doc/c-api/unicode.rst | 95 +- Doc/distutils/examples.rst | 2 +- Doc/extending/newtypes.rst | 2 +- Doc/glossary.rst | 20 +- Doc/howto/argparse.rst | 2 +- Doc/howto/clinic.rst | 6 +- Doc/howto/descriptor.rst | 2 +- Doc/howto/functional.rst | 29 +- Doc/howto/logging-cookbook.rst | 2 +- Doc/howto/unicode.rst | 6 +- Doc/howto/urllib2.rst | 10 +- Doc/library/2to3.rst | 7 - Doc/library/asyncio-protocol.rst | 2 +- Doc/library/asyncio-subprocess.rst | 4 +- Doc/library/asyncio-task.rst | 5 + Doc/library/binhex.rst | 2 - Doc/library/cmd.rst | 4 +- Doc/library/collections.rst | 8 +- Doc/library/configparser.rst | 4 +- Doc/library/constants.rst | 2 +- Doc/library/copy.rst | 4 +- Doc/library/csv.rst | 6 +- Doc/library/ctypes.rst | 60 +- Doc/library/curses.rst | 72 +- Doc/library/datetime.rst | 4 +- Doc/library/dis.rst | 106 +- Doc/library/doctest.rst | 6 +- Doc/library/email.compat32-message.rst | 2 +- Doc/library/email.contentmanager.rst | 4 +- Doc/library/email.errors.rst | 4 +- Doc/library/email.generator.rst | 4 +- Doc/library/email.headerregistry.rst | 2 +- Doc/library/email.message.rst | 10 +- Doc/library/email.mime.rst | 2 +- Doc/library/email.parser.rst | 2 +- Doc/library/email.policy.rst | 6 +- Doc/library/exceptions.rst | 2 +- Doc/library/fnmatch.rst | 8 +- Doc/library/fpectl.rst | 2 +- Doc/library/ftplib.rst | 6 +- Doc/library/functions.rst | 27 +- Doc/library/http.server.rst | 3 +- Doc/library/idle.rst | 60 +- Doc/library/inspect.rst | 383 ++--- Doc/library/json.rst | 2 +- Doc/library/marshal.rst | 19 +- Doc/library/mimetypes.rst | 86 +- Doc/library/othergui.rst | 18 +- Doc/library/pdb.rst | 2 +- Doc/library/profile.rst | 10 +- Doc/library/pyexpat.rst | 2 +- Doc/library/re.rst | 23 +- Doc/library/select.rst | 2 +- Doc/library/selectors.rst | 2 +- Doc/library/shutil.rst | 4 +- Doc/library/signal.rst | 6 +- Doc/library/sqlite3.rst | 5 + Doc/library/ssl.rst | 29 +- Doc/library/stdtypes.rst | 176 ++- Doc/library/sunau.rst | 2 +- Doc/library/sysconfig.rst | 1 - Doc/library/tabnanny.rst | 8 +- Doc/library/tarfile.rst | 28 +- Doc/library/tempfile.rst | 4 +- Doc/library/test.rst | 3 +- Doc/library/threading.rst | 5 +- Doc/library/traceback.rst | 14 +- Doc/library/turtle.rst | 2 +- Doc/library/typing.rst | 59 +- Doc/library/unicodedata.rst | 2 +- Doc/library/unittest.rst | 15 +- Doc/library/urllib.parse.rst | 6 + Doc/library/xml.dom.minidom.rst | 2 +- Doc/library/xml.dom.pulldom.rst | 2 +- Doc/library/xml.etree.elementtree.rst | 2 +- Doc/library/zipfile.rst | 30 +- Doc/library/zipimport.rst | 2 +- Doc/reference/datamodel.rst | 37 +- Doc/reference/executionmodel.rst | 24 +- Doc/reference/expressions.rst | 25 +- Doc/reference/import.rst | 14 +- Doc/reference/lexical_analysis.rst | 11 + Doc/reference/simple_stmts.rst | 24 +- Doc/tools/extensions/pyspecific.py | 14 +- Doc/tools/susp-ignored.csv | 11 +- Doc/tools/templates/download.html | 2 +- Doc/tools/templates/dummy.html | 6 + Doc/tutorial/classes.rst | 2 +- Doc/tutorial/controlflow.rst | 2 +- Doc/tutorial/datastructures.rst | 4 +- Doc/tutorial/inputoutput.rst | 51 +- Doc/tutorial/introduction.rst | 6 +- Doc/using/cmdline.rst | 8 +- Doc/using/index.rst | 2 +- Doc/using/unix.rst | 2 +- Doc/using/windows.rst | 6 +- Doc/whatsnew/3.6.rst | 38 +- Include/abstract.h | 16 +- Include/ceval.h | 1 + Include/fileutils.h | 2 +- Include/patchlevel.h | 4 +- Include/pystate.h | 16 +- Include/sliceobject.h | 2 + Include/unicodeobject.h | 49 +- Lib/_collections_abc.py | 5 +- Lib/_osx_support.py | 2 +- Lib/abc.py | 4 +- Lib/asyncio/base_events.py | 3 +- Lib/asyncio/futures.py | 1 + Lib/asyncio/locks.py | 17 +- Lib/asyncio/proactor_events.py | 5 +- Lib/asyncio/sslproto.py | 21 +- Lib/asyncio/tasks.py | 11 +- Lib/contextlib.py | 16 +- Lib/ctypes/test/test_structures.py | 5 +- Lib/ctypes/test/test_win32.py | 12 +- Lib/distutils/sysconfig.py | 7 +- Lib/distutils/tests/test_build_ext.py | 7 + Lib/email/_policybase.py | 8 +- Lib/email/architecture.rst | 2 +- Lib/idlelib/NEWS.txt | 43 +- Lib/idlelib/autocomplete_w.py | 61 +- Lib/idlelib/config.py | 2 +- Lib/idlelib/editor.py | 1 - Lib/idlelib/help.html | 91 +- Lib/idlelib/help.py | 2 +- Lib/idlelib/help_about.py | 210 +-- Lib/idlelib/idle_test/test_calltips.py | 8 +- Lib/idlelib/idle_test/test_config.py | 2 - Lib/idlelib/idle_test/test_config_key.py | 2 +- Lib/idlelib/idle_test/test_editor.py | 2 - Lib/idlelib/idle_test/test_help_about.py | 112 +- Lib/idlelib/idle_test/test_macosx.py | 5 +- Lib/idlelib/idle_test/test_parenmatch.py | 1 + Lib/idlelib/idle_test/test_query.py | 1 + Lib/idlelib/idle_test/test_replace.py | 4 +- Lib/idlelib/idle_test/test_searchbase.py | 2 +- Lib/idlelib/idle_test/test_textview.py | 83 +- Lib/idlelib/idle_test/test_tree.py | 1 - Lib/idlelib/macosx.py | 1 - Lib/idlelib/pyparse.py | 2 +- Lib/idlelib/pyshell.py | 14 +- Lib/idlelib/query.py | 2 +- Lib/idlelib/run.py | 19 +- Lib/idlelib/stackviewer.py | 1 - Lib/idlelib/textview.py | 98 +- Lib/imaplib.py | 9 +- Lib/importlib/__init__.py | 2 +- Lib/inspect.py | 66 +- Lib/ipaddress.py | 6 +- Lib/lib2to3/fixes/fix_filter.py | 21 +- Lib/lib2to3/fixes/fix_map.py | 37 +- Lib/lib2to3/fixes/fix_zip.py | 21 +- Lib/lib2to3/pgen2/tokenize.py | 44 +- Lib/lib2to3/tests/data/py3_test_grammar.py | 22 + Lib/lib2to3/tests/test_fixers.py | 56 +- Lib/lib2to3/tests/test_parser.py | 22 + Lib/logging/handlers.py | 34 +- Lib/multiprocessing/forkserver.py | 20 +- Lib/multiprocessing/pool.py | 86 +- Lib/multiprocessing/queues.py | 23 +- Lib/multiprocessing/util.py | 36 +- Lib/pathlib.py | 51 +- Lib/poplib.py | 9 +- Lib/pydoc_data/topics.py | 232 +-- Lib/smtplib.py | 1 + Lib/sre_constants.py | 11 + Lib/sre_parse.py | 35 +- Lib/subprocess.py | 27 +- Lib/tabnanny.py | 2 +- Lib/test/185test.db | Bin 16384 -> 0 bytes Lib/test/_test_multiprocessing.py | 201 ++- Lib/test/datetimetester.py | 5 + Lib/test/eintrdata/eintr_tester.py | 56 +- Lib/test/libregrtest/cmdline.py | 30 +- Lib/test/libregrtest/main.py | 119 +- Lib/test/libregrtest/refleak.py | 13 +- Lib/test/libregrtest/runtest.py | 14 +- Lib/test/libregrtest/runtest_mp.py | 11 +- Lib/test/libregrtest/save_env.py | 5 +- Lib/test/sndhdrdata/README | 7 - Lib/test/support/__init__.py | 55 +- Lib/test/test_abc.py | 12 + Lib/test/test_array.py | 39 +- Lib/test/test_asyncio/test_events.py | 4 +- Lib/test/test_asyncio/test_futures.py | 8 + Lib/test/test_asyncio/test_locks.py | 22 + Lib/test/test_asyncio/test_sslproto.py | 27 + Lib/test/test_asyncio/test_tasks.py | 37 + Lib/test/test_asyncore.py | 80 +- Lib/test/test_atexit.py | 11 + Lib/test/test_bytes.py | 10 + Lib/test/test_call.py | 176 +++ Lib/test/test_capi.py | 65 +- Lib/test/test_class.py | 27 + Lib/test/test_code.py | 103 +- Lib/test/test_codecencodings_cn.py | 10 + Lib/test/test_collections.py | 17 +- Lib/test/test_contextlib.py | 33 + Lib/test/test_coroutines.py | 16 + Lib/test/test_dict.py | 85 + Lib/test/test_email/test_generator.py | 7 + Lib/test/test_exceptions.py | 20 + Lib/test/test_faulthandler.py | 40 + Lib/test/test_fileio.py | 8 +- Lib/test/test_float.py | 28 +- Lib/test/test_fnmatch.py | 64 +- Lib/test/test_format.py | 8 +- Lib/test/test_fstring.py | 30 + Lib/test/test_functools.py | 26 + Lib/test/test_gdb.py | 22 +- Lib/test/test_generators.py | 29 + Lib/test/test_imaplib.py | 24 +- Lib/test/test_import/__init__.py | 2 +- Lib/test/test_importlib/test_util.py | 44 + Lib/test/test_inspect.py | 64 + Lib/test/test_io.py | 42 + Lib/test/test_ipaddress.py | 37 +- Lib/test/test_itertools.py | 8 + Lib/test/test_json/test_tool.py | 9 +- Lib/test/test_logging.py | 30 +- Lib/test/test_nntplib.py | 7 +- Lib/test/test_os.py | 11 +- Lib/test/test_parser.py | 81 + Lib/test/test_pathlib.py | 56 +- Lib/test/test_random.py | 5 + Lib/test/test_range.py | 17 +- Lib/test/test_re.py | 88 +- Lib/test/test_regrtest.py | 105 +- Lib/test/test_sax.py | 24 +- Lib/test/test_site.py | 37 +- Lib/test/test_socket.py | 40 +- Lib/test/test_ssl.py | 4 +- Lib/test/test_subprocess.py | 53 +- Lib/test/test_support.py | 29 +- Lib/test/test_sys.py | 8 +- Lib/test/test_tarfile.py | 76 + Lib/test/test_tempfile.py | 5 +- Lib/test/test_thread.py | 4 + Lib/test/test_threading.py | 8 +- Lib/test/test_tools/test_reindent.py | 7 + Lib/test/test_tracemalloc.py | 1 + Lib/test/test_typing.py | 90 +- Lib/test/test_urllib.py | 6 +- Lib/test/test_urllib2.py | 47 +- Lib/test/test_urllibnet.py | 6 +- Lib/test/test_urlparse.py | 63 +- Lib/test/test_xml_etree.py | 112 ++ Lib/test/test_xmlrpc.py | 90 +- Lib/test/test_zipfile.py | 159 +- Lib/tkinter/__init__.py | 12 +- Lib/typing.py | 96 +- Lib/unittest/case.py | 52 +- Lib/unittest/main.py | 1 + Lib/unittest/test/test_case.py | 13 + Lib/urllib/error.py | 4 - Lib/urllib/parse.py | 62 +- Lib/urllib/request.py | 9 +- Lib/xml/sax/expatreader.py | 33 +- Lib/xmlrpc/server.py | 43 +- Lib/zipfile.py | 51 +- Mac/BuildScript/build-installer.py | 34 +- Makefile.pre.in | 176 ++- Misc/ACKS | 18 + Misc/NEWS | 417 ++++- Modules/_asynciomodule.c | 30 +- Modules/_collectionsmodule.c | 4 +- Modules/_ctypes/_ctypes.c | 10 +- Modules/_ctypes/_ctypes_test.c | 6 +- Modules/_cursesmodule.c | 2 +- Modules/_datetimemodule.c | 21 +- Modules/_elementtree.c | 112 +- Modules/_functoolsmodule.c | 5 +- Modules/_io/bufferedio.c | 14 +- Modules/_io/fileio.c | 16 +- Modules/_io/iobase.c | 38 +- Modules/_io/textio.c | 23 +- Modules/_io/winconsoleio.c | 7 +- Modules/_json.c | 86 +- Modules/_posixsubprocess.c | 43 +- Modules/_randommodule.c | 5 +- Modules/_ssl.c | 15 +- Modules/_struct.c | 14 +- Modules/_testbuffer.c | 7 +- Modules/_testcapimodule.c | 131 +- Modules/_winapi.c | 40 +- Modules/arraymodule.c | 119 +- Modules/atexitmodule.c | 2 +- Modules/cjkcodecs/_codecs_cn.c | 29 +- Modules/cjkcodecs/multibytecodec.c | 3 + Modules/clinic/_ssl.c.h | 4 +- Modules/expat/COPYING | 5 +- Modules/expat/amigaconfig.h | 32 - Modules/expat/expat.h | 35 +- Modules/expat/expat_external.h | 19 +- Modules/expat/internal.h | 22 + Modules/expat/macconfig.h | 53 - Modules/expat/siphash.h | 344 ++++ Modules/expat/watcomconfig.h | 47 - Modules/expat/winconfig.h | 10 + Modules/expat/xmlparse.c | 700 ++++++-- Modules/expat/xmlrole.c | 230 ++- Modules/expat/xmltok.c | 244 ++- Modules/expat/xmltok.h | 10 +- Modules/expat/xmltok_impl.c | 226 ++- Modules/faulthandler.c | 38 +- Modules/itertoolsmodule.c | 52 +- Modules/main.c | 2 +- Modules/mmapmodule.c | 9 +- Modules/parsermodule.c | 131 +- Modules/posixmodule.c | 75 +- Modules/pyexpat.c | 6 +- Modules/signalmodule.c | 33 +- Modules/socketmodule.c | 36 +- Modules/socketmodule.h | 34 + Modules/timemodule.c | 6 +- Modules/zipimport.c | 6 +- Objects/abstract.c | 23 +- Objects/bytearrayobject.c | 14 +- Objects/bytes_methods.c | 13 +- Objects/bytesobject.c | 27 +- Objects/codeobject.c | 41 +- Objects/dictobject.c | 73 +- Objects/exceptions.c | 48 + Objects/genobject.c | 3 +- Objects/listobject.c | 14 +- Objects/lnotab_notes.txt | 50 +- Objects/memoryobject.c | 4 +- Objects/methodobject.c | 3 +- Objects/obmalloc.c | 2 +- Objects/rangeobject.c | 12 +- Objects/setobject.c | 23 +- Objects/sliceobject.c | 8 +- Objects/tupleobject.c | 10 +- Objects/typeobject.c | 39 +- Objects/unicodeobject.c | 18 +- PC/bdist_wininst/bdist_wininst.vcxproj | 210 +-- PC/bdist_wininst/bdist_wininst.vcxproj.filters | 120 +- PC/bdist_wininst/build.bat | 11 +- PC/getpathp.c | 4 +- PC/readme.txt | 162 +- PCbuild/_asyncio.vcxproj | 154 +- PCbuild/_asyncio.vcxproj.filters | 30 +- PCbuild/_bz2.vcxproj | 194 +-- PCbuild/_bz2.vcxproj.filters | 96 +- PCbuild/_ctypes.vcxproj | 218 +-- PCbuild/_ctypes.vcxproj.filters | 128 +- PCbuild/_ctypes_test.vcxproj | 158 +- PCbuild/_ctypes_test.vcxproj.filters | 40 +- PCbuild/_decimal.vcxproj | 252 +-- PCbuild/_decimal.vcxproj.filters | 230 +-- PCbuild/_elementtree.vcxproj | 212 +-- PCbuild/_elementtree.vcxproj.filters | 142 +- PCbuild/_freeze_importlib.vcxproj | 238 +-- PCbuild/_freeze_importlib.vcxproj.filters | 54 +- PCbuild/_hashlib.vcxproj | 186 +-- PCbuild/_hashlib.vcxproj.filters | 24 +- PCbuild/_lzma.vcxproj | 180 +-- PCbuild/_lzma.vcxproj.filters | 24 +- PCbuild/_msi.vcxproj | 164 +- PCbuild/_msi.vcxproj.filters | 24 +- PCbuild/_multiprocessing.vcxproj | 172 +- PCbuild/_multiprocessing.vcxproj.filters | 46 +- PCbuild/_overlapped.vcxproj | 164 +- PCbuild/_overlapped.vcxproj.filters | 24 +- PCbuild/_socket.vcxproj | 170 +- PCbuild/_socket.vcxproj.filters | 40 +- PCbuild/_sqlite3.vcxproj | 216 +-- PCbuild/_sqlite3.vcxproj.filters | 136 +- PCbuild/_ssl.vcxproj | 194 +-- PCbuild/_ssl.vcxproj.filters | 24 +- PCbuild/_testbuffer.vcxproj | 162 +- PCbuild/_testbuffer.vcxproj.filters | 24 +- PCbuild/_testcapi.vcxproj | 164 +- PCbuild/_testcapi.vcxproj.filters | 24 +- PCbuild/_testconsole.vcxproj | 164 +- PCbuild/_testconsole.vcxproj.filters | 42 +- PCbuild/_testembed.vcxproj | 158 +- PCbuild/_testembed.vcxproj.filters | 42 +- PCbuild/_testimportmultiple.vcxproj | 166 +- PCbuild/_testimportmultiple.vcxproj.filters | 26 +- PCbuild/_testmultiphase.vcxproj | 164 +- PCbuild/_testmultiphase.vcxproj.filters | 42 +- PCbuild/_tkinter.vcxproj | 186 +-- PCbuild/_tkinter.vcxproj.filters | 30 +- PCbuild/build.bat | 32 +- PCbuild/find_msbuild.bat | 51 + PCbuild/get_external.py | 60 + PCbuild/get_externals.bat | 126 +- PCbuild/libeay.vcxproj | 1812 ++++++++++----------- PCbuild/liblzma.vcxproj | 432 ++--- PCbuild/openssl.props | 152 +- PCbuild/pcbuild.proj | 238 +-- PCbuild/pcbuild.sln | 34 + PCbuild/pyexpat.vcxproj | 172 +- PCbuild/pyexpat.vcxproj.filters | 64 +- PCbuild/pylauncher.vcxproj | 164 +- PCbuild/pylauncher.vcxproj.filters | 62 +- PCbuild/pyproject.props | 324 ++-- PCbuild/pyshellext.vcxproj | 172 +- PCbuild/pyshellext.vcxproj.filters | 78 +- PCbuild/python.props | 344 ++-- PCbuild/python.vcxproj | 225 +-- PCbuild/python.vcxproj.filters | 52 +- PCbuild/python3dll.vcxproj | 272 ++-- PCbuild/python3dll.vcxproj.filters | 62 +- PCbuild/pythoncore.vcxproj | 868 +++++----- PCbuild/pythoncore.vcxproj.filters | 2014 ++++++++++++------------ PCbuild/pythonw.vcxproj | 156 +- PCbuild/pythonw.vcxproj.filters | 40 +- PCbuild/pywlauncher.vcxproj | 162 +- PCbuild/pywlauncher.vcxproj.filters | 62 +- PCbuild/readme.txt | 24 +- PCbuild/rmpyc.py | 20 +- PCbuild/rt.bat | 8 +- PCbuild/select.vcxproj | 162 +- PCbuild/select.vcxproj.filters | 24 +- PCbuild/sqlite3.vcxproj | 178 +-- PCbuild/sqlite3.vcxproj.filters | 46 +- PCbuild/ssleay.vcxproj | 236 +-- PCbuild/tcl.vcxproj | 184 +-- PCbuild/tcltk.props | 90 +- PCbuild/tix.vcxproj | 186 +-- PCbuild/tk.vcxproj | 192 +-- PCbuild/unicodedata.vcxproj | 170 +- PCbuild/unicodedata.vcxproj.filters | 46 +- PCbuild/winsound.vcxproj | 162 +- PCbuild/winsound.vcxproj.filters | 26 +- PCbuild/xxlimited.vcxproj | 164 +- PCbuild/xxlimited.vcxproj.filters | 24 +- Python/ast.c | 88 +- Python/bltinmodule.c | 21 +- Python/ceval.c | 53 +- Python/fileutils.c | 4 +- Python/import.c | 8 +- Python/marshal.c | 26 +- Python/pylifecycle.c | 11 +- Python/pystate.c | 48 +- Python/sysmodule.c | 8 +- Python/thread_pthread.h | 24 +- README.rst | 6 +- Tools/gdb/libpython.py | 66 +- Tools/hg/hgtouch.py | 129 -- Tools/msi/build.bat | 10 +- Tools/msi/buildrelease.bat | 15 +- Tools/msi/bundle/bootstrap/pythonba.sln | 44 +- Tools/msi/bundle/bootstrap/pythonba.vcxproj | 136 +- Tools/msi/get_externals.bat | 118 +- Tools/msi/make_zip.proj | 76 +- Tools/msi/msi.props | 374 ++--- Tools/msi/uploadrelease.bat | 15 +- Tools/msi/uploadrelease.proj | 224 +-- Tools/msi/wix.props | 22 +- Tools/nuget/build.bat | 7 +- Tools/nuget/make_pkg.proj | 110 +- Tools/scripts/patchcheck.py | 63 +- Tools/scripts/reindent.py | 6 +- Tools/ssl/sslspeed.vcxproj | 138 +- aclocal.m4 | 4 +- config.guess | 197 ++- config.sub | 90 +- configure | 129 +- configure.ac | 97 +- setup.py | 7 +- 474 files changed, 17877 insertions(+), 12316 deletions(-) delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md delete mode 100644 .gitignore create mode 100644 Doc/tools/templates/dummy.html delete mode 100644 Lib/test/185test.db delete mode 100644 Modules/expat/amigaconfig.h delete mode 100644 Modules/expat/macconfig.h create mode 100644 Modules/expat/siphash.h delete mode 100644 Modules/expat/watcomconfig.h create mode 100644 PCbuild/find_msbuild.bat create mode 100644 PCbuild/get_external.py delete mode 100644 Tools/hg/hgtouch.py diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 4ce80d8..0000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,9 +0,0 @@ -## CPython Mirror - -https://github.com/python/cpython is a cpython mirror repository. Pull requests -are not accepted on this repo and will be automatically closed. - -### Submit patches at https://bugs.python.org - -For additional information about contributing to CPython, see the -[developer's guide](https://docs.python.org/devguide/#contributing). diff --git a/.gitignore b/.gitignore deleted file mode 100644 index ed4ebfb..0000000 --- a/.gitignore +++ /dev/null @@ -1,96 +0,0 @@ -# Two-trick pony for OSX and other case insensitive file systems: -# Ignore ./python binary on Unix but still look into ./Python/ directory. -/python -!/Python/ -*.cover -*.o -*.orig -*.pyc -*.pyd -*.pyo -*.rej -*.swp -*~ -*.gc?? -*.profclang? -*.profraw -*.dyn -.gdb_history -Doc/build/ -Doc/venv/ -Lib/distutils/command/*.pdb -Lib/lib2to3/*.pickle -Lib/test/data/* -Makefile -Makefile.pre -Misc/python.pc -Misc/python-config.sh -Modules/Setup -Modules/Setup.config -Modules/Setup.local -Modules/config.c -Modules/ld_so_aix -Programs/_freeze_importlib -Programs/_testembed -PC/python_nt*.h -PC/pythonnt_rc*.h -PC/*/*.exe -PC/*/*.exp -PC/*/*.lib -PC/*/*.bsc -PC/*/*.dll -PC/*/*.pdb -PC/*/*.user -PC/*/*.ncb -PC/*/*.suo -PC/*/Win32-temp-* -PC/*/x64-temp-* -PC/*/amd64 -PCbuild/*.user -PCbuild/*.suo -PCbuild/*.*sdf -PCbuild/*-pgi -PCbuild/*-pgo -PCbuild/.vs/ -PCbuild/amd64/ -PCbuild/obj/ -PCBuild/win32/ -.purify -Parser/pgen -__pycache__ -autom4te.cache -build/ -buildno -config.cache -config.log -config.status -config.status.lineno -core -db_home -.hg/ -ipch/ -libpython*.a -libpython*.so* -libpython*.dylib -platform -pybuilddir.txt -pyconfig.h -python-config -python-config.py -python.bat -python.exe -python-gdb.py -python.exe-gdb.py -reflog.txt -.svn/ -tags -TAGS -.coverage -coverage/ -externals/ -htmlcov/ -Tools/msi/obj -Tools/ssl/amd64 -Tools/ssl/win32 -.vs/ -.vscode/ diff --git a/Doc/Makefile b/Doc/Makefile index 94697f9..ae59f32 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -159,15 +159,20 @@ serve: ../Tools/scripts/serve.py build/html # Targets for daily automated doc build +# By default, Sphinx only rebuilds pages where the page content has changed. +# This means it doesn't always pick up changes to preferred link targets, etc +# To ensure such changes are picked up, we build the published docs with +# `-E` (to ignore the cached environment) and `-a` (to ignore already existing +# output files) # for development releases: always build autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -A daily=1 -A versionswitcher=1' + make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A versionswitcher=1' -make suspicious # for quick rebuilds (HTML only) autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -A daily=1 -A versionswitcher=1' + make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A versionswitcher=1' # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 037b85c..2bc1bd8 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -291,16 +291,11 @@ an error value). is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, and so forth. - Warning categories must be subclasses of :c:data:`Warning`; the default warning - category is :c:data:`RuntimeWarning`. The standard Python warning categories are - available as global variables whose names are ``PyExc_`` followed by the Python - exception name. These have the type :c:type:`PyObject\*`; they are all class - objects. Their names are :c:data:`PyExc_Warning`, :c:data:`PyExc_UserWarning`, - :c:data:`PyExc_UnicodeWarning`, :c:data:`PyExc_DeprecationWarning`, - :c:data:`PyExc_SyntaxWarning`, :c:data:`PyExc_RuntimeWarning`, and - :c:data:`PyExc_FutureWarning`. :c:data:`PyExc_Warning` is a subclass of - :c:data:`PyExc_Exception`; the other warning categories are subclasses of - :c:data:`PyExc_Warning`. + Warning categories must be subclasses of :c:data:`PyExc_Warning`; + :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; + the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard + Python warning categories are available as global variables whose names are + enumerated at :ref:`standardwarningcategories`. For information about warning control, see the documentation for the :mod:`warnings` module and the :option:`-W` option in the command line @@ -750,6 +745,61 @@ All standard Python exceptions are available as global variables whose names are :c:type:`PyObject\*`; they are all class objects. For completeness, here are all the variables: +.. index:: + single: PyExc_BaseException + single: PyExc_Exception + single: PyExc_ArithmeticError + single: PyExc_AssertionError + single: PyExc_AttributeError + single: PyExc_BlockingIOError + single: PyExc_BrokenPipeError + single: PyExc_BufferError + single: PyExc_ChildProcessError + single: PyExc_ConnectionAbortedError + single: PyExc_ConnectionError + single: PyExc_ConnectionRefusedError + single: PyExc_ConnectionResetError + single: PyExc_EOFError + single: PyExc_FileExistsError + single: PyExc_FileNotFoundError + single: PyExc_FloatingPointError + single: PyExc_GeneratorExit + single: PyExc_ImportError + single: PyExc_IndentationError + single: PyExc_IndexError + single: PyExc_InterruptedError + single: PyExc_IsADirectoryError + single: PyExc_KeyError + single: PyExc_KeyboardInterrupt + single: PyExc_LookupError + single: PyExc_MemoryError + single: PyExc_ModuleNotFoundError + single: PyExc_NameError + single: PyExc_NotADirectoryError + single: PyExc_NotImplementedError + single: PyExc_OSError + single: PyExc_OverflowError + single: PyExc_PermissionError + single: PyExc_ProcessLookupError + single: PyExc_RecursionError + single: PyExc_ReferenceError + single: PyExc_RuntimeError + single: PyExc_StopAsyncIteration + single: PyExc_StopIteration + single: PyExc_SyntaxError + single: PyExc_SystemError + single: PyExc_SystemExit + single: PyExc_TabError + single: PyExc_TimeoutError + single: PyExc_TypeError + single: PyExc_UnboundLocalError + single: PyExc_UnicodeDecodeError + single: PyExc_UnicodeEncodeError + single: PyExc_UnicodeError + single: PyExc_UnicodeTranslateError + single: PyExc_ValueError + single: PyExc_ZeroDivisionError + +-----------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | +=========================================+=================================+==========+ @@ -759,8 +809,6 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | @@ -769,27 +817,31 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | +| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | +| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ImportError` | :exc:`ImportError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | +| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_IndexError` | :exc:`IndexError` | | +-----------------------------------------+---------------------------------+----------+ @@ -801,8 +853,12 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NameError` | :exc:`NameError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | @@ -823,16 +879,32 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SystemError` | :exc:`SystemError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TabError` | :exc:`TabError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TypeError` | :exc:`TypeError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ValueError` | :exc:`ValueError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | @@ -849,11 +921,18 @@ the variables: and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. .. versionadded:: 3.5 - :c:data:`PyExc_RecursionError`. + :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. +.. versionadded:: 3.6 + :c:data:`PyExc_ModuleNotFoundError`. These are compatibility aliases to :c:data:`PyExc_OSError`: +.. index:: + single: PyExc_EnvironmentError + single: PyExc_IOError + single: PyExc_WindowsError + +-------------------------------------+----------+ | C Name | Notes | +=====================================+==========+ @@ -867,52 +946,6 @@ These are compatibility aliases to :c:data:`PyExc_OSError`: .. versionchanged:: 3.3 These aliases used to be separate exception types. - -.. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_LookupError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_ConnectionError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_ImportError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_MemoryError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError - Notes: (1) @@ -924,3 +957,60 @@ Notes: (3) Only defined on Windows; protect code that uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. + +.. _standardwarningcategories: + +Standard Warning Categories +=========================== + +All standard Python warning categories are available as global variables whose +names are ``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +the variables: + +.. index:: + single: PyExc_Warning + single: PyExc_BytesWarning + single: PyExc_DeprecationWarning + single: PyExc_FutureWarning + single: PyExc_ImportWarning + single: PyExc_PendingDeprecationWarning + single: PyExc_ResourceWarning + single: PyExc_RuntimeWarning + single: PyExc_SyntaxWarning + single: PyExc_UnicodeWarning + single: PyExc_UserWarning + ++------------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++==========================================+=================================+==========+ +| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | ++------------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.2 + :c:data:`PyExc_ResourceWarning`. + +Notes: + +(1) + This is a base class for other standard warning categories. diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index f592cb6..f50680b 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -85,13 +85,12 @@ All integers are implemented as "long" integer objects of arbitrary size. Return a new :c:type:`PyLongObject` based on the string value in *str*, which is interpreted according to the radix in *base*. If *pend* is non-*NULL*, *\*pend* will point to the first character in *str* which follows the - representation of the number. If *base* is ``0``, the radix will be - determined based on the leading characters of *str*: if *str* starts with - ``'0x'`` or ``'0X'``, radix 16 will be used; if *str* starts with ``'0o'`` or - ``'0O'``, radix 8 will be used; if *str* starts with ``'0b'`` or ``'0B'``, - radix 2 will be used; otherwise radix 10 will be used. If *base* is not - ``0``, it must be between ``2`` and ``36``, inclusive. Leading spaces are - ignored. If there are no digits, :exc:`ValueError` will be raised. + representation of the number. If *base* is ``0``, *str* is interpreted using + the :ref:`integers` definition; in this case, leading zeros in a + non-zero decimal number raises a :exc:`ValueError`. If *base* is not ``0``, + it must be between ``2`` and ``36``, inclusive. Leading spaces and single + underscores after a base specifier and between digits are ignored. If there + are no digits, :exc:`ValueError` will be raised. .. c:function:: PyObject* PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index a6d0f46..c6d1d02 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -34,7 +34,7 @@ unmarshalling. Version 2 uses a binary format for floating point numbers. .. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version) - Return a string object containing the marshalled representation of *value*. + Return a bytes object containing the marshalled representation of *value*. *version* indicates the file format. @@ -88,10 +88,10 @@ written using these routines? :exc:`TypeError`) and returns *NULL*. -.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *string, Py_ssize_t len) +.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) - Return a Python object from the data stream in a character buffer - containing *len* bytes pointed to by *string*. + Return a Python object from the data stream in a byte buffer + containing *len* bytes pointed to by *data*. On error, sets the appropriate exception (:exc:`EOFError` or :exc:`TypeError`) and returns *NULL*. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 3ff5452..873fb2a 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -391,7 +391,7 @@ with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_MEM` (ex: -:c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: +:c:func:`PyMem_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. The arena allocator uses the following functions: diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 7724350..d3125b8 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -50,7 +50,7 @@ Module Objects .. c:function:: PyObject* PyModule_New(const char *name) - Similar to :c:func:`PyImport_NewObject`, but the name is a UTF-8 encoded + Similar to :c:func:`PyModule_NewObject`, but the name is a UTF-8 encoded string instead of a Unicode object. diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index a825164..8b695e0 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -56,3 +56,14 @@ Slice Objects .. versionchanged:: 3.2 The parameter type for the *slice* parameter was ``PySliceObject*`` before. + + +Ellipsis Object +--------------- + + +.. c:var:: PyObject *Py_Ellipsis + + The Python ``Ellipsis`` object. This object has no methods. It needs to be + treated just like any other object with respect to reference counts. Like + :c:data:`Py_None` it is a singleton object. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index f481193..c080f31 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -241,7 +241,7 @@ definition with the same method name. +==================+=============+===============================+ | :attr:`name` | char \* | name of the member | +------------------+-------------+-------------------------------+ - | :attr:`type` | int | the type of the member in the | + | :attr:`!type` | int | the type of the member in the | | | | C struct | +------------------+-------------+-------------------------------+ | :attr:`offset` | Py_ssize_t | the offset in bytes that the | @@ -256,7 +256,7 @@ definition with the same method name. | | | docstring | +------------------+-------------+-------------------------------+ - :attr:`type` can be one of many ``T_`` macros corresponding to various C + :attr:`!type` can be one of many ``T_`` macros corresponding to various C types. When the member is accessed in Python, it will be converted to the equivalent Python type. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 02f7ada..6e91576 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1393,77 +1393,78 @@ Character Map Codecs This codec is special in that it can be used to implement many different codecs (and this is in fact what was done to obtain most of the standard codecs included in the :mod:`encodings` package). The codec uses mapping to encode and -decode characters. - -Decoding mappings must map single string characters to single Unicode -characters, integers (which are then interpreted as Unicode ordinals) or ``None`` -(meaning "undefined mapping" and causing an error). - -Encoding mappings must map single Unicode characters to single string -characters, integers (which are then interpreted as Latin-1 ordinals) or ``None`` -(meaning "undefined mapping" and causing an error). - -The mapping objects provided must only support the __getitem__ mapping -interface. - -If a character lookup fails with a LookupError, the character is copied as-is -meaning that its ordinal value will be interpreted as Unicode or Latin-1 ordinal -resp. Because of this, mappings only need to contain those mappings which map -characters to different code points. +decode characters. The mapping objects provided must support the +:meth:`__getitem__` mapping interface; dictionaries and sequences work well. These are the mapping codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \ PyObject *mapping, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s* using - the given *mapping* object. Return *NULL* if an exception was raised by the - codec. If *mapping* is *NULL* latin-1 decoding will be done. Else it can be a - dictionary mapping byte or a unicode string, which is treated as a lookup table. - Byte values greater that the length of the string and U+FFFE "characters" are - treated as "undefined mapping". + Create a Unicode object by decoding *size* bytes of the encoded string *s* + using the given *mapping* object. Return *NULL* if an exception was raised + by the codec. + + If *mapping* is *NULL*, Latin-1 decoding will be applied. Else + *mapping* must map bytes ordinals (integers in the range from 0 to 255) + to Unicode strings, integers (which are then interpreted as Unicode + ordinals) or ``None``. Unmapped data bytes -- ones which cause a + :exc:`LookupError`, as well as ones which get mapped to ``None``, + ``0xFFFE`` or ``'\ufffe'``, are treated as undefined mappings and cause + an error. .. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping) - Encode a Unicode object using the given *mapping* object and return the result - as Python string object. Error handling is "strict". Return *NULL* if an + Encode a Unicode object using the given *mapping* object and return the + result as a bytes object. Error handling is "strict". Return *NULL* if an exception was raised by the codec. -The following codec API is special in that maps Unicode to Unicode. - + The *mapping* object must map Unicode ordinal integers to bytes objects, + integers in the range from 0 to 255 or ``None``. Unmapped character + ordinals (ones which cause a :exc:`LookupError`) as well as mapped to + ``None`` are treated as "undefined mapping" and cause an error. -.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ - PyObject *table, const char *errors) - - Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a - character mapping *table* to it and return the resulting Unicode object. Return - *NULL* when an exception was raised by the codec. - The *mapping* table must map Unicode ordinal integers to Unicode ordinal - integers or ``None`` (causing deletion of the character). +.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ + PyObject *mapping, const char *errors) - Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries - and sequences work well. Unmapped character ordinals (ones which cause a - :exc:`LookupError`) are left untouched and are copied as-is. + Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given + *mapping* object and return the result as a bytes object. Return *NULL* if + an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API - ` + :c:func:`PyUnicode_AsCharmapString` or + :c:func:`PyUnicode_AsEncodedString`. -.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ +The following codec API is special in that maps Unicode to Unicode. + +.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, \ PyObject *mapping, const char *errors) - Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given - *mapping* object and return a Python string object. Return *NULL* if an - exception was raised by the codec. + Translate a Unicode object using the given *mapping* object and return the + resulting Unicode object. Return *NULL* if an exception was raised by the + codec. + + The *mapping* object must map Unicode ordinal integers to Unicode strings, + integers (which are then interpreted as Unicode ordinals) or ``None`` + (causing deletion of the character). Unmapped character ordinals (ones + which cause a :exc:`LookupError`) are left untouched and are copied as-is. + + +.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ + PyObject *mapping, const char *errors) + + Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a + character *mapping* table to it and return the resulting Unicode object. + Return *NULL* when an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsCharmapString` or - :c:func:`PyUnicode_AsEncodedString`. + :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API + ` MBCS codecs for Windows diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst index 1f5be9c..4e2761d 100644 --- a/Doc/distutils/examples.rst +++ b/Doc/distutils/examples.rst @@ -321,7 +321,7 @@ You can read back this static file, by using the >>> metadata.description 'Easily download, build, install, upgrade, and uninstall Python packages' -Notice that the class can also be instanciated with a metadata file path to +Notice that the class can also be instantiated with a metadata file path to loads its values:: >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info' diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index b8ce437..003b4e5 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -124,7 +124,7 @@ our objects and in some error messages, for example:: >>> "" + noddy.new_noddy() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: cannot add type "noddy.Noddy" to string Note that the name is a dotted name that includes both the module name and the diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 41ee3d8..dba9186 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -131,6 +131,10 @@ Glossary binary file A :term:`file object` able to read and write :term:`bytes-like objects `. + Examples of binary files are files opened in binary mode (``'rb'``, + ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`, + :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and + :class:`gzip.GzipFile`. .. seealso:: A :term:`text file` reads and writes :class:`str` objects. @@ -155,7 +159,7 @@ Glossary bytecode Python source code is compiled into bytecode, the internal representation of a Python program in the CPython interpreter. The bytecode is also - cached in ``.pyc`` and ``.pyo`` files so that executing the same file is + cached in ``.pyc`` files so that executing the same file is faster the second time (recompilation from source to bytecode can be avoided). This "intermediate language" is said to run on a :term:`virtual machine` that executes the machine code corresponding to @@ -316,6 +320,11 @@ Glossary A module written in C or C++, using Python's C API to interact with the core and with user code. + f-string + String literals prefixed with ``'f'`` or ``'F'`` are commonly called + "f-strings" which is short for + :ref:`formatted string literals `. See also :pep:`498`. + file object An object exposing a file-oriented API (with methods such as :meth:`read()` or :meth:`write()`) to an underlying resource. Depending @@ -458,9 +467,9 @@ Glossary Hashability makes an object usable as a dictionary key and a set member, because these data structures use the hash value internally. - All of Python's immutable built-in objects are hashable, while no mutable - containers (such as lists or dictionaries) are. Objects which are - instances of user-defined classes are hashable by default; they all + All of Python's immutable built-in objects are hashable; mutable + containers (such as lists or dictionaries) are not. Objects which are + instances of user-defined classes are hashable by default. They all compare unequal (except with themselves), and their hash value is derived from their :func:`id`. @@ -966,6 +975,9 @@ Glossary A :term:`file object` able to read and write :class:`str` objects. Often, a text file actually accesses a byte-oriented datastream and handles the :term:`text encoding` automatically. + Examples of text files are files opened in text mode (``'r'`` or ``'w'``), + :data:`sys.stdin`, :data:`sys.stdout`, and instances of + :class:`io.StringIO`. .. seealso:: A :term:`binary file` reads and write :class:`bytes` objects. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 7e161a5..9d770f5 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -221,7 +221,7 @@ before proceeding. Introducing Optional arguments ============================== -So far we, have been playing with positional arguments. Let us +So far we have been playing with positional arguments. Let us have a look on how to add optional ones:: import argparse diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index eaab20a..083a299 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1407,8 +1407,8 @@ Let's start with defining some terminology: ``two-pass`` A buffer like ``buffer``. However, a two-pass buffer can only - be written once, and it prints out all text sent to it during - all of processing, even from Clinic blocks *after* the + be dumped once, and it prints out all text sent to it during + all processing, even from Clinic blocks *after* the dumping point. ``suppress`` The text is suppressed—thrown away. @@ -1471,7 +1471,7 @@ preset configurations, as follows: The default filename is ``"{dirname}/clinic/{basename}.h"``. ``buffer`` - Save up all most of the output from Clinic, to be written into + Save up most of the output from Clinic, to be written into your file near the end. For Python files implementing modules or builtin types, it's recommended that you dump the buffer just above the static structures for your module or diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index c2bf473..2dd6c34 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -282,7 +282,7 @@ this:: . . . def __get__(self, obj, objtype=None): "Simulate func_descr_get() in Objects/funcobject.c" - return types.MethodType(self, obj, objtype) + return types.MethodType(self, obj) Running the interpreter shows how the function descriptor works in practice:: diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 8ae9679..4060181 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -210,7 +210,7 @@ You can experiment with the iteration interface manually: 3 >>> next(it) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in StopIteration >>> @@ -474,7 +474,7 @@ Here's a sample usage of the ``generate_ints()`` generator: 2 >>> next(gen) Traceback (most recent call last): - File "stdin", line 1, in ? + File "stdin", line 1, in File "stdin", line 2, in generate_ints StopIteration @@ -577,7 +577,7 @@ And here's an example of changing the counter: 9 >>> next(it) #doctest: +SKIP Traceback (most recent call last): - File "t.py", line 15, in ? + File "t.py", line 15, in it.next() StopIteration @@ -653,8 +653,9 @@ This can also be written as a list comprehension: [0, 2, 4, 6, 8] -:func:`enumerate(iter) ` counts off the elements in the iterable, -returning 2-tuples containing the count and each element. :: +:func:`enumerate(iter, start=0) ` counts off the elements in the +iterable returning 2-tuples containing the count (from *start*) and +each element. :: >>> for item in enumerate(['subject', 'verb', 'object']): ... print(item) @@ -747,14 +748,16 @@ The module's functions fall into a few broad classes: Creating new iterators ---------------------- -:func:`itertools.count(n) ` returns an infinite stream of -integers, increasing by 1 each time. You can optionally supply the starting -number, which defaults to 0:: +:func:`itertools.count(start, step) ` returns an infinite +stream of evenly spaced values. You can optionally supply the starting number, +which defaults to 0, and the interval between numbers, which defaults to 1:: itertools.count() => 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ... itertools.count(10) => 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ... + itertools.count(10, 5) => + 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, ... :func:`itertools.cycle(iter) ` saves a copy of the contents of a provided iterable and returns a new iterator that returns its elements from @@ -1060,10 +1063,10 @@ write the obvious :keyword:`for` loop:: for i in [1,2,3]: product *= i -A related function is `itertools.accumulate(iterable, func=operator.add) `. It performs the same calculation, but instead of +returning only the final result, :func:`accumulate` returns an iterator that +also yields each partial result:: itertools.accumulate([1,2,3,4,5]) => 1, 3, 6, 10, 15 @@ -1235,6 +1238,8 @@ Python documentation Documentation for the :mod:`itertools` module. +Documentation for the :mod:`functools` module. + Documentation for the :mod:`operator` module. :pep:`289`: "Generator Expressions" diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index bb79bb1..6498ea5 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1683,7 +1683,7 @@ Implementing structured logging ------------------------------- Although most logging messages are intended for reading by humans, and thus not -readily machine-parseable, there might be cirumstances where you want to output +readily machine-parseable, there might be circumstances where you want to output messages in a structured format which *is* capable of being parsed by a program (without needing complex regular expressions to parse the log message). This is straightforward to achieve using the logging package. There are a number of diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index a48ae1f..9649b9c 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -43,9 +43,9 @@ hold values ranging from 0 to 255. ASCII codes only went up to 127, so some machines assigned values between 128 and 255 to accented characters. Different machines had different codes, however, which led to problems exchanging files. Eventually various commonly used sets of values for the 128--255 range emerged. -Some were true standards, defined by the International Standards Organization, -and some were *de facto* conventions that were invented by one company or -another and managed to catch on. +Some were true standards, defined by the International Organization for +Standardization, and some were *de facto* conventions that were invented by one +company or another and managed to catch on. 255 characters aren't very many. For example, you can't fit both the accented characters used in Western Europe and the Cyrillic alphabet used for Russian diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 18b5c65..8d383e0 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -34,8 +34,8 @@ handling common situations - like basic authentication, cookies, proxies and so on. These are provided by objects called handlers and openers. urllib.request supports fetching URLs for many "URL schemes" (identified by the string -before the ":" in URL - for example "ftp" is the URL scheme of -"ftp://python.org/") using their associated network protocols (e.g. FTP, HTTP). +before the ``":"`` in URL - for example ``"ftp"`` is the URL scheme of +``"ftp://python.org/"``) using their associated network protocols (e.g. FTP, HTTP). This tutorial focuses on the most common case, HTTP. For straightforward situations *urlopen* is very easy to use. But as soon as you @@ -511,10 +511,10 @@ than the URL you pass to .add_password() will also match. :: ``top_level_url`` is in fact *either* a full URL (including the 'http:' scheme component and the hostname and optionally the port number) -e.g. "http://example.com/" *or* an "authority" (i.e. the hostname, -optionally including the port number) e.g. "example.com" or "example.com:8080" +e.g. ``"http://example.com/"`` *or* an "authority" (i.e. the hostname, +optionally including the port number) e.g. ``"example.com"`` or ``"example.com:8080"`` (the latter example includes a port number). The authority, if present, must -NOT contain the "userinfo" component - for example "joe:password@example.com" is +NOT contain the "userinfo" component - for example ``"joe:password@example.com"`` is not correct. diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index ace1bfa..4c9a528 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -199,13 +199,6 @@ and off individually. They are described here in more detail. because the :class:`memoryview` API is similar but not exactly the same as that of :class:`buffer`. -.. 2to3fixer:: callable - - Converts ``callable(x)`` to ``isinstance(x, collections.Callable)``, adding - an import to :mod:`collections` if needed. Note ``callable(x)`` has returned - in Python 3.2, so if you do not intend to support Python 3.1, you can disable - this fixer. - .. 2to3fixer:: dict Fixes dictionary iteration methods. :meth:`dict.iteritems` is converted to diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 482ffbb..3f55506 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -384,7 +384,7 @@ The following callbacks are called on :class:`Protocol` instances: .. method:: Protocol.eof_received() - Calls when the other end signals it won't send any more data + Called when the other end signals it won't send any more data (for example by calling :meth:`write_eof`, if the other end also uses asyncio). diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index dc93a74..16ba9a3 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -80,7 +80,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module. however, where :class:`~subprocess.Popen` takes a single argument which is list of strings, :func:`subprocess_exec` takes multiple string arguments. - The *protocol_factory* must instanciate a subclass of the + The *protocol_factory* must instantiate a subclass of the :class:`asyncio.SubprocessProtocol` class. Other parameters: @@ -123,7 +123,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module. using the platform's "shell" syntax. This is similar to the standard library :class:`subprocess.Popen` class called with ``shell=True``. - The *protocol_factory* must instanciate a subclass of the + The *protocol_factory* must instantiate a subclass of the :class:`asyncio.SubprocessProtocol` class. See :meth:`~AbstractEventLoop.subprocess_exec` for more details about diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 558d17c..804f192 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -540,6 +540,11 @@ Task functions .. deprecated:: 3.4.4 +.. function:: wrap_future(future, \*, loop=None) + + Wrap a :class:`concurrent.futures.Future` object in a :class:`Future` + object. + .. function:: gather(\*coros_or_futures, loop=None, return_exceptions=False) Return a future aggregating results from the given coroutine objects or diff --git a/Doc/library/binhex.rst b/Doc/library/binhex.rst index 359ab23..2966e0d 100644 --- a/Doc/library/binhex.rst +++ b/Doc/library/binhex.rst @@ -55,5 +55,3 @@ the source for details. If you code or decode textfiles on non-Macintosh platforms they will still use the old Macintosh newline convention (carriage-return as end of line). -As of this writing, :func:`hexbin` appears to not work in all cases. - diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index f40cfdf..3b4a8ff 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -266,10 +266,10 @@ immediate playback:: 'Draw circle with given radius an options extent and steps: CIRCLE 50' circle(*parse(arg)) def do_position(self, arg): - 'Print the current turle position: POSITION' + 'Print the current turtle position: POSITION' print('Current position is %d %d\n' % position()) def do_heading(self, arg): - 'Print the current turle heading in degrees: HEADING' + 'Print the current turtle heading in degrees: HEADING' print('Current heading is %d\n' % (heading(),)) def do_color(self, arg): 'Set the color: COLOR BLUE' diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 8e2eb4d..d6d2056 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -771,9 +771,9 @@ they add the ability to access fields by name instead of position index. helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` method which lists the tuple contents in a ``name=value`` format. - The *field_names* are a single string with each fieldname separated by whitespace - and/or commas, for example ``'x y'`` or ``'x, y'``. Alternatively, *field_names* - can be a sequence of strings such as ``['x', 'y']``. + The *field_names* are a sequence of strings such as ``['x', 'y']``. + Alternatively, *field_names* can be a single string with each fieldname + separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``. Any valid Python identifier may be used for a fieldname except for names starting with an underscore. Valid identifiers consist of letters, digits, @@ -866,7 +866,7 @@ field names, the method and attribute names start with an underscore. .. versionchanged:: 3.1 Returns an :class:`OrderedDict` instead of a regular :class:`dict`. -.. method:: somenamedtuple._replace(kwargs) +.. method:: somenamedtuple._replace(**kwargs) Return a new instance of the named tuple replacing specified fields with new values:: diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index af57cba..04c2a82 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -34,8 +34,8 @@ can be customized by end users easily. .. seealso:: Module :mod:`shlex` - Support for a creating Unix shell-like mini-languages which can be used - as an alternate format for application configuration files. + Support for creating Unix shell-like mini-languages which can be used as + an alternate format for application configuration files. Module :mod:`json` The json module implements a subset of JavaScript syntax which can also diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index f0742ce..469a3ee 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -46,7 +46,7 @@ A small number of constants live in the built-in namespace. They are: .. note:: - ``NotImplentedError`` and ``NotImplemented`` are not interchangeable, + ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, even though they have similar names and purposes. See :exc:`NotImplementedError` for details on when to use it. diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index d0b861d..2041d91 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -47,8 +47,8 @@ copy operations: * Recursive objects (compound objects that, directly or indirectly, contain a reference to themselves) may cause a recursive loop. -* Because deep copy copies *everything* it may copy too much, e.g., - even administrative data structures that should be shared even between copies. +* Because deep copy copies everything it may copy too much, such as data + which is intended to be shared between copies. The :func:`deepcopy` function avoids these problems by: diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 52a8a31..43714f7 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -401,8 +401,10 @@ Reader objects (:class:`DictReader` instances and objects returned by the .. method:: csvreader.__next__() - Return the next row of the reader's iterable object as a list, parsed according - to the current dialect. Usually you should call this as ``next(reader)``. + Return the next row of the reader's iterable object as a list (if the object + was returned from :func:`reader`) or a dict (if it is a :class:`DictReader` + instance), parsed according to the current dialect. Usually you should call + this as ``next(reader)``. Reader objects have the following public attributes: diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 3840935..cdcbefa 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -97,7 +97,7 @@ Functions are accessed as attributes of dll objects:: <_FuncPtr object at 0x...> >>> print(windll.kernel32.MyOwnFunction) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "ctypes.py", line 239, in __getattr__ func = _StdcallFuncPtr(name, self) AttributeError: function 'MyOwnFunction' not found @@ -135,7 +135,7 @@ functions can be accessed by indexing the dll object with the ordinal number:: <_FuncPtr object at 0x...> >>> cdll.kernel32[0] # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "ctypes.py", line 310, in __getitem__ func = _StdcallFuncPtr(name, self) AttributeError: function ordinal 0 not found @@ -161,33 +161,25 @@ as the NULL pointer):: 0x1d000000 >>> -:mod:`ctypes` tries to protect you from calling functions with the wrong number -of arguments or the wrong calling convention. Unfortunately this only works on -Windows. It does this by examining the stack after the function returns, so -although an error is raised the function *has* been called:: +.. note:: - >>> windll.kernel32.GetModuleHandleA() # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in ? - ValueError: Procedure probably called with not enough arguments (4 bytes missing) - >>> windll.kernel32.GetModuleHandleA(0, 0) # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in ? - ValueError: Procedure probably called with too many arguments (4 bytes in excess) - >>> + :mod:`ctypes` may raise a :exc:`ValueError` after calling the function, if + it detects that an invalid number of arguments were passed. This behavior + should not be relied upon. It is deprecated in 3.6.2, and will be removed + in 3.7. -The same exception is raised when you call an ``stdcall`` function with the +:exc:`ValueError` is raised when you call an ``stdcall`` function with the ``cdecl`` calling convention, or vice versa:: >>> cdll.kernel32.GetModuleHandleA(None) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: Procedure probably called with not enough arguments (4 bytes missing) >>> >>> windll.msvcrt.printf(b"spam") # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: Procedure probably called with too many arguments (4 bytes in excess) >>> @@ -200,7 +192,7 @@ argument values:: >>> windll.kernel32.GetModuleHandleA(32) # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in OSError: exception: access violation reading 0x00000020 >>> @@ -284,7 +276,7 @@ the correct type and value:: >>> c_int() c_long(0) >>> c_wchar_p("Hello, World") - c_wchar_p('Hello, World') + c_wchar_p(140018365411392) >>> c_ushort(-3) c_ushort(65533) >>> @@ -309,11 +301,15 @@ bytes objects are immutable):: >>> s = "Hello, World" >>> c_s = c_wchar_p(s) >>> print(c_s) - c_wchar_p('Hello, World') + c_wchar_p(139966785747344) + >>> print(c_s.value) + Hello World >>> c_s.value = "Hi, there" - >>> print(c_s) - c_wchar_p('Hi, there') - >>> print(s) # first object is unchanged + >>> print(c_s) # the memory location has changed + c_wchar_p(139966783348904) + >>> print(c_s.value) + Hi, there + >>> print(s) # first object is unchanged Hello, World >>> @@ -369,7 +365,7 @@ from within *IDLE* or *PythonWin*:: 19 >>> printf(b"%f bottles of beer\n", 42.5) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ArgumentError: argument 2: exceptions.TypeError: Don't know how to convert parameter 2 >>> @@ -432,7 +428,7 @@ prototype for a C function), and tries to convert the arguments to valid types:: >>> printf(b"%d %d %d", 1, 2, 3) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ArgumentError: argument 2: exceptions.TypeError: wrong type >>> printf(b"%s %d %f\n", b"X", 2, 3) X 2 3.000000 @@ -482,7 +478,7 @@ single character Python bytes object into a C char:: 'def' >>> strchr(b"abcdef", b"def") Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ArgumentError: argument 2: exceptions.TypeError: one character string expected >>> print(strchr(b"abcdef", b"x")) None @@ -508,7 +504,7 @@ useful to check for error return values and automatically raise an exception:: 486539264 >>> GetModuleHandle("something silly") # doctest: +WINDOWS Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "", line 3, in ValidHandle OSError: [Errno 126] The specified module could not be found. >>> @@ -579,7 +575,7 @@ Here is a simple example of a POINT structure, which contains two integers named 0 5 >>> POINT(1, 2, 3) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: too many initializers >>> @@ -782,7 +778,7 @@ new type:: >>> PI(42) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: expected c_long instead of int >>> PI(c_int(42)) @@ -858,7 +854,7 @@ but not instances of other types:: >>> bar.values = (c_byte * 4)() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: incompatible types, c_byte_Array_4 instance instead of LP_c_long instance >>> @@ -909,7 +905,7 @@ work:: ... ("next", POINTER(cell))] ... Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "", line 2, in cell NameError: name 'cell' is not defined >>> diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index d746eaf..668d282 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -290,8 +290,8 @@ The module :mod:`curses` defines the following functions: .. function:: initscr() - Initialize the library. Return a :class:`WindowObject` which represents the - whole screen. + Initialize the library. Return a :ref:`window ` object + which represents the whole screen. .. note:: @@ -383,8 +383,8 @@ The module :mod:`curses` defines the following functions: .. function:: newwin(nlines, ncols) newwin(nlines, ncols, begin_y, begin_x) - Return a new window, whose left-upper corner is at ``(begin_y, begin_x)``, and - whose height/width is *nlines*/*ncols*. + Return a new :ref:`window `, whose left-upper corner + is at ``(begin_y, begin_x)``, and whose height/width is *nlines*/*ncols*. By default, the window will extend from the specified position to the lower right corner of the screen. @@ -1271,27 +1271,63 @@ The :mod:`curses` module defines the following data members: A string representing the current version of the module. Also available as :const:`__version__`. -Several constants are available to specify character cell attributes: +Some constants are available to specify character cell attributes. +The exact constants available are system dependent. +------------------+-------------------------------+ | Attribute | Meaning | +==================+===============================+ -| ``A_ALTCHARSET`` | Alternate character set mode. | +| ``A_ALTCHARSET`` | Alternate character set mode | +------------------+-------------------------------+ -| ``A_BLINK`` | Blink mode. | +| ``A_BLINK`` | Blink mode | +------------------+-------------------------------+ -| ``A_BOLD`` | Bold mode. | +| ``A_BOLD`` | Bold mode | +------------------+-------------------------------+ -| ``A_DIM`` | Dim mode. | +| ``A_DIM`` | Dim mode | +------------------+-------------------------------+ -| ``A_NORMAL`` | Normal attribute. | +| ``A_INVIS`` | Invisible or blank mode | ++------------------+-------------------------------+ +| ``A_NORMAL`` | Normal attribute | ++------------------+-------------------------------+ +| ``A_PROTECT`` | Protected mode | +------------------+-------------------------------+ | ``A_REVERSE`` | Reverse background and | -| | foreground colors. | +| | foreground colors | ++------------------+-------------------------------+ +| ``A_STANDOUT`` | Standout mode | ++------------------+-------------------------------+ +| ``A_UNDERLINE`` | Underline mode | ++------------------+-------------------------------+ +| ``A_HORIZONTAL`` | Horizontal highlight | ++------------------+-------------------------------+ +| ``A_LEFT`` | Left highlight | ++------------------+-------------------------------+ +| ``A_LOW`` | Low highlight | ++------------------+-------------------------------+ +| ``A_RIGHT`` | Right highlight | ++------------------+-------------------------------+ +| ``A_TOP`` | Top highlight | ++------------------+-------------------------------+ +| ``A_VERTICAL`` | Vertical highlight | ++------------------+-------------------------------+ +| ``A_CHARTEXT`` | Bit-mask to extract a | +| | character | ++------------------+-------------------------------+ + +Several constants are available to extract corresponding attributes returned +by some methods. + ++------------------+-------------------------------+ +| Bit-mask | Meaning | ++==================+===============================+ +| ``A_ATTRIBUTES`` | Bit-mask to extract | +| | attributes | +------------------+-------------------------------+ -| ``A_STANDOUT`` | Standout mode. | +| ``A_CHARTEXT`` | Bit-mask to extract a | +| | character | +------------------+-------------------------------+ -| ``A_UNDERLINE`` | Underline mode. | +| ``A_COLOR`` | Bit-mask to extract | +| | color-pair field information | +------------------+-------------------------------+ Keys are referred to by integer constants with names starting with ``KEY_``. @@ -1443,7 +1479,7 @@ The exact keycaps available are system dependent. +-------------------+--------------------------------------------+ | ``KEY_SEOL`` | Shifted Clear line | +-------------------+--------------------------------------------+ -| ``KEY_SEXIT`` | Shifted Dxit | +| ``KEY_SEXIT`` | Shifted Exit | +-------------------+--------------------------------------------+ | ``KEY_SFIND`` | Shifted Find | +-------------------+--------------------------------------------+ @@ -1679,10 +1715,10 @@ You can instantiate a :class:`Textbox` object as follows: .. class:: Textbox(win) Return a textbox widget object. The *win* argument should be a curses - :class:`WindowObject` in which the textbox is to be contained. The edit cursor - of the textbox is initially located at the upper left hand corner of the - containing window, with coordinates ``(0, 0)``. The instance's - :attr:`stripspaces` flag is initially on. + :ref:`window ` object in which the textbox is to + be contained. The edit cursor of the textbox is initially located at the + upper left hand corner of the containing window, with coordinates ``(0, 0)``. + The instance's :attr:`stripspaces` flag is initially on. :class:`Textbox` objects have the following methods: diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index c931855..98fe86e 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1849,11 +1849,11 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: - `datetuil.tz `_ + `dateutil.tz `_ The standard library has :class:`timezone` class for handling arbitrary fixed offsets from UTC and :attr:`timezone.utc` as UTC timezone instance. - *datetuil.tz* library brings the *IANA timezone database* (also known as the + *dateutil.tz* library brings the *IANA timezone database* (also known as the Olson database) to Python and its usage is recommended. `IANA timezone database `_ diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 1c1e1a2..c795782 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -20,6 +20,10 @@ interpreter. between versions of Python. Use of this module should not be considered to work across Python VMs or Python releases. + .. versionchanged:: 3.6 + Use 2 bytes for each instruction. Previously the number of bytes varied + by instruction. + Example: Given the function :func:`myfunc`:: @@ -210,6 +214,11 @@ operation is being performed, so the intermediate analysis object isn't useful: This generator function uses the ``co_firstlineno`` and ``co_lnotab`` attributes of the code object *code* to find the offsets which are starts of lines in the source code. They are generated as ``(offset, lineno)`` pairs. + See :source:`Objects/lnotab_notes.txt` for the ``co_lnotab`` format and + how to decode it. + + .. versionchanged:: 3.6 + Line numbers can be decreasing. Before, they were always increasing. .. function:: findlabels(code) @@ -807,6 +816,15 @@ All of the following opcodes use their arguments. .. versionadded:: 3.5 +.. opcode:: BUILD_TUPLE_UNPACK_WITH_CALL (count) + + This is similar to :opcode:`BUILD_TUPLE_UNPACK`, + but is used for ``f(*x, *y, *z)`` call syntax. The stack item at position + ``count + 1`` should be the corresponding callable ``f``. + + .. versionadded:: 3.6 + + .. opcode:: BUILD_LIST_UNPACK (count) This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a list @@ -834,14 +852,16 @@ All of the following opcodes use their arguments. .. versionadded:: 3.5 -.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (oparg) +.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (count) This is similar to :opcode:`BUILD_MAP_UNPACK`, - but is used for ``f(**x, **y, **z)`` call syntax. The lowest byte of - *oparg* is the count of mappings, the relative position of the - corresponding callable ``f`` is encoded in the second byte of *oparg*. + but is used for ``f(**x, **y, **z)`` call syntax. The stack item at + position ``count + 2`` should be the corresponding callable ``f``. .. versionadded:: 3.5 + .. versionchanged:: 3.6 + The position of the callable is determined by adding 2 to the opcode + argument instead of encoding it in the second byte of the argument. .. opcode:: LOAD_ATTR (namei) @@ -998,14 +1018,45 @@ All of the following opcodes use their arguments. .. opcode:: CALL_FUNCTION (argc) - Calls a function. The low byte of *argc* indicates the number of positional - parameters, the high byte the number of keyword parameters. On the stack, the - opcode finds the keyword parameters first. For each keyword argument, the - value is on top of the key. Below the keyword parameters, the positional - parameters are on the stack, with the right-most parameter on top. Below the - parameters, the function object to call is on the stack. Pops all function - arguments, and the function itself off the stack, and pushes the return - value. + Calls a function. *argc* indicates the number of positional arguments. + The positional arguments are on the stack, with the right-most argument + on top. Below the arguments, the function object to call is on the stack. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. + + .. versionchanged:: 3.6 + This opcode is used only for calls with positional arguments. + + +.. opcode:: CALL_FUNCTION_KW (argc) + + Calls a function. *argc* indicates the number of arguments (positional + and keyword). The top element on the stack contains a tuple of keyword + argument names. Below the tuple, keyword arguments are on the stack, in + the order corresponding to the tuple. Below the keyword arguments, the + positional arguments are on the stack, with the right-most parameter on + top. Below the arguments, the function object to call is on the stack. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. + + .. versionchanged:: 3.6 + Keyword arguments are packed in a tuple instead of a dictionary, + *argc* indicates the total number of arguments + + +.. opcode:: CALL_FUNCTION_EX (flags) + + Calls a function. The lowest bit of *flags* indicates whether the + var-keyword argument is placed at the top of the stack. Below the + var-keyword argument, the var-positional argument is on the stack. + Below the arguments, the function object to call is placed. + Pops all function arguments, and the function itself off the stack, and + pushes the return value. Note that this opcode pops at most three items + from the stack. Var-positional and var-keyword arguments are packed + by :opcode:`BUILD_MAP_UNPACK_WITH_CALL` and + :opcode:`BUILD_MAP_UNPACK_WITH_CALL`. + + .. versionadded:: 3.6 .. opcode:: MAKE_FUNCTION (argc) @@ -1038,28 +1089,6 @@ All of the following opcodes use their arguments. two most-significant bytes. -.. opcode:: CALL_FUNCTION_VAR (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the variable argument list, followed by - keyword and positional arguments. - - -.. opcode:: CALL_FUNCTION_KW (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by explicit keyword and positional arguments. - - -.. opcode:: CALL_FUNCTION_VAR_KW (argc) - - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by the variable-arguments tuple, followed by explicit keyword and positional - arguments. - - .. opcode:: FORMAT_VALUE (flags) Used for implementing formatted literal strings (f-strings). Pops @@ -1085,8 +1114,13 @@ All of the following opcodes use their arguments. .. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between - opcodes which don't take arguments ``< HAVE_ARGUMENT`` and those which do - ``>= HAVE_ARGUMENT``. + opcodes which don't use their argument and those that do + (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). + + .. versionchanged:: 3.6 + Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT`` + ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. + .. _opcode_collections: diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 15b12f7..587a0a0 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -408,7 +408,7 @@ Simple example:: >>> [1, 2, 3].remove(42) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: list.remove(x): x not in list That doctest succeeds if :exc:`ValueError` is raised, with the ``list.remove(x): @@ -432,7 +432,7 @@ multi-line detail:: >>> raise ValueError('multi\n line\ndetail') Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: multi line detail @@ -591,7 +591,7 @@ doctest decides whether actual output matches an example's expected output: >>> (1, 2)[3] = 'moo' Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: object doesn't support item assignment passes under Python 2.3 and later Python versions with the flag specified, diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index b070764..7e11782 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -67,7 +67,7 @@ Here are the methods of the :class:`Message` class: Return the entire message flattened as a string. When optional *unixfrom* is true, the envelope header is included in the returned string. - *unixfrom* defaults to ``False``. For backward compabitility reasons, + *unixfrom* defaults to ``False``. For backward compatibility reasons, *maxheaderlen* defaults to ``0``, so if you want a different value you must override it explicitly (the value specified for *max_line_length* in the policy will be ignored by this method). The *policy* argument may be diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index 57743d5..f56836a 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -157,7 +157,7 @@ Currently the email package provides only one concrete content manager, MIME charset name, use the standard charset instead. If *cte* is set, encode the payload using the specified content transfer - encoding, and set the :mailheader:`Content-Transfer-Endcoding` header to + encoding, and set the :mailheader:`Content-Transfer-Encoding` header to that value. Possible values for *cte* are ``quoted-printable``, ``base64``, ``7bit``, ``8bit``, and ``binary``. If the input cannot be encoded in the specified encoding (for example, specifying a *cte* of @@ -203,5 +203,5 @@ Currently the email package provides only one concrete content manager, .. rubric:: Footnotes -.. [1] Oringally added in 3.4 as a :term:`provisional module ` diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 2d0d192..5838767 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -102,9 +102,9 @@ All defect classes are subclassed from :class:`email.errors.MessageDefect`. return false even though its content type claims to be :mimetype:`multipart`. * :class:`InvalidBase64PaddingDefect` -- When decoding a block of base64 - enocded bytes, the padding was not correct. Enough padding is added to + encoded bytes, the padding was not correct. Enough padding is added to perform the decode, but the resulting decoded bytes may be invalid. * :class:`InvalidBase64CharactersDefect` -- When decoding a block of base64 - enocded bytes, characters outside the base64 alphebet were encountered. + encoded bytes, characters outside the base64 alphabet were encountered. The characters are ignored, but the resulting decoded bytes may be invalid. diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index ab0fbc2..1e64e10 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -88,8 +88,8 @@ over channels that are not "8 bit clean". If ``cte_type`` is ``7bit``, convert the bytes with the high bit set as needed using an ASCII-compatible :mailheader:`Content-Transfer-Encoding`. That is, transform parts with non-ASCII - :mailheader:`Cotnent-Transfer-Encoding` - (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatibile + :mailheader:`Content-Transfer-Encoding` + (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatible :mailheader:`Content-Transfer-Encoding`, and encode RFC-invalid non-ASCII bytes in headers using the MIME ``unknown-8bit`` character set, thus rendering them RFC-compliant. diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 2c830cf..ce283c6 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -451,5 +451,5 @@ construct structured values to assign to specific headers. .. rubric:: Footnotes -.. [1] Oringally added in 3.3 as a :term:`provisional module ` diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index d36e769..261d0d6 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -52,7 +52,7 @@ message objects. .. class:: EmailMessage(policy=default) - If *policy* is specified use the rules it specifies to udpate and serialize + If *policy* is specified use the rules it specifies to update and serialize the representation of the message. If *policy* is not set, use the :class:`~email.policy.default` policy, which follows the rules of the email RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses @@ -63,7 +63,7 @@ message objects. Return the entire message flattened as a string. When optional *unixfrom* is true, the envelope header is included in the returned - string. *unixfrom* defaults to ``False``. For backward compabitility + string. *unixfrom* defaults to ``False``. For backward compatibility with the base :class:`~email.message.Message` class *maxheaderlen* is accepted, but defaults to ``None``, which means that by default the line length is controlled by the @@ -213,7 +213,7 @@ message objects. del msg['subject'] msg['subject'] = 'Python roolz!' - If the :mod:`policy` defines certain haders to be unique (as the standard + If the :mod:`policy` defines certain headers to be unique (as the standard policies do), this method may raise a :exc:`ValueError` when an attempt is made to assign a value to such a header when one already exists. This behavior is intentional for consistency's sake, but do not depend on it @@ -558,7 +558,7 @@ message objects. the part a candidate match if the value of the header is ``inline``. If none of the candidates matches any of the preferences in - *preferneclist*, return ``None``. + *preferencelist*, return ``None``. Notes: (1) For most applications the only *preferencelist* combinations that really make sense are ``('plain',)``, ``('html', 'plain')``, and the @@ -746,6 +746,6 @@ message objects. .. rubric:: Footnotes -.. [1] Oringally added in 3.4 as a :term:`provisional module `. Docs for legacy message class moved to :ref:`compat32_message`. diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst index d9dae9f..f37f6aa 100644 --- a/Doc/library/email.mime.rst +++ b/Doc/library/email.mime.rst @@ -242,7 +242,7 @@ Here are the classes: Unless the *_charset* argument is explicitly set to ``None``, the MIMEText object created will have both a :mailheader:`Content-Type` header - with a ``charset`` parameter, and a :mailheader:`Content-Transfer-Endcoding` + with a ``charset`` parameter, and a :mailheader:`Content-Transfer-Encoding` header. This means that a subsequent ``set_payload`` call will not result in an encoded payload, even if a charset is passed in the ``set_payload`` command. You can "reset" this behavior by deleting the diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index c323ebc..dea409d 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -139,7 +139,7 @@ message body, instead setting the payload to the raw body. .. class:: BytesParser(_class=None, *, policy=policy.compat32) Create a :class:`BytesParser` instance. The *_class* and *policy* - arguments have the same meaning and sematnics as the *_factory* + arguments have the same meaning and semantics as the *_factory* and *policy* arguments of :class:`BytesFeedParser`. Note: **The policy keyword should always be specified**; The default will diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 8a41877..8e70762 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -276,7 +276,7 @@ added matters. To illustrate:: Called when a header is added to an :class:`~email.message.EmailMessage` or :class:`~email.message.Message` object. If the returned value is not ``0`` or ``None``, and there are already a number of headers with the - name *name* greather than or equal to the value returned, a + name *name* greater than or equal to the value returned, a :exc:`ValueError` is raised. Because the default behavior of ``Message.__setitem__`` is to append the @@ -533,7 +533,7 @@ more closely to the RFCs relevant to their domains. The same as ``SMTP`` except that :attr:`~EmailPolicy.utf8` is ``True``. Useful for serializing messages to a message store without using encoded - words in the headers. Should only be used for SMTP trasmission if the + words in the headers. Should only be used for SMTP transmission if the sender or recipient addresses have non-ASCII characters (the :meth:`smtplib.SMTP.send_message` method handles this automatically). @@ -647,5 +647,5 @@ The header objects and their attributes are described in .. rubric:: Footnotes -.. [1] Oringally added in 3.3 as a :term:`provisional feature `. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index a428f51..a6b20a5 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -243,7 +243,7 @@ The following exceptions are the exceptions that are usually raised. .. note:: - It should not be used to indicate that an operater or method is not + It should not be used to indicate that an operator or method is not meant to be supported at all -- in that case either leave the operator / method undefined or, if a subclass, set it to :data:`None`. diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index c03a9d3..634c26e 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -43,9 +43,8 @@ patterns. .. function:: fnmatch(filename, pattern) Test whether the *filename* string matches the *pattern* string, returning - :const:`True` or :const:`False`. If the operating system is case-insensitive, - then both parameters will be normalized to all lower- or upper-case before - the comparison is performed. :func:`fnmatchcase` can be used to perform a + :const:`True` or :const:`False`. Both parameters are case-normalized + using :func:`os.path.normcase`. :func:`fnmatchcase` can be used to perform a case-sensitive comparison, regardless of whether that's standard for the operating system. @@ -63,7 +62,8 @@ patterns. .. function:: fnmatchcase(filename, pattern) Test whether *filename* matches *pattern*, returning :const:`True` or - :const:`False`; the comparison is case-sensitive. + :const:`False`; the comparison is case-sensitive and does not apply + :func:`os.path.normcase`. .. function:: filter(names, pattern) diff --git a/Doc/library/fpectl.rst b/Doc/library/fpectl.rst index e4b528c..9660716 100644 --- a/Doc/library/fpectl.rst +++ b/Doc/library/fpectl.rst @@ -89,7 +89,7 @@ The following example demonstrates how to start up and test operation of the >>> import math >>> math.exp(1000) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in FloatingPointError: in math_1 diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index b8c1dcf..7291dfe 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -235,7 +235,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. Retrieve a file in binary transfer mode. *cmd* should be an appropriate ``RETR`` command: ``'RETR filename'``. The *callback* function is called for - each block of data received, with a single string argument giving the data + each block of data received, with a single bytes argument giving the data block. The optional *blocksize* argument specifies the maximum chunk size to read on the low-level socket object created to do the actual transfer (which will also be the largest size of the data blocks passed to *callback*). A @@ -255,9 +255,9 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. prints the line to ``sys.stdout``. -.. method:: FTP.set_pasv(boolean) +.. method:: FTP.set_pasv(val) - Enable "passive" mode if *boolean* is true, other disable passive mode. + Enable "passive" mode if *val* is true, otherwise disable passive mode. Passive mode is on by default. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index c26037b..6621f4a 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -16,8 +16,8 @@ are always available. They are listed here in alphabetical order. :func:`ascii` :func:`enumerate` :func:`input` :func:`oct` :func:`staticmethod` :func:`bin` :func:`eval` :func:`int` :func:`open` |func-str|_ :func:`bool` :func:`exec` :func:`isinstance` :func:`ord` :func:`sum` -:func:`bytearray` :func:`filter` :func:`issubclass` :func:`pow` :func:`super` -:func:`bytes` :func:`float` :func:`iter` :func:`print` |func-tuple|_ +|func-bytearray|_ :func:`filter` :func:`issubclass` :func:`pow` :func:`super` +|func-bytes|_ :func:`float` :func:`iter` :func:`print` |func-tuple|_ :func:`callable` :func:`format` :func:`len` :func:`property` :func:`type` :func:`chr` |func-frozenset|_ |func-list|_ |func-range|_ :func:`vars` :func:`classmethod` :func:`getattr` :func:`locals` :func:`repr` :func:`zip` @@ -37,7 +37,8 @@ are always available. They are listed here in alphabetical order. .. |func-str| replace:: ``str()`` .. |func-tuple| replace:: ``tuple()`` .. |func-range| replace:: ``range()`` - +.. |func-bytearray| replace:: ``bytearray()`` +.. |func-bytes| replace:: ``bytes()`` .. function:: abs(x) @@ -99,6 +100,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytearray: .. class:: bytearray([source[, encoding[, errors]]]) + :noindex: Return a new array of bytes. The :class:`bytearray` class is a mutable sequence of integers in the range 0 <= x < 256. It has most of the usual @@ -128,6 +130,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytes: .. class:: bytes([source[, encoding[, errors]]]) + :noindex: Return a new "bytes" object, which is an immutable sequence of integers in the range ``0 <= x < 256``. :class:`bytes` is an immutable version of @@ -610,7 +613,7 @@ are always available. They are listed here in alphabetical order. .. note:: - For object's with custom :meth:`__hash__` methods, note that :func:`hash` + For objects with custom :meth:`__hash__` methods, note that :func:`hash` truncates the return value based on the bit width of the host machine. See :meth:`__hash__` for details. @@ -1072,7 +1075,7 @@ are always available. They are listed here in alphabetical order. * The ``'x'`` mode was added. * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. * :exc:`FileExistsError` is now raised if the file opened in exclusive - * creation mode (``'x'``) already exists. + creation mode (``'x'``) already exists. .. versionchanged:: 3.4 @@ -1125,7 +1128,7 @@ are always available. They are listed here in alphabetical order. .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) Print *objects* to the text stream *file*, separated by *sep* and followed - by *end*. *sep*, *end* and *file*, if present, must be given as keyword + by *end*. *sep*, *end*, *file* and *flush*, if present, must be given as keyword arguments. All non-keyword arguments are converted to strings like :func:`str` does and @@ -1254,17 +1257,21 @@ are always available. They are listed here in alphabetical order. .. function:: round(number[, ndigits]) - Return the floating point value *number* rounded to *ndigits* digits after - the decimal point. If *ndigits* is omitted or is ``None``, it returns the - nearest integer to its input. Delegates to ``number.__round__(ndigits)``. + Return *number* rounded to *ndigits* precision after the decimal + point. If *ndigits* is omitted or is ``None``, it returns the + nearest integer to its input. For the built-in types supporting :func:`round`, values are rounded to the closest multiple of 10 to the power minus *ndigits*; if two multiples are equally close, rounding is done toward the even choice (so, for example, both ``round(0.5)`` and ``round(-0.5)`` are ``0``, and ``round(1.5)`` is - ``2``). The return value is an integer if called with one argument, + ``2``). Any integer value is valid for *ndigits* (positive, zero, or + negative). The return value is an integer if called with one argument, otherwise of the same type as *number*. + For a general Python object ``number``, ``round(number, ndigits)`` delegates to + ``number.__round__(ndigits)``. + .. note:: The behavior of :func:`round` for floats can be surprising: for example, diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index fb5c1df..b29020b 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -105,7 +105,8 @@ of which this module provides three different variants: Contains the output stream for writing a response back to the client. Proper adherence to the HTTP protocol must be used when writing to - this stream. + this stream in order to achieve successful interoperation with HTTP + clients. .. versionchanged:: 3.6 This is an :class:`io.BufferedIOBase` stream. diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 07c2a25..a945b6d 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -244,7 +244,7 @@ Go to File/Line single: stack viewer Debugger (toggle) - When actived, code entered in the Shell or run from an Editor will run + When activated, code entered in the Shell or run from an Editor will run under the debugger. In the Editor, breakpoints can be set with the context menu. This feature is still incomplete and somewhat experimental. @@ -372,7 +372,7 @@ the :kbd:`Command` key on Mac OSX. * :kbd:`C-l` center window around the insertion point - * :kbd:`C-b` go backwards one character without deleting (usually you can + * :kbd:`C-b` go backward one character without deleting (usually you can also use the cursor key for this) * :kbd:`C-f` go forward one character without deleting (usually you can @@ -394,7 +394,7 @@ After a block-opening statement, the next line is indented by 4 spaces (in the Python Shell window by one tab). After certain keywords (break, return etc.) the next line is dedented. In leading indentation, :kbd:`Backspace` deletes up to 4 spaces if they are there. :kbd:`Tab` inserts spaces (in the Python -Shell window one tab), number depends on Indent width. Currently tabs +Shell window one tab), number depends on Indent width. Currently, tabs are restricted to four spaces due to Tcl/Tk limitations. See also the indent/dedent region commands in the edit menu. @@ -418,7 +418,7 @@ If there is only one possible completion for the characters entered, a :kbd:`C-space` will open a completions window. In an empty string, this will contain the files in the current directory. On a blank line, it will contain the built-in and user-defined functions and -classes in the current name spaces, plus any modules imported. If some +classes in the current namespaces, plus any modules imported. If some characters have been entered, the ACW will attempt to be more specific. If a string of characters is typed, the ACW selection will jump to the @@ -552,12 +552,56 @@ If there are arguments: ``sys.argv`` reflects the arguments passed to IDLE itself. +Startup failure +^^^^^^^^^^^^^^^ + +IDLE uses a socket to communicate between the IDLE GUI process and the user +code execution process. A connection must be established whenever the Shell +starts or restarts. (The latter is indicated by a divider line that says +'RESTART'). If the user process fails to connect to the GUI process, it +displays a ``Tk`` error box with a 'cannot connect' message that directs the +user here. It then exits. + +A common cause of failure is a user-written file with the same name as a +standard library module, such as *random.py* and *tkinter.py*. When such a +file is located in the same directory as a file that is about to be run, +IDLE cannot import the stdlib file. The current fix is to rename the +user file. + +Though less common than in the past, an antivirus or firewall program may +stop the connection. If the program cannot be taught to allow the +connection, then it must be turned off for IDLE to work. It is safe to +allow this internal connection because no data is visible on external +ports. A similar problem is a network mis-configuration that blocks +connections. + +Python installation issues occasionally stop IDLE: multiple versions can +clash, or a single installation might need admin access. If one undo the +clash, or cannot or does not want to run as admin, it might be easiest to +completely remove Python and start over. + +A zombie pythonw.exe process could be a problem. On Windows, use Task +Manager to detect and stop one. Sometimes a restart initiated by a program +crash or Keyboard Interrupt (control-C) may fail to connect. Dismissing +the error box or Restart Shell on the Shell menu may fix a temporary problem. + +When IDLE first starts, it attempts to read user configuration files in +~/.idlerc/ (~ is one's home directory). If there is a problem, an error +message should be displayed. Leaving aside random disk glitches, this can +be prevented by never editing the files by hand, using the configuration +dialog, under Options, instead Options. Once it happens, the solution may +be to delete one or more of the configuration files. + +If IDLE quits with no message, and it was not started from a console, try +starting from a console (``python -m idlelib)`` and see if a message appears. + + IDLE-console differences ^^^^^^^^^^^^^^^^^^^^^^^^ As much as possible, the result of executing Python code with IDLE is the same as executing the same code in a console window. However, the different -interface and operation occasionally affects visible results. For instance, +interface and operation occasionally affect visible results. For instance, ``sys.modules`` starts with more entries. IDLE also replaces ``sys.stdin``, ``sys.stdout``, and ``sys.stderr`` with @@ -583,7 +627,7 @@ If firewall software complains anyway, you can ignore it. If the attempt to make the socket connection fails, Idle will notify you. Such failures are sometimes transient, but if persistent, the problem -may be either a firewall blocking the connecton or misconfiguration of +may be either a firewall blocking the connection or misconfiguration of a particular system. Until the problem is fixed, one can run Idle with the -n command line switch. @@ -619,14 +663,14 @@ Setting preferences The font preferences, highlighting, keys, and general preferences can be changed via Configure IDLE on the Option menu. Keys can be user defined; -IDLE ships with four built in key sets. In addition a user can create a +IDLE ships with four built-in key sets. In addition, a user can create a custom key set in the Configure IDLE dialog under the keys tab. Extensions ^^^^^^^^^^ -IDLE contains an extension facility. Peferences for extensions can be +IDLE contains an extension facility. Preferences for extensions can be changed with Configure Extensions. See the beginning of config-extensions.def in the idlelib directory for further information. The default extensions are currently: diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 3fa44a0..6be28a2 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -34,185 +34,198 @@ provided as convenient choices for the second argument to :func:`getmembers`. They also help you determine when you can expect to find the following special attributes: -+-----------+-----------------+---------------------------+ -| Type | Attribute | Description | -+===========+=================+===========================+ -| module | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __file__ | filename (missing for | -| | | built-in modules) | -+-----------+-----------------+---------------------------+ -| class | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | class was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __module__ | name of module in which | -| | | this class was defined | -+-----------+-----------------+---------------------------+ -| method | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | method was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __func__ | function object | -| | | containing implementation | -| | | of method | -+-----------+-----------------+---------------------------+ -| | __self__ | instance to which this | -| | | method is bound, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ -| function | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | name with which this | -| | | function was defined | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __code__ | code object containing | -| | | compiled function | -| | | :term:`bytecode` | -+-----------+-----------------+---------------------------+ -| | __defaults__ | tuple of any default | -| | | values for positional or | -| | | keyword parameters | -+-----------+-----------------+---------------------------+ -| | __kwdefaults__ | mapping of any default | -| | | values for keyword-only | -| | | parameters | -+-----------+-----------------+---------------------------+ -| | __globals__ | global namespace in which | -| | | this function was defined | -+-----------+-----------------+---------------------------+ -| | __annotations__ | mapping of parameters | -| | | names to annotations; | -| | | ``"return"`` key is | -| | | reserved for return | -| | | annotations. | -+-----------+-----------------+---------------------------+ -| traceback | tb_frame | frame object at this | -| | | level | -+-----------+-----------------+---------------------------+ -| | tb_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-----------------+---------------------------+ -| | tb_lineno | current line number in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | tb_next | next inner traceback | -| | | object (called by this | -| | | level) | -+-----------+-----------------+---------------------------+ -| frame | f_back | next outer frame object | -| | | (this frame's caller) | -+-----------+-----------------+---------------------------+ -| | f_builtins | builtins namespace seen | -| | | by this frame | -+-----------+-----------------+---------------------------+ -| | f_code | code object being | -| | | executed in this frame | -+-----------+-----------------+---------------------------+ -| | f_globals | global namespace seen by | -| | | this frame | -+-----------+-----------------+---------------------------+ -| | f_lasti | index of last attempted | -| | | instruction in bytecode | -+-----------+-----------------+---------------------------+ -| | f_lineno | current line number in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | f_locals | local namespace seen by | -| | | this frame | -+-----------+-----------------+---------------------------+ -| | f_restricted | 0 or 1 if frame is in | -| | | restricted execution mode | -+-----------+-----------------+---------------------------+ -| | f_trace | tracing function for this | -| | | frame, or ``None`` | -+-----------+-----------------+---------------------------+ -| code | co_argcount | number of arguments (not | -| | | including \* or \*\* | -| | | args) | -+-----------+-----------------+---------------------------+ -| | co_code | string of raw compiled | -| | | bytecode | -+-----------+-----------------+---------------------------+ -| | co_consts | tuple of constants used | -| | | in the bytecode | -+-----------+-----------------+---------------------------+ -| | co_filename | name of file in which | -| | | this code object was | -| | | created | -+-----------+-----------------+---------------------------+ -| | co_firstlineno | number of first line in | -| | | Python source code | -+-----------+-----------------+---------------------------+ -| | co_flags | bitmap of ``CO_*`` flags, | -| | | read more :ref:`here | -| | | `| -+-----------+-----------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------+-----------------+---------------------------+ -| | co_name | name with which this code | -| | | object was defined | -+-----------+-----------------+---------------------------+ -| | co_names | tuple of names of local | -| | | variables | -+-----------+-----------------+---------------------------+ -| | co_nlocals | number of local variables | -+-----------+-----------------+---------------------------+ -| | co_stacksize | virtual machine stack | -| | | space required | -+-----------+-----------------+---------------------------+ -| | co_varnames | tuple of names of | -| | | arguments and local | -| | | variables | -+-----------+-----------------+---------------------------+ -| generator | __name__ | name | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | gi_frame | frame | -+-----------+-----------------+---------------------------+ -| | gi_running | is the generator running? | -+-----------+-----------------+---------------------------+ -| | gi_code | code | -+-----------+-----------------+---------------------------+ -| | gi_yieldfrom | object being iterated by | -| | | ``yield from``, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ -| coroutine | __name__ | name | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | cr_await | object being awaited on, | -| | | or ``None`` | -+-----------+-----------------+---------------------------+ -| | cr_frame | frame | -+-----------+-----------------+---------------------------+ -| | cr_running | is the coroutine running? | -+-----------+-----------------+---------------------------+ -| | cr_code | code | -+-----------+-----------------+---------------------------+ -| builtin | __doc__ | documentation string | -+-----------+-----------------+---------------------------+ -| | __name__ | original name of this | -| | | function or method | -+-----------+-----------------+---------------------------+ -| | __qualname__ | qualified name | -+-----------+-----------------+---------------------------+ -| | __self__ | instance to which a | -| | | method is bound, or | -| | | ``None`` | -+-----------+-----------------+---------------------------+ ++-----------+-------------------+---------------------------+ +| Type | Attribute | Description | ++===========+===================+===========================+ +| module | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __file__ | filename (missing for | +| | | built-in modules) | ++-----------+-------------------+---------------------------+ +| class | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | class was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __module__ | name of module in which | +| | | this class was defined | ++-----------+-------------------+---------------------------+ +| method | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | method was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __func__ | function object | +| | | containing implementation | +| | | of method | ++-----------+-------------------+---------------------------+ +| | __self__ | instance to which this | +| | | method is bound, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ +| function | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | name with which this | +| | | function was defined | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __code__ | code object containing | +| | | compiled function | +| | | :term:`bytecode` | ++-----------+-------------------+---------------------------+ +| | __defaults__ | tuple of any default | +| | | values for positional or | +| | | keyword parameters | ++-----------+-------------------+---------------------------+ +| | __kwdefaults__ | mapping of any default | +| | | values for keyword-only | +| | | parameters | ++-----------+-------------------+---------------------------+ +| | __globals__ | global namespace in which | +| | | this function was defined | ++-----------+-------------------+---------------------------+ +| | __annotations__ | mapping of parameters | +| | | names to annotations; | +| | | ``"return"`` key is | +| | | reserved for return | +| | | annotations. | ++-----------+-------------------+---------------------------+ +| traceback | tb_frame | frame object at this | +| | | level | ++-----------+-------------------+---------------------------+ +| | tb_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------+-------------------+---------------------------+ +| | tb_lineno | current line number in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | tb_next | next inner traceback | +| | | object (called by this | +| | | level) | ++-----------+-------------------+---------------------------+ +| frame | f_back | next outer frame object | +| | | (this frame's caller) | ++-----------+-------------------+---------------------------+ +| | f_builtins | builtins namespace seen | +| | | by this frame | ++-----------+-------------------+---------------------------+ +| | f_code | code object being | +| | | executed in this frame | ++-----------+-------------------+---------------------------+ +| | f_globals | global namespace seen by | +| | | this frame | ++-----------+-------------------+---------------------------+ +| | f_lasti | index of last attempted | +| | | instruction in bytecode | ++-----------+-------------------+---------------------------+ +| | f_lineno | current line number in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | f_locals | local namespace seen by | +| | | this frame | ++-----------+-------------------+---------------------------+ +| | f_restricted | 0 or 1 if frame is in | +| | | restricted execution mode | ++-----------+-------------------+---------------------------+ +| | f_trace | tracing function for this | +| | | frame, or ``None`` | ++-----------+-------------------+---------------------------+ +| code | co_argcount | number of arguments (not | +| | | including keyword only | +| | | arguments, \* or \*\* | +| | | args) | ++-----------+-------------------+---------------------------+ +| | co_code | string of raw compiled | +| | | bytecode | ++-----------+-------------------+---------------------------+ +| | co_cellvars | tuple of names of cell | +| | | variables (referenced by | +| | | containing scopes) | ++-----------+-------------------+---------------------------+ +| | co_consts | tuple of constants used | +| | | in the bytecode | ++-----------+-------------------+---------------------------+ +| | co_filename | name of file in which | +| | | this code object was | +| | | created | ++-----------+-------------------+---------------------------+ +| | co_firstlineno | number of first line in | +| | | Python source code | ++-----------+-------------------+---------------------------+ +| | co_flags | bitmap of ``CO_*`` flags, | +| | | read more :ref:`here | +| | | `| ++-----------+-------------------+---------------------------+ +| | co_lnotab | encoded mapping of line | +| | | numbers to bytecode | +| | | indices | ++-----------+-------------------+---------------------------+ +| | co_freevars | tuple of names of free | +| | | variables (referenced via | +| | | a function's closure) | ++-----------+-------------------+---------------------------+ +| | co_kwonlyargcount | number of keyword only | +| | | arguments (not including | +| | | \*\* arg) | ++-----------+-------------------+---------------------------+ +| | co_name | name with which this code | +| | | object was defined | ++-----------+-------------------+---------------------------+ +| | co_names | tuple of names of local | +| | | variables | ++-----------+-------------------+---------------------------+ +| | co_nlocals | number of local variables | ++-----------+-------------------+---------------------------+ +| | co_stacksize | virtual machine stack | +| | | space required | ++-----------+-------------------+---------------------------+ +| | co_varnames | tuple of names of | +| | | arguments and local | +| | | variables | ++-----------+-------------------+---------------------------+ +| generator | __name__ | name | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | gi_frame | frame | ++-----------+-------------------+---------------------------+ +| | gi_running | is the generator running? | ++-----------+-------------------+---------------------------+ +| | gi_code | code | ++-----------+-------------------+---------------------------+ +| | gi_yieldfrom | object being iterated by | +| | | ``yield from``, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ +| coroutine | __name__ | name | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | cr_await | object being awaited on, | +| | | or ``None`` | ++-----------+-------------------+---------------------------+ +| | cr_frame | frame | ++-----------+-------------------+---------------------------+ +| | cr_running | is the coroutine running? | ++-----------+-------------------+---------------------------+ +| | cr_code | code | ++-----------+-------------------+---------------------------+ +| builtin | __doc__ | documentation string | ++-----------+-------------------+---------------------------+ +| | __name__ | original name of this | +| | | function or method | ++-----------+-------------------+---------------------------+ +| | __qualname__ | qualified name | ++-----------+-------------------+---------------------------+ +| | __self__ | instance to which a | +| | | method is bound, or | +| | | ``None`` | ++-----------+-------------------+---------------------------+ .. versionchanged:: 3.5 @@ -442,7 +455,9 @@ Retrieving source code Return in a single string any lines of comments immediately preceding the object's source code (for a class, function, or method), or at the top of the - Python source file (if the object is a module). + Python source file (if the object is a module). If the object's source code + is unavailable, return ``None``. This could happen if the object has been + defined in C or the interactive shell. .. function:: getfile(object) @@ -1266,6 +1281,10 @@ Code Objects Bit Flags Python code objects have a ``co_flags`` attribute, which is a bitmap of the following flags: +.. data:: CO_OPTIMIZED + + The code object is optimized, using fast locals. + .. data:: CO_NEWLOCALS If set, a new dict will be created for the frame's ``f_locals`` when @@ -1279,6 +1298,10 @@ the following flags: The code object has a variable keyword parameter (``**kwargs``-like). +.. data:: CO_NESTED + + The flag is set when the code object is a nested function. + .. data:: CO_GENERATOR The flag is set when the code object is a generator function, i.e. diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 8103c61..fdbdcb1 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -503,7 +503,7 @@ Encoders and Decoders Exceptions ---------- -.. exception:: JSONDecodeError(msg, doc, pos, end=None) +.. exception:: JSONDecodeError(msg, doc, pos) Subclass of :exc:`ValueError` with the following additional attributes: diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index 1ffc6ef..d65afc2 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -49,7 +49,7 @@ For format *version* lower than 3, recursive lists, sets and dictionaries cannot be written (see below). There are functions that read/write files as well as functions operating on -strings. +bytes-like objects. The module defines these functions: @@ -57,9 +57,7 @@ The module defines these functions: .. function:: dump(value, file[, version]) Write the value on the open file. The value must be a supported type. The - file must be an open file object such as ``sys.stdout`` or returned by - :func:`open` or :func:`os.popen`. It must be opened in binary mode (``'wb'`` - or ``'w+b'``). + file must be a writeable :term:`binary file`. If the value has (or contains an object that has) an unsupported type, a :exc:`ValueError` exception is raised --- but garbage data will also be written @@ -74,8 +72,7 @@ The module defines these functions: Read one value from the open file and return it. If no valid value is read (e.g. because the data has a different Python version's incompatible marshal format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. The - file must be an open file object opened in binary mode (``'rb'`` or - ``'r+b'``). + file must be a readable :term:`binary file`. .. note:: @@ -85,7 +82,7 @@ The module defines these functions: .. function:: dumps(value[, version]) - Return the string that would be written to a file by ``dump(value, file)``. The + Return the bytes object that would be written to a file by ``dump(value, file)``. The value must be a supported type. Raise a :exc:`ValueError` exception if value has (or contains an object that has) an unsupported type. @@ -93,11 +90,11 @@ The module defines these functions: (see below). -.. function:: loads(string) +.. function:: loads(bytes) - Convert the string to a value. If no valid value is found, raise - :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra characters in the - string are ignored. + Convert the :term:`bytes-like object` to a value. If no valid value is found, raise + :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra bytes in the + input are ignored. In addition, the following constants are defined: diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 464248c..67b7a71 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -186,78 +186,78 @@ than one MIME-type database; it provides an interface similar to the one of the loaded "on top" of the default database. -.. attribute:: MimeTypes.suffix_map + .. attribute:: MimeTypes.suffix_map - Dictionary mapping suffixes to suffixes. This is used to allow recognition of - encoded files for which the encoding and the type are indicated by the same - extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` - to allow the encoding and type to be recognized separately. This is initially a - copy of the global :data:`suffix_map` defined in the module. + Dictionary mapping suffixes to suffixes. This is used to allow recognition of + encoded files for which the encoding and the type are indicated by the same + extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` + to allow the encoding and type to be recognized separately. This is initially a + copy of the global :data:`suffix_map` defined in the module. -.. attribute:: MimeTypes.encodings_map + .. attribute:: MimeTypes.encodings_map - Dictionary mapping filename extensions to encoding types. This is initially a - copy of the global :data:`encodings_map` defined in the module. + Dictionary mapping filename extensions to encoding types. This is initially a + copy of the global :data:`encodings_map` defined in the module. -.. attribute:: MimeTypes.types_map + .. attribute:: MimeTypes.types_map - Tuple containing two dictionaries, mapping filename extensions to MIME types: - the first dictionary is for the non-standards types and the second one is for - the standard types. They are initialized by :data:`common_types` and - :data:`types_map`. + Tuple containing two dictionaries, mapping filename extensions to MIME types: + the first dictionary is for the non-standards types and the second one is for + the standard types. They are initialized by :data:`common_types` and + :data:`types_map`. -.. attribute:: MimeTypes.types_map_inv + .. attribute:: MimeTypes.types_map_inv - Tuple containing two dictionaries, mapping MIME types to a list of filename - extensions: the first dictionary is for the non-standards types and the - second one is for the standard types. They are initialized by - :data:`common_types` and :data:`types_map`. + Tuple containing two dictionaries, mapping MIME types to a list of filename + extensions: the first dictionary is for the non-standards types and the + second one is for the standard types. They are initialized by + :data:`common_types` and :data:`types_map`. -.. method:: MimeTypes.guess_extension(type, strict=True) + .. method:: MimeTypes.guess_extension(type, strict=True) - Similar to the :func:`guess_extension` function, using the tables stored as part - of the object. + Similar to the :func:`guess_extension` function, using the tables stored as part + of the object. -.. method:: MimeTypes.guess_type(url, strict=True) + .. method:: MimeTypes.guess_type(url, strict=True) - Similar to the :func:`guess_type` function, using the tables stored as part of - the object. + Similar to the :func:`guess_type` function, using the tables stored as part of + the object. -.. method:: MimeTypes.guess_all_extensions(type, strict=True) + .. method:: MimeTypes.guess_all_extensions(type, strict=True) - Similar to the :func:`guess_all_extensions` function, using the tables stored - as part of the object. + Similar to the :func:`guess_all_extensions` function, using the tables stored + as part of the object. -.. method:: MimeTypes.read(filename, strict=True) + .. method:: MimeTypes.read(filename, strict=True) - Load MIME information from a file named *filename*. This uses :meth:`readfp` to - parse the file. + Load MIME information from a file named *filename*. This uses :meth:`readfp` to + parse the file. - If *strict* is ``True``, information will be added to list of standard types, - else to the list of non-standard types. + If *strict* is ``True``, information will be added to list of standard types, + else to the list of non-standard types. -.. method:: MimeTypes.readfp(fp, strict=True) + .. method:: MimeTypes.readfp(fp, strict=True) - Load MIME type information from an open file *fp*. The file must have the format of - the standard :file:`mime.types` files. + Load MIME type information from an open file *fp*. The file must have the format of + the standard :file:`mime.types` files. - If *strict* is ``True``, information will be added to the list of standard - types, else to the list of non-standard types. + If *strict* is ``True``, information will be added to the list of standard + types, else to the list of non-standard types. -.. method:: MimeTypes.read_windows_registry(strict=True) + .. method:: MimeTypes.read_windows_registry(strict=True) - Load MIME type information from the Windows registry. Availability: Windows. + Load MIME type information from the Windows registry. Availability: Windows. - If *strict* is ``True``, information will be added to the list of standard - types, else to the list of non-standard types. + If *strict* is ``True``, information will be added to the list of standard + types, else to the list of non-standard types. - .. versionadded:: 3.2 + .. versionadded:: 3.2 diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst index ee1ce50..d40abe1 100644 --- a/Doc/library/othergui.rst +++ b/Doc/library/othergui.rst @@ -9,14 +9,15 @@ available for Python: .. seealso:: `PyGObject `_ - provides introspection bindings for C libraries using + PyGObject provides introspection bindings for C libraries using `GObject `_. One of these libraries is the `GTK+ 3 `_ widget set. GTK+ comes with many more widgets than Tkinter provides. An online `Python GTK+ 3 Tutorial `_ is available. - `PyGTK `_ provides bindings for an older version + `PyGTK `_ + PyGTK provides bindings for an older version of the library, GTK+ 2. It provides an object oriented interface that is slightly higher level than the C one. There are also bindings to `GNOME `_. An online `tutorial @@ -27,15 +28,10 @@ available for Python: extensive C++ GUI application development framework that is available for Unix, Windows and Mac OS X. :program:`sip` is a tool for generating bindings for C++ libraries as Python classes, and - is specifically designed for Python. The *PyQt3* bindings have a - book, `GUI Programming with Python: QT Edition - `_ by Boudewijn - Rempt. The *PyQt4* bindings also have a book, `Rapid GUI Programming - with Python and Qt `_, by Mark - Summerfield. + is specifically designed for Python. `PySide `_ - is a newer binding to the Qt toolkit, provided by Nokia. + PySide is a newer binding to the Qt toolkit, provided by Nokia. Compared to PyQt, its licensing scheme is friendlier to non-open source applications. @@ -49,9 +45,7 @@ available for Python: documentation and context sensitive help, printing, HTML viewing, low-level device context drawing, drag and drop, system clipboard access, an XML-based resource format and more, including an ever growing library - of user-contributed modules. wxPython has a book, `wxPython in Action - `_, by Noel Rappin and - Robin Dunn. + of user-contributed modules. PyGTK, PyQt, and wxPython, all have a modern look and feel and more widgets than Tkinter. In addition, there are many other GUI toolkits for diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 7c37bb7..6225a3a 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -76,7 +76,7 @@ The typical usage to inspect a crashed program is:: >>> import mymodule >>> mymodule.test() Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in File "./mymodule.py", line 4, in test test2() File "./mymodule.py", line 3, in test2 diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index bd67fe4..5796e3a 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -85,11 +85,11 @@ next line: ``Ordered by: standard name``, indicates that the text string in the far right column was used to sort the output. The column headings include: ncalls - for the number of calls, + for the number of calls. tottime - for the total time spent in the given function (and excluding time made in - calls to sub-functions) + for the total time spent in the given function (and excluding time made in + calls to sub-functions) percall is the quotient of ``tottime`` divided by ``ncalls`` @@ -215,11 +215,11 @@ functions: and gathers profiling statistics from the execution. If no file name is present, then this function automatically creates a :class:`~pstats.Stats` - instance and prints a simple profiling report. If the sort value is specified + instance and prints a simple profiling report. If the sort value is specified, it is passed to this :class:`~pstats.Stats` instance to control how the results are sorted. -.. function:: runctx(command, globals, locals, filename=None) +.. function:: runctx(command, globals, locals, filename=None, sort=-1) This function is similar to :func:`run`, with added arguments to supply the globals and locals dictionaries for the *command* string. This routine diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 075a8b5..e43b9ae 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -869,7 +869,7 @@ The ``errors`` module has the following attributes: .. rubric:: Footnotes -.. [#] The encoding string included in XML output should conform to the +.. [1] The encoding string included in XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 9cced51..5ae304d 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -784,11 +784,22 @@ form. Unmatched groups are replaced with an empty string. -.. function:: escape(string) +.. function:: escape(pattern) - Escape all the characters in pattern except ASCII letters, numbers and ``'_'``. + Escape all the characters in *pattern* except ASCII letters, numbers and ``'_'``. This is useful if you want to match an arbitrary literal string that may - have regular expression metacharacters in it. + have regular expression metacharacters in it. For example:: + + >>> print(re.escape('python.exe')) + python\.exe + + >>> legal_chars = string.ascii_lowercase + string.digits + "!#$%&'*+-.^_`|~:" + >>> print('[%s]+' % re.escape(legal_chars)) + [abcdefghijklmnopqrstuvwxyz0123456789\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\:]+ + + >>> operators = ['+', '-', '*', '/', '**'] + >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) + \/|\-|\+|\*\*|\* .. versionchanged:: 3.3 The ``'_'`` character is no longer escaped. @@ -817,15 +828,15 @@ form. .. attribute:: pos - The index of *pattern* where compilation failed. + The index in *pattern* where compilation failed (may be ``None``). .. attribute:: lineno - The line corresponding to *pos*. + The line corresponding to *pos* (may be ``None``). .. attribute:: colno - The column corresponding to *pos*. + The column corresponding to *pos* (may be ``None``). .. versionchanged:: 3.5 Added additional attributes. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index f97118e..bd5442c 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -290,7 +290,7 @@ Edge and Level Trigger Polling (epoll) Objects | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | | | associated fd has an event. The default (if | | | this flag is not set) is to wake all epoll | - | | objects polling on on a fd. | + | | objects polling on a fd. | +-------------------------+-----------------------------------------------+ | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | | | down writing half of connection. | diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index 1624d88..6d864a8 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -68,7 +68,7 @@ constants below: .. class:: SelectorKey A :class:`SelectorKey` is a :class:`~collections.namedtuple` used to - associate a file object to its underlying file decriptor, selected event + associate a file object to its underlying file descriptor, selected event mask and attached data. It is returned by several :class:`BaseSelector` methods. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index a85bf33..41e5baf 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -153,7 +153,7 @@ Directory and files operations is true and *src* is a symbolic link, *dst* will be a copy of the file *src* refers to. - :func:`copy` copies the file data and the file's permission + :func:`~shutil.copy` copies the file data and the file's permission mode (see :func:`os.chmod`). Other metadata, like the file's creation and modification times, is not preserved. To preserve all file metadata from the original, use @@ -302,7 +302,7 @@ Directory and files operations *src* and *dst*, and will be used to copy *src* to *dest* if :func:`os.rename` cannot be used. If the source is a directory, :func:`copytree` is called, passing it the :func:`copy_function`. The - default *copy_function* is :func:`copy2`. Using :func:`copy` as the + default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the *copy_function* allows the move to succeed when it is not possible to also copy the metadata, at the expense of not copying any of the metadata. diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 039b666..46d71de 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -306,8 +306,10 @@ The :mod:`signal` module defines the following functions: a library to wakeup a poll or select call, allowing the signal to be fully processed. - The old wakeup fd is returned. *fd* must be non-blocking. It is up to the - library to remove any bytes before calling poll or select again. + The old wakeup fd is returned (or -1 if file descriptor wakeup was not + enabled). If *fd* is -1, file descriptor wakeup is disabled. + If not -1, *fd* must be non-blocking. It is up to the library to remove + any bytes from *fd* before calling poll or select again. Use for example ``struct.unpack('%uB' % len(data), data)`` to decode the signal numbers list. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index d1f7a6f..9fef7d7 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -640,6 +640,11 @@ Cursor Objects .. versionchanged:: 3.6 Added support for the ``REPLACE`` statement. + .. attribute:: arraysize + + Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. + The default value is 1 which means a single row would be fetched per call. + .. attribute:: description This read-only attribute provides the column names of the last query. To diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index bbb1374..6c30a74 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -374,9 +374,9 @@ Certificate handling Verify that *cert* (in decoded format as returned by :meth:`SSLSocket.getpeercert`) matches the given *hostname*. The rules applied are those for checking the identity of HTTPS servers as outlined - in :rfc:`2818` and :rfc:`6125`. In addition to HTTPS, this function - should be suitable for checking the identity of servers in various - SSL-based protocols such as FTPS, IMAPS, POPS and others. + in :rfc:`2818`, :rfc:`5280` and :rfc:`6125`. In addition to HTTPS, this + function should be suitable for checking the identity of servers in + various SSL-based protocols such as FTPS, IMAPS, POPS and others. :exc:`CertificateError` is raised on failure. On success, the function returns nothing:: @@ -610,13 +610,13 @@ Constants .. data:: PROTOCOL_TLS Selects the highest protocol version that both the client and server support. - Despite the name, this option can select "TLS" protocols as well as "SSL". + Despite the name, this option can select both "SSL" and "TLS" protocols. .. versionadded:: 3.6 .. data:: PROTOCOL_TLS_CLIENT - Auto-negotiate the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_TLS`, but only support client-side :class:`SSLSocket` connections. The protocol enables :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` by default. @@ -625,7 +625,7 @@ Constants .. data:: PROTOCOL_TLS_SERVER - Auto-negotiate the highest protocol version like :data:`PROTOCOL_SSLv23`, + Auto-negotiate the highest protocol version like :data:`PROTOCOL_TLS`, but only support server-side :class:`SSLSocket` connections. .. versionadded:: 3.6 @@ -820,7 +820,7 @@ Constants .. data:: HAS_SNI Whether the OpenSSL library has built-in support for the *Server Name - Indication* extension (as defined in :rfc:`4366`). + Indication* extension (as defined in :rfc:`6066`). .. versionadded:: 3.2 @@ -2107,8 +2107,8 @@ provided. When compared to :class:`SSLSocket`, this object lacks the following features: - - Any form of network IO incluging methods such as ``recv()`` and - ``send()``. + - Any form of network IO; ``recv()`` and ``send()`` read and write only to + the underlying :class:`MemoryBIO` buffers. - There is no *do_handshake_on_connect* machinery. You must always manually call :meth:`~SSLSocket.do_handshake` to start the handshake. @@ -2306,14 +2306,11 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management `_ Steve Kent - `RFC 1750: Randomness Recommendations for Security `_ - D. Eastlake et. al. + `RFC 4086: Randomness Requirements for Security `_ + Donald E., Jeffrey I. Schiller - `RFC 3280: Internet X.509 Public Key Infrastructure Certificate and CRL Profile `_ - Housley et. al. - - `RFC 4366: Transport Layer Security (TLS) Extensions `_ - Blake-Wilson et. al. + `RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile `_ + D. Cooper `RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 `_ T. Dierks et. al. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 6729a20..083fe53 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -394,10 +394,12 @@ Bitwise Operations on Integer Types pair: bitwise; operations pair: shifting; operations pair: masking; operations + operator: | operator: ^ operator: & operator: << operator: >> + operator: ~ Bitwise operations only make sense for integers. Negative numbers are treated as their 2's complement value (this assumes that there are enough bits so that @@ -829,7 +831,7 @@ restrictions imposed by *s*. The ``in`` and ``not in`` operations have the same priorities as the comparison operations. The ``+`` (concatenation) and ``*`` (repetition) -operations have the same priority as the corresponding numeric operations. +operations have the same priority as the corresponding numeric operations. [3]_ .. index:: triple: operations on; sequence; types @@ -1719,10 +1721,10 @@ expression support in the :mod:`re` module). .. method:: str.join(iterable) - Return a string which is the concatenation of the strings in the - :term:`iterable` *iterable*. A :exc:`TypeError` will be raised if there are - any non-string values in *iterable*, including :class:`bytes` objects. The - separator between elements is the string providing this method. + Return a string which is the concatenation of the strings in *iterable*. + A :exc:`TypeError` will be raised if there are any non-string values in + *iterable*, including :class:`bytes` objects. The separator between + elements is the string providing this method. .. method:: str.ljust(width[, fillchar]) @@ -2264,8 +2266,8 @@ The :mod:`array` module supports efficient storage of basic data types like .. _typebytes: -Bytes ------ +Bytes Objects +------------- .. index:: object: bytes @@ -2274,65 +2276,67 @@ binary protocols are based on the ASCII text encoding, bytes objects offer several methods that are only valid when working with ASCII compatible data and are closely related to string objects in a variety of other ways. -Firstly, the syntax for bytes literals is largely the same as that for string -literals, except that a ``b`` prefix is added: +.. class:: bytes([source[, encoding[, errors]]]) -* Single quotes: ``b'still allows embedded "double" quotes'`` -* Double quotes: ``b"still allows embedded 'single' quotes"``. -* Triple quoted: ``b'''3 single quotes'''``, ``b"""3 double quotes"""`` + Firstly, the syntax for bytes literals is largely the same as that for string + literals, except that a ``b`` prefix is added: -Only ASCII characters are permitted in bytes literals (regardless of the -declared source code encoding). Any binary values over 127 must be entered -into bytes literals using the appropriate escape sequence. + * Single quotes: ``b'still allows embedded "double" quotes'`` + * Double quotes: ``b"still allows embedded 'single' quotes"``. + * Triple quoted: ``b'''3 single quotes'''``, ``b"""3 double quotes"""`` -As with string literals, bytes literals may also use a ``r`` prefix to disable -processing of escape sequences. See :ref:`strings` for more about the various -forms of bytes literal, including supported escape sequences. + Only ASCII characters are permitted in bytes literals (regardless of the + declared source code encoding). Any binary values over 127 must be entered + into bytes literals using the appropriate escape sequence. -While bytes literals and representations are based on ASCII text, bytes -objects actually behave like immutable sequences of integers, with each -value in the sequence restricted such that ``0 <= x < 256`` (attempts to -violate this restriction will trigger :exc:`ValueError`. This is done -deliberately to emphasise that while many binary formats include ASCII based -elements and can be usefully manipulated with some text-oriented algorithms, -this is not generally the case for arbitrary binary data (blindly applying -text processing algorithms to binary data formats that are not ASCII -compatible will usually lead to data corruption). + As with string literals, bytes literals may also use a ``r`` prefix to disable + processing of escape sequences. See :ref:`strings` for more about the various + forms of bytes literal, including supported escape sequences. -In addition to the literal forms, bytes objects can be created in a number of -other ways: + While bytes literals and representations are based on ASCII text, bytes + objects actually behave like immutable sequences of integers, with each + value in the sequence restricted such that ``0 <= x < 256`` (attempts to + violate this restriction will trigger :exc:`ValueError`. This is done + deliberately to emphasise that while many binary formats include ASCII based + elements and can be usefully manipulated with some text-oriented algorithms, + this is not generally the case for arbitrary binary data (blindly applying + text processing algorithms to binary data formats that are not ASCII + compatible will usually lead to data corruption). -* A zero-filled bytes object of a specified length: ``bytes(10)`` -* From an iterable of integers: ``bytes(range(20))`` -* Copying existing binary data via the buffer protocol: ``bytes(obj)`` + In addition to the literal forms, bytes objects can be created in a number of + other ways: -Also see the :ref:`bytes ` built-in. + * A zero-filled bytes object of a specified length: ``bytes(10)`` + * From an iterable of integers: ``bytes(range(20))`` + * Copying existing binary data via the buffer protocol: ``bytes(obj)`` -Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal -numbers are a commonly used format for describing binary data. Accordingly, -the bytes type has an additional class method to read data in that format: + Also see the :ref:`bytes ` built-in. -.. classmethod:: bytes.fromhex(string) + Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal + numbers are a commonly used format for describing binary data. Accordingly, + the bytes type has an additional class method to read data in that format: - This :class:`bytes` class method returns a bytes object, decoding the - given string object. The string must contain two hexadecimal digits per - byte, with ASCII spaces being ignored. + .. classmethod:: fromhex(string) - >>> bytes.fromhex('2Ef0 F1f2 ') - b'.\xf0\xf1\xf2' + This :class:`bytes` class method returns a bytes object, decoding the + given string object. The string must contain two hexadecimal digits per + byte, with ASCII whitespace being ignored. -A reverse conversion function exists to transform a bytes object into its -hexadecimal representation. + >>> bytes.fromhex('2Ef0 F1f2 ') + b'.\xf0\xf1\xf2' -.. method:: bytes.hex() + A reverse conversion function exists to transform a bytes object into its + hexadecimal representation. - Return a string object containing two hexadecimal digits for each - byte in the instance. + .. method:: hex() - >>> b'\xf0\xf1\xf2'.hex() - 'f0f1f2' + Return a string object containing two hexadecimal digits for each + byte in the instance. - .. versionadded:: 3.5 + >>> b'\xf0\xf1\xf2'.hex() + 'f0f1f2' + + .. versionadded:: 3.5 Since bytes objects are sequences of integers (akin to a tuple), for a bytes object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes @@ -2362,45 +2366,49 @@ Bytearray Objects .. index:: object: bytearray :class:`bytearray` objects are a mutable counterpart to :class:`bytes` -objects. There is no dedicated literal syntax for bytearray objects, instead -they are always created by calling the constructor: +objects. -* Creating an empty instance: ``bytearray()`` -* Creating a zero-filled instance with a given length: ``bytearray(10)`` -* From an iterable of integers: ``bytearray(range(20))`` -* Copying existing binary data via the buffer protocol: ``bytearray(b'Hi!')`` +.. class:: bytearray([source[, encoding[, errors]]]) -As bytearray objects are mutable, they support the -:ref:`mutable ` sequence operations in addition to the -common bytes and bytearray operations described in :ref:`bytes-methods`. + There is no dedicated literal syntax for bytearray objects, instead + they are always created by calling the constructor: -Also see the :ref:`bytearray ` built-in. + * Creating an empty instance: ``bytearray()`` + * Creating a zero-filled instance with a given length: ``bytearray(10)`` + * From an iterable of integers: ``bytearray(range(20))`` + * Copying existing binary data via the buffer protocol: ``bytearray(b'Hi!')`` -Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal -numbers are a commonly used format for describing binary data. Accordingly, -the bytearray type has an additional class method to read data in that format: + As bytearray objects are mutable, they support the + :ref:`mutable ` sequence operations in addition to the + common bytes and bytearray operations described in :ref:`bytes-methods`. -.. classmethod:: bytearray.fromhex(string) + Also see the :ref:`bytearray ` built-in. - This :class:`bytearray` class method returns bytearray object, decoding - the given string object. The string must contain two hexadecimal digits - per byte, with ASCII spaces being ignored. + Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal + numbers are a commonly used format for describing binary data. Accordingly, + the bytearray type has an additional class method to read data in that format: - >>> bytearray.fromhex('2Ef0 F1f2 ') - bytearray(b'.\xf0\xf1\xf2') + .. classmethod:: fromhex(string) -A reverse conversion function exists to transform a bytearray object into its -hexadecimal representation. + This :class:`bytearray` class method returns bytearray object, decoding + the given string object. The string must contain two hexadecimal digits + per byte, with ASCII whitespace being ignored. -.. method:: bytearray.hex() + >>> bytearray.fromhex('2Ef0 F1f2 ') + bytearray(b'.\xf0\xf1\xf2') - Return a string object containing two hexadecimal digits for each - byte in the instance. + A reverse conversion function exists to transform a bytearray object into its + hexadecimal representation. - >>> bytearray(b'\xf0\xf1\xf2').hex() - 'f0f1f2' + .. method:: hex() + + Return a string object containing two hexadecimal digits for each + byte in the instance. - .. versionadded:: 3.5 + >>> bytearray(b'\xf0\xf1\xf2').hex() + 'f0f1f2' + + .. versionadded:: 3.5 Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be @@ -2539,11 +2547,11 @@ arbitrary binary data. bytearray.join(iterable) Return a bytes or bytearray object which is the concatenation of the - binary data sequences in the :term:`iterable` *iterable*. A - :exc:`TypeError` will be raised if there are any values in *iterable* - that are not :term:`bytes-like objects `, including - :class:`str` objects. The separator between elements is the contents - of the bytes or bytearray object providing this method. + binary data sequences in *iterable*. A :exc:`TypeError` will be raised + if there are any values in *iterable* that are not :term:`bytes-like + objects `, including :class:`str` objects. The + separator between elements is the contents of the bytes or + bytearray object providing this method. .. staticmethod:: bytes.maketrans(from, to) @@ -3982,9 +3990,7 @@ The constructors for both classes work the same: Note, the *elem* argument to the :meth:`__contains__`, :meth:`remove`, and :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, the *elem* set is temporarily mutated during the search and then - restored. During the search, the *elem* set should not be read or mutated - since it does not have a meaningful value. + frozenset, a temporary one is created from *elem*. .. _typesmapping: diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst index 1ecc7a7..c8357e4 100644 --- a/Doc/library/sunau.rst +++ b/Doc/library/sunau.rst @@ -118,7 +118,7 @@ AU_read objects, as returned by :func:`.open` above, have the following methods: .. method:: AU_read.getnchannels() - Returns number of audio channels (1 for mone, 2 for stereo). + Returns number of audio channels (1 for mono, 2 for stereo). .. method:: AU_read.getsampwidth() diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 08b74a9..f066a76 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -255,7 +255,6 @@ You can use :mod:`sysconfig` as a script with Python's *-m* option: AIX_GENUINE_CPLUSPLUS = "0" AR = "ar" ARFLAGS = "rc" - ASDLGEN = "./Parser/asdl_c.py" ... This call will print in the standard output the information returned by diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index 1edb0fb..dfe688a 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -48,14 +48,14 @@ described below. .. exception:: NannyNag - Raised by :func:`tokeneater` if detecting an ambiguous indent. Captured and + Raised by :func:`process_tokens` if detecting an ambiguous indent. Captured and handled in :func:`check`. -.. function:: tokeneater(type, token, start, end, line) +.. function:: process_tokens(tokens) - This function is used by :func:`check` as a callback parameter to the function - :func:`tokenize.tokenize`. + This function is used by :func:`check` to process tokens generated by the + :mod:`tokenize` module. .. XXX document errprint, format_witnesses, Whitespace, check_equal, indents, reset_globals diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index d8f8097..337c061 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -146,6 +146,10 @@ Some facts and figures: .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + + .. class:: TarFile Class for reading and writing tar archives. Do not use this class directly: @@ -266,7 +270,8 @@ be finalized; only the internally used file object will be closed. See the All following arguments are optional and can be accessed as instance attributes as well. - *name* is the pathname of the archive. It can be omitted if *fileobj* is given. + *name* is the pathname of the archive. *name* may be a :term:`path-like object`. + It can be omitted if *fileobj* is given. In this case, the file object's :attr:`name` attribute is used if it exists. *mode* is either ``'r'`` to read from an existing archive, ``'a'`` to append @@ -319,6 +324,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + + .. classmethod:: TarFile.open(...) Alternative constructor. The :func:`tarfile.open` function is actually a @@ -390,14 +399,17 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 Added the *numeric_owner* parameter. + .. versionchanged:: 3.6 + The *path* parameter accepts a :term:`path-like object`. + .. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False) Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. *member* may be a filename or a :class:`TarInfo` object. You can specify a different - directory using *path*. File attributes (owner, mtime, mode) are set unless - *set_attrs* is false. + directory using *path*. *path* may be a :term:`path-like object`. + File attributes (owner, mtime, mode) are set unless *set_attrs* is false. If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile are used to set the owner/group for the extracted files. Otherwise, the named @@ -418,6 +430,10 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.5 Added the *numeric_owner* parameter. + .. versionchanged:: 3.6 + The *path* parameter accepts a :term:`path-like object`. + + .. method:: TarFile.extractfile(member) Extract a member from the archive as a file object. *member* may be a filename @@ -464,7 +480,8 @@ be finalized; only the internally used file object will be closed. See the Create a :class:`TarInfo` object from the result of :func:`os.stat` or equivalent on an existing file. The file is either named by *name*, or - specified as a :term:`file object` *fileobj* with a file descriptor. If + specified as a :term:`file object` *fileobj* with a file descriptor. + *name* may be a :term:`path-like object`. If given, *arcname* specifies an alternative name for the file in the archive, otherwise, the name is taken from *fileobj*’s :attr:`~io.FileIO.name` attribute, or the *name* argument. The name @@ -478,6 +495,9 @@ be finalized; only the internally used file object will be closed. See the The :attr:`~TarInfo.name` may also be modified, in which case *arcname* could be a dummy string. + .. versionchanged:: 3.6 + The *name* parameter accepts a :term:`path-like object`. + .. method:: TarFile.close() diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 665261f..c59aca1 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -245,12 +245,12 @@ The module uses a global variable to store the name of the directory used for temporary files returned by :func:`gettempdir`. It can be set directly to override the selection process, but this is discouraged. All functions in this module take a *dir* argument which can be used -to specify the directory and this is the recommend approach. +to specify the directory and this is the recommended approach. .. data:: tempdir When set to a value other than ``None``, this variable defines the - default value for the *dir* argument to all the functions defined in this + default value for the *dir* argument to the functions defined in this module. If ``tempdir`` is unset or ``None`` at any call to any of the above diff --git a/Doc/library/test.rst b/Doc/library/test.rst index fab3e1f..9d4ff7a 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -570,7 +570,8 @@ The :mod:`test.support` module defines the following functions: def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) -.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()): + +.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()) Returns the set of attributes, functions or methods of *ref_api* not found on *other_api*, except for a defined list of items to be diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 2792dfd..cda859f 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -371,8 +371,9 @@ All methods are executed atomically. lock, subsequent attempts to acquire it block, until it is released; any thread may release it. - .. versionchanged:: 3.3 - Changed from a factory function to a class. + Note that ``Lock`` is actually a factory function which returns an instance + of the most efficient version of the concrete Lock class that is supported + by the platform. .. method:: acquire(blocking=True, timeout=-1) diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 066ee96..55d331c 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -45,9 +45,9 @@ The module defines the following functions: * if *tb* is not ``None``, it prints a header ``Traceback (most recent call last):`` * it prints the exception *etype* and *value* after the stack trace - * if *etype* is :exc:`SyntaxError` and *value* has the appropriate format, it - prints the line where the syntax error occurred with a caret indicating the - approximate position of the error. + * if *type(value)* is :exc:`SyntaxError` and *value* has the appropriate + format, it prints the line where the syntax error occurred with a caret + indicating the approximate position of the error. The optional *limit* argument has the same meaning as for :func:`print_tb`. If *chain* is true (the default), then chained exceptions (the @@ -55,6 +55,9 @@ The module defines the following functions: printed as well, like the interpreter itself does when printing an unhandled exception. + .. versionchanged:: 3.5 + The *etype* argument is ignored and inferred from the type of *value*. + .. function:: print_exc(limit=None, file=None, chain=True) @@ -131,6 +134,9 @@ The module defines the following functions: containing internal newlines. When these lines are concatenated and printed, exactly the same text is printed as does :func:`print_exception`. + .. versionchanged:: 3.5 + The *etype* argument is ignored and inferred from the type of *value*. + .. function:: format_exc(limit=None, chain=True) @@ -372,6 +378,7 @@ exception and traceback: print("*** print_tb:") traceback.print_tb(exc_traceback, limit=1, file=sys.stdout) print("*** print_exception:") + # exc_type below is ignored on 3.5 and later traceback.print_exception(exc_type, exc_value, exc_traceback, limit=2, file=sys.stdout) print("*** print_exc:") @@ -381,6 +388,7 @@ exception and traceback: print(formatted_lines[0]) print(formatted_lines[-1]) print("*** format_exception:") + # exc_type below is ignored on 3.5 and later print(repr(traceback.format_exception(exc_type, exc_value, exc_traceback))) print("*** extract_tb:") diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 1986972..31761be 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -1797,7 +1797,7 @@ Input methods :param prompt: string Pop up a dialog window for input of a string. Parameter title is - the title of the dialog window, propmt is a text mostly describing + the title of the dialog window, prompt is a text mostly describing what information to input. Return the string input. If the dialog is canceled, return ``None``. :: diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index d130e17..1780739 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -508,6 +508,14 @@ The module defines the following classes, functions and decorators: An ABC with one abstract method ``__float__``. +.. class:: SupportsComplex + + An ABC with one abstract method ``__complex__``. + +.. class:: SupportsBytes + + An ABC with one abstract method ``__bytes__``. + .. class:: SupportsAbs An ABC with one abstract method ``__abs__`` that is covariant @@ -658,7 +666,19 @@ The module defines the following classes, functions and decorators: .. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) - A generic version of :class:`collections.defaultdict` + A generic version of :class:`collections.defaultdict`. + +.. class:: Counter(collections.Counter, Dict[T, int]) + + A generic version of :class:`collections.Counter`. + + .. versionadded:: 3.6.1 + +.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) + + A generic version of :class:`collections.ChainMap`. + + .. versionadded:: 3.6.1 .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -742,9 +762,12 @@ The module defines the following classes, functions and decorators: This defines the generic type ``IO[AnyStr]`` and aliases ``TextIO`` and ``BinaryIO`` for respectively ``IO[str]`` and ``IO[bytes]``. - These representing the types of I/O streams such as returned by + These represent the types of I/O streams such as returned by :func:`open`. + These types are also accessible directly as ``typing.IO``, + ``typing.TextIO``, and ``typing.BinaryIO``. + .. class:: re Wrapper namespace for regular expression matching types. @@ -756,6 +779,9 @@ The module defines the following classes, functions and decorators: ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or ``Match[bytes]``. + These types are also accessible directly as ``typing.Pattern`` + and ``typing.Match``. + .. class:: NamedTuple Typed version of namedtuple. @@ -782,10 +808,20 @@ The module defines the following classes, functions and decorators: Fields with a default value must come after any fields without a default. The resulting class has two extra attributes: ``_field_types``, - giving a dict mapping field names to types, and ``field_defaults``, a dict + giving a dict mapping field names to types, and ``_field_defaults``, a dict mapping field names to default values. (The field names are in the ``_fields`` attribute, which is part of the namedtuple API.) + ``NamedTuple`` subclasses can also have docstrings and methods:: + + class Employee(NamedTuple): + """Represents an employee.""" + name: str + id: int = 3 + + def __repr__(self) -> str: + return f'' + Backward-compatible usage:: Employee = NamedTuple('Employee', [('name', str), ('id', int)]) @@ -794,7 +830,7 @@ The module defines the following classes, functions and decorators: Added support for :pep:`526` variable annotation syntax. .. versionchanged:: 3.6.1 - Added support for default values. + Added support for default values, methods, and docstrings. .. function:: NewType(typ) @@ -972,9 +1008,9 @@ The module defines the following classes, functions and decorators: :data:`ClassVar` is not a class itself, and should not be used with :func:`isinstance` or :func:`issubclass`. - Note that :data:`ClassVar` does not change Python runtime behavior; - it can be used by 3rd party type checkers, so that the following - code might flagged as an error by those:: + :data:`ClassVar` does not change Python runtime behavior, but + it can be used by third-party type checkers. For example, a type checker + might flag the following code as an error:: enterprise_d = Starship(3000) enterprise_d.stats = {} # Error, setting class variable on instance @@ -1005,5 +1041,10 @@ The module defines the following classes, functions and decorators: if TYPE_CHECKING: import expensive_mod - def fun(): - local_var: expensive_mod.some_type = other_fun() + def fun(arg: 'expensive_mod.SomeType') -> None: + local_var: expensive_mod.AnotherType = other_fun() + + Note that the first type annotation must be enclosed in quotes, making it a + "forward reference", to hide the ``expensive_mod`` reference from the + interpreter runtime. Type annotations for local variables are not + evaluated, so the second annotation does not need to be enclosed in quotes. diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index 6431809..2a97776 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -158,7 +158,7 @@ Examples: 9 >>> unicodedata.decimal('a') Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in ValueError: not a decimal >>> unicodedata.category('A') # 'L'etter, 'u'ppercase 'Lu' diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 92e567d..2099bd1 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1170,6 +1170,9 @@ Test cases :meth:`.assertRegex`. .. versionadded:: 3.2 :meth:`.assertNotRegex`. + .. versionadded:: 3.5 + The name ``assertNotRegexpMatches`` is a deprecated alias + for :meth:`.assertNotRegex`. .. method:: assertCountEqual(first, second, msg=None) @@ -1435,9 +1438,9 @@ For historical reasons, some of the :class:`TestCase` methods had one or more aliases that are now deprecated. The following table lists the correct names along with their deprecated aliases: - ============================== ====================== ====================== + ============================== ====================== ======================= Method Name Deprecated alias Deprecated alias - ============================== ====================== ====================== + ============================== ====================== ======================= :meth:`.assertEqual` failUnlessEqual assertEquals :meth:`.assertNotEqual` failIfEqual assertNotEquals :meth:`.assertTrue` failUnless assert\_ @@ -1446,8 +1449,9 @@ along with their deprecated aliases: :meth:`.assertAlmostEqual` failUnlessAlmostEqual assertAlmostEquals :meth:`.assertNotAlmostEqual` failIfAlmostEqual assertNotAlmostEquals :meth:`.assertRegex` assertRegexpMatches + :meth:`.assertNotRegex` assertNotRegexpMatches :meth:`.assertRaisesRegex` assertRaisesRegexp - ============================== ====================== ====================== + ============================== ====================== ======================= .. deprecated:: 3.1 the fail* aliases listed in the second column. @@ -1455,8 +1459,9 @@ along with their deprecated aliases: the assert* aliases listed in the third column. .. deprecated:: 3.2 ``assertRegexpMatches`` and ``assertRaisesRegexp`` have been renamed to - :meth:`.assertRegex` and :meth:`.assertRaisesRegex` - + :meth:`.assertRegex` and :meth:`.assertRaisesRegex`. + .. deprecated:: 3.5 + the ``assertNotRegexpMatches`` name in favor of :meth:`.assertNotRegex`. .. _testsuite-objects: diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 676321b..1cc69e6 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -118,6 +118,9 @@ or on combining URL components into a URL string. an invalid port is specified in the URL. See section :ref:`urlparse-result-object` for more information on the result object. + Unmatched square brackets in the :attr:`netloc` attribute will raise a + :exc:`ValueError`. + .. versionchanged:: 3.2 Added IPv6 URL parsing capabilities. @@ -236,6 +239,9 @@ or on combining URL components into a URL string. an invalid port is specified in the URL. See section :ref:`urlparse-result-object` for more information on the result object. + Unmatched square brackets in the :attr:`netloc` attribute will raise a + :exc:`ValueError`. + .. versionchanged:: 3.6 Out-of-range port numbers now raise :exc:`ValueError`, instead of returning :const:`None`. diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 2e9e814..40470e8 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -248,7 +248,7 @@ utility to most DOM users. .. rubric:: Footnotes -.. [#] The encoding name included in the XML output should conform to +.. [1] The encoding name included in the XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not valid in an XML document's declaration, even though Python accepts it as an encoding name. diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index b502554..5c0f469 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -108,7 +108,7 @@ DOMEventStream Objects :class:`xml.dom.minidom.Element` if event equals :data:`START_ELEMENT` or :data:`END_ELEMENT` or :class:`xml.dom.minidom.Text` if event equals :data:`CHARACTERS`. - The current node does not contain informations about its children, unless + The current node does not contain information about its children, unless :func:`expandNode` is called. .. method:: expandNode(node) diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index b54eace..7d814ad 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -1192,7 +1192,7 @@ Exceptions .. rubric:: Footnotes -.. [#] The encoding string included in XML output should conform to the +.. [1] The encoding string included in XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 5eb6f10..a5d4211 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -132,8 +132,9 @@ ZipFile Objects .. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True) - Open a ZIP file, where *file* can be either a path to a file (a string) or a - file-like object. The *mode* parameter should be ``'r'`` to read an existing + Open a ZIP file, where *file* can be a path to a file (a string), a + file-like object or a :term:`path-like object`. + The *mode* parameter should be ``'r'`` to read an existing file, ``'w'`` to truncate and write a new file, ``'a'`` to append to an existing file, or ``'x'`` to exclusively create and write a new file. If *mode* is ``'x'`` and *file* refers to an existing file, @@ -183,6 +184,9 @@ ZipFile Objects Previously, a plain :exc:`RuntimeError` was raised for unrecognized compression values. + .. versionchanged:: 3.6.2 + The *file* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.close() @@ -284,6 +288,9 @@ ZipFile Objects Calling :meth:`extract` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.6.2 + The *path* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.extractall(path=None, members=None, pwd=None) @@ -304,6 +311,9 @@ ZipFile Objects Calling :meth:`extractall` on a closed ZipFile will raise a :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. + .. versionchanged:: 3.6.2 + The *path* parameter accepts a :term:`path-like object`. + .. method:: ZipFile.printdir() @@ -403,6 +413,9 @@ ZipFile Objects The following data attributes are also available: +.. attribute:: ZipFile.filename + + Name of the ZIP file. .. attribute:: ZipFile.debug @@ -451,12 +464,12 @@ The :class:`PyZipFile` constructor takes the same parameters as the added to the archive, compiling if necessary. If *pathname* is a file, the filename must end with :file:`.py`, and - just the (corresponding :file:`\*.py[co]`) file is added at the top level + just the (corresponding :file:`\*.pyc`) file is added at the top level (no path information). If *pathname* is a file that does not end with :file:`.py`, a :exc:`RuntimeError` will be raised. If it is a directory, and the directory is not a package directory, then all the files - :file:`\*.py[co]` are added at the top level. If the directory is a - package directory, then all :file:`\*.py[co]` are added under the package + :file:`\*.pyc` are added at the top level. If the directory is a + package directory, then all :file:`\*.pyc` are added under the package name as a file path, and if any subdirectories are package directories, all of these are added recursively. @@ -488,6 +501,9 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. versionadded:: 3.4 The *filterfunc* parameter. + .. versionchanged:: 3.6.2 + The *pathname* parameter accepts a :term:`path-like object`. + .. _zipinfo-objects: @@ -514,6 +530,10 @@ file: .. versionadded:: 3.6 + .. versionchanged:: 3.6.2 + The *filename* parameter accepts a :term:`path-like object`. + + Instances have the following methods and attributes: .. method:: ZipInfo.is_dir() diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 46b8c24..eaae2bb 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -9,7 +9,7 @@ -------------- This module adds the ability to import Python modules (:file:`\*.py`, -:file:`\*.py[co]`) and packages from ZIP-format archives. It is usually not +:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not needed to use the :mod:`zipimport` module explicitly; it is automatically used by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths to ZIP archives. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 095a238..7c140a3 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -320,9 +320,9 @@ Sequences A bytes object is an immutable array. The items are 8-bit bytes, represented by integers in the range 0 <= x < 256. Bytes literals - (like ``b'abc'``) and the built-in function :func:`bytes` can be used to - construct bytes objects. Also, bytes objects can be decoded to strings - via the :meth:`~bytes.decode` method. + (like ``b'abc'``) and the built-in :func:`bytes()` constructor + can be used to create bytes objects. Also, bytes objects can be + decoded to strings via the :meth:`~bytes.decode` method. Mutable sequences .. index:: @@ -349,9 +349,9 @@ Sequences .. index:: bytearray A bytearray object is a mutable array. They are created by the built-in - :func:`bytearray` constructor. Aside from being mutable (and hence - unhashable), byte arrays otherwise provide the same interface and - functionality as immutable bytes objects. + :func:`bytearray` constructor. Aside from being mutable + (and hence unhashable), byte arrays otherwise provide the same interface + and functionality as immutable :class:`bytes` objects. .. index:: module: array @@ -1119,9 +1119,9 @@ Basic customization (usually an instance of *cls*). Typical implementations create a new instance of the class by invoking the - superclass's :meth:`__new__` method using ``super(currentclass, - cls).__new__(cls[, ...])`` with appropriate arguments and then modifying the - newly-created instance as necessary before returning it. + superclass's :meth:`__new__` method using ``super().__new__(cls[, ...])`` + with appropriate arguments and then modifying the newly-created instance + as necessary before returning it. If :meth:`__new__` returns an instance of *cls*, then the new instance's :meth:`__init__` method will be invoked like ``__init__(self[, ...])``, where @@ -1145,7 +1145,7 @@ Basic customization class constructor expression. If a base class has an :meth:`__init__` method, the derived class's :meth:`__init__` method, if any, must explicitly call it to ensure proper initialization of the base class part of the - instance; for example: ``BaseClass.__init__(self, [args...])``. + instance; for example: ``super().__init__([args...])``. Because :meth:`__new__` and :meth:`__init__` work together in constructing objects (:meth:`__new__` to create it, and :meth:`__init__` to customize it), @@ -1253,8 +1253,8 @@ Basic customization .. index:: builtin: bytes - Called by :func:`bytes` to compute a byte-string representation of an - object. This should return a ``bytes`` object. + Called by :ref:`bytes ` to compute a byte-string representation + of an object. This should return a :class:`bytes` object. .. index:: single: string; __format__() (object method) @@ -1578,8 +1578,8 @@ Class Binding ``A.__dict__['x'].__get__(None, A)``. Super Binding - If ``a`` is an instance of :class:`super`, then the binding ``super(B, - obj).m()`` searches ``obj.__class__.__mro__`` for the base class ``A`` + If ``a`` is an instance of :class:`super`, then the binding ``super(B, obj).m()`` + searches ``obj.__class__.__mro__`` for the base class ``A`` immediately preceding ``B`` and then invokes the descriptor with the call: ``A.__dict__['m'].__get__(obj, obj.__class__)``. @@ -2011,6 +2011,14 @@ through the container; for mappings, :meth:`__iter__` should be the same as :meth:`__bool__` method and whose :meth:`__len__` method returns zero is considered to be false in a Boolean context. + .. impl-detail:: + + In CPython, the length is required to be at most :attr:`sys.maxsize`. + If the length is larger than :attr:`!sys.maxsize` some features (such as + :func:`len`) may raise :exc:`OverflowError`. To prevent raising + :exc:`!OverflowError` by truth value testing, an object must define a + :meth:`__bool__` method. + .. method:: object.__length_hint__(self) @@ -2021,6 +2029,7 @@ through the container; for mappings, :meth:`__iter__` should be the same as .. versionadded:: 3.4 + .. note:: Slicing is done exclusively with the following three methods. A call like :: diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 5f1ea92..d08abdf 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -164,15 +164,6 @@ Builtins and restricted execution .. index:: pair: restricted; execution -The builtins namespace associated with the execution of a code block is actually -found by looking up the name ``__builtins__`` in its global namespace; this -should be a dictionary or a module (in the latter case the module's dictionary -is used). By default, when in the :mod:`__main__` module, ``__builtins__`` is -the built-in module :mod:`builtins`; when in any other module, -``__builtins__`` is an alias for the dictionary of the :mod:`builtins` module -itself. ``__builtins__`` can be set to a user-created dictionary to create a -weak form of restricted execution. - .. impl-detail:: Users should not touch ``__builtins__``; it is strictly an implementation @@ -180,6 +171,15 @@ weak form of restricted execution. :keyword:`import` the :mod:`builtins` module and modify its attributes appropriately. +The builtins namespace associated with the execution of a code block +is actually found by looking up the name ``__builtins__`` in its +global namespace; this should be a dictionary or a module (in the +latter case the module's dictionary is used). By default, when in the +:mod:`__main__` module, ``__builtins__`` is the built-in module +:mod:`builtins`; when in any other module, ``__builtins__`` is an +alias for the dictionary of the :mod:`builtins` module itself. + + .. _dynamic-features: Interaction with dynamic features @@ -194,12 +194,6 @@ This means that the following code will print 42:: i = 42 f() -There are several cases where Python statements are illegal when used in -conjunction with nested scopes that contain free variables. - -If a variable is referenced in an enclosing scope, it is illegal to delete the -name. An error will be reported at compile time. - .. XXX from * also invalid with relative imports (at least currently) The :func:`eval` and :func:`exec` functions do not have access to the full diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index f4a8269..d92be97 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -636,7 +636,7 @@ which are used to control the execution of a generator function. without yielding another value, an :exc:`StopAsyncIteration` exception is raised by the awaitable. If the generator function does not catch the passed-in exception, or - raises a different exception, then when the awaitalbe is run that exception + raises a different exception, then when the awaitable is run that exception propagates to the caller of the awaitable. .. index:: exception: GeneratorExit @@ -905,7 +905,7 @@ keyword arguments (and any ``**expression`` arguments -- see below). So:: 2 1 >>> f(a=1, *(2,)) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: f() got multiple values for keyword argument 'a' >>> f(1, *(2,)) 1 2 @@ -1355,7 +1355,7 @@ built-in types. true). * Mappings (instances of :class:`dict`) compare equal if and only if they have - equal `(key, value)` pairs. Equality comparison of the keys and elements + equal `(key, value)` pairs. Equality comparison of the keys and values enforces reflexivity. Order comparisons (``<``, ``>``, ``<=``, and ``>=``) raise :exc:`TypeError`. @@ -1431,28 +1431,29 @@ Membership test operations -------------------------- The operators :keyword:`in` and :keyword:`not in` test for membership. ``x in -s`` evaluates to true if *x* is a member of *s*, and false otherwise. ``x not -in s`` returns the negation of ``x in s``. All built-in sequences and set types -support this as well as dictionary, for which :keyword:`in` tests whether the -dictionary has a given key. For container types such as list, tuple, set, -frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent +s`` evaluates to ``True`` if *x* is a member of *s*, and ``False`` otherwise. +``x not in s`` returns the negation of ``x in s``. All built-in sequences and +set types support this as well as dictionary, for which :keyword:`in` tests +whether the dictionary has a given key. For container types such as list, tuple, +set, frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent to ``any(x is e or x == e for e in y)``. -For the string and bytes types, ``x in y`` is true if and only if *x* is a +For the string and bytes types, ``x in y`` is ``True`` if and only if *x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``. Empty strings are always considered to be a substring of any other string, so ``"" in "abc"`` will return ``True``. For user-defined classes which define the :meth:`__contains__` method, ``x in -y`` is true if and only if ``y.__contains__(x)`` is true. +y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and +``False`` otherwise. For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is true if some value ``z`` with ``x == z`` is +:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z`` with ``x == z`` is produced while iterating over ``y``. If an exception is raised during the iteration, it is as if :keyword:`in` raised that exception. Lastly, the old-style iteration protocol is tried: if a class defines -:meth:`__getitem__`, ``x in y`` is true if and only if there is a non-negative +:meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative integer index *i* such that ``x == y[i]``, and all lower integer indices do not raise :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 4dbd9d8..d504f37 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -36,7 +36,7 @@ implement import semantics. When a module is first imported, Python searches for the module and if found, it creates a module object [#fnmo]_, initializing it. If the named module -cannot be found, an :exc:`ModuleNotFoundError` is raised. Python implements various +cannot be found, a :exc:`ModuleNotFoundError` is raised. Python implements various strategies to search for the named module when the import machinery is invoked. These strategies can be modified and extended by using various hooks described in the sections below. @@ -167,7 +167,7 @@ arguments to the :keyword:`import` statement, or from the parameters to the This name will be used in various phases of the import search, and it may be the dotted path to a submodule, e.g. ``foo.bar.baz``. In this case, Python first tries to import ``foo``, then ``foo.bar``, and finally ``foo.bar.baz``. -If any of the intermediate imports fail, an :exc:`ModuleNotFoundError` is raised. +If any of the intermediate imports fail, a :exc:`ModuleNotFoundError` is raised. The module cache @@ -185,7 +185,7 @@ object. During import, the module name is looked up in :data:`sys.modules` and if present, the associated value is the module satisfying the import, and the -process completes. However, if the value is ``None``, then an +process completes. However, if the value is ``None``, then a :exc:`ModuleNotFoundError` is raised. If the module name is missing, Python will continue searching for the module. @@ -194,7 +194,7 @@ associated module (as other modules may hold references to it), but it will invalidate the cache entry for the named module, causing Python to search anew for the named module upon its next import. The key can also be assigned to ``None``, forcing the next import -of the module to result in an :exc:`ModuleNotFoundError`. +of the module to result in a :exc:`ModuleNotFoundError`. Beware though, as if you keep a reference to the module object, invalidate its cache entry in :data:`sys.modules`, and then re-import the @@ -298,7 +298,7 @@ The second argument is the path entries to use for the module search. For top-level modules, the second argument is ``None``, but for submodules or subpackages, the second argument is the value of the parent package's ``__path__`` attribute. If the appropriate ``__path__`` attribute cannot -be accessed, an :exc:`ModuleNotFoundError` is raised. The third argument +be accessed, a :exc:`ModuleNotFoundError` is raised. The third argument is an existing module object that will be the target of loading later. The import system passes in a target module only during reload. @@ -616,7 +616,7 @@ the module. module.__path__ --------------- -By definition, if a module has an ``__path__`` attribute, it is a package, +By definition, if a module has a ``__path__`` attribute, it is a package, regardless of its value. A package's ``__path__`` attribute is used during imports of its subpackages. @@ -890,7 +890,7 @@ import statements within that module. To selectively prevent import of some modules from a hook early on the meta path (rather than disabling the standard import system entirely), -it is sufficient to raise :exc:`ModuleNoFoundError` directly from +it is sufficient to raise :exc:`ModuleNotFoundError` directly from :meth:`~importlib.abc.MetaPathFinder.find_spec` instead of returning ``None``. The latter indicates that the meta path search should continue, while raising an exception terminates it immediately. diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index da7017a..7f9c664 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -696,6 +696,17 @@ a temporary variable. >>> f"newline: {newline}" 'newline: 10' +Formatted string literals cannot be used as docstrings, even if they do not +include expressions. + +:: + + >>> def foo(): + ... f"Not a docstring" + ... + >>> foo.__doc__ is None + True + See also :pep:`498` for the proposal that added formatted string literals, and :meth:`str.format`, which uses a related format string mechanism. diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index e152b16..8786d73 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -587,7 +587,7 @@ printed:: ... Traceback (most recent call last): File "", line 2, in - ZeroDivisionError: int division or modulo by zero + ZeroDivisionError: division by zero The above exception was the direct cause of the following exception: @@ -606,7 +606,7 @@ attached as the new exception's :attr:`__context__` attribute:: ... Traceback (most recent call last): File "", line 2, in - ZeroDivisionError: int division or modulo by zero + ZeroDivisionError: division by zero During handling of the above exception, another exception occurred: @@ -614,9 +614,27 @@ attached as the new exception's :attr:`__context__` attribute:: File "", line 4, in RuntimeError: Something bad happened +Exception chaining can be explicitly suppressed by specifying :const:`None` in +the ``from`` clause:: + + >>> try: + ... print(1 / 0) + ... except: + ... raise RuntimeError("Something bad happened") from None + ... + Traceback (most recent call last): + File "", line 4, in + RuntimeError: Something bad happened + Additional information on exceptions can be found in section :ref:`exceptions`, and information about handling exceptions is in section :ref:`try`. +.. versionchanged:: 3.3 + :const:`None` is now permitted as ``Y`` in ``raise X from Y``. + +.. versionadded:: 3.3 + The ``__suppress_context__`` attribute to suppress automatic display of the + exception context. .. _break: @@ -922,7 +940,7 @@ annotation. builtin: eval builtin: compile -**Programmer's note:** the :keyword:`global` is a directive to the parser. It +**Programmer's note:** :keyword:`global` is a directive to the parser. It applies only to code parsed at the same time as the :keyword:`global` statement. In particular, a :keyword:`global` statement contained in a string or code object supplied to the built-in :func:`exec` function does not affect the code diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 1141d6c..865f2ed 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -21,6 +21,7 @@ from docutils import nodes, utils from sphinx import addnodes from sphinx.builders import Builder +from sphinx.locale import translators from sphinx.util.nodes import split_explicit_title from sphinx.util.compat import Directive from sphinx.writers.html import HTMLTranslator @@ -103,16 +104,25 @@ class ImplementationDetail(Directive): optional_arguments = 1 final_argument_whitespace = True + # This text is copied to templates/dummy.html + label_text = 'CPython implementation detail:' + def run(self): pnode = nodes.compound(classes=['impl-detail']) + label = translators['sphinx'].gettext(self.label_text) content = self.content - add_text = nodes.strong('CPython implementation detail:', - 'CPython implementation detail:') + add_text = nodes.strong(label, label) if self.arguments: n, m = self.state.inline_text(self.arguments[0], self.lineno) pnode.append(nodes.paragraph('', '', *(n + m))) self.state.nested_parse(content, self.content_offset, pnode) if pnode.children and isinstance(pnode[0], nodes.paragraph): + content = nodes.inline(pnode[0].rawsource, translatable=True) + content.source = pnode[0].source + content.line = pnode[0].line + content += pnode[0].children + pnode[0].replace_self(nodes.paragraph('', '', content, + translatable=False)) pnode[0].insert(0, add_text) pnode[0].insert(1, nodes.Text(' ')) else: diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index c6e0311..2dc5404 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -106,7 +106,7 @@ howto/pyporting,,::,Programming Language :: Python :: 2 howto/pyporting,,::,Programming Language :: Python :: 3 howto/regex,,::, howto/regex,,:foo,(?:foo) -howto/urllib2,,:password,"for example ""joe:password@example.com""" +howto/urllib2,,:password,"""joe:password@example.com""" library/audioop,,:ipos,"# factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)]," library/bisect,32,:hi,all(val >= x for val in a[i:hi]) library/bisect,42,:hi,all(val > x for val in a[i:hi]) @@ -302,6 +302,8 @@ whatsnew/3.2,,:feed,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe: whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') as tf:" whatsnew/3.2,,:location,zope9-location = ${zope9:location} whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf +library/re,,`,!#$%&'*+-.^_`|~: +library/re,,`,\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\: library/tarfile,,:xz,'x:xz' library/xml.etree.elementtree,,:sometag,prefix:sometag library/xml.etree.elementtree,,:fictional,">> m[::2].tolist() library/sys,,`,# ``wrapper`` creates a ``wrap(coro)`` coroutine: whatsnew/3.5,,:root,'WARNING:root:warning\n' @@ -324,6 +326,3 @@ whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1') whatsnew/3.5,,:root,ERROR:root:exception whatsnew/3.5,,:exception,ERROR:root:exception whatsnew/changelog,,:version,import sys; I = version[:version.index(' ')] -whatsnew/changelog,,:gz,": TarFile opened with external fileobj and ""w:gz"" mode didn't" -whatsnew/changelog,,::,": Use ""127.0.0.1"" or ""::1"" instead of ""localhost"" as much as" -whatsnew/changelog,,`,"for readability (was ""`"")." diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index de84ae3..3a05cb6 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -42,7 +42,7 @@ in the table are the size of the download files in megabytes.

These archives contain all the content in the documentation.

HTML Help (.chm) files are made available in the "Windows" section -on the Python +on the Python download page.

diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html new file mode 100644 index 0000000..6e43be2 --- /dev/null +++ b/Doc/tools/templates/dummy.html @@ -0,0 +1,6 @@ +This file is not an actual template, but used to add some +texts in extensions to sphinx.pot file. + +In extensions/pyspecific.py: + +{% trans %}CPython implementation detail:{% endtrans %} diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index e134d5d..073444c 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -784,7 +784,7 @@ using the :func:`next` built-in function; this example shows how it all works:: 'c' >>> next(it) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in next(it) StopIteration diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 6a9bb48..54171bc 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -475,7 +475,7 @@ Here's an example that fails due to this restriction:: ... >>> function(0, a=0) Traceback (most recent call last): - File "", line 1, in ? + File "", line 1, in TypeError: function() got multiple values for keyword argument 'a' When a final formal parameter of the form ``**name`` is present, it receives a diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 6140ece..f9ddf06 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -261,7 +261,7 @@ it must be parenthesized. :: [(0, 0), (1, 1), (2, 4), (3, 9), (4, 16), (5, 25)] >>> # the tuple must be parenthesized, otherwise an error is raised >>> [x, x**2 for x in range(6)] - File "", line 1, in ? + File "", line 1, in [x, x**2 for x in range(6)] ^ SyntaxError: invalid syntax @@ -466,7 +466,7 @@ Here is a brief demonstration:: {'a', 'r', 'b', 'c', 'd'} >>> a - b # letters in a but not in b {'r', 'd', 'b'} - >>> a | b # letters in either a or b + >>> a | b # letters in a or b or both {'a', 'c', 'r', 'd', 'b', 'm', 'z', 'l'} >>> a & b # letters in both a and b {'a', 'c'} diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index beeaac3..74d7bad 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -262,6 +262,35 @@ to file data is fine for text files, but will corrupt binary data like that in :file:`JPEG` or :file:`EXE` files. Be very careful to use binary mode when reading and writing such files. +It is good practice to use the :keyword:`with` keyword when dealing +with file objects. The advantage is that the file is properly closed +after its suite finishes, even if an exception is raised at some +point. Using :keyword:`with` is also much shorter than writing +equivalent :keyword:`try`\ -\ :keyword:`finally` blocks:: + + >>> with open('workfile') as f: + ... read_data = f.read() + >>> f.closed + True + +If you're not using the :keyword:`with` keyword, then you should call +``f.close()`` to close the file and immediately free up any system +resources used by it. If you don't explicitly close a file, Python's +garbage collector will eventually destroy the object and close the +open file for you, but the file may stay open for a while. Another +risk is that different Python implementations will do this clean-up at +different times. + +After a file object is closed, either by a :keyword:`with` statement +or by calling ``f.close()``, attempts to use the file object will +automatically fail. :: + + >>> f.close() + >>> f.read() + Traceback (most recent call last): + File "", line 1, in + ValueError: I/O operation on closed file + .. _tut-filemethods: @@ -354,27 +383,6 @@ to the very file end with ``seek(0, 2)``) and the only valid *offset* values are those returned from the ``f.tell()``, or zero. Any other *offset* value produces undefined behaviour. - -When you're done with a file, call ``f.close()`` to close it and free up any -system resources taken up by the open file. After calling ``f.close()``, -attempts to use the file object will automatically fail. :: - - >>> f.close() - >>> f.read() - Traceback (most recent call last): - File "", line 1, in ? - ValueError: I/O operation on closed file - -It is good practice to use the :keyword:`with` keyword when dealing with file -objects. This has the advantage that the file is properly closed after its -suite finishes, even if an exception is raised on the way. It is also much -shorter than writing equivalent :keyword:`try`\ -\ :keyword:`finally` blocks:: - - >>> with open('workfile', 'r') as f: - ... read_data = f.read() - >>> f.closed - True - File objects have some additional methods, such as :meth:`~file.isatty` and :meth:`~file.truncate` which are less frequently used; consult the Library Reference for a complete guide to file objects. @@ -412,6 +420,7 @@ sent over a network connection to some distant machine. If you have an object ``x``, you can view its JSON string representation with a simple line of code:: + >>> import json >>> json.dumps([1, 'simple', 'list']) '[1, "simple", "list"]' diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 52120a0..8956aa5 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -100,10 +100,8 @@ give you an error:: There is full support for floating point; operators with mixed type operands convert the integer operand to floating point:: - >>> 3 * 3.75 / 1.5 - 7.5 - >>> 7.0 / 2 - 3.5 + >>> 4 * 3.75 - 1 + 14.0 In interactive mode, the last printed expression is assigned to the variable ``_``. This means that when you are using Python as a desk calculator, it is diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 195f63f..40a06b9 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -532,8 +532,8 @@ conflict. .. envvar:: PYTHONDONTWRITEBYTECODE - If this is set to a non-empty string, Python won't try to write ``.pyc`` or - ``.pyo`` files on the import of source modules. This is equivalent to + If this is set to a non-empty string, Python won't try to write ``.pyc`` + files on the import of source modules. This is equivalent to specifying the :option:`-B` option. @@ -571,7 +571,7 @@ conflict. .. versionchanged:: 3.6 On Windows, the encoding specified by this variable is ignored for interactive - console buffers unless :envvar:`PYTHONLEGACYWINDOWSIOENCODING` is also specified. + console buffers unless :envvar:`PYTHONLEGACYWINDOWSSTDIO` is also specified. Files and pipes redirected through the standard streams are not affected. .. envvar:: PYTHONNOUSERSITE @@ -700,7 +700,7 @@ conflict. .. versionadded:: 3.6 See :pep:`529` for more details. -.. envvar:: PYTHONLEGACYWINDOWSIOENCODING +.. envvar:: PYTHONLEGACYWINDOWSSTDIO If set to a non-empty string, does not use the new console reader and writer. This means that Unicode characters will be encoded according to diff --git a/Doc/using/index.rst b/Doc/using/index.rst index a5df713..4f0aa7d 100644 --- a/Doc/using/index.rst +++ b/Doc/using/index.rst @@ -6,7 +6,7 @@ This part of the documentation is devoted to general information on the setup -of the Python environment on different platform, the invocation of the +of the Python environment on different platforms, the invocation of the interpreter and things that make working with Python easier. diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 97f0a49..604688c 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -77,7 +77,7 @@ The build process consists in the usual :: make install invocations. Configuration options and caveats for specific Unix platforms are -extensively documented in the :source:`README` file in the root of the Python +extensively documented in the :source:`README.rst` file in the root of the Python source tree. .. warning:: diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 3e4b70e..68687e9 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -550,9 +550,9 @@ Shebang Lines If the first line of a script file starts with ``#!``, it is known as a "shebang" line. Linux and other Unix like operating systems have native -support for such lines and are commonly used on such systems to indicate how -a script should be executed. This launcher allows the same facilities to be -using with Python scripts on Windows and the examples above demonstrate their +support for such lines and they are commonly used on such systems to indicate +how a script should be executed. This launcher allows the same facilities to +be used with Python scripts on Windows and the examples above demonstrate their use. To allow shebang lines in Python scripts to be portable between Unix and diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index a696af4..1e71bc3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -534,7 +534,7 @@ provide correctly read str objects to Python code. ``sys.stdin``, This change only applies when using an interactive console, and not when redirecting files or pipes. To revert to the previous behaviour for interactive -console use, set :envvar:`PYTHONLEGACYWINDOWSIOENCODING`. +console use, set :envvar:`PYTHONLEGACYWINDOWSSTDIO`. .. seealso:: @@ -2306,3 +2306,39 @@ There have been several major changes to the :term:`bytecode` in Python 3.6. * The new :opcode:`SETUP_ANNOTATIONS` and :opcode:`STORE_ANNOTATION` opcodes have been added to support the new :term:`variable annotation` syntax. (Contributed by Ivan Levkivskyi in :issue:`27985`.) + + +Notable changes in Python 3.6.2 +=============================== + +New ``make regen-all`` build target +----------------------------------- + +To simplify cross-compilation, and to ensure that CPython can reliably be +compiled without requiring an existing version of Python to already be +available, the autotools-based build system no longer attempts to implicitly +recompile generated files based on file modification times. + +Instead, a new ``make regen-all`` command has been added to force regeneration +of these files when desired (e.g. after an initial version of Python has +already been built based on the pregenerated versions). + +More selective regeneration targets are also defined - see +:source:`Makefile.pre.in` for details. + +(Contributed by Victor Stinner in :issue:`23404`.) + +.. versionadded:: 3.6.2 + + +Removal of ``make touch`` build target +-------------------------------------- + +The ``make touch`` build target previously used to request implicit regeneration +of generated files by updating their modification times has been removed. + +It has been replaced by the new ``make regen-all`` target. + +(Contributed by Victor Stinner in :issue:`23404`.) + +.. versionchanged:: 3.6.2 diff --git a/Include/abstract.h b/Include/abstract.h index 7d137a2..6bb76ec 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -290,21 +290,23 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/ PyObject **values, PyObject *kwnames); - /* Convert (args, nargs, kwargs) into a (stack, nargs, kwnames). + /* Convert (args, nargs, kwargs: dict) into (stack, nargs, kwnames: tuple). - Return a new stack which should be released by PyMem_Free(), or return - args unchanged if kwargs is NULL or an empty dictionary. + Return 0 on success, raise an exception and return -1 on error. + + Write the new stack into *p_stack. If *p_stack is differen than args, it + must be released by PyMem_Free(). The stack uses borrowed references. The type of keyword keys is not checked, these checks should be done - later (ex: _PyArg_ParseStack). */ - PyAPI_FUNC(PyObject **) _PyStack_UnpackDict( + later (ex: _PyArg_ParseStackAndKeywords). */ + PyAPI_FUNC(int) _PyStack_UnpackDict( PyObject **args, Py_ssize_t nargs, PyObject *kwargs, - PyObject **kwnames, - PyObject *func); + PyObject ***p_stack, + PyObject **p_kwnames); /* Call the callable object func with the "fast call" calling convention: args is a C array for positional arguments (nargs is the number of diff --git a/Include/ceval.h b/Include/ceval.h index 89c6062..1e48272 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -216,6 +216,7 @@ PyAPI_FUNC(Py_ssize_t) _PyEval_RequestCodeExtraIndex(freefunc); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); +PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); PyAPI_FUNC(void) _PyEval_SignalAsyncExc(void); #endif diff --git a/Include/fileutils.h b/Include/fileutils.h index b933e98..900c70f 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -22,7 +22,7 @@ PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #ifdef MS_WINDOWS struct _Py_stat_struct { unsigned long st_dev; - __int64 st_ino; + uint64_t st_ino; unsigned short st_mode; int st_nlink; int st_uid; diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 54a9ab6..312a6fb 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 6 -#define PY_MICRO_VERSION 1 +#define PY_MICRO_VERSION 2 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.6.1" +#define PY_VERSION "3.6.2" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/pystate.h b/Include/pystate.h index afc3c0c..1838fa4 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -51,6 +51,16 @@ typedef struct _is { } PyInterpreterState; #endif +typedef struct _co_extra_state { + struct _co_extra_state *next; + PyInterpreterState* interp; + + Py_ssize_t co_extra_user_count; + freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS]; +} __PyCodeExtraState; + +/* This is temporary for backwards compat in 3.6 and will be removed in 3.7 */ +__PyCodeExtraState* __PyCodeExtraState_Get(void); /* State unique per thread */ @@ -142,8 +152,10 @@ typedef struct _ts { PyObject *coroutine_wrapper; int in_coroutine_wrapper; - Py_ssize_t co_extra_user_count; - freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS]; + /* Now used from PyInterpreterState, kept here for ABI + compatibility with PyThreadState */ + Py_ssize_t _preserve_36_ABI_1; + freefunc _preserve_36_ABI_2[MAX_CO_EXTRA_USERS]; PyObject *async_gen_firstiter; PyObject *async_gen_finalizer; diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 3626354..579ac07 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -45,11 +45,13 @@ PyAPI_FUNC(int) PySlice_GetIndicesEx(PyObject *r, Py_ssize_t length, Py_ssize_t *step, Py_ssize_t *slicelength); #if !defined(Py_LIMITED_API) || (Py_LIMITED_API+0 >= 0x03050400 && Py_LIMITED_API+0 < 0x03060000) || Py_LIMITED_API+0 >= 0x03060100 +#ifdef Py_LIMITED_API #define PySlice_GetIndicesEx(slice, length, start, stop, step, slicelen) ( \ PySlice_Unpack((slice), (start), (stop), (step)) < 0 ? \ ((*(slicelen) = 0), -1) : \ ((*(slicelen) = PySlice_AdjustIndices((length), (start), (stop), *(step))), \ 0)) +#endif PyAPI_FUNC(int) PySlice_Unpack(PyObject *slice, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step); PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 587cf03..5b87718 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1609,50 +1609,41 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII( This codec uses mappings to encode and decode characters. - Decoding mappings must map single string characters to single - Unicode characters, integers (which are then interpreted as Unicode - ordinals) or None (meaning "undefined mapping" and causing an - error). - - Encoding mappings must map single Unicode characters to single - string characters, integers (which are then interpreted as Latin-1 - ordinals) or None (meaning "undefined mapping" and causing an - error). - - If a character lookup fails with a LookupError, the character is - copied as-is meaning that its ordinal value will be interpreted as - Unicode or Latin-1 ordinal resp. Because of this mappings only need - to contain those mappings which map characters to different code - points. + Decoding mappings must map byte ordinals (integers in the range from 0 to + 255) to Unicode strings, integers (which are then interpreted as Unicode + ordinals) or None. Unmapped data bytes (ones which cause a LookupError) + as well as mapped to None, 0xFFFE or '\ufffe' are treated as "undefined + mapping" and cause an error. + + Encoding mappings must map Unicode ordinal integers to bytes objects, + integers in the range from 0 to 255 or None. Unmapped character + ordinals (ones which cause a LookupError) as well as mapped to + None are treated as "undefined mapping" and cause an error. */ PyAPI_FUNC(PyObject*) PyUnicode_DecodeCharmap( const char *string, /* Encoded string */ Py_ssize_t length, /* size of string */ - PyObject *mapping, /* character mapping - (char ordinal -> unicode ordinal) */ + PyObject *mapping, /* decoding mapping */ const char *errors /* error handling */ ); PyAPI_FUNC(PyObject*) PyUnicode_AsCharmapString( PyObject *unicode, /* Unicode object */ - PyObject *mapping /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping /* encoding mapping */ ); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap( const Py_UNICODE *data, /* Unicode char buffer */ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ - PyObject *mapping, /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping, /* encoding mapping */ const char *errors /* error handling */ ); PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( PyObject *unicode, /* Unicode object */ - PyObject *mapping, /* character mapping - (unicode ordinal -> char ordinal) */ + PyObject *mapping, /* encoding mapping */ const char *errors /* error handling */ ); #endif @@ -1661,8 +1652,8 @@ PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( character mapping table to it and return the resulting Unicode object. - The mapping table must map Unicode ordinal integers to Unicode - ordinal integers or None (causing deletion of the character). + The mapping table must map Unicode ordinal integers to Unicode strings, + Unicode ordinal integers or None (causing deletion of the character). Mapping tables may be dictionaries or sequences. Unmapped character ordinals (ones which cause a LookupError) are left untouched and @@ -1960,8 +1951,8 @@ PyAPI_FUNC(PyObject*) PyUnicode_RSplit( /* Translate a string by applying a character mapping table to it and return the resulting Unicode object. - The mapping table must map Unicode ordinal integers to Unicode - ordinal integers or None (causing deletion of the character). + The mapping table must map Unicode ordinal integers to Unicode strings, + Unicode ordinal integers or None (causing deletion of the character). Mapping tables may be dictionaries or sequences. Unmapped character ordinals (ones which cause a LookupError) are left untouched and @@ -2318,6 +2309,10 @@ PyAPI_FUNC(Py_UNICODE*) PyUnicode_AsUnicodeCopy( PyAPI_FUNC(int) _PyUnicode_CheckConsistency( PyObject *op, int check_content); +#elif !defined(NDEBUG) +/* For asserts that call _PyUnicode_CheckConsistency(), which would + * otherwise be a problem when building with asserts but without Py_DEBUG. */ +#define _PyUnicode_CheckConsistency(op, check_content) PyUnicode_Check(op) #endif #ifndef Py_LIMITED_API diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index b172f3f..005d884 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -908,7 +908,8 @@ class Sequence(Reversible, Collection): i = start while stop is None or i < stop: try: - if self[i] == value: + v = self[i] + if v is value or v == value: return i except IndexError: break @@ -917,7 +918,7 @@ class Sequence(Reversible, Collection): def count(self, value): 'S.count(value) -> integer -- return number of occurrences of value' - return sum(1 for v in self if v == value) + return sum(1 for v in self if v is value or v == value) Sequence.register(tuple) Sequence.register(str) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index eadf06f..e37852e 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -210,7 +210,7 @@ def _remove_universal_flags(_config_vars): # Do not alter a config var explicitly overridden by env var if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] - flags = re.sub(r'-arch\s+\w+\s', ' ', flags, re.ASCII) + flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII) flags = re.sub('-isysroot [^ \t]*', ' ', flags) _save_modified_value(_config_vars, cv, flags) diff --git a/Lib/abc.py b/Lib/abc.py index 1cbf96a..43a34a0 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -129,8 +129,8 @@ class ABCMeta(type): # external code. _abc_invalidation_counter = 0 - def __new__(mcls, name, bases, namespace): - cls = super().__new__(mcls, name, bases, namespace) + def __new__(mcls, name, bases, namespace, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) # Compute set of abstract method names abstracts = {name for name, value in namespace.items() diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 0df58c5..a4967b8 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -459,7 +459,8 @@ class BaseEventLoop(events.AbstractEventLoop): # local task. future.exception() raise - future.remove_done_callback(_run_until_complete_cb) + finally: + future.remove_done_callback(_run_until_complete_cb) if not future.done(): raise RuntimeError('Event loop stopped before Future completed.') diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index d11d289..cff9590 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -191,6 +191,7 @@ class Future: change the future's state to cancelled, schedule the callbacks and return True. """ + self._log_traceback = False if self._state != _PENDING: return False self._state = _CANCELLED diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index deefc93..9266183 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -176,6 +176,10 @@ class Lock(_ContextManagerMixin): yield from fut self._locked = True return True + except futures.CancelledError: + if not self._locked: + self._wake_up_first() + raise finally: self._waiters.remove(fut) @@ -192,14 +196,17 @@ class Lock(_ContextManagerMixin): """ if self._locked: self._locked = False - # Wake up the first waiter who isn't cancelled. - for fut in self._waiters: - if not fut.done(): - fut.set_result(True) - break + self._wake_up_first() else: raise RuntimeError('Lock is not acquired.') + def _wake_up_first(self): + """Wake up the first waiter who isn't cancelled.""" + for fut in self._waiters: + if not fut.done(): + fut.set_result(True) + break + class Event: """Asynchronous equivalent to threading.Event. diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index ff12877..a81645d 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -232,8 +232,9 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport, def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be byte-ish (%r)', - type(data)) + msg = ("data argument must be a bytes-like object, not '%s'" % + type(data).__name__) + raise TypeError(msg) if self._eof_written: raise RuntimeError('write_eof() already called') diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 7ad28d6..7948c4c 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -543,14 +543,19 @@ class SSLProtocol(protocols.Protocol): def _get_extra_info(self, name, default=None): if name in self._extra: return self._extra[name] - else: + elif self._transport is not None: return self._transport.get_extra_info(name, default) + else: + return default def _start_shutdown(self): if self._in_shutdown: return - self._in_shutdown = True - self._write_appdata(b'') + if self._in_handshake: + self._abort() + else: + self._in_shutdown = True + self._write_appdata(b'') def _write_appdata(self, data): self._write_backlog.append((data, 0)) @@ -681,12 +686,14 @@ class SSLProtocol(protocols.Protocol): self._transport._force_close(exc) def _finalize(self): + self._sslpipe = None + if self._transport is not None: self._transport.close() def _abort(self): - if self._transport is not None: - try: + try: + if self._transport is not None: self._transport.abort() - finally: - self._finalize() + finally: + self._finalize() diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 4d79367..9fe2a2f 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -148,6 +148,7 @@ class Task(futures.Future): terminates with a CancelledError exception (even if cancel() was not called). """ + self._log_traceback = False if self.done(): return False if self._fut_waiter is not None: @@ -180,7 +181,12 @@ class Task(futures.Future): else: result = coro.throw(exc) except StopIteration as exc: - self.set_result(exc.value) + if self._must_cancel: + # Task is cancelled right before coro stops. + self._must_cancel = False + self.set_exception(futures.CancelledError()) + else: + self.set_result(exc.value) except futures.CancelledError: super().cancel() # I.e., Future.cancel(self). except Exception as exc: @@ -517,7 +523,8 @@ def ensure_future(coro_or_future, *, loop=None): elif compat.PY35 and inspect.isawaitable(coro_or_future): return ensure_future(_wrap_awaitable(coro_or_future), loop=loop) else: - raise TypeError('A Future, a coroutine or an awaitable is required') + raise TypeError('An asyncio.Future, a coroutine or an awaitable is ' + 'required') @coroutine diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 8421968..6fcba9c 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -1,6 +1,7 @@ """Utilities for with-statement contexts. See PEP 343.""" import abc import sys +import _collections_abc from collections import deque from functools import wraps @@ -25,9 +26,7 @@ class AbstractContextManager(abc.ABC): @classmethod def __subclasshook__(cls, C): if cls is AbstractContextManager: - if (any("__enter__" in B.__dict__ for B in C.__mro__) and - any("__exit__" in B.__dict__ for B in C.__mro__)): - return True + return _collections_abc._check_methods(C, "__enter__", "__exit__") return NotImplemented @@ -88,7 +87,7 @@ class _GeneratorContextManager(ContextDecorator, AbstractContextManager): try: next(self.gen) except StopIteration: - return + return False else: raise RuntimeError("generator didn't stop") else: @@ -98,7 +97,6 @@ class _GeneratorContextManager(ContextDecorator, AbstractContextManager): value = type() try: self.gen.throw(type, value, traceback) - raise RuntimeError("generator didn't stop after throw()") except StopIteration as exc: # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration @@ -111,7 +109,7 @@ class _GeneratorContextManager(ContextDecorator, AbstractContextManager): # Likewise, avoid suppressing if a StopIteration exception # was passed to throw() and later wrapped into a RuntimeError # (see PEP 479). - if exc.__cause__ is value: + if type is StopIteration and exc.__cause__ is value: return False raise except: @@ -122,8 +120,10 @@ class _GeneratorContextManager(ContextDecorator, AbstractContextManager): # fixes the impedance mismatch between the throw() protocol # and the __exit__() protocol. # - if sys.exc_info()[1] is not value: - raise + if sys.exc_info()[1] is value: + return False + raise + raise RuntimeError("generator didn't stop after throw()") def contextmanager(func): diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 3eded77..2e778fb 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -2,8 +2,8 @@ import unittest from ctypes import * from ctypes.test import need_symbol from struct import calcsize -import _testcapi import _ctypes_test +import test.support class SubclassesTest(unittest.TestCase): def test_subclass(self): @@ -202,7 +202,10 @@ class StructureTestCase(unittest.TestCase): "_pack_": -1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + @test.support.cpython_only + def test_packed_c_limits(self): # Issue 15989 + import _testcapi d = {"_fields_": [("a", c_byte)], "_pack_": _testcapi.INT_MAX + 1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) diff --git a/Lib/ctypes/test/test_win32.py b/Lib/ctypes/test/test_win32.py index da16240..5d85ad6 100644 --- a/Lib/ctypes/test/test_win32.py +++ b/Lib/ctypes/test/test_win32.py @@ -41,15 +41,19 @@ class FunctionCallTestCase(unittest.TestCase): @unittest.skipIf(sys.executable.lower().endswith('_d.exe'), "SEH not enabled in debug builds") def test_SEH(self): - # Call functions with invalid arguments, and make sure - # that access violations are trapped and raise an - # exception. - self.assertRaises(OSError, windll.kernel32.GetModuleHandleA, 32) + # Disable faulthandler to prevent logging the warning: + # "Windows fatal exception: access violation" + with support.disable_faulthandler(): + # Call functions with invalid arguments, and make sure + # that access violations are trapped and raise an + # exception. + self.assertRaises(OSError, windll.kernel32.GetModuleHandleA, 32) def test_noargs(self): # This is a special case on win32 x64 windll.user32.GetDesktopWindow() + @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class TestWintypes(unittest.TestCase): def test_HWND(self): diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 8bf1a70..2bcd1dd 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -93,14 +93,11 @@ def get_python_inc(plat_specific=0, prefix=None): # the build directory may not be the source directory, we # must use "srcdir" from the makefile to find the "Include" # directory. - base = _sys_home or project_base if plat_specific: - return base - if _sys_home: - incdir = os.path.join(_sys_home, get_config_var('AST_H_DIR')) + return _sys_home or project_base else: incdir = os.path.join(get_config_var('srcdir'), 'Include') - return os.path.normpath(incdir) + return os.path.normpath(incdir) python_dir = 'python' + get_python_version() + build_flags return os.path.join(prefix, "include", python_dir) elif os.name == "nt": diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index be7f5f3..c6502d6 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -37,6 +37,13 @@ class BuildExtTestCase(TempdirManager, from distutils.command import build_ext build_ext.USER_BASE = site.USER_BASE + # bpo-30132: On Windows, a .pdb file may be created in the current + # working directory. Create a temporary working directory to cleanup + # everything at the end of the test. + self.temp_cwd = support.temp_cwd() + self.temp_cwd.__enter__() + self.addCleanup(self.temp_cwd.__exit__, None, None, None) + def build_ext(self, *args, **kwargs): return build_ext(*args, **kwargs) diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index df46496..c9cbadd 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -361,8 +361,12 @@ class Compat32(Policy): # Assume it is a Header-like object. h = value if h is not None: - parts.append(h.encode(linesep=self.linesep, - maxlinelen=self.max_line_length)) + # The Header class interprets a value of None for maxlinelen as the + # default value of 78, as recommended by RFC 2822. + maxlinelen = 0 + if self.max_line_length is not None: + maxlinelen = self.max_line_length + parts.append(h.encode(linesep=self.linesep, maxlinelen=maxlinelen)) parts.append(self.linesep) return ''.join(parts) diff --git a/Lib/email/architecture.rst b/Lib/email/architecture.rst index 78572ae..fcd10bd 100644 --- a/Lib/email/architecture.rst +++ b/Lib/email/architecture.rst @@ -66,7 +66,7 @@ data payloads. Message Lifecycle ----------------- -The general lifecyle of a message is: +The general lifecycle of a message is: Creation A `Message` object can be created by a Parser, or it can be diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index a3fc501..f12c185 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -1,6 +1,43 @@ -What's New in IDLE 3.6.0? -=========================== -*Release date: 2016-12-16?* +What's New in IDLE 3.6.2 +Released on 2017-06-30? +======================== + +bpo-15786: Fix several problems with IDLE's autocompletion box. +The following should now work: clicking on selection box items; +using the scrollbar; selecting an item by hitting Return. +Hangs on MacOSX should no longer happen. Patch by Louie Lu. + +bpo-25514: Add doc subsubsection about IDLE failure to start. +Popup no-connection message directs users to this section. + +bpo-30642: Fix reference leaks in IDLE tests. +Patches by Louie Lu and Terry Jan Reedy. + +bpo-30495: Add docstrings for textview.py and use PEP8 names. +Patches by Cheryl Sabella and Terry Jan Reedy. + +bpo-30290: Help-about: use pep8 names and add tests. +Increase coverage to 100%. +Patches by Louie Lu, Cheryl Sabella, and Terry Jan Reedy. + +bpo-30303: Add _utest option to textview; add new tests. +Increase coverage to 100%. +Patches by Louie Lu and Terry Jan Reedy. + + +What's New in IDLE 3.6.1 +Released on 2017-03-21 +======================== + +Issue #29071: IDLE colors f-string prefixes (but not invalid ur prefixes). + +Issue #28572: Add 10% to coverage of IDLE's test_configdialog. +Update and augment description of the configuration system. + + +What's New in IDLE 3.6.0 +Released on 2016-12-23 +======================== - Issue #15308: Add 'interrupt execution' (^C) to Shell menu. Patch by Roger Serwy, updated by Bayard Randel. diff --git a/Lib/idlelib/autocomplete_w.py b/Lib/idlelib/autocomplete_w.py index 3374c6e..7c3138f 100644 --- a/Lib/idlelib/autocomplete_w.py +++ b/Lib/idlelib/autocomplete_w.py @@ -1,6 +1,8 @@ """ An auto-completion window for IDLE, used by the autocomplete extension """ +import platform + from tkinter import * from tkinter.ttk import Scrollbar @@ -8,7 +10,8 @@ from idlelib.autocomplete import COMPLETE_FILES, COMPLETE_ATTRIBUTES from idlelib.multicall import MC_SHIFT HIDE_VIRTUAL_EVENT_NAME = "<>" -HIDE_SEQUENCES = ("", "") +HIDE_FOCUS_OUT_SEQUENCE = "" +HIDE_SEQUENCES = (HIDE_FOCUS_OUT_SEQUENCE, "") KEYPRESS_VIRTUAL_EVENT_NAME = "<>" # We need to bind event beyond so that the function will be called # before the default specific IDLE function @@ -201,10 +204,12 @@ class AutoCompleteWindow: self._selection_changed() # bind events - self.hideid = self.widget.bind(HIDE_VIRTUAL_EVENT_NAME, - self.hide_event) + self.hideaid = acw.bind(HIDE_VIRTUAL_EVENT_NAME, self.hide_event) + self.hidewid = self.widget.bind(HIDE_VIRTUAL_EVENT_NAME, self.hide_event) + acw.event_add(HIDE_VIRTUAL_EVENT_NAME, HIDE_FOCUS_OUT_SEQUENCE) for seq in HIDE_SEQUENCES: self.widget.event_add(HIDE_VIRTUAL_EVENT_NAME, seq) + self.keypressid = self.widget.bind(KEYPRESS_VIRTUAL_EVENT_NAME, self.keypress_event) for seq in KEYPRESS_SEQUENCES: @@ -240,9 +245,37 @@ class AutoCompleteWindow: new_y -= acw_height acw.wm_geometry("+%d+%d" % (new_x, new_y)) + if platform.system().startswith('Windows'): + # See issue 15786. When on windows platform, Tk will misbehaive + # to call winconfig_event multiple times, we need to prevent this, + # otherwise mouse button double click will not be able to used. + acw.unbind(WINCONFIG_SEQUENCE, self.winconfigid) + self.winconfigid = None + + def _hide_event_check(self): + if not self.autocompletewindow: + return + + try: + if not self.autocompletewindow.focus_get(): + self.hide_window() + except KeyError: + # See issue 734176, when user click on menu, acw.focus_get() + # will get KeyError. + self.hide_window() + def hide_event(self, event): + # Hide autocomplete list if it exists and does not have focus or + # mouse click on widget / text area. if self.is_active(): - self.hide_window() + if event.type == EventType.FocusOut: + # On windows platform, it will need to delay the check for + # acw.focus_get() when click on acw, otherwise it will return + # None and close the window + self.widget.after(1, self._hide_event_check) + elif event.type == EventType.ButtonPress: + # ButtonPress event only bind to self.widget + self.hide_window() def listselect_event(self, event): if self.is_active(): @@ -292,8 +325,9 @@ class AutoCompleteWindow: return "break" elif keysym == "Return": + self.complete() self.hide_window() - return None + return 'break' elif (self.mode == COMPLETE_ATTRIBUTES and keysym in ("period", "space", "parenleft", "parenright", "bracketleft", @@ -391,10 +425,15 @@ class AutoCompleteWindow: return # unbind events + self.autocompletewindow.event_delete(HIDE_VIRTUAL_EVENT_NAME, + HIDE_FOCUS_OUT_SEQUENCE) for seq in HIDE_SEQUENCES: self.widget.event_delete(HIDE_VIRTUAL_EVENT_NAME, seq) - self.widget.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hideid) - self.hideid = None + + self.autocompletewindow.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hideaid) + self.widget.unbind(HIDE_VIRTUAL_EVENT_NAME, self.hidewid) + self.hideaid = None + self.hidewid = None for seq in KEYPRESS_SEQUENCES: self.widget.event_delete(KEYPRESS_VIRTUAL_EVENT_NAME, seq) self.widget.unbind(KEYPRESS_VIRTUAL_EVENT_NAME, self.keypressid) @@ -405,8 +444,12 @@ class AutoCompleteWindow: self.keyreleaseid = None self.listbox.unbind(LISTUPDATE_SEQUENCE, self.listupdateid) self.listupdateid = None - self.autocompletewindow.unbind(WINCONFIG_SEQUENCE, self.winconfigid) - self.winconfigid = None + if self.winconfigid: + self.autocompletewindow.unbind(WINCONFIG_SEQUENCE, self.winconfigid) + self.winconfigid = None + + # Re-focusOn frame.text (See issue #15786) + self.widget.focus_set() # destroy widgets self.scrollbar.destroy() diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 358bee4..6a9fc61 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -29,7 +29,7 @@ from configparser import ConfigParser import os import sys -from tkinter.font import Font, nametofont +from tkinter.font import Font class InvalidConfigType(Exception): pass class InvalidConfigSet(Exception): pass diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index ae475cb..ab4f1a3 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -26,7 +26,6 @@ from idlelib import pyparse from idlelib import query from idlelib import replace from idlelib import search -from idlelib import textview from idlelib import windows # The default tab setting for a Text widget, in average-width characters. diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index f10cd34..0a3062e 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -6,7 +6,7 @@ - 25.5. IDLE — Python 3.5.2 documentation + 25.5. IDLE — Python 3.7.0a0 documentation @@ -14,7 +14,7 @@ - + + + +

25.5.2.3. Calltips¶

-

A calltip is shown when one types ( after the name of an acccessible +

A calltip is shown when one types ( after the name of an accessible function. A name expression may include dots and subscripts. A calltip remains until it is clicked, the cursor is moved out of the argument area, or ) is typed. When the cursor is in the argument part of a definition, @@ -532,11 +535,47 @@ set in the Options dialog. sys.argv reflects the arguments passed to IDLE itself.

+
+

25.5.3.2. Startup failure¶

+

IDLE uses a socket to communicate between the IDLE GUI process and the user +code execution process. A connection must be established whenever the Shell +starts or restarts. (The latter is indicated by a divider line that says +‘RESTART’). If the user process fails to connect to the GUI process, it +displays a Tk error box with a ‘cannot connect’ message that directs the +user here. It then exits.

+

A common cause of failure is a user-written file with the same name as a +standard library module, such as random.py and tkinter.py. When such a +file is located in the same directory as a file that is about to be run, +IDLE cannot import the stdlib file. The current fix is to rename the +user file.

+

Though less common than in the past, an antivirus or firewall program may +stop the connection. If the program cannot be taught to allow the +connection, then it must be turned off for IDLE to work. It is safe to +allow this internal connection because no data is visible on external +ports. A similar problem is a network mis-configuration that blocks +connections.

+

Python installation issues occasionally stop IDLE: multiple versions can +clash, or a single installation might need admin access. If one undo the +clash, or cannot or does not want to run as admin, it might be easiest to +completely remove Python and start over.

+

A zombie pythonw.exe process could be a problem. On Windows, use Task +Manager to detect and stop one. Sometimes a restart initiated by a program +crash or Keyboard Interrupt (control-C) may fail to connect. Dismissing +the error box or Restart Shell on the Shell menu may fix a temporary problem.

+

When IDLE first starts, it attempts to read user configuration files in +~/.idlerc/ (~ is one’s home directory). If there is a problem, an error +message should be displayed. Leaving aside random disk glitches, this can +be prevented by never editing the files by hand, using the configuration +dialog, under Options, instead Options. Once it happens, the solution may +be to delete one or more of the configuration files.

+

If IDLE quits with no message, and it was not started from a console, try +starting from a console (python -m idlelib) and see if a message appears.

+
-

25.5.3.2. IDLE-console differences¶

+

25.5.3.3. IDLE-console differences¶

As much as possible, the result of executing Python code with IDLE is the same as executing the same code in a console window. However, the different -interface and operation occasionally affects visible results. For instance, +interface and operation occasionally affect visible results. For instance, sys.modules starts with more entries.

IDLE also replaces sys.stdin, sys.stdout, and sys.stderr with objects that get input from and send output to the Shell window. @@ -551,14 +590,14 @@ Some consoles only work with a single physical line at a time. IDLE uses defined for each statement.

-

25.5.3.3. Running without a subprocess¶

+

25.5.3.4. Running without a subprocess¶

By default, IDLE executes user code in a separate subprocess via a socket, which uses the internal loopback interface. This connection is not externally visible and no data is sent to or received from the Internet. If firewall software complains anyway, you can ignore it.

If the attempt to make the socket connection fails, Idle will notify you. Such failures are sometimes transient, but if persistent, the problem -may be either a firewall blocking the connecton or misconfiguration of +may be either a firewall blocking the connection or misconfiguration of a particular system. Until the problem is fixed, one can run Idle with the -n command line switch.

If IDLE is started with the -n command line switch it will run in a @@ -590,12 +629,12 @@ more information.

25.5.4.2. Setting preferences¶

The font preferences, highlighting, keys, and general preferences can be changed via Configure IDLE on the Option menu. Keys can be user defined; -IDLE ships with four built in key sets. In addition a user can create a +IDLE ships with four built-in key sets. In addition, a user can create a custom key set in the Configure IDLE dialog under the keys tab.

25.5.4.3. Extensions¶

-

IDLE contains an extension facility. Peferences for extensions can be +

IDLE contains an extension facility. Preferences for extensions can be changed with Configure Extensions. See the beginning of config-extensions.def in the idlelib directory for further information. The default extensions are currently:

@@ -646,8 +685,9 @@ are currently:

  • 25.5.3. Startup and code execution
  • 25.5.4. Help and preferences
    • @@ -697,7 +740,7 @@ are currently:

    style="vertical-align: middle; margin-top: -1px"/>
  • Python »
  • - 3.5.2 Documentation » + 3.7.0a0 Documentation »
  • The Python Standard Library »
    • @@ -720,12 +763,12 @@ are currently:

    - © Copyright 2001-2016, Python Software Foundation. + © Copyright 2001-2017, Python Software Foundation.
    The Python Software Foundation is a non-profit corporation. Please donate.
    - Last updated on Sep 12, 2016. + Last updated on Jun 13, 2017. Found a bug?
    Created using Sphinx 1.3.6. diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index 77e01a3..fa6112a 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -260,7 +260,7 @@ def copy_strip(): open(dst, 'wb') as out: for line in inn: out.write(line.rstrip() + b'\n') - print('idle.html copied to help.html') + print(f'{src} copied to {dst}') def show_idlehelp(parent): "Create HelpWindow; called from Idle Help event handler." diff --git a/Lib/idlelib/help_about.py b/Lib/idlelib/help_about.py index 071bd3e..f0a40e9 100644 --- a/Lib/idlelib/help_about.py +++ b/Lib/idlelib/help_about.py @@ -4,7 +4,8 @@ import os from sys import version -from tkinter import * +from tkinter import Toplevel, Frame, Label, Button +from tkinter import SUNKEN, TOP, BOTTOM, LEFT, X, BOTH, W, EW, NSEW from idlelib import textview @@ -13,9 +14,13 @@ class AboutDialog(Toplevel): """Modal about dialog for idle """ - def __init__(self, parent, title, _htest=False): - """ + def __init__(self, parent, title, _htest=False, _utest=False): + """Create popup, do not return until tk widget destroyed. + + parent - parent of this dialog + title - string which is title of popup dialog _htest - bool, change box location when running htest + _utest - bool, don't wait_window when running unittest """ Toplevel.__init__(self, parent) self.configure(borderwidth=5) @@ -25,125 +30,152 @@ class AboutDialog(Toplevel): parent.winfo_rooty()+(30 if not _htest else 100))) self.bg = "#707070" self.fg = "#ffffff" - self.CreateWidgets() - self.resizable(height=FALSE, width=FALSE) + self.create_widgets() + self.resizable(height=False, width=False) self.title(title) self.transient(parent) self.grab_set() - self.protocol("WM_DELETE_WINDOW", self.Ok) + self.protocol("WM_DELETE_WINDOW", self.ok) self.parent = parent - self.buttonOk.focus_set() - self.bind('',self.Ok) #dismiss dialog - self.bind('',self.Ok) #dismiss dialog - self.wait_window() + self.button_ok.focus_set() + self.bind('', self.ok) # dismiss dialog + self.bind('', self.ok) # dismiss dialog + self._current_textview = None + self._utest = _utest - def CreateWidgets(self): + if not _utest: + self.deiconify() + self.wait_window() + + def create_widgets(self): release = version[:version.index(' ')] - frameMain = Frame(self, borderwidth=2, relief=SUNKEN) - frameButtons = Frame(self) - frameButtons.pack(side=BOTTOM, fill=X) - frameMain.pack(side=TOP, expand=TRUE, fill=BOTH) - self.buttonOk = Button(frameButtons, text='Close', - command=self.Ok) - self.buttonOk.pack(padx=5, pady=5) - #self.picture = Image('photo', data=self.pictureData) - frameBg = Frame(frameMain, bg=self.bg) - frameBg.pack(expand=TRUE, fill=BOTH) - labelTitle = Label(frameBg, text='IDLE', fg=self.fg, bg=self.bg, - font=('courier', 24, 'bold')) - labelTitle.grid(row=0, column=0, sticky=W, padx=10, pady=10) - #labelPicture = Label(frameBg, text='[picture]') - #image=self.picture, bg=self.bg) - #labelPicture.grid(row=1, column=1, sticky=W, rowspan=2, - # padx=0, pady=3) - byline = "Python's Integrated DeveLopment Environment" + 5*'\n' - labelDesc = Label(frameBg, text=byline, justify=LEFT, - fg=self.fg, bg=self.bg) - labelDesc.grid(row=2, column=0, sticky=W, columnspan=3, padx=10, pady=5) - labelEmail = Label(frameBg, text='email: idle-dev@python.org', - justify=LEFT, fg=self.fg, bg=self.bg) - labelEmail.grid(row=6, column=0, columnspan=2, - sticky=W, padx=10, pady=0) - labelWWW = Label(frameBg, text='https://docs.python.org/' + - version[:3] + '/library/idle.html', - justify=LEFT, fg=self.fg, bg=self.bg) - labelWWW.grid(row=7, column=0, columnspan=2, sticky=W, padx=10, pady=0) - Frame(frameBg, borderwidth=1, relief=SUNKEN, + frame = Frame(self, borderwidth=2, relief=SUNKEN) + frame_buttons = Frame(self) + frame_buttons.pack(side=BOTTOM, fill=X) + frame.pack(side=TOP, expand=True, fill=BOTH) + self.button_ok = Button(frame_buttons, text='Close', + command=self.ok) + self.button_ok.pack(padx=5, pady=5) + + frame_background = Frame(frame, bg=self.bg) + frame_background.pack(expand=True, fill=BOTH) + + header = Label(frame_background, text='IDLE', fg=self.fg, + bg=self.bg, font=('courier', 24, 'bold')) + header.grid(row=0, column=0, sticky=W, padx=10, pady=10) + byline_text = "Python's Integrated DeveLopment Environment" + 5*'\n' + byline = Label(frame_background, text=byline_text, justify=LEFT, + fg=self.fg, bg=self.bg) + byline.grid(row=2, column=0, sticky=W, columnspan=3, padx=10, pady=5) + email = Label(frame_background, text='email: idle-dev@python.org', + justify=LEFT, fg=self.fg, bg=self.bg) + email.grid(row=6, column=0, columnspan=2, sticky=W, padx=10, pady=0) + docs = Label(frame_background, text='https://docs.python.org/' + + version[:3] + '/library/idle.html', + justify=LEFT, fg=self.fg, bg=self.bg) + docs.grid(row=7, column=0, columnspan=2, sticky=W, padx=10, pady=0) + + Frame(frame_background, borderwidth=1, relief=SUNKEN, height=2, bg=self.bg).grid(row=8, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - labelPythonVer = Label(frameBg, text='Python version: ' + - release, fg=self.fg, bg=self.bg) - labelPythonVer.grid(row=9, column=0, sticky=W, padx=10, pady=0) - tkVer = self.tk.call('info', 'patchlevel') - labelTkVer = Label(frameBg, text='Tk version: '+ - tkVer, fg=self.fg, bg=self.bg) - labelTkVer.grid(row=9, column=1, sticky=W, padx=2, pady=0) - py_button_f = Frame(frameBg, bg=self.bg) - py_button_f.grid(row=10, column=0, columnspan=2, sticky=NSEW) - buttonLicense = Button(py_button_f, text='License', width=8, - highlightbackground=self.bg, - command=self.ShowLicense) - buttonLicense.pack(side=LEFT, padx=10, pady=10) - buttonCopyright = Button(py_button_f, text='Copyright', width=8, + + pyver = Label(frame_background, text='Python version: ' + release, + fg=self.fg, bg=self.bg) + pyver.grid(row=9, column=0, sticky=W, padx=10, pady=0) + tk_patchlevel = self.tk.call('info', 'patchlevel') + tkver = Label(frame_background, text='Tk version: ' + tk_patchlevel, + fg=self.fg, bg=self.bg) + tkver.grid(row=9, column=1, sticky=W, padx=2, pady=0) + py_buttons = Frame(frame_background, bg=self.bg) + py_buttons.grid(row=10, column=0, columnspan=2, sticky=NSEW) + self.py_license = Button(py_buttons, text='License', width=8, + highlightbackground=self.bg, + command=self.show_py_license) + self.py_license.pack(side=LEFT, padx=10, pady=10) + self.py_copyright = Button(py_buttons, text='Copyright', width=8, + highlightbackground=self.bg, + command=self.show_py_copyright) + self.py_copyright.pack(side=LEFT, padx=10, pady=10) + self.py_credits = Button(py_buttons, text='Credits', width=8, highlightbackground=self.bg, - command=self.ShowCopyright) - buttonCopyright.pack(side=LEFT, padx=10, pady=10) - buttonCredits = Button(py_button_f, text='Credits', width=8, - highlightbackground=self.bg, - command=self.ShowPythonCredits) - buttonCredits.pack(side=LEFT, padx=10, pady=10) - Frame(frameBg, borderwidth=1, relief=SUNKEN, + command=self.show_py_credits) + self.py_credits.pack(side=LEFT, padx=10, pady=10) + + Frame(frame_background, borderwidth=1, relief=SUNKEN, height=2, bg=self.bg).grid(row=11, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - idle_v = Label(frameBg, text='IDLE version: ' + release, - fg=self.fg, bg=self.bg) - idle_v.grid(row=12, column=0, sticky=W, padx=10, pady=0) - idle_button_f = Frame(frameBg, bg=self.bg) - idle_button_f.grid(row=13, column=0, columnspan=3, sticky=NSEW) - idle_about_b = Button(idle_button_f, text='README', width=8, - highlightbackground=self.bg, - command=self.ShowIDLEAbout) - idle_about_b.pack(side=LEFT, padx=10, pady=10) - idle_news_b = Button(idle_button_f, text='NEWS', width=8, - highlightbackground=self.bg, - command=self.ShowIDLENEWS) - idle_news_b.pack(side=LEFT, padx=10, pady=10) - idle_credits_b = Button(idle_button_f, text='Credits', width=8, - highlightbackground=self.bg, - command=self.ShowIDLECredits) - idle_credits_b.pack(side=LEFT, padx=10, pady=10) - # License, et all, are of type _sitebuiltins._Printer - def ShowLicense(self): + idlever = Label(frame_background, text='IDLE version: ' + release, + fg=self.fg, bg=self.bg) + idlever.grid(row=12, column=0, sticky=W, padx=10, pady=0) + idle_buttons = Frame(frame_background, bg=self.bg) + idle_buttons.grid(row=13, column=0, columnspan=3, sticky=NSEW) + self.readme = Button(idle_buttons, text='README', width=8, + highlightbackground=self.bg, + command=self.show_readme) + self.readme.pack(side=LEFT, padx=10, pady=10) + self.idle_news = Button(idle_buttons, text='NEWS', width=8, + highlightbackground=self.bg, + command=self.show_idle_news) + self.idle_news.pack(side=LEFT, padx=10, pady=10) + self.idle_credits = Button(idle_buttons, text='Credits', width=8, + highlightbackground=self.bg, + command=self.show_idle_credits) + self.idle_credits.pack(side=LEFT, padx=10, pady=10) + + # License, copyright, and credits are of type _sitebuiltins._Printer + def show_py_license(self): + "Handle License button event." self.display_printer_text('About - License', license) - def ShowCopyright(self): + def show_py_copyright(self): + "Handle Copyright button event." self.display_printer_text('About - Copyright', copyright) - def ShowPythonCredits(self): + def show_py_credits(self): + "Handle Python Credits button event." self.display_printer_text('About - Python Credits', credits) # Encode CREDITS.txt to utf-8 for proper version of Loewis. # Specify others as ascii until need utf-8, so catch errors. - def ShowIDLECredits(self): + def show_idle_credits(self): + "Handle Idle Credits button event." self.display_file_text('About - Credits', 'CREDITS.txt', 'utf-8') - def ShowIDLEAbout(self): + def show_readme(self): + "Handle Readme button event." self.display_file_text('About - Readme', 'README.txt', 'ascii') - def ShowIDLENEWS(self): + def show_idle_news(self): + "Handle News button event." self.display_file_text('About - NEWS', 'NEWS.txt', 'utf-8') def display_printer_text(self, title, printer): + """Create textview for built-in constants. + + Built-in constants have type _sitebuiltins._Printer. The + text is extracted from the built-in and then sent to a text + viewer with self as the parent and title as the title of + the popup. + """ printer._Printer__setup() text = '\n'.join(printer._Printer__lines) - textview.view_text(self, title, text) + self._current_textview = textview.view_text( + self, title, text, _utest=self._utest) def display_file_text(self, title, filename, encoding=None): + """Create textview for filename. + + The filename needs to be in the current directory. The path + is sent to a text viewer with self as the parent, title as + the title of the popup, and the file encoding. + """ fn = os.path.join(os.path.abspath(os.path.dirname(__file__)), filename) - textview.view_file(self, title, fn, encoding) + self._current_textview = textview.view_file( + self, title, fn, encoding, _utest=self._utest) - def Ok(self, event=None): + def ok(self, event=None): + "Dismiss help_about dialog." self.destroy() diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py index 0b11602..97e15fe 100644 --- a/Lib/idlelib/idle_test/test_calltips.py +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -58,7 +58,7 @@ class Get_signatureTest(unittest.TestCase): 'Create and return a new object. See help(type) for accurate signature.') gtest(list.__init__, 'Initialize self. See help(type(self)) for accurate signature.') - append_doc = "L.append(object) -> None -- append object to end" + append_doc = "L.append(object) -> None -- append object to end" #see3.7 gtest(list.append, append_doc) gtest([].append, append_doc) gtest(List.append, append_doc) @@ -81,9 +81,9 @@ class Get_signatureTest(unittest.TestCase): def test_multiline_docstring(self): # Test fewer lines than max. - self.assertEqual(signature(list), - "list() -> new empty list\n" - "list(iterable) -> new list initialized from iterable's items") + self.assertEqual(signature(range), + "range(stop) -> range object\n" + "range(start, stop[, step]) -> range object") # Test max lines self.assertEqual(signature(bytes), '''\ diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index a3fa1a3..e678cc6 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -3,9 +3,7 @@ Much is tested by opening config dialog live or in test_configdialog. Coverage: 27% ''' -from sys import modules from test.support import captured_stderr -from tkinter import Tk import unittest from idlelib import config diff --git a/Lib/idlelib/idle_test/test_config_key.py b/Lib/idlelib/idle_test/test_config_key.py index ee3f2c8..8a24c96 100644 --- a/Lib/idlelib/idle_test/test_config_key.py +++ b/Lib/idlelib/idle_test/test_config_key.py @@ -6,7 +6,7 @@ from idlelib import config_key from test.support import requires requires('gui') import unittest -from tkinter import Tk, Text +from tkinter import Tk class GetKeysTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index e9d29d4..64a2a88 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -1,7 +1,5 @@ import unittest -from tkinter import Tk, Text from idlelib.editor import EditorWindow -from test.support import requires class Editor_func_test(unittest.TestCase): def test_filename_to_unicode(self): diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index 843efb9..15d1b6b 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -1,24 +1,117 @@ '''Test idlelib.help_about. -Coverage: +Coverage: 100% ''' -from idlelib import help_about -from idlelib import textview +from test.support import requires, findfile +from tkinter import Tk, TclError +import unittest from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Mbox_func -import unittest +from idlelib.help_about import AboutDialog as About +from idlelib import textview + +class LiveDialogTest(unittest.TestCase): + """Simulate user clicking buttons other than [Close]. + + Test that invoked textview has text from source. + """ + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.dialog = About(cls.root, 'About IDLE', _utest=True) + + @classmethod + def tearDownClass(cls): + del cls.dialog + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_dialog_title(self): + """Test about dialog title""" + self.assertEqual(self.dialog.title(), 'About IDLE') + + def test_printer_buttons(self): + """Test buttons whose commands use printer function.""" + dialog = self.dialog + button_sources = [(self.dialog.py_license, license), + (self.dialog.py_copyright, copyright), + (self.dialog.py_credits, credits)] + + for button, printer in button_sources: + printer._Printer__setup() + button.invoke() + self.assertEqual( + printer._Printer__lines[0], + dialog._current_textview.text.get('1.0', '1.end')) + self.assertEqual( + printer._Printer__lines[1], + dialog._current_textview.text.get('2.0', '2.end')) + dialog._current_textview.destroy() + + def test_file_buttons(self): + """Test buttons that display files.""" + dialog = self.dialog + button_sources = [(self.dialog.readme, 'README.txt'), + (self.dialog.idle_news, 'NEWS.txt'), + (self.dialog.idle_credits, 'CREDITS.txt')] + + for button, filename in button_sources: + button.invoke() + fn = findfile(filename, subdir='idlelib') + with open(fn) as f: + self.assertEqual( + f.readline().strip(), + dialog._current_textview.text.get('1.0', '1.end')) + f.readline() + self.assertEqual( + f.readline().strip(), + dialog._current_textview.text.get('3.0', '3.end')) + dialog._current_textview.destroy() + + +class CloseTest(unittest.TestCase): + """Simulate user clicking [Close] button""" + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.dialog = About(cls.root, 'About IDLE', _utest=True) + + @classmethod + def tearDownClass(cls): + del cls.dialog + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def test_close(self): + self.assertEqual(self.dialog.winfo_class(), 'Toplevel') + self.dialog.button_ok.invoke() + with self.assertRaises(TclError): + self.dialog.winfo_class() + -About = help_about.AboutDialog class Dummy_about_dialog(): # Dummy class for testing file display functions. - idle_credits = About.ShowIDLECredits - idle_readme = About.ShowIDLEAbout - idle_news = About.ShowIDLENEWS + idle_credits = About.show_idle_credits + idle_readme = About.show_readme + idle_news = About.show_idle_news # Called by the above display_file_text = About.display_file_text + _utest = True class DisplayFileTest(unittest.TestCase): + """Test functions that display files. + + While somewhat redundant with gui-based test_file_dialog, + these unit tests run on all buildbots, not just a few. + """ dialog = Dummy_about_dialog() @classmethod @@ -29,14 +122,13 @@ class DisplayFileTest(unittest.TestCase): cls.view = Func() textview.showerror = cls.error textview.view_text = cls.view - cls.About = Dummy_about_dialog() @classmethod def tearDownClass(cls): textview.showerror = cls.orig_error textview.view_text = cls.orig_view - def test_file_isplay(self): + def test_file_display(self): for handler in (self.dialog.idle_credits, self.dialog.idle_readme, self.dialog.idle_news): diff --git a/Lib/idlelib/idle_test/test_macosx.py b/Lib/idlelib/idle_test/test_macosx.py index fae75d8..3d85f3c 100644 --- a/Lib/idlelib/idle_test/test_macosx.py +++ b/Lib/idlelib/idle_test/test_macosx.py @@ -4,7 +4,6 @@ Coverage: 71% on Windows. ''' from idlelib import macosx from test.support import requires -import sys import tkinter as tk import unittest import unittest.mock as mock @@ -78,6 +77,10 @@ class SetupTest(unittest.TestCase): requires('gui') cls.root = tk.Tk() cls.root.withdraw() + def cmd(tkpath, func): + assert isinstance(tkpath, str) + assert isinstance(func, type(cmd)) + cls.root.createcommand = cmd @classmethod def tearDownClass(cls): diff --git a/Lib/idlelib/idle_test/test_parenmatch.py b/Lib/idlelib/idle_test/test_parenmatch.py index 051f7ea..b5fd0d1 100644 --- a/Lib/idlelib/idle_test/test_parenmatch.py +++ b/Lib/idlelib/idle_test/test_parenmatch.py @@ -24,6 +24,7 @@ class ParenMatchTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.root = Tk() + cls.root.withdraw() cls.text = Text(cls.root) cls.editwin = DummyEditwin(cls.text) cls.editwin.text_frame = Mock() diff --git a/Lib/idlelib/idle_test/test_query.py b/Lib/idlelib/idle_test/test_query.py index 66af8eb..1210afe 100644 --- a/Lib/idlelib/idle_test/test_query.py +++ b/Lib/idlelib/idle_test/test_query.py @@ -258,6 +258,7 @@ class QueryGuiTest(unittest.TestCase): @classmethod def tearDownClass(cls): + del cls.dialog.destroy del cls.dialog cls.root.destroy() del cls.root diff --git a/Lib/idlelib/idle_test/test_replace.py b/Lib/idlelib/idle_test/test_replace.py index 9913ed2..2ecbd34 100644 --- a/Lib/idlelib/idle_test/test_replace.py +++ b/Lib/idlelib/idle_test/test_replace.py @@ -221,8 +221,8 @@ class ReplaceDialogTest(unittest.TestCase): self.assertIn('Invalid Replace Expression', showerror.message) # test access method - self.engine.setcookedpat("\'") - equal(pv.get(), "\\'") + self.engine.setcookedpat("?") + equal(pv.get(), "\\?") def test_replace_backwards(self): equal = self.assertEqual diff --git a/Lib/idlelib/idle_test/test_searchbase.py b/Lib/idlelib/idle_test/test_searchbase.py index d769fa2..27b02fb 100644 --- a/Lib/idlelib/idle_test/test_searchbase.py +++ b/Lib/idlelib/idle_test/test_searchbase.py @@ -5,7 +5,7 @@ testing skipping of suite when self.needwrapbutton is false. ''' import unittest from test.support import requires -from tkinter import Tk, Toplevel, Frame ##, BooleanVar, StringVar +from tkinter import Tk, Frame ##, BooleanVar, StringVar from idlelib import searchengine as se from idlelib import searchbase as sdb from idlelib.idle_test.mock_idle import Func diff --git a/Lib/idlelib/idle_test/test_textview.py b/Lib/idlelib/idle_test/test_textview.py index f018f5e..7a2f7e4 100644 --- a/Lib/idlelib/idle_test/test_textview.py +++ b/Lib/idlelib/idle_test/test_textview.py @@ -5,7 +5,7 @@ is a widget containing multiple widgets, all tests must be gui tests. Using mock Text would not change this. Other mocks are used to retrieve information about calls. -Coverage: 94%. +Coverage: 100%. ''' from idlelib import textview as tv from test.support import requires @@ -13,7 +13,7 @@ requires('gui') import unittest import os -from tkinter import Tk +from tkinter import Tk, Button from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Mbox_func @@ -28,12 +28,18 @@ def tearDownModule(): root.destroy() # Pyflakes falsely sees root as undefined. del root +# If we call TextViewer or wrapper functions with defaults +# modal=True, _utest=False, test hangs on call to wait_window. +# Have also gotten tk error 'can't invoke "event" command'. + class TV(tv.TextViewer): # Used in TextViewTest. transient = Func() grab_set = Func() wait_window = Func() + +# Call wrapper class with mock wait_window. class TextViewTest(unittest.TestCase): def setUp(self): @@ -46,24 +52,25 @@ class TextViewTest(unittest.TestCase): self.assertTrue(TV.transient.called) self.assertTrue(TV.grab_set.called) self.assertTrue(TV.wait_window.called) - view.Ok() + view.ok() def test_init_nonmodal(self): view = TV(root, 'Title', 'test text', modal=False) self.assertFalse(TV.transient.called) self.assertFalse(TV.grab_set.called) self.assertFalse(TV.wait_window.called) - view.Ok() + view.ok() def test_ok(self): view = TV(root, 'Title', 'test text', modal=False) view.destroy = Func() - view.Ok() + view.ok() self.assertTrue(view.destroy.called) del view.destroy # Unmask real function. view.destroy() +# Call TextViewer with modal=False. class ViewFunctionTest(unittest.TestCase): @classmethod @@ -77,23 +84,71 @@ class ViewFunctionTest(unittest.TestCase): del cls.orig_error def test_view_text(self): - # If modal True, get tk error 'can't invoke "event" command'. view = tv.view_text(root, 'Title', 'test text', modal=False) self.assertIsInstance(view, tv.TextViewer) - view.Ok() + view.ok() def test_view_file(self): - test_dir = os.path.dirname(__file__) - testfile = os.path.join(test_dir, 'test_textview.py') - view = tv.view_file(root, 'Title', testfile, modal=False) + view = tv.view_file(root, 'Title', __file__, modal=False) self.assertIsInstance(view, tv.TextViewer) - self.assertIn('Test', view.textView.get('1.0', '1.end')) - view.Ok() + self.assertIn('Test', view.text.get('1.0', '1.end')) + view.ok() + def test_bad_file(self): # Mock showerror will be used; view_file will return None. - testfile = os.path.join(test_dir, '../notthere.py') - view = tv.view_file(root, 'Title', testfile, modal=False) + view = tv.view_file(root, 'Title', 'abc.xyz', modal=False) self.assertIsNone(view) + self.assertEqual(tv.showerror.title, 'File Load Error') + + def test_bad_encoding(self): + p = os.path + fn = p.abspath(p.join(p.dirname(__file__), '..', 'CREDITS.txt')) + tv.showerror.title = None + view = tv.view_file(root, 'Title', fn, 'ascii', modal=False) + self.assertIsNone(view) + self.assertEqual(tv.showerror.title, 'Unicode Decode Error') + + + +# Call TextViewer with _utest=True. +class ButtonClickTest(unittest.TestCase): + + def setUp(self): + self.view = None + self.called = False + + def tearDown(self): + if self.view: + self.view.destroy() + + def test_view_text_bind_with_button(self): + def _command(): + self.called = True + self.view = tv.view_text(root, 'TITLE_TEXT', 'COMMAND', _utest=True) + button = Button(root, text='BUTTON', command=_command) + button.invoke() + self.addCleanup(button.destroy) + + self.assertEqual(self.called, True) + self.assertEqual(self.view.title(), 'TITLE_TEXT') + self.assertEqual(self.view.text.get('1.0', '1.end'), 'COMMAND') + + def test_view_file_bind_with_button(self): + def _command(): + self.called = True + self.view = tv.view_file(root, 'TITLE_FILE', __file__, _utest=True) + button = Button(root, text='BUTTON', command=_command) + button.invoke() + self.addCleanup(button.destroy) + + self.assertEqual(self.called, True) + self.assertEqual(self.view.title(), 'TITLE_FILE') + with open(__file__) as f: + self.assertEqual(self.view.text.get('1.0', '1.end'), + f.readline().strip()) + f.readline() + self.assertEqual(self.view.text.get('3.0', '3.end'), + f.readline().strip()) if __name__ == '__main__': diff --git a/Lib/idlelib/idle_test/test_tree.py b/Lib/idlelib/idle_test/test_tree.py index 09ba964..bb597d8 100644 --- a/Lib/idlelib/idle_test/test_tree.py +++ b/Lib/idlelib/idle_test/test_tree.py @@ -5,7 +5,6 @@ Coverage: 56% from idlelib import tree from test.support import requires requires('gui') -import os import unittest from tkinter import Tk diff --git a/Lib/idlelib/macosx.py b/Lib/idlelib/macosx.py index c225dd9..d4566cd 100644 --- a/Lib/idlelib/macosx.py +++ b/Lib/idlelib/macosx.py @@ -2,7 +2,6 @@ A number of functions that enhance IDLE on Mac OSX. """ from sys import platform # Used in _init_tk_type, changed by test. -import warnings import tkinter diff --git a/Lib/idlelib/pyparse.py b/Lib/idlelib/pyparse.py index 6739dfd..536b2d7 100644 --- a/Lib/idlelib/pyparse.py +++ b/Lib/idlelib/pyparse.py @@ -1,4 +1,4 @@ -from collections import Mapping +from collections.abc import Mapping import re import sys diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index f3ee391..5b0e5b2 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -1,5 +1,7 @@ #! /usr/bin/env python3 +import sys + try: from tkinter import * except ImportError: @@ -16,29 +18,24 @@ if TkVersion < 8.5: raise SystemExit(1) from code import InteractiveInterpreter -import getopt -import io import linecache import os import os.path -from platform import python_version, system +from platform import python_version import re import socket import subprocess -import sys import threading import time import tokenize import warnings -from idlelib import testing # bool value from idlelib.colorizer import ColorDelegator from idlelib.config import idleConf from idlelib import debugger from idlelib import debugger_r from idlelib.editor import EditorWindow, fixwordbreaks from idlelib.filelist import FileList -from idlelib import macosx from idlelib.outwin import OutputWindow from idlelib import rpc from idlelib.run import idle_formatwarning, PseudoInputFile, PseudoOutputFile @@ -1371,6 +1368,11 @@ echo "import sys; print(sys.argv)" | idle - "foobar" """ def main(): + import getopt + from platform import system + from idlelib import testing # bool value + from idlelib import macosx + global flist, root, use_subprocess capture_warnings(True) diff --git a/Lib/idlelib/query.py b/Lib/idlelib/query.py index 3b1f1e2..5935063 100644 --- a/Lib/idlelib/query.py +++ b/Lib/idlelib/query.py @@ -24,7 +24,7 @@ import importlib import os from sys import executable, platform # Platform is set for one test. -from tkinter import Toplevel, StringVar, W, E, N, S +from tkinter import Toplevel, StringVar, W, E, S from tkinter.ttk import Frame, Button, Entry, Label from tkinter import filedialog from tkinter.font import Font diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index afa9744..9f6604b 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -52,7 +52,7 @@ def idle_showwarning_subproc( try: file.write(idle_formatwarning( message, category, filename, lineno, line)) - except IOError: + except OSError: pass # the file (probably stderr) is invalid - this warning gets lost. _warnings_showwarning = None @@ -180,19 +180,16 @@ def manage_socket(address): server.handle_request() # A single request only def show_socket_error(err, address): + "Display socket error from manage_socket." import tkinter - import tkinter.messagebox as tkMessageBox + from tkinter.messagebox import showerror root = tkinter.Tk() root.withdraw() - if err.args[0] == 61: # connection refused - msg = "IDLE's subprocess can't connect to %s:%d. This may be due "\ - "to your personal firewall configuration. It is safe to "\ - "allow this internal connection because no data is visible on "\ - "external ports." % address - tkMessageBox.showerror("IDLE Subprocess Error", msg, parent=root) - else: - tkMessageBox.showerror("IDLE Subprocess Error", - "Socket Error: %s" % err.args[1], parent=root) + msg = f"IDLE's subprocess can't connect to {address[0]}:{address[1]}.\n"\ + f"Fatal OSError #{err.errno}: {err.strerror}.\n"\ + f"See the 'Startup failure' section of the IDLE doc, online at\n"\ + f"https://docs.python.org/3/library/idle.html#startup-failure" + showerror("IDLE Subprocess Error", msg, parent=root) root.destroy() def print_exception(): diff --git a/Lib/idlelib/stackviewer.py b/Lib/idlelib/stackviewer.py index 0698def..400fa63 100644 --- a/Lib/idlelib/stackviewer.py +++ b/Lib/idlelib/stackviewer.py @@ -1,6 +1,5 @@ import linecache import os -import re import sys import tkinter as tk diff --git a/Lib/idlelib/textview.py b/Lib/idlelib/textview.py index adee326..ab653a9 100644 --- a/Lib/idlelib/textview.py +++ b/Lib/idlelib/textview.py @@ -1,7 +1,9 @@ """Simple text browser for IDLE """ -from tkinter import * +from tkinter import Toplevel, Frame, Button, Text +from tkinter import DISABLED, SUNKEN, VERTICAL, WORD +from tkinter import RIGHT, LEFT, TOP, BOTTOM, BOTH, X, Y from tkinter.ttk import Scrollbar from tkinter.messagebox import showerror @@ -9,14 +11,18 @@ from tkinter.messagebox import showerror class TextViewer(Toplevel): "A simple text viewer dialog for IDLE." - def __init__(self, parent, title, text, modal=True, _htest=False): - """Show the given text in a scrollable window with a 'close' button + def __init__(self, parent, title, text, modal=True, + _htest=False, _utest=False): + """Show the given text in a scrollable window with a 'close' button. - If modal option set to False, user can interact with other windows, - otherwise they will be unable to interact with other windows until - the textview window is closed. + If modal is left True, users cannot interact with other windows + until the textview window is closed. + parent - parent of this dialog + title - string which is title of popup dialog + text - text to display in dialog _htest - bool; change box location when running htest. + _utest - bool; don't wait_window when running unittest. """ Toplevel.__init__(self, parent) self.configure(borderwidth=5) @@ -28,47 +34,65 @@ class TextViewer(Toplevel): self.bg = '#ffffff' self.fg = '#000000' - self.CreateWidgets() + self.create_widgets() self.title(title) - self.protocol("WM_DELETE_WINDOW", self.Ok) + self.protocol("WM_DELETE_WINDOW", self.ok) self.parent = parent - self.textView.focus_set() + self.text.focus_set() # Bind keys for closing this dialog. - self.bind('',self.Ok) - self.bind('',self.Ok) - self.textView.insert(0.0, text) - self.textView.config(state=DISABLED) + self.bind('', self.ok) + self.bind('', self.ok) + self.text.insert(0.0, text) + self.text.config(state=DISABLED) if modal: self.transient(parent) self.grab_set() - self.wait_window() - - def CreateWidgets(self): - frameText = Frame(self, relief=SUNKEN, height=700) - frameButtons = Frame(self) - self.buttonOk = Button(frameButtons, text='Close', - command=self.Ok, takefocus=FALSE) - self.scrollbarView = Scrollbar(frameText, orient=VERTICAL, - takefocus=FALSE) - self.textView = Text(frameText, wrap=WORD, highlightthickness=0, + if not _utest: + self.wait_window() + + def create_widgets(self): + "Create Frame with Text (with vertical Scrollbar) and Button." + frame = Frame(self, relief=SUNKEN, height=700) + frame_buttons = Frame(self) + self.button_ok = Button(frame_buttons, text='Close', + command=self.ok, takefocus=False) + self.scrollbar = Scrollbar(frame, orient=VERTICAL, takefocus=False) + self.text = Text(frame, wrap=WORD, highlightthickness=0, fg=self.fg, bg=self.bg) - self.scrollbarView.config(command=self.textView.yview) - self.textView.config(yscrollcommand=self.scrollbarView.set) - self.buttonOk.pack() - self.scrollbarView.pack(side=RIGHT,fill=Y) - self.textView.pack(side=LEFT,expand=TRUE,fill=BOTH) - frameButtons.pack(side=BOTTOM,fill=X) - frameText.pack(side=TOP,expand=TRUE,fill=BOTH) - - def Ok(self, event=None): + self.scrollbar.config(command=self.text.yview) + self.text.config(yscrollcommand=self.scrollbar.set) + + self.button_ok.pack() + self.scrollbar.pack(side=RIGHT, fill=Y) + self.text.pack(side=LEFT, expand=True, fill=BOTH) + frame_buttons.pack(side=BOTTOM, fill=X) + frame.pack(side=TOP, expand=True, fill=BOTH) + + def ok(self, event=None): + """Dismiss text viewer dialog.""" self.destroy() -def view_text(parent, title, text, modal=True): - return TextViewer(parent, title, text, modal) +def view_text(parent, title, text, modal=True, _utest=False): + """Create TextViewer for given text. + + parent - parent of this dialog + title - string which is the title of popup dialog + text - text to display in this dialog + modal - controls if users can interact with other windows while this + dialog is displayed + _utest - bool; controls wait_window on unittest + """ + return TextViewer(parent, title, text, modal, _utest=_utest) -def view_file(parent, title, filename, encoding=None, modal=True): + +def view_file(parent, title, filename, encoding=None, modal=True, _utest=False): + """Create TextViewer for text in filename. + + Return error message if file cannot be read. Otherwise calls view_text + with contents of the file. + """ try: with open(filename, 'r', encoding=encoding) as file: contents = file.read() @@ -81,7 +105,9 @@ def view_file(parent, title, filename, encoding=None, modal=True): message=str(err), parent=parent) else: - return view_text(parent, title, contents, modal) + return view_text(parent, title, contents, modal, _utest=_utest) + return None + if __name__ == '__main__': import unittest diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 2fa9012..1c0b03b 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -318,9 +318,12 @@ class IMAP4: self.file.close() try: self.sock.shutdown(socket.SHUT_RDWR) - except OSError as e: - # The server might already have closed the connection - if e.errno != errno.ENOTCONN: + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): raise finally: self.sock.close() diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index b6a9f82..8b11d22 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -136,7 +136,7 @@ def reload(module): """ if not module or not isinstance(module, types.ModuleType): - raise TypeError("reload() argument must be module") + raise TypeError("reload() argument must be a module") try: name = module.__spec__.name except AttributeError: diff --git a/Lib/inspect.py b/Lib/inspect.py index 4d56ef5..3317f58 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -31,6 +31,7 @@ Here are some of the useful functions provided by this module: __author__ = ('Ka-Ping Yee ', 'Yury Selivanov ') +import abc import ast import dis import collections.abc @@ -253,18 +254,24 @@ def iscode(object): """Return true if the object is a code object. Code objects provide these attributes: - co_argcount number of arguments (not including * or ** args) - co_code string of raw compiled bytecode - co_consts tuple of constants used in the bytecode - co_filename name of file in which this code object was created - co_firstlineno number of first line in Python source code - co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg - co_lnotab encoded mapping of line numbers to bytecode indices - co_name name with which this code object was defined - co_names tuple of names of local variables - co_nlocals number of local variables - co_stacksize virtual machine stack space required - co_varnames tuple of names of arguments and local variables""" + co_argcount number of arguments (not including *, ** args + or keyword only arguments) + co_code string of raw compiled bytecode + co_cellvars tuple of names of cell variables + co_consts tuple of constants used in the bytecode + co_filename name of file in which this code object was created + co_firstlineno number of first line in Python source code + co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg + | 16=nested | 32=generator | 64=nofree | 128=coroutine + | 256=iterable_coroutine | 512=async_generator + co_freevars tuple of names of free variables + co_kwonlyargcount number of keyword only arguments (not including ** arg) + co_lnotab encoded mapping of line numbers to bytecode indices + co_name name with which this code object was defined + co_names tuple of names of local variables + co_nlocals number of local variables + co_stacksize virtual machine stack space required + co_varnames tuple of names of arguments and local variables""" return isinstance(object, types.CodeType) def isbuiltin(object): @@ -285,7 +292,27 @@ def isroutine(object): def isabstract(object): """Return true if the object is an abstract base class (ABC).""" - return bool(isinstance(object, type) and object.__flags__ & TPFLAGS_IS_ABSTRACT) + if not isinstance(object, type): + return False + if object.__flags__ & TPFLAGS_IS_ABSTRACT: + return True + if not issubclass(type(object), abc.ABCMeta): + return False + if hasattr(object, '__abstractmethods__'): + # It looks like ABCMeta.__new__ has finished running; + # TPFLAGS_IS_ABSTRACT should have been accurate. + return False + # It looks like ABCMeta.__new__ has not finished running yet; we're + # probably in __init_subclass__. We'll look for abstractmethods manually. + for name, value in object.__dict__.items(): + if getattr(value, "__isabstractmethod__", False): + return True + for base in object.__bases__: + for name in getattr(base, "__abstractmethods__", ()): + value = getattr(object, name, None) + if getattr(value, "__isabstractmethod__", False): + return True + return False def getmembers(object, predicate=None): """Return all members of an object as (name, value) pairs sorted by name. @@ -2214,11 +2241,16 @@ def _signature_from_callable(obj, *, sigcls=sigcls) sig = _signature_get_partial(wrapped_sig, partialmethod, (None,)) - first_wrapped_param = tuple(wrapped_sig.parameters.values())[0] - new_params = (first_wrapped_param,) + tuple(sig.parameters.values()) - - return sig.replace(parameters=new_params) + if first_wrapped_param.kind is Parameter.VAR_POSITIONAL: + # First argument of the wrapped callable is `*args`, as in + # `partialmethod(lambda *args)`. + return sig + else: + sig_params = tuple(sig.parameters.values()) + assert first_wrapped_param is not sig_params[0] + new_params = (first_wrapped_param,) + sig_params + return sig.replace(parameters=new_params) if isfunction(obj) or _signature_is_functionlike(obj): # If it's a pure Python function, or an object that is duck type diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 20f33cb..70746f8 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1410,7 +1410,8 @@ class IPv4Interface(IPv4Address): if address_less is NotImplemented: return NotImplemented try: - return self.network < other.network + return (self.network < other.network or + self.network == other.network and address_less) except AttributeError: # We *do* allow addresses and interfaces to be sorted. The # unassociated address is considered less than all interfaces. @@ -2100,7 +2101,8 @@ class IPv6Interface(IPv6Address): if address_less is NotImplemented: return NotImplemented try: - return self.network < other.network + return (self.network < other.network or + self.network == other.network and address_less) except AttributeError: # We *do* allow addresses and interfaces to be sorted. The # unassociated address is considered less than all interfaces. diff --git a/Lib/lib2to3/fixes/fix_filter.py b/Lib/lib2to3/fixes/fix_filter.py index bb6718c..a7a5a15 100644 --- a/Lib/lib2to3/fixes/fix_filter.py +++ b/Lib/lib2to3/fixes/fix_filter.py @@ -15,7 +15,10 @@ Python 2.6 figure it out. # Local imports from .. import fixer_base -from ..fixer_util import Name, Call, ListComp, in_special_context +from ..pytree import Node +from ..pygram import python_symbols as syms +from ..fixer_util import Name, ArgList, ListComp, in_special_context + class FixFilter(fixer_base.ConditionalFix): BM_compatible = True @@ -34,16 +37,19 @@ class FixFilter(fixer_base.ConditionalFix): > ')' > + [extra_trailers=trailer*] > | power< 'filter' trailer< '(' arglist< none='None' ',' seq=any > ')' > + [extra_trailers=trailer*] > | power< 'filter' args=trailer< '(' [any] ')' > + [extra_trailers=trailer*] > """ @@ -53,23 +59,32 @@ class FixFilter(fixer_base.ConditionalFix): if self.should_skip(node): return + trailers = [] + if 'extra_trailers' in results: + for t in results['extra_trailers']: + trailers.append(t.clone()) + if "filter_lambda" in results: new = ListComp(results.get("fp").clone(), results.get("fp").clone(), results.get("it").clone(), results.get("xp").clone()) + new = Node(syms.power, [new] + trailers, prefix="") elif "none" in results: new = ListComp(Name("_f"), Name("_f"), results["seq"].clone(), Name("_f")) + new = Node(syms.power, [new] + trailers, prefix="") else: if in_special_context(node): return None - new = node.clone() + + args = results['args'].clone() + new = Node(syms.power, [Name("filter"), args], prefix="") + new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) new.prefix = "" - new = Call(Name("list"), [new]) new.prefix = node.prefix return new diff --git a/Lib/lib2to3/fixes/fix_map.py b/Lib/lib2to3/fixes/fix_map.py index 9f966fe..78cf81c 100644 --- a/Lib/lib2to3/fixes/fix_map.py +++ b/Lib/lib2to3/fixes/fix_map.py @@ -22,8 +22,10 @@ soon as the shortest argument is exhausted. # Local imports from ..pgen2 import token from .. import fixer_base -from ..fixer_util import Name, Call, ListComp, in_special_context +from ..fixer_util import Name, ArgList, Call, ListComp, in_special_context from ..pygram import python_symbols as syms +from ..pytree import Node + class FixMap(fixer_base.ConditionalFix): BM_compatible = True @@ -32,6 +34,7 @@ class FixMap(fixer_base.ConditionalFix): map_none=power< 'map' trailer< '(' arglist< 'None' ',' arg=any [','] > ')' > + [extra_trailers=trailer*] > | map_lambda=power< @@ -47,10 +50,12 @@ class FixMap(fixer_base.ConditionalFix): > ')' > + [extra_trailers=trailer*] > | power< - 'map' trailer< '(' [arglist=any] ')' > + 'map' args=trailer< '(' [any] ')' > + [extra_trailers=trailer*] > """ @@ -60,6 +65,11 @@ class FixMap(fixer_base.ConditionalFix): if self.should_skip(node): return + trailers = [] + if 'extra_trailers' in results: + for t in results['extra_trailers']: + trailers.append(t.clone()) + if node.parent.type == syms.simple_stmt: self.warning(node, "You should use a for loop here") new = node.clone() @@ -69,23 +79,32 @@ class FixMap(fixer_base.ConditionalFix): new = ListComp(results["xp"].clone(), results["fp"].clone(), results["it"].clone()) + new = Node(syms.power, [new] + trailers, prefix="") + else: if "map_none" in results: new = results["arg"].clone() + new.prefix = "" else: - if "arglist" in results: - args = results["arglist"] - if args.type == syms.arglist and \ - args.children[0].type == token.NAME and \ - args.children[0].value == "None": + if "args" in results: + args = results["args"] + if args.type == syms.trailer and \ + args.children[1].type == syms.arglist and \ + args.children[1].children[0].type == token.NAME and \ + args.children[1].children[0].value == "None": self.warning(node, "cannot convert map(None, ...) " "with multiple arguments because map() " "now truncates to the shortest sequence") return + + new = Node(syms.power, [Name("map"), args.clone()]) + new.prefix = "" + if in_special_context(node): return None - new = node.clone() + + new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) new.prefix = "" - new = Call(Name("list"), [new]) + new.prefix = node.prefix return new diff --git a/Lib/lib2to3/fixes/fix_zip.py b/Lib/lib2to3/fixes/fix_zip.py index 8f36a94..52c28df 100644 --- a/Lib/lib2to3/fixes/fix_zip.py +++ b/Lib/lib2to3/fixes/fix_zip.py @@ -9,13 +9,16 @@ iter(<>), list(<>), tuple(<>), sorted(<>), ...join(<>), or for V in <>:. # Local imports from .. import fixer_base -from ..fixer_util import Name, Call, in_special_context +from ..pytree import Node +from ..pygram import python_symbols as syms +from ..fixer_util import Name, ArgList, in_special_context + class FixZip(fixer_base.ConditionalFix): BM_compatible = True PATTERN = """ - power< 'zip' args=trailer< '(' [any] ')' > + power< 'zip' args=trailer< '(' [any] ')' > [trailers=trailer*] > """ @@ -28,8 +31,16 @@ class FixZip(fixer_base.ConditionalFix): if in_special_context(node): return None - new = node.clone() - new.prefix = "" - new = Call(Name("list"), [new]) + args = results['args'].clone() + args.prefix = "" + + trailers = [] + if 'trailers' in results: + trailers = [n.clone() for n in results['trailers']] + for n in trailers: + n.prefix = "" + + new = Node(syms.power, [Name("zip"), args], prefix="") + new = Node(syms.power, [Name("list"), ArgList([new])] + trailers) new.prefix = node.prefix return new diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index d14db60..45afc5f 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -54,16 +54,16 @@ Comment = r'#[^\r\n]*' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Binnumber = r'0[bB][01]*' -Hexnumber = r'0[xX][\da-fA-F]*[lL]?' -Octnumber = r'0[oO]?[0-7]*[lL]?' -Decnumber = r'[1-9]\d*[lL]?' +Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' +Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' +Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' +Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+' -Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) -Expfloat = r'\d+' + Exponent +Exponent = r'[eE][-+]?\d+(?:_\d+)*' +Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) +Expfloat = r'\d+(?:_\d+)*' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Tail end of ' string. @@ -74,10 +74,11 @@ Double = r'[^"\\]*(?:\\.[^"\\]*)*"' Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" # Tail end of """ string. Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' -Triple = group("[ubUB]?[rR]?'''", '[ubUB]?[rR]?"""') +_litprefix = r"(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?" +Triple = group(_litprefix + "'''", _litprefix + '"""') # Single-line ' or " string. -String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'", - r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"') +String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"') # Because of leftmost-then-longest match semantics, be sure to put the # longest operators first (e.g., if = came before ==, == would get @@ -95,9 +96,9 @@ PlainToken = group(Number, Funny, String, Name) Token = Ignore + PlainToken # First (or only) line of ' or " string. -ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + +ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), - r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + _litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) PseudoExtras = group(r'\\\r?\n', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) @@ -109,19 +110,26 @@ endprogs = {"'": re.compile(Single), '"': re.compile(Double), "r'''": single3prog, 'r"""': double3prog, "u'''": single3prog, 'u"""': double3prog, "b'''": single3prog, 'b"""': double3prog, + "f'''": single3prog, 'f"""': double3prog, "ur'''": single3prog, 'ur"""': double3prog, "br'''": single3prog, 'br"""': double3prog, + "rb'''": single3prog, 'rb"""': double3prog, "R'''": single3prog, 'R"""': double3prog, "U'''": single3prog, 'U"""': double3prog, "B'''": single3prog, 'B"""': double3prog, + "F'''": single3prog, 'F"""': double3prog, "uR'''": single3prog, 'uR"""': double3prog, "Ur'''": single3prog, 'Ur"""': double3prog, "UR'''": single3prog, 'UR"""': double3prog, "bR'''": single3prog, 'bR"""': double3prog, "Br'''": single3prog, 'Br"""': double3prog, "BR'''": single3prog, 'BR"""': double3prog, + "rB'''": single3prog, 'rB"""': double3prog, + "Rb'''": single3prog, 'Rb"""': double3prog, + "RB'''": single3prog, 'RB"""': double3prog, 'r': None, 'R': None, 'u': None, 'U': None, + 'f': None, 'F': None, 'b': None, 'B': None} triple_quoted = {} @@ -129,20 +137,26 @@ for t in ("'''", '"""', "r'''", 'r"""', "R'''", 'R"""', "u'''", 'u"""', "U'''", 'U"""', "b'''", 'b"""', "B'''", 'B"""', + "f'''", 'f"""', "F'''", 'F"""', "ur'''", 'ur"""', "Ur'''", 'Ur"""', "uR'''", 'uR"""', "UR'''", 'UR"""', "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""',): + "bR'''", 'bR"""', "BR'''", 'BR"""', + "rb'''", 'rb"""', "Rb'''", 'Rb"""', + "rB'''", 'rB"""', "RB'''", 'RB"""',): triple_quoted[t] = t single_quoted = {} for t in ("'", '"', "r'", 'r"', "R'", 'R"', "u'", 'u"', "U'", 'U"', "b'", 'b"', "B'", 'B"', + "f'", 'f"', "F'", 'F"', "ur'", 'ur"', "Ur'", 'Ur"', "uR'", 'uR"', "UR'", 'UR"', "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"', ): + "bR'", 'bR"', "BR'", 'BR"', + "rb'", 'rb"', "Rb'", 'Rb"', + "rB'", 'rB"', "RB'", 'RB"',): single_quoted[t] = t tabsize = 8 diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index cf31a54..0b9bee0 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -72,6 +72,28 @@ class TokenTests(unittest.TestCase): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 + def testUnderscoresInNumbers(self): + # Integers + x = 1_0 + x = 123_456_7_89 + x = 0xabc_123_4_5 + x = 0X_abc_123 + x = 0B11_01 + x = 0b_11_01 + x = 0o45_67 + x = 0O_45_67 + + # Floats + x = 3_1.4 + x = 03_1.4 + x = 3_1. + x = .3_1 + x = 3.1_4 + x = 0_3.1_4 + x = 3e1_4 + x = 3_1e+4_1 + x = 3_1E-4_1 + def testFloats(self): x = 3.14 x = 314. diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index b3f2680..3e1a255 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -2954,10 +2954,23 @@ class Test_filter(FixerTestCase): a = """x = [x for x in range(10) if x%2 == 0]""" self.check(b, a) - # XXX This (rare) case is not supported -## b = """x = filter(f, 'abc')[0]""" -## a = """x = list(filter(f, 'abc'))[0]""" -## self.check(b, a) + def test_filter_trailers(self): + b = """x = filter(None, 'abc')[0]""" + a = """x = [_f for _f in 'abc' if _f][0]""" + self.check(b, a) + + b = """x = len(filter(f, 'abc')[0])""" + a = """x = len(list(filter(f, 'abc'))[0])""" + self.check(b, a) + + b = """x = filter(lambda x: x%2 == 0, range(10))[0]""" + a = """x = [x for x in range(10) if x%2 == 0][0]""" + self.check(b, a) + + # Note the parens around x + b = """x = filter(lambda (x): x%2 == 0, range(10))[0]""" + a = """x = [x for x in range(10) if x%2 == 0][0]""" + self.check(b, a) def test_filter_nochange(self): a = """b.join(filter(f, 'abc'))""" @@ -3022,6 +3035,23 @@ class Test_map(FixerTestCase): a = """x = list(map( f, 'abc' ))""" self.check(b, a) + def test_map_trailers(self): + b = """x = map(f, 'abc')[0]""" + a = """x = list(map(f, 'abc'))[0]""" + self.check(b, a) + + b = """x = map(None, l)[0]""" + a = """x = list(l)[0]""" + self.check(b, a) + + b = """x = map(lambda x:x, l)[0]""" + a = """x = [x for x in l][0]""" + self.check(b, a) + + b = """x = map(f, 'abc')[0][1]""" + a = """x = list(map(f, 'abc'))[0][1]""" + self.check(b, a) + def test_trailing_comment(self): b = """x = map(f, 'abc') # foo""" a = """x = list(map(f, 'abc')) # foo""" @@ -3066,11 +3096,6 @@ class Test_map(FixerTestCase): """ self.warns(b, a, "You should use a for loop here") - # XXX This (rare) case is not supported -## b = """x = map(f, 'abc')[0]""" -## a = """x = list(map(f, 'abc'))[0]""" -## self.check(b, a) - def test_map_nochange(self): a = """b.join(map(f, 'abc'))""" self.unchanged(a) @@ -3130,6 +3155,10 @@ class Test_zip(FixerTestCase): super(Test_zip, self).check(b, a) def test_zip_basic(self): + b = """x = zip()""" + a = """x = list(zip())""" + self.check(b, a) + b = """x = zip(a, b, c)""" a = """x = list(zip(a, b, c))""" self.check(b, a) @@ -3138,6 +3167,15 @@ class Test_zip(FixerTestCase): a = """x = len(list(zip(a, b)))""" self.check(b, a) + def test_zip_trailers(self): + b = """x = zip(a, b, c)[0]""" + a = """x = list(zip(a, b, c))[0]""" + self.check(b, a) + + b = """x = zip(a, b, c)[0][1]""" + a = """x = list(zip(a, b, c))[0][1]""" + self.check(b, a) + def test_zip_nochange(self): a = """b.join(zip(a, b))""" self.unchanged(a) diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 9a969e8..3f7ab97 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -320,6 +320,7 @@ class TestVarAnnotations(GrammarTest): def test_6(self): self.validate("lst: List[int] = []") + class TestExcept(GrammarTest): def test_new(self): s = """ @@ -338,6 +339,27 @@ class TestExcept(GrammarTest): self.validate(s) +class TestStringLiterals(GrammarTest): + prefixes = ("'", '"', + "r'", 'r"', "R'", 'R"', + "u'", 'u"', "U'", 'U"', + "b'", 'b"', "B'", 'B"', + "f'", 'f"', "F'", 'F"', + "ur'", 'ur"', "Ur'", 'Ur"', + "uR'", 'uR"', "UR'", 'UR"', + "br'", 'br"', "Br'", 'Br"', + "bR'", 'bR"', "BR'", 'BR"', + "rb'", 'rb"', "Rb'", 'Rb"', + "rB'", 'rB"', "RB'", 'RB"',) + + def test_lit(self): + for pre in self.prefixes: + single = "{p}spamspamspam{s}".format(p=pre, s=pre[-1]) + self.validate(single) + triple = "{p}{s}{s}eggs{s}{s}{s}".format(p=pre, s=pre[-1]) + self.validate(triple) + + # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms class TestSetLiteral(GrammarTest): def test_1(self): diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 7d77973..2f934b3 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -815,16 +815,38 @@ class SysLogHandler(logging.Handler): if isinstance(address, str): self.unixsocket = True - self._connect_unixsocket(address) + # Syslog server may be unavailable during handler initialisation. + # C's openlog() function also ignores connection errors. + # Moreover, we ignore these errors while logging, so it not worse + # to ignore it also here. + try: + self._connect_unixsocket(address) + except OSError: + pass else: self.unixsocket = False if socktype is None: socktype = socket.SOCK_DGRAM - self.socket = socket.socket(socket.AF_INET, socktype) - if socktype == socket.SOCK_STREAM: - self.socket.connect(address) + host, port = address + ress = socket.getaddrinfo(host, port, 0, socktype) + if not ress: + raise OSError("getaddrinfo returns an empty list") + for res in ress: + af, socktype, proto, _, sa = res + err = sock = None + try: + sock = socket.socket(af, socktype, proto) + if socktype == socket.SOCK_STREAM: + sock.connect(sa) + break + except OSError as exc: + err = exc + if sock is not None: + sock.close() + if err is not None: + raise err + self.socket = sock self.socktype = socktype - self.formatter = None def _connect_unixsocket(self, address): use_socktype = self.socktype @@ -863,7 +885,7 @@ class SysLogHandler(logging.Handler): priority = self.priority_names[priority] return (facility << 3) | priority - def close (self): + def close(self): """ Closes the socket. """ diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index f2c179e..d5ce625 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -149,8 +149,15 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): util._close_stdin() - # ignoring SIGCHLD means no need to reap zombie processes - handler = signal.signal(signal.SIGCHLD, signal.SIG_IGN) + # ignoring SIGCHLD means no need to reap zombie processes; + # letting SIGINT through avoids KeyboardInterrupt tracebacks + handlers = { + signal.SIGCHLD: signal.SIG_IGN, + signal.SIGINT: signal.SIG_DFL, + } + old_handlers = {sig: signal.signal(sig, val) + for (sig, val) in handlers.items()} + with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \ selectors.DefaultSelector() as selector: _forkserver._forkserver_address = listener.getsockname() @@ -175,7 +182,7 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): code = 1 if os.fork() == 0: try: - _serve_one(s, listener, alive_r, handler) + _serve_one(s, listener, alive_r, old_handlers) except Exception: sys.excepthook(*sys.exc_info()) sys.stderr.flush() @@ -186,11 +193,12 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): if e.errno != errno.ECONNABORTED: raise -def _serve_one(s, listener, alive_r, handler): - # close unnecessary stuff and reset SIGCHLD handler +def _serve_one(s, listener, alive_r, handlers): + # close unnecessary stuff and reset signal handlers listener.close() os.close(alive_r) - signal.signal(signal.SIGCHLD, handler) + for sig, val in handlers.items(): + signal.signal(sig, val) # receive fds from parent process fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index ffdf426..a545f3c 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -118,7 +118,7 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, try: result = (True, func(*args, **kwds)) except Exception as e: - if wrap_exception: + if wrap_exception and func is not _helper_reraises_exception: e = ExceptionWithTraceback(e, e.__traceback__) result = (False, e) try: @@ -128,9 +128,15 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, util.debug("Possible encoding error while sending result: %s" % ( wrapped)) put((job, i, (False, wrapped))) + + task = job = result = func = args = kwds = None completed += 1 util.debug('worker exiting after %d tasks' % completed) +def _helper_reraises_exception(ex): + 'Pickle-able helper function for use by _guarded_task_generation.' + raise ex + # # Class representing a process pool # @@ -275,6 +281,17 @@ class Pool(object): return self._map_async(func, iterable, starmapstar, chunksize, callback, error_callback) + def _guarded_task_generation(self, result_job, func, iterable): + '''Provides a generator of tasks for imap and imap_unordered with + appropriate handling for iterables which throw exceptions during + iteration.''' + try: + i = -1 + for i, x in enumerate(iterable): + yield (result_job, i, func, (x,), {}) + except Exception as e: + yield (result_job, i+1, _helper_reraises_exception, (e,), {}) + def imap(self, func, iterable, chunksize=1): ''' Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. @@ -283,15 +300,23 @@ class Pool(object): raise ValueError("Pool not running") if chunksize == 1: result = IMapIterator(self._cache) - self._taskqueue.put((((result._job, i, func, (x,), {}) - for i, x in enumerate(iterable)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapIterator(self._cache) - self._taskqueue.put((((result._job, i, mapstar, (x,), {}) - for i, x in enumerate(task_batches)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) return (item for chunk in result for item in chunk) def imap_unordered(self, func, iterable, chunksize=1): @@ -302,15 +327,23 @@ class Pool(object): raise ValueError("Pool not running") if chunksize == 1: result = IMapUnorderedIterator(self._cache) - self._taskqueue.put((((result._job, i, func, (x,), {}) - for i, x in enumerate(iterable)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapUnorderedIterator(self._cache) - self._taskqueue.put((((result._job, i, mapstar, (x,), {}) - for i, x in enumerate(task_batches)), result._set_length)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) return (item for chunk in result for item in chunk) def apply_async(self, func, args=(), kwds={}, callback=None, @@ -321,7 +354,7 @@ class Pool(object): if self._state != RUN: raise ValueError("Pool not running") result = ApplyResult(self._cache, callback, error_callback) - self._taskqueue.put(([(result._job, None, func, args, kwds)], None)) + self._taskqueue.put(([(result._job, 0, func, args, kwds)], None)) return result def map_async(self, func, iterable, chunksize=None, callback=None, @@ -352,8 +385,14 @@ class Pool(object): task_batches = Pool._get_tasks(func, iterable, chunksize) result = MapResult(self._cache, chunksize, len(iterable), callback, error_callback=error_callback) - self._taskqueue.put((((result._job, i, mapper, (x,), {}) - for i, x in enumerate(task_batches)), None)) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapper, + task_batches), + None + ) + ) return result @staticmethod @@ -375,37 +414,32 @@ class Pool(object): for taskseq, set_length in iter(taskqueue.get, None): task = None - i = -1 try: - for i, task in enumerate(taskseq): + # iterating taskseq cannot fail + for task in taskseq: if thread._state: util.debug('task handler found thread._state != RUN') break try: put(task) except Exception as e: - job, ind = task[:2] + job, idx = task[:2] try: - cache[job]._set(ind, (False, e)) + cache[job]._set(idx, (False, e)) except KeyError: pass else: if set_length: util.debug('doing set_length()') - set_length(i+1) + idx = task[1] if task else -1 + set_length(idx + 1) continue break - except Exception as ex: - job, ind = task[:2] if task else (0, 0) - if job in cache: - cache[job]._set(ind + 1, (False, ex)) - if set_length: - util.debug('doing set_length()') - set_length(i+1) + finally: + task = taskseq = job = None else: util.debug('task handler got sentinel') - try: # tell result handler to finish when cache is empty util.debug('task handler sending sentinel to result handler') @@ -445,6 +479,7 @@ class Pool(object): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None while cache and thread._state != TERMINATE: try: @@ -461,6 +496,7 @@ class Pool(object): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None if hasattr(outqueue, '_reader'): util.debug('ensuring that outqueue is not full') diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index dda03dd..7f77837 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -221,8 +221,8 @@ class Queue(object): else: wacquire = None - try: - while 1: + while 1: + try: nacquire() try: if not buffer: @@ -249,21 +249,19 @@ class Queue(object): wrelease() except IndexError: pass - except Exception as e: - if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: - return - # Since this runs in a daemon thread the resources it uses - # may be become unusable while the process is cleaning up. - # We ignore errors which happen after the process has - # started to cleanup. - try: + except Exception as e: + if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: + return + # Since this runs in a daemon thread the resources it uses + # may be become unusable while the process is cleaning up. + # We ignore errors which happen after the process has + # started to cleanup. if is_exiting(): info('error in queue thread: %s', e) + return else: import traceback traceback.print_exc() - except Exception: - pass _sentinel = object() @@ -337,6 +335,7 @@ class SimpleQueue(object): def __setstate__(self, state): (self._reader, self._writer, self._rlock, self._wlock) = state + self._poll = self._reader.poll def get(self): with self._rlock: diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 1a2c0db..b490caa 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -241,20 +241,28 @@ def _run_finalizers(minpriority=None): return if minpriority is None: - f = lambda p : p[0][0] is not None + f = lambda p : p[0] is not None else: - f = lambda p : p[0][0] is not None and p[0][0] >= minpriority - - items = [x for x in list(_finalizer_registry.items()) if f(x)] - items.sort(reverse=True) - - for key, finalizer in items: - sub_debug('calling %s', finalizer) - try: - finalizer() - except Exception: - import traceback - traceback.print_exc() + f = lambda p : p[0] is not None and p[0] >= minpriority + + # Careful: _finalizer_registry may be mutated while this function + # is running (either by a GC run or by another thread). + + # list(_finalizer_registry) should be atomic, while + # list(_finalizer_registry.items()) is not. + keys = [key for key in list(_finalizer_registry) if f(key)] + keys.sort(reverse=True) + + for key in keys: + finalizer = _finalizer_registry.get(key) + # key may have been removed from the registry + if finalizer is not None: + sub_debug('calling %s', finalizer) + try: + finalizer() + except Exception: + import traceback + traceback.print_exc() if minpriority is None: _finalizer_registry.clear() @@ -386,7 +394,7 @@ def _close_stdin(): def spawnv_passfds(path, args, passfds): import _posixsubprocess - passfds = sorted(passfds) + passfds = tuple(sorted(map(int, passfds))) errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 9f34721..48b566d 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -187,19 +187,18 @@ class _WindowsFlavour(_Flavour): if strict: return self._ext_to_normal(_getfinalpathname(s)) else: + tail_parts = [] # End of the path after the first one not found while True: try: s = self._ext_to_normal(_getfinalpathname(s)) except FileNotFoundError: previous_s = s - s = os.path.dirname(s) + s, tail = os.path.split(s) + tail_parts.append(tail) if previous_s == s: return path else: - if previous_s is None: - return s - else: - return s + os.path.sep + os.path.basename(previous_s) + return os.path.join(s, *reversed(tail_parts)) # Means fallback on absolute return None @@ -330,12 +329,10 @@ class _PosixFlavour(_Flavour): try: target = accessor.readlink(newpath) except OSError as e: - if e.errno != EINVAL: - if strict: - raise - else: - return newpath - # Not a symlink + if e.errno != EINVAL and strict: + raise + # Not a symlink, or non-strict mode. We just leave the path + # untouched. path = newpath else: seen[newpath] = None # not resolved symlink @@ -1220,25 +1217,23 @@ class Path(PurePath): os.close(fd) def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ if self._closed: self._raise_closed() - if not parents: - try: - self._accessor.mkdir(self, mode) - except FileExistsError: - if not exist_ok or not self.is_dir(): - raise - else: - try: - self._accessor.mkdir(self, mode) - except FileExistsError: - if not exist_ok or not self.is_dir(): - raise - except OSError as e: - if e.errno != ENOENT or self.parent == self: - raise - self.parent.mkdir(parents=True) - self._accessor.mkdir(self, mode) + try: + self._accessor.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise def chmod(self, mode): """ diff --git a/Lib/poplib.py b/Lib/poplib.py index cae6950..6bcfa5c 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -288,9 +288,12 @@ class POP3: if sock is not None: try: sock.shutdown(socket.SHUT_RDWR) - except OSError as e: - # The server might already have closed the connection - if e.errno != errno.ENOTCONN: + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): raise finally: sock.close() diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 2caab63..015afb3 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Sat Mar 4 12:14:44 2017 +# Autogenerated by Sphinx on Sat Jun 17 04:32:54 2017 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -1300,7 +1300,7 @@ topics = {'assert': 'The "assert" statement\n' ' 2 1\n' ' >>> f(a=1, *(2,))\n' ' Traceback (most recent call last):\n' - ' File "", line 1, in ?\n' + ' File "", line 1, in \n' " TypeError: f() got multiple values for keyword argument 'a'\n" ' >>> f(1, *(2,))\n' ' 1 2\n' @@ -1669,7 +1669,7 @@ topics = {'assert': 'The "assert" statement\n' 'they\n' ' have equal *(key, value)* pairs. Equality comparison of the ' 'keys and\n' - ' elements enforces reflexivity.\n' + ' values enforces reflexivity.\n' '\n' ' Order comparisons ("<", ">", "<=", and ">=") raise ' '"TypeError".\n' @@ -1761,9 +1761,9 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'The operators "in" and "not in" test for membership. "x in ' 's"\n' - 'evaluates to true if *x* is a member of *s*, and false ' - 'otherwise. "x\n' - 'not in s" returns the negation of "x in s". All built-in ' + 'evaluates to "True" if *x* is a member of *s*, and "False" ' + 'otherwise.\n' + '"x not in s" returns the negation of "x in s". All built-in ' 'sequences\n' 'and set types support this as well as dictionary, for which ' '"in" tests\n' @@ -1774,30 +1774,32 @@ topics = {'assert': 'The "assert" statement\n' 'for e in\n' 'y)".\n' '\n' - 'For the string and bytes types, "x in y" is true if and only ' - 'if *x* is\n' - 'a substring of *y*. An equivalent test is "y.find(x) != ' - '-1". Empty\n' - 'strings are always considered to be a substring of any other ' - 'string,\n' - 'so """ in "abc"" will return "True".\n' + 'For the string and bytes types, "x in y" is "True" if and ' + 'only if *x*\n' + 'is a substring of *y*. An equivalent test is "y.find(x) != ' + '-1".\n' + 'Empty strings are always considered to be a substring of any ' + 'other\n' + 'string, so """ in "abc"" will return "True".\n' '\n' 'For user-defined classes which define the "__contains__()" ' 'method, "x\n' - 'in y" is true if and only if "y.__contains__(x)" is true.\n' + 'in y" returns "True" if "y.__contains__(x)" returns a true ' + 'value, and\n' + '"False" otherwise.\n' '\n' 'For user-defined classes which do not define "__contains__()" ' 'but do\n' - 'define "__iter__()", "x in y" is true if some value "z" with ' - '"x == z"\n' - 'is produced while iterating over "y". If an exception is ' + 'define "__iter__()", "x in y" is "True" if some value "z" ' + 'with "x ==\n' + 'z" is produced while iterating over "y". If an exception is ' 'raised\n' 'during the iteration, it is as if "in" raised that ' 'exception.\n' '\n' 'Lastly, the old-style iteration protocol is tried: if a class ' 'defines\n' - '"__getitem__()", "x in y" is true if and only if there is a ' + '"__getitem__()", "x in y" is "True" if and only if there is a ' 'non-\n' 'negative integer index *i* such that "x == y[i]", and all ' 'lower\n' @@ -2839,11 +2841,11 @@ topics = {'assert': 'The "assert" statement\n' ' Typical implementations create a new instance of the ' 'class by\n' ' invoking the superclass\'s "__new__()" method using\n' - ' "super(currentclass, cls).__new__(cls[, ...])" with ' - 'appropriate\n' - ' arguments and then modifying the newly-created instance ' - 'as\n' - ' necessary before returning it.\n' + ' "super().__new__(cls[, ...])" with appropriate arguments ' + 'and then\n' + ' modifying the newly-created instance as necessary before ' + 'returning\n' + ' it.\n' '\n' ' If "__new__()" returns an instance of *cls*, then the ' 'new\n' @@ -2878,7 +2880,7 @@ topics = {'assert': 'The "assert" statement\n' ' any, must explicitly call it to ensure proper ' 'initialization of the\n' ' base class part of the instance; for example:\n' - ' "BaseClass.__init__(self, [args...])".\n' + ' "super().__init__([args...])".\n' '\n' ' Because "__new__()" and "__init__()" work together in ' 'constructing\n' @@ -3027,8 +3029,8 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'object.__bytes__(self)\n' '\n' - ' Called by "bytes()" to compute a byte-string ' - 'representation of an\n' + ' Called by bytes to compute a byte-string representation ' + 'of an\n' ' object. This should return a "bytes" object.\n' '\n' 'object.__format__(self, format_spec)\n' @@ -3346,7 +3348,7 @@ topics = {'assert': 'The "assert" statement\n' ' >>> import mymodule\n' ' >>> mymodule.test()\n' ' Traceback (most recent call last):\n' - ' File "", line 1, in ?\n' + ' File "", line 1, in \n' ' File "./mymodule.py", line 4, in test\n' ' test2()\n' ' File "./mymodule.py", line 3, in test2\n' @@ -4010,16 +4012,6 @@ topics = {'assert': 'The "assert" statement\n' ' i = 42\n' ' f()\n' '\n' - 'There are several cases where Python statements are ' - 'illegal when used\n' - 'in conjunction with nested scopes that contain free ' - 'variables.\n' - '\n' - 'If a variable is referenced in an enclosing scope, it is ' - 'illegal to\n' - 'delete the name. An error will be reported at compile ' - 'time.\n' - '\n' 'The "eval()" and "exec()" functions do not have access ' 'to the full\n' 'environment for resolving names. Names may be resolved ' @@ -4323,6 +4315,13 @@ topics = {'assert': 'The "assert" statement\n' 'Builtins and restricted execution\n' '---------------------------------\n' '\n' + '**CPython implementation detail:** Users should not touch\n' + '"__builtins__"; it is strictly an implementation detail. ' + 'Users\n' + 'wanting to override values in the builtins namespace should ' + '"import"\n' + 'the "builtins" module and modify its attributes appropriately.\n' + '\n' 'The builtins namespace associated with the execution of a code ' 'block\n' 'is actually found by looking up the name "__builtins__" in its ' @@ -4335,16 +4334,7 @@ topics = {'assert': 'The "assert" statement\n' 'in any\n' 'other module, "__builtins__" is an alias for the dictionary of ' 'the\n' - '"builtins" module itself. "__builtins__" can be set to a ' - 'user-created\n' - 'dictionary to create a weak form of restricted execution.\n' - '\n' - '**CPython implementation detail:** Users should not touch\n' - '"__builtins__"; it is strictly an implementation detail. ' - 'Users\n' - 'wanting to override values in the builtins namespace should ' - '"import"\n' - 'the "builtins" module and modify its attributes appropriately.\n' + '"builtins" module itself.\n' '\n' '\n' 'Interaction with dynamic features\n' @@ -4360,14 +4350,6 @@ topics = {'assert': 'The "assert" statement\n' ' i = 42\n' ' f()\n' '\n' - 'There are several cases where Python statements are illegal ' - 'when used\n' - 'in conjunction with nested scopes that contain free variables.\n' - '\n' - 'If a variable is referenced in an enclosing scope, it is ' - 'illegal to\n' - 'delete the name. An error will be reported at compile time.\n' - '\n' 'The "eval()" and "exec()" functions do not have access to the ' 'full\n' 'environment for resolving names. Names may be resolved in the ' @@ -5521,8 +5503,8 @@ topics = {'assert': 'The "assert" statement\n' 'change\n' 'the meaning of the program.\n' '\n' - '**Programmer\'s note:** the "global" is a directive to the ' - 'parser. It\n' + '**Programmer\'s note:** "global" is a directive to the parser. ' + 'It\n' 'applies only to code parsed at the same time as the "global"\n' 'statement. In particular, a "global" statement contained in a ' 'string\n' @@ -6031,8 +6013,9 @@ topics = {'assert': 'The "assert" statement\n' '**************************\n' '\n' 'The operators "in" and "not in" test for membership. "x in s"\n' - 'evaluates to true if *x* is a member of *s*, and false otherwise. "x\n' - 'not in s" returns the negation of "x in s". All built-in sequences\n' + 'evaluates to "True" if *x* is a member of *s*, and "False" otherwise.\n' + '"x not in s" returns the negation of "x in s". All built-in ' + 'sequences\n' 'and set types support this as well as dictionary, for which "in" ' 'tests\n' 'whether the dictionary has a given key. For container types such as\n' @@ -6040,22 +6023,22 @@ topics = {'assert': 'The "assert" statement\n' 'expression "x in y" is equivalent to "any(x is e or x == e for e in\n' 'y)".\n' '\n' - 'For the string and bytes types, "x in y" is true if and only if *x* ' - 'is\n' - 'a substring of *y*. An equivalent test is "y.find(x) != -1". Empty\n' - 'strings are always considered to be a substring of any other string,\n' - 'so """ in "abc"" will return "True".\n' + 'For the string and bytes types, "x in y" is "True" if and only if *x*\n' + 'is a substring of *y*. An equivalent test is "y.find(x) != -1".\n' + 'Empty strings are always considered to be a substring of any other\n' + 'string, so """ in "abc"" will return "True".\n' '\n' 'For user-defined classes which define the "__contains__()" method, "x\n' - 'in y" is true if and only if "y.__contains__(x)" is true.\n' + 'in y" returns "True" if "y.__contains__(x)" returns a true value, and\n' + '"False" otherwise.\n' '\n' 'For user-defined classes which do not define "__contains__()" but do\n' - 'define "__iter__()", "x in y" is true if some value "z" with "x == z"\n' - 'is produced while iterating over "y". If an exception is raised\n' + 'define "__iter__()", "x in y" is "True" if some value "z" with "x ==\n' + 'z" is produced while iterating over "y". If an exception is raised\n' 'during the iteration, it is as if "in" raised that exception.\n' '\n' 'Lastly, the old-style iteration protocol is tried: if a class defines\n' - '"__getitem__()", "x in y" is true if and only if there is a non-\n' + '"__getitem__()", "x in y" is "True" if and only if there is a non-\n' 'negative integer index *i* such that "x == y[i]", and all lower\n' 'integer indices do not raise "IndexError" exception. (If any other\n' 'exception is raised, it is as if "in" raised that exception).\n' @@ -6305,6 +6288,12 @@ topics = {'assert': 'The "assert" statement\n' 'Builtins and restricted execution\n' '=================================\n' '\n' + '**CPython implementation detail:** Users should not touch\n' + '"__builtins__"; it is strictly an implementation detail. Users\n' + 'wanting to override values in the builtins namespace should ' + '"import"\n' + 'the "builtins" module and modify its attributes appropriately.\n' + '\n' 'The builtins namespace associated with the execution of a code ' 'block\n' 'is actually found by looking up the name "__builtins__" in its ' @@ -6317,15 +6306,7 @@ topics = {'assert': 'The "assert" statement\n' 'any\n' 'other module, "__builtins__" is an alias for the dictionary of ' 'the\n' - '"builtins" module itself. "__builtins__" can be set to a ' - 'user-created\n' - 'dictionary to create a weak form of restricted execution.\n' - '\n' - '**CPython implementation detail:** Users should not touch\n' - '"__builtins__"; it is strictly an implementation detail. Users\n' - 'wanting to override values in the builtins namespace should ' - '"import"\n' - 'the "builtins" module and modify its attributes appropriately.\n' + '"builtins" module itself.\n' '\n' '\n' 'Interaction with dynamic features\n' @@ -6341,14 +6322,6 @@ topics = {'assert': 'The "assert" statement\n' ' i = 42\n' ' f()\n' '\n' - 'There are several cases where Python statements are illegal when ' - 'used\n' - 'in conjunction with nested scopes that contain free variables.\n' - '\n' - 'If a variable is referenced in an enclosing scope, it is illegal ' - 'to\n' - 'delete the name. An error will be reported at compile time.\n' - '\n' 'The "eval()" and "exec()" functions do not have access to the ' 'full\n' 'environment for resolving names. Names may be resolved in the ' @@ -6980,7 +6953,7 @@ topics = {'assert': 'The "assert" statement\n' ' ...\n' ' Traceback (most recent call last):\n' ' File "", line 2, in \n' - ' ZeroDivisionError: int division or modulo by zero\n' + ' ZeroDivisionError: division by zero\n' '\n' ' The above exception was the direct cause of the following ' 'exception:\n' @@ -7002,7 +6975,7 @@ topics = {'assert': 'The "assert" statement\n' ' ...\n' ' Traceback (most recent call last):\n' ' File "", line 2, in \n' - ' ZeroDivisionError: int division or modulo by zero\n' + ' ZeroDivisionError: division by zero\n' '\n' ' During handling of the above exception, another exception ' 'occurred:\n' @@ -7011,10 +6984,30 @@ topics = {'assert': 'The "assert" statement\n' ' File "", line 4, in \n' ' RuntimeError: Something bad happened\n' '\n' + 'Exception chaining can be explicitly suppressed by specifying ' + '"None"\n' + 'in the "from" clause:\n' + '\n' + ' >>> try:\n' + ' ... print(1 / 0)\n' + ' ... except:\n' + ' ... raise RuntimeError("Something bad happened") from None\n' + ' ...\n' + ' Traceback (most recent call last):\n' + ' File "", line 4, in \n' + ' RuntimeError: Something bad happened\n' + '\n' 'Additional information on exceptions can be found in section\n' 'Exceptions, and information about handling exceptions is in ' 'section\n' - 'The try statement.\n', + 'The try statement.\n' + '\n' + 'Changed in version 3.3: "None" is now permitted as "Y" in "raise X\n' + 'from Y".\n' + '\n' + 'New in version 3.3: The "__suppress_context__" attribute to ' + 'suppress\n' + 'automatic display of the exception context.\n', 'return': 'The "return" statement\n' '**********************\n' '\n' @@ -7122,6 +7115,16 @@ topics = {'assert': 'The "assert" statement\n' ' returns zero is considered to be false in a Boolean ' 'context.\n' '\n' + ' **CPython implementation detail:** In CPython, the ' + 'length is\n' + ' required to be at most "sys.maxsize". If the length is ' + 'larger than\n' + ' "sys.maxsize" some features (such as "len()") may ' + 'raise\n' + ' "OverflowError". To prevent raising "OverflowError" by ' + 'truth value\n' + ' testing, an object must define a "__bool__()" method.\n' + '\n' 'object.__length_hint__(self)\n' '\n' ' Called to implement "operator.length_hint()". Should ' @@ -7497,11 +7500,11 @@ topics = {'assert': 'The "assert" statement\n' ' Typical implementations create a new instance of the ' 'class by\n' ' invoking the superclass\'s "__new__()" method using\n' - ' "super(currentclass, cls).__new__(cls[, ...])" with ' - 'appropriate\n' - ' arguments and then modifying the newly-created instance ' - 'as\n' - ' necessary before returning it.\n' + ' "super().__new__(cls[, ...])" with appropriate arguments ' + 'and then\n' + ' modifying the newly-created instance as necessary before ' + 'returning\n' + ' it.\n' '\n' ' If "__new__()" returns an instance of *cls*, then the ' 'new\n' @@ -7536,7 +7539,7 @@ topics = {'assert': 'The "assert" statement\n' ' any, must explicitly call it to ensure proper ' 'initialization of the\n' ' base class part of the instance; for example:\n' - ' "BaseClass.__init__(self, [args...])".\n' + ' "super().__init__([args...])".\n' '\n' ' Because "__new__()" and "__init__()" work together in ' 'constructing\n' @@ -7683,8 +7686,8 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'object.__bytes__(self)\n' '\n' - ' Called by "bytes()" to compute a byte-string ' - 'representation of an\n' + ' Called by bytes to compute a byte-string representation ' + 'of an\n' ' object. This should return a "bytes" object.\n' '\n' 'object.__format__(self, format_spec)\n' @@ -8740,6 +8743,15 @@ topics = {'assert': 'The "assert" statement\n' ' returns zero is considered to be false in a Boolean ' 'context.\n' '\n' + ' **CPython implementation detail:** In CPython, the length ' + 'is\n' + ' required to be at most "sys.maxsize". If the length is ' + 'larger than\n' + ' "sys.maxsize" some features (such as "len()") may raise\n' + ' "OverflowError". To prevent raising "OverflowError" by ' + 'truth value\n' + ' testing, an object must define a "__bool__()" method.\n' + '\n' 'object.__length_hint__(self)\n' '\n' ' Called to implement "operator.length_hint()". Should ' @@ -9564,13 +9576,13 @@ topics = {'assert': 'The "assert" statement\n' 'str.join(iterable)\n' '\n' ' Return a string which is the concatenation of the ' - 'strings in the\n' - ' *iterable* *iterable*. A "TypeError" will be raised if ' - 'there are\n' - ' any non-string values in *iterable*, including "bytes" ' - 'objects.\n' - ' The separator between elements is the string providing ' - 'this method.\n' + 'strings in\n' + ' *iterable*. A "TypeError" will be raised if there are ' + 'any non-\n' + ' string values in *iterable*, including "bytes" ' + 'objects. The\n' + ' separator between elements is the string providing this ' + 'method.\n' '\n' 'str.ljust(width[, fillchar])\n' '\n' @@ -10695,11 +10707,11 @@ topics = {'assert': 'The "assert" statement\n' '8-bit\n' ' bytes, represented by integers in the range 0 <= x < 256.\n' ' Bytes literals (like "b\'abc\'") and the built-in ' - 'function\n' - ' "bytes()" can be used to construct bytes objects. Also,\n' - ' bytes objects can be decoded to strings via the ' - '"decode()"\n' - ' method.\n' + '"bytes()"\n' + ' constructor can be used to create bytes objects. Also, ' + 'bytes\n' + ' objects can be decoded to strings via the "decode()" ' + 'method.\n' '\n' ' Mutable sequences\n' ' Mutable sequences can be changed after they are created. ' @@ -10725,7 +10737,7 @@ topics = {'assert': 'The "assert" statement\n' ' the built-in "bytearray()" constructor. Aside from being\n' ' mutable (and hence unhashable), byte arrays otherwise ' 'provide\n' - ' the same interface and functionality as immutable bytes\n' + ' the same interface and functionality as immutable "bytes"\n' ' objects.\n' '\n' ' The extension module "array" provides an additional example ' @@ -11998,7 +12010,7 @@ topics = {'assert': 'The "assert" statement\n' 'comparison operations. The "+" (concatenation) and "*" ' '(repetition)\n' 'operations have the same priority as the corresponding numeric\n' - 'operations.\n' + 'operations. [3]\n' '\n' '+----------------------------+----------------------------------+------------+\n' '| Operation | Result ' diff --git a/Lib/smtplib.py b/Lib/smtplib.py index f7c2c77..5e422b7 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -250,6 +250,7 @@ class SMTP: if host: (code, msg) = self.connect(host, port) if code != 220: + self.close() raise SMTPConnectError(code, msg) if local_hostname is not None: self.local_hostname = local_hostname diff --git a/Lib/sre_constants.py b/Lib/sre_constants.py index fc684ae..a6e8a1f 100644 --- a/Lib/sre_constants.py +++ b/Lib/sre_constants.py @@ -21,6 +21,17 @@ from _sre import MAXREPEAT, MAXGROUPS # should this really be here? class error(Exception): + """Exception raised for invalid regular expressions. + + Attributes: + + msg: The unformatted error message + pattern: The regular expression pattern + pos: The index in the pattern where compilation failed (may be None) + lineno: The line corresponding to pos (may be None) + colno: The column corresponding to pos (may be None) + """ + def __init__(self, msg, pattern=None, pos=None): self.msg = msg self.pattern = pattern diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index 6aa49c3..b505e11 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -404,7 +404,7 @@ def _escape(source, escape, state): pass raise source.error("bad escape %s" % escape, len(escape)) -def _parse_sub(source, state, verbose, nested=True): +def _parse_sub(source, state, verbose, nested): # parse an alternation: a|b|c items = [] @@ -412,7 +412,8 @@ def _parse_sub(source, state, verbose, nested=True): sourcematch = source.match start = source.tell() while True: - itemsappend(_parse(source, state, verbose)) + itemsappend(_parse(source, state, verbose, nested + 1, + not nested and not items)) if not sourcematch("|"): break @@ -454,10 +455,10 @@ def _parse_sub(source, state, verbose, nested=True): subpattern.append((BRANCH, (None, items))) return subpattern -def _parse_sub_cond(source, state, condgroup, verbose): - item_yes = _parse(source, state, verbose) +def _parse_sub_cond(source, state, condgroup, verbose, nested): + item_yes = _parse(source, state, verbose, nested + 1) if source.match("|"): - item_no = _parse(source, state, verbose) + item_no = _parse(source, state, verbose, nested + 1) if source.next == "|": raise source.error("conditional backref with more than two branches") else: @@ -466,7 +467,7 @@ def _parse_sub_cond(source, state, condgroup, verbose): subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no))) return subpattern -def _parse(source, state, verbose): +def _parse(source, state, verbose, nested, first=False): # parse a simple pattern subpattern = SubPattern(state) @@ -692,7 +693,7 @@ def _parse(source, state, verbose): lookbehindgroups = state.lookbehindgroups if lookbehindgroups is None: state.lookbehindgroups = state.groups - p = _parse_sub(source, state, verbose) + p = _parse_sub(source, state, verbose, nested + 1) if dir < 0: if lookbehindgroups is None: state.lookbehindgroups = None @@ -730,18 +731,19 @@ def _parse(source, state, verbose): state.checklookbehindgroup(condgroup, source) elif char in FLAGS or char == "-": # flags - pos = source.pos flags = _parse_flags(source, state, char) if flags is None: # global flags - if pos != 3: # "(?x" + if not first or subpattern: import warnings warnings.warn( - 'Flags not at the start of the expression %s%s' % ( + 'Flags not at the start of the expression %r%s' % ( source.string[:20], # truncate long regexes ' (truncated)' if len(source.string) > 20 else '', ), - DeprecationWarning, stacklevel=7 + DeprecationWarning, stacklevel=nested + 6 ) + if (state.flags & SRE_FLAG_VERBOSE) and not verbose: + raise Verbose continue add_flags, del_flags = flags group = None @@ -756,11 +758,11 @@ def _parse(source, state, verbose): except error as err: raise source.error(err.msg, len(name) + 1) from None if condgroup: - p = _parse_sub_cond(source, state, condgroup, verbose) + p = _parse_sub_cond(source, state, condgroup, verbose, nested + 1) else: sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and not (del_flags & SRE_FLAG_VERBOSE)) - p = _parse_sub(source, state, sub_verbose) + p = _parse_sub(source, state, sub_verbose, nested + 1) if not source.match(")"): raise source.error("missing ), unterminated subpattern", source.tell() - start) @@ -795,9 +797,6 @@ def _parse_flags(source, state, char): msg = "unknown flag" if char.isalpha() else "missing -, : or )" raise source.error(msg, len(char)) if char == ")": - if ((add_flags & SRE_FLAG_VERBOSE) and - not (state.flags & SRE_FLAG_VERBOSE)): - raise Verbose state.flags |= add_flags return None if add_flags & GLOBAL_FLAGS: @@ -853,7 +852,7 @@ def parse(str, flags=0, pattern=None): pattern.str = str try: - p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, False) + p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, 0) except Verbose: # the VERBOSE flag was switched on inside the pattern. to be # on the safe side, we'll parse the whole thing again... @@ -861,7 +860,7 @@ def parse(str, flags=0, pattern=None): pattern.flags = flags | SRE_FLAG_VERBOSE pattern.str = str source.seek(0) - p = _parse_sub(source, pattern, True, False) + p = _parse_sub(source, pattern, True, 0) p.pattern.flags = fix_flags(str, p.pattern.flags) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 822ddb4..b790cbd 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -776,19 +776,21 @@ class Popen(object): self.stdin.write(input) except BrokenPipeError: pass # communicate() must ignore broken pipe errors. - except OSError as e: - if e.errno == errno.EINVAL and self.poll() is not None: - # Issue #19612: On Windows, stdin.write() fails with EINVAL - # if the process already exited before the write + except OSError as exc: + if exc.errno == errno.EINVAL: + # bpo-19612, bpo-30418: On Windows, stdin.write() fails + # with EINVAL if the child process exited or if the child + # process is still running but closed the pipe. pass else: raise + try: self.stdin.close() except BrokenPipeError: pass # communicate() must ignore broken pipe errors. - except OSError as e: - if e.errno == errno.EINVAL and self.poll() is not None: + except OSError as exc: + if exc.errno == errno.EINVAL: pass else: raise @@ -986,7 +988,7 @@ class Popen(object): int(not close_fds), creationflags, env, - cwd, + os.fspath(cwd) if cwd is not None else None, startupinfo) finally: # Child is launched. Close the parent's copy of those pipe @@ -1237,8 +1239,12 @@ class Popen(object): # and pass it to fork_exec() if env is not None: - env_list = [os.fsencode(k) + b'=' + os.fsencode(v) - for k, v in env.items()] + env_list = [] + for k, v in env.items(): + k = os.fsencode(k) + if b'=' in k: + raise ValueError("illegal environment variable name") + env_list.append(k + b'=' + os.fsencode(v)) else: env_list = None # Use execv instead of execve. executable = os.fsencode(executable) @@ -1253,7 +1259,8 @@ class Popen(object): fds_to_keep.add(errpipe_write) self.pid = _posixsubprocess.fork_exec( args, executable_list, - close_fds, sorted(fds_to_keep), cwd, env_list, + close_fds, tuple(sorted(map(int, fds_to_keep))), + cwd, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index 46e0f56..bfb670c 100755 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -59,7 +59,7 @@ def main(): class NannyNag(Exception): """ - Raised by tokeneater() if detecting an ambiguous indent. + Raised by process_tokens() if detecting an ambiguous indent. Captured and handled in check(). """ def __init__(self, lineno, msg, line): diff --git a/Lib/test/185test.db b/Lib/test/185test.db deleted file mode 100644 index 14cb5e258bc0961fa22527dcb5d947a6a0fd5480..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI!F$%&k6vpwdEl39?E*_+VxVZETo'. For example, to run all the tests except for the gui tests, give the option '-uall,-gui'. + +--matchfile filters tests using a text file, one pattern per line. +Pattern examples: + +- test method: test_stat_attributes +- test class: FileTests +- test identifier: test_os.FileTests.test_stat_attributes """ @@ -189,8 +196,12 @@ def _create_parser(): help='single step through a set of tests.' + more_details) group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', + dest='match_tests', action='append', help='match test cases and methods with glob pattern PAT') + group.add_argument('--matchfile', metavar='FILENAME', + dest='match_filename', + help='similar to --match but get patterns from a ' + 'text file, one pattern per line') group.add_argument('-G', '--failfast', action='store_true', help='fail as soon as a test fails (only with -v or -W)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', @@ -239,6 +250,9 @@ def _create_parser(): group.add_argument('--list-tests', action='store_true', help="only write the name of tests that will be run, " "don't execute them") + group.add_argument('--list-cases', action='store_true', + help='only write the name of test cases that will be run' + ' , don\'t execute them') group.add_argument('-P', '--pgo', dest='pgo', action='store_true', help='enable Profile Guided Optimization training') @@ -343,5 +357,19 @@ def _parse_args(args, **kwargs): ns.use_resources.append(r) if ns.random_seed is not None: ns.randomize = True + if ns.verbose: + ns.header = True + if ns.huntrleaks and ns.verbose3: + ns.verbose3 = False + print("WARNING: Disable --verbose3 because it's incompatible with " + "--huntrleaks: see http://bugs.python.org/issue27103", + file=sys.stderr) + if ns.match_filename: + if ns.match_tests is None: + ns.match_tests = [] + filename = os.path.join(support.SAVEDCWD, ns.match_filename) + with open(filename) as fp: + for line in fp: + ns.match_tests.append(line.strip()) return ns diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f0effc9..527de17 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -10,9 +10,10 @@ import sysconfig import tempfile import textwrap import time +import unittest from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( - findtests, runtest, + findtests, runtest, get_abs_module, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, INTERRUPTED, CHILD_ERROR, PROGRESS_MIN_TIME, format_test_result) @@ -28,7 +29,13 @@ except ImportError: # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. if sysconfig.is_python_build(): - TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') + TEMPDIR = sysconfig.get_config_var('abs_builddir') + if TEMPDIR is None: + # bpo-30284: On Windows, only srcdir is available. Using abs_builddir + # mostly matters on UNIX when building Python out of the source tree, + # especially when the source tree is read only. + TEMPDIR = sysconfig.get_config_var('srcdir') + TEMPDIR = os.path.join(TEMPDIR, 'build') else: TEMPDIR = tempfile.gettempdir() TEMPDIR = os.path.abspath(TEMPDIR) @@ -107,7 +114,7 @@ class Regrtest: self.test_times.append((test_time, test)) if ok == PASSED: self.good.append(test) - elif ok == FAILED: + elif ok in (FAILED, CHILD_ERROR): self.bad.append(test) elif ok == ENV_CHANGED: self.environment_changed.append(test) @@ -116,22 +123,28 @@ class Regrtest: elif ok == RESOURCE_DENIED: self.skipped.append(test) self.resource_denieds.append(test) + elif ok != INTERRUPTED: + raise ValueError("invalid test result: %r" % ok) def display_progress(self, test_index, test): if self.ns.quiet: return + + # "[ 51/405/1] test_tcl passed" + line = f"{test_index:{self.test_count_width}}{self.test_count}" if self.bad and not self.ns.pgo: - fmt = "{time} [{test_index:{count_width}}{test_count}/{nbad}] {test_name}" - else: - fmt = "{time} [{test_index:{count_width}}{test_count}] {test_name}" + line = f"{line}/{len(self.bad)}" + line = f"[{line}] {test}" + + # add the system load prefix: "load avg: 1.80 " + if hasattr(os, 'getloadavg'): + load_avg_1min = os.getloadavg()[0] + line = f"load avg: {load_avg_1min:.2f} {line}" + + # add the timestamp prefix: "0:01:05 " test_time = time.monotonic() - self.start_time test_time = datetime.timedelta(seconds=int(test_time)) - line = fmt.format(count_width=self.test_count_width, - test_index=test_index, - test_count=self.test_count, - nbad=len(self.bad), - test_name=test, - time=test_time) + line = f"{test_time} {line}" print(line, flush=True) def parse_args(self, kwargs): @@ -179,19 +192,14 @@ class Regrtest: self.tests = [] # regex to match 'test_builtin' in line: # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' - regex = (r'^(?:[0-9]+:[0-9]+:[0-9]+ *)?' - r'(?:\[[0-9/ ]+\] *)?' - r'(test_[a-zA-Z0-9_]+)') - regex = re.compile(regex) + regex = re.compile(r'\btest_[a-zA-Z0-9_]+\b') with open(os.path.join(support.SAVEDCWD, self.ns.fromfile)) as fp: for line in fp: + line = line.split('#', 1)[0] line = line.strip() - if line.startswith('#'): - continue - match = regex.match(line) - if match is None: - continue - self.tests.append(match.group(1)) + match = regex.search(line) + if match is not None: + self.tests.append(match.group()) removepy(self.tests) @@ -241,6 +249,29 @@ class Regrtest: for name in self.selected: print(name) + def _list_cases(self, suite): + for test in suite: + if isinstance(test, unittest.loader._FailedTest): + continue + if isinstance(test, unittest.TestSuite): + self._list_cases(test) + elif isinstance(test, unittest.TestCase): + print(test.id()) + + def list_cases(self): + for test in self.selected: + abstest = get_abs_module(self.ns, test) + try: + suite = unittest.defaultTestLoader.loadTestsFromName(abstest) + self._list_cases(suite) + except unittest.SkipTest: + self.skipped.append(test) + + if self.skipped: + print(file=sys.stderr) + print(count(len(self.skipped), "test"), "skipped:", file=sys.stderr) + printlist(self.skipped, file=sys.stderr) + def rerun_failed_tests(self): self.ns.verbose = True self.ns.failfast = False @@ -381,23 +412,28 @@ class Regrtest: if self.bad: return + def display_header(self): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("==", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== hash algorithm:", sys.hash_info.algorithm, + "64bit" if sys.maxsize > 2**32 else "32bit") + print("== cwd:", os.getcwd()) + cpu_count = os.cpu_count() + if cpu_count: + print("== CPU count:", cpu_count) + print("== encodings: locale=%s, FS=%s" + % (locale.getpreferredencoding(False), + sys.getfilesystemencoding())) + print("Testing with flags:", sys.flags) + def run_tests(self): # For a partial run, we do not need to clutter the output. - if (self.ns.verbose - or self.ns.header - or not (self.ns.pgo or self.ns.quiet or self.ns.single - or self.tests or self.ns.args)): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== cwd:", os.getcwd()) - print("== encodings: locale=%s, FS=%s" - % (locale.getpreferredencoding(False), - sys.getfilesystemencoding())) - print("Testing with flags:", sys.flags) + if (self.ns.header + or not(self.ns.pgo or self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): + self.display_header() if self.ns.randomize: print("Using random seed", self.ns.random_seed) @@ -487,6 +523,10 @@ class Regrtest: self.list_tests() sys.exit(0) + if self.ns.list_cases: + self.list_cases() + sys.exit(0) + self.run_tests() self.display_result() @@ -513,7 +553,7 @@ def count(n, word): return "%d %ss" % (n, word) -def printlist(x, width=70, indent=4): +def printlist(x, width=70, indent=4, file=None): """Print the elements of iterable x to stdout. Optional arg width (default 70) is the maximum line length. @@ -524,7 +564,8 @@ def printlist(x, width=70, indent=4): blanks = ' ' * indent # Print the sorted list: 'x' may be a '--random' list or a set() print(textwrap.fill(' '.join(str(elt) for elt in sorted(x)), width, - initial_indent=blanks, subsequent_indent=blanks)) + initial_indent=blanks, subsequent_indent=blanks), + file=file) def main(tests=None, **kwargs): diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 3b3d245..c69fc0f 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -84,7 +84,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks): indirect_test() alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) - print('.', end='', flush=True) + print('.', end='', file=sys.stderr, flush=True) if i >= nwarmup: rc_deltas[i] = rc_after - rc_before alloc_deltas[i] = alloc_after - alloc_before @@ -143,9 +143,14 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue + abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] + abs_classes = filter(isabstract, abs_classes) + if 'typing' in sys.modules: + t = sys.modules['typing'] + # These classes require special treatment because they do not appear + # in direct subclasses of collections.abc classes + abs_classes = list(abs_classes) + [t.ChainMap, t.Counter, t.DefaultDict] + for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: obj._abc_registry = abcs.get(obj, WeakSet()).copy() obj._abc_cache.clear() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index ba0df0a..fda4ca1 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -71,6 +71,14 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): return stdtests + sorted(tests) +def get_abs_module(ns, test): + if test.startswith('test.') or ns.testdir: + return test + else: + # Always import it from the test package + return 'test.' + test + + def runtest(ns, test): """Run a single test. @@ -141,11 +149,7 @@ def runtest_inner(ns, test, display_failure=True): test_time = 0.0 refleak = False # True if the test leaked references. try: - if test.startswith('test.') or ns.testdir: - abstest = test - else: - # Always import it from the test package - abstest = 'test.' + test + abstest = get_abs_module(ns, test) clear_caches() with saved_test_environment(test, ns.verbose, ns.quiet, pgo=ns.pgo) as environment: start_time = time.time() diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 9604c16..779ff01 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -41,7 +41,7 @@ def run_test_in_subprocess(testname, ns): slaveargs = json.dumps(slaveargs) cmd = [sys.executable, *support.args_from_interpreter_flags(), - '-X', 'faulthandler', + '-u', # Unbuffered stdout and stderr '-m', 'test.regrtest', '--slaveargs', slaveargs] if ns.pgo: @@ -124,13 +124,13 @@ class MultiprocessThread(threading.Thread): finally: self.current_test = None - stdout, _, result = stdout.strip().rpartition("\n") if retcode != 0: result = (CHILD_ERROR, "Exit code %s" % retcode) self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - return True + return False + stdout, _, result = stdout.strip().rpartition("\n") if not result: self.output.put((None, None, None, None)) return True @@ -203,6 +203,8 @@ def run_tests_multiprocess(regrtest): and test_time >= PROGRESS_MIN_TIME and not regrtest.ns.pgo): text += ' (%.0f sec)' % test_time + elif ok == CHILD_ERROR: + text = '%s (%s)' % (text, test_time) running = get_running(workers) if running and not regrtest.ns.pgo: text += ' -- running: %s' % ', '.join(running) @@ -216,9 +218,6 @@ def run_tests_multiprocess(regrtest): if result[0] == INTERRUPTED: raise KeyboardInterrupt - if result[0] == CHILD_ERROR: - msg = "Child error on {}: {}".format(test, result[1]) - raise Exception(msg) test_index += 1 except KeyboardInterrupt: regrtest.interrupted = True diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 96ad3af..8309f26 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -279,7 +279,6 @@ class saved_test_environment: if not self.quiet and not self.pgo: print(f"Warning -- {name} was modified by {self.testname}", file=sys.stderr, flush=True) - if self.verbose > 1: - print(f" Before: {original}\n After: {current} ", - file=sys.stderr, flush=True) + print(f" Before: {original}\n After: {current} ", + file=sys.stderr, flush=True) return False diff --git a/Lib/test/sndhdrdata/README b/Lib/test/sndhdrdata/README index 8a17c00..b2cb664 100644 --- a/Lib/test/sndhdrdata/README +++ b/Lib/test/sndhdrdata/README @@ -3,10 +3,3 @@ following commands: dd if=/dev/zero of=sndhdr.raw bs=20 count=1 sox -s -2 -c 2 -r 44100 sndhdr.raw sndhdr. - -Sound file samples used by Lib/test/test_sndhdr.py and generated using the -following commands: - - dd if=/dev/zero of=sndhdr.raw bs=20 count=1 - sox -s -2 -c 2 -r 44100 sndhdr.raw sndhdr. - diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 15d8fc8..be926ea 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1915,9 +1915,15 @@ def run_unittest(*classes): def case_pred(test): if match_tests is None: return True - for name in test.id().split("."): - if fnmatch.fnmatchcase(name, match_tests): + test_id = test.id() + + for match_test in match_tests: + if fnmatch.fnmatchcase(test_id, match_test): return True + + for name in test_id.split("."): + if fnmatch.fnmatchcase(name, match_test): + return True return False _filter_suite(suite, case_pred) _run_suite(suite) @@ -2105,12 +2111,15 @@ def swap_attr(obj, attr, new_val): restoring the old value at the end of the block. If `attr` doesn't exist on `obj`, it will be created and then deleted at the end of the block. + + The old value (or None if it doesn't exist) will be assigned to the + target of the "as" clause, if there is one. """ if hasattr(obj, attr): real_val = getattr(obj, attr) setattr(obj, attr, new_val) try: - yield + yield real_val finally: setattr(obj, attr, real_val) else: @@ -2118,7 +2127,8 @@ def swap_attr(obj, attr, new_val): try: yield finally: - delattr(obj, attr) + if hasattr(obj, attr): + delattr(obj, attr) @contextlib.contextmanager def swap_item(obj, item, new_val): @@ -2132,12 +2142,15 @@ def swap_item(obj, item, new_val): restoring the old value at the end of the block. If `item` doesn't exist on `obj`, it will be created and then deleted at the end of the block. + + The old value (or None if it doesn't exist) will be assigned to the + target of the "as" clause, if there is one. """ if item in obj: real_val = obj[item] obj[item] = new_val try: - yield + yield real_val finally: obj[item] = real_val else: @@ -2145,7 +2158,8 @@ def swap_item(obj, item, new_val): try: yield finally: - del obj[item] + if item in obj: + del obj[item] def strip_python_stderr(stderr): """Strip the stderr of a Python process from potential debug output @@ -2433,6 +2447,7 @@ class SuppressCrashReport: (0, self.old_value[1])) except (ValueError, OSError): pass + if sys.platform == 'darwin': # Check if the 'Crash Reporter' on OSX was configured # in 'Developer' mode and warn that it will get triggered @@ -2440,10 +2455,14 @@ class SuppressCrashReport: # # This assumes that this context manager is used in tests # that might trigger the next manager. - value = subprocess.Popen(['/usr/bin/defaults', 'read', - 'com.apple.CrashReporter', 'DialogType'], - stdout=subprocess.PIPE).communicate()[0] - if value.strip() == b'developer': + cmd = ['/usr/bin/defaults', 'read', + 'com.apple.CrashReporter', 'DialogType'] + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + with proc: + stdout = proc.communicate()[0] + if stdout.strip() == b'developer': print("this test triggers the Crash Reporter, " "that is intentional", end='', flush=True) @@ -2581,3 +2600,19 @@ def setswitchinterval(interval): if _is_android_emulator: interval = minimum_interval return sys.setswitchinterval(interval) + + +@contextlib.contextmanager +def disable_faulthandler(): + # use sys.__stderr__ instead of sys.stderr, since regrtest replaces + # sys.stderr with a StringIO which has no file descriptor when a test + # is run with -W/--verbose3. + fd = sys.__stderr__.fileno() + + is_enabled = faulthandler.is_enabled() + try: + faulthandler.disable() + yield + finally: + if is_enabled: + faulthandler.enable(file=fd, all_threads=True) diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index e1765f0..4bc8382 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -404,5 +404,17 @@ class TestABC(unittest.TestCase): self.assertEqual(B.counter, 1) +class TestABCWithInitSubclass(unittest.TestCase): + def test_works_with_init_subclass(self): + saved_kwargs = {} + class ReceivesClassKwargs: + def __init_subclass__(cls, **kwargs): + super().__init_subclass__() + saved_kwargs.update(kwargs) + class Receiver(ReceivesClassKwargs, abc.ABC, x=1, y=2, z=3): + pass + self.assertEqual(saved_kwargs, dict(x=1, y=2, z=3)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 1f8967c..d67f919 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -14,14 +14,6 @@ import warnings import array from array import _array_reconstructor as array_reconstructor -try: - # Try to determine availability of long long independently - # of the array module under test - struct.calcsize('@q') - have_long_long = True -except struct.error: - have_long_long = False - sizeof_wchar = array.array('u').itemsize @@ -32,9 +24,7 @@ class ArraySubclassWithKwargs(array.array): def __init__(self, typecode, newarg=None): array.array.__init__(self) -typecodes = "ubBhHiIlLfd" -if have_long_long: - typecodes += 'qQ' +typecodes = 'ubBhHiIlLfdqQ' class MiscTest(unittest.TestCase): @@ -1240,7 +1230,26 @@ class NumberTest(BaseTest): b = array.array(self.typecode, a) self.assertEqual(a, b) -class SignedNumberTest(NumberTest): +class IntegerNumberTest(NumberTest): + def test_type_error(self): + a = array.array(self.typecode) + a.append(42) + with self.assertRaises(TypeError): + a.append(42.0) + with self.assertRaises(TypeError): + a[0] = 42.0 + +class Intable: + def __init__(self, num): + self._num = num + def __int__(self): + return self._num + def __sub__(self, other): + return Intable(int(self) - int(other)) + def __add__(self, other): + return Intable(int(self) + int(other)) + +class SignedNumberTest(IntegerNumberTest): example = [-1, 0, 1, 42, 0x7f] smallerexample = [-1, 0, 1, 42, 0x7e] biggerexample = [-1, 0, 1, 43, 0x7f] @@ -1251,8 +1260,9 @@ class SignedNumberTest(NumberTest): lower = -1 * int(pow(2, a.itemsize * 8 - 1)) upper = int(pow(2, a.itemsize * 8 - 1)) - 1 self.check_overflow(lower, upper) + self.check_overflow(Intable(lower), Intable(upper)) -class UnsignedNumberTest(NumberTest): +class UnsignedNumberTest(IntegerNumberTest): example = [0, 1, 17, 23, 42, 0xff] smallerexample = [0, 1, 17, 23, 42, 0xfe] biggerexample = [0, 1, 17, 23, 43, 0xff] @@ -1263,6 +1273,7 @@ class UnsignedNumberTest(NumberTest): lower = 0 upper = int(pow(2, a.itemsize * 8)) - 1 self.check_overflow(lower, upper) + self.check_overflow(Intable(lower), Intable(upper)) def test_bytes_extend(self): s = bytes(self.example) @@ -1314,12 +1325,10 @@ class UnsignedLongTest(UnsignedNumberTest, unittest.TestCase): typecode = 'L' minitemsize = 4 -@unittest.skipIf(not have_long_long, 'need long long support') class LongLongTest(SignedNumberTest, unittest.TestCase): typecode = 'q' minitemsize = 8 -@unittest.skipIf(not have_long_long, 'need long long support') class UnsignedLongLongTest(UnsignedNumberTest, unittest.TestCase): typecode = 'Q' minitemsize = 8 diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 802763b..492a84a 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2194,8 +2194,10 @@ else: def test_get_event_loop_new_process(self): async def main(): pool = concurrent.futures.ProcessPoolExecutor() - return await self.loop.run_in_executor( + result = await self.loop.run_in_executor( pool, _test_get_event_loop_new_process__sub_proc) + pool.shutdown() + return result self.unpatch_get_running_loop() diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 99336f8..5d4b2d2 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -319,6 +319,14 @@ class BaseFutureTests: self.assertFalse(m_log.error.called) @mock.patch('asyncio.base_events.logger') + def test_tb_logger_not_called_after_cancel(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_exception(Exception()) + fut.cancel() + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') def test_tb_logger_result_unretrieved(self, m_log): fut = self._new_future(loop=self.loop) fut.set_result(42) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 152948c..c85e8b1 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -176,6 +176,28 @@ class LockTests(test_utils.TestCase): self.assertTrue(tb.cancelled()) self.assertTrue(tc.done()) + def test_finished_waiter_cancelled(self): + lock = asyncio.Lock(loop=self.loop) + + ta = asyncio.Task(lock.acquire(), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertTrue(lock.locked()) + + tb = asyncio.Task(lock.acquire(), loop=self.loop) + test_utils.run_briefly(self.loop) + self.assertEqual(len(lock._waiters), 1) + + # Create a second waiter, wake up the first, and cancel it. + # Without the fix, the second was not woken up. + tc = asyncio.Task(lock.acquire(), loop=self.loop) + lock.release() + tb.cancel() + test_utils.run_briefly(self.loop) + + self.assertTrue(lock.locked()) + self.assertTrue(ta.done()) + self.assertTrue(tb.cancelled()) + def test_release_not_acquired(self): lock = asyncio.Lock(loop=self.loop) diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 59ff0f6..bcd236e 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -42,6 +42,7 @@ class SslProtoHandshakeTests(test_utils.TestCase): sslpipe.do_handshake.side_effect = mock_handshake with mock.patch('asyncio.sslproto._SSLPipe', return_value=sslpipe): ssl_proto.connection_made(transport) + return transport def test_cancel_handshake(self): # Python issue #23197: cancelling a handshake must not raise an @@ -95,5 +96,31 @@ class SslProtoHandshakeTests(test_utils.TestCase): test_utils.run_briefly(self.loop) self.assertIsInstance(waiter.exception(), ConnectionAbortedError) + def test_close_during_handshake(self): + # bpo-29743 Closing transport during handshake process leaks socket + waiter = asyncio.Future(loop=self.loop) + ssl_proto = self.ssl_protocol(waiter) + + def do_handshake(callback): + return [] + + transport = self.connection_made(ssl_proto) + test_utils.run_briefly(self.loop) + + ssl_proto._app_transport.close() + self.assertTrue(transport.abort.called) + + def test_get_extra_info_on_closed_connection(self): + waiter = asyncio.Future(loop=self.loop) + ssl_proto = self.ssl_protocol(waiter) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + default = object() + self.assertIs(ssl_proto._get_extra_info('socket', default), default) + self.connection_made(ssl_proto) + self.assertIsNotNone(ssl_proto._get_extra_info('socket')) + ssl_proto.connection_lost(None) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 4f05319..195a1ed 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -588,6 +588,24 @@ class BaseTaskTests: self.assertFalse(t._must_cancel) # White-box test. self.assertFalse(t.cancel()) + def test_cancel_at_end(self): + """coroutine end right after task is cancelled""" + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + @asyncio.coroutine + def task(): + t.cancel() + self.assertTrue(t._must_cancel) # White-box test. + return 12 + + t = self.new_task(loop, task()) + self.assertRaises( + asyncio.CancelledError, loop.run_until_complete, t) + self.assertTrue(t.done()) + self.assertFalse(t._must_cancel) # White-box test. + self.assertFalse(t.cancel()) + def test_stop_while_run_in_complete(self): def gen(): @@ -1847,6 +1865,25 @@ class BaseTaskTests: }) mock_handler.reset_mock() + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_not_called_after_cancel(self, m_log): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + @asyncio.coroutine + def coro(): + raise TypeError + + @asyncio.coroutine + def runner(): + task = self.new_task(loop, coro()) + yield from asyncio.sleep(0.05, loop=loop) + task.cancel() + task = None + + loop.run_until_complete(runner()) + self.assertFalse(m_log.error.called) + @mock.patch('asyncio.coroutines.logger') def test_coroutine_never_yielded(self, m_log): with set_coroutine_debug(True): diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index d05462b..dc2f716 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -502,7 +502,7 @@ class BaseClient(BaseTestHandler): class BaseTestAPI: def tearDown(self): - asyncore.close_all() + asyncore.close_all(ignore_all=True) def loop_waiting_for_flag(self, instance, timeout=5): timeout = float(timeout) / 100 @@ -755,50 +755,50 @@ class BaseTestAPI: def test_set_reuse_addr(self): if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: self.skipTest("Not applicable to AF_UNIX sockets.") - sock = socket.socket(self.family) - try: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except OSError: - unittest.skip("SO_REUSEADDR not supported on this platform") - else: - # if SO_REUSEADDR succeeded for sock we expect asyncore - # to do the same - s = asyncore.dispatcher(socket.socket(self.family)) - self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR)) - s.socket.close() - s.create_socket(self.family) - s.set_reuse_addr() - self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR)) - finally: - sock.close() + + with socket.socket(self.family) as sock: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except OSError: + unittest.skip("SO_REUSEADDR not supported on this platform") + else: + # if SO_REUSEADDR succeeded for sock we expect asyncore + # to do the same + s = asyncore.dispatcher(socket.socket(self.family)) + self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + s.socket.close() + s.create_socket(self.family) + s.set_reuse_addr() + self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) @unittest.skipUnless(threading, 'Threading required for this test.') @support.reap_threads def test_quick_connect(self): # see: http://bugs.python.org/issue10340 - if self.family in (socket.AF_INET, getattr(socket, "AF_INET6", object())): - server = BaseServer(self.family, self.addr) - t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, - count=500)) - t.start() - def cleanup(): - t.join(timeout=TIMEOUT) - if t.is_alive(): - self.fail("join() timed out") - self.addCleanup(cleanup) - - s = socket.socket(self.family, socket.SOCK_STREAM) - s.settimeout(.2) - s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, - struct.pack('ii', 1, 0)) - try: - s.connect(server.address) - except OSError: - pass - finally: - s.close() + if self.family not in (socket.AF_INET, getattr(socket, "AF_INET6", object())): + self.skipTest("test specific to AF_INET and AF_INET6") + + server = BaseServer(self.family, self.addr) + # run the thread 500 ms: the socket should be connected in 200 ms + t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, + count=5)) + t.start() + try: + with socket.socket(self.family, socket.SOCK_STREAM) as s: + s.settimeout(.2) + s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, + struct.pack('ii', 1, 0)) + + try: + s.connect(server.address) + except OSError: + pass + finally: + t.join(timeout=TIMEOUT) + if t.is_alive(): + self.fail("join() timed out") class TestAPI_UseIPv4Sockets(BaseTestAPI): family = socket.AF_INET diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 172bd25..1d0b018 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -23,6 +23,9 @@ def raise1(): def raise2(): raise SystemError +def exit(): + raise SystemExit + class GeneralTest(unittest.TestCase): @@ -76,6 +79,13 @@ class GeneralTest(unittest.TestCase): self.assertRaises(ZeroDivisionError, atexit._run_exitfuncs) self.assertIn("ZeroDivisionError", self.stream.getvalue()) + def test_exit(self): + # be sure a SystemExit is handled properly + atexit.register(exit) + + self.assertRaises(SystemExit, atexit._run_exitfuncs) + self.assertEqual(self.stream.getvalue(), '') + def test_print_tracebacks(self): # Issue #18776: the tracebacks should be printed when errors occur. def f(): @@ -143,6 +153,7 @@ class GeneralTest(unittest.TestCase): self.assertEqual(l, [5]) +@support.cpython_only class SubinterpreterTest(unittest.TestCase): def test_callbacks_leak(self): diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index a103a7d..cd82fa6 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -507,6 +507,11 @@ class BaseBytesTest: a = b % (b'seventy-nine', 79) self.assertEqual(a, b'seventy-nine / 100 = 79%') self.assertIs(type(a), self.type2test) + # issue 29714 + b = self.type2test(b'hello,\x00%b!') + b = b % b'world' + self.assertEqual(b, b'hello,\x00world!') + self.assertIs(type(b), self.type2test) def test_imod(self): b = self.type2test(b'hello, %b!') @@ -519,6 +524,11 @@ class BaseBytesTest: b %= (b'seventy-nine', 79) self.assertEqual(b, b'seventy-nine / 100 = 79%') self.assertIs(type(b), self.type2test) + # issue 29714 + b = self.type2test(b'hello,\x00%b!') + b %= b'world' + self.assertEqual(b, b'hello,\x00world!') + self.assertIs(type(b), self.type2test) def test_rmod(self): with self.assertRaises(TypeError): diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index e2b8e0f..2e8819b 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,4 +1,10 @@ +import datetime import unittest +from test.support import cpython_only +try: + import _testcapi +except ImportError: + _testcapi = None # The test cases here cover several paths through the function calling # code. They depend on the METH_XXX flag that is used to define a C @@ -122,5 +128,175 @@ class CFunctionCalls(unittest.TestCase): self.assertRaises(TypeError, [].count, x=2, y=2) +def pyfunc(arg1, arg2): + return [arg1, arg2] + + +def pyfunc_noarg(): + return "noarg" + + +class PythonClass: + def method(self, arg1, arg2): + return [arg1, arg2] + + def method_noarg(self): + return "noarg" + + @classmethod + def class_method(cls): + return "classmethod" + + @staticmethod + def static_method(): + return "staticmethod" + + +PYTHON_INSTANCE = PythonClass() + + +IGNORE_RESULT = object() + + +@cpython_only +class FastCallTests(unittest.TestCase): + # Test calls with positional arguments + CALLS_POSARGS = ( + # (func, args: tuple, result) + + # Python function with 2 arguments + (pyfunc, (1, 2), [1, 2]), + + # Python function without argument + (pyfunc_noarg, (), "noarg"), + + # Python class methods + (PythonClass.class_method, (), "classmethod"), + (PythonClass.static_method, (), "staticmethod"), + + # Python instance methods + (PYTHON_INSTANCE.method, (1, 2), [1, 2]), + (PYTHON_INSTANCE.method_noarg, (), "noarg"), + (PYTHON_INSTANCE.class_method, (), "classmethod"), + (PYTHON_INSTANCE.static_method, (), "staticmethod"), + + # C function: METH_NOARGS + (globals, (), IGNORE_RESULT), + + # C function: METH_O + (id, ("hello",), IGNORE_RESULT), + + # C function: METH_VARARGS + (dir, (1,), IGNORE_RESULT), + + # C function: METH_VARARGS | METH_KEYWORDS + (min, (5, 9), 5), + + # C function: METH_FASTCALL + (divmod, (1000, 33), (30, 10)), + + # C type static method: METH_FASTCALL | METH_CLASS + (int.from_bytes, (b'\x01\x00', 'little'), 1), + + # bpo-30524: Test that calling a C type static method with no argument + # doesn't crash (ignore the result): METH_FASTCALL | METH_CLASS + (datetime.datetime.now, (), IGNORE_RESULT), + ) + + # Test calls with positional and keyword arguments + CALLS_KWARGS = ( + # (func, args: tuple, kwargs: dict, result) + + # Python function with 2 arguments + (pyfunc, (1,), {'arg2': 2}, [1, 2]), + (pyfunc, (), {'arg1': 1, 'arg2': 2}, [1, 2]), + + # Python instance methods + (PYTHON_INSTANCE.method, (1,), {'arg2': 2}, [1, 2]), + (PYTHON_INSTANCE.method, (), {'arg1': 1, 'arg2': 2}, [1, 2]), + + # C function: METH_VARARGS | METH_KEYWORDS + (max, ([],), {'default': 9}, 9), + + # C type static method: METH_FASTCALL | METH_CLASS + (int.from_bytes, (b'\x01\x00',), {'byteorder': 'little'}, 1), + (int.from_bytes, (), {'bytes': b'\x01\x00', 'byteorder': 'little'}, 1), + ) + + def check_result(self, result, expected): + if expected is IGNORE_RESULT: + return + self.assertEqual(result, expected) + + def test_fastcall(self): + # Test _PyObject_FastCall() + + for func, args, expected in self.CALLS_POSARGS: + with self.subTest(func=func, args=args): + result = _testcapi.pyobject_fastcall(func, args) + self.check_result(result, expected) + + if not args: + # args=NULL, nargs=0 + result = _testcapi.pyobject_fastcall(func, None) + self.check_result(result, expected) + + def test_fastcall_dict(self): + # Test _PyObject_FastCallDict() + + for func, args, expected in self.CALLS_POSARGS: + with self.subTest(func=func, args=args): + # kwargs=NULL + result = _testcapi.pyobject_fastcalldict(func, args, None) + self.check_result(result, expected) + + # kwargs={} + result = _testcapi.pyobject_fastcalldict(func, args, {}) + self.check_result(result, expected) + + if not args: + # args=NULL, nargs=0, kwargs=NULL + result = _testcapi.pyobject_fastcalldict(func, None, None) + self.check_result(result, expected) + + # args=NULL, nargs=0, kwargs={} + result = _testcapi.pyobject_fastcalldict(func, None, {}) + self.check_result(result, expected) + + for func, args, kwargs, expected in self.CALLS_KWARGS: + with self.subTest(func=func, args=args, kwargs=kwargs): + result = _testcapi.pyobject_fastcalldict(func, args, kwargs) + self.check_result(result, expected) + + def test_fastcall_keywords(self): + # Test _PyObject_FastCallKeywords() + + for func, args, expected in self.CALLS_POSARGS: + with self.subTest(func=func, args=args): + # kwnames=NULL + result = _testcapi.pyobject_fastcallkeywords(func, args, None) + self.check_result(result, expected) + + # kwnames=() + result = _testcapi.pyobject_fastcallkeywords(func, args, ()) + self.check_result(result, expected) + + if not args: + # kwnames=NULL + result = _testcapi.pyobject_fastcallkeywords(func, None, None) + self.check_result(result, expected) + + # kwnames=() + result = _testcapi.pyobject_fastcallkeywords(func, None, ()) + self.check_result(result, expected) + + for func, args, kwargs, expected in self.CALLS_KWARGS: + with self.subTest(func=func, args=args, kwargs=kwargs): + kwnames = tuple(kwargs.keys()) + args = args + tuple(kwargs.values()) + result = _testcapi.pyobject_fastcallkeywords(func, args, kwnames) + self.check_result(result, expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 6c3625d..216851c 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -98,7 +98,7 @@ class CAPITest(unittest.TestCase): def __len__(self): return 1 self.assertRaises(TypeError, _posixsubprocess.fork_exec, - 1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) + 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) # Issue #15736: overflow in _PySequence_BytesToCharpArray() class Z(object): def __len__(self): @@ -106,7 +106,7 @@ class CAPITest(unittest.TestCase): def __getitem__(self, i): return b'x' self.assertRaises(MemoryError, _posixsubprocess.fork_exec, - 1,Z(),3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) + 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') def test_subprocess_fork_exec(self): @@ -116,7 +116,7 @@ class CAPITest(unittest.TestCase): # Issue #15738: crash in subprocess_fork_exec() self.assertRaises(TypeError, _posixsubprocess.fork_exec, - Z(),[b'1'],3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) + Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17) @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") @@ -490,9 +490,8 @@ class SkipitemTest(unittest.TestCase): # test the format unit when not skipped format = c + "i" try: - # (note: the format string must be bytes!) _testcapi.parse_tuple_and_keywords(tuple_1, dict_b, - format.encode("ascii"), keywords) + format, keywords) when_not_skipped = False except SystemError as e: s = "argument 1 (impossible)" @@ -504,7 +503,7 @@ class SkipitemTest(unittest.TestCase): optional_format = "|" + format try: _testcapi.parse_tuple_and_keywords(empty_tuple, dict_b, - optional_format.encode("ascii"), keywords) + optional_format, keywords) when_skipped = False except SystemError as e: s = "impossible: '{}'".format(format) @@ -517,40 +516,64 @@ class SkipitemTest(unittest.TestCase): self.assertIs(when_skipped, when_not_skipped, message) def test_parse_tuple_and_keywords(self): - # parse_tuple_and_keywords error handling tests + # Test handling errors in the parse_tuple_and_keywords helper itself self.assertRaises(TypeError, _testcapi.parse_tuple_and_keywords, (), {}, 42, []) self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, b'', 42) + (), {}, '', 42) self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, b'', [''] * 42) + (), {}, '', [''] * 42) self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, - (), {}, b'', [42]) + (), {}, '', [42]) + + def test_bad_use(self): + # Test handling invalid format and keywords in + # PyArg_ParseTupleAndKeywords() + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '||O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1, 2), {}, '|O|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1}, '$$O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1, 'b': 2}, '$O$O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1}, '$|O', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {'a': 1, 'b': 2}, '$O|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '|O', ['a', 'b']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (1,), {}, '|OO', ['a']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {}, '|$O', ['']) + self.assertRaises(SystemError, _testcapi.parse_tuple_and_keywords, + (), {}, '|OO', ['a', '']) def test_positional_only(self): parse = _testcapi.parse_tuple_and_keywords - parse((1, 2, 3), {}, b'OOO', ['', '', 'a']) - parse((1, 2), {'a': 3}, b'OOO', ['', '', 'a']) + parse((1, 2, 3), {}, 'OOO', ['', '', 'a']) + parse((1, 2), {'a': 3}, 'OOO', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes at least 2 positional arguments \(1 given\)'): - parse((1,), {'a': 3}, b'OOO', ['', '', 'a']) - parse((1,), {}, b'O|OO', ['', '', 'a']) + parse((1,), {'a': 3}, 'OOO', ['', '', 'a']) + parse((1,), {}, 'O|OO', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes at least 1 positional arguments \(0 given\)'): - parse((), {}, b'O|OO', ['', '', 'a']) - parse((1, 2), {'a': 3}, b'OO$O', ['', '', 'a']) + parse((), {}, 'O|OO', ['', '', 'a']) + parse((1, 2), {'a': 3}, 'OO$O', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes exactly 2 positional arguments \(1 given\)'): - parse((1,), {'a': 3}, b'OO$O', ['', '', 'a']) - parse((1,), {}, b'O|O$O', ['', '', 'a']) + parse((1,), {'a': 3}, 'OO$O', ['', '', 'a']) + parse((1,), {}, 'O|O$O', ['', '', 'a']) with self.assertRaisesRegex(TypeError, r'Function takes at least 1 positional arguments \(0 given\)'): - parse((), {}, b'O|O$O', ['', '', 'a']) + parse((), {}, 'O|O$O', ['', '', 'a']) with self.assertRaisesRegex(SystemError, r'Empty parameter name after \$'): - parse((1,), {}, b'O|$OO', ['', '', 'a']) + parse((1,), {}, 'O|$OO', ['', '', 'a']) with self.assertRaisesRegex(SystemError, 'Empty keyword'): - parse((1,), {}, b'O|OO', ['', 'a', '']) + parse((1,), {}, 'O|OO', ['', 'a', '']) @unittest.skipUnless(threading, 'Threading required for this test.') diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 4d554a3..ecc01f2 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -568,5 +568,32 @@ class ClassTests(unittest.TestCase): a = A(hash(A.f)^(-1)) hash(a.f) + def testSetattrWrapperNameIntern(self): + # Issue #25794: __setattr__ should intern the attribute name + class A: + pass + + def add(self, other): + return 'summa' + + name = str(b'__add__', 'ascii') # shouldn't be optimized + self.assertIsNot(name, '__add__') # not interned + type.__setattr__(A, name, add) + self.assertEqual(A() + 1, 'summa') + + name2 = str(b'__add__', 'ascii') + self.assertIsNot(name2, '__add__') + self.assertIsNot(name2, name) + type.__delattr__(A, name2) + with self.assertRaises(TypeError): + A() + 1 + + def testSetattrNonStringName(self): + class A: + pass + + with self.assertRaises(TypeError): + type.__setattr__(A, b'x', None) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 7975ea0..9f9df9d 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -103,9 +103,11 @@ consts: ('None',) """ import sys +import threading import unittest import weakref -from test.support import run_doctest, run_unittest, cpython_only +from test.support import (run_doctest, run_unittest, cpython_only, + check_impl_detail) def consts(t): @@ -212,11 +214,106 @@ class CodeWeakRefTest(unittest.TestCase): self.assertTrue(self.called) +if check_impl_detail(cpython=True): + import ctypes + py = ctypes.pythonapi + freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) + + RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex + RequestCodeExtraIndex.argtypes = (freefunc,) + RequestCodeExtraIndex.restype = ctypes.c_ssize_t + + SetExtra = py._PyCode_SetExtra + SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) + SetExtra.restype = ctypes.c_int + + GetExtra = py._PyCode_GetExtra + GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, + ctypes.POINTER(ctypes.c_voidp)) + GetExtra.restype = ctypes.c_int + + LAST_FREED = None + def myfree(ptr): + global LAST_FREED + LAST_FREED = ptr + + FREE_FUNC = freefunc(myfree) + FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC) + + class CoExtra(unittest.TestCase): + def get_func(self): + # Defining a function causes the containing function to have a + # reference to the code object. We need the code objects to go + # away, so we eval a lambda. + return eval('lambda:42') + + def test_get_non_code(self): + f = self.get_func() + + self.assertRaises(SystemError, SetExtra, 42, FREE_INDEX, + ctypes.c_voidp(100)) + self.assertRaises(SystemError, GetExtra, 42, FREE_INDEX, + ctypes.c_voidp(100)) + + def test_bad_index(self): + f = self.get_func() + self.assertRaises(SystemError, SetExtra, f.__code__, + FREE_INDEX+100, ctypes.c_voidp(100)) + self.assertEqual(GetExtra(f.__code__, FREE_INDEX+100, + ctypes.c_voidp(100)), 0) + + def test_free_called(self): + # Verify that the provided free function gets invoked + # when the code object is cleaned up. + f = self.get_func() + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(100)) + del f + self.assertEqual(LAST_FREED, 100) + + def test_get_set(self): + # Test basic get/set round tripping. + f = self.get_func() + + extra = ctypes.c_voidp() + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(200)) + # reset should free... + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(300)) + self.assertEqual(LAST_FREED, 200) + + extra = ctypes.c_voidp() + GetExtra(f.__code__, FREE_INDEX, extra) + self.assertEqual(extra.value, 300) + del f + + def test_free_different_thread(self): + # Freeing a code object on a different thread then + # where the co_extra was set should be safe. + f = self.get_func() + class ThreadTest(threading.Thread): + def __init__(self, f, test): + super().__init__() + self.f = f + self.test = test + def run(self): + del self.f + self.test.assertEqual(LAST_FREED, 500) + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) + tt = ThreadTest(f, self) + del f + tt.start() + tt.join() + self.assertEqual(LAST_FREED, 500) + def test_main(verbose=None): from test import test_code run_doctest(test_code, verbose) - run_unittest(CodeTest, CodeConstsTest, CodeWeakRefTest) - + tests = [CodeTest, CodeConstsTest, CodeWeakRefTest] + if check_impl_detail(cpython=True): + tests.append(CoExtra) + run_unittest(*tests) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py index 3bdf7d0..2a45071 100644 --- a/Lib/test/test_codecencodings_cn.py +++ b/Lib/test/test_codecencodings_cn.py @@ -48,6 +48,12 @@ class Test_GB18030(multibytecodec_support.TestBase, unittest.TestCase): (b"abc\x84\x32\x80\x80def", "replace", 'abc\ufffd2\ufffd\ufffddef'), (b"abc\x81\x30\x81\x30def", "strict", 'abc\x80def'), (b"abc\x86\x30\x81\x30def", "replace", 'abc\ufffd0\ufffd0def'), + # issue29990 + (b"\xff\x30\x81\x30", "strict", None), + (b"\x81\x30\xff\x30", "strict", None), + (b"abc\x81\x39\xff\x39\xc1\xc4", "replace", "abc\ufffd\x39\ufffd\x39\u804a"), + (b"abc\xab\x36\xff\x30def", "replace", 'abc\ufffd\x36\ufffd\x30def'), + (b"abc\xbf\x38\xff\x32\xc1\xc4", "ignore", "abc\x38\x32\u804a"), ) has_iso10646 = True @@ -80,6 +86,10 @@ class Test_HZ(multibytecodec_support.TestBase, unittest.TestCase): (b'ab~{\x81\x81\x41\x44~}cd', 'replace', 'ab\uFFFD\uFFFD\u804Acd'), (b'ab~{\x41\x44~}cd', 'replace', 'ab\u804Acd'), (b"ab~{\x79\x79\x41\x44~}cd", "replace", "ab\ufffd\ufffd\u804acd"), + # issue 30003 + ('ab~cd', 'strict', b'ab~~cd'), # escape ~ + (b'~{Dc~~:C~}', 'strict', None), # ~~ only in ASCII mode + (b'~{Dc~\n:C~}', 'strict', None), # ~\n only in ASCII mode ) if __name__ == "__main__": diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 87454cc..47f7562 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1310,20 +1310,29 @@ class TestCollectionABCs(ABCTestCase): class CustomEqualObject: def __eq__(self, other): return False - class CustomSequence(list): - def __contains__(self, value): - return Sequence.__contains__(self, value) + class CustomSequence(Sequence): + def __init__(self, seq): + self._seq = seq + def __getitem__(self, index): + return self._seq[index] + def __len__(self): + return len(self._seq) nan = float('nan') obj = CustomEqualObject() + seq = CustomSequence([nan, obj, nan]) containers = [ - CustomSequence([nan, obj]), + seq, ItemsView({1: nan, 2: obj}), ValuesView({1: nan, 2: obj}) ] for container in containers: for elem in container: self.assertIn(elem, container) + self.assertEqual(seq.index(nan), 0) + self.assertEqual(seq.index(obj), 1) + self.assertEqual(seq.count(nan), 2) + self.assertEqual(seq.count(obj), 1) def assertSameSet(self, s1, s2): # coerce both to a real set then check equality diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index c04c804..2301f75 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -44,6 +44,16 @@ class TestAbstractContextManager(unittest.TestCase): self.assertTrue(issubclass(DefaultEnter, AbstractContextManager)) + class NoEnter(ManagerFromScratch): + __enter__ = None + + self.assertFalse(issubclass(NoEnter, AbstractContextManager)) + + class NoExit(ManagerFromScratch): + __exit__ = None + + self.assertFalse(issubclass(NoExit, AbstractContextManager)) + class ContextManagerTestCase(unittest.TestCase): @@ -152,6 +162,29 @@ def woohoo(): else: self.fail('StopIteration was suppressed') + def test_contextmanager_do_not_unchain_non_stopiteration_exceptions(self): + @contextmanager + def test_issue29692(): + try: + yield + except Exception as exc: + raise RuntimeError('issue29692:Chained') from exc + try: + with test_issue29692(): + raise ZeroDivisionError + except Exception as ex: + self.assertIs(type(ex), RuntimeError) + self.assertEqual(ex.args[0], 'issue29692:Chained') + self.assertIsInstance(ex.__cause__, ZeroDivisionError) + + try: + with test_issue29692(): + raise StopIteration('issue29692:Unchained') + except Exception as ex: + self.assertIs(type(ex), StopIteration) + self.assertEqual(ex.args[0], 'issue29692:Unchained') + self.assertIsNone(ex.__cause__) + def _create_contextmanager_attribs(self): def attribs(**kw): def decorate(func): diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index b4c7b5b..2b79a17 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1103,6 +1103,21 @@ class CoroutineTest(unittest.TestCase): "coroutine is being awaited already"): waiter(coro).send(None) + def test_await_16(self): + # See https://bugs.python.org/issue29600 for details. + + async def f(): + return ValueError() + + async def g(): + try: + raise KeyError + except: + return await f() + + _, result = run_async(g()) + self.assertIsNone(result.__context__) + def test_with_1(self): class Manager: def __init__(self, name): @@ -2102,6 +2117,7 @@ class SysSetCoroWrapperTest(unittest.TestCase): sys.set_coroutine_wrapper(None) +@support.cpython_only class CAPITest(unittest.TestCase): def test_tp_await_1(self): diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 832bb9c..8013f37 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1085,6 +1085,91 @@ class DictTest(unittest.TestCase): support.check_free_after_iterating(self, lambda d: iter(d.values()), dict) support.check_free_after_iterating(self, lambda d: iter(d.items()), dict) + def test_equal_operator_modifying_operand(self): + # test fix for seg fault reported in issue 27945 part 3. + class X(): + def __del__(self): + dict_b.clear() + + def __eq__(self, other): + dict_a.clear() + return True + + def __hash__(self): + return 13 + + dict_a = {X(): 0} + dict_b = {X(): X()} + self.assertTrue(dict_a == dict_b) + + def test_fromkeys_operator_modifying_dict_operand(self): + # test fix for seg fault reported in issue 27945 part 4a. + class X(int): + def __hash__(self): + return 13 + + def __eq__(self, other): + if len(d) > 1: + d.clear() + return False + + d = {} # this is required to exist so that d can be constructed! + d = {X(1): 1, X(2): 2} + try: + dict.fromkeys(d) # shouldn't crash + except RuntimeError: # implementation defined + pass + + def test_fromkeys_operator_modifying_set_operand(self): + # test fix for seg fault reported in issue 27945 part 4b. + class X(int): + def __hash__(self): + return 13 + + def __eq__(self, other): + if len(d) > 1: + d.clear() + return False + + d = {} # this is required to exist so that d can be constructed! + d = {X(1), X(2)} + try: + dict.fromkeys(d) # shouldn't crash + except RuntimeError: # implementation defined + pass + + def test_dictitems_contains_use_after_free(self): + class X: + def __eq__(self, other): + d.clear() + return NotImplemented + + d = {0: set()} + (0, X()) in d.items() + + def test_init_use_after_free(self): + class X: + def __hash__(self): + pair[:] = [] + return 13 + + pair = [X(), 123] + dict([pair]) + + def test_oob_indexing_dictiter_iternextitem(self): + class X(int): + def __del__(self): + d.clear() + + d = {i: X(i) for i in range(8)} + + def iter_and_mutate(): + for result in d.items(): + if result[0] == 2: + d[2] = None # free d[2] --> X(2).__del__ was called + + self.assertRaises(RuntimeError, iter_and_mutate) + class CAPITest(unittest.TestCase): diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index 7c8877f..c4f1829 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -162,6 +162,13 @@ class TestGeneratorBase: g.flatten(msg) self.assertEqual(s.getvalue(), self.typ(expected)) + def test_compat32_max_line_length_does_not_fold_when_none(self): + msg = self.msgmaker(self.typ(self.refold_long_expected[0])) + s = self.ioclass() + g = self.genclass(s, policy=policy.compat32.clone(max_line_length=None)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[0])) + class TestGenerator(TestGeneratorBase, TestEmailBase): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 4837922..960fc0f 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1,5 +1,6 @@ # Python test set -- part 5, built-in exceptions +import copy import os import sys import unittest @@ -1119,6 +1120,25 @@ class ImportErrorTests(unittest.TestCase): exc = ImportError(arg) self.assertEqual(str(arg), str(exc)) + def test_copy_pickle(self): + for kwargs in (dict(), + dict(name='somename'), + dict(path='somepath'), + dict(name='somename', path='somepath')): + orig = ImportError('test', **kwargs) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + exc = pickle.loads(pickle.dumps(orig, proto)) + self.assertEqual(exc.args, ('test',)) + self.assertEqual(exc.msg, 'test') + self.assertEqual(exc.name, orig.name) + self.assertEqual(exc.path, orig.path) + for c in copy.copy, copy.deepcopy: + exc = c(orig) + self.assertEqual(exc.args, ('test',)) + self.assertEqual(exc.msg, 'test') + self.assertEqual(exc.name, orig.name) + self.assertEqual(exc.path, orig.path) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index bdd8d1a..20fe1e2 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -755,6 +755,46 @@ class FaultHandlerTests(unittest.TestCase): 3, name) + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') + def test_raise_nonfatal_exception(self): + # These exceptions are not strictly errors. Letting + # faulthandler display the traceback when they are + # raised is likely to result in noise. However, they + # may still terminate the process if there is no + # handler installed for them (which there typically + # is, e.g. for debug messages). + for exc in ( + 0x00000000, + 0x34567890, + 0x40000000, + 0x40001000, + 0x70000000, + 0x7FFFFFFF, + ): + output, exitcode = self.get_output(f""" + import faulthandler + faulthandler.enable() + faulthandler._raise_exception(0x{exc:x}) + """ + ) + self.assertEqual(output, []) + # On Windows older than 7 SP1, the actual exception code has + # bit 29 cleared. + self.assertIn(exitcode, + (exc, exc & ~0x10000000)) + + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') + def test_disable_windows_exc_handler(self): + code = dedent(""" + import faulthandler + faulthandler.enable() + faulthandler.disable() + code = faulthandler._EXCEPTION_ACCESS_VIOLATION + faulthandler._raise_exception(code) + """) + output, exitcode = self.get_output(code) + self.assertEqual(output, []) + self.assertEqual(exitcode, 0xC0000005) if __name__ == "__main__": diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 3da210a..57a0265 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -10,7 +10,7 @@ from weakref import proxy from functools import wraps from test.support import (TESTFN, TESTFN_UNICODE, check_warnings, run_unittest, - make_bad_fd, cpython_only) + make_bad_fd, cpython_only, swap_attr) from collections import UserList import _io # C implementation of io @@ -176,6 +176,12 @@ class AutoFileTests: finally: os.close(fd) + def testRecursiveRepr(self): + # Issue #25455 + with swap_attr(self.f, 'name', self.f): + with self.assertRaises(RuntimeError): + repr(self.f) # Should not crash + def testErrors(self): f = self.f self.assertFalse(f.isatty()) diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index ac8473d..6491f45 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -119,15 +119,27 @@ class GeneralFloatCases(unittest.TestCase): self.assertEqual(float(memoryview(b'12.34')[1:4]), 2.3) def test_error_message(self): - testlist = ('\xbd', '123\xbd', ' 123 456 ') - for s in testlist: - try: + def check(s): + with self.assertRaises(ValueError, msg='float(%r)' % (s,)) as cm: float(s) - except ValueError as e: - self.assertIn(s.strip(), e.args[0]) - else: - self.fail("Expected int(%r) to raise a ValueError", s) - + self.assertEqual(str(cm.exception), + 'could not convert string to float: %r' % (s,)) + + check('\xbd') + check('123\xbd') + check(' 123 456 ') + check(b' 123 456 ') + + # non-ascii digits (error came from non-digit '!') + check('\u0663\u0661\u0664!') + # embedded NUL + check('123\x00') + check('123\x00 245') + check('123\x00245') + # byte string with embedded NUL + check(b'123\x00') + # non-UTF-8 byte string + check(b'123\xa0') @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE') def test_float_with_comma(self): diff --git a/Lib/test/test_fnmatch.py b/Lib/test/test_fnmatch.py index fb74246..78245c3 100644 --- a/Lib/test/test_fnmatch.py +++ b/Lib/test/test_fnmatch.py @@ -1,18 +1,19 @@ """Test cases for the fnmatch module.""" import unittest +import os from fnmatch import fnmatch, fnmatchcase, translate, filter class FnmatchTestCase(unittest.TestCase): - def check_match(self, filename, pattern, should_match=1, fn=fnmatch): + def check_match(self, filename, pattern, should_match=True, fn=fnmatch): if should_match: self.assertTrue(fn(filename, pattern), "expected %r to match pattern %r" % (filename, pattern)) else: - self.assertTrue(not fn(filename, pattern), + self.assertFalse(fn(filename, pattern), "expected %r not to match pattern %r" % (filename, pattern)) @@ -26,15 +27,15 @@ class FnmatchTestCase(unittest.TestCase): check('abc', '*') check('abc', 'ab[cd]') check('abc', 'ab[!de]') - check('abc', 'ab[de]', 0) - check('a', '??', 0) - check('a', 'b', 0) + check('abc', 'ab[de]', False) + check('a', '??', False) + check('a', 'b', False) # these test that '\' is handled correctly in character sets; # see SF bug #409651 check('\\', r'[\]') check('a', r'[!\]') - check('\\', r'[!\]', 0) + check('\\', r'[!\]', False) # test that filenames with newlines in them are handled correctly. # http://bugs.python.org/issue6665 @@ -51,14 +52,38 @@ class FnmatchTestCase(unittest.TestCase): def test_fnmatchcase(self): check = self.check_match - check('AbC', 'abc', 0, fnmatchcase) - check('abc', 'AbC', 0, fnmatchcase) + check('abc', 'abc', True, fnmatchcase) + check('AbC', 'abc', False, fnmatchcase) + check('abc', 'AbC', False, fnmatchcase) + check('AbC', 'AbC', True, fnmatchcase) + + check('usr/bin', 'usr/bin', True, fnmatchcase) + check('usr\\bin', 'usr/bin', False, fnmatchcase) + check('usr/bin', 'usr\\bin', False, fnmatchcase) + check('usr\\bin', 'usr\\bin', True, fnmatchcase) def test_bytes(self): self.check_match(b'test', b'te*') self.check_match(b'test\xff', b'te*\xff') self.check_match(b'foo\nbar', b'foo*') + def test_case(self): + ignorecase = os.path.normcase('ABC') == os.path.normcase('abc') + check = self.check_match + check('abc', 'abc') + check('AbC', 'abc', ignorecase) + check('abc', 'AbC', ignorecase) + check('AbC', 'AbC') + + def test_sep(self): + normsep = os.path.normcase('\\') == os.path.normcase('/') + check = self.check_match + check('usr/bin', 'usr/bin') + check('usr\\bin', 'usr/bin', normsep) + check('usr/bin', 'usr\\bin', normsep) + check('usr\\bin', 'usr\\bin') + + class TranslateTestCase(unittest.TestCase): def test_translate(self): @@ -75,7 +100,28 @@ class TranslateTestCase(unittest.TestCase): class FilterTestCase(unittest.TestCase): def test_filter(self): - self.assertEqual(filter(['a', 'b'], 'a'), ['a']) + self.assertEqual(filter(['Python', 'Ruby', 'Perl', 'Tcl'], 'P*'), + ['Python', 'Perl']) + self.assertEqual(filter([b'Python', b'Ruby', b'Perl', b'Tcl'], b'P*'), + [b'Python', b'Perl']) + + def test_mix_bytes_str(self): + self.assertRaises(TypeError, filter, ['test'], b'*') + self.assertRaises(TypeError, filter, [b'test'], '*') + + def test_case(self): + ignorecase = os.path.normcase('P') == os.path.normcase('p') + self.assertEqual(filter(['Test.py', 'Test.rb', 'Test.PL'], '*.p*'), + ['Test.py', 'Test.PL'] if ignorecase else ['Test.py']) + self.assertEqual(filter(['Test.py', 'Test.rb', 'Test.PL'], '*.P*'), + ['Test.py', 'Test.PL'] if ignorecase else ['Test.PL']) + + def test_sep(self): + normsep = os.path.normcase('\\') == os.path.normcase('/') + self.assertEqual(filter(['usr/bin', 'usr', 'usr\\lib'], 'usr/*'), + ['usr/bin', 'usr\\lib'] if normsep else ['usr/bin']) + self.assertEqual(filter(['usr/bin', 'usr', 'usr\\lib'], 'usr\\*'), + ['usr/bin', 'usr\\lib'] if normsep else ['usr\\lib']) if __name__ == "__main__": diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 83eb29f..b6ba2e5 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -315,10 +315,12 @@ class FormatTest(unittest.TestCase): testcommon(b"%b", b"abc", b"abc") testcommon(b"%b", bytearray(b"def"), b"def") testcommon(b"%b", fb, b"123") + testcommon(b"%b", memoryview(b"abc"), b"abc") # # %s is an alias for %b -- should only be used for Py2/3 code testcommon(b"%s", b"abc", b"abc") testcommon(b"%s", bytearray(b"def"), b"def") testcommon(b"%s", fb, b"123") + testcommon(b"%s", memoryview(b"abc"), b"abc") # %a will give the equivalent of # repr(some_obj).encode('ascii', 'backslashreplace') testcommon(b"%a", 3.14, b"3.14") @@ -377,9 +379,11 @@ class FormatTest(unittest.TestCase): test_exc(b"%c", 3.14, TypeError, "%c requires an integer in range(256) or a single byte") test_exc(b"%b", "Xc", TypeError, - "%b requires bytes, or an object that implements __bytes__, not 'str'") + "%b requires a bytes-like object, " + "or an object that implements __bytes__, not 'str'") test_exc(b"%s", "Wd", TypeError, - "%b requires bytes, or an object that implements __bytes__, not 'str'") + "%b requires a bytes-like object, " + "or an object that implements __bytes__, not 'str'") if maxsize == 2**31-1: # crashes 2.2.1 and earlier: diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 708ed25..3d762b5 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -280,6 +280,10 @@ f'{a * x()}'""" "f'{10:{ }}'", "f' { } '", + # The Python parser ignores also the following + # whitespace characters in additional to a space. + "f'''{\t\f\r\n}'''", + # Catch the empty expression before the # invalid conversion. "f'{!x}'", @@ -300,6 +304,12 @@ f'{a * x()}'""" "f'{:x'", ]) + # Different error message is raised for other whitespace characters. + self.assertAllRaise(SyntaxError, 'invalid character in identifier', + ["f'''{\xa0}'''", + "\xa0", + ]) + def test_parens_in_expressions(self): self.assertEqual(f'{3,}', '(3,)') @@ -361,6 +371,20 @@ f'{a * x()}'""" self.assertEqual(f'2\x203', '2 3') self.assertEqual(f'\x203', ' 3') + with self.assertWarns(DeprecationWarning): # invalid escape sequence + value = eval(r"f'\{6*7}'") + self.assertEqual(value, '\\42') + self.assertEqual(f'\\{6*7}', '\\42') + self.assertEqual(fr'\{6*7}', '\\42') + + AMPERSAND = 'spam' + # Get the right unicode character (&), or pick up local variable + # depending on the number of backslashes. + self.assertEqual(f'\N{AMPERSAND}', '&') + self.assertEqual(f'\\N{AMPERSAND}', '\\Nspam') + self.assertEqual(fr'\N{AMPERSAND}', '\\Nspam') + self.assertEqual(f'\\\N{AMPERSAND}', '\\&') + def test_misformed_unicode_character_name(self): # These test are needed because unicode names are parsed # differently inside f-strings. @@ -756,5 +780,11 @@ f'{a * x()}'""" self.assertEqual(f'{d["foo"]}', 'bar') self.assertEqual(f"{d['foo']}", 'bar') + def test_backslash_char(self): + # Check eval of a backslash followed by a control char. + # See bpo-30682: this used to raise an assert in pydebug mode. + self.assertEqual(eval('f"\\\n"'), '') + self.assertEqual(eval('f"\\\r"'), '') + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index b7d648d..cd4664c 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -402,6 +402,32 @@ class TestPartialC(TestPartial, unittest.TestCase): else: self.fail('partial object allowed __dict__ to be deleted') + def test_manually_adding_non_string_keyword(self): + p = self.partial(capture) + # Adding a non-string/unicode keyword to partial kwargs + p.keywords[1234] = 'value' + r = repr(p) + self.assertIn('1234', r) + self.assertIn("'value'", r) + with self.assertRaises(TypeError): + p() + + def test_keystr_replaces_value(self): + p = self.partial(capture) + + class MutatesYourDict(object): + def __str__(self): + p.keywords[self] = ['sth2'] + return 'astr' + + # Raplacing the value during key formatting should keep the original + # value alive (at least long enough). + p.keywords[MutatesYourDict()] = ['sth'] + r = repr(p) + self.assertIn('astr', r) + self.assertIn("['sth']", r) + + class TestPartialPy(TestPartial, unittest.TestCase): partial = py_functools.partial diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 60f1d92..b7554d6 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -3,13 +3,14 @@ # The code for testing gdb was adapted from similar work in Unladen Swallow's # Lib/test/test_jit_gdb.py +import locale import os import re import subprocess import sys import sysconfig +import textwrap import unittest -import locale # Is this Python configured to support threads? try: @@ -845,7 +846,24 @@ id(42) breakpoint='time_gmtime', cmds_after_breakpoint=['py-bt-full'], ) - self.assertIn('#0 " % clsname) + def test_recursive_repr(self): + # Issue #25455 + raw = self.MockRawIO() + b = self.tp(raw) + with support.swap_attr(raw, 'name', b): + try: + repr(b) # Should not crash + except RuntimeError: + pass + def test_flush_error_on_close(self): # Test that buffered file is closed despite failed flush # and that flush() is called before file closed. @@ -2424,6 +2450,16 @@ class TextIOWrapperTest(unittest.TestCase): t.buffer.detach() repr(t) # Should not raise an exception + def test_recursive_repr(self): + # Issue #25455 + raw = self.BytesIO() + t = self.TextIOWrapper(raw) + with support.swap_attr(raw, 'name', t): + try: + repr(t) # Should not crash + except RuntimeError: + pass + def test_line_buffering(self): r = self.BytesIO() b = self.BufferedWriter(r, 1000) @@ -3478,6 +3514,7 @@ class MiscIOTest(unittest.TestCase): self.assertRaises(ValueError, f.readinto1, bytearray(1024)) self.assertRaises(ValueError, f.readline) self.assertRaises(ValueError, f.readlines) + self.assertRaises(ValueError, f.readlines, 1) self.assertRaises(ValueError, f.seek, 0) self.assertRaises(ValueError, f.tell) self.assertRaises(ValueError, f.truncate) @@ -3683,6 +3720,7 @@ class CMiscIOTest(MiscIOTest): import sys import time import threading + from test.support import SuppressCrashReport file = sys.{stream_name} @@ -3691,6 +3729,10 @@ class CMiscIOTest(MiscIOTest): file.write('.') file.flush() + crash = SuppressCrashReport() + crash.__enter__() + # don't call __exit__(): the crash occurs at Python shutdown + thread = threading.Thread(target=run) thread.daemon = True thread.start() diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 0e39516..5d96330 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -1405,14 +1405,35 @@ class IpaddrUnitTest(unittest.TestCase): ipaddress.ip_address('::2')) def testInterfaceComparison(self): - self.assertTrue(ipaddress.ip_interface('1.1.1.1') <= - ipaddress.ip_interface('1.1.1.1')) - self.assertTrue(ipaddress.ip_interface('1.1.1.1') <= - ipaddress.ip_interface('1.1.1.2')) - self.assertTrue(ipaddress.ip_interface('::1') <= - ipaddress.ip_interface('::1')) - self.assertTrue(ipaddress.ip_interface('::1') <= - ipaddress.ip_interface('::2')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/24') == + ipaddress.ip_interface('1.1.1.1/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/16') < + ipaddress.ip_interface('1.1.1.1/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/24') < + ipaddress.ip_interface('1.1.1.2/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.2/16') < + ipaddress.ip_interface('1.1.1.1/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/24') > + ipaddress.ip_interface('1.1.1.1/16')) + self.assertTrue(ipaddress.ip_interface('1.1.1.2/24') > + ipaddress.ip_interface('1.1.1.1/24')) + self.assertTrue(ipaddress.ip_interface('1.1.1.1/24') > + ipaddress.ip_interface('1.1.1.2/16')) + + self.assertTrue(ipaddress.ip_interface('::1/64') == + ipaddress.ip_interface('::1/64')) + self.assertTrue(ipaddress.ip_interface('::1/64') < + ipaddress.ip_interface('::1/80')) + self.assertTrue(ipaddress.ip_interface('::1/64') < + ipaddress.ip_interface('::2/64')) + self.assertTrue(ipaddress.ip_interface('::2/48') < + ipaddress.ip_interface('::1/64')) + self.assertTrue(ipaddress.ip_interface('::1/80') > + ipaddress.ip_interface('::1/64')) + self.assertTrue(ipaddress.ip_interface('::2/64') > + ipaddress.ip_interface('::1/64')) + self.assertTrue(ipaddress.ip_interface('::1/64') > + ipaddress.ip_interface('::2/48')) def testNetworkComparison(self): # ip1 and ip2 have the same network address diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index ea1f57c..c431f0d 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1976,6 +1976,14 @@ class RegressionTests(unittest.TestCase): self.assertRaises(AssertionError, list, cycle(gen1())) self.assertEqual(hist, [0,1]) + def test_long_chain_of_empty_iterables(self): + # Make sure itertools.chain doesn't run into recursion limits when + # dealing with long chains of empty iterables. Even with a high + # number this would probably only fail in Py_DEBUG mode. + it = chain.from_iterable(() for unused in range(10000000)) + with self.assertRaises(StopIteration): + next(it) + class SubclassWithKwargsTest(unittest.TestCase): def test_keywords_in_subclass(self): # count is not subclassable... diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 15f3736..9d93f93 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -2,7 +2,7 @@ import os import sys import textwrap import unittest -import subprocess +from subprocess import Popen, PIPE from test import support from test.support.script_helper import assert_python_ok @@ -61,12 +61,11 @@ class TestTool(unittest.TestCase): """) def test_stdin_stdout(self): - with subprocess.Popen( - (sys.executable, '-m', 'json.tool'), - stdin=subprocess.PIPE, stdout=subprocess.PIPE) as proc: + args = sys.executable, '-m', 'json.tool' + with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc: out, err = proc.communicate(self.data.encode()) self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) - self.assertEqual(err, None) + self.assertEqual(err, b'') def _create_infile(self): infile = support.TESTFN diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 9dedc09..474affd 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1663,7 +1663,7 @@ class SysLogHandlerTest(BaseTest): server.ready.wait() hcls = logging.handlers.SysLogHandler if isinstance(server.server_address, tuple): - self.sl_hdlr = hcls(('localhost', server.port)) + self.sl_hdlr = hcls((server.server_address[0], server.port)) else: self.sl_hdlr = hcls(server.server_address) self.log_output = '' @@ -1723,6 +1723,24 @@ class UnixSysLogHandlerTest(SysLogHandlerTest): SysLogHandlerTest.tearDown(self) support.unlink(self.address) +@unittest.skipUnless(support.IPV6_ENABLED, + 'IPv6 support required for this test.') +@unittest.skipUnless(threading, 'Threading required for this test.') +class IPv6SysLogHandlerTest(SysLogHandlerTest): + + """Test for SysLogHandler with IPv6 host.""" + + server_class = TestUDPServer + address = ('::1', 0) + + def setUp(self): + self.server_class.address_family = socket.AF_INET6 + super(IPv6SysLogHandlerTest, self).setUp() + + def tearDown(self): + self.server_class.address_family = socket.AF_INET + super(IPv6SysLogHandlerTest, self).tearDown() + @unittest.skipUnless(threading, 'Threading required for this test.') class HTTPHandlerTest(BaseTest): """Test for HTTPHandler.""" @@ -3163,6 +3181,7 @@ if hasattr(logging.handlers, 'QueueListener'): handler.close() @patch.object(logging.handlers.QueueListener, 'handle') + @support.reap_threads def test_handle_called_with_queue_queue(self, mock_handle): for i in range(self.repeat): log_queue = queue.Queue() @@ -3172,10 +3191,13 @@ if hasattr(logging.handlers, 'QueueListener'): @support.requires_multiprocessing_queue @patch.object(logging.handlers.QueueListener, 'handle') + @support.reap_threads def test_handle_called_with_mp_queue(self, mock_handle): for i in range(self.repeat): log_queue = multiprocessing.Queue() self.setup_and_log(log_queue, '%s_%s' % (self.id(), i)) + log_queue.close() + log_queue.join_thread() self.assertEqual(mock_handle.call_count, 5 * self.repeat, 'correct number of handled log messages') @@ -3188,6 +3210,7 @@ if hasattr(logging.handlers, 'QueueListener'): return [] @support.requires_multiprocessing_queue + @support.reap_threads def test_no_messages_in_queue_after_stop(self): """ Five messages are logged then the QueueListener is stopped. This @@ -3200,6 +3223,9 @@ if hasattr(logging.handlers, 'QueueListener'): self.setup_and_log(queue, '%s_%s' %(self.id(), i)) # time.sleep(1) items = list(self.get_all_from_queue(queue)) + queue.close() + queue.join_thread() + expected = [[], [logging.handlers.QueueListener._sentinel]] self.assertIn(items, expected, 'Found unexpected messages in queue: %s' % ( @@ -4370,7 +4396,7 @@ def test_main(): QueueHandlerTest, ShutdownTest, ModuleLevelMiscTest, BasicConfigTest, LoggerAdapterTest, LoggerTest, SMTPHandlerTest, FileHandlerTest, RotatingFileHandlerTest, LastResortTest, LogRecordTest, - ExceptionTest, SysLogHandlerTest, HTTPHandlerTest, + ExceptionTest, SysLogHandlerTest, IPv6SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest, TimedRotatingFileHandlerTest, UnixSocketHandlerTest, UnixDatagramHandlerTest, UnixSysLogHandlerTest, MiscTestCase diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py index c46ded1..3e84f34 100644 --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -286,7 +286,12 @@ class NetworkedNNTPTests(NetworkedNNTPTestsMixin, unittest.TestCase): def setUpClass(cls): support.requires("network") with support.transient_internet(cls.NNTP_HOST): - cls.server = cls.NNTP_CLASS(cls.NNTP_HOST, timeout=TIMEOUT, usenetrc=False) + try: + cls.server = cls.NNTP_CLASS(cls.NNTP_HOST, timeout=TIMEOUT, + usenetrc=False) + except EOFError: + raise unittest.SkipTest(f"{cls} got EOF error on connecting " + f"to {cls.NNTP_HOST!r}") @classmethod def tearDownClass(cls): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e9fdb07..8612ec9 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -473,7 +473,9 @@ class StatAttributeTests(unittest.TestCase): # force CreateFile to fail with ERROR_ACCESS_DENIED. DETACHED_PROCESS = 8 subprocess.check_call( - ['icacls.exe', fname, '/deny', 'Users:(S)'], + # bpo-30584: Use security identifier *S-1-5-32-545 instead + # of localized "Users" to not depend on the locale. + ['icacls.exe', fname, '/deny', '*S-1-5-32-545:(S)'], creationflags=DETACHED_PROCESS ) result = os.stat(fname) @@ -620,8 +622,13 @@ class UtimeTests(unittest.TestCase): if not self.support_subsecond(self.fname): delta = 1.0 + elif os.name == 'nt': + # On Windows, the usual resolution of time.time() is 15.6 ms. + # bpo-30649: Tolerate 50 ms for slow Windows buildbots. + delta = 0.050 else: - # On Windows, the usual resolution of time.time() is 15.6 ms + # bpo-30649: PPC64 Fedora 3.x buildbot requires + # at least a delta of 14 ms delta = 0.020 st = os.stat(self.fname) msg = ("st_time=%r, current=%r, dt=%r" diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index d6e6f71..70cabb2 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -1,4 +1,6 @@ +import copy import parser +import pickle import unittest import operator import struct @@ -424,6 +426,52 @@ class IllegalSyntaxTestCase(unittest.TestCase): # not even remotely valid: self.check_bad_tree((1, 2, 3), "") + def test_illegal_terminal(self): + tree = \ + (257, + (269, + (270, + (271, + (277, + (1,))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "too small items in terminal node") + tree = \ + (257, + (269, + (270, + (271, + (277, + (1, b'pass'))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "non-string second item in terminal node") + tree = \ + (257, + (269, + (270, + (271, + (277, + (1, 'pass', '0', 0))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "non-integer third item in terminal node") + tree = \ + (257, + (269, + (270, + (271, + (277, + (1, 'pass', 0, 0))), + (4, ''))), + (4, ''), + (0, '')) + self.check_bad_tree(tree, "too many items in terminal node") + def test_illegal_yield_1(self): # Illegal yield statement: def f(): return 1; yield 1 tree = \ @@ -628,6 +676,24 @@ class IllegalSyntaxTestCase(unittest.TestCase): (4, ''), (0, '')) self.check_bad_tree(tree, "from import fred") + def test_illegal_encoding(self): + # Illegal encoding declaration + tree = \ + (339, + (257, (0, ''))) + self.check_bad_tree(tree, "missed encoding") + tree = \ + (339, + (257, (0, '')), + b'iso-8859-1') + self.check_bad_tree(tree, "non-string encoding") + tree = \ + (339, + (257, (0, '')), + '\udcff') + with self.assertRaises(UnicodeEncodeError): + parser.sequence2st(tree) + class CompileTestCase(unittest.TestCase): @@ -772,6 +838,21 @@ class STObjectTestCase(unittest.TestCase): self.assertRaises(TypeError, operator.lt, st1, 1815) self.assertRaises(TypeError, operator.gt, b'waterloo', st2) + def test_copy_pickle(self): + sts = [ + parser.expr('2 + 3'), + parser.suite('x = 2; y = x + 3'), + parser.expr('list(x**3 for x in range(20))') + ] + for st in sts: + st_copy = copy.copy(st) + self.assertEqual(st_copy.totuple(), st.totuple()) + st_copy = copy.deepcopy(st) + self.assertEqual(st_copy.totuple(), st.totuple()) + for proto in range(pickle.HIGHEST_PROTOCOL+1): + st_copy = pickle.loads(pickle.dumps(st, proto)) + self.assertEqual(st_copy.totuple(), st.totuple()) + check_sizeof = support.check_sizeof @support.cpython_only diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 88a93e5..6502413 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -8,6 +8,7 @@ import socket import stat import tempfile import unittest +from unittest import mock from test import support android_not_root = support.android_not_root @@ -1506,10 +1507,10 @@ class _BasePathTest(object): os.path.join(BASE, 'foo')) p = P(BASE, 'foo', 'in', 'spam') self.assertEqual(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo')) + os.path.join(BASE, 'foo', 'in', 'spam')) p = P(BASE, '..', 'foo', 'in', 'spam') self.assertEqual(str(p.resolve(strict=False)), - os.path.abspath(os.path.join('foo'))) + os.path.abspath(os.path.join('foo', 'in', 'spam'))) # These are all relative symlinks p = P(BASE, 'dirB', 'fileB') self._check_resolve_relative(p, p) @@ -1521,16 +1522,18 @@ class _BasePathTest(object): self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) # Non-strict p = P(BASE, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo', 'in', + 'spam'), False) p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') if os.name == 'nt': # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(BASE, 'dirA', 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'dirA', 'foo', 'in', + 'spam'), False) else: # In Posix, if linkY points to dirB, 'dirA/linkY/..' # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) # Now create absolute symlinks d = tempfile.mkdtemp(suffix='-dirD') self.addCleanup(support.rmtree, d) @@ -1540,16 +1543,17 @@ class _BasePathTest(object): self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB')) # Non-strict p = P(BASE, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'dirB', 'foo', 'in', 'spam'), + False) p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') if os.name == 'nt': # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(d, 'foo'), False) + self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) else: # In Posix, if linkY points to dirB, 'dirA/linkY/..' # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo'), False) + self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) @with_symlinks def test_resolve_dot(self): @@ -1563,7 +1567,7 @@ class _BasePathTest(object): r = q / '3' / '4' self.assertRaises(FileNotFoundError, r.resolve, strict=True) # Non-strict - self.assertEqual(r.resolve(strict=False), p / '3') + self.assertEqual(r.resolve(strict=False), p / '3' / '4') def test_with(self): p = self.cls(BASE) @@ -1776,6 +1780,11 @@ class _BasePathTest(object): self.assertTrue(p.exists()) self.assertEqual(p.stat().st_ctime, st_ctime_first) + def test_mkdir_exist_ok_root(self): + # Issue #25803: A drive root could raise PermissionError on Windows + self.cls('/').resolve().mkdir(exist_ok=True) + self.cls('/').resolve().mkdir(parents=True, exist_ok=True) + @only_nt # XXX: not sure how to test this on POSIX def test_mkdir_with_unknown_drive(self): for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': @@ -1811,6 +1820,35 @@ class _BasePathTest(object): p.mkdir(exist_ok=True) self.assertEqual(cm.exception.errno, errno.EEXIST) + def test_mkdir_concurrent_parent_creation(self): + for pattern_num in range(32): + p = self.cls(BASE, 'dirCPC%d' % pattern_num) + self.assertFalse(p.exists()) + + def my_mkdir(path, mode=0o777): + path = str(path) + # Emulate another process that would create the directory + # just before we try to create it ourselves. We do it + # in all possible pattern combinations, assuming that this + # function is called at most 5 times (dirCPC/dir1/dir2, + # dirCPC/dir1, dirCPC, dirCPC/dir1, dirCPC/dir1/dir2). + if pattern.pop(): + os.mkdir(path, mode) # from another process + concurrently_created.add(path) + os.mkdir(path, mode) # our real call + + pattern = [bool(pattern_num & (1 << n)) for n in range(5)] + concurrently_created = set() + p12 = p / 'dir1' / 'dir2' + try: + with mock.patch("pathlib._normal_accessor.mkdir", my_mkdir): + p12.mkdir(parents=True, exist_ok=False) + except FileExistsError: + self.assertIn(str(p12), concurrently_created) + else: + self.assertNotIn(str(p12), concurrently_created) + self.assertTrue(p.exists()) + @with_symlinks def test_symlink_to(self): P = self.cls(BASE) diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 5b6a4f0..03adc0a 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -427,6 +427,7 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): self.assertRaises(ValueError, self.gen.setstate, (1, None, None)) def test_setstate_middle_arg(self): + start_state = self.gen.getstate() # Wrong type, s/b tuple self.assertRaises(TypeError, self.gen.setstate, (2, None, None)) # Wrong length, s/b 625 @@ -440,6 +441,10 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): self.gen.setstate((2, (1,)*624+(625,), None)) with self.assertRaises((ValueError, OverflowError)): self.gen.setstate((2, (1,)*624+(-1,), None)) + # Failed calls to setstate() should not have changed the state. + bits100 = self.gen.getrandbits(100) + self.gen.setstate(start_state) + self.assertEqual(self.gen.getrandbits(100), bits100) # Little trick to make "tuple(x % (2**32) for x in internalstate)" # raise ValueError. I cannot think of a simple way to achieve this, so diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 9e11e51..679759e 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -99,20 +99,24 @@ class RangeTest(unittest.TestCase): x = range(10**20+10, 10**20, 3) self.assertEqual(len(x), 0) self.assertEqual(len(list(x)), 0) + self.assertFalse(x) x = range(10**20, 10**20+10, -3) self.assertEqual(len(x), 0) self.assertEqual(len(list(x)), 0) + self.assertFalse(x) x = range(10**20+10, 10**20, -3) self.assertEqual(len(x), 4) self.assertEqual(len(list(x)), 4) + self.assertTrue(x) # Now test range() with longs - self.assertEqual(list(range(-2**100)), []) - self.assertEqual(list(range(0, -2**100)), []) - self.assertEqual(list(range(0, 2**100, -1)), []) - self.assertEqual(list(range(0, 2**100, -1)), []) + for x in [range(-2**100), + range(0, -2**100), + range(0, 2**100, -1)]: + self.assertEqual(list(x), []) + self.assertFalse(x) a = int(10 * sys.maxsize) b = int(100 * sys.maxsize) @@ -153,6 +157,7 @@ class RangeTest(unittest.TestCase): step = x[1] - x[0] length = 1 + ((x[-1] - x[0]) // step) return length + a = -sys.maxsize b = sys.maxsize expected_len = b - a @@ -160,6 +165,7 @@ class RangeTest(unittest.TestCase): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -177,6 +183,7 @@ class RangeTest(unittest.TestCase): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -195,6 +202,7 @@ class RangeTest(unittest.TestCase): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 @@ -213,6 +221,7 @@ class RangeTest(unittest.TestCase): self.assertIn(a, x) self.assertNotIn(b, x) self.assertRaises(OverflowError, len, x) + self.assertTrue(x) self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index a506b98..a6cbbd0 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1303,48 +1303,102 @@ class ReTests(unittest.TestCase): upper_char = '\u1ea0' # Latin Capital Letter A with Dot Below lower_char = '\u1ea1' # Latin Small Letter A with Dot Below - p = re.compile(upper_char, re.I | re.U) - q = p.match(lower_char) + p = re.compile('.' + upper_char, re.I | re.S) + q = p.match('\n' + lower_char) self.assertTrue(q) - p = re.compile(lower_char, re.I | re.U) - q = p.match(upper_char) + p = re.compile('.' + lower_char, re.I | re.S) + q = p.match('\n' + upper_char) self.assertTrue(q) - p = re.compile('(?i)' + upper_char, re.U) - q = p.match(lower_char) + p = re.compile('(?i).' + upper_char, re.S) + q = p.match('\n' + lower_char) self.assertTrue(q) - p = re.compile('(?i)' + lower_char, re.U) - q = p.match(upper_char) + p = re.compile('(?i).' + lower_char, re.S) + q = p.match('\n' + upper_char) self.assertTrue(q) - p = re.compile('(?iu)' + upper_char) - q = p.match(lower_char) + p = re.compile('(?is).' + upper_char) + q = p.match('\n' + lower_char) self.assertTrue(q) - p = re.compile('(?iu)' + lower_char) - q = p.match(upper_char) + p = re.compile('(?is).' + lower_char) + q = p.match('\n' + upper_char) self.assertTrue(q) - self.assertTrue(re.match('(?ixu) ' + upper_char, lower_char)) - self.assertTrue(re.match('(?ixu) ' + lower_char, upper_char)) + p = re.compile('(?s)(?i).' + upper_char) + q = p.match('\n' + lower_char) + self.assertTrue(q) + + p = re.compile('(?s)(?i).' + lower_char) + q = p.match('\n' + upper_char) + self.assertTrue(q) + + self.assertTrue(re.match('(?ix) ' + upper_char, lower_char)) + self.assertTrue(re.match('(?ix) ' + lower_char, upper_char)) + self.assertTrue(re.match(' (?i) ' + upper_char, lower_char, re.X)) + self.assertTrue(re.match('(?x) (?i) ' + upper_char, lower_char)) + self.assertTrue(re.match(' (?x) (?i) ' + upper_char, lower_char, re.X)) p = upper_char + '(?i)' with self.assertWarns(DeprecationWarning) as warns: self.assertTrue(re.match(p, lower_char)) self.assertEqual( str(warns.warnings[0].message), - 'Flags not at the start of the expression %s' % p + 'Flags not at the start of the expression %r' % p ) + self.assertEqual(warns.warnings[0].filename, __file__) p = upper_char + '(?i)%s' % ('.?' * 100) with self.assertWarns(DeprecationWarning) as warns: self.assertTrue(re.match(p, lower_char)) self.assertEqual( str(warns.warnings[0].message), - 'Flags not at the start of the expression %s (truncated)' % p[:20] + 'Flags not at the start of the expression %r (truncated)' % p[:20] ) + self.assertEqual(warns.warnings[0].filename, __file__) + + # bpo-30605: Compiling a bytes instance regex was throwing a BytesWarning + with warnings.catch_warnings(): + warnings.simplefilter('error', BytesWarning) + p = b'A(?i)' + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.match(p, b'a')) + self.assertEqual( + str(warns.warnings[0].message), + 'Flags not at the start of the expression %r' % p + ) + self.assertEqual(warns.warnings[0].filename, __file__) + + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('(?s).(?i)' + upper_char, '\n' + lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('(?i) ' + upper_char + ' (?x)', lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match(' (?x) (?i) ' + upper_char, lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('^(?i)' + upper_char, lower_char)) + with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('$|(?i)' + upper_char, lower_char)) + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.match('(?:(?i)' + upper_char + ')', lower_char)) + self.assertRegex(str(warns.warnings[0].message), + 'Flags not at the start') + self.assertEqual(warns.warnings[0].filename, __file__) + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.fullmatch('(^)?(?(1)(?i)' + upper_char + ')', + lower_char)) + self.assertRegex(str(warns.warnings[0].message), + 'Flags not at the start') + self.assertEqual(warns.warnings[0].filename, __file__) + with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.fullmatch('($)?(?(1)|(?i)' + upper_char + ')', + lower_char)) + self.assertRegex(str(warns.warnings[0].message), + 'Flags not at the start') + self.assertEqual(warns.warnings[0].filename, __file__) + def test_dollar_matches_twice(self): "$ matches the end of string, and just before the terminating \n" @@ -1402,7 +1456,7 @@ class ReTests(unittest.TestCase): def test_locale_flag(self): import locale - enc = locale.getpreferredencoding(False) + _, enc = locale.getlocale(locale.LC_CTYPE) # Search non-ASCII letter for i in range(128, 256): try: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 751df15..5c6154a 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -118,6 +118,9 @@ class ParseArgsTestCase(unittest.TestCase): ns = libregrtest._parse_args(['--header']) self.assertTrue(ns.header) + ns = libregrtest._parse_args(['--verbose']) + self.assertTrue(ns.header) + def test_randomize(self): for opt in '-r', '--randomize': with self.subTest(opt=opt): @@ -156,9 +159,24 @@ class ParseArgsTestCase(unittest.TestCase): for opt in '-m', '--match': with self.subTest(opt=opt): ns = libregrtest._parse_args([opt, 'pattern']) - self.assertEqual(ns.match_tests, 'pattern') + self.assertEqual(ns.match_tests, ['pattern']) self.checkError([opt], 'expected one argument') + ns = libregrtest._parse_args(['-m', 'pattern1', + '-m', 'pattern2']) + self.assertEqual(ns.match_tests, ['pattern1', 'pattern2']) + + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "w") as fp: + print('matchfile1', file=fp) + print('matchfile2', file=fp) + + filename = os.path.abspath(support.TESTFN) + ns = libregrtest._parse_args(['-m', 'match', + '--matchfile', filename]) + self.assertEqual(ns.match_tests, + ['match', 'matchfile1', 'matchfile2']) + def test_failfast(self): for opt in '-G', '--failfast': with self.subTest(opt=opt): @@ -272,7 +290,6 @@ class ParseArgsTestCase(unittest.TestCase): ns = libregrtest._parse_args([opt]) self.assertTrue(ns.forever) - def test_unrecognized_argument(self): self.checkError(['--xxx'], 'usage:') @@ -354,7 +371,7 @@ class BaseTestCase(unittest.TestCase): self.assertRegex(output, regex) def parse_executed_tests(self, output): - regex = (r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)?\] (%s)' + regex = (r'^[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?\[ *[0-9]+(?:/ *[0-9]+)*\] (%s)' % self.TESTNAME_REGEX) parser = re.finditer(regex, output, re.MULTILINE) return list(match.group(1) for match in parser) @@ -454,7 +471,6 @@ class BaseTestCase(unittest.TestCase): self.fail(msg) return proc - def run_python(self, args, **kw): args = [sys.executable, '-X', 'faulthandler', '-I', *args] proc = self.run_command(args, **kw) @@ -676,6 +692,14 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('--fromfile', filename) self.check_executed_tests(output, tests) + # test format 'Lib/test/test_opcodes.py' + with open(filename, "w") as fp: + for name in tests: + print('Lib/test/%s.py' % name, file=fp) + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + def test_interrupted(self): code = TEST_INTERRUPTED test = self.create_test('sigint', code=code) @@ -801,6 +825,79 @@ class ArgsTestCase(BaseTestCase): self.assertEqual(output.rstrip().splitlines(), tests) + def test_list_cases(self): + # test --list-cases + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_method1(self): + pass + def test_method2(self): + pass + """) + testname = self.create_test(code=code) + all_methods = ['%s.Tests.test_method1' % testname, + '%s.Tests.test_method2' % testname] + output = self.run_tests('--list-cases', testname) + self.assertEqual(output.splitlines(), all_methods) + + def test_crashed(self): + # Any code which causes a crash + code = 'import faulthandler; faulthandler._sigsegv()' + crash_test = self.create_test(name="crash", code=code) + ok_test = self.create_test(name="ok") + + tests = [crash_test, ok_test] + output = self.run_tests("-j2", *tests, exitcode=1) + self.check_executed_tests(output, tests, failed=crash_test, + randomize=True) + + def parse_methods(self, output): + regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE) + return [match.group(1) for match in regex.finditer(output)] + + def test_matchfile(self): + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_method1(self): + pass + def test_method2(self): + pass + def test_method3(self): + pass + def test_method4(self): + pass + """) + all_methods = ['test_method1', 'test_method2', + 'test_method3', 'test_method4'] + testname = self.create_test(code=code) + + # by default, all methods should be run + output = self.run_tests("-v", testname) + methods = self.parse_methods(output) + self.assertEqual(methods, all_methods) + + # only run a subset + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + subset = [ + # only match the method name + 'test_method1', + # match the full identifier + '%s.Tests.test_method3' % testname] + with open(filename, "w") as fp: + for name in subset: + print(name, file=fp) + + output = self.run_tests("-v", "--matchfile", filename, testname) + methods = self.parse_methods(output) + subset = ['test_method1', 'test_method3'] + self.assertEqual(methods, subset) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index 2411895..2eb6290 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -4,6 +4,7 @@ from xml.sax import make_parser, ContentHandler, \ SAXException, SAXReaderNotAvailable, SAXParseException import unittest +from unittest import mock try: make_parser() except SAXReaderNotAvailable: @@ -175,12 +176,8 @@ class ParseTest(unittest.TestCase): with self.assertRaises(SAXException): self.check_parse(BytesIO(xml_bytes(self.data, 'iso-8859-1', None))) make_xml_file(self.data, 'iso-8859-1', None) - with support.check_warnings(('unclosed file', ResourceWarning)): - # XXX Failed parser leaks an opened file. - with self.assertRaises(SAXException): - self.check_parse(TESTFN) - # Collect leaked file. - gc.collect() + with self.assertRaises(SAXException): + self.check_parse(TESTFN) with open(TESTFN, 'rb') as f: with self.assertRaises(SAXException): self.check_parse(f) @@ -194,6 +191,21 @@ class ParseTest(unittest.TestCase): input.setEncoding('iso-8859-1') self.check_parse(input) + def test_parse_close_source(self): + builtin_open = open + fileobj = None + + def mock_open(*args): + nonlocal fileobj + fileobj = builtin_open(*args) + return fileobj + + with mock.patch('xml.sax.saxutils.open', side_effect=mock_open): + make_xml_file(self.data, 'iso-8859-1', None) + with self.assertRaises(SAXException): + self.check_parse(TESTFN) + self.assertTrue(fileobj.closed) + def check_parseString(self, s): from xml.sax import parseString result = StringIO() diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 0620b24..4029617 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -27,14 +27,27 @@ if sys.flags.no_site: import site -if site.ENABLE_USER_SITE and not os.path.isdir(site.USER_SITE): - # need to add user site directory for tests - try: - os.makedirs(site.USER_SITE) - site.addsitedir(site.USER_SITE) - except PermissionError as exc: - raise unittest.SkipTest('unable to create user site directory (%r): %s' - % (site.USER_SITE, exc)) + +OLD_SYS_PATH = None + + +def setUpModule(): + global OLD_SYS_PATH + OLD_SYS_PATH = sys.path[:] + + if site.ENABLE_USER_SITE and not os.path.isdir(site.USER_SITE): + # need to add user site directory for tests + try: + os.makedirs(site.USER_SITE) + # modify sys.path: will be restored by tearDownModule() + site.addsitedir(site.USER_SITE) + except PermissionError as exc: + raise unittest.SkipTest('unable to create user site directory (%r): %s' + % (site.USER_SITE, exc)) + + +def tearDownModule(): + sys.path[:] = OLD_SYS_PATH class HelperFunctionsTests(unittest.TestCase): @@ -501,15 +514,15 @@ class StartupImportTests(unittest.TestCase): print(line, file=f) return exe_file except: - os.unlink(_pth_file) - os.unlink(exe_file) + test.support.unlink(_pth_file) + test.support.unlink(exe_file) raise @classmethod def _cleanup_underpth_exe(self, exe_file): _pth_file = os.path.splitext(exe_file)[0] + '._pth' - os.unlink(_pth_file) - os.unlink(exe_file) + test.support.unlink(_pth_file) + test.support.unlink(exe_file) @classmethod def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 97dc3cd..80dfc40 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -803,11 +803,6 @@ class GeneralModuleTests(unittest.TestCase): self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) def test_host_resolution(self): - for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', - '1:1:1:1:1:1:1:1:1']: - self.assertRaises(OSError, socket.gethostbyname, addr) - self.assertRaises(OSError, socket.gethostbyaddr, addr) - for addr in [support.HOST, '10.0.0.1', '255.255.255.255']: self.assertEqual(socket.gethostbyname(addr), addr) @@ -816,6 +811,21 @@ class GeneralModuleTests(unittest.TestCase): for host in [support.HOST]: self.assertIn(host, socket.gethostbyaddr(host)[2]) + def test_host_resolution_bad_address(self): + # These are all malformed IP addresses and expected not to resolve to + # any result. But some ISPs, e.g. AWS, may successfully resolve these + # IPs. + explanation = ( + "resolving an invalid IP address did not raise OSError; " + "can be caused by a broken DNS server" + ) + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + with self.assertRaises(OSError): + socket.gethostbyname(addr) + with self.assertRaises(OSError, msg=explanation): + socket.gethostbyaddr(addr) + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") def test_sethostname(self): @@ -896,6 +906,7 @@ class GeneralModuleTests(unittest.TestCase): self.assertEqual(swapped & mask, mask) self.assertRaises(OverflowError, func, 1<<34) + @support.cpython_only def testNtoHErrors(self): good_values = [ 1, 2, 3, 1, 2, 3 ] bad_values = [ -1, -2, -3, -1, -2, -3 ] @@ -4649,6 +4660,10 @@ class TestUnixDomain(unittest.TestCase): else: raise + def testUnbound(self): + # Issue #30205 + self.assertIn(self.sock.getsockname(), ('', None)) + def testStrAddr(self): # Test binding to and retrieving a normal string pathname. path = os.path.abspath(support.TESTFN) @@ -5469,7 +5484,7 @@ class LinuxKernelCryptoAPI(unittest.TestCase): self.assertEqual(len(dec), msglen * multiplier) self.assertEqual(dec, msg * multiplier) - @support.requires_linux_version(4, 3) # see test_aes_cbc + @support.requires_linux_version(4, 9) # see issue29324 def test_aead_aes_gcm(self): key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') @@ -5492,8 +5507,7 @@ class LinuxKernelCryptoAPI(unittest.TestCase): op.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, iv=iv, assoclen=assoclen, flags=socket.MSG_MORE) op.sendall(assoc, socket.MSG_MORE) - op.sendall(plain, socket.MSG_MORE) - op.sendall(b'\x00' * taglen) + op.sendall(plain) res = op.recv(assoclen + len(plain) + taglen) self.assertEqual(expected_ct, res[assoclen:-taglen]) self.assertEqual(expected_tag, res[-taglen:]) @@ -5501,7 +5515,7 @@ class LinuxKernelCryptoAPI(unittest.TestCase): # now with msg op, _ = algo.accept() with op: - msg = assoc + plain + b'\x00' * taglen + msg = assoc + plain op.sendmsg_afalg([msg], op=socket.ALG_OP_ENCRYPT, iv=iv, assoclen=assoclen) res = op.recv(assoclen + len(plain) + taglen) @@ -5512,7 +5526,7 @@ class LinuxKernelCryptoAPI(unittest.TestCase): pack_uint32 = struct.Struct('I').pack op, _ = algo.accept() with op: - msg = assoc + plain + b'\x00' * taglen + msg = assoc + plain op.sendmsg( [msg], ([socket.SOL_ALG, socket.ALG_SET_OP, pack_uint32(socket.ALG_OP_ENCRYPT)], @@ -5520,7 +5534,7 @@ class LinuxKernelCryptoAPI(unittest.TestCase): [socket.SOL_ALG, socket.ALG_SET_AEAD_ASSOCLEN, pack_uint32(assoclen)], ) ) - res = op.recv(len(msg)) + res = op.recv(len(msg) + taglen) self.assertEqual(expected_ct, res[assoclen:-taglen]) self.assertEqual(expected_tag, res[-taglen:]) @@ -5530,8 +5544,8 @@ class LinuxKernelCryptoAPI(unittest.TestCase): msg = assoc + expected_ct + expected_tag op.sendmsg_afalg([msg], op=socket.ALG_OP_DECRYPT, iv=iv, assoclen=assoclen) - res = op.recv(len(msg)) - self.assertEqual(plain, res[assoclen:-taglen]) + res = op.recv(len(msg) - taglen) + self.assertEqual(plain, res[assoclen:]) @support.requires_linux_version(4, 3) # see test_aes_cbc def test_drbg_pr_sha256(self): diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index d203cdd..85c59a6 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2065,7 +2065,7 @@ if _have_threads: class EchoServer (asyncore.dispatcher): - class ConnectionHandler (asyncore.dispatcher_with_send): + class ConnectionHandler(asyncore.dispatcher_with_send): def __init__(self, conn, certfile): self.socket = test_wrap_socket(conn, server_side=True, @@ -2156,6 +2156,8 @@ if _have_threads: self.join() if support.verbose: sys.stdout.write(" cleanup: successfully joined.\n") + # make sure that ConnectionHandler is removed from socket_map + asyncore.close_all(ignore_all=True) def start (self, flag=None): self.flag = flag diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 3c871dd..9a9e3ca 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -644,6 +644,46 @@ class ProcessTestCase(BaseTestCase): # environment b"['__CF_USER_TEXT_ENCODING']")) + def test_invalid_cmd(self): + # null character in the command name + cmd = sys.executable + '\0' + with self.assertRaises(ValueError): + subprocess.Popen([cmd, "-c", "pass"]) + + # null character in the command argument + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass#\0"]) + + def test_invalid_env(self): + # null character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # null character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange=lemon" + with subprocess.Popen([sys.executable, "-c", + 'import sys, os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, stderr = p.communicate() + self.assertEqual(stdout, b"orange=lemon") + def test_communicate_stdin(self): p = subprocess.Popen([sys.executable, "-c", 'import sys;' @@ -2431,7 +2471,7 @@ class POSIXProcessTestCase(BaseTestCase): with self.assertRaises(TypeError): _posixsubprocess.fork_exec( args, exe_list, - True, [], cwd, env_list, + True, (), cwd, env_list, -1, -1, -1, -1, 1, 2, 3, 4, True, True, func) @@ -2443,6 +2483,16 @@ class POSIXProcessTestCase(BaseTestCase): def test_fork_exec_sorted_fd_sanity_check(self): # Issue #23564: sanity check the fork_exec() fds_to_keep sanity check. import _posixsubprocess + class BadInt: + first = True + def __init__(self, value): + self.value = value + def __int__(self): + if self.first: + self.first = False + return self.value + raise ValueError + gc_enabled = gc.isenabled() try: gc.enable() @@ -2453,6 +2503,7 @@ class POSIXProcessTestCase(BaseTestCase): (18, 23, 42, 2**63), # Out of range. (5, 4), # Not sorted. (6, 7, 7, 8), # Duplicate. + (BadInt(1), BadInt(2)), ): with self.assertRaises( ValueError, diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index e83a4d6..ddd6509 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -282,17 +282,34 @@ class TestSupport(unittest.TestCase): def test_swap_attr(self): class Obj: - x = 1 + pass obj = Obj() - with support.swap_attr(obj, "x", 5): + obj.x = 1 + with support.swap_attr(obj, "x", 5) as x: self.assertEqual(obj.x, 5) + self.assertEqual(x, 1) self.assertEqual(obj.x, 1) + with support.swap_attr(obj, "y", 5) as y: + self.assertEqual(obj.y, 5) + self.assertIsNone(y) + self.assertFalse(hasattr(obj, 'y')) + with support.swap_attr(obj, "y", 5): + del obj.y + self.assertFalse(hasattr(obj, 'y')) def test_swap_item(self): - D = {"item":1} - with support.swap_item(D, "item", 5): - self.assertEqual(D["item"], 5) - self.assertEqual(D["item"], 1) + D = {"x":1} + with support.swap_item(D, "x", 5) as x: + self.assertEqual(D["x"], 5) + self.assertEqual(x, 1) + self.assertEqual(D["x"], 1) + with support.swap_item(D, "y", 5) as y: + self.assertEqual(D["y"], 5) + self.assertIsNone(y) + self.assertNotIn("y", D) + with support.swap_item(D, "y", 5): + del D["y"] + self.assertNotIn("y", D) class RefClass: attribute1 = None diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index df9ebd4..e151f49 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -913,13 +913,15 @@ class SizeofTest(unittest.TestCase): return inner check(get_cell().__closure__[0], size('P')) # code - check(get_cell().__code__, size('6i13P')) - check(get_cell.__code__, size('6i13P')) + def check_code_size(a, expected_size): + self.assertGreaterEqual(sys.getsizeof(a), expected_size) + check_code_size(get_cell().__code__, size('6i13P')) + check_code_size(get_cell.__code__, size('6i13P')) def get_cell2(x): def inner(): return x return inner - check(get_cell2.__code__, size('6i13P') + 1) + check_code_size(get_cell2.__code__, size('6i13P') + calcsize('n')) # complex check(complex(0,1), size('2d')) # method_descriptor (descriptor object) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 619cbc0..fc79055 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -4,6 +4,7 @@ import io from hashlib import md5 from contextlib import contextmanager from random import Random +import pathlib import unittest import unittest.mock @@ -440,6 +441,22 @@ class MiscReadTestBase(CommonReadTest): self.assertIsInstance(tar.name, bytes) self.assertEqual(tar.name, os.path.abspath(fobj.name)) + def test_pathlike_name(self): + tarname = pathlib.Path(self.tarname) + with tarfile.open(tarname, mode=self.mode) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + with self.taropen(tarname) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + with tarfile.TarFile.open(tarname, mode=self.mode) as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + if self.suffix == '': + with tarfile.TarFile(tarname, mode='r') as tar: + self.assertIsInstance(tar.name, str) + self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + def test_illegal_mode_arg(self): with open(tmpname, 'wb'): pass @@ -582,6 +599,26 @@ class MiscReadTestBase(CommonReadTest): finally: support.rmtree(DIR) + def test_extractall_pathlike_name(self): + DIR = pathlib.Path(TEMPDIR) / "extractall" + with support.temp_dir(DIR), \ + tarfile.open(tarname, encoding="iso8859-1") as tar: + directories = [t for t in tar if t.isdir()] + tar.extractall(DIR, directories) + for tarinfo in directories: + path = DIR / tarinfo.name + self.assertEqual(os.path.getmtime(path), tarinfo.mtime) + + def test_extract_pathlike_name(self): + dirtype = "ustar/dirtype" + DIR = pathlib.Path(TEMPDIR) / "extractall" + with support.temp_dir(DIR), \ + tarfile.open(tarname, encoding="iso8859-1") as tar: + tarinfo = tar.getmember(dirtype) + tar.extract(tarinfo, path=DIR) + extracted = DIR / dirtype + self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) + def test_init_close_fobj(self): # Issue #7341: Close the internal file object in the TarFile # constructor in case of an error. For the test we rely on @@ -1092,6 +1129,17 @@ class WriteTest(WriteTestBase, unittest.TestCase): finally: support.rmdir(path) + def test_gettarinfo_pathlike_name(self): + with tarfile.open(tmpname, self.mode) as tar: + path = pathlib.Path(TEMPDIR) / "file" + with open(path, "wb") as fobj: + fobj.write(b"aaa") + tarinfo = tar.gettarinfo(path) + tarinfo2 = tar.gettarinfo(os.fspath(path)) + self.assertIsInstance(tarinfo.name, str) + self.assertEqual(tarinfo.name, tarinfo2.name) + self.assertEqual(tarinfo.size, 3) + @unittest.skipUnless(hasattr(os, "link"), "Missing hardlink implementation") def test_link_size(self): @@ -1528,6 +1576,34 @@ class CreateTest(WriteTestBase, unittest.TestCase): self.assertEqual(len(names), 1) self.assertIn("spameggs42", names[0]) + def test_create_pathlike_name(self): + with tarfile.open(pathlib.Path(tmpname), self.mode) as tobj: + self.assertIsInstance(tobj.name, str) + self.assertEqual(tobj.name, os.path.abspath(tmpname)) + tobj.add(pathlib.Path(self.file_path)) + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + with self.taropen(tmpname) as tobj: + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + def test_create_taropen_pathlike_name(self): + with self.taropen(pathlib.Path(tmpname), "x") as tobj: + self.assertIsInstance(tobj.name, str) + self.assertEqual(tobj.name, os.path.abspath(tmpname)) + tobj.add(pathlib.Path(self.file_path)) + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + + with self.taropen(tmpname) as tobj: + names = tobj.getnames() + self.assertEqual(len(names), 1) + self.assertIn('spameggs42', names[0]) + class GzipCreateTest(GzipTest, CreateTest): pass diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 51df1ec..d0cf04b 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -273,13 +273,12 @@ class TestGetDefaultTempdir(BaseTestCase): tempfile._get_default_tempdir() self.assertEqual(os.listdir(our_temp_directory), []) - open = io.open def bad_writer(*args, **kwargs): - fp = open(*args, **kwargs) + fp = orig_open(*args, **kwargs) fp.write = raise_OSError return fp - with support.swap_attr(io, "open", bad_writer): + with support.swap_attr(io, "open", bad_writer) as orig_open: # test again with failing write() with self.assertRaises(FileNotFoundError): tempfile._get_default_tempdir() diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index ef3059b..3909b75 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -20,6 +20,7 @@ def verbose_print(arg): with _print_mutex: print(arg) + class BasicThreadTest(unittest.TestCase): def setUp(self): @@ -31,6 +32,9 @@ class BasicThreadTest(unittest.TestCase): self.running = 0 self.next_ident = 0 + key = support.threading_setup() + self.addCleanup(support.threading_cleanup, *key) + class ThreadRunningTests(BasicThreadTest): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 6b6c4d2..0db0288 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -470,13 +470,15 @@ class ThreadTests(BaseTestCase): for i in range(20): t = threading.Thread(target=lambda: None) t.start() - self.addCleanup(t.join) pid = os.fork() if pid == 0: - os._exit(1 if t.is_alive() else 0) + os._exit(11 if t.is_alive() else 10) else: + t.join() + pid, status = os.waitpid(pid, 0) - self.assertEqual(0, status) + self.assertTrue(os.WIFEXITED(status)) + self.assertEqual(10, os.WEXITSTATUS(status)) def test_main_thread(self): main = threading.main_thread() diff --git a/Lib/test/test_tools/test_reindent.py b/Lib/test/test_tools/test_reindent.py index d7c20e1..34df0c5 100644 --- a/Lib/test/test_tools/test_reindent.py +++ b/Lib/test/test_tools/test_reindent.py @@ -7,6 +7,7 @@ Tools directory of a Python checkout or tarball, such as reindent.py. import os import unittest from test.support.script_helper import assert_python_ok +from test.support import findfile from test.test_tools import scriptsdir, skip_if_missing @@ -23,6 +24,12 @@ class ReindentTests(unittest.TestCase): self.assertEqual(out, b'') self.assertGreater(err, b'') + def test_reindent_file_with_bad_encoding(self): + bad_coding_path = findfile('bad_coding.py') + rc, out, err = assert_python_ok(self.script, '-r', bad_coding_path) + self.assertEqual(out, b'') + self.assertNotEqual(err, b'') + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 790ab7e..742259b 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -865,6 +865,7 @@ class TestCommandLine(unittest.TestCase): b'number of frames', stderr) + @unittest.skipIf(_testcapi is None, 'need _testcapi') def test_pymem_alloc0(self): # Issue #21639: Check that PyMem_Malloc(0) with tracemalloc enabled # does not crash. diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f0070ec..6d0b265 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -6,7 +6,7 @@ import sys from unittest import TestCase, main, skipUnless, SkipTest from copy import copy, deepcopy -from typing import Any +from typing import Any, NoReturn from typing import TypeVar, AnyStr from typing import T, KT, VT # Not in __all__. from typing import Union, Optional @@ -102,10 +102,6 @@ class AnyTests(BaseTestCase): with self.assertRaises(TypeError): type(Any)() - def test_cannot_subscript(self): - with self.assertRaises(TypeError): - Any[int] - def test_any_works_with_alias(self): # These expressions must simply not fail. typing.Match[Any] @@ -113,6 +109,40 @@ class AnyTests(BaseTestCase): typing.IO[Any] +class NoReturnTests(BaseTestCase): + + def test_noreturn_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, NoReturn) + + def test_noreturn_subclass_type_error(self): + with self.assertRaises(TypeError): + issubclass(Employee, NoReturn) + with self.assertRaises(TypeError): + issubclass(NoReturn, Employee) + + def test_repr(self): + self.assertEqual(repr(NoReturn), 'typing.NoReturn') + + def test_not_generic(self): + with self.assertRaises(TypeError): + NoReturn[int] + + def test_cannot_subclass(self): + with self.assertRaises(TypeError): + class A(NoReturn): + pass + with self.assertRaises(TypeError): + class A(type(NoReturn)): + pass + + def test_cannot_instantiate(self): + with self.assertRaises(TypeError): + NoReturn() + with self.assertRaises(TypeError): + type(NoReturn)() + + class TypeVarTests(BaseTestCase): def test_basic_plain(self): @@ -1522,6 +1552,12 @@ class AsyncIteratorWrapper(typing.AsyncIterator[T_a]): return data else: raise StopAsyncIteration + +class ACM: + async def __aenter__(self) -> int: + return 42 + async def __aexit__(self, etype, eval, tb): + return None """ if ASYNCIO: @@ -1532,12 +1568,13 @@ if ASYNCIO: else: # fake names for the sake of static analysis asyncio = None - AwaitableWrapper = AsyncIteratorWrapper = object + AwaitableWrapper = AsyncIteratorWrapper = ACM = object PY36 = sys.version_info[:2] >= (3, 6) PY36_TESTS = """ from test import ann_module, ann_module2, ann_module3 +from typing import AsyncContextManager class A: y: float @@ -1574,6 +1611,16 @@ class XRepr(NamedTuple): return f'{self.x} -> {self.y}' def __add__(self, other): return 0 + +async def g_with(am: AsyncContextManager[int]): + x: int + async with am as x: + return x + +try: + g_with(ACM()).send(None) +except StopIteration as e: + assert e.args[0] == 42 """ if PY36: @@ -2126,8 +2173,6 @@ class CollectionsAbcTests(BaseTestCase): class OtherABCTests(BaseTestCase): - @skipUnless(hasattr(typing, 'ContextManager'), - 'requires typing.ContextManager') def test_contextmanager(self): @contextlib.contextmanager def manager(): @@ -2137,6 +2182,24 @@ class OtherABCTests(BaseTestCase): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) + @skipUnless(ASYNCIO, 'Python 3.5 required') + def test_async_contextmanager(self): + class NotACM: + pass + self.assertIsInstance(ACM(), typing.AsyncContextManager) + self.assertNotIsInstance(NotACM(), typing.AsyncContextManager) + @contextlib.contextmanager + def manager(): + yield 42 + + cm = manager() + self.assertNotIsInstance(cm, typing.AsyncContextManager) + self.assertEqual(typing.AsyncContextManager[int].__args__, (int,)) + with self.assertRaises(TypeError): + isinstance(42, typing.AsyncContextManager[int]) + with self.assertRaises(TypeError): + typing.AsyncContextManager[int, str] + class TypeTests(BaseTestCase): @@ -2273,6 +2336,14 @@ class XMethBad(NamedTuple): return 'no chance for this' """) + with self.assertRaises(AttributeError): + exec(""" +class XMethBad2(NamedTuple): + x: int + def _source(self): + return 'no chance for this as well' +""") + @skipUnless(PY36, 'Python 3.6 required') def test_namedtuple_keyword_usage(self): LocalEmployee = NamedTuple("LocalEmployee", name=str, age=int) @@ -2420,6 +2491,9 @@ class AllTests(BaseTestCase): self.assertNotIn('sys', a) # Check that Text is defined. self.assertIn('Text', a) + # Check previously missing classes. + self.assertIn('SupportsBytes', a) + self.assertIn('SupportsComplex', a) if __name__ == '__main__': diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 5084486..fa3757c 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -206,6 +206,7 @@ class urlopen_FileTests(unittest.TestCase): def test_relativelocalfile(self): self.assertRaises(ValueError,urllib.request.urlopen,'./' + self.pathname) + class ProxyTests(unittest.TestCase): def setUp(self): @@ -259,6 +260,7 @@ class ProxyTests(unittest.TestCase): self.assertFalse(bypass('newdomain.com')) # no port self.assertFalse(bypass('newdomain.com:1235')) # wrong port + class ProxyTests_withOrderedEnv(unittest.TestCase): def setUp(self): @@ -294,6 +296,7 @@ class ProxyTests_withOrderedEnv(unittest.TestCase): proxies = urllib.request.getproxies_environment() self.assertEqual('http://somewhere:3128', proxies['http']) + class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): """Test urlopen() opening a fake http connection.""" @@ -432,7 +435,6 @@ Connection: close finally: self.unfakeftp() - def test_userpass_inurl(self): self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") try: @@ -476,6 +478,7 @@ Connection: close "https://localhost", cafile="/nonexistent/path", context=context ) + class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" @@ -549,6 +552,7 @@ class urlopen_DataTests(unittest.TestCase): # missing padding character self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=') + class urlretrieve_FileTests(unittest.TestCase): """Test urllib.urlretrieve() on local files""" diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 34329f8..876fcd4 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -141,44 +141,55 @@ class RequestHdrsTests(unittest.TestCase): mgr = urllib.request.HTTPPasswordMgr() add = mgr.add_password find_user_pass = mgr.find_user_password + add("Some Realm", "http://example.com/", "joe", "password") add("Some Realm", "http://example.com/ni", "ni", "ni") - add("c", "http://example.com/foo", "foo", "ni") - add("c", "http://example.com/bar", "bar", "nini") - add("b", "http://example.com/", "first", "blah") - add("b", "http://example.com/", "second", "spam") - add("a", "http://example.com", "1", "a") add("Some Realm", "http://c.example.com:3128", "3", "c") add("Some Realm", "d.example.com", "4", "d") add("Some Realm", "e.example.com:3128", "5", "e") + # For the same realm, password set the highest path is the winner. self.assertEqual(find_user_pass("Some Realm", "example.com"), ('joe', 'password')) - - #self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), - # ('ni', 'ni')) - + self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), + ('joe', 'password')) self.assertEqual(find_user_pass("Some Realm", "http://example.com"), ('joe', 'password')) self.assertEqual(find_user_pass("Some Realm", "http://example.com/"), ('joe', 'password')) - self.assertEqual( - find_user_pass("Some Realm", "http://example.com/spam"), - ('joe', 'password')) - self.assertEqual( - find_user_pass("Some Realm", "http://example.com/spam/spam"), - ('joe', 'password')) + self.assertEqual(find_user_pass("Some Realm", + "http://example.com/spam"), + ('joe', 'password')) + + self.assertEqual(find_user_pass("Some Realm", + "http://example.com/spam/spam"), + ('joe', 'password')) + + # You can have different passwords for different paths. + + add("c", "http://example.com/foo", "foo", "ni") + add("c", "http://example.com/bar", "bar", "nini") + self.assertEqual(find_user_pass("c", "http://example.com/foo"), ('foo', 'ni')) + self.assertEqual(find_user_pass("c", "http://example.com/bar"), ('bar', 'nini')) + + # For the same path, newer password should be considered. + + add("b", "http://example.com/", "first", "blah") + add("b", "http://example.com/", "second", "spam") + self.assertEqual(find_user_pass("b", "http://example.com/"), ('second', 'spam')) # No special relationship between a.example.com and example.com: + add("a", "http://example.com", "1", "a") self.assertEqual(find_user_pass("a", "http://example.com/"), ('1', 'a')) + self.assertEqual(find_user_pass("a", "http://a.example.com/"), (None, None)) @@ -830,7 +841,6 @@ class HandlerTests(unittest.TestCase): for url, ftp in [ ("file://ftp.example.com//foo.txt", False), ("file://ftp.example.com///foo.txt", False), -# XXXX bug: fails with OSError, should be URLError ("file://ftp.example.com/foo.txt", False), ("file://somehost//foo/something.txt", False), ("file://localhost//foo/something.txt", False), @@ -838,8 +848,7 @@ class HandlerTests(unittest.TestCase): req = Request(url) try: h.file_open(req) - # XXXX remove OSError when bug fixed - except (urllib.error.URLError, OSError): + except urllib.error.URLError: self.assertFalse(ftp) else: self.assertIs(o.req, req) @@ -1414,7 +1423,6 @@ class HandlerTests(unittest.TestCase): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") - # TODO: This should be only for OSX @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { @@ -1690,7 +1698,6 @@ class HandlerTests(unittest.TestCase): self.assertTrue(conn.fakesock.closed, "Connection not closed") - class MiscTests(unittest.TestCase): def opener_has_handler(self, opener, handler_class): diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 949716c..4103b6c 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -11,6 +11,7 @@ import time support.requires('network') + class URLTimeoutTest(unittest.TestCase): # XXX this test doesn't seem to test anything useful. @@ -25,11 +26,11 @@ class URLTimeoutTest(unittest.TestCase): def testURLread(self): with support.transient_internet("www.example.com"): f = urllib.request.urlopen("http://www.example.com/") - x = f.read() + f.read() class urlopenNetworkTests(unittest.TestCase): - """Tests urllib.reqest.urlopen using the network. + """Tests urllib.request.urlopen using the network. These tests are not exhaustive. Assuming that testing using files does a good job overall of some of the basic interface features. There are no @@ -188,6 +189,7 @@ class urlretrieveNetworkTests(unittest.TestCase): def test_reporthook(self): records = [] + def recording_reporthook(blocks, block_size, total_size): records.append((blocks, block_size, total_size)) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 99c5c03..e5f6130 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -6,8 +6,8 @@ RFC2396_BASE = "http://a/b/c/d;p?q" RFC3986_BASE = 'http://a/b/c/d;p?q' SIMPLE_BASE = 'http://a/b/c/d' -# A list of test cases. Each test case is a two-tuple that contains -# a string with the query and a dictionary with the expected result. +# Each parse_qsl testcase is a two-tuple that contains +# a string with the query and a list with the expected result. parse_qsl_test_cases = [ ("", []), @@ -42,6 +42,9 @@ parse_qsl_test_cases = [ (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), ] +# Each parse_qs testcase is a two-tuple that contains +# a string with the query and a dictionary with the expected result. + parse_qs_test_cases = [ ("", {}), ("&", {}), @@ -290,7 +293,6 @@ class UrlParseTestCase(unittest.TestCase): def test_RFC2396(self): # cases from RFC 2396 - self.checkJoin(RFC2396_BASE, 'g:h', 'g:h') self.checkJoin(RFC2396_BASE, 'g', 'http://a/b/c/g') self.checkJoin(RFC2396_BASE, './g', 'http://a/b/c/g') @@ -333,9 +335,7 @@ class UrlParseTestCase(unittest.TestCase): # self.checkJoin(RFC2396_BASE, '/./g', 'http://a/./g') # self.checkJoin(RFC2396_BASE, '/../g', 'http://a/../g') - def test_RFC3986(self): - # Test cases from RFC3986 self.checkJoin(RFC3986_BASE, '?y','http://a/b/c/d;p?y') self.checkJoin(RFC3986_BASE, ';x', 'http://a/b/c/;x') self.checkJoin(RFC3986_BASE, 'g:h','g:h') @@ -363,7 +363,7 @@ class UrlParseTestCase(unittest.TestCase): self.checkJoin(RFC3986_BASE, '../../g','http://a/g') self.checkJoin(RFC3986_BASE, '../../../g', 'http://a/g') - #Abnormal Examples + # Abnormal Examples # The 'abnormal scenarios' are incompatible with RFC2986 parsing # Tests are here for reference. @@ -755,28 +755,35 @@ class UrlParseTestCase(unittest.TestCase): def test_parse_fragments(self): # Exercise the allow_fragments parameter of urlparse() and urlsplit() tests = ( - ("http:#frag", "path"), - ("//example.net#frag", "path"), - ("index.html#frag", "path"), - (";a=b#frag", "params"), - ("?a=b#frag", "query"), - ("#frag", "path"), + ("http:#frag", "path", "frag"), + ("//example.net#frag", "path", "frag"), + ("index.html#frag", "path", "frag"), + (";a=b#frag", "params", "frag"), + ("?a=b#frag", "query", "frag"), + ("#frag", "path", "frag"), + ("abc#@frag", "path", "@frag"), + ("//abc#@frag", "path", "@frag"), + ("//abc:80#@frag", "path", "@frag"), + ("//abc#@frag:80", "path", "@frag:80"), ) - for url, attr in tests: + for url, attr, expected_frag in tests: for func in (urllib.parse.urlparse, urllib.parse.urlsplit): if attr == "params" and func is urllib.parse.urlsplit: attr = "path" with self.subTest(url=url, function=func): result = func(url, allow_fragments=False) self.assertEqual(result.fragment, "") - self.assertTrue(getattr(result, attr).endswith("#frag")) + self.assertTrue( + getattr(result, attr).endswith("#" + expected_frag)) self.assertEqual(func(url, "", False).fragment, "") result = func(url, allow_fragments=True) - self.assertEqual(result.fragment, "frag") - self.assertFalse(getattr(result, attr).endswith("frag")) - self.assertEqual(func(url, "", True).fragment, "frag") - self.assertEqual(func(url).fragment, "frag") + self.assertEqual(result.fragment, expected_frag) + self.assertFalse( + getattr(result, attr).endswith(expected_frag)) + self.assertEqual(func(url, "", True).fragment, + expected_frag) + self.assertEqual(func(url).fragment, expected_frag) def test_mixed_types_rejected(self): # Several functions that process either strings or ASCII encoded bytes @@ -983,6 +990,26 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splithost('/foo/bar/baz.html'), (None, '/foo/bar/baz.html')) + # bpo-30500: # starts a fragment. + self.assertEqual(splithost('//127.0.0.1#@host.com'), + ('127.0.0.1', '/#@host.com')) + self.assertEqual(splithost('//127.0.0.1#@host.com:80'), + ('127.0.0.1', '/#@host.com:80')) + self.assertEqual(splithost('//127.0.0.1:80#@host.com'), + ('127.0.0.1:80', '/#@host.com')) + + # Empty host is returned as empty string. + self.assertEqual(splithost("///file"), + ('', '/file')) + + # Trailing semicolon, question mark and hash symbol are kept. + self.assertEqual(splithost("//example.net/file;"), + ('example.net', '/file;')) + self.assertEqual(splithost("//example.net/file?"), + ('example.net', '/file?')) + self.assertEqual(splithost("//example.net/file#"), + ('example.net', '/file#')) + def test_splituser(self): splituser = urllib.parse.splituser self.assertEqual(splituser('User:Pass@www.python.org:080'), diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index c0144d1..dbdad23 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1879,6 +1879,118 @@ class BadElementTest(ElementTestCase, unittest.TestCase): with self.assertRaises(RuntimeError): repr(e) # Should not crash + def test_element_get_text(self): + # Issue #27863 + class X(str): + def __del__(self): + try: + elem.text + except NameError: + pass + + b = ET.TreeBuilder() + b.start('tag', {}) + b.data('ABCD') + b.data(X('EFGH')) + b.data('IJKL') + b.end('tag') + + elem = b.close() + self.assertEqual(elem.text, 'ABCDEFGHIJKL') + + def test_element_get_tail(self): + # Issue #27863 + class X(str): + def __del__(self): + try: + elem[0].tail + except NameError: + pass + + b = ET.TreeBuilder() + b.start('root', {}) + b.start('tag', {}) + b.end('tag') + b.data('ABCD') + b.data(X('EFGH')) + b.data('IJKL') + b.end('root') + + elem = b.close() + self.assertEqual(elem[0].tail, 'ABCDEFGHIJKL') + + def test_element_iter(self): + # Issue #27863 + state = { + 'tag': 'tag', + '_children': [None], # non-Element + 'attrib': 'attr', + 'tail': 'tail', + 'text': 'text', + } + + e = ET.Element('tag') + try: + e.__setstate__(state) + except AttributeError: + e.__dict__ = state + + it = e.iter() + self.assertIs(next(it), e) + self.assertRaises(AttributeError, next, it) + + def test_subscr(self): + # Issue #27863 + class X: + def __index__(self): + del e[:] + return 1 + + e = ET.Element('elem') + e.append(ET.Element('child')) + e[:X()] # shouldn't crash + + e.append(ET.Element('child')) + e[0:10:X()] # shouldn't crash + + def test_ass_subscr(self): + # Issue #27863 + class X: + def __index__(self): + e[:] = [] + return 1 + + e = ET.Element('elem') + for _ in range(10): + e.insert(0, ET.Element('child')) + + e[0:10:X()] = [] # shouldn't crash + + def test_treebuilder_start(self): + # Issue #27863 + def element_factory(x, y): + return [] + b = ET.TreeBuilder(element_factory=element_factory) + + b.start('tag', {}) + b.data('ABCD') + self.assertRaises(AttributeError, b.start, 'tag2', {}) + del b + gc_collect() + + def test_treebuilder_end(self): + # Issue #27863 + def element_factory(x, y): + return [] + b = ET.TreeBuilder(element_factory=element_factory) + + b.start('tag', {}) + b.data('ABCD') + self.assertRaises(AttributeError, b.end, 'tag') + del b + gc_collect() + + class MutatingElementPath(str): def __new__(cls, elem, *args): self = str.__new__(cls, *args) diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index df9c79e..30025e3 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -343,6 +343,94 @@ class XMLRPCTestCase(unittest.TestCase): self.assertEqual(p.method(), 5) self.assertEqual(p.method(), 5) + +class SimpleXMLRPCDispatcherTestCase(unittest.TestCase): + class DispatchExc(Exception): + """Raised inside the dispatched functions when checking for + chained exceptions""" + + def test_call_registered_func(self): + """Calls explicitly registered function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_params = 1, 2, 3 + + def dispatched_func(*params): + raise self.DispatchExc(params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_function(dispatched_func) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch('dispatched_func', exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_params,)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_call_instance_func(self): + """Calls a registered instance attribute as a function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_params = 1, 2, 3 + + class DispatchedClass: + def dispatched_func(self, *params): + raise SimpleXMLRPCDispatcherTestCase.DispatchExc(params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(DispatchedClass()) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch('dispatched_func', exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_params,)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_call_dispatch_func(self): + """Calls the registered instance's `_dispatch` function""" + # Makes sure any exception raised inside the function has no other + # exception chained to it + + exp_method = 'method' + exp_params = 1, 2, 3 + + class TestInstance: + def _dispatch(self, method, params): + raise SimpleXMLRPCDispatcherTestCase.DispatchExc( + method, params) + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(TestInstance()) + with self.assertRaises(self.DispatchExc) as exc_ctx: + dispatcher._dispatch(exp_method, exp_params) + self.assertEqual(exc_ctx.exception.args, (exp_method, exp_params)) + self.assertIsNone(exc_ctx.exception.__cause__) + self.assertIsNone(exc_ctx.exception.__context__) + + def test_registered_func_is_none(self): + """Calls explicitly registered function which is None""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_function(None, name='method') + with self.assertRaisesRegex(Exception, 'method'): + dispatcher._dispatch('method', ('param',)) + + def test_instance_has_no_func(self): + """Attempts to call nonexistent function on a registered instance""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + dispatcher.register_instance(object()) + with self.assertRaisesRegex(Exception, 'method'): + dispatcher._dispatch('method', ('param',)) + + def test_cannot_locate_func(self): + """Calls a function that the dispatcher cannot locate""" + + dispatcher = xmlrpc.server.SimpleXMLRPCDispatcher() + with self.assertRaisesRegex(Exception, 'method'): + dispatcher._dispatch('method', ('param',)) + + class HelperTestCase(unittest.TestCase): def test_escape(self): self.assertEqual(xmlrpclib.escape("a&b"), "a&b") @@ -1312,7 +1400,7 @@ def test_main(): KeepaliveServerTestCase1, KeepaliveServerTestCase2, GzipServerTestCase, GzipUtilTestCase, MultiPathServerTestCase, ServerProxyTestCase, FailingServerTestCase, - CGIHandlerTestCase) + CGIHandlerTestCase, SimpleXMLRPCDispatcherTestCase) if __name__ == "__main__": diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 0a43b20..d09ad96 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -2,6 +2,7 @@ import contextlib import io import os import importlib.util +import pathlib import posixpath import time import struct @@ -13,7 +14,7 @@ from tempfile import TemporaryFile from random import randint, random, getrandbits from test.support import script_helper -from test.support import (TESTFN, findfile, unlink, rmtree, temp_dir, +from test.support import (TESTFN, findfile, unlink, rmtree, temp_dir, temp_cwd, requires_zlib, requires_bz2, requires_lzma, captured_stdout, check_warnings) @@ -148,6 +149,12 @@ class AbstractTestsWithSourceFile: for f in get_files(self): self.zip_open_test(f, self.compression) + def test_open_with_pathlike(self): + path = pathlib.Path(TESTFN2) + self.zip_open_test(path, self.compression) + with zipfile.ZipFile(path, "r", self.compression) as zipfp: + self.assertIsInstance(zipfp.filename, str) + def zip_random_open_test(self, f, compression): self.make_test_archive(f, compression) @@ -727,6 +734,48 @@ class LzmaTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, compression = zipfile.ZIP_LZMA +class AbstractWriterTests: + + def tearDown(self): + unlink(TESTFN2) + + def test_close_after_close(self): + data = b'content' + with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipf: + w = zipf.open('test', 'w') + w.write(data) + w.close() + self.assertTrue(w.closed) + w.close() + self.assertTrue(w.closed) + self.assertEqual(zipf.read('test'), data) + + def test_write_after_close(self): + data = b'content' + with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipf: + w = zipf.open('test', 'w') + w.write(data) + w.close() + self.assertTrue(w.closed) + self.assertRaises(ValueError, w.write, b'') + self.assertEqual(zipf.read('test'), data) + +class StoredWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_STORED + +@requires_zlib +class DeflateWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + +@requires_bz2 +class Bzip2WriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + +@requires_lzma +class LzmaWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_LZMA + + class PyZipFileTests(unittest.TestCase): def assertCompiledIn(self, name, namelist): if name + 'o' not in namelist: @@ -906,22 +955,56 @@ class PyZipFileTests(unittest.TestCase): finally: rmtree(TESTFN2) + def test_write_pathlike(self): + os.mkdir(TESTFN2) + try: + with open(os.path.join(TESTFN2, "mod1.py"), "w") as fp: + fp.write("print(42)\n") + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") + names = zipfp.namelist() + self.assertCompiledIn('mod1.py', names) + finally: + rmtree(TESTFN2) + class ExtractTests(unittest.TestCase): - def test_extract(self): + + def make_test_file(self): with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: for fpath, fdata in SMALL_TEST_DATA: zipfp.writestr(fpath, fdata) + def test_extract(self): + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + for fpath, fdata in SMALL_TEST_DATA: + writtenfile = zipfp.extract(fpath) + + # make sure it was written to the right place + correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.normpath(correctfile) + + self.assertEqual(writtenfile, correctfile) + + # make sure correct data is in correct file + with open(writtenfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(writtenfile) + + def _test_extract_with_target(self, target): + self.make_test_file() with zipfile.ZipFile(TESTFN2, "r") as zipfp: for fpath, fdata in SMALL_TEST_DATA: - writtenfile = zipfp.extract(fpath) + writtenfile = zipfp.extract(fpath, target) # make sure it was written to the right place - correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.join(target, fpath) correctfile = os.path.normpath(correctfile) - - self.assertEqual(writtenfile, correctfile) + self.assertTrue(os.path.samefile(writtenfile, correctfile), (writtenfile, target)) # make sure correct data is in correct file with open(writtenfile, "rb") as f: @@ -929,26 +1012,50 @@ class ExtractTests(unittest.TestCase): unlink(writtenfile) - # remove the test file subdirectories - rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + unlink(TESTFN2) + + def test_extract_with_target(self): + with temp_dir() as extdir: + self._test_extract_with_target(extdir) + + def test_extract_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_with_target(pathlib.Path(extdir)) def test_extract_all(self): - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: - for fpath, fdata in SMALL_TEST_DATA: - zipfp.writestr(fpath, fdata) + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + zipfp.extractall() + for fpath, fdata in SMALL_TEST_DATA: + outfile = os.path.join(os.getcwd(), fpath) + + with open(outfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(outfile) + def _test_extract_all_with_target(self, target): + self.make_test_file() with zipfile.ZipFile(TESTFN2, "r") as zipfp: - zipfp.extractall() + zipfp.extractall(target) for fpath, fdata in SMALL_TEST_DATA: - outfile = os.path.join(os.getcwd(), fpath) + outfile = os.path.join(target, fpath) with open(outfile, "rb") as f: self.assertEqual(fdata.encode(), f.read()) unlink(outfile) - # remove the test file subdirectories - rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + unlink(TESTFN2) + + def test_extract_all_with_target(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(extdir) + + def test_extract_all_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(pathlib.Path(extdir)) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -1188,6 +1295,8 @@ class OtherTests(unittest.TestCase): with open(TESTFN, "w") as fp: fp.write("this is not a legal zip file\n") self.assertFalse(zipfile.is_zipfile(TESTFN)) + # - passing a path-like object + self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) # - passing a file object with open(TESTFN, "rb") as fp: self.assertFalse(zipfile.is_zipfile(fp)) @@ -2033,6 +2142,26 @@ class ZipInfoTests(unittest.TestCase): zi = zipfile.ZipInfo.from_file(__file__) self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_pathlike(self): + zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) + self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_bytes(self): + zi = zipfile.ZipInfo.from_file(os.fsencode(__file__), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_fileno(self): + with open(__file__, 'rb') as f: + zi = zipfile.ZipInfo.from_file(f.fileno(), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_dir(self): dirpath = os.path.dirname(os.path.abspath(__file__)) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index ee2415d..deea791 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1004,7 +1004,7 @@ class Misc: return self.tk.getint( self.tk.call('winfo', 'ismapped', self._w)) def winfo_manager(self): - """Return the window mananger name for this widget.""" + """Return the window manager name for this widget.""" return self.tk.call('winfo', 'manager', self._w) def winfo_name(self): """Return the name of this widget.""" @@ -1679,7 +1679,7 @@ class Misc: return self.tk.splitlist(self.tk.call('image', 'names')) def image_types(self): - """Return a list of all available image types (e.g. phote bitmap).""" + """Return a list of all available image types (e.g. photo bitmap).""" return self.tk.splitlist(self.tk.call('image', 'types')) @@ -1818,7 +1818,7 @@ class Wm: return self.tk.call('wm', 'focusmodel', self._w, model) focusmodel = wm_focusmodel def wm_forget(self, window): # new in Tk 8.5 - """The window will be unmappend from the screen and will no longer + """The window will be unmapped from the screen and will no longer be managed by wm. toplevel windows will be treated like frame windows once they are no longer managed by wm, however, the menu option configuration will be remembered and the menus will return @@ -2527,7 +2527,7 @@ class Canvas(Widget, XView, YView): """Return item which is closest to pixel at X, Y. If several match take the top-most. All items closer than HALO are considered overlapping (all are - closests). If START is specified the next below this tag is taken.""" + closest). If START is specified the next below this tag is taken.""" return self.find('closest', x, y, halo, start) def find_enclosed(self, x1, y1, x2, y2): """Return all items in rectangle defined @@ -2587,7 +2587,7 @@ class Canvas(Widget, XView, YView): """Print the contents of the canvas to a postscript file. Valid options: colormap, colormode, file, fontmap, height, pageanchor, pageheight, pagewidth, pagex, pagey, - rotate, witdh, x, y.""" + rotate, width, x, y.""" return self.tk.call((self._w, 'postscript') + self._options(cnf, kw)) def tag_raise(self, *args): @@ -3522,7 +3522,7 @@ class Image: return self.tk.getint( self.tk.call('image', 'height', self.name)) def type(self): - """Return the type of the imgage, e.g. "photo" or "bitmap".""" + """Return the type of the image, e.g. "photo" or "bitmap".""" return self.tk.call('image', 'type', self.name) def width(self): """Return the width of the image.""" diff --git a/Lib/typing.py b/Lib/typing.py index 9a0f490..c487afc 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -10,10 +10,12 @@ try: import collections.abc as collections_abc except ImportError: import collections as collections_abc # Fallback for PY3.2. +if sys.version_info[:2] >= (3, 6): + import _collections_abc # Needed for private function _check_methods # noqa try: - from types import SlotWrapperType, MethodWrapperType, MethodDescriptorType + from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType except ImportError: - SlotWrapperType = type(object.__init__) + WrapperDescriptorType = type(object.__init__) MethodWrapperType = type(object().__str__) MethodDescriptorType = type(str.join) @@ -37,6 +39,7 @@ __all__ = [ # for 'Generic' and ABCs below. 'ByteString', 'Container', + 'ContextManager', 'Hashable', 'ItemsView', 'Iterable', @@ -57,12 +60,14 @@ __all__ = [ # AsyncIterable, # Coroutine, # Collection, - # ContextManager, # AsyncGenerator, + # AsyncContextManager # Structural checks, a.k.a. protocols. 'Reversible', 'SupportsAbs', + 'SupportsBytes', + 'SupportsComplex', 'SupportsFloat', 'SupportsInt', 'SupportsRound', @@ -420,6 +425,31 @@ class _Any(_FinalTypingBase, _root=True): Any = _Any(_root=True) +class _NoReturn(_FinalTypingBase, _root=True): + """Special type indicating functions that never return. + Example:: + + from typing import NoReturn + + def stop() -> NoReturn: + raise Exception('no way') + + This type is invalid in other positions, e.g., ``List[NoReturn]`` + will fail in static type checkers. + """ + + __slots__ = () + + def __instancecheck__(self, obj): + raise TypeError("NoReturn cannot be used with isinstance().") + + def __subclasscheck__(self, cls): + raise TypeError("NoReturn cannot be used with issubclass().") + + +NoReturn = _NoReturn(_root=True) + + class TypeVar(_TypingBase, _root=True): """Type variable. @@ -1450,7 +1480,7 @@ def _get_defaults(func): _allowed_types = (types.FunctionType, types.BuiltinFunctionType, types.MethodType, types.ModuleType, - SlotWrapperType, MethodWrapperType, MethodDescriptorType) + WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) def get_type_hints(obj, globalns=None, localns=None): @@ -1922,7 +1952,61 @@ class ValuesView(MappingView[VT_co], extra=collections_abc.ValuesView): if hasattr(contextlib, 'AbstractContextManager'): class ContextManager(Generic[T_co], extra=contextlib.AbstractContextManager): __slots__ = () - __all__.append('ContextManager') +else: + class ContextManager(Generic[T_co]): + __slots__ = () + + def __enter__(self): + return self + + @abc.abstractmethod + def __exit__(self, exc_type, exc_value, traceback): + return None + + @classmethod + def __subclasshook__(cls, C): + if cls is ContextManager: + # In Python 3.6+, it is possible to set a method to None to + # explicitly indicate that the class does not implement an ABC + # (https://bugs.python.org/issue25958), but we do not support + # that pattern here because this fallback class is only used + # in Python 3.5 and earlier. + if (any("__enter__" in B.__dict__ for B in C.__mro__) and + any("__exit__" in B.__dict__ for B in C.__mro__)): + return True + return NotImplemented + + +if hasattr(contextlib, 'AbstractAsyncContextManager'): + class AsyncContextManager(Generic[T_co], + extra=contextlib.AbstractAsyncContextManager): + __slots__ = () + + __all__.append('AsyncContextManager') +elif sys.version_info[:2] >= (3, 5): + exec(""" +class AsyncContextManager(Generic[T_co]): + __slots__ = () + + async def __aenter__(self): + return self + + @abc.abstractmethod + async def __aexit__(self, exc_type, exc_value, traceback): + return None + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncContextManager: + if sys.version_info[:2] >= (3, 6): + return _collections_abc._check_methods(C, "__aenter__", "__aexit__") + if (any("__aenter__" in B.__dict__ for B in C.__mro__) and + any("__aexit__" in B.__dict__ for B in C.__mro__)): + return True + return NotImplemented + +__all__.append('AsyncContextManager') +""") class Dict(dict, MutableMapping[KT, VT], extra=dict): @@ -2051,7 +2135,7 @@ _PY36 = sys.version_info[:2] >= (3, 6) # attributes prohibited to set in NamedTuple class syntax _prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', '_fields', '_field_defaults', '_field_types', - '_make', '_replace', '_asdict') + '_make', '_replace', '_asdict', '_source') _special = ('__module__', '__name__', '__qualname__', '__annotations__') diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index b523f73..f4dbc52 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -153,28 +153,32 @@ class _AssertRaisesBaseContext(_BaseTestCaseContext): If args is not empty, call a callable passing positional and keyword arguments. """ - if not _is_subtype(self.expected, self._base_type): - raise TypeError('%s() arg 1 must be %s' % - (name, self._base_type_str)) - if args and args[0] is None: - warnings.warn("callable is None", - DeprecationWarning, 3) - args = () - if not args: - self.msg = kwargs.pop('msg', None) - if kwargs: - warnings.warn('%r is an invalid keyword argument for ' - 'this function' % next(iter(kwargs)), - DeprecationWarning, 3) - return self - - callable_obj, *args = args try: - self.obj_name = callable_obj.__name__ - except AttributeError: - self.obj_name = str(callable_obj) - with self: - callable_obj(*args, **kwargs) + if not _is_subtype(self.expected, self._base_type): + raise TypeError('%s() arg 1 must be %s' % + (name, self._base_type_str)) + if args and args[0] is None: + warnings.warn("callable is None", + DeprecationWarning, 3) + args = () + if not args: + self.msg = kwargs.pop('msg', None) + if kwargs: + warnings.warn('%r is an invalid keyword argument for ' + 'this function' % next(iter(kwargs)), + DeprecationWarning, 3) + return self + + callable_obj, *args = args + try: + self.obj_name = callable_obj.__name__ + except AttributeError: + self.obj_name = str(callable_obj) + with self: + callable_obj(*args, **kwargs) + finally: + # bpo-23890: manually break a reference cycle + self = None class _AssertRaisesContext(_AssertRaisesBaseContext): @@ -725,7 +729,11 @@ class TestCase(object): self.assertEqual(the_exception.error_code, 3) """ context = _AssertRaisesContext(expected_exception, self) - return context.handle('assertRaises', args, kwargs) + try: + return context.handle('assertRaises', args, kwargs) + finally: + # bpo-23890: manually break a reference cycle + context = None def assertWarns(self, expected_warning, *args, **kwargs): """Fail unless a warning of class warnClass is triggered diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 09fefe1..807604f 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -14,6 +14,7 @@ Examples: %(prog)s test_module - run tests from test_module %(prog)s module.TestClass - run tests from module.TestClass %(prog)s module.Class.test_method - run specified test method + %(prog)s path/to/test_file.py - run tests from test_file.py """ MODULE_EXAMPLES = """\ diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py index 8f752b8..b849591 100644 --- a/Lib/unittest/test/test_case.py +++ b/Lib/unittest/test/test_case.py @@ -1273,6 +1273,19 @@ test case with self.assertRaises(TypeError): self.assertRaises((ValueError, object)) + def testAssertRaisesRefcount(self): + # bpo-23890: assertRaises() must not keep objects alive longer + # than expected + def func() : + try: + raise ValueError + except ValueError: + raise ValueError + + refcount = sys.getrefcount(func) + self.assertRaises(ValueError, func) + self.assertEqual(refcount, sys.getrefcount(func)) + def testAssertRaisesRegex(self): class ExceptionMock(Exception): pass diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index c5b675d..851515b 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -16,10 +16,6 @@ import urllib.response __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] -# do these error classes make sense? -# make sure all of the OSError stuff is overridden. we just want to be -# subtypes. - class URLError(OSError): # URLError is a sub-type of OSError, but it doesn't share any of # the implementation. need to override __init__ and __str__. diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 958767a..3cab2d1 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -38,29 +38,37 @@ __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "DefragResult", "ParseResult", "SplitResult", "DefragResultBytes", "ParseResultBytes", "SplitResultBytes"] -# A classification of schemes ('' means apply by default) -uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap', +# A classification of schemes. +# The empty string classifies URLs with no scheme specified, +# being the default value returned by “urlsplit” and “urlparse”. + +uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', - 'prospero', 'rtsp', 'rtspu', '', 'sftp', + 'prospero', 'rtsp', 'rtspu', 'sftp', 'svn', 'svn+ssh', 'ws', 'wss'] -uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet', + +uses_netloc = ['', 'ftp', 'http', 'gopher', 'nntp', 'telnet', 'imap', 'wais', 'file', 'mms', 'https', 'shttp', - 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', '', + 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', 'ws', 'wss'] -uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap', + +uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', - 'mms', '', 'sftp', 'tel'] + 'mms', 'sftp', 'tel'] # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) + non_hierarchical = ['gopher', 'hdl', 'mailto', 'news', 'telnet', 'wais', 'imap', 'snews', 'sip', 'sips'] -uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms', - 'gopher', 'rtsp', 'rtspu', 'sip', 'sips', ''] -uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news', + +uses_query = ['', 'http', 'wais', 'imap', 'https', 'shttp', 'mms', + 'gopher', 'rtsp', 'rtspu', 'sip', 'sips'] + +uses_fragment = ['', 'ftp', 'hdl', 'http', 'gopher', 'news', 'nntp', 'wais', 'https', 'shttp', 'snews', - 'file', 'prospero', ''] + 'file', 'prospero'] # Characters valid in scheme names scheme_chars = ('abcdefghijklmnopqrstuvwxyz' @@ -612,6 +620,7 @@ def unquote(string, encoding='utf-8', errors='replace'): append(bits[i + 1]) return ''.join(res) + def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace'): """Parse a query given as a string argument. @@ -633,6 +642,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding and errors: specify how to decode percent-encoded sequences into Unicode characters, as accepted by the bytes.decode() method. + + Returns a dictionary. """ parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, @@ -644,28 +655,29 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, parsed_result[name] = [value] return parsed_result + def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace'): """Parse a query given as a string argument. - Arguments: + Arguments: - qs: percent-encoded query string to be parsed + qs: percent-encoded query string to be parsed - keep_blank_values: flag indicating whether blank values in - percent-encoded queries should be treated as blank strings. A - true value indicates that blanks should be retained as blank - strings. The default false value indicates that blank values - are to be ignored and treated as if they were not included. + keep_blank_values: flag indicating whether blank values in + percent-encoded queries should be treated as blank strings. + A true value indicates that blanks should be retained as blank + strings. The default false value indicates that blank values + are to be ignored and treated as if they were not included. - strict_parsing: flag indicating what to do with parsing errors. If - false (the default), errors are silently ignored. If true, - errors raise a ValueError exception. + strict_parsing: flag indicating what to do with parsing errors. If + false (the default), errors are silently ignored. If true, + errors raise a ValueError exception. - encoding and errors: specify how to decode percent-encoded sequences - into Unicode characters, as accepted by the bytes.decode() method. + encoding and errors: specify how to decode percent-encoded sequences + into Unicode characters, as accepted by the bytes.decode() method. - Returns a list, as G-d intended. + Returns a list, as G-d intended. """ qs, _coerce_result = _coerce_args(qs) pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] @@ -932,7 +944,7 @@ def splithost(url): """splithost('//host[:port]/path') --> 'host[:port]', '/path'.""" global _hostprog if _hostprog is None: - _hostprog = re.compile('//([^/?]*)(.*)', re.DOTALL) + _hostprog = re.compile('//([^/#?]*)(.*)', re.DOTALL) match = _hostprog.match(url) if match: diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index b6690c3..9692144 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1488,7 +1488,6 @@ class FileHandler(BaseHandler): origurl = 'file://' + filename return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: - # users shouldn't expect OSErrors coming from urlopen() raise URLError(exp) raise URLError('file not on local host') @@ -1658,14 +1657,10 @@ else: of the 'file' scheme; not recommended for general use.""" return quote(pathname) -# This really consists of two pieces: -# (1) a class which handles opening of all sorts of URLs -# (plus assorted utilities etc.) -# (2) a set of functions for parsing URLs -# XXX Should these be separated out into different modules? - ftpcache = {} + + class URLopener: """Class to open URLs. This is a class rather than just a subroutine because we may need diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index 98b5ca9..421358f 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -105,9 +105,16 @@ class ExpatParser(xmlreader.IncrementalParser, xmlreader.Locator): source = saxutils.prepare_input_source(source) self._source = source - self.reset() - self._cont_handler.setDocumentLocator(ExpatLocator(self)) - xmlreader.IncrementalParser.parse(self, source) + try: + self.reset() + self._cont_handler.setDocumentLocator(ExpatLocator(self)) + xmlreader.IncrementalParser.parse(self, source) + except: + # bpo-30264: Close the source on error to not leak resources: + # xml.sax.parse() doesn't give access to the underlying parser + # to the caller + self._close_source() + raise def prepareParser(self, source): if source.getSystemId() is not None: @@ -213,6 +220,17 @@ class ExpatParser(xmlreader.IncrementalParser, xmlreader.Locator): # FIXME: when to invoke error()? self._err_handler.fatalError(exc) + def _close_source(self): + source = self._source + try: + file = source.getCharacterStream() + if file is not None: + file.close() + finally: + file = source.getByteStream() + if file is not None: + file.close() + def close(self): if (self._entity_stack or self._parser is None or isinstance(self._parser, _ClosedParser)): @@ -232,14 +250,7 @@ class ExpatParser(xmlreader.IncrementalParser, xmlreader.Locator): parser.ErrorColumnNumber = self._parser.ErrorColumnNumber parser.ErrorLineNumber = self._parser.ErrorLineNumber self._parser = parser - try: - file = self._source.getCharacterStream() - if file is not None: - file.close() - finally: - file = self._source.getByteStream() - if file is not None: - file.close() + self._close_source() def _reset_cont_handler(self): self._parser.ProcessingInstructionHandler = \ diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 849bfdd..6faa2d6 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -386,31 +386,36 @@ class SimpleXMLRPCDispatcher: not be called. """ - func = None try: - # check to see if a matching function has been registered + # call the matching registered function func = self.funcs[method] except KeyError: - if self.instance is not None: - # check for a _dispatch method - if hasattr(self.instance, '_dispatch'): - return self.instance._dispatch(method, params) - else: - # call instance method directly - try: - func = resolve_dotted_attribute( - self.instance, - method, - self.allow_dotted_names - ) - except AttributeError: - pass - - if func is not None: - return func(*params) + pass else: + if func is not None: + return func(*params) raise Exception('method "%s" is not supported' % method) + if self.instance is not None: + if hasattr(self.instance, '_dispatch'): + # call the `_dispatch` method on the instance + return self.instance._dispatch(method, params) + + # call the instance's method directly + try: + func = resolve_dotted_attribute( + self.instance, + method, + self.allow_dotted_names + ) + except AttributeError: + pass + else: + if func is not None: + return func(*params) + + raise Exception('method "%s" is not supported' % method) + class SimpleXMLRPCRequestHandler(BaseHTTPRequestHandler): """Simple XML-RPC request handler class. diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 7f2b43c..9164f8a 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -479,6 +479,8 @@ class ZipInfo (object): this will be the same as filename, but without a drive letter and with leading path separators removed). """ + if isinstance(filename, os.PathLike): + filename = os.fspath(filename) st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) @@ -979,6 +981,8 @@ class _ZipWriteFile(io.BufferedIOBase): return True def write(self, data): + if self.closed: + raise ValueError('I/O operation on closed file.') nbytes = len(data) self._file_size += nbytes self._crc = crc32(data, self._crc) @@ -989,6 +993,8 @@ class _ZipWriteFile(io.BufferedIOBase): return nbytes def close(self): + if self.closed: + return super().close() # Flush any data from the compressor, and update header info if self._compressor: @@ -1070,6 +1076,8 @@ class ZipFile: self._comment = b'' # Check if we were passed a file-like object + if isinstance(file, os.PathLike): + file = os.fspath(file) if isinstance(file, str): # No, it's a filename self._filePassed = 0 @@ -1102,7 +1110,6 @@ class ZipFile: # set the modified flag so central directory gets written # even if no files are added to the archive self._didModify = True - self._start_disk = 0 try: self.start_dir = self.fp.tell() except (AttributeError, OSError): @@ -1128,7 +1135,7 @@ class ZipFile: # set the modified flag so central directory gets written # even if no files are added to the archive self._didModify = True - self.start_dir = self._start_disk = self.fp.tell() + self.start_dir = self.fp.tell() else: raise ValueError("Mode must be 'r', 'w', 'x', or 'a'") except: @@ -1172,18 +1179,17 @@ class ZipFile: offset_cd = endrec[_ECD_OFFSET] # offset of central directory self._comment = endrec[_ECD_COMMENT] # archive comment - # self._start_disk: Position of the start of ZIP archive - # It is zero, unless ZIP was concatenated to another file - self._start_disk = endrec[_ECD_LOCATION] - size_cd - offset_cd + # "concat" is zero, unless zip was concatenated to another file + concat = endrec[_ECD_LOCATION] - size_cd - offset_cd if endrec[_ECD_SIGNATURE] == stringEndArchive64: # If Zip64 extension structures are present, account for them - self._start_disk -= (sizeEndCentDir64 + sizeEndCentDir64Locator) + concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) if self.debug > 2: - inferred = self._start_disk + offset_cd - print("given, inferred, offset", offset_cd, inferred, self._start_disk) + inferred = concat + offset_cd + print("given, inferred, offset", offset_cd, inferred, concat) # self.start_dir: Position of start of central directory - self.start_dir = offset_cd + self._start_disk + self.start_dir = offset_cd + concat fp.seek(self.start_dir, 0) data = fp.read(size_cd) fp = io.BytesIO(data) @@ -1223,7 +1229,7 @@ class ZipFile: t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) x._decodeExtra() - x.header_offset = x.header_offset + self._start_disk + x.header_offset = x.header_offset + concat self.filelist.append(x) self.NameToInfo[x.filename] = x @@ -1470,11 +1476,10 @@ class ZipFile: as possible. `member' may be a filename or a ZipInfo object. You can specify a different directory using `path'. """ - if not isinstance(member, ZipInfo): - member = self.getinfo(member) - if path is None: path = os.getcwd() + else: + path = os.fspath(path) return self._extract_member(member, path, pwd) @@ -1487,8 +1492,13 @@ class ZipFile: if members is None: members = self.namelist() + if path is None: + path = os.getcwd() + else: + path = os.fspath(path) + for zipinfo in members: - self.extract(zipinfo, path, pwd) + self._extract_member(zipinfo, path, pwd) @classmethod def _sanitize_windows_name(cls, arcname, pathsep): @@ -1509,6 +1519,9 @@ class ZipFile: """Extract the ZipInfo object 'member' to a physical file on the path targetpath. """ + if not isinstance(member, ZipInfo): + member = self.getinfo(member) + # build the destination pathname, replacing # forward slashes to platform specific separators. arcname = member.filename.replace('/', os.path.sep) @@ -1687,10 +1700,11 @@ class ZipFile: file_size = zinfo.file_size compress_size = zinfo.compress_size - header_offset = zinfo.header_offset - self._start_disk - if header_offset > ZIP64_LIMIT: - extra.append(header_offset) + if zinfo.header_offset > ZIP64_LIMIT: + extra.append(zinfo.header_offset) header_offset = 0xffffffff + else: + header_offset = zinfo.header_offset extra_data = zinfo.extra min_version = 0 @@ -1737,7 +1751,7 @@ class ZipFile: # Write end-of-zip-archive record centDirCount = len(self.filelist) centDirSize = pos2 - self.start_dir - centDirOffset = self.start_dir - self._start_disk + centDirOffset = self.start_dir requires_zip64 = None if centDirCount > ZIP_FILECOUNT_LIMIT: requires_zip64 = "Files count" @@ -1801,6 +1815,7 @@ class PyZipFile(ZipFile): If filterfunc(pathname) is given, it is called with every argument. When it is False, the file or directory is skipped. """ + pathname = os.fspath(pathname) if filterfunc and not filterfunc(pathname): if self.debug: label = 'path' if os.path.isdir(pathname) else 'file' diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index f59ae34..1fff052 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -13,7 +13,7 @@ sphinx-build and the current versions of Sphinx now require at least Python 2.6. In addition to what is supplied with OS X 10.5+ and Xcode 3+, the script -requires an installed version of git and a third-party version of +requires an installed third-party version of Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets) or Tcl/TK 8.5 (for 10.6 or later) installed in /Library/Frameworks. When installed, the Python built by this script will attempt to dynamically link first to @@ -23,7 +23,7 @@ installing the most recent ActiveTcl 8.4 or 8.5 version. 32-bit-only installer builds are still possible on OS X 10.4 with Xcode 2.5 and the installation of additional components, such as a newer Python -(2.5 is needed for Python parser updates), git, and for the documentation +(2.5 is needed for Python parser updates) and for the documentation build either svn (pre-3.4.1) or sphinx-build (3.4.1 and later). Usage: see USAGE variable in the script. @@ -635,9 +635,8 @@ def checkEnvironment(): base_path = base_path + ':' + OLD_DEVELOPER_TOOLS os.environ['PATH'] = base_path print("Setting default PATH: %s"%(os.environ['PATH'])) - # Ensure ws have access to git and to sphinx-build. - # You may have to create links in /usr/bin for them. - runCommand('git --version') + # Ensure we have access to sphinx-build. + # You may have to create a link in /usr/bin for it. runCommand('sphinx-build --version') def parseOptions(args=None): @@ -1142,12 +1141,25 @@ def buildPython(): shellQuote(WORKDIR)[1:-1], shellQuote(WORKDIR)[1:-1])) - # bpo-29550: avoid using make touch until it is fixed for git - # print("Running make touch") - # runCommand("make touch") - - print("Running make") - runCommand("make") + # Look for environment value BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS + # and, if defined, append its value to the make command. This allows + # us to pass in version control tags, like GITTAG, to a build from a + # tarball rather than from a vcs checkout, thus eliminating the need + # to have a working copy of the vcs program on the build machine. + # + # A typical use might be: + # export BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS=" \ + # GITVERSION='echo 123456789a' \ + # GITTAG='echo v3.6.0' \ + # GITBRANCH='echo 3.6'" + + make_extras = os.getenv("BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS") + if make_extras: + make_cmd = "make " + make_extras + else: + make_cmd = "make" + print("Running " + make_cmd) + runCommand(make_cmd) print("Running make install") runCommand("make install DESTDIR=%s"%( diff --git a/Makefile.pre.in b/Makefile.pre.in index 8f27d73..82e8307 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -107,6 +107,8 @@ ARFLAGS= @ARFLAGS@ CFLAGSFORSHARED=@CFLAGSFORSHARED@ # C flags used for building the interpreter object files PY_CORE_CFLAGS= $(PY_CFLAGS) $(PY_CFLAGS_NODIST) $(PY_CPPFLAGS) $(CFLAGSFORSHARED) -DPy_BUILD_CORE +# Strict or non-strict aliasing flags used to compile dtoa.c, see above +CFLAGS_ALIASING=@CFLAGS_ALIASING@ # Machine-dependent subdirectories @@ -227,7 +229,7 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) -PYTHON_FOR_GEN=@PYTHON_FOR_GEN@ +PYTHON_FOR_REGEN=@PYTHON_FOR_REGEN@ PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ BUILD_GNU_TYPE= @build@ @@ -271,11 +273,6 @@ IO_OBJS= \ Modules/_io/stringio.o ########################################################################## -# Grammar -GRAMMAR_H= Include/graminit.h -GRAMMAR_C= Python/graminit.c -GRAMMAR_INPUT= $(srcdir)/Grammar/Grammar - LIBFFI_INCLUDEDIR= @LIBFFI_INCLUDEDIR@ @@ -315,37 +312,8 @@ PARSER_HEADERS= \ PGENOBJS= $(POBJS) $(PGOBJS) ########################################################################## -# opcode.h generation -OPCODE_H_DIR= $(srcdir)/Include -OPCODE_H_SCRIPT= $(srcdir)/Tools/scripts/generate_opcode_h.py -OPCODE_H= $(OPCODE_H_DIR)/opcode.h -OPCODE_H_GEN= $(PYTHON_FOR_GEN) $(OPCODE_H_SCRIPT) $(srcdir)/Lib/opcode.py $(OPCODE_H) - -########################################################################## -# AST -AST_H_DIR= Include -AST_H= $(AST_H_DIR)/Python-ast.h -AST_C_DIR= Python -AST_C= $(AST_C_DIR)/Python-ast.c -AST_ASDL= $(srcdir)/Parser/Python.asdl - -ASDLGEN_FILES= $(srcdir)/Parser/asdl.py $(srcdir)/Parser/asdl_c.py -# Note that a build now requires Python to exist before the build starts. -# Use "hg touch" to fix up screwed up file mtimes in a checkout. -ASDLGEN= $(PYTHON_FOR_GEN) $(srcdir)/Parser/asdl_c.py - -########################################################################## # Python -OPCODETARGETS_H= \ - Python/opcode_targets.h - -OPCODETARGETGEN= \ - $(srcdir)/Python/makeopcodetargets.py - -OPCODETARGETGEN_FILES= \ - $(OPCODETARGETGEN) $(srcdir)/Lib/opcode.py - PYTHON_OBJS= \ Python/_warnings.o \ Python/Python-ast.o \ @@ -545,7 +513,8 @@ coverage-lcov: @echo "lcov report at $(COVERAGE_REPORT)/index.html" @echo -coverage-report: +# Force regeneration of parser and importlib +coverage-report: regen-grammar regen-importlib : # force rebuilding of parser and importlib @touch $(GRAMMAR_INPUT) @touch $(srcdir)/Lib/importlib/_bootstrap.py @@ -721,14 +690,24 @@ Programs/_freeze_importlib.o: Programs/_freeze_importlib.c Makefile Programs/_freeze_importlib: Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LINKCC) $(PY_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) -Python/importlib_external.h: @GENERATED_COMMENT@ $(srcdir)/Lib/importlib/_bootstrap_external.py Programs/_freeze_importlib Python/marshal.c +.PHONY: regen-importlib +regen-importlib: Programs/_freeze_importlib + # Regenerate Python/importlib_external.h + # from Lib/importlib/_bootstrap_external.py using _freeze_importlib ./Programs/_freeze_importlib \ - $(srcdir)/Lib/importlib/_bootstrap_external.py Python/importlib_external.h - -Python/importlib.h: @GENERATED_COMMENT@ $(srcdir)/Lib/importlib/_bootstrap.py Programs/_freeze_importlib Python/marshal.c + $(srcdir)/Lib/importlib/_bootstrap_external.py \ + $(srcdir)/Python/importlib_external.h + # Regenerate Python/importlib.h from Lib/importlib/_bootstrap.py + # using _freeze_importlib ./Programs/_freeze_importlib \ - $(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h + $(srcdir)/Lib/importlib/_bootstrap.py \ + $(srcdir)/Python/importlib.h + + +############################################################################ +# Regenerate all generated files +regen-all: regen-opcode regen-opcode-targets regen-typeslots regen-grammar regen-ast regen-importlib ############################################################################ # Special rules for object files @@ -787,15 +766,18 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(IO_OBJS): $(IO_H) -$(GRAMMAR_H): @GENERATED_COMMENT@ $(GRAMMAR_INPUT) $(PGEN) - @$(MKDIR_P) Include - $(PGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C) -$(GRAMMAR_C): @GENERATED_COMMENT@ $(GRAMMAR_H) - touch $(GRAMMAR_C) - $(PGEN): $(PGENOBJS) $(CC) $(OPT) $(PY_LDFLAGS) $(PGENOBJS) $(LIBS) -o $(PGEN) +.PHONY: regen-grammar +regen-grammar: $(PGEN) + # Regenerate Include/graminit.h and Python/graminit.c + # from Grammar/Grammar using pgen + @$(MKDIR_P) Include + $(PGEN) $(srcdir)/Grammar/Grammar \ + $(srcdir)/Include/graminit.h \ + $(srcdir)/Python/graminit.c + Parser/grammar.o: $(srcdir)/Parser/grammar.c \ $(srcdir)/Include/token.h \ $(srcdir)/Include/grammar.h @@ -807,18 +789,28 @@ Parser/printgrammar.o: $(srcdir)/Parser/printgrammar.c Parser/pgenmain.o: $(srcdir)/Include/parsetok.h -$(AST_H): $(AST_ASDL) $(ASDLGEN_FILES) - $(MKDIR_P) $(AST_H_DIR) - $(ASDLGEN) -h $(AST_H_DIR) $(AST_ASDL) - -$(AST_C): $(AST_H) $(AST_ASDL) $(ASDLGEN_FILES) - $(MKDIR_P) $(AST_C_DIR) - $(ASDLGEN) -c $(AST_C_DIR) $(AST_ASDL) - -$(OPCODE_H): $(srcdir)/Lib/opcode.py $(OPCODE_H_SCRIPT) - $(OPCODE_H_GEN) - -Python/compile.o Python/symtable.o Python/ast.o: $(GRAMMAR_H) $(AST_H) +.PHONY=regen-ast +regen-ast: + # Regenerate Include/Python-ast.h using Parser/asdl_c.py -h + $(MKDIR_P) $(srcdir)/Include + $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ + -h $(srcdir)/Include \ + $(srcdir)/Parser/Python.asdl + # Regenerate Python/Python-ast.c using Parser/asdl_c.py -c + $(MKDIR_P) $(srcdir)/Python + $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ + -c $(srcdir)/Python \ + $(srcdir)/Parser/Python.asdl + +.PHONY: regen-opcode +regen-opcode: + # Regenerate Include/opcode.h from Lib/opcode.py + # using Tools/scripts/generate_opcode_h.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_opcode_h.py \ + $(srcdir)/Lib/opcode.py \ + $(srcdir)/Include/opcode.h + +Python/compile.o Python/symtable.o Python/ast.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h Python/getplatform.o: $(srcdir)/Python/getplatform.c $(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c @@ -868,17 +860,22 @@ Objects/odictobject.o: $(srcdir)/Objects/dict-common.h Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h $(srcdir)/Objects/dict-common.h Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h -$(OPCODETARGETS_H): $(OPCODETARGETGEN_FILES) - $(PYTHON_FOR_GEN) $(OPCODETARGETGEN) $(OPCODETARGETS_H) +.PHONY: regen-opcode-targets +regen-opcode-targets: + # Regenerate Python/opcode_targets.h from Lib/opcode.py + # using Python/makeopcodetargets.py + $(PYTHON_FOR_REGEN) $(srcdir)/Python/makeopcodetargets.py \ + $(srcdir)/Python/opcode_targets.h -Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h +Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h -Python/frozen.o: Python/importlib.h Python/importlib_external.h +Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h # Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to # follow our naming conventions. dtrace(1) uses the output filename to generate # an include guard, so we can't use a pipeline to transform its output. Include/pydtrace_probes.h: $(srcdir)/Include/pydtrace.d + $(MKDIR_P) Include $(DTRACE) $(DFLAGS) -o $@ -h -s $< : sed in-place edit with POSIX-only tools sed 's/PYTHON_/PyDTrace_/' $@ > $@.tmp @@ -888,8 +885,14 @@ Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) $(DTRACE) $(DFLAGS) -o $@ -G -s $< $(DTRACE_DEPS) Objects/typeobject.o: Objects/typeslots.inc -Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py - $(PYTHON_FOR_GEN) $(srcdir)/Objects/typeslots.py < $(srcdir)/Include/typeslots.h Objects/typeslots.inc + +.PHONY: regen-typeslots +regen-typeslots: + # Regenerate Objects/typeslots.inc from Include/typeslotsh + # using Objects/typeslots.py + $(PYTHON_FOR_REGEN) $(srcdir)/Objects/typeslots.py \ + < $(srcdir)/Include/typeslots.h \ + $(srcdir)/Objects/typeslots.inc ############################################################################ # Header files @@ -942,7 +945,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/node.h \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ - $(OPCODE_H) \ + $(srcdir)/Include/opcode.h \ $(srcdir)/Include/osdefs.h \ $(srcdir)/Include/osmodule.h \ $(srcdir)/Include/patchlevel.h \ @@ -985,7 +988,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/weakrefobject.h \ pyconfig.h \ $(PARSER_HEADERS) \ - $(AST_H) \ + $(srcdir)/Include/Python-ast.h \ $(DTRACE_HEADERS) $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) @@ -1000,7 +1003,7 @@ TESTTIMEOUT= 1200 # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. -test: all platform +test: @DEF_MAKE_RULE@ platform $(TESTRUNNER) $(TESTOPTS) # Run the full test suite twice - once without .pyc files, and once with. @@ -1010,7 +1013,7 @@ test: all platform # the bytecode read from a .pyc file had the bug, sometimes the directly # generated bytecode. This is sometimes a very shy bug needing a lot of # sample data. -testall: all platform +testall: @DEF_MAKE_RULE@ platform -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f @@ -1019,7 +1022,7 @@ testall: all platform # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. -testuniversal: all platform +testuniversal: @DEF_MAKE_RULE@ platform if [ `arch` != 'i386' ];then \ echo "This can only be used on OSX/i386" ;\ exit 1 ;\ @@ -1042,7 +1045,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ test_multiprocessing_forkserver \ test_mailbox test_socket test_poll \ test_select test_zipfile test_concurrent_futures -quicktest: all platform +quicktest: @DEF_MAKE_RULE@ platform $(TESTRUNNER) $(QUICKTESTOPTS) @@ -1379,7 +1382,7 @@ LIBPL= @LIBPL@ # pkgconfig directory LIBPC= $(LIBDIR)/pkgconfig -libainstall: all python-config +libainstall: @DEF_MAKE_RULE@ python-config @for i in $(LIBDIR) $(LIBPL) $(LIBPC); \ do \ if test ! -d $(DESTDIR)$$i; then \ @@ -1535,6 +1538,13 @@ config.status: $(srcdir)/configure .c.o: $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< +# bpo-30104: dtoa.c uses union to cast double to unsigned long[2]. clang 4.0 +# with -O2 or higher and strict aliasing miscompiles the ratio() function +# causing rounding issues. Compile dtoa.c using -fno-strict-aliasing on clang. +# https://bugs.llvm.org//show_bug.cgi?id=31928 +Python/dtoa.o: Python/dtoa.c + $(CC) -c $(PY_CORE_CFLAGS) $(CFLAGS_ALIASING) -o $@ $< + # Run reindent on the library reindent: ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/reindent.py -r $(srcdir)/Lib @@ -1545,9 +1555,12 @@ recheck: $(SHELL) config.status --recheck $(SHELL) config.status -# Rebuild the configure script from configure.ac; also rebuild pyconfig.h.in +# Regenerate configure and pyconfig.h.in +.PHONY: autoconf autoconf: + # Regenerate the configure script from configure.ac using autoconf (cd $(srcdir); autoconf -Wall) + # Regenerate pyconfig.h.in from configure.ac using autoheader (cd $(srcdir); autoheader -Wall) # Create a tags file for vi @@ -1564,14 +1577,6 @@ TAGS:: etags Include/*.h; \ for i in $(SRCDIRS); do etags -a $$i/*.[ch]; done -# This fixes up the mtimes of checked-in generated files, assuming that they -# only *appear* to be outdated because of checkout order. -# This is run while preparing a source release tarball, and can be run manually -# to avoid bootstrap issues. -touch: - cd $(srcdir); \ - hg --config extensions.touch=Tools/hg/hgtouch.py touch -v - # Sanitation targets -- clean leaves libraries, executables and tags # files, which clobber removes as well pycremoval: @@ -1626,7 +1631,8 @@ distclean: clobber done -rm -f core Makefile Makefile.pre config.status \ Modules/Setup Modules/Setup.local Modules/Setup.config \ - Modules/ld_so_aix Modules/python.exp Misc/python.pc + Modules/ld_so_aix Modules/python.exp Misc/python.pc \ + Misc/python-config.sh -rm -f python*-gdb.py # Issue #28258: set LC_ALL to avoid issues with Estonian locale. # Expansion is performed here by shell (spawned by make) itself before @@ -1639,7 +1645,7 @@ distclean: clobber -exec rm -f {} ';' # Check for smelly exported symbols (not starting with Py/_Py) -smelly: all +smelly: @DEF_MAKE_RULE@ nm -p $(LIBRARY) | \ sed -n "/ [TDB] /s/.* //p" | grep -v "^_*Py" | sort -u; \ @@ -1676,7 +1682,7 @@ funny: -o -print # Perform some verification checks on any modified files. -patchcheck: all +patchcheck: @DEF_MAKE_RULE@ $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py # Dependencies @@ -1689,7 +1695,7 @@ Python/thread.o: @THREADHEADERS@ .PHONY: maninstall libinstall inclinstall libainstall sharedinstall .PHONY: frameworkinstall frameworkinstallframework frameworkinstallstructure .PHONY: frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools -.PHONY: frameworkaltinstallunixtools recheck autoconf clean clobber distclean +.PHONY: frameworkaltinstallunixtools recheck clean clobber distclean .PHONY: smelly funny patchcheck touch altmaninstall commoninstall .PHONY: gdbhooks diff --git a/Misc/ACKS b/Misc/ACKS index c3b29a4..03f9ea6 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -128,6 +128,7 @@ Andrew Bennetts Andy Bensky Bennett Benson Ezra Berch +Stuart Berg Michel Van den Bergh Julian Berman Brice Berna @@ -308,6 +309,7 @@ Garrett Cooper Greg Copeland Ian Cordasco Aldo Cortesi +Mircea Cosbuc David Costanzo Scott Cotton Greg Couch @@ -542,6 +544,7 @@ Tim Graham Kim Gräsman Nathaniel Gray Eddy De Greef +Duane Griffin Grant Griffin Andrea Griffini Duncan Grisby @@ -551,6 +554,8 @@ Eric Groo Daniel Andrade Groppe Dag Gruneau Filip Gruszczyński +Andrii Grynenko +Grzegorz Grzywacz Thomas Guettler Yuyang Guo Anuj Gupta @@ -770,11 +775,13 @@ Lawrence Kesteloot Vivek Khera Dhiru Kholia Akshit Khurana +Sanyam Khurana Mads Kiilerich Jason Killen Jan Kim Taek Joo Kim Sam Kimbrel +Tomohiko Kinebuchi James King W. Trevor King Paul Kippes @@ -958,6 +965,7 @@ David Marek Doug Marien Sven Marnach Alex Martelli +Dennis MÃ¥rtensson Anthony Martin Owen Martin Sidney San Martín @@ -1058,6 +1066,7 @@ R. David Murray Matti Mäki Jörg Müller Kaushik N +Dong-hee Na Dale Nagata John Nagle Takahiro Nakayama @@ -1072,6 +1081,7 @@ Max Neunhöffer Anthon van der Neut George Neville-Neil Hieu Nguyen +Nam Nguyen Johannes Nicolai Samuel Nicolary Jonathan Niehof @@ -1103,6 +1113,7 @@ Milan Oberkirch Pascal Oberndoerfer Jeffrey Ollie Adam Olsen +Bryan Olson Grant Olson Koray Oner Piet van Oostrum @@ -1111,7 +1122,9 @@ Jason Orendorff Douglas Orr William Orr Michele Orrù +Tomáš Orsava Oleg Oshmyan +Denis Osipov Denis S. Otkidach Peter Otten Michael Otteneder @@ -1181,6 +1194,7 @@ Steve Piercy Jim St. Pierre Dan Pierson Martijn Pieters +Antoine Pietri Anand B. Pillai François Pinard Tom Pinckney @@ -1371,6 +1385,7 @@ Federico Schwindt Barry Scott Steven Scott Nick Seidenman +Michael Seifert Žiga Seilnacht Yury Selivanov Fred Sells @@ -1471,6 +1486,7 @@ Daniel Stokes Michael Stone Serhiy Storchaka Ken Stox +Charalampos Stratakis Dan Stromberg Donald Stufft Daniel Stutzbach @@ -1650,6 +1666,7 @@ Jakub Wilk Gerald S. Williams Jason Williams John Williams +Roy Williams Sue Williams Carol Willing Steven Willis @@ -1708,6 +1725,7 @@ Artur Zaprzala Mike Zarnstorff Yury V. Zaytsev Siebren van der Zee +Christophe Zeitouny Nickolai Zeldovich Yuxiao Zeng Uwe Zessin diff --git a/Misc/NEWS b/Misc/NEWS index 4807bd0..644b4c3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,377 @@ Python News +++++++++++ +What's New in Python 3.6.2? +=========================== + +*Release date: 2017-07-17* + +- No changes since release candidate 2 + + +What's New in Python 3.6.2 release candidate 2? +=============================================== + +*Release date: 2017-07-07* + +Core and Builtins +----------------- + +Library +------- + +- [Security] bpo-30730: Prevent environment variables injection in subprocess on + Windows. Prevent passing other environment variables and command arguments. + +- [Security] bpo-30694: Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes + of multiple security vulnerabilities including: CVE-2017-9233 (External + entity infinite loop DoS), CVE-2016-9063 (Integer overflow, re-fix), + CVE-2016-0718 (Fix regression bugs from 2.2.0's fix to CVE-2016-0718) + and CVE-2012-0876 (Counter hash flooding with SipHash). + Note: the CVE-2016-5300 (Use os-specific entropy sources like getrandom) + doesn't impact Python, since Python already gets entropy from the OS to set + the expat secret using ``XML_SetHashSalt()``. + +- [Security] bpo-30500: Fix urllib.parse.splithost() to correctly parse + fragments. For example, ``splithost('//127.0.0.1#@evil.com/')`` now + correctly returns the ``127.0.0.1`` host, instead of treating ``@evil.com`` + as the host in an authentification (``login@host``). + + +What's New in Python 3.6.2 release candidate 1? +=============================================== + +*Release date: 2017-06-17* + +Core and Builtins +----------------- + +- bpo-30682: Removed a too-strict assertion that failed for certain f-strings, + such as eval("f'\\\n'") and eval("f'\\\r'"). + +- bpo-30604: Move co_extra_freefuncs to not be per-thread to avoid crashes + +- bpo-29104: Fixed parsing backslashes in f-strings. + +- bpo-27945: Fixed various segfaults with dict when input collections are + mutated during searching, inserting or comparing. Based on patches by + Duane Griffin and Tim Mitchell. + +- bpo-25794: Fixed type.__setattr__() and type.__delattr__() for + non-interned attribute names. Based on patch by Eryk Sun. + +- bpo-30039: If a KeyboardInterrupt happens when the interpreter is in + the middle of resuming a chain of nested 'yield from' or 'await' + calls, it's now correctly delivered to the innermost frame. + +- bpo-12414: sys.getsizeof() on a code object now returns the sizes + which includes the code struct and sizes of objects which it references. + Patch by Dong-hee Na. + +- bpo-29949: Fix memory usage regression of set and frozenset object. + +- bpo-29935: Fixed error messages in the index() method of tuple, list and deque + when pass indices of wrong type. + +- bpo-29859: Show correct error messages when any of the pthread_* calls in + thread_pthread.h fails. + +- bpo-28876: ``bool(range)`` works even if ``len(range)`` + raises :exc:`OverflowError`. + +- bpo-29600: Fix wrapping coroutine return values in StopIteration. + +- bpo-28856: Fix an oversight that %b format for bytes should support objects + follow the buffer protocol. + +- bpo-29714: Fix a regression that bytes format may fail when containing zero + bytes inside. + +- bpo-29478: If max_line_length=None is specified while using the Compat32 policy, + it is no longer ignored. Patch by Mircea Cosbuc. + +Library +------- + +- bpo-30038: Fix race condition between signal delivery and wakeup file + descriptor. Patch by Nathaniel Smith. + +- bpo-23894: lib2to3 now recognizes ``rb'...'`` and ``f'...'`` strings. + +- bpo-23890: unittest.TestCase.assertRaises() now manually breaks a reference + cycle to not keep objects alive longer than expected. + +- bpo-30149: inspect.signature() now supports callables with + variable-argument parameters wrapped with partialmethod. + Patch by Dong-hee Na. + +- bpo-29931: Fixed comparison check for ipaddress.ip_interface objects. + Patch by Sanjay Sundaresan. + +- bpo-30605: re.compile() no longer raises a BytesWarning when compiling a + bytes instance with misplaced inline modifier. Patch by Roy Williams. + +- [Security] bpo-29591: Update expat copy from 2.1.1 to 2.2.0 to get fixes + of CVE-2016-0718 and CVE-2016-4472. See + https://sourceforge.net/p/expat/bugs/537/ for more information. + +- bpo-24484: Avoid race condition in multiprocessing cleanup (#2159) + +- bpo-28994: The traceback no longer displayed for SystemExit raised in + a callback registered by atexit. + +- bpo-30508: Don't log exceptions if Task/Future "cancel()" method was + called. + +- bpo-28556: Updates to typing module: Add generic AsyncContextManager, add + support for ContextManager on all versions. Original PRs by Jelle Zijlstra + and Ivan Levkivskyi + +- bpo-29870: Fix ssl sockets leaks when connection is aborted in asyncio/ssl + implementation. Patch by Michaël Sghaïer. + +- bpo-29743: Closing transport during handshake process leaks open socket. + Patch by Nikolay Kim + +- bpo-27585: Fix waiter cancellation in asyncio.Lock. + Patch by Mathieu Sornay. + +- bpo-30418: On Windows, subprocess.Popen.communicate() now also ignore EINVAL + on stdin.write() if the child process is still running but closed the pipe. + +- bpo-29822: inspect.isabstract() now works during __init_subclass__. Patch + by Nate Soares. + +- bpo-29581: ABCMeta.__new__ now accepts ``**kwargs``, allowing abstract base + classes to use keyword parameters in __init_subclass__. Patch by Nate Soares. + +- bpo-30557: faulthandler now correctly filters and displays exception codes + on Windows + +- bpo-30378: Fix the problem that logging.handlers.SysLogHandler cannot + handle IPv6 addresses. + +- bpo-29960: Preserve generator state when _random.Random.setstate() + raises an exception. Patch by Bryan Olson. + +- bpo-30414: multiprocessing.Queue._feed background running + thread do not break from main loop on exception. + +- bpo-30003: Fix handling escape characters in HZ codec. Based on patch + by Ma Lin. + +- bpo-30301: Fix AttributeError when using SimpleQueue.empty() under + *spawn* and *forkserver* start methods. + +- bpo-30329: imaplib and poplib now catch the Windows socket WSAEINVAL error + (code 10022) on shutdown(SHUT_RDWR): An invalid operation was attempted. + This error occurs sometimes on SSL connections. + +- bpo-30375: Warnings emitted when compile a regular expression now always + point to the line in the user code. Previously they could point into inners + of the re module if emitted from inside of groups or conditionals. + +- bpo-30048: Fixed ``Task.cancel()`` can be ignored when the task is + running coroutine and the coroutine returned without any more ``await``. + +- bpo-30266: contextlib.AbstractContextManager now supports anti-registration + by setting __enter__ = None or __exit__ = None, following the pattern + introduced in bpo-25958. Patch by Jelle Zijlstra. + +- bpo-30298: Weaken the condition of deprecation warnings for inline modifiers. + Now allowed several subsequential inline modifiers at the start of the + pattern (e.g. ``'(?i)(?s)...'``). In verbose mode whitespaces and comments + now are allowed before and between inline modifiers (e.g. + ``'(?x) (?i) (?s)...'``). + +- bpo-29990: Fix range checking in GB18030 decoder. Original patch by Ma Lin. + +- Revert bpo-26293 for zipfile breakage. See also bpo-29094. + +- bpo-30243: Removed the __init__ methods of _json's scanner and encoder. + Misusing them could cause memory leaks or crashes. Now scanner and encoder + objects are completely initialized in the __new__ methods. + +- bpo-30185: Avoid KeyboardInterrupt tracebacks in forkserver helper process + when Ctrl-C is received. + +- bpo-28556: Various updates to typing module: add typing.NoReturn type, use + WrapperDescriptorType, minor bug-fixes. Original PRs by + Jim Fasarakis-Hilliard and Ivan Levkivskyi. + +- bpo-30205: Fix getsockname() for unbound AF_UNIX sockets on Linux. + +- bpo-30070: Fixed leaks and crashes in errors handling in the parser module. + +- bpo-30061: Fixed crashes in IOBase methods __next__() and readlines() when + readline() or __next__() respectively return non-sizeable object. + Fixed possible other errors caused by not checking results of PyObject_Size(), + PySequence_Size(), or PyMapping_Size(). + +- bpo-30017: Allowed calling the close() method of the zip entry writer object + multiple times. Writing to a closed writer now always produces a ValueError. + +- bpo-30068: _io._IOBase.readlines will check if it's closed first when + hint is present. + +- bpo-29694: Fixed race condition in pathlib mkdir with flags + parents=True. Patch by Armin Rigo. + +- bpo-29692: Fixed arbitrary unchaining of RuntimeError exceptions in + contextlib.contextmanager. Patch by Siddharth Velankar. + +- bpo-29998: Pickling and copying ImportError now preserves name and path + attributes. + +- bpo-29953: Fixed memory leaks in the replace() method of datetime and time + objects when pass out of bound fold argument. + +- bpo-29942: Fix a crash in itertools.chain.from_iterable when encountering + long runs of empty iterables. + +- bpo-27863: Fixed multiple crashes in ElementTree caused by race conditions + and wrong types. + +- bpo-28699: Fixed a bug in pools in multiprocessing.pool that raising an + exception at the very first of an iterable may swallow the exception or + make the program hang. Patch by Davin Potts and Xiang Zhang. + +- bpo-25803: Avoid incorrect errors raised by Path.mkdir(exist_ok=True) + when the OS gives priority to errors such as EACCES over EEXIST. + +- bpo-29861: Release references to tasks, their arguments and their results + as soon as they are finished in multiprocessing.Pool. + +- bpo-29884: faulthandler: Restore the old sigaltstack during teardown. + Patch by Christophe Zeitouny. + +- bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. + +- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords + are not strings. Patch by Michael Seifert. + +- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. + Patch by Nikolay Kim. + +- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" + or "errors" of sys.stdin or sys.stdout are not set or are not strings. + +- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big + intables (objects that have __int__) as elements. Patch by Oren Milman. + +- bpo-28231: The zipfile module now accepts path-like objects for external + paths. + +- bpo-26915: index() and count() methods of collections.abc.Sequence now + check identity before checking equality when do comparisons. + +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + +- bpo-30177: path.resolve(strict=False) no longer cuts the path after the first + element not present in the filesystem. Patch by Antoine Pietri. + +IDLE +---- + +- bpo-15786: Fix several problems with IDLE's autocompletion box. + The following should now work: clicking on selection box items; + using the scrollbar; selecting an item by hitting Return. + Hangs on MacOSX should no longer happen. Patch by Louie Lu. + +- bpo-25514: Add doc subsubsection about IDLE failure to start. + Popup no-connection message directs users to this section. + +- bpo-30642: Fix reference leaks in IDLE tests. + Patches by Louie Lu and Terry Jan Reedy. + +- bpo-30495: Add docstrings for textview.py and use PEP8 names. + Patches by Cheryl Sabella and Terry Jan Reedy. + +- bpo-30290: Help-about: use pep8 names and add tests. + Increase coverage to 100%. + Patches by Louie Lu, Cheryl Sabella, and Terry Jan Reedy. + +- bpo-30303: Add _utest option to textview; add new tests. + Increase coverage to 100%. + Patches by Louie Lu and Terry Jan Reedy. + +C API +----- + +- Issue #27867: Function PySlice_GetIndicesEx() no longer replaced with a macro + if Py_LIMITED_API is not set. + + +Build +----- + +- bpo-29941: Add ``--with-assertions`` configure flag to explicitly enable + C ``assert()`` checks. Defaults to off. ``--with-pydebug`` implies + ``--with-assertions``. + +- bpo-28787: Fix out-of-tree builds of Python when configured with + ``--with--dtrace``. + +- bpo-29243: Prevent unnecessary rebuilding of Python during ``make test``, + ``make install`` and some other make targets when configured with + ``--enable-optimizations``. + +- bpo-23404: Don't regenerate generated files based on file modification time + anymore: the action is now explicit. Replace ``make touch`` with + ``make regen-all``. + +- bpo-29643: Fix ``--enable-optimization`` didn't work. + +Documentation +------------- + +- bpo-30176: Add missing attribute related constants in curses documentation. + +- Issue #30052: the link targets for :func:`bytes` and + :func:`bytearray` are now their respective type definitions, rather + than the corresponding builtin function entries. Use :ref:`bytes ` + and :ref:`bytearray ` to reference the latter. + + In order to ensure this and future cross-reference updates are applied + automatically, the daily documentation builds now disable the default + output caching features in Sphinx. + +- bpo-26985: Add missing info of code object in inspect documentation. + +Tools/Demos +----------- + +- Issue #29367: python-gdb.py now supports also ``method-wrapper`` + (``wrapperobject``) objects. + +Tests +----- + +- bpo-30357: test_thread: setUp() now uses support.threading_setup() and + support.threading_cleanup() to wait until threads complete to avoid + random side effects on following tests. Initial patch written by Grzegorz + Grzywacz. + +- bpo-30197: Enhanced functions swap_attr() and swap_item() in the + test.support module. They now work when delete replaced attribute or item + inside the with statement. The old value of the attribute or item (or None + if it doesn't exist) now will be assigned to the target of the "as" clause, + if there is one. + +Windows +------- + +- bpo-30687: Locate msbuild.exe on Windows when building rather than + vcvarsall.bat + +- bpo-30450: The build process on Windows no longer depends on Subversion, + instead pulling external code from GitHub via a Python script. If Python 3.6 + is not found on the system (via ``py -3.6``), NuGet is used to download a + copy of 32-bit Python. + + What's New in Python 3.6.1? =========================== @@ -47,8 +418,6 @@ Core and Builtins - Issue #28598: Support __rmod__ for subclasses of str being called before str.__mod__. Patch by Martijn Pieters. -- bpo-29572: Update Windows build and OS X installers to use OpenSSL 1.0.2k. - - bpo-29607: Fix stack_effect computation for CALL_FUNCTION_EX. Patch by Matthieu Dartiailh. @@ -210,7 +579,7 @@ Library - Issue #28427: old keys should not remove new values from WeakValueDictionary when collecting from another thread. -- Issue 28923: Remove editor artifacts from Tix.py. +- Issue #28923: Remove editor artifacts from Tix.py. - Issue #29055: Neaten-up empty population error on random.choice() by suppressing the upstream exception. @@ -235,6 +604,14 @@ Library now when the grp module cannot be imported, as for example on Android platforms. +IDLE +---- + +- Issue #29071: IDLE colors f-string prefixes (but not invalid ur prefixes). + +- Issue #28572: Add 10% to coverage of IDLE's test_configdialog. + Update and augment description of the configuration system. + Windows ------- @@ -616,7 +993,7 @@ Library non-None value is passed to it.send(val). - Issue #27025: Generated names for Tkinter widgets now start by the "!" prefix - for readability (was "`"). + for readability. - Issue #25464: Fixed HList.header_exists() in tkinter.tix module by addin a workaround to Tix library bug. @@ -978,7 +1355,7 @@ Core and Builtins alpha releases, where backslashes are allowed anywhere in an f-string. Also, require that expressions inside f-strings be enclosed within literal braces, and not escapes like - f'\x7b"hi"\x7d'. + ``f'\x7b"hi"\x7d'``. - Issue #28046: Remove platform-specific directories from sys.path. @@ -1197,7 +1574,7 @@ Library - Issue #24277: The new email API is no longer provisional, and the docs have been reorganized and rewritten to emphasize the new API. -- Issue #22450: urllib now includes an "Accept: */*" header among the +- Issue #22450: urllib now includes an ``Accept: */*`` header among the default headers. This makes the results of REST API requests more consistent and predictable especially when proxy servers are involved. @@ -1216,9 +1593,9 @@ Library characters, not on arbitrary unicode line breaks. This also fixes a bug in HTTP header parsing. -- Issue 27331: The email.mime classes now all accept an optional policy keyword. +- Issue #27331: The email.mime classes now all accept an optional policy keyword. -- Issue 27988: Fix email iter_attachments incorrect mutation of payload list. +- Issue #27988: Fix email iter_attachments incorrect mutation of payload list. - Issue #16113: Add SHA-3 and SHAKE support to hashlib module. @@ -2326,7 +2703,7 @@ Core and Builtins in ``def f(): 1.0``. - Issue #4806: Avoid masking the original TypeError exception when using star - (*) unpacking in function calls. Based on patch by Hagen Fürstenau and + (``*``) unpacking in function calls. Based on patch by Hagen Fürstenau and Daniel Urban. - Issue #26146: Add a new kind of AST node: ``ast.Constant``. It can be used @@ -2884,7 +3261,7 @@ Library - Issue #25672: In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode option if it is safe to do so. -- Issue #26012: Don't traverse into symlinks for ** pattern in +- Issue #26012: Don't traverse into symlinks for ``**`` pattern in pathlib.Path.[r]glob(). - Issue #24120: Ignore PermissionError when traversing a tree with @@ -3020,7 +3397,7 @@ Library - Issue #25584: Added "escape" to the __all__ list in the glob module. -- Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. +- Issue #25584: Fixed recursive glob() with patterns starting with ``**``. - Issue #25446: Fix regression in smtplib's AUTH LOGIN support. @@ -3655,7 +4032,7 @@ Library - Issue #28427: old keys should not remove new values from WeakValueDictionary when collecting from another thread. -- Issue 28923: Remove editor artifacts from Tix.py. +- Issue #28923: Remove editor artifacts from Tix.py. - Issue #28871: Fixed a crash when deallocate deep ElementTree. @@ -3778,7 +4155,7 @@ Library - Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). -- Issue #19003:m email.generator now replaces only \r and/or \n line +- Issue #19003:m email.generator now replaces only ``\r`` and/or ``\n`` line endings, per the RFC, instead of all unicode line endings. - Issue #28019: itertools.count() no longer rounds non-integer step in range @@ -3801,7 +4178,7 @@ Library - Issue #27445: Don't pass str(_charset) to MIMEText.set_payload(). Patch by Claude Paroz. -- Issue #22450: urllib now includes an "Accept: */*" header among the +- Issue #22450: urllib now includes an ``Accept: */*`` header among the default headers. This makes the results of REST API requests more consistent and predictable especially when proxy servers are involved. @@ -3816,7 +4193,7 @@ Library characters, not on arbitrary unicode line breaks. This also fixes a bug in HTTP header parsing. -- Issue 27988: Fix email iter_attachments incorrect mutation of payload list. +- Issue #27988: Fix email iter_attachments incorrect mutation of payload list. - Issue #27691: Fix ssl module's parsing of GEN_RID subject alternative name fields in X.509 certs. @@ -4267,7 +4644,7 @@ Core and Builtins cookie names. - Issue #4806: Avoid masking the original TypeError exception when using star - (*) unpacking in function calls. Based on patch by Hagen Fürstenau and + (``*``) unpacking in function calls. Based on patch by Hagen Fürstenau and Daniel Urban. - Issue #27138: Fix the doc comment for FileFinder.find_spec(). @@ -4647,7 +5024,7 @@ Library - Issue #25672: In the ssl module, enable the SSL_MODE_RELEASE_BUFFERS mode option if it is safe to do so. -- Issue #26012: Don't traverse into symlinks for ** pattern in +- Issue #26012: Don't traverse into symlinks for ``**`` pattern in pathlib.Path.[r]glob(). - Issue #24120: Ignore PermissionError when traversing a tree with @@ -5057,7 +5434,7 @@ Library - Issue #25584: Added "escape" to the __all__ list in the glob module. -- Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. +- Issue #25584: Fixed recursive glob() with patterns starting with ``**``. - Issue #25446: Fix regression in smtplib's AUTH LOGIN support. @@ -6600,7 +6977,7 @@ Library - Issue #23521: Corrected pure python implementation of timedelta division. - * Eliminated OverflowError from timedelta * float for some floats; + * Eliminated OverflowError from ``timedelta * float`` for some floats; * Corrected rounding in timedlta true division. - Issue #21619: Popen objects no longer leave a zombie after exit in the with @@ -7399,7 +7776,7 @@ Library character instead of truncating it. Based on patch by Victor Stinner. - Issue #13968: The glob module now supports recursive search in - subdirectories using the "**" pattern. + subdirectories using the ``**`` pattern. - Issue #21951: Fixed a crash in Tkinter on AIX when called Tcl command with empty string or tuple argument. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index a77ff96..492b983 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -305,6 +305,8 @@ future_add_done_callback(FutureObj *fut, PyObject *arg) static PyObject * future_cancel(FutureObj *fut) { + fut->fut_log_tb = 0; + if (fut->fut_state != STATE_PENDING) { Py_RETURN_FALSE; } @@ -638,6 +640,17 @@ FutureObj_get_log_traceback(FutureObj *fut) } } +static int +FutureObj_set_log_traceback(FutureObj *fut, PyObject *val) +{ + int is_true = PyObject_IsTrue(val); + if (is_true < 0) { + return -1; + } + fut->fut_log_tb = is_true; + return 0; +} + static PyObject * FutureObj_get_loop(FutureObj *fut) { @@ -882,7 +895,8 @@ static PyMethodDef FutureType_methods[] = { {"_callbacks", (getter)FutureObj_get_callbacks, NULL, NULL}, \ {"_result", (getter)FutureObj_get_result, NULL, NULL}, \ {"_exception", (getter)FutureObj_get_exception, NULL, NULL}, \ - {"_log_traceback", (getter)FutureObj_get_log_traceback, NULL, NULL}, \ + {"_log_traceback", (getter)FutureObj_get_log_traceback, \ + (setter)FutureObj_set_log_traceback, NULL}, \ {"_source_traceback", (getter)FutureObj_get_source_traceback, NULL, NULL}, static PyGetSetDef FutureType_getsetlist[] = { @@ -1568,6 +1582,8 @@ static PyObject * _asyncio_Task_cancel_impl(TaskObj *self) /*[clinic end generated code: output=6bfc0479da9d5757 input=13f9bf496695cb52]*/ { + self->task_log_tb = 0; + if (self->task_state != STATE_PENDING) { Py_RETURN_FALSE; } @@ -1984,6 +2000,16 @@ task_step_impl(TaskObj *task, PyObject *exc) if (_PyGen_FetchStopIterationValue(&o) == 0) { /* The error is StopIteration and that means that the underlying coroutine has resolved */ + if (task->task_must_cancel) { + // Task is cancelled right before coro stops. + Py_DECREF(o); + task->task_must_cancel = 0; + et = asyncio_CancelledError; + Py_INCREF(et); + ev = NULL; + tb = NULL; + goto set_exception; + } PyObject *res = future_set_result((FutureObj*)task, o); Py_DECREF(o); if (res == NULL) { @@ -2001,6 +2027,8 @@ task_step_impl(TaskObj *task, PyObject *exc) /* Some other exception; pop it and call Task.set_exception() */ PyErr_Fetch(&et, &ev, &tb); + +set_exception: assert(et); if (!ev || !PyObject_TypeCheck(ev, (PyTypeObject *) et)) { PyErr_NormalizeException(&et, &ev, &tb); diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index e6111c6..3015770 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1051,8 +1051,8 @@ deque_index(dequeobject *deque, PyObject *args) int cmp; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(deque); diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index df3aede..12234e2 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -4273,11 +4273,10 @@ Array_subscript(PyObject *myself, PyObject *item) PyObject *np; Py_ssize_t start, stop, step, slicelen, cur, i; - if (PySlice_GetIndicesEx(item, - self->b_length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); stgdict = PyObject_stgdict((PyObject *)self); assert(stgdict); /* Cannot be NULL for array object instances */ @@ -4414,11 +4413,10 @@ Array_ass_subscript(PyObject *myself, PyObject *item, PyObject *value) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen, otherlen, i, cur; - if (PySlice_GetIndicesEx(item, - self->b_length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); if ((step < 0 && start < stop) || (step > 0 && start > stop)) stop = start; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 59d56d0..6119ecd 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -52,9 +52,9 @@ _testfunc_cbk_large_struct(Test in, void (*func)(Test)) EXPORT(void) _testfunc_large_struct_update_value(Test in) { - in.first = 0x0badf00d; - in.second = 0x0badf00d; - in.third = 0x0badf00d; + ((volatile Test *)&in)->first = 0x0badf00d; + ((volatile Test *)&in)->second = 0x0badf00d; + ((volatile Test *)&in)->third = 0x0badf00d; } EXPORT(void)testfunc_array(int values[4]) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 3bf2ca7..d88d06e 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -2082,7 +2082,7 @@ static PyGetSetDef PyCursesWindow_getsets[] = { PyTypeObject PyCursesWindow_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "_curses.curses window", /*tp_name*/ + "_curses.window", /*tp_name*/ sizeof(PyCursesWindowObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index c784d0f..c2ad9a2 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3926,16 +3926,16 @@ time_replace(PyDateTime_Time *self, PyObject *args, PyObject *kw) time_kws, &hh, &mm, &ss, &us, &tzinfo, &fold)) return NULL; + if (fold != 0 && fold != 1) { + PyErr_SetString(PyExc_ValueError, + "fold must be either 0 or 1"); + return NULL; + } tuple = Py_BuildValue("iiiiO", hh, mm, ss, us, tzinfo); if (tuple == NULL) return NULL; clone = time_new(Py_TYPE(self), tuple, NULL); if (clone != NULL) { - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } TIME_SET_FOLD(clone, fold); } Py_DECREF(tuple); @@ -5019,17 +5019,16 @@ datetime_replace(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) &y, &m, &d, &hh, &mm, &ss, &us, &tzinfo, &fold)) return NULL; + if (fold != 0 && fold != 1) { + PyErr_SetString(PyExc_ValueError, + "fold must be either 0 or 1"); + return NULL; + } tuple = Py_BuildValue("iiiiiiiO", y, m, d, hh, mm, ss, us, tzinfo); if (tuple == NULL) return NULL; clone = datetime_new(Py_TYPE(self), tuple, NULL); - if (clone != NULL) { - if (fold != 0 && fold != 1) { - PyErr_SetString(PyExc_ValueError, - "fold must be either 0 or 1"); - return NULL; - } DATE_SET_FOLD(clone, fold); } Py_DECREF(tuple); diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 2cda98e..bef702e 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -131,7 +131,7 @@ elementtree_free(void *m) LOCAL(PyObject*) list_join(PyObject* list) { - /* join list elements (destroying the list in the process) */ + /* join list elements */ PyObject* joiner; PyObject* result; @@ -140,8 +140,6 @@ list_join(PyObject* list) return NULL; result = PyUnicode_Join(joiner, list); Py_DECREF(joiner); - if (result) - Py_DECREF(list); return result; } @@ -508,15 +506,17 @@ element_get_text(ElementObject* self) { /* return borrowed reference to text attribute */ - PyObject* res = self->text; + PyObject *res = self->text; if (JOIN_GET(res)) { res = JOIN_OBJ(res); if (PyList_CheckExact(res)) { - res = list_join(res); - if (!res) + PyObject *tmp = list_join(res); + if (!tmp) return NULL; - self->text = res; + self->text = tmp; + Py_DECREF(res); + res = tmp; } } @@ -528,15 +528,17 @@ element_get_tail(ElementObject* self) { /* return borrowed reference to text attribute */ - PyObject* res = self->tail; + PyObject *res = self->tail; if (JOIN_GET(res)) { res = JOIN_OBJ(res); if (PyList_CheckExact(res)) { - res = list_join(res); - if (!res) + PyObject *tmp = list_join(res); + if (!tmp) return NULL; - self->tail = res; + self->tail = tmp; + Py_DECREF(res); + res = tmp; } } @@ -1710,11 +1712,11 @@ element_subscr(PyObject* self_, PyObject* item) if (!self->extra) return PyList_New(0); - if (PySlice_GetIndicesEx(item, - self->extra->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (slicelen <= 0) return PyList_New(0); @@ -1766,11 +1768,11 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) return -1; } - if (PySlice_GetIndicesEx(item, - self->extra->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (value == NULL) { /* Delete slice */ @@ -2147,6 +2149,12 @@ elementiter_next(ElementIterObject *it) continue; } + if (!PyObject_TypeCheck(extra->children[child_index], &Element_Type)) { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute 'iter'", + Py_TYPE(extra->children[child_index])->tp_name); + return NULL; + } elem = (ElementObject *)extra->children[child_index]; item->child_index++; Py_INCREF(elem); @@ -2396,40 +2404,51 @@ treebuilder_dealloc(TreeBuilderObject *self) /* helpers for handling of arbitrary element-like objects */ static int -treebuilder_set_element_text_or_tail(PyObject *element, PyObject *data, +treebuilder_set_element_text_or_tail(PyObject *element, PyObject **data, PyObject **dest, _Py_Identifier *name) { if (Element_CheckExact(element)) { - Py_DECREF(JOIN_OBJ(*dest)); - *dest = JOIN_SET(data, PyList_CheckExact(data)); + PyObject *tmp = JOIN_OBJ(*dest); + *dest = JOIN_SET(*data, PyList_CheckExact(*data)); + *data = NULL; + Py_DECREF(tmp); return 0; } else { - PyObject *joined = list_join(data); + PyObject *joined = list_join(*data); int r; if (joined == NULL) return -1; r = _PyObject_SetAttrId(element, name, joined); Py_DECREF(joined); - return r; + if (r < 0) + return -1; + Py_CLEAR(*data); + return 0; } } -/* These two functions steal a reference to data */ -static int -treebuilder_set_element_text(PyObject *element, PyObject *data) +LOCAL(int) +treebuilder_flush_data(TreeBuilderObject* self) { - _Py_IDENTIFIER(text); - return treebuilder_set_element_text_or_tail( - element, data, &((ElementObject *) element)->text, &PyId_text); -} + PyObject *element = self->last; -static int -treebuilder_set_element_tail(PyObject *element, PyObject *data) -{ - _Py_IDENTIFIER(tail); - return treebuilder_set_element_text_or_tail( - element, data, &((ElementObject *) element)->tail, &PyId_tail); + if (!self->data) { + return 0; + } + + if (self->this == element) { + _Py_IDENTIFIER(text); + return treebuilder_set_element_text_or_tail( + element, &self->data, + &((ElementObject *) element)->text, &PyId_text); + } + else { + _Py_IDENTIFIER(tail); + return treebuilder_set_element_text_or_tail( + element, &self->data, + &((ElementObject *) element)->tail, &PyId_tail); + } } static int @@ -2479,16 +2498,8 @@ treebuilder_handle_start(TreeBuilderObject* self, PyObject* tag, PyObject* this; elementtreestate *st = ET_STATE_GLOBAL; - if (self->data) { - if (self->this == self->last) { - if (treebuilder_set_element_text(self->last, self->data)) - return NULL; - } - else { - if (treebuilder_set_element_tail(self->last, self->data)) - return NULL; - } - self->data = NULL; + if (treebuilder_flush_data(self) < 0) { + return NULL; } if (!self->element_factory || self->element_factory == Py_None) { @@ -2591,15 +2602,8 @@ treebuilder_handle_end(TreeBuilderObject* self, PyObject* tag) { PyObject* item; - if (self->data) { - if (self->this == self->last) { - if (treebuilder_set_element_text(self->last, self->data)) - return NULL; - } else { - if (treebuilder_set_element_tail(self->last, self->data)) - return NULL; - } - self->data = NULL; + if (treebuilder_flush_data(self) < 0) { + return NULL; } if (self->index == 0) { diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 7abc9f4..1bcf16a 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -250,8 +250,11 @@ partial_repr(partialobject *pto) /* Pack keyword arguments */ assert (PyDict_Check(pto->kw)); for (i = 0; PyDict_Next(pto->kw, &i, &key, &value);) { - Py_SETREF(arglist, PyUnicode_FromFormat("%U, %U=%R", arglist, + /* Prevent key.__str__ from deleting the value. */ + Py_INCREF(value); + Py_SETREF(arglist, PyUnicode_FromFormat("%U, %S=%R", arglist, key, value)); + Py_DECREF(value); if (arglist == NULL) goto done; } diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index cbe7425..efc7d05 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1416,8 +1416,18 @@ buffered_repr(buffered *self) res = PyUnicode_FromFormat("<%s>", Py_TYPE(self)->tp_name); } else { - res = PyUnicode_FromFormat("<%s name=%R>", - Py_TYPE(self)->tp_name, nameobj); + int status = Py_ReprEnter((PyObject *)self); + res = NULL; + if (status == 0) { + res = PyUnicode_FromFormat("<%s name=%R>", + Py_TYPE(self)->tp_name, nameobj); + Py_ReprLeave((PyObject *)self); + } + else if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } Py_DECREF(nameobj); } return res; diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 7f3bcab..833ea8e 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -1082,9 +1082,19 @@ fileio_repr(fileio *self) self->fd, mode_string(self), self->closefd ? "True" : "False"); } else { - res = PyUnicode_FromFormat( - "<_io.FileIO name=%R mode='%s' closefd=%s>", - nameobj, mode_string(self), self->closefd ? "True" : "False"); + int status = Py_ReprEnter((PyObject *)self); + res = NULL; + if (status == 0) { + res = PyUnicode_FromFormat( + "<_io.FileIO name=%R mode='%s' closefd=%s>", + nameobj, mode_string(self), self->closefd ? "True" : "False"); + Py_ReprLeave((PyObject *)self); + } + else if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } Py_DECREF(nameobj); } return res; diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 472ef3b..c864204 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -625,7 +625,8 @@ iobase_iternext(PyObject *self) if (line == NULL) return NULL; - if (PyObject_Size(line) == 0) { + if (PyObject_Size(line) <= 0) { + /* Error or empty */ Py_DECREF(line); return NULL; } @@ -650,7 +651,7 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) /*[clinic end generated code: output=2f50421677fa3dea input=1961c4a95e96e661]*/ { Py_ssize_t length = 0; - PyObject *result; + PyObject *result, *it = NULL; result = PyList_New(0); if (result == NULL) @@ -664,19 +665,23 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) PyObject *ret = _PyObject_CallMethodId(result, &PyId_extend, "O", self); if (ret == NULL) { - Py_DECREF(result); - return NULL; + goto error; } Py_DECREF(ret); return result; } + it = PyObject_GetIter(self); + if (it == NULL) { + goto error; + } + while (1) { - PyObject *line = PyIter_Next(self); + Py_ssize_t line_length; + PyObject *line = PyIter_Next(it); if (line == NULL) { if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; + goto error; } else break; /* StopIteration raised */ @@ -684,16 +689,25 @@ _io__IOBase_readlines_impl(PyObject *self, Py_ssize_t hint) if (PyList_Append(result, line) < 0) { Py_DECREF(line); - Py_DECREF(result); - return NULL; + goto error; } - length += PyObject_Size(line); + line_length = PyObject_Size(line); Py_DECREF(line); - - if (length > hint) + if (line_length < 0) { + goto error; + } + if (line_length > hint - length) break; + length += line_length; } + + Py_DECREF(it); return result; + + error: + Py_XDECREF(it); + Py_DECREF(result); + return NULL; } /*[clinic input] diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 4df5562..bc8d11e 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2483,6 +2483,7 @@ static PyObject * textiowrapper_repr(textio *self) { PyObject *nameobj, *modeobj, *res, *s; + int status; CHECK_INITIALIZED(self); @@ -2490,6 +2491,15 @@ textiowrapper_repr(textio *self) if (res == NULL) return NULL; + status = Py_ReprEnter((PyObject *)self); + if (status != 0) { + if (status > 0) { + PyErr_Format(PyExc_RuntimeError, + "reentrant call inside %s.__repr__", + Py_TYPE(self)->tp_name); + } + goto error; + } nameobj = _PyObject_GetAttrId((PyObject *) self, &PyId_name); if (nameobj == NULL) { if (PyErr_ExceptionMatches(PyExc_Exception)) @@ -2504,7 +2514,7 @@ textiowrapper_repr(textio *self) goto error; PyUnicode_AppendAndDel(&res, s); if (res == NULL) - return NULL; + goto error; } modeobj = _PyObject_GetAttrId((PyObject *) self, &PyId_mode); if (modeobj == NULL) { @@ -2520,14 +2530,21 @@ textiowrapper_repr(textio *self) goto error; PyUnicode_AppendAndDel(&res, s); if (res == NULL) - return NULL; + goto error; } s = PyUnicode_FromFormat("%U encoding=%R>", res, self->encoding); Py_DECREF(res); + if (status == 0) { + Py_ReprLeave((PyObject *)self); + } return s; -error: + + error: Py_XDECREF(res); + if (status == 0) { + Py_ReprLeave((PyObject *)self); + } return NULL; } diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 1d169e2..94dfe3e 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -101,7 +101,7 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { DWORD length; wchar_t name_buf[MAX_PATH], *pname_buf = name_buf; - + length = GetFullPathNameW(decoded_wstr, MAX_PATH, pname_buf, NULL); if (length > MAX_PATH) { pname_buf = PyMem_New(wchar_t, length); @@ -305,8 +305,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, self->fd = fd; if (fd < 0) { - PyObject *decodedname = Py_None; - Py_INCREF(decodedname); + PyObject *decodedname; int d = PyUnicode_FSDecoder(nameobj, (void*)&decodedname); if (!d) @@ -1017,7 +1016,7 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b) wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, wbuf, wlen); if (wlen) { res = WriteConsoleW(self->handle, wbuf, wlen, &n, NULL); - if (n < wlen) { + if (res && n < wlen) { /* Wrote fewer characters than expected, which means our * len value may be wrong. So recalculate it from the * characters that were written. As this could potentially diff --git a/Modules/_json.c b/Modules/_json.c index faa2134..a84b085 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -89,16 +89,12 @@ static PyObject * _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); static PyObject * scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds); static void scanner_dealloc(PyObject *self); static int scanner_clear(PyObject *self); static PyObject * encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds); static void encoder_dealloc(PyObject *self); static int @@ -1203,38 +1199,21 @@ static PyObject * scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyScannerObject *s; - s = (PyScannerObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->strict = NULL; - s->object_hook = NULL; - s->object_pairs_hook = NULL; - s->parse_float = NULL; - s->parse_int = NULL; - s->parse_constant = NULL; - } - return (PyObject *)s; -} - -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Initialize Scanner object */ PyObject *ctx; static char *kwlist[] = {"context", NULL}; - PyScannerObject *s; - - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) - return -1; + return NULL; - if (s->memo == NULL) { - s->memo = PyDict_New(); - if (s->memo == NULL) - goto bail; + s = (PyScannerObject *)type->tp_alloc(type, 0); + if (s == NULL) { + return NULL; } + s->memo = PyDict_New(); + if (s->memo == NULL) + goto bail; + /* All of these will fail "gracefully" so we don't need to verify them */ s->strict = PyObject_GetAttrString(ctx, "strict"); if (s->strict == NULL) @@ -1255,16 +1234,11 @@ scanner_init(PyObject *self, PyObject *args, PyObject *kwds) if (s->parse_constant == NULL) goto bail; - return 0; + return (PyObject *)s; bail: - Py_CLEAR(s->strict); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->object_pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - return -1; + Py_DECREF(s); + return NULL; } PyDoc_STRVAR(scanner_doc, "JSON scanner object"); @@ -1306,7 +1280,7 @@ PyTypeObject PyScannerType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - scanner_init, /* tp_init */ + 0, /* tp_init */ 0,/* PyType_GenericAlloc, */ /* tp_alloc */ scanner_new, /* tp_new */ 0,/* PyObject_GC_Del, */ /* tp_free */ @@ -1315,25 +1289,6 @@ PyTypeObject PyScannerType = { static PyObject * encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyEncoderObject *s; - s = (PyEncoderObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->markers = NULL; - s->defaultfn = NULL; - s->encoder = NULL; - s->indent = NULL; - s->key_separator = NULL; - s->item_separator = NULL; - s->sort_keys = NULL; - s->skipkeys = NULL; - } - return (PyObject *)s; -} - -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* initialize Encoder object */ static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", NULL}; PyEncoderObject *s; @@ -1341,22 +1296,23 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds) PyObject *item_separator, *sort_keys, *skipkeys; int allow_nan; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOUUOOp:make_encoder", kwlist, &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, &sort_keys, &skipkeys, &allow_nan)) - return -1; + return NULL; if (markers != Py_None && !PyDict_Check(markers)) { PyErr_Format(PyExc_TypeError, "make_encoder() argument 1 must be dict or None, " "not %.200s", Py_TYPE(markers)->tp_name); - return -1; + return NULL; } + s = (PyEncoderObject *)type->tp_alloc(type, 0); + if (s == NULL) + return NULL; + s->markers = markers; s->defaultfn = defaultfn; s->encoder = encoder; @@ -1383,7 +1339,7 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds) Py_INCREF(s->item_separator); Py_INCREF(s->sort_keys); Py_INCREF(s->skipkeys); - return 0; + return (PyObject *)s; } static PyObject * @@ -1914,7 +1870,7 @@ PyTypeObject PyEncoderType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - encoder_init, /* tp_init */ + 0, /* tp_init */ 0, /* tp_alloc */ encoder_new, /* tp_new */ 0, /* tp_free */ @@ -1957,10 +1913,8 @@ PyInit__json(void) PyObject *m = PyModule_Create(&jsonmodule); if (!m) return NULL; - PyScannerType.tp_new = PyType_GenericNew; if (PyType_Ready(&PyScannerType) < 0) goto fail; - PyEncoderType.tp_new = PyType_GenericNew; if (PyType_Ready(&PyEncoderType) < 0) goto fail; Py_INCREF((PyObject*)&PyScannerType); diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 5007a39..d1434d5 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -111,13 +111,17 @@ _is_fdescfs_mounted_on_dev_fd(void) static int _sanity_check_python_fd_sequence(PyObject *fd_sequence) { - Py_ssize_t seq_idx, seq_len = PySequence_Length(fd_sequence); + Py_ssize_t seq_idx; long prev_fd = -1; - for (seq_idx = 0; seq_idx < seq_len; ++seq_idx) { - PyObject* py_fd = PySequence_Fast_GET_ITEM(fd_sequence, seq_idx); - long iter_fd = PyLong_AsLong(py_fd); + for (seq_idx = 0; seq_idx < PyTuple_GET_SIZE(fd_sequence); ++seq_idx) { + PyObject* py_fd = PyTuple_GET_ITEM(fd_sequence, seq_idx); + long iter_fd; + if (!PyLong_Check(py_fd)) { + return 1; + } + iter_fd = PyLong_AsLong(py_fd); if (iter_fd < 0 || iter_fd <= prev_fd || iter_fd > INT_MAX) { - /* Negative, overflow, not a Long, unsorted, too big for a fd. */ + /* Negative, overflow, unsorted, too big for a fd. */ return 1; } prev_fd = iter_fd; @@ -132,13 +136,12 @@ _is_fd_in_sorted_fd_sequence(int fd, PyObject *fd_sequence) { /* Binary search. */ Py_ssize_t search_min = 0; - Py_ssize_t search_max = PySequence_Length(fd_sequence) - 1; + Py_ssize_t search_max = PyTuple_GET_SIZE(fd_sequence) - 1; if (search_max < 0) return 0; do { long middle = (search_min + search_max) / 2; - long middle_fd = PyLong_AsLong( - PySequence_Fast_GET_ITEM(fd_sequence, middle)); + long middle_fd = PyLong_AsLong(PyTuple_GET_ITEM(fd_sequence, middle)); if (fd == middle_fd) return 1; if (fd > middle_fd) @@ -154,9 +157,9 @@ make_inheritable(PyObject *py_fds_to_keep, int errpipe_write) { Py_ssize_t i, len; - len = PySequence_Length(py_fds_to_keep); + len = PyTuple_GET_SIZE(py_fds_to_keep); for (i = 0; i < len; ++i) { - PyObject* fdobj = PySequence_Fast_GET_ITEM(py_fds_to_keep, i); + PyObject* fdobj = PyTuple_GET_ITEM(py_fds_to_keep, i); long fd = PyLong_AsLong(fdobj); assert(!PyErr_Occurred()); assert(0 <= fd && fd <= INT_MAX); @@ -213,14 +216,13 @@ static void _close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) { long end_fd = safe_get_max_fd(); - Py_ssize_t num_fds_to_keep = PySequence_Length(py_fds_to_keep); + Py_ssize_t num_fds_to_keep = PyTuple_GET_SIZE(py_fds_to_keep); Py_ssize_t keep_seq_idx; int fd_num; /* As py_fds_to_keep is sorted we can loop through the list closing * fds inbetween any in the keep list falling within our range. */ for (keep_seq_idx = 0; keep_seq_idx < num_fds_to_keep; ++keep_seq_idx) { - PyObject* py_keep_fd = PySequence_Fast_GET_ITEM(py_fds_to_keep, - keep_seq_idx); + PyObject* py_keep_fd = PyTuple_GET_ITEM(py_fds_to_keep, keep_seq_idx); int keep_fd = PyLong_AsLong(py_keep_fd); if (keep_fd < start_fd) continue; @@ -306,7 +308,7 @@ _close_open_fds_safe(int start_fd, PyObject* py_fds_to_keep) /* Close all open file descriptors from start_fd and higher. - * Do not close any in the sorted py_fds_to_keep list. + * Do not close any in the sorted py_fds_to_keep tuple. * * This function violates the strict use of async signal safe functions. :( * It calls opendir(), readdir() and closedir(). Of these, the one most @@ -562,8 +564,9 @@ subprocess_fork_exec(PyObject* self, PyObject *args) #endif if (!PyArg_ParseTuple( - args, "OOpOOOiiiiiiiiiiO:fork_exec", - &process_args, &executable_list, &close_fds, &py_fds_to_keep, + args, "OOpO!OOiiiiiiiiiiO:fork_exec", + &process_args, &executable_list, + &close_fds, &PyTuple_Type, &py_fds_to_keep, &cwd_obj, &env_list, &p2cread, &p2cwrite, &c2pread, &c2pwrite, &errread, &errwrite, &errpipe_read, &errpipe_write, @@ -574,10 +577,6 @@ subprocess_fork_exec(PyObject* self, PyObject *args) PyErr_SetString(PyExc_ValueError, "errpipe_write must be >= 3"); return NULL; } - if (PySequence_Length(py_fds_to_keep) < 0) { - PyErr_SetString(PyExc_ValueError, "cannot get length of fds_to_keep"); - return NULL; - } if (_sanity_check_python_fd_sequence(py_fds_to_keep)) { PyErr_SetString(PyExc_ValueError, "bad value(s) in fds_to_keep"); return NULL; @@ -631,6 +630,10 @@ subprocess_fork_exec(PyObject* self, PyObject *args) goto cleanup; for (arg_num = 0; arg_num < num_args; ++arg_num) { PyObject *borrowed_arg, *converted_arg; + if (PySequence_Fast_GET_SIZE(fast_args) != num_args) { + PyErr_SetString(PyExc_RuntimeError, "args changed during iteration"); + goto cleanup; + } borrowed_arg = PySequence_Fast_GET_ITEM(fast_args, arg_num); if (PyUnicode_FSConverter(borrowed_arg, &converted_arg) == 0) goto cleanup; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index e5dd2c9..d006aeb 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -348,6 +348,7 @@ random_setstate(RandomObject *self, PyObject *state) int i; unsigned long element; long index; + uint32_t new_state[N]; if (!PyTuple_Check(state)) { PyErr_SetString(PyExc_TypeError, @@ -364,7 +365,7 @@ random_setstate(RandomObject *self, PyObject *state) element = PyLong_AsUnsignedLong(PyTuple_GET_ITEM(state, i)); if (element == (unsigned long)-1 && PyErr_Occurred()) return NULL; - self->state[i] = (uint32_t)element; + new_state[i] = (uint32_t)element; } index = PyLong_AsLong(PyTuple_GET_ITEM(state, i)); @@ -375,6 +376,8 @@ random_setstate(RandomObject *self, PyObject *state) return NULL; } self->index = (int)index; + for (i = 0; i < N; i++) + self->state[i] = new_state[i]; Py_INCREF(Py_None); return Py_None; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index c0a7b8e..0fffaac 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -599,6 +599,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->ssl = NULL; self->Socket = NULL; self->ctx = sslctx; + Py_INCREF(sslctx); self->shutdown_seen_zero = 0; self->handshake_done = 0; self->owner = NULL; @@ -613,8 +614,6 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->server_hostname = hostname; } - Py_INCREF(sslctx); - /* Make sure the SSL error state is initialized */ (void) ERR_get_state(); ERR_clear_error(); @@ -1210,10 +1209,6 @@ _get_crl_dp(X509 *certificate) { int i, j; PyObject *lst, *res = NULL; -#if OPENSSL_VERSION_NUMBER >= 0x10001000L - /* Calls x509v3_cache_extensions and sets up crldp */ - X509_check_ca(certificate); -#endif dps = X509_get_ext_d2i(certificate, NID_crl_distribution_points, NULL, NULL); if (dps == NULL) @@ -1258,9 +1253,7 @@ _get_crl_dp(X509 *certificate) { done: Py_XDECREF(lst); -#if OPENSSL_VERSION_NUMBER < 0x10001000L - sk_DIST_POINT_free(dps); -#endif + CRL_DIST_POINTS_free(dps); return res; } @@ -4482,12 +4475,12 @@ _ssl.RAND_add Mix string into the OpenSSL PRNG state. entropy (a float) is a lower bound on the entropy contained in -string. See RFC 1750. +string. See RFC 4086. [clinic start generated code]*/ static PyObject * _ssl_RAND_add_impl(PyObject *module, Py_buffer *view, double entropy) -/*[clinic end generated code: output=e6dd48df9c9024e9 input=580c85e6a3a4fe29]*/ +/*[clinic end generated code: output=e6dd48df9c9024e9 input=5c33017422828f5c]*/ { const char *buf; Py_ssize_t len, written; diff --git a/Modules/_struct.c b/Modules/_struct.c index 796d168..2635af9 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -539,7 +539,7 @@ np_ubyte(char *p, PyObject *v, const formatdef *f) "ubyte format requires 0 <= number <= 255"); return -1; } - *p = (char)x; + *(unsigned char *)p = (unsigned char)x; return 0; } @@ -868,6 +868,7 @@ bp_int(char *p, PyObject *v, const formatdef *f) { long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_long(v, &x) < 0) return -1; i = f->size; @@ -880,7 +881,7 @@ bp_int(char *p, PyObject *v, const formatdef *f) #endif } do { - p[--i] = (char)x; + q[--i] = (unsigned char)(x & 0xffL); x >>= 8; } while (i > 0); return 0; @@ -891,6 +892,7 @@ bp_uint(char *p, PyObject *v, const formatdef *f) { unsigned long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_ulong(v, &x) < 0) return -1; i = f->size; @@ -901,7 +903,7 @@ bp_uint(char *p, PyObject *v, const formatdef *f) RANGE_ERROR(x, f, 1, maxint - 1); } do { - p[--i] = (char)x; + q[--i] = (unsigned char)(x & 0xffUL); x >>= 8; } while (i > 0); return 0; @@ -1087,6 +1089,7 @@ lp_int(char *p, PyObject *v, const formatdef *f) { long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_long(v, &x) < 0) return -1; i = f->size; @@ -1099,7 +1102,7 @@ lp_int(char *p, PyObject *v, const formatdef *f) #endif } do { - *p++ = (char)x; + *q++ = (unsigned char)(x & 0xffL); x >>= 8; } while (--i > 0); return 0; @@ -1110,6 +1113,7 @@ lp_uint(char *p, PyObject *v, const formatdef *f) { unsigned long x; Py_ssize_t i; + unsigned char *q = (unsigned char *)p; if (get_ulong(v, &x) < 0) return -1; i = f->size; @@ -1120,7 +1124,7 @@ lp_uint(char *p, PyObject *v, const formatdef *f) RANGE_ERROR(x, f, 1, maxint - 1); } do { - *p++ = (char)x; + *q++ = (unsigned char)(x & 0xffUL); x >>= 8; } while (--i > 0); return 0; diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 4e1ce68..6b8ab34 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -1715,10 +1715,10 @@ init_slice(Py_buffer *base, PyObject *key, int dim) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, base->shape[dim], - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(key, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(base->shape[dim], &start, &stop, step); if (base->suboffsets == NULL || dim == 0) { @@ -1935,9 +1935,10 @@ slice_indices(PyObject *self, PyObject *args) "first argument must be a slice object"); return NULL; } - if (PySlice_GetIndicesEx(key, len, &s[0], &s[1], &s[2], &s[3]) < 0) { + if (PySlice_Unpack(key, &s[0], &s[1], &s[2]) < 0) { return NULL; } + s[3] = PySlice_AdjustIndices(len, &s[0], &s[1], s[2]); ret = PyTuple_New(4); if (ret == NULL) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f09205f..c3d829c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1571,7 +1571,7 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) { PyObject *sub_args; PyObject *sub_kwargs; - char *sub_format; + const char *sub_format; PyObject *sub_keywords; Py_ssize_t i, size; @@ -1584,7 +1584,7 @@ parse_tuple_and_keywords(PyObject *self, PyObject *args) double buffers[8][4]; /* double ensures alignment where necessary */ - if (!PyArg_ParseTuple(args, "OOyO:parse_tuple_and_keywords", + if (!PyArg_ParseTuple(args, "OOsO:parse_tuple_and_keywords", &sub_args, &sub_kwargs, &sub_format, &sub_keywords)) return NULL; @@ -3812,7 +3812,7 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) if (_PyTime_AsTimeval(t, &tv, round) < 0) return NULL; - seconds = PyLong_FromLong((long long)tv.tv_sec); + seconds = PyLong_FromLongLong(tv.tv_sec); if (seconds == NULL) return NULL; return Py_BuildValue("Nl", seconds, tv.tv_usec); @@ -4027,6 +4027,127 @@ dict_get_version(PyObject *self, PyObject *args) } +static int +fastcall_args(PyObject *args, PyObject ***stack, Py_ssize_t *nargs) +{ + if (args == Py_None) { + *stack = NULL; + *nargs = 0; + } + else if (PyTuple_Check(args)) { + *stack = &PyTuple_GET_ITEM(args, 0); + *nargs = PyTuple_GET_SIZE(args); + } + else { + PyErr_SetString(PyExc_TypeError, "args must be None or a tuple"); + return -1; + } + return 0; +} + + +static PyObject * +test_pyobject_fastcall(PyObject *self, PyObject *args) +{ + PyObject *func, *func_args; + PyObject **stack; + Py_ssize_t nargs; + + if (!PyArg_ParseTuple(args, "OO", &func, &func_args)) { + return NULL; + } + + if (fastcall_args(func_args, &stack, &nargs) < 0) { + return NULL; + } + return _PyObject_FastCall(func, stack, nargs); +} + + +static PyObject * +test_pyobject_fastcalldict(PyObject *self, PyObject *args) +{ + PyObject *func, *func_args, *kwargs; + PyObject **stack; + Py_ssize_t nargs; + + if (!PyArg_ParseTuple(args, "OOO", &func, &func_args, &kwargs)) { + return NULL; + } + + if (fastcall_args(func_args, &stack, &nargs) < 0) { + return NULL; + } + + if (kwargs == Py_None) { + kwargs = NULL; + } + else if (!PyDict_Check(kwargs)) { + PyErr_SetString(PyExc_TypeError, "kwnames must be None or a dict"); + return NULL; + } + + return _PyObject_FastCallDict(func, stack, nargs, kwargs); +} + + +static PyObject * +test_pyobject_fastcallkeywords(PyObject *self, PyObject *args) +{ + PyObject *func, *func_args, *kwnames = NULL; + PyObject **stack; + Py_ssize_t nargs, nkw; + + if (!PyArg_ParseTuple(args, "OOO", &func, &func_args, &kwnames)) { + return NULL; + } + + if (fastcall_args(func_args, &stack, &nargs) < 0) { + return NULL; + } + + if (kwnames == Py_None) { + kwnames = NULL; + } + else if (PyTuple_Check(kwnames)) { + nkw = PyTuple_GET_SIZE(kwnames); + if (nargs < nkw) { + PyErr_SetString(PyExc_ValueError, "kwnames longer than args"); + return NULL; + } + nargs -= nkw; + } + else { + PyErr_SetString(PyExc_TypeError, "kwnames must be None or a tuple"); + return NULL; + } + return _PyObject_FastCallKeywords(func, stack, nargs, kwnames); +} + + +static PyObject * +raise_SIGINT_then_send_None(PyObject *self, PyObject *args) +{ + PyGenObject *gen; + + if (!PyArg_ParseTuple(args, "O!", &PyGen_Type, &gen)) + return NULL; + + /* This is used in a test to check what happens if a signal arrives just + as we're in the process of entering a yield from chain (see + bpo-30039). + + Needs to be done in C, because: + - we don't have a Python wrapper for raise() + - we need to make sure that the Python-level signal handler doesn't run + *before* we enter the generator frame, which is impossible in Python + because we check for signals before every bytecode operation. + */ + raise(SIGINT); + return _PyGen_Send(gen, Py_None); +} + + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", (PyCFunction)raise_memoryerror, METH_NOARGS}, @@ -4230,6 +4351,10 @@ static PyMethodDef TestMethods[] = { {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {"dict_get_version", dict_get_version, METH_VARARGS}, + {"pyobject_fastcall", test_pyobject_fastcall, METH_VARARGS}, + {"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS}, + {"pyobject_fastcallkeywords", test_pyobject_fastcallkeywords, METH_VARARGS}, + {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 91d4f01..1606f0d 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -722,23 +722,42 @@ getenvironment(PyObject* environment) return NULL; } - envsize = PyMapping_Length(environment); - keys = PyMapping_Keys(environment); values = PyMapping_Values(environment); if (!keys || !values) goto error; + envsize = PySequence_Fast_GET_SIZE(keys); + if (PySequence_Fast_GET_SIZE(values) != envsize) { + PyErr_SetString(PyExc_RuntimeError, + "environment changed size during iteration"); + goto error; + } + totalsize = 1; /* trailing null character */ for (i = 0; i < envsize; i++) { - PyObject* key = PyList_GET_ITEM(keys, i); - PyObject* value = PyList_GET_ITEM(values, i); + PyObject* key = PySequence_Fast_GET_ITEM(keys, i); + PyObject* value = PySequence_Fast_GET_ITEM(values, i); if (! PyUnicode_Check(key) || ! PyUnicode_Check(value)) { PyErr_SetString(PyExc_TypeError, "environment can only contain strings"); goto error; } + if (PyUnicode_FindChar(key, '\0', 0, PyUnicode_GET_LENGTH(key), 1) != -1 || + PyUnicode_FindChar(value, '\0', 0, PyUnicode_GET_LENGTH(value), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto error; + } + /* Search from index 1 because on Windows starting '=' is allowed for + defining hidden environment variables. */ + if (PyUnicode_GET_LENGTH(key) == 0 || + PyUnicode_FindChar(key, '=', 1, PyUnicode_GET_LENGTH(key), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + goto error; + } if (totalsize > PY_SSIZE_T_MAX - PyUnicode_GET_LENGTH(key) - 1) { PyErr_SetString(PyExc_OverflowError, "environment too long"); goto error; @@ -760,8 +779,8 @@ getenvironment(PyObject* environment) end = buffer + totalsize; for (i = 0; i < envsize; i++) { - PyObject* key = PyList_GET_ITEM(keys, i); - PyObject* value = PyList_GET_ITEM(values, i); + PyObject* key = PySequence_Fast_GET_ITEM(keys, i); + PyObject* value = PySequence_Fast_GET_ITEM(values, i); if (!PyUnicode_AsUCS4(key, p, end - p, 0)) goto error; p += PyUnicode_GET_LENGTH(key); @@ -841,12 +860,13 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, if (env_mapping != Py_None) { environment = getenvironment(env_mapping); - if (! environment) + if (environment == NULL) { return NULL; + } + /* contains embedded null characters */ wenvironment = PyUnicode_AsUnicode(environment); - if (wenvironment == NULL) - { - Py_XDECREF(environment); + if (wenvironment == NULL) { + Py_DECREF(environment); return NULL; } } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 2caa8ee..64e0f17 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -331,35 +331,51 @@ II_getitem(arrayobject *ap, Py_ssize_t i) (unsigned long) ((unsigned int *)ap->ob_item)[i]); } +static PyObject * +get_int_unless_float(PyObject *v) +{ + if (PyFloat_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "array item must be integer"); + return NULL; + } + return (PyObject *)_PyLong_FromNbInt(v); +} + static int II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long) -1 && PyErr_Occurred()) + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; + } + do_decref = 1; } - else { - long y; - if (!PyArg_Parse(v, "l;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned int is less than minimum"); - return -1; + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); } - x = (unsigned long)y; - + return -1; } if (x > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, - "unsigned int is greater than maximum"); + "unsigned int is greater than maximum"); + if (do_decref) { + Py_DECREF(v); + } return -1; } - if (i >= 0) ((unsigned int *)ap->ob_item)[i] = (unsigned int)x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -390,31 +406,28 @@ static int LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLong(v); - if (x == (unsigned long) -1 && PyErr_Occurred()) - return -1; - } - else { - long y; - if (!PyArg_Parse(v, "l;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long is less than minimum"); + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; } - x = (unsigned long)y; - + do_decref = 1; } - if (x > ULONG_MAX) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long is greater than maximum"); + x = PyLong_AsUnsignedLong(v); + if (x == (unsigned long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); + } return -1; } - if (i >= 0) ((unsigned long *)ap->ob_item)[i] = x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -446,25 +459,28 @@ static int QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { unsigned long long x; - if (PyLong_Check(v)) { - x = PyLong_AsUnsignedLongLong(v); - if (x == (unsigned long long) -1 && PyErr_Occurred()) + int do_decref = 0; /* if nb_int was called */ + + if (!PyLong_Check(v)) { + v = get_int_unless_float(v); + if (NULL == v) { return -1; + } + do_decref = 1; } - else { - long long y; - if (!PyArg_Parse(v, "L;array item must be integer", &y)) - return -1; - if (y < 0) { - PyErr_SetString(PyExc_OverflowError, - "unsigned long long is less than minimum"); - return -1; + x = PyLong_AsUnsignedLongLong(v); + if (x == (unsigned long long)-1 && PyErr_Occurred()) { + if (do_decref) { + Py_DECREF(v); } - x = (unsigned long long)y; + return -1; } - if (i >= 0) ((unsigned long long *)ap->ob_item)[i] = x; + + if (do_decref) { + Py_DECREF(v); + } return 0; } @@ -2281,10 +2297,11 @@ array_subscr(arrayobject* self, PyObject* item) arrayobject* ar; int itemsize = self->ob_descr->itemsize; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (slicelength <= 0) { return newarrayobject(&Arraytype, 0, self->ob_descr); @@ -2352,11 +2369,11 @@ array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value) return (*self->ob_descr->setitem)(self, i, value); } else if (PySlice_Check(item)) { - if (PySlice_GetIndicesEx(item, - Py_SIZE(self), &start, &stop, - &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); } else { PyErr_SetString(PyExc_TypeError, diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 3cdf2d7..35ebf08 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -97,7 +97,7 @@ atexit_callfuncs(void) Py_XDECREF(exc_tb); } PyErr_Fetch(&exc_type, &exc_value, &exc_tb); - if (!PyErr_ExceptionMatches(PyExc_SystemExit)) { + if (!PyErr_GivenExceptionMatches(exc_type, PyExc_SystemExit)) { PySys_WriteStderr("Error in atexit._run_exitfuncs:\n"); PyErr_NormalizeException(&exc_type, &exc_value, &exc_tb); PyErr_Display(exc_type, exc_value, exc_tb); diff --git a/Modules/cjkcodecs/_codecs_cn.c b/Modules/cjkcodecs/_codecs_cn.c index 1a070f2..1fcc220 100644 --- a/Modules/cjkcodecs/_codecs_cn.c +++ b/Modules/cjkcodecs/_codecs_cn.c @@ -279,7 +279,9 @@ DECODER(gb18030) REQUIRE_INBUF(4); c3 = INBYTE3; c4 = INBYTE4; - if (c < 0x81 || c3 < 0x81 || c4 < 0x30 || c4 > 0x39) + if (c < 0x81 || c > 0xFE || + c3 < 0x81 || c3 > 0xFE || + c4 < 0x30 || c4 > 0x39) return 1; c -= 0x81; c2 -= 0x30; c3 -= 0x81; c4 -= 0x30; @@ -348,15 +350,17 @@ ENCODER(hz) DBCHAR code; if (c < 0x80) { - if (state->i == 0) { - WRITEBYTE1((unsigned char)c); - NEXT(1, 1); - } - else { - WRITEBYTE3('~', '}', (unsigned char)c); - NEXT(1, 3); + if (state->i) { + WRITEBYTE2('~', '}'); + NEXT_OUT(2); state->i = 0; } + WRITEBYTE1((unsigned char)c); + NEXT(1, 1); + if (c == '~') { + WRITEBYTE1('~'); + NEXT_OUT(1); + } continue; } @@ -407,17 +411,14 @@ DECODER(hz) unsigned char c2 = INBYTE2; REQUIRE_INBUF(2); - if (c2 == '~') { + if (c2 == '~' && state->i == 0) OUTCHAR('~'); - NEXT_IN(2); - continue; - } else if (c2 == '{' && state->i == 0) state->i = 1; /* set GB */ + else if (c2 == '\n' && state->i == 0) + ; /* line-continuation */ else if (c2 == '}' && state->i == 1) state->i = 0; /* set ASCII */ - else if (c2 == '\n') - ; /* line-continuation */ else return 1; NEXT_IN(2); diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index d1da189..d6efc77 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -1670,6 +1670,9 @@ _multibytecodec_MultibyteStreamWriter_writelines(MultibyteStreamWriterObject *se if (r == -1) return NULL; } + /* PySequence_Length() can fail */ + if (PyErr_Occurred()) + return NULL; Py_RETURN_NONE; } diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 29f5838..75f8f5a 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -839,7 +839,7 @@ PyDoc_STRVAR(_ssl_RAND_add__doc__, "Mix string into the OpenSSL PRNG state.\n" "\n" "entropy (a float) is a lower bound on the entropy contained in\n" -"string. See RFC 1750."); +"string. See RFC 4086."); #define _SSL_RAND_ADD_METHODDEF \ {"RAND_add", (PyCFunction)_ssl_RAND_add, METH_VARARGS, _ssl_RAND_add__doc__}, @@ -1168,4 +1168,4 @@ exit: #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=a859b21fe68a6115 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=56cead8610faa505 input=a9049054013a1b77]*/ diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index dcb4506..8d288f0 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,6 +1,5 @@ -Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd - and Clark Cooper -Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Expat maintainers. +Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper +Copyright (c) 2001-2017 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/amigaconfig.h b/Modules/expat/amigaconfig.h deleted file mode 100644 index 86c6115..0000000 --- a/Modules/expat/amigaconfig.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef AMIGACONFIG_H -#define AMIGACONFIG_H - -/* 1234 = LIL_ENDIAN, 4321 = BIGENDIAN */ -#define BYTEORDER 4321 - -/* Define to 1 if you have the `bcopy' function. */ -#define HAVE_BCOPY 1 - -/* Define to 1 if you have the header file. */ -#undef HAVE_CHECK_H - -/* Define to 1 if you have the `memmove' function. */ -#define HAVE_MEMMOVE 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_UNISTD_H 1 - -/* whether byteorder is bigendian */ -#define WORDS_BIGENDIAN - -/* Define to specify how much context to retain around the current parse - point. */ -#define XML_CONTEXT_BYTES 1024 - -/* Define to make parameter entity parsing functionality available. */ -#define XML_DTD - -/* Define to make XML Namespaces functionality available. */ -#define XML_NS - -#endif /* AMIGACONFIG_H */ diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index e8eefdd..28b0f95 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -95,7 +95,9 @@ enum XML_Error { /* Added in 2.0. */ XML_ERROR_RESERVED_PREFIX_XML, XML_ERROR_RESERVED_PREFIX_XMLNS, - XML_ERROR_RESERVED_NAMESPACE_URI + XML_ERROR_RESERVED_NAMESPACE_URI, + /* Added in 2.2.1. */ + XML_ERROR_INVALID_ARGUMENT }; enum XML_Content_Type { @@ -342,7 +344,7 @@ XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler); /* OBSOLETE -- OBSOLETE -- OBSOLETE - This handler has been superceded by the EntityDeclHandler above. + This handler has been superseded by the EntityDeclHandler above. It is provided here for backward compatibility. This is called for a declaration of an unparsed (NDATA) entity. @@ -706,6 +708,7 @@ XML_UseParserAsHandlerArg(XML_Parser parser); be called, despite an external subset being parsed. Note: If XML_DTD is not defined when Expat is compiled, returns XML_ERROR_FEATURE_REQUIRES_XML_DTD. + Note: If parser == NULL, returns XML_ERROR_INVALID_ARGUMENT. */ XMLPARSEAPI(enum XML_Error) XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD); @@ -729,15 +732,16 @@ XML_GetBase(XML_Parser parser); to the XML_StartElementHandler that were specified in the start-tag rather than defaulted. Each attribute/value pair counts as 2; thus this correspondds to an index into the atts array passed to the - XML_StartElementHandler. + XML_StartElementHandler. Returns -1 if parser == NULL. */ XMLPARSEAPI(int) XML_GetSpecifiedAttributeCount(XML_Parser parser); /* Returns the index of the ID attribute passed in the last call to - XML_StartElementHandler, or -1 if there is no ID attribute. Each - attribute/value pair counts as 2; thus this correspondds to an - index into the atts array passed to the XML_StartElementHandler. + XML_StartElementHandler, or -1 if there is no ID attribute or + parser == NULL. Each attribute/value pair counts as 2; thus this + correspondds to an index into the atts array passed to the + XML_StartElementHandler. */ XMLPARSEAPI(int) XML_GetIdAttributeIndex(XML_Parser parser); @@ -901,6 +905,7 @@ enum XML_ParamEntityParsing { entities is requested; otherwise it will return non-zero. Note: If XML_SetParamEntityParsing is called after XML_Parse or XML_ParseBuffer, then it has no effect and will always return 0. + Note: If parser == NULL, the function will do nothing and return 0. */ XMLPARSEAPI(int) XML_SetParamEntityParsing(XML_Parser parser, @@ -910,13 +915,12 @@ XML_SetParamEntityParsing(XML_Parser parser, Helps in preventing DoS attacks based on predicting hash function behavior. This must be called before parsing is started. Returns 1 if successful, 0 when called after parsing has started. + Note: If parser == NULL, the function will do nothing and return 0. */ XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt); -#define XML_HAS_SET_HASH_SALT /* Python Only: Defined for pyexpat.c. */ - /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then XML_GetErrorCode returns information about the error. */ @@ -938,6 +942,10 @@ XML_GetErrorCode(XML_Parser parser); the location is the location of the character at which the error was detected; otherwise the location is the location of the last parse event, as described above. + + Note: XML_GetCurrentLineNumber and XML_GetCurrentColumnNumber + return 0 to indicate an error. + Note: XML_GetCurrentByteIndex returns -1 to indicate an error. */ XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser); XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser); @@ -975,9 +983,12 @@ XML_FreeContentModel(XML_Parser parser, XML_Content *model); /* Exposing the memory handling functions used in Expat */ XMLPARSEAPI(void *) +XML_ATTR_MALLOC +XML_ATTR_ALLOC_SIZE(2) XML_MemMalloc(XML_Parser parser, size_t size); XMLPARSEAPI(void *) +XML_ATTR_ALLOC_SIZE(3) XML_MemRealloc(XML_Parser parser, void *ptr, size_t size); XMLPARSEAPI(void) @@ -1033,13 +1044,11 @@ XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); -/* Expat follows the GNU/Linux convention of odd number minor version for - beta/development releases and even number minor version for stable - releases. Micro is bumped with each release, and set to 0 with each - change to major or minor version. +/* Expat follows the semantic versioning convention. + See http://semver.org. */ #define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 1 +#define XML_MINOR_VERSION 2 #define XML_MICRO_VERSION 1 #ifdef __cplusplus diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index f337e1c..4c9e5ea 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -69,12 +69,26 @@ #endif #endif /* not defined XML_STATIC */ +#if !defined(XMLIMPORT) && defined(__GNUC__) && (__GNUC__ >= 4) +#define XMLIMPORT __attribute__ ((visibility ("default"))) +#endif /* If we didn't define it above, define it away: */ #ifndef XMLIMPORT #define XMLIMPORT #endif +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)) +#define XML_ATTR_MALLOC __attribute__((__malloc__)) +#else +#define XML_ATTR_MALLOC +#endif + +#if defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +#define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x))) +#else +#define XML_ATTR_ALLOC_SIZE(x) +#endif #define XMLPARSEAPI(type) XMLIMPORT type XMLCALL @@ -83,7 +97,10 @@ extern "C" { #endif #ifdef XML_UNICODE_WCHAR_T -#define XML_UNICODE +# define XML_UNICODE +# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) +# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" +# endif #endif #ifdef XML_UNICODE /* Information is UTF-16 encoded. */ diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index dd54548..94cb98e 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -71,3 +71,25 @@ #define inline #endif #endif + +#ifndef UNUSED_P +# ifdef __GNUC__ +# define UNUSED_P(p) UNUSED_ ## p __attribute__((__unused__)) +# else +# define UNUSED_P(p) UNUSED_ ## p +# endif +#endif + + +#ifdef __cplusplus +extern "C" { +#endif + + +void +align_limit_to_full_utf8_characters(const char * from, const char ** fromLimRef); + + +#ifdef __cplusplus +} +#endif diff --git a/Modules/expat/macconfig.h b/Modules/expat/macconfig.h deleted file mode 100644 index 2725caa..0000000 --- a/Modules/expat/macconfig.h +++ /dev/null @@ -1,53 +0,0 @@ -/*================================================================ -** Copyright 2000, Clark Cooper -** All rights reserved. -** -** This is free software. You are permitted to copy, distribute, or modify -** it under the terms of the MIT/X license (contained in the COPYING file -** with this distribution.) -** -*/ - -#ifndef MACCONFIG_H -#define MACCONFIG_H - - -/* 1234 = LIL_ENDIAN, 4321 = BIGENDIAN */ -#define BYTEORDER 4321 - -/* Define to 1 if you have the `bcopy' function. */ -#undef HAVE_BCOPY - -/* Define to 1 if you have the `memmove' function. */ -#define HAVE_MEMMOVE - -/* Define to 1 if you have a working `mmap' system call. */ -#undef HAVE_MMAP - -/* Define to 1 if you have the header file. */ -#undef HAVE_UNISTD_H - -/* whether byteorder is bigendian */ -#define WORDS_BIGENDIAN - -/* Define to specify how much context to retain around the current parse - point. */ -#undef XML_CONTEXT_BYTES - -/* Define to make parameter entity parsing functionality available. */ -#define XML_DTD - -/* Define to make XML Namespaces functionality available. */ -#define XML_NS - -/* Define to empty if `const' does not conform to ANSI C. */ -#undef const - -/* Define to `long' if does not define. */ -#define off_t long - -/* Define to `unsigned' if does not define. */ -#undef size_t - - -#endif /* ifndef MACCONFIG_H */ diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h new file mode 100644 index 0000000..23b56d2 --- /dev/null +++ b/Modules/expat/siphash.h @@ -0,0 +1,344 @@ +/* ========================================================================== + * siphash.h - SipHash-2-4 in a single header file + * -------------------------------------------------------------------------- + * Derived by William Ahern from the reference implementation[1] published[2] + * by Jean-Philippe Aumasson and Daniel J. Berstein. Licensed in kind. + * by Jean-Philippe Aumasson and Daniel J. Berstein. + * Minimal changes by Sebastian Pipping on top, details below. + * Licensed under the CC0 Public Domain Dedication license. + * + * 1. https://www.131002.net/siphash/siphash24.c + * 2. https://www.131002.net/siphash/ + * -------------------------------------------------------------------------- + * HISTORY: + * + * 2017-06-10 (Sebastian Pipping) + * - Clarify license note in the header + * - Address C89 issues: + * - Stop using inline keyword (and let compiler decide) + * - Turn integer suffix ULL to UL + * - Replace _Bool by int + * - Turn macro siphash24 into a function + * - Address invalid conversion (void pointer) by explicit cast + * - Always expose sip24_valid (for self-tests) + * + * 2012-11-04 - Born. (William Ahern) + * -------------------------------------------------------------------------- + * USAGE: + * + * SipHash-2-4 takes as input two 64-bit words as the key, some number of + * message bytes, and outputs a 64-bit word as the message digest. This + * implementation employs two data structures: a struct sipkey for + * representing the key, and a struct siphash for representing the hash + * state. + * + * For converting a 16-byte unsigned char array to a key, use either the + * macro sip_keyof or the routine sip_tokey. The former instantiates a + * compound literal key, while the latter requires a key object as a + * parameter. + * + * unsigned char secret[16]; + * arc4random_buf(secret, sizeof secret); + * struct sipkey *key = sip_keyof(secret); + * + * For hashing a message, use either the convenience macro siphash24 or the + * routines sip24_init, sip24_update, and sip24_final. + * + * struct siphash state; + * void *msg; + * size_t len; + * uint64_t hash; + * + * sip24_init(&state, key); + * sip24_update(&state, msg, len); + * hash = sip24_final(&state); + * + * or + * + * hash = siphash24(msg, len, key); + * + * To convert the 64-bit hash value to a canonical 8-byte little-endian + * binary representation, use either the macro sip_binof or the routine + * sip_tobin. The former instantiates and returns a compound literal array, + * while the latter requires an array object as a parameter. + * -------------------------------------------------------------------------- + * NOTES: + * + * o Neither sip_keyof, sip_binof, nor siphash24 will work with compilers + * lacking compound literal support. Instead, you must use the lower-level + * interfaces which take as parameters the temporary state objects. + * + * o Uppercase macros may evaluate parameters more than once. Lowercase + * macros should not exhibit any such side effects. + * ========================================================================== + */ +#ifndef SIPHASH_H +#define SIPHASH_H + +#include /* size_t */ +#include /* uint64_t uint32_t uint8_t */ + + +#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b)))) + +#define SIP_U32TO8_LE(p, v) \ + (p)[0] = (uint8_t)((v) >> 0); (p)[1] = (uint8_t)((v) >> 8); \ + (p)[2] = (uint8_t)((v) >> 16); (p)[3] = (uint8_t)((v) >> 24); + +#define SIP_U64TO8_LE(p, v) \ + SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >> 0)); \ + SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32)); + +#define SIP_U8TO64_LE(p) \ + (((uint64_t)((p)[0]) << 0) | \ + ((uint64_t)((p)[1]) << 8) | \ + ((uint64_t)((p)[2]) << 16) | \ + ((uint64_t)((p)[3]) << 24) | \ + ((uint64_t)((p)[4]) << 32) | \ + ((uint64_t)((p)[5]) << 40) | \ + ((uint64_t)((p)[6]) << 48) | \ + ((uint64_t)((p)[7]) << 56)) + + +#define SIPHASH_INITIALIZER { 0, 0, 0, 0, { 0 }, 0, 0 } + +struct siphash { + uint64_t v0, v1, v2, v3; + + unsigned char buf[8], *p; + uint64_t c; +}; /* struct siphash */ + + +#define SIP_KEYLEN 16 + +struct sipkey { + uint64_t k[2]; +}; /* struct sipkey */ + +#define sip_keyof(k) sip_tokey(&(struct sipkey){ { 0 } }, (k)) + +static struct sipkey *sip_tokey(struct sipkey *key, const void *src) { + key->k[0] = SIP_U8TO64_LE((const unsigned char *)src); + key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8); + return key; +} /* sip_tokey() */ + + +#define sip_binof(v) sip_tobin((unsigned char[8]){ 0 }, (v)) + +static void *sip_tobin(void *dst, uint64_t u64) { + SIP_U64TO8_LE((unsigned char *)dst, u64); + return dst; +} /* sip_tobin() */ + + +static void sip_round(struct siphash *H, const int rounds) { + int i; + + for (i = 0; i < rounds; i++) { + H->v0 += H->v1; + H->v1 = SIP_ROTL(H->v1, 13); + H->v1 ^= H->v0; + H->v0 = SIP_ROTL(H->v0, 32); + + H->v2 += H->v3; + H->v3 = SIP_ROTL(H->v3, 16); + H->v3 ^= H->v2; + + H->v0 += H->v3; + H->v3 = SIP_ROTL(H->v3, 21); + H->v3 ^= H->v0; + + H->v2 += H->v1; + H->v1 = SIP_ROTL(H->v1, 17); + H->v1 ^= H->v2; + H->v2 = SIP_ROTL(H->v2, 32); + } +} /* sip_round() */ + + +static struct siphash *sip24_init(struct siphash *H, const struct sipkey *key) { + H->v0 = 0x736f6d6570736575UL ^ key->k[0]; + H->v1 = 0x646f72616e646f6dUL ^ key->k[1]; + H->v2 = 0x6c7967656e657261UL ^ key->k[0]; + H->v3 = 0x7465646279746573UL ^ key->k[1]; + + H->p = H->buf; + H->c = 0; + + return H; +} /* sip24_init() */ + + +#define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)]) + +static struct siphash *sip24_update(struct siphash *H, const void *src, size_t len) { + const unsigned char *p = (const unsigned char *)src, *pe = p + len; + uint64_t m; + + do { + while (p < pe && H->p < sip_endof(H->buf)) + *H->p++ = *p++; + + if (H->p < sip_endof(H->buf)) + break; + + m = SIP_U8TO64_LE(H->buf); + H->v3 ^= m; + sip_round(H, 2); + H->v0 ^= m; + + H->p = H->buf; + H->c += 8; + } while (p < pe); + + return H; +} /* sip24_update() */ + + +static uint64_t sip24_final(struct siphash *H) { + char left = H->p - H->buf; + uint64_t b = (H->c + left) << 56; + + switch (left) { + case 7: b |= (uint64_t)H->buf[6] << 48; + case 6: b |= (uint64_t)H->buf[5] << 40; + case 5: b |= (uint64_t)H->buf[4] << 32; + case 4: b |= (uint64_t)H->buf[3] << 24; + case 3: b |= (uint64_t)H->buf[2] << 16; + case 2: b |= (uint64_t)H->buf[1] << 8; + case 1: b |= (uint64_t)H->buf[0] << 0; + case 0: break; + } + + H->v3 ^= b; + sip_round(H, 2); + H->v0 ^= b; + H->v2 ^= 0xff; + sip_round(H, 4); + + return H->v0 ^ H->v1 ^ H->v2 ^ H->v3; +} /* sip24_final() */ + + +static uint64_t siphash24(const void *src, size_t len, const struct sipkey *key) { + struct siphash state = SIPHASH_INITIALIZER; + return sip24_final(sip24_update(sip24_init(&state, key), src, len)); +} /* siphash24() */ + + +/* + * SipHash-2-4 output with + * k = 00 01 02 ... + * and + * in = (empty string) + * in = 00 (1 byte) + * in = 00 01 (2 bytes) + * in = 00 01 02 (3 bytes) + * ... + * in = 00 01 02 ... 3e (63 bytes) + */ +static int sip24_valid(void) { + static const unsigned char vectors[64][8] = { + { 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, }, + { 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, }, + { 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, }, + { 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, }, + { 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, }, + { 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, }, + { 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, }, + { 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, }, + { 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, }, + { 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, }, + { 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, }, + { 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, }, + { 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, }, + { 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, }, + { 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, }, + { 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, }, + { 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, }, + { 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, }, + { 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, }, + { 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, }, + { 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, }, + { 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, }, + { 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, }, + { 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, }, + { 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, }, + { 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, }, + { 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, }, + { 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, }, + { 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, }, + { 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, }, + { 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, }, + { 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, }, + { 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, }, + { 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, }, + { 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, }, + { 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, }, + { 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, }, + { 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, }, + { 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, }, + { 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, }, + { 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, }, + { 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, }, + { 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, }, + { 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, }, + { 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, }, + { 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, }, + { 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, }, + { 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, }, + { 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, }, + { 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, }, + { 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, }, + { 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, }, + { 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, }, + { 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, }, + { 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, }, + { 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, }, + { 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, }, + { 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, }, + { 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, }, + { 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, }, + { 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, }, + { 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, }, + { 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, }, + { 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, } + }; + unsigned char in[64]; + struct sipkey k; + size_t i; + + sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017"); + + for (i = 0; i < sizeof in; ++i) { + in[i] = i; + + if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i])) + return 0; + } + + return 1; +} /* sip24_valid() */ + + +#if SIPHASH_MAIN + +#include + +int main(void) { + int ok = sip24_valid(); + + if (ok) + puts("OK"); + else + puts("FAIL"); + + return !ok; +} /* main() */ + +#endif /* SIPHASH_MAIN */ + + +#endif /* SIPHASH_H */ diff --git a/Modules/expat/watcomconfig.h b/Modules/expat/watcomconfig.h deleted file mode 100644 index 2f05e3f..0000000 --- a/Modules/expat/watcomconfig.h +++ /dev/null @@ -1,47 +0,0 @@ -/* expat_config.h for use with Open Watcom 1.5 and above. */ - -#ifndef WATCOMCONFIG_H -#define WATCOMCONFIG_H - -#ifdef __NT__ -#define WIN32_LEAN_AND_MEAN -#include -#undef WIN32_LEAN_AND_MEAN -#endif - -/* 1234 = LIL_ENDIAN, 4321 = BIGENDIAN */ -#define BYTEORDER 1234 - -/* Define to 1 if you have the `memmove' function. */ -#define HAVE_MEMMOVE 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_UNISTD_H 1 - -/* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "expat-bugs@mail.libexpat.org" - -/* Define to the full name of this package. */ -#define PACKAGE_NAME "expat" - -/* Define to the full name and version of this package. */ -#define PACKAGE_STRING "expat 2.0.0" - -/* Define to the one symbol short name of this package. */ -#undef PACKAGE_TARNAME - -/* Define to the version of this package. */ -#define PACKAGE_VERSION "2.0.0" - -/* Define to specify how much context to retain around the current parse - point. */ -#define XML_CONTEXT_BYTES 1024 - -/* Define to make parameter entity parsing functionality available. */ -#define XML_DTD 1 - -/* Define to make XML Namespaces functionality available. */ -#define XML_NS 1 - -#endif - diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index c1b791d..9bf014d 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -17,6 +17,12 @@ #include #include + +#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */ +# include +#else /* !defined(HAVE_EXPAT_CONFIG_H) */ + + #define XML_NS 1 #define XML_DTD 1 #define XML_CONTEXT_BYTES 1024 @@ -27,4 +33,8 @@ /* Windows has memmove() available. */ #define HAVE_MEMMOVE + +#endif /* !defined(HAVE_EXPAT_CONFIG_H) */ + + #endif /* ndef WINCONFIG_H */ diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 4128387..daec151 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,29 +1,37 @@ /* Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd See the file COPYING for copying permission. -*/ -#define XML_BUILDING_EXPAT 1 + 77fea421d361dca90041d0040ecf1dca651167fadf2af79e990e35168d70d933 (2.2.1+) +*/ -#ifdef COMPILED_FROM_DSP -#include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" -#elif defined(HAVE_EXPAT_CONFIG_H) -#include -#endif /* ndef COMPILED_FROM_DSP */ +#define _GNU_SOURCE 1 /* syscall prototype */ #include #include /* memset(), memcpy() */ #include #include /* UINT_MAX */ -#include /* time() */ +#include /* fprintf */ +#include /* getenv */ + +#ifdef _WIN32 +#define getpid GetCurrentProcessId +#else +#include /* gettimeofday() */ +#include /* getpid() */ +#include /* getpid() */ +#endif + +#define XML_BUILDING_EXPAT 1 + +#ifdef _WIN32 +#include "winconfig.h" +#elif defined(HAVE_EXPAT_CONFIG_H) +#include +#endif /* ndef _WIN32 */ #include "ascii.h" #include "expat.h" +#include "siphash.h" #ifdef XML_UNICODE #define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX @@ -102,17 +110,11 @@ typedef struct { const XML_Memory_Handling_Suite *mem; } HASH_TABLE; -/* Basic character hash algorithm, taken from Python's string hash: - h = h * 1000003 ^ character, the constant being a prime number. +static size_t +keylen(KEY s); -*/ -#ifdef XML_UNICODE -#define CHAR_HASH(h, c) \ - (((h) * 0xF4243) ^ (unsigned short)(c)) -#else -#define CHAR_HASH(h, c) \ - (((h) * 0xF4243) ^ (unsigned char)(c)) -#endif +static void +copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key); /* For probing (after a collision) we need a step size relative prime to the hash table size, which is a power of 2. We use double-hashing, @@ -348,6 +350,8 @@ doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore); #endif /* XML_DTD */ +static void +freeBindings(XML_Parser parser, BINDING *bindings); static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *, const char *s, TAG_NAME *tagNamePtr, BINDING **bindingsPtr); @@ -432,7 +436,7 @@ static ELEMENT_TYPE * getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, const char *end); -static unsigned long generate_hash_secret_salt(void); +static unsigned long generate_hash_secret_salt(XML_Parser parser); static XML_Bool startParsing(XML_Parser parser); static XML_Parser @@ -690,12 +694,155 @@ static const XML_Char implicitContext[] = { ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0' }; + +#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) +# include + +# if defined(HAVE_GETRANDOM) +# include /* getrandom */ +# else +# include /* syscall */ +# include /* SYS_getrandom */ +# endif + +/* Obtain entropy on Linux 3.17+ */ +static int +writeRandomBytes_getrandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + size_t bytesWrittenTotal = 0; + const unsigned int getrandomFlags = 0; + + do { + void * const currentTarget = (void*)((char*)target + bytesWrittenTotal); + const size_t bytesToWrite = count - bytesWrittenTotal; + + const int bytesWrittenMore = +#if defined(HAVE_GETRANDOM) + getrandom(currentTarget, bytesToWrite, getrandomFlags); +#else + syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags); +#endif + + if (bytesWrittenMore > 0) { + bytesWrittenTotal += bytesWrittenMore; + if (bytesWrittenTotal >= count) + success = 1; + } + } while (! success && (errno == EINTR || errno == EAGAIN)); + + return success; +} + +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + + +#ifdef _WIN32 + +typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG); + +/* Obtain entropy on Windows XP / Windows Server 2003 and later. + * Hint on RtlGenRandom and the following article from libsodioum. + * + * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI + * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographically-secure-random-number-on-windows-without-using-cryptoapi/ + */ +static int +writeRandomBytes_RtlGenRandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + const HMODULE advapi32 = LoadLibrary("ADVAPI32.DLL"); + + if (advapi32) { + const RTLGENRANDOM_FUNC RtlGenRandom + = (RTLGENRANDOM_FUNC)GetProcAddress(advapi32, "SystemFunction036"); + if (RtlGenRandom) { + if (RtlGenRandom((PVOID)target, (ULONG)count) == TRUE) { + success = 1; + } + } + FreeLibrary(advapi32); + } + + return success; +} + +#endif /* _WIN32 */ + + static unsigned long -generate_hash_secret_salt(void) +gather_time_entropy(void) { - unsigned int seed = time(NULL) % UINT_MAX; - srand(seed); - return rand(); +#ifdef _WIN32 + FILETIME ft; + GetSystemTimeAsFileTime(&ft); /* never fails */ + return ft.dwHighDateTime ^ ft.dwLowDateTime; +#else + struct timeval tv; + int gettimeofday_res; + + gettimeofday_res = gettimeofday(&tv, NULL); + assert (gettimeofday_res == 0); + + /* Microseconds time is <20 bits entropy */ + return tv.tv_usec; +#endif +} + +#if defined(HAVE_ARC4RANDOM_BUF) && defined(HAVE_LIBBSD) +# include +#endif + +static unsigned long +ENTROPY_DEBUG(const char * label, unsigned long entropy) { + const char * const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG"); + if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) { + fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", + label, + (int)sizeof(entropy) * 2, entropy, + (unsigned long)sizeof(entropy)); + } + return entropy; +} + +static unsigned long +generate_hash_secret_salt(XML_Parser parser) +{ + unsigned long entropy; + (void)parser; +#if defined(HAVE_ARC4RANDOM_BUF) || defined(__CloudABI__) + (void)gather_time_entropy; + arc4random_buf(&entropy, sizeof(entropy)); + return ENTROPY_DEBUG("arc4random_buf", entropy); +#else + /* Try high quality providers first .. */ +#ifdef _WIN32 + if (writeRandomBytes_RtlGenRandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("RtlGenRandom", entropy); + } +#elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) + if (writeRandomBytes_getrandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("getrandom", entropy); + } +#endif + /* .. and self-made low quality for backup: */ + + /* Process ID is 0 bits entropy if attacker has local access */ + entropy = gather_time_entropy() ^ getpid(); + + /* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */ + if (sizeof(unsigned long) == 4) { + return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); + } else { + return ENTROPY_DEBUG("fallback(8)", + entropy * (unsigned long)2305843009213693951); + } +#endif +} + +static unsigned long +get_hash_secret_salt(XML_Parser parser) { + if (parser->m_parentParser != NULL) + return get_hash_secret_salt(parser->m_parentParser); + return parser->m_hash_secret_salt; } static XML_Bool /* only valid for root parser */ @@ -703,7 +850,7 @@ startParsing(XML_Parser parser) { /* hash functions must be initialized before setContext() is called */ if (hash_secret_salt == 0) - hash_secret_salt = generate_hash_secret_salt(); + hash_secret_salt = generate_hash_secret_salt(parser); if (ns) { /* implicit context only set for root parser, since child parsers (i.e. external entity parsers) will inherit it @@ -926,6 +1073,10 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { TAG *tStk; OPEN_INTERNAL_ENTITY *openEntityList; + + if (parser == NULL) + return XML_FALSE; + if (parentParser) return XML_FALSE; /* move tagStack to freeTagList */ @@ -960,6 +1111,8 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) enum XML_Status XMLCALL XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { + if (parser == NULL) + return XML_STATUS_ERROR; /* Block after XML_Parse()/XML_ParseBuffer() has been called. XXX There's no way for the caller to determine which of the XXX possible error cases caused the XML_STATUS_ERROR return. @@ -983,52 +1136,88 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, { XML_Parser parser = oldParser; DTD *newDtd = NULL; - DTD *oldDtd = _dtd; - XML_StartElementHandler oldStartElementHandler = startElementHandler; - XML_EndElementHandler oldEndElementHandler = endElementHandler; - XML_CharacterDataHandler oldCharacterDataHandler = characterDataHandler; - XML_ProcessingInstructionHandler oldProcessingInstructionHandler - = processingInstructionHandler; - XML_CommentHandler oldCommentHandler = commentHandler; - XML_StartCdataSectionHandler oldStartCdataSectionHandler - = startCdataSectionHandler; - XML_EndCdataSectionHandler oldEndCdataSectionHandler - = endCdataSectionHandler; - XML_DefaultHandler oldDefaultHandler = defaultHandler; - XML_UnparsedEntityDeclHandler oldUnparsedEntityDeclHandler - = unparsedEntityDeclHandler; - XML_NotationDeclHandler oldNotationDeclHandler = notationDeclHandler; - XML_StartNamespaceDeclHandler oldStartNamespaceDeclHandler - = startNamespaceDeclHandler; - XML_EndNamespaceDeclHandler oldEndNamespaceDeclHandler - = endNamespaceDeclHandler; - XML_NotStandaloneHandler oldNotStandaloneHandler = notStandaloneHandler; - XML_ExternalEntityRefHandler oldExternalEntityRefHandler - = externalEntityRefHandler; - XML_SkippedEntityHandler oldSkippedEntityHandler = skippedEntityHandler; - XML_UnknownEncodingHandler oldUnknownEncodingHandler - = unknownEncodingHandler; - XML_ElementDeclHandler oldElementDeclHandler = elementDeclHandler; - XML_AttlistDeclHandler oldAttlistDeclHandler = attlistDeclHandler; - XML_EntityDeclHandler oldEntityDeclHandler = entityDeclHandler; - XML_XmlDeclHandler oldXmlDeclHandler = xmlDeclHandler; - ELEMENT_TYPE * oldDeclElementType = declElementType; - - void *oldUserData = userData; - void *oldHandlerArg = handlerArg; - XML_Bool oldDefaultExpandInternalEntities = defaultExpandInternalEntities; - XML_Parser oldExternalEntityRefHandlerArg = externalEntityRefHandlerArg; + DTD *oldDtd; + XML_StartElementHandler oldStartElementHandler; + XML_EndElementHandler oldEndElementHandler; + XML_CharacterDataHandler oldCharacterDataHandler; + XML_ProcessingInstructionHandler oldProcessingInstructionHandler; + XML_CommentHandler oldCommentHandler; + XML_StartCdataSectionHandler oldStartCdataSectionHandler; + XML_EndCdataSectionHandler oldEndCdataSectionHandler; + XML_DefaultHandler oldDefaultHandler; + XML_UnparsedEntityDeclHandler oldUnparsedEntityDeclHandler; + XML_NotationDeclHandler oldNotationDeclHandler; + XML_StartNamespaceDeclHandler oldStartNamespaceDeclHandler; + XML_EndNamespaceDeclHandler oldEndNamespaceDeclHandler; + XML_NotStandaloneHandler oldNotStandaloneHandler; + XML_ExternalEntityRefHandler oldExternalEntityRefHandler; + XML_SkippedEntityHandler oldSkippedEntityHandler; + XML_UnknownEncodingHandler oldUnknownEncodingHandler; + XML_ElementDeclHandler oldElementDeclHandler; + XML_AttlistDeclHandler oldAttlistDeclHandler; + XML_EntityDeclHandler oldEntityDeclHandler; + XML_XmlDeclHandler oldXmlDeclHandler; + ELEMENT_TYPE * oldDeclElementType; + + void *oldUserData; + void *oldHandlerArg; + XML_Bool oldDefaultExpandInternalEntities; + XML_Parser oldExternalEntityRefHandlerArg; #ifdef XML_DTD - enum XML_ParamEntityParsing oldParamEntityParsing = paramEntityParsing; - int oldInEntityValue = prologState.inEntityValue; + enum XML_ParamEntityParsing oldParamEntityParsing; + int oldInEntityValue; +#endif + XML_Bool oldns_triplets; + /* Note that the new parser shares the same hash secret as the old + parser, so that dtdCopy and copyEntityTable can lookup values + from hash tables associated with either parser without us having + to worry which hash secrets each table has. + */ + unsigned long oldhash_secret_salt; + + /* Validate the oldParser parameter before we pull everything out of it */ + if (oldParser == NULL) + return NULL; + + /* Stash the original parser contents on the stack */ + oldDtd = _dtd; + oldStartElementHandler = startElementHandler; + oldEndElementHandler = endElementHandler; + oldCharacterDataHandler = characterDataHandler; + oldProcessingInstructionHandler = processingInstructionHandler; + oldCommentHandler = commentHandler; + oldStartCdataSectionHandler = startCdataSectionHandler; + oldEndCdataSectionHandler = endCdataSectionHandler; + oldDefaultHandler = defaultHandler; + oldUnparsedEntityDeclHandler = unparsedEntityDeclHandler; + oldNotationDeclHandler = notationDeclHandler; + oldStartNamespaceDeclHandler = startNamespaceDeclHandler; + oldEndNamespaceDeclHandler = endNamespaceDeclHandler; + oldNotStandaloneHandler = notStandaloneHandler; + oldExternalEntityRefHandler = externalEntityRefHandler; + oldSkippedEntityHandler = skippedEntityHandler; + oldUnknownEncodingHandler = unknownEncodingHandler; + oldElementDeclHandler = elementDeclHandler; + oldAttlistDeclHandler = attlistDeclHandler; + oldEntityDeclHandler = entityDeclHandler; + oldXmlDeclHandler = xmlDeclHandler; + oldDeclElementType = declElementType; + + oldUserData = userData; + oldHandlerArg = handlerArg; + oldDefaultExpandInternalEntities = defaultExpandInternalEntities; + oldExternalEntityRefHandlerArg = externalEntityRefHandlerArg; +#ifdef XML_DTD + oldParamEntityParsing = paramEntityParsing; + oldInEntityValue = prologState.inEntityValue; #endif - XML_Bool oldns_triplets = ns_triplets; + oldns_triplets = ns_triplets; /* Note that the new parser shares the same hash secret as the old parser, so that dtdCopy and copyEntityTable can lookup values from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - unsigned long oldhash_secret_salt = hash_secret_salt; + oldhash_secret_salt = hash_secret_salt; #ifdef XML_DTD if (!context) @@ -1194,12 +1383,15 @@ XML_ParserFree(XML_Parser parser) void XMLCALL XML_UseParserAsHandlerArg(XML_Parser parser) { - handlerArg = parser; + if (parser != NULL) + handlerArg = parser; } enum XML_Error XMLCALL XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) { + if (parser == NULL) + return XML_ERROR_INVALID_ARGUMENT; #ifdef XML_DTD /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) @@ -1214,6 +1406,8 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) void XMLCALL XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) { + if (parser == NULL) + return; /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return; @@ -1223,6 +1417,8 @@ XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) void XMLCALL XML_SetUserData(XML_Parser parser, void *p) { + if (parser == NULL) + return; if (handlerArg == userData) handlerArg = userData = p; else @@ -1232,6 +1428,8 @@ XML_SetUserData(XML_Parser parser, void *p) enum XML_Status XMLCALL XML_SetBase(XML_Parser parser, const XML_Char *p) { + if (parser == NULL) + return XML_STATUS_ERROR; if (p) { p = poolCopyString(&_dtd->pool, p); if (!p) @@ -1246,18 +1444,24 @@ XML_SetBase(XML_Parser parser, const XML_Char *p) const XML_Char * XMLCALL XML_GetBase(XML_Parser parser) { + if (parser == NULL) + return NULL; return curBase; } int XMLCALL XML_GetSpecifiedAttributeCount(XML_Parser parser) { + if (parser == NULL) + return -1; return nSpecifiedAtts; } int XMLCALL XML_GetIdAttributeIndex(XML_Parser parser) { + if (parser == NULL) + return -1; return idAttIndex; } @@ -1265,6 +1469,8 @@ XML_GetIdAttributeIndex(XML_Parser parser) const XML_AttrInfo * XMLCALL XML_GetAttributeInfo(XML_Parser parser) { + if (parser == NULL) + return NULL; return attInfo; } #endif @@ -1274,6 +1480,8 @@ XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start, XML_EndElementHandler end) { + if (parser == NULL) + return; startElementHandler = start; endElementHandler = end; } @@ -1281,34 +1489,39 @@ XML_SetElementHandler(XML_Parser parser, void XMLCALL XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler start) { - startElementHandler = start; + if (parser != NULL) + startElementHandler = start; } void XMLCALL XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler end) { - endElementHandler = end; + if (parser != NULL) + endElementHandler = end; } void XMLCALL XML_SetCharacterDataHandler(XML_Parser parser, XML_CharacterDataHandler handler) { - characterDataHandler = handler; + if (parser != NULL) + characterDataHandler = handler; } void XMLCALL XML_SetProcessingInstructionHandler(XML_Parser parser, XML_ProcessingInstructionHandler handler) { - processingInstructionHandler = handler; + if (parser != NULL) + processingInstructionHandler = handler; } void XMLCALL XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler) { - commentHandler = handler; + if (parser != NULL) + commentHandler = handler; } void XMLCALL @@ -1316,6 +1529,8 @@ XML_SetCdataSectionHandler(XML_Parser parser, XML_StartCdataSectionHandler start, XML_EndCdataSectionHandler end) { + if (parser == NULL) + return; startCdataSectionHandler = start; endCdataSectionHandler = end; } @@ -1323,19 +1538,23 @@ XML_SetCdataSectionHandler(XML_Parser parser, void XMLCALL XML_SetStartCdataSectionHandler(XML_Parser parser, XML_StartCdataSectionHandler start) { - startCdataSectionHandler = start; + if (parser != NULL) + startCdataSectionHandler = start; } void XMLCALL XML_SetEndCdataSectionHandler(XML_Parser parser, XML_EndCdataSectionHandler end) { - endCdataSectionHandler = end; + if (parser != NULL) + endCdataSectionHandler = end; } void XMLCALL XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler) { + if (parser == NULL) + return; defaultHandler = handler; defaultExpandInternalEntities = XML_FALSE; } @@ -1344,6 +1563,8 @@ void XMLCALL XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler) { + if (parser == NULL) + return; defaultHandler = handler; defaultExpandInternalEntities = XML_TRUE; } @@ -1353,6 +1574,8 @@ XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start, XML_EndDoctypeDeclHandler end) { + if (parser == NULL) + return; startDoctypeDeclHandler = start; endDoctypeDeclHandler = end; } @@ -1360,27 +1583,31 @@ XML_SetDoctypeDeclHandler(XML_Parser parser, void XMLCALL XML_SetStartDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start) { - startDoctypeDeclHandler = start; + if (parser != NULL) + startDoctypeDeclHandler = start; } void XMLCALL XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end) { - endDoctypeDeclHandler = end; + if (parser != NULL) + endDoctypeDeclHandler = end; } void XMLCALL XML_SetUnparsedEntityDeclHandler(XML_Parser parser, XML_UnparsedEntityDeclHandler handler) { - unparsedEntityDeclHandler = handler; + if (parser != NULL) + unparsedEntityDeclHandler = handler; } void XMLCALL XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler) { - notationDeclHandler = handler; + if (parser != NULL) + notationDeclHandler = handler; } void XMLCALL @@ -1388,6 +1615,8 @@ XML_SetNamespaceDeclHandler(XML_Parser parser, XML_StartNamespaceDeclHandler start, XML_EndNamespaceDeclHandler end) { + if (parser == NULL) + return; startNamespaceDeclHandler = start; endNamespaceDeclHandler = end; } @@ -1395,32 +1624,38 @@ XML_SetNamespaceDeclHandler(XML_Parser parser, void XMLCALL XML_SetStartNamespaceDeclHandler(XML_Parser parser, XML_StartNamespaceDeclHandler start) { - startNamespaceDeclHandler = start; + if (parser != NULL) + startNamespaceDeclHandler = start; } void XMLCALL XML_SetEndNamespaceDeclHandler(XML_Parser parser, XML_EndNamespaceDeclHandler end) { - endNamespaceDeclHandler = end; + if (parser != NULL) + endNamespaceDeclHandler = end; } void XMLCALL XML_SetNotStandaloneHandler(XML_Parser parser, XML_NotStandaloneHandler handler) { - notStandaloneHandler = handler; + if (parser != NULL) + notStandaloneHandler = handler; } void XMLCALL XML_SetExternalEntityRefHandler(XML_Parser parser, XML_ExternalEntityRefHandler handler) { - externalEntityRefHandler = handler; + if (parser != NULL) + externalEntityRefHandler = handler; } void XMLCALL XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg) { + if (parser == NULL) + return; if (arg) externalEntityRefHandlerArg = (XML_Parser)arg; else @@ -1431,7 +1666,8 @@ void XMLCALL XML_SetSkippedEntityHandler(XML_Parser parser, XML_SkippedEntityHandler handler) { - skippedEntityHandler = handler; + if (parser != NULL) + skippedEntityHandler = handler; } void XMLCALL @@ -1439,6 +1675,8 @@ XML_SetUnknownEncodingHandler(XML_Parser parser, XML_UnknownEncodingHandler handler, void *data) { + if (parser == NULL) + return; unknownEncodingHandler = handler; unknownEncodingHandlerData = data; } @@ -1447,33 +1685,39 @@ void XMLCALL XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl) { - elementDeclHandler = eldecl; + if (parser != NULL) + elementDeclHandler = eldecl; } void XMLCALL XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl) { - attlistDeclHandler = attdecl; + if (parser != NULL) + attlistDeclHandler = attdecl; } void XMLCALL XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler) { - entityDeclHandler = handler; + if (parser != NULL) + entityDeclHandler = handler; } void XMLCALL XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler handler) { - xmlDeclHandler = handler; + if (parser != NULL) + xmlDeclHandler = handler; } int XMLCALL XML_SetParamEntityParsing(XML_Parser parser, enum XML_ParamEntityParsing peParsing) { + if (parser == NULL) + return 0; /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return 0; @@ -1489,6 +1733,10 @@ int XMLCALL XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { + if (parser == NULL) + return 0; + if (parser->m_parentParser) + return XML_SetHashSalt(parser->m_parentParser, hash_salt); /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return 0; @@ -1499,6 +1747,10 @@ XML_SetHashSalt(XML_Parser parser, enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { + if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { + errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } switch (ps_parsing) { case XML_SUSPENDED: errorCode = XML_ERROR_SUSPENDED; @@ -1551,6 +1803,13 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) const char *end; int nLeftOver; enum XML_Status result; + /* Detect overflow (a+b > MAX <==> b > MAX-a) */ + if (len > ((XML_Size)-1) / 2 - parseEndByteIndex) { + errorCode = XML_ERROR_NO_MEMORY; + eventPtr = eventEndPtr = NULL; + processor = errorProcessor; + return XML_STATUS_ERROR; + } parseEndByteIndex += len; positionPtr = s; ps_finalBuffer = (XML_Bool)isFinal; @@ -1583,11 +1842,14 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) nLeftOver = s + len - end; if (nLeftOver) { if (buffer == NULL || nLeftOver > bufferLim - buffer) { - /* FIXME avoid integer overflow */ - char *temp; - temp = (buffer == NULL - ? (char *)MALLOC(len * 2) - : (char *)REALLOC(buffer, len * 2)); + /* avoid _signed_ integer overflow */ + char *temp = NULL; + const int bytesToAllocate = (int)((unsigned)len * 2U); + if (bytesToAllocate > 0) { + temp = (buffer == NULL + ? (char *)MALLOC(bytesToAllocate) + : (char *)REALLOC(buffer, bytesToAllocate)); + } if (temp == NULL) { errorCode = XML_ERROR_NO_MEMORY; eventPtr = eventEndPtr = NULL; @@ -1595,7 +1857,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) return XML_STATUS_ERROR; } buffer = temp; - bufferLim = buffer + len * 2; + bufferLim = buffer + bytesToAllocate; } memcpy(buffer, end, nLeftOver); } @@ -1625,6 +1887,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) const char *start; enum XML_Status result = XML_STATUS_OK; + if (parser == NULL) + return XML_STATUS_ERROR; switch (ps_parsing) { case XML_SUSPENDED: errorCode = XML_ERROR_SUSPENDED; @@ -1678,6 +1942,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) void * XMLCALL XML_GetBuffer(XML_Parser parser, int len) { + if (parser == NULL) + return NULL; if (len < 0) { errorCode = XML_ERROR_NO_MEMORY; return NULL; @@ -1695,15 +1961,15 @@ XML_GetBuffer(XML_Parser parser, int len) if (len > bufferLim - bufferEnd) { #ifdef XML_CONTEXT_BYTES int keep; -#endif - int neededSize = len + (int)(bufferEnd - bufferPtr); +#endif /* defined XML_CONTEXT_BYTES */ + /* Do not invoke signed arithmetic overflow: */ + int neededSize = (int) ((unsigned)len + (unsigned)(bufferEnd - bufferPtr)); if (neededSize < 0) { errorCode = XML_ERROR_NO_MEMORY; return NULL; } #ifdef XML_CONTEXT_BYTES keep = (int)(bufferPtr - buffer); - if (keep > XML_CONTEXT_BYTES) keep = XML_CONTEXT_BYTES; neededSize += keep; @@ -1728,7 +1994,8 @@ XML_GetBuffer(XML_Parser parser, int len) if (bufferSize == 0) bufferSize = INIT_BUFFER_SIZE; do { - bufferSize *= 2; + /* Do not invoke signed arithmetic overflow: */ + bufferSize = (int) (2U * (unsigned) bufferSize); } while (bufferSize < neededSize && bufferSize > 0); if (bufferSize <= 0) { errorCode = XML_ERROR_NO_MEMORY; @@ -1773,6 +2040,8 @@ XML_GetBuffer(XML_Parser parser, int len) enum XML_Status XMLCALL XML_StopParser(XML_Parser parser, XML_Bool resumable) { + if (parser == NULL) + return XML_STATUS_ERROR; switch (ps_parsing) { case XML_SUSPENDED: if (resumable) { @@ -1805,6 +2074,8 @@ XML_ResumeParser(XML_Parser parser) { enum XML_Status result = XML_STATUS_OK; + if (parser == NULL) + return XML_STATUS_ERROR; if (ps_parsing != XML_SUSPENDED) { errorCode = XML_ERROR_NOT_SUSPENDED; return XML_STATUS_ERROR; @@ -1841,6 +2112,8 @@ XML_ResumeParser(XML_Parser parser) void XMLCALL XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) { + if (parser == NULL) + return; assert(status != NULL); *status = parser->m_parsingStatus; } @@ -1848,20 +2121,26 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) enum XML_Error XMLCALL XML_GetErrorCode(XML_Parser parser) { + if (parser == NULL) + return XML_ERROR_INVALID_ARGUMENT; return errorCode; } XML_Index XMLCALL XML_GetCurrentByteIndex(XML_Parser parser) { + if (parser == NULL) + return -1; if (eventPtr) - return parseEndByteIndex - (parseEndPtr - eventPtr); + return (XML_Index)(parseEndByteIndex - (parseEndPtr - eventPtr)); return -1; } int XMLCALL XML_GetCurrentByteCount(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventEndPtr && eventPtr) return (int)(eventEndPtr - eventPtr); return 0; @@ -1871,11 +2150,19 @@ const char * XMLCALL XML_GetInputContext(XML_Parser parser, int *offset, int *size) { #ifdef XML_CONTEXT_BYTES + if (parser == NULL) + return NULL; if (eventPtr && buffer) { - *offset = (int)(eventPtr - buffer); - *size = (int)(bufferEnd - buffer); + if (offset != NULL) + *offset = (int)(eventPtr - buffer); + if (size != NULL) + *size = (int)(bufferEnd - buffer); return buffer; } +#else + (void)parser; + (void)offset; + (void)size; #endif /* defined XML_CONTEXT_BYTES */ return (char *) 0; } @@ -1883,6 +2170,8 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) XML_Size XMLCALL XML_GetCurrentLineNumber(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventPtr && eventPtr >= positionPtr) { XmlUpdatePosition(encoding, positionPtr, eventPtr, &position); positionPtr = eventPtr; @@ -1893,6 +2182,8 @@ XML_GetCurrentLineNumber(XML_Parser parser) XML_Size XMLCALL XML_GetCurrentColumnNumber(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventPtr && eventPtr >= positionPtr) { XmlUpdatePosition(encoding, positionPtr, eventPtr, &position); positionPtr = eventPtr; @@ -1903,30 +2194,38 @@ XML_GetCurrentColumnNumber(XML_Parser parser) void XMLCALL XML_FreeContentModel(XML_Parser parser, XML_Content *model) { - FREE(model); + if (parser != NULL) + FREE(model); } void * XMLCALL XML_MemMalloc(XML_Parser parser, size_t size) { + if (parser == NULL) + return NULL; return MALLOC(size); } void * XMLCALL XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) { + if (parser == NULL) + return NULL; return REALLOC(ptr, size); } void XMLCALL XML_MemFree(XML_Parser parser, void *ptr) { - FREE(ptr); + if (parser != NULL) + FREE(ptr); } void XMLCALL XML_DefaultCurrent(XML_Parser parser) { + if (parser == NULL) + return; if (defaultHandler) { if (openInternalEntities) reportDefault(parser, @@ -2429,11 +2728,11 @@ doContent(XML_Parser parser, for (;;) { int bufSize; int convLen; - XmlConvert(enc, + const enum XML_Convert_Result convert_res = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); convLen = (int)(toPtr - (XML_Char *)tag->buf); - if (fromPtr == rawNameEnd) { + if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; break; } @@ -2476,8 +2775,10 @@ doContent(XML_Parser parser, return XML_ERROR_NO_MEMORY; poolFinish(&tempPool); result = storeAtts(parser, enc, s, &name, &bindings); - if (result) + if (result != XML_ERROR_NONE) { + freeBindings(parser, bindings); return result; + } poolFinish(&tempPool); if (startElementHandler) { startElementHandler(handlerArg, name.str, (const XML_Char **)atts); @@ -2492,15 +2793,7 @@ doContent(XML_Parser parser, if (noElmHandlers && defaultHandler) reportDefault(parser, enc, s, next); poolClear(&tempPool); - while (bindings) { - BINDING *b = bindings; - if (endNamespaceDeclHandler) - endNamespaceDeclHandler(handlerArg, b->prefix->name); - bindings = bindings->nextTagBinding; - b->nextTagBinding = freeBindingList; - freeBindingList = b; - b->prefix->binding = b->prevPrefixBinding; - } + freeBindings(parser, bindings); } if (tagLevel == 0) return epilogProcessor(parser, next, end, nextPtr); @@ -2654,11 +2947,11 @@ doContent(XML_Parser parser, if (MUST_CONVERT(enc, s)) { for (;;) { ICHAR *dataPtr = (ICHAR *)dataBuf; - XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)dataBufEnd); + const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)dataBufEnd); *eventEndPP = s; charDataHandler(handlerArg, dataBuf, (int)(dataPtr - (ICHAR *)dataBuf)); - if (s == next) + if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } @@ -2698,6 +2991,29 @@ doContent(XML_Parser parser, /* not reached */ } +/* This function does not call free() on the allocated memory, merely + * moving it to the parser's freeBindingList where it can be freed or + * reused as appropriate. + */ +static void +freeBindings(XML_Parser parser, BINDING *bindings) +{ + while (bindings) { + BINDING *b = bindings; + + /* startNamespaceDeclHandler will have been called for this + * binding in addBindings(), so call the end handler now. + */ + if (endNamespaceDeclHandler) + endNamespaceDeclHandler(handlerArg, b->prefix->name); + + bindings = bindings->nextTagBinding; + b->nextTagBinding = freeBindingList; + freeBindingList = b; + b->prefix->binding = b->prevPrefixBinding; + } +} + /* Precondition: all arguments must be non-NULL; Purpose: - normalize attributes @@ -2922,7 +3238,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc, if (s[-1] == 2) { /* prefixed */ ATTRIBUTE_ID *id; const BINDING *b; - unsigned long uriHash = hash_secret_salt; + unsigned long uriHash; + struct siphash sip_state; + struct sipkey sip_key; + + copy_salt_to_sipkey(parser, &sip_key); + sip24_init(&sip_state, &sip_key); + ((XML_Char *)s)[-1] = 0; /* clear flag */ id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); if (!id || !id->prefix) @@ -2931,22 +3253,26 @@ storeAtts(XML_Parser parser, const ENCODING *enc, if (!b) return XML_ERROR_UNBOUND_PREFIX; - /* as we expand the name we also calculate its hash value */ for (j = 0; j < b->uriLen; j++) { const XML_Char c = b->uri[j]; if (!poolAppendChar(&tempPool, c)) return XML_ERROR_NO_MEMORY; - uriHash = CHAR_HASH(uriHash, c); } + + sip24_update(&sip_state, b->uri, b->uriLen * sizeof(XML_Char)); + while (*s++ != XML_T(ASCII_COLON)) ; + + sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char)); + do { /* copies null terminator */ - const XML_Char c = *s; if (!poolAppendChar(&tempPool, *s)) return XML_ERROR_NO_MEMORY; - uriHash = CHAR_HASH(uriHash, c); } while (*s++); + uriHash = (unsigned long)sip24_final(&sip_state); + { /* Check hash table for duplicate of expanded name (uriName). Derived from code in lookup(parser, HASH_TABLE *table, ...). */ @@ -3264,11 +3590,11 @@ doCdataSection(XML_Parser parser, if (MUST_CONVERT(enc, s)) { for (;;) { ICHAR *dataPtr = (ICHAR *)dataBuf; - XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)dataBufEnd); + const enum XML_Convert_Result convert_res = XmlConvert(enc, &s, next, &dataPtr, (ICHAR *)dataBufEnd); *eventEndPP = next; charDataHandler(handlerArg, dataBuf, (int)(dataPtr - (ICHAR *)dataBuf)); - if (s == next) + if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } @@ -3660,6 +3986,14 @@ entityValueInitProcessor(XML_Parser parser, *nextPtr = next; return XML_ERROR_NONE; } + /* If we get this token, we have the start of what might be a + normal tag, but not a declaration (i.e. it doesn't begin with + "internalEventEndPtr = NULL; textStart = (char *)entity->textPtr; textEnd = (char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + next = textStart; #ifdef XML_DTD if (entity->is_param) { @@ -4881,6 +5217,8 @@ internalEntityProcessor(XML_Parser parser, entity = openEntity->entity; textStart = ((char *)entity->textPtr) + entity->processed; textEnd = (char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + next = textStart; #ifdef XML_DTD if (entity->is_param) { @@ -4927,9 +5265,9 @@ internalEntityProcessor(XML_Parser parser, static enum XML_Error PTRCALL errorProcessor(XML_Parser parser, - const char *s, - const char *end, - const char **nextPtr) + const char *UNUSED_P(s), + const char *UNUSED_P(end), + const char **UNUSED_P(nextPtr)) { return errorCode; } @@ -5345,6 +5683,7 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, const char *end) { if (MUST_CONVERT(enc, s)) { + enum XML_Convert_Result convert_res; const char **eventPP; const char **eventEndPP; if (enc == encoding) { @@ -5357,11 +5696,11 @@ reportDefault(XML_Parser parser, const ENCODING *enc, } do { ICHAR *dataPtr = (ICHAR *)dataBuf; - XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)dataBufEnd); + convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)dataBufEnd); *eventEndPP = s; defaultHandler(handlerArg, dataBuf, (int)(dataPtr - (ICHAR *)dataBuf)); *eventPP = s; - } while (s != end); + } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE)); } else defaultHandler(handlerArg, (XML_Char *)s, (int)((XML_Char *)end - (XML_Char *)s)); @@ -5840,7 +6179,6 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H newE->defaultAtts = (DEFAULT_ATTRIBUTE *) ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (!newE->defaultAtts) { - ms->free_fcn(newE); return 0; } } @@ -5975,13 +6313,32 @@ keyeq(KEY s1, KEY s2) return XML_FALSE; } +static size_t +keylen(KEY s) +{ + size_t len = 0; + for (; *s; s++, len++); + return len; +} + +static void +copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key) +{ + key->k[0] = 0; + key->k[1] = get_hash_secret_salt(parser); +} + static unsigned long FASTCALL hash(XML_Parser parser, KEY s) { - unsigned long h = hash_secret_salt; - while (*s) - h = CHAR_HASH(h, *s++); - return h; + struct siphash state; + struct sipkey key; + (void)sip_tobin; + (void)sip24_valid; + copy_salt_to_sipkey(parser, &key); + sip24_init(&state, &key); + sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + return (unsigned long)sip24_final(&state); } static NAMED * @@ -6166,8 +6523,8 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, if (!pool->ptr && !poolGrow(pool)) return NULL; for (;;) { - XmlConvert(enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end); - if (ptr == end) + const enum XML_Convert_Result convert_res = XmlConvert(enc, &ptr, end, (ICHAR **)&(pool->ptr), (ICHAR *)pool->end); + if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; if (!poolGrow(pool)) return NULL; @@ -6224,6 +6581,35 @@ poolStoreString(STRING_POOL *pool, const ENCODING *enc, return pool->start; } +static size_t +poolBytesToAllocateFor(int blockSize) +{ + /* Unprotected math would be: + ** return offsetof(BLOCK, s) + blockSize * sizeof(XML_Char); + ** + ** Detect overflow, avoiding _signed_ overflow undefined behavior + ** For a + b * c we check b * c in isolation first, so that addition of a + ** on top has no chance of making us accept a small non-negative number + */ + const size_t stretch = sizeof(XML_Char); /* can be 4 bytes */ + + if (blockSize <= 0) + return 0; + + if (blockSize > (int)(INT_MAX / stretch)) + return 0; + + { + const int stretchedBlockSize = blockSize * (int)stretch; + const int bytesToAllocate = (int)( + offsetof(BLOCK, s) + (unsigned)stretchedBlockSize); + if (bytesToAllocate < 0) + return 0; + + return (size_t)bytesToAllocate; + } +} + static XML_Bool FASTCALL poolGrow(STRING_POOL *pool) { @@ -6251,11 +6637,19 @@ poolGrow(STRING_POOL *pool) } } if (pool->blocks && pool->start == pool->blocks->s) { - int blockSize = (int)(pool->end - pool->start)*2; - BLOCK *temp = (BLOCK *) - pool->mem->realloc_fcn(pool->blocks, - (offsetof(BLOCK, s) - + blockSize * sizeof(XML_Char))); + BLOCK *temp; + int blockSize = (int)((unsigned)(pool->end - pool->start)*2U); + size_t bytesToAllocate; + + if (blockSize < 0) + return XML_FALSE; + + bytesToAllocate = poolBytesToAllocateFor(blockSize); + if (bytesToAllocate == 0) + return XML_FALSE; + + temp = (BLOCK *) + pool->mem->realloc_fcn(pool->blocks, (unsigned)bytesToAllocate); if (temp == NULL) return XML_FALSE; pool->blocks = temp; @@ -6267,12 +6661,26 @@ poolGrow(STRING_POOL *pool) else { BLOCK *tem; int blockSize = (int)(pool->end - pool->start); + size_t bytesToAllocate; + + if (blockSize < 0) + return XML_FALSE; + if (blockSize < INIT_BLOCK_SIZE) blockSize = INIT_BLOCK_SIZE; - else + else { + /* Detect overflow, avoiding _signed_ overflow undefined behavior */ + if ((int)((unsigned)blockSize * 2U) < 0) { + return XML_FALSE; + } blockSize *= 2; - tem = (BLOCK *)pool->mem->malloc_fcn(offsetof(BLOCK, s) - + blockSize * sizeof(XML_Char)); + } + + bytesToAllocate = poolBytesToAllocateFor(blockSize); + if (bytesToAllocate == 0) + return XML_FALSE; + + tem = (BLOCK *)pool->mem->malloc_fcn(bytesToAllocate); if (!tem) return XML_FALSE; tem->size = blockSize; diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 44772e2..a7c5630 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -4,19 +4,13 @@ #include -#ifdef COMPILED_FROM_DSP +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #else #ifdef HAVE_EXPAT_CONFIG_H #include #endif -#endif /* ndef COMPILED_FROM_DSP */ +#endif /* ndef _WIN32 */ #include "expat_external.h" #include "internal.h" @@ -195,9 +189,9 @@ prolog1(PROLOG_STATE *state, static int PTRCALL prolog2(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -216,9 +210,9 @@ prolog2(PROLOG_STATE *state, static int PTRCALL doctype0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -264,9 +258,9 @@ doctype1(PROLOG_STATE *state, static int PTRCALL doctype2(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -281,9 +275,9 @@ doctype2(PROLOG_STATE *state, static int PTRCALL doctype3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -298,9 +292,9 @@ doctype3(PROLOG_STATE *state, static int PTRCALL doctype4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -318,9 +312,9 @@ doctype4(PROLOG_STATE *state, static int PTRCALL doctype5(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -437,9 +431,9 @@ externalSubset1(PROLOG_STATE *state, static int PTRCALL entity0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -457,9 +451,9 @@ entity0(PROLOG_STATE *state, static int PTRCALL entity1(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -502,9 +496,9 @@ entity2(PROLOG_STATE *state, static int PTRCALL entity3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -519,9 +513,9 @@ entity3(PROLOG_STATE *state, static int PTRCALL entity4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -559,9 +553,9 @@ entity5(PROLOG_STATE *state, static int PTRCALL entity6(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -605,9 +599,9 @@ entity7(PROLOG_STATE *state, static int PTRCALL entity8(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -622,9 +616,9 @@ entity8(PROLOG_STATE *state, static int PTRCALL entity9(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -639,9 +633,9 @@ entity9(PROLOG_STATE *state, static int PTRCALL entity10(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -656,9 +650,9 @@ entity10(PROLOG_STATE *state, static int PTRCALL notation0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -697,9 +691,9 @@ notation1(PROLOG_STATE *state, static int PTRCALL notation2(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -714,9 +708,9 @@ notation2(PROLOG_STATE *state, static int PTRCALL notation3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -732,9 +726,9 @@ notation3(PROLOG_STATE *state, static int PTRCALL notation4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -753,9 +747,9 @@ notation4(PROLOG_STATE *state, static int PTRCALL attlist0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -771,9 +765,9 @@ attlist0(PROLOG_STATE *state, static int PTRCALL attlist1(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -833,9 +827,9 @@ attlist2(PROLOG_STATE *state, static int PTRCALL attlist3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -852,9 +846,9 @@ attlist3(PROLOG_STATE *state, static int PTRCALL attlist4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -872,9 +866,9 @@ attlist4(PROLOG_STATE *state, static int PTRCALL attlist5(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -889,9 +883,9 @@ attlist5(PROLOG_STATE *state, static int PTRCALL attlist6(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -906,9 +900,9 @@ attlist6(PROLOG_STATE *state, static int PTRCALL attlist7(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -967,9 +961,9 @@ attlist8(PROLOG_STATE *state, static int PTRCALL attlist9(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -984,9 +978,9 @@ attlist9(PROLOG_STATE *state, static int PTRCALL element0(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1072,9 +1066,9 @@ element2(PROLOG_STATE *state, static int PTRCALL element3(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1097,9 +1091,9 @@ element3(PROLOG_STATE *state, static int PTRCALL element4(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1115,9 +1109,9 @@ element4(PROLOG_STATE *state, static int PTRCALL element5(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1136,9 +1130,9 @@ element5(PROLOG_STATE *state, static int PTRCALL element6(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1166,9 +1160,9 @@ element6(PROLOG_STATE *state, static int PTRCALL element7(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1240,9 +1234,9 @@ condSect0(PROLOG_STATE *state, static int PTRCALL condSect1(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1258,9 +1252,9 @@ condSect1(PROLOG_STATE *state, static int PTRCALL condSect2(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1277,9 +1271,9 @@ condSect2(PROLOG_STATE *state, static int PTRCALL declClose(PROLOG_STATE *state, int tok, - const char *ptr, - const char *end, - const ENCODING *enc) + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { switch (tok) { case XML_TOK_PROLOG_S: @@ -1292,11 +1286,11 @@ declClose(PROLOG_STATE *state, } static int PTRCALL -error(PROLOG_STATE *state, - int tok, - const char *ptr, - const char *end, - const ENCODING *enc) +error(PROLOG_STATE *UNUSED_P(state), + int UNUSED_P(tok), + const char *UNUSED_P(ptr), + const char *UNUSED_P(end), + const ENCODING *UNUSED_P(enc)) { return XML_ROLE_NONE; } diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index bf09dfc..cdf0720 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -4,19 +4,13 @@ #include -#ifdef COMPILED_FROM_DSP +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #else #ifdef HAVE_EXPAT_CONFIG_H #include #endif -#endif /* ndef COMPILED_FROM_DSP */ +#endif /* ndef _WIN32 */ #include "expat_external.h" #include "internal.h" @@ -46,7 +40,7 @@ #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16) #define UCS2_GET_NAMING(pages, hi, lo) \ - (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1 << ((lo) & 0x1F))) + (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1u << ((lo) & 0x1F))) /* A 2 byte UTF-8 representation splits the characters 11 bits between the bottom 5 and 6 bits of the bytes. We need 8 bits to index into @@ -56,7 +50,7 @@ (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3) \ + ((((byte)[0]) & 3) << 1) \ + ((((byte)[1]) >> 5) & 1)] \ - & (1 << (((byte)[1]) & 0x1F))) + & (1u << (((byte)[1]) & 0x1F))) /* A 3 byte UTF-8 representation splits the characters 16 bits between the bottom 4, 6 and 6 bits of the bytes. We need 8 bits to index @@ -69,7 +63,7 @@ << 3) \ + ((((byte)[1]) & 3) << 1) \ + ((((byte)[2]) >> 5) & 1)] \ - & (1 << (((byte)[2]) & 0x1F))) + & (1u << (((byte)[2]) & 0x1F))) #define UTF8_GET_NAMING(pages, p, n) \ ((n) == 2 \ @@ -122,19 +116,19 @@ ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0))) static int PTRFASTCALL -isNever(const ENCODING *enc, const char *p) +isNever(const ENCODING *UNUSED_P(enc), const char *UNUSED_P(p)) { return 0; } static int PTRFASTCALL -utf8_isName2(const ENCODING *enc, const char *p) +utf8_isName2(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_GET_NAMING2(namePages, (const unsigned char *)p); } static int PTRFASTCALL -utf8_isName3(const ENCODING *enc, const char *p) +utf8_isName3(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_GET_NAMING3(namePages, (const unsigned char *)p); } @@ -142,13 +136,13 @@ utf8_isName3(const ENCODING *enc, const char *p) #define utf8_isName4 isNever static int PTRFASTCALL -utf8_isNmstrt2(const ENCODING *enc, const char *p) +utf8_isNmstrt2(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p); } static int PTRFASTCALL -utf8_isNmstrt3(const ENCODING *enc, const char *p) +utf8_isNmstrt3(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p); } @@ -156,19 +150,19 @@ utf8_isNmstrt3(const ENCODING *enc, const char *p) #define utf8_isNmstrt4 isNever static int PTRFASTCALL -utf8_isInvalid2(const ENCODING *enc, const char *p) +utf8_isInvalid2(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_INVALID2((const unsigned char *)p); } static int PTRFASTCALL -utf8_isInvalid3(const ENCODING *enc, const char *p) +utf8_isInvalid3(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_INVALID3((const unsigned char *)p); } static int PTRFASTCALL -utf8_isInvalid4(const ENCODING *enc, const char *p) +utf8_isInvalid4(const ENCODING *UNUSED_P(enc), const char *p) { return UTF8_INVALID4((const unsigned char *)p); } @@ -222,6 +216,17 @@ struct normal_encoding { E ## isInvalid3, \ E ## isInvalid4 +#define NULL_VTABLE \ + /* isName2 */ NULL, \ + /* isName3 */ NULL, \ + /* isName4 */ NULL, \ + /* isNmstrt2 */ NULL, \ + /* isNmstrt3 */ NULL, \ + /* isNmstrt4 */ NULL, \ + /* isInvalid2 */ NULL, \ + /* isInvalid3 */ NULL, \ + /* isInvalid4 */ NULL + static int FASTCALL checkCharRefNumber(int); #include "xmltok_impl.h" @@ -318,39 +323,89 @@ enum { /* UTF8_cvalN is value of masked first byte of N byte sequence */ UTF8_cval4 = 0xf0 }; -static void PTRCALL -utf8_toUtf8(const ENCODING *enc, +void +align_limit_to_full_utf8_characters(const char * from, const char ** fromLimRef) +{ + const char * fromLim = *fromLimRef; + size_t walked = 0; + for (; fromLim > from; fromLim--, walked++) { + const unsigned char prev = (unsigned char)fromLim[-1]; + if ((prev & 0xf8u) == 0xf0u) { /* 4-byte character, lead by 0b11110xxx byte */ + if (walked + 1 >= 4) { + fromLim += 4 - 1; + break; + } else { + walked = 0; + } + } else if ((prev & 0xf0u) == 0xe0u) { /* 3-byte character, lead by 0b1110xxxx byte */ + if (walked + 1 >= 3) { + fromLim += 3 - 1; + break; + } else { + walked = 0; + } + } else if ((prev & 0xe0u) == 0xc0u) { /* 2-byte character, lead by 0b110xxxxx byte */ + if (walked + 1 >= 2) { + fromLim += 2 - 1; + break; + } else { + walked = 0; + } + } else if ((prev & 0x80u) == 0x00u) { /* 1-byte character, matching 0b0xxxxxxx */ + break; + } + } + *fromLimRef = fromLim; +} + +static enum XML_Convert_Result PTRCALL +utf8_toUtf8(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, char **toP, const char *toLim) { char *to; const char *from; - if (fromLim - *fromP > toLim - *toP) { - /* Avoid copying partial characters. */ - for (fromLim = *fromP + (toLim - *toP); fromLim > *fromP; fromLim--) - if (((unsigned char)fromLim[-1] & 0xc0) != 0x80) - break; - } - for (to = *toP, from = *fromP; from != fromLim; from++, to++) + const char *fromLimInitial = fromLim; + + /* Avoid copying partial characters. */ + align_limit_to_full_utf8_characters(*fromP, &fromLim); + + for (to = *toP, from = *fromP; (from < fromLim) && (to < toLim); from++, to++) *to = *from; *fromP = from; *toP = to; + + if (fromLim < fromLimInitial) + return XML_CONVERT_INPUT_INCOMPLETE; + else if ((to == toLim) && (from < fromLim)) + return XML_CONVERT_OUTPUT_EXHAUSTED; + else + return XML_CONVERT_COMPLETED; } -static void PTRCALL +static enum XML_Convert_Result PTRCALL utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { + enum XML_Convert_Result res = XML_CONVERT_COMPLETED; unsigned short *to = *toP; const char *from = *fromP; - while (from != fromLim && to != toLim) { + while (from < fromLim && to < toLim) { switch (((struct normal_encoding *)enc)->type[(unsigned char)*from]) { case BT_LEAD2: + if (fromLim - from < 2) { + res = XML_CONVERT_INPUT_INCOMPLETE; + goto after; + } *to++ = (unsigned short)(((from[0] & 0x1f) << 6) | (from[1] & 0x3f)); from += 2; break; case BT_LEAD3: + if (fromLim - from < 3) { + res = XML_CONVERT_INPUT_INCOMPLETE; + goto after; + } *to++ = (unsigned short)(((from[0] & 0xf) << 12) | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f)); from += 3; @@ -358,8 +413,14 @@ utf8_toUtf16(const ENCODING *enc, case BT_LEAD4: { unsigned long n; - if (to + 1 == toLim) + if (toLim - to < 2) { + res = XML_CONVERT_OUTPUT_EXHAUSTED; + goto after; + } + if (fromLim - from < 4) { + res = XML_CONVERT_INPUT_INCOMPLETE; goto after; + } n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12) | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f); n -= 0x10000; @@ -374,9 +435,12 @@ utf8_toUtf16(const ENCODING *enc, break; } } + if (from < fromLim) + res = XML_CONVERT_OUTPUT_EXHAUSTED; after: *fromP = from; *toP = to; + return res; } #ifdef XML_NS @@ -425,38 +489,43 @@ static const struct normal_encoding internal_utf8_encoding = { STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_) }; -static void PTRCALL -latin1_toUtf8(const ENCODING *enc, +static enum XML_Convert_Result PTRCALL +latin1_toUtf8(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, char **toP, const char *toLim) { for (;;) { unsigned char c; if (*fromP == fromLim) - break; + return XML_CONVERT_COMPLETED; c = (unsigned char)**fromP; if (c & 0x80) { if (toLim - *toP < 2) - break; + return XML_CONVERT_OUTPUT_EXHAUSTED; *(*toP)++ = (char)((c >> 6) | UTF8_cval2); *(*toP)++ = (char)((c & 0x3f) | 0x80); (*fromP)++; } else { if (*toP == toLim) - break; + return XML_CONVERT_OUTPUT_EXHAUSTED; *(*toP)++ = *(*fromP)++; } } } -static void PTRCALL -latin1_toUtf16(const ENCODING *enc, +static enum XML_Convert_Result PTRCALL +latin1_toUtf16(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { - while (*fromP != fromLim && *toP != toLim) + while (*fromP < fromLim && *toP < toLim) *(*toP)++ = (unsigned char)*(*fromP)++; + + if ((*toP == toLim) && (*fromP < fromLim)) + return XML_CONVERT_OUTPUT_EXHAUSTED; + else + return XML_CONVERT_COMPLETED; } #ifdef XML_NS @@ -467,7 +536,7 @@ static const struct normal_encoding latin1_encoding_ns = { #include "asciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(sb_) + STANDARD_VTABLE(sb_) NULL_VTABLE }; #endif @@ -480,16 +549,21 @@ static const struct normal_encoding latin1_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(sb_) + STANDARD_VTABLE(sb_) NULL_VTABLE }; -static void PTRCALL -ascii_toUtf8(const ENCODING *enc, +static enum XML_Convert_Result PTRCALL +ascii_toUtf8(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, char **toP, const char *toLim) { - while (*fromP != fromLim && *toP != toLim) + while (*fromP < fromLim && *toP < toLim) *(*toP)++ = *(*fromP)++; + + if ((*toP == toLim) && (*fromP < fromLim)) + return XML_CONVERT_OUTPUT_EXHAUSTED; + else + return XML_CONVERT_COMPLETED; } #ifdef XML_NS @@ -500,7 +574,7 @@ static const struct normal_encoding ascii_encoding_ns = { #include "asciitab.h" /* BT_NONXML == 0 */ }, - STANDARD_VTABLE(sb_) + STANDARD_VTABLE(sb_) NULL_VTABLE }; #endif @@ -513,7 +587,7 @@ static const struct normal_encoding ascii_encoding = { #undef BT_COLON /* BT_NONXML == 0 */ }, - STANDARD_VTABLE(sb_) + STANDARD_VTABLE(sb_) NULL_VTABLE }; static int PTRFASTCALL @@ -536,13 +610,14 @@ unicode_byte_type(char hi, char lo) } #define DEFINE_UTF16_TO_UTF8(E) \ -static void PTRCALL \ -E ## toUtf8(const ENCODING *enc, \ +static enum XML_Convert_Result PTRCALL \ +E ## toUtf8(const ENCODING *UNUSED_P(enc), \ const char **fromP, const char *fromLim, \ char **toP, const char *toLim) \ { \ - const char *from; \ - for (from = *fromP; from != fromLim; from += 2) { \ + const char *from = *fromP; \ + fromLim = from + (((fromLim - from) >> 1) << 1); /* shrink to even */ \ + for (; from < fromLim; from += 2) { \ int plane; \ unsigned char lo2; \ unsigned char lo = GET_LO(from); \ @@ -552,7 +627,7 @@ E ## toUtf8(const ENCODING *enc, \ if (lo < 0x80) { \ if (*toP == toLim) { \ *fromP = from; \ - return; \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ } \ *(*toP)++ = lo; \ break; \ @@ -562,7 +637,7 @@ E ## toUtf8(const ENCODING *enc, \ case 0x4: case 0x5: case 0x6: case 0x7: \ if (toLim - *toP < 2) { \ *fromP = from; \ - return; \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ } \ *(*toP)++ = ((lo >> 6) | (hi << 2) | UTF8_cval2); \ *(*toP)++ = ((lo & 0x3f) | 0x80); \ @@ -570,7 +645,7 @@ E ## toUtf8(const ENCODING *enc, \ default: \ if (toLim - *toP < 3) { \ *fromP = from; \ - return; \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ } \ /* 16 bits divided 4, 6, 6 amongst 3 bytes */ \ *(*toP)++ = ((hi >> 4) | UTF8_cval3); \ @@ -580,7 +655,11 @@ E ## toUtf8(const ENCODING *enc, \ case 0xD8: case 0xD9: case 0xDA: case 0xDB: \ if (toLim - *toP < 4) { \ *fromP = from; \ - return; \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ + } \ + if (fromLim - from < 4) { \ + *fromP = from; \ + return XML_CONVERT_INPUT_INCOMPLETE; \ } \ plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1; \ *(*toP)++ = ((plane >> 2) | UTF8_cval4); \ @@ -596,20 +675,32 @@ E ## toUtf8(const ENCODING *enc, \ } \ } \ *fromP = from; \ + if (from < fromLim) \ + return XML_CONVERT_INPUT_INCOMPLETE; \ + else \ + return XML_CONVERT_COMPLETED; \ } #define DEFINE_UTF16_TO_UTF16(E) \ -static void PTRCALL \ -E ## toUtf16(const ENCODING *enc, \ +static enum XML_Convert_Result PTRCALL \ +E ## toUtf16(const ENCODING *UNUSED_P(enc), \ const char **fromP, const char *fromLim, \ unsigned short **toP, const unsigned short *toLim) \ { \ + enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \ + fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */ \ /* Avoid copying first half only of surrogate */ \ if (fromLim - *fromP > ((toLim - *toP) << 1) \ - && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) \ + && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \ fromLim -= 2; \ - for (; *fromP != fromLim && *toP != toLim; *fromP += 2) \ + res = XML_CONVERT_INPUT_INCOMPLETE; \ + } \ + for (; *fromP < fromLim && *toP < toLim; *fromP += 2) \ *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP); \ + if ((*toP == toLim) && (*fromP < fromLim)) \ + return XML_CONVERT_OUTPUT_EXHAUSTED; \ + else \ + return res; \ } #define SET2(ptr, ch) \ @@ -726,7 +817,7 @@ static const struct normal_encoding little2_encoding_ns = { #include "asciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) + STANDARD_VTABLE(little2_) NULL_VTABLE }; #endif @@ -745,7 +836,7 @@ static const struct normal_encoding little2_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) + STANDARD_VTABLE(little2_) NULL_VTABLE }; #if BYTEORDER != 4321 @@ -758,7 +849,7 @@ static const struct normal_encoding internal_little2_encoding_ns = { #include "iasciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) + STANDARD_VTABLE(little2_) NULL_VTABLE }; #endif @@ -771,7 +862,7 @@ static const struct normal_encoding internal_little2_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) + STANDARD_VTABLE(little2_) NULL_VTABLE }; #endif @@ -867,7 +958,7 @@ static const struct normal_encoding big2_encoding_ns = { #include "asciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) + STANDARD_VTABLE(big2_) NULL_VTABLE }; #endif @@ -886,7 +977,7 @@ static const struct normal_encoding big2_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) + STANDARD_VTABLE(big2_) NULL_VTABLE }; #if BYTEORDER != 1234 @@ -899,7 +990,7 @@ static const struct normal_encoding internal_big2_encoding_ns = { #include "iasciitab.h" #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) + STANDARD_VTABLE(big2_) NULL_VTABLE }; #endif @@ -912,7 +1003,7 @@ static const struct normal_encoding internal_big2_encoding = { #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) + STANDARD_VTABLE(big2_) NULL_VTABLE }; #endif @@ -938,7 +1029,7 @@ streqci(const char *s1, const char *s2) } static void PTRCALL -initUpdatePosition(const ENCODING *enc, const char *ptr, +initUpdatePosition(const ENCODING *UNUSED_P(enc), const char *ptr, const char *end, POSITION *pos) { normal_updatePosition(&utf8_encoding.enc, ptr, end, pos); @@ -1288,7 +1379,7 @@ unknown_isInvalid(const ENCODING *enc, const char *p) return (c & ~0xFFFF) || checkCharRefNumber(c) < 0; } -static void PTRCALL +static enum XML_Convert_Result PTRCALL unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) @@ -1299,21 +1390,21 @@ unknown_toUtf8(const ENCODING *enc, const char *utf8; int n; if (*fromP == fromLim) - break; + return XML_CONVERT_COMPLETED; utf8 = uenc->utf8[(unsigned char)**fromP]; n = *utf8++; if (n == 0) { int c = uenc->convert(uenc->userData, *fromP); n = XmlUtf8Encode(c, buf); if (n > toLim - *toP) - break; + return XML_CONVERT_OUTPUT_EXHAUSTED; utf8 = buf; *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP] - (BT_LEAD2 - 2)); } else { if (n > toLim - *toP) - break; + return XML_CONVERT_OUTPUT_EXHAUSTED; (*fromP)++; } do { @@ -1322,13 +1413,13 @@ unknown_toUtf8(const ENCODING *enc, } } -static void PTRCALL +static enum XML_Convert_Result PTRCALL unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); - while (*fromP != fromLim && *toP != toLim) { + while (*fromP < fromLim && *toP < toLim) { unsigned short c = uenc->utf16[(unsigned char)**fromP]; if (c == 0) { c = (unsigned short) @@ -1340,6 +1431,11 @@ unknown_toUtf16(const ENCODING *enc, (*fromP)++; *(*toP)++ = c; } + + if ((*toP == toLim) && (*fromP < fromLim)) + return XML_CONVERT_OUTPUT_EXHAUSTED; + else + return XML_CONVERT_COMPLETED; } ENCODING * @@ -1503,7 +1599,7 @@ initScan(const ENCODING * const *encodingTable, { const ENCODING **encPtr; - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; encPtr = enc->encPtr; if (ptr + 1 == end) { diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index ca867aa..752007e 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -130,6 +130,12 @@ typedef int (PTRCALL *SCANNER)(const ENCODING *, const char *, const char **); +enum XML_Convert_Result { + XML_CONVERT_COMPLETED = 0, + XML_CONVERT_INPUT_INCOMPLETE = 1, + XML_CONVERT_OUTPUT_EXHAUSTED = 2 /* and therefore potentially input remaining as well */ +}; + struct encoding { SCANNER scanners[XML_N_STATES]; SCANNER literalScanners[XML_N_LITERAL_TYPES]; @@ -158,12 +164,12 @@ struct encoding { const char *ptr, const char *end, const char **badPtr); - void (PTRCALL *utf8Convert)(const ENCODING *enc, + enum XML_Convert_Result (PTRCALL *utf8Convert)(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim); - void (PTRCALL *utf16Convert)(const ENCODING *enc, + enum XML_Convert_Result (PTRCALL *utf16Convert)(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 9c2895b..5f779c0 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -87,27 +87,45 @@ #define PREFIX(ident) ident #endif + +#define HAS_CHARS(enc, ptr, end, count) \ + (end - ptr >= count * MINBPC(enc)) + +#define HAS_CHAR(enc, ptr, end) \ + HAS_CHARS(enc, ptr, end, 1) + +#define REQUIRE_CHARS(enc, ptr, end, count) \ + { \ + if (! HAS_CHARS(enc, ptr, end, count)) { \ + return XML_TOK_PARTIAL; \ + } \ + } + +#define REQUIRE_CHAR(enc, ptr, end) \ + REQUIRE_CHARS(enc, ptr, end, 1) + + /* ptr points to character following " */ switch (BYTE_TYPE(enc, ptr + MINBPC(enc))) { case BT_S: case BT_CR: case BT_LF: case BT_PERCNT: @@ -175,7 +191,7 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, } static int PTRCALL -PREFIX(checkPiTarget)(const ENCODING *enc, const char *ptr, +PREFIX(checkPiTarget)(const ENCODING *UNUSED_P(enc), const char *ptr, const char *end, int *tokPtr) { int upper = 0; @@ -225,15 +241,14 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, { int tok; const char *target = ptr; - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_S: case BT_CR: case BT_LF: @@ -242,13 +257,12 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, return XML_TOK_INVALID; } ptr += MINBPC(enc); - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { INVALID_CASES(ptr, nextTokPtr) case BT_QUEST: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_GT)) { *nextTokPtr = ptr + MINBPC(enc); return tok; @@ -266,8 +280,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, return XML_TOK_INVALID; } ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_GT)) { *nextTokPtr = ptr + MINBPC(enc); return tok; @@ -282,15 +295,14 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, } static int PTRCALL -PREFIX(scanCdataSection)(const ENCODING *enc, const char *ptr, +PREFIX(scanCdataSection)(const ENCODING *UNUSED_P(enc), const char *ptr, const char *end, const char **nextTokPtr) { static const char CDATA_LSQB[] = { ASCII_C, ASCII_D, ASCII_A, ASCII_T, ASCII_A, ASCII_LSQB }; int i; /* CDATA[ */ - if (end - ptr < 6 * MINBPC(enc)) - return XML_TOK_PARTIAL; + REQUIRE_CHARS(enc, ptr, end, 6); for (i = 0; i < 6; i++, ptr += MINBPC(enc)) { if (!CHAR_MATCHES(enc, ptr, CDATA_LSQB[i])) { *nextTokPtr = ptr; @@ -305,7 +317,7 @@ static int PTRCALL PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; if (MINBPC(enc) > 1) { size_t n = end - ptr; @@ -319,13 +331,11 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, switch (BYTE_TYPE(enc, ptr)) { case BT_RSQB: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB)) break; ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) { ptr -= MINBPC(enc); break; @@ -334,8 +344,7 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, return XML_TOK_CDATA_SECT_CLOSE; case BT_CR: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); *nextTokPtr = ptr; @@ -348,7 +357,7 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, ptr += MINBPC(enc); break; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: \ @@ -383,19 +392,18 @@ static int PTRCALL PREFIX(scanEndTag)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_S: case BT_CR: case BT_LF: - for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) { + for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) { switch (BYTE_TYPE(enc, ptr)) { case BT_S: case BT_CR: case BT_LF: break; @@ -432,7 +440,7 @@ static int PTRCALL PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr != end) { + if (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { case BT_DIGIT: case BT_HEX: @@ -441,7 +449,7 @@ PREFIX(scanHexCharRef)(const ENCODING *enc, const char *ptr, *nextTokPtr = ptr; return XML_TOK_INVALID; } - for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) { + for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) { switch (BYTE_TYPE(enc, ptr)) { case BT_DIGIT: case BT_HEX: @@ -464,7 +472,7 @@ static int PTRCALL PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr != end) { + if (HAS_CHAR(enc, ptr, end)) { if (CHAR_MATCHES(enc, ptr, ASCII_x)) return PREFIX(scanHexCharRef)(enc, ptr + MINBPC(enc), end, nextTokPtr); switch (BYTE_TYPE(enc, ptr)) { @@ -474,7 +482,7 @@ PREFIX(scanCharRef)(const ENCODING *enc, const char *ptr, *nextTokPtr = ptr; return XML_TOK_INVALID; } - for (ptr += MINBPC(enc); ptr != end; ptr += MINBPC(enc)) { + for (ptr += MINBPC(enc); HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) { switch (BYTE_TYPE(enc, ptr)) { case BT_DIGIT: break; @@ -496,8 +504,7 @@ static int PTRCALL PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_NUM: @@ -506,7 +513,7 @@ PREFIX(scanRef)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_SEMI: @@ -529,7 +536,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, #ifdef XML_NS int hadColon = 0; #endif - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) #ifdef XML_NS @@ -540,8 +547,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, } hadColon = 1; ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: @@ -555,8 +561,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, int t; ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); t = BYTE_TYPE(enc, ptr); if (t == BT_EQUALS) break; @@ -579,8 +584,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, #endif for (;;) { ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); open = BYTE_TYPE(enc, ptr); if (open == BT_QUOT || open == BT_APOS) break; @@ -598,8 +602,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, /* in attribute value */ for (;;) { int t; - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); t = BYTE_TYPE(enc, ptr); if (t == open) break; @@ -624,8 +627,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, } } ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { case BT_S: case BT_CR: @@ -642,8 +644,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, /* ptr points to closing quote */ for (;;) { ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_S: case BT_CR: case BT_LF: @@ -655,8 +656,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, case BT_SOL: sol: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) { *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -688,13 +688,12 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, #ifdef XML_NS int hadColon; #endif - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_EXCL: - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { case BT_MINUS: return PREFIX(scanComment)(enc, ptr + MINBPC(enc), end, nextTokPtr); @@ -716,7 +715,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, hadColon = 0; #endif /* we have a start-tag */ - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) #ifdef XML_NS @@ -727,8 +726,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, } hadColon = 1; ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: @@ -740,7 +738,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, case BT_S: case BT_CR: case BT_LF: { ptr += MINBPC(enc); - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_GT: @@ -765,8 +763,7 @@ PREFIX(scanLt)(const ENCODING *enc, const char *ptr, const char *end, case BT_SOL: sol: ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) { *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -785,7 +782,7 @@ static int PTRCALL PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; if (MINBPC(enc) > 1) { size_t n = end - ptr; @@ -803,7 +800,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, return PREFIX(scanRef)(enc, ptr + MINBPC(enc), end, nextTokPtr); case BT_CR: ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_CR; if (BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); @@ -814,12 +811,12 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_DATA_NEWLINE; case BT_RSQB: ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_RSQB; if (!CHAR_MATCHES(enc, ptr, ASCII_RSQB)) break; ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_RSQB; if (!CHAR_MATCHES(enc, ptr, ASCII_GT)) { ptr -= MINBPC(enc); @@ -832,7 +829,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, ptr += MINBPC(enc); break; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: \ @@ -845,12 +842,12 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4) #undef LEAD_CASE case BT_RSQB: - if (ptr + MINBPC(enc) != end) { + if (HAS_CHARS(enc, ptr, end, 2)) { if (!CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_RSQB)) { ptr += MINBPC(enc); break; } - if (ptr + 2*MINBPC(enc) != end) { + if (HAS_CHARS(enc, ptr, end, 3)) { if (!CHAR_MATCHES(enc, ptr + 2*MINBPC(enc), ASCII_GT)) { ptr += MINBPC(enc); break; @@ -884,8 +881,7 @@ static int PTRCALL PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) case BT_S: case BT_LF: case BT_CR: case BT_PERCNT: @@ -895,7 +891,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_SEMI: @@ -913,15 +909,14 @@ static int PTRCALL PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { CHECK_NMSTRT_CASES(enc, ptr, end, nextTokPtr) default: *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_CR: case BT_LF: case BT_S: @@ -941,7 +936,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { int t = BYTE_TYPE(enc, ptr); switch (t) { INVALID_CASES(ptr, nextTokPtr) @@ -950,7 +945,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc, ptr += MINBPC(enc); if (t != open) break; - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return -XML_TOK_LITERAL; *nextTokPtr = ptr; switch (BYTE_TYPE(enc, ptr)) { @@ -973,7 +968,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { int tok; - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; if (MINBPC(enc) > 1) { size_t n = end - ptr; @@ -992,8 +987,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, case BT_LT: { ptr += MINBPC(enc); - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); switch (BYTE_TYPE(enc, ptr)) { case BT_EXCL: return PREFIX(scanDecl)(enc, ptr + MINBPC(enc), end, nextTokPtr); @@ -1021,7 +1015,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, case BT_S: case BT_LF: for (;;) { ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) break; switch (BYTE_TYPE(enc, ptr)) { case BT_S: case BT_LF: @@ -1048,11 +1042,10 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_OPEN_BRACKET; case BT_RSQB: ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return -XML_TOK_CLOSE_BRACKET; if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) { - if (ptr + MINBPC(enc) == end) - return XML_TOK_PARTIAL; + REQUIRE_CHARS(enc, ptr, end, 2); if (CHAR_MATCHES(enc, ptr + MINBPC(enc), ASCII_GT)) { *nextTokPtr = ptr + 2*MINBPC(enc); return XML_TOK_COND_SECT_CLOSE; @@ -1065,7 +1058,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_OPEN_PAREN; case BT_RPAR: ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return -XML_TOK_CLOSE_PAREN; switch (BYTE_TYPE(enc, ptr)) { case BT_AST: @@ -1141,7 +1134,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) case BT_GT: case BT_RPAR: case BT_COMMA: @@ -1154,8 +1147,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, ptr += MINBPC(enc); switch (tok) { case XML_TOK_NAME: - if (ptr == end) - return XML_TOK_PARTIAL; + REQUIRE_CHAR(enc, ptr, end); tok = XML_TOK_PREFIXED_NAME; switch (BYTE_TYPE(enc, ptr)) { CHECK_NAME_CASES(enc, ptr, end, nextTokPtr) @@ -1204,10 +1196,12 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; + else if (! HAS_CHAR(enc, ptr, end)) + return XML_TOK_PARTIAL; start = ptr; - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: ptr += n; break; @@ -1232,7 +1226,7 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, case BT_CR: if (ptr == start) { ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_CR; if (BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); @@ -1262,10 +1256,12 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; - if (ptr == end) + if (ptr >= end) return XML_TOK_NONE; + else if (! HAS_CHAR(enc, ptr, end)) + return XML_TOK_PARTIAL; start = ptr; - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: ptr += n; break; @@ -1294,7 +1290,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, case BT_CR: if (ptr == start) { ptr += MINBPC(enc); - if (ptr == end) + if (! HAS_CHAR(enc, ptr, end)) return XML_TOK_TRAILING_CR; if (BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); @@ -1326,15 +1322,15 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, end = ptr + n; } } - while (ptr != end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { INVALID_CASES(ptr, nextTokPtr) case BT_LT: - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_EXCL)) { - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_LSQB)) { ++level; ptr += MINBPC(enc); @@ -1342,11 +1338,11 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, } break; case BT_RSQB: - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_RSQB)) { - if ((ptr += MINBPC(enc)) == end) - return XML_TOK_PARTIAL; + ptr += MINBPC(enc); + REQUIRE_CHAR(enc, ptr, end); if (CHAR_MATCHES(enc, ptr, ASCII_GT)) { ptr += MINBPC(enc); if (level == 0) { @@ -1373,7 +1369,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, { ptr += MINBPC(enc); end -= MINBPC(enc); - for (; ptr != end; ptr += MINBPC(enc)) { + for (; HAS_CHAR(enc, ptr, end); ptr += MINBPC(enc)) { switch (BYTE_TYPE(enc, ptr)) { case BT_DIGIT: case BT_HEX: @@ -1521,7 +1517,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr, } static int PTRFASTCALL -PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) +PREFIX(charRefNumber)(const ENCODING *UNUSED_P(enc), const char *ptr) { int result = 0; /* skip &# */ @@ -1565,7 +1561,7 @@ PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) } static int PTRCALL -PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr, +PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr, const char *end) { switch ((end - ptr)/MINBPC(enc)) { @@ -1683,11 +1679,11 @@ PREFIX(sameName)(const ENCODING *enc, const char *ptr1, const char *ptr2) } static int PTRCALL -PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1, +PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1, const char *end1, const char *ptr2) { for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) { - if (ptr1 == end1) + if (end1 - ptr1 < MINBPC(enc)) return 0; if (!CHAR_MATCHES(enc, ptr1, *ptr2)) return 0; @@ -1744,7 +1740,7 @@ PREFIX(updatePosition)(const ENCODING *enc, const char *end, POSITION *pos) { - while (ptr < end) { + while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { #define LEAD_CASE(n) \ case BT_LEAD ## n: \ @@ -1760,7 +1756,7 @@ PREFIX(updatePosition)(const ENCODING *enc, case BT_CR: pos->lineNumber++; ptr += MINBPC(enc); - if (ptr != end && BYTE_TYPE(enc, ptr) == BT_LF) + if (HAS_CHAR(enc, ptr, end) && BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); pos->columnNumber = (XML_Size)-1; break; diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 1c1e4fb..2efa998 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -55,6 +55,9 @@ static struct { int fd; int all_threads; PyInterpreterState *interp; +#ifdef MS_WINDOWS + void *exc_handler; +#endif } fatal_error = {0, NULL, -1, 0}; #ifdef FAULTHANDLER_LATER @@ -124,6 +127,7 @@ static const size_t faulthandler_nsignals = \ #ifdef HAVE_SIGALTSTACK static stack_t stack; +static stack_t old_stack; #endif @@ -369,8 +373,8 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) DWORD code = exc_info->ExceptionRecord->ExceptionCode; DWORD flags = exc_info->ExceptionRecord->ExceptionFlags; - /* only log fatal exceptions */ - if (flags & EXCEPTION_NONCONTINUABLE) { + /* bpo-30557: only log fatal exceptions */ + if (!(code & 0x80000000)) { /* call the next exception handler */ return EXCEPTION_CONTINUE_SEARCH; } @@ -387,8 +391,8 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) case EXCEPTION_IN_PAGE_ERROR: PUTS(fd, "page error"); break; case EXCEPTION_STACK_OVERFLOW: PUTS(fd, "stack overflow"); break; default: - PUTS(fd, "code "); - _Py_DumpDecimal(fd, code); + PUTS(fd, "code 0x"); + _Py_DumpHexadecimal(fd, code, 8); } PUTS(fd, "\n\n"); @@ -461,7 +465,8 @@ faulthandler_enable(void) } #ifdef MS_WINDOWS - AddVectoredExceptionHandler(1, faulthandler_exc_handler); + assert(fatal_error.exc_handler == NULL); + fatal_error.exc_handler = AddVectoredExceptionHandler(1, faulthandler_exc_handler); #endif return 0; } @@ -513,7 +518,12 @@ faulthandler_disable(void) faulthandler_disable_fatal_handler(handler); } } - +#ifdef MS_WINDOWS + if (fatal_error.exc_handler != NULL) { + RemoveVectoredExceptionHandler(fatal_error.exc_handler); + fatal_error.exc_handler = NULL; + } +#endif Py_CLEAR(fatal_error.file); } @@ -1310,7 +1320,7 @@ int _PyFaulthandler_Init(void) stack.ss_size = SIGSTKSZ; stack.ss_sp = PyMem_Malloc(stack.ss_size); if (stack.ss_sp != NULL) { - err = sigaltstack(&stack, NULL); + err = sigaltstack(&stack, &old_stack); if (err) { PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; @@ -1366,6 +1376,20 @@ void _PyFaulthandler_Fini(void) faulthandler_disable(); #ifdef HAVE_SIGALTSTACK if (stack.ss_sp != NULL) { + /* Fetch the current alt stack */ + stack_t current_stack; + if (sigaltstack(NULL, ¤t_stack) == 0) { + if (current_stack.ss_sp == stack.ss_sp) { + /* The current alt stack is the one that we installed. + It is safe to restore the old stack that we found when + we installed ours */ + sigaltstack(&old_stack, NULL); + } else { + /* Someone switched to a different alt stack and didn't + restore ours when they were done (if they're done). + There's not much we can do in this unlikely case */ + } + } PyMem_Free(stack.ss_sp); stack.ss_sp = NULL; } diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 6bf04cb..8e22ec9 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1864,33 +1864,37 @@ chain_next(chainobject *lz) { PyObject *item; - if (lz->source == NULL) - return NULL; /* already stopped */ - - if (lz->active == NULL) { - PyObject *iterable = PyIter_Next(lz->source); - if (iterable == NULL) { - Py_CLEAR(lz->source); - return NULL; /* no more input sources */ - } - lz->active = PyObject_GetIter(iterable); - Py_DECREF(iterable); + /* lz->source is the iterator of iterables. If it's NULL, we've already + * consumed them all. lz->active is the current iterator. If it's NULL, + * we should grab a new one from lz->source. */ + while (lz->source != NULL) { if (lz->active == NULL) { - Py_CLEAR(lz->source); - return NULL; /* input not iterable */ + PyObject *iterable = PyIter_Next(lz->source); + if (iterable == NULL) { + Py_CLEAR(lz->source); + return NULL; /* no more input sources */ + } + lz->active = PyObject_GetIter(iterable); + Py_DECREF(iterable); + if (lz->active == NULL) { + Py_CLEAR(lz->source); + return NULL; /* input not iterable */ + } } + item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); + if (item != NULL) + return item; + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_StopIteration)) + PyErr_Clear(); + else + return NULL; /* input raised an exception */ + } + /* lz->active is consumed, try with the next iterable. */ + Py_CLEAR(lz->active); } - item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); - if (item != NULL) - return item; - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else - return NULL; /* input raised an exception */ - } - Py_CLEAR(lz->active); - return chain_next(lz); /* recurse and use next active */ + /* Everything had been consumed already. */ + return NULL; } static PyObject * diff --git a/Modules/main.c b/Modules/main.c index dd50211..475a2fd 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -53,7 +53,7 @@ static const char usage_1[] = "\ Options and arguments (and corresponding environment variables):\n\ -b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ and comparing bytes/bytearray with str. (-bb: issue errors)\n\ --B : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x\n\ +-B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ -c cmd : program passed in as string (terminates option list)\n\ -d : debug output from parser; also PYTHONDEBUG=x\n\ -E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 5f1615f..426b7ca 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -806,10 +806,10 @@ mmap_subscript(mmap_object *self, PyObject *item) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, self->size, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (slicelen <= 0) return PyBytes_FromStringAndSize("", 0); @@ -932,11 +932,10 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) Py_ssize_t start, stop, step, slicelen; Py_buffer vbuf; - if (PySlice_GetIndicesEx(item, - self->size, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "mmap object doesn't support slice deletion"); diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index b256695..a444335 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -698,7 +698,7 @@ validate_node(node *tree) short a_label = dfa_state->s_arc[arc].a_lbl; assert(a_label < _PyParser_Grammar.g_ll.ll_nlabels); if (_PyParser_Grammar.g_ll.ll_label[a_label].lb_type == ch_type) { - /* The child is acceptable; if non-terminal, validate it recursively. */ + /* The child is acceptable; if non-terminal, validate it recursively. */ if (ISNONTERMINAL(ch_type) && !validate_node(ch)) return 0; @@ -776,32 +776,35 @@ parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw) */ tree = build_node_tree(tuple); if (tree != 0) { - node *validation_root = tree; + node *validation_root = NULL; int tree_type = 0; switch (TYPE(tree)) { case eval_input: /* Might be an eval form. */ tree_type = PyST_EXPR; + validation_root = tree; break; case encoding_decl: /* This looks like an encoding_decl so far. */ - if (NCH(tree) != 1) + if (NCH(tree) == 1) { + tree_type = PyST_SUITE; + validation_root = CHILD(tree, 0); + } + else { err_string("Error Parsing encoding_decl"); - validation_root = CHILD(tree, 0); - /* Fall through */ + } + break; case file_input: /* This looks like an exec form so far. */ - tree_type = PyST_SUITE; + validation_root = tree; break; default: /* This is a fragment, at best. */ - PyNode_Free(tree); err_string("parse tree does not use a valid start symbol"); - return (0); } - if (validate_node(validation_root)) + if (validation_root != NULL && validate_node(validation_root)) st = parser_newstobject(tree, tree_type); else PyNode_Free(tree); @@ -831,6 +834,9 @@ build_node_children(PyObject *tuple, node *root, int *line_num) Py_ssize_t i; int err; + if (len < 0) { + return NULL; + } for (i = 1; i < len; ++i) { /* elem must always be a sequence, however simple */ PyObject* elem = PySequence_GetItem(tuple, i); @@ -851,7 +857,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num) if (type == -1 && PyErr_Occurred()) { Py_DECREF(temp); Py_DECREF(elem); - return 0; + return NULL; } } Py_DECREF(temp); @@ -863,7 +869,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num) PyErr_SetObject(parser_error, err); Py_XDECREF(err); Py_XDECREF(elem); - return (0); + return NULL; } if (ISTERMINAL(type)) { Py_ssize_t len = PyObject_Size(elem); @@ -872,11 +878,14 @@ build_node_children(PyObject *tuple, node *root, int *line_num) if ((len != 2) && (len != 3)) { err_string("terminal nodes must have 2 or 3 entries"); - return 0; + Py_DECREF(elem); + return NULL; } temp = PySequence_GetItem(elem, 1); - if (temp == NULL) - return 0; + if (temp == NULL) { + Py_DECREF(elem); + return NULL; + } if (!PyUnicode_Check(temp)) { PyErr_Format(parser_error, "second item in terminal node must be a string," @@ -884,46 +893,49 @@ build_node_children(PyObject *tuple, node *root, int *line_num) Py_TYPE(temp)->tp_name); Py_DECREF(temp); Py_DECREF(elem); - return 0; + return NULL; } if (len == 3) { PyObject *o = PySequence_GetItem(elem, 2); - if (o != NULL) { - if (PyLong_Check(o)) { - int num = _PyLong_AsInt(o); - if (num == -1 && PyErr_Occurred()) { - Py_DECREF(o); - Py_DECREF(temp); - Py_DECREF(elem); - return 0; - } - *line_num = num; - } - else { - PyErr_Format(parser_error, - "third item in terminal node must be an" - " integer, found %s", - Py_TYPE(temp)->tp_name); + if (o == NULL) { + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; + } + if (PyLong_Check(o)) { + int num = _PyLong_AsInt(o); + if (num == -1 && PyErr_Occurred()) { Py_DECREF(o); Py_DECREF(temp); Py_DECREF(elem); - return 0; + return NULL; } + *line_num = num; + } + else { + PyErr_Format(parser_error, + "third item in terminal node must be an" + " integer, found %s", + Py_TYPE(temp)->tp_name); Py_DECREF(o); + Py_DECREF(temp); + Py_DECREF(elem); + return NULL; } + Py_DECREF(o); } temp_str = PyUnicode_AsUTF8AndSize(temp, &len); if (temp_str == NULL) { Py_DECREF(temp); - Py_XDECREF(elem); - return 0; + Py_DECREF(elem); + return NULL; } strn = (char *)PyObject_MALLOC(len + 1); if (strn == NULL) { Py_DECREF(temp); - Py_XDECREF(elem); + Py_DECREF(elem); PyErr_NoMemory(); - return 0; + return NULL; } (void) memcpy(strn, temp_str, len + 1); Py_DECREF(temp); @@ -933,20 +945,21 @@ build_node_children(PyObject *tuple, node *root, int *line_num) * It has to be one or the other; this is an error. * Raise an exception. */ - PyObject *err = Py_BuildValue("os", elem, "unknown node type."); + PyObject *err = Py_BuildValue("Os", elem, "unknown node type."); PyErr_SetObject(parser_error, err); Py_XDECREF(err); - Py_XDECREF(elem); - return (0); + Py_DECREF(elem); + return NULL; } err = PyNode_AddChild(root, type, strn, *line_num, 0); if (err == E_NOMEM) { - Py_XDECREF(elem); + Py_DECREF(elem); PyObject_FREE(strn); - return (node *) PyErr_NoMemory(); + PyErr_NoMemory(); + return NULL; } if (err == E_OVERFLOW) { - Py_XDECREF(elem); + Py_DECREF(elem); PyObject_FREE(strn); PyErr_SetString(PyExc_ValueError, "unsupported number of child nodes"); @@ -957,14 +970,14 @@ build_node_children(PyObject *tuple, node *root, int *line_num) node* new_child = CHILD(root, i - 1); if (new_child != build_node_children(elem, new_child, line_num)) { - Py_XDECREF(elem); - return (0); + Py_DECREF(elem); + return NULL; } } else if (type == NEWLINE) { /* It's true: we increment the */ ++(*line_num); /* line number *after* the newline! */ } - Py_XDECREF(elem); + Py_DECREF(elem); } return root; } @@ -999,10 +1012,23 @@ build_node_tree(PyObject *tuple) if (num == encoding_decl) { encoding = PySequence_GetItem(tuple, 2); + if (encoding == NULL) { + PyErr_SetString(parser_error, "missed encoding"); + return NULL; + } + if (!PyUnicode_Check(encoding)) { + PyErr_Format(parser_error, + "encoding must be a string, found %.200s", + Py_TYPE(encoding)->tp_name); + Py_DECREF(encoding); + return NULL; + } /* tuple isn't borrowed anymore here, need to DECREF */ tuple = PySequence_GetSlice(tuple, 0, 2); - if (tuple == NULL) + if (tuple == NULL) { + Py_DECREF(encoding); return NULL; + } } res = PyNode_New(num); if (res != NULL) { @@ -1015,31 +1041,33 @@ build_node_tree(PyObject *tuple) const char *temp; temp = PyUnicode_AsUTF8AndSize(encoding, &len); if (temp == NULL) { - Py_DECREF(res); + PyNode_Free(res); Py_DECREF(encoding); Py_DECREF(tuple); return NULL; } res->n_str = (char *)PyObject_MALLOC(len + 1); if (res->n_str == NULL) { - Py_DECREF(res); + PyNode_Free(res); Py_DECREF(encoding); Py_DECREF(tuple); PyErr_NoMemory(); return NULL; } (void) memcpy(res->n_str, temp, len + 1); - Py_DECREF(encoding); - Py_DECREF(tuple); } } + if (encoding != NULL) { + Py_DECREF(encoding); + Py_DECREF(tuple); + } } else { /* The tuple is illegal -- if the number is neither TERMINAL nor * NONTERMINAL, we can't use it. Not sure the implementation * allows this condition, but the API doesn't preclude it. */ - PyObject *err = Py_BuildValue("os", tuple, + PyObject *err = Py_BuildValue("Os", tuple, "Illegal component tuple."); PyErr_SetObject(parser_error, err); Py_XDECREF(err); @@ -1074,7 +1102,6 @@ parser__pickler(PyObject *self, PyObject *args) result = Py_BuildValue("O(O)", pickle_constructor, tuple); Py_DECREF(tuple); } - Py_DECREF(empty_dict); Py_DECREF(newargs); } finally: diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8f8ba25..0a9123b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1932,11 +1932,13 @@ _pystat_fromstructstat(STRUCT_STAT *st) return NULL; PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long)st->st_mode)); -#ifdef HAVE_LARGEFILE_SUPPORT +#if defined(HAVE_LARGEFILE_SUPPORT) || defined(MS_WINDOWS) + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(st->st_ino)); PyStructSequence_SET_ITEM(v, 1, - PyLong_FromLongLong((long long)st->st_ino)); + PyLong_FromUnsignedLongLong(st->st_ino)); #else - PyStructSequence_SET_ITEM(v, 1, PyLong_FromLong((long)st->st_ino)); + Py_BUILD_ASSERT(sizeof(unsigned long) >= sizeof(st->st_ino)); + PyStructSequence_SET_ITEM(v, 1, PyLong_FromUnsignedLong(st->st_ino)); #endif #ifdef MS_WINDOWS PyStructSequence_SET_ITEM(v, 2, PyLong_FromUnsignedLong(st->st_dev)); @@ -6648,7 +6650,7 @@ static PyObject * os_setgroups(PyObject *module, PyObject *groups) /*[clinic end generated code: output=3fcb32aad58c5ecd input=fa742ca3daf85a7e]*/ { - int i, len; + Py_ssize_t i, len; gid_t grouplist[MAX_GROUPS]; if (!PySequence_Check(groups)) { @@ -6656,6 +6658,9 @@ os_setgroups(PyObject *module, PyObject *groups) return NULL; } len = PySequence_Size(groups); + if (len < 0) { + return NULL; + } if (len > MAX_GROUPS) { PyErr_SetString(PyExc_ValueError, "too many groups"); return NULL; @@ -7884,9 +7889,9 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length) #if (defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__) \ || defined(__APPLE__))) || defined(HAVE_READV) || defined(HAVE_WRITEV) static Py_ssize_t -iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, int cnt, int type) +iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, Py_ssize_t cnt, int type) { - int i, j; + Py_ssize_t i, j; Py_ssize_t blen, total = 0; *iov = PyMem_New(struct iovec, cnt); @@ -7963,8 +7968,7 @@ static Py_ssize_t os_readv_impl(PyObject *module, int fd, PyObject *buffers) /*[clinic end generated code: output=792da062d3fcebdb input=e679eb5dbfa0357d]*/ { - int cnt; - Py_ssize_t n; + Py_ssize_t cnt, n; int async_err = 0; struct iovec *iov; Py_buffer *buf; @@ -7976,6 +7980,8 @@ os_readv_impl(PyObject *module, int fd, PyObject *buffers) } cnt = PySequence_Size(buffers); + if (cnt < 0) + return -1; if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_WRITABLE) < 0) return -1; @@ -8114,15 +8120,24 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) "sendfile() headers must be a sequence"); return NULL; } else { - Py_ssize_t i = 0; /* Avoid uninitialized warning */ - sf.hdr_cnt = PySequence_Size(headers); - if (sf.hdr_cnt > 0 && - (i = iov_setup(&(sf.headers), &hbuf, - headers, sf.hdr_cnt, PyBUF_SIMPLE)) < 0) + Py_ssize_t i = PySequence_Size(headers); + if (i < 0) + return NULL; + if (i > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "sendfile() header is too large"); return NULL; + } + if (i > 0) { + sf.hdr_cnt = (int)i; + i = iov_setup(&(sf.headers), &hbuf, + headers, sf.hdr_cnt, PyBUF_SIMPLE); + if (i < 0) + return NULL; #ifdef __APPLE__ - sbytes += i; + sbytes += i; #endif + } } } if (trailers != NULL) { @@ -8131,15 +8146,24 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) "sendfile() trailers must be a sequence"); return NULL; } else { - Py_ssize_t i = 0; /* Avoid uninitialized warning */ - sf.trl_cnt = PySequence_Size(trailers); - if (sf.trl_cnt > 0 && - (i = iov_setup(&(sf.trailers), &tbuf, - trailers, sf.trl_cnt, PyBUF_SIMPLE)) < 0) + Py_ssize_t i = PySequence_Size(trailers); + if (i < 0) + return NULL; + if (i > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "sendfile() trailer is too large"); return NULL; + } + if (i > 0) { + sf.trl_cnt = (int)i; + i = iov_setup(&(sf.trailers), &tbuf, + trailers, sf.trl_cnt, PyBUF_SIMPLE); + if (i < 0) + return NULL; #ifdef __APPLE__ - sbytes += i; + sbytes += i; #endif + } } } @@ -8409,7 +8433,7 @@ static Py_ssize_t os_writev_impl(PyObject *module, int fd, PyObject *buffers) /*[clinic end generated code: output=56565cfac3aac15b input=5b8d17fe4189d2fe]*/ { - int cnt; + Py_ssize_t cnt; Py_ssize_t result; int async_err = 0; struct iovec *iov; @@ -8421,6 +8445,8 @@ os_writev_impl(PyObject *module, int fd, PyObject *buffers) return -1; } cnt = PySequence_Size(buffers); + if (cnt < 0) + return -1; if (iov_setup(&iov, &buf, buffers, cnt, PyBUF_SIMPLE) < 0) { return -1; @@ -11156,7 +11182,7 @@ typedef struct { PyObject *lstat; #ifdef MS_WINDOWS struct _Py_stat_struct win32_lstat; - __int64 win32_file_index; + uint64_t win32_file_index; int got_file_index; #else /* POSIX */ #ifdef HAVE_DIRENT_D_TYPE @@ -11419,7 +11445,8 @@ DirEntry_inode(DirEntry *self) self->win32_file_index = stat.st_ino; self->got_file_index = 1; } - return PyLong_FromLongLong((long long)self->win32_file_index); + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(self->win32_file_index)); + return PyLong_FromUnsignedLongLong(self->win32_file_index); #else /* POSIX */ #ifdef HAVE_LARGEFILE_SUPPORT return PyLong_FromLongLong((long long)self->d_ino); @@ -12729,7 +12756,7 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, SCHED_RR)) return -1; #endif #ifdef SCHED_SPORADIC - if (PyModule_AddIntMacro(m, SCHED_SPORADIC) return -1; + if (PyModule_AddIntMacro(m, SCHED_SPORADIC)) return -1; #endif #ifdef SCHED_BATCH if (PyModule_AddIntMacro(m, SCHED_BATCH)) return -1; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index b3259d5..47c3e86 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1191,10 +1191,8 @@ newxmlparseobject(const char *encoding, const char *namespace_separator, PyObjec Py_DECREF(self); return NULL; } -#if ((XML_MAJOR_VERSION >= 2) && (XML_MINOR_VERSION >= 1)) || defined(XML_HAS_SET_HASH_SALT) - /* This feature was added upstream in libexpat 2.1.0. Our expat copy - * has a backport of this feature where we also define XML_HAS_SET_HASH_SALT - * to indicate that we can still use it. */ +#if XML_COMBINED_VERSION >= 20100 + /* This feature was added upstream in libexpat 2.1.0. */ XML_SetHashSalt(self->itself, (unsigned long)_Py_HashSecret.expat.hashsalt); #endif diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index e27075b..ed2f4e8 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -244,6 +244,32 @@ trip_signal(int sig_num) Handlers[sig_num].tripped = 1; + if (!is_tripped) { + /* Set is_tripped after setting .tripped, as it gets + cleared in PyErr_CheckSignals() before .tripped. */ + is_tripped = 1; + Py_AddPendingCall(checksignals_witharg, NULL); + } + + /* And then write to the wakeup fd *after* setting all the globals and + doing the Py_AddPendingCall. We used to write to the wakeup fd and then + set the flag, but this allowed the following sequence of events + (especially on windows, where trip_signal runs in a new thread): + + - main thread blocks on select([wakeup_fd], ...) + - signal arrives + - trip_signal writes to the wakeup fd + - the main thread wakes up + - the main thread checks the signal flags, sees that they're unset + - the main thread empties the wakeup fd + - the main thread goes back to sleep + - trip_signal sets the flags to request the Python-level signal handler + be run + - the main thread doesn't notice, because it's asleep + + See bpo-30038 for more details. + */ + #ifdef MS_WINDOWS fd = Py_SAFE_DOWNCAST(wakeup.fd, SOCKET_T, int); #else @@ -281,13 +307,6 @@ trip_signal(int sig_num) } } } - - if (!is_tripped) { - /* Set is_tripped after setting .tripped, as it gets - cleared in PyErr_CheckSignals() before .tripped. */ - is_tripped = 1; - Py_AddPendingCall(checksignals_witharg, NULL); - } } static void diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f4edc06..42aec59 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -288,36 +288,6 @@ http://cvsweb.netbsd.org/bsdweb.cgi/src/lib/libc/net/getaddrinfo.c.diff?r1=1.82& #include #endif -#ifdef HAVE_SOCKADDR_ALG -#include -#ifndef AF_ALG -#define AF_ALG 38 -#endif -#ifndef SOL_ALG -#define SOL_ALG 279 -#endif - -/* Linux 3.19 */ -#ifndef ALG_SET_AEAD_ASSOCLEN -#define ALG_SET_AEAD_ASSOCLEN 4 -#endif -#ifndef ALG_SET_AEAD_AUTHSIZE -#define ALG_SET_AEAD_AUTHSIZE 5 -#endif -/* Linux 4.8 */ -#ifndef ALG_SET_PUBKEY -#define ALG_SET_PUBKEY 6 -#endif - -#ifndef ALG_OP_SIGN -#define ALG_OP_SIGN 2 -#endif -#ifndef ALG_OP_VERIFY -#define ALG_OP_VERIFY 3 -#endif - -#endif /* HAVE_SOCKADDR_ALG */ - /* Generic socket object definitions and includes */ #define PySocket_BUILDING_SOCKET #include "socketmodule.h" @@ -1242,9 +1212,9 @@ makesockaddr(SOCKET_T sockfd, struct sockaddr *addr, size_t addrlen, int proto) { struct sockaddr_un *a = (struct sockaddr_un *) addr; #ifdef __linux__ - if (a->sun_path[0] == 0) { /* Linux abstract namespace */ - addrlen -= offsetof(struct sockaddr_un, sun_path); - return PyBytes_FromStringAndSize(a->sun_path, addrlen); + size_t linuxaddrlen = addrlen - offsetof(struct sockaddr_un, sun_path); + if (linuxaddrlen > 0 && a->sun_path[0] == 0) { /* Linux abstract namespace */ + return PyBytes_FromStringAndSize(a->sun_path, linuxaddrlen); } else #endif /* linux */ diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 3cce927..03f982b 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -98,6 +98,37 @@ typedef int socklen_t; #include #endif +#ifdef HAVE_SOCKADDR_ALG +#include +#ifndef AF_ALG +#define AF_ALG 38 +#endif +#ifndef SOL_ALG +#define SOL_ALG 279 +#endif + +/* Linux 3.19 */ +#ifndef ALG_SET_AEAD_ASSOCLEN +#define ALG_SET_AEAD_ASSOCLEN 4 +#endif +#ifndef ALG_SET_AEAD_AUTHSIZE +#define ALG_SET_AEAD_AUTHSIZE 5 +#endif +/* Linux 4.8 */ +#ifndef ALG_SET_PUBKEY +#define ALG_SET_PUBKEY 6 +#endif + +#ifndef ALG_OP_SIGN +#define ALG_OP_SIGN 2 +#endif +#ifndef ALG_OP_VERIFY +#define ALG_OP_VERIFY 3 +#endif + +#endif /* HAVE_SOCKADDR_ALG */ + + #ifndef Py__SOCKET_H #define Py__SOCKET_H #ifdef __cplusplus @@ -159,6 +190,9 @@ typedef union sock_addr { #ifdef HAVE_SYS_KERN_CONTROL_H struct sockaddr_ctl ctl; #endif +#ifdef HAVE_SOCKADDR_ALG + struct sockaddr_alg alg; +#endif } sock_addr_t; /* The object holding a socket. It holds some extra information, diff --git a/Modules/timemodule.c b/Modules/timemodule.c index ebd44ad..328b84f 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -279,7 +279,7 @@ static PyTypeObject StructTimeType; static PyObject * tmtotuple(struct tm *p #ifndef HAVE_STRUCT_TM_TM_ZONE - , const char *zone, int gmtoff + , const char *zone, time_t gmtoff #endif ) { @@ -305,7 +305,7 @@ tmtotuple(struct tm *p #else PyStructSequence_SET_ITEM(v, 9, PyUnicode_DecodeLocale(zone, "surrogateescape")); - SET(10, gmtoff); + PyStructSequence_SET_ITEM(v, 10, _PyLong_FromTime_t(gmtoff)); #endif /* HAVE_STRUCT_TM_TM_ZONE */ #undef SET if (PyErr_Occurred()) { @@ -397,7 +397,7 @@ time_localtime(PyObject *self, PyObject *args) { struct tm local = buf; char zone[100]; - int gmtoff; + time_t gmtoff; strftime(zone, sizeof(zone), "%Z", &buf); gmtoff = timegm(&buf) - when; return tmtotuple(&local, zone, gmtoff); diff --git a/Modules/zipimport.c b/Modules/zipimport.c index 59046aa..cccc033 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -1102,7 +1102,7 @@ get_decompress_func(void) _Py_IDENTIFIER(decompress); if (importing_zlib != 0) - /* Someone has a zlib.py[co] in their Zip file; + /* Someone has a zlib.pyc in their Zip file; let's avoid a stack overflow. */ return NULL; importing_zlib = 1; @@ -1261,7 +1261,7 @@ eq_mtime(time_t t1, time_t t2) return d <= 1; } -/* Given the contents of a .py[co] file in a buffer, unmarshal the data +/* Given the contents of a .pyc file in a buffer, unmarshal the data and return the code object. Return None if it the magic word doesn't match (we do this instead of raising an exception as we fall back to .py if available and we don't want to mask other errors). @@ -1403,7 +1403,7 @@ get_mtime_of_source(ZipImporter *self, PyObject *path) PyObject *toc_entry, *stripped; time_t mtime; - /* strip 'c' or 'o' from *.py[co] */ + /* strip 'c' from *.pyc */ if (PyUnicode_READY(path) == -1) return (time_t)-1; stripped = PyUnicode_FromKindAndData(PyUnicode_KIND(path), diff --git a/Objects/abstract.c b/Objects/abstract.c index d838856..38de774 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2389,9 +2389,9 @@ _PyStack_AsDict(PyObject **values, PyObject *kwnames) return kwdict; } -PyObject ** +int _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, - PyObject **p_kwnames, PyObject *func) + PyObject ***p_stack, PyObject **p_kwnames) { PyObject **stack, **kwstack; Py_ssize_t nkwargs; @@ -2402,27 +2402,27 @@ _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, assert(nargs >= 0); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - nkwargs = (kwargs != NULL) ? PyDict_Size(kwargs) : 0; - if (!nkwargs) { + if (kwargs == NULL || (nkwargs = PyDict_Size(kwargs)) == 0) { + *p_stack = args; *p_kwnames = NULL; - return args; + return 0; } if ((size_t)nargs > PY_SSIZE_T_MAX / sizeof(stack[0]) - (size_t)nkwargs) { PyErr_NoMemory(); - return NULL; + return -1; } stack = PyMem_Malloc((nargs + nkwargs) * sizeof(stack[0])); if (stack == NULL) { PyErr_NoMemory(); - return NULL; + return -1; } kwnames = PyTuple_New(nkwargs); if (kwnames == NULL) { PyMem_Free(stack); - return NULL; + return -1; } /* Copy position arguments (borrowed references) */ @@ -2441,8 +2441,9 @@ _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, i++; } + *p_stack = stack; *p_kwnames = kwnames; - return stack; + return 0; } PyObject * @@ -3192,8 +3193,8 @@ _PySequence_BytesToCharpArray(PyObject* self) array[i] = NULL; goto fail; } - data = PyBytes_AsString(item); - if (data == NULL) { + /* check for embedded null bytes */ + if (PyBytes_AsStringAndSize(item, &data, NULL) < 0) { /* NULL terminate before freeing. */ array[i] = NULL; goto fail; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a8d6980..a9c8ca6 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -254,7 +254,7 @@ PyByteArray_Concat(PyObject *a, PyObject *b) if (PyObject_GetBuffer(a, &va, PyBUF_SIMPLE) != 0 || PyObject_GetBuffer(b, &vb, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + Py_TYPE(b)->tp_name, Py_TYPE(a)->tp_name); goto done; } @@ -400,11 +400,11 @@ bytearray_subscript(PyByteArrayObject *self, PyObject *index) } else if (PySlice_Check(index)) { Py_ssize_t start, stop, step, slicelength, cur, i; - if (PySlice_GetIndicesEx(index, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(index, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyByteArray_GET_SIZE(self), + &start, &stop, step); if (slicelength <= 0) return PyByteArray_FromStringAndSize("", 0); @@ -630,11 +630,11 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu } } else if (PySlice_Check(index)) { - if (PySlice_GetIndicesEx(index, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_Unpack(index, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(PyByteArray_GET_SIZE(self), &start, + &stop, step); } else { PyErr_Format(PyExc_TypeError, diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index d5c4fe6..625e242 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -542,7 +542,11 @@ _Py_bytes_find(const char *str, Py_ssize_t len, PyObject *args) PyDoc_STRVAR_shared(_Py_index__doc__, "B.index(sub[, start[, end]]) -> int\n\ \n\ -Like B.find() but raise ValueError when the subsection is not found."); +Return the lowest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the subsection is not found."); PyObject * _Py_bytes_index(const char *str, Py_ssize_t len, PyObject *args) @@ -579,7 +583,11 @@ _Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args) PyDoc_STRVAR_shared(_Py_rindex__doc__, "B.rindex(sub[, start[, end]]) -> int\n\ \n\ -Like B.rfind() but raise ValueError when the subsection is not found."); +Return the highest index in B where subsection sub is found,\n\ +such that sub is contained within B[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raise ValueError when the subsection is not found."); PyObject * _Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args) @@ -811,4 +819,3 @@ PyDoc_STRVAR_shared(_Py_zfill__doc__, "\n" "Pad a numeric string B with zeros on the left, to fill a field\n" "of the specified width. B is never truncated."); - diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 5d48440..4c55294 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -528,6 +528,8 @@ byte_converter(PyObject *arg, char *p) return 0; } +static PyObject *_PyBytes_FromBuffer(PyObject *x); + static PyObject * format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) { @@ -564,8 +566,19 @@ format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) *plen = PyBytes_GET_SIZE(result); return result; } + /* does it support buffer protocol? */ + if (PyObject_CheckBuffer(v)) { + /* maybe we can avoid making a copy of the buffer object here? */ + result = _PyBytes_FromBuffer(v); + if (result == NULL) + return NULL; + *pbuf = PyBytes_AS_STRING(result); + *plen = PyBytes_GET_SIZE(result); + return result; + } PyErr_Format(PyExc_TypeError, - "%%b requires bytes, or an object that implements __bytes__, not '%.100s'", + "%%b requires a bytes-like object, " + "or an object that implements __bytes__, not '%.100s'", Py_TYPE(v)->tp_name); return NULL; } @@ -619,11 +632,11 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, Py_ssize_t len; char *pos; - pos = strchr(fmt + 1, '%'); + pos = (char *)memchr(fmt + 1, '%', fmtcnt); if (pos != NULL) len = pos - fmt; else - len = format_len - (fmt - format); + len = fmtcnt + 1; assert(len != 0); memcpy(res, fmt, len); @@ -1425,7 +1438,7 @@ bytes_concat(PyObject *a, PyObject *b) if (PyObject_GetBuffer(a, &va, PyBUF_SIMPLE) != 0 || PyObject_GetBuffer(b, &vb, PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + Py_TYPE(b)->tp_name, Py_TYPE(a)->tp_name); goto done; } @@ -1670,11 +1683,11 @@ bytes_subscript(PyBytesObject* self, PyObject* item) char* result_buf; PyObject* result; - if (PySlice_GetIndicesEx(item, - PyBytes_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyBytes_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { return PyBytes_FromStringAndSize("", 0); diff --git a/Objects/codeobject.c b/Objects/codeobject.c index df8b953..6de697a 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -411,18 +411,19 @@ static void code_dealloc(PyCodeObject *co) { if (co->co_extra != NULL) { - PyThreadState *tstate = PyThreadState_Get(); + __PyCodeExtraState *state = __PyCodeExtraState_Get(); _PyCodeObjectExtra *co_extra = co->co_extra; for (Py_ssize_t i = 0; i < co_extra->ce_size; i++) { - freefunc free_extra = tstate->co_extra_freefuncs[i]; + freefunc free_extra = state->co_extra_freefuncs[i]; if (free_extra != NULL) { free_extra(co_extra->ce_extras[i]); } } - PyMem_FREE(co->co_extra); + PyMem_Free(co_extra->ce_extras); + PyMem_Free(co_extra); } Py_XDECREF(co->co_code); @@ -446,11 +447,15 @@ code_dealloc(PyCodeObject *co) static PyObject * code_sizeof(PyCodeObject *co, void *unused) { - Py_ssize_t res; + Py_ssize_t res = _PyObject_SIZE(Py_TYPE(co)); + _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra; - res = _PyObject_SIZE(Py_TYPE(co)); if (co->co_cell2arg != NULL && co->co_cellvars != NULL) - res += PyTuple_GET_SIZE(co->co_cellvars) * sizeof(unsigned char); + res += PyTuple_GET_SIZE(co->co_cellvars) * sizeof(Py_ssize_t); + + if (co_extra != NULL) + res += co_extra->ce_size * sizeof(co_extra->ce_extras[0]); + return PyLong_FromSsize_t(res); } @@ -821,8 +826,6 @@ _PyCode_CheckLineNumber(PyCodeObject* co, int lasti, PyAddrPair *bounds) int _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) { - assert(*extra == NULL); - if (!PyCode_Check(code)) { PyErr_BadInternalCall(); return -1; @@ -833,6 +836,7 @@ _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) if (co_extra == NULL || co_extra->ce_size <= index) { + *extra = NULL; return 0; } @@ -844,10 +848,10 @@ _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) int _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) { - PyThreadState *tstate = PyThreadState_Get(); + __PyCodeExtraState *state = __PyCodeExtraState_Get(); if (!PyCode_Check(code) || index < 0 || - index >= tstate->co_extra_user_count) { + index >= state->co_extra_user_count) { PyErr_BadInternalCall(); return -1; } @@ -862,13 +866,13 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) } co_extra->ce_extras = PyMem_Malloc( - tstate->co_extra_user_count * sizeof(void*)); + state->co_extra_user_count * sizeof(void*)); if (co_extra->ce_extras == NULL) { PyMem_Free(co_extra); return -1; } - co_extra->ce_size = tstate->co_extra_user_count; + co_extra->ce_size = state->co_extra_user_count; for (Py_ssize_t i = 0; i < co_extra->ce_size; i++) { co_extra->ce_extras[i] = NULL; @@ -878,20 +882,27 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) } else if (co_extra->ce_size <= index) { void** ce_extras = PyMem_Realloc( - co_extra->ce_extras, tstate->co_extra_user_count * sizeof(void*)); + co_extra->ce_extras, state->co_extra_user_count * sizeof(void*)); if (ce_extras == NULL) { return -1; } for (Py_ssize_t i = co_extra->ce_size; - i < tstate->co_extra_user_count; + i < state->co_extra_user_count; i++) { ce_extras[i] = NULL; } co_extra->ce_extras = ce_extras; - co_extra->ce_size = tstate->co_extra_user_count; + co_extra->ce_size = state->co_extra_user_count; + } + + if (co_extra->ce_extras[index] != NULL) { + freefunc free = state->co_extra_freefuncs[index]; + if (free != NULL) { + free(co_extra->ce_extras[index]); + } } co_extra->ce_extras[index] = extra; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a0c1977..b0f583a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -436,7 +436,7 @@ static PyObject *empty_values[1] = { NULL }; /* #define DEBUG_PYDICT */ -#ifdef Py_DEBUG +#ifndef NDEBUG static int _PyDict_CheckConsistency(PyDictObject *mp) { @@ -1115,18 +1115,18 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) PyDictKeyEntry *ep, *ep0; Py_ssize_t hashpos, ix; + Py_INCREF(key); + Py_INCREF(value); if (mp->ma_values != NULL && !PyUnicode_CheckExact(key)) { if (insertion_resize(mp) < 0) - return -1; + goto Fail; } ix = mp->ma_keys->dk_lookup(mp, key, hash, &value_addr, &hashpos); - if (ix == DKIX_ERROR) { - return -1; - } + if (ix == DKIX_ERROR) + goto Fail; assert(PyUnicode_CheckExact(key) || mp->ma_keys->dk_lookup == lookdict); - Py_INCREF(value); MAINTAIN_TRACKING(mp, key, value); /* When insertion order is different from shared key, we can't share @@ -1135,10 +1135,8 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) if (_PyDict_HasSplitTable(mp) && ((ix >= 0 && *value_addr == NULL && mp->ma_used != ix) || (ix == DKIX_EMPTY && mp->ma_used != mp->ma_keys->dk_nentries))) { - if (insertion_resize(mp) < 0) { - Py_DECREF(value); - return -1; - } + if (insertion_resize(mp) < 0) + goto Fail; find_empty_slot(mp, key, hash, &value_addr, &hashpos); ix = DKIX_EMPTY; } @@ -1147,16 +1145,13 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) /* Insert into new slot. */ if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ - if (insertion_resize(mp) < 0) { - Py_DECREF(value); - return -1; - } + if (insertion_resize(mp) < 0) + goto Fail; find_empty_slot(mp, key, hash, &value_addr, &hashpos); } ep0 = DK_ENTRIES(mp->ma_keys); ep = &ep0[mp->ma_keys->dk_nentries]; dk_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries); - Py_INCREF(key); ep->me_key = key; ep->me_hash = hash; if (mp->ma_values) { @@ -1184,6 +1179,7 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) assert(_PyDict_CheckConsistency(mp)); Py_DECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ + Py_DECREF(key); return 0; } @@ -1194,7 +1190,13 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) mp->ma_used++; mp->ma_version_tag = DICT_NEXT_VERSION(); assert(_PyDict_CheckConsistency(mp)); + Py_DECREF(key); return 0; + +Fail: + Py_DECREF(value); + Py_DECREF(key); + return -1; } /* @@ -1929,7 +1931,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; - if (dictresize(mp, ESTIMATE_SIZE(Py_SIZE(iterable)))) { + if (dictresize(mp, ESTIMATE_SIZE(((PyDictObject *)iterable)->ma_used))) { Py_DECREF(d); return NULL; } @@ -2432,11 +2434,18 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) /* Update/merge with this (key, value) pair. */ key = PySequence_Fast_GET_ITEM(fast, 0); value = PySequence_Fast_GET_ITEM(fast, 1); + Py_INCREF(key); + Py_INCREF(value); if (override || PyDict_GetItem(d, key) == NULL) { int status = PyDict_SetItem(d, key, value); - if (status < 0) + if (status < 0) { + Py_DECREF(key); + Py_DECREF(value); goto Fail; + } } + Py_DECREF(key); + Py_DECREF(value); Py_DECREF(fast); Py_DECREF(item); } @@ -2737,14 +2746,15 @@ dict_equal(PyDictObject *a, PyDictObject *b) bval = NULL; else bval = *vaddr; - Py_DECREF(key); if (bval == NULL) { + Py_DECREF(key); Py_DECREF(aval); if (PyErr_Occurred()) return -1; return 0; } cmp = PyObject_RichCompareBool(aval, bval, Py_EQ); + Py_DECREF(key); Py_DECREF(aval); if (cmp <= 0) /* error or not equal */ return cmp; @@ -3633,7 +3643,7 @@ PyTypeObject PyDictIterValue_Type = { static PyObject * dictiter_iternextitem(dictiterobject *di) { - PyObject *key, *value, *result = di->di_result; + PyObject *key, *value, *result; Py_ssize_t i, n; PyDictObject *d = di->di_dict; @@ -3674,20 +3684,25 @@ dictiter_iternextitem(dictiterobject *di) } di->di_pos = i+1; di->len--; - if (result->ob_refcnt == 1) { + Py_INCREF(key); + Py_INCREF(value); + result = di->di_result; + if (Py_REFCNT(result) == 1) { + PyObject *oldkey = PyTuple_GET_ITEM(result, 0); + PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ Py_INCREF(result); - Py_DECREF(PyTuple_GET_ITEM(result, 0)); - Py_DECREF(PyTuple_GET_ITEM(result, 1)); + Py_DECREF(oldkey); + Py_DECREF(oldvalue); } else { result = PyTuple_New(2); if (result == NULL) return NULL; + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ } - Py_INCREF(key); - Py_INCREF(value); - PyTuple_SET_ITEM(result, 0, key); /* steals reference */ - PyTuple_SET_ITEM(result, 1, value); /* steals reference */ return result; fail: @@ -4180,6 +4195,7 @@ dictitems_iter(_PyDictViewObject *dv) static int dictitems_contains(_PyDictViewObject *dv, PyObject *obj) { + int result; PyObject *key, *value, *found; if (dv->dv_dict == NULL) return 0; @@ -4193,7 +4209,10 @@ dictitems_contains(_PyDictViewObject *dv, PyObject *obj) return -1; return 0; } - return PyObject_RichCompareBool(value, found, Py_EQ); + Py_INCREF(found); + result = PyObject_RichCompareBool(value, found, Py_EQ); + Py_DECREF(found); + return result; } static PySequenceMethods dictitems_as_sequence = { diff --git a/Objects/exceptions.c b/Objects/exceptions.c index f63f06a..d158b97 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -686,6 +686,53 @@ ImportError_str(PyImportErrorObject *self) } } +static PyObject * +ImportError_getstate(PyImportErrorObject *self) +{ + PyObject *dict = ((PyBaseExceptionObject *)self)->dict; + if (self->name || self->path) { + _Py_IDENTIFIER(name); + _Py_IDENTIFIER(path); + dict = dict ? PyDict_Copy(dict) : PyDict_New(); + if (dict == NULL) + return NULL; + if (self->name && _PyDict_SetItemId(dict, &PyId_name, self->name) < 0) { + Py_DECREF(dict); + return NULL; + } + if (self->path && _PyDict_SetItemId(dict, &PyId_path, self->path) < 0) { + Py_DECREF(dict); + return NULL; + } + return dict; + } + else if (dict) { + Py_INCREF(dict); + return dict; + } + else { + Py_RETURN_NONE; + } +} + +/* Pickling support */ +static PyObject * +ImportError_reduce(PyImportErrorObject *self) +{ + PyObject *res; + PyObject *args; + PyObject *state = ImportError_getstate(self); + if (state == NULL) + return NULL; + args = ((PyBaseExceptionObject *)self)->args; + if (state == Py_None) + res = PyTuple_Pack(2, Py_TYPE(self), args); + else + res = PyTuple_Pack(3, Py_TYPE(self), args, state); + Py_DECREF(state); + return res; +} + static PyMemberDef ImportError_members[] = { {"msg", T_OBJECT, offsetof(PyImportErrorObject, msg), 0, PyDoc_STR("exception message")}, @@ -697,6 +744,7 @@ static PyMemberDef ImportError_members[] = { }; static PyMethodDef ImportError_methods[] = { + {"__reduce__", (PyCFunction)ImportError_reduce, METH_NOARGS}, {NULL} }; diff --git a/Objects/genobject.c b/Objects/genobject.c index 2680ab0..1c29e29 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -575,8 +575,7 @@ _PyGen_SetStopIterationValue(PyObject *value) PyObject *e; if (value == NULL || - (!PyTuple_Check(value) && - !PyObject_TypeCheck(value, (PyTypeObject *) PyExc_StopIteration))) + (!PyTuple_Check(value) && !PyExceptionInstance_Check(value))) { /* Delay exception instantiation if we can */ PyErr_SetObject(PyExc_StopIteration, value); diff --git a/Objects/listobject.c b/Objects/listobject.c index dcd7b5e..547bdf0 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2153,8 +2153,8 @@ listindex(PyListObject *self, PyObject *args) PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(self); @@ -2420,10 +2420,11 @@ list_subscript(PyListObject* self, PyObject* item) PyObject* it; PyObject **src, **dest; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (slicelength <= 0) { return PyList_New(0); @@ -2469,10 +2470,11 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(item, Py_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + step); if (step == 1) return list_ass_slice(self, start, stop, value); diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 5153757..3dab2b9 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -1,17 +1,18 @@ All about co_lnotab, the line number table. Code objects store a field named co_lnotab. This is an array of unsigned bytes -disguised as a Python string. It is used to map bytecode offsets to source code -line #s for tracebacks and to identify line number boundaries for line tracing. +disguised as a Python bytes object. It is used to map bytecode offsets to +source code line #s for tracebacks and to identify line number boundaries for +line tracing. The array is conceptually a compressed list of (bytecode offset increment, line number increment) pairs. The details are important and delicate, best illustrated by example: byte code offset source code line number - 0 1 - 6 2 - 50 7 + 0 1 + 6 2 + 50 7 350 207 361 208 @@ -24,7 +25,8 @@ look like: The above doesn't really work, but it's a start. An unsigned byte (byte code offset) can't hold negative values, or values larger than 255, a signed byte (line number) can't hold values larger than 127 or less than -128, and the -above example contains two such values. So we make two tweaks: +above example contains two such values. (Note that before 3.6, line number +was also encoded by an unsigned byte.) So we make two tweaks: (a) there's a deep assumption that byte code offsets increase monotonically, and @@ -52,7 +54,7 @@ the example above, assemble_lnotab in compile.c should not (as was actually done until 2.2) expand 300, 200 to 255, 255, 45, 45, but to - 255, 0, 45, 128, 0, 72. + 255, 0, 45, 127, 0, 73. The above is sufficient to reconstruct line numbers for tracebacks, but not for line tracing. Tracing is handled by PyCode_CheckLineNumber() in codeobject.c @@ -83,30 +85,34 @@ Consider this code: 1: def f(a): 2: while a: -3: print 1, +3: print(1) 4: break 5: else: -6: print 2, +6: print(2) which compiles to this: - 2 0 SETUP_LOOP 19 (to 22) - >> 3 LOAD_FAST 0 (a) - 6 POP_JUMP_IF_FALSE 17 + 2 0 SETUP_LOOP 26 (to 28) + >> 2 LOAD_FAST 0 (a) + 4 POP_JUMP_IF_FALSE 18 - 3 9 LOAD_CONST 1 (1) - 12 PRINT_ITEM + 3 6 LOAD_GLOBAL 0 (print) + 8 LOAD_CONST 1 (1) + 10 CALL_FUNCTION 1 + 12 POP_TOP - 4 13 BREAK_LOOP - 14 JUMP_ABSOLUTE 3 - >> 17 POP_BLOCK + 4 14 BREAK_LOOP + 16 JUMP_ABSOLUTE 2 + >> 18 POP_BLOCK - 6 18 LOAD_CONST 2 (2) - 21 PRINT_ITEM - >> 22 LOAD_CONST 0 (None) - 25 RETURN_VALUE + 6 20 LOAD_GLOBAL 0 (print) + 22 LOAD_CONST 2 (2) + 24 CALL_FUNCTION 1 + 26 POP_TOP + >> 28 LOAD_CONST 0 (None) + 30 RETURN_VALUE -If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 17 +If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 18 and the co_lnotab will claim that execution has moved to line 4, which is wrong. In this case, we could instead associate the POP_BLOCK with line 5, but that would break jumps around loops without else clauses. diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index b1798a2..e1ac728 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2285,10 +2285,10 @@ init_slice(Py_buffer *base, PyObject *key, int dim) { Py_ssize_t start, stop, step, slicelength; - if (PySlice_GetIndicesEx(key, base->shape[dim], - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(key, &start, &stop, &step) < 0) { return -1; } + slicelength = PySlice_AdjustIndices(base->shape[dim], &start, &stop, step); if (base->suboffsets == NULL || dim == 0) { diff --git a/Objects/methodobject.c b/Objects/methodobject.c index c2001f0..fe52545 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -243,8 +243,7 @@ _PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs, PyObject *kwnames; _PyCFunctionFast fastmeth = (_PyCFunctionFast)meth; - stack = _PyStack_UnpackDict(args, nargs, kwargs, &kwnames, func_obj); - if (stack == NULL) { + if (_PyStack_UnpackDict(args, nargs, kwargs, &stack, &kwnames) < 0) { return NULL; } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index a1142f3..32e7ecb 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1227,7 +1227,7 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) _Py_AllocatedBlocks++; - assert(nelem <= PY_SSIZE_T_MAX / elsize); + assert(elsize == 0 || nelem <= PY_SSIZE_T_MAX / elsize); nbytes = nelem * elsize; #ifdef WITH_VALGRIND diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 8449fc7..8f5fc43 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -668,6 +668,16 @@ static PyMappingMethods range_as_mapping = { (objobjargproc)0, /* mp_ass_subscript */ }; +static int +range_bool(rangeobject* self) +{ + return PyObject_IsTrue(self->length); +} + +static PyNumberMethods range_as_number = { + .nb_bool = (inquiry)range_bool, +}; + static PyObject * range_iter(PyObject *seq); static PyObject * range_reverse(PyObject *seq); @@ -707,7 +717,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_setattr */ 0, /* tp_reserved */ (reprfunc)range_repr, /* tp_repr */ - 0, /* tp_as_number */ + &range_as_number, /* tp_as_number */ &range_as_sequence, /* tp_as_sequence */ &range_as_mapping, /* tp_as_mapping */ (hashfunc)range_hash, /* tp_hash */ diff --git a/Objects/setobject.c b/Objects/setobject.c index fdb9d36..c1bc1e1 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -236,7 +236,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) entry->hash = hash; if ((size_t)so->fill*3 < mask*2) return 0; - return set_table_resize(so, so->used); + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); found_active: Py_DECREF(key); @@ -304,7 +304,6 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) setentry small_copy[PySet_MINSIZE]; assert(minused >= 0); - minused = (minused > 50000) ? minused * 2 : minused * 4; /* Find the smallest table size > minused. */ /* XXX speed-up with intrinsics */ @@ -646,8 +645,8 @@ set_merge(PySetObject *so, PyObject *otherset) * that there will be no (or few) overlapping keys. */ if ((so->fill + other->used)*3 >= so->mask*2) { - if (set_table_resize(so, so->used + other->used) != 0) - return -1; + if (set_table_resize(so, (so->used + other->used)*2) != 0) + return -1; } so_entry = so->table; other_entry = other->table; @@ -990,7 +989,7 @@ set_update_internal(PySetObject *so, PyObject *other) if (dictsize < 0) return -1; if ((so->fill + dictsize)*3 >= so->mask*2) { - if (set_table_resize(so, so->used + dictsize) != 0) + if (set_table_resize(so, (so->used + dictsize)*2) != 0) return -1; } while (_PyDict_Next(other, &pos, &key, &value, &hash)) { @@ -1511,7 +1510,7 @@ set_difference_update_internal(PySetObject *so, PyObject *other) /* If more than 1/4th are dummies, then resize them away. */ if ((size_t)(so->fill - so->used) <= (size_t)so->mask / 4) return 0; - return set_table_resize(so, so->used); + return set_table_resize(so, so->used>50000 ? so->used*2 : so->used*4); } static PyObject * @@ -1551,20 +1550,26 @@ set_difference(PySetObject *so, PyObject *other) PyObject *key; Py_hash_t hash; setentry *entry; - Py_ssize_t pos = 0; + Py_ssize_t pos = 0, other_size; int rv; if (PySet_GET_SIZE(so) == 0) { return set_copy(so); } - if (!PyAnySet_Check(other) && !PyDict_CheckExact(other)) { + if (PyAnySet_Check(other)) { + other_size = PySet_GET_SIZE(other); + } + else if (PyDict_CheckExact(other)) { + other_size = PyDict_Size(other); + } + else { return set_copy_and_difference(so, other); } /* If len(so) much more than len(other), it's more efficient to simply copy * so and then iterate other looking for common elements. */ - if ((PySet_GET_SIZE(so) >> 2) > PyObject_Size(other)) { + if ((PySet_GET_SIZE(so) >> 2) > other_size) { return set_copy_and_difference(so, other); } diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 6a69021..4e4b32d 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -197,6 +197,8 @@ PySlice_Unpack(PyObject *_r, PySliceObject *r = (PySliceObject*)_r; /* this is harder to get right than you might think */ + Py_BUILD_ASSERT(PY_SSIZE_T_MIN + 1 <= -PY_SSIZE_T_MAX); + if (r->step == Py_None) { *step = 1; } @@ -217,14 +219,14 @@ PySlice_Unpack(PyObject *_r, } if (r->start == Py_None) { - *start = *step < 0 ? PY_SSIZE_T_MAX-1 : 0;; + *start = *step < 0 ? PY_SSIZE_T_MAX : 0; } else { if (!_PyEval_SliceIndex(r->start, start)) return -1; } if (r->stop == Py_None) { - *stop = *step < 0 ? -PY_SSIZE_T_MAX : PY_SSIZE_T_MAX; + *stop = *step < 0 ? PY_SSIZE_T_MIN : PY_SSIZE_T_MAX; } else { if (!_PyEval_SliceIndex(r->stop, stop)) return -1; @@ -258,7 +260,7 @@ PySlice_AdjustIndices(Py_ssize_t length, *stop = (step < 0) ? -1 : 0; } } - else if (*stop >= length) { + else if (*stop >= length) { *stop = (step < 0) ? length - 1 : length; } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index c0ff499..0dada74 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -522,8 +522,8 @@ tupleindex(PyTupleObject *self, PyObject *args) PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, - _PyEval_SliceIndex, &start, - _PyEval_SliceIndex, &stop)) + _PyEval_SliceIndexNotNone, &start, + _PyEval_SliceIndexNotNone, &stop)) return NULL; if (start < 0) { start += Py_SIZE(self); @@ -720,11 +720,11 @@ tuplesubscript(PyTupleObject* self, PyObject* item) PyObject* it; PyObject **src, **dest; - if (PySlice_GetIndicesEx(item, - PyTuple_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyTuple_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { return PyTuple_New(0); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 329261b..271b935 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3075,6 +3075,7 @@ type_getattro(PyTypeObject *type, PyObject *name) static int type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) { + int res; if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { PyErr_Format( PyExc_TypeError, @@ -3082,9 +3083,35 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) type->tp_name); return -1; } - if (PyObject_GenericSetAttr((PyObject *)type, name, value) < 0) - return -1; - return update_slot(type, name); + if (PyUnicode_Check(name)) { + if (PyUnicode_CheckExact(name)) { + if (PyUnicode_READY(name) == -1) + return -1; + Py_INCREF(name); + } + else { + name = _PyUnicode_Copy(name); + if (name == NULL) + return -1; + } + PyUnicode_InternInPlace(&name); + if (!PyUnicode_CHECK_INTERNED(name)) { + PyErr_SetString(PyExc_MemoryError, + "Out of memory interning an attribute name"); + Py_DECREF(name); + return -1; + } + } + else { + /* Will fail in _PyObject_GenericSetAttrWithDict. */ + Py_INCREF(name); + } + res = PyObject_GenericSetAttr((PyObject *)type, name, value); + if (res == 0) { + res = update_slot(type, name); + } + Py_DECREF(name); + return res; } extern void @@ -6929,7 +6956,7 @@ init_slotdefs(void) /* Slots must be ordered by their offset in the PyHeapTypeObject. */ assert(!p[1].name || p->offset <= p[1].offset); p->name_strobj = PyUnicode_InternFromString(p->name); - if (!p->name_strobj) + if (!p->name_strobj || !PyUnicode_CHECK_INTERNED(p->name_strobj)) Py_FatalError("Out of memory interning slotdef names"); } slotdefs_initialized = 1; @@ -6954,6 +6981,9 @@ update_slot(PyTypeObject *type, PyObject *name) slotdef **pp; int offset; + assert(PyUnicode_CheckExact(name)); + assert(PyUnicode_CHECK_INTERNED(name)); + /* Clear the VALID_VERSION flag of 'type' and all its subclasses. This could possibly be unified with the update_subclasses() recursion below, but carefully: @@ -6964,7 +6994,6 @@ update_slot(PyTypeObject *type, PyObject *name) init_slotdefs(); pp = ptrs; for (p = slotdefs; p->name; p++) { - /* XXX assume name is interned! */ if (p->name_strobj == name) *pp++ = p; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a5ae454..4e0c663 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3902,6 +3902,7 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) PyObject *output = NULL; if (arg == NULL) { Py_DECREF(*(PyObject**)addr); + *(PyObject**)addr = NULL; return 1; } @@ -11701,7 +11702,11 @@ unicode_hash(PyObject *self) PyDoc_STRVAR(index__doc__, "S.index(sub[, start[, end]]) -> int\n\ \n\ -Like S.find() but raise ValueError when the substring is not found."); +Return the lowest index in S where substring sub is found, \n\ +such that sub is contained within S[start:end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the substring is not found."); static PyObject * unicode_index(PyObject *self, PyObject *args) @@ -12758,7 +12763,11 @@ unicode_rfind(PyObject *self, PyObject *args) PyDoc_STRVAR(rindex__doc__, "S.rindex(sub[, start[, end]]) -> int\n\ \n\ -Like S.rfind() but raise ValueError when the substring is not found."); +Return the highest index in S where substring sub is found,\n\ +such that sub is contained within S[start:end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Raises ValueError when the substring is not found."); static PyObject * unicode_rindex(PyObject *self, PyObject *args) @@ -13915,10 +13924,11 @@ unicode_subscript(PyObject* self, PyObject* item) int src_kind, dest_kind; Py_UCS4 ch, max_char, kind_limit; - if (PySlice_GetIndicesEx(item, PyUnicode_GET_LENGTH(self), - &start, &stop, &step, &slicelength) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelength = PySlice_AdjustIndices(PyUnicode_GET_LENGTH(self), + &start, &stop, step); if (slicelength <= 0) { _Py_RETURN_UNICODE_EMPTY(); diff --git a/PC/bdist_wininst/bdist_wininst.vcxproj b/PC/bdist_wininst/bdist_wininst.vcxproj index 4dc5009..16213fb 100644 --- a/PC/bdist_wininst/bdist_wininst.vcxproj +++ b/PC/bdist_wininst/bdist_wininst.vcxproj @@ -1,104 +1,108 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C} - wininst - false - - - - - Application - false - NotSet - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - $(PySourcePath)lib\distutils\command\ - false - wininst-$(VisualStudioVersion) - $(TargetName)-amd64 - .exe - - - - $(OutDir)wininst.tlb - - - MinSpace - $(PySourcePath)Modules\zlib;%(AdditionalIncludeDirectories) - _CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions) - MultiThreadedDebug - MultiThreaded - - - $(PySourcePath)PC\bdist_wininst;%(AdditionalIncludeDirectories) - - - comctl32.lib;imagehlp.lib;%(AdditionalDependencies) - - - - - - - - - - - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {EB1C19C1-1F18-421E-9735-CAEE69DC6A3C} + wininst + false + + + + + Application + false + NotSet + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + $(PySourcePath)lib\distutils\command\ + false + wininst-$(VisualStudioVersion) + wininst-14.0 + wininst-12.0 + wininst-11.0 + wininst-10.0 + $(TargetName)-amd64 + .exe + + + + $(OutDir)wininst.tlb + + + MinSpace + $(PySourcePath)Modules\zlib;%(AdditionalIncludeDirectories) + _CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions) + MultiThreadedDebug + MultiThreaded + + + $(PySourcePath)PC\bdist_wininst;%(AdditionalIncludeDirectories) + + + comctl32.lib;imagehlp.lib;%(AdditionalDependencies) + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/PC/bdist_wininst/bdist_wininst.vcxproj.filters b/PC/bdist_wininst/bdist_wininst.vcxproj.filters index b4b6a4d..a47ac86 100644 --- a/PC/bdist_wininst/bdist_wininst.vcxproj.filters +++ b/PC/bdist_wininst/bdist_wininst.vcxproj.filters @@ -1,61 +1,61 @@ - - - - - {293b1092-03ad-4b7c-acb9-c4ab62e52f55} - cpp;c;cxx;rc;def;r;odl;idl;hpj;bat - - - {0edc0406-282f-4dbc-b60e-a867c34a2a31} - - - {ea0c0f0e-3b73-474e-a999-e9689d032ccc} - h;hpp;hxx;hm;inl - - - {0c77c1cf-3f87-4f87-bd86-b425211c2181} - ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe - - - - - Source Files - - - Source Files - - - Source Files\zlib - - - Source Files\zlib - - - Source Files\zlib - - - Source Files\zlib - - - Source Files\zlib - - - Source Files\zlib - - - - - Header Files - - - - - Resource Files - - - - - Resource Files - - + + + + + {293b1092-03ad-4b7c-acb9-c4ab62e52f55} + cpp;c;cxx;rc;def;r;odl;idl;hpj;bat + + + {0edc0406-282f-4dbc-b60e-a867c34a2a31} + + + {ea0c0f0e-3b73-474e-a999-e9689d032ccc} + h;hpp;hxx;hm;inl + + + {0c77c1cf-3f87-4f87-bd86-b425211c2181} + ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe + + + + + Source Files + + + Source Files + + + Source Files\zlib + + + Source Files\zlib + + + Source Files\zlib + + + Source Files\zlib + + + Source Files\zlib + + + Source Files\zlib + + + + + Header Files + + + + + Resource Files + + + + + Resource Files + + \ No newline at end of file diff --git a/PC/bdist_wininst/build.bat b/PC/bdist_wininst/build.bat index 25f565c..744267f 100644 --- a/PC/bdist_wininst/build.bat +++ b/PC/bdist_wininst/build.bat @@ -7,16 +7,13 @@ set PCBUILD=%~dp0..\..\PCBuild\ echo Building Lib\distutils\command\wininst-xx.0.exe -call "%PCBUILD%env.bat" x86 -if errorlevel 1 goto :eof +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) -msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32 +%MSBUILD% "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32 if errorlevel 1 goto :eof echo Building Lib\distutils\command\wininst-xx.0-amd64.exe -call "%PCBUILD%env.bat" x86_amd64 -if errorlevel 1 goto :eof - -msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64 +%MSBUILD% "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64 diff --git a/PC/getpathp.c b/PC/getpathp.c index 1eeebfe..e7be704 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -185,7 +185,7 @@ exists(wchar_t *filename) may extend 'filename' by one character. */ static int -ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/.pyo too */ +ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc too */ { size_t n; @@ -196,7 +196,7 @@ ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/ n = wcsnlen_s(filename, MAXPATHLEN+1); if (n < MAXPATHLEN) { int exist = 0; - filename[n] = Py_OptimizeFlag ? L'o' : L'c'; + filename[n] = L'c'; filename[n + 1] = L'\0'; exist = exists(filename); if (!update_filename) diff --git a/PC/readme.txt b/PC/readme.txt index 0a96d26..b796901 100644 --- a/PC/readme.txt +++ b/PC/readme.txt @@ -1,81 +1,81 @@ -Welcome to the "PC" subdirectory of the Python distribution -*********************************************************** - -This "PC" subdirectory contains complete project files to make -several older PC ports of Python, as well as all the PC-specific -Python source files. It should be located in the root of the -Python distribution, and there should be directories "Modules", -"Objects", "Python", etc. in the parent directory of this "PC" -subdirectory. Be sure to read the documentation in the Python -distribution. - -Python requires library files such as string.py to be available in -one or more library directories. The search path of libraries is -set up when Python starts. To see the current Python library search -path, start Python and enter "import sys" and "print sys.path". - -All PC ports use this scheme to try to set up a module search path: - - 1) The script location; the current directory without script. - 2) The PYTHONPATH variable, if set. - 3) For Win32 platforms (NT/95), paths specified in the Registry. - 4) Default directories lib, lib/win, lib/test, lib/tkinter; - these are searched relative to the environment variable - PYTHONHOME, if set, or relative to the executable and its - ancestors, if a landmark file (Lib/string.py) is found , - or the current directory (not useful). - 5) The directory containing the executable. - -The best installation strategy is to put the Python executable (and -DLL, for Win32 platforms) in some convenient directory such as -C:/python, and copy all library files and subdirectories (using XCOPY) -to C:/python/lib. Then you don't need to set PYTHONPATH. Otherwise, -set the environment variable PYTHONPATH to your Python search path. -For example, - set PYTHONPATH=.;d:\python\lib;d:\python\lib\win;d:\python\lib\dos-8x3 - -There are several add-in modules to build Python programs which use -the native Windows operating environment. The ports here just make -"QuickWin" and DOS Python versions which support a character-mode -(console) environment. Look in www.python.org for Tkinter, PythonWin, -WPY and wxPython. - -To make a Python port, start the Integrated Development Environment -(IDE) of your compiler, and read in the native "project file" -(or makefile) provided. This will enable you to change any source -files or build settings so you can make custom builds. - -pyconfig.h An important configuration file specific to PC's. - -config.c The list of C modules to include in the Python PC - version. Manually edit this file to add or - remove Python modules. - -testpy.py A Python test program. Run this to test your - Python port. It should produce copious output, - ending in a report on how many tests were OK, how many - failed, and how many were skipped. Don't worry about - skipped tests (these test unavailable optional features). - - -Additional files and subdirectories for 32-bit Windows -====================================================== - -python_nt.rc Resource compiler input for python15.dll. - -dl_nt.c - Additional sources used for 32-bit Windows features. - -getpathp.c Default sys.path calculations (for all PC platforms). - -dllbase_nt.txt A (manually maintained) list of base addresses for - various DLLs, to avoid run-time relocation. - - -Note for Windows 3.x and DOS users -================================== - -Neither Windows 3.x nor DOS is supported any more. The last Python -version that supported these was Python 1.5.2; the support files were -present in Python 2.0 but weren't updated, and it is not our intention -to support these platforms for Python 2.x. +Welcome to the "PC" subdirectory of the Python distribution +*********************************************************** + +This "PC" subdirectory contains complete project files to make +several older PC ports of Python, as well as all the PC-specific +Python source files. It should be located in the root of the +Python distribution, and there should be directories "Modules", +"Objects", "Python", etc. in the parent directory of this "PC" +subdirectory. Be sure to read the documentation in the Python +distribution. + +Python requires library files such as string.py to be available in +one or more library directories. The search path of libraries is +set up when Python starts. To see the current Python library search +path, start Python and enter "import sys" and "print sys.path". + +All PC ports use this scheme to try to set up a module search path: + + 1) The script location; the current directory without script. + 2) The PYTHONPATH variable, if set. + 3) For Win32 platforms (NT/95), paths specified in the Registry. + 4) Default directories lib, lib/win, lib/test, lib/tkinter; + these are searched relative to the environment variable + PYTHONHOME, if set, or relative to the executable and its + ancestors, if a landmark file (Lib/string.py) is found , + or the current directory (not useful). + 5) The directory containing the executable. + +The best installation strategy is to put the Python executable (and +DLL, for Win32 platforms) in some convenient directory such as +C:/python, and copy all library files and subdirectories (using XCOPY) +to C:/python/lib. Then you don't need to set PYTHONPATH. Otherwise, +set the environment variable PYTHONPATH to your Python search path. +For example, + set PYTHONPATH=.;d:\python\lib;d:\python\lib\win;d:\python\lib\dos-8x3 + +There are several add-in modules to build Python programs which use +the native Windows operating environment. The ports here just make +"QuickWin" and DOS Python versions which support a character-mode +(console) environment. Look in www.python.org for Tkinter, PythonWin, +WPY and wxPython. + +To make a Python port, start the Integrated Development Environment +(IDE) of your compiler, and read in the native "project file" +(or makefile) provided. This will enable you to change any source +files or build settings so you can make custom builds. + +pyconfig.h An important configuration file specific to PC's. + +config.c The list of C modules to include in the Python PC + version. Manually edit this file to add or + remove Python modules. + +testpy.py A Python test program. Run this to test your + Python port. It should produce copious output, + ending in a report on how many tests were OK, how many + failed, and how many were skipped. Don't worry about + skipped tests (these test unavailable optional features). + + +Additional files and subdirectories for 32-bit Windows +====================================================== + +python_nt.rc Resource compiler input for python15.dll. + +dl_nt.c + Additional sources used for 32-bit Windows features. + +getpathp.c Default sys.path calculations (for all PC platforms). + +dllbase_nt.txt A (manually maintained) list of base addresses for + various DLLs, to avoid run-time relocation. + + +Note for Windows 3.x and DOS users +================================== + +Neither Windows 3.x nor DOS is supported any more. The last Python +version that supported these was Python 1.5.2; the support files were +present in Python 2.0 but weren't updated, and it is not our intention +to support these platforms for Python 2.x. diff --git a/PCbuild/_asyncio.vcxproj b/PCbuild/_asyncio.vcxproj index 3cca000..2d23e83 100644 --- a/PCbuild/_asyncio.vcxproj +++ b/PCbuild/_asyncio.vcxproj @@ -1,77 +1,77 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {384C224A-7474-476E-A01B-750EA7DE918C} - _asyncio - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {384C224A-7474-476E-A01B-750EA7DE918C} + _asyncio + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + + diff --git a/PCbuild/_asyncio.vcxproj.filters b/PCbuild/_asyncio.vcxproj.filters index 10a186c..4c4a22b 100644 --- a/PCbuild/_asyncio.vcxproj.filters +++ b/PCbuild/_asyncio.vcxproj.filters @@ -1,16 +1,16 @@ - - - - - - - - {2422278e-eeeb-4241-8182-433e2bc5a7fc} - - - - - Source Files - - + + + + + + + + {2422278e-eeeb-4241-8182-433e2bc5a7fc} + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_bz2.vcxproj b/PCbuild/_bz2.vcxproj index 9efb0d9..f53634a 100644 --- a/PCbuild/_bz2.vcxproj +++ b/PCbuild/_bz2.vcxproj @@ -1,98 +1,98 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {73FCD2BD-F133-46B7-8EC1-144CD82A59D5} - bz2 - Win32Proj - - - - - DynamicLibrary - NotSet - - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - $(bz2Dir);%(AdditionalIncludeDirectories) - WIN32;_FILE_OFFSET_BITS=64;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions) - - - 0x1D170000 - - - - - - - - - - - - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {73FCD2BD-F133-46B7-8EC1-144CD82A59D5} + bz2 + Win32Proj + + + + + DynamicLibrary + NotSet + + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + $(bz2Dir);%(AdditionalIncludeDirectories) + WIN32;_FILE_OFFSET_BITS=64;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions) + + + 0x1D170000 + + + + + + + + + + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/_bz2.vcxproj.filters b/PCbuild/_bz2.vcxproj.filters index eac64dd..382f6b6 100644 --- a/PCbuild/_bz2.vcxproj.filters +++ b/PCbuild/_bz2.vcxproj.filters @@ -1,48 +1,48 @@ - - - - - {f53a859d-dad2-4d5b-ae41-f28d8b571f5a} - - - {7e0bed05-ae33-43b7-8797-656455bbb7f3} - - - {ed574b89-6983-4cdf-9f98-fe7048d9e89c} - - - - - Source Files - - - bzip2 1.0.6 Source Files - - - bzip2 1.0.6 Source Files - - - bzip2 1.0.6 Source Files - - - bzip2 1.0.6 Source Files - - - bzip2 1.0.6 Source Files - - - bzip2 1.0.6 Source Files - - - bzip2 1.0.6 Source Files - - - - - bzip2 1.0.6 Header Files - - - bzip2 1.0.6 Header Files - - - + + + + + {f53a859d-dad2-4d5b-ae41-f28d8b571f5a} + + + {7e0bed05-ae33-43b7-8797-656455bbb7f3} + + + {ed574b89-6983-4cdf-9f98-fe7048d9e89c} + + + + + Source Files + + + bzip2 1.0.6 Source Files + + + bzip2 1.0.6 Source Files + + + bzip2 1.0.6 Source Files + + + bzip2 1.0.6 Source Files + + + bzip2 1.0.6 Source Files + + + bzip2 1.0.6 Source Files + + + bzip2 1.0.6 Source Files + + + + + bzip2 1.0.6 Header Files + + + bzip2 1.0.6 Header Files + + + diff --git a/PCbuild/_ctypes.vcxproj b/PCbuild/_ctypes.vcxproj index 70bea45..0b2c00e 100644 --- a/PCbuild/_ctypes.vcxproj +++ b/PCbuild/_ctypes.vcxproj @@ -1,110 +1,110 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {0E9791DB-593A-465F-98BC-681011311618} - _ctypes - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.40219.1 - - - - ..\Modules\_ctypes\libffi_msvc;%(AdditionalIncludeDirectories) - - - 0x1D1A0000 - /EXPORT:DllGetClassObject,PRIVATE /EXPORT:DllCanUnloadNow,PRIVATE %(AdditionalOptions) - - - - - - - - - - - - - - - - - - - - - true - - - - - true - ml64 /nologo /c /Zi /Fo "$(IntDir)win64.obj" "%(FullPath)" - $(IntDir)win64.obj;%(Outputs) - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {0E9791DB-593A-465F-98BC-681011311618} + _ctypes + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.40219.1 + + + + ..\Modules\_ctypes\libffi_msvc;%(AdditionalIncludeDirectories) + + + 0x1D1A0000 + /EXPORT:DllGetClassObject,PRIVATE /EXPORT:DllCanUnloadNow,PRIVATE %(AdditionalOptions) + + + + + + + + + + + + + + + + + + + + + true + + + + + true + ml64 /nologo /c /Zi /Fo "$(IntDir)win64.obj" "%(FullPath)" + $(IntDir)win64.obj;%(Outputs) + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + + + + + \ No newline at end of file diff --git a/PCbuild/_ctypes.vcxproj.filters b/PCbuild/_ctypes.vcxproj.filters index 83d7a7b..0a2c4bf 100644 --- a/PCbuild/_ctypes.vcxproj.filters +++ b/PCbuild/_ctypes.vcxproj.filters @@ -1,65 +1,65 @@ - - - - - {806081ee-2af0-48d0-a83e-ee02a74baa0f} - - - {dbdea1f2-ad8b-44ca-b782-fcf65d91559b} - - - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - - - Source Files - - + + + + + {806081ee-2af0-48d0-a83e-ee02a74baa0f} + + + {dbdea1f2-ad8b-44ca-b782-fcf65d91559b} + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_ctypes_test.vcxproj b/PCbuild/_ctypes_test.vcxproj index b62407f..06a9301 100644 --- a/PCbuild/_ctypes_test.vcxproj +++ b/PCbuild/_ctypes_test.vcxproj @@ -1,80 +1,80 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {9EC7190A-249F-4180-A900-548FDCF3055F} - _ctypes_test - Win32Proj - false - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {9EC7190A-249F-4180-A900-548FDCF3055F} + _ctypes_test + Win32Proj + false + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + + + + + \ No newline at end of file diff --git a/PCbuild/_ctypes_test.vcxproj.filters b/PCbuild/_ctypes_test.vcxproj.filters index c1e94ce..1a5a82b 100644 --- a/PCbuild/_ctypes_test.vcxproj.filters +++ b/PCbuild/_ctypes_test.vcxproj.filters @@ -1,21 +1,21 @@ - - - - - {8fd70119-5481-4e5d-b187-d0b14eb27e38} - - - {38abc486-e143-49dc-8cf0-8aefab0e0d3d} - - - - - Header Files - - - - - Source Files - - + + + + + {8fd70119-5481-4e5d-b187-d0b14eb27e38} + + + {38abc486-e143-49dc-8cf0-8aefab0e0d3d} + + + + + Header Files + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_decimal.vcxproj b/PCbuild/_decimal.vcxproj index 9c9c19a..d2868c8 100644 --- a/PCbuild/_decimal.vcxproj +++ b/PCbuild/_decimal.vcxproj @@ -1,127 +1,127 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {0E9791DB-593A-465F-98BC-681011311617} - _decimal - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.40219.1 - - - - _CRT_SECURE_NO_WARNINGS;MASM;%(PreprocessorDefinitions) - CONFIG_32;PPRO;%(PreprocessorDefinitions) - CONFIG_64;%(PreprocessorDefinitions) - ..\Modules\_decimal;..\Modules\_decimal\libmpdec;%(AdditionalIncludeDirectories) - - - 0x1D1A0000 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - ml64 /nologo /c /Zi /Fo "$(IntDir)vcdiv64.obj" "%(FullPath)" - $(IntDir)vcdiv64.obj;%(Outputs) - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {0E9791DB-593A-465F-98BC-681011311617} + _decimal + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.40219.1 + + + + _CRT_SECURE_NO_WARNINGS;MASM;%(PreprocessorDefinitions) + CONFIG_32;PPRO;%(PreprocessorDefinitions) + CONFIG_64;%(PreprocessorDefinitions) + ..\Modules\_decimal;..\Modules\_decimal\libmpdec;%(AdditionalIncludeDirectories) + + + 0x1D1A0000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + ml64 /nologo /c /Zi /Fo "$(IntDir)vcdiv64.obj" "%(FullPath)" + $(IntDir)vcdiv64.obj;%(Outputs) + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + + + + + \ No newline at end of file diff --git a/PCbuild/_decimal.vcxproj.filters b/PCbuild/_decimal.vcxproj.filters index 7e19aa2..6f1d520 100644 --- a/PCbuild/_decimal.vcxproj.filters +++ b/PCbuild/_decimal.vcxproj.filters @@ -1,116 +1,116 @@ - - - - - {f35a78a6-3ef0-4e36-bd8b-afaef22fbb7c} - - - {138089f8-faba-494f-b6ed-051f31fbaf2d} - - - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - - - Source Files - - + + + + + {f35a78a6-3ef0-4e36-bd8b-afaef22fbb7c} + + + {138089f8-faba-494f-b6ed-051f31fbaf2d} + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 725b5a9..e7cdf57 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -1,106 +1,106 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {17E1E049-C309-4D79-843F-AE483C264AEA} - _elementtree - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - ..\Modules\expat;%(AdditionalIncludeDirectories) - XML_NS;XML_DTD;BYTEORDER=1234;XML_CONTEXT_BYTES=1024;USE_PYEXPAT_CAPI;XML_STATIC;HAVE_MEMMOVE;%(PreprocessorDefinitions) - - - 0x1D100000 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - - \ No newline at end of file + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {17E1E049-C309-4D79-843F-AE483C264AEA} + _elementtree + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + ..\Modules\expat;%(AdditionalIncludeDirectories) + _CRT_SECURE_NO_WARNINGS;USE_PYEXPAT_CAPI;XML_STATIC;%(PreprocessorDefinitions) + + + 0x1D100000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + + diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters index ee78295..acb722d 100644 --- a/PCbuild/_elementtree.vcxproj.filters +++ b/PCbuild/_elementtree.vcxproj.filters @@ -1,72 +1,72 @@ - - - - - {643d8607-d024-40fe-8583-1823c96430f0} - - - {7b5335ad-059f-486f-85e4-f4757e26a9bf} - - - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - + + + + + {643d8607-d024-40fe-8583-1823c96430f0} + + + {7b5335ad-059f-486f-85e4-f4757e26a9bf} + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_freeze_importlib.vcxproj b/PCbuild/_freeze_importlib.vcxproj index c732663..b6ad0ba 100644 --- a/PCbuild/_freeze_importlib.vcxproj +++ b/PCbuild/_freeze_importlib.vcxproj @@ -1,119 +1,119 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {19C0C13F-47CA-4432-AFF3-799A296A4DDC} - Win32Proj - _freeze_importlib - false - - - - - Application - Unicode - - - - - - - - - - - - _CONSOLE;%(PreprocessorDefinitions) - - - Console - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - true - true - false - true - false - - - - - $(IntDir)importlib.g.h - $(PySourcePath)Python\importlib.h - - - $(IntDir)importlib_external.g.h - $(PySourcePath)Python\importlib_external.h - - - - - - - - - - <_OldContent Condition="Exists($(OutTargetPath))"> - <_NewContent Condition="Exists($(IntTargetPath))">$([System.IO.File]::ReadAllText($(IntTargetPath)).Replace(` `, ` `)) - - - - - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {19C0C13F-47CA-4432-AFF3-799A296A4DDC} + Win32Proj + _freeze_importlib + false + + + + + Application + Unicode + + + + + + + + + + + + _CONSOLE;%(PreprocessorDefinitions) + + + Console + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + true + true + false + true + false + + + + + $(IntDir)importlib.g.h + $(PySourcePath)Python\importlib.h + + + $(IntDir)importlib_external.g.h + $(PySourcePath)Python\importlib_external.h + + + + + + + + + + <_OldContent Condition="Exists($(OutTargetPath))"> + <_NewContent Condition="Exists($(IntTargetPath))">$([System.IO.File]::ReadAllText($(IntTargetPath)).Replace(` `, ` `)) + + + + + + + + + + + + + + + + + + diff --git a/PCbuild/_freeze_importlib.vcxproj.filters b/PCbuild/_freeze_importlib.vcxproj.filters index ccad053..80b7fa9 100644 --- a/PCbuild/_freeze_importlib.vcxproj.filters +++ b/PCbuild/_freeze_importlib.vcxproj.filters @@ -1,27 +1,27 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - - - Source Files - - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + + + Source Files + + + diff --git a/PCbuild/_hashlib.vcxproj b/PCbuild/_hashlib.vcxproj index b1300cb..76862ea 100644 --- a/PCbuild/_hashlib.vcxproj +++ b/PCbuild/_hashlib.vcxproj @@ -1,93 +1,93 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {447F05A8-F581-4CAC-A466-5AC7936E207E} - _hashlib - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - $(opensslIncludeDir);%(AdditionalIncludeDirectories) - - - ws2_32.lib;$(OutDir)libeay$(PyDebugExt).lib;$(OutDir)ssleay$(PyDebugExt).lib;%(AdditionalDependencies) - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - {10615b24-73bf-4efa-93aa-236916321317} - false - - - {e5b04cc0-eb4c-42ab-b4dc-18ef95f864b0} - false - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {447F05A8-F581-4CAC-A466-5AC7936E207E} + _hashlib + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + $(opensslIncludeDir);%(AdditionalIncludeDirectories) + + + ws2_32.lib;$(OutDir)libeay$(PyDebugExt).lib;$(OutDir)ssleay$(PyDebugExt).lib;%(AdditionalDependencies) + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + {10615b24-73bf-4efa-93aa-236916321317} + false + + + {e5b04cc0-eb4c-42ab-b4dc-18ef95f864b0} + false + + + + + + diff --git a/PCbuild/_hashlib.vcxproj.filters b/PCbuild/_hashlib.vcxproj.filters index a9d993e..0f26370 100644 --- a/PCbuild/_hashlib.vcxproj.filters +++ b/PCbuild/_hashlib.vcxproj.filters @@ -1,13 +1,13 @@ - - - - - {cc45963d-bd25-4eb8-bdba-a5507090bca4} - - - - - Source Files - - + + + + + {cc45963d-bd25-4eb8-bdba-a5507090bca4} + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_lzma.vcxproj b/PCbuild/_lzma.vcxproj index 7ec2692..c684921 100644 --- a/PCbuild/_lzma.vcxproj +++ b/PCbuild/_lzma.vcxproj @@ -1,90 +1,90 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {F9D71780-F393-11E0-BE50-0800200C9A66} - lzma - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - $(lzmaDir)src/liblzma/api;%(AdditionalIncludeDirectories) - WIN32;_FILE_OFFSET_BITS=64;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;LZMA_API_STATIC;%(PreprocessorDefinitions) - - - $(OutDir)/liblzma$(PyDebugExt).lib - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - {12728250-16eC-4dc6-94d7-e21dd88947f8} - false - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {F9D71780-F393-11E0-BE50-0800200C9A66} + lzma + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + $(lzmaDir)src/liblzma/api;%(AdditionalIncludeDirectories) + WIN32;_FILE_OFFSET_BITS=64;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;LZMA_API_STATIC;%(PreprocessorDefinitions) + + + $(OutDir)/liblzma$(PyDebugExt).lib + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + {12728250-16eC-4dc6-94d7-e21dd88947f8} + false + + + + + + diff --git a/PCbuild/_lzma.vcxproj.filters b/PCbuild/_lzma.vcxproj.filters index 0c5ed74..bb0ae1f 100644 --- a/PCbuild/_lzma.vcxproj.filters +++ b/PCbuild/_lzma.vcxproj.filters @@ -1,13 +1,13 @@ - - - - - {53e68eda-39fc-4336-a658-dc5f5d598760} - - - - - Source Files - - + + + + + {53e68eda-39fc-4336-a658-dc5f5d598760} + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_msi.vcxproj b/PCbuild/_msi.vcxproj index 3895d45..b61a528 100644 --- a/PCbuild/_msi.vcxproj +++ b/PCbuild/_msi.vcxproj @@ -1,83 +1,83 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {31FFC478-7B4A-43E8-9954-8D03E2187E9C} - _msi - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - cabinet.lib;msi.lib;rpcrt4.lib;%(AdditionalDependencies) - 0x1D160000 - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {31FFC478-7B4A-43E8-9954-8D03E2187E9C} + _msi + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + cabinet.lib;msi.lib;rpcrt4.lib;%(AdditionalDependencies) + 0x1D160000 + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/_msi.vcxproj.filters b/PCbuild/_msi.vcxproj.filters index 7dec52d..a40f405 100644 --- a/PCbuild/_msi.vcxproj.filters +++ b/PCbuild/_msi.vcxproj.filters @@ -1,13 +1,13 @@ - - - - - {bdef7710-e433-4ac0-84e0-14f34454bd3e} - - - - - Source Files - - + + + + + {bdef7710-e433-4ac0-84e0-14f34454bd3e} + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_multiprocessing.vcxproj b/PCbuild/_multiprocessing.vcxproj index bb2bb41..0673f26 100644 --- a/PCbuild/_multiprocessing.vcxproj +++ b/PCbuild/_multiprocessing.vcxproj @@ -1,87 +1,87 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {9E48B300-37D1-11DD-8C41-005056C00008} - _multiprocessing - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - ws2_32.lib;%(AdditionalDependencies) - 0x1e1D0000 - - - - - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {9E48B300-37D1-11DD-8C41-005056C00008} + _multiprocessing + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + ws2_32.lib;%(AdditionalDependencies) + 0x1e1D0000 + + + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/_multiprocessing.vcxproj.filters b/PCbuild/_multiprocessing.vcxproj.filters index 78133ec..0b5291d 100644 --- a/PCbuild/_multiprocessing.vcxproj.filters +++ b/PCbuild/_multiprocessing.vcxproj.filters @@ -1,24 +1,24 @@ - - - - - {623c956c-1893-43d9-a7dc-96e4fef20f93} - - - {34615a62-f999-4659-83f5-19d17a644530} - - - - - Header Files - - - - - Source Files - - - Source Files - - + + + + + {623c956c-1893-43d9-a7dc-96e4fef20f93} + + + {34615a62-f999-4659-83f5-19d17a644530} + + + + + Header Files + + + + + Source Files + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_overlapped.vcxproj b/PCbuild/_overlapped.vcxproj index 8cf8a86..ce946a5 100644 --- a/PCbuild/_overlapped.vcxproj +++ b/PCbuild/_overlapped.vcxproj @@ -1,83 +1,83 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E} - _overlapped - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - ws2_32.lib;%(AdditionalDependencies) - 0x1D110000 - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E} + _overlapped + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + ws2_32.lib;%(AdditionalDependencies) + 0x1D110000 + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/_overlapped.vcxproj.filters b/PCbuild/_overlapped.vcxproj.filters index 78de895..5ed9a09 100644 --- a/PCbuild/_overlapped.vcxproj.filters +++ b/PCbuild/_overlapped.vcxproj.filters @@ -1,13 +1,13 @@ - - - - - {6f67c8db-7de7-4714-a967-2b0d4bc71f2e} - - - - - Source Files - - + + + + + {6f67c8db-7de7-4714-a967-2b0d4bc71f2e} + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_socket.vcxproj b/PCbuild/_socket.vcxproj index d5c4d1b..0c1d05a 100644 --- a/PCbuild/_socket.vcxproj +++ b/PCbuild/_socket.vcxproj @@ -1,86 +1,86 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {86937F53-C189-40EF-8CE8-8759D8E7D480} - _socket - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - ws2_32.lib;%(AdditionalDependencies) - 0x1e1D0000 - - - - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {86937F53-C189-40EF-8CE8-8759D8E7D480} + _socket + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + ws2_32.lib;%(AdditionalDependencies) + 0x1e1D0000 + + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/_socket.vcxproj.filters b/PCbuild/_socket.vcxproj.filters index 790ffa8..8fe3e6b 100644 --- a/PCbuild/_socket.vcxproj.filters +++ b/PCbuild/_socket.vcxproj.filters @@ -1,21 +1,21 @@ - - - - - {1452207f-707c-4e84-b532-307193a0fd85} - - - {1edfe0d0-7b9d-4dc8-a335-b21fef7cc77a} - - - - - Header Files - - - - - Source Files - - + + + + + {1452207f-707c-4e84-b532-307193a0fd85} + + + {1edfe0d0-7b9d-4dc8-a335-b21fef7cc77a} + + + + + Header Files + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj index 5456bb5..39f7de8 100644 --- a/PCbuild/_sqlite3.vcxproj +++ b/PCbuild/_sqlite3.vcxproj @@ -1,109 +1,109 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {13CECB97-4119-4316-9D42-8534019A5A44} - _sqlite3 - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - $(sqlite3Dir);%(AdditionalIncludeDirectories) - MODULE_NAME="sqlite3";%(PreprocessorDefinitions) - - - 0x1e180000 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - {a1a295e5-463c-437f-81ca-1f32367685da} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {13CECB97-4119-4316-9D42-8534019A5A44} + _sqlite3 + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + $(sqlite3Dir);%(AdditionalIncludeDirectories) + MODULE_NAME="sqlite3";%(PreprocessorDefinitions) + + + 0x1e180000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + {a1a295e5-463c-437f-81ca-1f32367685da} + false + + + + + \ No newline at end of file diff --git a/PCbuild/_sqlite3.vcxproj.filters b/PCbuild/_sqlite3.vcxproj.filters index dce77c9..269a798 100644 --- a/PCbuild/_sqlite3.vcxproj.filters +++ b/PCbuild/_sqlite3.vcxproj.filters @@ -1,69 +1,69 @@ - - - - - {dac8ab3b-ce16-4bef-bef9-76463a01f5c4} - - - {814b187d-44ad-4f2b-baa7-18ca8a8a6a77} - - - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - + + + + + {dac8ab3b-ce16-4bef-bef9-76463a01f5c4} + + + {814b187d-44ad-4f2b-baa7-18ca8a8a6a77} + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_ssl.vcxproj b/PCbuild/_ssl.vcxproj index d75ebd6..02f9812 100644 --- a/PCbuild/_ssl.vcxproj +++ b/PCbuild/_ssl.vcxproj @@ -1,97 +1,97 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {C6E20F84-3247-4AD6-B051-B073268F73BA} - _ssl - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - $(opensslIncludeDir);%(AdditionalIncludeDirectories) - - - ws2_32.lib;crypt32.lib;$(OutDir)libeay$(PyDebugExt).lib;$(OutDir)ssleay$(PyDebugExt).lib;%(AdditionalDependencies) - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - {e5b04cc0-eb4c-42ab-b4dc-18ef95f864b0} - false - - - {10615b24-73bf-4efa-93aa-236916321317} - false - - - {86937f53-c189-40ef-8ce8-8759d8e7d480} - false - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {C6E20F84-3247-4AD6-B051-B073268F73BA} + _ssl + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + $(opensslIncludeDir);%(AdditionalIncludeDirectories) + + + ws2_32.lib;crypt32.lib;$(OutDir)libeay$(PyDebugExt).lib;$(OutDir)ssleay$(PyDebugExt).lib;%(AdditionalDependencies) + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + {e5b04cc0-eb4c-42ab-b4dc-18ef95f864b0} + false + + + {10615b24-73bf-4efa-93aa-236916321317} + false + + + {86937f53-c189-40ef-8ce8-8759d8e7d480} + false + + + + + + diff --git a/PCbuild/_ssl.vcxproj.filters b/PCbuild/_ssl.vcxproj.filters index 78b1459..7cd1b5d 100644 --- a/PCbuild/_ssl.vcxproj.filters +++ b/PCbuild/_ssl.vcxproj.filters @@ -1,13 +1,13 @@ - - - - - {695348f7-e9f6-4fe1-bc03-5f08ffc8095b} - - - - - Source Files - - + + + + + {695348f7-e9f6-4fe1-bc03-5f08ffc8095b} + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_testbuffer.vcxproj b/PCbuild/_testbuffer.vcxproj index 1f45b29..cdc4a9b 100644 --- a/PCbuild/_testbuffer.vcxproj +++ b/PCbuild/_testbuffer.vcxproj @@ -1,82 +1,82 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {A2697BD3-28C1-4AEC-9106-8B748639FD16} - _testbuffer - Win32Proj - false - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.40219.1 - - - - 0x1e1F0000 - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {A2697BD3-28C1-4AEC-9106-8B748639FD16} + _testbuffer + Win32Proj + false + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.40219.1 + + + + 0x1e1F0000 + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + + + + + \ No newline at end of file diff --git a/PCbuild/_testbuffer.vcxproj.filters b/PCbuild/_testbuffer.vcxproj.filters index eadf282..e3ff428 100644 --- a/PCbuild/_testbuffer.vcxproj.filters +++ b/PCbuild/_testbuffer.vcxproj.filters @@ -1,13 +1,13 @@ - - - - - {8d232240-921a-4bc2-87c3-93ffd3462f0a} - - - - - Source Files - - + + + + + {8d232240-921a-4bc2-87c3-93ffd3462f0a} + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 365b07c..8d3d321 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -1,83 +1,83 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {6901D91C-6E48-4BB7-9FEC-700C8131DF1D} - _testcapi - Win32Proj - false - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - 0x1e1F0000 - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {6901D91C-6E48-4BB7-9FEC-700C8131DF1D} + _testcapi + Win32Proj + false + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + 0x1e1F0000 + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 7a6ad1a..04735f7 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -1,13 +1,13 @@ - - - - - {a76a90d8-8e8b-4c36-8f58-8bd46abe9f5e} - - - - - Source Files - - + + + + + {a76a90d8-8e8b-4c36-8f58-8bd46abe9f5e} + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_testconsole.vcxproj b/PCbuild/_testconsole.vcxproj index d351ced..59dd426 100644 --- a/PCbuild/_testconsole.vcxproj +++ b/PCbuild/_testconsole.vcxproj @@ -1,83 +1,83 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {B244E787-C445-441C-BDF4-5A4F1A3A1E51} - Win32Proj - _testconsole - false - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - - _CONSOLE;%(PreprocessorDefinitions) - - - Console - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {B244E787-C445-441C-BDF4-5A4F1A3A1E51} + Win32Proj + _testconsole + false + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + + _CONSOLE;%(PreprocessorDefinitions) + + + Console + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/_testconsole.vcxproj.filters b/PCbuild/_testconsole.vcxproj.filters index 0c25101..f78f3ce 100644 --- a/PCbuild/_testconsole.vcxproj.filters +++ b/PCbuild/_testconsole.vcxproj.filters @@ -1,22 +1,22 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_testembed.vcxproj b/PCbuild/_testembed.vcxproj index 14a926e..5f6fd6e 100644 --- a/PCbuild/_testembed.vcxproj +++ b/PCbuild/_testembed.vcxproj @@ -1,80 +1,80 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {6DAC66D9-E703-4624-BE03-49112AB5AA62} - Win32Proj - _testembed - false - - - - - Application - Unicode - - - - - - - - - - - - _CONSOLE;%(PreprocessorDefinitions) - - - Console - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {6DAC66D9-E703-4624-BE03-49112AB5AA62} + Win32Proj + _testembed + false + + + + + Application + Unicode + + + + + + + + + + + + _CONSOLE;%(PreprocessorDefinitions) + + + Console + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/_testembed.vcxproj.filters b/PCbuild/_testembed.vcxproj.filters index f7f9abe..5a1b9f4 100644 --- a/PCbuild/_testembed.vcxproj.filters +++ b/PCbuild/_testembed.vcxproj.filters @@ -1,22 +1,22 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_testimportmultiple.vcxproj b/PCbuild/_testimportmultiple.vcxproj index 37c1a64..531868a 100644 --- a/PCbuild/_testimportmultiple.vcxproj +++ b/PCbuild/_testimportmultiple.vcxproj @@ -1,83 +1,83 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781} - _testimportmultiple - Win32Proj - false - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - 0x1e1F0000 - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {36D0C52C-DF4E-45D0-8BC7-E294C3ABC781} + _testimportmultiple + Win32Proj + false + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + 0x1e1F0000 + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + + diff --git a/PCbuild/_testimportmultiple.vcxproj.filters b/PCbuild/_testimportmultiple.vcxproj.filters index db0d0a6..42fa8cf 100644 --- a/PCbuild/_testimportmultiple.vcxproj.filters +++ b/PCbuild/_testimportmultiple.vcxproj.filters @@ -1,13 +1,13 @@ - - - - - {1ec38ad9-1abf-4b80-8628-ac43ccba324b} - - - - - Source Files - - - + + + + + {1ec38ad9-1abf-4b80-8628-ac43ccba324b} + + + + + Source Files + + + diff --git a/PCbuild/_testmultiphase.vcxproj b/PCbuild/_testmultiphase.vcxproj index 106927c..0245a72 100644 --- a/PCbuild/_testmultiphase.vcxproj +++ b/PCbuild/_testmultiphase.vcxproj @@ -1,83 +1,83 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {16BFE6F0-22EF-40B5-B831-7E937119EF10} - Win32Proj - _testmultiphase - false - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - - _CONSOLE;%(PreprocessorDefinitions) - - - Console - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {16BFE6F0-22EF-40B5-B831-7E937119EF10} + Win32Proj + _testmultiphase + false + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + + _CONSOLE;%(PreprocessorDefinitions) + + + Console + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/_testmultiphase.vcxproj.filters b/PCbuild/_testmultiphase.vcxproj.filters index 0c25101..f78f3ce 100644 --- a/PCbuild/_testmultiphase.vcxproj.filters +++ b/PCbuild/_testmultiphase.vcxproj.filters @@ -1,22 +1,22 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/_tkinter.vcxproj b/PCbuild/_tkinter.vcxproj index 6793106..9766115 100644 --- a/PCbuild/_tkinter.vcxproj +++ b/PCbuild/_tkinter.vcxproj @@ -1,94 +1,94 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {4946ECAC-2E69-4BF8-A90A-F5136F5094DF} - _tkinter - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - $(tcltkDir)include;%(AdditionalIncludeDirectories) - WITH_APPINIT;%(PreprocessorDefinitions) - Py_TCLTK_DIR="$(tcltkDir.TrimEnd('\').Replace('\', '\\'))";%(PreprocessorDefinitions) - - - $(tcltkLib);%(AdditionalDependencies) - - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - {b5fd6f1d-129e-4bff-9340-03606fac7283} - - - {7e85eccf-a72c-4da4-9e52-884508e80ba1} - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {4946ECAC-2E69-4BF8-A90A-F5136F5094DF} + _tkinter + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + $(tcltkDir)include;%(AdditionalIncludeDirectories) + WITH_APPINIT;%(PreprocessorDefinitions) + Py_TCLTK_DIR="$(tcltkDir.TrimEnd('\').Replace('\', '\\'))";%(PreprocessorDefinitions) + + + $(tcltkLib);%(AdditionalDependencies) + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + {b5fd6f1d-129e-4bff-9340-03606fac7283} + + + {7e85eccf-a72c-4da4-9e52-884508e80ba1} + + + + + \ No newline at end of file diff --git a/PCbuild/_tkinter.vcxproj.filters b/PCbuild/_tkinter.vcxproj.filters index 4e5cd47..c63b5fe 100644 --- a/PCbuild/_tkinter.vcxproj.filters +++ b/PCbuild/_tkinter.vcxproj.filters @@ -1,16 +1,16 @@ - - - - - {b9ce64dd-cb95-472d-bbe8-5583b2cd375b} - - - - - Source Files - - - Source Files - - + + + + + {b9ce64dd-cb95-472d-bbe8-5583b2cd375b} + + + + + Source Files + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/build.bat b/PCbuild/build.bat index f7f2858..41be720 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -48,7 +48,6 @@ exit /b 127 :Run setlocal set platf=Win32 -set vs_platf=x86 set conf=Release set target=Build set dir=%~dp0 @@ -57,10 +56,6 @@ set verbose=/nologo /v:m set kill= set do_pgo= set pgo_job=-m test --pgo -set on_64_bit=true - -rem This may not be 100% accurate, but close enough. -if "%ProgramFiles(x86)%"=="" (set on_64_bit=false) :CheckOpts if "%~1"=="-h" goto Usage @@ -90,18 +85,12 @@ if "%IncludeTkinter%"=="" set IncludeTkinter=true if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" -if "%platf%"=="x64" ( - if "%on_64_bit%"=="true" ( - rem This ought to always be correct these days... - set vs_platf=amd64 - ) else ( - if "%do_pgo%"=="true" ( - echo.ERROR: Cannot cross-compile with PGO - echo. 32bit operating system detected, if this is incorrect, - echo. make sure the ProgramFiles(x86^) environment variable is set - exit /b 1 - ) - set vs_platf=x86_amd64 +if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" ( + if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" ( + echo.ERROR: Cannot cross-compile with PGO + echo. 32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE + echo. and PROCESSOR_ARCHITEW6432 environment variables are correct. + exit /b 1 ) ) @@ -110,7 +99,8 @@ if exist "%GIT%" set GITProperty=/p:GIT="%GIT%" if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty= rem Setup the environment -call "%dir%env.bat" %vs_platf% >nul +call "%dir%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if "%kill%"=="true" call :Kill @@ -129,7 +119,7 @@ if "%do_pgo%"=="true" ( goto Build :Kill echo on -msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ +%MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ /p:Configuration=%conf% /p:Platform=%platf%^ /p:KillPython=true @@ -141,7 +131,7 @@ rem Call on MSBuild to do the work, echo the command. rem Passing %1-9 is not the preferred option, but argument parsing in rem batch is, shall we say, "lackluster" echo on -msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ +%MSBUILD% "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ /p:Configuration=%conf% /p:Platform=%platf%^ /p:IncludeExternals=%IncludeExternals%^ /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ @@ -153,4 +143,4 @@ goto :eof :Version rem Display the current build version information -msbuild "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 +%MSBUILD% "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/PCbuild/find_msbuild.bat b/PCbuild/find_msbuild.bat new file mode 100644 index 0000000..1a8bfa8 --- /dev/null +++ b/PCbuild/find_msbuild.bat @@ -0,0 +1,51 @@ +@rem +@rem Searches for MSBuild.exe. This is the only tool we need to initiate +@rem a build, so we no longer search for the full VC toolset. +@rem +@rem This file is supposed to modify the state of the caller (specifically +@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid +@rem changing any other persistent state. +@rem + +@rem No arguments provided means do full search +@if '%1' EQU '' goto :begin_search + +@rem One argument may be the full path. Use a goto so we don't try to +@rem parse the next if statement - incorrect quoting in the multi-arg +@rem case can cause us to break immediately. +@if '%2' EQU '' goto :one_arg + +@rem Entire command line may represent the full path if quoting failed. +@if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found +@goto :begin_search + +:one_arg +@if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found + +:begin_search +@set MSBUILD= + +@rem If msbuild.exe is on the PATH, assume that the user wants that one. +@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc" +@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found + +@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there. +@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul +@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @( + @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe") +) +@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found + +@rem VS 2015 and earlier register MSBuild separately, so we can find it. +@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul +@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @( + @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe") +) +@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found + + +@exit /b 1 + +:found +@echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%) +@set _Py_MSBuild_Source= diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py new file mode 100644 index 0000000..a682d38 --- /dev/null +++ b/PCbuild/get_external.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +import argparse +import os +import pathlib +import zipfile +from urllib.request import urlretrieve + + +def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): + repo = f'cpython-{"bin" if binary else "source"}-deps' + url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip' + reporthook = None + if verbose: + reporthook = print + zip_dir.mkdir(parents=True, exist_ok=True) + filename, headers = urlretrieve( + url, + zip_dir / f'{commit_hash}.zip', + reporthook=reporthook, + ) + return filename + + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-b', '--binary', action='store_true', + help='Is the dependency in the binary repo?') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='python') + p.add_argument('-e', '--externals-dir', type=pathlib.Path, + help='Directory in which to store dependencies', + default=pathlib.Path(__file__).parent.parent / 'externals') + p.add_argument('tag', + help='tag of the dependency') + return p.parse_args() + + +def main(): + args = parse_args() + zip_path = fetch_zip( + args.tag, + args.externals_dir / 'zips', + org=args.organization, + binary=args.binary, + verbose=args.verbose, + ) + final_name = args.externals_dir / args.tag + extract_zip(args.externals_dir, zip_path).replace(final_name) + + +if __name__ == '__main__': + main() diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 98a755d..a04a81e 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -2,58 +2,59 @@ setlocal rem Simple script to fetch source for external libraries -if not exist "%~dp0..\externals" mkdir "%~dp0..\externals" -pushd "%~dp0..\externals" +if "%PCBUILD%"=="" (set PCBUILD=%~dp0) +if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals) +if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\nuget.exe) +if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl) -if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/ +set DO_FETCH=true +set DO_CLEAN=false -rem Optionally clean up first. Be warned that this can be very destructive! -if not "%1"=="" ( - for %%c in (-c --clean --clean-only) do ( - if "%1"=="%%c" goto clean - ) - goto usage -) -goto fetch +:CheckOpts +if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts +if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts +if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts +if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts +if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts +if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts +if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean +if "x%~1" NEQ "x" goto usage +if "%DO_CLEAN%"=="false" goto fetch :clean echo.Cleaning up external libraries. -for /D %%d in ( - bzip2-* - db-* - nasm-* - openssl-* - tcl-* - tcltk* - tk-* - tix-* - sqlite-* - xz-* - ) do ( - echo.Removing %%d - rmdir /s /q %%d -) -if "%1"=="--clean-only" ( - goto end +if exist "%EXTERNALS_DIR%" ( + rem Sometimes this fails the first time; try it twice + rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%" ) +if "%DO_FETCH%"=="false" goto end :fetch -rem Fetch current versions - -svn --version > nul 2>&1 -if ERRORLEVEL 9009 ( - echo.svn.exe must be on your PATH. - echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the - echo.command line tools option. - popd - exit /b 1 + +if "%ORG%"=="" (set ORG=python) + +if "%PYTHON_FOR_BUILD%"=="" ( + echo Checking for installed python... + py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6) +) +if "%PYTHON_FOR_BUILD%"=="" ( + if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%" + if NOT exist "%NUGET%" ( + echo Downloading nuget... + rem NB: Must use single quotes around NUGET here, NOT double! + rem Otherwise, a space in the path would break things + powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%' + ) + echo Installing Python via nuget... + "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%" + rem Quote it here; it's not quoted later because "py -3.6" wouldn't work + set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe" ) echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.6 -if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06 if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k set libraries=%libraries% sqlite-3.14.2.0 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0 @@ -62,43 +63,48 @@ if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.6 set libraries=%libraries% xz-5.2.2 for %%e in (%libraries%) do ( - if exist %%e ( + if exist "%EXTERNALS_DIR%\%%e" ( echo.%%e already exists, skipping. ) else ( echo.Fetching %%e... - svn export %SVNROOT%%%e + %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -O %ORG% %%e + ) +) + +echo.Fetching external binaries... + +set binaries= +set binaries=%binaries% +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% nasm-2.11.06 + +for %%b in (%binaries%) do ( + if exist "%EXTERNALS_DIR%\%%b" ( + echo.%%b already exists, skipping. + ) else ( + echo.Fetching %%b... + %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -b -O %ORG% %%b ) ) +echo Finished. goto end :usage -echo.invalid argument: %1 -echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ] +echo.Valid options: -c, --clean, --clean-only, --organization, --python, +echo.--no-tkinter, --no-openssl echo. -echo.Pull all sources necessary for compiling optional extension modules -echo.that rely on external libraries. Requires svn.exe to be on your PATH -echo.and pulls sources from %SVNROOT%. +echo.Pull all sources and binaries necessary for compiling optional extension +echo.modules that rely on external libraries. echo. -echo.Use the -c or --clean option to clean up all external library sources -echo.before pulling in the current versions. +echo.The --organization option determines which github organization to download +echo.from, the --python option determines which Python 3.6+ interpreter to use +echo.with PCbuild\get_external.py. +echo. +echo.Use the -c or --clean option to remove the entire externals directory. echo. echo.Use the --clean-only option to do the same cleaning, without pulling in echo.anything new. echo. -echo.Only the first argument is checked, all others are ignored. -echo. -echo.**WARNING**: the cleaning options unconditionally remove any directory -echo.that is a child of -echo. %CD% -echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-, -echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential -echo.to be very destructive if you are not aware of what it is doing. Use with -echo.caution! -popd exit /b -1 - :end -echo Finished. -popd diff --git a/PCbuild/libeay.vcxproj b/PCbuild/libeay.vcxproj index 9662cd4..cd59401 100644 --- a/PCbuild/libeay.vcxproj +++ b/PCbuild/libeay.vcxproj @@ -1,907 +1,907 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Debug - x64 - - - Release - x64 - - - - {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0} - libeay - - - - - - - StaticLibrary - - - - - - - - <_DATEValue>#define DATE "$([System.DateTime]::Now.ToString(`ddd MMM dd HH':'mm':'ss yyyy`))" - <_CFLAGSValue>#define CFLAGS "cl /MD /Ox -W3 -Gs0 -Gy -nologo @(PreprocessorDefinitions->'-D%(Identity)',' ')" - <_PLATFORMValue Condition="$(Platform)=='Win32'">#define PLATFORM "VC-WIN32" - <_PLATFORMValue Condition="$(Platform)=='x64'">#define PLATFORM "VC-WIN64A" - - - - - - - - - - - - $(IntDir);%(AdditionalIncludeDirectories) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Release + Win32 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Debug + x64 + + + Release + x64 + + + + {E5B04CC0-EB4C-42AB-B4DC-18EF95F864B0} + libeay + + + + + + + StaticLibrary + + + + + + + + <_DATEValue>#define DATE "$([System.DateTime]::Now.ToString(`ddd MMM dd HH':'mm':'ss yyyy`))" + <_CFLAGSValue>#define CFLAGS "cl /MD /Ox -W3 -Gs0 -Gy -nologo @(PreprocessorDefinitions->'-D%(Identity)',' ')" + <_PLATFORMValue Condition="$(Platform)=='Win32'">#define PLATFORM "VC-WIN32" + <_PLATFORMValue Condition="$(Platform)=='x64'">#define PLATFORM "VC-WIN64A" + + + + + + + + + + + + $(IntDir);%(AdditionalIncludeDirectories) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/PCbuild/liblzma.vcxproj b/PCbuild/liblzma.vcxproj index 711f9bd..b33c2c2 100644 --- a/PCbuild/liblzma.vcxproj +++ b/PCbuild/liblzma.vcxproj @@ -1,216 +1,216 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Debug - x64 - - - Release - x64 - - - - {12728250-16EC-4DC6-94D7-E21DD88947F8} - liblzma - true - - - - - - - StaticLibrary - - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - WIN32;HAVE_CONFIG_H;_DEBUG;_LIB;%(PreprocessorDefinitions) - Level3 - ProgramDatabase - Disabled - $(lzmaDir)windows;$(lzmaDir)src/liblzma/common;$(lzmaDir)src/common;$(lzmaDir)src/liblzma/api;$(lzmaDir)src/liblzma/check;$(lzmaDir)src/liblzma/delta;$(lzmaDir)src/liblzma/lz;$(lzmaDir)src/liblzma/lzma;$(lzmaDir)src/liblzma/rangecoder;$(lzmaDir)src/liblzma/simple - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Release + Win32 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Debug + x64 + + + Release + x64 + + + + {12728250-16EC-4DC6-94D7-E21DD88947F8} + liblzma + true + + + + + + + StaticLibrary + + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + WIN32;HAVE_CONFIG_H;_DEBUG;_LIB;%(PreprocessorDefinitions) + Level3 + ProgramDatabase + Disabled + $(lzmaDir)windows;$(lzmaDir)src/liblzma/common;$(lzmaDir)src/common;$(lzmaDir)src/liblzma/api;$(lzmaDir)src/liblzma/check;$(lzmaDir)src/liblzma/delta;$(lzmaDir)src/liblzma/lz;$(lzmaDir)src/liblzma/lzma;$(lzmaDir)src/liblzma/rangecoder;$(lzmaDir)src/liblzma/simple + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/PCbuild/openssl.props b/PCbuild/openssl.props index 0de4e43..3d695c8 100644 --- a/PCbuild/openssl.props +++ b/PCbuild/openssl.props @@ -1,76 +1,76 @@ - - - - $(opensslDir)tmp\ - - - - - - StaticLibrary - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <_PreprocessorDefinitionList>@(PreprocessorDefinitions) - - - - - - 4244;4267 - $(opensslDir);$(opensslIncludeDir);$(opensslDir)crypto;$(opensslDir)crypto\asn1;$(opensslDir)crypto\evp;$(opensslDir)crypto\modes - $(_PreprocessorDefinitionList);%(PreprocessorDefinitions) - - - - - - nasm.exe -f win32 - nasm.exe -f win64 -DNEAR -Ox -g - - - - - - - - - - - + + + + $(opensslDir)tmp\ + + + + + + StaticLibrary + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_PreprocessorDefinitionList>@(PreprocessorDefinitions) + + + + + + 4244;4267 + $(opensslDir);$(opensslIncludeDir);$(opensslDir)crypto;$(opensslDir)crypto\asn1;$(opensslDir)crypto\evp;$(opensslDir)crypto\modes + $(_PreprocessorDefinitionList);%(PreprocessorDefinitions) + + + + + + nasm.exe -f win32 + nasm.exe -f win64 -DNEAR -Ox -g + + + + + + + + + + + diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index c6b8487..3b6a134 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -1,120 +1,120 @@ - - - - {CC9B93A2-439D-4058-9D29-6DCF43774405} - Win32 - Release - true - true - true - true - true - - - - - $(Platform) - $(Configuration) - - Build - Clean - CleanAll - true - - - $(Platform) - $(Configuration) - - Build - Clean - CleanAll - false - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - - - - + + + + {CC9B93A2-439D-4058-9D29-6DCF43774405} + Win32 + Release + true + true + true + true + true + + + + + $(Platform) + $(Configuration) + + Build + Clean + CleanAll + true + + + $(Platform) + $(Configuration) + + Build + Clean + CleanAll + false + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 0e65811..5808799 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -96,6 +96,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsol EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblzma", "liblzma.vcxproj", "{12728250-16EC-4DC6-94D7-E21DD88947F8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -728,6 +730,38 @@ Global {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|Win32.Build.0 = Release|Win32 {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.ActiveCfg = Release|x64 {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Release|x64.Build.0 = Release|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.ActiveCfg = Debug|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.Build.0 = Debug|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|x64.ActiveCfg = Debug|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|x64.Build.0 = Debug|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.ActiveCfg = Release|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.Build.0 = Release|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.ActiveCfg = Release|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.Build.0 = Release|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.ActiveCfg = Debug|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.Build.0 = Debug|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.ActiveCfg = Debug|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.Build.0 = Debug|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.ActiveCfg = Release|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|Win32.Build.0 = Release|Win32 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.ActiveCfg = Release|x64 + {12728250-16EC-4DC6-94D7-E21DD88947F8}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index 99c7286..5e28748 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -1,87 +1,87 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - <_ProjectFileVersion>10.0.30319.1 - {D06B6426-4762-44CC-8BAD-D79052507F2F} - pyexpat - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - - $(PySourcePath)Modules\expat;%(AdditionalIncludeDirectories) - PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_NS;XML_DTD;BYTEORDER=1234;XML_CONTEXT_BYTES=1024;XML_STATIC;HAVE_MEMMOVE;%(PreprocessorDefinitions) - - - - - - - - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + <_ProjectFileVersion>10.0.30319.1 + {D06B6426-4762-44CC-8BAD-D79052507F2F} + pyexpat + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + + $(PySourcePath)Modules\expat;%(AdditionalIncludeDirectories) + _CRT_SECURE_NO_WARNINGS;PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_STATIC;%(PreprocessorDefinitions) + + + + + + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters index abf362f..bfa04d5 100644 --- a/PCbuild/pyexpat.vcxproj.filters +++ b/PCbuild/pyexpat.vcxproj.filters @@ -1,33 +1,33 @@ - - - - - {ddae77a6-7ca0-4a1b-b71c-deea5f4025de} - - - {5af9d40c-fc46-4640-ad84-3d1dd34a71d7} - - - - - Header Files - - - Header Files - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - + + + + + {ddae77a6-7ca0-4a1b-b71c-deea5f4025de} + + + {5af9d40c-fc46-4640-ad84-3d1dd34a71d7} + + + + + Header Files + + + Header Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/pylauncher.vcxproj b/PCbuild/pylauncher.vcxproj index 2d4b5f6..298be3f 100644 --- a/PCbuild/pylauncher.vcxproj +++ b/PCbuild/pylauncher.vcxproj @@ -1,82 +1,82 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {7B2727B5-5A3F-40EE-A866-43A13CD31446} - pylauncher - py - false - - - - - Application - MultiByte - - - - - - ClCompile - - - - - - - - - _CONSOLE;%(PreprocessorDefinitions) - MultiThreaded - - - version.lib;%(AdditionalDependencies) - Console - - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {7B2727B5-5A3F-40EE-A866-43A13CD31446} + pylauncher + py + false + + + + + Application + MultiByte + + + + + + ClCompile + + + + + + + + + _CONSOLE;%(PreprocessorDefinitions) + MultiThreaded + + + version.lib;%(AdditionalDependencies) + Console + + + + + + + + + + + + + + + diff --git a/PCbuild/pylauncher.vcxproj.filters b/PCbuild/pylauncher.vcxproj.filters index e4b23d2..05a93d0 100644 --- a/PCbuild/pylauncher.vcxproj.filters +++ b/PCbuild/pylauncher.vcxproj.filters @@ -1,32 +1,32 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - - - Resource Files - - - - - Resource Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + + + Resource Files + + + + + Resource Files + + \ No newline at end of file diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 7012170..fd25809 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -1,163 +1,163 @@ - - - - <_ProjectFileVersion>10.0.30319.1 - 10.0 - $(BuildPath) - $(OutDir)\ - $(MSBuildThisFileDirectory)obj\ - $(Py_IntDir)\$(ArchName)_$(Configuration)\$(ProjectName)\ - $(ProjectName) - $(TargetName)$(PyDebugExt) - false - false - true - true - false - false - - - - <_DebugPreprocessorDefinition>NDEBUG; - <_DebugPreprocessorDefinition Condition="$(Configuration) == 'Debug'">_DEBUG; - <_PlatformPreprocessorDefinition>_WIN32; - <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64;_M_X64; - <_PydPreprocessorDefinition Condition="$(TargetExt) == '.pyd'">Py_BUILD_CORE_MODULE; - - - - $(PySourcePath)Include;$(PySourcePath)PC;$(IntDir);%(AdditionalIncludeDirectories) - WIN32;$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) - - MaxSpeed - true - true - - MultiThreadedDLL - true - Level3 - ProgramDatabase - Default - true - true - NoExtensions - - - Disabled - false - MultiThreadedDebugDLL - - - Strict - - - $(OutDir);%(AdditionalLibraryDirectories) - true - $(OutDir)$(TargetName).pdb - Windows - true - true - true - LIBC;%(IgnoreSpecificDefaultLibraries) - MachineX86 - MachineX64 - $(OutDir)$(TargetName).pgd - UseLinkTimeCodeGeneration - PGInstrument - PGUpdate - - - true - true - true - - - $(PySourcePath)PC;$(PySourcePath)Include;$(IntDir);%(AdditionalIncludeDirectories) - $(_DebugPreprocessorDefinition)%(PreprocessorDefinitions) - 0x0409 - - - $(_DebugPreprocessorDefinition)%(PreprocessorDefinitions) - true - true - Win32 - X64 - $(IntDir) - $(MSBuildProjectName)_i.c - $(MSBuildProjectName)_p.c - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot10)\bin\x86 - $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot81)\bin\x86 - $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot)\bin\x86 - $(registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A@InstallationFolder)\Bin\ - <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" - <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe" - - - - - - + + + + <_ProjectFileVersion>10.0.30319.1 + 10.0 + $(BuildPath) + $(OutDir)\ + $(MSBuildThisFileDirectory)obj\ + $(Py_IntDir)\$(ArchName)_$(Configuration)\$(ProjectName)\ + $(ProjectName) + $(TargetName)$(PyDebugExt) + false + false + true + true + false + false + + + + <_DebugPreprocessorDefinition>NDEBUG; + <_DebugPreprocessorDefinition Condition="$(Configuration) == 'Debug'">_DEBUG; + <_PlatformPreprocessorDefinition>_WIN32; + <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64;_M_X64; + <_PydPreprocessorDefinition Condition="$(TargetExt) == '.pyd'">Py_BUILD_CORE_MODULE; + + + + $(PySourcePath)Include;$(PySourcePath)PC;$(IntDir);%(AdditionalIncludeDirectories) + WIN32;$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) + + MaxSpeed + true + true + + MultiThreadedDLL + true + Level3 + ProgramDatabase + Default + true + true + NoExtensions + + + Disabled + false + MultiThreadedDebugDLL + + + Strict + + + $(OutDir);%(AdditionalLibraryDirectories) + true + $(OutDir)$(TargetName).pdb + Windows + true + true + true + LIBC;%(IgnoreSpecificDefaultLibraries) + MachineX86 + MachineX64 + $(OutDir)$(TargetName).pgd + UseLinkTimeCodeGeneration + PGInstrument + PGUpdate + + + true + true + true + + + $(PySourcePath)PC;$(PySourcePath)Include;$(IntDir);%(AdditionalIncludeDirectories) + $(_DebugPreprocessorDefinition)%(PreprocessorDefinitions) + 0x0409 + + + $(_DebugPreprocessorDefinition)%(PreprocessorDefinitions) + true + true + Win32 + X64 + $(IntDir) + $(MSBuildProjectName)_i.c + $(MSBuildProjectName)_p.c + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot10)\bin\x86 + $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot81)\bin\x86 + $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot)\bin\x86 + $(registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A@InstallationFolder)\Bin\ + <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" + <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe" + + + + + + \ No newline at end of file diff --git a/PCbuild/pyshellext.vcxproj b/PCbuild/pyshellext.vcxproj index 0293935..b5b0d13 100644 --- a/PCbuild/pyshellext.vcxproj +++ b/PCbuild/pyshellext.vcxproj @@ -1,87 +1,87 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} - pyshellext - pyshellext - false - - - - - DynamicLibrary - Unicode - - - - - - ClCompile - - - - - - - - - _CONSOLE;%(PreprocessorDefinitions) - - - version.lib;shlwapi.lib;%(AdditionalDependencies) - Console - ..\PC\pyshellext$(PyDebugExt).def - - - true - - - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} + pyshellext + pyshellext + false + + + + + DynamicLibrary + Unicode + + + + + + ClCompile + + + + + + + + + _CONSOLE;%(PreprocessorDefinitions) + + + version.lib;shlwapi.lib;%(AdditionalDependencies) + Console + ..\PC\pyshellext$(PyDebugExt).def + + + true + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/PCbuild/pyshellext.vcxproj.filters b/PCbuild/pyshellext.vcxproj.filters index 648e499..d07e249 100644 --- a/PCbuild/pyshellext.vcxproj.filters +++ b/PCbuild/pyshellext.vcxproj.filters @@ -1,40 +1,40 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - - - Source Files - - - - - Resource Files - - - - - Source Files - - - Source Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + + + Source Files + + + + + Resource Files + + + + + Source Files + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/python.props b/PCbuild/python.props index d6bfd08..a74135e 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -1,172 +1,172 @@ - - - - Win32 - Release - - v140 - v120 - v110 - v100 - - $(BasePlatformToolset) - false - true - - - amd64 - win32 - - - $([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\)) - $(PySourcePath)\ - - - $(PySourcePath)PCBuild\win32\ - $(Py_OutDir)\win32\ - $(PySourcePath)PCBuild\amd64\ - $(Py_OutDir)\amd64\ - $(BuildPath32) - $(BuildPath64) - $(PySourcePath)PCBuild\$(ArchName)\ - $(BuildPath)\ - - - $([System.IO.Path]::GetFullPath(`$(PySourcePath)externals\`)) - $(ExternalsDir)sqlite-3.14.2.0\ - $(ExternalsDir)bzip2-1.0.6\ - $(ExternalsDir)xz-5.2.2\ - $(ExternalsDir)openssl-1.0.2k\ - $(opensslDir)include32 - $(opensslDir)include64 - $(ExternalsDir)\nasm-2.11.06\ - - - _d - - - -test - - - -32 - - - $(BuildPath)python$(PyDebugExt).exe - - - - - <_PatchLevelContent>$([System.IO.File]::ReadAllText(`$(PySourcePath)Include\patchlevel.h`)) - $([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_MAJOR_VERSION\s+(\d+)`).Groups[1].Value) - $([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_MINOR_VERSION\s+(\d+)`).Groups[1].Value) - $([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_MICRO_VERSION\s+(\d+)`).Groups[1].Value) - <_ReleaseLevel>$([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_RELEASE_LEVEL\s+PY_RELEASE_LEVEL_(\w+)`).Groups[1].Value) - $([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_RELEASE_SERIAL\s+(\d+)`).Groups[1].Value) - 15 - 10 - 11 - 12 - a$(ReleaseSerial) - b$(ReleaseSerial) - rc$(ReleaseSerial) - - - - - $([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[1].Value) - $([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[2].Value) - $([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[3].Value) - $([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[4].Value) - <_ReleaseLevel>$([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[5].Value) - $([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[6].Value) - 0 - 15 - 10 - 11 - 12 - - - - $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber) - $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)$(ReleaseLevelName) - $([msbuild]::BitwiseOr( - $([msbuild]::Multiply($(MajorVersionNumber), 16777216)), - $([msbuild]::BitwiseOr( - $([msbuild]::Multiply($(MinorVersionNumber), 65536)), - $([msbuild]::BitwiseOr( - $([msbuild]::Multiply($(MicroVersionNumber), 256)), - $([msbuild]::BitwiseOr( - $([msbuild]::Multiply($(ReleaseLevelNumber), 16)), - $(ReleaseSerial) - )) - )) - )) - )) - $([msbuild]::Add( - $(ReleaseSerial), - $([msbuild]::Add( - $([msbuild]::Multiply($(ReleaseLevelNumber), 10)), - $([msbuild]::Multiply($(MicroVersionNumber), 1000)) - )) - )) - $([msbuild]::Add($(Field3Value), 9000)) - - - python$(MajorVersionNumber)$(MinorVersionNumber)$(PyDebugExt) - - - .cp$(MajorVersionNumber)$(MinorVersionNumber)-win32 - .cp$(MajorVersionNumber)$(MinorVersionNumber)-win_amd64 - - - $(MajorVersionNumber).$(MinorVersionNumber)$(PyArchExt)$(PyTestExt) - - - - - - - - - - - - + + + + Win32 + Release + + v140 + v120 + v110 + v100 + + $(BasePlatformToolset) + false + true + + + amd64 + win32 + + + $([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\)) + $(PySourcePath)\ + + + $(PySourcePath)PCBuild\win32\ + $(Py_OutDir)\win32\ + $(PySourcePath)PCBuild\amd64\ + $(Py_OutDir)\amd64\ + $(BuildPath32) + $(BuildPath64) + $(PySourcePath)PCBuild\$(ArchName)\ + $(BuildPath)\ + + + $([System.IO.Path]::GetFullPath(`$(PySourcePath)externals\`)) + $(ExternalsDir)sqlite-3.14.2.0\ + $(ExternalsDir)bzip2-1.0.6\ + $(ExternalsDir)xz-5.2.2\ + $(ExternalsDir)openssl-1.0.2k\ + $(opensslDir)include32 + $(opensslDir)include64 + $(ExternalsDir)\nasm-2.11.06\ + + + _d + + + -test + + + -32 + + + $(BuildPath)python$(PyDebugExt).exe + + + + + <_PatchLevelContent>$([System.IO.File]::ReadAllText(`$(PySourcePath)Include\patchlevel.h`)) + $([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_MAJOR_VERSION\s+(\d+)`).Groups[1].Value) + $([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_MINOR_VERSION\s+(\d+)`).Groups[1].Value) + $([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_MICRO_VERSION\s+(\d+)`).Groups[1].Value) + <_ReleaseLevel>$([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_RELEASE_LEVEL\s+PY_RELEASE_LEVEL_(\w+)`).Groups[1].Value) + $([System.Text.RegularExpressions.Regex]::Match($(_PatchLevelContent), `define\s+PY_RELEASE_SERIAL\s+(\d+)`).Groups[1].Value) + 15 + 10 + 11 + 12 + a$(ReleaseSerial) + b$(ReleaseSerial) + rc$(ReleaseSerial) + + + + + $([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[1].Value) + $([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[2].Value) + $([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[3].Value) + $([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[4].Value) + <_ReleaseLevel>$([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[5].Value) + $([System.Text.RegularExpressions.Regex]::Match($(OverrideVersion), `(\d+)\.(\d+)\.(\d+)((a|b|rc)(\d))?`).Groups[6].Value) + 0 + 15 + 10 + 11 + 12 + + + + $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber) + $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)$(ReleaseLevelName) + $([msbuild]::BitwiseOr( + $([msbuild]::Multiply($(MajorVersionNumber), 16777216)), + $([msbuild]::BitwiseOr( + $([msbuild]::Multiply($(MinorVersionNumber), 65536)), + $([msbuild]::BitwiseOr( + $([msbuild]::Multiply($(MicroVersionNumber), 256)), + $([msbuild]::BitwiseOr( + $([msbuild]::Multiply($(ReleaseLevelNumber), 16)), + $(ReleaseSerial) + )) + )) + )) + )) + $([msbuild]::Add( + $(ReleaseSerial), + $([msbuild]::Add( + $([msbuild]::Multiply($(ReleaseLevelNumber), 10)), + $([msbuild]::Multiply($(MicroVersionNumber), 1000)) + )) + )) + $([msbuild]::Add($(Field3Value), 9000)) + + + python$(MajorVersionNumber)$(MinorVersionNumber)$(PyDebugExt) + + + .cp$(MajorVersionNumber)$(MinorVersionNumber)-win32 + .cp$(MajorVersionNumber)$(MinorVersionNumber)-win_amd64 + + + $(MajorVersionNumber).$(MinorVersionNumber)$(PyArchExt)$(PyTestExt) + + + + + + + + + + + + diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 807213f..edbc3c5 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -1,112 +1,115 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {B11D750F-CD1F-4A96-85CE-E69A5C5259F9} - - - - - Application - false - MultiByte - - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - _CONSOLE;%(PreprocessorDefinitions) - - - Console - 2000000 - 0x1d000000 - - - - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - - - - ucrtbase - ucrtbased - - - - - - <_Content>@rem This script invokes the most recently built Python with all arguments -@rem passed through to the interpreter. This file is generated by the -@rem build process and any changes *will* be thrown away by the next -@rem rebuild. -@rem This is only meant as a convenience for developing CPython -@rem and using it outside of that context is ill-advised. -@echo Running $(Configuration)^|$(Platform) interpreter... -@setlocal -@set PYTHONHOME=$(PySourcePath) -@"$(OutDir)python$(PyDebugExt).exe" %* - - <_ExistingContent Condition="Exists('$(PySourcePath)python.bat')">$([System.IO.File]::ReadAllText('$(PySourcePath)python.bat')) - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {B11D750F-CD1F-4A96-85CE-E69A5C5259F9} + + + + + Application + false + MultiByte + + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + _CONSOLE;%(PreprocessorDefinitions) + + + Console + 2000000 + 0x1d000000 + + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + + + + ucrtbase + ucrtbased + + + + + + <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'Win32'">@set PATH=%PATH%%3B$(VCInstallDir)bin + <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'x64'">@set PATH=%PATH%%3B$(VCInstallDir)bin\amd64 + <_Content>@rem This script invokes the most recently built Python with all arguments +@rem passed through to the interpreter. This file is generated by the +@rem build process and any changes *will* be thrown away by the next +@rem rebuild. +@rem This is only meant as a convenience for developing CPython +@rem and using it outside of that context is ill-advised. +@echo Running $(Configuration)^|$(Platform) interpreter... +@setlocal +@set PYTHONHOME=$(PySourcePath) +$(_PGOPath) +@"$(OutDir)python$(PyDebugExt).exe" %* + + <_ExistingContent Condition="Exists('$(PySourcePath)python.bat')">$([System.IO.File]::ReadAllText('$(PySourcePath)python.bat')) + + + \ No newline at end of file diff --git a/PCbuild/python.vcxproj.filters b/PCbuild/python.vcxproj.filters index 0662a4e..958eee0 100644 --- a/PCbuild/python.vcxproj.filters +++ b/PCbuild/python.vcxproj.filters @@ -1,26 +1,26 @@ - - - - - {2d690795-de83-4a33-8235-3c5dafe45efa} - - - {8b010a19-5b29-45f1-a8a0-f672e2bbb11a} - - - - - Resource Files - - - - - Resource Files - - - - - Source Files - - - + + + + + {2d690795-de83-4a33-8235-3c5dafe45efa} + + + {8b010a19-5b29-45f1-a8a0-f672e2bbb11a} + + + + + Resource Files + + + + + Resource Files + + + + + Source Files + + + diff --git a/PCbuild/python3dll.vcxproj b/PCbuild/python3dll.vcxproj index c66c8ef..125e071 100644 --- a/PCbuild/python3dll.vcxproj +++ b/PCbuild/python3dll.vcxproj @@ -1,137 +1,137 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {885D4898-D08D-4091-9C40-C700CFE3FC5A} - python3dll - Win32Proj - python3 - false - - - - - DynamicLibrary - - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - <_Machine>X86 - <_Machine Condition="$(Platform) == 'x64'">X64 - $(ExtensionsToDeleteOnClean);$(IntDir)python3_d.def;$(IntDir)python3stub.def - - - - false - - - $(OutDir)$(TargetName)stub.lib - $(PySourcePath)PC\python3.def - $(IntDir)python3_d.def - DllMain - - - lib /nologo /def:"$(IntDir)python3stub.def" /out:"$(OutDir)$(TargetName)stub.lib" /MACHINE:$(_Machine) - Rebuilding $(TargetName)stub.lib - $(OutDir)$(TargetName)stub.lib - - - - - - - - - - - - - - - - - - <_DefLines Remove="@(_DefLines)" /> - <_Lines Remove="@(_Lines)" /> - - - - - - <_Pattern1>(=python$(MajorVersionNumber)$(MinorVersionNumber))\. - <_Sub1>$1_d. - <_Pattern2>"python3" - <_Sub2>"python3_d" - - - <_Lines Include="@(_DefLines)"> - $([System.Text.RegularExpressions.Regex]::Replace($([System.Text.RegularExpressions.Regex]::Replace(`%(Identity)`, `$(_Pattern1)`, `$(_Sub1)`)), `$(_Pattern2)`, `$(_Sub2)`)) - - - - - - - - - <_DefLines Remove="@(_DefLines)" /> - <_Lines Remove="@(_Lines)" /> - - - - - - <_Pattern>^[\w.]+=.+?\.([^ ]+).*$ - <_Sub>$1 - - - <_Lines Include="EXPORTS" /> - <_Symbols Include="@(_DefLines)" Condition="$([System.Text.RegularExpressions.Regex]::IsMatch(`%(Identity)`, `$(_Pattern)`))"> - $([System.Text.RegularExpressions.Regex]::Replace(`%(Identity)`, `$(_Pattern)`, `$(_Sub)`)) - - <_Lines Include="@(_Symbols->'%(Symbol)')" /> - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {885D4898-D08D-4091-9C40-C700CFE3FC5A} + python3dll + Win32Proj + python3 + false + + + + + DynamicLibrary + + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + <_Machine>X86 + <_Machine Condition="$(Platform) == 'x64'">X64 + $(ExtensionsToDeleteOnClean);$(IntDir)python3_d.def;$(IntDir)python3stub.def + + + + false + + + $(OutDir)$(TargetName)stub.lib + $(PySourcePath)PC\python3.def + $(IntDir)python3_d.def + DllMain + + + lib /nologo /def:"$(IntDir)python3stub.def" /out:"$(OutDir)$(TargetName)stub.lib" /MACHINE:$(_Machine) + Rebuilding $(TargetName)stub.lib + $(OutDir)$(TargetName)stub.lib + + + + + + + + + + + + + + + + + + <_DefLines Remove="@(_DefLines)" /> + <_Lines Remove="@(_Lines)" /> + + + + + + <_Pattern1>(=python$(MajorVersionNumber)$(MinorVersionNumber))\. + <_Sub1>$1_d. + <_Pattern2>"python3" + <_Sub2>"python3_d" + + + <_Lines Include="@(_DefLines)"> + $([System.Text.RegularExpressions.Regex]::Replace($([System.Text.RegularExpressions.Regex]::Replace(`%(Identity)`, `$(_Pattern1)`, `$(_Sub1)`)), `$(_Pattern2)`, `$(_Sub2)`)) + + + + + + + + + <_DefLines Remove="@(_DefLines)" /> + <_Lines Remove="@(_Lines)" /> + + + + + + <_Pattern>^[\w.]+=.+?\.([^ ]+).*$ + <_Sub>$1 + + + <_Lines Include="EXPORTS" /> + <_Symbols Include="@(_DefLines)" Condition="$([System.Text.RegularExpressions.Regex]::IsMatch(`%(Identity)`, `$(_Pattern)`))"> + $([System.Text.RegularExpressions.Regex]::Replace(`%(Identity)`, `$(_Pattern)`, `$(_Sub)`)) + + <_Lines Include="@(_Symbols->'%(Symbol)')" /> + + + + \ No newline at end of file diff --git a/PCbuild/python3dll.vcxproj.filters b/PCbuild/python3dll.vcxproj.filters index d250c45..ff48bea 100644 --- a/PCbuild/python3dll.vcxproj.filters +++ b/PCbuild/python3dll.vcxproj.filters @@ -1,32 +1,32 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav - - - - - Source Files - - - - - Source Files - - - - - Resource Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav + + + + + Source Files + + + + + Source Files + + + + + Resource Files + + \ No newline at end of file diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 6ea1848..d7e8386 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -1,434 +1,434 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} - pythoncore - - - - - DynamicLibrary - false - - - - - - true - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - $(PyDllName) - - - Link - - - - /Zm200 %(AdditionalOptions) - $(PySourcePath)Python;$(PySourcePath)Modules\zlib;%(AdditionalIncludeDirectories) - _USRDLL;Py_BUILD_CORE;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions) - - - version.lib;shlwapi.lib;ws2_32.lib;%(AdditionalDependencies) - 0x1e000000 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - git - <_GIT>$(GIT) - <_GIT Condition="$(GIT.Contains(` `))">"$(GIT)" - - - - - - - - $([System.IO.File]::ReadAllText('$(IntDir)gitbranch.txt').Trim()) - $([System.IO.File]::ReadAllText('$(IntDir)gitversion.txt').Trim()) - $([System.IO.File]::ReadAllText('$(IntDir)gittag.txt').Trim()) - - - - - GITVERSION="$(GitVersion)";GITTAG="$(GitTag)";GITBRANCH="$(GitBranch)";%(PreprocessorDefinitions) - - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} + pythoncore + + + + + DynamicLibrary + false + + + + + + true + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + $(PyDllName) + + + Link + + + + /Zm200 %(AdditionalOptions) + $(PySourcePath)Python;$(PySourcePath)Modules\zlib;%(AdditionalIncludeDirectories) + _USRDLL;Py_BUILD_CORE;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions) + + + version.lib;shlwapi.lib;ws2_32.lib;%(AdditionalDependencies) + 0x1e000000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + git + <_GIT>$(GIT) + <_GIT Condition="$(GIT.Contains(` `))">"$(GIT)" + + + + + + + + $([System.IO.File]::ReadAllText('$(IntDir)gitbranch.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gitversion.txt').Trim()) + $([System.IO.File]::ReadAllText('$(IntDir)gittag.txt').Trim()) + + + + + GITVERSION="$(GitVersion)";GITTAG="$(GitTag)";GITBRANCH="$(GitBranch)";%(PreprocessorDefinitions) + + + + + + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 762210d..9448cb7 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1,1007 +1,1007 @@ - - - - - {086b0afb-270c-4603-a02a-63d46f0b2b92} - - - {8e81609f-13ca-4eae-9fdb-f8af20c710c7} - - - {8787c5bb-bab6-4586-a42e-4a27c7b3ffb6} - - - {5d6d2d6c-9e61-4a1d-b0b2-5cc2f446d69e} - - - {9f12c4b1-322e-431e-abf1-e02550f50032} - - - {ab29a558-143d-4fe7-a039-b431fb429856} - - - {97349fee-0abf-48b0-a8f5-771bf39b8aee} - - - {ea21fc98-de89-4746-a979-c5616964329a} - - - {f2696406-14bc-48bd-90c5-e93ab82a21ac} - - - {c3e03a5c-56c7-45fd-8543-e5d2326b907d} - - - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Include - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules\_io - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Parser - - - Parser - - - PC - - - PC - - - Python - - - Python - - - Python - - - Python - - - Python - - - Include - - - Include - - - Modules - - - Include - - - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules - - - Modules\_io - - - Modules\_io - - - Modules\_io - - - Modules\_io - - - Modules\_io - - - Modules\_io - - - Modules\_io - - - Modules\_io - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\zlib - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Modules\cjkcodecs - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Objects - - - Parser - - - Parser - - - Parser - - - Parser - - - Parser - - - Parser - - - Parser - - - Parser - - - Parser - - - Parser - - - Parser - - - Parser - - - PC - - - PC - - - PC - - - PC - - - PC - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - Python - - - PC - - - Modules - - - Python - - - Objects - - - Modules - - - Modules - - - Modules - - - PC - - - Objects - - - - - Resource Files - - - + + + + + {086b0afb-270c-4603-a02a-63d46f0b2b92} + + + {8e81609f-13ca-4eae-9fdb-f8af20c710c7} + + + {8787c5bb-bab6-4586-a42e-4a27c7b3ffb6} + + + {5d6d2d6c-9e61-4a1d-b0b2-5cc2f446d69e} + + + {9f12c4b1-322e-431e-abf1-e02550f50032} + + + {ab29a558-143d-4fe7-a039-b431fb429856} + + + {97349fee-0abf-48b0-a8f5-771bf39b8aee} + + + {ea21fc98-de89-4746-a979-c5616964329a} + + + {f2696406-14bc-48bd-90c5-e93ab82a21ac} + + + {c3e03a5c-56c7-45fd-8543-e5d2326b907d} + + + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Include + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules\_io + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Parser + + + Parser + + + PC + + + PC + + + Python + + + Python + + + Python + + + Python + + + Python + + + Include + + + Include + + + Modules + + + Include + + + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules + + + Modules\_io + + + Modules\_io + + + Modules\_io + + + Modules\_io + + + Modules\_io + + + Modules\_io + + + Modules\_io + + + Modules\_io + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Modules\cjkcodecs + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Objects + + + Parser + + + Parser + + + Parser + + + Parser + + + Parser + + + Parser + + + Parser + + + Parser + + + Parser + + + Parser + + + Parser + + + Parser + + + PC + + + PC + + + PC + + + PC + + + PC + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + Python + + + PC + + + Modules + + + Python + + + Objects + + + Modules + + + Modules + + + Modules + + + PC + + + Objects + + + + + Resource Files + + + diff --git a/PCbuild/pythonw.vcxproj b/PCbuild/pythonw.vcxproj index e40f66c..28ac70e 100644 --- a/PCbuild/pythonw.vcxproj +++ b/PCbuild/pythonw.vcxproj @@ -1,79 +1,79 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {F4229CC3-873C-49AE-9729-DD308ED4CD4A} - false - - - - - Application - false - - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - 2000000 - 0x1d000000 - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {F4229CC3-873C-49AE-9729-DD308ED4CD4A} + false + + + + + Application + false + + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + 2000000 + 0x1d000000 + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/pythonw.vcxproj.filters b/PCbuild/pythonw.vcxproj.filters index 259866f..6032117 100644 --- a/PCbuild/pythonw.vcxproj.filters +++ b/PCbuild/pythonw.vcxproj.filters @@ -1,21 +1,21 @@ - - - - - {0434cf11-a311-4a92-8a6c-4164aa79a7f2} - - - {e1d8ea6b-c65d-42f4-9eed-6010846ed378} - - - - - Resource Files - - - - - Source Files - - + + + + + {0434cf11-a311-4a92-8a6c-4164aa79a7f2} + + + {e1d8ea6b-c65d-42f4-9eed-6010846ed378} + + + + + Resource Files + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/pywlauncher.vcxproj b/PCbuild/pywlauncher.vcxproj index eabf883..e836ccc 100644 --- a/PCbuild/pywlauncher.vcxproj +++ b/PCbuild/pywlauncher.vcxproj @@ -1,82 +1,82 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6} - pywlauncher - pyw - false - - - - - Application - Unicode - - - - - - ClCompile - - - - - - - - - _WINDOWS;%(PreprocessorDefinitions) - MultiThreaded - - - version.lib;%(AdditionalDependencies) - Windows - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {1D4B18D3-7C12-4ECB-9179-8531FF876CE6} + pywlauncher + pyw + false + + + + + Application + Unicode + + + + + + ClCompile + + + + + + + + + _WINDOWS;%(PreprocessorDefinitions) + MultiThreaded + + + version.lib;%(AdditionalDependencies) + Windows + + + + + + + + + + + + + + \ No newline at end of file diff --git a/PCbuild/pywlauncher.vcxproj.filters b/PCbuild/pywlauncher.vcxproj.filters index e4b23d2..05a93d0 100644 --- a/PCbuild/pywlauncher.vcxproj.filters +++ b/PCbuild/pywlauncher.vcxproj.filters @@ -1,32 +1,32 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - - - Resource Files - - - - - Resource Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + + + Resource Files + + + + + Resource Files + + \ No newline at end of file diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 8924b3f..b873ce0 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -2,9 +2,11 @@ Quick Start Guide ----------------- 1. Install Microsoft Visual Studio 2015, any edition. -2. Install Subversion, and make sure 'svn.exe' is on your PATH. -3. Run "build.bat -e" to build Python in 32-bit Release configuration. -4. (Optional, but recommended) Run the test suite with "rt.bat -q". +1a. Optionally install Python 3.6 or later. If not installed, + get_externals.bat (build.bat -e) will download and use Python via + NuGet. +2. Run "build.bat -e" to build Python in 32-bit Release configuration. +3. (Optional, but recommended) Run the test suite with "rt.bat -q". Building Python using Microsoft Visual C++ @@ -164,8 +166,7 @@ _bz2 Homepage: http://www.bzip.org/ _lzma - Python wrapper for the liblzma compression library, using pre-built - binaries of XZ Utils version 5.0.5 + Python wrapper for version 5.2.2 of the liblzma compression library Homepage: http://tukaani.org/xz/ _ssl @@ -236,9 +237,16 @@ order to download the relevant source files for each project before they can be built. However, a simple script is provided to make this as painless as possible, called "get_externals.bat" and located in this directory. This script extracts all the external sub-projects from - http://svn.python.org/projects/external -via Subversion (so you'll need svn.exe on your PATH) and places them -in ..\externals (relative to this directory). + https://github.com/python/cpython-source-deps +and + https://github.com/python/cpython-bin-deps +via a Python script called "get_external.py", located in this directory. +If Python 3.6 or later is not available via the "py.exe" launcher, the +path or command to use for Python can be provided in the PYTHON_FOR_BUILD +environment variable, or get_externals.bat will download the latest +version of NuGet and use it to download the latest "pythonx86" package +for use with get_external.py. Everything downloaded by these scripts is +stored in ..\externals (relative to this directory). It is also possible to download sources from each project's homepage, though you may have to change folder names or pass the names to MSBuild diff --git a/PCbuild/rmpyc.py b/PCbuild/rmpyc.py index a1e75bb..0b58f68 100644 --- a/PCbuild/rmpyc.py +++ b/PCbuild/rmpyc.py @@ -1,25 +1,19 @@ -# Remove all the .pyc and .pyo files under ../Lib. +# Remove all the .pyc files under ../Lib. def deltree(root): import os from os.path import join - npyc = npyo = 0 + npyc = 0 for root, dirs, files in os.walk(root): for name in files: - delete = False - if name.endswith('.pyc'): - delete = True + # to be thorough + if name.endswith(('.pyc', '.pyo')): npyc += 1 - elif name.endswith('.pyo'): - delete = True - npyo += 1 - - if delete: os.remove(join(root, name)) - return npyc, npyo + return npyc -npyc, npyo = deltree("../Lib") -print(npyc, ".pyc deleted,", npyo, ".pyo deleted") +npyc = deltree("../Lib") +print(npyc, ".pyc deleted") diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 3582672..e73ac04 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -4,8 +4,8 @@ rem Usage: rt [-d] [-O] [-q] [-x64] regrtest_args rem -d Run Debug build (python_d.exe). Else release build. rem -O Run python.exe or python_d.exe (see -d) with -O. rem -q "quick" -- normally the tests are run twice, the first time -rem after deleting all the .py[co] files reachable from Lib/. -rem -q runs the tests just once, and without deleting .py[co] files. +rem after deleting all the .pyc files reachable from Lib/. +rem -q runs the tests just once, and without deleting .pyc files. rem -x64 Run the 64-bit build of python (or python_d if -d was specified) rem from the 'amd64' dir instead of the 32-bit build in this dir. rem All leading instances of these switches are shifted off, and @@ -45,7 +45,7 @@ set exe=%prefix%python%suffix%.exe set cmd="%exe%" %dashO% -Wd -E -bb -m test %regrtestargs% if defined qmode goto Qmode -echo Deleting .pyc/.pyo files ... +echo Deleting .pyc files ... "%exe%" "%pcbuild%rmpyc.py" echo Cleaning _pth files ... @@ -55,7 +55,7 @@ echo on %cmd% @echo off -echo About to run again without deleting .pyc/.pyo first: +echo About to run again without deleting .pyc first: pause :Qmode diff --git a/PCbuild/select.vcxproj b/PCbuild/select.vcxproj index d6112ab..8ac16b7 100644 --- a/PCbuild/select.vcxproj +++ b/PCbuild/select.vcxproj @@ -1,82 +1,82 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {18CAE28C-B454-46C1-87A0-493D91D97F03} - select - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - ws2_32.lib;%(AdditionalDependencies) - 0x1D110000 - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {18CAE28C-B454-46C1-87A0-493D91D97F03} + select + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + ws2_32.lib;%(AdditionalDependencies) + 0x1D110000 + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/select.vcxproj.filters b/PCbuild/select.vcxproj.filters index 55a2fc8..9a994e6 100644 --- a/PCbuild/select.vcxproj.filters +++ b/PCbuild/select.vcxproj.filters @@ -1,13 +1,13 @@ - - - - - {98346077-900c-4c7a-852f-a23470e37b40} - - - - - Source Files - - + + + + + {98346077-900c-4c7a-852f-a23470e37b40} + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/sqlite3.vcxproj b/PCbuild/sqlite3.vcxproj index 4f5b196..ee703e3 100644 --- a/PCbuild/sqlite3.vcxproj +++ b/PCbuild/sqlite3.vcxproj @@ -1,89 +1,89 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {A1A295E5-463C-437F-81CA-1F32367685DA} - sqlite3 - .pyd - false - - - - - DynamicLibrary - NotSet - - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - <_SqliteVersion>$([System.Text.RegularExpressions.Regex]::Match(`$(sqlite3Dir)`, `((\d+)\.(\d+)\.(\d+)\.(\d+))\\?$`).Groups) - $(_SqliteVersion.Split(`;`)[1]) - $(_SqliteVersion.Split(`;`)[2]) - $(_SqliteVersion.Split(`;`)[3]) - $(_SqliteVersion.Split(`;`)[4]) - $(_SqliteVersion.Split(`;`)[5]) - - - - $(sqlite3Dir);%(AdditionalIncludeDirectories) - SQLITE_ENABLE_FTS4;SQLITE_ENABLE_FTS5;SQLITE_API=__declspec(dllexport);%(PreprocessorDefinitions) - Level1 - - - SQLITE_VERSION=$(SqliteVersion);SQLITE_MAJOR_VERSION=$(SqliteMajorVersion);SQLITE_MINOR_VERSION=$(SqliteMinorVersion);SQLITE_MICRO_VERSION=$(SqliteMicroVersion);SQLITE_PATCH_VERSION=$(SqlitePatchVersion);%(PreprocessorDefinitions) - - - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {A1A295E5-463C-437F-81CA-1F32367685DA} + sqlite3 + .pyd + false + + + + + DynamicLibrary + NotSet + + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + <_SqliteVersion>$([System.Text.RegularExpressions.Regex]::Match(`$(sqlite3Dir)`, `((\d+)\.(\d+)\.(\d+)\.(\d+))\\?$`).Groups) + $(_SqliteVersion.Split(`;`)[1]) + $(_SqliteVersion.Split(`;`)[2]) + $(_SqliteVersion.Split(`;`)[3]) + $(_SqliteVersion.Split(`;`)[4]) + $(_SqliteVersion.Split(`;`)[5]) + + + + $(sqlite3Dir);%(AdditionalIncludeDirectories) + SQLITE_ENABLE_FTS4;SQLITE_ENABLE_FTS5;SQLITE_API=__declspec(dllexport);%(PreprocessorDefinitions) + Level1 + + + SQLITE_VERSION=$(SqliteVersion);SQLITE_MAJOR_VERSION=$(SqliteMajorVersion);SQLITE_MINOR_VERSION=$(SqliteMinorVersion);SQLITE_MICRO_VERSION=$(SqliteMicroVersion);SQLITE_PATCH_VERSION=$(SqlitePatchVersion);%(PreprocessorDefinitions) + + + + + + + + + + + + + + + + diff --git a/PCbuild/sqlite3.vcxproj.filters b/PCbuild/sqlite3.vcxproj.filters index 86680c1..8d79004 100644 --- a/PCbuild/sqlite3.vcxproj.filters +++ b/PCbuild/sqlite3.vcxproj.filters @@ -1,24 +1,24 @@ - - - - - {ce5b649d-a6f7-4459-9425-c883795d79df} - - - {0e842fe2-176b-4e83-9d1f-0ad13a859efd} - - - - - Header Files - - - Header Files - - - - - Source Files - - + + + + + {ce5b649d-a6f7-4459-9425-c883795d79df} + + + {0e842fe2-176b-4e83-9d1f-0ad13a859efd} + + + + + Header Files + + + Header Files + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/ssleay.vcxproj b/PCbuild/ssleay.vcxproj index 439e3ac..1499fbd 100644 --- a/PCbuild/ssleay.vcxproj +++ b/PCbuild/ssleay.vcxproj @@ -1,119 +1,119 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Debug - x64 - - - Release - x64 - - - - {10615B24-73BF-4EFA-93AA-236916321317} - ssleay - - - - - - - StaticLibrary - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Release + Win32 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Debug + x64 + + + Release + x64 + + + + {10615B24-73BF-4EFA-93AA-236916321317} + ssleay + + + + + + + StaticLibrary + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/PCbuild/tcl.vcxproj b/PCbuild/tcl.vcxproj index 3dfd155..69d98be 100644 --- a/PCbuild/tcl.vcxproj +++ b/PCbuild/tcl.vcxproj @@ -1,93 +1,93 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Debug - x64 - - - Release - x64 - - - - {B5FD6F1D-129E-4BFF-9340-03606FAC7283} - - - - - - - - Makefile - $(tcltkDir) - $(OutDir)bin\$(tclDLLName) - - - - - - - - - - msvcrt - symbols,msvcrt - INSTALLDIR="$(OutDir.TrimEnd(`\`))" INSTALL_DIR="$(OutDir.TrimEnd(`\`))" - DEBUGFLAGS="-wd4456 -wd4457 -wd4458 -wd4459 -wd4996" - setlocal -@(ExpectedOutputs->'if not exist "%(FullPath)" goto build',' -') -goto :eof -:build -set VCINSTALLDIR=$(VCInstallDir) -cd /D "$(tclDir)win" -nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) core shell dlls -nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) install-binaries install-libraries - - - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Release + Win32 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Debug + x64 + + + Release + x64 + + + + {B5FD6F1D-129E-4BFF-9340-03606FAC7283} + + + + + + + + Makefile + $(tcltkDir) + $(OutDir)bin\$(tclDLLName) + + + + + + + + + + msvcrt + symbols,msvcrt + INSTALLDIR="$(OutDir.TrimEnd(`\`))" INSTALL_DIR="$(OutDir.TrimEnd(`\`))" + DEBUGFLAGS="-wd4456 -wd4457 -wd4458 -wd4459 -wd4996" + setlocal +@(ExpectedOutputs->'if not exist "%(FullPath)" goto build',' +') +goto :eof +:build +set VCINSTALLDIR=$(VCInstallDir) +cd /D "$(tclDir)win" +nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) core shell dlls +nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) install-binaries install-libraries + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index 57bb98a..a67debd 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -1,45 +1,45 @@ - - - - - 8 - 6 - 6 - 0 - $(TclMajorVersion) - $(TclMinorVersion) - $(TclPatchLevel) - $(TclRevision) - 8 - 4 - 3 - 6 - $(ExternalsDir)tcl-core-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\ - $(ExternalsDir)tk-$(TkMajorVersion).$(TkMinorVersion).$(TkPatchLevel).$(TkRevision)\ - $(ExternalsDir)tix-$(TixMajorVersion).$(TixMinorVersion).$(TixPatchLevel).$(TixRevision)\ - $(ExternalsDir)tcltk\ - $(ExternalsDir)tcltk64\ - g - tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).dll - tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).lib - tclsh$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).exe - tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).dll - tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).lib - tix$(TixMajorVersion)$(TixMinorVersion)$(TclDebugExt).dll - $(tcltkDir)lib\tix$(TixMajorVersion).$(TixMinorVersion).$(TixPatchLevel)\$(tixDLLName) - $(tcltkDir)lib\tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).lib;$(tcltkDir)lib\tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).lib - IX86 - AMD64 - TCL_MAJOR_VERSION=$(TclMajorVersion) TCL_MINOR_VERSION=$(TclMinorVersion) TCL_PATCH_LEVEL=$(TclPatchLevel) - TCL_MAJOR=$(TclMajorVersion) TCL_MINOR=$(TclMinorVersion) TCL_PATCH=$(TclPatchLevel) - TK_MAJOR_VERSION=$(TkMajorVersion) TK_MINOR_VERSION=$(TkMinorVersion) TK_PATCH_LEVEL=$(TkPatchLevel) - - Release - Debug - $(BuildDirTop)_$(TclMachine) - $(BuildDirTop)_VC13 - $(BuildDirTop)_VC12 - $(BuildDirTop)_VC11 - $(BuildDirTop)_VC10 - - + + + + + 8 + 6 + 6 + 0 + $(TclMajorVersion) + $(TclMinorVersion) + $(TclPatchLevel) + $(TclRevision) + 8 + 4 + 3 + 6 + $(ExternalsDir)tcl-core-$(TclMajorVersion).$(TclMinorVersion).$(TclPatchLevel).$(TclRevision)\ + $(ExternalsDir)tk-$(TkMajorVersion).$(TkMinorVersion).$(TkPatchLevel).$(TkRevision)\ + $(ExternalsDir)tix-$(TixMajorVersion).$(TixMinorVersion).$(TixPatchLevel).$(TixRevision)\ + $(ExternalsDir)tcltk\ + $(ExternalsDir)tcltk64\ + g + tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).dll + tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).lib + tclsh$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).exe + tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).dll + tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).lib + tix$(TixMajorVersion)$(TixMinorVersion)$(TclDebugExt).dll + $(tcltkDir)lib\tix$(TixMajorVersion).$(TixMinorVersion).$(TixPatchLevel)\$(tixDLLName) + $(tcltkDir)lib\tcl$(TclMajorVersion)$(TclMinorVersion)t$(TclDebugExt).lib;$(tcltkDir)lib\tk$(TkMajorVersion)$(TkMinorVersion)t$(TclDebugExt).lib + IX86 + AMD64 + TCL_MAJOR_VERSION=$(TclMajorVersion) TCL_MINOR_VERSION=$(TclMinorVersion) TCL_PATCH_LEVEL=$(TclPatchLevel) + TCL_MAJOR=$(TclMajorVersion) TCL_MINOR=$(TclMinorVersion) TCL_PATCH=$(TclPatchLevel) + TK_MAJOR_VERSION=$(TkMajorVersion) TK_MINOR_VERSION=$(TkMinorVersion) TK_PATCH_LEVEL=$(TkPatchLevel) + + Release + Debug + $(BuildDirTop)_$(TclMachine) + $(BuildDirTop)_VC13 + $(BuildDirTop)_VC12 + $(BuildDirTop)_VC11 + $(BuildDirTop)_VC10 + + diff --git a/PCbuild/tix.vcxproj b/PCbuild/tix.vcxproj index d1bc0ab..407acdd 100644 --- a/PCbuild/tix.vcxproj +++ b/PCbuild/tix.vcxproj @@ -1,94 +1,94 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Debug - x64 - - - Release - x64 - - - - {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555} - tix - - - - - - - - Makefile - $(tcltkDir) - $(tixDLLPath) - - - - - - - - - - BUILDDIRTOP="$(BuildDirTop)" TCL_DIR="$(tclDir.TrimEnd(`\`))" TK_DIR="$(tkDir.TrimEnd(`\`))" INSTALL_DIR="$(OutDir.TrimEnd(`\`))" - DEBUG=1 NODEBUG=0 TCL_DBGX=g TK_DBGX=g - DEBUG=0 NODEBUG=1 - setlocal -@(ExpectedOutputs->'if not exist "%(FullPath)" goto build',' -') -goto :eof -:build -set VCINSTALLDIR=$(VCInstallDir) -cd /D "$(tixDir)win" -nmake /nologo -f makefile.vc MACHINE=$(TclMachine) $(DebugFlags) $(TclShortVersions) $(TixDirs) all install - - rmdir /q/s "$(OutDir.TrimEnd(`\`))" - - - - - {b5fd6f1d-129e-4bff-9340-03606fac7283} - false - - - {7e85eccf-a72c-4da4-9e52-884508e80ba1} - false - - - - - - - - - - - - + + + + + Debug + Win32 + + + Release + Win32 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Debug + x64 + + + Release + x64 + + + + {C5A3E7FB-9695-4B2E-960B-1D9F43F1E555} + tix + + + + + + + + Makefile + $(tcltkDir) + $(tixDLLPath) + + + + + + + + + + BUILDDIRTOP="$(BuildDirTop)" TCL_DIR="$(tclDir.TrimEnd(`\`))" TK_DIR="$(tkDir.TrimEnd(`\`))" INSTALL_DIR="$(OutDir.TrimEnd(`\`))" + DEBUG=1 NODEBUG=0 TCL_DBGX=g TK_DBGX=g + DEBUG=0 NODEBUG=1 + setlocal +@(ExpectedOutputs->'if not exist "%(FullPath)" goto build',' +') +goto :eof +:build +set VCINSTALLDIR=$(VCInstallDir) +cd /D "$(tixDir)win" +nmake /nologo -f makefile.vc MACHINE=$(TclMachine) $(DebugFlags) $(TclShortVersions) $(TixDirs) all install + + rmdir /q/s "$(OutDir.TrimEnd(`\`))" + + + + + {b5fd6f1d-129e-4bff-9340-03606fac7283} + false + + + {7e85eccf-a72c-4da4-9e52-884508e80ba1} + false + + + + + + + + + + + + \ No newline at end of file diff --git a/PCbuild/tk.vcxproj b/PCbuild/tk.vcxproj index a26318b..8d1aacb 100644 --- a/PCbuild/tk.vcxproj +++ b/PCbuild/tk.vcxproj @@ -1,97 +1,97 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Debug - x64 - - - Release - x64 - - - - {7E85ECCF-A72C-4DA4-9E52-884508E80BA1} - tk - - - - - - - - Makefile - $(tcltkDir) - $(OutDir)bin\$(tkDLLName) - - - - - - - - - - msvcrt - symbols,msvcrt - TCLDIR="$(tclDir.TrimEnd(`\`))" INSTALLDIR="$(OutDir.TrimEnd(`\`))" - DEBUGFLAGS="-wd4456 -wd4457 -wd4458 -wd4459 -wd4996" - setlocal -@(ExpectedOutputs->'if not exist "%(FullPath)" goto build',' -') -goto :eof -:build -set VCINSTALLDIR=$(VCInstallDir) -cd /D "$(tkDir)win" -nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) all -nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) install-binaries install-libraries - - - - - {b5fd6f1d-129e-4bff-9340-03606fac7283} - false - - - - - - - - - - - - - - - + + + + + Debug + Win32 + + + Release + Win32 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Debug + x64 + + + Release + x64 + + + + {7E85ECCF-A72C-4DA4-9E52-884508E80BA1} + tk + + + + + + + + Makefile + $(tcltkDir) + $(OutDir)bin\$(tkDLLName) + + + + + + + + + + msvcrt + symbols,msvcrt + TCLDIR="$(tclDir.TrimEnd(`\`))" INSTALLDIR="$(OutDir.TrimEnd(`\`))" + DEBUGFLAGS="-wd4456 -wd4457 -wd4458 -wd4459 -wd4996" + setlocal +@(ExpectedOutputs->'if not exist "%(FullPath)" goto build',' +') +goto :eof +:build +set VCINSTALLDIR=$(VCInstallDir) +cd /D "$(tkDir)win" +nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) all +nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) install-binaries install-libraries + + + + + {b5fd6f1d-129e-4bff-9340-03606fac7283} + false + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/PCbuild/unicodedata.vcxproj b/PCbuild/unicodedata.vcxproj index 317c1a8..cf99b95 100644 --- a/PCbuild/unicodedata.vcxproj +++ b/PCbuild/unicodedata.vcxproj @@ -1,86 +1,86 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881} - unicodedata - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - 0x1D120000 - - - - - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881} + unicodedata + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + 0x1D120000 + + + + + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/unicodedata.vcxproj.filters b/PCbuild/unicodedata.vcxproj.filters index 20e640e..74f35f6 100644 --- a/PCbuild/unicodedata.vcxproj.filters +++ b/PCbuild/unicodedata.vcxproj.filters @@ -1,24 +1,24 @@ - - - - - {b939a8f1-ccd7-420a-974a-243606dccd74} - - - {e2c055bb-ec62-4bbc-aa1c-d88da4d4ad1c} - - - - - Header Files - - - Header Files - - - - - Source Files - - + + + + + {b939a8f1-ccd7-420a-974a-243606dccd74} + + + {e2c055bb-ec62-4bbc-aa1c-d88da4d4ad1c} + + + + + Header Files + + + Header Files + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/winsound.vcxproj b/PCbuild/winsound.vcxproj index 12913ef..2720cfa 100644 --- a/PCbuild/winsound.vcxproj +++ b/PCbuild/winsound.vcxproj @@ -1,82 +1,82 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {28B5D777-DDF2-4B6B-B34F-31D938813856} - winsound - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - winmm.lib;%(AdditionalDependencies) - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {28B5D777-DDF2-4B6B-B34F-31D938813856} + winsound + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + winmm.lib;%(AdditionalDependencies) + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + \ No newline at end of file diff --git a/PCbuild/winsound.vcxproj.filters b/PCbuild/winsound.vcxproj.filters index 7d08931..d91fdbf 100644 --- a/PCbuild/winsound.vcxproj.filters +++ b/PCbuild/winsound.vcxproj.filters @@ -1,14 +1,14 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - - - Source Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + + + Source Files + + \ No newline at end of file diff --git a/PCbuild/xxlimited.vcxproj b/PCbuild/xxlimited.vcxproj index 9dbdc77..8d4b906 100644 --- a/PCbuild/xxlimited.vcxproj +++ b/PCbuild/xxlimited.vcxproj @@ -1,83 +1,83 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {F749B822-B489-4CA5-A3AD-CE078F5F338A} - xxlimited - Win32Proj - - - - - DynamicLibrary - NotSet - false - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - %(PreprocessorDefinitions);Py_LIMITED_API=0x03060000 - - - wsock32.lib;%(AdditionalDependencies) - 0x1D110000 - - - - - - - - {885d4898-d08d-4091-9c40-c700cfe3fc5a} - - - - - + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {F749B822-B489-4CA5-A3AD-CE078F5F338A} + xxlimited + Win32Proj + + + + + DynamicLibrary + NotSet + false + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + %(PreprocessorDefinitions);Py_LIMITED_API=0x03060000 + + + wsock32.lib;%(AdditionalDependencies) + 0x1D110000 + + + + + + + + {885d4898-d08d-4091-9c40-c700cfe3fc5a} + + + + + \ No newline at end of file diff --git a/PCbuild/xxlimited.vcxproj.filters b/PCbuild/xxlimited.vcxproj.filters index 21db468..807279a 100644 --- a/PCbuild/xxlimited.vcxproj.filters +++ b/PCbuild/xxlimited.vcxproj.filters @@ -1,13 +1,13 @@ - - - - - {eb5eebc7-3787-4f8f-b282-204db2962cc3} - - - - - Source Files - - + + + + + {eb5eebc7-3787-4f8f-b282-204db2962cc3} + + + + + Source Files + + \ No newline at end of file diff --git a/Python/ast.c b/Python/ast.c index 217ea14..8d53736 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4170,9 +4170,11 @@ decode_unicode_with_escapes(struct compiling *c, const node *n, const char *s, while (s < end) { if (*s == '\\') { *p++ = *s++; - if (*s & 0x80) { + if (s >= end || *s & 0x80) { strcpy(p, "u005c"); p += 5; + if (s >= end) + break; } } if (*s & 0x80) { /* XXX inefficient */ @@ -4245,49 +4247,32 @@ fstring_compile_expr(const char *expr_start, const char *expr_end, struct compiling *c, const node *n) { - int all_whitespace = 1; - int kind; - void *data; PyCompilerFlags cf; mod_ty mod; char *str; - PyObject *o; Py_ssize_t len; - Py_ssize_t i; + const char *s; assert(expr_end >= expr_start); assert(*(expr_start-1) == '{'); assert(*expr_end == '}' || *expr_end == '!' || *expr_end == ':'); - /* We know there are no escapes here, because backslashes are not allowed, - and we know it's utf-8 encoded (per PEP 263). But, in order to check - that each char is not whitespace, we need to decode it to unicode. - Which is unfortunate, but such is life. */ - /* If the substring is all whitespace, it's an error. We need to catch this here, and not when we call PyParser_ASTFromString, because turning the expression '' in to '()' would go from being invalid to valid. */ - /* Note that this code says an empty string is all whitespace. That's - important. There's a test for it: f'{}'. */ - o = PyUnicode_DecodeUTF8(expr_start, expr_end-expr_start, NULL); - if (o == NULL) - return NULL; - len = PyUnicode_GET_LENGTH(o); - kind = PyUnicode_KIND(o); - data = PyUnicode_DATA(o); - for (i = 0; i < len; i++) { - if (!Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, i))) { - all_whitespace = 0; + for (s = expr_start; s != expr_end; s++) { + char c = *s; + /* The Python parser ignores only the following whitespace + characters (\r already is converted to \n). */ + if (!(c == ' ' || c == '\t' || c == '\n' || c == '\f')) { break; } } - Py_DECREF(o); - if (all_whitespace) { + if (s == expr_end) { ast_error(c, n, "f-string: empty expression not allowed"); return NULL; } - /* Reuse len to be the length of the utf-8 input string. */ len = expr_end - expr_start; /* Allocate 3 extra bytes: open paren, close paren, null byte. */ str = PyMem_RawMalloc(len + 3); @@ -4325,30 +4310,37 @@ fstring_find_literal(const char **str, const char *end, int raw, brace (which isn't part of a unicode name escape such as "\N{EULER CONSTANT}"), or the end of the string. */ - const char *literal_start = *str; - const char *literal_end; - int in_named_escape = 0; + const char *s = *str; + const char *literal_start = s; int result = 0; assert(*literal == NULL); - for (; *str < end; (*str)++) { - char ch = **str; - if (!in_named_escape && ch == '{' && (*str)-literal_start >= 2 && - *(*str-2) == '\\' && *(*str-1) == 'N') { - in_named_escape = 1; - } else if (in_named_escape && ch == '}') { - in_named_escape = 0; - } else if (ch == '{' || ch == '}') { + while (s < end) { + char ch = *s++; + if (!raw && ch == '\\' && s < end) { + ch = *s++; + if (ch == 'N') { + if (s < end && *s++ == '{') { + while (s < end && *s++ != '}') { + } + continue; + } + break; + } + if (ch == '{' && warn_invalid_escape_sequence(c, n, ch) < 0) { + return -1; + } + } + if (ch == '{' || ch == '}') { /* Check for doubled braces, but only at the top level. If we checked at every level, then f'{0:{3}}' would fail with the two closing braces. */ if (recurse_lvl == 0) { - if (*str+1 < end && *(*str+1) == ch) { + if (s < end && *s == ch) { /* We're going to tell the caller that the literal ends here, but that they should continue scanning. But also skip over the second brace when we resume scanning. */ - literal_end = *str+1; - *str += 2; + *str = s + 1; result = 1; goto done; } @@ -4356,6 +4348,7 @@ fstring_find_literal(const char **str, const char *end, int raw, /* Where a single '{' is the start of a new expression, a single '}' is not allowed. */ if (ch == '}') { + *str = s - 1; ast_error(c, n, "f-string: single '}' is not allowed"); return -1; } @@ -4363,21 +4356,22 @@ fstring_find_literal(const char **str, const char *end, int raw, /* We're either at a '{', which means we're starting another expression; or a '}', which means we're at the end of this f-string (for a nested format_spec). */ + s--; break; } } - literal_end = *str; - assert(*str <= end); - assert(*str == end || **str == '{' || **str == '}'); + *str = s; + assert(s <= end); + assert(s == end || *s == '{' || *s == '}'); done: - if (literal_start != literal_end) { + if (literal_start != s) { if (raw) *literal = PyUnicode_DecodeUTF8Stateful(literal_start, - literal_end-literal_start, + s - literal_start, NULL, NULL); else *literal = decode_unicode_with_escapes(c, n, literal_start, - literal_end-literal_start); + s - literal_start); if (!*literal) return -1; } @@ -4893,6 +4887,8 @@ FstringParser_ConcatFstring(FstringParser *state, const char **str, /* Do nothing. Just leave last_str alone (and possibly NULL). */ } else if (!state->last_str) { + /* Note that the literal can be zero length, if the + input string is "\\\n" or "\\\r", among others. */ state->last_str = literal; literal = NULL; } else { @@ -4902,8 +4898,6 @@ FstringParser_ConcatFstring(FstringParser *state, const char **str, return -1; literal = NULL; } - assert(!state->last_str || - PyUnicode_GET_LENGTH(state->last_str) != 0); /* We've dealt with the literal now. It can't be leaked on further errors. */ diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index ef5a34c..597e26e 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1927,12 +1927,15 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyObject *result; size_t len; + /* stdin is a text stream, so it must have an encoding. */ stdin_encoding = _PyObject_GetAttrId(fin, &PyId_encoding); stdin_errors = _PyObject_GetAttrId(fin, &PyId_errors); - if (!stdin_encoding || !stdin_errors) - /* stdin is a text stream, so it must have an - encoding. */ + if (!stdin_encoding || !stdin_errors || + !PyUnicode_Check(stdin_encoding) || + !PyUnicode_Check(stdin_errors)) { + tty = 0; goto _readline_errors; + } stdin_encoding_str = PyUnicode_AsUTF8(stdin_encoding); stdin_errors_str = PyUnicode_AsUTF8(stdin_errors); if (!stdin_encoding_str || !stdin_errors_str) @@ -1948,8 +1951,12 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyObject *stringpo; stdout_encoding = _PyObject_GetAttrId(fout, &PyId_encoding); stdout_errors = _PyObject_GetAttrId(fout, &PyId_errors); - if (!stdout_encoding || !stdout_errors) + if (!stdout_encoding || !stdout_errors || + !PyUnicode_Check(stdout_encoding) || + !PyUnicode_Check(stdout_errors)) { + tty = 0; goto _readline_errors; + } stdout_encoding_str = PyUnicode_AsUTF8(stdout_encoding); stdout_errors_str = PyUnicode_AsUTF8(stdout_errors); if (!stdout_encoding_str || !stdout_errors_str) @@ -2003,13 +2010,17 @@ builtin_input_impl(PyObject *module, PyObject *prompt) Py_XDECREF(po); PyMem_FREE(s); return result; + _readline_errors: Py_XDECREF(stdin_encoding); Py_XDECREF(stdout_encoding); Py_XDECREF(stdin_errors); Py_XDECREF(stdout_errors); Py_XDECREF(po); - return NULL; + if (tty) + return NULL; + + PyErr_Clear(); } /* Fallback if we're not interactive */ diff --git a/Python/ceval.c b/Python/ceval.c index 9cac771..ea79f5f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1119,9 +1119,20 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) Py_MakePendingCalls() above. */ if (_Py_atomic_load_relaxed(&eval_breaker)) { - if (_Py_OPCODE(*next_instr) == SETUP_FINALLY) { - /* Make the last opcode before - a try: finally: block uninterruptible. */ + if (_Py_OPCODE(*next_instr) == SETUP_FINALLY || + _Py_OPCODE(*next_instr) == YIELD_FROM) { + /* Two cases where we skip running signal handlers and other + pending calls: + - If we're about to enter the try: of a try/finally (not + *very* useful, but might help in some cases and it's + traditional) + - If we're resuming a chain of nested 'yield from' or + 'await' calls, then each frame is parked with YIELD_FROM + as its next opcode. If the user hit control-C we want to + wait until we've reached the innermost frame before + running the signal handler and raising KeyboardInterrupt + (see bpo-30039). + */ goto fast_next_opcode; } if (_Py_atomic_load_relaxed(&pendingcalls_to_do)) { @@ -5071,17 +5082,13 @@ do_call_core(PyObject *func, PyObject *callargs, PyObject *kwdict) /* Extract a slice index from a PyLong or an object with the nb_index slot defined, and store in *pi. Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX, - and silently boost values less than -PY_SSIZE_T_MAX-1 to -PY_SSIZE_T_MAX-1. + and silently boost values less than PY_SSIZE_T_MIN to PY_SSIZE_T_MIN. Return 0 on error, 1 on success. */ -/* Note: If v is NULL, return success without storing into *pi. This - is because_PyEval_SliceIndex() is called by apply_slice(), which can be - called by the SLICE opcode with v and/or w equal to NULL. -*/ int _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) { - if (v != NULL) { + if (v != Py_None) { Py_ssize_t x; if (PyIndex_Check(v)) { x = PyNumber_AsSsize_t(v, NULL); @@ -5099,6 +5106,26 @@ _PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi) return 1; } +int +_PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi) +{ + Py_ssize_t x; + if (PyIndex_Check(v)) { + x = PyNumber_AsSsize_t(v, NULL); + if (x == -1 && PyErr_Occurred()) + return 0; + } + else { + PyErr_SetString(PyExc_TypeError, + "slice indices must be integers or " + "have an __index__ method"); + return 0; + } + *pi = x; + return 1; +} + + #define CANNOT_CATCH_MSG "catching classes that do not inherit from "\ "BaseException is not allowed" @@ -5426,14 +5453,14 @@ _Py_GetDXProfile(PyObject *self, PyObject *args) Py_ssize_t _PyEval_RequestCodeExtraIndex(freefunc free) { - PyThreadState *tstate = PyThreadState_Get(); + __PyCodeExtraState *state = __PyCodeExtraState_Get(); Py_ssize_t new_index; - if (tstate->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) { + if (state->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) { return -1; } - new_index = tstate->co_extra_user_count++; - tstate->co_extra_freefuncs[new_index] = free; + new_index = state->co_extra_user_count++; + state->co_extra_freefuncs[new_index] = free; return new_index; } diff --git a/Python/fileutils.c b/Python/fileutils.c index e84d66e..f3764e4 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -583,7 +583,7 @@ _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); result->st_nlink = info->nNumberOfLinks; - result->st_ino = (((__int64)info->nFileIndexHigh)<<32) + info->nFileIndexLow; + result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow; if (reparse_tag == IO_REPARSE_TAG_SYMLINK) { /* first clear the S_IFMT bits */ result->st_mode ^= (result->st_mode & S_IFMT); @@ -653,7 +653,7 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) _Py_attribute_data_to_stat(&info, 0, status); /* specific to fstat() */ - status->st_ino = (((__int64)info.nFileIndexHigh)<<32) + info.nFileIndexLow; + status->st_ino = (((uint64_t)info.nFileIndexHigh) << 32) + info.nFileIndexLow; return 0; #else return fstat(fd, status); diff --git a/Python/import.c b/Python/import.c index cd865a5..a23a102 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1785,9 +1785,13 @@ PyImport_Import(PyObject *module_name) Py_DECREF(r); modules = PyImport_GetModuleDict(); - r = PyDict_GetItem(modules, module_name); - if (r != NULL) + r = PyDict_GetItemWithError(modules, module_name); + if (r != NULL) { Py_INCREF(r); + } + else if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, module_name); + } err: Py_XDECREF(globals); diff --git a/Python/marshal.c b/Python/marshal.c index 87a4b24..7b12ab7 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -549,7 +549,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_lnotab, p); } else if (PyObject_CheckBuffer(v)) { - /* Write unknown bytes-like objects as a byte string */ + /* Write unknown bytes-like objects as a bytes object */ Py_buffer view; if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) != 0) { w_byte(TYPE_UNKNOWN, p); @@ -1086,7 +1086,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (bytes object size out of range)"); break; } v = PyBytes_FromStringAndSize((char *)NULL, n); @@ -1110,7 +1110,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); break; } goto _read_ascii; @@ -1150,7 +1150,7 @@ r_object(RFILE *p) if (PyErr_Occurred()) break; if (n < 0 || n > SIZE32_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); break; } if (n != 0) { @@ -1612,7 +1612,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) if (wf.ptr - base > PY_SSIZE_T_MAX) { Py_DECREF(wf.str); PyErr_SetString(PyExc_OverflowError, - "too much marshal data for a string"); + "too much marshal data for a bytes object"); return NULL; } if (_PyBytes_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)) < 0) @@ -1658,8 +1658,7 @@ PyDoc_STRVAR(dump_doc, "dump(value, file[, version])\n\ \n\ Write the value on the open file. The value must be a supported type.\n\ -The file must be an open file object such as sys.stdout or returned by\n\ -open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\ +The file must be a writeable binary file.\n\ \n\ If the value has (or contains an object that has) an unsupported type, a\n\ ValueError exception is raised - but garbage data will also be written\n\ @@ -1715,8 +1714,7 @@ PyDoc_STRVAR(load_doc, Read one value from the open file and return it. If no valid value is\n\ read (e.g. because the data has a different Python version's\n\ incompatible marshal format), raise EOFError, ValueError or TypeError.\n\ -The file must be an open file object opened in binary mode ('rb' or\n\ -'r+b').\n\ +The file must be a readable binary file.\n\ \n\ Note: If an object containing an unsupported type was marshalled with\n\ dump(), load() will substitute None for the unmarshallable type."); @@ -1735,7 +1733,7 @@ marshal_dumps(PyObject *self, PyObject *args) PyDoc_STRVAR(dumps_doc, "dumps(value[, version])\n\ \n\ -Return the string that would be written to a file by dump(value, file).\n\ +Return the bytes object that would be written to a file by dump(value, file).\n\ The value must be a supported type. Raise a ValueError exception if\n\ value has (or contains an object that has) an unsupported type.\n\ \n\ @@ -1771,8 +1769,8 @@ marshal_loads(PyObject *self, PyObject *args) PyDoc_STRVAR(loads_doc, "loads(bytes)\n\ \n\ -Convert the bytes object to a value. If no valid value is found, raise\n\ -EOFError, ValueError or TypeError. Extra characters in the input are\n\ +Convert the bytes-like object to a value. If no valid value is found,\n\ +raise EOFError, ValueError or TypeError. Extra bytes in the input are\n\ ignored."); static PyMethodDef marshal_methods[] = { @@ -1810,8 +1808,8 @@ Functions:\n\ \n\ dump() -- write value to a file\n\ load() -- read value from a file\n\ -dumps() -- write value to a string\n\ -loads() -- read value from a string"); +dumps() -- marshal value as a bytes object\n\ +loads() -- read value from a bytes-like object"); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a4f7f82..640271f 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -88,7 +88,7 @@ int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ int Py_FrozenFlag; /* Needed by getpath.c */ int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ -int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.py[co]) */ +int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.pyc) */ int Py_NoUserSiteDirectory = 0; /* for -s and site.py */ int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */ int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ @@ -1044,6 +1044,14 @@ initsite(void) static int is_valid_fd(int fd) { +#ifdef __APPLE__ + /* bpo-30225: On macOS Tiger, when stdout is redirected to a pipe + and the other side of the pipe is closed, dup(1) succeed, whereas + fstat(1, &st) fails with EBADF. Prefer fstat() over dup() to detect + such error. */ + struct stat st; + return (fstat(fd, &st) == 0); +#else int fd2; if (fd < 0) return 0; @@ -1056,6 +1064,7 @@ is_valid_fd(int fd) close(fd2); _Py_END_SUPPRESS_IPH return fd2 >= 0; +#endif } /* returns Py_None if the fd is not valid */ diff --git a/Python/pystate.c b/Python/pystate.c index 65c244e..65f9b7e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -55,6 +55,7 @@ static int autoTLSkey = -1; #endif static PyInterpreterState *interp_head = NULL; +static __PyCodeExtraState *coextra_head = NULL; /* Assuming the current thread holds the GIL, this is the PyThreadState for the current thread. */ @@ -73,6 +74,12 @@ PyInterpreterState_New(void) PyMem_RawMalloc(sizeof(PyInterpreterState)); if (interp != NULL) { + __PyCodeExtraState* coextra = PyMem_RawMalloc(sizeof(__PyCodeExtraState)); + if (coextra == NULL) { + PyMem_RawFree(interp); + return NULL; + } + HEAD_INIT(); #ifdef WITH_THREAD if (head_mutex == NULL) @@ -92,6 +99,8 @@ PyInterpreterState_New(void) interp->importlib = NULL; interp->import_func = NULL; interp->eval_frame = _PyEval_EvalFrameDefault; + coextra->co_extra_user_count = 0; + coextra->interp = interp; #ifdef HAVE_DLOPEN #if HAVE_DECL_RTLD_NOW interp->dlopenflags = RTLD_NOW; @@ -103,6 +112,8 @@ PyInterpreterState_New(void) HEAD_LOCK(); interp->next = interp_head; interp_head = interp; + coextra->next = coextra_head; + coextra_head = coextra; HEAD_UNLOCK(); } @@ -147,9 +158,10 @@ void PyInterpreterState_Delete(PyInterpreterState *interp) { PyInterpreterState **p; + __PyCodeExtraState **pextra; zapthreads(interp); HEAD_LOCK(); - for (p = &interp_head; ; p = &(*p)->next) { + for (p = &interp_head; /* N/A */; p = &(*p)->next) { if (*p == NULL) Py_FatalError( "PyInterpreterState_Delete: invalid interp"); @@ -159,6 +171,18 @@ PyInterpreterState_Delete(PyInterpreterState *interp) if (interp->tstate_head != NULL) Py_FatalError("PyInterpreterState_Delete: remaining threads"); *p = interp->next; + + for (pextra = &coextra_head; ; pextra = &(*pextra)->next) { + if (*pextra == NULL) + Py_FatalError( + "PyInterpreterState_Delete: invalid extra"); + __PyCodeExtraState* extra = *pextra; + if (extra->interp == interp) { + *pextra = extra->next; + PyMem_RawFree(extra); + break; + } + } HEAD_UNLOCK(); PyMem_RawFree(interp); #ifdef WITH_THREAD @@ -224,7 +248,6 @@ new_threadstate(PyInterpreterState *interp, int init) tstate->coroutine_wrapper = NULL; tstate->in_coroutine_wrapper = 0; - tstate->co_extra_user_count = 0; tstate->async_gen_firstiter = NULL; tstate->async_gen_finalizer = NULL; @@ -548,6 +571,23 @@ PyThreadState_Swap(PyThreadState *newts) return oldts; } +__PyCodeExtraState* +__PyCodeExtraState_Get(void) { + PyInterpreterState* interp = PyThreadState_Get()->interp; + + HEAD_LOCK(); + for (__PyCodeExtraState* cur = coextra_head; cur != NULL; cur = cur->next) { + if (cur->interp == interp) { + HEAD_UNLOCK(); + return cur; + } + } + HEAD_UNLOCK(); + + Py_FatalError("__PyCodeExtraState_Get: no code state for interpreter"); + return NULL; +} + /* An extension mechanism to store arbitrary additional per-thread state. PyThreadState_GetDict() returns a dictionary that can be used to hold such state; the caller should pick a unique key and store its state there. If @@ -743,6 +783,10 @@ _PyGILState_Fini(void) void _PyGILState_Reinit(void) { +#ifdef WITH_THREAD + head_mutex = NULL; + HEAD_INIT(); +#endif PyThreadState *tstate = PyGILState_GetThisThreadState(); PyThread_delete_key(autoTLSkey); if ((autoTLSkey = PyThread_create_key()) == -1) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 99e6b5e..b09268b 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -985,10 +985,10 @@ sys_getwindowsversion(PyObject *self) } PyMem_RawFree(verblock); } - PyStructSequence_SET_ITEM(version, pos++, PyTuple_Pack(3, - PyLong_FromLong(realMajor), - PyLong_FromLong(realMinor), - PyLong_FromLong(realBuild) + PyStructSequence_SET_ITEM(version, pos++, Py_BuildValue("(kkk)", + realMajor, + realMinor, + realBuild )); if (PyErr_Occurred()) { diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 27e0dc8..ba7393f 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -143,6 +143,8 @@ typedef struct { } pthread_lock; #define CHECK_STATUS(name) if (status != 0) { perror(name); error = 1; } +#define CHECK_STATUS_PTHREAD(name) if (status != 0) { fprintf(stderr, \ + "%s: %s\n", name, strerror(status)); error = 1; } /* * Initialization. @@ -417,7 +419,7 @@ PyThread_allocate_lock(void) status = pthread_mutex_init(&lock->mut, pthread_mutexattr_default); - CHECK_STATUS("pthread_mutex_init"); + CHECK_STATUS_PTHREAD("pthread_mutex_init"); /* Mark the pthread mutex underlying a Python mutex as pure happens-before. We can't simply mark the Python-level mutex as a mutex because it can be @@ -427,7 +429,7 @@ PyThread_allocate_lock(void) status = pthread_cond_init(&lock->lock_released, pthread_condattr_default); - CHECK_STATUS("pthread_cond_init"); + CHECK_STATUS_PTHREAD("pthread_cond_init"); if (error) { PyMem_RawFree((void *)lock); @@ -452,10 +454,10 @@ PyThread_free_lock(PyThread_type_lock lock) * and must have the cond destroyed first. */ status = pthread_cond_destroy( &thelock->lock_released ); - CHECK_STATUS("pthread_cond_destroy"); + CHECK_STATUS_PTHREAD("pthread_cond_destroy"); status = pthread_mutex_destroy( &thelock->mut ); - CHECK_STATUS("pthread_mutex_destroy"); + CHECK_STATUS_PTHREAD("pthread_mutex_destroy"); PyMem_RawFree((void *)thelock); } @@ -472,7 +474,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, lock, microseconds, intr_flag)); status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_lock[1]"); + CHECK_STATUS_PTHREAD("pthread_mutex_lock[1]"); if (thelock->locked == 0) { success = PY_LOCK_ACQUIRED; @@ -494,13 +496,13 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, &thelock->mut, &ts); if (status == ETIMEDOUT) break; - CHECK_STATUS("pthread_cond_timed_wait"); + CHECK_STATUS_PTHREAD("pthread_cond_timed_wait"); } else { status = pthread_cond_wait( &thelock->lock_released, &thelock->mut); - CHECK_STATUS("pthread_cond_wait"); + CHECK_STATUS_PTHREAD("pthread_cond_wait"); } if (intr_flag && status == 0 && thelock->locked) { @@ -518,7 +520,7 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, } if (success == PY_LOCK_ACQUIRED) thelock->locked = 1; status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_unlock[1]"); + CHECK_STATUS_PTHREAD("pthread_mutex_unlock[1]"); if (error) success = PY_LOCK_FAILURE; dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n", @@ -536,16 +538,16 @@ PyThread_release_lock(PyThread_type_lock lock) dprintf(("PyThread_release_lock(%p) called\n", lock)); status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_lock[3]"); + CHECK_STATUS_PTHREAD("pthread_mutex_lock[3]"); thelock->locked = 0; /* wake up someone (anyone, if any) waiting on the lock */ status = pthread_cond_signal( &thelock->lock_released ); - CHECK_STATUS("pthread_cond_signal"); + CHECK_STATUS_PTHREAD("pthread_cond_signal"); status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS("pthread_mutex_unlock[3]"); + CHECK_STATUS_PTHREAD("pthread_mutex_unlock[3]"); } #endif /* USE_SEMAPHORES */ diff --git a/README.rst b/README.rst index 9ea0360..c43cd44 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,14 @@ -This is Python version 3.6.1 +This is Python version 3.6.2 ============================ .. image:: https://travis-ci.org/python/cpython.svg?branch=3.6 :alt: CPython build status on Travis CI :target: https://travis-ci.org/python/cpython +.. image:: https://ci.appveyor.com/api/projects/status/4mew1a93xdkbf5ua/branch/3.6?svg=true + :alt: CPython build status on Appveyor + :target: https://ci.appveyor.com/project/python/cpython/branch/3.6 + .. image:: https://codecov.io/gh/python/cpython/branch/3.6/graph/badge.svg :alt: CPython code coverage on Codecov :target: https://codecov.io/gh/python/cpython diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index cc1afbe..31ae811 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -362,6 +362,7 @@ class PyObjectPtr(object): 'set' : PySetObjectPtr, 'frozenset' : PySetObjectPtr, 'builtin_function_or_method' : PyCFunctionObjectPtr, + 'method-wrapper': wrapperobject, } if tp_name in name_map: return name_map[tp_name] @@ -714,7 +715,7 @@ class PyDictObjectPtr(PyObjectPtr): try: # <= Python 3.5 return keys['dk_entries'], dk_size - except gdb.error: + except RuntimeError: # >= Python 3.6 pass @@ -1330,6 +1331,39 @@ class PyUnicodeObjectPtr(PyObjectPtr): out.write(quote) +class wrapperobject(PyObjectPtr): + _typename = 'wrapperobject' + + def safe_name(self): + try: + name = self.field('descr')['d_base']['name'].string() + return repr(name) + except (NullPyObjectPtr, RuntimeError): + return '' + + def safe_tp_name(self): + try: + return self.field('self')['ob_type']['tp_name'].string() + except (NullPyObjectPtr, RuntimeError): + return '' + + def safe_self_addresss(self): + try: + address = long(self.field('self')) + return '%#x' % address + except (NullPyObjectPtr, RuntimeError): + return '' + + def proxyval(self, visited): + name = self.safe_name() + tp_name = self.safe_tp_name() + self_address = self.safe_self_addresss() + return ("" + % (name, tp_name, self_address)) + + def write_repr(self, out, visited): + proxy = self.proxyval(visited) + out.write(proxy) def int_from_int(gdbval): @@ -1364,11 +1398,13 @@ class PyObjectPtrPrinter: def pretty_printer_lookup(gdbval): type = gdbval.type.unqualified() - if type.code == gdb.TYPE_CODE_PTR: - type = type.target().unqualified() - t = str(type) - if t in ("PyObject", "PyFrameObject", "PyUnicodeObject"): - return PyObjectPtrPrinter(gdbval) + if type.code != gdb.TYPE_CODE_PTR: + return None + + type = type.target().unqualified() + t = str(type) + if t in ("PyObject", "PyFrameObject", "PyUnicodeObject", "wrapperobject"): + return PyObjectPtrPrinter(gdbval) """ During development, I've been manually invoking the code in this way: @@ -1497,11 +1533,8 @@ class Frame(object): return 'Garbage-collecting' # Detect invocations of PyCFunction instances: - older = self.older() - if not older: - return False - - caller = older._gdbframe.name() + frame = self._gdbframe + caller = frame.name() if not caller: return False @@ -1513,18 +1546,25 @@ class Frame(object): # "self" is the (PyObject*) of the 'self' try: # Use the prettyprinter for the func: - func = older._gdbframe.read_var('func') + func = frame.read_var('func') return str(func) except RuntimeError: return 'PyCFunction invocation (unable to read "func")' elif caller == '_PyCFunction_FastCallDict': try: - func = older._gdbframe.read_var('func_obj') + func = frame.read_var('func_obj') return str(func) except RuntimeError: return 'PyCFunction invocation (unable to read "func_obj")' + if caller == 'wrapper_call': + try: + func = frame.read_var('wp') + return str(func) + except RuntimeError: + return '' + # This frame isn't worth reporting: return False diff --git a/Tools/hg/hgtouch.py b/Tools/hg/hgtouch.py deleted file mode 100644 index fbca469..0000000 --- a/Tools/hg/hgtouch.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Bring time stamps of generated checked-in files into the right order - -A versioned configuration file .hgtouch specifies generated files, in the -syntax of make rules. - - output: input1 input2 - -In addition to the dependency syntax, #-comments are supported. -""" -import errno -import os -import time - -def parse_config(repo): - try: - fp = repo.wfile(".hgtouch") - except IOError, e: - if e.errno != errno.ENOENT: - raise - return {} - result = {} - with fp: - for line in fp: - # strip comments - line = line.split('#')[0].strip() - if ':' not in line: - continue - outputs, inputs = line.split(':', 1) - outputs = outputs.split() - inputs = inputs.split() - for o in outputs: - try: - result[o].extend(inputs) - except KeyError: - result[o] = inputs - return result - -def check_rule(ui, repo, modified, basedir, output, inputs): - """Verify that the output is newer than any of the inputs. - Return (status, stamp), where status is True if the update succeeded, - and stamp is the newest time stamp assigned to any file (might be in - the future). - - If basedir is nonempty, it gives a directory in which the tree is to - be checked. - """ - f_output = repo.wjoin(os.path.join(basedir, output)) - try: - o_time = os.stat(f_output).st_mtime - except OSError: - ui.warn("Generated file %s does not exist\n" % output) - return False, 0 - youngest = 0 # youngest dependency - backdate = None - backdate_source = None - for i in inputs: - f_i = repo.wjoin(os.path.join(basedir, i)) - try: - i_time = os.stat(f_i).st_mtime - except OSError: - ui.warn(".hgtouch input file %s does not exist\n" % i) - return False, 0 - if i in modified: - # input is modified. Need to backdate at least to i_time - if backdate is None or backdate > i_time: - backdate = i_time - backdate_source = i - continue - youngest = max(i_time, youngest) - if backdate is not None: - ui.warn("Input %s for file %s locally modified\n" % (backdate_source, output)) - # set to 1s before oldest modified input - backdate -= 1 - os.utime(f_output, (backdate, backdate)) - return False, 0 - if youngest >= o_time: - ui.note("Touching %s\n" % output) - youngest += 1 - os.utime(f_output, (youngest, youngest)) - return True, youngest - else: - # Nothing to update - return True, 0 - -def do_touch(ui, repo, basedir): - if basedir: - if not os.path.isdir(repo.wjoin(basedir)): - ui.warn("Abort: basedir %r does not exist\n" % basedir) - return - modified = [] - else: - modified = repo.status()[0] - dependencies = parse_config(repo) - success = True - tstamp = 0 # newest time stamp assigned - # try processing all rules in topological order - hold_back = {} - while dependencies: - output, inputs = dependencies.popitem() - # check whether any of the inputs is generated - for i in inputs: - if i in dependencies: - hold_back[output] = inputs - continue - _success, _tstamp = check_rule(ui, repo, modified, basedir, output, inputs) - success = success and _success - tstamp = max(tstamp, _tstamp) - # put back held back rules - dependencies.update(hold_back) - hold_back = {} - now = time.time() - if tstamp > now: - # wait until real time has passed the newest time stamp, to - # avoid having files dated in the future - time.sleep(tstamp-now) - if hold_back: - ui.warn("Cyclic dependency involving %s\n" % (' '.join(hold_back.keys()))) - return False - return success - -def touch(ui, repo, basedir): - "touch generated files that are older than their sources after an update." - do_touch(ui, repo, basedir) - -cmdtable = { - "touch": (touch, - [('b', 'basedir', '', 'base dir of the tree to apply touching')], - "hg touch [-b BASEDIR]") -} diff --git a/Tools/msi/build.bat b/Tools/msi/build.bat index 69f00c0..fe5158d 100644 --- a/Tools/msi/build.bat +++ b/Tools/msi/build.bat @@ -22,8 +22,8 @@ if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) call "%D%get_externals.bat" - -call "%PCBUILD%env.bat" x86 +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if defined BUILDX86 ( call "%PCBUILD%build.bat" -d -e %REBUILD% %BUILDTEST% @@ -44,7 +44,7 @@ if defined BUILDDOC ( ) rem Build the launcher MSI separately -msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 +%MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 set BUILD_CMD="%D%bundle\snapshot.wixproj" if defined BUILDTEST ( @@ -58,11 +58,11 @@ if defined REBUILD ( ) if defined BUILDX86 ( - msbuild %BUILD_CMD% + %MSBUILD% %BUILD_CMD% if errorlevel 1 goto :eof ) if defined BUILDX64 ( - msbuild /p:Platform=x64 %BUILD_CMD% + %MSBUILD% /p:Platform=x64 %BUILD_CMD% if errorlevel 1 goto :eof ) diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index 81a3f86..d4500bc 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -69,6 +69,8 @@ if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP if not exist "%GIT%" echo Cannot find Git on PATH && exit /B 1 call "%D%get_externals.bat" +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) :builddoc if "%SKIPBUILD%" EQU "1" goto skipdoc @@ -165,28 +167,27 @@ if not "%SKIPBUILD%" EQU "1" ( @echo off ) -call "%PCBUILD%env.bat" if "%OUTDIR_PLAT%" EQU "win32" ( - msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% + %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% if errorlevel 1 exit /B ) else if not exist "%PCBUILD%win32\en-us\launcher.msi" ( - msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% + %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% if errorlevel 1 exit /B ) set BUILDOPTS=/p:Platform=%1 /p:BuildForRelease=true /p:DownloadUrl=%DOWNLOAD_URL% /p:DownloadUrlBase=%DOWNLOAD_URL_BASE% /p:ReleaseUri=%RELEASE_URI% -msbuild "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true +%MSBUILD% "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true if errorlevel 1 exit /B -msbuild "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false +%MSBUILD% "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false if errorlevel 1 exit /B if defined BUILDZIP ( - msbuild "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us" + %MSBUILD% "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us" if errorlevel 1 exit /B ) if defined BUILDNUGET ( - msbuild "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us" + %MSBUILD% "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us" if errorlevel 1 exit /B ) diff --git a/Tools/msi/bundle/bootstrap/pythonba.sln b/Tools/msi/bundle/bootstrap/pythonba.sln index bf43fed..4857180 100644 --- a/Tools/msi/bundle/bootstrap/pythonba.sln +++ b/Tools/msi/bundle/bootstrap/pythonba.sln @@ -1,22 +1,22 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2013 -VisualStudioVersion = 12.0.30501.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythonba", "pythonba.vcxproj", "{7A09B132-B3EE-499B-A700-A4B2157FEA3D}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Release|Win32 = Release|Win32 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {7A09B132-B3EE-499B-A700-A4B2157FEA3D}.Debug|Win32.ActiveCfg = Debug|Win32 - {7A09B132-B3EE-499B-A700-A4B2157FEA3D}.Debug|Win32.Build.0 = Debug|Win32 - {7A09B132-B3EE-499B-A700-A4B2157FEA3D}.Release|Win32.ActiveCfg = Release|Win32 - {7A09B132-B3EE-499B-A700-A4B2157FEA3D}.Release|Win32.Build.0 = Release|Win32 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.30501.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythonba", "pythonba.vcxproj", "{7A09B132-B3EE-499B-A700-A4B2157FEA3D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {7A09B132-B3EE-499B-A700-A4B2157FEA3D}.Debug|Win32.ActiveCfg = Debug|Win32 + {7A09B132-B3EE-499B-A700-A4B2157FEA3D}.Debug|Win32.Build.0 = Debug|Win32 + {7A09B132-B3EE-499B-A700-A4B2157FEA3D}.Release|Win32.ActiveCfg = Release|Win32 + {7A09B132-B3EE-499B-A700-A4B2157FEA3D}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/Tools/msi/bundle/bootstrap/pythonba.vcxproj b/Tools/msi/bundle/bootstrap/pythonba.vcxproj index bcd3951..3b101d7 100644 --- a/Tools/msi/bundle/bootstrap/pythonba.vcxproj +++ b/Tools/msi/bundle/bootstrap/pythonba.vcxproj @@ -1,69 +1,69 @@ - - - - - - Debug - Win32 - - - Release - Win32 - - - - Release - Win32 - v140 - v120 - {7A09B132-B3EE-499B-A700-A4B2157FEA3D} - PythonBA - - - - - DynamicLibrary - Unicode - $(Py_IntDir)\$(Configuration)_$(Platform)_Setup\Bootstrap\ - $(IntDir) - - - - - _CRT_STDIO_LEGACY_WIDE_SPECIFIERS=1;%(PreprocessorDefinitions) - $(WixInstallPath)sdk\inc - Use - pch.h - MultiThreaded - - - comctl32.lib;gdiplus.lib;msimg32.lib;shlwapi.lib;wininet.lib;dutil.lib;balutil.lib;version.lib;uxtheme.lib;%(AdditionalDependencies) - $(WixInstallPath)sdk\vs2015\lib\x86 - $(WixInstallPath)sdk\vs2013\lib\x86 - pythonba.def - true - - - - - - - Create - - - - - - - - - - + + + + + + Debug + Win32 + + + Release + Win32 + + + + Release + Win32 + v140 + v120 + {7A09B132-B3EE-499B-A700-A4B2157FEA3D} + PythonBA + + + + + DynamicLibrary + Unicode + $(Py_IntDir)\$(Configuration)_$(Platform)_Setup\Bootstrap\ + $(IntDir) + + + + + _CRT_STDIO_LEGACY_WIDE_SPECIFIERS=1;%(PreprocessorDefinitions) + $(WixInstallPath)sdk\inc + Use + pch.h + MultiThreaded + + + comctl32.lib;gdiplus.lib;msimg32.lib;shlwapi.lib;wininet.lib;dutil.lib;balutil.lib;version.lib;uxtheme.lib;%(AdditionalDependencies) + $(WixInstallPath)sdk\vs2015\lib\x86 + $(WixInstallPath)sdk\vs2013\lib\x86 + pythonba.def + true + + + + + + + Create + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/get_externals.bat b/Tools/msi/get_externals.bat index 4ead75e..c7d13c6 100644 --- a/Tools/msi/get_externals.bat +++ b/Tools/msi/get_externals.bat @@ -1,27 +1,107 @@ @echo off setlocal -rem Simple script to fetch source for external tools - -where /Q svn -if ERRORLEVEL 1 ( - echo.svn.exe must be on your PATH to get external tools. - echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the - echo.command line tools option. - popd - exit /b 1 +rem Simple script to fetch source for external libraries + +set HERE=%~dp0 +if "%PCBUILD%"=="" (set PCBUILD=%HERE%..\..\PCbuild\) +if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%HERE%..\..\externals\windows-installer) +if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\..\nuget.exe) +if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl) + +set DO_FETCH=true +set DO_CLEAN=false + +:CheckOpts +if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts +if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts +if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts +if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts +if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean +if "x%~1" NEQ "x" goto usage + +if "%DO_CLEAN%"=="false" goto fetch +:clean +echo.Cleaning up external libraries. +if exist "%EXTERNALS_DIR%" ( + rem Sometimes this fails the first time; try it twice + rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%" ) -if not exist "%~dp0..\..\externals" mkdir "%~dp0..\..\externals" -pushd "%~dp0..\..\externals" +if "%DO_FETCH%"=="false" goto end +:fetch -if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/ +if "%ORG%"=="" (set ORG=python) -if not exist "windows-installer\.svn" ( - echo.Checking out installer dependencies to %CD%\windows-installer - svn co %SVNROOT%windows-installer -) else ( - echo.Updating installer dependencies in %CD%\windows-installer - svn up windows-installer +if "%PYTHON_FOR_BUILD%"=="" ( + echo Checking for installed python... + py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6) ) +if "%PYTHON_FOR_BUILD%"=="" ( + if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%" + if NOT exist "%NUGET%" ( + echo Downloading nuget... + rem NB: Must use single quotes around NUGET here, NOT double! + rem Otherwise, a space in the path would break things + powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%' + ) + echo Installing Python via nuget... + "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%" + rem Quote it here; it's not quoted later because "py -3.6" wouldn't work + set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe" +) + +echo.Fetching external libraries... + +set libraries= + +for %%e in (%libraries%) do ( + if exist "%EXTERNALS_DIR%\%%e" ( + echo.%%e already exists, skipping. + ) else ( + echo.Fetching %%e... + %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e + ) +) + +echo.Fetching external tools... + +set binaries= +rem We always use whatever's latest in the repo for these +set binaries=%binaries% binutils +set binaries=%binaries% gpg +set binaries=%binaries% htmlhelp +set binaries=%binaries% nuget +set binaries=%binaries% redist +set binaries=%binaries% wix + +for %%b in (%binaries%) do ( + if exist "%EXTERNALS_DIR%\%%b" ( + echo.%%b already exists, skipping. + ) else ( + echo.Fetching %%b... + %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b + ) +) + +echo Finished. +goto end + +:usage +echo.Valid options: -c, --clean, --clean-only, --organization, --python, +echo.--no-tkinter, --no-openssl +echo. +echo.Pull all sources and binaries necessary for compiling optional extension +echo.modules that rely on external libraries. +echo. +echo.The --organization option determines which github organization to download +echo.from, the --python option determines which Python 3.6+ interpreter to use +echo.with PCbuild\get_external.py. +echo. +echo.Use the -c or --clean option to remove the entire externals directory. +echo. +echo.Use the --clean-only option to do the same cleaning, without pulling in +echo.anything new. +echo. +exit /b -1 -popd +:end diff --git a/Tools/msi/make_zip.proj b/Tools/msi/make_zip.proj index b3588b7..eb74565 100644 --- a/Tools/msi/make_zip.proj +++ b/Tools/msi/make_zip.proj @@ -1,38 +1,38 @@ - - - - {10487945-15D1-4092-A214-338395C4116B} - python - - false - - - - - - false - python-$(PythonVersion)-embed-$(ArchName) - .zip - $(OutputPath)\$(TargetName)$(TargetExt) - rmdir /q/s "$(IntermediateOutputPath)\zip_$(ArchName)" - "$(PythonExe)" "$(MSBuildThisFileDirectory)\make_zip.py" - $(Arguments) -e -o "$(TargetPath)" -t "$(IntermediateOutputPath)\zip_$(ArchName)" -a $(ArchName) - set DOC_FILENAME=python$(PythonVersion).chm - $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) - - - - - - - - - - - - - - - - - + + + + {10487945-15D1-4092-A214-338395C4116B} + python + + false + + + + + + false + python-$(PythonVersion)-embed-$(ArchName) + .zip + $(OutputPath)\$(TargetName)$(TargetExt) + rmdir /q/s "$(IntermediateOutputPath)\zip_$(ArchName)" + "$(PythonExe)" "$(MSBuildThisFileDirectory)\make_zip.py" + $(Arguments) -e -o "$(TargetPath)" -t "$(IntermediateOutputPath)\zip_$(ArchName)" -a $(ArchName) + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) + + + + + + + + + + + + + + + + + diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props index 60abba1..40ad904 100644 --- a/Tools/msi/msi.props +++ b/Tools/msi/msi.props @@ -1,188 +1,188 @@ - - - - $(OutputName) - false - false - $(SuppressIces);ICE03;ICE57;ICE61 - 1026 - false - true - Release - x86 - perUser - <_MakeCatCommand Condition="'$(_MakeCatCommand)' == ''">makecat - - - - - - - - $(ComputerName)/$(ArchName)/ - $(ReleaseUri)/ - - - - - - - - WixUtilExtension - WixUtilExtension - - - - - $(Py_IntDir)\$(Configuration)_$(Platform)_Setup\$(OutputName) - $(IntermediateOutputPath)_$(OutputSuffix) - $(BuildPath) - $(OutputPath)\ - $(OutputPath) - true - $(ExternalsDir)\windows-installer\redist - - python$(MajorVersionNumber)$(MinorVersionNumber)$(MicroVersionNumber)$(ReleaseLevelName).chm - - $(MajorVersionNumber).$(MinorVersionNumber).$(Field3Value).0 - - - - $([System.Math]::Floor($([System.DateTime]::Now.Subtract($([System.DateTime]::new(2001, 1, 1))).TotalDays))) - $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)dev$(RevisionNumber) - $(MajorVersionNumber).$(MinorVersionNumber).$(RevisionNumber).0 - - - - 32-bit - 64-bit - 32bit - 64bit - - $(DefineConstants); - Version=$(InstallerVersion); - ShortVersion=$(MajorVersionNumber).$(MinorVersionNumber); - LongVersion=$(PythonVersion); - MajorVersionNumber=$(MajorVersionNumber); - MinorVersionNumber=$(MinorVersionNumber); - UpgradeMinimumVersion=$(MajorVersionNumber).$(MinorVersionNumber).0.0; - NextMajorVersionNumber=$(MajorVersionNumber).$([msbuild]::Add($(MinorVersionNumber), 1)).0.0; - Bitness=$(Bitness); - PlatformArchitecture=$(PlatformArchitecture); - PyDebugExt=$(PyDebugExt); - PyArchExt=$(PyArchExt); - PyTestExt=$(PyTestExt); - OptionalFeatureName=$(OutputName); - - - $(DefineConstants);CRTRedist=$(CRTRedist); - - - $(DefineConstants);Suffix32=-32; - - - $(DefineConstants);Suffix32=; - - - - - - generated_filelist - - - false - - - false - - - - - - - - - - - src - - - tcltk - - - redist - - - redist - - - build32 - - - build64 - - - - - - - - - <_Uuid Include="CoreUpgradeCode"> - upgradecode - - <_Uuid Include="UpgradeCode"> - upgradecode/$(OutputName) - - <_Uuid Include="InstallDirectoryGuidSeed"> - installdirectoryseed - - <_Uuid Include="PythonExeComponentGuid"> - python.exe - - <_Uuid Include="PythonwExeComponentGuid"> - pythonw.exe - - <_Uuid Include="RemoveLib2to3PickleComponentGuid"> - lib2to3/pickles - - <_Uuid Include="CommonPythonRegComponentGuid"> - registry - - <_Uuid Include="PythonRegComponentGuid"> - registry/$(OutputName) - - - - - <_Uuids>@(_Uuid->'("%(Identity)", "$(MajorVersionNumber).$(MinorVersionNumber)/%(Uri)")',',') - <_GenerateCommand>import uuid; print('\n'.join('{}={}'.format(i, uuid.uuid5(uuid.UUID('c8d9733e-a70c-43ff-ab0c-e26456f11083'), '$(ReleaseUri.Replace(`{arch}`, `$(ArchName)`))' + j)) for i,j in [$(_Uuids.Replace(`"`,`'`))])) - - - - - - - - - - $(DefineConstants);@(_UuidValue,';'); - - + + + + $(OutputName) + false + false + $(SuppressIces);ICE03;ICE57;ICE61 + 1026 + false + true + Release + x86 + perUser + <_MakeCatCommand Condition="'$(_MakeCatCommand)' == ''">makecat + + + + + + + + $(ComputerName)/$(ArchName)/ + $(ReleaseUri)/ + + + + + + + + WixUtilExtension + WixUtilExtension + + + + + $(Py_IntDir)\$(Configuration)_$(Platform)_Setup\$(OutputName) + $(IntermediateOutputPath)_$(OutputSuffix) + $(BuildPath) + $(OutputPath)\ + $(OutputPath) + true + $(ExternalsDir)\windows-installer\redist + + python$(MajorVersionNumber)$(MinorVersionNumber)$(MicroVersionNumber)$(ReleaseLevelName).chm + + $(MajorVersionNumber).$(MinorVersionNumber).$(Field3Value).0 + + + + $([System.Math]::Floor($([System.DateTime]::Now.Subtract($([System.DateTime]::new(2001, 1, 1))).TotalDays))) + $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)dev$(RevisionNumber) + $(MajorVersionNumber).$(MinorVersionNumber).$(RevisionNumber).0 + + + + 32-bit + 64-bit + 32bit + 64bit + + $(DefineConstants); + Version=$(InstallerVersion); + ShortVersion=$(MajorVersionNumber).$(MinorVersionNumber); + LongVersion=$(PythonVersion); + MajorVersionNumber=$(MajorVersionNumber); + MinorVersionNumber=$(MinorVersionNumber); + UpgradeMinimumVersion=$(MajorVersionNumber).$(MinorVersionNumber).0.0; + NextMajorVersionNumber=$(MajorVersionNumber).$([msbuild]::Add($(MinorVersionNumber), 1)).0.0; + Bitness=$(Bitness); + PlatformArchitecture=$(PlatformArchitecture); + PyDebugExt=$(PyDebugExt); + PyArchExt=$(PyArchExt); + PyTestExt=$(PyTestExt); + OptionalFeatureName=$(OutputName); + + + $(DefineConstants);CRTRedist=$(CRTRedist); + + + $(DefineConstants);Suffix32=-32; + + + $(DefineConstants);Suffix32=; + + + + + + generated_filelist + + + false + + + false + + + + + + + + + + + src + + + tcltk + + + redist + + + redist + + + build32 + + + build64 + + + + + + + + + <_Uuid Include="CoreUpgradeCode"> + upgradecode + + <_Uuid Include="UpgradeCode"> + upgradecode/$(OutputName) + + <_Uuid Include="InstallDirectoryGuidSeed"> + installdirectoryseed + + <_Uuid Include="PythonExeComponentGuid"> + python.exe + + <_Uuid Include="PythonwExeComponentGuid"> + pythonw.exe + + <_Uuid Include="RemoveLib2to3PickleComponentGuid"> + lib2to3/pickles + + <_Uuid Include="CommonPythonRegComponentGuid"> + registry + + <_Uuid Include="PythonRegComponentGuid"> + registry/$(OutputName) + + + + + <_Uuids>@(_Uuid->'("%(Identity)", "$(MajorVersionNumber).$(MinorVersionNumber)/%(Uri)")',',') + <_GenerateCommand>import uuid; print('\n'.join('{}={}'.format(i, uuid.uuid5(uuid.UUID('c8d9733e-a70c-43ff-ab0c-e26456f11083'), '$(ReleaseUri.Replace(`{arch}`, `$(ArchName)`))' + j)) for i,j in [$(_Uuids.Replace(`"`,`'`))])) + + + + + + + + + + $(DefineConstants);@(_UuidValue,';'); + + \ No newline at end of file diff --git a/Tools/msi/uploadrelease.bat b/Tools/msi/uploadrelease.bat index 670836b..07b2573 100644 --- a/Tools/msi/uploadrelease.bat +++ b/Tools/msi/uploadrelease.bat @@ -49,16 +49,17 @@ if defined NOGPG ( echo Found gpg2.exe at %GPG% ) -call "%PCBUILD%env.bat" > nul 2> nul +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) pushd "%D%" -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% +%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% +%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% if not defined NOTEST ( - msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 - msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 ) -msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 -msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false +%MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 +%MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false popd exit /B 0 diff --git a/Tools/msi/uploadrelease.proj b/Tools/msi/uploadrelease.proj index 75840f2..dd82ee0 100644 --- a/Tools/msi/uploadrelease.proj +++ b/Tools/msi/uploadrelease.proj @@ -1,112 +1,112 @@ - - - - {2D69F2AB-D5D0-4344-84B5-EF6DB34A9BC9} - python - - - $(TARGET) - /srv/www.python.org/ftp/python - true - true - false - false - - - - - - - $(DownloadUrlBase.TrimEnd(`/`))/$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber) - $(DownloadUrl.Replace(`{version}`, `$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)`).Replace(`{arch}`, `$(ArchName)`).Replace(`{releasename}`, `$(ReleaseLevelName)`).Replace(`{msi}`, ``).TrimEnd(`/`)) - - - - - $(MSITarget) - - - $(EXETarget) - - - $(EXETarget) - - - - - - - - - - - - - - - - - %(File.CopyTo) - - - - - - - - - - - - - - - - - - - - - - $(TEMP)\%(Filename)_source - $(TEMP)\%(Filename)_source\%(Filename)%(Extension) - $(TEMP)\%(Filename)_layout - $(OutputPath)\%(Filename)_layoutlog - $(OutputPath)\%(Filename)_layoutlog\%(Filename).log - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + {2D69F2AB-D5D0-4344-84B5-EF6DB34A9BC9} + python + + + $(TARGET) + /srv/www.python.org/ftp/python + true + true + false + false + + + + + + + $(DownloadUrlBase.TrimEnd(`/`))/$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber) + $(DownloadUrl.Replace(`{version}`, `$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)`).Replace(`{arch}`, `$(ArchName)`).Replace(`{releasename}`, `$(ReleaseLevelName)`).Replace(`{msi}`, ``).TrimEnd(`/`)) + + + + + $(MSITarget) + + + $(EXETarget) + + + $(EXETarget) + + + + + + + + + + + + + + + + + %(File.CopyTo) + + + + + + + + + + + + + + + + + + + + + + $(TEMP)\%(Filename)_source + $(TEMP)\%(Filename)_source\%(Filename)%(Extension) + $(TEMP)\%(Filename)_layout + $(OutputPath)\%(Filename)_layoutlog + $(OutputPath)\%(Filename)_layoutlog\%(Filename).log + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Tools/msi/wix.props b/Tools/msi/wix.props index fbb2d10..18a304d 100644 --- a/Tools/msi/wix.props +++ b/Tools/msi/wix.props @@ -1,12 +1,12 @@ - - - - - - $(MSBuildThisFileDirectory)\Wix\ - $(ExternalsDir)\windows-installer\wix\ - $(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows Installer XML\3.10@InstallRoot) - $(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Windows Installer XML\3.10@InstallRoot) - $(WixInstallPath)\Wix.targets - + + + + + + $(MSBuildThisFileDirectory)\Wix\ + $(ExternalsDir)\windows-installer\wix\ + $(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows Installer XML\3.10@InstallRoot) + $(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Windows Installer XML\3.10@InstallRoot) + $(WixInstallPath)\Wix.targets + \ No newline at end of file diff --git a/Tools/nuget/build.bat b/Tools/nuget/build.bat index 120b38c..b02e770 100644 --- a/Tools/nuget/build.bat +++ b/Tools/nuget/build.bat @@ -21,7 +21,8 @@ if "%~1" EQU "-p" (set PACKAGES=%PACKAGES% %~2) && shift && shift && goto CheckO if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) call "%D%..\msi\get_externals.bat" -call "%PCBUILD%env.bat" x86 +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if defined PACKAGES set PACKAGES="/p:Packages=%PACKAGES%" @@ -30,7 +31,7 @@ if defined BUILDX86 ( ) else if not exist "%PCBUILD%win32\python.exe" call "%PCBUILD%build.bat" -e if errorlevel 1 goto :eof - msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES% + %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES% if errorlevel 1 goto :eof ) @@ -39,7 +40,7 @@ if defined BUILDX64 ( ) else if not exist "%PCBUILD%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e if errorlevel 1 goto :eof - msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES% + %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES% if errorlevel 1 goto :eof ) diff --git a/Tools/nuget/make_pkg.proj b/Tools/nuget/make_pkg.proj index 464ef04..284f593 100644 --- a/Tools/nuget/make_pkg.proj +++ b/Tools/nuget/make_pkg.proj @@ -1,55 +1,55 @@ - - - - {10487945-15D1-4092-A214-338395C4116B} - python - $(OutputName)x86 - - false - true - - - - - - $(ExternalsDir)\windows-installer\nuget\nuget.exe - $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber) - $(NuspecVersion)-$(ReleaseLevelName) - false - $(OutputName).$(NuspecVersion) - .nupkg - $(IntermediateOutputPath)\nuget_$(ArchName) - - rmdir /q/s "$(IntermediateOutputPath)" - - "$(PythonExe)" "$(MSBuildThisFileDirectory)\..\msi\make_zip.py" - $(PythonArguments) -t "$(IntermediateOutputPath)" -a $(ArchName) - - "$(IntermediateOutputPath)\python.exe" -B -c "import sys; sys.path.append(r'$(PySourcePath)\Lib'); import ensurepip; ensurepip._main()" - "$(IntermediateOutputPath)\python.exe" -B -m pip install -U $(Packages) - - "$(Nuget)" pack "$(MSBuildThisFileDirectory)\$(OutputName).nuspec" - $(NugetArguments) -BasePath "$(IntermediateOutputPath)" - $(NugetArguments) -OutputDirectory "$(OutputPath.Trim(`\`))" - $(NugetArguments) -Version "$(NuspecVersion)" - $(NugetArguments) -NoPackageAnalysis -NonInteractive - - set DOC_FILENAME=python$(PythonVersion).chm - $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) - - - - - - - - - - - - - - - - - + + + + {10487945-15D1-4092-A214-338395C4116B} + python + $(OutputName)x86 + + false + true + + + + + + $(ExternalsDir)\windows-installer\nuget\nuget.exe + $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber) + $(NuspecVersion)-$(ReleaseLevelName) + false + $(OutputName).$(NuspecVersion) + .nupkg + $(IntermediateOutputPath)\nuget_$(ArchName) + + rmdir /q/s "$(IntermediateOutputPath)" + + "$(PythonExe)" "$(MSBuildThisFileDirectory)\..\msi\make_zip.py" + $(PythonArguments) -t "$(IntermediateOutputPath)" -a $(ArchName) + + "$(IntermediateOutputPath)\python.exe" -B -c "import sys; sys.path.append(r'$(PySourcePath)\Lib'); import ensurepip; ensurepip._main()" + "$(IntermediateOutputPath)\python.exe" -B -m pip install -U $(Packages) + + "$(Nuget)" pack "$(MSBuildThisFileDirectory)\$(OutputName).nuspec" + $(NugetArguments) -BasePath "$(IntermediateOutputPath)" + $(NugetArguments) -OutputDirectory "$(OutputPath.Trim(`\`))" + $(NugetArguments) -Version "$(NuspecVersion)" + $(NugetArguments) -NoPackageAnalysis -NonInteractive + + set DOC_FILENAME=python$(PythonVersion).chm + $(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform) + + + + + + + + + + + + + + + + + diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index 58b081a..33a9fea 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -12,7 +12,6 @@ import untabify SRCDIR = sysconfig.get_config_var('srcdir') - def n_files_str(count): """Return 'N file(s)' with the proper plurality on 'file'.""" return "{} file{}".format(count, "s" if count != 1 else "") @@ -46,27 +45,76 @@ def mq_patches_applied(): return st.returncode == 0 and bstdout +def get_git_branch(): + """Get the symbolic name for the current git branch""" + cmd = "git rev-parse --abbrev-ref HEAD".split() + try: + return subprocess.check_output(cmd, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return None + + +def get_git_upstream_remote(): + """Get the remote name to use for upstream branches + + Uses "upstream" if it exists, "origin" otherwise + """ + cmd = "git remote get-url upstream".split() + try: + subprocess.check_output(cmd, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return "origin" + return "upstream" + + +@status("Getting base branch for PR", + info=lambda x: x if x is not None else "not a PR branch") +def get_base_branch(): + if not os.path.exists(os.path.join(SRCDIR, '.git')): + # Not a git checkout, so there's no base branch + return None + version = sys.version_info + if version.releaselevel == 'alpha': + base_branch = "master" + else: + base_branch = "{0.major}.{0.minor}".format(version) + this_branch = get_git_branch() + if this_branch is None or this_branch == base_branch: + # Not on a git PR branch, so there's no base branch + return None + upstream_remote = get_git_upstream_remote() + return upstream_remote + "/" + base_branch + + @status("Getting the list of files that have been added/changed", info=lambda x: n_files_str(len(x))) -def changed_files(): +def changed_files(base_branch=None): """Get the list of changed or added files from Mercurial or git.""" if os.path.isdir(os.path.join(SRCDIR, '.hg')): + if base_branch is not None: + sys.exit('need a git checkout to check PR status') cmd = 'hg status --added --modified --no-status' if mq_patches_applied(): cmd += ' --rev qparent' with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: return [x.decode().rstrip() for x in st.stdout] - elif os.path.isdir(os.path.join(SRCDIR, '.git')): - cmd = 'git status --porcelain' + elif os.path.exists(os.path.join(SRCDIR, '.git')): + # We just use an existence check here as: + # directory = normal git checkout/clone + # file = git worktree directory + if base_branch: + cmd = 'git diff --name-status ' + base_branch + else: + cmd = 'git status --porcelain' filenames = [] with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: for line in st.stdout: line = line.decode().rstrip() - status = set(line[:2]) + status_text, filename = line.split(maxsplit=1) + status = set(status_text) # modified, added or unmerged files if not status.intersection('MAU'): continue - filename = line[3:] if ' -> ' in filename: # file is renamed filename = filename.split(' -> ', 2)[1].strip() @@ -165,7 +213,8 @@ def regenerated_pyconfig_h_in(file_paths): return "not needed" def main(): - file_paths = changed_files() + base_branch = get_base_branch() + file_paths = changed_files(base_branch) python_files = [fn for fn in file_paths if fn.endswith('.py')] c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] doc_files = [fn for fn in file_paths if fn.startswith('Doc') and diff --git a/Tools/scripts/reindent.py b/Tools/scripts/reindent.py index 18424de..f6dadaa 100755 --- a/Tools/scripts/reindent.py +++ b/Tools/scripts/reindent.py @@ -118,7 +118,11 @@ def check(file): if verbose: print("checking", file, "...", end=' ') with open(file, 'rb') as f: - encoding, _ = tokenize.detect_encoding(f.readline) + try: + encoding, _ = tokenize.detect_encoding(f.readline) + except SyntaxError as se: + errprint("%s: SyntaxError: %s" % (file, str(se))) + return try: with open(file, encoding=encoding) as f: r = Reindenter(f) diff --git a/Tools/ssl/sslspeed.vcxproj b/Tools/ssl/sslspeed.vcxproj index 8ec4106..5761399 100644 --- a/Tools/ssl/sslspeed.vcxproj +++ b/Tools/ssl/sslspeed.vcxproj @@ -1,70 +1,70 @@ - - - - - Debug - Win32 - - - Release - Win32 - - - Debug - x64 - - - Release - x64 - - - - - {F068BCCF-C0D6-478D-A2C5-26BA3237C992} - sslspeed - - - - - - - - $(OutDir) - $(MSBuildProjectDirectory)\$(ArchName)\ - $(MSBuildProjectDirectory)\$(ArchName)\obj\ - Application - MultiByte - - - - - - - - - - - - _CONSOLE;%(PreprocessorDefinitions) - $(opensslIncDir);%(AdditionalIncludeDirectories) - - - ws2_32.lib;crypt32.lib;libeay$(PyDebugExt).lib;ssleay$(PyDebugExt).lib;%(AdditionalDependencies) - $(OriginalOutDir);%(AdditionalLibraryDirectories) - Console - - - - - - {10615b24-73bf-4efa-93aa-236916321317} - false - - - {e5b04cc0-eb4c-42ab-b4dc-18ef95f864b0} - false - - - - + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + {F068BCCF-C0D6-478D-A2C5-26BA3237C992} + sslspeed + + + + + + + + $(OutDir) + $(MSBuildProjectDirectory)\$(ArchName)\ + $(MSBuildProjectDirectory)\$(ArchName)\obj\ + Application + MultiByte + + + + + + + + + + + + _CONSOLE;%(PreprocessorDefinitions) + $(opensslIncDir);%(AdditionalIncludeDirectories) + + + ws2_32.lib;crypt32.lib;libeay$(PyDebugExt).lib;ssleay$(PyDebugExt).lib;%(AdditionalDependencies) + $(OriginalOutDir);%(AdditionalLibraryDirectories) + Console + + + + + + {10615b24-73bf-4efa-93aa-236916321317} + false + + + {e5b04cc0-eb4c-42ab-b4dc-18ef95f864b0} + false + + + + \ No newline at end of file diff --git a/aclocal.m4 b/aclocal.m4 index 9a9cc55..2a745e5 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -13,7 +13,7 @@ m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])]) dnl pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- -dnl serial 11 (pkg-config-0.29) +dnl serial 11 (pkg-config-0.29.1) dnl dnl Copyright © 2004 Scott James Remnant . dnl Copyright © 2012-2015 Dan Nicholson @@ -55,7 +55,7 @@ dnl dnl See the "Since" comment for each macro you use to see what version dnl of the macros you require. m4_defun([PKG_PREREQ], -[m4_define([PKG_MACROS_VERSION], [0.29]) +[m4_define([PKG_MACROS_VERSION], [0.29.1]) m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1, [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])]) ])dnl PKG_PREREQ diff --git a/config.guess b/config.guess index 1f5c50c..2193702 100755 --- a/config.guess +++ b/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2014 Free Software Foundation, Inc. +# Copyright 1992-2017 Free Software Foundation, Inc. -timestamp='2014-03-23' +timestamp='2017-05-27' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -24,12 +24,12 @@ timestamp='2014-03-23' # program. This Exception is an additional permission under section 7 # of the GNU General Public License, version 3 ("GPLv3"). # -# Originally written by Per Bothner. +# Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess # -# Please send patches with a ChangeLog entry to config-patches@gnu.org. +# Please send patches to . me=`echo "$0" | sed -e 's,.*/,,'` @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2014 Free Software Foundation, Inc. +Copyright 1992-2017 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -168,19 +168,29 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" - UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \ - /usr/sbin/$sysctl 2>/dev/null || echo unknown)` + UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ + /sbin/$sysctl 2>/dev/null || \ + /usr/sbin/$sysctl 2>/dev/null || \ + echo unknown)` case "${UNAME_MACHINE_ARCH}" in armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; + earmv*) + arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'` + endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'` + machine=${arch}${endian}-unknown + ;; *) machine=${UNAME_MACHINE_ARCH}-unknown ;; esac # The Operating System including object format, if it has switched - # to ELF recently, or will in the future. + # to ELF recently (or will in the future) and ABI. case "${UNAME_MACHINE_ARCH}" in + earm*) + os=netbsdelf + ;; arm*|i386|m68k|ns32k|sh3*|sparc|vax) eval $set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ @@ -197,6 +207,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in os=netbsd ;; esac + # Determine ABI tags. + case "${UNAME_MACHINE_ARCH}" in + earm*) + expr='s/^earmv[0-9]/-eabi/;s/eb$//' + abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"` + ;; + esac # The OS release # Debian GNU/NetBSD machines have a different userland, and # thus, need a distinct triplet. However, they do not need @@ -207,13 +224,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in release='-gnu' ;; *) - release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` + release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d. -f1,2` ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "${machine}-${os}${release}" + echo "${machine}-${os}${release}${abi}" exit ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` @@ -223,6 +240,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} exit ;; + *:LibertyBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-libertybsd${UNAME_RELEASE} + exit ;; *:ekkoBSD:*:*) echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} exit ;; @@ -235,6 +256,9 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in *:MirBSD:*:*) echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} exit ;; + *:Sortix:*:*) + echo ${UNAME_MACHINE}-unknown-sortix + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) @@ -251,42 +275,42 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` case "$ALPHA_CPU_TYPE" in "EV4 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV4.5 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "LCA4 (21066/21068)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV5 (21164)") - UNAME_MACHINE="alphaev5" ;; + UNAME_MACHINE=alphaev5 ;; "EV5.6 (21164A)") - UNAME_MACHINE="alphaev56" ;; + UNAME_MACHINE=alphaev56 ;; "EV5.6 (21164PC)") - UNAME_MACHINE="alphapca56" ;; + UNAME_MACHINE=alphapca56 ;; "EV5.7 (21164PC)") - UNAME_MACHINE="alphapca57" ;; + UNAME_MACHINE=alphapca57 ;; "EV6 (21264)") - UNAME_MACHINE="alphaev6" ;; + UNAME_MACHINE=alphaev6 ;; "EV6.7 (21264A)") - UNAME_MACHINE="alphaev67" ;; + UNAME_MACHINE=alphaev67 ;; "EV6.8CB (21264C)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8AL (21264B)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8CX (21264D)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.9A (21264/EV69A)") - UNAME_MACHINE="alphaev69" ;; + UNAME_MACHINE=alphaev69 ;; "EV7 (21364)") - UNAME_MACHINE="alphaev7" ;; + UNAME_MACHINE=alphaev7 ;; "EV7.9 (21364A)") - UNAME_MACHINE="alphaev79" ;; + UNAME_MACHINE=alphaev79 ;; esac # A Pn.n version is a patched version. # A Vn.n version is a released version. # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. - echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 @@ -359,16 +383,16 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) eval $set_cc_for_build - SUN_ARCH="i386" + SUN_ARCH=i386 # If there is a compiler, see if it is configured for 64-bit objects. # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. # This test works for both compilers. - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then - SUN_ARCH="x86_64" + SUN_ARCH=x86_64 fi fi echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` @@ -393,7 +417,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` - test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 + test "x${UNAME_RELEASE}" = x && UNAME_RELEASE=3 case "`/bin/arch`" in sun3) echo m68k-sun-sunos${UNAME_RELEASE} @@ -579,8 +603,9 @@ EOF else IBM_ARCH=powerpc fi - if [ -x /usr/bin/oslevel ] ; then - IBM_REV=`/usr/bin/oslevel` + if [ -x /usr/bin/lslpp ] ; then + IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | + awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` else IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} fi @@ -617,13 +642,13 @@ EOF sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 + 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 532) # CPU_PA_RISC2_0 case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; - '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 + 32) HP_ARCH=hppa2.0n ;; + 64) HP_ARCH=hppa2.0w ;; + '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 esac ;; esac fi @@ -662,11 +687,11 @@ EOF exit (0); } EOF - (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + (CCOPTS="" $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac - if [ ${HP_ARCH} = "hppa2.0w" ] + if [ ${HP_ARCH} = hppa2.0w ] then eval $set_cc_for_build @@ -679,12 +704,12 @@ EOF # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess # => hppa64-hp-hpux11.23 - if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | grep -q __LP64__ then - HP_ARCH="hppa2.0w" + HP_ARCH=hppa2.0w else - HP_ARCH="hppa64" + HP_ARCH=hppa64 fi fi echo ${HP_ARCH}-hp-hpux${HPUX_REV} @@ -789,14 +814,14 @@ EOF echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) @@ -812,10 +837,11 @@ EOF UNAME_PROCESSOR=`/usr/bin/uname -p` case ${UNAME_PROCESSOR} in amd64) - echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; - *) - echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + UNAME_PROCESSOR=x86_64 ;; + i386) + UNAME_PROCESSOR=i586 ;; esac + echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` exit ;; i*:CYGWIN*:*) echo ${UNAME_MACHINE}-pc-cygwin @@ -878,7 +904,7 @@ EOF exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} + echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} exit ;; i*86:Minix:*:*) echo ${UNAME_MACHINE}-pc-minix @@ -901,7 +927,7 @@ EOF EV68*) UNAME_MACHINE=alphaev68 ;; esac objdump --private-headers /bin/sh | grep -q ld.so.1 - if test "$?" = 0 ; then LIBC="gnulibc1" ; fi + if test "$?" = 0 ; then LIBC=gnulibc1 ; fi echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; arc:Linux:*:* | arceb:Linux:*:*) @@ -932,6 +958,9 @@ EOF crisv32:Linux:*:*) echo ${UNAME_MACHINE}-axis-linux-${LIBC} exit ;; + e2k:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; frv:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; @@ -944,6 +973,9 @@ EOF ia64:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; + k1om:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; m32r*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; @@ -969,6 +1001,9 @@ EOF eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } ;; + mips64el:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; openrisc*:Linux:*:*) echo or1k-unknown-linux-${LIBC} exit ;; @@ -1001,6 +1036,9 @@ EOF ppcle:Linux:*:*) echo powerpcle-unknown-linux-${LIBC} exit ;; + riscv32:Linux:*:* | riscv64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; s390:Linux:*:* | s390x:Linux:*:*) echo ${UNAME_MACHINE}-ibm-linux-${LIBC} exit ;; @@ -1020,7 +1058,7 @@ EOF echo ${UNAME_MACHINE}-dec-linux-${LIBC} exit ;; x86_64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo ${UNAME_MACHINE}-pc-linux-${LIBC} exit ;; xtensa*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} @@ -1099,7 +1137,7 @@ EOF # uname -m prints for DJGPP always 'pc', but it prints nothing about # the processor, so we play safe by assuming i586. # Note: whatever this is, it MUST be the same as what config.sub - # prints for the "djgpp" host, or else GDB configury will decide that + # prints for the "djgpp" host, or else GDB configure will decide that # this is a cross-build. echo i586-pc-msdosdjgpp exit ;; @@ -1248,6 +1286,9 @@ EOF SX-8R:SUPER-UX:*:*) echo sx8r-nec-superux${UNAME_RELEASE} exit ;; + SX-ACE:SUPER-UX:*:*) + echo sxace-nec-superux${UNAME_RELEASE} + exit ;; Power*:Rhapsody:*:*) echo powerpc-apple-rhapsody${UNAME_RELEASE} exit ;; @@ -1261,16 +1302,23 @@ EOF UNAME_PROCESSOR=powerpc fi if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null then case $UNAME_PROCESSOR in i386) UNAME_PROCESSOR=x86_64 ;; powerpc) UNAME_PROCESSOR=powerpc64 ;; esac fi + # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc + if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_PPC >/dev/null + then + UNAME_PROCESSOR=powerpc + fi fi elif test "$UNAME_PROCESSOR" = i386 ; then # Avoid executing cc on OS X 10.9, as it ships with a stub @@ -1285,7 +1333,7 @@ EOF exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` - if test "$UNAME_PROCESSOR" = "x86"; then + if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc fi @@ -1294,15 +1342,18 @@ EOF *:QNX:*:4*) echo i386-pc-qnx exit ;; - NEO-?:NONSTOP_KERNEL:*:*) + NEO-*:NONSTOP_KERNEL:*:*) echo neo-tandem-nsk${UNAME_RELEASE} exit ;; NSE-*:NONSTOP_KERNEL:*:*) echo nse-tandem-nsk${UNAME_RELEASE} exit ;; - NSR-?:NONSTOP_KERNEL:*:*) + NSR-*:NONSTOP_KERNEL:*:*) echo nsr-tandem-nsk${UNAME_RELEASE} exit ;; + NSX-*:NONSTOP_KERNEL:*:*) + echo nsx-tandem-nsk${UNAME_RELEASE} + exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux exit ;; @@ -1316,7 +1367,7 @@ EOF # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 # operating systems. - if test "$cputype" = "386"; then + if test "$cputype" = 386; then UNAME_MACHINE=i386 else UNAME_MACHINE="$cputype" @@ -1358,7 +1409,7 @@ EOF echo i386-pc-xenix exit ;; i*86:skyos:*:*) - echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE} | sed -e 's/ .*$//'` exit ;; i*86:rdos:*:*) echo ${UNAME_MACHINE}-pc-rdos @@ -1369,23 +1420,25 @@ EOF x86_64:VMkernel:*:*) echo ${UNAME_MACHINE}-unknown-esx exit ;; + amd64:Isilon\ OneFS:*:*) + echo x86_64-unknown-onefs + exit ;; esac cat >&2 < in order to provide the needed -information to handle your system. +If $0 has already been updated, send the following data and any +information you think might be pertinent to config-patches@gnu.org to +provide the necessary information to handle your system. config.guess timestamp = $timestamp diff --git a/config.sub b/config.sub index d654d03..40ea5df 100755 --- a/config.sub +++ b/config.sub @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2014 Free Software Foundation, Inc. +# Copyright 1992-2017 Free Software Foundation, Inc. -timestamp='2014-05-01' +timestamp='2017-04-02' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -25,7 +25,7 @@ timestamp='2014-05-01' # of the GNU General Public License, version 3 ("GPLv3"). -# Please send patches with a ChangeLog entry to config-patches@gnu.org. +# Please send patches to . # # Configuration subroutine to validate and canonicalize a configuration type. # Supply the specified configuration type as an argument. @@ -33,7 +33,7 @@ timestamp='2014-05-01' # Otherwise, we print the canonical config type on stdout and succeed. # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases @@ -53,8 +53,7 @@ timestamp='2014-05-01' me=`echo "$0" | sed -e 's,.*/,,'` usage="\ -Usage: $0 [OPTION] CPU-MFR-OPSYS - $0 [OPTION] ALIAS +Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS Canonicalize a configuration name. @@ -68,7 +67,7 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright 1992-2014 Free Software Foundation, Inc. +Copyright 1992-2017 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -117,8 +116,8 @@ maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` case $maybe_os in nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ - knetbsd*-gnu* | netbsd*-gnu* | \ - kopensolaris*-gnu* | \ + knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \ + kopensolaris*-gnu* | cloudabi*-eabi* | \ storm-chaos* | os2-emx* | rtmk-nova*) os=-$maybe_os basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` @@ -255,15 +254,16 @@ case $basic_machine in | arc | arceb \ | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ | avr | avr32 \ + | ba \ | be32 | be64 \ | bfin \ | c4x | c8051 | clipper \ | d10v | d30v | dlx | dsp16xx \ - | epiphany \ - | fido | fr30 | frv \ + | e2k | epiphany \ + | fido | fr30 | frv | ft32 \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ - | i370 | i860 | i960 | ia64 \ + | i370 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ | k1om \ | le32 | le64 \ @@ -301,10 +301,12 @@ case $basic_machine in | open8 | or1k | or1knd | or32 \ | pdp10 | pdp11 | pj | pjl \ | powerpc | powerpc64 | powerpc64le | powerpcle \ + | pru \ | pyramid \ + | riscv32 | riscv64 \ | rl78 | rx \ | score \ - | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ | sh64 | sh64le \ | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ @@ -312,6 +314,8 @@ case $basic_machine in | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ | ubicom32 \ | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ + | visium \ + | wasm32 \ | we32k \ | x86 | xc16x | xstormy16 | xtensa \ | z8k | z80) @@ -326,6 +330,9 @@ case $basic_machine in c6x) basic_machine=tic6x-unknown ;; + leon|leon[3-9]) + basic_machine=sparc-$basic_machine + ;; m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip) basic_machine=$basic_machine-unknown os=-none @@ -371,17 +378,18 @@ case $basic_machine in | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ | avr-* | avr32-* \ + | ba-* \ | be32-* | be64-* \ | bfin-* | bs2000-* \ | c[123]* | c30-* | [cjt]90-* | c4x-* \ | c8051-* | clipper-* | craynv-* | cydra-* \ | d10v-* | d30v-* | dlx-* \ - | elxsi-* \ + | e2k-* | elxsi-* \ | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ | hexagon-* \ - | i*86-* | i860-* | i960-* | ia64-* \ + | i*86-* | i860-* | i960-* | ia16-* | ia64-* \ | ip2k-* | iq2000-* \ | k1om-* \ | le32-* | le64-* \ @@ -422,13 +430,15 @@ case $basic_machine in | orion-* \ | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ + | pru-* \ | pyramid-* \ + | riscv32-* | riscv64-* \ | rl78-* | romp-* | rs6000-* | rx-* \ | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ | sparclite-* \ - | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \ | tahoe-* \ | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ | tile*-* \ @@ -436,6 +446,8 @@ case $basic_machine in | ubicom32-* \ | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ | vax-* \ + | visium-* \ + | wasm32-* \ | we32k-* \ | x86-* | x86_64-* | xc16x-* | xps100-* \ | xstormy16-* | xtensa*-* \ @@ -512,6 +524,9 @@ case $basic_machine in basic_machine=i386-pc os=-aros ;; + asmjs) + basic_machine=asmjs-unknown + ;; aux) basic_machine=m68k-apple os=-aux @@ -632,6 +647,14 @@ case $basic_machine in basic_machine=m68k-bull os=-sysv3 ;; + e500v[12]) + basic_machine=powerpc-unknown + os=$os"spe" + ;; + e500v[12]-*) + basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + os=$os"spe" + ;; ebmon29k) basic_machine=a29k-amd os=-ebmon @@ -773,6 +796,9 @@ case $basic_machine in basic_machine=m68k-isi os=-sysv ;; + leon-*|leon[3-9]-*) + basic_machine=sparc-`echo $basic_machine | sed 's/-.*//'` + ;; m68knommu) basic_machine=m68k-unknown os=-linux @@ -828,6 +854,10 @@ case $basic_machine in basic_machine=powerpc-unknown os=-morphos ;; + moxiebox) + basic_machine=moxie-unknown + os=-moxiebox + ;; msdos) basic_machine=i386-pc os=-msdos @@ -920,6 +950,9 @@ case $basic_machine in nsr-tandem) basic_machine=nsr-tandem ;; + nsx-tandem) + basic_machine=nsx-tandem + ;; op50n-* | op60c-*) basic_machine=hppa1.1-oki os=-proelf @@ -1004,7 +1037,7 @@ case $basic_machine in ppc-* | ppcbe-*) basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` ;; - ppcle | powerpclittle | ppc-le | powerpc-little) + ppcle | powerpclittle) basic_machine=powerpcle-unknown ;; ppcle-* | powerpclittle-*) @@ -1014,7 +1047,7 @@ case $basic_machine in ;; ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` ;; - ppc64le | powerpc64little | ppc64-le | powerpc64-little) + ppc64le | powerpc64little) basic_machine=powerpc64le-unknown ;; ppc64le-* | powerpc64little-*) @@ -1215,6 +1248,9 @@ case $basic_machine in basic_machine=a29k-wrs os=-vxworks ;; + wasm32) + basic_machine=wasm32-unknown + ;; w65*) basic_machine=w65-wdc os=-none @@ -1360,27 +1396,28 @@ case $os in | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ | -sym* | -kopensolaris* | -plan9* \ | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* | -aros* \ + | -aos* | -aros* | -cloudabi* | -sortix* \ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ - | -bitrig* | -openbsd* | -solidbsd* \ + | -bitrig* | -openbsd* | -solidbsd* | -libertybsd* \ | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ - | -chorusos* | -chorusrdb* | -cegcc* \ + | -chorusos* | -chorusrdb* | -cegcc* | -glidix* \ | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ + | -midipix* | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ | -linux-newlib* | -linux-musl* | -linux-uclibc* \ - | -uxpv* | -beos* | -mpeix* | -udk* \ + | -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \ | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ - | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* | -tirtos*) + | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \ + | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox*) # Remember, each alternative MUST END IN *, to match a version number. ;; -qnx*) @@ -1512,6 +1549,8 @@ case $os in ;; -nacl*) ;; + -ios) + ;; -none) ;; *) @@ -1607,6 +1646,9 @@ case $basic_machine in sparc-* | *-sun) os=-sunos4.1.1 ;; + pru-*) + os=-elf + ;; *-be) os=-beos ;; diff --git a/configure b/configure index abe1dc5..10ac675 100755 --- a/configure +++ b/configure @@ -668,6 +668,7 @@ OTHER_LIBTOOL_OPT UNIVERSAL_ARCH_FLAGS CFLAGS_NODIST BASECFLAGS +CFLAGS_ALIASING OPT LLVM_PROF_FOUND target_os @@ -749,9 +750,8 @@ UNIVERSALSDK CONFIG_ARGS SOVERSION VERSION -GENERATED_COMMENT PYTHON_FOR_BUILD -PYTHON_FOR_GEN +PYTHON_FOR_REGEN host_os host_vendor host_cpu @@ -817,6 +817,7 @@ with_suffix enable_shared enable_profiling with_pydebug +with_assertions enable_optimizations with_lto with_hash_algorithm @@ -1476,8 +1477,8 @@ Optional Features: Build (MacOSX|Darwin) framework --enable-shared disable/enable building shared python library --enable-profiling enable C-level code profiling - --enable-optimizations Enable expensive optimizations (PGO, etc). Disabled - by default. + --enable-optimizations Enable expensive, stable optimizations (PGO, etc). + Disabled by default. --enable-loadable-sqlite-extensions support loadable extensions in _sqlite module --enable-ipv6 Enable ipv6 (with ipv4) support @@ -1501,6 +1502,7 @@ Optional Packages: compiler --with-suffix=.exe set executable suffix --with-pydebug build with Py_DEBUG defined + --with-assertions build with C assertions enabled --with-lto Enable Link Time Optimization in PGO builds. Disabled by default. --with-hash-algorithm=[fnv|siphash24] @@ -2698,7 +2700,7 @@ fi -if test -e $srcdir/.git/HEAD +if test -e $srcdir/.git then # Extract the first word of "git", so it can be a program name with args. set dummy git; ac_word=$2 @@ -2868,11 +2870,11 @@ do set dummy $ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_PYTHON_FOR_GEN+:} false; then : +if ${ac_cv_prog_PYTHON_FOR_REGEN+:} false; then : $as_echo_n "(cached) " >&6 else - if test -n "$PYTHON_FOR_GEN"; then - ac_cv_prog_PYTHON_FOR_GEN="$PYTHON_FOR_GEN" # Let the user override the test. + if test -n "$PYTHON_FOR_REGEN"; then + ac_cv_prog_PYTHON_FOR_REGEN="$PYTHON_FOR_REGEN" # Let the user override the test. else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR for as_dir in $PATH @@ -2881,7 +2883,7 @@ do test -z "$as_dir" && as_dir=. for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_PYTHON_FOR_GEN="$ac_prog" + ac_cv_prog_PYTHON_FOR_REGEN="$ac_prog" $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 break 2 fi @@ -2891,25 +2893,20 @@ IFS=$as_save_IFS fi fi -PYTHON_FOR_GEN=$ac_cv_prog_PYTHON_FOR_GEN -if test -n "$PYTHON_FOR_GEN"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON_FOR_GEN" >&5 -$as_echo "$PYTHON_FOR_GEN" >&6; } +PYTHON_FOR_REGEN=$ac_cv_prog_PYTHON_FOR_REGEN +if test -n "$PYTHON_FOR_REGEN"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON_FOR_REGEN" >&5 +$as_echo "$PYTHON_FOR_REGEN" >&6; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } fi - test -n "$PYTHON_FOR_GEN" && break + test -n "$PYTHON_FOR_REGEN" && break done -test -n "$PYTHON_FOR_GEN" || PYTHON_FOR_GEN="not-found" +test -n "$PYTHON_FOR_REGEN" || PYTHON_FOR_REGEN="python3" -if test "$PYTHON_FOR_GEN" = not-found; then - PYTHON_FOR_GEN='@echo "Cannot generate $@, python not found !" && \ - echo "To skip re-generation of $@ run or ." && \ - echo "Otherwise, set python in PATH and run configure or run ." && false &&' -fi if test "$cross_compiling" = yes; then @@ -2930,18 +2927,14 @@ $as_echo_n "checking for python interpreter for cross build... " >&6; } $as_echo "$interp" >&6; } PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$interp fi - # Used to comment out stuff for rebuilding generated files - GENERATED_COMMENT='#' elif test "$cross_compiling" = maybe; then as_fn_error $? "Cross compiling required --host=HOST-TUPLE and --build=ARCH" "$LINENO" 5 else PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E' - GENERATED_COMMENT='' fi - if test "$prefix" != "/"; then prefix=`echo "$prefix" | sed -e 's/\/$//g'` fi @@ -6512,6 +6505,33 @@ $as_echo "no" >&6; } fi +# Check for --with-assertions. Py_DEBUG implies assertions, but also changes +# the ABI. This allows enabling assertions without changing the ABI. +assertions='false' +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-assertions" >&5 +$as_echo_n "checking for --with-assertions... " >&6; } + +# Check whether --with-assertions was given. +if test "${with_assertions+set}" = set; then : + withval=$with_assertions; +if test "$withval" != no +then + assertions='true' +fi +fi + +if test "$assertions" = 'true'; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +elif test "$Py_DEBUG" = 'true'; then + assertions='true' + { $as_echo "$as_me:${as_lineno-$LINENO}: result: implied by --with-pydebug" >&5 +$as_echo "implied by --with-pydebug" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + # Enable optimization flags @@ -6521,7 +6541,7 @@ $as_echo_n "checking for --enable-optimizations... " >&6; } # Check whether --enable-optimizations was given. if test "${enable_optimizations+set}" = set; then : enableval=$enable_optimizations; -if test "$withval" != no +if test "$enableval" != no then Py_OPT='true' { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 @@ -6839,6 +6859,7 @@ esac # tweak OPT based on compiler and platform, only if the user didn't set # it on the command line + if test "${OPT-unset}" = "unset" then case $GCC in @@ -6851,30 +6872,49 @@ then WRAP="-fwrapv" fi - # Clang also needs -fwrapv case $CC in - *clang*) WRAP="-fwrapv" - ;; + *clang*) + cc_is_clang=1 + ;; + *) + if $CC --version 2>&1 | grep -q clang + then + cc_is_clang=1 + else + cc_is_clang= + fi esac + if test -n "${cc_is_clang}" + then + # Clang also needs -fwrapv + WRAP="-fwrapv" + # bpo-30104: disable strict aliasing to compile correctly dtoa.c, + # see Makefile.pre.in for more information + CFLAGS_ALIASING="-fno-strict-aliasing" + fi + case $ac_cv_prog_cc_g in yes) if test "$Py_DEBUG" = 'true' ; then # Optimization messes up debuggers, so turn it off for # debug builds. if "$CC" -v --help 2>/dev/null |grep -- -Og > /dev/null; then - OPT="-g -Og -Wall $STRICT_PROTO" + OPT="-g -Og -Wall" else - OPT="-g -O0 -Wall $STRICT_PROTO" + OPT="-g -O0 -Wall" fi else - OPT="-g $WRAP -O3 -Wall $STRICT_PROTO" + OPT="-g $WRAP -O3 -Wall" fi ;; *) - OPT="-O3 -Wall $STRICT_PROTO" + OPT="-O3 -Wall" ;; esac + + OPT="$OPT $STRICT_PROTO" + case $ac_sys_system in SCO_SV*) OPT="$OPT -m486 -DSCO5" ;; @@ -7401,7 +7441,7 @@ case "$CC" in ;; esac -if test "$Py_DEBUG" = 'true'; then +if test "$assertions" = 'true'; then : else OPT="-DNDEBUG $OPT" @@ -9697,8 +9737,11 @@ esac # check for systems that require aligned memory access { $as_echo "$as_me:${as_lineno-$LINENO}: checking aligned memory access is required" >&5 $as_echo_n "checking aligned memory access is required... " >&6; } -if test "$cross_compiling" = yes; then : - aligned_required=yes +if ${ac_cv_aligned_required+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "$cross_compiling" = yes; then : + ac_cv_aligned_required=yes else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -9715,26 +9758,26 @@ int main() return 1; return 0; } - _ACEOF if ac_fn_c_try_run "$LINENO"; then : - aligned_required=no + ac_cv_aligned_required=no else - aligned_required=yes + ac_cv_aligned_required=yes fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext fi -if test "$aligned_required" = yes ; then +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_aligned_required" >&5 +$as_echo "$ac_cv_aligned_required" >&6; } +if test "$ac_cv_aligned_required" = yes ; then $as_echo "#define HAVE_ALIGNED_REQUIRED 1" >>confdefs.h fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $aligned_required" >&5 -$as_echo "$aligned_required" >&6; } - # str, bytes and memoryview hash algorithm @@ -17928,7 +17971,7 @@ mv config.c Modules if test "$Py_OPT" = 'false' -a "$Py_DEBUG" != 'true'; then echo "" >&6 echo "" >&6 - echo "If you want a release build with all optimizations active (LTO, PGO, etc)," >&6 + echo "If you want a release build with all stable optimizations active (PGO, etc)," >&6 echo "please run ./configure --enable-optimizations" >&6 echo "" >&6 echo "" >&6 diff --git a/configure.ac b/configure.ac index 9eacf52..9620067 100644 --- a/configure.ac +++ b/configure.ac @@ -29,7 +29,7 @@ AC_SUBST(GITVERSION) AC_SUBST(GITTAG) AC_SUBST(GITBRANCH) -if test -e $srcdir/.git/HEAD +if test -e $srcdir/.git then AC_CHECK_PROG(HAS_GIT, git, found, not-found) else @@ -56,13 +56,8 @@ AC_SUBST(host) # pybuilddir.txt will be created by --generate-posix-vars in the Makefile rm -f pybuilddir.txt -AC_CHECK_PROGS(PYTHON_FOR_GEN, python$PACKAGE_VERSION python3 python, not-found) -if test "$PYTHON_FOR_GEN" = not-found; then - PYTHON_FOR_GEN='@echo "Cannot generate $@, python not found !" && \ - echo "To skip re-generation of $@ run or ." && \ - echo "Otherwise, set python in PATH and run configure or run ." && false &&' -fi -AC_SUBST(PYTHON_FOR_GEN) +AC_CHECK_PROGS(PYTHON_FOR_REGEN, python$PACKAGE_VERSION python3 python, python3) +AC_SUBST(PYTHON_FOR_REGEN) if test "$cross_compiling" = yes; then AC_MSG_CHECKING([for python interpreter for cross build]) @@ -80,16 +75,12 @@ if test "$cross_compiling" = yes; then AC_MSG_RESULT($interp) PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$interp fi - # Used to comment out stuff for rebuilding generated files - GENERATED_COMMENT='#' elif test "$cross_compiling" = maybe; then AC_MSG_ERROR([Cross compiling required --host=HOST-TUPLE and --build=ARCH]) else PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E' - GENERATED_COMMENT='' fi AC_SUBST(PYTHON_FOR_BUILD) -AC_SUBST(GENERATED_COMMENT) dnl Ensure that if prefix is specified, it does not end in a slash. If dnl it does, we get path names containing '//' which is both ugly and @@ -1267,14 +1258,35 @@ else AC_MSG_RESULT(no); Py_DEBUG='false' fi], [AC_MSG_RESULT(no)]) +# Check for --with-assertions. Py_DEBUG implies assertions, but also changes +# the ABI. This allows enabling assertions without changing the ABI. +assertions='false' +AC_MSG_CHECKING(for --with-assertions) +AC_ARG_WITH(assertions, + AC_HELP_STRING([--with-assertions], [build with C assertions enabled]), +[ +if test "$withval" != no +then + assertions='true' +fi], +[]) +if test "$assertions" = 'true'; then + AC_MSG_RESULT(yes) +elif test "$Py_DEBUG" = 'true'; then + assertions='true' + AC_MSG_RESULT(implied by --with-pydebug) +else + AC_MSG_RESULT(no) +fi + # Enable optimization flags AC_SUBST(DEF_MAKE_ALL_RULE) AC_SUBST(DEF_MAKE_RULE) Py_OPT='false' AC_MSG_CHECKING(for --enable-optimizations) -AC_ARG_ENABLE(optimizations, AS_HELP_STRING([--enable-optimizations], [Enable expensive optimizations (PGO, etc). Disabled by default.]), +AC_ARG_ENABLE(optimizations, AS_HELP_STRING([--enable-optimizations], [Enable expensive, stable optimizations (PGO, etc). Disabled by default.]), [ -if test "$withval" != no +if test "$enableval" != no then Py_OPT='true' AC_MSG_RESULT(yes); @@ -1447,6 +1459,7 @@ esac # tweak OPT based on compiler and platform, only if the user didn't set # it on the command line AC_SUBST(OPT) +AC_SUBST(CFLAGS_ALIASING) if test "${OPT-unset}" = "unset" then case $GCC in @@ -1459,30 +1472,49 @@ then WRAP="-fwrapv" fi - # Clang also needs -fwrapv case $CC in - *clang*) WRAP="-fwrapv" - ;; + *clang*) + cc_is_clang=1 + ;; + *) + if $CC --version 2>&1 | grep -q clang + then + cc_is_clang=1 + else + cc_is_clang= + fi esac + if test -n "${cc_is_clang}" + then + # Clang also needs -fwrapv + WRAP="-fwrapv" + # bpo-30104: disable strict aliasing to compile correctly dtoa.c, + # see Makefile.pre.in for more information + CFLAGS_ALIASING="-fno-strict-aliasing" + fi + case $ac_cv_prog_cc_g in yes) if test "$Py_DEBUG" = 'true' ; then # Optimization messes up debuggers, so turn it off for # debug builds. if "$CC" -v --help 2>/dev/null |grep -- -Og > /dev/null; then - OPT="-g -Og -Wall $STRICT_PROTO" + OPT="-g -Og -Wall" else - OPT="-g -O0 -Wall $STRICT_PROTO" + OPT="-g -O0 -Wall" fi else - OPT="-g $WRAP -O3 -Wall $STRICT_PROTO" + OPT="-g $WRAP -O3 -Wall" fi ;; *) - OPT="-O3 -Wall $STRICT_PROTO" + OPT="-O3 -Wall" ;; esac + + OPT="$OPT $STRICT_PROTO" + case $ac_sys_system in SCO_SV*) OPT="$OPT -m486 -DSCO5" ;; @@ -1844,7 +1876,7 @@ case "$CC" in ;; esac -if test "$Py_DEBUG" = 'true'; then +if test "$assertions" = 'true'; then : else OPT="-DNDEBUG $OPT" @@ -2696,7 +2728,8 @@ esac # check for systems that require aligned memory access AC_MSG_CHECKING(aligned memory access is required) -AC_TRY_RUN([ +AC_CACHE_VAL(ac_cv_aligned_required, +[AC_RUN_IFELSE([AC_LANG_SOURCE([[ int main() { char s[16]; @@ -2708,18 +2741,16 @@ int main() if (*p1 == *p2) return 1; return 0; -} - ], - [aligned_required=no], - [aligned_required=yes], - [aligned_required=yes]) - -if test "$aligned_required" = yes ; then +}]])], +[ac_cv_aligned_required=no], +[ac_cv_aligned_required=yes], +[ac_cv_aligned_required=yes]) +]) +AC_MSG_RESULT($ac_cv_aligned_required) +if test "$ac_cv_aligned_required" = yes ; then AC_DEFINE([HAVE_ALIGNED_REQUIRED], [1], [Define if aligned memory access is required]) fi -AC_MSG_RESULT($aligned_required) - # str, bytes and memoryview hash algorithm AH_TEMPLATE(Py_HASH_ALGORITHM, @@ -5421,7 +5452,7 @@ mv config.c Modules if test "$Py_OPT" = 'false' -a "$Py_DEBUG" != 'true'; then echo "" >&AS_MESSAGE_FD echo "" >&AS_MESSAGE_FD - echo "If you want a release build with all optimizations active (LTO, PGO, etc)," >&AS_MESSAGE_FD + echo "If you want a release build with all stable optimizations active (PGO, etc)," >&AS_MESSAGE_FD echo "please run ./configure --enable-optimizations" >&AS_MESSAGE_FD echo "" >&AS_MESSAGE_FD echo "" >&AS_MESSAGE_FD diff --git a/setup.py b/setup.py index f04bf22..fe47797 100644 --- a/setup.py +++ b/setup.py @@ -897,8 +897,11 @@ class PyBuildExt(build_ext): blake2_deps.append('hashlib.h') blake2_macros = [] - if not cross_compiling and os.uname().machine == "x86_64": - # Every x86_64 machine has at least SSE2. + if (not cross_compiling and + os.uname().machine == "x86_64" and + sys.maxsize > 2**32): + # Every x86_64 machine has at least SSE2. Check for sys.maxsize + # in case that kernel is 64-bit but userspace is 32-bit. blake2_macros.append(('BLAKE2_USE_SSE', '1')) exts.append( Extension('_blake2', -- 2.7.4