From a473ceb4bf6703a7725a2ae8899b0e3f76f6a73b Mon Sep 17 00:00:00 2001 From: JinWang An Date: Mon, 21 Aug 2023 16:53:44 +0900 Subject: [PATCH] Imported Upstream version 7.45.2 --- AUTHORS | 119 ++ COPYING-LGPL | 502 +++++ COPYING-MIT | 23 + ChangeLog | 1762 +++++++++++++++++ INSTALL.rst | 313 +++ MANIFEST.in | 55 + Makefile | 200 ++ PKG-INFO | 112 ++ README.rst | 190 ++ RELEASE-NOTES.rst | 237 +++ doc/callbacks.rst | 447 +++++ doc/conf.py | 182 ++ doc/curl.rst | 10 + doc/curlmultiobject.rst | 36 + doc/curlobject.rst | 43 + doc/curlshareobject.rst | 12 + doc/docstrings/curl.rst | 9 + doc/docstrings/curl_close.rst | 10 + doc/docstrings/curl_duphandle.rst | 23 + doc/docstrings/curl_errstr.rst | 11 + doc/docstrings/curl_errstr_raw.rst | 12 + doc/docstrings/curl_getinfo.rst | 74 + doc/docstrings/curl_getinfo_raw.rst | 70 + doc/docstrings/curl_pause.rst | 12 + doc/docstrings/curl_perform.rst | 10 + doc/docstrings/curl_perform_rb.rst | 17 + doc/docstrings/curl_perform_rs.rst | 25 + doc/docstrings/curl_reset.rst | 8 + doc/docstrings/curl_set_ca_certs.rst | 5 + doc/docstrings/curl_setopt.rst | 110 + doc/docstrings/curl_setopt_string.rst | 31 + doc/docstrings/curl_unsetopt.rst | 15 + doc/docstrings/multi.rst | 4 + doc/docstrings/multi_add_handle.rst | 12 + doc/docstrings/multi_assign.rst | 7 + doc/docstrings/multi_close.rst | 8 + doc/docstrings/multi_fdset.rst | 26 + doc/docstrings/multi_info_read.rst | 17 + doc/docstrings/multi_perform.rst | 6 + doc/docstrings/multi_remove_handle.rst | 7 + doc/docstrings/multi_select.rst | 24 + doc/docstrings/multi_setopt.rst | 38 + doc/docstrings/multi_socket_action.rst | 17 + doc/docstrings/multi_socket_all.rst | 4 + doc/docstrings/multi_timeout.rst | 6 + doc/docstrings/pycurl_global_cleanup.rst | 7 + doc/docstrings/pycurl_global_init.rst | 10 + doc/docstrings/pycurl_module.rst | 13 + doc/docstrings/pycurl_version_info.rst | 18 + doc/docstrings/share.rst | 5 + doc/docstrings/share_close.rst | 10 + doc/docstrings/share_setopt.rst | 26 + doc/files.rst | 31 + doc/index.rst | 157 ++ doc/install.rst | 1 + doc/internals.rst | 15 + doc/pycurl.rst | 34 + doc/quickstart.rst | 442 +++++ doc/release-notes.rst | 1 + doc/release-process.rst | 33 + doc/static/favicon.ico | Bin 0 -> 318 bytes doc/thread-safety.rst | 34 + doc/troubleshooting.rst | 128 ++ doc/unicode.rst | 277 +++ doc/unimplemented.rst | 65 + examples/basicfirst.py | 29 + examples/file_upload.py | 45 + examples/linksys.py | 568 ++++++ examples/multi-socket_action-select.py | 197 ++ examples/opensocketexception.py | 30 + examples/quickstart/file_upload_buffer.py | 18 + examples/quickstart/file_upload_real.py | 18 + examples/quickstart/file_upload_real_fancy.py | 22 + examples/quickstart/follow_redirect.py | 13 + examples/quickstart/form_post.py | 25 + examples/quickstart/get.py | 24 + examples/quickstart/get_python2.py | 20 + examples/quickstart/get_python2_https.py | 22 + examples/quickstart/get_python3.py | 19 + examples/quickstart/get_python3_https.py | 21 + examples/quickstart/put_buffer.py | 22 + examples/quickstart/put_file.py | 17 + examples/quickstart/response_headers.py | 68 + examples/quickstart/response_info.py | 23 + examples/quickstart/write_file.py | 14 + examples/retriever-multi.py | 123 ++ examples/retriever.py | 103 + examples/sfquery.py | 65 + examples/smtp.py | 46 + examples/ssh_keyfunction.py | 15 + examples/tests/test_build_config.py | 65 + examples/tests/test_gtk.py | 98 + examples/tests/test_xmlrpc.py | 31 + examples/xmlrpc_curl.py | 77 + pycurl.egg-info/PKG-INFO | 112 ++ pycurl.egg-info/SOURCES.txt | 234 +++ pycurl.egg-info/dependency_links.txt | 1 + pycurl.egg-info/top_level.txt | 2 + pytest.ini | 9 + python/curl/__init__.py | 194 ++ requirements-dev.txt | 7 + setup.cfg | 4 + setup.py | 975 +++++++++ src/docstrings.c | 748 +++++++ src/docstrings.h | 39 + src/easy.c | 875 ++++++++ src/easycb.c | 941 +++++++++ src/easyinfo.c | 381 ++++ src/easyopt.c | 1188 +++++++++++ src/easyperform.c | 125 ++ src/module.c | 1569 +++++++++++++++ src/multi.c | 1081 ++++++++++ src/oscompat.c | 57 + src/pycurl.h | 705 +++++++ src/pythoncompat.c | 128 ++ src/share.c | 348 ++++ src/stringcompat.c | 86 + src/threadsupport.c | 365 ++++ src/util.c | 31 + tests/__init__.py | 13 + tests/app.py | 145 ++ tests/appmanager.py | 57 + tests/cadata_test.py | 43 + tests/certinfo_test.py | 94 + tests/certs/ca.crt | 23 + tests/certs/ca.key | 27 + tests/certs/server.crt | 21 + tests/certs/server.key | 27 + tests/close_socket_cb_test.py | 79 + tests/curl_object_test.py | 153 ++ tests/debug_test.py | 66 + tests/default_write_cb_test.py | 85 + tests/duphandle_test.py | 144 ++ tests/error_constants_test.py | 25 + tests/error_test.py | 85 + tests/ext/test-lib.sh | 69 + tests/ext/test-suite.sh | 27 + tests/failonerror_test.py | 89 + tests/fake-curl/curl-config-empty | 15 + .../curl-config-libs-and-static-libs | 17 + tests/fake-curl/curl-config-ssl-feature-only | 18 + tests/fake-curl/curl-config-ssl-in-libs | 17 + .../fake-curl/curl-config-ssl-in-static-libs | 20 + tests/fake-curl/libcurl/Makefile | 28 + tests/fake-curl/libcurl/with_gnutls.c | 29 + tests/fake-curl/libcurl/with_nss.c | 29 + tests/fake-curl/libcurl/with_openssl.c | 29 + tests/fake-curl/libcurl/with_unknown_ssl.c | 29 + tests/fake-curl/libcurl/without_ssl.c | 29 + tests/fixtures/form_submission.txt | 1 + tests/ftp_test.py | 53 + tests/getinfo_test.py | 130 ++ tests/global_init_test.py | 30 + tests/header_cb_test.py | 50 + tests/header_test.py | 55 + tests/high_level_curl_test.py | 35 + tests/info_constants_test.py | 16 + tests/info_test.py | 19 + tests/internals_test.py | 225 +++ tests/matrix.py | 165 ++ .../curl-7.19.0-sslv2-2b0e09b0f98.patch | 40 + ...rl-7.19.0-sslv2-c66b0b32fba-modified.patch | 29 + .../openssl-1.0.1e-fix_pod_syntax-1.patch | 393 ++++ tests/memory_mgmt_test.py | 380 ++++ tests/multi_callback_test.py | 100 + tests/multi_memory_mgmt_test.py | 58 + tests/multi_option_constants_test.py | 89 + tests/multi_socket_select_test.py | 123 ++ tests/multi_socket_test.py | 98 + tests/multi_test.py | 380 ++++ tests/multi_timer_test.py | 91 + tests/open_socket_cb_test.py | 141 ++ tests/option_constants_test.py | 544 +++++ tests/pause_test.py | 97 + tests/perform_test.py | 66 + tests/post_test.py | 200 ++ tests/procmgr.py | 103 + tests/protocol_constants_test.py | 48 + tests/read_cb_test.py | 127 ++ tests/readdata_test.py | 253 +++ tests/relative_url_test.py | 23 + tests/reload_test.py | 19 + tests/reset_test.py | 85 + tests/resolve_test.py | 25 + tests/run-quickstart.sh | 38 + tests/run.sh | 29 + tests/runwsgi.py | 119 ++ tests/seek_cb_constants_test.py | 31 + tests/seek_cb_test.py | 77 + tests/setopt_lifecycle_test.py | 59 + tests/setopt_string_test.py | 31 + tests/setopt_test.py | 113 ++ tests/setopt_unicode_test.py | 38 + tests/setup_test.py | 285 +++ tests/share_test.py | 68 + tests/sockopt_cb_test.py | 85 + tests/ssh_key_cb_test.py | 90 + tests/subclass_test.py | 88 + tests/unset_range_test.py | 52 + tests/user_agent_string_test.py | 28 + tests/util.py | 283 +++ tests/version_comparison_test.py | 15 + tests/version_constants_test.py | 81 + tests/version_test.py | 13 + tests/vsftpd.conf | 13 + tests/weakref_test.py | 23 + tests/write_abort_test.py | 42 + tests/write_cb_bogus_test.py | 48 + tests/write_test.py | 257 +++ tests/write_to_stringio_test.py | 42 + tests/xferinfo_cb_test.py | 75 + winbuild.py | 412 ++++ winbuild/__init__.py | 0 winbuild/builder.py | 146 ++ winbuild/c-ares-vs2015.patch | 13 + winbuild/cares.py | 27 + winbuild/config.py | 155 ++ winbuild/curl.py | 110 + winbuild/iconv.py | 11 + winbuild/idn.py | 10 + winbuild/libcurl-fix-zlib-references.patch | 11 + winbuild/libssh2-vs2015.patch | 10 + winbuild/nghttp_cmake.py | 121 ++ winbuild/nghttp_gmake.py | 22 + winbuild/openssl-fix-crt-1.0.2.patch | 36 + winbuild/openssl-fix-crt-1.1.0.patch | 29 + winbuild/openssl-fix-crt-1.1.1.patch | 29 + winbuild/openssl.py | 76 + winbuild/pycurl.py | 136 ++ winbuild/pythons.py | 20 + winbuild/ssh.py | 40 + winbuild/tools.py | 11 + winbuild/utils.py | 114 ++ winbuild/vcvars-vc14-32.sh | 25 + winbuild/vcvars-vc14-64.sh | 25 + winbuild/zlib.py | 25 + 236 files changed, 27771 insertions(+) create mode 100644 AUTHORS create mode 100644 COPYING-LGPL create mode 100644 COPYING-MIT create mode 100644 ChangeLog create mode 100644 INSTALL.rst create mode 100644 MANIFEST.in create mode 100644 Makefile create mode 100644 PKG-INFO create mode 100644 README.rst create mode 100644 RELEASE-NOTES.rst create mode 100644 doc/callbacks.rst create mode 100644 doc/conf.py create mode 100644 doc/curl.rst create mode 100644 doc/curlmultiobject.rst create mode 100644 doc/curlobject.rst create mode 100644 doc/curlshareobject.rst create mode 100644 doc/docstrings/curl.rst create mode 100644 doc/docstrings/curl_close.rst create mode 100644 doc/docstrings/curl_duphandle.rst create mode 100644 doc/docstrings/curl_errstr.rst create mode 100644 doc/docstrings/curl_errstr_raw.rst create mode 100644 doc/docstrings/curl_getinfo.rst create mode 100644 doc/docstrings/curl_getinfo_raw.rst create mode 100644 doc/docstrings/curl_pause.rst create mode 100644 doc/docstrings/curl_perform.rst create mode 100644 doc/docstrings/curl_perform_rb.rst create mode 100644 doc/docstrings/curl_perform_rs.rst create mode 100644 doc/docstrings/curl_reset.rst create mode 100644 doc/docstrings/curl_set_ca_certs.rst create mode 100644 doc/docstrings/curl_setopt.rst create mode 100644 doc/docstrings/curl_setopt_string.rst create mode 100644 doc/docstrings/curl_unsetopt.rst create mode 100644 doc/docstrings/multi.rst create mode 100644 doc/docstrings/multi_add_handle.rst create mode 100644 doc/docstrings/multi_assign.rst create mode 100644 doc/docstrings/multi_close.rst create mode 100644 doc/docstrings/multi_fdset.rst create mode 100644 doc/docstrings/multi_info_read.rst create mode 100644 doc/docstrings/multi_perform.rst create mode 100644 doc/docstrings/multi_remove_handle.rst create mode 100644 doc/docstrings/multi_select.rst create mode 100644 doc/docstrings/multi_setopt.rst create mode 100644 doc/docstrings/multi_socket_action.rst create mode 100644 doc/docstrings/multi_socket_all.rst create mode 100644 doc/docstrings/multi_timeout.rst create mode 100644 doc/docstrings/pycurl_global_cleanup.rst create mode 100644 doc/docstrings/pycurl_global_init.rst create mode 100644 doc/docstrings/pycurl_module.rst create mode 100644 doc/docstrings/pycurl_version_info.rst create mode 100644 doc/docstrings/share.rst create mode 100644 doc/docstrings/share_close.rst create mode 100644 doc/docstrings/share_setopt.rst create mode 100644 doc/files.rst create mode 100644 doc/index.rst create mode 100644 doc/install.rst create mode 100644 doc/internals.rst create mode 100644 doc/pycurl.rst create mode 100644 doc/quickstart.rst create mode 100644 doc/release-notes.rst create mode 100644 doc/release-process.rst create mode 100644 doc/static/favicon.ico create mode 100644 doc/thread-safety.rst create mode 100644 doc/troubleshooting.rst create mode 100644 doc/unicode.rst create mode 100644 doc/unimplemented.rst create mode 100644 examples/basicfirst.py create mode 100644 examples/file_upload.py create mode 100644 examples/linksys.py create mode 100644 examples/multi-socket_action-select.py create mode 100644 examples/opensocketexception.py create mode 100644 examples/quickstart/file_upload_buffer.py create mode 100644 examples/quickstart/file_upload_real.py create mode 100644 examples/quickstart/file_upload_real_fancy.py create mode 100644 examples/quickstart/follow_redirect.py create mode 100644 examples/quickstart/form_post.py create mode 100644 examples/quickstart/get.py create mode 100644 examples/quickstart/get_python2.py create mode 100644 examples/quickstart/get_python2_https.py create mode 100644 examples/quickstart/get_python3.py create mode 100644 examples/quickstart/get_python3_https.py create mode 100644 examples/quickstart/put_buffer.py create mode 100644 examples/quickstart/put_file.py create mode 100644 examples/quickstart/response_headers.py create mode 100644 examples/quickstart/response_info.py create mode 100644 examples/quickstart/write_file.py create mode 100644 examples/retriever-multi.py create mode 100644 examples/retriever.py create mode 100644 examples/sfquery.py create mode 100644 examples/smtp.py create mode 100644 examples/ssh_keyfunction.py create mode 100644 examples/tests/test_build_config.py create mode 100644 examples/tests/test_gtk.py create mode 100644 examples/tests/test_xmlrpc.py create mode 100644 examples/xmlrpc_curl.py create mode 100644 pycurl.egg-info/PKG-INFO create mode 100644 pycurl.egg-info/SOURCES.txt create mode 100644 pycurl.egg-info/dependency_links.txt create mode 100644 pycurl.egg-info/top_level.txt create mode 100644 pytest.ini create mode 100644 python/curl/__init__.py create mode 100644 requirements-dev.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 src/docstrings.c create mode 100644 src/docstrings.h create mode 100644 src/easy.c create mode 100644 src/easycb.c create mode 100644 src/easyinfo.c create mode 100644 src/easyopt.c create mode 100644 src/easyperform.c create mode 100644 src/module.c create mode 100644 src/multi.c create mode 100644 src/oscompat.c create mode 100644 src/pycurl.h create mode 100644 src/pythoncompat.c create mode 100644 src/share.c create mode 100644 src/stringcompat.c create mode 100644 src/threadsupport.c create mode 100644 src/util.c create mode 100644 tests/__init__.py create mode 100644 tests/app.py create mode 100644 tests/appmanager.py create mode 100644 tests/cadata_test.py create mode 100644 tests/certinfo_test.py create mode 100644 tests/certs/ca.crt create mode 100644 tests/certs/ca.key create mode 100644 tests/certs/server.crt create mode 100644 tests/certs/server.key create mode 100644 tests/close_socket_cb_test.py create mode 100644 tests/curl_object_test.py create mode 100644 tests/debug_test.py create mode 100644 tests/default_write_cb_test.py create mode 100644 tests/duphandle_test.py create mode 100644 tests/error_constants_test.py create mode 100644 tests/error_test.py create mode 100644 tests/ext/test-lib.sh create mode 100755 tests/ext/test-suite.sh create mode 100644 tests/failonerror_test.py create mode 100755 tests/fake-curl/curl-config-empty create mode 100755 tests/fake-curl/curl-config-libs-and-static-libs create mode 100755 tests/fake-curl/curl-config-ssl-feature-only create mode 100755 tests/fake-curl/curl-config-ssl-in-libs create mode 100755 tests/fake-curl/curl-config-ssl-in-static-libs create mode 100644 tests/fake-curl/libcurl/Makefile create mode 100644 tests/fake-curl/libcurl/with_gnutls.c create mode 100644 tests/fake-curl/libcurl/with_nss.c create mode 100644 tests/fake-curl/libcurl/with_openssl.c create mode 100644 tests/fake-curl/libcurl/with_unknown_ssl.c create mode 100644 tests/fake-curl/libcurl/without_ssl.c create mode 100644 tests/fixtures/form_submission.txt create mode 100644 tests/ftp_test.py create mode 100644 tests/getinfo_test.py create mode 100644 tests/global_init_test.py create mode 100644 tests/header_cb_test.py create mode 100644 tests/header_test.py create mode 100644 tests/high_level_curl_test.py create mode 100644 tests/info_constants_test.py create mode 100644 tests/info_test.py create mode 100644 tests/internals_test.py create mode 100644 tests/matrix.py create mode 100644 tests/matrix/curl-7.19.0-sslv2-2b0e09b0f98.patch create mode 100644 tests/matrix/curl-7.19.0-sslv2-c66b0b32fba-modified.patch create mode 100644 tests/matrix/openssl-1.0.1e-fix_pod_syntax-1.patch create mode 100644 tests/memory_mgmt_test.py create mode 100644 tests/multi_callback_test.py create mode 100644 tests/multi_memory_mgmt_test.py create mode 100644 tests/multi_option_constants_test.py create mode 100644 tests/multi_socket_select_test.py create mode 100644 tests/multi_socket_test.py create mode 100644 tests/multi_test.py create mode 100644 tests/multi_timer_test.py create mode 100644 tests/open_socket_cb_test.py create mode 100644 tests/option_constants_test.py create mode 100644 tests/pause_test.py create mode 100644 tests/perform_test.py create mode 100644 tests/post_test.py create mode 100644 tests/procmgr.py create mode 100644 tests/protocol_constants_test.py create mode 100644 tests/read_cb_test.py create mode 100644 tests/readdata_test.py create mode 100644 tests/relative_url_test.py create mode 100644 tests/reload_test.py create mode 100644 tests/reset_test.py create mode 100644 tests/resolve_test.py create mode 100755 tests/run-quickstart.sh create mode 100755 tests/run.sh create mode 100644 tests/runwsgi.py create mode 100644 tests/seek_cb_constants_test.py create mode 100644 tests/seek_cb_test.py create mode 100644 tests/setopt_lifecycle_test.py create mode 100644 tests/setopt_string_test.py create mode 100644 tests/setopt_test.py create mode 100644 tests/setopt_unicode_test.py create mode 100644 tests/setup_test.py create mode 100644 tests/share_test.py create mode 100644 tests/sockopt_cb_test.py create mode 100644 tests/ssh_key_cb_test.py create mode 100644 tests/subclass_test.py create mode 100644 tests/unset_range_test.py create mode 100644 tests/user_agent_string_test.py create mode 100644 tests/util.py create mode 100644 tests/version_comparison_test.py create mode 100644 tests/version_constants_test.py create mode 100644 tests/version_test.py create mode 100644 tests/vsftpd.conf create mode 100644 tests/weakref_test.py create mode 100644 tests/write_abort_test.py create mode 100644 tests/write_cb_bogus_test.py create mode 100644 tests/write_test.py create mode 100644 tests/write_to_stringio_test.py create mode 100644 tests/xferinfo_cb_test.py create mode 100644 winbuild.py create mode 100644 winbuild/__init__.py create mode 100644 winbuild/builder.py create mode 100644 winbuild/c-ares-vs2015.patch create mode 100644 winbuild/cares.py create mode 100644 winbuild/config.py create mode 100644 winbuild/curl.py create mode 100644 winbuild/iconv.py create mode 100644 winbuild/idn.py create mode 100644 winbuild/libcurl-fix-zlib-references.patch create mode 100644 winbuild/libssh2-vs2015.patch create mode 100644 winbuild/nghttp_cmake.py create mode 100644 winbuild/nghttp_gmake.py create mode 100644 winbuild/openssl-fix-crt-1.0.2.patch create mode 100644 winbuild/openssl-fix-crt-1.1.0.patch create mode 100644 winbuild/openssl-fix-crt-1.1.1.patch create mode 100644 winbuild/openssl.py create mode 100644 winbuild/pycurl.py create mode 100644 winbuild/pythons.py create mode 100644 winbuild/ssh.py create mode 100644 winbuild/tools.py create mode 100644 winbuild/utils.py create mode 100644 winbuild/vcvars-vc14-32.sh create mode 100644 winbuild/vcvars-vc14-64.sh create mode 100644 winbuild/zlib.py diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..10893e1 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,119 @@ +Copyright (C) 2001-2008 by Kjetil Jacobsen +Copyright (C) 2001-2008 by Markus F.X.J. Oberhumer +Copyright (C) 2013-2022 by Oleg Pudeyev + +Please see README, COPYING-LGPL and COPYING-MIT for license information. + +The following individuals contributed code to PycURL: + +Aaron Hill +Adam Guthrie +Adam Jacob Muller +Akiomi Kamakura +Alexandre Pion +Amir Rossert +Amit Mongia +Andjelko Horvat +Arshad Khan +Artur Sobierak +Ashley Whetter +Barry Warsaw +Bastian Kleineidam +Benjamin Peterson +Bill Collins +Bo Anderson +Casey Miller +Chih-Hsuan Yen +Christian Clauss +Christopher Warner +Clint Clayton +Conrad Steenberg +Daniel Pena Arteaga +Daniel Stenberg +decitre +Dima Tisnek +Dmitriy Taychenachev +Dmitry Ketov +Domenico Andreoli +Dominique +Eneas U de Queiroz +Eric S. Raymond +Felix Yan +Francisco Alves +Gabi Davar +Gisle Vanem +Gregory Petukhov +Hasan +Hugo +Iain R. Learmonth +ideal +Jakob Truelsen +Jakub Wilk +James Deucker +Jan Kryl +Jayne +James Deucker +JiCiT +Jim Patterson +Josef Schlehofer +Jozef Melicher +K.S.Sreeram +Kamil Dudka +Kevin Ko +Kevin Schlosser +Khavish Anshudass Bhundoo +Kian-Meng Ang +kxrd +Lipin Dmitriy +Léo El Amri +Marc Labranche +Marcelo Jorge Vieira +Marien Zwart +Mark Eichin +Markus +Martin Muenstermann +Matt King +Michael Coughlin +Michael Treanor <26148512+skeptycal at users.noreply.github.com> +Michał Górny +Nelson Chen +Nick Pilon +Nicolas Pauss +Oren +Orion Poplawski +Oskari Saarenmaa +Paul Pacheco +Pierre Grimaud +René Dudfield +resokou +Roland Sommer +Romuald Brunet +Romulo A. Ceccon +Russell McConnachie +Russell McConnachie +Samuel Dion-Girardeau +Samuel Henrique +Scott Talbert +Simon Legner +Srinivas +Steve Kowalik +Subin +Tal Einat +Thomas Hunger +Tino Lange +toddrme2178 +Tom Pierce +Victor Lascurain +Vincent Philippon +Vitaly Murashev +Vitezslav Cizek +vmurashev +Wei C +Whitney Sorenson +Wim Lewis +Yiteng Zhang +Yuhui H +Yuri Ushakov +Yves Bastide +Zdenek Pavlas +ziggy diff --git a/COPYING-LGPL b/COPYING-LGPL new file mode 100644 index 0000000..4362b49 --- /dev/null +++ b/COPYING-LGPL @@ -0,0 +1,502 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/COPYING-MIT b/COPYING-MIT new file mode 100644 index 0000000..f796099 --- /dev/null +++ b/COPYING-MIT @@ -0,0 +1,23 @@ +COPYRIGHT AND PERMISSION NOTICE + +Copyright (C) 2001-2008 by Kjetil Jacobsen +Copyright (C) 2001-2008 by Markus F.X.J. Oberhumer +Copyright (C) 2013-2022 by Oleg Pudeyev + +All rights reserved. + +Permission to use, copy, modify, and distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright +notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall not +be used in advertising or otherwise to promote the sale, use or other dealings +in this Software without prior written authorization of the copyright holder. diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..f8d9bbe --- /dev/null +++ b/ChangeLog @@ -0,0 +1,1762 @@ +Version 7.45.2 [requires libcurl-7.19.0 or better] - 2022-12-16 +--------------------------------------------------------------- + + * Python 3.9 compatibility for Py_TRASHCAN_SAFE_BEGIN + (patch by Scott Talbert). + * Add support for CURL_HTTP_VERSION_3 (patch by Scott Talbert). + * Add CURLOPT_TLS13_CIPHERS and CURLOPT_PROXY_TLS13_CIPHERS options + (patch by Scott Talbert). + * Added HTTP09_ALLOWED option (patch by Scott Talbert). + * Removed use of distutils (patch by Scott Talbert). + + +Version 7.45.1 [requires libcurl-7.19.0 or better] - 2022-03-13 +--------------------------------------------------------------- + + * Fixed build against libcurl < 7.64.1 (patch by Scott Talbert). + + +Version 7.45.0 [requires libcurl-7.64.1 or better] - 2022-03-09 +--------------------------------------------------------------- + + * Add CURLOPT_MAXLIFETIME_CONN (patch by fsbs). + + * Easy handle duplication support (patch by fsbs). + + * Support for unsetting a number of multi options (patch by fsbs). + + * pycurl classes can now be subclassed (patch by fsbs). + + * Multi callbacks' thread state management fixed (patch by fsbs). + + * Add CURL_LOCK_DATA_PSL (patch by fsbs). + + * Add support for SecureTransport SSL backend (MacOS) + (patch by Scott Talbert). + + +Version 7.44.1 [requires libcurl-7.19.0 or better] - 2021-08-15 +--------------------------------------------------------------- + + * Fixed Python thread initialization causing hangs on operations + (patch by Scott Talbert). + + +Version 7.44.0 [requires libcurl-7.19.0 or better] - 2021-08-08 +--------------------------------------------------------------- + + * getinfo(CURLINFO_FTP_ENTRY_PATH) now handles NULL return from + libcurl, returning None in this case. + + * Python 3.9 is now officially supported (patch by Bill Collins). + + * Added CURLOPT_DOH_URL (patch by resokou). + + * Best effort Python 2 support has been reinstated. + + * Added missing fields to curl_version_info struct (patch by Hasan). + + * Added CURLINFO_CONDITION_UNMET (patch by Dima Tisnek). + + * Exposed MAX_CONCURRENT_STREAMS in CurlMulti (patch by Alexandre Pion). + + * Compilation fixed against Python 3.10 alpha (patch by Kamil Dudka). + + +Version 7.43.0.6 [requires libcurl-7.19.0 or better] - 2020-09-02 +----------------------------------------------------------------- + + * Fixed offset parameter usage in seek callback (patch by Scott Talbert). + + * Added support for libcurl SSL backend detection via + `curl-config --ssl-backends` (patch by Scott Talbert). + + * Added support for libcurl MultiSSL (patch by Bo Anderson). + + * Added ability to unset CURLOPT_PROXY. + + * Added support for CURLOPT_UPLOAD_BUFFERSIZE (patch by Artur Sobierak). + + * Added support for CURLOPT_MAXAGE_CONN (patch by Artur Sobierak). + + * Added support for sharing connection cache in libcurl (patch by + Artur Sobierak). + + * Added support for CURLOPT_HAPROXYPROTOCOL (patch by + Russell McConnachie). + + * CC and CFLAGS environment variables are now respected when building + (patch by Michał Górny). + + * Fixed OpenSSL detection on CentOS 7 and 8 (patch by Nicolas Pauss). + + * surrogateescape error handler is used in multi_info_read to handle + invalid UTF-8. + + +Version 7.43.0.5 [requires libcurl-7.19.0 or better] - 2020-01-29 +----------------------------------------------------------------- + + * Fixed build with recent Pythons on RHEL/CentOS. + + +Version 7.43.0.4 [requires libcurl-7.19.0 or better] - 2020-01-15 +----------------------------------------------------------------- + + * Minimum supported Python 3 version is now 3.5. + + * Python 2 is no longer officially supported. + + * Improved thread safety of multi code. + + * Added Python 3.8 support (patch by Michael Treanor). + + * Fixed link order when linking statically against OpenSSL (patch by + Ashley Whetter). + + * Fixed Darwin detection. + + * Added support for wolfSSL (patch by Eneas U de Queiroz). + + * Added PROXY_SSL_VERIFYHOST (patch by Amir Rossert). + + +Version 7.43.0.3 [requires libcurl-7.19.0 or better] - 2019-06-17 +----------------------------------------------------------------- + + * Fixed use with libcurl 7.65+ when FTP support is disabled. + + * Added support for mbedTLS (patch by Josef Schlehofer). + + * Fixed string processing on Python 3 (patch by Dmitriy Taychenachev). + + * Added CURLOPT_TCP_FASTOPEN and CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE + (patch by Khavish Anshudass Bhundoo). + + * Repaired inability to install PycURL when libcurl is using an SSL + backend other than the ones PycURL explicitly recognizes and + handles (OpenSSL, LibreSSL, BoringSSL, GnuTLS, NSS). + The requirement for setup.py to detect an SSL backend if libcurl + is configured to use SSL, added in 7.43.0.2, has been changed + to a warning to allow this. + + +Version 7.43.0.2 [requires libcurl-7.19.0 or better] - 2018-06-02 +----------------------------------------------------------------- + + * Official Windows builds now include HTTP 2 support via + libnghttp2 and international domain name support via WINIDN. + + * Added perform_rb and perform_rs methods to Curl objects to + return response body as byte string and string, respectively. + + * Added OPT_COOKIELIST constant for consistency with other + option constants. + + * PycURL is now able to report errors triggered by libcurl + via CURLOPT_FAILONERROR mechanism when the error messages are + not decodable in Python's default encoding (GitHub issue #259). + + * Added getinfo_raw method to Curl objects to return byte strings + as is from libcurl without attempting to decode them + (GitHub issue #493). + + * When adding a Curl easy object to CurlMulti via add_handle, + the easy objects now have their reference counts increased so that + the application is no longer required to keep references to them + to keep them from being garbage collected (GitHub issue #171). + + * PycURL easy, multi and share objects can now be weak referenced. + + * Python 3.2 and 3.3 support officially dropped as those versions + are end of lifed. + + * set_ca_certs now accepts byte strings as it should have been + all along. + + * PycURL now skips automatic SSL backend detection if curl-config + indicates that libcurl is not built with SSL support, and will warn + if an SSL backend is explicitly specified in this case. + + * PycURL now requires that SSL backend is determined by setup.py + to provide earlier failure compared to the existing warning + during compilation and failing during module import on mismatched + SSL backends. + + * Use OpenSSL 1.1 and 1.0 specific APIs for controlling thread locks + depending on OpenSSL version (patch by Vitaly Murashev). + + * Fixed a crash when closesocket callback failed (patch by + Gisle Vanem and toddrme2178). + + * Added CURLOPT_PROXY_SSLCERT, CURLOPT_PROXY_SSLCERTTYPE, + CURLOPT_PROXY_SSLKEY, CURLOPT_PROXY_SSLKEYTYPE, + CURLOPT_PROXY_SSL_VERIFYPEER (libcurl 7.52.0+, + patch by Casey Miller). + + * Added CURLOPT_PRE_PROXY (libcurl 7.52.0+, patch by ziggy). + + * Support for Python 2.6 officially dropped. + + * Added SOCKET_BAD constant and it is now recognized as a valid + return value from OPENSOCKET callback. + + * BoringSSL is now recognized as equivalent to OpenSSL backend + (patch by Gisle Vanem). + + +Version 7.43.0.1 [requires libcurl-7.19.0 or better] - 2017-12-07 +----------------------------------------------------------------- + + * WRITEHEADER/WRITEFUNCTION and WRITEDATA/WRITEFUNCTION can now + be set on the same handle. The last call will take precedence over + previous calls. Previously some combinations were not allowed. + + * Fixed a crash when using WRITEDATA with a file-like object followed + by WRITEDATA with a real file object (patch by Léo El Amri). + + * Fixed a theoretical memory leak in module initialization (patch by + ideal). + + * Added support for CURL_SSLVERSION_MAX_* constants (libcurl 7.52.0+, + patch by Jozef Melicher). + + * Added support for CURLSSH_AUTH_AGENT (libcurl 7.28.0+, + patch by kxrd). + + * Added support for CURLOPT_CONNECT_TO (patch by Iain R. Learmonth). + + * Added support for CURLINFO_HTTP_VERSION (patch by Iain R. Learmonth). + + * Fixed build against OpenSSL l.1 on Windows. + + * Added set_ca_certs method to the Easy object to set CA certificates + from a string (OpenSSL only, patch by Lipin Dmitriy). + + * Python 3.6 is now officially supported (patch by Samuel + Dion-Girardeau). + + * Added support for CURLOPT_PROXY_CAPATH (libcurl 7.52.0+, + patch by Jan Kryl). + + * C-Ares updated to 1.12.0 in Windows builds, fixing DNS resolution + issues on Windows (patch by Wei C). + + * Added --openssl-lib-name="" option to support building against + OpenSSL 1.1.0 on Windows. + + * Fixed a possible double free situation in all Curl objects + due to a misuse of the trashcan API (patch by Benjamin Peterson). + + * High level Curl objects can now be reused. + + * LARGE options fixed under Windows and Python 3 (INFILESIZE, + MAX_RECV_SPEED_LARGE, MAX_SEND_SPEED_LARGE, MAXFILESIZE, + POSTFILESIZE, RESUME_FROM). + + * Fixed compilation on Solaris (patch by Yiteng Zhang). + + * ENCODING option can now be unset (patch by Yves Bastide). + + +Version 7.43.0 [requires libcurl-7.19.0 or better] - 2016-02-02 +--------------------------------------------------------------- + + * Added CURLINFO_RTSP_* constants (libcurl 7.20.0+). + + * Added CURLOPT_XOAUTH2_BEARER (libcurl 7.33.0+). + + * Added CURLOPT_SASL_IR (libcurl 7.31.0+). + + * Added CURLOPT_LOGIN_OPTIONS (libcurl 7.34.0+). + + * Added CURLOPT_FTP_USE_PRET (libcurl 7.20.0+). + + * Added setopt_string method to Curl objects to set arbitrary + string options. + + * Switched to Bintray for hosting release distributions. + + * Added CURLOPT_DEFAULT_PROTOCOL (libcurl 7.45.0+). + + * Added CURLOPT_TLSAUTH_* options (libcurl 7.21.4+). + + * Added CURLPROTO_SMB and CURLPROTO_SMBS constants (libcurl 7.40.0+). + + * Added CURL_SOCKOPT_* constants (libcurl 7.21.5+). + + * Added CURL_HTTP_VERSION_2_0, CURL_HTTP_VERSION_2 and + CURL_HTTP_VERSION_2TLS constants for CURLOPT_HTTP_VERSION + (various libcurl versions required for these). + + * winbuild.py can now build binary wheels on Windows. + + * Added failed memory allocation handling during SSL lock initialization. + + * CURLOPT_IOCTLDATA option support has been removed. + This option is used internally by PycURL and is not settable by + applications. + + * HTTPHEADER and PROXYHEADER options can now be unset. + + * Added CURLPIPE_* constants (libcurl 7.43.0+). + + * Added CURLOPT_PIPEWAIT (libcurl 7.43.0+). + + * Added CURLOPT_PATH_AS_IS (libcurl 7.42.0+). + + * Added CURLOPT_PROXYHEADER and CURLOPT_HEADEROPT as well as + CURLHEADER_UNIFIED and CURLHEADER_SEPARATE (libcurl 7.37.0+). + + * Added CURLOPT_EXPECT_100_TIMEOUT_MS (libcurl 7.36.0+). + + * Added CURLOPT_XFERINFOFUNCTION (libcurl 7.32.0+). + + * Added CURLM_ADDED_ALREADY error constant (libcurl 7.32.1+). + + * Added remaining CURLE_* constants through libcurl 7.46.0. + + * Unbroken `curl' module import on Windows - apparently Windows now + has a `signal' Python module but no `SIGPIPE' (patch by Gabi Davar). + + * Added CURLMOPT_PIPELINING_SITE_BL and CURLMOPT_PIPELINING_SERVER_BL + options (libcurl 7.30.0+). + + * Added CURLOPT_TCP_KEEPALIVE, CURLOPT_TCP_KEEPIDLE and + CURLOPT_TCP_KEEPINTVL options (libcurl 7.25.0+). + + * Added CURLOPT_ACCEPTTIMEOUT_MS (libcurl 7.24.0+). + + * Added CURLOPT_ACCEPT_ENCODING and CURLOPT_TRANSFER_ENCODING + options (libcurl 7.21.6+). + + * OPENSOCKETFUNCTION callback for AF_UNIX sockets was mistakenly + invoked with the address as a `string' rather than `bytes' on + Python 3. The callback now receives a `bytes' instance as was + documented. + + +Version 7.21.5 [requires libcurl-7.19.0 or better] - 2016-01-05 +--------------------------------------------------------------- + + * --with-openssl and its --win-ssl alias setup.py options are now + accepted under Windows in order to use OpenSSL's crypto locks + when building against OpenSSL. + + * --with-openssl added as an alias for --with-ssl option to setup.py. + + * Official Windows builds are now linked against C-Ares and libssh2. + + * Official Windows builds are now linked against OpenSSL instead of + WinSSL. + + * Official Windows builds are now statically linked against + their dependencies (libcurl and zlib). + + * Added CURLOPT_USE_SSL and CURLUSESSL_* constants. + + * Added CURLOPT_APPEND, CURLOPT_COOKIESESSION, CURLOPT_DIRLISTONLY, + CURLOPT_KEYPASSWD, CURLOPT_TELNETOPTIONS. + + * Several CURLE_* and CURLM_* constants added. + + * Add VERSION_* constants, corresponding to CURL_VERSION_*. + + * Breaking change: OPENSOCKETFUNCTION callback API now mirrors that + of libcurl: + 1. The callback now takes two arguments, `purpose' and `address`. + Previously the callback took `family', `socktype', `protocol` + and `addr' arguments. + 2. The second argument to the callback, `address', is a + `namedtuple' with `family', `socktype', `protocol' and + `addr' fields. + 3. `addr' field on `address' for AF_INET6 family addresses is a + 4-tuple of (address, port, flow info, scope id) which matches + Python's `socket.getaddrinfo' API. + + It seems that libcurl may mishandle error return from an + opensocket callback, as would happen when code written for + pre-PycURL 7.21.5 API is run with PycURL 7.21.5 or newer, + resulting in the application hanging. + + * OPENSOCKETFUNCTION callback can now be unset. + + * Added CURLOPT_CLOSESOCKETFUNCTION (libcurl 7.21.7+). + CURLOPT_CLOSESOCKETDATA is used internally by PycURL. + + * Added CURLOPT_SOCKOPTFUNCTION. CURLOPT_SOCKOPTDATA is used + internally by PycURL. + + * Added CURLOPT_SSH_KEYFUNCTION (libcurl 7.19.6+). + CURLOPT_SSH_KEYDATA is used internally by PycURL. + + * Added CURLOPT_SSL_OPTIONS (libcurl 7.25.0+). + + * Added CURLOPT_KRBLEVEL. + + * Added CURLOPT_SSL_FALSESTART (libcurl 7.42.0+). + + * Added CURLOPT_SSL_ENABLE_NPN (libcurl 7.36.0+). + + * Added CURLOPT_SSL_ENABLE_ALPN (libcurl 7.36.0+). + + * Added CURLOPT_UNIX_SOCKET_PATH (libcurl 7.40.0+). + + * Added CURLOPT_WILDCARDMATCH (libcurl 7.21.0+). + + * C module initialization changed to raise exceptions on failure + rather than trigger a fatal error and abort the Python interpreter. + + * Added CURLOPT_PINNEDPUBLICKEY (libcurl 7.39.0-7.44.0+ + depending on SSL backend and encoding algorithm). + + * Fixed incorrect detection of libcurl 7.19.5 and 7.19.6 + (thanks to bataniya). + + +Version 7.19.5.3 [requires libcurl-7.19.0 or better] - 2015-11-03 +----------------------------------------------------------------- + + * python and nosetests binaries can now be overridden when running + the test suite (patch by Kamil Dudka). + + * Files needed to run the test suite are distributed in sdist + (patch by Kamil Dudka). + + +Version 7.19.5.2 [requires libcurl-7.19.0 or better] - 2015-11-02 +----------------------------------------------------------------- + + * C sources made 64-bit clean on Windows. + + * Support for building against Python 3.5 added to winbuild.py. + + * Fixed build on Windows when using MS SDK 8.1+ or MSVC 14/2015 + (patch by Gisle Vanem). + + * Added automatic SSL library detection on CentOS 6 by loading + libcurl shared library in setup.py. This automatic detection is + meant to permit installing pycurl seamlessly via `pip install pycurl` + on CentOS; as such, it is only employed when no other configuration + options or configuration environment variables are given to setup.py + (original patch by Francisco Alves). + + * Added --libcurl-dll option to setup.py to take SSL library + information out of libcurl shared library (original patch by + Francisco Alves). This option is only usable + with Python 2.5 or higher. + + * --with-ssl, --with-gnutls and --with-nss options to setup.py now + result in PycURL explicitly linking against the respective SSL + library. Previously setup.py relied on curl-config to supply the + needed libraries in this case. + + * List and tuples are now accepted in all positions of HTTPPOST + option values. + + * Tuples are now accepted for options taking list values (e.g. + HTTPHEADER). + + * Fixed a use after free in HTTPPOST when using FORM_BUFFERPTR with + a Unicode string (patch by Clint Clayton). + + * Fixed a memory leak in HTTPPOST for multiple FORM_BUFFERPTR + (patch by Clint Clayton). + + * CURLMOPT_* option constants were mistakenly defined on Curl + instances but not on CurlMulti instances. These option constants + are now defined on CurlMulti instances and on pycurl module, + but not on Curl instances. + + * Fixed several memory leaks when setting string options to + Unicode values failed. + + * Fixed a memory leak when using POSTFIELDS with unicode objects + on Python 2 (patch by Clint Clayton). + + * Official support for Python 2.4 and 2.5 dropped. PycURL is no + longer tested against these Python versions on Travis. + + * Added CURLAUTH_NEGOTIATE (libcurl 7.38.0+), CURLAUTH_NTLM_WB + (libcurl 7.22.0+), CURLAUTH_ONLY (libcurl 7.21.3+), + + * Added CURLOPT_SERVICE_NAME (libcurl 7.43.0+). + + * Added CURLOPT_PROXY_SERVICE_NAME (libcurl 7.43.0+). + + * Added CURLE_SSL_CRL_BADFILE, CURLE_SSL_INVALIDCERTSTATUS + (libcurl 7.41.0+), CURLE_SSL_ISSUER_ERROR and + CURLE_SSL_PINNEDPUBKEYNOTMATCH (libcurl 7.39.0+). + + * Added CURLOPT_SSL_VERIFYSTATUS (libcurl 7.41.0+). + + * Added CURL_SSLVERSION_TLSv1_0, CURL_SSLVERSION_TLSv1_1 + and CURL_SSLVERSION_TLSv1_2 (libcurl 7.34.0+). + + * The second argument of DEBUGFUNCTION callback is now of type bytes on + Python 3. When response body contains non-ASCII data and + DEBUGFUNCTION is enabled, this argument would receive non-ASCII data. + Which encoding this data is in is unknown by PycURL, and e.g. in + the case of HTTP requires parsing response headers. GitHub issue + #210, patch by Barry Warsaw with help from Gregory Petukhov. + + * Fixed build on GCC 4.4.5 (patch by Travis Jensen). + + * Added CURLOPT_GSSAPI_DELEGATION, CURLGSSAPI_DELEGATION_FLAG, + CURLGSSAPI_DELEGATION_NONE and CURLGSSAPI_DELEGATION_POLICY_FLAG + (libcurl 7.22.0+, patch by Dmitry Ketov). + + +Version 7.19.5.1 [requires libcurl-7.19.0 or better] - 2015-01-06 +----------------------------------------------------------------- + + * Added CURLPROXY_SOCKS4A and CURLPROXY_SOCKS5_HOSTNAME. + + * setup.py now prints PycURL-specific option help when -h is used. + + * LibreSSL is now supported (patch by JiCiT). + + * Fixed an oversight that broke PycURL building against libcurl 7.19.4 + through 7.21.1. The bug was introduced in PycURL 7.19.5. + + * Tests are now included in source distributions again, thanks to + Kamil Dudka and Johan Bergstroem. + + * Added CURLOPT_MAIL_FROM and CURLOPT_MAIL_RCPT (libcurl 7.20.0+) + and CURLOPT_MAIL_AUTH (libcurl 7.25.0+). + + +Version 7.19.5 [requires libcurl-7.21.2 or better] - 2014-07-12 +--------------------------------------------------------------- + + * Tests removed from source and binary distributions. + + * Documentation greatly improved. Quickstart guide added. + + * pycurl.Curl, pycurl.CurlMulti and pycurl.CurlShare are now classes + rather than factory functions. Previously, the classes were "hidden" + (they were accessible as e.g. type(pycurl.Curl()), but could not be + instantiated, nor could class methods be obtained from the classes. + Please see this mailing list post for further information: + https://curl.haxx.se/mail/curlpython-2014-06/0004.html + + * When passing a file-like object to READDATA option, PycURL was + mistakenly looking for write method on this object. Now read method + is looked up, as would be expected. + + * Python 3.4 is now officially supported. + + * Windows packages now build libcurl against zlib. + + * CherryPy is no longer required for the test suite, ssl module from + the Python standard library is used instead. + + * Fixed a reference leak of SOCKET and TIMER callbacks on + CurlMulti instances, thanks to Ben Darnell. + + * Fixed build against openssl on cygwin, where pycurl needs to link + against libcrypto rather than libssl. + + * Added CURLOPT_SSH_KNOWNHOSTS (libcurl 7.19.6+). + + * Added CURLE_FTP_ACCEPT_FAILED (libcurl 7.24.0+). + + * Added CURLE_NOT_BUILT_IN and CURLE_UNKNOWN_OPTION (libcurl 7.21.5+). + + * Added CURL_SEEKFUNC_OK, CURL_SEEKFUNC_FAIL and + CURL_SEEKFUNC_CANTSEEK. All constants require libcurl 7.19.5+; + numeric values of CURL_SEEKFUNC_OK and CURL_SEEKFUNC_FAIL were + understood earlier but constants only exist as of libcurl 7.19.5. + + * Added CURLINFO_CONDITION_UNMET (libcurl 7.19.4+). + + * Added CURLPROXY_HTTP_1_0 (libcurl 7.19.4+). + + * Added CURLOPT_SOCKS5_GSSAPI_SERVICE and + CURLOPT_SOCKS5_GSSAPI_NEC (libcurl 7.19.4+). + + * Added CURLOPT_TFTP_BLKSIZE (libcurl 7.19.4+). + + * Added CURLOPT_PROTOCOLS, CURLOPT_REDIR_PROTOCOLS and associated + CURLPROTO_* constants, which require libcurl 7.19.4+. + + * Fixed a reference leak of OPENSOCKET and SEEK callbacks, thanks to + Ben Darnell. + + * C source is now split into several files. + + * Documentation is now processed by sphinx. + + +Version 7.19.3.1 [requires libcurl-7.19.0 or better] - 2014-02-05 +----------------------------------------------------------------- + + * Added --avoid-stdio setup.py option to avoid passing FILE + pointers from Python to libcurl. Applies to Python 2 only. + + * Added CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE, + CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE, CURLMOPT_MAX_HOST_CONNECTIONS + CURLMOPT_MAX_PIPELINE_LENGTH, CURLMOPT_MAX_TOTAL_CONNECTIONS + multi options (patch by Jakob Truelsen). + + * SSL detection logic changed to consult `curl-config --static-libs` + even if `curl-config --libs` succeeded. This should achieve + pre-7.19.3 behavior with respect to automatic SSL detection + (patch by Andjelko Horvat). + + +Version 7.19.3 [requires libcurl-7.19.0 or better] - 2014-01-09 +--------------------------------------------------------------- + + * Added CURLOPT_NOPROXY. + + * Added CURLINFO_LOCAL_PORT, CURLINFO_PRIMARY_PORT and + CURLINFO_LOCAL_IP (patch by Adam Jacob Muller). + + * When running on Python 2.x, for compatibility with Python 3.x, + Unicode strings containing ASCII code points only are now accepted + in setopt() calls. + + * PycURL now requires that compile time SSL backend used by libcurl + is the same as the one used at runtime. setup.py supports + --with-ssl, --with-gnutls and --with-nss options like libcurl does, + to specify which backend libcurl uses. On some systems PycURL can + automatically figure out libcurl's backend. + If the backend is not one for which PycURL provides crypto locks + (i.e., any of the other backends supported by libcurl), + no runtime SSL backend check is performed. + + * Default PycURL user agent string is now built at runtime, and will + include the user agent string of libcurl loaded at runtime rather + than the one present at compile time. + + * PycURL will now use WSAduplicateSocket rather than dup on Windows + to duplicate sockets obtained from OPENSOCKETFUNCTION. + Using dup may have caused crashes, OPENSOCKETFUNCTION should + now be usable on Windows. + + * A new script, winbuild.py, was added to build PycURL on Windows + against Python 2.6, 2.7, 3.2 and 3.3. + + * Added CURL_LOCK_DATA_SSL_SESSION (patch by Tom Pierce). + + * Added E_OPERATION_TIMEDOUT (patch by Romuald Brunet). + + * setup.py now handles --help argument and will print PycURL-specific + configuration options in addition to distutils help. + + * Windows build configuration has been redone: + PYCURL_USE_LIBCURL_DLL #define is gone, use --use-libcurl-dll + argument to setup.py to build against a libcurl DLL. + CURL_STATICLIB is now #defined only when --use-libcurl-dll is not + given to setup.py, and PycURL is built against libcurl statically. + --libcurl-lib-name option can be used to override libcurl import + library name. + + * Added CURLAUTH_DIGEST_IE as pycurl.HTTPAUTH_DIGEST_IE. + + * Added CURLOPT_POSTREDIR option and CURL_REDIR_POST_301, + CURL_REDIR_POST_302, CURL_REDIR_POST_303 and CURL_REDIR_POST_ALL + constants. CURL_REDIR_POST_303 requires libcurl 7.26.0 or higher, + all others require libcurl 7.19.1 or higher. + + * As part of Python 3 support, WRITEDATA option now accepts + any object with a write method on Python 2 and Python 3. + For non-file objects, c.setopt(c.WRITEDATA, buf) is equivalent to + c.setopt(c.WRITEFUNCTION, buf.write). + + * PycURL now supports Python 3.1 through 3.3. Python 3.0 might + work but it appears to ship with broken distutils, making virtualenv + not function on it. + + * PycURL multi objects now have the multi constants defined on them. + Previously the constants were only available on pycurl module. + The new behavior matches that of curl and share objects. + + * PycURL share objects can now be closed via the close() method. + + * PycURL will no longer call `curl-config --static-libs` if + `curl-config --libs` succeeds and returns output. + Systems on which neither `curl-config --libs` nor + `curl-config --static-libs` do the right thing should provide + a `curl-config` wrapper that is sane. + + * Added CURLFORM_BUFFER and CURLFORM_BUFFERPTR. + + * pycurl.version and user agent string now include both + PycURL version and libcurl version as separate items. + + * Added CURLOPT_DNS_SERVERS. + + * PycURL can now be dynamically linked against libcurl on Windows + if PYCURL_USE_LIBCURL_DLL is #defined during compilation. + + * Breaking change: opensocket callback now takes an additional + (address, port) tuple argument. Existing callbacks will need to + be modified to accept this new argument. + https://github.com/pycurl/pycurl/pull/18 + + +Version 7.19.0.3 [requires libcurl-7.19.0 or better] - 2013-12-24 +----------------------------------------------------------------- + + * Re-release of 7.19.0.2 with minor changes to build Windows packages + due to botched 7.19.0.2 files on PyPi. + https://curl.haxx.se/mail/curlpython-2013-12/0021.html + + +Version 7.19.0.2 [requires libcurl-7.19.0 or better] - 2013-10-08 +----------------------------------------------------------------- + + * Fixed a bug in a commit made in 2008 but not released until 7.19.0.1 + which caused CURLOPT_POSTFIELDS to not correctly increment reference + count of the object being given as its argument, despite libcurl not + copying the data provided by said object. + + * Added support for libcurl pause/unpause functionality, + via curl_easy_pause call and returning READFUNC_PAUSE from + read callback function. + + +Version 7.19.0.1 [requires libcurl-7.19.0 or better] - 2013-09-23 +----------------------------------------------------------------- + + * Test matrix tool added to test against all supported Python and + libcurl versions. + + * Python 2.4 is now the minimum required version. + + * Source code, bugs and patches are now kept on GitHub. + + * Added CURLINFO_CERTINFO and CURLOPT_CERTINFO. + + * Added CURLOPT_RESOLVE. + + * PycURL can now be used with Python binaries without thread + support. + + * gcrypt is no longer initialized when a newer version of gnutls + is used. + + * Marked NSS as supported. + + * Fixed relative URL request logic. + + * Fixed a memory leak in util_curl_init. + + * Added CURLOPT_USERNAME and CURLOPT_PASSWORD. + + * Fixed handling of big timeout values. + + * Added GLOBAL_ACK_EINTR. + + * setopt(..., None) can be used as unsetopt(). + + * CURLOPT_RANGE can now be unset. + + * Write callback can return -1 to signal user abort. + + * Reorganized tests into an automated test suite. + + * Added CURLOPT_SEEKFUNCTION and CURLOPT_SEEKDATA. + + * Cleaned up website. + + * Fix pycurl.reset() (patch by ). + + * Fix install routine in setup.py where + certain platforms (Solaris, Mac OSX, etc) + would search for a static copy of libcurl (dbp). + + * Fixed build on OpenSolaris 0906 and other platforms on which + curl-config does not have a --static-libs option. + + * No longer keep string options copies in the + Curl Python objects, since string options are + now managed by libcurl. + + +Version 7.19.0 [requires libcurl-7.19.0 or better] +-------------------------------------------------- + + * Added CURLFILE, ADDRESS_SCOPE and ISSUERCERT options, + as well as the APPCONNECT_TIME info. + + * Added PRIMARY_IP info (patch by + Yuhui H ). + + * Added support for curl_easy_reset through a + new 'reset' method on curl objects + (patch by Nick Pilon ). + + * Added support for OPENSOCKET callbacks. + See 'tests/test_opensocket.py' for example + usage (patch by Thomas Hunger ). + + +Version 7.18.2 +-------------- + + * Added REDIRECT_URL info and M_MAXCONNECTS option + (patch by Yuhui H ). + + * Added socket_action() method to CurlMulti objects. + See 'tests/test_multi_socket_select.py' for example + usage (patch by Yuhui H ). + + * Added AUTOREFERER option. + + * Allow resetting some list operations (HTTPHEADER, + QUOTE, POSTQUOTE, PREQUOTE) by passing an empty + list to setopt (patch by Jim Patterson). + + +Version 7.18.1 +-------------- + + * Added POST301, SSH_HOST_PUBLIC_KEY_MD5, + COPYPOSTFIELDS and PROXY_TRANSFER_MODE options. + + * Check for static libs in setup.py to better detect + whether libcurl was linked with OpenSSL or GNUTLS. + + * PycURL is now dual licensed under the LGPL and + a license similar to the cURL license (an MIT/X + derivative). + + +Version 7.16.4 +-------------- + + * Allow any callable object as the callback function. + This change comes handy when you would like to use objects + which are callable but are not functions or methods, for + example those objects created by the functions in the functools + module (patch by Daniel Pena Arteaga ). + + * Added NEW_DIRECTORY_PERMS and NEW_FILE_PERMS options. + + +Version 7.16.2.1 +---------------- + + * Added IOCMD_NOP and IOCMD_RESTARTREAD for ioctl callback + handling (patch by Mark Eichin). + + * Use Py_ssize_t where appropriate for Python 2.5 and 64-bit + compatibility. This fixes the problem reported by Aaron + Hill, where the exception "pycurl.error: (2, '')" is thrown + when calling setopt(pycurl.POSTFIELDS,...) on 64-bit + platforms. + + +Version 7.16.2 +-------------- + + * Added options HTTP_TRANSFER_DECODING, HTTP_CONTENT_DECODING, + TIMEOUT_MS, CONNECTTIMEOUT_MS from libcurl 7.16.2. + + * Right-strip URLs read from files in the test scripts + to avoid sending requests with '\n' at the end. + + +Version 7.16.1 +-------------- + + * Added constants for all libcurl (error) return codes. They + are named the same as the macro constants in curl.h but prefixed + with E_ instead of CURLE. Return codes for the multi API are + prefixed with M_ instead of CURLM. + + * Added CURLOPT_FTP_SSL_CCC, CURLOPT_SSH_PUBLIC_KEYFILE, + CURLOPT_SSH_PRIVATE_KEYFILE, CURLOPT_SSH_AUTH_TYPES. + + * Removed CLOSEPOLICY and friends since this option is now + deprecated in libcurl. + + * Set the _use_datetime attribute on the CURLTransport class + to unbreak xmlrpc_curl.py on Python 2.5. + + +Version 7.16.0 [no public release] +-------------- + + * Added CURLOPT_SSL_SESSIONID_CACHE. + + * Removed SOURCE_* options since they are no longer + supported by libcurl. + + +Version 7.15.5.1 +---------------- + + * Added test for basic ftp usage (tests/test_ftp.py). + + * Fix broken ssl mutex lock function when using + GNU TLS (Debian bug #380156, fix by Bastian Kleineidam) + + +Version 7.15.5 +-------------- + + * Added CURLOPT_FTP_ALTERNATIVE_TO_USER, + CURLOPT_MAX_SEND_SPEED_LARGE, + and CURLOPT_MAX_RECV_SPEED_LARGE. + + +Version 7.15.4.2 +---------------- + + * Use SSL locking callbacks, fixes random + crashes for multithreaded SSL connections + (patch by Jayne ). + + +Version 7.15.4.1 +---------------- + + * Fixed compilation problem with C compilers + not allowing declarations in the middle of + code blocks (patch by + K.S.Sreeram ). + + * Fixed bug in curl_multi_fdset wrapping, + max_fd < 0 is not an error (patch by + K.S.Sreeram ). + + +Version 7.15.4 +-------------- + + * Added support for libcurl shares, patch from + Victor Lascurain . See the + file tests/test_share.py for example usage. + + * Added support for CURLINFO_FTP_ENTRY_PATH. + + +Version 7.15.2 +-------------- + + * Added CURLOPT_CONNECT_ONLY, CURLINFO_LASTSOCKET, + CURLOPT_LOCALPORT and CURLOPT_LOCALPORTRANGE. + + +Version 7.15.1 +-------------- + +2006-01-31 Kjetil Jacobsen + + * Fixed memory leak for getinfo calls that return a + list as result. Patch by Paul Pacheco. + + +Version 7.15.0 +-------------- + +2005-10-18 Kjetil Jacobsen + + * Added CURLOPT_FTP_SKIP_PASV_IP. + + +Version 7.14.1 +-------------- + +2005-09-05 Kjetil Jacobsen + + * Added CURLOPT_IGNORE_CONTENT_LENGTH, CURLOPT_COOKIELIST as + COOKIELIST and CURLINFO_COOKIELIST as INFO_COOKIELIST. + + +Version 7.14.0 +-------------- + +2005-05-18 Kjetil Jacobsen + + * Added missing information returned from the info() method + in the high-level interface. + + * Added the FORM_FILENAME option to the CURLFORM API + with HTTPPOST. + + +Version 7.13.2 +-------------- + +2005-03-30 Kjetil Jacobsen + + * Unbreak tests/test_gtk.py and require pygtk >= 2.0. + +2005-03-15 Kjetil Jacobsen + + * Cleaned up several of the examples. + +2005-03-11 Kjetil Jacobsen + + * WARNING: multi.select() now requires the previously optional + timeout parameter. Updated the tests and examples to reflect + this change. If the timeout is not set, select could block + infinitely and cause problems for the internal timeout handling + in the multi stack. The problem was identified by + . + + +Version 7.13.1 +-------------- + +2005-03-04 Kjetil Jacobsen + + * Use METH_NOARGS where appropriate. + +2005-03-03 Kjetil Jacobsen + + * Added support for CURLFORM API with HTTPPOST: Supports a + a tuple with pairs of options and values instead of just + supporting string contents. See tests/test_post2.py + for example usage. Options are FORM_CONTENTS, FORM_FILE and + FORM_CONTENTTYPE, corresponding to the CURLFORM_* options, + and values are strings. + +2005-02-13 Markus F.X.J. Oberhumer + + * Read callbacks (pycurl.READFUNCTION) can now return + pycurl.READFUNC_ABORT to immediately abort the current transfer. + + * The INFILESIZE, MAXFILESIZE, POSTFIELDSIZE and RESUME_FROM + options now automatically use the largefile version to handle + files > 2GB. + + * Added missing pycurl.PORT constant. + + +Version 7.13.0 +-------------- + +2005-02-10 Kjetil Jacobsen + + * Added file_upload.py to examples, shows how to upload + a file. + + * Added CURLOPT_IOCTLFUNCTION/DATA. + + * Added options from libcurl 7.13.0: FTP_ACCOUNT, SOURCE_URL, + SOURCE_QUOTE. + + * Obsoleted options: SOURCE_HOST, SOURCE_PATH, SOURCE_PORT, + PASV_HOST. + + +Version 7.12.3 +-------------- + +2004-12-22 Markus F.X.J. Oberhumer + + * Added CURLINFO_NUM_CONNECTS and CURLINFO_SSL_ENGINES. + + * Added some other missing constants. + + * Updated pycurl.version_info() to return a 12-tuple + instead of a 9-tuple. + + +Version 7.12.2 +-------------- + +2004-10-15 Kjetil Jacobsen + + * Added CURLOPT_FTPSSLAUTH (and CURLFTPAUTH_*). + + * Added CURLINFO_OS_ERRNO. + +2004-08-17 Kjetil Jacobsen + + * Use LONG_LONG instead of PY_LONG_LONG to make pycurl compile + on Python versions < 2.3 (fix from Domenico Andreoli + ). + + +Version 7.12.1 +-------------- + +2004-08-02 Kjetil Jacobsen + + * Added INFOTYPE_SSL_DATA_IN/OUT. + +2004-07-16 Markus F.X.J. Oberhumer + + * WARNING: removed deprecated PROXY_, TIMECOND_ and non-prefixed + INFOTYPE constant names. See ChangeLog entry 2003-06-10. + +2004-06-21 Kjetil Jacobsen + + * Added test program for HTTP post using the read callback (see + tests/test_post3.py for details). + + * Use the new CURL_READFUNC_ABORT return code where appropriate + to avoid hanging in perform() when read callbacks are used. + + * Added support for libcurl 7.12.1 CURLOPT features: + SOURCE_HOST, SOURCE_USERPWD, SOURCE_PATH, SOURCE_PORT, + PASV_HOST, SOURCE_PREQUOTE, SOURCE_POSTQUOTE. + +2004-06-08 Markus F.X.J. Oberhumer + + * Setting CURLOPT_POSTFIELDS now allows binary data and + automatically sets CURLOPT_POSTFIELDSIZE for you. If you really + want a different size you have to manually set POSTFIELDSIZE + after setting POSTFIELDS. + (Based on a patch by Martin Muenstermann). + +2004-06-05 Markus F.X.J. Oberhumer + + * Added stricter checks within the callback handlers. + + * Unify the behaviour of int and long parameters where appropriate. + + +Version 7.12 +------------ + +2004-05-18 Kjetil Jacobsen + + * WARNING: To simplify code maintenance pycurl now requires + libcurl 7.11.2 and Python 2.2 or newer to work. + + * GC support is now always enabled. + + +Version 7.11.3 +-------------- + +2004-04-30 Kjetil Jacobsen + + * Do not use the deprecated curl_formparse function. + API CHANGE: HTTPPOST now takes a list of tuples where each + tuple contains a form name and a form value, both strings + (see test/test_post2.py for example usage). + + * Found a possible reference count bug in the multithreading + code which may have contributed to the long-standing GC + segfault which has haunted pycurl. Fingers crossed. + + +Version 7.11.2 +-------------- + +2004-04-21 Kjetil Jacobsen + + * Added support for libcurl 7.11.2 CURLOPT features: + CURLOPT_TCP_NODELAY. + +2004-03-25 Kjetil Jacobsen + + * Store Python longs in off_t with PyLong_AsLongLong instead + of PyLong_AsLong. Should make the options which deal + with large files behave a little better. Note that this + requires the long long support in Python 2.2 or newer to + work properly. + + +Version 7.11.1 +-------------- + +2004-03-16 Kjetil Jacobsen + + * WARNING: Removed support for the PASSWDFUNCTION callback, which + is no longer supported by libcurl. + +2004-03-15 Kjetil Jacobsen + + * Added support for libcurl 7.11.1 CURLOPT features: + CURLOPT_POSTFIELDSIZE_LARGE. + + +Version 7.11.0 +-------------- + +2004-02-11 Kjetil Jacobsen + + * Added support for libcurl 7.11.0 CURLOPT features: + INFILESIZE_LARGE, RESUME_FROM_LARGE, MAXFILESIZE_LARGE + and FTP_SSL. + + * Circular garbage collection support can now be enabled or + disabled by passing the '--use-gc=[1|0]' parameter to setup.py + when building pycurl. + + * HTTP_VERSION options are known as CURL_HTTP_VERSION_NONE, + CURL_HTTP_VERSION_1_0, CURL_HTTP_VERSION_1_1 and + CURL_HTTP_VERSION_LAST. + +2003-11-16 Markus F.X.J. Oberhumer + + * Added support for these new libcurl 7.11.0 features: + CURLOPT_NETRC_FILE. + + +Version 7.10.8 +-------------- + +2003-11-04 Markus F.X.J. Oberhumer + + * Added support for these new libcurl 7.10.8 features: + CURLOPT_FTP_RESPONSE_TIMEOUT, CURLOPT_IPRESOLVE, + CURLOPT_MAXFILESIZE, + CURLINFO_HTTPAUTH_AVAIL, CURLINFO_PROXYAUTH_AVAIL, + CURL_IPRESOLVE_* constants. + + * Added support for these new libcurl 7.10.7 features: + CURLOPT_FTP_CREATE_MISSING_DIRS, CURLOPT_PROXYAUTH, + CURLINFO_HTTP_CONNECTCODE. + + +2003-10-28 Kjetil Jacobsen + + * Added missing CURLOPT_ENCODING option (patch by Martijn + Boerwinkel ) + + +Version 7.10.6 +-------------- + +2003-07-29 Markus F.X.J. Oberhumer + + * Started working on support for CURLOPT_SSL_CTX_FUNCTION and + CURLOPT_SSL_CTX_DATA (libcurl-7.10.6) - not yet finished. + +2003-06-10 Markus F.X.J. Oberhumer + + * Added support for CURLOPT_HTTPAUTH (libcurl-7.10.6), including + the new HTTPAUTH_BASIC, HTTPAUTH_DIGEST, HTTPAUTH_GSSNEGOTIATE + and HTTPAUTH_NTML constants. + + * Some constants were renamed for consistency: + + All curl_infotype constants are now prefixed with "INFOTYPE_", + all curl_proxytype constants are prefixed with "PROXYTYPE_" instead + of "PROXY_", and all curl_TimeCond constants are now prefixed + with "TIMECONDITION_" instead of "TIMECOND_". + + (The old names are still available but will get removed + in a future release.) + + * WARNING: Removed the deprecated pycurl.init() and pycurl.multi_init() + names - use pycurl.Curl() and pycurl.CurlMulti() instead. + + * WARNING: Removed the deprecated Curl.cleanup() and + CurlMulti.cleanup() methods - use Curl.close() and + CurlMulti.close() instead. + + +Version 7.10.5 +-------------- + +2003-05-15 Markus F.X.J. Oberhumer + + * Added support for CURLOPT_FTP_USE_EPRT (libcurl-7.10.5). + + * Documentation updates. + +2003-05-07 Eric S. Raymond + + * Lifted all HTML docs to clean XHTML, verified by tidy. + +2003-05-02 Markus F.X.J. Oberhumer + + * Fixed some `int' vs. `long' mismatches that affected 64-bit systems. + + * Fixed wrong pycurl.CAPATH constant. + +2003-05-01 Markus F.X.J. Oberhumer + + * Added new method Curl.errstr() which returns the internal + libcurl error buffer string of the handle. + + +Version 7.10.4.2 +---------------- + +2003-04-15 Markus F.X.J. Oberhumer + + * Allow compilation against the libcurl-7.10.3 release - some + recent Linux distributions (e.g. Mandrake 9.1) ship with 7.10.3, + and apart from the new CURLOPT_UNRESTRICTED_AUTH option there is + no need that we require libcurl-7.10.4. + + +Version 7.10.4 +-------------- + +2003-04-01 Kjetil Jacobsen + + * Markus added CURLOPT_UNRESTRICTED_AUTH (libcurl-7.10.4). + +2003-02-25 Kjetil Jacobsen + + * Fixed some broken test code and removed the fileupload test + since it didn't work properly. + +2003-01-28 Kjetil Jacobsen + + * Some documentation updates by Markus and me. + +2003-01-22 Kjetil Jacobsen + + * API CHANGE: the CurlMulti.info_read() method now returns + a separate array with handles that failed. Each entry in this array + is a tuple with (curl object, error number, error message). + This addition makes it simpler to do error checking of individual + curl objects when using the multi interface. + + +Version 7.10.3 +-------------- + +2003-01-13 Kjetil Jacobsen + + * PycURL memory usage has been reduced. + +2003-01-10 Kjetil Jacobsen + + * Added 'examples/retriever-multi.py' which shows how to retrieve + a set of URLs concurrently using the multi interface. + +2003-01-09 Kjetil Jacobsen + + * Added support for CURLOPT_HTTP200ALIASES. + +2002-11-22 Kjetil Jacobsen + + * Updated pycurl documentation in the 'doc' directory. + +2002-11-21 Kjetil Jacobsen + + * Updated and improved 'examples/curl.py'. + + * Added 'tests/test_multi6.py' which shows how to use the + info_read method with CurlMulti. + +2002-11-19 Kjetil Jacobsen + + * Added new method CurlMulti.info_read(). + + +Version 7.10.2 +-------------- + +2002-11-14 Kjetil Jacobsen + + * Free options set with setopt after cleanup is called, as cleanup + assumes that options are still valid when invoked. This fixes the + bug with COOKIEJAR reported by Bastiaan Naber + . + +2002-11-06 Markus F.X.J. Oberhumer + + * Install documentation under /usr/share/doc instead of /usr/doc. + Also, start shipping the (unfinished) HTML docs and some + basic test scripts. + +2002-10-30 Markus F.X.J. Oberhumer + + * API CHANGE: For integral values, Curl.getinfo() now returns a + Python-int instead of a Python-long. + + +Version 7.10.1 +-------------- + +2002-10-03 Markus F.X.J. Oberhumer + + * Added new module-level function version_info() from + libcurl-7.10. + + +Version 7.10 +------------ + +2002-09-13 Kjetil Jacobsen + + * Added commandline options to setup.py for specifying the path to + 'curl-config' (non-windows) and the curl installation directory + (windows). See the 'INSTALL' file for details. + + * Added CURLOPT_ENCODING, CURLOPT_NOSIGNAL and CURLOPT_BUFFERSIZE + from libcurl-7.10 (by Markus Oberhumer). + + +Version 7.9.8.4 +--------------- + +2002-08-28 Kjetil Jacobsen + + * Added a simple web-browser example based on gtkhtml and pycurl. + See the file 'examples/gtkhtml_demo.py' for details. The example + requires a working installation of gnome-python with gtkhtml + bindings enabled (pass --with-gtkhtml to gnome-python configure). + +2002-08-14 Kjetil Jacobsen + + * Added new method 'select' on CurlMulti objects. Example usage + in 'tests/test_multi5.py'. This method is just an optimization of + the combined use of fdset and select. + +2002-08-12 Kjetil Jacobsen + + * Added support for curl_multi_fdset. See the file + 'tests/test_multi4.py' for example usage. Contributed by Conrad + Steenberg . + + * perform() on multi objects now returns a tuple (result, number + of handles) like the libcurl interface does. + +2002-08-08 Kjetil Jacobsen + + * Added the 'sfquery' script which retrieves a SourceForge XML + export object for a given project. See the file 'examples/sfquery.py' + for details and usage. 'sfquery' was contributed by Eric + S. Raymond . + +2002-07-20 Markus F.X.J. Oberhumer + + * API enhancements: added Curl() and CurlMulti() as aliases for + init() and multi_init(), and added close() methods as aliases + for the cleanup() methods. The new names much better match + the actual intended use of the objects, and they also nicely + correspond to Python's file object. + + * Also, all constants for Curl.setopt() and Curl.getinfo() are now + visible from within Curl objects. + + All changes are fully backward-compatible. + + +Version 7.9.8.3 +--------------- + +2002-07-16 Markus F.X.J. Oberhumer + + * Under Python 2.2 or better, Curl and CurlMulti objects now + automatically participate in cyclic garbage collection + (using the gc module). + + +Version 7.9.8.2 +--------------- + +2002-07-05 Markus F.X.J. Oberhumer + + * Curl and CurlMulti objects now support standard Python attributes. + See tests/test_multi2.py for an example. + +2002-07-02 Kjetil Jacobsen + + * Added support for the multi-interface. + + +Version 7.9.8.1 +--------------- + +2002-06-25 Markus F.X.J. Oberhumer + + * Fixed a couple of `int' vs. `size_t' mismatches in callbacks + and Py_BuildValue() calls. + +2002-06-25 Kjetil Jacobsen + + * Use 'double' type instead of 'size_t' for progress callbacks + (by Conrad Steenberg ). Also cleaned up + some other type mismatches in the callback interfaces. + +2002-06-24 Kjetil Jacobsen + + * Added example code on how to upload a file using HTTPPOST in + pycurl (code by Amit Mongia ). See the + file 'test_fileupload.py' for details. + + +Version 7.9.8 +------------- + +2002-06-24 Kjetil Jacobsen + + * Resolved some build problems on Windows (by Markus Oberhumer). + +2002-06-19 Kjetil Jacobsen + + * Added CURLOPT_CAPATH. + + * Added option constants for CURLOPT_NETRC: CURL_NETRC_OPTIONAL, + CURL_NETRC_IGNORED and CURL_NETRC_REQUIRED. + + * Added option constants for CURLOPT_TIMECONDITION: + TIMECOND_IFMODSINCE and TIMECOND_IFUNMODSINCE. + + * Added an simple example crawler, which downloads documents + listed in a file with a configurable number of worker threads. + See the file 'crawler.py' in the 'tests' directory for details. + + * Removed the redundant 'test_xmlrpc2.py' test script. + + * Disallow recursive callback invocations (by Markus Oberhumer). + +2002-06-18 Kjetil Jacobsen + + * Made some changes to setup.py which should fix the build + problems on RedHat 7.3 (suggested by Benji ). + + * Use CURLOPT_READDATA instead of CURLOPT_INFILE, and + CURLOPT_WRITEDATA instead of CURLOPT_FILE. Also fixed some + reference counting bugs with file objects. + + * CURLOPT_FILETIME and CURLINFO_FILETIME had a namespace clash + which caused them not to work. Use OPT_FILETIME for setopt() and + INFO_FILETIME for getinfo(). See example usage in + 'test_getinfo.py' for details. + + +Version 7.9.7 +------------- + +2002-05-20 Kjetil Jacobsen + + * New versioning scheme. Pycurl now has the same version number + as the libcurl version it was built with. The pycurl version + number thus indicates which version of libcurl is required to run. + +2002-05-17 Kjetil Jacobsen + + * Added CURLINFO_REDIRECT_TIME and CURLINFO_REDIRECT_COUNT. + +2002-04-27 Kjetil Jacobsen + + * Fixed potential memory leak and thread race (by Markus + Oberhumer). + + +Version 0.4.9 +------------- + +2002-04-15 Kjetil Jacobsen + + * Added CURLOPT_DEBUGFUNCTION to allow debug callbacks to be + specified (see the file 'test_debug.py' for details on how to use + debug callbacks). + + * Added CURLOPT_DNS_USE_GLOBAL_CACHE and + CURLOPT_DNS_CACHE_TIMEOUT. + + * Fixed a segfault when finalizing curl objects in Python 1.5.2. + + * Now requires libcurl 7.9.6 or greater. + +2002-04-12 Kjetil Jacobsen + + * Added 'test_post2.py' file which is another example on how to + issue POST requests. + +2002-04-11 Markus F.X.J. Oberhumer + + * Added the 'test_post.py' file which demonstrates the use of + POST requests. + + +Version 0.4.8 +------------- + +2002-03-07 Kjetil Jacobsen + + * Added CURLOPT_PREQUOTE. + + * Now requires libcurl 7.9.5 or greater. + + * Other minor code cleanups and bugfixes. + +2002-03-05 Kjetil Jacobsen + + * Do not allow WRITEFUNCTION and WRITEHEADER on the same handle. + + +Version 0.4.7 +------------- + +2002-02-27 Kjetil Jacobsen + + * Abort callback if the thread state of the calling thread cannot + be determined. + + * Check that the installed version of libcurl matches the + requirements of pycurl. + +2002-02-26 Kjetil Jacobsen + + * Clarence Garnder found a bug where string + arguments to setopt sometimes were prematurely deallocated, this + should now be fixed. + +2002-02-21 Kjetil Jacobsen + + * Added the 'xmlrpc_curl.py' file which implements a transport + for xmlrpclib (xmlrpclib is part of Python 2.2). + + * Added CURLINFO_CONTENT_TYPE. + + * Added CURLOPT_SSLCERTTYPE, CURLOPT_SSLKEY, CURLOPT_SSLKEYTYPE, + CURLOPT_SSLKEYPASSWD, CURLOPT_SSLENGINE and + CURLOPT_SSLENGINE_DEFAULT. + + * When thrown, the pycurl.error exception is now a tuple consisting + of the curl error code and the error message. + + * Now requires libcurl 7.9.4 or greater. + +2002-02-19 Kjetil Jacobsen + + * Fixed docstring for getopt() function. + +2001-12-18 Kjetil Jacobsen + + * Updated the INSTALL information for Win32. + +2001-12-12 Kjetil Jacobsen + + * Added missing link flag to make pycurl build on MacOS X (by Matt + King ). + +2001-12-06 Kjetil Jacobsen + + * Added CURLINFO_STARTTRANSFER_TIME and CURLOPT_FTP_USE_EPSV from + libcurl 7.9.2. + +2001-12-01 Markus F.X.J. Oberhumer + + * Added the 'test_stringio.py' file which demonstrates the use of + StringIO objects as callback. + +2001-12-01 Markus F.X.J. Oberhumer + + * setup.py: Do not remove entries from a list while iterating + over it. + +2001-11-29 Kjetil Jacobsen + + * Added code in setup.py to install on Windows. Requires some + manual configuration (by Tino Lange ). + +2001-11-27 Kjetil Jacobsen + + * Improved detection of where libcurl is installed in setup.py. + Should make it easier to install pycurl when libcurl is not + located in regular lib/include paths. + +2001-11-05 Kjetil Jacobsen + + * Some of the newer options to setopt were missing, this should + now be fixed. + +2001-11-04 Kjetil Jacobsen + + * Exception handling has been improved and should no longer throw + spurious exceptions (by Markus F.X.J. Oberhumer + ). + +2001-10-15 Kjetil Jacobsen + + * Refactored the test_gtk.py script to avoid global variables. + +2001-10-12 Kjetil Jacobsen + + * Added module docstrings, terse perhaps, but better than nothing. + + * Added the 'basicfirst.py' file which is a Python version of the + corresponding Perl script by Daniel. + + * PycURL now works properly under Python 1.5 and 1.6 (by Markus + F.X.J. Oberhumer ). + + * Allow C-functions and Python methods as callbacks (by Markus + F.X.J. Oberhumer ). + + * Allow None as success result of write, header and progress + callback invocations (by Markus F.X.J. Oberhumer + ). + + * Added the 'basicfirst2.py' file which demonstrates the use of a + class method as callback instead of just a function. + +2001-08-21 Kjetil Jacobsen + + * Cleaned up the script with GNOME/PycURL integration. + +2001-08-20 Kjetil Jacobsen + + * Added another test script for shipping XML-RPC requests which + uses py-xmlrpc to encode the arguments (tests/test_xmlrpc2.py). + +2001-08-20 Kjetil Jacobsen + + * Added test script for using PycURL and GNOME (tests/test_gtk.py). + +2001-08-20 Kjetil Jacobsen + + * Added test script for using XML-RPC (tests/test_xmlrpc.py). + + * Added more comments to the test sources. + +2001-08-06 Kjetil Jacobsen + + * Renamed module namespace to pycurl instead of curl. + +2001-08-06 Kjetil Jacobsen + + * Set CURLOPT_VERBOSE to 0 by default. + +2001-06-29 Kjetil Jacobsen + + * Updated INSTALL, curl version 7.8 or greater is now mandatory to + use pycurl. + +2001-06-13 Kjetil Jacobsen + + * Set NOPROGRESS to 1 by default. + +2001-06-07 Kjetil Jacobsen + + * Added global_init/cleanup. + +2001-06-06 Kjetil Jacobsen + + * Added HEADER/PROGRESSFUNCTION callbacks (see files in tests/). + + * Added PASSWDFUNCTION callback (untested). + + * Added READFUNCTION callback (untested). + +2001-06-05 Kjetil Jacobsen + + * WRITEFUNCTION callbacks now work (see tests/test_cb.py for details). + + * Preliminary distutils installation. + + * Added CLOSEPOLICY constants to module namespace. + +2001-06-04 Kjetil Jacobsen + + * Return -1 on error from Python callback in WRITEFUNCTION callback. + +2001-06-01 Kjetil Jacobsen + + * Moved source to src and tests to tests directory. + +2001-05-31 Kjetil Jacobsen + + * Added better type checking for setopt. + +2001-05-30 Kjetil Jacobsen + + * Moved code to sourceforge. + + * Added getinfo support. + + +# vi:ts=8:et diff --git a/INSTALL.rst b/INSTALL.rst new file mode 100644 index 0000000..89b868e --- /dev/null +++ b/INSTALL.rst @@ -0,0 +1,313 @@ +.. _install: + +PycURL Installation +=================== + +NOTE: You need Python and libcurl installed on your system to use or +build pycurl. Some RPM distributions of curl/libcurl do not include +everything necessary to build pycurl, in which case you need to +install the developer specific RPM which is usually called curl-dev. + + +Distutils +--------- + +Build and install pycurl with the following commands:: + + (if necessary, become root) + tar -zxvf pycurl-$VER.tar.gz + cd pycurl-$VER + python setup.py install + +$VER should be substituted with the pycurl version number, e.g. 7.10.5. + +Note that the installation script assumes that 'curl-config' can be +located in your path setting. If curl-config is installed outside +your path or you want to force installation to use a particular +version of curl-config, use the '--curl-config' command line option to +specify the location of curl-config. Example:: + + python setup.py install --curl-config=/usr/local/bin/curl-config + +If libcurl is linked dynamically with pycurl, you may have to alter the +LD_LIBRARY_PATH environment variable accordingly. This normally +applies only if there is more than one version of libcurl installed, +e.g. one in /usr/lib and one in /usr/local/lib. + + +SSL +^^^ + +PycURL requires that the SSL library that it is built against is the same +one libcurl, and therefore PycURL, uses at runtime. PycURL's ``setup.py`` +uses ``curl-config`` to attempt to figure out which SSL library libcurl +was compiled against, however this does not always work. If PycURL is unable +to determine the SSL library in use it will print a warning similar to +the following:: + + src/pycurl.c:137:4: warning: #warning "libcurl was compiled with SSL support, but configure could not determine which " "library was used; thus no SSL crypto locking callbacks will be set, which may " "cause random crashes on SSL requests" [-Wcpp] + +It will then fail at runtime as follows:: + + ImportError: pycurl: libcurl link-time ssl backend (openssl) is different from compile-time ssl backend (none/other) + +To fix this, you need to tell ``setup.py`` what SSL backend is used:: + + python setup.py --with-[openssl|gnutls|nss|mbedtls|wolfssl|sectransp] install + +Note: as of PycURL 7.21.5, setup.py accepts ``--with-openssl`` option to +indicate that libcurl is built against OpenSSL/LibreSSL/BoringSSL. +``--with-ssl`` is an alias +for ``--with-openssl`` and continues to be accepted for backwards compatibility. + +You can also ask ``setup.py`` to obtain SSL backend information from installed +libcurl shared library, as follows: + + python setup.py --libcurl-dll=libcurl.so + +An unqualified ``libcurl.so`` would use the system libcurl, or you can +specify a full path. + + +easy_install / pip +------------------ + +:: + + easy_install pycurl + pip install pycurl + +If you need to specify an alternate curl-config, it can be done via an +environment variable:: + + export PYCURL_CURL_CONFIG=/usr/local/bin/curl-config + easy_install pycurl + +The same applies to the SSL backend, if you need to specify it (see the SSL +note above):: + + export PYCURL_SSL_LIBRARY=[openssl|gnutls|nss|mbedtls|sectransp] + easy_install pycurl + + +pip and cached pycurl package +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you have already installed pycurl and are trying to reinstall it via +pip with different SSL options for example, pip may reinstall the package it +has previously compiled instead of recompiling pycurl with newly specified +options. More details are given in `this Stack Overflow post`_. + +To force pip to recompile pycurl, run:: + + # upgrade pip if necessary + pip install --upgrade pip + + # remove current pycurl + pip uninstall pycurl + + # set PYCURL_SSL_LIBRARY + export PYCURL_SSL_LIBRARY=nss + + # recompile and install pycurl + pip install --compile pycurl + +.. _this Stack Overflow post: http://stackoverflow.com/questions/21487278/ssl-error-installing-pycurl-after-ssl-is-set + + +Windows +------- + +There are currently no official binary Windows packages. You can build PycURL +from source or use third-party binary packages. + + +Building From Source +^^^^^^^^^^^^^^^^^^^^ + +Building PycURL from source is not for the faint of heart due to the multitude +of possible dependencies and each of these dependencies having its own +directory structure, configuration style, parameters and quirks. +Additionally different dependencies have different +settings for MSVCRT usage, and an application must have all of its parts +agreeing on a single setting. If you decide to build PycURL from source +it is advisable to look through the ``winbuild.py`` +script - it is used to build the official binaries and contains a wealth +of information for compiling PycURL's dependencies on Windows. + +If you are compiling PycURL from source it is recommended to compile all of its +dependencies from source as well. Using precompiled libraries may lead to +multiple MSVCRT versions mixed in the resulting PycURL binary, which will +not be good. + +If PycURL is to be linked statically against its dependencies, OpenSSL must +be patched to link to the DLL version of MSVCRT. There is a patch for this in +``winbuild`` directory of PycURL source. + +For a minimum build you will just need libcurl source. Follow its Windows +build instructions to build either a static or a DLL version of the library, +then configure PycURL as follows to use it:: + + python setup.py --curl-dir=c:\dev\curl-7.33.0\builds\libcurl-vc-x86-release-dll-ipv6-sspi-spnego-winssl --use-libcurl-dll + +Note that ``--curl-dir`` must point not to libcurl source but rather to headers +and compiled libraries. + +If libcurl and Python are not linked against the same exact C runtime +(version number, static/dll, single-threaded/multi-threaded) you must use +``--avoid-stdio`` option (see below). + +Additional Windows setup.py options: + +- ``--use-libcurl-dll``: build against libcurl DLL, if not given PycURL will + be built against libcurl statically. +- ``--libcurl-lib-name=libcurl_imp.lib``: specify a different name for libcurl + import library. The default is ``libcurl.lib`` which is appropriate for + static linking and is sometimes the correct choice for dynamic linking as + well. The other possibility for dynamic linking is ``libcurl_imp.lib``. +- ``--with-openssl``: use OpenSSL/LibreSSL/BoringSSL crypto locks when libcurl + was built against these SSL backends. +- ``--with-ssl``: legacy alias for ``--with-openssl``. +- ``--openssl-lib-name=""``: specify a different name for OpenSSL import + library containing CRYPTO_num_locks. For OpenSSL 1.1.0+ this should be set + to an empty string as given here. +- ``--avoid-stdio``: on Windows, a process and each library it is using + may be linked to its own version of the C runtime (MSVCRT). + FILE pointers from one C runtime may not be passed to another C runtime. + This option prevents direct passing of FILE pointers from Python to libcurl, + thus permitting Python and libcurl to be linked against different C runtimes. + This option may carry a performance penalty when Python file objects are + given directly to PycURL in CURLOPT_READDATA, CURLOPT_WRITEDATA or + CURLOPT_WRITEHEADER options. This option applies only on Python 2; on + Python 3, file objects no longer expose C library FILE pointers and the + C runtime issue does not exist. On Python 3, this option is recognized but + does nothing. You can also give ``--avoid-stdio`` option in + PYCURL_SETUP_OPTIONS environment variable as follows:: + + PYCURL_SETUP_OPTIONS=--avoid-stdio pip install pycurl + +A good ``setup.py`` target to use is ``bdist_wininst`` which produces an +executable installer that you can run to install PycURL. + +You may find the following mailing list posts helpful: + +- https://curl.haxx.se/mail/curlpython-2009-11/0010.html +- https://curl.haxx.se/mail/curlpython-2013-11/0002.html + + +winbuild.py +^^^^^^^^^^^ + +This script is used to build official PycURL Windows packages. You can +use it to build a full complement of packages with your own options or modify +it to build a single package you need. + +Prerequisites: + +- `Git for Windows`_. +- Appropriate `Python versions`_ installed. +- MS Visual C++ 9/2008 for Python <= 3.2, MS Visual C++ 10/2010 for + Python 3.3 or 3.4, MS Visual C++ 14/2015 for Python 3.5 through 3.8. + Express versions of Visual Studio work fine for this, + although getting 64 bit compilers to wok in some Express versions involves + jumping through several hoops. +- NASM if building libcurl against OpenSSL. +- ActivePerl if building libcurl against OpenSSL. The perl shipping with + Git for Windows handles forward and backslashes in paths in a way that is + incompatible with OpenSSL's build scripts. + +.. _Git for Windows: https://git-for-windows.github.io/ +.. _Python versions: http://python.org/download/ + +``winbuild.py`` assumes all programs are installed in their default locations, +if this is not the case edit it as needed. ``winbuild.py`` itself can be run +with any Python it supports. + + +Using PycURL With Custom Python Builds +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As of version 7.21.5, the official binary packages of PycURL are linked +statically against all of its dependencies except MSVCRT. This means that +as long as your custom Python build uses the same version of MSVC as the +corresponding official Python build as well as the same MSVCRT linking setting +(/MD et. al.), an official PycURL package should work. + +If your Python build uses different MSVCRT settings or a different MSVC +version from the official Python builds, you will need to compile PycURL +from source. + +If the C runtime library (MSVCRT.DLL) versions used by PycURL and Python +do not match, you will receive a message +like the following one when trying to import ``pycurl`` module:: + + ImportError: DLL load failed: The specified procedure could not be found. + +To identify which MSVCRT version your Python uses use the +`application profiling feature`_ of +`Dependency Walker`_ and look for `msvcrt.dll variants`_ being loaded. +You may find `the entire thread starting here`_ helpful. + +.. _application profiling feature: https://curl.haxx.se/mail/curlpython-2014-05/0007.html +.. _Dependency Walker: http://www.dependencywalker.com/ +.. _msvcrt.dll variants: https://curl.haxx.se/mail/curlpython-2014-05/0010.html +.. _the entire thread starting here: https://curl.haxx.se/mail/curlpython-2014-05/0000.html + + +Git Checkout +------------ + +In order to build PycURL from a Git checkout, some files need to be +generated. On Unix systems it is easiest to build PycURL with ``make``:: + + make + +To specify which curl or SSL backend to compile against, use the same +environment variables as easy_install/pip, namely ``PYCURL_CURL_CONFIG`` +and ``PYCURL_SSL_LIBRARY``. + +To generate generated files only you may run:: + + make gen + +This might be handy if you are on Windows. Remember to run ``make gen`` +whenever you change sources. + +To generate documentation, run:: + + make docs + +Generating documentation requires `Sphinx`_ to be installed. + +.. _Sphinx: http://sphinx-doc.org/ + + +A Note Regarding SSL Backends +----------------------------- + +libcurl's functionality varies depending on which SSL backend it is compiled +against. For example, users have `reported`_ `problems`_ with GnuTLS backend. +As of this writing, generally speaking, OpenSSL backend has the most +functionality as well as the best compatibility with other software. + +If you experience SSL issues, especially if you are not using OpenSSL +backend, you can try rebuilding libcurl and PycURL against another SSL backend. + +.. _reported: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=515200 +.. _problems: https://bugs.launchpad.net/ubuntu/+source/pycurl/+bug/1111673 + + +SSL Certificate Bundle +---------------------- + +libcurl, and PycURL, by default verify validity of HTTPS servers' SSL +certificates. Doing so requires a CA certificate bundle, which libcurl +and most SSL libraries do not provide. + +Here_ is a good resource on how to build your own certificate bundle. +certifie.com also has a `prebuilt certificate bundle`_. +To use the certificate bundle, use ``CAINFO`` or ``CAPATH`` PycURL +options. + +.. _Here: http://certifie.com/ca-bundle/ +.. _prebuilt certificate bundle: http://certifie.com/ca-bundle/ca-bundle.crt.txt diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..d957246 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,55 @@ +# +# MANIFEST.in +# Manifest template for creating the source distribution. +# + +include AUTHORS +include COPYING-LGPL +include COPYING-MIT +include ChangeLog +include INSTALL.rst +include MANIFEST.in +include Makefile +include pytest.ini +include README.rst +include RELEASE-NOTES.rst +include doc/*.py +include doc/*.rst +include doc/docstrings/*.rst +include doc/static/favicon.ico +include examples/*.py +include examples/quickstart/*.py +include examples/tests/*.py +include src/Makefile +include src/docstrings.c +include src/docstrings.h +include src/easy.c +include src/easycb.c +include src/easyinfo.c +include src/easyopt.c +include src/easyperform.c +include src/module.c +include src/multi.c +include src/oscompat.c +include src/pycurl.h +include src/pythoncompat.c +include src/share.c +include src/stringcompat.c +include src/threadsupport.c +include src/util.c +include python/curl/*.py +include requirements*.txt +include tests/*.py +include tests/certs/*.crt +include tests/certs/*.key +include tests/ext/*.sh +include tests/fake-curl/* +include tests/fake-curl/libcurl/* +include tests/fixtures/form_submission.txt +include tests/matrix/*.patch +include tests/run.sh +include tests/run-quickstart.sh +include tests/vsftpd.conf +include winbuild.py +include winbuild/* +exclude tests/fake-curl/libcurl/*.so diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1365444 --- /dev/null +++ b/Makefile @@ -0,0 +1,200 @@ +# +# to use a specific python version call +# `make PYTHON=python2.7' +# + +SHELL = /bin/sh + +PYTHON = python +PYTEST = pytest +PYFLAKES = pyflakes + +PYTHONMAJOR=$$($(PYTHON) -V 2>&1 |awk '{print $$2}' |awk -F. '{print $$1}') +PYTHONMINOR=$$($(PYTHON) -V 2>&1 |awk '{print $$2}' |awk -F. '{print $$2}') + +# -c on linux +# freebsd does not understand -c +CHMOD_VERBOSE=-v + +BUILD_WWW = build/www + +RSYNC = rsync +##RSYNC_FLAGS = -av --relative -e ssh +RSYNC_FLAGS = -av --relative --delete --delete-after -e ssh + +RSYNC_FILES = \ + htdocs \ + htdocs/download/.htaccess \ + upload + +RSYNC_EXCLUDES = \ + '--exclude=htdocs/download/' \ + '--exclude=upload/Ignore/' \ + '--exclude=htdocs/travis-deps/' + +RSYNC_TARGET = /home/groups/p/py/pycurl/ + +RSYNC_USER = armco@web.sourceforge.net + +# src/module.c is first because it declares global variables +# which other files reference; important for single source build +SOURCES = src/easy.c src/easycb.c src/easyinfo.c src/easyopt.c src/easyperform.c \ + src/module.c src/multi.c src/oscompat.c src/pythoncompat.c \ + src/share.c src/stringcompat.c src/threadsupport.c src/util.c + +GEN_SOURCES = src/docstrings.c src/docstrings.h + +ALL_SOURCES = src/pycurl.h $(GEN_SOURCES) $(SOURCES) + +RELEASE_SOURCES = src/allpycurl.c + +DOCSTRINGS_SOURCES = \ + doc/docstrings/curl.rst \ + doc/docstrings/curl_close.rst \ + doc/docstrings/curl_errstr.rst \ + doc/docstrings/curl_errstr_raw.rst \ + doc/docstrings/curl_getinfo.rst \ + doc/docstrings/curl_getinfo_raw.rst \ + doc/docstrings/curl_pause.rst \ + doc/docstrings/curl_perform.rst \ + doc/docstrings/curl_reset.rst \ + doc/docstrings/curl_setopt.rst \ + doc/docstrings/curl_unsetopt.rst \ + doc/docstrings/curl_set_ca_certs.rst \ + doc/docstrings/multi.rst \ + doc/docstrings/multi_add_handle.rst \ + doc/docstrings/multi_assign.rst \ + doc/docstrings/multi_close.rst \ + doc/docstrings/multi_fdset.rst \ + doc/docstrings/multi_info_read.rst \ + doc/docstrings/multi_perform.rst \ + doc/docstrings/multi_remove_handle.rst \ + doc/docstrings/multi_select.rst \ + doc/docstrings/multi_setopt.rst \ + doc/docstrings/multi_socket_action.rst \ + doc/docstrings/multi_socket_all.rst \ + doc/docstrings/multi_timeout.rst \ + doc/docstrings/pycurl_global_cleanup.rst \ + doc/docstrings/pycurl_global_init.rst \ + doc/docstrings/pycurl_module.rst \ + doc/docstrings/pycurl_version_info.rst \ + doc/docstrings/share.rst \ + doc/docstrings/share_close.rst \ + doc/docstrings/share_setopt.rst + +all: build +src-release: $(RELEASE_SOURCES) + +src/docstrings.c src/docstrings.h: $(DOCSTRINGS_SOURCES) + $(PYTHON) setup.py docstrings + +src/allpycurl.c: $(ALL_SOURCES) + echo '#define PYCURL_SINGLE_FILE' >src/.tmp.allpycurl.c + cat src/pycurl.h >>src/.tmp.allpycurl.c + cat src/docstrings.c $(SOURCES) |sed -e 's/#include "pycurl.h"//' -e 's/#include "docstrings.h"//' >>src/.tmp.allpycurl.c + mv src/.tmp.allpycurl.c src/allpycurl.c + +gen: $(ALL_SOURCES) + +build: $(ALL_SOURCES) + $(PYTHON) setup.py build + +build-release: $(RELEASE_SOURCES) + PYCURL_RELEASE=1 $(PYTHON) setup.py build + +do-test: + make -C tests/fake-curl/libcurl + ./tests/run.sh + ./tests/ext/test-suite.sh + $(PYFLAKES) python examples tests setup.py + +test: build do-test +test-release: build-release do-test + +# rails-style alias +c: console +console: + PYTHONPATH=$$(ls -d build/lib.*$$PYTHONMAJOR*$$PYTHONMINOR):$$PYTHONPATH \ + $(PYTHON) + +# (needs GNU binutils) +strip: build + strip -p --strip-unneeded build/lib*/*.so + chmod -x build/lib*/*.so + +install install_lib: + $(PYTHON) setup.py $@ + +clean: + -rm -rf build dist + -rm -f *.pyc *.pyo */*.pyc */*.pyo */*/*.pyc */*/*.pyo + -rm -f MANIFEST + -rm -f src/allpycurl.c $(GEN_SOURCES) + +distclean: clean + +maintainer-clean: distclean + +dist sdist: distclean + $(PYTHON) setup.py sdist + +run-quickstart: + ./tests/run-quickstart.sh + +# Rebuild missing or changed documentation. +# Editing docstrings in Python or C source will not cause the documentation +# to be rebuilt with this target, use docs-force instead. +docs: build + mkdir -p build/docstrings + for file in doc/docstrings/*.rst; do tail -n +3 $$file >build/docstrings/`basename $$file`; done + PYTHONPATH=$$(ls -d build/lib.*$$PYTHONMAJOR*$$PYTHONMINOR):$$PYTHONPATH \ + $(PYTHON) -m sphinx doc build/doc + cp ChangeLog build/doc + +# Rebuild all documentation. +# As sphinx extracts documentation from pycurl modules, docs targets +# depend on build target. +docs-force: build + # sphinx-docs has an -a option but it does not seem to always + # rebuild everything + rm -rf build/doc + PYTHONPATH=$$(ls -d build/lib.*$$PYTHONMAJOR*$$PYTHONMINOR):$$PYTHONPATH \ + $(PYTHON) -m sphinx doc build/doc + cp ChangeLog build/doc + +www: docs + mkdir -p build + rsync -a www build --delete + rsync -a build/doc/ build/www/htdocs/doc --exclude .buildinfo --exclude .doctrees + cp doc/static/favicon.ico build/www/htdocs + cp ChangeLog build/www/htdocs + +rsync: rsync-prepare + cd $(BUILD_WWW) && \ + $(RSYNC) $(RSYNC_FLAGS) $(RSYNC_EXCLUDES) $(RSYNC_FILES) $(RSYNC_USER):$(RSYNC_TARGET) + +rsync-dry: + $(MAKE) rsync 'RSYNC=rsync --dry-run' + +rsync-check: + $(MAKE) rsync 'RSYNC=rsync --dry-run -c' + +# NOTE: Git does not maintain metadata like owners and file permissions, +# so we have to care manually. +# NOTE: rsync targets depend on www. +rsync-prepare: + chgrp $(CHMOD_VERBOSE) -R pycurl $(BUILD_WWW) + chmod $(CHMOD_VERBOSE) g+r `find $(BUILD_WWW) -perm +400 -print` + chmod $(CHMOD_VERBOSE) g+w `find $(BUILD_WWW) -perm +200 -print` + chmod $(CHMOD_VERBOSE) g+s `find $(BUILD_WWW) -type d -print` +## chmod $(CHMOD_VERBOSE) g+rws `find $(BUILD_WWW) -type d -perm -770 -print` + chmod $(CHMOD_VERBOSE) g+rws `find $(BUILD_WWW) -type d -print` + chmod $(CHMOD_VERBOSE) o-rwx $(BUILD_WWW)/upload + #-rm -rf `find $(BUILD_WWW) -name .xvpics -type d -print` + +.PHONY: all build test do-test strip install install_lib \ + clean distclean maintainer-clean dist sdist \ + docs docs-force \ + rsync rsync-dry rsync-check rsync-prepare + +.NOEXPORT: diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..ef2a1a0 --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,112 @@ +Metadata-Version: 2.1 +Name: pycurl +Version: 7.45.2 +Summary: PycURL -- A Python Interface To The cURL library +Home-page: http://pycurl.io/ +Author: Kjetil Jacobsen, Markus F.X.J. Oberhumer, Oleg Pudeyev +Author-email: kjetilja@gmail.com, markus@oberhumer.com, oleg@bsdpower.com +Maintainer: Oleg Pudeyev +Maintainer-email: oleg@bsdpower.com +License: LGPL/MIT +Keywords: curl,libcurl,urllib,wget,download,file transfer,http,www +Platform: All +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Web Environment +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Topic :: Internet :: File Transfer Protocol (FTP) +Classifier: Topic :: Internet :: WWW/HTTP +Requires-Python: >=3.5 +License-File: COPYING-LGPL +License-File: COPYING-MIT +License-File: AUTHORS + +PycURL -- A Python Interface To The cURL library +================================================ + +PycURL is a Python interface to `libcurl`_, the multiprotocol file +transfer library. Similarly to the urllib_ Python module, +PycURL can be used to fetch objects identified by a URL from a Python program. +Beyond simple fetches however PycURL exposes most of the functionality of +libcurl, including: + +- Speed - libcurl is very fast and PycURL, being a thin wrapper above + libcurl, is very fast as well. PycURL `was benchmarked`_ to be several + times faster than requests_. +- Features including multiple protocol support, SSL, authentication and + proxy options. PycURL supports most of libcurl's callbacks. +- Multi_ and share_ interfaces. +- Sockets used for network operations, permitting integration of PycURL + into the application's I/O loop (e.g., using Tornado_). + +.. _was benchmarked: http://stackoverflow.com/questions/15461995/python-requests-vs-pycurl-performance +.. _requests: http://python-requests.org/ +.. _Multi: https://curl.haxx.se/libcurl/c/libcurl-multi.html +.. _share: https://curl.haxx.se/libcurl/c/libcurl-share.html +.. _Tornado: http://www.tornadoweb.org/ + + +Requirements +------------ + +- Python 3.5-3.10. +- libcurl 7.19.0 or better. + + +Installation +------------ + +Download the source distribution from `PyPI`_. + +Please see `the installation documentation`_ for installation instructions. + +.. _PyPI: https://pypi.python.org/pypi/pycurl +.. _the installation documentation: http://pycurl.io/docs/latest/install.html + + +Documentation +------------- + +Documentation for the most recent PycURL release is available on +`PycURL website `_. + + +Support +------- + +For support questions please use `curl-and-python mailing list`_. +`Mailing list archives`_ are available for your perusal as well. + +Although not an official support venue, `Stack Overflow`_ has been +popular with some PycURL users. + +Bugs can be reported `via GitHub`_. Please use GitHub only for bug +reports and direct questions to our mailing list instead. + +.. _curl-and-python mailing list: http://cool.haxx.se/mailman/listinfo/curl-and-python +.. _Stack Overflow: http://stackoverflow.com/questions/tagged/pycurl +.. _Mailing list archives: https://curl.haxx.se/mail/list.cgi?list=curl-and-python +.. _via GitHub: https://github.com/pycurl/pycurl/issues + + +License +------- + +PycURL is dual licensed under the LGPL and an MIT/X derivative license +based on the libcurl license. The complete text of the licenses is available +in COPYING-LGPL_ and COPYING-MIT_ files in the source distribution. + +.. _libcurl: https://curl.haxx.se/libcurl/ +.. _urllib: http://docs.python.org/library/urllib.html +.. _COPYING-LGPL: https://raw.githubusercontent.com/pycurl/pycurl/master/COPYING-LGPL +.. _COPYING-MIT: https://raw.githubusercontent.com/pycurl/pycurl/master/COPYING-MIT diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..f474157 --- /dev/null +++ b/README.rst @@ -0,0 +1,190 @@ +PycURL -- A Python Interface To The cURL library +================================================ + +.. image:: https://github.com/pycurl/pycurl/workflows/CI/badge.svg + :target: https://github.com/pycurl/pycurl/actions + +PycURL is a Python interface to `libcurl`_, the multiprotocol file +transfer library. Similarly to the urllib_ Python module, +PycURL can be used to fetch objects identified by a URL from a Python program. +Beyond simple fetches however PycURL exposes most of the functionality of +libcurl, including: + +- Speed - libcurl is very fast and PycURL, being a thin wrapper above + libcurl, is very fast as well. PycURL `was benchmarked`_ to be several + times faster than requests_. +- Features including multiple protocol support, SSL, authentication and + proxy options. PycURL supports most of libcurl's callbacks. +- Multi_ and share_ interfaces. +- Sockets used for network operations, permitting integration of PycURL + into the application's I/O loop (e.g., using Tornado_). + +.. _was benchmarked: http://stackoverflow.com/questions/15461995/python-requests-vs-pycurl-performance +.. _requests: http://python-requests.org/ +.. _Multi: https://curl.haxx.se/libcurl/c/libcurl-multi.html +.. _share: https://curl.haxx.se/libcurl/c/libcurl-share.html +.. _Tornado: http://www.tornadoweb.org/ + + +Requirements +------------ + +- Python 3.5-3.10. +- libcurl 7.19.0 or better. + + +Installation +------------ + +Download source and binary distributions from `PyPI`_. +Binary wheels are now available for 32 and 64 bit Windows versions. + +Please see `INSTALL.rst`_ for installation instructions. If installing from +a Git checkout, please follow instruction in the `Git Checkout`_ section +of INSTALL.rst. + +.. _PyPI: https://pypi.python.org/pypi/pycurl +.. _INSTALL.rst: http://pycurl.io/docs/latest/install.html +.. _Git Checkout: http://pycurl.io/docs/latest/install.html#git-checkout + + +Documentation +------------- + +Documentation for the most recent PycURL release is available on +`PycURL website `_. + +Documentation for the development version of PycURL +is available `here `_. + +To build documentation from source, run ``make docs``. +Building documentation requires `Sphinx `_ to +be installed, as well as pycurl extension module built as docstrings are +extracted from it. Built documentation is stored in ``build/doc`` +subdirectory. + + +Support +------- + +For support questions please use `curl-and-python mailing list`_. +`Mailing list archives`_ are available for your perusal as well. + +Although not an official support venue, `Stack Overflow`_ has been +popular with some PycURL users. + +Bugs can be reported `via GitHub`_. Please use GitHub only for bug +reports and direct questions to our mailing list instead. + +.. _curl-and-python mailing list: http://cool.haxx.se/mailman/listinfo/curl-and-python +.. _Stack Overflow: http://stackoverflow.com/questions/tagged/pycurl +.. _Mailing list archives: https://curl.haxx.se/mail/list.cgi?list=curl-and-python +.. _via GitHub: https://github.com/pycurl/pycurl/issues + + +Automated Tests +--------------- + +PycURL comes with an automated test suite. To run the tests, execute:: + + make test + +The suite depends on packages `pytest`_ and `bottle`_, as well as `vsftpd`_. + +Some tests use vsftpd configured to accept anonymous uploads. These tests +are not run by default. As configured, vsftpd will allow reads and writes to +anything the user running the tests has read and write access. To run +vsftpd tests you must explicitly set PYCURL_VSFTPD_PATH variable like so:: + + # use vsftpd in PATH + export PYCURL_VSFTPD_PATH=vsftpd + + # specify full path to vsftpd + export PYCURL_VSFTPD_PATH=/usr/local/libexec/vsftpd + +.. _pytest: https://pytest.org/ +.. _bottle: http://bottlepy.org/ +.. _vsftpd: http://vsftpd.beasts.org/ + + +Test Matrix +----------- + +The test matrix is a separate framework that runs tests on more esoteric +configurations. It supports: + +- Testing against Python 2.4, which bottle does not support. +- Testing against Python compiled without threads, which requires an out of + process test server. +- Testing against locally compiled libcurl with arbitrary options. + +To use the test matrix, first start the test server from Python 2.5+ by +running:: + + python -m tests.appmanager + +Then in a different shell, and preferably in a separate user account, +run the test matrix:: + + # run ftp tests, etc. + export PYCURL_VSFTPD_PATH=vsftpd + # create a new work directory, preferably not under pycurl tree + mkdir testmatrix + cd testmatrix + # run the matrix specifying absolute path + python /path/to/pycurl/tests/matrix.py + +The test matrix will download, build and install supported Python versions +and supported libcurl versions, then run pycurl tests against each combination. +To see what the combinations are, look in +`tests/matrix.py `_. + + +Contribute +---------- + +For smaller changes: + +#. Fork `the repository`_ on Github. +#. Create a branch off **master**. +#. Make your changes. +#. Write a test which shows that the bug was fixed or that the feature + works as expected. +#. Send a pull request. +#. Check back after 10-15 minutes to see if tests passed on Travis CI. + PycURL supports old Python and libcurl releases and their support is tested + on Travis. + +For larger changes: + +#. Join the `mailing list`_. +#. Discuss your proposal on the mailing list. +#. When consensus is reached, implement it as described above. + +Please contribute binary distributions for your system to the +`downloads repository`_. + + +License +------- + +:: + + Copyright (C) 2001-2008 by Kjetil Jacobsen + Copyright (C) 2001-2008 by Markus F.X.J. Oberhumer + Copyright (C) 2013-2022 by Oleg Pudeyev + + All rights reserved. + + PycURL is dual licensed under the LGPL and an MIT/X derivative license + based on the cURL license. A full copy of the LGPL license is included + in the file COPYING-LGPL. A full copy of the MIT/X derivative license is + included in the file COPYING-MIT. You can redistribute and/or modify PycURL + according to the terms of either license. + +.. _PycURL: http://pycurl.io/ +.. _libcurl: https://curl.haxx.se/libcurl/ +.. _urllib: http://docs.python.org/library/urllib.html +.. _`the repository`: https://github.com/pycurl/pycurl +.. _`mailing list`: http://cool.haxx.se/mailman/listinfo/curl-and-python +.. _`downloads repository`: https://github.com/pycurl/downloads diff --git a/RELEASE-NOTES.rst b/RELEASE-NOTES.rst new file mode 100644 index 0000000..e364d63 --- /dev/null +++ b/RELEASE-NOTES.rst @@ -0,0 +1,237 @@ +Release Notes +============= + +PycURL 7.45.2 - 2022-12-16 +-------------------------- + +This release fixes several minor issues and adds support for several libcurl +options. + +PycURL 7.45.1 - 2022-03-13 +-------------------------- + +This release fixes build when libcurl < 7.64.1 is used. + +PycURL 7.45.0 - 2022-03-09 +-------------------------- + +This release adds support for SecureTransport SSL backend (MacOS), adds +ability to unset a number of multi options, adds ability to duplicate easy +handles and permits pycurl classes to be subclassed. + +PycURL 7.44.1 - 2021-08-15 +-------------------------- + +This release repairs incorrect Python thread initialization logic which +caused operations to hang. + +PycURL 7.44.0 - 2021-08-08 +-------------------------- + +This release reinstates best effort Python 2 support, adds Python 3.9 and +Python 3.10 alpha support and implements support for several libcurl options. + +Official Windows builds are currently not being produced. + +PycURL 7.43.0.6 - 2020-09-02 +---------------------------- + +This release improves SSL backend detection on various systems, adds support +for libcurl's multiple SSL backend functionality and adds support for several +libcurl options. + +PycURL 7.43.0.5 - 2020-01-29 +---------------------------- + +This release fixes a build issue on recent Pythons on CentOS/RHEL distributions. + +It also brings back Windows binaries. Special thank you to Gisle Vanem for +contributing the nghttp2 makefile. + + +PycURL 7.43.0.4 - 2020-01-15 +---------------------------- + +This release improves compatibility with Python 3.8 and removes support for +Python 2 and Python 3.4. It also adds wolfSSL support and thread safety of +the multi interface. + + +PycURL 7.43.0.3 - 2019-06-17 +---------------------------- + +This release primarily fixes an OpenSSL-related installation issue, and +repairs the ability to use PycURL with newer libcurls compiled without FTP +support. Also, mbedTLS support has been contributed by Josef Schlehofer. + + +PycURL 7.43.0.2 - 2018-06-02 +---------------------------- + +Highlights of this release: + +1. Experimental perform_rs and perform_rb methods have been added to Curl + objects. They return response body as a string and a byte string, + respectively. The goal of these methods is to improve PycURL's usability + for typical use cases, specifically removing the need to set up + StringIO/BytesIO objects to store the response body. + +2. getinfo_raw and errstr_raw methods have been added to Curl objects to + return transfer information as byte strings, permitting applications to + retrieve transfer information that is not decodable using Python's + default encoding. + +3. errstr and "fail or error" exceptions now replace undecodable bytes + so as to provide usable strings; use errstr_raw to retrieve original + byte strings. + +4. There is no longer a need to keep references to Curl objects when they + are used in CurlMulti objects - PycURL now maintains such references + internally. + +5. Official Windows builds now include HTTP/2 and international domain + name support. + +6. PycURL now officially supports BoringSSL. + +7. A number of smaller improvements have been made and bugs fixed. + + +PycURL 7.43.0.1 - 2017-12-07 +---------------------------- + +This release collects fixes and improvements made over the past two years, +notably updating Windows dependencies to address DNS resolution and +TLS connection issues. + + +PycURL 7.43.0 - 2016-02-02 +-------------------------- + +Highlights of this release: + +1. Binary wheels are now built for Windows systems. + +2. setopt_string method added to Curl objects to permit setting string libcurl + options that PycURL does not know about. + +3. curl module can now be imported on Windows again. + +4. OPENSOCKETFUNCTION callback is now invoked with the address as bytes on + Python 3 as was documented. + +5. Support for many libcurl options and constants was added. + + +PycURL 7.21.5 - 2016-01-05 +-------------------------- + +Highlights of this release: + +1. Socket callbacks are now fully implemented (``CURLOPT_OPENSOCKETFUNCTION``, + ``CURLOPT_SOCKOPTFUNCTION``, ``CURLOPT_CLOSESOCKETFUNCTION``). Unfortunately + this required changing ``OPENSOCKETFUNCTION`` API once again in a + backwards-incompatible manner. Support for ``SOCKOPTFUNCTION`` and + ``CLOSESOCKETFUNCTION`` was added in this release. ``OPENSOCKETFUNCTION`` + now supports Unix sockets. + +2. Many other libcurl options and constants have been added to PycURL. + +3. When ``pycurl`` module initialization fails, ``ImportError`` is raised + instead of a fatal error terminating the process. + +4. Usability of official Windows builds has been greatly improved: + + * Dependencies are linked statically, eliminating possible DLL conflicts. + * OpenSSL is used instead of WinSSL. + * libcurl is linked against C-Ares and libssh2. + + +PycURL 7.19.5.3 - 2015-11-03 +---------------------------- + +PycURL 7.19.5.2 release did not include some of the test suite files in +its manifest, leading to inability to run the test suite from the sdist +tarball. This is now fixed thanks to Kamil Dudka. + + +PycURL 7.19.5.2 - 2015-11-02 +---------------------------- + +Breaking change: DEBUGFUNCTION now takes bytes rather than (Unicode) string +as its argument on Python 3. + +Breaking change: CURLMOPT_* option constants moved from Easy to Multi +class. They remain available in pycurl module. + +SSL library detection improved again, --libcurl-dll option to setup.py added. + +Options that required tuples now also accept lists, and vice versa. + +This release fixes several memory leaks and one use after free issue. + +Support for several new libcurl options and constants has been added. + + +PycURL 7.19.5.1 - 2015-01-06 +---------------------------- + +This release primarily fixes build breakage against libcurl 7.19.4 through +7.21.1, such as versions shipped with CentOS. + + +PycURL 7.19.5 - 2014-07-12 +-------------------------- + +PycURL C code has been significantly reorganized. Curl, CurlMulti and +CurlShare classes are now properly exported, instead of factory functions for +the respective objects. PycURL API has not changed. + +Documentation has been transitioned to Sphinx and reorganized as well. +Both docstrings and standalone documentation are now more informative. + +Documentation is no longer included in released distributions. It can be +generated from source by running `make docs`. + +Tests are no longer included in released distributions. Instead the +documentation and quickstart examples should be consulted for sample code. + +Official Windows builds now are linked against zlib. + + +PycURL 7.19.3.1 - 2014-02-05 +---------------------------- + +This release restores PycURL's ability to automatically detect SSL library +in use in most circumstances, thanks to Andjelko Horvat. + + +PycURL 7.19.3 - 2014-01-09 +-------------------------- + +This release brings official Python 3 support to PycURL. +Several GNU/Linux distributions provided Python 3 packages of PycURL +previously; these packages were based on patches that were incomplete and +in some places incorrect. Behavior of PycURL 7.19.3 and later may therefore +differ from behavior of unofficial Python 3 packages of previous PycURL +versions. + +To summarize the behavior under Python 3, PycURL will accept ``bytes`` where +it accepted strings under Python 2, and will also accept Unicode strings +containing ASCII codepoints only for convenience. Please refer to +`Unicode`_ and `file`_ documentation for further details. + +In the interests of compatibility, PycURL will also accept Unicode data on +Python 2 given the same constraints as under Python 3. + +While Unicode and file handling rules are expected to be sensible for +all use cases, and retain backwards compatibility with previous PycURL +versions, please treat behavior of this versions under Python 3 as experimental +and subject to change. + +Another potentially disruptive change in PycURL is the requirement for +compile time and runtime SSL backends to match. Please see the readme for +how to indicate the SSL backend to setup.py. + +.. _Unicode: doc/unicode.html +.. _file: doc/files.html diff --git a/doc/callbacks.rst b/doc/callbacks.rst new file mode 100644 index 0000000..afe2594 --- /dev/null +++ b/doc/callbacks.rst @@ -0,0 +1,447 @@ +.. _callbacks: + +Callbacks +========= + +For more fine-grained control, libcurl allows a number of callbacks to be +associated with each connection. In pycurl, callbacks are defined using the +``setopt()`` method for Curl objects with options ``WRITEFUNCTION``, +``READFUNCTION``, ``HEADERFUNCTION``, ``PROGRESSFUNCTION``, +``XFERINFOFUNCTION``, ``IOCTLFUNCTION``, or +``DEBUGFUNCTION``. These options correspond to the libcurl options with ``CURLOPT_`` +prefix removed. A callback in pycurl must be either a regular Python +function, a class method or an extension type function. + +There are some limitations to some of the options which can be used +concurrently with the pycurl callbacks compared to the libcurl callbacks. +This is to allow different callback functions to be associated with different +Curl objects. More specifically, ``WRITEDATA`` cannot be used with +``WRITEFUNCTION``, ``READDATA`` cannot be used with ``READFUNCTION``, +``WRITEHEADER`` cannot be used with ``HEADERFUNCTION``. +In practice, these limitations can be overcome by having a +callback function be a class instance method and rather use the class +instance attributes to store per object data such as files used in the +callbacks. + +The signature of each callback used in PycURL is documented below. + + +Error Reporting +--------------- + +PycURL callbacks are invoked as follows: + +Python application -> ``perform()`` -> libcurl (C code) -> Python callback + +Because callbacks are invoked by libcurl, they should not raise exceptions +on failure but instead return appropriate values indicating failure. +The documentation for individual callbacks below specifies expected success and +failure return values. + +Unhandled exceptions propagated out of Python callbacks will be intercepted +by PycURL or the Python runtime. This will fail the callback with a +generic failure status, in turn failing the ``perform()`` operation. +A failing ``perform()`` will raise ``pycurl.error``, but the error code +used depends on the specific callback. + +Rich context information like exception objects can be stored in various ways, +for example the following example stores OPENSOCKET callback exception on the +Curl object:: + + import pycurl, random, socket + + class ConnectionRejected(Exception): + pass + + def opensocket(curl, purpose, curl_address): + # always fail + curl.exception = ConnectionRejected('Rejecting connection attempt in opensocket callback') + return pycurl.SOCKET_BAD + + # the callback must create a socket if it does not fail, + # see examples/opensocketexception.py + + c = pycurl.Curl() + c.setopt(c.URL, 'http://pycurl.io') + c.exception = None + c.setopt(c.OPENSOCKETFUNCTION, + lambda purpose, address: opensocket(c, purpose, address)) + + try: + c.perform() + except pycurl.error as e: + if e.args[0] == pycurl.E_COULDNT_CONNECT and c.exception: + print(c.exception) + else: + print(e) + + +WRITEFUNCTION +------------- + +.. function:: WRITEFUNCTION(byte string) -> number of characters written + + Callback for writing data. Corresponds to `CURLOPT_WRITEFUNCTION`_ + in libcurl. + + On Python 3, the argument is of type ``bytes``. + + The ``WRITEFUNCTION`` callback may return the number of bytes written. + If this number is not equal to the size of the byte string, this signifies + an error and libcurl will abort the request. Returning ``None`` is an + alternate way of indicating that the callback has consumed all of the + string passed to it and, hence, succeeded. + + `write_test.py test`_ shows how to use ``WRITEFUNCTION``. + + +Example: Callbacks for document header and body +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This example prints the header data to stderr and the body data to stdout. +Also note that neither callback returns the number of bytes written. For +WRITEFUNCTION and HEADERFUNCTION callbacks, returning None implies that all +bytes where written. + +:: + + ## Callback function invoked when body data is ready + def body(buf): + # Print body data to stdout + import sys + sys.stdout.write(buf) + # Returning None implies that all bytes were written + + ## Callback function invoked when header data is ready + def header(buf): + # Print header data to stderr + import sys + sys.stderr.write(buf) + # Returning None implies that all bytes were written + + c = pycurl.Curl() + c.setopt(pycurl.URL, "http://www.python.org/") + c.setopt(pycurl.WRITEFUNCTION, body) + c.setopt(pycurl.HEADERFUNCTION, header) + c.perform() + + +HEADERFUNCTION +-------------- + +.. function:: HEADERFUNCTION(byte string) -> number of characters written + + Callback for writing received headers. Corresponds to + `CURLOPT_HEADERFUNCTION`_ in libcurl. + + On Python 3, the argument is of type ``bytes``. + + The ``HEADERFUNCTION`` callback may return the number of bytes written. + If this number is not equal to the size of the byte string, this signifies + an error and libcurl will abort the request. Returning ``None`` is an + alternate way of indicating that the callback has consumed all of the + string passed to it and, hence, succeeded. + + `header_test.py test`_ shows how to use ``WRITEFUNCTION``. + + +READFUNCTION +------------ + +.. function:: READFUNCTION(number of characters to read) -> byte string + + Callback for reading data. Corresponds to `CURLOPT_READFUNCTION`_ in + libcurl. + + On Python 3, the callback must return either a byte string or a Unicode + string consisting of ASCII code points only. + + In addition, ``READFUNCTION`` may return ``READFUNC_ABORT`` or + ``READFUNC_PAUSE``. See the libcurl documentation for an explanation + of these values. + + The `file_upload.py example`_ in the distribution contains example code for + using ``READFUNCTION``. + + +.. _SEEKFUNCTION: + +SEEKFUNCTION +------------ + +.. function:: SEEKFUNCTION(offset, origin) -> status + + Callback for seek operations. Corresponds to `CURLOPT_SEEKFUNCTION`_ + in libcurl. + + +IOCTLFUNCTION +------------- + +.. function:: IOCTLFUNCTION(ioctl cmd) -> status + + Callback for I/O operations. Corresponds to `CURLOPT_IOCTLFUNCTION`_ + in libcurl. + + *Note:* this callback is deprecated. Use :ref:`SEEKFUNCTION ` instead. + + +DEBUGFUNCTION +------------- + +.. function:: DEBUGFUNCTION(debug message type, debug message byte string) -> None + + Callback for debug information. Corresponds to `CURLOPT_DEBUGFUNCTION`_ + in libcurl. + + *Changed in version 7.19.5.2:* The second argument to a ``DEBUGFUNCTION`` + callback is now of type ``bytes`` on Python 3. Previously the argument was + of type ``str``. + + `debug_test.py test`_ shows how to use ``DEBUGFUNCTION``. + + +Example: Debug callbacks +~~~~~~~~~~~~~~~~~~~~~~~~ + +This example shows how to use the debug callback. The debug message type is +an integer indicating the type of debug message. The VERBOSE option must be +enabled for this callback to be invoked. + +:: + + def test(debug_type, debug_msg): + print "debug(%d): %s" % (debug_type, debug_msg) + + c = pycurl.Curl() + c.setopt(pycurl.URL, "https://curl.haxx.se/") + c.setopt(pycurl.VERBOSE, 1) + c.setopt(pycurl.DEBUGFUNCTION, test) + c.perform() + + +PROGRESSFUNCTION +---------------- + +.. function:: PROGRESSFUNCTION(download total, downloaded, upload total, uploaded) -> status + + Callback for progress meter. Corresponds to `CURLOPT_PROGRESSFUNCTION`_ + in libcurl. + + ``PROGRESSFUNCTION`` receives amounts as floating point arguments to the + callback. Since libcurl 7.32.0 ``PROGRESSFUNCTION`` is deprecated; + ``XFERINFOFUNCTION`` should be used instead which receives amounts as + long integers. + + ``NOPROGRESS`` option must be set for False libcurl to invoke a + progress callback, as PycURL by default sets ``NOPROGRESS`` to True. + + +XFERINFOFUNCTION +---------------- + +.. function:: XFERINFOFUNCTION(download total, downloaded, upload total, uploaded) -> status + + Callback for progress meter. Corresponds to `CURLOPT_XFERINFOFUNCTION`_ + in libcurl. + + ``XFERINFOFUNCTION`` receives amounts as long integers. + + ``NOPROGRESS`` option must be set for False libcurl to invoke a + progress callback, as PycURL by default sets ``NOPROGRESS`` to True. + + +Example: Download/upload progress callback +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This example shows how to use the progress callback. When downloading a +document, the arguments related to uploads are zero, and vice versa. + +:: + + ## Callback function invoked when download/upload has progress + def progress(download_t, download_d, upload_t, upload_d): + print "Total to download", download_t + print "Total downloaded", download_d + print "Total to upload", upload_t + print "Total uploaded", upload_d + + c = pycurl.Curl() + c.setopt(c.URL, "http://slashdot.org/") + c.setopt(c.NOPROGRESS, False) + c.setopt(c.XFERINFOFUNCTION, progress) + c.perform() + + +OPENSOCKETFUNCTION +------------------ + +.. function:: OPENSOCKETFUNCTION(purpose, address) -> int + + Callback for opening sockets. Corresponds to + `CURLOPT_OPENSOCKETFUNCTION`_ in libcurl. + + *purpose* is a ``SOCKTYPE_*`` value. + + *address* is a `namedtuple`_ with ``family``, ``socktype``, ``protocol`` + and ``addr`` fields, per `CURLOPT_OPENSOCKETFUNCTION`_ documentation. + + *addr* is an object representing the address. Currently the following + address families are supported: + + - ``AF_INET``: *addr* is a 2-tuple of ``(host, port)``. + - ``AF_INET6``: *addr* is a 4-tuple of ``(host, port, flow info, scope id)``. + - ``AF_UNIX``: *addr* is a byte string containing path to the Unix socket. + + Availability: Unix. + + This behavior matches that of Python's `socket module`_. + + The callback should return a socket object, a socket file descriptor + or a Python object with a ``fileno`` property containing the socket + file descriptor. + + The callback may be unset by calling :ref:`setopt ` with ``None`` + as the value or by calling :ref:`unsetopt `. + + `open_socket_cb_test.py test`_ shows how to use ``OPENSOCKETFUNCTION``. + + *Changed in version 7.21.5:* Previously, the callback received ``family``, + ``socktype``, ``protocol`` and ``addr`` parameters (``purpose`` was + not passed and ``address`` was flattened). Also, ``AF_INET6`` addresses + were exposed as 2-tuples of ``(host, port)`` rather than 4-tuples. + + *Changed in version 7.19.3:* ``addr`` parameter added to the callback. + + +CLOSESOCKETFUNCTION +------------------- + +.. function:: CLOSESOCKETFUNCTION(curlfd) -> int + + Callback for setting socket options. Corresponds to + `CURLOPT_CLOSESOCKETFUNCTION`_ in libcurl. + + *curlfd* is the file descriptor to be closed. + + The callback should return an ``int``. + + The callback may be unset by calling :ref:`setopt ` with ``None`` + as the value or by calling :ref:`unsetopt `. + + `close_socket_cb_test.py test`_ shows how to use ``CLOSESOCKETFUNCTION``. + + +SOCKOPTFUNCTION +--------------- + +.. function:: SOCKOPTFUNCTION(curlfd, purpose) -> int + + Callback for setting socket options. Corresponds to `CURLOPT_SOCKOPTFUNCTION`_ + in libcurl. + + *curlfd* is the file descriptor of the newly created socket. + + *purpose* is a ``SOCKTYPE_*`` value. + + The callback should return an ``int``. + + The callback may be unset by calling :ref:`setopt ` with ``None`` + as the value or by calling :ref:`unsetopt `. + + `sockopt_cb_test.py test`_ shows how to use ``SOCKOPTFUNCTION``. + + +SSH_KEYFUNCTION +--------------- + +.. function:: SSH_KEYFUNCTION(known_key, found_key, match) -> int + + Callback for known host matching logic. Corresponds to + `CURLOPT_SSH_KEYFUNCTION`_ in libcurl. + + *known_key* and *found_key* are instances of ``KhKey`` class which is a + `namedtuple`_ with ``key`` and ``keytype`` fields, corresponding to + libcurl's ``struct curl_khkey``:: + + KhKey = namedtuple('KhKey', ('key', 'keytype')) + + On Python 2, the *key* field of ``KhKey`` is a ``str``. On Python 3, the + *key* field is ``bytes``. *keytype* is an ``int``. + + *known_key* may be ``None`` when there is no known matching host key. + + ``SSH_KEYFUNCTION`` callback should return a ``KHSTAT_*`` value. + + The callback may be unset by calling :ref:`setopt ` with ``None`` + as the value or by calling :ref:`unsetopt `. + + `ssh_key_cb_test.py test`_ shows how to use ``SSH_KEYFUNCTION``. + + +TIMERFUNCTION +------------- + +.. function:: TIMERFUNCTION(timeout_ms) -> None + + Callback for installing a timer requested by libcurl. Corresponds to + `CURLMOPT_TIMERFUNCTION`_. + + The application should arrange for a non-repeating timer to fire in + ``timeout_ms`` milliseconds, at which point the application should call + either :ref:`socket_action ` or + :ref:`perform `. + + See ``examples/multi-socket_action-select.py`` for an example program + that uses the timer function and the socket function. + + +SOCKETFUNCTION +-------------- + +.. function:: SOCKETFUNCTION(what, sock_fd, multi, socketp) -> None + + Callback notifying the application about activity on libcurl sockets. + Corresponds to `CURLMOPT_SOCKETFUNCTION`_. + + Note that the PycURL callback takes ``what`` as the first argument and + ``sock_fd`` as the second argument, whereas the libcurl callback takes + ``sock_fd`` as the first argument and ``what`` as the second argument. + + The ``userp`` ("private callback pointer") argument, as described in the + ``CURLMOPT_SOCKETFUNCTION`` documentation, is set to the ``CurlMulti`` + instance. + + The ``socketp`` ("private socket pointer") argument, as described in the + ``CURLMOPT_SOCKETFUNCTION`` documentation, is set to the value provided + to the :ref:`assign ` method for the corresponding + ``sock_fd``, or ``None`` if no value was assigned. + + See ``examples/multi-socket_action-select.py`` for an example program + that uses the timer function and the socket function. + + +.. _CURLOPT_HEADERFUNCTION: https://curl.haxx.se/libcurl/c/CURLOPT_HEADERFUNCTION.html +.. _CURLOPT_WRITEFUNCTION: https://curl.haxx.se/libcurl/c/CURLOPT_WRITEFUNCTION.html +.. _CURLOPT_READFUNCTION: https://curl.haxx.se/libcurl/c/CURLOPT_READFUNCTION.html +.. _CURLOPT_PROGRESSFUNCTION: https://curl.haxx.se/libcurl/c/CURLOPT_PROGRESSFUNCTION.html +.. _CURLOPT_XFERINFOFUNCTION: https://curl.haxx.se/libcurl/c/CURLOPT_XFERINFOFUNCTION.html +.. _CURLOPT_DEBUGFUNCTION: https://curl.haxx.se/libcurl/c/CURLOPT_DEBUGFUNCTION.html +.. _CURLOPT_SEEKFUNCTION: https://curl.haxx.se/libcurl/c/CURLOPT_SEEKFUNCTION.html +.. _CURLOPT_IOCTLFUNCTION: https://curl.haxx.se/libcurl/c/CURLOPT_IOCTLFUNCTION.html +.. _file_upload.py example: https://github.com/pycurl/pycurl/blob/master/examples/file_upload.py +.. _write_test.py test: https://github.com/pycurl/pycurl/blob/master/tests/write_test.py +.. _header_test.py test: https://github.com/pycurl/pycurl/blob/master/tests/header_test.py +.. _debug_test.py test: https://github.com/pycurl/pycurl/blob/master/tests/debug_test.py +.. _CURLOPT_SSH_KEYFUNCTION: https://curl.haxx.se/libcurl/c/CURLOPT_SSH_KEYFUNCTION.html +.. _namedtuple: https://docs.python.org/library/collections.html#collections.namedtuple +.. _CURLOPT_SOCKOPTFUNCTION: https://curl.haxx.se/libcurl/c/CURLOPT_SOCKOPTFUNCTION.html +.. _sockopt_cb_test.py test: https://github.com/pycurl/pycurl/blob/master/tests/sockopt_cb_test.py +.. _ssh_key_cb_test.py test: https://github.com/pycurl/pycurl/blob/master/tests/ssh_key_cb_test.py +.. _CURLOPT_CLOSESOCKETFUNCTION: https://curl.haxx.se/libcurl/c/CURLOPT_CLOSESOCKETFUNCTION.html +.. _close_socket_cb_test.py test: https://github.com/pycurl/pycurl/blob/master/tests/close_socket_cb_test.py +.. _CURLOPT_OPENSOCKETFUNCTION: https://curl.haxx.se/libcurl/c/CURLOPT_OPENSOCKETFUNCTION.html +.. _open_socket_cb_test.py test: https://github.com/pycurl/pycurl/blob/master/tests/open_socket_cb_test.py +.. _socket module: https://docs.python.org/library/socket.html +.. _CURLMOPT_TIMERFUNCTION: https://curl.se/libcurl/c/CURLMOPT_TIMERFUNCTION.html +.. _CURLMOPT_SOCKETFUNCTION: https://curl.se/libcurl/c/CURLMOPT_SOCKETFUNCTION.html diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000..9f98d71 --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +# +# PycURL documentation build configuration file, created by +# sphinx-quickstart on Tue Feb 4 03:14:18 2014. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.coverage', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'PycURL' +copyright = u'2001-2022 Kjetil Jacobsen, Markus F.X.J. Oberhumer, Oleg Pudeyev' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '7.45.2' +# The full version, including alpha/beta/rc tags. +release = '7.45.2' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['docstrings'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +html_favicon = 'favicon.ico' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'PycURLdoc' diff --git a/doc/curl.rst b/doc/curl.rst new file mode 100644 index 0000000..49b401d --- /dev/null +++ b/doc/curl.rst @@ -0,0 +1,10 @@ +curl Module Functionality +========================= + +.. automodule:: curl + +High Level Curl Object +---------------------- + +.. autoclass:: curl.Curl + :members: diff --git a/doc/curlmultiobject.rst b/doc/curlmultiobject.rst new file mode 100644 index 0000000..c83e9e3 --- /dev/null +++ b/doc/curlmultiobject.rst @@ -0,0 +1,36 @@ +.. _curlmultiobject: + +CurlMulti Object +================ + +.. autoclass:: pycurl.CurlMulti + + CurlMulti objects have the following methods: + + .. automethod:: pycurl.CurlMulti.close + + .. automethod:: pycurl.CurlMulti.add_handle + + .. automethod:: pycurl.CurlMulti.remove_handle + + .. _multi-perform: + .. automethod:: pycurl.CurlMulti.perform + + .. _multi-socket_action: + .. automethod:: pycurl.CurlMulti.socket_action + + .. _multi-socket_all: + .. automethod:: pycurl.CurlMulti.socket_all + + .. automethod:: pycurl.CurlMulti.setopt + + .. automethod:: pycurl.CurlMulti.fdset + + .. automethod:: pycurl.CurlMulti.select + + .. automethod:: pycurl.CurlMulti.info_read + + .. automethod:: pycurl.CurlMulti.timeout + + .. _multi-assign: + .. automethod:: pycurl.CurlMulti.assign diff --git a/doc/curlobject.rst b/doc/curlobject.rst new file mode 100644 index 0000000..54ca11e --- /dev/null +++ b/doc/curlobject.rst @@ -0,0 +1,43 @@ +.. _curlobject: + +Curl Object +=========== + +.. autoclass:: pycurl.Curl + + Curl objects have the following methods: + + .. automethod:: pycurl.Curl.close + + .. _setopt: + .. automethod:: pycurl.Curl.setopt + + .. _perform: + .. automethod:: pycurl.Curl.perform + + .. _perform_rb: + .. automethod:: pycurl.Curl.perform_rb + + .. _perform_rs: + .. automethod:: pycurl.Curl.perform_rs + + .. _getinfo: + .. automethod:: pycurl.Curl.getinfo + + .. _getinfo_raw: + .. automethod:: pycurl.Curl.getinfo_raw + + .. automethod:: pycurl.Curl.reset + + .. _unsetopt: + .. automethod:: pycurl.Curl.unsetopt + + .. automethod:: pycurl.Curl.pause + + .. _errstr: + .. automethod:: pycurl.Curl.errstr + + .. _errstr_raw: + .. automethod:: pycurl.Curl.errstr_raw + + .. automethod:: pycurl.Curl.setopt_string diff --git a/doc/curlshareobject.rst b/doc/curlshareobject.rst new file mode 100644 index 0000000..e40bc08 --- /dev/null +++ b/doc/curlshareobject.rst @@ -0,0 +1,12 @@ +.. _curlshareobject: + +CurlShare Object +================ + +.. autoclass:: pycurl.CurlShare + + CurlShare objects have the following methods: + + .. automethod:: pycurl.CurlShare.close + + .. automethod:: pycurl.CurlShare.setopt diff --git a/doc/docstrings/curl.rst b/doc/docstrings/curl.rst new file mode 100644 index 0000000..96e1be8 --- /dev/null +++ b/doc/docstrings/curl.rst @@ -0,0 +1,9 @@ +Curl() -> New Curl object + +Creates a new :ref:`curlobject` which corresponds to a +``CURL`` handle in libcurl. Curl objects automatically set +CURLOPT_VERBOSE to 0, CURLOPT_NOPROGRESS to 1, provide a default +CURLOPT_USERAGENT and setup CURLOPT_ERRORBUFFER to point to a +private error buffer. + +Implicitly calls :py:func:`pycurl.global_init` if the latter has not yet been called. diff --git a/doc/docstrings/curl_close.rst b/doc/docstrings/curl_close.rst new file mode 100644 index 0000000..54bf85b --- /dev/null +++ b/doc/docstrings/curl_close.rst @@ -0,0 +1,10 @@ +close() -> None + +Close handle and end curl session. + +Corresponds to `curl_easy_cleanup`_ in libcurl. This method is +automatically called by pycurl when a Curl object no longer has any +references to it, but can also be called explicitly. + +.. _curl_easy_cleanup: + https://curl.haxx.se/libcurl/c/curl_easy_cleanup.html diff --git a/doc/docstrings/curl_duphandle.rst b/doc/docstrings/curl_duphandle.rst new file mode 100644 index 0000000..ca6e7a8 --- /dev/null +++ b/doc/docstrings/curl_duphandle.rst @@ -0,0 +1,23 @@ +duphandle() -> Curl + +Clone a curl handle. This function will return a new curl handle, +a duplicate, using all the options previously set in the input curl handle. +Both handles can subsequently be used independently. + +The new handle will not inherit any state information, no connections, +no SSL sessions and no cookies. It also will not inherit any share object +states or options (it will be made as if SHARE was unset). + +Corresponds to `curl_easy_duphandle`_ in libcurl. + +Example usage:: + + import pycurl + curl = pycurl.Curl() + curl.setopt(pycurl.URL, "https://python.org") + dup = curl.duphandle() + curl.perform() + dup.perform() + +.. _curl_easy_duphandle: + https://curl.se/libcurl/c/curl_easy_duphandle.html diff --git a/doc/docstrings/curl_errstr.rst b/doc/docstrings/curl_errstr.rst new file mode 100644 index 0000000..198c1fc --- /dev/null +++ b/doc/docstrings/curl_errstr.rst @@ -0,0 +1,11 @@ +errstr() -> string + +Return the internal libcurl error buffer of this handle as a string. + +Return value is a ``str`` instance on all Python versions. +On Python 3, error buffer data is decoded using Python's default encoding +at the time of the call. If this decoding fails, ``UnicodeDecodeError`` is +raised. Use :ref:`errstr_raw ` to retrieve the error buffer +as a byte string in this case. + +On Python 2, ``errstr`` and ``errstr_raw`` behave identically. diff --git a/doc/docstrings/curl_errstr_raw.rst b/doc/docstrings/curl_errstr_raw.rst new file mode 100644 index 0000000..864734e --- /dev/null +++ b/doc/docstrings/curl_errstr_raw.rst @@ -0,0 +1,12 @@ +errstr_raw() -> byte string + +Return the internal libcurl error buffer of this handle as a byte string. + +Return value is a ``str`` instance on Python 2 and ``bytes`` instance +on Python 3. Unlike :ref:`errstr_raw `, ``errstr_raw`` +allows reading libcurl error buffer in Python 3 when its contents is not +valid in Python's default encoding. + +On Python 2, ``errstr`` and ``errstr_raw`` behave identically. + +*Added in version 7.43.0.2.* diff --git a/doc/docstrings/curl_getinfo.rst b/doc/docstrings/curl_getinfo.rst new file mode 100644 index 0000000..984d782 --- /dev/null +++ b/doc/docstrings/curl_getinfo.rst @@ -0,0 +1,74 @@ +getinfo(option) -> Result + +Extract and return information from a curl session, +decoding string data in Python's default encoding at the time of the call. +Corresponds to `curl_easy_getinfo`_ in libcurl. +The ``getinfo`` method should not be called unless +``perform`` has been called and finished. + +*option* is a constant corresponding to one of the +``CURLINFO_*`` constants in libcurl. Most option constant names match +the respective ``CURLINFO_*`` constant names with the ``CURLINFO_`` prefix +removed, for example ``CURLINFO_CONTENT_TYPE`` is accessible as +``pycurl.CONTENT_TYPE``. Exceptions to this rule are as follows: + +- ``CURLINFO_FILETIME`` is mapped as ``pycurl.INFO_FILETIME`` +- ``CURLINFO_COOKIELIST`` is mapped as ``pycurl.INFO_COOKIELIST`` +- ``CURLINFO_CERTINFO`` is mapped as ``pycurl.INFO_CERTINFO`` +- ``CURLINFO_RTSP_CLIENT_CSEQ`` is mapped as ``pycurl.INFO_RTSP_CLIENT_CSEQ`` +- ``CURLINFO_RTSP_CSEQ_RECV`` is mapped as ``pycurl.INFO_RTSP_CSEQ_RECV`` +- ``CURLINFO_RTSP_SERVER_CSEQ`` is mapped as ``pycurl.INFO_RTSP_SERVER_CSEQ`` +- ``CURLINFO_RTSP_SESSION_ID`` is mapped as ``pycurl.INFO_RTSP_SESSION_ID`` + +The type of return value depends on the option, as follows: + +- Options documented by libcurl to return an integer value return a + Python integer (``long`` on Python 2, ``int`` on Python 3). +- Options documented by libcurl to return a floating point value + return a Python ``float``. +- Options documented by libcurl to return a string value + return a Python string (``str`` on Python 2 and Python 3). + On Python 2, the string contains whatever data libcurl returned. + On Python 3, the data returned by libcurl is decoded using the + default string encoding at the time of the call. + If the data cannot be decoded using the default encoding, ``UnicodeDecodeError`` + is raised. Use :ref:`getinfo_raw ` + to retrieve the data as ``bytes`` in these + cases. +- ``SSL_ENGINES`` and ``INFO_COOKIELIST`` return a list of strings. + The same encoding caveats apply; use :ref:`getinfo_raw ` + to retrieve the + data as a list of byte strings. +- ``INFO_CERTINFO`` returns a list with one element + per certificate in the chain, starting with the leaf; each element is a + sequence of *(key, value)* tuples where both ``key`` and ``value`` are + strings. String encoding caveats apply; use :ref:`getinfo_raw ` + to retrieve + certificate data as byte strings. + +On Python 2, ``getinfo`` and ``getinfo_raw`` behave identically. + +Example usage:: + + import pycurl + c = pycurl.Curl() + c.setopt(pycurl.OPT_CERTINFO, 1) + c.setopt(pycurl.URL, "https://python.org") + c.setopt(pycurl.FOLLOWLOCATION, 1) + c.perform() + print(c.getinfo(pycurl.HTTP_CODE)) + # --> 200 + print(c.getinfo(pycurl.EFFECTIVE_URL)) + # --> "https://www.python.org/" + certinfo = c.getinfo(pycurl.INFO_CERTINFO) + print(certinfo) + # --> [(('Subject', 'C = AU, ST = Some-State, O = PycURL test suite, + CN = localhost'), ('Issuer', 'C = AU, ST = Some-State, + O = PycURL test suite, OU = localhost, CN = localhost'), + ('Version', '0'), ...)] + + +Raises pycurl.error exception upon failure. + +.. _curl_easy_getinfo: + https://curl.haxx.se/libcurl/c/curl_easy_getinfo.html diff --git a/doc/docstrings/curl_getinfo_raw.rst b/doc/docstrings/curl_getinfo_raw.rst new file mode 100644 index 0000000..bca661d --- /dev/null +++ b/doc/docstrings/curl_getinfo_raw.rst @@ -0,0 +1,70 @@ +getinfo_raw(option) -> Result + +Extract and return information from a curl session, +returning string data as byte strings. +Corresponds to `curl_easy_getinfo`_ in libcurl. +The ``getinfo_raw`` method should not be called unless +``perform`` has been called and finished. + +*option* is a constant corresponding to one of the +``CURLINFO_*`` constants in libcurl. Most option constant names match +the respective ``CURLINFO_*`` constant names with the ``CURLINFO_`` prefix +removed, for example ``CURLINFO_CONTENT_TYPE`` is accessible as +``pycurl.CONTENT_TYPE``. Exceptions to this rule are as follows: + +- ``CURLINFO_FILETIME`` is mapped as ``pycurl.INFO_FILETIME`` +- ``CURLINFO_COOKIELIST`` is mapped as ``pycurl.INFO_COOKIELIST`` +- ``CURLINFO_CERTINFO`` is mapped as ``pycurl.INFO_CERTINFO`` +- ``CURLINFO_RTSP_CLIENT_CSEQ`` is mapped as ``pycurl.INFO_RTSP_CLIENT_CSEQ`` +- ``CURLINFO_RTSP_CSEQ_RECV`` is mapped as ``pycurl.INFO_RTSP_CSEQ_RECV`` +- ``CURLINFO_RTSP_SERVER_CSEQ`` is mapped as ``pycurl.INFO_RTSP_SERVER_CSEQ`` +- ``CURLINFO_RTSP_SESSION_ID`` is mapped as ``pycurl.INFO_RTSP_SESSION_ID`` + +The type of return value depends on the option, as follows: + +- Options documented by libcurl to return an integer value return a + Python integer (``long`` on Python 2, ``int`` on Python 3). +- Options documented by libcurl to return a floating point value + return a Python ``float``. +- Options documented by libcurl to return a string value + return a Python byte string (``str`` on Python 2, ``bytes`` on Python 3). + The string contains whatever data libcurl returned. + Use :ref:`getinfo ` to retrieve this data as a Unicode string on Python 3. +- ``SSL_ENGINES`` and ``INFO_COOKIELIST`` return a list of byte strings. + The same encoding caveats apply; use :ref:`getinfo ` to retrieve the + data as a list of potentially Unicode strings. +- ``INFO_CERTINFO`` returns a list with one element + per certificate in the chain, starting with the leaf; each element is a + sequence of *(key, value)* tuples where both ``key`` and ``value`` are + byte strings. String encoding caveats apply; use :ref:`getinfo ` + to retrieve + certificate data as potentially Unicode strings. + +On Python 2, ``getinfo`` and ``getinfo_raw`` behave identically. + +Example usage:: + + import pycurl + c = pycurl.Curl() + c.setopt(pycurl.OPT_CERTINFO, 1) + c.setopt(pycurl.URL, "https://python.org") + c.setopt(pycurl.FOLLOWLOCATION, 1) + c.perform() + print(c.getinfo_raw(pycurl.HTTP_CODE)) + # --> 200 + print(c.getinfo_raw(pycurl.EFFECTIVE_URL)) + # --> b"https://www.python.org/" + certinfo = c.getinfo_raw(pycurl.INFO_CERTINFO) + print(certinfo) + # --> [((b'Subject', b'C = AU, ST = Some-State, O = PycURL test suite, + CN = localhost'), (b'Issuer', b'C = AU, ST = Some-State, + O = PycURL test suite, OU = localhost, CN = localhost'), + (b'Version', b'0'), ...)] + + +Raises pycurl.error exception upon failure. + +*Added in version 7.43.0.2.* + +.. _curl_easy_getinfo: + https://curl.haxx.se/libcurl/c/curl_easy_getinfo.html diff --git a/doc/docstrings/curl_pause.rst b/doc/docstrings/curl_pause.rst new file mode 100644 index 0000000..8fd16bc --- /dev/null +++ b/doc/docstrings/curl_pause.rst @@ -0,0 +1,12 @@ +pause(bitmask) -> None + +Pause or unpause a curl handle. Bitmask should be a value such as +PAUSE_RECV or PAUSE_CONT. + +Corresponds to `curl_easy_pause`_ in libcurl. The argument should be +derived from the ``PAUSE_RECV``, ``PAUSE_SEND``, ``PAUSE_ALL`` and +``PAUSE_CONT`` constants. + +Raises pycurl.error exception upon failure. + +.. _curl_easy_pause: https://curl.haxx.se/libcurl/c/curl_easy_pause.html diff --git a/doc/docstrings/curl_perform.rst b/doc/docstrings/curl_perform.rst new file mode 100644 index 0000000..ab5439d --- /dev/null +++ b/doc/docstrings/curl_perform.rst @@ -0,0 +1,10 @@ +perform() -> None + +Perform a file transfer. + +Corresponds to `curl_easy_perform`_ in libcurl. + +Raises pycurl.error exception upon failure. + +.. _curl_easy_perform: + https://curl.haxx.se/libcurl/c/curl_easy_perform.html diff --git a/doc/docstrings/curl_perform_rb.rst b/doc/docstrings/curl_perform_rb.rst new file mode 100644 index 0000000..7ecd0f9 --- /dev/null +++ b/doc/docstrings/curl_perform_rb.rst @@ -0,0 +1,17 @@ +perform_rb() -> response_body + +Perform a file transfer and return response body as a byte string. + +This method arranges for response body to be saved in a StringIO +(Python 2) or BytesIO (Python 3) instance, then invokes :ref:`perform ` +to perform the file transfer, then returns the value of the StringIO/BytesIO +instance which is a ``str`` instance on Python 2 and ``bytes`` instance +on Python 3. Errors during transfer raise ``pycurl.error`` exceptions +just like in :ref:`perform `. + +Use :ref:`perform_rs ` to retrieve response body as a string +(``str`` instance on both Python 2 and 3). + +Raises ``pycurl.error`` exception upon failure. + +*Added in version 7.43.0.2.* diff --git a/doc/docstrings/curl_perform_rs.rst b/doc/docstrings/curl_perform_rs.rst new file mode 100644 index 0000000..3ebeaa5 --- /dev/null +++ b/doc/docstrings/curl_perform_rs.rst @@ -0,0 +1,25 @@ +perform_rs() -> response_body + +Perform a file transfer and return response body as a string. + +On Python 2, this method arranges for response body to be saved in a StringIO +instance, then invokes :ref:`perform ` +to perform the file transfer, then returns the value of the StringIO instance. +This behavior is identical to :ref:`perform_rb `. + +On Python 3, this method arranges for response body to be saved in a BytesIO +instance, then invokes :ref:`perform ` +to perform the file transfer, then decodes the response body in Python's +default encoding and returns the decoded body as a Unicode string +(``str`` instance). *Note:* decoding happens after the transfer finishes, +thus an encoding error implies the transfer/network operation succeeded. + +Any transfer errors raise ``pycurl.error`` exception, +just like in :ref:`perform `. + +Use :ref:`perform_rb ` to retrieve response body as a byte +string (``bytes`` instance on Python 3) without attempting to decode it. + +Raises ``pycurl.error`` exception upon failure. + +*Added in version 7.43.0.2.* diff --git a/doc/docstrings/curl_reset.rst b/doc/docstrings/curl_reset.rst new file mode 100644 index 0000000..b72c893 --- /dev/null +++ b/doc/docstrings/curl_reset.rst @@ -0,0 +1,8 @@ +reset() -> None + +Reset all options set on curl handle to default values, but preserves +live connections, session ID cache, DNS cache, cookies, and shares. + +Corresponds to `curl_easy_reset`_ in libcurl. + +.. _curl_easy_reset: https://curl.haxx.se/libcurl/c/curl_easy_reset.html diff --git a/doc/docstrings/curl_set_ca_certs.rst b/doc/docstrings/curl_set_ca_certs.rst new file mode 100644 index 0000000..49fa0a5 --- /dev/null +++ b/doc/docstrings/curl_set_ca_certs.rst @@ -0,0 +1,5 @@ +set_ca_certs() -> None + +Load ca certs from provided unicode string. + +Note that certificates will be added only when cURL starts new connection. diff --git a/doc/docstrings/curl_setopt.rst b/doc/docstrings/curl_setopt.rst new file mode 100644 index 0000000..2bb0ecb --- /dev/null +++ b/doc/docstrings/curl_setopt.rst @@ -0,0 +1,110 @@ +setopt(option, value) -> None + +Set curl session option. Corresponds to `curl_easy_setopt`_ in libcurl. + +*option* specifies which option to set. PycURL defines constants +corresponding to ``CURLOPT_*`` constants in libcurl, except that +the ``CURLOPT_`` prefix is removed. For example, ``CURLOPT_URL`` is +exposed in PycURL as ``pycurl.URL``, with some exceptions as detailed below. +For convenience, ``CURLOPT_*`` +constants are also exposed on the Curl objects themselves:: + + import pycurl + c = pycurl.Curl() + c.setopt(pycurl.URL, "http://www.python.org/") + # Same as: + c.setopt(c.URL, "http://www.python.org/") + +The following are exceptions to option constant naming convention: + +- ``CURLOPT_FILETIME`` is mapped as ``pycurl.OPT_FILETIME`` +- ``CURLOPT_CERTINFO`` is mapped as ``pycurl.OPT_CERTINFO`` +- ``CURLOPT_COOKIELIST`` is mapped as ``pycurl.COOKIELIST`` + and, as of PycURL 7.43.0.2, also as ``pycurl.OPT_COOKIELIST`` +- ``CURLOPT_RTSP_CLIENT_CSEQ`` is mapped as ``pycurl.OPT_RTSP_CLIENT_CSEQ`` +- ``CURLOPT_RTSP_REQUEST`` is mapped as ``pycurl.OPT_RTSP_REQUEST`` +- ``CURLOPT_RTSP_SERVER_CSEQ`` is mapped as ``pycurl.OPT_RTSP_SERVER_CSEQ`` +- ``CURLOPT_RTSP_SESSION_ID`` is mapped as ``pycurl.OPT_RTSP_SESSION_ID`` +- ``CURLOPT_RTSP_STREAM_URI`` is mapped as ``pycurl.OPT_RTSP_STREAM_URI`` +- ``CURLOPT_RTSP_TRANSPORT`` is mapped as ``pycurl.OPT_RTSP_TRANSPORT`` + +*value* specifies the value to set the option to. Different options accept +values of different types: + +- Options specified by `curl_easy_setopt`_ as accepting ``1`` or an + integer value accept Python integers, long integers (on Python 2.x) and + booleans:: + + c.setopt(pycurl.FOLLOWLOCATION, True) + c.setopt(pycurl.FOLLOWLOCATION, 1) + # Python 2.x only: + c.setopt(pycurl.FOLLOWLOCATION, 1L) + +- Options specified as accepting strings by ``curl_easy_setopt`` accept + byte strings (``str`` on Python 2, ``bytes`` on Python 3) and + Unicode strings with ASCII code points only. + For more information, please refer to :ref:`unicode`. Example:: + + c.setopt(pycurl.URL, "http://www.python.org/") + c.setopt(pycurl.URL, u"http://www.python.org/") + # Python 3.x only: + c.setopt(pycurl.URL, b"http://www.python.org/") + +- ``HTTP200ALIASES``, ``HTTPHEADER``, ``POSTQUOTE``, ``PREQUOTE``, + ``PROXYHEADER`` and + ``QUOTE`` accept a list or tuple of strings. The same rules apply to these + strings as do to string option values. Example:: + + c.setopt(pycurl.HTTPHEADER, ["Accept:"]) + c.setopt(pycurl.HTTPHEADER, ("Accept:",)) + +- ``READDATA`` accepts a file object or any Python object which has + a ``read`` method. On Python 2, a file object will be passed directly + to libcurl and may result in greater transfer efficiency, unless + PycURL has been compiled with ``AVOID_STDIO`` option. + On Python 3 and on Python 2 when the value is not a true file object, + ``READDATA`` is emulated in PycURL via ``READFUNCTION``. + The file should generally be opened in binary mode. Example:: + + f = open('file.txt', 'rb') + c.setopt(c.READDATA, f) + +- ``WRITEDATA`` and ``WRITEHEADER`` accept a file object or any Python + object which has a ``write`` method. On Python 2, a file object will + be passed directly to libcurl and may result in greater transfer efficiency, + unless PycURL has been compiled with ``AVOID_STDIO`` option. + On Python 3 and on Python 2 when the value is not a true file object, + ``WRITEDATA`` is emulated in PycURL via ``WRITEFUNCTION``. + The file should generally be opened in binary mode. Example:: + + f = open('/dev/null', 'wb') + c.setopt(c.WRITEDATA, f) + +- ``*FUNCTION`` options accept a function. Supported callbacks are documented + in :ref:`callbacks`. Example:: + + # Python 2 + import StringIO + b = StringIO.StringIO() + c.setopt(pycurl.WRITEFUNCTION, b.write) + +- ``SHARE`` option accepts a :ref:`curlshareobject`. + +It is possible to set integer options - and only them - that PycURL does +not know about by using the numeric value of the option constant directly. +For example, ``pycurl.VERBOSE`` has the value 42, and may be set as follows:: + + c.setopt(42, 1) + +*setopt* can reset some options to their default value, performing the job of +:py:meth:`pycurl.Curl.unsetopt`, if ``None`` is passed +for the option value. The following two calls are equivalent:: + + c.setopt(c.URL, None) + c.unsetopt(c.URL) + +Raises TypeError when the option value is not of a type accepted by the +respective option, and pycurl.error exception when libcurl rejects the +option or its value. + +.. _curl_easy_setopt: https://curl.haxx.se/libcurl/c/curl_easy_setopt.html diff --git a/doc/docstrings/curl_setopt_string.rst b/doc/docstrings/curl_setopt_string.rst new file mode 100644 index 0000000..230749e --- /dev/null +++ b/doc/docstrings/curl_setopt_string.rst @@ -0,0 +1,31 @@ +setopt_string(option, value) -> None + +Set curl session option to a string value. + +This method allows setting string options that are not officially supported +by PycURL, for example because they did not exist when the version of PycURL +being used was released. +:py:meth:`pycurl.Curl.setopt` should be used for setting options that +PycURL knows about. + +**Warning:** No checking is performed that *option* does, in fact, +expect a string value. Using this method incorrectly can crash the program +and may lead to a security vulnerability. +Furthermore, it is on the application to ensure that the *value* object +does not get garbage collected while libcurl is using it. +libcurl copies most string options but not all; one option whose value +is not copied by libcurl is `CURLOPT_POSTFIELDS`_. + +*option* would generally need to be given as an integer literal rather than +a symbolic constant. + +*value* can be a binary string or a Unicode string using ASCII code points, +same as with string options given to PycURL elsewhere. + +Example setting URL via ``setopt_string``:: + + import pycurl + c = pycurl.Curl() + c.setopt_string(10002, "http://www.python.org/") + +.. _CURLOPT_POSTFIELDS: https://curl.haxx.se/libcurl/c/CURLOPT_POSTFIELDS.html diff --git a/doc/docstrings/curl_unsetopt.rst b/doc/docstrings/curl_unsetopt.rst new file mode 100644 index 0000000..dfb1439 --- /dev/null +++ b/doc/docstrings/curl_unsetopt.rst @@ -0,0 +1,15 @@ +unsetopt(option) -> None + +Reset curl session option to its default value. + +Only some curl options may be reset via this method. + +libcurl does not provide a general way to reset a single option to its default value; +:py:meth:`pycurl.Curl.reset` resets all options to their default values, +otherwise :py:meth:`pycurl.Curl.setopt` must be called with whatever value +is the default. For convenience, PycURL provides this unsetopt method +to reset some of the options to their default values. + +Raises pycurl.error exception on failure. + +``c.unsetopt(option)`` is equivalent to ``c.setopt(option, None)``. diff --git a/doc/docstrings/multi.rst b/doc/docstrings/multi.rst new file mode 100644 index 0000000..c1b54c7 --- /dev/null +++ b/doc/docstrings/multi.rst @@ -0,0 +1,4 @@ +CurlMulti() -> New CurlMulti object + +Creates a new :ref:`curlmultiobject` which corresponds to +a ``CURLM`` handle in libcurl. diff --git a/doc/docstrings/multi_add_handle.rst b/doc/docstrings/multi_add_handle.rst new file mode 100644 index 0000000..64ed7c7 --- /dev/null +++ b/doc/docstrings/multi_add_handle.rst @@ -0,0 +1,12 @@ +add_handle(Curl object) -> None + +Corresponds to `curl_multi_add_handle`_ in libcurl. This method adds an +existing and valid Curl object to the CurlMulti object. + +*Changed in version 7.43.0.2:* add_handle now ensures that the Curl object +is not garbage collected while it is being used by a CurlMulti object. +Previously application had to maintain an outstanding reference to the Curl +object to keep it from being garbage collected. + +.. _curl_multi_add_handle: + https://curl.haxx.se/libcurl/c/curl_multi_add_handle.html diff --git a/doc/docstrings/multi_assign.rst b/doc/docstrings/multi_assign.rst new file mode 100644 index 0000000..b494d2f --- /dev/null +++ b/doc/docstrings/multi_assign.rst @@ -0,0 +1,7 @@ +assign(sock_fd, object) -> None + +Creates an association in the multi handle between the given socket and +a private object in the application. +Corresponds to `curl_multi_assign`_ in libcurl. + +.. _curl_multi_assign: https://curl.haxx.se/libcurl/c/curl_multi_assign.html diff --git a/doc/docstrings/multi_close.rst b/doc/docstrings/multi_close.rst new file mode 100644 index 0000000..d08ba3a --- /dev/null +++ b/doc/docstrings/multi_close.rst @@ -0,0 +1,8 @@ +close() -> None + +Corresponds to `curl_multi_cleanup`_ in libcurl. This method is +automatically called by pycurl when a CurlMulti object no longer has any +references to it, but can also be called explicitly. + +.. _curl_multi_cleanup: + https://curl.haxx.se/libcurl/c/curl_multi_cleanup.html diff --git a/doc/docstrings/multi_fdset.rst b/doc/docstrings/multi_fdset.rst new file mode 100644 index 0000000..4f01316 --- /dev/null +++ b/doc/docstrings/multi_fdset.rst @@ -0,0 +1,26 @@ +fdset() -> tuple of lists with active file descriptors, readable, writeable, exceptions + +Returns a tuple of three lists that can be passed to the select.select() method. + +Corresponds to `curl_multi_fdset`_ in libcurl. This method extracts the +file descriptor information from a CurlMulti object. The returned lists can +be used with the ``select`` module to poll for events. + +Example usage:: + + import pycurl + c = pycurl.Curl() + c.setopt(pycurl.URL, "https://curl.haxx.se") + m = pycurl.CurlMulti() + m.add_handle(c) + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: break + while num_handles: + apply(select.select, m.fdset() + (1,)) + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: break + +.. _curl_multi_fdset: + https://curl.haxx.se/libcurl/c/curl_multi_fdset.html diff --git a/doc/docstrings/multi_info_read.rst b/doc/docstrings/multi_info_read.rst new file mode 100644 index 0000000..cccaf57 --- /dev/null +++ b/doc/docstrings/multi_info_read.rst @@ -0,0 +1,17 @@ +info_read([max_objects]) -> tuple(number of queued messages, a list of successful objects, a list of failed objects) + +Corresponds to the `curl_multi_info_read`_ function in libcurl. + +This method extracts at most *max* messages from the multi stack and returns +them in two lists. The first list contains the handles which completed +successfully and the second list contains a tuple *(curl object, curl error +number, curl error message)* for each failed curl object. The curl error +message is returned as a Python string which is decoded from the curl error +string using the `surrogateescape`_ error handler. The number of +queued messages after this method has been called is also returned. + +.. _curl_multi_info_read: + https://curl.haxx.se/libcurl/c/curl_multi_info_read.html + +.. _surrogateescape: + https://www.python.org/dev/peps/pep-0383/ diff --git a/doc/docstrings/multi_perform.rst b/doc/docstrings/multi_perform.rst new file mode 100644 index 0000000..4c78e81 --- /dev/null +++ b/doc/docstrings/multi_perform.rst @@ -0,0 +1,6 @@ +perform() -> tuple of status and the number of active Curl objects + +Corresponds to `curl_multi_perform`_ in libcurl. + +.. _curl_multi_perform: + https://curl.haxx.se/libcurl/c/curl_multi_perform.html diff --git a/doc/docstrings/multi_remove_handle.rst b/doc/docstrings/multi_remove_handle.rst new file mode 100644 index 0000000..8548ad4 --- /dev/null +++ b/doc/docstrings/multi_remove_handle.rst @@ -0,0 +1,7 @@ +remove_handle(Curl object) -> None + +Corresponds to `curl_multi_remove_handle`_ in libcurl. This method +removes an existing and valid Curl object from the CurlMulti object. + +.. _curl_multi_remove_handle: + https://curl.haxx.se/libcurl/c/curl_multi_remove_handle.html diff --git a/doc/docstrings/multi_select.rst b/doc/docstrings/multi_select.rst new file mode 100644 index 0000000..8849603 --- /dev/null +++ b/doc/docstrings/multi_select.rst @@ -0,0 +1,24 @@ +select([timeout]) -> number of ready file descriptors or 0 on timeout + +Returns result from doing a select() on the curl multi file descriptor +with the given timeout. + +This is a convenience function which simplifies the combined use of +``fdset()`` and the ``select`` module. + +Example usage:: + + import pycurl + c = pycurl.Curl() + c.setopt(pycurl.URL, "https://curl.haxx.se") + m = pycurl.CurlMulti() + m.add_handle(c) + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: break + while num_handles: + ret = m.select(1.0) + if ret == 0: continue + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: break diff --git a/doc/docstrings/multi_setopt.rst b/doc/docstrings/multi_setopt.rst new file mode 100644 index 0000000..fbb2c95 --- /dev/null +++ b/doc/docstrings/multi_setopt.rst @@ -0,0 +1,38 @@ +setopt(option, value) -> None + +Set curl multi option. Corresponds to `curl_multi_setopt`_ in libcurl. + +*option* specifies which option to set. PycURL defines constants +corresponding to ``CURLMOPT_*`` constants in libcurl, except that +the ``CURLMOPT_`` prefix is replaced with ``M_`` prefix. +For example, ``CURLMOPT_PIPELINING`` is +exposed in PycURL as ``pycurl.M_PIPELINING``. For convenience, ``CURLMOPT_*`` +constants are also exposed on CurlMulti objects:: + + import pycurl + m = pycurl.CurlMulti() + m.setopt(pycurl.M_PIPELINING, 1) + # Same as: + m.setopt(m.M_PIPELINING, 1) + +*value* specifies the value to set the option to. Different options accept +values of different types: + +- Options specified by `curl_multi_setopt`_ as accepting ``1`` or an + integer value accept Python integers, long integers (on Python 2.x) and + booleans:: + + m.setopt(pycurl.M_PIPELINING, True) + m.setopt(pycurl.M_PIPELINING, 1) + # Python 2.x only: + m.setopt(pycurl.M_PIPELINING, 1L) + +- ``*FUNCTION`` options accept a function. Supported callbacks are + ``CURLMOPT_SOCKETFUNCTION`` AND ``CURLMOPT_TIMERFUNCTION``. Please refer to + the PycURL test suite for examples on using the callbacks. + +Raises TypeError when the option value is not of a type accepted by the +respective option, and pycurl.error exception when libcurl rejects the +option or its value. + +.. _curl_multi_setopt: https://curl.haxx.se/libcurl/c/curl_multi_setopt.html diff --git a/doc/docstrings/multi_socket_action.rst b/doc/docstrings/multi_socket_action.rst new file mode 100644 index 0000000..d2fb1cc --- /dev/null +++ b/doc/docstrings/multi_socket_action.rst @@ -0,0 +1,17 @@ +socket_action(sock_fd, ev_bitmask) -> (result, num_running_handles) + +Returns result from doing a socket_action() on the curl multi file descriptor +with the given timeout. +Corresponds to `curl_multi_socket_action`_ in libcurl. + +The return value is a two-element tuple. The first element is the return +value of the underlying ``curl_multi_socket_action`` function, and it is +always zero (``CURLE_OK``) because any other return value would cause +``socket_action`` to raise an exception. The second element is the number of +running easy handles within this multi handle. When the number of running +handles reaches zero, all transfers have completed. Note that if the number +of running handles has decreased by one compared to the previous invocation, +this is not mean the handle corresponding to the ``sock_fd`` provided as +the argument to this function was the completed handle. + +.. _curl_multi_socket_action: https://curl.haxx.se/libcurl/c/curl_multi_socket_action.html diff --git a/doc/docstrings/multi_socket_all.rst b/doc/docstrings/multi_socket_all.rst new file mode 100644 index 0000000..b9de921 --- /dev/null +++ b/doc/docstrings/multi_socket_all.rst @@ -0,0 +1,4 @@ +socket_all() -> tuple + +Returns result from doing a socket_all() on the curl multi file descriptor +with the given timeout. diff --git a/doc/docstrings/multi_timeout.rst b/doc/docstrings/multi_timeout.rst new file mode 100644 index 0000000..aea5a37 --- /dev/null +++ b/doc/docstrings/multi_timeout.rst @@ -0,0 +1,6 @@ +timeout() -> int + +Returns how long to wait for action before proceeding. +Corresponds to `curl_multi_timeout`_ in libcurl. + +.. _curl_multi_timeout: https://curl.haxx.se/libcurl/c/curl_multi_timeout.html diff --git a/doc/docstrings/pycurl_global_cleanup.rst b/doc/docstrings/pycurl_global_cleanup.rst new file mode 100644 index 0000000..4970bf9 --- /dev/null +++ b/doc/docstrings/pycurl_global_cleanup.rst @@ -0,0 +1,7 @@ +global_cleanup() -> None + +Cleanup curl environment. + +Corresponds to `curl_global_cleanup`_ in libcurl. + +.. _curl_global_cleanup: https://curl.haxx.se/libcurl/c/curl_global_cleanup.html diff --git a/doc/docstrings/pycurl_global_init.rst b/doc/docstrings/pycurl_global_init.rst new file mode 100644 index 0000000..bef09d8 --- /dev/null +++ b/doc/docstrings/pycurl_global_init.rst @@ -0,0 +1,10 @@ +global_init(option) -> None + +Initialize curl environment. + +*option* is one of the constants pycurl.GLOBAL_SSL, pycurl.GLOBAL_WIN32, +pycurl.GLOBAL_ALL, pycurl.GLOBAL_NOTHING, pycurl.GLOBAL_DEFAULT. + +Corresponds to `curl_global_init`_ in libcurl. + +.. _curl_global_init: https://curl.haxx.se/libcurl/c/curl_global_init.html diff --git a/doc/docstrings/pycurl_module.rst b/doc/docstrings/pycurl_module.rst new file mode 100644 index 0000000..55156f3 --- /dev/null +++ b/doc/docstrings/pycurl_module.rst @@ -0,0 +1,13 @@ +This module implements an interface to the cURL library. + +Types: + +Curl() -> New object. Create a new curl object. +CurlMulti() -> New object. Create a new curl multi object. +CurlShare() -> New object. Create a new curl share object. + +Functions: + +global_init(option) -> None. Initialize curl environment. +global_cleanup() -> None. Cleanup curl environment. +version_info() -> tuple. Return version information. diff --git a/doc/docstrings/pycurl_version_info.rst b/doc/docstrings/pycurl_version_info.rst new file mode 100644 index 0000000..98ee759 --- /dev/null +++ b/doc/docstrings/pycurl_version_info.rst @@ -0,0 +1,18 @@ +version_info() -> tuple + +Returns a 12-tuple with the version info. + +Corresponds to `curl_version_info`_ in libcurl. Returns a tuple of +information which is similar to the ``curl_version_info_data`` struct +returned by ``curl_version_info()`` in libcurl. + +Example usage:: + + >>> import pycurl + >>> pycurl.version_info() + (3, '7.33.0', 467200, 'amd64-portbld-freebsd9.1', 33436, 'OpenSSL/0.9.8x', + 0, '1.2.7', ('dict', 'file', 'ftp', 'ftps', 'gopher', 'http', 'https', + 'imap', 'imaps', 'pop3', 'pop3s', 'rtsp', 'smtp', 'smtps', 'telnet', + 'tftp'), None, 0, None) + +.. _curl_version_info: https://curl.haxx.se/libcurl/c/curl_version_info.html diff --git a/doc/docstrings/share.rst b/doc/docstrings/share.rst new file mode 100644 index 0000000..6be1368 --- /dev/null +++ b/doc/docstrings/share.rst @@ -0,0 +1,5 @@ +CurlShare() -> New CurlShare object + +Creates a new :ref:`curlshareobject` which corresponds to a +``CURLSH`` handle in libcurl. CurlShare objects is what you pass as an +argument to the SHARE option on :ref:`Curl objects `. diff --git a/doc/docstrings/share_close.rst b/doc/docstrings/share_close.rst new file mode 100644 index 0000000..df5f658 --- /dev/null +++ b/doc/docstrings/share_close.rst @@ -0,0 +1,10 @@ +close() -> None + +Close shared handle. + +Corresponds to `curl_share_cleanup`_ in libcurl. This method is +automatically called by pycurl when a CurlShare object no longer has +any references to it, but can also be called explicitly. + +.. _curl_share_cleanup: + https://curl.haxx.se/libcurl/c/curl_share_cleanup.html diff --git a/doc/docstrings/share_setopt.rst b/doc/docstrings/share_setopt.rst new file mode 100644 index 0000000..abc39d1 --- /dev/null +++ b/doc/docstrings/share_setopt.rst @@ -0,0 +1,26 @@ +setopt(option, value) -> None + +Set curl share option. + +Corresponds to `curl_share_setopt`_ in libcurl, where *option* is +specified with the ``CURLSHOPT_*`` constants in libcurl, except that the +``CURLSHOPT_`` prefix has been changed to ``SH_``. Currently, *value* must be +one of: ``LOCK_DATA_COOKIE``, ``LOCK_DATA_DNS``, ``LOCK_DATA_SSL_SESSION`` or +``LOCK_DATA_CONNECT``. + +Example usage:: + + import pycurl + curl = pycurl.Curl() + s = pycurl.CurlShare() + s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE) + s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS) + curl.setopt(pycurl.URL, 'https://curl.haxx.se') + curl.setopt(pycurl.SHARE, s) + curl.perform() + curl.close() + +Raises pycurl.error exception upon failure. + +.. _curl_share_setopt: + https://curl.haxx.se/libcurl/c/curl_share_setopt.html diff --git a/doc/files.rst b/doc/files.rst new file mode 100644 index 0000000..5bc0b08 --- /dev/null +++ b/doc/files.rst @@ -0,0 +1,31 @@ +File Handling +============= + +In PycURL 7.19.0.3 and below, ``CURLOPT_READDATA``, ``CURLOPT_WRITEDATA`` and +``CURLOPT_WRITEHEADER`` options accepted file objects and directly passed +the underlying C library ``FILE`` pointers to libcurl. + +Python 3 no longer implements files as C library ``FILE`` objects. +In PycURL 7.19.3 and above, when running on Python 3, these options +are implemented as calls to ``CURLOPT_READFUNCTION``, ``CURLOPT_WRITEFUNCTION`` +and ``CURLOPT_HEADERFUNCTION``, respectively, with the write method of the +Python file object as the parameter. As a result, any Python file-like +object implementing a ``read`` method can be passed to ``CURLOPT_READDATA``, +and any Python file-like object implementing a ``write`` method can be +passed to ``CURLOPT_WRITEDATA`` or ``CURLOPT_WRITEHEADER`` options. + +When running PycURL 7.19.3 and above on Python 2, the old behavior of +passing ``FILE`` pointers to libcurl remains when a true file object is given +to ``CURLOPT_READDATA``, ``CURLOPT_WRITEDATA`` and ``CURLOPT_WRITEHEADER`` +options. For consistency with Python 3 behavior these options also accept +file-like objects implementing a ``read`` or ``write`` method, as appropriate, +as arguments, in which case the Python 3 code path is used converting these +options to ``CURLOPT_*FUNCTION`` option calls. + +Files given to PycURL as arguments to ``CURLOPT_READDATA``, +``CURLOPT_WRITEDATA`` or ``CURLOPT_WRITEHEADER`` must be opened for reading or +writing in binary mode. Files opened in text mode (without ``"b"`` flag to +``open()``) expect string objects and reading from or writing to them from +PycURL will fail. Similarly when passing ``f.write`` method of an open file to +``CURLOPT_WRITEFUNCTION`` or ``CURLOPT_HEADERFUNCTION``, or ``f.read`` to +``CURLOPT_READFUNCTION``, the file must have been be opened in binary mode. diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 0000000..874958f --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,157 @@ +PycURL -- A Python Interface To The cURL library +================================================ + +PycURL is a Python interface to `libcurl`_, the multiprotocol file +transfer library. Similarly to the urllib_ Python module, +PycURL can be used to fetch objects identified by a URL from a Python program. +Beyond simple fetches however PycURL exposes most of the functionality of +libcurl, including: + +- Speed - libcurl is very fast and PycURL, being a thin wrapper above + libcurl, is very fast as well. PycURL `was benchmarked`_ to be several + times faster than Requests_. +- Features including multiple protocol support, SSL, authentication and + proxy options. PycURL supports most of libcurl's callbacks. +- Multi_ and share_ interfaces. +- Sockets used for network operations, permitting integration of PycURL + into the application's I/O loop (e.g., using Tornado_). + +.. _was benchmarked: http://stackoverflow.com/questions/15461995/python-requests-vs-pycurl-performance +.. _Requests: http://python-requests.org/ +.. _Multi: https://curl.haxx.se/libcurl/c/libcurl-multi.html +.. _share: https://curl.haxx.se/libcurl/c/libcurl-share.html +.. _Tornado: http://www.tornadoweb.org/ + + +PycURL vs Requests +------------------ + +Requests_ is another popular Python library that is frequently compared to +PycURL. + +Advantages of PycURL: + +- PycURL can be `several times faster than Requests + `_. + The performance difference is larger when there are multiple requests + performed and connections are reused. +- PycURL makes it possible to take advantage of I/O multiplexing via the + `libcurl multi interface `_. +- PycURL supports many protocols, not just HTTP. +- PycURL generally provides more features, for example ability to use several + TLS backends, more authentication options, etc. + +Advantages of Requests: + +- Requests is written in pure Python and does not require C extensions. + As a result, Requests is trivial to install while PycURL's installation + can be complex (though operating system-specific packages, if available, + negate this drawback). +- Requests' API is generally easier to learn and use than PycURL's. + + +About libcurl +------------- + +- libcurl is a free and easy-to-use client-side URL transfer library, supporting + DICT, FILE, FTP, FTPS, Gopher, HTTP, HTTPS, IMAP, IMAPS, LDAP, LDAPS, POP3, + POP3S, RTMP, RTSP, SCP, SFTP, SMTP, SMTPS, Telnet and TFTP. + libcurl supports SSL certificates, HTTP POST, HTTP PUT, + FTP uploading, HTTP form based upload, proxies, cookies, user+password + authentication (Basic, Digest, NTLM, Negotiate, Kerberos4), file transfer + resume, http proxy tunneling and more! + +- libcurl is highly portable, it builds and works identically on numerous + platforms, including Solaris, NetBSD, FreeBSD, OpenBSD, Darwin, HPUX, IRIX, + AIX, Tru64, Linux, UnixWare, HURD, Windows, Amiga, OS/2, BeOs, Mac OS X, + Ultrix, QNX, OpenVMS, RISC OS, Novell NetWare, DOS and more... + +- libcurl is `free`_, :ref:`thread-safe `, `IPv6 compatible`_, `feature rich`_, + `well supported`_, `fast`_, `thoroughly documented`_ and is already used by + many known, big and successful `companies`_ and numerous `applications`_. + +.. _free: https://curl.haxx.se/docs/copyright.html +.. _thread-safe: :ref:`thread-safety` +.. _`IPv6 compatible`: https://curl.haxx.se/libcurl/features.html#ipv6 +.. _`feature rich`: https://curl.haxx.se/libcurl/features.html#features +.. _`well supported`: https://curl.haxx.se/libcurl/features.html#support +.. _`fast`: https://curl.haxx.se/libcurl/features.html#fast +.. _`thoroughly documented`: https://curl.haxx.se/libcurl/features.html#docs +.. _companies: https://curl.haxx.se/docs/companies.html +.. _applications: https://curl.haxx.se/libcurl/using/apps.html + + +Requirements +------------ + +- Python 3. +- libcurl 7.19.0 or better. + + +Installation +------------ + +On Unix, PycURL is easiest to install using your operating system's package +manager. This will also install libcurl and other dependencies as needed. + +Installation via easy_install and pip is also supported:: + + easy_install pycurl + pip install pycurl + +If this does not work, please see :ref:`install`. + +On Windows, build from source or use a third-party binary package. + + +Support +------- + +For support questions, please use `curl-and-python mailing list`_. +`Mailing list archives`_ are available for your perusal as well. + +Although not an official support venue, `Stack Overflow`_ has been +popular with PycURL users as well. + +Bugs can be reported `via GitHub`_. Please only use GitHub issues when you are +certain you have found a bug in PycURL. If you do not have a patch to fix +the bug, or at least a specific code fragment in PycURL that you believe is +the cause, you should instead post your inquiry to the mailing list. + +.. _curl-and-python mailing list: http://cool.haxx.se/mailman/listinfo/curl-and-python +.. _Stack Overflow: http://stackoverflow.com/questions/tagged/pycurl +.. _Mailing list archives: https://curl.haxx.se/mail/list.cgi?list=curl-and-python +.. _via GitHub: https://github.com/pycurl/pycurl/issues + + +Documentation Contents +---------------------- + +.. toctree:: + :maxdepth: 2 + + release-notes + install + quickstart + troubleshooting + pycurl + curlobject + curlmultiobject + curlshareobject + callbacks + curl + unicode + files + thread-safety + unimplemented + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + +.. _libcurl: https://curl.haxx.se/libcurl/ +.. _urllib: http://docs.python.org/library/urllib.html diff --git a/doc/install.rst b/doc/install.rst new file mode 100644 index 0000000..a23a75e --- /dev/null +++ b/doc/install.rst @@ -0,0 +1 @@ +.. include:: ../INSTALL.rst diff --git a/doc/internals.rst b/doc/internals.rst new file mode 100644 index 0000000..9729a2b --- /dev/null +++ b/doc/internals.rst @@ -0,0 +1,15 @@ +Internals +========= + +Cleanup sequence: + +x=curl/multi/share + +x.close() -> do_x_close -> util_x_close +del x -> do_x_dealloc -> util_x_close + +do_* functions are directly invoked by user code. +They check pycurl object state. + +util_* functions are only invoked by other pycurl C functions. +They do not check pycurl object state. diff --git a/doc/pycurl.rst b/doc/pycurl.rst new file mode 100644 index 0000000..0113acd --- /dev/null +++ b/doc/pycurl.rst @@ -0,0 +1,34 @@ +pycurl Module Functionality +=========================== + +.. module:: pycurl + +.. autofunction:: pycurl.global_init + +.. autofunction:: pycurl.global_cleanup + +.. data:: version + + This is a string with version information on libcurl, corresponding to + `curl_version`_ in libcurl. + + Example usage: + + :: + + >>> import pycurl + >>> pycurl.version + 'PycURL/7.19.3 libcurl/7.33.0 OpenSSL/0.9.8x zlib/1.2.7' + +.. autofunction:: pycurl.version_info + +.. autoclass:: pycurl.Curl + :noindex: + +.. autoclass:: pycurl.CurlMulti + :noindex: + +.. autoclass:: pycurl.CurlShare + :noindex: + +.. _curl_version: https://curl.haxx.se/libcurl/c/curl_version.html diff --git a/doc/quickstart.rst b/doc/quickstart.rst new file mode 100644 index 0000000..22607f9 --- /dev/null +++ b/doc/quickstart.rst @@ -0,0 +1,442 @@ +PycURL Quick Start +================== + +Retrieving A Network Resource +----------------------------- + +Once PycURL is installed we can perform network operations. The simplest +one is retrieving a resource by its URL. To issue a network request with +PycURL, the following steps are required: + + 1. Create a ``pycurl.Curl`` instance. + 2. Use ``setopt`` to set options. + 3. Call ``perform`` to perform the operation. + +Here is how we can retrieve a network resource in Python 3:: + + import pycurl + import certifi + from io import BytesIO + + buffer = BytesIO() + c = pycurl.Curl() + c.setopt(c.URL, 'http://pycurl.io/') + c.setopt(c.WRITEDATA, buffer) + c.setopt(c.CAINFO, certifi.where()) + c.perform() + c.close() + + body = buffer.getvalue() + # Body is a byte string. + # We have to know the encoding in order to print it to a text file + # such as standard output. + print(body.decode('iso-8859-1')) + +This code is available as ``examples/quickstart/get_python3.py``. +For a Python 2 only example, see ``examples/quickstart/get_python2.py``. +For an example targeting Python 2 and 3, see ``examples/quickstart/get.py``. + +PycURL does not provide storage for the network response - that is the +application's job. Therefore we must setup a buffer (in the form of a +StringIO object) and instruct PycURL to write to that buffer. + +Most of the existing PycURL code uses WRITEFUNCTION instead of WRITEDATA +as follows:: + + c.setopt(c.WRITEFUNCTION, buffer.write) + +While the WRITEFUNCTION idiom continues to work, it is now unnecessary. +As of PycURL 7.19.3 WRITEDATA accepts any Python object with a ``write`` +method. + +Working With HTTPS +------------------ + +Most web sites today use HTTPS which is HTTP over TLS/SSL. In order to +take advantage of security that HTTPS provides, PycURL needs to utilize +a *certificate bundle*. As certificates change over time PycURL does not +provide such a bundle; one may be supplied by your operating system, but +if not, consider using the `certifi`_ Python package:: + + import pycurl + import certifi + from io import BytesIO + + buffer = BytesIO() + c = pycurl.Curl() + c.setopt(c.URL, 'https://python.org/') + c.setopt(c.WRITEDATA, buffer) + c.setopt(c.CAINFO, certifi.where()) + c.perform() + c.close() + + body = buffer.getvalue() + # Body is a byte string. + # We have to know the encoding in order to print it to a text file + # such as standard output. + print(body.decode('iso-8859-1')) + +This code is available as ``examples/quickstart/get_python3_https.py``. +For a Python 2 example, see ``examples/quickstart/get_python2_https.py``. + + +Troubleshooting +--------------- + +When things don't work as expected, use libcurl's ``VERBOSE`` option to +receive lots of debugging output pertaining to the request:: + + c.setopt(c.VERBOSE, True) + +It is often helpful to compare verbose output from the program using PycURL +with that of ``curl`` command line tool when the latter is invoked with +``-v`` option:: + + curl -v http://pycurl.io/ + + +Examining Response Headers +-------------------------- + +In reality we want to decode the response using the encoding specified by +the server rather than assuming an encoding. To do this we need to +examine the response headers:: + + import pycurl + import re + try: + from io import BytesIO + except ImportError: + from StringIO import StringIO as BytesIO + + headers = {} + def header_function(header_line): + # HTTP standard specifies that headers are encoded in iso-8859-1. + # On Python 2, decoding step can be skipped. + # On Python 3, decoding step is required. + header_line = header_line.decode('iso-8859-1') + + # Header lines include the first status line (HTTP/1.x ...). + # We are going to ignore all lines that don't have a colon in them. + # This will botch headers that are split on multiple lines... + if ':' not in header_line: + return + + # Break the header line into header name and value. + name, value = header_line.split(':', 1) + + # Remove whitespace that may be present. + # Header lines include the trailing newline, and there may be whitespace + # around the colon. + name = name.strip() + value = value.strip() + + # Header names are case insensitive. + # Lowercase name here. + name = name.lower() + + # Now we can actually record the header name and value. + # Note: this only works when headers are not duplicated, see below. + headers[name] = value + + buffer = BytesIO() + c = pycurl.Curl() + c.setopt(c.URL, 'http://pycurl.io') + c.setopt(c.WRITEFUNCTION, buffer.write) + # Set our header function. + c.setopt(c.HEADERFUNCTION, header_function) + c.perform() + c.close() + + # Figure out what encoding was sent with the response, if any. + # Check against lowercased header name. + encoding = None + if 'content-type' in headers: + content_type = headers['content-type'].lower() + match = re.search('charset=(\S+)', content_type) + if match: + encoding = match.group(1) + print('Decoding using %s' % encoding) + if encoding is None: + # Default encoding for HTML is iso-8859-1. + # Other content types may have different default encoding, + # or in case of binary data, may have no encoding at all. + encoding = 'iso-8859-1' + print('Assuming encoding is %s' % encoding) + + body = buffer.getvalue() + # Decode using the encoding we figured out. + print(body.decode(encoding)) + +This code is available as ``examples/quickstart/response_headers.py``. + +That was a lot of code for something very straightforward. Unfortunately, +as libcurl refrains from allocating memory for response data, it is on our +application to perform this grunt work. + +One caveat with the above code is that if there are multiple headers +for the same name, such as Set-Cookie, only the last header value will be +stored. To record all values in multi-valued headers as a list the following +code can be used instead of ``headers[name] = value`` line:: + + if name in headers: + if isinstance(headers[name], list): + headers[name].append(value) + else: + headers[name] = [headers[name], value] + else: + headers[name] = value + + +Writing To A File +----------------- + +Suppose we want to save response body to a file. This is actually easy +for a change:: + + import pycurl + + # As long as the file is opened in binary mode, both Python 2 and Python 3 + # can write response body to it without decoding. + with open('out.html', 'wb') as f: + c = pycurl.Curl() + c.setopt(c.URL, 'http://pycurl.io/') + c.setopt(c.WRITEDATA, f) + c.perform() + c.close() + +This code is available as ``examples/quickstart/write_file.py``. + +The important part is opening the file in binary mode - then response body +can be written bytewise without decoding or encoding steps. + + +Following Redirects +------------------- + +By default libcurl, and PycURL, do not follow redirects. Changing this +behavior involves using ``setopt`` like so:: + + import pycurl + + c = pycurl.Curl() + # Redirects to https://www.python.org/. + c.setopt(c.URL, 'http://www.python.org/') + # Follow redirect. + c.setopt(c.FOLLOWLOCATION, True) + c.perform() + c.close() + +This code is available as ``examples/quickstart/follow_redirect.py``. + +As we did not set a write callback, the default libcurl and PycURL behavior +to write response body to standard output takes effect. + + +Setting Options +--------------- + +Following redirects is one option that libcurl provides. There are many more +such options, and they are documented on `curl_easy_setopt`_ page. +With very few exceptions, PycURL option names are derived from libcurl +option names by removing the ``CURLOPT_`` prefix. Thus, ``CURLOPT_URL`` +becomes simply ``URL``. + +.. _curl_easy_setopt: https://curl.haxx.se/libcurl/c/curl_easy_setopt.html + + +Examining Response +------------------ + +We already covered examining response headers. Other response information is +accessible via ``getinfo`` call as follows:: + + import pycurl + try: + from io import BytesIO + except ImportError: + from StringIO import StringIO as BytesIO + + buffer = BytesIO() + c = pycurl.Curl() + c.setopt(c.URL, 'http://pycurl.io/') + c.setopt(c.WRITEDATA, buffer) + c.perform() + + # HTTP response code, e.g. 200. + print('Status: %d' % c.getinfo(c.RESPONSE_CODE)) + # Elapsed time for the transfer. + print('Time: %f' % c.getinfo(c.TOTAL_TIME)) + + # getinfo must be called before close. + c.close() + +This code is available as ``examples/quickstart/response_info.py``. + +Here we write the body to a buffer to avoid printing uninteresting output +to standard out. + +Response information that libcurl exposes is documented on +`curl_easy_getinfo`_ page. With very few exceptions, PycURL constants +are derived from libcurl constants by removing the ``CURLINFO_`` prefix. +Thus, ``CURLINFO_RESPONSE_CODE`` becomes simply ``RESPONSE_CODE``. + +.. _curl_easy_getinfo: https://curl.haxx.se/libcurl/c/curl_easy_getinfo.html + + +Sending Form Data +----------------- + +To send form data, use ``POSTFIELDS`` option. Form data must be URL-encoded +beforehand:: + + import pycurl + try: + # python 3 + from urllib.parse import urlencode + except ImportError: + # python 2 + from urllib import urlencode + + c = pycurl.Curl() + c.setopt(c.URL, 'https://httpbin.org/post') + + post_data = {'field': 'value'} + # Form data must be provided already urlencoded. + postfields = urlencode(post_data) + # Sets request method to POST, + # Content-Type header to application/x-www-form-urlencoded + # and data to send in request body. + c.setopt(c.POSTFIELDS, postfields) + + c.perform() + c.close() + +This code is available as ``examples/quickstart/form_post.py``. + +``POSTFIELDS`` automatically sets HTTP request method to POST. Other request +methods can be specified via ``CUSTOMREQUEST`` option:: + + c.setopt(c.CUSTOMREQUEST, 'PATCH') + + +File Upload - Multipart POST +---------------------------- + +To replicate the behavior of file upload in an HTML form (specifically, +a multipart form), +use ``HTTPPOST`` option. Such an upload is performed with a ``POST`` request. +See the next example for how to upload a file with a ``PUT`` request. + +If the data to be uploaded is located in a physical file, +use ``FORM_FILE``:: + + import pycurl + + c = pycurl.Curl() + c.setopt(c.URL, 'https://httpbin.org/post') + + c.setopt(c.HTTPPOST, [ + ('fileupload', ( + # upload the contents of this file + c.FORM_FILE, __file__, + )), + ]) + + c.perform() + c.close() + +This code is available as ``examples/quickstart/file_upload_real.py``. + +``libcurl`` provides a number of options to tweak file uploads and multipart +form submissions in general. These are documented on `curl_formadd page`_. +For example, to set a different filename and content type:: + + import pycurl + + c = pycurl.Curl() + c.setopt(c.URL, 'https://httpbin.org/post') + + c.setopt(c.HTTPPOST, [ + ('fileupload', ( + # upload the contents of this file + c.FORM_FILE, __file__, + # specify a different file name for the upload + c.FORM_FILENAME, 'helloworld.py', + # specify a different content type + c.FORM_CONTENTTYPE, 'application/x-python', + )), + ]) + + c.perform() + c.close() + +This code is available as ``examples/quickstart/file_upload_real_fancy.py``. + +If the file data is in memory, use ``BUFFER``/``BUFFERPTR`` as follows:: + + import pycurl + + c = pycurl.Curl() + c.setopt(c.URL, 'https://httpbin.org/post') + + c.setopt(c.HTTPPOST, [ + ('fileupload', ( + c.FORM_BUFFER, 'readme.txt', + c.FORM_BUFFERPTR, 'This is a fancy readme file', + )), + ]) + + c.perform() + c.close() + +This code is available as ``examples/quickstart/file_upload_buffer.py``. + + +File Upload - PUT +----------------- + +A file can also be uploaded in request body, via a ``PUT`` request. +Here is how this can be arranged with a physical file:: + + import pycurl + + c = pycurl.Curl() + c.setopt(c.URL, 'https://httpbin.org/put') + + c.setopt(c.UPLOAD, 1) + file = open('body.json') + c.setopt(c.READDATA, file) + + c.perform() + c.close() + # File must be kept open while Curl object is using it + file.close() + +This code is available as ``examples/quickstart/put_file.py``. + +And if the data is stored in a buffer:: + + import pycurl + try: + from io import BytesIO + except ImportError: + from StringIO import StringIO as BytesIO + + c = pycurl.Curl() + c.setopt(c.URL, 'https://httpbin.org/put') + + c.setopt(c.UPLOAD, 1) + data = '{"json":true}' + # READDATA requires an IO-like object; a string is not accepted + # encode() is necessary for Python 3 + buffer = BytesIO(data.encode('utf-8')) + c.setopt(c.READDATA, buffer) + + c.perform() + c.close() + +This code is available as ``examples/quickstart/put_buffer.py``. + +.. _curl_formadd page: https://curl.haxx.se/libcurl/c/curl_formadd.html +.. _certifi: https://pypi.org/project/certifi/ diff --git a/doc/release-notes.rst b/doc/release-notes.rst new file mode 100644 index 0000000..302b4c6 --- /dev/null +++ b/doc/release-notes.rst @@ -0,0 +1 @@ +.. include:: ../RELEASE-NOTES.rst diff --git a/doc/release-process.rst b/doc/release-process.rst new file mode 100644 index 0000000..e05e64e --- /dev/null +++ b/doc/release-process.rst @@ -0,0 +1,33 @@ +Release Process +=============== + +1. Ensure changelog is up to date with commits in master. +2. Run ``python setup.py authors`` and review the updated AUTHORS file. +3. Run ``git shortlog REL_...`` and add new contributors + missed by the authors script to AUTHORS. +4. Run ``python setup.py manifest``, check that none of the listed files + should be in MANIFEST.in. +5. Check ``get_data_files()`` in ``setup.py`` to see if any new files should + be included in binary distributions. +6. Make sure Travis and AppVeyor are green for master. +7. Update version numbers in: + - Changelog (also record release date) + - doc/conf.py + - setup.py + - winbuild.py +8. Update copyright years if necessary. +9. Draft release notes, add to RELEASE-NOTES.rst. +10. ``make gen docs``. +11. ``python setup.py sdist``. +12. Manually test install the built package. +13. Build windows packages using winbuild.py. +14. Add sdist and windows packages to downloads repo on github. +15. Tag the new version. +16. Upload source distribution to pypi using twine. +17. Upload windows wheels to pypi using twine. +18. Upload windows exe installers to pypi using twine. +19. Upload release files to bintray. +20. Push tag to github pycurl repo. +21. Generate and upload documentation to web site. +22. Update web site home page. +23. Announce release on mailing list. diff --git a/doc/static/favicon.ico b/doc/static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..9ad4b6e0cf64810acaced390d263696a33ea0d4d GIT binary patch literal 318 zcmZXOI}QRd3`8dqQrc|8wv-$%UD`+-fdg=rTxF&0(VieY8zX^mG&9cllSqkEN|D7i zNoP}F&IKf$V5vL*llxGQq+2JKF$U5#ML;u>8JB5Bt}VC5BXv#7`qoN+T558||IzSm tr$?(kpYeU(S3r$<_R0E9!~atA_FLc~ZD0>rZr?koZ{T@H!2#w6)L%R;JWv1t literal 0 HcmV?d00001 diff --git a/doc/thread-safety.rst b/doc/thread-safety.rst new file mode 100644 index 0000000..6b24b8e --- /dev/null +++ b/doc/thread-safety.rst @@ -0,0 +1,34 @@ +.. _thread-safety: + +Thread Safety +============= + +Per `libcurl thread safety documentation`_, libcurl is thread-safe but +has no internal thread synchronization. + +For Python programs using PycURL, this means: + +* Accessing the same PycURL object from different threads is OK when + this object is not involved in active transfers, as Python internally + has a Global Interpreter Lock and only one operating system thread can + be executing Python code at a time. + +* Accessing a PycURL object that is involved in an active transfer from + Python code *inside a libcurl callback for the PycURL object in question* + is OK, because PycURL takes out the appropriate locks. + +* Accessing a PycURL object that is involved in an active transfer from + Python code *outside of a libcurl callback for the PycURL object in question* + is unsafe. + +PycURL handles the necessary SSL locks for OpenSSL/LibreSSL/BoringSSL, +GnuTLS, NSS, mbedTLS and wolfSSL. + +A special situation exists when libcurl uses the standard C library +name resolver (i.e., not threaded nor c-ares resolver). By default libcurl +uses signals for timeouts with the C library resolver, and signals do not +work properly in multi-threaded programs. When using PycURL objects from +multiple Python threads ``NOSIGNAL`` option `must be given`_. + +.. _libcurl thread safety documentation: https://curl.haxx.se/libcurl/c/threadsafe.html +.. _must be given: https://github.com/curl/curl/issues/1003 diff --git a/doc/troubleshooting.rst b/doc/troubleshooting.rst new file mode 100644 index 0000000..6f8218c --- /dev/null +++ b/doc/troubleshooting.rst @@ -0,0 +1,128 @@ +Troubleshooting +=============== + +The first step of troubleshooting issues in programs using PycURL is +identifying which piece of software is responsible for the misbehavior. +PycURL is a thin wrapper around libcurl; libcurl performs most of the +network operations and transfer-related issues are generally the domain +of libcurl. + +``setopt``-Related Issues +------------------------- + +:ref:`setopt ` is one method that is used for setting most +of the libcurl options, as such calls to it can fail in a wide variety +of ways. + +``TypeError: invalid arguments to setopt`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This usually means the *type* of argument passed to ``setopt`` does not +match what the option expects. Recent versions of PycURL have improved +error reporting when this happens and they also accept more data types +(for example tuples in addition to lists). If you are using an old version of +PycURL, upgrading to the last version may help troubleshoot the situation. + +The next step is carefully reading libcurl documentation for the option +in question and verifying that the type, structure and format of data +you are passing matches what the option expects. + +``pycurl.error: (1, '')`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +An exception like this means PycURL accepted the structure and values +in the option parameter and sent them on to libcurl, and +libcurl rejected the attempt to set the option. + +Until PycURL implements an error code to symbol mapping, +you have to perform this mapping by hand. Error codes are +found in the file `curl.h`_ in libcurl source; look for ``CURLE_OK``. +For example, error code 1 means ``CURLE_UNSUPPORTED_PROTOCOL``. + +libcurl can reject a ``setopt`` call for a variety of reasons of its own, +including but not limited to the requested functionality +`not being compiled in`_ or being not supported with the SSL backend +being used. + +Transfer-Related Issues +----------------------- + +If your issue is transfer-related (timeout, connection failure, transfer +failure, ``perform`` hangs, etc.) the first step in troubleshooting is +setting the ``VERBOSE`` flag for the operation. libcurl will then output +debugging information as the transfer executes:: + + >>> import pycurl + >>> curl = pycurl.Curl() + >>> curl.setopt(curl.VERBOSE, True) + >>> curl.setopt(curl.URL, 'https://www.python.org') + >>> curl.setopt(curl.WRITEDATA, open('/dev/null', 'w')) + >>> curl.perform() + * Hostname www.python.org was found in DNS cache + * Trying 151.101.208.223... + * TCP_NODELAY set + * Connected to www.python.org (151.101.208.223) port 443 (#1) + * found 173 certificates in /etc/ssl/certs/ca-certificates.crt + * found 696 certificates in /etc/ssl/certs + * ALPN, offering http/1.1 + * SSL re-using session ID + * SSL connection using TLS1.2 / ECDHE_RSA_AES_128_GCM_SHA256 + * server certificate verification OK + * server certificate status verification SKIPPED + * common name: www.python.org (matched) + * server certificate expiration date OK + * server certificate activation date OK + * certificate public key: RSA + * certificate version: #3 + * subject: + * start date: Sat, 17 Jun 2017 00:00:00 GMT + * expire date: Thu, 27 Sep 2018 12:00:00 GMT + * issuer: C=US,O=DigiCert Inc,OU=www.digicert.com,CN=DigiCert SHA2 Extended Validation Server CA + * compression: NULL + * ALPN, server accepted to use http/1.1 + > GET / HTTP/1.1 + Host: www.python.org + User-Agent: PycURL/7.43.0.1 libcurl/7.52.1 GnuTLS/3.5.8 zlib/1.2.8 libidn2/0.16 libpsl/0.17.0 (+libidn2/0.16) libssh2/1.7.0 nghttp2/1.18.1 librtmp/2.3 + Accept: */* + + < HTTP/1.1 200 OK + < Server: nginx + < Content-Type: text/html; charset=utf-8 + < X-Frame-Options: SAMEORIGIN + < x-xss-protection: 1; mode=block + < X-Clacks-Overhead: GNU Terry Pratchett + < Via: 1.1 varnish + < Fastly-Debug-Digest: a63ab819df3b185a89db37a59e39f0dd85cf8ee71f54bbb42fae41670ae56fd2 + < Content-Length: 48893 + < Accept-Ranges: bytes + < Date: Thu, 07 Dec 2017 07:28:32 GMT + < Via: 1.1 varnish + < Age: 2497 + < Connection: keep-alive + < X-Served-By: cache-iad2146-IAD, cache-ewr18146-EWR + < X-Cache: HIT, HIT + < X-Cache-Hits: 2, 2 + < X-Timer: S1512631712.274059,VS0,VE0 + < Vary: Cookie + < Strict-Transport-Security: max-age=63072000; includeSubDomains + < + * Curl_http_done: called premature == 0 + * Connection #1 to host www.python.org left intact + >>> + +The verbose output in the above example includes: + +- DNS resolution +- SSL connection +- SSL certificate verification +- Headers sent to the server +- Headers received from the server + +If the verbose output indicates something you believe is incorrect, +the next step is to perform an identical transfer using ``curl`` command-line +utility and verify that the behavior is PycURL-specific, as in most cases +it is not. This is also a good time to check the behavior of the latest +version of libcurl. + +.. _curl.h: https://github.com/curl/curl/blob/master/include/curl/curl.h#L456 +.. _not being compiled in: https://github.com/pycurl/pycurl/issues/477 diff --git a/doc/unicode.rst b/doc/unicode.rst new file mode 100644 index 0000000..9c4b39d --- /dev/null +++ b/doc/unicode.rst @@ -0,0 +1,277 @@ +.. _unicode: + +String And Unicode Handling +=========================== + +Generally speaking, libcurl does not perform data encoding or decoding. +In particular, libcurl is not Unicode-aware, but operates on byte streams. +libcurl leaves it up to the application - PycURL library or an application +using PycURL in this case - to encode and decode Unicode data into byte streams. + +PycURL, being a thin wrapper around libcurl, generally does not perform +this encoding and decoding either, leaving it up to the application. +Specifically: + +- Data that PycURL passes to an application, such as via callback functions, + is normally byte strings. The application must decode them to obtain text + (Unicode) data. +- Data that an application passes to PycURL, such as via ``setopt`` calls, + must normally be byte strings appropriately encoded. For convenience and + compatibility with existing code, PycURL will accept Unicode strings that + contain ASCII code points only [#ascii]_, and transparently encode these to + byte strings. + +Why doesn't PycURL automatically encode and decode, say, HTTP request or +response data? The key to remember is that libcurl supports over 20 protocols, +and PycURL generally has no knowledge of what protocol is being used by +a particular request as PycURL does not track application state. Having +to manually encode and decode data is unfortunately the price of libcurl's +flexibility. + + +Setting Options - Python 2.x +---------------------------- + +Under Python 2, the ``str`` type can hold arbitrary encoded byte strings. +PycURL will pass whatever byte strings it is given verbatim to libcurl. +The following code will work:: + + >>> import pycurl + >>> c = pycurl.Curl() + >>> c.setopt(c.USERAGENT, 'Foo\xa9') + # ok + +Unicode strings can be used but must contain ASCII code points only:: + + >>> c.setopt(c.USERAGENT, u'Foo') + # ok + + >>> c.setopt(c.USERAGENT, u'Foo\xa9') + Traceback (most recent call last): + File "", line 1, in + UnicodeEncodeError: 'ascii' codec can't encode character u'\xa9' in position 3: ordinal not in range(128) + + >>> c.setopt(c.USERAGENT, u'Foo\xa9'.encode('iso-8859-1')) + # ok + + +Setting Options - Python 3.x +---------------------------- + +Under Python 3, the ``bytes`` type holds arbitrary encoded byte strings. +PycURL will accept ``bytes`` values for all options where libcurl specifies +a "string" argument:: + + >>> import pycurl + >>> c = pycurl.Curl() + >>> c.setopt(c.USERAGENT, b'Foo\xa9') + # ok + +The ``str`` type holds Unicode data. PycURL will accept ``str`` values +containing ASCII code points only:: + + >>> c.setopt(c.USERAGENT, 'Foo') + # ok + + >>> c.setopt(c.USERAGENT, 'Foo\xa9') + Traceback (most recent call last): + File "", line 1, in + UnicodeEncodeError: 'ascii' codec can't encode character '\xa9' in position 3: ordinal not in range(128) + + >>> c.setopt(c.USERAGENT, 'Foo\xa9'.encode('iso-8859-1')) + # ok + + +Writing To Files +---------------- + +PycURL will return all data read from the network as byte strings. On Python 2, +this means the write callbacks will receive ``str`` objects, and +on Python 3, write callbacks will receive ``bytes`` objects. + +Under Python 2, when using e.g. ``WRITEDATA`` or ``WRITEFUNCTION`` options, +files being written to *should* be opened in binary mode. Writing to files +opened in text mode will not raise exceptions but may corrupt data. + +Under Python 3, PycURL passes strings and binary data to the application +using ``bytes`` instances. When writing to files, the files must be opened +in binary mode for the writes to work:: + + import pycurl + c = pycurl.Curl() + c.setopt(c.URL,'http://pycurl.io') + # File opened in binary mode. + with open('/dev/null','wb') as f: + c.setopt(c.WRITEDATA, f) + # Same result if using WRITEFUNCTION instead: + #c.setopt(c.WRITEFUNCTION, f.write) + c.perform() + # ok + +If a file is opened in text mode (``w`` instead of ``wb`` mode), an error +similar to the following will result:: + + TypeError: must be str, not bytes + Traceback (most recent call last): + File "/tmp/test.py", line 8, in + c.perform() + pycurl.error: (23, 'Failed writing body (0 != 168)') + +The TypeError is actually an exception raised by Python which will be printed, +but not propagated, by PycURL. PycURL will raise a ``pycurl.error`` to +signify operation failure. + + +Writing To StringIO/BytesIO +--------------------------- + +Under Python 2, response can be saved in memory by using a ``StringIO`` +object:: + + import pycurl + from StringIO import StringIO + c = pycurl.Curl() + c.setopt(c.URL,'http://pycurl.io') + buffer = StringIO() + c.setopt(c.WRITEDATA, buffer) + # Same result if using WRITEFUNCTION instead: + #c.setopt(c.WRITEFUNCTION, buffer.write) + c.perform() + # ok + +Under Python 3, as PycURL invokes the write callback with ``bytes`` argument, +the response must be written to a ``BytesIO`` object:: + + import pycurl + from io import BytesIO + c = pycurl.Curl() + c.setopt(c.URL,'http://pycurl.io') + buffer = BytesIO() + c.setopt(c.WRITEDATA, buffer) + # Same result if using WRITEFUNCTION instead: + #c.setopt(c.WRITEFUNCTION, buffer.write) + c.perform() + # ok + +Attempting to use a ``StringIO`` object will produce an error:: + + import pycurl + from io import StringIO + c = pycurl.Curl() + c.setopt(c.URL,'http://pycurl.io') + buffer = StringIO() + c.setopt(c.WRITEDATA, buffer) + c.perform() + + TypeError: string argument expected, got 'bytes' + Traceback (most recent call last): + File "/tmp/test.py", line 9, in + c.perform() + pycurl.error: (23, 'Failed writing body (0 != 168)') + +The following idiom can be used for code that needs to be compatible with both +Python 2 and Python 3:: + + import pycurl + try: + # Python 3 + from io import BytesIO + except ImportError: + # Python 2 + from StringIO import StringIO as BytesIO + c = pycurl.Curl() + c.setopt(c.URL,'http://pycurl.io') + buffer = BytesIO() + c.setopt(c.WRITEDATA, buffer) + c.perform() + # ok + # Decode the response body: + string_body = buffer.getvalue().decode('utf-8') + + +Header Functions +---------------- + +Although headers are often ASCII text, they are still returned as +``bytes`` instances on Python 3 and thus require appropriate decoding. +HTTP headers are encoded in ISO/IEC 8859-1 according to the standards. + +When using ``WRITEHEADER`` option to write headers to files, the files +should be opened in binary mode in Python 2 and must be opened in binary +mode in Python 3, same as with ``WRITEDATA``. + + +Read Functions +-------------- + +Read functions are expected to provide data in the same fashion as +string options expect it: + +- On Python 2, the data can be given as ``str`` instances, appropriately + encoded. +- On Python 2, the data can be given as ``unicode`` instances containing + ASCII code points only. +- On Python 3, the data can be given as ``bytes`` instances. +- On Python 3. the data can be given as ``str`` instances containing + ASCII code points only. + +Caution: when using CURLOPT_READFUNCTION in tandem with CURLOPT_POSTFIELDSIZE, +as would be done for HTTP for example, take care to pass the length of +*encoded* data to CURLOPT_POSTFIELDSIZE if you are performing the encoding. +If you pass the number of Unicode characters rather than +encoded bytes to libcurl, the server will receive wrong Content-Length. +Alternatively you can return Unicode strings from a CURLOPT_READFUNCTION +function, if your data contains only ASCII code points, +and let PycURL encode them for you. + + +How PycURL Handles Unicode Strings +---------------------------------- + +If PycURL is given a Unicode string which contains non-ASCII code points, +and as such cannot be encoded to ASCII, PycURL will return an error to libcurl, +and libcurl in turn will fail the request with an error like +"read function error/data error". PycURL will then raise ``pycurl.error`` +with this latter message. The encoding exception that was the +underlying cause of the problem is stored as ``sys.last_value``. + + +Figuring Out Correct Encoding +----------------------------- + +What encoding should be used when is a complicated question. For example, +when working with HTTP: + +- URLs and POSTFIELDS data must be URL-encoded. A URL-encoded string has + only ASCII code points. +- Headers must be ISO/IEC 8859-1 encoded. +- Encoding for bodies is specified in Content-Type and Content-Encoding headers. + + +Legacy PycURL Versions +---------------------- + +The Unicode handling documented here was implemented in PycURL 7.19.3 +along with Python 3 support. Prior to PycURL 7.19.3 Unicode data was not +accepted at all:: + + >>> import pycurl + >>> c = pycurl.Curl() + >>> c.setopt(c.USERAGENT, u'Foo\xa9') + Traceback (most recent call last): + File "", line 1, in + TypeError: invalid arguments to setopt + +Some GNU/Linux distributions provided Python 3 packages of PycURL prior to +PycURL 7.19.3. These packages included unofficial patches +([#patch1]_, [#patch2]_) which did not handle Unicode correctly, and did not behave +as described in this document. Such unofficial versions of PycURL should +be avoided. + + +.. rubric:: Footnotes + +.. [#ascii] Only ASCII is accepted; ISO-8859-1/Latin 1, for example, will be + rejected. +.. [#patch1] http://sourceforge.net/p/pycurl/patches/5/ +.. [#patch2] http://sourceforge.net/p/pycurl/patches/12/ diff --git a/doc/unimplemented.rst b/doc/unimplemented.rst new file mode 100644 index 0000000..9af7f17 --- /dev/null +++ b/doc/unimplemented.rst @@ -0,0 +1,65 @@ +Unimplemented Options And Constants +=================================== + +PycURL intentionally does not expose some of the libcurl options and constants. +This document explains libcurl symbols that were omitted from PycURL. + + +``*DATA`` options +----------------- + +In libcurl, the ``*aDATA`` options set *client data* for various callbacks. +Each callback has a corresponding ``*DATA`` option. + +In Python - a language with closures - such options are unnecessary. +For example, the following code invokes an instance's ``write`` method +which has full access to its class instance:: + + class Writer(object): + def __init__(self): + self.foo = True + + def write(chunk): + # can use self.foo + + writer = Writer() + curl = pycurl.Curl() + curl.setopt(curl.WRITEFUNCTION, writer.write) + +As of version 7.19.3, PycURL does implement three ``*DATA`` options for +convenience: +``WRITEDATA``, ``HEADERDATA`` and ``READDATA``. These are equivalent to +setting the respective callback option with either a ``write`` or ``read`` +method, as appropriate:: + + # equivalent pairs: + curl.setopt(curl.WRITEDATA, writer) + curl.setopt(curl.WRITEFUNCTION, writer.write) + + curl.setopt(curl.HEADERDATA, writer) + curl.setopt(curl.HEADERFUNCTION, writer.write) + + curl.setopt(curl.READDATA, reader) + curl.setopt(curl.READFUNCTION, reader.read) + + +``CURLINFO_TLS_SESSION`` +------------------------ + +It is unclear how the SSL context should be exposed to Python code. +This option can be implemented if it finds a use case. + + + +Undocumented symbols +-------------------- + +Some symbols are present in libcurl's `symbols in versions`_ document but +are not documented by libcurl. These symbols are not implemented by PycURL. + +As of this writing, the following symbols are thusly omitted: + +- ``CURLPAUSE_RECV_CONT`` +- ``CURLPAUSE_SEND_CONT`` + +.. _symbols in versions: https://curl.haxx.se/libcurl/c/symbols-in-versions.html diff --git a/examples/basicfirst.py b/examples/basicfirst.py new file mode 100644 index 0000000..ebbbb8b --- /dev/null +++ b/examples/basicfirst.py @@ -0,0 +1,29 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et +import sys +import pycurl + +PY3 = sys.version_info[0] > 2 + + +class Test: + def __init__(self): + self.contents = '' + if PY3: + self.contents = self.contents.encode('ascii') + + def body_callback(self, buf): + self.contents = self.contents + buf + + +sys.stderr.write("Testing %s\n" % pycurl.version) + +t = Test() +c = pycurl.Curl() +c.setopt(c.URL, 'https://curl.haxx.se/dev/') +c.setopt(c.WRITEFUNCTION, t.body_callback) +c.perform() +c.close() + +print(t.contents) diff --git a/examples/file_upload.py b/examples/file_upload.py new file mode 100644 index 0000000..a3b769a --- /dev/null +++ b/examples/file_upload.py @@ -0,0 +1,45 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import os, sys +import pycurl + +# Class which holds a file reference and the read callback +class FileReader: + def __init__(self, fp): + self.fp = fp + def read_callback(self, size): + return self.fp.read(size) + +# Check commandline arguments +if len(sys.argv) < 3: + print("Usage: %s " % sys.argv[0]) + raise SystemExit +url = sys.argv[1] +filename = sys.argv[2] + +if not os.path.exists(filename): + print("Error: the file '%s' does not exist" % filename) + raise SystemExit + +# Initialize pycurl +c = pycurl.Curl() +c.setopt(pycurl.URL, url) +c.setopt(pycurl.UPLOAD, 1) + +# Two versions with the same semantics here, but the filereader version +# is useful when you have to process the data which is read before returning +if 1: + c.setopt(pycurl.READFUNCTION, FileReader(open(filename, 'rb')).read_callback) +else: + c.setopt(pycurl.READFUNCTION, open(filename, 'rb').read) + +# Set size of file to be uploaded. +filesize = os.path.getsize(filename) +c.setopt(pycurl.INFILESIZE, filesize) + +# Start transfer +print('Uploading file %s to url %s' % (filename, url)) +c.perform() +c.close() diff --git a/examples/linksys.py b/examples/linksys.py new file mode 100644 index 0000000..116e8ed --- /dev/null +++ b/examples/linksys.py @@ -0,0 +1,568 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et +# +# linksys.py -- program settings on a Linkys router +# +# This tool is designed to help you recover from the occasional episodes +# of catatonia that afflict Linksys boxes. It allows you to batch-program +# them rather than manually entering values to the Web interface. Commands +# are taken from the command line first, then standard input. +# +# The somewhat spotty coverage of status queries is because I only did the +# ones that were either (a) easy, or (b) necessary. If you want to know the +# status of the box, look at the web interface. +# +# This code has been tested against the following hardware: +# +# Hardware Firmware +# ---------- --------------------- +# BEFW11S4v2 1.44.2.1, Dec 20 2002 +# +# The code is, of course, sensitive to changes in the names of CGI pages +# and field names. +# +# Note: to make the no-arguments form work, you'll need to have the following +# entry in your ~/.netrc file. If you have changed the router IP address or +# name/password, modify accordingly. +# +# machine 192.168.1.1 +# login "" +# password admin +# +# By Eric S. Raymond, August April 2003. All rites reversed. + +import sys, re, curl, exceptions + +def print_stderr(arg): + sys.stderr.write(arg) + sys.stderr.write("\n") + +class LinksysError(exceptions.Exception): + def __init__(self, *args): + self.args = args + +class LinksysSession: + months = 'Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec' + + WAN_CONNECT_AUTO = '1' + WAN_CONNECT_STATIC = '2' + WAN_CONNECT_PPOE = '3' + WAN_CONNECT_RAS = '4' + WAN_CONNECT_PPTP = '5' + WAN_CONNECT_HEARTBEAT = '6' + + # Substrings to check for on each page load. + # This may enable us to detect when a firmware change has hosed us. + check_strings = { + "": "basic setup functions", + "Passwd.htm": "For security reasons,", + "DHCP.html": "You can configure the router to act as a DHCP", + "Log.html": "There are some log settings and lists in this page.", + "Forward.htm":"Port forwarding can be used to set up public services", + } + + def __init__(self): + self.actions = [] + self.host = "http://192.168.1.1" + self.verbosity = False + self.pagecache = {} + + def set_verbosity(self, flag): + self.verbosity = flag + + # This is not a performance hack -- we need the page cache to do + # sanity checks at configure time. + def cache_load(self, page): + if page not in self.pagecache: + fetch = curl.Curl(self.host) + fetch.set_verbosity(self.verbosity) + fetch.get(page) + self.pagecache[page] = fetch.body() + if fetch.answered("401"): + raise LinksysError("authorization failure.", True) + elif not fetch.answered(LinksysSession.check_strings[page]): + del self.pagecache[page] + raise LinksysError("check string for page %s missing!" % os.path.join(self.host, page), False) + fetch.close() + def cache_flush(self): + self.pagecache = {} + + # Primitives + def screen_scrape(self, page, template): + self.cache_load(page) + match = re.compile(template).search(self.pagecache[page]) + if match: + result = match.group(1) + else: + result = None + return result + def get_MAC_address(self, page, prefix): + return self.screen_scrape("", prefix+r":[^M]*\(MAC Address: *([^)]*)") + def set_flag(self, page, flag, value): + if value: + self.actions.append(page, flag, "1") + else: + self.actions.append(page, flag, "0") + def set_IP_address(self, page, cgi, role, ip): + ind = 0 + for octet in ip.split("."): + self.actions.append(("", "F1", role + repr(ind+1), octet)) + ind += 1 + + # Scrape configuration data off the main page + def get_firmware_version(self): + # This is fragile. There is no distinguishing tag before the firmware + # version, so we have to key off the pattern of the version number. + # Our model is ">1.44.2.1, Dec 20 2002<" + return self.screen_scrape("", ">([0-9.v]*, (" + \ + LinksysSession.months + ")[^<]*)<", ) + def get_LAN_MAC(self): + return self.get_MAC_address("", r"LAN IP Address") + def get_Wireless_MAC(self): + return self.get_MAC_address("", r"Wireless") + def get_WAN_MAC(self): + return self.get_MAC_address("", r"WAN Connection Type") + + # Set configuration data on the main page + def set_host_name(self, name): + self.actions.append(("", "hostName", name)) + def set_domain_name(self, name): + self.actions.append(("", "DomainName", name)) + def set_LAN_IP(self, ip): + self.set_IP_address("", "ipAddr", ip) + def set_LAN_netmask(self, ip): + if not ip.startswith("255.255.255."): + raise ValueError + lastquad = ip.split(".")[-1] + if lastquad not in ("0", "128", "192", "240", "252"): + raise ValueError + self.actions.append("", "netMask", lastquad) + def set_wireless(self, flag): + self.set_flag("", "wirelessStatus") + def set_SSID(self, ssid): + self.actions.append(("", "wirelessESSID", ssid)) + def set_SSID_broadcast(self, flag): + self.set_flag("", "broadcastSSID") + def set_channel(self, channel): + self.actions.append(("", "wirelessChannel", channel)) + def set_WEP(self, flag): + self.set_flag("", "WepType") + # FIXME: Add support for setting WEP keys + def set_connection_type(self, type): + self.actions.append(("", "WANConnectionType", type)) + def set_WAN_IP(self, ip): + self.set_IP_address("", "aliasIP", ip) + def set_WAN_netmask(self, ip): + self.set_IP_address("", "aliasMaskIP", ip) + def set_WAN_gateway_address(self, ip): + self.set_IP_address("", "routerIP", ip) + def set_DNS_server(self, index, ip): + self.set_IP_address("", "dns" + "ABC"[index], ip) + + # Set configuration data on the password page + def set_password(self, str): + self.actions.append("Passwd.htm","sysPasswd", str) + self.actions.append("Passwd.htm","sysPasswdConfirm", str) + def set_UPnP(self, flag): + self.set_flag("Passwd.htm", "UPnP_Work") + def reset(self): + self.actions.append("Passwd.htm", "FactoryDefaults") + + # DHCP features + def set_DHCP(self, flag): + if flag: + self.actions.append("DHCP.htm","dhcpStatus","Enable") + else: + self.actions.append("DHCP.htm","dhcpStatus","Disable") + def set_DHCP_starting_IP(self, val): + self.actions.append("DHCP.htm","dhcpS4", str(val)) + def set_DHCP_users(self, val): + self.actions.append("DHCP.htm","dhcpLen", str(val)) + def set_DHCP_lease_time(self, val): + self.actions.append("DHCP.htm","leaseTime", str(val)) + def set_DHCP_DNS_server(self, index, ip): + self.set_IP_address("DHCP.htm", "dns" + "ABC"[index], ip) + # FIXME: add support for setting WINS key + + # Logging features + def set_logging(self, flag): + if flag: + self.actions.append("Log.htm", "rLog", "Enable") + else: + self.actions.append("Log.htm", "rLog", "Disable") + def set_log_address(self, val): + self.actions.append("DHCP.htm","trapAddr3", str(val)) + + # The AOL parental control flag is not supported by design. + + # FIXME: add Filters and other advanced features + + def configure(self): + "Write configuration changes to the Linksys." + if self.actions: + fields = [] + self.cache_flush() + for (page, field, value) in self.actions: + self.cache_load(page) + if self.pagecache[page].find(field) == -1: + print_stderr("linksys: field %s not found where expected in page %s!" % (field, os.path.join(self.host, page))) + continue + else: + fields.append((field, value)) + # Clearing the action list before fieldsping is deliberate. + # Otherwise we could get permanently wedged by a 401. + self.actions = [] + transaction = curl.Curl(self.host) + transaction.set_verbosity(self.verbosity) + transaction.get("Gozila.cgi", tuple(fields)) + transaction.close() + +if __name__ == "__main__": + import os, cmd + + class LinksysInterpreter(cmd.Cmd): + """Interpret commands to perform LinkSys programming actions.""" + def __init__(self): + cmd.Cmd.__init__(self) + self.session = LinksysSession() + if os.isatty(0): + print("Type ? or `help' for help.") + self.prompt = self.session.host + ": " + else: + self.prompt = "" + print("Bar1") + + def flag_command(self, func, line): + if line.strip() in ("on", "enable", "yes"): + func(True) + elif line.strip() in ("off", "disable", "no"): + func(False) + else: + print_stderr("linksys: unknown switch value") + return 0 + + def do_connect(self, line): + newhost = line.strip() + if newhost: + self.session.host = newhost + self.session.cache_flush() + self.prompt = self.session.host + ": " + else: + print(self.session.host) + return 0 + def help_connect(self): + print("Usage: connect []") + print("Connect to a Linksys by name or IP address.") + print("If no argument is given, print the current host.") + + def do_status(self, line): + self.session.cache_load("") + if "" in self.session.pagecache: + print("Firmware:", self.session.get_firmware_version()) + print("LAN MAC:", self.session.get_LAN_MAC()) + print("Wireless MAC:", self.session.get_Wireless_MAC()) + print("WAN MAC:", self.session.get_WAN_MAC()) + print(".") + return 0 + def help_status(self): + print("Usage: status") + print("The status command shows the status of the Linksys.") + print("It is mainly useful as a sanity check to make sure") + print("the box is responding correctly.") + + def do_verbose(self, line): + self.flag_command(self.session.set_verbosity, line) + def help_verbose(self): + print("Usage: verbose {on|off|enable|disable|yes|no}") + print("Enables display of HTTP requests.") + + def do_host(self, line): + self.session.set_host_name(line) + return 0 + def help_host(self): + print("Usage: host ") + print("Sets the Host field to be queried by the ISP.") + + def do_domain(self, line): + print("Usage: host ") + self.session.set_domain_name(line) + return 0 + def help_domain(self): + print("Sets the Domain field to be queried by the ISP.") + + def do_lan_address(self, line): + self.session.set_LAN_IP(line) + return 0 + def help_lan_address(self): + print("Usage: lan_address ") + print("Sets the LAN IP address.") + + def do_lan_netmask(self, line): + self.session.set_LAN_netmask(line) + return 0 + def help_lan_netmask(self): + print("Usage: lan_netmask ") + print("Sets the LAN subnetwork mask.") + + def do_wireless(self, line): + self.flag_command(self.session.set_wireless, line) + return 0 + def help_wireless(self): + print("Usage: wireless {on|off|enable|disable|yes|no}") + print("Switch to enable or disable wireless features.") + + def do_ssid(self, line): + self.session.set_SSID(line) + return 0 + def help_ssid(self): + print("Usage: ssid ") + print("Sets the SSID used to control wireless access.") + + def do_ssid_broadcast(self, line): + self.flag_command(self.session.set_SSID_broadcast, line) + return 0 + def help_ssid_broadcast(self): + print("Usage: ssid_broadcast {on|off|enable|disable|yes|no}") + print("Switch to enable or disable SSID broadcast.") + + def do_channel(self, line): + self.session.set_channel(line) + return 0 + def help_channel(self): + print("Usage: channel ") + print("Sets the wireless channel.") + + def do_wep(self, line): + self.flag_command(self.session.set_WEP, line) + return 0 + def help_wep(self): + print("Usage: wep {on|off|enable|disable|yes|no}") + print("Switch to enable or disable WEP security.") + + def do_wan_type(self, line): + try: + type=eval("LinksysSession.WAN_CONNECT_"+line.strip().upper()) + self.session.set_connection_type(type) + except ValueError: + print_stderr("linksys: unknown connection type.") + return 0 + def help_wan_type(self): + print("Usage: wan_type {auto|static|ppoe|ras|pptp|heartbeat}") + print("Set the WAN connection type.") + + def do_wan_address(self, line): + self.session.set_WAN_IP(line) + return 0 + def help_wan_address(self): + print("Usage: wan_address ") + print("Sets the WAN IP address.") + + def do_wan_netmask(self, line): + self.session.set_WAN_netmask(line) + return 0 + def help_wan_netmask(self): + print("Usage: wan_netmask ") + print("Sets the WAN subnetwork mask.") + + def do_wan_gateway(self, line): + self.session.set_WAN_gateway(line) + return 0 + def help_wan_gateway(self): + print("Usage: wan_gateway ") + print("Sets the LAN subnetwork mask.") + + def do_dns(self, line): + (index, address) = line.split() + if index in ("1", "2", "3"): + self.session.set_DNS_server(eval(index), address) + else: + print_stderr("linksys: server index out of bounds.") + return 0 + def help_dns(self): + print("Usage: dns {1|2|3} ") + print("Sets a primary, secondary, or tertiary DNS server address.") + + def do_password(self, line): + self.session.set_password(line) + return 0 + def help_password(self): + print("Usage: password ") + print("Sets the router password.") + + def do_upnp(self, line): + self.flag_command(self.session.set_UPnP, line) + return 0 + def help_upnp(self): + print("Usage: upnp {on|off|enable|disable|yes|no}") + print("Switch to enable or disable Universal Plug and Play.") + + def do_reset(self, line): + self.session.reset() + def help_reset(self): + print("Usage: reset") + print("Reset Linksys settings to factory defaults.") + + def do_dhcp(self, line): + self.flag_command(self.session.set_DHCP, line) + def help_dhcp(self): + print("Usage: dhcp {on|off|enable|disable|yes|no}") + print("Switch to enable or disable DHCP features.") + + def do_dhcp_start(self, line): + self.session.set_DHCP_starting_IP(line) + def help_dhcp_start(self): + print("Usage: dhcp_start ") + print("Set the start address of the DHCP pool.") + + def do_dhcp_users(self, line): + self.session.set_DHCP_users(line) + def help_dhcp_users(self): + print("Usage: dhcp_users ") + print("Set number of address slots to allocate in the DHCP pool.") + + def do_dhcp_lease(self, line): + self.session.set_DHCP_lease(line) + def help_dhcp_lease(self): + print("Usage: dhcp_lease ") + print("Set number of address slots to allocate in the DHCP pool.") + + def do_dhcp_dns(self, line): + (index, address) = line.split() + if index in ("1", "2", "3"): + self.session.set_DHCP_DNS_server(eval(index), address) + else: + print_stderr("linksys: server index out of bounds.") + return 0 + def help_dhcp_dns(self): + print("Usage: dhcp_dns {1|2|3} ") + print("Sets primary, secondary, or tertiary DNS server address.") + + def do_logging(self, line): + self.flag_command(self.session.set_logging, line) + def help_logging(self): + print("Usage: logging {on|off|enable|disable|yes|no}") + print("Switch to enable or disable session logging.") + + def do_log_address(self, line): + self.session.set_Log_address(line) + def help_log_address(self): + print("Usage: log_address ") + print("Set the last quad of the address to which to log.") + + def do_configure(self, line): + self.session.configure() + return 0 + def help_configure(self): + print("Usage: configure") + print("Writes the configuration to the Linksys.") + + def do_cache(self, line): + print(self.session.pagecache) + def help_cache(self): + print("Usage: cache") + print("Display the page cache.") + + def do_quit(self, line): + return 1 + def help_quit(self, line): + print("The quit command ends your linksys session without") + print("writing configuration changes to the Linksys.") + def do_EOF(self, line): + print("") + self.session.configure() + return 1 + def help_EOF(self): + print("The EOF command writes the configuration to the linksys") + print("and ends your session.") + + def default(self, line): + """Pass the command through to be executed by the shell.""" + os.system(line) + return 0 + + def help_help(self): + print("On-line help is available through this command.") + print("? is a convenience alias for help.") + + def help_introduction(self): + print("""\ + +This program supports changing the settings on Linksys blue-box routers. This +capability may come in handy when they freeze up and have to be reset. Though +it can be used interactively (and will command-prompt when standard input is a +terminal) it is really designed to be used in batch mode. Commands are taken +from the command line first, then standard input. + +By default, it is assumed that the Linksys is at http://192.168.1.1, the +default LAN address. You can connect to a different address or IP with the +'connect' command. Note that your .netrc must contain correct user/password +credentials for the router. The entry corresponding to the defaults is: + +machine 192.168.1.1 + login "" + password admin + +Most commands queue up changes but don't actually send them to the Linksys. +You can force pending changes to be written with 'configure'. Otherwise, they +will be shipped to the Linksys at the end of session (e.g. when the program +running in batch mode encounters end-of-file or you type a control-D). If you +end the session with `quit', pending changes will be discarded. + +For more help, read the topics 'wan', 'lan', and 'wireless'.""") + + def help_lan(self): + print("""\ +The `lan_address' and `lan_netmask' commands let you set the IP location of +the Linksys on your LAN, or inside. Normally you'll want to leave these +untouched.""") + + def help_wan(self): + print("""\ +The WAN commands become significant if you are using the BEFSR41 or any of +the other Linksys boxes designed as DSL or cable-modem gateways. You will +need to use `wan_type' to declare how you expect to get your address. + +If your ISP has issued you a static address, you'll need to use the +`wan_address', `wan_netmask', and `wan_gateway' commands to set the address +of the router as seen from the WAN, the outside. In this case you will also +need to use the `dns' command to declare which remote servers your DNS +requests should be forwarded to. + +Some ISPs may require you to set host and domain for use with dynamic-address +allocation.""") + + def help_wireless_desc(self): + print("""\ +The channel, ssid, ssid_broadcast, wep, and wireless commands control +wireless routing.""") + + def help_switches(self): + print("Switches may be turned on with 'on', 'enable', or 'yes'.") + print("Switches may be turned off with 'off', 'disable', or 'no'.") + print("Switch commands include: wireless, ssid_broadcast.") + + def help_addresses(self): + print("An address argument must be a valid IP address;") + print("four decimal numbers separated by dots, each ") + print("between 0 and 255.") + + def emptyline(self): + pass + + interpreter = LinksysInterpreter() + for arg in sys.argv[1:]: + interpreter.onecmd(arg) + fatal = False + while not fatal: + try: + interpreter.cmdloop() + fatal = True + except LinksysError: + message, fatal = sys.exc_info()[1].args + print("linksys: " + message) + +# The following sets edit modes for GNU EMACS +# Local Variables: +# mode:python +# End: diff --git a/examples/multi-socket_action-select.py b/examples/multi-socket_action-select.py new file mode 100644 index 0000000..969447c --- /dev/null +++ b/examples/multi-socket_action-select.py @@ -0,0 +1,197 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +# Retrieves a single URL using the CurlMulti.socket_action calls, using +# select as the I/O polling mechanism: +# +# First, create a Multi object, and set socket and timer callbacks on it. +# Observed side effect: this causes the timer callback to be immediately +# invoked with the zero value for the timeout. +# +# The timer callback is very simple - it stores the timeout value passed +# into it in the global state for future use by the select calls that +# we will be making. +# +# The socket callback is more complicated. Its job is to add and remove +# socket handles to/from the data structure that we use for waiting for +# activity on them. The callback is invoked with a socket handle and the +# needed action (add for reading, add for writing or remove). +# Since this script utilizes the select call for waiting for activity, +# the socket callback updates the list of sockets which we should be +# polling for readability and the list that we should be polling for +# writability, which are then passed to the select call (and both of the +# sets are passed as the sockets to wait for errors/exceptions on). +# +# Next, create a Curl object (mapping to a libcurl easy handle), set the URL +# we are going to retrieve as well as any transfer options. This script sets +# the timeout to 5 seconds to be able to test failing transfers easily. +# +# Add the Curl object to the Multi object. +# +# Invoke Multi.socket_action to start the retrieval operation. +# Observed side effect: this causes the timer callback to be invoked +# with a greater than zero value for the timeout. +# +# By now we should have initialized our own state, which this script does +# prior to invoking any libcurl functions. Importantly, the state includes +# the timeout value that was communicated to us by libcurl. +# +# Run a loop which waits for activity on any of the sockets used by libcurl. +# The sockets are set that the socket callback has produced as of the +# present moment; the timeout is the most recent timeout value received by +# the timer callback. +# +# Importantly, the loop should not simply sleep for the entire +# timeout interval, as that would cause the transfer to take a very long time. +# It is *required* to use something like a select call to wait for activity +# on any of the sockets currently active for *up to* the timeout value. +# +# The loop terminates when the number of active transfers (handles in libcurl +# parlance) reaches zero. This number is provided by each socket_action +# call, which is why each call (even the ones that are called due to +# timeout being reached, as opposed to any socket activity) must update +# the number of running handles. +# +# After the loop terminates, clean up everything: remove the easy object from +# the multi object, close the easy object, close the multi object. + +import sys, select +import pycurl +from io import BytesIO + +if len(sys.argv) > 1: + url = sys.argv[1] +else: + url = 'https://www.python.org' + +state = { + 'rlist': [], + 'wlist': [], + 'running': None, + 'timeout': None, + 'result': None, + # If the transfer failed, code and msg will be filled in. + 'code': None, + 'msg': None, +} + +def socket_fn(what, sock_fd, multi, socketp): + if what == pycurl.POLL_IN or what == pycurl.POLL_INOUT: + state['rlist'].append(sock_fd) + elif what == pycurl.POLL_OUT or what == pycurl.POLL_INOUT: + state['wlist'].append(sock_fd) + elif what == pycurl.POLL_REMOVE: + if sock_fd in state['rlist']: + state['rlist'].remove(sock_fd) + if sock_fd in state['wlist']: + state['wlist'].remove(sock_fd) + else: + raise Exception("Unknown value of what: %s" % what) + +def work(timeout): + rready, wready, xready = select.select( + state['rlist'], state['wlist'], set(state['rlist']) | set(state['wlist']), timeout) + + if len(rready) == 0 and len(wready) == 0 and len(xready) == 0: + # The number of running handles must be updated after each + # call to socket_action, which includes those with the SOCKET_TIMEOUT + # argument (otherwise e.g. a transfer which failed due to + # exceeding the connection timeout would hang). + _, running = multi.socket_action(pycurl.SOCKET_TIMEOUT, 0) + else: + for sock_fd in rready: + # socket_action returns a tuple whose first element is always the + # CURLE_OK value (0), ignore it and use the second element only. + _, running = multi.socket_action(sock_fd, pycurl.CSELECT_IN) + for sock_fd in wready: + _, running = multi.socket_action(sock_fd, pycurl.CSELECT_OUT) + for sock_fd in xready: + _, running = multi.socket_action(sock_fd, pycurl.CSELECT_ERR) + + # Since we are only performing a single transfer, we could call + # Multi.info_read after the I/O loop terminates. + # In practice, you would probably use socket_action with multiple + # transfers, and you may want to be notified about transfer completion + # as soon as the result is available. + if state['running'] is not None and running != state['running']: + # Some handle has completed. + # + # Note that socket_action was potentially called multiple times + # in this function (e.g. if both a read handle became ready and a + # different write handle became ready), therefore it is possible + # that multiple handles have completed. In this particular script + # we are only performing a single transfer (one + # Curl object / easy handle), therefore only one transfer can ever + # possibly complete. + qmsg, successes, failures = multi.info_read() + # We should have retrieved all of the available statuses, leaving + # none in the queue. + assert qmsg == 0 + + # We have only one transfer. + assert len(successes) == 1 and len(failures) == 0 or \ + len(successes) == 0 and len(failures) == 1 + + if successes: + state['result'] = True + if failures: + state['result'] = False + # The failures array contains tuples of + # (easy object, CURLE code, error message). + _easy, state['code'], state['msg'] = failures[0] + + state['running'] = running + +def timer_fn(timeout_ms): + if timeout_ms < 0: + # libcurl passes a negative timeout value when no further + # calls should be made. + state['timeout'] = None + state['timeout'] = timeout_ms / 1000.0 + +multi = pycurl.CurlMulti() +multi.setopt(pycurl.M_SOCKETFUNCTION, socket_fn) +multi.setopt(pycurl.M_TIMERFUNCTION, timer_fn) + +easy = pycurl.Curl() +easy.setopt(pycurl.URL, url) +# Uncomment to see what libcurl is doing throughout the transfer. +#easy.setopt(pycurl.VERBOSE, 1) +easy.setopt(pycurl.CONNECTTIMEOUT, 5) +easy.setopt(pycurl.LOW_SPEED_TIME, 5) +easy.setopt(pycurl.LOW_SPEED_LIMIT, 1) +_io = BytesIO() +easy.setopt(pycurl.WRITEDATA, _io) + +multi.add_handle(easy) + +handles = multi.socket_action(pycurl.SOCKET_TIMEOUT, 0) +# This should invoke the timer function with a timeout value. + +while True: + if state['running'] == 0: + break + else: + # By the time we get here, timer function should have been already + # invoked at least once so that we have a libcurl-supplied + # timeout value. But in case this hasn't happened, default the timeout + # to 1 second. + timeout = state['timeout'] + if timeout is None: + raise Exception('Need to poll for I/O but the timeout is not set!') + work(timeout) + +multi.remove_handle(easy) +easy.close() +multi.close() + +# Uncomment to print the retrieved contents. +#print(_io.getvalue().decode()) + +if state['result'] is None: + raise Exception('Script finished without a result!') +if state['result']: + print('Transfer successful, retrieved %d bytes' % len(_io.getvalue())) +else: + print('Transfer failed with code %d: %s' % (state['code'], state['msg'])) diff --git a/examples/opensocketexception.py b/examples/opensocketexception.py new file mode 100644 index 0000000..5c50344 --- /dev/null +++ b/examples/opensocketexception.py @@ -0,0 +1,30 @@ +# Exposing rich exception information from callbacks example + +import pycurl, random, socket + +class ConnectionRejected(Exception): + pass + +def opensocket(curl, purpose, curl_address): + if random.random() < 0.5: + curl.exception = ConnectionRejected('Rejecting connection attempt in opensocket callback') + return pycurl.SOCKET_BAD + + family, socktype, protocol, address = curl_address + s = socket.socket(family, socktype, protocol) + s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + return s + +c = pycurl.Curl() +c.setopt(c.URL, 'http://pycurl.io') +c.exception = None +c.setopt(c.OPENSOCKETFUNCTION, + lambda purpose, address: opensocket(c, purpose, address)) + +try: + c.perform() +except pycurl.error as e: + if e.args[0] == pycurl.E_COULDNT_CONNECT and c.exception: + print(c.exception) + else: + print(e) diff --git a/examples/quickstart/file_upload_buffer.py b/examples/quickstart/file_upload_buffer.py new file mode 100644 index 0000000..8834da4 --- /dev/null +++ b/examples/quickstart/file_upload_buffer.py @@ -0,0 +1,18 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl + +c = pycurl.Curl() +c.setopt(c.URL, 'https://httpbin.org/post') + +c.setopt(c.HTTPPOST, [ + ('fileupload', ( + c.FORM_BUFFER, 'readme.txt', + c.FORM_BUFFERPTR, 'This is a fancy readme file', + )), +]) + +c.perform() +c.close() diff --git a/examples/quickstart/file_upload_real.py b/examples/quickstart/file_upload_real.py new file mode 100644 index 0000000..9ea4d13 --- /dev/null +++ b/examples/quickstart/file_upload_real.py @@ -0,0 +1,18 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl + +c = pycurl.Curl() +c.setopt(c.URL, 'https://httpbin.org/post') + +c.setopt(c.HTTPPOST, [ + ('fileupload', ( + # upload the contents of this file + c.FORM_FILE, __file__, + )), +]) + +c.perform() +c.close() diff --git a/examples/quickstart/file_upload_real_fancy.py b/examples/quickstart/file_upload_real_fancy.py new file mode 100644 index 0000000..ce7414c --- /dev/null +++ b/examples/quickstart/file_upload_real_fancy.py @@ -0,0 +1,22 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl + +c = pycurl.Curl() +c.setopt(c.URL, 'https://httpbin.org/post') + +c.setopt(c.HTTPPOST, [ + ('fileupload', ( + # upload the contents of this file + c.FORM_FILE, __file__, + # specify a different file name for the upload + c.FORM_FILENAME, 'helloworld.py', + # specify a different content type + c.FORM_CONTENTTYPE, 'application/x-python', + )), +]) + +c.perform() +c.close() diff --git a/examples/quickstart/follow_redirect.py b/examples/quickstart/follow_redirect.py new file mode 100644 index 0000000..9e3e3e4 --- /dev/null +++ b/examples/quickstart/follow_redirect.py @@ -0,0 +1,13 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl + +c = pycurl.Curl() +# Redirects to https://www.python.org/. +c.setopt(c.URL, 'http://www.python.org/') +# Follow redirect. +c.setopt(c.FOLLOWLOCATION, True) +c.perform() +c.close() diff --git a/examples/quickstart/form_post.py b/examples/quickstart/form_post.py new file mode 100644 index 0000000..c366b37 --- /dev/null +++ b/examples/quickstart/form_post.py @@ -0,0 +1,25 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +try: + # python 3 + from urllib.parse import urlencode +except ImportError: + # python 2 + from urllib import urlencode + +c = pycurl.Curl() +c.setopt(c.URL, 'https://httpbin.org/post') + +post_data = {'field': 'value'} +# Form data must be provided already urlencoded. +postfields = urlencode(post_data) +# Sets request method to POST, +# Content-Type header to application/x-www-form-urlencoded +# and data to send in request body. +c.setopt(c.POSTFIELDS, postfields) + +c.perform() +c.close() diff --git a/examples/quickstart/get.py b/examples/quickstart/get.py new file mode 100644 index 0000000..e3496f9 --- /dev/null +++ b/examples/quickstart/get.py @@ -0,0 +1,24 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +try: + from io import BytesIO +except ImportError: + from StringIO import StringIO as BytesIO + +buffer = BytesIO() +c = pycurl.Curl() +c.setopt(c.URL, 'http://pycurl.io/') +c.setopt(c.WRITEDATA, buffer) +# For older PycURL versions: +#c.setopt(c.WRITEFUNCTION, buffer.write) +c.perform() +c.close() + +body = buffer.getvalue() +# Body is a string on Python 2 and a byte string on Python 3. +# If we know the encoding, we can always decode the body and +# end up with a Unicode string. +print(body.decode('iso-8859-1')) diff --git a/examples/quickstart/get_python2.py b/examples/quickstart/get_python2.py new file mode 100644 index 0000000..91c14b7 --- /dev/null +++ b/examples/quickstart/get_python2.py @@ -0,0 +1,20 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +from StringIO import StringIO + +buffer = StringIO() +c = pycurl.Curl() +c.setopt(c.URL, 'http://pycurl.io/') +c.setopt(c.WRITEDATA, buffer) +# For older PycURL versions: +#c.setopt(c.WRITEFUNCTION, buffer.write) +c.perform() +c.close() + +body = buffer.getvalue() +# Body is a string in some encoding. +# In Python 2, we can print it without knowing what the encoding is. +print(body) diff --git a/examples/quickstart/get_python2_https.py b/examples/quickstart/get_python2_https.py new file mode 100644 index 0000000..f51f368 --- /dev/null +++ b/examples/quickstart/get_python2_https.py @@ -0,0 +1,22 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import certifi +from StringIO import StringIO + +buffer = StringIO() +c = pycurl.Curl() +c.setopt(c.URL, 'http://pycurl.io/') +c.setopt(c.WRITEDATA, buffer) +# For older PycURL versions: +#c.setopt(c.WRITEFUNCTION, buffer.write) +c.setopt(c.CAINFO, certifi.where()) +c.perform() +c.close() + +body = buffer.getvalue() +# Body is a string in some encoding. +# In Python 2, we can print it without knowing what the encoding is. +print(body) diff --git a/examples/quickstart/get_python3.py b/examples/quickstart/get_python3.py new file mode 100644 index 0000000..31a1123 --- /dev/null +++ b/examples/quickstart/get_python3.py @@ -0,0 +1,19 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +from io import BytesIO + +buffer = BytesIO() +c = pycurl.Curl() +c.setopt(c.URL, 'http://pycurl.io/') +c.setopt(c.WRITEDATA, buffer) +c.perform() +c.close() + +body = buffer.getvalue() +# Body is a byte string. +# We have to know the encoding in order to print it to a text file +# such as standard output. +print(body.decode('iso-8859-1')) diff --git a/examples/quickstart/get_python3_https.py b/examples/quickstart/get_python3_https.py new file mode 100644 index 0000000..e26fbd5 --- /dev/null +++ b/examples/quickstart/get_python3_https.py @@ -0,0 +1,21 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import certifi +from io import BytesIO + +buffer = BytesIO() +c = pycurl.Curl() +c.setopt(c.URL, 'http://pycurl.io/') +c.setopt(c.WRITEDATA, buffer) +c.setopt(c.CAINFO, certifi.where()) +c.perform() +c.close() + +body = buffer.getvalue() +# Body is a byte string. +# We have to know the encoding in order to print it to a text file +# such as standard output. +print(body.decode('iso-8859-1')) diff --git a/examples/quickstart/put_buffer.py b/examples/quickstart/put_buffer.py new file mode 100644 index 0000000..4ff753b --- /dev/null +++ b/examples/quickstart/put_buffer.py @@ -0,0 +1,22 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +try: + from io import BytesIO +except ImportError: + from StringIO import StringIO as BytesIO + +c = pycurl.Curl() +c.setopt(c.URL, 'https://httpbin.org/put') + +c.setopt(c.UPLOAD, 1) +data = '{"json":true}' +# READDATA requires an IO-like object; a string is not accepted +# encode() is necessary for Python 3 +buffer = BytesIO(data.encode('utf-8')) +c.setopt(c.READDATA, buffer) + +c.perform() +c.close() diff --git a/examples/quickstart/put_file.py b/examples/quickstart/put_file.py new file mode 100644 index 0000000..533c569 --- /dev/null +++ b/examples/quickstart/put_file.py @@ -0,0 +1,17 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl + +c = pycurl.Curl() +c.setopt(c.URL, 'https://httpbin.org/put') + +c.setopt(c.UPLOAD, 1) +file = open(__file__) +c.setopt(c.READDATA, file) + +c.perform() +c.close() +# File must be kept open while Curl object is using it +file.close() diff --git a/examples/quickstart/response_headers.py b/examples/quickstart/response_headers.py new file mode 100644 index 0000000..ca98562 --- /dev/null +++ b/examples/quickstart/response_headers.py @@ -0,0 +1,68 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import re +try: + from io import BytesIO +except ImportError: + from StringIO import StringIO as BytesIO + +headers = {} +def header_function(header_line): + # HTTP standard specifies that headers are encoded in iso-8859-1. + # On Python 2, decoding step can be skipped. + # On Python 3, decoding step is required. + header_line = header_line.decode('iso-8859-1') + + # Header lines include the first status line (HTTP/1.x ...). + # We are going to ignore all lines that don't have a colon in them. + # This will botch headers that are split on multiple lines... + if ':' not in header_line: + return + + # Break the header line into header name and value. + name, value = header_line.split(':', 1) + + # Remove whitespace that may be present. + # Header lines include the trailing newline, and there may be whitespace + # around the colon. + name = name.strip() + value = value.strip() + + # Header names are case insensitive. + # Lowercase name here. + name = name.lower() + + # Now we can actually record the header name and value. + headers[name] = value + +buffer = BytesIO() +c = pycurl.Curl() +c.setopt(c.URL, 'http://pycurl.io') +c.setopt(c.WRITEFUNCTION, buffer.write) +# Set our header function. +c.setopt(c.HEADERFUNCTION, header_function) +c.perform() +c.close() + +# Figure out what encoding was sent with the response, if any. +# Check against lowercased header name. +encoding = None +if 'content-type' in headers: + content_type = headers['content-type'].lower() + match = re.search('charset=(\S+)', content_type) + if match: + encoding = match.group(1) + print('Decoding using %s' % encoding) +if encoding is None: + # Default encoding for HTML is iso-8859-1. + # Other content types may have different default encoding, + # or in case of binary data, may have no encoding at all. + encoding = 'iso-8859-1' + print('Assuming encoding is %s' % encoding) + +body = buffer.getvalue() +# Decode using the encoding we figured out. +print(body.decode(encoding)) diff --git a/examples/quickstart/response_info.py b/examples/quickstart/response_info.py new file mode 100644 index 0000000..875d7ff --- /dev/null +++ b/examples/quickstart/response_info.py @@ -0,0 +1,23 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +try: + from io import BytesIO +except ImportError: + from StringIO import StringIO as BytesIO + +buffer = BytesIO() +c = pycurl.Curl() +c.setopt(c.URL, 'http://pycurl.io/') +c.setopt(c.WRITEDATA, buffer) +c.perform() + +# HTTP response code, e.g. 200. +print('Status: %d' % c.getinfo(c.RESPONSE_CODE)) +# Elapsed time for the transfer. +print('Time: %f' % c.getinfo(c.TOTAL_TIME)) + +# getinfo must be called before close. +c.close() diff --git a/examples/quickstart/write_file.py b/examples/quickstart/write_file.py new file mode 100644 index 0000000..91cdf8b --- /dev/null +++ b/examples/quickstart/write_file.py @@ -0,0 +1,14 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl + +# As long as the file is opened in binary mode, both Python 2 and Python 3 +# can write response body to it without decoding. +with open('out.html', 'wb') as f: + c = pycurl.Curl() + c.setopt(c.URL, 'http://pycurl.io/') + c.setopt(c.WRITEDATA, f) + c.perform() + c.close() diff --git a/examples/retriever-multi.py b/examples/retriever-multi.py new file mode 100644 index 0000000..8f5bef0 --- /dev/null +++ b/examples/retriever-multi.py @@ -0,0 +1,123 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +# +# Usage: python retriever-multi.py [<# of +# concurrent connections>] +# + +import sys +import pycurl + +# We should ignore SIGPIPE when using pycurl.NOSIGNAL - see +# the libcurl tutorial for more info. +try: + import signal + from signal import SIGPIPE, SIG_IGN +except ImportError: + pass +else: + signal.signal(SIGPIPE, SIG_IGN) + + + +# Get args +num_conn = 10 +try: + if sys.argv[1] == "-": + urls = sys.stdin.readlines() + else: + urls = open(sys.argv[1]).readlines() + if len(sys.argv) >= 3: + num_conn = int(sys.argv[2]) +except: + print("Usage: %s [<# of concurrent connections>]" % sys.argv[0]) + raise SystemExit + + +# Make a queue with (url, filename) tuples +queue = [] +for url in urls: + url = url.strip() + if not url or url[0] == "#": + continue + filename = "doc_%03d.dat" % (len(queue) + 1) + queue.append((url, filename)) + + +# Check args +assert queue, "no URLs given" +num_urls = len(queue) +num_conn = min(num_conn, num_urls) +assert 1 <= num_conn <= 10000, "invalid number of concurrent connections" +print("PycURL %s (compiled against 0x%x)" % (pycurl.version, pycurl.COMPILE_LIBCURL_VERSION_NUM)) +print("----- Getting", num_urls, "URLs using", num_conn, "connections -----") + + +# Pre-allocate a list of curl objects +m = pycurl.CurlMulti() +m.handles = [] +for i in range(num_conn): + c = pycurl.Curl() + c.fp = None + c.setopt(pycurl.FOLLOWLOCATION, 1) + c.setopt(pycurl.MAXREDIRS, 5) + c.setopt(pycurl.CONNECTTIMEOUT, 30) + c.setopt(pycurl.TIMEOUT, 300) + c.setopt(pycurl.NOSIGNAL, 1) + m.handles.append(c) + + +# Main loop +freelist = m.handles[:] +num_processed = 0 +while num_processed < num_urls: + # If there is an url to process and a free curl object, add to multi stack + while queue and freelist: + url, filename = queue.pop(0) + c = freelist.pop() + c.fp = open(filename, "wb") + c.setopt(pycurl.URL, url) + c.setopt(pycurl.WRITEDATA, c.fp) + m.add_handle(c) + # store some info + c.filename = filename + c.url = url + # Run the internal curl state machine for the multi stack + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + # Check for curl objects which have terminated, and add them to the freelist + while 1: + num_q, ok_list, err_list = m.info_read() + for c in ok_list: + c.fp.close() + c.fp = None + m.remove_handle(c) + print("Success:", c.filename, c.url, c.getinfo(pycurl.EFFECTIVE_URL)) + freelist.append(c) + for c, errno, errmsg in err_list: + c.fp.close() + c.fp = None + m.remove_handle(c) + print("Failed: ", c.filename, c.url, errno, errmsg) + freelist.append(c) + num_processed = num_processed + len(ok_list) + len(err_list) + if num_q == 0: + break + # Currently no more I/O is pending, could do something in the meantime + # (display a progress bar, etc.). + # We just call select() to sleep until some more data is available. + m.select(1.0) + + +# Cleanup +for c in m.handles: + if c.fp is not None: + c.fp.close() + c.fp = None + c.close() +m.close() + diff --git a/examples/retriever.py b/examples/retriever.py new file mode 100644 index 0000000..ff939c0 --- /dev/null +++ b/examples/retriever.py @@ -0,0 +1,103 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +# +# Usage: python retriever.py [<# of +# concurrent connections>] +# + +import sys, threading +try: + import Queue +except ImportError: + import queue as Queue +import pycurl + +# We should ignore SIGPIPE when using pycurl.NOSIGNAL - see +# the libcurl tutorial for more info. +try: + import signal + from signal import SIGPIPE, SIG_IGN +except ImportError: + pass +else: + signal.signal(SIGPIPE, SIG_IGN) + + +# Get args +num_conn = 10 +try: + if sys.argv[1] == "-": + urls = sys.stdin.readlines() + else: + urls = open(sys.argv[1]).readlines() + if len(sys.argv) >= 3: + num_conn = int(sys.argv[2]) +except: + print("Usage: %s [<# of concurrent connections>]" % sys.argv[0]) + raise SystemExit + + +# Make a queue with (url, filename) tuples +queue = Queue.Queue() +for url in urls: + url = url.strip() + if not url or url[0] == "#": + continue + filename = "doc_%03d.dat" % (len(queue.queue) + 1) + queue.put((url, filename)) + + +# Check args +assert queue.queue, "no URLs given" +num_urls = len(queue.queue) +num_conn = min(num_conn, num_urls) +assert 1 <= num_conn <= 10000, "invalid number of concurrent connections" +print("PycURL %s (compiled against 0x%x)" % (pycurl.version, pycurl.COMPILE_LIBCURL_VERSION_NUM)) +print("----- Getting", num_urls, "URLs using", num_conn, "connections -----") + + +class WorkerThread(threading.Thread): + def __init__(self, queue): + threading.Thread.__init__(self) + self.queue = queue + + def run(self): + while 1: + try: + url, filename = self.queue.get_nowait() + except Queue.Empty: + raise SystemExit + fp = open(filename, "wb") + curl = pycurl.Curl() + curl.setopt(pycurl.URL, url) + curl.setopt(pycurl.FOLLOWLOCATION, 1) + curl.setopt(pycurl.MAXREDIRS, 5) + curl.setopt(pycurl.CONNECTTIMEOUT, 30) + curl.setopt(pycurl.TIMEOUT, 300) + curl.setopt(pycurl.NOSIGNAL, 1) + curl.setopt(pycurl.WRITEDATA, fp) + try: + curl.perform() + except: + import traceback + traceback.print_exc(file=sys.stderr) + sys.stderr.flush() + curl.close() + fp.close() + sys.stdout.write(".") + sys.stdout.flush() + + +# Start a bunch of threads +threads = [] +for dummy in range(num_conn): + t = WorkerThread(queue) + t.start() + threads.append(t) + + +# Wait for all threads to finish +for thread in threads: + thread.join() diff --git a/examples/sfquery.py b/examples/sfquery.py new file mode 100644 index 0000000..0005748 --- /dev/null +++ b/examples/sfquery.py @@ -0,0 +1,65 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et +# +# sfquery -- Source Forge query script using the ClientCGI high-level interface +# +# Retrieves a SourceForge XML export object for a given project. +# Specify the *numeric* project ID. the user name, and the password, +# as arguments. If you have a valid ~/.netrc entry for sourceforge.net, +# you can just give the project ID. +# +# By Eric S. Raymond, August 2002. All rites reversed. + +import sys, netrc +import curl + +class SourceForgeUserSession(curl.Curl): + # SourceForge-specific methods. Sensitive to changes in site design. + def login(self, name, password): + "Establish a login session." + self.post("account/login.php", (("form_loginname", name), + ("form_pw", password), + ("return_to", ""), + ("stay_in_ssl", "1"), + ("login", "Login With SSL"))) + def logout(self): + "Log out of SourceForge." + self.get("account/logout.php") + def fetch_xml(self, numid): + self.get("export/xml_export.php?group_id=%s" % numid) + +if __name__ == "__main__": + if len(sys.argv) == 1: + project_id = '28236' # PyCurl project ID + else: + project_id = sys.argv[1] + # Try to grab authenticators out of your .netrc + try: + auth = netrc.netrc().authenticators("sourceforge.net") + name, account, password = auth + except: + if len(sys.argv) < 4: + print("Usage: %s " % sys.argv[0]) + raise SystemExit + name = sys.argv[2] + password = sys.argv[3] + session = SourceForgeUserSession("https://sourceforge.net/") + session.set_verbosity(0) + session.login(name, password) + # Login could fail. + if session.answered("Invalid Password or User Name"): + sys.stderr.write("Login/password not accepted (%d bytes)\n" % len(session.body())) + sys.exit(1) + # We'll see this if we get the right thing. + elif session.answered("Personal Page For: " + name): + session.fetch_xml(project_id) + sys.stdout.write(session.body()) + session.logout() + sys.exit(0) + # Or maybe SourceForge has changed its site design so our check strings + # are no longer valid. + else: + sys.stderr.write("Unexpected page (%d bytes)\n"%len(session.body())) + sys.exit(1) + diff --git a/examples/smtp.py b/examples/smtp.py new file mode 100644 index 0000000..7bbd1bc --- /dev/null +++ b/examples/smtp.py @@ -0,0 +1,46 @@ +# Based on the simple libcurl SMTP example: +# https://github.com/bagder/curl/blob/master/docs/examples/smtp-mail.c +# There are other SMTP examples in that directory that you may find helpful. + +from . import localhost +import pycurl +try: + from io import BytesIO +except ImportError: + from StringIO import StringIO as BytesIO +import sys + +PY3 = sys.version_info[0] > 2 + +mail_server = 'smtp://%s' % localhost +mail_from = 'sender@example.org' +mail_to = 'addressee@example.net' + +c = pycurl.Curl() +c.setopt(c.URL, mail_server) +c.setopt(c.MAIL_FROM, mail_from) +c.setopt(c.MAIL_RCPT, [mail_to]) + +message = '''\ +From: %s +To: %s +Subject: PycURL SMTP example + +SMTP example via PycURL +''' % (mail_from, mail_to) + +if PY3: + message = message.encode('ascii') + +# libcurl does not perform buffering, therefore +# we need to wrap the message string into a BytesIO or StringIO. +io = BytesIO(message) +c.setopt(c.READDATA, io) + +# If UPLOAD is not set, libcurl performs SMTP VRFY. +# Setting UPLOAD to True sends a message. +c.setopt(c.UPLOAD, True) + +# Observe SMTP conversation. +c.setopt(c.VERBOSE, True) +c.perform() diff --git a/examples/ssh_keyfunction.py b/examples/ssh_keyfunction.py new file mode 100644 index 0000000..93fcc77 --- /dev/null +++ b/examples/ssh_keyfunction.py @@ -0,0 +1,15 @@ +import pycurl + +sftp_server = 'sftp://web.sourceforge.net' + +c = pycurl.Curl() +c.setopt(c.URL, sftp_server) +c.setopt(c.VERBOSE, True) + +def keyfunction(known_key, found_key, match): + return c.KHSTAT_FINE + +c.setopt(c.SSH_KNOWNHOSTS, '.known_hosts') +c.setopt(c.SSH_KEYFUNCTION, keyfunction) + +c.perform() diff --git a/examples/tests/test_build_config.py b/examples/tests/test_build_config.py new file mode 100644 index 0000000..5bec022 --- /dev/null +++ b/examples/tests/test_build_config.py @@ -0,0 +1,65 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import zlib +try: + from io import BytesIO +except ImportError: + try: + from cStringIO import StringIO as BytesIO + except ImportError: + from StringIO import StringIO as BytesIO + +c = pycurl.Curl() +c.setopt(c.URL, 'http://pycurl.io') +#c.setopt(c.ENCODING, 'deflate') +c.setopt(c.HTTPHEADER, ['Accept-Encoding: deflate']) +body = BytesIO() +c.setopt(c.WRITEFUNCTION, body.write) +encoding_found = False +def header_function(header): + global encoding_found + if header.decode('iso-8859-1').lower().startswith('content-encoding: deflate'): + encoding_found = True +c.setopt(c.HEADERFUNCTION, header_function) +c.perform() +assert encoding_found +print('Server supports deflate encoding') +encoded = body.getvalue() +# should not raise exceptions +zlib.decompress(encoded, -zlib.MAX_WBITS) +print('Server served deflated body') + +c.reset() +c.setopt(c.URL, 'http://pycurl.io') +c.setopt(c.ENCODING, 'deflate') +body = BytesIO() +c.setopt(c.WRITEFUNCTION, body.write) +encoding_found = False +def header_function(header): + global encoding_found + if header.decode('iso-8859-1').lower().startswith('content-encoding: deflate'): + encoding_found = True +c.setopt(c.HEADERFUNCTION, header_function) +c.perform() +assert encoding_found +print('Server claimed deflate encoding as expected') +# body should be decoded +encoded = body.getvalue() +if '= 1.0: self.round = 0.0 + else: + self.round = float(download_d) / float(download_t) + gtk.threads_enter() + self.pbar.set_fraction(self.round) + gtk.threads_leave() + + def mainloop(self): + gtk.threads_enter() + gtk.main() + gtk.threads_leave() + + def close_app(self, *args): + args[0].destroy() + gtk.main_quit() + + +class Test(threading.Thread): + def __init__(self, url, target_file, progress): + threading.Thread.__init__(self) + self.target_file = target_file + self.progress = progress + self.curl = pycurl.Curl() + self.curl.setopt(pycurl.URL, url) + self.curl.setopt(pycurl.WRITEDATA, self.target_file) + self.curl.setopt(pycurl.FOLLOWLOCATION, 1) + self.curl.setopt(pycurl.NOPROGRESS, 0) + self.curl.setopt(pycurl.PROGRESSFUNCTION, self.progress) + self.curl.setopt(pycurl.MAXREDIRS, 5) + self.curl.setopt(pycurl.NOSIGNAL, 1) + + def run(self): + self.curl.perform() + self.curl.close() + self.target_file.close() + self.progress(1.0, 1.0, 0, 0) + + +# Check command line args +if len(sys.argv) < 3: + print("Usage: %s " % sys.argv[0]) + raise SystemExit + +# Make a progress bar window +p = ProgressBar(sys.argv[1]) +# Start thread for fetching url +Test(sys.argv[1], open(sys.argv[2], 'wb'), p.progress).start() +# Enter the GTK mainloop +gtk.threads_init() +try: + p.mainloop() +except KeyboardInterrupt: + pass diff --git a/examples/tests/test_xmlrpc.py b/examples/tests/test_xmlrpc.py new file mode 100644 index 0000000..9fb0f94 --- /dev/null +++ b/examples/tests/test_xmlrpc.py @@ -0,0 +1,31 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +## XML-RPC lib included in python2.2 +try: + import xmlrpclib +except ImportError: + import xmlrpc.client as xmlrpclib +import pycurl + +# Header fields passed in request +xmlrpc_header = [ + "User-Agent: PycURL XML-RPC Test", "Content-Type: text/xml" + ] + +# XML-RPC request template +xmlrpc_template = """ +%s%s +""" + +# Engage +c = pycurl.Curl() +c.setopt(c.URL, 'http://betty.userland.com/RPC2') +c.setopt(c.POST, 1) +c.setopt(c.HTTPHEADER, xmlrpc_header) +c.setopt(c.POSTFIELDS, xmlrpc_template % ("examples.getStateName", xmlrpclib.dumps((5,)))) + +print('Response from http://betty.userland.com/') +c.perform() +c.close() diff --git a/examples/xmlrpc_curl.py b/examples/xmlrpc_curl.py new file mode 100644 index 0000000..653d6ce --- /dev/null +++ b/examples/xmlrpc_curl.py @@ -0,0 +1,77 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +# We should ignore SIGPIPE when using pycurl.NOSIGNAL - see +# the libcurl tutorial for more info. +try: + import signal + from signal import SIGPIPE, SIG_IGN +except ImportError: + pass +else: + signal.signal(SIGPIPE, SIG_IGN) + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO +try: + import xmlrpclib +except ImportError: + import xmlrpc.client as xmlrpclib +import pycurl +import sys + +PY3 = sys.version_info[0] > 2 + + +class CURLTransport(xmlrpclib.Transport): + """Handles a cURL HTTP transaction to an XML-RPC server.""" + + xmlrpc_h = [ "Content-Type: text/xml" ] + + def __init__(self, username=None, password=None): + self.c = pycurl.Curl() + self.c.setopt(pycurl.POST, 1) + self.c.setopt(pycurl.NOSIGNAL, 1) + self.c.setopt(pycurl.CONNECTTIMEOUT, 30) + self.c.setopt(pycurl.HTTPHEADER, self.xmlrpc_h) + if username != None and password != None: + self.c.setopt(pycurl.USERPWD, '%s:%s' % (username, password)) + self._use_datetime = False + + def request(self, host, handler, request_body, verbose=0): + b = StringIO() + self.c.setopt(pycurl.URL, 'http://%s%s' % (host, handler)) + self.c.setopt(pycurl.POSTFIELDS, request_body) + self.c.setopt(pycurl.WRITEFUNCTION, b.write) + self.c.setopt(pycurl.VERBOSE, verbose) + self.verbose = verbose + try: + self.c.perform() + except pycurl.error: + v = sys.exc_info()[1] + if PY3: + v = v.args + raise xmlrpclib.ProtocolError( + host + handler, + v[0], v[1], None + ) + b.seek(0) + return self.parse_response(b) + + +if __name__ == "__main__": + ## Test + server = xmlrpclib.ServerProxy("http://betty.userland.com", + transport=CURLTransport()) + print(server) + try: + print(server.examples.getStateName(41)) + except xmlrpclib.Error: + v = sys.exc_info()[1] + print("ERROR", v) diff --git a/pycurl.egg-info/PKG-INFO b/pycurl.egg-info/PKG-INFO new file mode 100644 index 0000000..ef2a1a0 --- /dev/null +++ b/pycurl.egg-info/PKG-INFO @@ -0,0 +1,112 @@ +Metadata-Version: 2.1 +Name: pycurl +Version: 7.45.2 +Summary: PycURL -- A Python Interface To The cURL library +Home-page: http://pycurl.io/ +Author: Kjetil Jacobsen, Markus F.X.J. Oberhumer, Oleg Pudeyev +Author-email: kjetilja@gmail.com, markus@oberhumer.com, oleg@bsdpower.com +Maintainer: Oleg Pudeyev +Maintainer-email: oleg@bsdpower.com +License: LGPL/MIT +Keywords: curl,libcurl,urllib,wget,download,file transfer,http,www +Platform: All +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Web Environment +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Topic :: Internet :: File Transfer Protocol (FTP) +Classifier: Topic :: Internet :: WWW/HTTP +Requires-Python: >=3.5 +License-File: COPYING-LGPL +License-File: COPYING-MIT +License-File: AUTHORS + +PycURL -- A Python Interface To The cURL library +================================================ + +PycURL is a Python interface to `libcurl`_, the multiprotocol file +transfer library. Similarly to the urllib_ Python module, +PycURL can be used to fetch objects identified by a URL from a Python program. +Beyond simple fetches however PycURL exposes most of the functionality of +libcurl, including: + +- Speed - libcurl is very fast and PycURL, being a thin wrapper above + libcurl, is very fast as well. PycURL `was benchmarked`_ to be several + times faster than requests_. +- Features including multiple protocol support, SSL, authentication and + proxy options. PycURL supports most of libcurl's callbacks. +- Multi_ and share_ interfaces. +- Sockets used for network operations, permitting integration of PycURL + into the application's I/O loop (e.g., using Tornado_). + +.. _was benchmarked: http://stackoverflow.com/questions/15461995/python-requests-vs-pycurl-performance +.. _requests: http://python-requests.org/ +.. _Multi: https://curl.haxx.se/libcurl/c/libcurl-multi.html +.. _share: https://curl.haxx.se/libcurl/c/libcurl-share.html +.. _Tornado: http://www.tornadoweb.org/ + + +Requirements +------------ + +- Python 3.5-3.10. +- libcurl 7.19.0 or better. + + +Installation +------------ + +Download the source distribution from `PyPI`_. + +Please see `the installation documentation`_ for installation instructions. + +.. _PyPI: https://pypi.python.org/pypi/pycurl +.. _the installation documentation: http://pycurl.io/docs/latest/install.html + + +Documentation +------------- + +Documentation for the most recent PycURL release is available on +`PycURL website `_. + + +Support +------- + +For support questions please use `curl-and-python mailing list`_. +`Mailing list archives`_ are available for your perusal as well. + +Although not an official support venue, `Stack Overflow`_ has been +popular with some PycURL users. + +Bugs can be reported `via GitHub`_. Please use GitHub only for bug +reports and direct questions to our mailing list instead. + +.. _curl-and-python mailing list: http://cool.haxx.se/mailman/listinfo/curl-and-python +.. _Stack Overflow: http://stackoverflow.com/questions/tagged/pycurl +.. _Mailing list archives: https://curl.haxx.se/mail/list.cgi?list=curl-and-python +.. _via GitHub: https://github.com/pycurl/pycurl/issues + + +License +------- + +PycURL is dual licensed under the LGPL and an MIT/X derivative license +based on the libcurl license. The complete text of the licenses is available +in COPYING-LGPL_ and COPYING-MIT_ files in the source distribution. + +.. _libcurl: https://curl.haxx.se/libcurl/ +.. _urllib: http://docs.python.org/library/urllib.html +.. _COPYING-LGPL: https://raw.githubusercontent.com/pycurl/pycurl/master/COPYING-LGPL +.. _COPYING-MIT: https://raw.githubusercontent.com/pycurl/pycurl/master/COPYING-MIT diff --git a/pycurl.egg-info/SOURCES.txt b/pycurl.egg-info/SOURCES.txt new file mode 100644 index 0000000..1c92627 --- /dev/null +++ b/pycurl.egg-info/SOURCES.txt @@ -0,0 +1,234 @@ +AUTHORS +COPYING-LGPL +COPYING-MIT +ChangeLog +INSTALL.rst +MANIFEST.in +Makefile +README.rst +RELEASE-NOTES.rst +pytest.ini +requirements-dev.txt +setup.py +winbuild.py +doc/callbacks.rst +doc/conf.py +doc/curl.rst +doc/curlmultiobject.rst +doc/curlobject.rst +doc/curlshareobject.rst +doc/files.rst +doc/index.rst +doc/install.rst +doc/internals.rst +doc/pycurl.rst +doc/quickstart.rst +doc/release-notes.rst +doc/release-process.rst +doc/thread-safety.rst +doc/troubleshooting.rst +doc/unicode.rst +doc/unimplemented.rst +doc/docstrings/curl.rst +doc/docstrings/curl_close.rst +doc/docstrings/curl_duphandle.rst +doc/docstrings/curl_errstr.rst +doc/docstrings/curl_errstr_raw.rst +doc/docstrings/curl_getinfo.rst +doc/docstrings/curl_getinfo_raw.rst +doc/docstrings/curl_pause.rst +doc/docstrings/curl_perform.rst +doc/docstrings/curl_perform_rb.rst +doc/docstrings/curl_perform_rs.rst +doc/docstrings/curl_reset.rst +doc/docstrings/curl_set_ca_certs.rst +doc/docstrings/curl_setopt.rst +doc/docstrings/curl_setopt_string.rst +doc/docstrings/curl_unsetopt.rst +doc/docstrings/multi.rst +doc/docstrings/multi_add_handle.rst +doc/docstrings/multi_assign.rst +doc/docstrings/multi_close.rst +doc/docstrings/multi_fdset.rst +doc/docstrings/multi_info_read.rst +doc/docstrings/multi_perform.rst +doc/docstrings/multi_remove_handle.rst +doc/docstrings/multi_select.rst +doc/docstrings/multi_setopt.rst +doc/docstrings/multi_socket_action.rst +doc/docstrings/multi_socket_all.rst +doc/docstrings/multi_timeout.rst +doc/docstrings/pycurl_global_cleanup.rst +doc/docstrings/pycurl_global_init.rst +doc/docstrings/pycurl_module.rst +doc/docstrings/pycurl_version_info.rst +doc/docstrings/share.rst +doc/docstrings/share_close.rst +doc/docstrings/share_setopt.rst +doc/static/favicon.ico +examples/basicfirst.py +examples/file_upload.py +examples/linksys.py +examples/multi-socket_action-select.py +examples/opensocketexception.py +examples/retriever-multi.py +examples/retriever.py +examples/sfquery.py +examples/smtp.py +examples/ssh_keyfunction.py +examples/xmlrpc_curl.py +examples/quickstart/file_upload_buffer.py +examples/quickstart/file_upload_real.py +examples/quickstart/file_upload_real_fancy.py +examples/quickstart/follow_redirect.py +examples/quickstart/form_post.py +examples/quickstart/get.py +examples/quickstart/get_python2.py +examples/quickstart/get_python2_https.py +examples/quickstart/get_python3.py +examples/quickstart/get_python3_https.py +examples/quickstart/put_buffer.py +examples/quickstart/put_file.py +examples/quickstart/response_headers.py +examples/quickstart/response_info.py +examples/quickstart/write_file.py +examples/tests/test_build_config.py +examples/tests/test_gtk.py +examples/tests/test_xmlrpc.py +pycurl.egg-info/PKG-INFO +pycurl.egg-info/SOURCES.txt +pycurl.egg-info/dependency_links.txt +pycurl.egg-info/top_level.txt +python/curl/__init__.py +src/docstrings.c +src/docstrings.h +src/easy.c +src/easycb.c +src/easyinfo.c +src/easyopt.c +src/easyperform.c +src/module.c +src/multi.c +src/oscompat.c +src/pycurl.h +src/pythoncompat.c +src/share.c +src/stringcompat.c +src/threadsupport.c +src/util.c +tests/__init__.py +tests/app.py +tests/appmanager.py +tests/cadata_test.py +tests/certinfo_test.py +tests/close_socket_cb_test.py +tests/curl_object_test.py +tests/debug_test.py +tests/default_write_cb_test.py +tests/duphandle_test.py +tests/error_constants_test.py +tests/error_test.py +tests/failonerror_test.py +tests/ftp_test.py +tests/getinfo_test.py +tests/global_init_test.py +tests/header_cb_test.py +tests/header_test.py +tests/high_level_curl_test.py +tests/info_constants_test.py +tests/info_test.py +tests/internals_test.py +tests/matrix.py +tests/memory_mgmt_test.py +tests/multi_callback_test.py +tests/multi_memory_mgmt_test.py +tests/multi_option_constants_test.py +tests/multi_socket_select_test.py +tests/multi_socket_test.py +tests/multi_test.py +tests/multi_timer_test.py +tests/open_socket_cb_test.py +tests/option_constants_test.py +tests/pause_test.py +tests/perform_test.py +tests/post_test.py +tests/procmgr.py +tests/protocol_constants_test.py +tests/read_cb_test.py +tests/readdata_test.py +tests/relative_url_test.py +tests/reload_test.py +tests/reset_test.py +tests/resolve_test.py +tests/run-quickstart.sh +tests/run.sh +tests/runwsgi.py +tests/seek_cb_constants_test.py +tests/seek_cb_test.py +tests/setopt_lifecycle_test.py +tests/setopt_string_test.py +tests/setopt_test.py +tests/setopt_unicode_test.py +tests/setup_test.py +tests/share_test.py +tests/sockopt_cb_test.py +tests/ssh_key_cb_test.py +tests/subclass_test.py +tests/unset_range_test.py +tests/user_agent_string_test.py +tests/util.py +tests/version_comparison_test.py +tests/version_constants_test.py +tests/version_test.py +tests/vsftpd.conf +tests/weakref_test.py +tests/write_abort_test.py +tests/write_cb_bogus_test.py +tests/write_test.py +tests/write_to_stringio_test.py +tests/xferinfo_cb_test.py +tests/certs/ca.crt +tests/certs/ca.key +tests/certs/server.crt +tests/certs/server.key +tests/ext/test-lib.sh +tests/ext/test-suite.sh +tests/fake-curl/curl-config-empty +tests/fake-curl/curl-config-libs-and-static-libs +tests/fake-curl/curl-config-ssl-feature-only +tests/fake-curl/curl-config-ssl-in-libs +tests/fake-curl/curl-config-ssl-in-static-libs +tests/fake-curl/libcurl/Makefile +tests/fake-curl/libcurl/with_gnutls.c +tests/fake-curl/libcurl/with_nss.c +tests/fake-curl/libcurl/with_openssl.c +tests/fake-curl/libcurl/with_unknown_ssl.c +tests/fake-curl/libcurl/without_ssl.c +tests/fixtures/form_submission.txt +tests/matrix/curl-7.19.0-sslv2-2b0e09b0f98.patch +tests/matrix/curl-7.19.0-sslv2-c66b0b32fba-modified.patch +tests/matrix/openssl-1.0.1e-fix_pod_syntax-1.patch +winbuild/__init__.py +winbuild/builder.py +winbuild/c-ares-vs2015.patch +winbuild/cares.py +winbuild/config.py +winbuild/curl.py +winbuild/iconv.py +winbuild/idn.py +winbuild/libcurl-fix-zlib-references.patch +winbuild/libssh2-vs2015.patch +winbuild/nghttp_cmake.py +winbuild/nghttp_gmake.py +winbuild/openssl-fix-crt-1.0.2.patch +winbuild/openssl-fix-crt-1.1.0.patch +winbuild/openssl-fix-crt-1.1.1.patch +winbuild/openssl.py +winbuild/pycurl.py +winbuild/pythons.py +winbuild/ssh.py +winbuild/tools.py +winbuild/utils.py +winbuild/vcvars-vc14-32.sh +winbuild/vcvars-vc14-64.sh +winbuild/zlib.py \ No newline at end of file diff --git a/pycurl.egg-info/dependency_links.txt b/pycurl.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/pycurl.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/pycurl.egg-info/top_level.txt b/pycurl.egg-info/top_level.txt new file mode 100644 index 0000000..62b667f --- /dev/null +++ b/pycurl.egg-info/top_level.txt @@ -0,0 +1,2 @@ +curl +pycurl diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..f1bce5c --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +python_files = tests/*.py +norecursedirs = examples win +markers = + ssh: mark a test as requiring ssh + online: mark a test as requiring internet access + gssapi: mark a test as requiring GSSAPI + http2: mark a test as requiring HTTP/2 + standalone: mark a test as being standalone diff --git a/python/curl/__init__.py b/python/curl/__init__.py new file mode 100644 index 0000000..0c95e2b --- /dev/null +++ b/python/curl/__init__.py @@ -0,0 +1,194 @@ +'''A high-level interface to the pycurl extension''' + +# ** mfx NOTE: the CGI class uses "black magic" using COOKIEFILE in +# combination with a non-existent file name. See the libcurl docs +# for more info. + +import sys, pycurl + +py3 = sys.version_info[0] == 3 + +# python 2/3 compatibility +if py3: + import urllib.parse as urllib_parse + from urllib.parse import urljoin + from io import BytesIO +else: + import urllib as urllib_parse + from urlparse import urljoin + try: + from cStringIO import StringIO as BytesIO + except ImportError: + from StringIO import StringIO as BytesIO + +# We should ignore SIGPIPE when using pycurl.NOSIGNAL - see +# the libcurl tutorial for more info. +try: + import signal + from signal import SIGPIPE, SIG_IGN +except ImportError: + pass +else: + signal.signal(SIGPIPE, SIG_IGN) + + +class Curl: + "High-level interface to pycurl functions." + def __init__(self, base_url="", fakeheaders=None): + self.handle = pycurl.Curl() + # These members might be set. + self.set_url(base_url) + self.verbosity = 0 + self.fakeheaders = fakeheaders or [] + # Nothing past here should be modified by the caller. + self.payload = None + self.payload_io = BytesIO() + self.hdr = "" + # Verify that we've got the right site; harmless on a non-SSL connect. + self.set_option(pycurl.SSL_VERIFYHOST, 2) + # Follow redirects in case it wants to take us to a CGI... + self.set_option(pycurl.FOLLOWLOCATION, 1) + self.set_option(pycurl.MAXREDIRS, 5) + self.set_option(pycurl.NOSIGNAL, 1) + # Setting this option with even a nonexistent file makes libcurl + # handle cookie capture and playback automatically. + self.set_option(pycurl.COOKIEFILE, "/dev/null") + # Set timeouts to avoid hanging too long + self.set_timeout(30) + # Use password identification from .netrc automatically + self.set_option(pycurl.NETRC, 1) + self.set_option(pycurl.WRITEFUNCTION, self.payload_io.write) + def header_callback(x): + self.hdr += x.decode('ascii') + self.set_option(pycurl.HEADERFUNCTION, header_callback) + + def set_timeout(self, timeout): + "Set timeout for a retrieving an object" + self.set_option(pycurl.TIMEOUT, timeout) + + def set_url(self, url): + "Set the base URL to be retrieved." + self.base_url = url + self.set_option(pycurl.URL, self.base_url) + + def set_option(self, *args): + "Set an option on the retrieval." + self.handle.setopt(*args) + + def set_verbosity(self, level): + "Set verbosity to 1 to see transactions." + self.set_option(pycurl.VERBOSE, level) + + def __request(self, relative_url=None): + "Perform the pending request." + if self.fakeheaders: + self.set_option(pycurl.HTTPHEADER, self.fakeheaders) + if relative_url: + self.set_option(pycurl.URL, urljoin(self.base_url, relative_url)) + self.payload = None + self.payload_io.seek(0) + self.payload_io.truncate() + self.hdr = "" + self.handle.perform() + self.payload = self.payload_io.getvalue() + return self.payload + + def get(self, url="", params=None): + "Ship a GET request for a specified URL, capture the response." + if params: + url += "?" + urllib_parse.urlencode(params) + self.set_option(pycurl.HTTPGET, 1) + return self.__request(url) + + def head(self, url="", params=None): + "Ship a HEAD request for a specified URL, capture the response." + if params: + url += "?" + urllib_parse.urlencode(params) + self.set_option(pycurl.NOBODY, 1) + return self.__request(url) + + def post(self, cgi, params): + "Ship a POST request to a specified CGI, capture the response." + self.set_option(pycurl.POST, 1) + self.set_option(pycurl.POSTFIELDS, urllib_parse.urlencode(params)) + return self.__request(cgi) + + def body(self): + "Return the body from the last response." + return self.payload + + def header(self): + "Return the header from the last response." + return self.hdr + + def get_info(self, *args): + "Get information about retrieval." + return self.handle.getinfo(*args) + + def info(self): + "Return a dictionary with all info on the last response." + m = {} + m['effective-url'] = self.handle.getinfo(pycurl.EFFECTIVE_URL) + m['http-code'] = self.handle.getinfo(pycurl.HTTP_CODE) + m['total-time'] = self.handle.getinfo(pycurl.TOTAL_TIME) + m['namelookup-time'] = self.handle.getinfo(pycurl.NAMELOOKUP_TIME) + m['connect-time'] = self.handle.getinfo(pycurl.CONNECT_TIME) + m['pretransfer-time'] = self.handle.getinfo(pycurl.PRETRANSFER_TIME) + m['redirect-time'] = self.handle.getinfo(pycurl.REDIRECT_TIME) + m['redirect-count'] = self.handle.getinfo(pycurl.REDIRECT_COUNT) + m['size-upload'] = self.handle.getinfo(pycurl.SIZE_UPLOAD) + m['size-download'] = self.handle.getinfo(pycurl.SIZE_DOWNLOAD) + m['speed-upload'] = self.handle.getinfo(pycurl.SPEED_UPLOAD) + m['header-size'] = self.handle.getinfo(pycurl.HEADER_SIZE) + m['request-size'] = self.handle.getinfo(pycurl.REQUEST_SIZE) + m['content-length-download'] = self.handle.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD) + m['content-length-upload'] = self.handle.getinfo(pycurl.CONTENT_LENGTH_UPLOAD) + m['content-type'] = self.handle.getinfo(pycurl.CONTENT_TYPE) + m['response-code'] = self.handle.getinfo(pycurl.RESPONSE_CODE) + m['speed-download'] = self.handle.getinfo(pycurl.SPEED_DOWNLOAD) + m['ssl-verifyresult'] = self.handle.getinfo(pycurl.SSL_VERIFYRESULT) + m['filetime'] = self.handle.getinfo(pycurl.INFO_FILETIME) + m['starttransfer-time'] = self.handle.getinfo(pycurl.STARTTRANSFER_TIME) + m['redirect-time'] = self.handle.getinfo(pycurl.REDIRECT_TIME) + m['redirect-count'] = self.handle.getinfo(pycurl.REDIRECT_COUNT) + m['http-connectcode'] = self.handle.getinfo(pycurl.HTTP_CONNECTCODE) + m['httpauth-avail'] = self.handle.getinfo(pycurl.HTTPAUTH_AVAIL) + m['proxyauth-avail'] = self.handle.getinfo(pycurl.PROXYAUTH_AVAIL) + m['os-errno'] = self.handle.getinfo(pycurl.OS_ERRNO) + m['num-connects'] = self.handle.getinfo(pycurl.NUM_CONNECTS) + m['ssl-engines'] = self.handle.getinfo(pycurl.SSL_ENGINES) + m['cookielist'] = self.handle.getinfo(pycurl.INFO_COOKIELIST) + m['lastsocket'] = self.handle.getinfo(pycurl.LASTSOCKET) + m['ftp-entry-path'] = self.handle.getinfo(pycurl.FTP_ENTRY_PATH) + return m + + def answered(self, check): + "Did a given check string occur in the last payload?" + return self.payload.find(check) >= 0 + + def close(self): + "Close a session, freeing resources." + if self.handle: + self.handle.close() + self.handle = None + self.hdr = "" + self.payload = "" + + def __del__(self): + self.close() + + +if __name__ == "__main__": + if len(sys.argv) < 2: + url = 'https://curl.haxx.se' + else: + url = sys.argv[1] + c = Curl() + c.get(url) + print(c.body()) + print('='*74 + '\n') + import pprint + pprint.pprint(c.info()) + print(c.get_info(pycurl.OS_ERRNO)) + print(c.info()['os-errno']) + c.close() diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..da562d5 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,7 @@ +# bottle 0.12.17 changed behavior +# https://github.com/pycurl/pycurl/issues/573 +bottle +flaky +pyflakes +pytest>=5 +sphinx diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..8bfd5a1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[egg_info] +tag_build = +tag_date = 0 + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..a5b115b --- /dev/null +++ b/setup.py @@ -0,0 +1,975 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +"""Setup script for the PycURL module distribution.""" + +PACKAGE = "pycurl" +PY_PACKAGE = "curl" +VERSION = "7.45.2" + +import glob, os, re, shlex, sys, subprocess +from setuptools import setup +from setuptools.extension import Extension + +py3 = sys.version_info[0] == 3 + +try: + # python 2 + exception_base = StandardError +except NameError: + # python 3 + exception_base = Exception +class ConfigurationError(exception_base): + pass + + +def fail(msg): + sys.stderr.write(msg + "\n") + exit(10) + + +def scan_argv(argv, s, default=None): + p = default + i = 1 + while i < len(argv): + arg = argv[i] + if s.endswith('='): + if str.find(arg, s) == 0: + # --option=value + p = arg[len(s):] + if s != '--openssl-lib-name=': + assert p, arg + del argv[i] + else: + i += 1 + else: + if s == arg: + # --option + # set value to True + p = True + del argv[i] + else: + i = i + 1 + ##print argv + return p + + +def scan_argvs(argv, s): + if not s.endswith('='): + raise Exception('specification must end with =') + p = [] + i = 1 + while i < len(argv): + arg = argv[i] + if str.find(arg, s) == 0: + # --option=value + p.append(arg[len(s):]) + if s != '--openssl-lib-name=': + assert p[-1], arg + del argv[i] + else: + i = i + 1 + ##print argv + return p + + +class ExtensionConfiguration(object): + def __init__(self, argv=[]): + # we mutate argv, this is necessary because + # setuptools does not recognize pycurl-specific options + self.argv = argv + self.original_argv = argv[:] + self.include_dirs = [] + self.define_macros = [("PYCURL_VERSION", '"%s"' % VERSION)] + self.library_dirs = [] + self.libraries = [] + self.runtime_library_dirs = [] + self.extra_objects = [] + self.extra_compile_args = [] + self.extra_link_args = [] + self.ssl_lib_detected = None + + self.configure() + + @property + def define_symbols(self): + return [symbol for symbol, expansion in self.define_macros] + + # append contents of an environment variable to library_dirs[] + def add_libdirs(self, envvar, sep, fatal=False): + v = os.environ.get(envvar) + if not v: + return + for dir in str.split(v, sep): + dir = str.strip(dir) + if not dir: + continue + dir = os.path.normpath(dir) + if os.path.isdir(dir): + if not dir in self.library_dirs: + self.library_dirs.append(dir) + elif fatal: + fail("FATAL: bad directory %s in environment variable %s" % (dir, envvar)) + + def detect_features(self): + p = subprocess.Popen((self.curl_config(), '--features'), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0: + msg = "Problem running `%s' --features" % self.curl_config() + if stderr: + msg += ":\n" + stderr.decode() + raise ConfigurationError(msg) + curl_has_ssl = False + for feature in shlex.split(stdout.decode()): + if feature == 'SSL': + # this means any ssl library, not just openssl. + # we set the ssl flag to check for ssl library mismatch + # at link time and run time + self.define_macros.append(('HAVE_CURL_SSL', 1)) + curl_has_ssl = True + self.curl_has_ssl = curl_has_ssl + + def ssl_options(self): + return { + '--with-openssl': self.using_openssl, + '--with-ssl': self.using_openssl, + '--with-wolfssl': self.using_wolfssl, + '--with-gnutls': self.using_gnutls, + '--with-nss': self.using_nss, + '--with-mbedtls': self.using_mbedtls, + '--with-sectransp': self.using_sectransp, + } + + def detect_ssl_option(self): + for option in self.ssl_options(): + if scan_argv(self.argv, option) is not None: + for other_option in self.ssl_options(): + if option != other_option: + if scan_argv(self.argv, other_option) is not None: + raise ConfigurationError('Cannot give both %s and %s' % (option, other_option)) + + return option + + def detect_ssl_backend(self): + ssl_lib_detected = None + + if 'PYCURL_SSL_LIBRARY' in os.environ: + ssl_lib = os.environ['PYCURL_SSL_LIBRARY'] + if ssl_lib in ['openssl', 'wolfssl', 'gnutls', 'nss', 'mbedtls', 'sectransp']: + ssl_lib_detected = ssl_lib + getattr(self, 'using_%s' % ssl_lib)() + else: + raise ConfigurationError('Invalid value "%s" for PYCURL_SSL_LIBRARY' % ssl_lib) + + option = self.detect_ssl_option() + if option: + ssl_lib_detected = option.replace('--with-', '') + self.ssl_options()[option]() + + # ssl detection - ssl libraries are added + if not ssl_lib_detected: + libcurl_dll_path = scan_argv(self.argv, "--libcurl-dll=") + if libcurl_dll_path is not None: + ssl_lib_detected = self.detect_ssl_lib_from_libcurl_dll(libcurl_dll_path) + + if not ssl_lib_detected: + ssl_lib_detected = self.detect_ssl_lib_using_curl_config() + + if not ssl_lib_detected: + # self.sslhintbuf is a hack + for arg in shlex.split(self.sslhintbuf): + if arg[:2] == "-l": + if arg[2:] == 'ssl': + self.using_openssl() + ssl_lib_detected = 'openssl' + break + if arg[2:] == 'wolfssl': + self.using_wolfssl() + ssl_lib_detected = 'wolfssl' + break + if arg[2:] == 'gnutls': + self.using_gnutls() + ssl_lib_detected = 'gnutls' + break + if arg[2:] == 'ssl3': + self.using_nss() + ssl_lib_detected = 'nss' + break + if arg[2:] == 'mbedtls': + self.using_mbedtls() + ssl_lib_detected = 'mbedtls' + break + + if not ssl_lib_detected and len(self.argv) == len(self.original_argv) \ + and not os.environ.get('PYCURL_CURL_CONFIG') \ + and not os.environ.get('PYCURL_SSL_LIBRARY'): + # this path should only be taken when no options or + # configuration environment variables are given to setup.py + ssl_lib_detected = self.detect_ssl_lib_on_centos6_plus() + + self.ssl_lib_detected = ssl_lib_detected + + def curl_config(self): + try: + return self._curl_config + except AttributeError: + curl_config = os.environ.get('PYCURL_CURL_CONFIG', "curl-config") + curl_config = scan_argv(self.argv, "--curl-config=", curl_config) + self._curl_config = curl_config + return curl_config + + def configure_unix(self): + OPENSSL_DIR = scan_argv(self.argv, "--openssl-dir=") + if OPENSSL_DIR is not None: + self.include_dirs.append(os.path.join(OPENSSL_DIR, "include")) + self.library_dirs.append(os.path.join(OPENSSL_DIR, "lib")) + try: + p = subprocess.Popen((self.curl_config(), '--version'), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError: + exc = sys.exc_info()[1] + msg = 'Could not run curl-config: %s' % str(exc) + raise ConfigurationError(msg) + stdout, stderr = p.communicate() + if p.wait() != 0: + msg = "`%s' not found -- please install the libcurl development files or specify --curl-config=/path/to/curl-config" % self.curl_config() + if stderr: + msg += ":\n" + stderr.decode() + raise ConfigurationError(msg) + libcurl_version = stdout.decode().strip() + print("Using %s (%s)" % (self.curl_config(), libcurl_version)) + p = subprocess.Popen((self.curl_config(), '--cflags'), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0: + msg = "Problem running `%s' --cflags" % self.curl_config() + if stderr: + msg += ":\n" + stderr.decode() + raise ConfigurationError(msg) + for arg in shlex.split(stdout.decode()): + if arg[:2] == "-I": + # do not add /usr/include + if not re.search(r"^\/+usr\/+include\/*$", arg[2:]): + self.include_dirs.append(arg[2:]) + else: + self.extra_compile_args.append(arg) + + # Obtain linker flags/libraries to link against. + # In theory, all we should need is `curl-config --libs`. + # Apparently on some platforms --libs fails and --static-libs works, + # so try that. + # If --libs succeeds do not try --static-libs; see + # https://github.com/pycurl/pycurl/issues/52 for more details. + # If neither --libs nor --static-libs work, fail. + # + # --libs/--static-libs are also used for SSL detection. + # libcurl may be configured such that --libs only includes -lcurl + # without any of libcurl's dependent libraries, but the dependent + # libraries would be included in --static-libs (unless libcurl + # was built with static libraries disabled). + # Therefore we largely ignore (see below) --static-libs output for + # libraries and flags if --libs succeeded, but consult both outputs + # for hints as to which SSL library libcurl is linked against. + # More information: https://github.com/pycurl/pycurl/pull/147 + # + # The final point is we should link against the SSL library in use + # even if libcurl does not tell us to, because *we* invoke functions + # in that SSL library. This means any SSL libraries found in + # --static-libs are forwarded to our libraries. + optbuf = '' + sslhintbuf = '' + errtext = '' + for option in ["--libs", "--static-libs"]: + p = subprocess.Popen((self.curl_config(), option), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if p.wait() == 0: + if optbuf == '': + # first successful call + optbuf = stdout.decode() + # optbuf only has output from this call + sslhintbuf += optbuf + else: + # second successful call + sslhintbuf += stdout.decode() + else: + if optbuf == '': + # no successful call yet + errtext += stderr.decode() + else: + # first call succeeded and second call failed + # ignore stderr and the error exit + pass + if optbuf == "": + msg = "Neither curl-config --libs nor curl-config --static-libs" +\ + " succeeded and produced output" + if errtext: + msg += ":\n" + errtext + raise ConfigurationError(msg) + + # hack + self.sslhintbuf = sslhintbuf + + self.detect_features() + self.ssl_lib_detected = None + if self.curl_has_ssl: + self.detect_ssl_backend() + + if not self.ssl_lib_detected: + sys.stderr.write('''\ +Warning: libcurl is configured to use SSL, but we have not been able to \ +determine which SSL backend it is using. If your Curl is built against \ +OpenSSL, LibreSSL, BoringSSL, GnuTLS, NSS, mbedTLS, or Secure Transport \ +please specify the SSL backend manually. For other SSL backends please \ +ignore this message.''') + else: + if self.detect_ssl_option(): + sys.stderr.write("Warning: SSL backend specified manually but libcurl does not use SSL\n") + + # libraries and options - all libraries and options are forwarded + # but if --libs succeeded, --static-libs output is ignored + for arg in shlex.split(optbuf): + if arg[:2] == "-l": + self.libraries.append(arg[2:]) + elif arg[:2] == "-L": + self.library_dirs.append(arg[2:]) + else: + self.extra_link_args.append(arg) + + if not self.libraries: + self.libraries.append("curl") + + # Add extra compile flag for MacOS X + if sys.platform.startswith('darwin'): + self.extra_link_args.append("-flat_namespace") + + # Recognize --avoid-stdio on Unix so that it can be tested + self.check_avoid_stdio() + + def detect_ssl_lib_from_libcurl_dll(self, libcurl_dll_path): + ssl_lib_detected = None + curl_version_info = self.get_curl_version_info(libcurl_dll_path) + ssl_version = curl_version_info.ssl_version + if py3: + # ssl_version is bytes on python 3 + ssl_version = ssl_version.decode('ascii') + if ssl_version.startswith('OpenSSL/') or ssl_version.startswith('LibreSSL/'): + self.using_openssl() + ssl_lib_detected = 'openssl' + elif ssl_version.startswith('GnuTLS/'): + self.using_gnutls() + ssl_lib_detected = 'gnutls' + elif ssl_version.startswith('NSS/'): + self.using_nss() + ssl_lib_detected = 'nss' + elif ssl_version.startswith('mbedTLS/'): + self.using_mbedtls() + ssl_lib_detected = 'mbedtls' + elif ssl_version.startswith('SecureTransport'): + self.using_sectransp() + ssl_lib_detected = 'sectransp' + return ssl_lib_detected + + def detect_ssl_lib_on_centos6_plus(self): + import platform + from ctypes.util import find_library + os_name = platform.system() + if os_name != 'Linux' or not hasattr(platform, 'dist'): + return False + dist_name, dist_version, _ = platform.dist() + dist_version = dist_version.split('.')[0] + if dist_name != 'centos' or int(dist_version) < 6: + return False + libcurl_dll_path = find_library('curl') + print('libcurl_dll_path = "%s"' % libcurl_dll_path) + return self.detect_ssl_lib_from_libcurl_dll(libcurl_dll_path) + + def detect_ssl_lib_using_curl_config(self): + ssl_lib_detected = None + p = subprocess.Popen((self.curl_config(), '--ssl-backends'), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0: + # curl-config --ssl-backends is not supported on older curl versions + return None + ssl_version = stdout.decode() + if ssl_version.startswith('OpenSSL') or ssl_version.startswith('LibreSSL'): + self.using_openssl() + ssl_lib_detected = 'openssl' + elif ssl_version.startswith('GnuTLS'): + self.using_gnutls() + ssl_lib_detected = 'gnutls' + elif ssl_version.startswith('NSS'): + self.using_nss() + ssl_lib_detected = 'nss' + elif ssl_version.startswith('mbedTLS'): + self.using_mbedtls() + ssl_lib_detected = 'mbedtls' + return ssl_lib_detected + + def configure_windows(self): + OPENSSL_DIR = scan_argv(self.argv, "--openssl-dir=") + if OPENSSL_DIR is not None: + self.include_dirs.append(os.path.join(OPENSSL_DIR, "include")) + self.library_dirs.append(os.path.join(OPENSSL_DIR, "lib")) + # Windows users have to pass --curl-dir parameter to specify path + # to libcurl, because there is no curl-config on windows at all. + curl_dir = scan_argv(self.argv, "--curl-dir=") + if curl_dir is None: + fail("Please specify --curl-dir=/path/to/built/libcurl") + if not os.path.exists(curl_dir): + fail("Curl directory does not exist: %s" % curl_dir) + if not os.path.isdir(curl_dir): + fail("Curl directory is not a directory: %s" % curl_dir) + print("Using curl directory: %s" % curl_dir) + self.include_dirs.append(os.path.join(curl_dir, "include")) + + # libcurl windows documentation states that for linking against libcurl + # dll, the import library name is libcurl_imp.lib. + # For libcurl 7.46.0, the library name is libcurl.lib. + # And static library name is libcurl_a.lib by default as of libcurl 7.46.0. + # override with: --libcurl-lib-name=libcurl_imp.lib + curl_lib_name = scan_argv(self.argv, '--libcurl-lib-name=', 'libcurl.lib') + + # openssl 1.1.0 changed its library names + # from libeay32.lib/ssleay32.lib to libcrypto.lib/libssl.lib. + # at the same time they dropped thread locking callback interface, + # meaning the correct usage of this option is --openssl-lib-name="" + self.openssl_lib_name = scan_argv(self.argv, '--openssl-lib-name=', 'libeay32.lib') + + for lib in scan_argvs(self.argv, '--link-arg='): + self.extra_link_args.append(lib) + + if scan_argv(self.argv, "--use-libcurl-dll") is not None: + libcurl_lib_path = os.path.join(curl_dir, "lib", curl_lib_name) + self.extra_link_args.extend(["ws2_32.lib"]) + if str.find(sys.version, "MSC") >= 0: + # build a dll + self.extra_compile_args.append("-MD") + else: + self.extra_compile_args.append("-DCURL_STATICLIB") + libcurl_lib_path = os.path.join(curl_dir, "lib", curl_lib_name) + self.extra_link_args.extend(["gdi32.lib", "wldap32.lib", "winmm.lib", "ws2_32.lib",]) + + if not os.path.exists(libcurl_lib_path): + fail("libcurl.lib does not exist at %s.\nCurl directory must point to compiled libcurl (bin/include/lib subdirectories): %s" %(libcurl_lib_path, curl_dir)) + self.extra_objects.append(libcurl_lib_path) + + if scan_argv(self.argv, '--with-openssl') is not None or scan_argv(self.argv, '--with-ssl') is not None: + self.using_openssl() + + self.check_avoid_stdio() + + # make pycurl binary work on windows xp. + # we use inet_ntop which was added in vista and implement a fallback. + # our implementation will not be compiled with _WIN32_WINNT targeting + # vista or above, thus said binary won't work on xp. + # https://curl.haxx.se/mail/curlpython-2013-12/0007.html + self.extra_compile_args.append("-D_WIN32_WINNT=0x0501") + + if str.find(sys.version, "MSC") >= 0: + self.extra_compile_args.append("-O2") + self.extra_compile_args.append("-GF") # enable read-only string pooling + self.extra_compile_args.append("-WX") # treat warnings as errors + p = subprocess.Popen(['cl.exe'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + match = re.search(r'Version (\d+)', err.decode().split("\n")[0]) + if match and int(match.group(1)) < 16: + # option removed in vs 2010: + # connect.microsoft.com/VisualStudio/feedback/details/475896/link-fatal-error-lnk1117-syntax-error-in-option-opt-nowin98/ + self.extra_link_args.append("/opt:nowin98") # use small section alignment + + if sys.platform == "win32": + configure = configure_windows + else: + configure = configure_unix + + + def check_avoid_stdio(self): + if 'PYCURL_SETUP_OPTIONS' in os.environ and '--avoid-stdio' in os.environ['PYCURL_SETUP_OPTIONS']: + self.extra_compile_args.append("-DPYCURL_AVOID_STDIO") + if scan_argv(self.argv, '--avoid-stdio') is not None: + self.extra_compile_args.append("-DPYCURL_AVOID_STDIO") + + def get_curl_version_info(self, dll_path): + import ctypes + + class curl_version_info_struct(ctypes.Structure): + _fields_ = [ + ('age', ctypes.c_int), + ('version', ctypes.c_char_p), + ('version_num', ctypes.c_uint), + ('host', ctypes.c_char_p), + ('features', ctypes.c_int), + ('ssl_version', ctypes.c_char_p), + ('ssl_version_num', ctypes.c_long), + ('libz_version', ctypes.c_char_p), + ('protocols', ctypes.c_void_p), + ('ares', ctypes.c_char_p), + ('ares_num', ctypes.c_int), + ('libidn', ctypes.c_char_p), + ('iconv_ver_num', ctypes.c_int), + ('libssh_version', ctypes.c_char_p), + ('brotli_ver_num', ctypes.c_uint), + ('brotli_version', ctypes.c_char_p), + ('nghttp2_ver_num', ctypes.c_uint), + ('nghttp2_version', ctypes.c_char_p), + ('quic_version', ctypes.c_char_p), + ('cainfo', ctypes.c_char_p), + ('capath', ctypes.c_char_p), + ('zstd_ver_num', ctypes.c_uint), + ('zstd_version', ctypes.c_char_p), + ('hyper_version', ctypes.c_char_p), + ('gsasl_version', ctypes.c_char_p), + ] + + dll = ctypes.CDLL(dll_path) + fn = dll.curl_version_info + fn.argtypes = [ctypes.c_int] + fn.restype = ctypes.POINTER(curl_version_info_struct) + + # current version is 3 + return fn(3)[0] + + def using_openssl(self): + self.define_macros.append(('HAVE_CURL_OPENSSL', 1)) + if sys.platform == "win32": + # CRYPTO_num_locks is defined in libeay32.lib + # for openssl < 1.1.0; it is a noop for openssl >= 1.1.0 + self.extra_link_args.append(self.openssl_lib_name) + else: + # we also need ssl for the certificate functions + # (SSL_CTX_get_cert_store) + self.libraries.append('ssl') + # the actual library that defines CRYPTO_num_locks etc. + # is crypto, and on cygwin linking against ssl does not + # link against crypto as of May 2014. + # http://stackoverflow.com/questions/23687488/cant-get-pycurl-to-install-on-cygwin-missing-openssl-symbols-crypto-num-locks + self.libraries.append('crypto') + self.define_macros.append(('HAVE_CURL_SSL', 1)) + self.ssl_lib_detected = 'openssl' + + def using_wolfssl(self): + self.define_macros.append(('HAVE_CURL_WOLFSSL', 1)) + self.libraries.append('wolfssl') + self.define_macros.append(('HAVE_CURL_SSL', 1)) + self.ssl_lib_detected = 'wolfssl' + + def using_gnutls(self): + self.define_macros.append(('HAVE_CURL_GNUTLS', 1)) + self.libraries.append('gnutls') + self.define_macros.append(('HAVE_CURL_SSL', 1)) + self.ssl_lib_detected = 'gnutls' + + def using_nss(self): + self.define_macros.append(('HAVE_CURL_NSS', 1)) + self.libraries.append('ssl3') + self.define_macros.append(('HAVE_CURL_SSL', 1)) + self.ssl_lib_detected = 'nss' + + def using_mbedtls(self): + self.define_macros.append(('HAVE_CURL_MBEDTLS', 1)) + self.libraries.append('mbedtls') + self.define_macros.append(('HAVE_CURL_SSL', 1)) + self.ssl_lib_detected = 'mbedtls' + + def using_sectransp(self): + self.define_macros.append(('HAVE_CURL_SECTRANSP', 1)) + self.define_macros.append(('HAVE_CURL_SSL', 1)) + self.ssl_lib_detected = 'sectransp' + + +def strip_pycurl_options(argv): + if sys.platform == 'win32': + options = [ + '--curl-dir=', '--libcurl-lib-name=', '--use-libcurl-dll', + '--avoid-stdio', '--with-openssl', '--openssl-dir=', + ] + else: + options = ['--openssl-dir=', '--curl-config=', '--avoid-stdio'] + for option in options: + scan_argv(argv, option) + + +############################################################################### + +PRETTY_SSL_LIBS = { + # setup.py may be detecting BoringSSL properly, need to test + 'openssl': 'OpenSSL/LibreSSL/BoringSSL', + 'wolfssl': 'wolfSSL', + 'gnutls': 'GnuTLS', + 'nss': 'NSS', + 'mbedtls': 'mbedTLS', + 'sectransp': 'Secure Transport', +} + +def get_extension(argv, split_extension_source=False): + if split_extension_source: + sources = [ + os.path.join("src", "docstrings.c"), + os.path.join("src", "easy.c"), + os.path.join("src", "easycb.c"), + os.path.join("src", "easyinfo.c"), + os.path.join("src", "easyopt.c"), + os.path.join("src", "easyperform.c"), + os.path.join("src", "module.c"), + os.path.join("src", "multi.c"), + os.path.join("src", "oscompat.c"), + os.path.join("src", "pythoncompat.c"), + os.path.join("src", "share.c"), + os.path.join("src", "stringcompat.c"), + os.path.join("src", "threadsupport.c"), + os.path.join("src", "util.c"), + ] + depends = [ + os.path.join("src", "pycurl.h"), + ] + else: + sources = [ + os.path.join("src", "allpycurl.c"), + ] + depends = [] + ext_config = ExtensionConfiguration(argv) + + if ext_config.ssl_lib_detected: + print('Using SSL library: %s' % PRETTY_SSL_LIBS[ext_config.ssl_lib_detected]) + else: + print('Not using an SSL library') + + ext = Extension( + name=PACKAGE, + sources=sources, + depends=depends, + include_dirs=ext_config.include_dirs, + define_macros=ext_config.define_macros, + library_dirs=ext_config.library_dirs, + libraries=ext_config.libraries, + runtime_library_dirs=ext_config.runtime_library_dirs, + extra_objects=ext_config.extra_objects, + extra_compile_args=ext_config.extra_compile_args, + extra_link_args=ext_config.extra_link_args, + ) + ##print(ext.__dict__); sys.exit(1) + return ext + + +############################################################################### + +# prepare data_files + +def get_data_files(): + # a list of tuples with (path to install to, a list of local files) + data_files = [] + if sys.platform == "win32": + datadir = os.path.join("doc", PACKAGE) + else: + datadir = os.path.join("share", "doc", PACKAGE) + # + files = ["AUTHORS", "ChangeLog", "COPYING-LGPL", "COPYING-MIT", + "INSTALL.rst", "README.rst", "RELEASE-NOTES.rst"] + if files: + data_files.append((os.path.join(datadir), files)) + files = glob.glob(os.path.join("examples", "*.py")) + if files: + data_files.append((os.path.join(datadir, "examples"), files)) + files = glob.glob(os.path.join("examples", "quickstart", "*.py")) + if files: + data_files.append((os.path.join(datadir, "examples", "quickstart"), files)) + # + assert data_files + for install_dir, files in data_files: + assert files + for f in files: + assert os.path.isfile(f), (f, install_dir) + return data_files + + +############################################################################### + +def check_manifest(): + import fnmatch + + f = open('MANIFEST.in') + globs = [] + try: + for line in f.readlines(): + stripped = line.strip() + if stripped == '' or stripped.startswith('#'): + continue + assert stripped.startswith('include ') + glob = stripped[8:] + globs.append(glob) + finally: + f.close() + + paths = [] + start = os.path.abspath(os.path.dirname(__file__)) + for root, dirs, files in os.walk(start): + if '.git' in dirs: + dirs.remove('.git') + for file in files: + if file.endswith('.pyc'): + continue + rel = os.path.join(root, file)[len(start)+1:] + paths.append(rel) + + for path in paths: + included = False + for glob in globs: + if fnmatch.fnmatch(path, glob): + included = True + break + if not included: + print(path) + +AUTHORS_PARAGRAPH = 3 + +def check_authors(): + f = open('AUTHORS') + try: + contents = f.read() + finally: + f.close() + + paras = contents.split("\n\n") + authors_para = paras[AUTHORS_PARAGRAPH] + authors = [author for author in authors_para.strip().split("\n")] + + log = subprocess.check_output(['git', 'log', '--format=%an (%ae)']).decode() + for author in log.strip().split("\n"): + author = author.replace('@', ' at ').replace('(', '<').replace(')', '>') + if author not in authors: + authors.append(author) + authors.sort(key=lambda s: s.lower()) + paras[AUTHORS_PARAGRAPH] = "\n".join(authors) + f = open('AUTHORS', 'w') + try: + f.write("\n\n".join(paras)) + finally: + f.close() + + +def convert_docstrings(): + docstrings = [] + for entry in sorted(os.listdir('doc/docstrings')): + if not entry.endswith('.rst'): + continue + + name = entry.replace('.rst', '') + f = open('doc/docstrings/%s' % entry) + try: + text = f.read().strip() + finally: + f.close() + docstrings.append((name, text)) + f = open('src/docstrings.c', 'w') + try: + f.write("/* Generated file - do not edit. */\n") + # space to avoid having /* inside a C comment + f.write("/* See doc/docstrings/ *.rst. */\n\n") + f.write("#include \"pycurl.h\"\n\n") + for name, text in docstrings: + text = text.replace("\"", "\\\"").replace("\n", "\\n\\\n") + f.write("PYCURL_INTERNAL const char %s_doc[] = \"%s\";\n\n" % (name, text)) + finally: + f.close() + f = open('src/docstrings.h', 'w') + try: + f.write("/* Generated file - do not edit. */\n") + # space to avoid having /* inside a C comment + f.write("/* See doc/docstrings/ *.rst. */\n\n") + for name, text in docstrings: + f.write("extern const char %s_doc[];\n" % name) + finally: + f.close() + + +def gen_docstrings_sources(): + sources = 'DOCSTRINGS_SOURCES =' + for entry in sorted(os.listdir('doc/docstrings')): + if entry.endswith('.rst'): + sources += " \\\n\tdoc/docstrings/%s" % entry + print(sources) + +############################################################################### + +setup_args = dict( + name=PACKAGE, + version=VERSION, + description='PycURL -- A Python Interface To The cURL library', + long_description='''\ +PycURL -- A Python Interface To The cURL library +================================================ + +PycURL is a Python interface to `libcurl`_, the multiprotocol file +transfer library. Similarly to the urllib_ Python module, +PycURL can be used to fetch objects identified by a URL from a Python program. +Beyond simple fetches however PycURL exposes most of the functionality of +libcurl, including: + +- Speed - libcurl is very fast and PycURL, being a thin wrapper above + libcurl, is very fast as well. PycURL `was benchmarked`_ to be several + times faster than requests_. +- Features including multiple protocol support, SSL, authentication and + proxy options. PycURL supports most of libcurl's callbacks. +- Multi_ and share_ interfaces. +- Sockets used for network operations, permitting integration of PycURL + into the application's I/O loop (e.g., using Tornado_). + +.. _was benchmarked: http://stackoverflow.com/questions/15461995/python-requests-vs-pycurl-performance +.. _requests: http://python-requests.org/ +.. _Multi: https://curl.haxx.se/libcurl/c/libcurl-multi.html +.. _share: https://curl.haxx.se/libcurl/c/libcurl-share.html +.. _Tornado: http://www.tornadoweb.org/ + + +Requirements +------------ + +- Python 3.5-3.10. +- libcurl 7.19.0 or better. + + +Installation +------------ + +Download the source distribution from `PyPI`_. + +Please see `the installation documentation`_ for installation instructions. + +.. _PyPI: https://pypi.python.org/pypi/pycurl +.. _the installation documentation: http://pycurl.io/docs/latest/install.html + + +Documentation +------------- + +Documentation for the most recent PycURL release is available on +`PycURL website `_. + + +Support +------- + +For support questions please use `curl-and-python mailing list`_. +`Mailing list archives`_ are available for your perusal as well. + +Although not an official support venue, `Stack Overflow`_ has been +popular with some PycURL users. + +Bugs can be reported `via GitHub`_. Please use GitHub only for bug +reports and direct questions to our mailing list instead. + +.. _curl-and-python mailing list: http://cool.haxx.se/mailman/listinfo/curl-and-python +.. _Stack Overflow: http://stackoverflow.com/questions/tagged/pycurl +.. _Mailing list archives: https://curl.haxx.se/mail/list.cgi?list=curl-and-python +.. _via GitHub: https://github.com/pycurl/pycurl/issues + + +License +------- + +PycURL is dual licensed under the LGPL and an MIT/X derivative license +based on the libcurl license. The complete text of the licenses is available +in COPYING-LGPL_ and COPYING-MIT_ files in the source distribution. + +.. _libcurl: https://curl.haxx.se/libcurl/ +.. _urllib: http://docs.python.org/library/urllib.html +.. _COPYING-LGPL: https://raw.githubusercontent.com/pycurl/pycurl/master/COPYING-LGPL +.. _COPYING-MIT: https://raw.githubusercontent.com/pycurl/pycurl/master/COPYING-MIT +''', + author="Kjetil Jacobsen, Markus F.X.J. Oberhumer, Oleg Pudeyev", + author_email="kjetilja@gmail.com, markus@oberhumer.com, oleg@bsdpower.com", + maintainer="Oleg Pudeyev", + maintainer_email="oleg@bsdpower.com", + url="http://pycurl.io/", + license="LGPL/MIT", + keywords=['curl', 'libcurl', 'urllib', 'wget', 'download', 'file transfer', + 'http', 'www'], + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Web Environment', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)', + 'License :: OSI Approved :: MIT License', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Topic :: Internet :: File Transfer Protocol (FTP)', + 'Topic :: Internet :: WWW/HTTP', + ], + packages=[PY_PACKAGE], + package_dir={ PY_PACKAGE: os.path.join('python', 'curl') }, + python_requires='>=3.5', + platforms='All', +) + +unix_help = '''\ +PycURL Unix options: + --curl-config=/path/to/curl-config use specified curl-config binary + --libcurl-dll=[/path/to/]libcurl.so obtain SSL library from libcurl.so + --openssl-dir=/path/to/openssl/dir path to OpenSSL/LibreSSL/BoringSSL headers and libraries + --with-openssl libcurl is linked against OpenSSL/LibreSSL/BoringSSL + --with-ssl legacy alias for --with-openssl + --with-gnutls libcurl is linked against GnuTLS + --with-nss libcurl is linked against NSS + --with-mbedtls libcurl is linked against mbedTLS + --with-wolfssl libcurl is linked against wolfSSL + --with-sectransp libcurl is linked against Secure Transport +''' + +windows_help = '''\ +PycURL Windows options: + --curl-dir=/path/to/compiled/libcurl path to libcurl headers and libraries + --use-libcurl-dll link against libcurl DLL, if not given + link against libcurl statically + --libcurl-lib-name=libcurl_imp.lib override libcurl import library name + --openssl-dir=/path/to/openssl/dir path to OpenSSL/LibreSSL/BoringSSL headers and libraries + --with-openssl libcurl is linked against OpenSSL/LibreSSL/BoringSSL + --with-ssl legacy alias for --with-openssl + --link-arg=foo.lib also link against specified library +''' + +if __name__ == "__main__": + if '--help' in sys.argv or '-h' in sys.argv: + # unfortunately this help precedes distutils help + if sys.platform == "win32": + print(windows_help) + else: + print(unix_help) + # invoke setup without configuring pycurl because + # configuration might fail, and we want to display help anyway. + # we need to remove our options because distutils complains about them + strip_pycurl_options(sys.argv) + setup(**setup_args) + elif len(sys.argv) > 1 and sys.argv[1] == 'manifest': + check_manifest() + elif len(sys.argv) > 1 and sys.argv[1] == 'docstrings': + convert_docstrings() + elif len(sys.argv) > 1 and sys.argv[1] == 'authors': + check_authors() + elif len(sys.argv) > 1 and sys.argv[1] == 'docstrings-sources': + gen_docstrings_sources() + else: + if sys.argv[1] not in ['clean'] and (not os.path.exists('src/docstrings.c') or not os.path.exists('src/docstrings.h')): + convert_docstrings() + + setup_args['data_files'] = get_data_files() + if 'PYCURL_RELEASE' in os.environ and os.environ['PYCURL_RELEASE'].lower() in ['1', 'yes', 'true']: + split_extension_source = False + else: + split_extension_source = True + ext = get_extension(sys.argv, split_extension_source=split_extension_source) + setup_args['ext_modules'] = [ext] + + for o in ext.extra_objects: + assert os.path.isfile(o), o + setup(**setup_args) diff --git a/src/docstrings.c b/src/docstrings.c new file mode 100644 index 0000000..3f56578 --- /dev/null +++ b/src/docstrings.c @@ -0,0 +1,748 @@ +/* Generated file - do not edit. */ +/* See doc/docstrings/ *.rst. */ + +#include "pycurl.h" + +PYCURL_INTERNAL const char curl_doc[] = "Curl() -> New Curl object\n\ +\n\ +Creates a new :ref:`curlobject` which corresponds to a\n\ +``CURL`` handle in libcurl. Curl objects automatically set\n\ +CURLOPT_VERBOSE to 0, CURLOPT_NOPROGRESS to 1, provide a default\n\ +CURLOPT_USERAGENT and setup CURLOPT_ERRORBUFFER to point to a\n\ +private error buffer.\n\ +\n\ +Implicitly calls :py:func:`pycurl.global_init` if the latter has not yet been called."; + +PYCURL_INTERNAL const char curl_close_doc[] = "close() -> None\n\ +\n\ +Close handle and end curl session.\n\ +\n\ +Corresponds to `curl_easy_cleanup`_ in libcurl. This method is\n\ +automatically called by pycurl when a Curl object no longer has any\n\ +references to it, but can also be called explicitly.\n\ +\n\ +.. _curl_easy_cleanup:\n\ + https://curl.haxx.se/libcurl/c/curl_easy_cleanup.html"; + +PYCURL_INTERNAL const char curl_duphandle_doc[] = "duphandle() -> Curl\n\ +\n\ +Clone a curl handle. This function will return a new curl handle,\n\ +a duplicate, using all the options previously set in the input curl handle.\n\ +Both handles can subsequently be used independently.\n\ +\n\ +The new handle will not inherit any state information, no connections,\n\ +no SSL sessions and no cookies. It also will not inherit any share object\n\ +states or options (it will be made as if SHARE was unset).\n\ +\n\ +Corresponds to `curl_easy_duphandle`_ in libcurl.\n\ +\n\ +Example usage::\n\ +\n\ + import pycurl\n\ + curl = pycurl.Curl()\n\ + curl.setopt(pycurl.URL, \"https://python.org\")\n\ + dup = curl.duphandle()\n\ + curl.perform()\n\ + dup.perform()\n\ +\n\ +.. _curl_easy_duphandle:\n\ + https://curl.se/libcurl/c/curl_easy_duphandle.html"; + +PYCURL_INTERNAL const char curl_errstr_doc[] = "errstr() -> string\n\ +\n\ +Return the internal libcurl error buffer of this handle as a string.\n\ +\n\ +Return value is a ``str`` instance on all Python versions.\n\ +On Python 3, error buffer data is decoded using Python's default encoding\n\ +at the time of the call. If this decoding fails, ``UnicodeDecodeError`` is\n\ +raised. Use :ref:`errstr_raw ` to retrieve the error buffer\n\ +as a byte string in this case.\n\ +\n\ +On Python 2, ``errstr`` and ``errstr_raw`` behave identically."; + +PYCURL_INTERNAL const char curl_errstr_raw_doc[] = "errstr_raw() -> byte string\n\ +\n\ +Return the internal libcurl error buffer of this handle as a byte string.\n\ +\n\ +Return value is a ``str`` instance on Python 2 and ``bytes`` instance\n\ +on Python 3. Unlike :ref:`errstr_raw `, ``errstr_raw``\n\ +allows reading libcurl error buffer in Python 3 when its contents is not\n\ +valid in Python's default encoding.\n\ +\n\ +On Python 2, ``errstr`` and ``errstr_raw`` behave identically.\n\ +\n\ +*Added in version 7.43.0.2.*"; + +PYCURL_INTERNAL const char curl_getinfo_doc[] = "getinfo(option) -> Result\n\ +\n\ +Extract and return information from a curl session,\n\ +decoding string data in Python's default encoding at the time of the call.\n\ +Corresponds to `curl_easy_getinfo`_ in libcurl.\n\ +The ``getinfo`` method should not be called unless\n\ +``perform`` has been called and finished.\n\ +\n\ +*option* is a constant corresponding to one of the\n\ +``CURLINFO_*`` constants in libcurl. Most option constant names match\n\ +the respective ``CURLINFO_*`` constant names with the ``CURLINFO_`` prefix\n\ +removed, for example ``CURLINFO_CONTENT_TYPE`` is accessible as\n\ +``pycurl.CONTENT_TYPE``. Exceptions to this rule are as follows:\n\ +\n\ +- ``CURLINFO_FILETIME`` is mapped as ``pycurl.INFO_FILETIME``\n\ +- ``CURLINFO_COOKIELIST`` is mapped as ``pycurl.INFO_COOKIELIST``\n\ +- ``CURLINFO_CERTINFO`` is mapped as ``pycurl.INFO_CERTINFO``\n\ +- ``CURLINFO_RTSP_CLIENT_CSEQ`` is mapped as ``pycurl.INFO_RTSP_CLIENT_CSEQ``\n\ +- ``CURLINFO_RTSP_CSEQ_RECV`` is mapped as ``pycurl.INFO_RTSP_CSEQ_RECV``\n\ +- ``CURLINFO_RTSP_SERVER_CSEQ`` is mapped as ``pycurl.INFO_RTSP_SERVER_CSEQ``\n\ +- ``CURLINFO_RTSP_SESSION_ID`` is mapped as ``pycurl.INFO_RTSP_SESSION_ID``\n\ +\n\ +The type of return value depends on the option, as follows:\n\ +\n\ +- Options documented by libcurl to return an integer value return a\n\ + Python integer (``long`` on Python 2, ``int`` on Python 3).\n\ +- Options documented by libcurl to return a floating point value\n\ + return a Python ``float``.\n\ +- Options documented by libcurl to return a string value\n\ + return a Python string (``str`` on Python 2 and Python 3).\n\ + On Python 2, the string contains whatever data libcurl returned.\n\ + On Python 3, the data returned by libcurl is decoded using the\n\ + default string encoding at the time of the call.\n\ + If the data cannot be decoded using the default encoding, ``UnicodeDecodeError``\n\ + is raised. Use :ref:`getinfo_raw `\n\ + to retrieve the data as ``bytes`` in these\n\ + cases.\n\ +- ``SSL_ENGINES`` and ``INFO_COOKIELIST`` return a list of strings.\n\ + The same encoding caveats apply; use :ref:`getinfo_raw `\n\ + to retrieve the\n\ + data as a list of byte strings.\n\ +- ``INFO_CERTINFO`` returns a list with one element\n\ + per certificate in the chain, starting with the leaf; each element is a\n\ + sequence of *(key, value)* tuples where both ``key`` and ``value`` are\n\ + strings. String encoding caveats apply; use :ref:`getinfo_raw `\n\ + to retrieve\n\ + certificate data as byte strings.\n\ +\n\ +On Python 2, ``getinfo`` and ``getinfo_raw`` behave identically.\n\ +\n\ +Example usage::\n\ +\n\ + import pycurl\n\ + c = pycurl.Curl()\n\ + c.setopt(pycurl.OPT_CERTINFO, 1)\n\ + c.setopt(pycurl.URL, \"https://python.org\")\n\ + c.setopt(pycurl.FOLLOWLOCATION, 1)\n\ + c.perform()\n\ + print(c.getinfo(pycurl.HTTP_CODE))\n\ + # --> 200\n\ + print(c.getinfo(pycurl.EFFECTIVE_URL))\n\ + # --> \"https://www.python.org/\"\n\ + certinfo = c.getinfo(pycurl.INFO_CERTINFO)\n\ + print(certinfo)\n\ + # --> [(('Subject', 'C = AU, ST = Some-State, O = PycURL test suite,\n\ + CN = localhost'), ('Issuer', 'C = AU, ST = Some-State,\n\ + O = PycURL test suite, OU = localhost, CN = localhost'),\n\ + ('Version', '0'), ...)]\n\ +\n\ +\n\ +Raises pycurl.error exception upon failure.\n\ +\n\ +.. _curl_easy_getinfo:\n\ + https://curl.haxx.se/libcurl/c/curl_easy_getinfo.html"; + +PYCURL_INTERNAL const char curl_getinfo_raw_doc[] = "getinfo_raw(option) -> Result\n\ +\n\ +Extract and return information from a curl session,\n\ +returning string data as byte strings.\n\ +Corresponds to `curl_easy_getinfo`_ in libcurl.\n\ +The ``getinfo_raw`` method should not be called unless\n\ +``perform`` has been called and finished.\n\ +\n\ +*option* is a constant corresponding to one of the\n\ +``CURLINFO_*`` constants in libcurl. Most option constant names match\n\ +the respective ``CURLINFO_*`` constant names with the ``CURLINFO_`` prefix\n\ +removed, for example ``CURLINFO_CONTENT_TYPE`` is accessible as\n\ +``pycurl.CONTENT_TYPE``. Exceptions to this rule are as follows:\n\ +\n\ +- ``CURLINFO_FILETIME`` is mapped as ``pycurl.INFO_FILETIME``\n\ +- ``CURLINFO_COOKIELIST`` is mapped as ``pycurl.INFO_COOKIELIST``\n\ +- ``CURLINFO_CERTINFO`` is mapped as ``pycurl.INFO_CERTINFO``\n\ +- ``CURLINFO_RTSP_CLIENT_CSEQ`` is mapped as ``pycurl.INFO_RTSP_CLIENT_CSEQ``\n\ +- ``CURLINFO_RTSP_CSEQ_RECV`` is mapped as ``pycurl.INFO_RTSP_CSEQ_RECV``\n\ +- ``CURLINFO_RTSP_SERVER_CSEQ`` is mapped as ``pycurl.INFO_RTSP_SERVER_CSEQ``\n\ +- ``CURLINFO_RTSP_SESSION_ID`` is mapped as ``pycurl.INFO_RTSP_SESSION_ID``\n\ +\n\ +The type of return value depends on the option, as follows:\n\ +\n\ +- Options documented by libcurl to return an integer value return a\n\ + Python integer (``long`` on Python 2, ``int`` on Python 3).\n\ +- Options documented by libcurl to return a floating point value\n\ + return a Python ``float``.\n\ +- Options documented by libcurl to return a string value\n\ + return a Python byte string (``str`` on Python 2, ``bytes`` on Python 3).\n\ + The string contains whatever data libcurl returned.\n\ + Use :ref:`getinfo ` to retrieve this data as a Unicode string on Python 3.\n\ +- ``SSL_ENGINES`` and ``INFO_COOKIELIST`` return a list of byte strings.\n\ + The same encoding caveats apply; use :ref:`getinfo ` to retrieve the\n\ + data as a list of potentially Unicode strings.\n\ +- ``INFO_CERTINFO`` returns a list with one element\n\ + per certificate in the chain, starting with the leaf; each element is a\n\ + sequence of *(key, value)* tuples where both ``key`` and ``value`` are\n\ + byte strings. String encoding caveats apply; use :ref:`getinfo `\n\ + to retrieve\n\ + certificate data as potentially Unicode strings.\n\ +\n\ +On Python 2, ``getinfo`` and ``getinfo_raw`` behave identically.\n\ +\n\ +Example usage::\n\ +\n\ + import pycurl\n\ + c = pycurl.Curl()\n\ + c.setopt(pycurl.OPT_CERTINFO, 1)\n\ + c.setopt(pycurl.URL, \"https://python.org\")\n\ + c.setopt(pycurl.FOLLOWLOCATION, 1)\n\ + c.perform()\n\ + print(c.getinfo_raw(pycurl.HTTP_CODE))\n\ + # --> 200\n\ + print(c.getinfo_raw(pycurl.EFFECTIVE_URL))\n\ + # --> b\"https://www.python.org/\"\n\ + certinfo = c.getinfo_raw(pycurl.INFO_CERTINFO)\n\ + print(certinfo)\n\ + # --> [((b'Subject', b'C = AU, ST = Some-State, O = PycURL test suite,\n\ + CN = localhost'), (b'Issuer', b'C = AU, ST = Some-State,\n\ + O = PycURL test suite, OU = localhost, CN = localhost'),\n\ + (b'Version', b'0'), ...)]\n\ +\n\ +\n\ +Raises pycurl.error exception upon failure.\n\ +\n\ +*Added in version 7.43.0.2.*\n\ +\n\ +.. _curl_easy_getinfo:\n\ + https://curl.haxx.se/libcurl/c/curl_easy_getinfo.html"; + +PYCURL_INTERNAL const char curl_pause_doc[] = "pause(bitmask) -> None\n\ +\n\ +Pause or unpause a curl handle. Bitmask should be a value such as\n\ +PAUSE_RECV or PAUSE_CONT.\n\ +\n\ +Corresponds to `curl_easy_pause`_ in libcurl. The argument should be\n\ +derived from the ``PAUSE_RECV``, ``PAUSE_SEND``, ``PAUSE_ALL`` and\n\ +``PAUSE_CONT`` constants.\n\ +\n\ +Raises pycurl.error exception upon failure.\n\ +\n\ +.. _curl_easy_pause: https://curl.haxx.se/libcurl/c/curl_easy_pause.html"; + +PYCURL_INTERNAL const char curl_perform_doc[] = "perform() -> None\n\ +\n\ +Perform a file transfer.\n\ +\n\ +Corresponds to `curl_easy_perform`_ in libcurl.\n\ +\n\ +Raises pycurl.error exception upon failure.\n\ +\n\ +.. _curl_easy_perform:\n\ + https://curl.haxx.se/libcurl/c/curl_easy_perform.html"; + +PYCURL_INTERNAL const char curl_perform_rb_doc[] = "perform_rb() -> response_body\n\ +\n\ +Perform a file transfer and return response body as a byte string.\n\ +\n\ +This method arranges for response body to be saved in a StringIO\n\ +(Python 2) or BytesIO (Python 3) instance, then invokes :ref:`perform `\n\ +to perform the file transfer, then returns the value of the StringIO/BytesIO\n\ +instance which is a ``str`` instance on Python 2 and ``bytes`` instance\n\ +on Python 3. Errors during transfer raise ``pycurl.error`` exceptions\n\ +just like in :ref:`perform `.\n\ +\n\ +Use :ref:`perform_rs ` to retrieve response body as a string\n\ +(``str`` instance on both Python 2 and 3).\n\ +\n\ +Raises ``pycurl.error`` exception upon failure.\n\ +\n\ +*Added in version 7.43.0.2.*"; + +PYCURL_INTERNAL const char curl_perform_rs_doc[] = "perform_rs() -> response_body\n\ +\n\ +Perform a file transfer and return response body as a string.\n\ +\n\ +On Python 2, this method arranges for response body to be saved in a StringIO\n\ +instance, then invokes :ref:`perform `\n\ +to perform the file transfer, then returns the value of the StringIO instance.\n\ +This behavior is identical to :ref:`perform_rb `.\n\ +\n\ +On Python 3, this method arranges for response body to be saved in a BytesIO\n\ +instance, then invokes :ref:`perform `\n\ +to perform the file transfer, then decodes the response body in Python's\n\ +default encoding and returns the decoded body as a Unicode string\n\ +(``str`` instance). *Note:* decoding happens after the transfer finishes,\n\ +thus an encoding error implies the transfer/network operation succeeded.\n\ +\n\ +Any transfer errors raise ``pycurl.error`` exception,\n\ +just like in :ref:`perform `.\n\ +\n\ +Use :ref:`perform_rb ` to retrieve response body as a byte\n\ +string (``bytes`` instance on Python 3) without attempting to decode it.\n\ +\n\ +Raises ``pycurl.error`` exception upon failure.\n\ +\n\ +*Added in version 7.43.0.2.*"; + +PYCURL_INTERNAL const char curl_reset_doc[] = "reset() -> None\n\ +\n\ +Reset all options set on curl handle to default values, but preserves\n\ +live connections, session ID cache, DNS cache, cookies, and shares.\n\ +\n\ +Corresponds to `curl_easy_reset`_ in libcurl.\n\ +\n\ +.. _curl_easy_reset: https://curl.haxx.se/libcurl/c/curl_easy_reset.html"; + +PYCURL_INTERNAL const char curl_set_ca_certs_doc[] = "set_ca_certs() -> None\n\ +\n\ +Load ca certs from provided unicode string.\n\ +\n\ +Note that certificates will be added only when cURL starts new connection."; + +PYCURL_INTERNAL const char curl_setopt_doc[] = "setopt(option, value) -> None\n\ +\n\ +Set curl session option. Corresponds to `curl_easy_setopt`_ in libcurl.\n\ +\n\ +*option* specifies which option to set. PycURL defines constants\n\ +corresponding to ``CURLOPT_*`` constants in libcurl, except that\n\ +the ``CURLOPT_`` prefix is removed. For example, ``CURLOPT_URL`` is\n\ +exposed in PycURL as ``pycurl.URL``, with some exceptions as detailed below.\n\ +For convenience, ``CURLOPT_*``\n\ +constants are also exposed on the Curl objects themselves::\n\ +\n\ + import pycurl\n\ + c = pycurl.Curl()\n\ + c.setopt(pycurl.URL, \"http://www.python.org/\")\n\ + # Same as:\n\ + c.setopt(c.URL, \"http://www.python.org/\")\n\ +\n\ +The following are exceptions to option constant naming convention:\n\ +\n\ +- ``CURLOPT_FILETIME`` is mapped as ``pycurl.OPT_FILETIME``\n\ +- ``CURLOPT_CERTINFO`` is mapped as ``pycurl.OPT_CERTINFO``\n\ +- ``CURLOPT_COOKIELIST`` is mapped as ``pycurl.COOKIELIST``\n\ + and, as of PycURL 7.43.0.2, also as ``pycurl.OPT_COOKIELIST``\n\ +- ``CURLOPT_RTSP_CLIENT_CSEQ`` is mapped as ``pycurl.OPT_RTSP_CLIENT_CSEQ``\n\ +- ``CURLOPT_RTSP_REQUEST`` is mapped as ``pycurl.OPT_RTSP_REQUEST``\n\ +- ``CURLOPT_RTSP_SERVER_CSEQ`` is mapped as ``pycurl.OPT_RTSP_SERVER_CSEQ``\n\ +- ``CURLOPT_RTSP_SESSION_ID`` is mapped as ``pycurl.OPT_RTSP_SESSION_ID``\n\ +- ``CURLOPT_RTSP_STREAM_URI`` is mapped as ``pycurl.OPT_RTSP_STREAM_URI``\n\ +- ``CURLOPT_RTSP_TRANSPORT`` is mapped as ``pycurl.OPT_RTSP_TRANSPORT``\n\ +\n\ +*value* specifies the value to set the option to. Different options accept\n\ +values of different types:\n\ +\n\ +- Options specified by `curl_easy_setopt`_ as accepting ``1`` or an\n\ + integer value accept Python integers, long integers (on Python 2.x) and\n\ + booleans::\n\ +\n\ + c.setopt(pycurl.FOLLOWLOCATION, True)\n\ + c.setopt(pycurl.FOLLOWLOCATION, 1)\n\ + # Python 2.x only:\n\ + c.setopt(pycurl.FOLLOWLOCATION, 1L)\n\ +\n\ +- Options specified as accepting strings by ``curl_easy_setopt`` accept\n\ + byte strings (``str`` on Python 2, ``bytes`` on Python 3) and\n\ + Unicode strings with ASCII code points only.\n\ + For more information, please refer to :ref:`unicode`. Example::\n\ +\n\ + c.setopt(pycurl.URL, \"http://www.python.org/\")\n\ + c.setopt(pycurl.URL, u\"http://www.python.org/\")\n\ + # Python 3.x only:\n\ + c.setopt(pycurl.URL, b\"http://www.python.org/\")\n\ +\n\ +- ``HTTP200ALIASES``, ``HTTPHEADER``, ``POSTQUOTE``, ``PREQUOTE``,\n\ + ``PROXYHEADER`` and\n\ + ``QUOTE`` accept a list or tuple of strings. The same rules apply to these\n\ + strings as do to string option values. Example::\n\ +\n\ + c.setopt(pycurl.HTTPHEADER, [\"Accept:\"])\n\ + c.setopt(pycurl.HTTPHEADER, (\"Accept:\",))\n\ +\n\ +- ``READDATA`` accepts a file object or any Python object which has\n\ + a ``read`` method. On Python 2, a file object will be passed directly\n\ + to libcurl and may result in greater transfer efficiency, unless\n\ + PycURL has been compiled with ``AVOID_STDIO`` option.\n\ + On Python 3 and on Python 2 when the value is not a true file object,\n\ + ``READDATA`` is emulated in PycURL via ``READFUNCTION``.\n\ + The file should generally be opened in binary mode. Example::\n\ +\n\ + f = open('file.txt', 'rb')\n\ + c.setopt(c.READDATA, f)\n\ +\n\ +- ``WRITEDATA`` and ``WRITEHEADER`` accept a file object or any Python\n\ + object which has a ``write`` method. On Python 2, a file object will\n\ + be passed directly to libcurl and may result in greater transfer efficiency,\n\ + unless PycURL has been compiled with ``AVOID_STDIO`` option.\n\ + On Python 3 and on Python 2 when the value is not a true file object,\n\ + ``WRITEDATA`` is emulated in PycURL via ``WRITEFUNCTION``.\n\ + The file should generally be opened in binary mode. Example::\n\ +\n\ + f = open('/dev/null', 'wb')\n\ + c.setopt(c.WRITEDATA, f)\n\ +\n\ +- ``*FUNCTION`` options accept a function. Supported callbacks are documented\n\ + in :ref:`callbacks`. Example::\n\ +\n\ + # Python 2\n\ + import StringIO\n\ + b = StringIO.StringIO()\n\ + c.setopt(pycurl.WRITEFUNCTION, b.write)\n\ +\n\ +- ``SHARE`` option accepts a :ref:`curlshareobject`.\n\ +\n\ +It is possible to set integer options - and only them - that PycURL does\n\ +not know about by using the numeric value of the option constant directly.\n\ +For example, ``pycurl.VERBOSE`` has the value 42, and may be set as follows::\n\ +\n\ + c.setopt(42, 1)\n\ +\n\ +*setopt* can reset some options to their default value, performing the job of\n\ +:py:meth:`pycurl.Curl.unsetopt`, if ``None`` is passed\n\ +for the option value. The following two calls are equivalent::\n\ +\n\ + c.setopt(c.URL, None)\n\ + c.unsetopt(c.URL)\n\ +\n\ +Raises TypeError when the option value is not of a type accepted by the\n\ +respective option, and pycurl.error exception when libcurl rejects the\n\ +option or its value.\n\ +\n\ +.. _curl_easy_setopt: https://curl.haxx.se/libcurl/c/curl_easy_setopt.html"; + +PYCURL_INTERNAL const char curl_setopt_string_doc[] = "setopt_string(option, value) -> None\n\ +\n\ +Set curl session option to a string value.\n\ +\n\ +This method allows setting string options that are not officially supported\n\ +by PycURL, for example because they did not exist when the version of PycURL\n\ +being used was released.\n\ +:py:meth:`pycurl.Curl.setopt` should be used for setting options that\n\ +PycURL knows about.\n\ +\n\ +**Warning:** No checking is performed that *option* does, in fact,\n\ +expect a string value. Using this method incorrectly can crash the program\n\ +and may lead to a security vulnerability.\n\ +Furthermore, it is on the application to ensure that the *value* object\n\ +does not get garbage collected while libcurl is using it.\n\ +libcurl copies most string options but not all; one option whose value\n\ +is not copied by libcurl is `CURLOPT_POSTFIELDS`_.\n\ +\n\ +*option* would generally need to be given as an integer literal rather than\n\ +a symbolic constant.\n\ +\n\ +*value* can be a binary string or a Unicode string using ASCII code points,\n\ +same as with string options given to PycURL elsewhere.\n\ +\n\ +Example setting URL via ``setopt_string``::\n\ +\n\ + import pycurl\n\ + c = pycurl.Curl()\n\ + c.setopt_string(10002, \"http://www.python.org/\")\n\ +\n\ +.. _CURLOPT_POSTFIELDS: https://curl.haxx.se/libcurl/c/CURLOPT_POSTFIELDS.html"; + +PYCURL_INTERNAL const char curl_unsetopt_doc[] = "unsetopt(option) -> None\n\ +\n\ +Reset curl session option to its default value.\n\ +\n\ +Only some curl options may be reset via this method.\n\ +\n\ +libcurl does not provide a general way to reset a single option to its default value;\n\ +:py:meth:`pycurl.Curl.reset` resets all options to their default values,\n\ +otherwise :py:meth:`pycurl.Curl.setopt` must be called with whatever value\n\ +is the default. For convenience, PycURL provides this unsetopt method\n\ +to reset some of the options to their default values.\n\ +\n\ +Raises pycurl.error exception on failure.\n\ +\n\ +``c.unsetopt(option)`` is equivalent to ``c.setopt(option, None)``."; + +PYCURL_INTERNAL const char multi_doc[] = "CurlMulti() -> New CurlMulti object\n\ +\n\ +Creates a new :ref:`curlmultiobject` which corresponds to\n\ +a ``CURLM`` handle in libcurl."; + +PYCURL_INTERNAL const char multi_add_handle_doc[] = "add_handle(Curl object) -> None\n\ +\n\ +Corresponds to `curl_multi_add_handle`_ in libcurl. This method adds an\n\ +existing and valid Curl object to the CurlMulti object.\n\ +\n\ +*Changed in version 7.43.0.2:* add_handle now ensures that the Curl object\n\ +is not garbage collected while it is being used by a CurlMulti object.\n\ +Previously application had to maintain an outstanding reference to the Curl\n\ +object to keep it from being garbage collected.\n\ +\n\ +.. _curl_multi_add_handle:\n\ + https://curl.haxx.se/libcurl/c/curl_multi_add_handle.html"; + +PYCURL_INTERNAL const char multi_assign_doc[] = "assign(sock_fd, object) -> None\n\ +\n\ +Creates an association in the multi handle between the given socket and\n\ +a private object in the application.\n\ +Corresponds to `curl_multi_assign`_ in libcurl.\n\ +\n\ +.. _curl_multi_assign: https://curl.haxx.se/libcurl/c/curl_multi_assign.html"; + +PYCURL_INTERNAL const char multi_close_doc[] = "close() -> None\n\ +\n\ +Corresponds to `curl_multi_cleanup`_ in libcurl. This method is\n\ +automatically called by pycurl when a CurlMulti object no longer has any\n\ +references to it, but can also be called explicitly.\n\ +\n\ +.. _curl_multi_cleanup:\n\ + https://curl.haxx.se/libcurl/c/curl_multi_cleanup.html"; + +PYCURL_INTERNAL const char multi_fdset_doc[] = "fdset() -> tuple of lists with active file descriptors, readable, writeable, exceptions\n\ +\n\ +Returns a tuple of three lists that can be passed to the select.select() method.\n\ +\n\ +Corresponds to `curl_multi_fdset`_ in libcurl. This method extracts the\n\ +file descriptor information from a CurlMulti object. The returned lists can\n\ +be used with the ``select`` module to poll for events.\n\ +\n\ +Example usage::\n\ +\n\ + import pycurl\n\ + c = pycurl.Curl()\n\ + c.setopt(pycurl.URL, \"https://curl.haxx.se\")\n\ + m = pycurl.CurlMulti()\n\ + m.add_handle(c)\n\ + while 1:\n\ + ret, num_handles = m.perform()\n\ + if ret != pycurl.E_CALL_MULTI_PERFORM: break\n\ + while num_handles:\n\ + apply(select.select, m.fdset() + (1,))\n\ + while 1:\n\ + ret, num_handles = m.perform()\n\ + if ret != pycurl.E_CALL_MULTI_PERFORM: break\n\ +\n\ +.. _curl_multi_fdset:\n\ + https://curl.haxx.se/libcurl/c/curl_multi_fdset.html"; + +PYCURL_INTERNAL const char multi_info_read_doc[] = "info_read([max_objects]) -> tuple(number of queued messages, a list of successful objects, a list of failed objects)\n\ +\n\ +Corresponds to the `curl_multi_info_read`_ function in libcurl.\n\ +\n\ +This method extracts at most *max* messages from the multi stack and returns\n\ +them in two lists. The first list contains the handles which completed\n\ +successfully and the second list contains a tuple *(curl object, curl error\n\ +number, curl error message)* for each failed curl object. The curl error\n\ +message is returned as a Python string which is decoded from the curl error\n\ +string using the `surrogateescape`_ error handler. The number of\n\ +queued messages after this method has been called is also returned.\n\ +\n\ +.. _curl_multi_info_read:\n\ + https://curl.haxx.se/libcurl/c/curl_multi_info_read.html\n\ +\n\ +.. _surrogateescape:\n\ + https://www.python.org/dev/peps/pep-0383/"; + +PYCURL_INTERNAL const char multi_perform_doc[] = "perform() -> tuple of status and the number of active Curl objects\n\ +\n\ +Corresponds to `curl_multi_perform`_ in libcurl.\n\ +\n\ +.. _curl_multi_perform:\n\ + https://curl.haxx.se/libcurl/c/curl_multi_perform.html"; + +PYCURL_INTERNAL const char multi_remove_handle_doc[] = "remove_handle(Curl object) -> None\n\ +\n\ +Corresponds to `curl_multi_remove_handle`_ in libcurl. This method\n\ +removes an existing and valid Curl object from the CurlMulti object.\n\ +\n\ +.. _curl_multi_remove_handle:\n\ + https://curl.haxx.se/libcurl/c/curl_multi_remove_handle.html"; + +PYCURL_INTERNAL const char multi_select_doc[] = "select([timeout]) -> number of ready file descriptors or 0 on timeout\n\ +\n\ +Returns result from doing a select() on the curl multi file descriptor\n\ +with the given timeout.\n\ +\n\ +This is a convenience function which simplifies the combined use of\n\ +``fdset()`` and the ``select`` module.\n\ +\n\ +Example usage::\n\ +\n\ + import pycurl\n\ + c = pycurl.Curl()\n\ + c.setopt(pycurl.URL, \"https://curl.haxx.se\")\n\ + m = pycurl.CurlMulti()\n\ + m.add_handle(c)\n\ + while 1:\n\ + ret, num_handles = m.perform()\n\ + if ret != pycurl.E_CALL_MULTI_PERFORM: break\n\ + while num_handles:\n\ + ret = m.select(1.0)\n\ + if ret == 0: continue\n\ + while 1:\n\ + ret, num_handles = m.perform()\n\ + if ret != pycurl.E_CALL_MULTI_PERFORM: break"; + +PYCURL_INTERNAL const char multi_setopt_doc[] = "setopt(option, value) -> None\n\ +\n\ +Set curl multi option. Corresponds to `curl_multi_setopt`_ in libcurl.\n\ +\n\ +*option* specifies which option to set. PycURL defines constants\n\ +corresponding to ``CURLMOPT_*`` constants in libcurl, except that\n\ +the ``CURLMOPT_`` prefix is replaced with ``M_`` prefix.\n\ +For example, ``CURLMOPT_PIPELINING`` is\n\ +exposed in PycURL as ``pycurl.M_PIPELINING``. For convenience, ``CURLMOPT_*``\n\ +constants are also exposed on CurlMulti objects::\n\ +\n\ + import pycurl\n\ + m = pycurl.CurlMulti()\n\ + m.setopt(pycurl.M_PIPELINING, 1)\n\ + # Same as:\n\ + m.setopt(m.M_PIPELINING, 1)\n\ +\n\ +*value* specifies the value to set the option to. Different options accept\n\ +values of different types:\n\ +\n\ +- Options specified by `curl_multi_setopt`_ as accepting ``1`` or an\n\ + integer value accept Python integers, long integers (on Python 2.x) and\n\ + booleans::\n\ +\n\ + m.setopt(pycurl.M_PIPELINING, True)\n\ + m.setopt(pycurl.M_PIPELINING, 1)\n\ + # Python 2.x only:\n\ + m.setopt(pycurl.M_PIPELINING, 1L)\n\ +\n\ +- ``*FUNCTION`` options accept a function. Supported callbacks are\n\ + ``CURLMOPT_SOCKETFUNCTION`` AND ``CURLMOPT_TIMERFUNCTION``. Please refer to\n\ + the PycURL test suite for examples on using the callbacks.\n\ +\n\ +Raises TypeError when the option value is not of a type accepted by the\n\ +respective option, and pycurl.error exception when libcurl rejects the\n\ +option or its value.\n\ +\n\ +.. _curl_multi_setopt: https://curl.haxx.se/libcurl/c/curl_multi_setopt.html"; + +PYCURL_INTERNAL const char multi_socket_action_doc[] = "socket_action(sock_fd, ev_bitmask) -> (result, num_running_handles)\n\ +\n\ +Returns result from doing a socket_action() on the curl multi file descriptor\n\ +with the given timeout.\n\ +Corresponds to `curl_multi_socket_action`_ in libcurl.\n\ +\n\ +The return value is a two-element tuple. The first element is the return\n\ +value of the underlying ``curl_multi_socket_action`` function, and it is\n\ +always zero (``CURLE_OK``) because any other return value would cause\n\ +``socket_action`` to raise an exception. The second element is the number of\n\ +running easy handles within this multi handle. When the number of running\n\ +handles reaches zero, all transfers have completed. Note that if the number\n\ +of running handles has decreased by one compared to the previous invocation,\n\ +this is not mean the handle corresponding to the ``sock_fd`` provided as\n\ +the argument to this function was the completed handle.\n\ +\n\ +.. _curl_multi_socket_action: https://curl.haxx.se/libcurl/c/curl_multi_socket_action.html"; + +PYCURL_INTERNAL const char multi_socket_all_doc[] = "socket_all() -> tuple\n\ +\n\ +Returns result from doing a socket_all() on the curl multi file descriptor\n\ +with the given timeout."; + +PYCURL_INTERNAL const char multi_timeout_doc[] = "timeout() -> int\n\ +\n\ +Returns how long to wait for action before proceeding.\n\ +Corresponds to `curl_multi_timeout`_ in libcurl.\n\ +\n\ +.. _curl_multi_timeout: https://curl.haxx.se/libcurl/c/curl_multi_timeout.html"; + +PYCURL_INTERNAL const char pycurl_global_cleanup_doc[] = "global_cleanup() -> None\n\ +\n\ +Cleanup curl environment.\n\ +\n\ +Corresponds to `curl_global_cleanup`_ in libcurl.\n\ +\n\ +.. _curl_global_cleanup: https://curl.haxx.se/libcurl/c/curl_global_cleanup.html"; + +PYCURL_INTERNAL const char pycurl_global_init_doc[] = "global_init(option) -> None\n\ +\n\ +Initialize curl environment.\n\ +\n\ +*option* is one of the constants pycurl.GLOBAL_SSL, pycurl.GLOBAL_WIN32,\n\ +pycurl.GLOBAL_ALL, pycurl.GLOBAL_NOTHING, pycurl.GLOBAL_DEFAULT.\n\ +\n\ +Corresponds to `curl_global_init`_ in libcurl.\n\ +\n\ +.. _curl_global_init: https://curl.haxx.se/libcurl/c/curl_global_init.html"; + +PYCURL_INTERNAL const char pycurl_module_doc[] = "This module implements an interface to the cURL library.\n\ +\n\ +Types:\n\ +\n\ +Curl() -> New object. Create a new curl object.\n\ +CurlMulti() -> New object. Create a new curl multi object.\n\ +CurlShare() -> New object. Create a new curl share object.\n\ +\n\ +Functions:\n\ +\n\ +global_init(option) -> None. Initialize curl environment.\n\ +global_cleanup() -> None. Cleanup curl environment.\n\ +version_info() -> tuple. Return version information."; + +PYCURL_INTERNAL const char pycurl_version_info_doc[] = "version_info() -> tuple\n\ +\n\ +Returns a 12-tuple with the version info.\n\ +\n\ +Corresponds to `curl_version_info`_ in libcurl. Returns a tuple of\n\ +information which is similar to the ``curl_version_info_data`` struct\n\ +returned by ``curl_version_info()`` in libcurl.\n\ +\n\ +Example usage::\n\ +\n\ + >>> import pycurl\n\ + >>> pycurl.version_info()\n\ + (3, '7.33.0', 467200, 'amd64-portbld-freebsd9.1', 33436, 'OpenSSL/0.9.8x',\n\ + 0, '1.2.7', ('dict', 'file', 'ftp', 'ftps', 'gopher', 'http', 'https',\n\ + 'imap', 'imaps', 'pop3', 'pop3s', 'rtsp', 'smtp', 'smtps', 'telnet',\n\ + 'tftp'), None, 0, None)\n\ +\n\ +.. _curl_version_info: https://curl.haxx.se/libcurl/c/curl_version_info.html"; + +PYCURL_INTERNAL const char share_doc[] = "CurlShare() -> New CurlShare object\n\ +\n\ +Creates a new :ref:`curlshareobject` which corresponds to a\n\ +``CURLSH`` handle in libcurl. CurlShare objects is what you pass as an\n\ +argument to the SHARE option on :ref:`Curl objects `."; + +PYCURL_INTERNAL const char share_close_doc[] = "close() -> None\n\ +\n\ +Close shared handle.\n\ +\n\ +Corresponds to `curl_share_cleanup`_ in libcurl. This method is\n\ +automatically called by pycurl when a CurlShare object no longer has\n\ +any references to it, but can also be called explicitly.\n\ +\n\ +.. _curl_share_cleanup:\n\ + https://curl.haxx.se/libcurl/c/curl_share_cleanup.html"; + +PYCURL_INTERNAL const char share_setopt_doc[] = "setopt(option, value) -> None\n\ +\n\ +Set curl share option.\n\ +\n\ +Corresponds to `curl_share_setopt`_ in libcurl, where *option* is\n\ +specified with the ``CURLSHOPT_*`` constants in libcurl, except that the\n\ +``CURLSHOPT_`` prefix has been changed to ``SH_``. Currently, *value* must be\n\ +one of: ``LOCK_DATA_COOKIE``, ``LOCK_DATA_DNS``, ``LOCK_DATA_SSL_SESSION`` or\n\ +``LOCK_DATA_CONNECT``.\n\ +\n\ +Example usage::\n\ +\n\ + import pycurl\n\ + curl = pycurl.Curl()\n\ + s = pycurl.CurlShare()\n\ + s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)\n\ + s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)\n\ + curl.setopt(pycurl.URL, 'https://curl.haxx.se')\n\ + curl.setopt(pycurl.SHARE, s)\n\ + curl.perform()\n\ + curl.close()\n\ +\n\ +Raises pycurl.error exception upon failure.\n\ +\n\ +.. _curl_share_setopt:\n\ + https://curl.haxx.se/libcurl/c/curl_share_setopt.html"; + diff --git a/src/docstrings.h b/src/docstrings.h new file mode 100644 index 0000000..1932120 --- /dev/null +++ b/src/docstrings.h @@ -0,0 +1,39 @@ +/* Generated file - do not edit. */ +/* See doc/docstrings/ *.rst. */ + +extern const char curl_doc[]; +extern const char curl_close_doc[]; +extern const char curl_duphandle_doc[]; +extern const char curl_errstr_doc[]; +extern const char curl_errstr_raw_doc[]; +extern const char curl_getinfo_doc[]; +extern const char curl_getinfo_raw_doc[]; +extern const char curl_pause_doc[]; +extern const char curl_perform_doc[]; +extern const char curl_perform_rb_doc[]; +extern const char curl_perform_rs_doc[]; +extern const char curl_reset_doc[]; +extern const char curl_set_ca_certs_doc[]; +extern const char curl_setopt_doc[]; +extern const char curl_setopt_string_doc[]; +extern const char curl_unsetopt_doc[]; +extern const char multi_doc[]; +extern const char multi_add_handle_doc[]; +extern const char multi_assign_doc[]; +extern const char multi_close_doc[]; +extern const char multi_fdset_doc[]; +extern const char multi_info_read_doc[]; +extern const char multi_perform_doc[]; +extern const char multi_remove_handle_doc[]; +extern const char multi_select_doc[]; +extern const char multi_setopt_doc[]; +extern const char multi_socket_action_doc[]; +extern const char multi_socket_all_doc[]; +extern const char multi_timeout_doc[]; +extern const char pycurl_global_cleanup_doc[]; +extern const char pycurl_global_init_doc[]; +extern const char pycurl_module_doc[]; +extern const char pycurl_version_info_doc[]; +extern const char share_doc[]; +extern const char share_close_doc[]; +extern const char share_setopt_doc[]; diff --git a/src/easy.c b/src/easy.c new file mode 100644 index 0000000..1b3464a --- /dev/null +++ b/src/easy.c @@ -0,0 +1,875 @@ +#include "pycurl.h" +#include "docstrings.h" + + +/************************************************************************* +// CurlSlistObject +**************************************************************************/ + +PYCURL_INTERNAL void +util_curlslist_update(CurlSlistObject **old, struct curl_slist *slist) +{ + /* Decref previous object */ + Py_XDECREF(*old); + /* Create a new object */ + *old = PyObject_New(CurlSlistObject, p_CurlSlist_Type); + assert(*old != NULL); + /* Store curl_slist into the new object */ + (*old)->slist = slist; +} + +PYCURL_INTERNAL void +do_curlslist_dealloc(CurlSlistObject *self) { + if (self->slist != NULL) { + curl_slist_free_all(self->slist); + self->slist = NULL; + } + CurlSlist_Type.tp_free(self); +} + +PYCURL_INTERNAL PyTypeObject CurlSlist_Type = { +#if PY_MAJOR_VERSION >= 3 + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ +#endif + "pycurl.CurlSlist", /* tp_name */ + sizeof(CurlSlistObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)do_curlslist_dealloc, /* tp_dealloc */ + 0, /* tp_print / tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved / tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + 0, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ +#if PY_MAJOR_VERSION >= 3 + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0, /* tp_finalize */ +#if PY_VERSION_HEX >= 0x03080000 + 0, /* tp_vectorcall */ +#endif +#endif +}; + + +/************************************************************************* +// CurlHttppostObject +**************************************************************************/ + +PYCURL_INTERNAL void +util_curlhttppost_update(CurlObject *obj, struct curl_httppost *httppost, PyObject *reflist) +{ + /* Decref previous object */ + Py_XDECREF(obj->httppost); + /* Create a new object */ + obj->httppost = PyObject_New(CurlHttppostObject, p_CurlHttppost_Type); + assert(obj->httppost != NULL); + /* Store curl_httppost and reflist into the new object */ + obj->httppost->httppost = httppost; + obj->httppost->reflist = reflist; +} + +PYCURL_INTERNAL void +do_curlhttppost_dealloc(CurlHttppostObject *self) { + if (self->httppost != NULL) { + curl_formfree(self->httppost); + self->httppost = NULL; + } + Py_CLEAR(self->reflist); + CurlHttppost_Type.tp_free(self); +} + +PYCURL_INTERNAL PyTypeObject CurlHttppost_Type = { +#if PY_MAJOR_VERSION >= 3 + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ +#endif + "pycurl.CurlHttppost", /* tp_name */ + sizeof(CurlHttppostObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)do_curlhttppost_dealloc, /* tp_dealloc */ + 0, /* tp_print / tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved / tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + 0, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ +#if PY_MAJOR_VERSION >= 3 + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0, /* tp_finalize */ +#if PY_VERSION_HEX >= 0x03080000 + 0, /* tp_vectorcall */ +#endif +#endif +}; + + +/************************************************************************* +// static utility functions +**************************************************************************/ + + +/* assert some CurlObject invariants */ +PYCURL_INTERNAL void +assert_curl_state(const CurlObject *self) +{ + assert(self != NULL); + assert(PyObject_IsInstance((PyObject *) self, (PyObject *) p_Curl_Type) == 1); +#ifdef WITH_THREAD + (void) pycurl_get_thread_state(self); +#endif +} + + +/* check state for methods */ +PYCURL_INTERNAL int +check_curl_state(const CurlObject *self, int flags, const char *name) +{ + assert_curl_state(self); + if ((flags & 1) && self->handle == NULL) { + PyErr_Format(ErrorObject, "cannot invoke %s() - no curl handle", name); + return -1; + } +#ifdef WITH_THREAD + if ((flags & 2) && pycurl_get_thread_state(self) != NULL) { + PyErr_Format(ErrorObject, "cannot invoke %s() - perform() is currently running", name); + return -1; + } +#endif + return 0; +} + + +/************************************************************************* +// CurlObject +**************************************************************************/ + +/* --------------- construct/destruct (i.e. open/close) --------------- */ + +/* initializer - used to initialize curl easy handles for use with pycurl */ +static int +util_curl_init(CurlObject *self) +{ + int res; + + /* Set curl error buffer and zero it */ + res = curl_easy_setopt(self->handle, CURLOPT_ERRORBUFFER, self->error); + if (res != CURLE_OK) { + return (-1); + } + memset(self->error, 0, sizeof(self->error)); + + /* Set backreference */ + res = curl_easy_setopt(self->handle, CURLOPT_PRIVATE, (char *) self); + if (res != CURLE_OK) { + return (-1); + } + + /* Enable NOPROGRESS by default, i.e. no progress output */ + res = curl_easy_setopt(self->handle, CURLOPT_NOPROGRESS, (long)1); + if (res != CURLE_OK) { + return (-1); + } + + /* Disable VERBOSE by default, i.e. no verbose output */ + res = curl_easy_setopt(self->handle, CURLOPT_VERBOSE, (long)0); + if (res != CURLE_OK) { + return (-1); + } + + /* Set default USERAGENT */ + assert(g_pycurl_useragent); + res = curl_easy_setopt(self->handle, CURLOPT_USERAGENT, g_pycurl_useragent); + if (res != CURLE_OK) { + return (-1); + } + return (0); +} + +/* constructor */ +PYCURL_INTERNAL CurlObject * +do_curl_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) +{ + CurlObject *self; + int res; + int *ptr; + + if (subtype == p_Curl_Type && !PyArg_ParseTupleAndKeywords(args, kwds, "", empty_keywords)) { + return NULL; + } + + /* Allocate python curl object */ + self = (CurlObject *) subtype->tp_alloc(subtype, 0); + if (self == NULL) + return NULL; + + /* tp_alloc is expected to return zeroed memory */ + for (ptr = (int *) &self->dict; + ptr < (int *) (((char *) self) + sizeof(CurlObject)); + ++ptr) + assert(*ptr == 0); + + /* Initialize curl handle */ + self->handle = curl_easy_init(); + if (self->handle == NULL) + goto error; + + res = util_curl_init(self); + if (res < 0) + goto error; + /* Success - return new object */ + return self; + +error: + Py_DECREF(self); /* this also closes self->handle */ + PyErr_SetString(ErrorObject, "initializing curl failed"); + return NULL; +} + +/* duphandle */ +PYCURL_INTERNAL CurlObject * +do_curl_duphandle(CurlObject *self) +{ + PyTypeObject *subtype; + CurlObject *dup; + int res; + int *ptr; + + /* Allocate python curl object */ + subtype = Py_TYPE(self); + dup = (CurlObject *) subtype->tp_alloc(subtype, 0); + if (dup == NULL) + return NULL; + + /* tp_alloc is expected to return zeroed memory */ + for (ptr = (int *) &dup->dict; + ptr < (int *) (((char *) dup) + sizeof(CurlObject)); + ++ptr) + assert(*ptr == 0); + + /* Clone the curl handle */ + dup->handle = curl_easy_duphandle(self->handle); + if (dup->handle == NULL) + goto error; + + /* Set curl error buffer and zero it */ + res = curl_easy_setopt(dup->handle, CURLOPT_ERRORBUFFER, dup->error); + if (res != CURLE_OK) { + goto error; + } + memset(dup->error, 0, sizeof(dup->error)); + + /* Set backreference */ + res = curl_easy_setopt(dup->handle, CURLOPT_PRIVATE, (char *) dup); + if (res != CURLE_OK) { + goto error; + } + + /* Copy attribute dictionary */ + if (self->dict != NULL) { + dup->dict = PyDict_Copy(self->dict); + if (dup->dict == NULL) { + goto error; + } + } + + /* Checking for CURLE_OK is not required here. + * All values have already been successfully setopt'ed with self->handle. */ + + /* Assign and incref python callback and update data pointers */ + if (self->w_cb != NULL) { + dup->w_cb = my_Py_NewRef(self->w_cb); + curl_easy_setopt(dup->handle, CURLOPT_WRITEDATA, dup); + } + if (self->h_cb != NULL) { + dup->h_cb = my_Py_NewRef(self->h_cb); + curl_easy_setopt(dup->handle, CURLOPT_WRITEHEADER, dup); + } + if (self->r_cb != NULL) { + dup->r_cb = my_Py_NewRef(self->r_cb); + curl_easy_setopt(dup->handle, CURLOPT_READDATA, dup); + } + if (self->pro_cb != NULL) { + dup->pro_cb = my_Py_NewRef(self->pro_cb); + curl_easy_setopt(dup->handle, CURLOPT_PROGRESSDATA, dup); + } +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 32, 0) + if (self->xferinfo_cb != NULL) { + dup->xferinfo_cb = my_Py_NewRef(self->xferinfo_cb); + curl_easy_setopt(dup->handle, CURLOPT_XFERINFODATA, dup); + } +#endif + if (self->debug_cb != NULL) { + dup->debug_cb = my_Py_NewRef(self->debug_cb); + curl_easy_setopt(dup->handle, CURLOPT_DEBUGDATA, dup); + } + if (self->ioctl_cb != NULL) { + dup->ioctl_cb = my_Py_NewRef(self->ioctl_cb); + curl_easy_setopt(dup->handle, CURLOPT_IOCTLDATA, dup); + } + if (self->opensocket_cb != NULL) { + dup->opensocket_cb = my_Py_NewRef(self->opensocket_cb); + curl_easy_setopt(dup->handle, CURLOPT_OPENSOCKETDATA, dup); + } +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 7) + if (self->closesocket_cb != NULL) { + dup->closesocket_cb = my_Py_NewRef(self->closesocket_cb); + curl_easy_setopt(dup->handle, CURLOPT_CLOSESOCKETDATA, dup); + } +#endif + if (self->sockopt_cb != NULL) { + dup->sockopt_cb = my_Py_NewRef(self->sockopt_cb); + curl_easy_setopt(dup->handle, CURLOPT_SOCKOPTDATA, dup); + } +#ifdef HAVE_CURL_7_19_6_OPTS + if (self->ssh_key_cb != NULL) { + dup->ssh_key_cb = my_Py_NewRef(self->ssh_key_cb); + curl_easy_setopt(dup->handle, CURLOPT_SSH_KEYDATA, dup); + } +#endif + if (self->seek_cb != NULL) { + dup->seek_cb = my_Py_NewRef(self->seek_cb); + curl_easy_setopt(dup->handle, CURLOPT_SEEKDATA, dup); + } + + /* Assign and incref python file objects */ + dup->readdata_fp = my_Py_XNewRef(self->readdata_fp); + dup->writedata_fp = my_Py_XNewRef(self->writedata_fp); + dup->writeheader_fp = my_Py_XNewRef(self->writeheader_fp); + + /* Assign and incref postfields object */ + dup->postfields_obj = my_Py_XNewRef(self->postfields_obj); + + /* Assign and incref ca certs related references */ + dup->ca_certs_obj = my_Py_XNewRef(self->ca_certs_obj); + + /* Assign and incref every curl_slist allocated by setopt */ + dup->httpheader = (CurlSlistObject *)my_Py_XNewRef((PyObject *)self->httpheader); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 37, 0) + dup->proxyheader = (CurlSlistObject *)my_Py_XNewRef((PyObject *)self->proxyheader); +#endif + dup->http200aliases = (CurlSlistObject *)my_Py_XNewRef((PyObject *)self->http200aliases); + dup->quote = (CurlSlistObject *)my_Py_XNewRef((PyObject *)self->quote); + dup->postquote = (CurlSlistObject *)my_Py_XNewRef((PyObject *)self->postquote); + dup->prequote = (CurlSlistObject *)my_Py_XNewRef((PyObject *)self->prequote); + dup->telnetoptions = (CurlSlistObject *)my_Py_XNewRef((PyObject *)self->telnetoptions); +#ifdef HAVE_CURLOPT_RESOLVE + dup->resolve = (CurlSlistObject *)my_Py_XNewRef((PyObject *)self->resolve); +#endif +#ifdef HAVE_CURL_7_20_0_OPTS + dup->mail_rcpt = (CurlSlistObject *)my_Py_XNewRef((PyObject *)self->mail_rcpt); +#endif +#ifdef HAVE_CURLOPT_CONNECT_TO + dup->connect_to = (CurlSlistObject *)my_Py_XNewRef((PyObject *)self->connect_to); +#endif + + /* Assign and incref httppost */ + dup->httppost = (CurlHttppostObject *)my_Py_XNewRef((PyObject *)self->httppost); + + /* Success - return cloned object */ + return dup; + +error: + Py_CLEAR(dup->dict); + Py_DECREF(dup); /* this also closes dup->handle */ + PyErr_SetString(ErrorObject, "cloning curl failed"); + return NULL; +} + + +/* util function shared by close() and clear() */ +PYCURL_INTERNAL void +util_curl_xdecref(CurlObject *self, int flags, CURL *handle) +{ + if (flags & PYCURL_MEMGROUP_ATTRDICT) { + /* Decrement refcount for attributes dictionary. */ + Py_CLEAR(self->dict); + } + + if (flags & PYCURL_MEMGROUP_MULTI) { + /* Decrement refcount for multi_stack. */ + if (self->multi_stack != NULL) { + CurlMultiObject *multi_stack = self->multi_stack; + if (multi_stack->multi_handle != NULL && handle != NULL) { + /* TODO this is where we could remove the easy object + from the multi object's easy_object_dict, but this + requires us to have a reference to the multi object + which right now we don't. */ + /* Allow threads because callbacks can be invoked */ + PYCURL_BEGIN_ALLOW_THREADS_EASY + (void) curl_multi_remove_handle(multi_stack->multi_handle, handle); + PYCURL_END_ALLOW_THREADS_EASY + } + self->multi_stack = NULL; + Py_DECREF(multi_stack); + } + } + + if (flags & PYCURL_MEMGROUP_CALLBACK) { + /* Decrement refcount for python callbacks. */ + Py_CLEAR(self->w_cb); + Py_CLEAR(self->h_cb); + Py_CLEAR(self->r_cb); + Py_CLEAR(self->pro_cb); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 32, 0) + Py_CLEAR(self->xferinfo_cb); +#endif + Py_CLEAR(self->debug_cb); + Py_CLEAR(self->ioctl_cb); + Py_CLEAR(self->seek_cb); + Py_CLEAR(self->opensocket_cb); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 7) + Py_CLEAR(self->closesocket_cb); +#endif + Py_CLEAR(self->sockopt_cb); + Py_CLEAR(self->ssh_key_cb); + } + + if (flags & PYCURL_MEMGROUP_FILE) { + /* Decrement refcount for python file objects. */ + Py_CLEAR(self->readdata_fp); + Py_CLEAR(self->writedata_fp); + Py_CLEAR(self->writeheader_fp); + } + + if (flags & PYCURL_MEMGROUP_POSTFIELDS) { + /* Decrement refcount for postfields object */ + Py_CLEAR(self->postfields_obj); + } + + if (flags & PYCURL_MEMGROUP_SHARE) { + /* Decrement refcount for share objects. */ + if (self->share != NULL) { + CurlShareObject *share = self->share; + self->share = NULL; + if (share->share_handle != NULL && handle != NULL) { + curl_easy_setopt(handle, CURLOPT_SHARE, NULL); + } + Py_DECREF(share); + } + } + + if (flags & PYCURL_MEMGROUP_HTTPPOST) { + /* Decrement refcounts for httppost object. */ + Py_CLEAR(self->httppost); + } + + if (flags & PYCURL_MEMGROUP_CACERTS) { + /* Decrement refcounts for ca certs related references. */ + Py_CLEAR(self->ca_certs_obj); + } + + if (flags & PYCURL_MEMGROUP_SLIST) { + /* Decrement refcounts for slist objects. */ + Py_CLEAR(self->httpheader); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 37, 0) + Py_CLEAR(self->proxyheader); +#endif + Py_CLEAR(self->http200aliases); + Py_CLEAR(self->quote); + Py_CLEAR(self->postquote); + Py_CLEAR(self->prequote); + Py_CLEAR(self->telnetoptions); +#ifdef HAVE_CURLOPT_RESOLVE + Py_CLEAR(self->resolve); +#endif +#ifdef HAVE_CURL_7_20_0_OPTS + Py_CLEAR(self->mail_rcpt); +#endif +#ifdef HAVE_CURLOPT_CONNECT_TO + Py_CLEAR(self->connect_to); +#endif + } +} + + +static void +util_curl_close(CurlObject *self) +{ + CURL *handle; + + /* Zero handle and thread-state to disallow any operations to be run + * from now on */ + assert(self != NULL); + assert(PyObject_IsInstance((PyObject *) self, (PyObject *) p_Curl_Type) == 1); + handle = self->handle; + self->handle = NULL; + if (handle == NULL) { + /* Some paranoia assertions just to make sure the object + * deallocation problem is finally really fixed... */ +#ifdef WITH_THREAD + assert(self->state == NULL); +#endif + assert(self->multi_stack == NULL); + assert(self->share == NULL); + return; /* already closed */ + } +#ifdef WITH_THREAD + self->state = NULL; +#endif + + /* Decref multi stuff which uses this handle */ + util_curl_xdecref(self, PYCURL_MEMGROUP_MULTI, handle); + /* Decref share which uses this handle */ + util_curl_xdecref(self, PYCURL_MEMGROUP_SHARE, handle); + + /* Cleanup curl handle - must be done without the gil */ + Py_BEGIN_ALLOW_THREADS + curl_easy_cleanup(handle); + Py_END_ALLOW_THREADS + handle = NULL; + + /* Decref easy related objects */ + util_curl_xdecref(self, PYCURL_MEMGROUP_EASY, handle); + + if (self->weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject *) self); + } +} + + +PYCURL_INTERNAL void +do_curl_dealloc(CurlObject *self) +{ + PyObject_GC_UnTrack(self); + CPy_TRASHCAN_BEGIN(self, do_curl_dealloc); + + Py_CLEAR(self->dict); + util_curl_close(self); + + Curl_Type.tp_free(self); + CPy_TRASHCAN_END(self); +} + + +static PyObject * +do_curl_close(CurlObject *self) +{ + if (check_curl_state(self, 2, "close") != 0) { + return NULL; + } + util_curl_close(self); + Py_RETURN_NONE; +} + + +/* --------------- GC support --------------- */ + +/* Drop references that may have created reference cycles. */ +PYCURL_INTERNAL int +do_curl_clear(CurlObject *self) +{ +#ifdef WITH_THREAD + assert(pycurl_get_thread_state(self) == NULL); +#endif + util_curl_xdecref(self, PYCURL_MEMGROUP_ALL, self->handle); + return 0; +} + +/* Traverse all refcounted objects. */ +PYCURL_INTERNAL int +do_curl_traverse(CurlObject *self, visitproc visit, void *arg) +{ + int err; +#undef VISIT +#define VISIT(v) if ((v) != NULL && ((err = visit(v, arg)) != 0)) return err + + VISIT(self->dict); + VISIT((PyObject *) self->multi_stack); + VISIT((PyObject *) self->share); + + VISIT(self->w_cb); + VISIT(self->h_cb); + VISIT(self->r_cb); + VISIT(self->pro_cb); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 32, 0) + VISIT(self->xferinfo_cb); +#endif + VISIT(self->debug_cb); + VISIT(self->ioctl_cb); + VISIT(self->seek_cb); + VISIT(self->opensocket_cb); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 7) + VISIT(self->closesocket_cb); +#endif + VISIT(self->sockopt_cb); + VISIT(self->ssh_key_cb); + + VISIT(self->readdata_fp); + VISIT(self->writedata_fp); + VISIT(self->writeheader_fp); + + VISIT(self->postfields_obj); + + VISIT(self->ca_certs_obj); + + VISIT((PyObject *) self->httpheader); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 37, 0) + VISIT((PyObject *) self->proxyheader); +#endif + VISIT((PyObject *) self->http200aliases); + VISIT((PyObject *) self->quote); + VISIT((PyObject *) self->postquote); + VISIT((PyObject *) self->prequote); + VISIT((PyObject *) self->telnetoptions); +#ifdef HAVE_CURLOPT_RESOLVE + VISIT((PyObject *) self->resolve); +#endif +#ifdef HAVE_CURL_7_20_0_OPTS + VISIT((PyObject *) self->mail_rcpt); +#endif +#ifdef HAVE_CURLOPT_CONNECT_TO + VISIT((PyObject *) self->connect_to); +#endif + + VISIT((PyObject *) self->httppost); + + return 0; +#undef VISIT +} + + +/* ------------------------ reset ------------------------ */ + +static PyObject* +do_curl_reset(CurlObject *self) +{ + int res; + + curl_easy_reset(self->handle); + + /* Decref easy interface related objects */ + util_curl_xdecref(self, PYCURL_MEMGROUP_EASY, self->handle); + + res = util_curl_init(self); + if (res < 0) { + Py_DECREF(self); /* this also closes self->handle */ + PyErr_SetString(ErrorObject, "resetting curl failed"); + return NULL; + } + + Py_RETURN_NONE; +} + + +static PyObject *do_curl_getstate(CurlObject *self) +{ + PyErr_SetString(PyExc_TypeError, "Curl objects do not support serialization"); + return NULL; +} + + +static PyObject *do_curl_setstate(CurlObject *self, PyObject *args) +{ + PyErr_SetString(PyExc_TypeError, "Curl objects do not support deserialization"); + return NULL; +} + + +/************************************************************************* +// type definitions +**************************************************************************/ + +/* --------------- methods --------------- */ + +PYCURL_INTERNAL PyMethodDef curlobject_methods[] = { + {"close", (PyCFunction)do_curl_close, METH_NOARGS, curl_close_doc}, + {"errstr", (PyCFunction)do_curl_errstr, METH_NOARGS, curl_errstr_doc}, + {"errstr_raw", (PyCFunction)do_curl_errstr_raw, METH_NOARGS, curl_errstr_raw_doc}, + {"getinfo", (PyCFunction)do_curl_getinfo, METH_VARARGS, curl_getinfo_doc}, + {"getinfo_raw", (PyCFunction)do_curl_getinfo_raw, METH_VARARGS, curl_getinfo_raw_doc}, + {"pause", (PyCFunction)do_curl_pause, METH_VARARGS, curl_pause_doc}, + {"perform", (PyCFunction)do_curl_perform, METH_NOARGS, curl_perform_doc}, + {"perform_rb", (PyCFunction)do_curl_perform_rb, METH_NOARGS, curl_perform_rb_doc}, + {"perform_rs", (PyCFunction)do_curl_perform_rs, METH_NOARGS, curl_perform_rs_doc}, + {"setopt", (PyCFunction)do_curl_setopt, METH_VARARGS, curl_setopt_doc}, + {"setopt_string", (PyCFunction)do_curl_setopt_string, METH_VARARGS, curl_setopt_string_doc}, + {"unsetopt", (PyCFunction)do_curl_unsetopt, METH_VARARGS, curl_unsetopt_doc}, + {"reset", (PyCFunction)do_curl_reset, METH_NOARGS, curl_reset_doc}, + {"duphandle", (PyCFunction)do_curl_duphandle, METH_NOARGS, curl_duphandle_doc}, +#if defined(HAVE_CURL_OPENSSL) + {"set_ca_certs", (PyCFunction)do_curl_set_ca_certs, METH_VARARGS, curl_set_ca_certs_doc}, +#endif + {"__getstate__", (PyCFunction)do_curl_getstate, METH_NOARGS, NULL}, + {"__setstate__", (PyCFunction)do_curl_setstate, METH_VARARGS, NULL}, + {NULL, NULL, 0, NULL} +}; + + +/* --------------- setattr/getattr --------------- */ + + +#if PY_MAJOR_VERSION >= 3 + +PYCURL_INTERNAL PyObject * +do_curl_getattro(PyObject *o, PyObject *n) +{ + PyObject *v = PyObject_GenericGetAttr(o, n); + if( !v && PyErr_ExceptionMatches(PyExc_AttributeError) ) + { + PyErr_Clear(); + v = my_getattro(o, n, ((CurlObject *)o)->dict, + curlobject_constants, curlobject_methods); + } + return v; +} + +PYCURL_INTERNAL int +do_curl_setattro(PyObject *o, PyObject *name, PyObject *v) +{ + assert_curl_state((CurlObject *)o); + return my_setattro(&((CurlObject *)o)->dict, name, v); +} + +#else /* PY_MAJOR_VERSION >= 3 */ + +PYCURL_INTERNAL PyObject * +do_curl_getattr(CurlObject *co, char *name) +{ + assert_curl_state(co); + return my_getattr((PyObject *)co, name, co->dict, + curlobject_constants, curlobject_methods); +} + +PYCURL_INTERNAL int +do_curl_setattr(CurlObject *co, char *name, PyObject *v) +{ + assert_curl_state(co); + return my_setattr(&co->dict, name, v); +} + +#endif /* PY_MAJOR_VERSION >= 3 */ + +PYCURL_INTERNAL PyTypeObject Curl_Type = { +#if PY_MAJOR_VERSION >= 3 + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ +#endif + "pycurl.Curl", /* tp_name */ + sizeof(CurlObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)do_curl_dealloc, /* tp_dealloc */ + 0, /* tp_print */ +#if PY_MAJOR_VERSION >= 3 + 0, /* tp_getattr */ + 0, /* tp_setattr */ +#else + (getattrfunc)do_curl_getattr, /* tp_getattr */ + (setattrfunc)do_curl_setattr, /* tp_setattr */ +#endif + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ +#if PY_MAJOR_VERSION >= 3 + (getattrofunc)do_curl_getattro, /* tp_getattro */ + (setattrofunc)do_curl_setattro, /* tp_setattro */ +#else + 0, /* tp_getattro */ + 0, /* tp_setattro */ +#endif + 0, /* tp_as_buffer */ + PYCURL_TYPE_FLAGS, /* tp_flags */ + curl_doc, /* tp_doc */ + (traverseproc)do_curl_traverse, /* tp_traverse */ + (inquiry)do_curl_clear, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(CurlObject, weakreflist), /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + curlobject_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + (newfunc)do_curl_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; + +/* vi:ts=4:et:nowrap + */ diff --git a/src/easycb.c b/src/easycb.c new file mode 100644 index 0000000..d0a330f --- /dev/null +++ b/src/easycb.c @@ -0,0 +1,941 @@ +#include "pycurl.h" + + +/* IMPORTANT NOTE: due to threading issues, we cannot call _any_ Python + * function without acquiring the thread state in the callback handlers. + */ + + +static size_t +util_write_callback(int flags, char *ptr, size_t size, size_t nmemb, void *stream) +{ + CurlObject *self; + PyObject *arglist; + PyObject *result = NULL; + size_t ret = 0; /* assume error */ + PyObject *cb; + int total_size; + PYCURL_DECLARE_THREAD_STATE; + + /* acquire thread */ + self = (CurlObject *)stream; + if (!PYCURL_ACQUIRE_THREAD()) + return ret; + + /* check args */ + cb = flags ? self->h_cb : self->w_cb; + if (cb == NULL) + goto silent_error; + if (size <= 0 || nmemb <= 0) + goto done; + total_size = (int)(size * nmemb); + if (total_size < 0 || (size_t)total_size / size != nmemb) { + PyErr_SetString(ErrorObject, "integer overflow in write callback"); + goto verbose_error; + } + + /* run callback */ +#if PY_MAJOR_VERSION >= 3 + arglist = Py_BuildValue("(y#)", ptr, total_size); +#else + arglist = Py_BuildValue("(s#)", ptr, total_size); +#endif + if (arglist == NULL) + goto verbose_error; + result = PyObject_Call(cb, arglist, NULL); + Py_DECREF(arglist); + if (result == NULL) + goto verbose_error; + + /* handle result */ + if (result == Py_None) { + ret = total_size; /* None means success */ + } + else if (PyInt_Check(result) || PyLong_Check(result)) { + /* if the cast to long fails, PyLong_AsLong() returns -1L */ + ret = (size_t) PyLong_AsLong(result); + } + else { + PyErr_SetString(ErrorObject, "write callback must return int or None"); + goto verbose_error; + } + +done: +silent_error: + Py_XDECREF(result); + PYCURL_RELEASE_THREAD(); + return ret; +verbose_error: + PyErr_Print(); + goto silent_error; +} + + +PYCURL_INTERNAL size_t +write_callback(char *ptr, size_t size, size_t nmemb, void *stream) +{ + return util_write_callback(0, ptr, size, nmemb, stream); +} + +PYCURL_INTERNAL size_t +header_callback(char *ptr, size_t size, size_t nmemb, void *stream) +{ + return util_write_callback(1, ptr, size, nmemb, stream); +} + + +/* convert protocol address from C to python, returns a tuple of protocol + specific values */ +static PyObject * +convert_protocol_address(struct sockaddr* saddr, unsigned int saddrlen) +{ + PyObject *res_obj = NULL; + + switch (saddr->sa_family) + { + case AF_INET: + { + struct sockaddr_in* sin = (struct sockaddr_in*)saddr; + char *addr_str = PyMem_New(char, INET_ADDRSTRLEN); + + if (addr_str == NULL) { + PyErr_NoMemory(); + goto error; + } + + if (inet_ntop(saddr->sa_family, &sin->sin_addr, addr_str, INET_ADDRSTRLEN) == NULL) { + PyErr_SetFromErrno(ErrorObject); + PyMem_Free(addr_str); + goto error; + } + res_obj = Py_BuildValue("(si)", addr_str, ntohs(sin->sin_port)); + PyMem_Free(addr_str); + } + break; + case AF_INET6: + { + struct sockaddr_in6* sin6 = (struct sockaddr_in6*)saddr; + char *addr_str = PyMem_New(char, INET6_ADDRSTRLEN); + + if (addr_str == NULL) { + PyErr_NoMemory(); + goto error; + } + + if (inet_ntop(saddr->sa_family, &sin6->sin6_addr, addr_str, INET6_ADDRSTRLEN) == NULL) { + PyErr_SetFromErrno(ErrorObject); + PyMem_Free(addr_str); + goto error; + } + res_obj = Py_BuildValue("(siii)", addr_str, (int) ntohs(sin6->sin6_port), + (int) ntohl(sin6->sin6_flowinfo), (int) ntohl(sin6->sin6_scope_id)); + PyMem_Free(addr_str); + } + break; +#if !defined(WIN32) + case AF_UNIX: + { + struct sockaddr_un* s_un = (struct sockaddr_un*)saddr; + +#if PY_MAJOR_VERSION >= 3 + res_obj = Py_BuildValue("y", s_un->sun_path); +#else + res_obj = Py_BuildValue("s", s_un->sun_path); +#endif + } + break; +#endif + default: + /* We (currently) only support IPv4/6 addresses. Can curl even be used + with anything else? */ + PyErr_SetString(ErrorObject, "Unsupported address family"); + } + +error: + return res_obj; +} + + +/* curl_socket_t is just an int on unix/windows (with limitations that + * are not important here) */ +PYCURL_INTERNAL curl_socket_t +opensocket_callback(void *clientp, curlsocktype purpose, + struct curl_sockaddr *address) +{ + PyObject *arglist; + PyObject *result = NULL; + PyObject *fileno_result = NULL; + CurlObject *self; + int ret = CURL_SOCKET_BAD; + PyObject *converted_address; + PyObject *python_address; + PYCURL_DECLARE_THREAD_STATE; + + self = (CurlObject *)clientp; + PYCURL_ACQUIRE_THREAD(); + + converted_address = convert_protocol_address(&address->addr, address->addrlen); + if (converted_address == NULL) { + goto verbose_error; + } + + arglist = Py_BuildValue("(iiiN)", address->family, address->socktype, address->protocol, converted_address); + if (arglist == NULL) { + Py_DECREF(converted_address); + goto verbose_error; + } + python_address = PyObject_Call(curl_sockaddr_type, arglist, NULL); + Py_DECREF(arglist); + if (python_address == NULL) { + goto verbose_error; + } + + arglist = Py_BuildValue("(iN)", purpose, python_address); + if (arglist == NULL) { + Py_DECREF(python_address); + goto verbose_error; + } + result = PyObject_Call(self->opensocket_cb, arglist, NULL); + Py_DECREF(arglist); + if (result == NULL) { + goto verbose_error; + } + + if (PyInt_Check(result) && PyInt_AsLong(result) == CURL_SOCKET_BAD) { + ret = CURL_SOCKET_BAD; + } else if (PyObject_HasAttrString(result, "fileno")) { + fileno_result = PyObject_CallMethod(result, "fileno", NULL); + + if (fileno_result == NULL) { + ret = CURL_SOCKET_BAD; + goto verbose_error; + } + // normal operation: + if (PyInt_Check(fileno_result)) { + int sock_fd = PyInt_AsLong(fileno_result); +#if defined(WIN32) + ret = dup_winsock(sock_fd, address); +#else + ret = dup(sock_fd); +#endif + goto done; + } else { + PyErr_SetString(ErrorObject, "Open socket callback returned an object whose fileno method did not return an integer"); + ret = CURL_SOCKET_BAD; + } + } else { + PyErr_SetString(ErrorObject, "Open socket callback's return value must be a socket"); + ret = CURL_SOCKET_BAD; + goto verbose_error; + } + +silent_error: +done: + Py_XDECREF(result); + Py_XDECREF(fileno_result); + PYCURL_RELEASE_THREAD(); + return ret; +verbose_error: + PyErr_Print(); + goto silent_error; +} + + +PYCURL_INTERNAL int +sockopt_cb(void *clientp, curl_socket_t curlfd, curlsocktype purpose) +{ + PyObject *arglist; + CurlObject *self; + int ret = -1; + PyObject *ret_obj = NULL; + PYCURL_DECLARE_THREAD_STATE; + + self = (CurlObject *)clientp; + PYCURL_ACQUIRE_THREAD(); + + arglist = Py_BuildValue("(ii)", (int) curlfd, (int) purpose); + if (arglist == NULL) + goto verbose_error; + + ret_obj = PyObject_Call(self->sockopt_cb, arglist, NULL); + Py_DECREF(arglist); + if (!PyInt_Check(ret_obj) && !PyLong_Check(ret_obj)) { + PyObject *ret_repr = PyObject_Repr(ret_obj); + if (ret_repr) { + PyObject *encoded_obj; + char *str = PyText_AsString_NoNUL(ret_repr, &encoded_obj); + fprintf(stderr, "sockopt callback returned %s which is not an integer\n", str); + /* PyErr_Format(PyExc_TypeError, "sockopt callback returned %s which is not an integer", str); */ + Py_XDECREF(encoded_obj); + Py_DECREF(ret_repr); + } + goto silent_error; + } + if (PyInt_Check(ret_obj)) { + /* long to int cast */ + ret = (int) PyInt_AsLong(ret_obj); + } else { + /* long to int cast */ + ret = (int) PyLong_AsLong(ret_obj); + } + goto done; + +silent_error: + ret = -1; +done: + Py_XDECREF(ret_obj); + PYCURL_RELEASE_THREAD(); + return ret; +verbose_error: + PyErr_Print(); + goto silent_error; +} + + +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 7) +PYCURL_INTERNAL int +closesocket_callback(void *clientp, curl_socket_t curlfd) +{ + PyObject *arglist; + CurlObject *self; + int ret = -1; + PyObject *ret_obj = NULL; + PYCURL_DECLARE_THREAD_STATE; + + self = (CurlObject *)clientp; + PYCURL_ACQUIRE_THREAD(); + + arglist = Py_BuildValue("(i)", (int) curlfd); + if (arglist == NULL) + goto verbose_error; + + ret_obj = PyObject_Call(self->closesocket_cb, arglist, NULL); + Py_DECREF(arglist); + if (!ret_obj) + goto silent_error; + if (!PyInt_Check(ret_obj) && !PyLong_Check(ret_obj)) { + PyObject *ret_repr = PyObject_Repr(ret_obj); + if (ret_repr) { + PyObject *encoded_obj; + char *str = PyText_AsString_NoNUL(ret_repr, &encoded_obj); + fprintf(stderr, "closesocket callback returned %s which is not an integer\n", str); + /* PyErr_Format(PyExc_TypeError, "closesocket callback returned %s which is not an integer", str); */ + Py_XDECREF(encoded_obj); + Py_DECREF(ret_repr); + } + goto silent_error; + } + if (PyInt_Check(ret_obj)) { + /* long to int cast */ + ret = (int) PyInt_AsLong(ret_obj); + } else { + /* long to int cast */ + ret = (int) PyLong_AsLong(ret_obj); + } + goto done; + +silent_error: + ret = -1; +done: + Py_XDECREF(ret_obj); + PYCURL_RELEASE_THREAD(); + return ret; +verbose_error: + PyErr_Print(); + goto silent_error; +} +#endif + + +#ifdef HAVE_CURL_7_19_6_OPTS +static PyObject * +khkey_to_object(const struct curl_khkey *khkey) +{ + PyObject *arglist, *ret; + + if (khkey == NULL) { + Py_INCREF(Py_None); + return Py_None; + } + + if (khkey->len) { +#if PY_MAJOR_VERSION >= 3 + arglist = Py_BuildValue("(y#i)", khkey->key, khkey->len, khkey->keytype); +#else + arglist = Py_BuildValue("(s#i)", khkey->key, khkey->len, khkey->keytype); +#endif + } else { +#if PY_MAJOR_VERSION >= 3 + arglist = Py_BuildValue("(yi)", khkey->key, khkey->keytype); +#else + arglist = Py_BuildValue("(si)", khkey->key, khkey->keytype); +#endif + } + + if (arglist == NULL) { + return NULL; + } + + ret = PyObject_Call(khkey_type, arglist, NULL); + Py_DECREF(arglist); + return ret; +} + + +PYCURL_INTERNAL int +ssh_key_cb(CURL *easy, const struct curl_khkey *knownkey, + const struct curl_khkey *foundkey, int khmatch, void *clientp) +{ + PyObject *arglist; + CurlObject *self; + int ret = -1; + PyObject *knownkey_obj = NULL; + PyObject *foundkey_obj = NULL; + PyObject *ret_obj = NULL; + PYCURL_DECLARE_THREAD_STATE; + + self = (CurlObject *)clientp; + PYCURL_ACQUIRE_THREAD(); + + knownkey_obj = khkey_to_object(knownkey); + if (knownkey_obj == NULL) { + goto silent_error; + } + foundkey_obj = khkey_to_object(foundkey); + if (foundkey_obj == NULL) { + goto silent_error; + } + + arglist = Py_BuildValue("(OOi)", knownkey_obj, foundkey_obj, khmatch); + if (arglist == NULL) + goto verbose_error; + + ret_obj = PyObject_Call(self->ssh_key_cb, arglist, NULL); + Py_DECREF(arglist); + if (!PyInt_Check(ret_obj) && !PyLong_Check(ret_obj)) { + PyObject *ret_repr = PyObject_Repr(ret_obj); + if (ret_repr) { + PyObject *encoded_obj; + char *str = PyText_AsString_NoNUL(ret_repr, &encoded_obj); + fprintf(stderr, "ssh key callback returned %s which is not an integer\n", str); + /* PyErr_Format(PyExc_TypeError, "ssh key callback returned %s which is not an integer", str); */ + Py_XDECREF(encoded_obj); + Py_DECREF(ret_repr); + } + goto silent_error; + } + if (PyInt_Check(ret_obj)) { + /* long to int cast */ + ret = (int) PyInt_AsLong(ret_obj); + } else { + /* long to int cast */ + ret = (int) PyLong_AsLong(ret_obj); + } + goto done; + +silent_error: + ret = -1; +done: + Py_XDECREF(knownkey_obj); + Py_XDECREF(foundkey_obj); + Py_XDECREF(ret_obj); + PYCURL_RELEASE_THREAD(); + return ret; +verbose_error: + PyErr_Print(); + goto silent_error; +} +#endif + + +PYCURL_INTERNAL int +seek_callback(void *stream, curl_off_t offset, int origin) +{ + CurlObject *self; + PyObject *arglist; + PyObject *result = NULL; + int ret = 2; /* assume error 2 (can't seek, libcurl free to work around). */ + PyObject *cb; + int source = 0; /* assume beginning */ + PYCURL_DECLARE_THREAD_STATE; + + /* acquire thread */ + self = (CurlObject *)stream; + if (!PYCURL_ACQUIRE_THREAD()) + return ret; + + /* check arguments */ + switch (origin) + { + case SEEK_SET: + source = 0; + break; + case SEEK_CUR: + source = 1; + break; + case SEEK_END: + source = 2; + break; + default: + source = origin; + break; + } + + /* run callback */ + cb = self->seek_cb; + if (cb == NULL) + goto silent_error; + arglist = Py_BuildValue("(L,i)", (PY_LONG_LONG) offset, source); + if (arglist == NULL) + goto verbose_error; + result = PyObject_Call(cb, arglist, NULL); + Py_DECREF(arglist); + if (result == NULL) + goto verbose_error; + + /* handle result */ + if (result == Py_None) { + ret = 0; /* None means success */ + } + else if (PyInt_Check(result)) { + int ret_code = PyInt_AsLong(result); + if (ret_code < 0 || ret_code > 2) { + PyErr_Format(ErrorObject, "invalid return value for seek callback %d not in (0, 1, 2)", ret_code); + goto verbose_error; + } + ret = ret_code; /* pass the return code from the callback */ + } + else { + PyErr_SetString(ErrorObject, "seek callback must return 0 (CURL_SEEKFUNC_OK), 1 (CURL_SEEKFUNC_FAIL), 2 (CURL_SEEKFUNC_CANTSEEK) or None"); + goto verbose_error; + } + +silent_error: + Py_XDECREF(result); + PYCURL_RELEASE_THREAD(); + return ret; +verbose_error: + PyErr_Print(); + goto silent_error; +} + + + + +PYCURL_INTERNAL size_t +read_callback(char *ptr, size_t size, size_t nmemb, void *stream) +{ + CurlObject *self; + PyObject *arglist; + PyObject *result = NULL; + + size_t ret = CURL_READFUNC_ABORT; /* assume error, this actually works */ + int total_size; + + PYCURL_DECLARE_THREAD_STATE; + + /* acquire thread */ + self = (CurlObject *)stream; + if (!PYCURL_ACQUIRE_THREAD()) + return ret; + + /* check args */ + if (self->r_cb == NULL) + goto silent_error; + if (size <= 0 || nmemb <= 0) + goto done; + total_size = (int)(size * nmemb); + if (total_size < 0 || (size_t)total_size / size != nmemb) { + PyErr_SetString(ErrorObject, "integer overflow in read callback"); + goto verbose_error; + } + + /* run callback */ + arglist = Py_BuildValue("(i)", total_size); + if (arglist == NULL) + goto verbose_error; + result = PyObject_Call(self->r_cb, arglist, NULL); + Py_DECREF(arglist); + if (result == NULL) + goto verbose_error; + + /* handle result */ + if (PyByteStr_Check(result)) { + char *buf = NULL; + Py_ssize_t obj_size = -1; + Py_ssize_t r; + r = PyByteStr_AsStringAndSize(result, &buf, &obj_size); + if (r != 0 || obj_size < 0 || obj_size > total_size) { + PyErr_Format(ErrorObject, "invalid return value for read callback (%ld bytes returned when at most %ld bytes were wanted)", (long)obj_size, (long)total_size); + goto verbose_error; + } + memcpy(ptr, buf, obj_size); + ret = obj_size; /* success */ + } + else if (PyUnicode_Check(result)) { + char *buf = NULL; + Py_ssize_t obj_size = -1; + Py_ssize_t r; + /* + Encode with ascii codec. + + HTTP requires sending content-length for request body to the server + before the request body is sent, therefore typically content length + is given via POSTFIELDSIZE before read function is invoked to + provide the data. + + However, if we encode the string using any encoding other than ascii, + the length of encoded string may not match the length of unicode + string we are encoding. Therefore, if client code does a simple + len(source_string) to determine the value to supply in content-length, + the length of bytes read may be different. + + To avoid this situation, we only accept ascii bytes in the string here. + + Encode data yourself to bytes when dealing with non-ascii data. + */ + PyObject *encoded = PyUnicode_AsEncodedString(result, "ascii", "strict"); + if (encoded == NULL) { + goto verbose_error; + } + r = PyByteStr_AsStringAndSize(encoded, &buf, &obj_size); + if (r != 0 || obj_size < 0 || obj_size > total_size) { + Py_DECREF(encoded); + PyErr_Format(ErrorObject, "invalid return value for read callback (%ld bytes returned after encoding to utf-8 when at most %ld bytes were wanted)", (long)obj_size, (long)total_size); + goto verbose_error; + } + memcpy(ptr, buf, obj_size); + Py_DECREF(encoded); + ret = obj_size; /* success */ + } +#if PY_MAJOR_VERSION < 3 + else if (PyInt_Check(result)) { + long r = PyInt_AsLong(result); + if (r != CURL_READFUNC_ABORT && r != CURL_READFUNC_PAUSE) + goto type_error; + ret = r; /* either CURL_READFUNC_ABORT or CURL_READFUNC_PAUSE */ + } +#endif + else if (PyLong_Check(result)) { + long r = PyLong_AsLong(result); + if (r != CURL_READFUNC_ABORT && r != CURL_READFUNC_PAUSE) + goto type_error; + ret = r; /* either CURL_READFUNC_ABORT or CURL_READFUNC_PAUSE */ + } + else { + type_error: + PyErr_SetString(ErrorObject, "read callback must return a byte string or Unicode string with ASCII code points only"); + goto verbose_error; + } + +done: +silent_error: + Py_XDECREF(result); + PYCURL_RELEASE_THREAD(); + return ret; +verbose_error: + PyErr_Print(); + goto silent_error; +} + + +PYCURL_INTERNAL int +progress_callback(void *stream, + double dltotal, double dlnow, double ultotal, double ulnow) +{ + CurlObject *self; + PyObject *arglist; + PyObject *result = NULL; + int ret = 1; /* assume error */ + PYCURL_DECLARE_THREAD_STATE; + + /* acquire thread */ + self = (CurlObject *)stream; + if (!PYCURL_ACQUIRE_THREAD()) + return ret; + + /* check args */ + if (self->pro_cb == NULL) + goto silent_error; + + /* run callback */ + arglist = Py_BuildValue("(dddd)", dltotal, dlnow, ultotal, ulnow); + if (arglist == NULL) + goto verbose_error; + result = PyObject_Call(self->pro_cb, arglist, NULL); + Py_DECREF(arglist); + if (result == NULL) + goto verbose_error; + + /* handle result */ + if (result == Py_None) { + ret = 0; /* None means success */ + } + else if (PyInt_Check(result)) { + ret = (int) PyInt_AsLong(result); + } + else { + ret = PyObject_IsTrue(result); /* FIXME ??? */ + } + +silent_error: + Py_XDECREF(result); + PYCURL_RELEASE_THREAD(); + return ret; +verbose_error: + PyErr_Print(); + goto silent_error; +} + + +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 32, 0) +PYCURL_INTERNAL int +xferinfo_callback(void *stream, + curl_off_t dltotal, curl_off_t dlnow, + curl_off_t ultotal, curl_off_t ulnow) +{ + CurlObject *self; + PyObject *arglist; + PyObject *result = NULL; + int ret = 1; /* assume error */ + PYCURL_DECLARE_THREAD_STATE; + + /* acquire thread */ + self = (CurlObject *)stream; + if (!PYCURL_ACQUIRE_THREAD()) + return ret; + + /* check args */ + if (self->xferinfo_cb == NULL) + goto silent_error; + + /* run callback */ + arglist = Py_BuildValue("(LLLL)", + (PY_LONG_LONG) dltotal, (PY_LONG_LONG) dlnow, + (PY_LONG_LONG) ultotal, (PY_LONG_LONG) ulnow); + if (arglist == NULL) + goto verbose_error; + result = PyObject_Call(self->xferinfo_cb, arglist, NULL); + Py_DECREF(arglist); + if (result == NULL) + goto verbose_error; + + /* handle result */ + if (result == Py_None) { + ret = 0; /* None means success */ + } + else if (PyInt_Check(result)) { + ret = (int) PyInt_AsLong(result); + } + else { + ret = PyObject_IsTrue(result); /* FIXME ??? */ + } + +silent_error: + Py_XDECREF(result); + PYCURL_RELEASE_THREAD(); + return ret; +verbose_error: + PyErr_Print(); + goto silent_error; +} +#endif + + +PYCURL_INTERNAL int +debug_callback(CURL *curlobj, curl_infotype type, + char *buffer, size_t total_size, void *stream) +{ + CurlObject *self; + PyObject *arglist; + PyObject *result = NULL; + int ret = 0; /* always success */ + PYCURL_DECLARE_THREAD_STATE; + + UNUSED(curlobj); + + /* acquire thread */ + self = (CurlObject *)stream; + if (!PYCURL_ACQUIRE_THREAD()) + return ret; + + /* check args */ + if (self->debug_cb == NULL) + goto silent_error; + if ((int)total_size < 0 || (size_t)((int)total_size) != total_size) { + PyErr_SetString(ErrorObject, "integer overflow in debug callback"); + goto verbose_error; + } + + /* run callback */ +#if PY_MAJOR_VERSION >= 3 + arglist = Py_BuildValue("(iy#)", (int)type, buffer, (int)total_size); +#else + arglist = Py_BuildValue("(is#)", (int)type, buffer, (int)total_size); +#endif + if (arglist == NULL) + goto verbose_error; + result = PyObject_Call(self->debug_cb, arglist, NULL); + Py_DECREF(arglist); + if (result == NULL) + goto verbose_error; + + /* return values from debug callbacks should be ignored */ + +silent_error: + Py_XDECREF(result); + PYCURL_RELEASE_THREAD(); + return ret; +verbose_error: + PyErr_Print(); + goto silent_error; +} + + +PYCURL_INTERNAL curlioerr +ioctl_callback(CURL *curlobj, int cmd, void *stream) +{ + CurlObject *self; + PyObject *arglist; + PyObject *result = NULL; + int ret = CURLIOE_FAILRESTART; /* assume error */ + PYCURL_DECLARE_THREAD_STATE; + + UNUSED(curlobj); + + /* acquire thread */ + self = (CurlObject *)stream; + if (!PYCURL_ACQUIRE_THREAD()) + return (curlioerr) ret; + + /* check args */ + if (self->ioctl_cb == NULL) + goto silent_error; + + /* run callback */ + arglist = Py_BuildValue("(i)", cmd); + if (arglist == NULL) + goto verbose_error; + result = PyObject_Call(self->ioctl_cb, arglist, NULL); + Py_DECREF(arglist); + if (result == NULL) + goto verbose_error; + + /* handle result */ + if (result == Py_None) { + ret = CURLIOE_OK; /* None means success */ + } + else if (PyInt_Check(result)) { + ret = (int) PyInt_AsLong(result); + if (ret >= CURLIOE_LAST || ret < 0) { + PyErr_SetString(ErrorObject, "ioctl callback returned invalid value"); + goto verbose_error; + } + } + +silent_error: + Py_XDECREF(result); + PYCURL_RELEASE_THREAD(); + return (curlioerr) ret; +verbose_error: + PyErr_Print(); + goto silent_error; +} + + +#if defined(HAVE_CURL_OPENSSL) +/* internal helper that load certificates from buffer, returns -1 on error */ +static int +add_ca_certs(SSL_CTX *context, void *data, Py_ssize_t len) +{ + // this code was copied from _ssl module + BIO *biobuf = NULL; + X509_STORE *store; + int retval = 0, err, loaded = 0; + + if (len <= 0) { + PyErr_SetString(PyExc_ValueError, + "Empty certificate data"); + return -1; + } else if (len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "Certificate data is too long."); + return -1; + } + + biobuf = BIO_new_mem_buf(data, (int)len); + if (biobuf == NULL) { + PyErr_SetString(PyExc_MemoryError, "Can't allocate buffer"); + ERR_clear_error(); + return -1; + } + + store = SSL_CTX_get_cert_store(context); + assert(store != NULL); + + while (1) { + X509 *cert = NULL; + int r; + + cert = PEM_read_bio_X509(biobuf, NULL, 0, NULL); + if (cert == NULL) { + break; + } + r = X509_STORE_add_cert(store, cert); + X509_free(cert); + if (!r) { + err = ERR_peek_last_error(); + if ((ERR_GET_LIB(err) == ERR_LIB_X509) && + (ERR_GET_REASON(err) == X509_R_CERT_ALREADY_IN_HASH_TABLE)) { + /* cert already in hash table, not an error */ + ERR_clear_error(); + } else { + break; + } + } + loaded++; + } + + err = ERR_peek_last_error(); + if ((loaded > 0) && + (ERR_GET_LIB(err) == ERR_LIB_PEM) && + (ERR_GET_REASON(err) == PEM_R_NO_START_LINE)) { + /* EOF PEM file, not an error */ + ERR_clear_error(); + retval = 0; + } else { + PyErr_SetString(ErrorObject, ERR_reason_error_string(err)); + ERR_clear_error(); + retval = -1; + } + + BIO_free(biobuf); + return retval; +} + + +PYCURL_INTERNAL CURLcode +ssl_ctx_callback(CURL *curl, void *ssl_ctx, void *ptr) +{ + CurlObject *self; + PYCURL_DECLARE_THREAD_STATE; + int r; + + UNUSED(curl); + + /* acquire thread */ + self = (CurlObject *)ptr; + if (!PYCURL_ACQUIRE_THREAD()) + return CURLE_FAILED_INIT; + + r = add_ca_certs((SSL_CTX*)ssl_ctx, + PyBytes_AS_STRING(self->ca_certs_obj), + PyBytes_GET_SIZE(self->ca_certs_obj)); + + if (r != 0) + PyErr_Print(); + + PYCURL_RELEASE_THREAD(); + return r == 0 ? CURLE_OK : CURLE_FAILED_INIT; +} +#endif diff --git a/src/easyinfo.c b/src/easyinfo.c new file mode 100644 index 0000000..1666814 --- /dev/null +++ b/src/easyinfo.c @@ -0,0 +1,381 @@ +#include "pycurl.h" + + +/* Convert a curl slist (a list of strings) to a Python list. + * In case of error return NULL with an exception set. + */ +static PyObject *convert_slist(struct curl_slist *slist, int free_flags) +{ + PyObject *ret = NULL; + struct curl_slist *slist_start = slist; + + ret = PyList_New((Py_ssize_t)0); + if (ret == NULL) goto error; + + for ( ; slist != NULL; slist = slist->next) { + PyObject *v = NULL; + + if (slist->data == NULL) { + v = Py_None; Py_INCREF(v); + } else { + v = PyByteStr_FromString(slist->data); + } + if (v == NULL || PyList_Append(ret, v) != 0) { + Py_XDECREF(v); + goto error; + } + Py_DECREF(v); + } + + if ((free_flags & 1) && slist_start) + curl_slist_free_all(slist_start); + return ret; + +error: + Py_XDECREF(ret); + if ((free_flags & 2) && slist_start) + curl_slist_free_all(slist_start); + return NULL; +} + + +#ifdef HAVE_CURLOPT_CERTINFO +/* Convert a struct curl_certinfo into a Python data structure. + * In case of error return NULL with an exception set. + */ +static PyObject *convert_certinfo(struct curl_certinfo *cinfo, int decode) +{ + PyObject *certs; + int cert_index; + + if (!cinfo) + Py_RETURN_NONE; + + certs = PyList_New((Py_ssize_t)(cinfo->num_of_certs)); + if (!certs) + return NULL; + + for (cert_index = 0; cert_index < cinfo->num_of_certs; cert_index ++) { + struct curl_slist *fields = cinfo->certinfo[cert_index]; + struct curl_slist *field_cursor; + int field_count, field_index; + PyObject *cert; + + field_count = 0; + field_cursor = fields; + while (field_cursor != NULL) { + field_cursor = field_cursor->next; + field_count ++; + } + + + cert = PyTuple_New((Py_ssize_t)field_count); + if (!cert) + goto error; + PyList_SetItem(certs, cert_index, cert); /* Eats the ref from New() */ + + for(field_index = 0, field_cursor = fields; + field_cursor != NULL; + field_index ++, field_cursor = field_cursor->next) { + const char *field = field_cursor->data; + PyObject *field_tuple; + + if (!field) { + field_tuple = Py_None; Py_INCREF(field_tuple); + } else { + const char *sep = strchr(field, ':'); + if (!sep) { + if (decode) { + field_tuple = PyText_FromString(field); + } else { + field_tuple = PyByteStr_FromString(field); + } + } else { + /* XXX check */ + if (decode) { + field_tuple = Py_BuildValue("s#s", field, (int)(sep - field), sep+1); + } else { +#if PY_MAJOR_VERSION >= 3 + field_tuple = Py_BuildValue("y#y", field, (int)(sep - field), sep+1); +#else + field_tuple = Py_BuildValue("s#s", field, (int)(sep - field), sep+1); +#endif + } + } + if (!field_tuple) + goto error; + } + PyTuple_SET_ITEM(cert, field_index, field_tuple); /* Eats the ref */ + } + } + + return certs; + + error: + Py_DECREF(certs); + return NULL; +} +#endif + +PYCURL_INTERNAL PyObject * +do_curl_getinfo_raw(CurlObject *self, PyObject *args) +{ + int option; + int res; + + if (!PyArg_ParseTuple(args, "i:getinfo_raw", &option)) { + return NULL; + } + if (check_curl_state(self, 1 | 2, "getinfo") != 0) { + return NULL; + } + + switch (option) { + case CURLINFO_FILETIME: + case CURLINFO_HEADER_SIZE: + case CURLINFO_RESPONSE_CODE: + case CURLINFO_REDIRECT_COUNT: + case CURLINFO_REQUEST_SIZE: + case CURLINFO_SSL_VERIFYRESULT: + case CURLINFO_HTTP_CONNECTCODE: + case CURLINFO_HTTPAUTH_AVAIL: + case CURLINFO_PROXYAUTH_AVAIL: + case CURLINFO_OS_ERRNO: + case CURLINFO_NUM_CONNECTS: + case CURLINFO_LASTSOCKET: +#ifdef HAVE_CURLINFO_LOCAL_PORT + case CURLINFO_LOCAL_PORT: +#endif +#ifdef HAVE_CURLINFO_PRIMARY_PORT + case CURLINFO_PRIMARY_PORT: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 20, 0) + case CURLINFO_RTSP_CLIENT_CSEQ: + case CURLINFO_RTSP_SERVER_CSEQ: + case CURLINFO_RTSP_CSEQ_RECV: +#endif +#ifdef HAVE_CURLINFO_HTTP_VERSION + case CURLINFO_HTTP_VERSION: +#endif +#ifdef HAVE_CURL_7_19_4_OPTS + case CURLINFO_CONDITION_UNMET: +#endif + { + /* Return PyInt as result */ + long l_res = -1; + + res = curl_easy_getinfo(self->handle, (CURLINFO)option, &l_res); + /* Check for errors and return result */ + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } + return PyInt_FromLong(l_res); + } + + case CURLINFO_CONTENT_TYPE: + case CURLINFO_EFFECTIVE_URL: + case CURLINFO_FTP_ENTRY_PATH: + case CURLINFO_REDIRECT_URL: + case CURLINFO_PRIMARY_IP: +#ifdef HAVE_CURLINFO_LOCAL_IP + case CURLINFO_LOCAL_IP: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 20, 0) + case CURLINFO_RTSP_SESSION_ID: +#endif + { + /* Return PyString as result */ + char *s_res = NULL; + + res = curl_easy_getinfo(self->handle, (CURLINFO)option, &s_res); + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } + /* If the resulting string is NULL, return None */ + if (s_res == NULL) { + Py_RETURN_NONE; + } + return PyByteStr_FromString(s_res); + + } + + case CURLINFO_CONNECT_TIME: + case CURLINFO_APPCONNECT_TIME: + case CURLINFO_CONTENT_LENGTH_DOWNLOAD: + case CURLINFO_CONTENT_LENGTH_UPLOAD: + case CURLINFO_NAMELOOKUP_TIME: + case CURLINFO_PRETRANSFER_TIME: + case CURLINFO_REDIRECT_TIME: + case CURLINFO_SIZE_DOWNLOAD: + case CURLINFO_SIZE_UPLOAD: + case CURLINFO_SPEED_DOWNLOAD: + case CURLINFO_SPEED_UPLOAD: + case CURLINFO_STARTTRANSFER_TIME: + case CURLINFO_TOTAL_TIME: + { + /* Return PyFloat as result */ + double d_res = 0.0; + + res = curl_easy_getinfo(self->handle, (CURLINFO)option, &d_res); + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } + return PyFloat_FromDouble(d_res); + } + + case CURLINFO_SSL_ENGINES: + case CURLINFO_COOKIELIST: + { + /* Return a list of strings */ + struct curl_slist *slist = NULL; + + res = curl_easy_getinfo(self->handle, (CURLINFO)option, &slist); + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } + return convert_slist(slist, 1 | 2); + } + +#ifdef HAVE_CURLOPT_CERTINFO + case CURLINFO_CERTINFO: + { + /* Return a list of lists of 2-tuples */ + struct curl_certinfo *clist = NULL; + res = curl_easy_getinfo(self->handle, CURLINFO_CERTINFO, &clist); + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } else { + return convert_certinfo(clist, 0); + } + } +#endif + } + + /* Got wrong option on the method call */ + PyErr_SetString(PyExc_ValueError, "invalid argument to getinfo"); + return NULL; +} + + +#if PY_MAJOR_VERSION >= 3 +static PyObject * +decode_string_list(PyObject *list) +{ + PyObject *decoded_list = NULL; + Py_ssize_t size = PyList_Size(list); + int i; + + decoded_list = PyList_New(size); + if (decoded_list == NULL) { + return NULL; + } + + for (i = 0; i < size; ++i) { + PyObject *decoded_item = PyUnicode_FromEncodedObject( + PyList_GET_ITEM(list, i), + NULL, + NULL); + + if (decoded_item == NULL) { + goto err; + } + PyList_SetItem(decoded_list, i, decoded_item); + } + + return decoded_list; + +err: + Py_DECREF(decoded_list); + return NULL; +} + +PYCURL_INTERNAL PyObject * +do_curl_getinfo(CurlObject *self, PyObject *args) +{ + int option, res; + PyObject *rv; + + if (!PyArg_ParseTuple(args, "i:getinfo", &option)) { + return NULL; + } + +#ifdef HAVE_CURLOPT_CERTINFO + if (option == CURLINFO_CERTINFO) { + /* Return a list of lists of 2-tuples */ + struct curl_certinfo *clist = NULL; + res = curl_easy_getinfo(self->handle, CURLINFO_CERTINFO, &clist); + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } else { + return convert_certinfo(clist, 1); + } + } +#endif + + rv = do_curl_getinfo_raw(self, args); + if (rv == NULL) { + return rv; + } + + switch (option) { + case CURLINFO_CONTENT_TYPE: + case CURLINFO_EFFECTIVE_URL: + case CURLINFO_FTP_ENTRY_PATH: + case CURLINFO_REDIRECT_URL: + case CURLINFO_PRIMARY_IP: +#ifdef HAVE_CURLINFO_LOCAL_IP + case CURLINFO_LOCAL_IP: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 20, 0) + case CURLINFO_RTSP_SESSION_ID: +#endif + if (rv != Py_None) { + PyObject *decoded; + + // Decode bytes into a Unicode string using default encoding + decoded = PyUnicode_FromEncodedObject(rv, NULL, NULL); + // success and failure paths both need to free bytes object + Py_DECREF(rv); + return decoded; + } + return rv; + + case CURLINFO_SSL_ENGINES: + case CURLINFO_COOKIELIST: + { + PyObject *decoded = decode_string_list(rv); + Py_DECREF(rv); + return decoded; + } + + default: + return rv; + } +} +#endif + + +PYCURL_INTERNAL PyObject * +do_curl_errstr(CurlObject *self) +{ + if (check_curl_state(self, 1 | 2, "errstr") != 0) { + return NULL; + } + self->error[sizeof(self->error) - 1] = 0; + + return PyText_FromString(self->error); +} + + +#if PY_MAJOR_VERSION >= 3 +PYCURL_INTERNAL PyObject * +do_curl_errstr_raw(CurlObject *self) +{ + if (check_curl_state(self, 1 | 2, "errstr") != 0) { + return NULL; + } + self->error[sizeof(self->error) - 1] = 0; + + return PyByteStr_FromString(self->error); +} +#endif diff --git a/src/easyopt.c b/src/easyopt.c new file mode 100644 index 0000000..faa0649 --- /dev/null +++ b/src/easyopt.c @@ -0,0 +1,1188 @@ +#include "pycurl.h" + + +static struct curl_slist * +pycurl_list_or_tuple_to_slist(int which, PyObject *obj, Py_ssize_t len) +{ + struct curl_slist *slist = NULL; + Py_ssize_t i; + + for (i = 0; i < len; i++) { + PyObject *listitem = PyListOrTuple_GetItem(obj, i, which); + struct curl_slist *nlist; + char *str; + PyObject *sencoded_obj; + + if (!PyText_Check(listitem)) { + curl_slist_free_all(slist); + PyErr_SetString(PyExc_TypeError, "list items must be byte strings or Unicode strings with ASCII code points only"); + return NULL; + } + /* INFO: curl_slist_append() internally does strdup() the data, so + * no embedded NUL characters allowed here. */ + str = PyText_AsString_NoNUL(listitem, &sencoded_obj); + if (str == NULL) { + curl_slist_free_all(slist); + return NULL; + } + nlist = curl_slist_append(slist, str); + PyText_EncodedDecref(sencoded_obj); + if (nlist == NULL || nlist->data == NULL) { + curl_slist_free_all(slist); + PyErr_NoMemory(); + return NULL; + } + slist = nlist; + } + return slist; +} + + +static PyObject * +util_curl_unsetopt(CurlObject *self, int option) +{ + int res; + +#define SETOPT2(o,x) \ + if ((res = curl_easy_setopt(self->handle, (o), (x))) != CURLE_OK) goto error +#define SETOPT(x) SETOPT2((CURLoption)option, (x)) +#define CLEAR_OBJECT(object_option, object_field) \ + case object_option: \ + if ((res = curl_easy_setopt(self->handle, object_option, NULL)) != CURLE_OK) \ + goto error; \ + Py_CLEAR(object_field); \ + break +#define CLEAR_CALLBACK(callback_option, data_option, callback_field) \ + case callback_option: \ + if ((res = curl_easy_setopt(self->handle, callback_option, NULL)) != CURLE_OK) \ + goto error; \ + if ((res = curl_easy_setopt(self->handle, data_option, NULL)) != CURLE_OK) \ + goto error; \ + Py_CLEAR(callback_field); \ + break + + /* FIXME: implement more options. Have to carefully check lib/url.c in the + * libcurl source code to see if it's actually safe to simply + * unset the option. */ + switch (option) + { + case CURLOPT_SHARE: + SETOPT((CURLSH *) NULL); + Py_XDECREF(self->share); + self->share = NULL; + break; + case CURLOPT_INFILESIZE: + SETOPT((long) -1); + break; + case CURLOPT_WRITEHEADER: + SETOPT((void *) 0); + Py_CLEAR(self->writeheader_fp); + break; + case CURLOPT_CAINFO: + case CURLOPT_CAPATH: + case CURLOPT_COOKIE: + case CURLOPT_COOKIEJAR: + case CURLOPT_CUSTOMREQUEST: + case CURLOPT_EGDSOCKET: + case CURLOPT_ENCODING: + case CURLOPT_FTPPORT: + case CURLOPT_PROXYUSERPWD: +#ifdef HAVE_CURLOPT_PROXYUSERNAME + case CURLOPT_PROXYUSERNAME: + case CURLOPT_PROXYPASSWORD: +#endif + case CURLOPT_RANDOM_FILE: + case CURLOPT_SSL_CIPHER_LIST: + case CURLOPT_USERPWD: +#ifdef HAVE_CURLOPT_USERNAME + case CURLOPT_USERNAME: + case CURLOPT_PASSWORD: +#endif + case CURLOPT_RANGE: +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 10, 0) + /* added by 0ff89b9c3c02a911e1e5ea9a4182c373a6e0f1c7 */ + case CURLOPT_PROXY: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 43, 0) + case CURLOPT_SERVICE_NAME: + case CURLOPT_PROXY_SERVICE_NAME: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 52, 0) + case CURLOPT_PROXY_CAPATH: + case CURLOPT_PROXY_CAINFO: + case CURLOPT_PRE_PROXY: + case CURLOPT_PROXY_SSLCERT: + case CURLOPT_PROXY_SSLCERTTYPE: + case CURLOPT_PROXY_SSLKEY: + case CURLOPT_PROXY_SSLKEYTYPE: +#endif + SETOPT((char *) NULL); + break; + +#ifdef HAVE_CURLOPT_CERTINFO + case CURLOPT_CERTINFO: + SETOPT((long) 0); + break; +#endif + + CLEAR_OBJECT(CURLOPT_HTTPHEADER, self->httpheader); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 37, 0) + CLEAR_OBJECT(CURLOPT_PROXYHEADER, self->proxyheader); +#endif + CLEAR_OBJECT(CURLOPT_HTTP200ALIASES, self->http200aliases); + CLEAR_OBJECT(CURLOPT_QUOTE, self->quote); + CLEAR_OBJECT(CURLOPT_POSTQUOTE, self->postquote); + CLEAR_OBJECT(CURLOPT_PREQUOTE, self->prequote); + CLEAR_OBJECT(CURLOPT_TELNETOPTIONS, self->telnetoptions); +#ifdef HAVE_CURLOPT_RESOLVE + CLEAR_OBJECT(CURLOPT_RESOLVE, self->resolve); +#endif +#ifdef HAVE_CURL_7_20_0_OPTS + CLEAR_OBJECT(CURLOPT_MAIL_RCPT, self->mail_rcpt); +#endif +#ifdef HAVE_CURLOPT_CONNECT_TO + CLEAR_OBJECT(CURLOPT_CONNECT_TO, self->connect_to); +#endif + /* FIXME: what about data->set.httpreq ?? */ + CLEAR_OBJECT(CURLOPT_HTTPPOST, self->httppost); + + CLEAR_CALLBACK(CURLOPT_OPENSOCKETFUNCTION, CURLOPT_OPENSOCKETDATA, self->opensocket_cb); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 7) + CLEAR_CALLBACK(CURLOPT_CLOSESOCKETFUNCTION, CURLOPT_CLOSESOCKETDATA, self->closesocket_cb); +#endif + CLEAR_CALLBACK(CURLOPT_SOCKOPTFUNCTION, CURLOPT_SOCKOPTDATA, self->sockopt_cb); +#ifdef HAVE_CURL_7_19_6_OPTS + CLEAR_CALLBACK(CURLOPT_SSH_KEYFUNCTION, CURLOPT_SSH_KEYDATA, self->ssh_key_cb); +#endif + + /* info: we explicitly list unsupported options here */ + case CURLOPT_COOKIEFILE: + default: + PyErr_SetString(PyExc_TypeError, "unsetopt() is not supported for this option"); + return NULL; + } + + Py_RETURN_NONE; + +error: + CURLERROR_RETVAL(); + +#undef SETOPT +#undef SETOPT2 +#undef CLEAR_CALLBACK +} + + +PYCURL_INTERNAL PyObject * +do_curl_unsetopt(CurlObject *self, PyObject *args) +{ + int option; + + if (!PyArg_ParseTuple(args, "i:unsetopt", &option)) { + return NULL; + } + if (check_curl_state(self, 1 | 2, "unsetopt") != 0) { + return NULL; + } + + /* early checks of option value */ + if (option <= 0) + goto error; + if (option >= (int)CURLOPTTYPE_OFF_T + OPTIONS_SIZE) + goto error; + if (option % 10000 >= OPTIONS_SIZE) + goto error; + + return util_curl_unsetopt(self, option); + +error: + PyErr_SetString(PyExc_TypeError, "invalid arguments to unsetopt"); + return NULL; +} + + +static PyObject * +do_curl_setopt_string_impl(CurlObject *self, int option, PyObject *obj) +{ + char *str = NULL; + Py_ssize_t len = -1; + PyObject *encoded_obj; + int res; + + /* Check that the option specified a string as well as the input */ + switch (option) { + case CURLOPT_CAINFO: + case CURLOPT_CAPATH: + case CURLOPT_COOKIE: + case CURLOPT_COOKIEFILE: + case CURLOPT_COOKIELIST: + case CURLOPT_COOKIEJAR: + case CURLOPT_CUSTOMREQUEST: + case CURLOPT_EGDSOCKET: + /* use CURLOPT_ENCODING instead of CURLOPT_ACCEPT_ENCODING + for compatibility with older libcurls */ + case CURLOPT_ENCODING: + case CURLOPT_FTPPORT: + case CURLOPT_INTERFACE: + case CURLOPT_KEYPASSWD: + case CURLOPT_NETRC_FILE: + case CURLOPT_PROXY: + case CURLOPT_PROXYUSERPWD: +#ifdef HAVE_CURLOPT_PROXYUSERNAME + case CURLOPT_PROXYUSERNAME: + case CURLOPT_PROXYPASSWORD: +#endif + case CURLOPT_RANDOM_FILE: + case CURLOPT_RANGE: + case CURLOPT_REFERER: + case CURLOPT_SSLCERT: + case CURLOPT_SSLCERTTYPE: + case CURLOPT_SSLENGINE: + case CURLOPT_SSLKEY: + case CURLOPT_SSLKEYTYPE: + case CURLOPT_SSL_CIPHER_LIST: + case CURLOPT_URL: + case CURLOPT_USERAGENT: + case CURLOPT_USERPWD: +#ifdef HAVE_CURLOPT_USERNAME + case CURLOPT_USERNAME: + case CURLOPT_PASSWORD: +#endif + case CURLOPT_FTP_ALTERNATIVE_TO_USER: + case CURLOPT_SSH_PUBLIC_KEYFILE: + case CURLOPT_SSH_PRIVATE_KEYFILE: + case CURLOPT_COPYPOSTFIELDS: + case CURLOPT_SSH_HOST_PUBLIC_KEY_MD5: + case CURLOPT_CRLFILE: + case CURLOPT_ISSUERCERT: +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 20, 0) + case CURLOPT_RTSP_STREAM_URI: + case CURLOPT_RTSP_SESSION_ID: + case CURLOPT_RTSP_TRANSPORT: +#endif +#ifdef HAVE_CURLOPT_DNS_SERVERS + case CURLOPT_DNS_SERVERS: +#endif +#ifdef HAVE_CURLOPT_NOPROXY + case CURLOPT_NOPROXY: +#endif +#ifdef HAVE_CURL_7_19_4_OPTS + case CURLOPT_SOCKS5_GSSAPI_SERVICE: +#endif +#ifdef HAVE_CURL_7_19_6_OPTS + case CURLOPT_SSH_KNOWNHOSTS: +#endif +#ifdef HAVE_CURL_7_20_0_OPTS + case CURLOPT_MAIL_FROM: +#endif +#ifdef HAVE_CURL_7_25_0_OPTS + case CURLOPT_MAIL_AUTH: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 39, 0) + case CURLOPT_PINNEDPUBLICKEY: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 43, 0) + case CURLOPT_SERVICE_NAME: + case CURLOPT_PROXY_SERVICE_NAME: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 0) + case CURLOPT_WILDCARDMATCH: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 40, 0) + case CURLOPT_UNIX_SOCKET_PATH: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 4) + case CURLOPT_TLSAUTH_TYPE: + case CURLOPT_TLSAUTH_USERNAME: + case CURLOPT_TLSAUTH_PASSWORD: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 45, 0) + case CURLOPT_DEFAULT_PROTOCOL: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 34, 0) + case CURLOPT_LOGIN_OPTIONS: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 33, 0) + case CURLOPT_XOAUTH2_BEARER: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 52, 0) + case CURLOPT_PROXY_CAPATH: + case CURLOPT_PROXY_CAINFO: + case CURLOPT_PRE_PROXY: + case CURLOPT_PROXY_SSLCERT: + case CURLOPT_PROXY_SSLCERTTYPE: + case CURLOPT_PROXY_SSLKEY: + case CURLOPT_PROXY_SSLKEYTYPE: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 61, 0) + case CURLOPT_TLS13_CIPHERS: + case CURLOPT_PROXY_TLS13_CIPHERS: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 62, 0) + case CURLOPT_DOH_URL: +#endif + case CURLOPT_KRBLEVEL: + str = PyText_AsString_NoNUL(obj, &encoded_obj); + if (str == NULL) + return NULL; + break; + case CURLOPT_POSTFIELDS: + if (PyText_AsStringAndSize(obj, &str, &len, &encoded_obj) != 0) + return NULL; + /* automatically set POSTFIELDSIZE */ + if (len <= INT_MAX) { + res = curl_easy_setopt(self->handle, CURLOPT_POSTFIELDSIZE, (long)len); + } else { + res = curl_easy_setopt(self->handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)len); + } + if (res != CURLE_OK) { + PyText_EncodedDecref(encoded_obj); + CURLERROR_RETVAL(); + } + break; + default: + PyErr_SetString(PyExc_TypeError, "strings are not supported for this option"); + return NULL; + } + assert(str != NULL); + /* Call setopt */ + res = curl_easy_setopt(self->handle, (CURLoption)option, str); + /* Check for errors */ + if (res != CURLE_OK) { + PyText_EncodedDecref(encoded_obj); + CURLERROR_RETVAL(); + } + /* libcurl does not copy the value of CURLOPT_POSTFIELDS */ + if (option == CURLOPT_POSTFIELDS) { + PyObject *store_obj; + + /* if obj was bytes, it was not encoded, and we need to incref obj. + * if obj was unicode, it was encoded, and we need to incref + * encoded_obj - except we can simply transfer ownership. + */ + if (encoded_obj) { + store_obj = encoded_obj; + } else { + /* no encoding is performed, incref the original object. */ + store_obj = obj; + Py_INCREF(store_obj); + } + + util_curl_xdecref(self, PYCURL_MEMGROUP_POSTFIELDS, self->handle); + self->postfields_obj = store_obj; + } else { + PyText_EncodedDecref(encoded_obj); + } + Py_RETURN_NONE; +} + + +#define IS_LONG_OPTION(o) (o < CURLOPTTYPE_OBJECTPOINT) +#define IS_OFF_T_OPTION(o) (o >= CURLOPTTYPE_OFF_T) + + +static PyObject * +do_curl_setopt_int(CurlObject *self, int option, PyObject *obj) +{ + long d; + PY_LONG_LONG ld; + int res; + + if (IS_LONG_OPTION(option)) { + d = PyInt_AsLong(obj); + res = curl_easy_setopt(self->handle, (CURLoption)option, (long)d); + } else if (IS_OFF_T_OPTION(option)) { + /* this path should only be taken in Python 3 */ + ld = PyLong_AsLongLong(obj); + res = curl_easy_setopt(self->handle, (CURLoption)option, (curl_off_t)ld); + } else { + PyErr_SetString(PyExc_TypeError, "integers are not supported for this option"); + return NULL; + } + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } + Py_RETURN_NONE; +} + + +static PyObject * +do_curl_setopt_long(CurlObject *self, int option, PyObject *obj) +{ + int res; + PY_LONG_LONG d = PyLong_AsLongLong(obj); + if (d == -1 && PyErr_Occurred()) + return NULL; + + if (IS_LONG_OPTION(option) && (long)d == d) + res = curl_easy_setopt(self->handle, (CURLoption)option, (long)d); + else if (IS_OFF_T_OPTION(option) && (curl_off_t)d == d) + res = curl_easy_setopt(self->handle, (CURLoption)option, (curl_off_t)d); + else { + PyErr_SetString(PyExc_TypeError, "longs are not supported for this option"); + return NULL; + } + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } + Py_RETURN_NONE; +} + + +#undef IS_LONG_OPTION +#undef IS_OFF_T_OPTION + + +#if PY_MAJOR_VERSION < 3 && !defined(PYCURL_AVOID_STDIO) +static PyObject * +do_curl_setopt_file_passthrough(CurlObject *self, int option, PyObject *obj) +{ + FILE *fp; + int res; + + fp = PyFile_AsFile(obj); + if (fp == NULL) { + PyErr_SetString(PyExc_TypeError, "second argument must be open file"); + return NULL; + } + + switch (option) { + case CURLOPT_READDATA: + res = curl_easy_setopt(self->handle, CURLOPT_READFUNCTION, fread); + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } + break; + case CURLOPT_WRITEDATA: + res = curl_easy_setopt(self->handle, CURLOPT_WRITEFUNCTION, fwrite); + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } + break; + case CURLOPT_WRITEHEADER: + res = curl_easy_setopt(self->handle, CURLOPT_HEADERFUNCTION, fwrite); + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } + break; + default: + PyErr_SetString(PyExc_TypeError, "files are not supported for this option"); + return NULL; + } + + res = curl_easy_setopt(self->handle, (CURLoption)option, fp); + if (res != CURLE_OK) { + /* + If we get here fread/fwrite are set as callbacks but the file pointer + is not set, program will crash if it does not reset read/write + callback. Also, we won't do the memory management later in this + function. + */ + CURLERROR_RETVAL(); + } + Py_INCREF(obj); + + switch (option) { + case CURLOPT_READDATA: + Py_CLEAR(self->readdata_fp); + self->readdata_fp = obj; + break; + case CURLOPT_WRITEDATA: + Py_CLEAR(self->writedata_fp); + self->writedata_fp = obj; + break; + case CURLOPT_WRITEHEADER: + Py_CLEAR(self->writeheader_fp); + self->writeheader_fp = obj; + break; + default: + assert(0); + break; + } + /* Return success */ + Py_RETURN_NONE; +} +#endif + + +static PyObject * +do_curl_setopt_httppost(CurlObject *self, int option, int which, PyObject *obj) +{ + struct curl_httppost *post = NULL; + struct curl_httppost *last = NULL; + /* List of all references that have been INCed as a result of + * this operation */ + PyObject *ref_params = NULL; + PyObject *nencoded_obj, *cencoded_obj, *oencoded_obj; + int which_httppost_item, which_httppost_option; + PyObject *httppost_option; + Py_ssize_t i, len; + int res; + + len = PyListOrTuple_Size(obj, which); + if (len == 0) + Py_RETURN_NONE; + + for (i = 0; i < len; i++) { + char *nstr = NULL, *cstr = NULL; + Py_ssize_t nlen = -1, clen = -1; + PyObject *listitem = PyListOrTuple_GetItem(obj, i, which); + + which_httppost_item = PyListOrTuple_Check(listitem); + if (!which_httppost_item) { + PyErr_SetString(PyExc_TypeError, "list items must be list or tuple objects"); + goto error; + } + if (PyListOrTuple_Size(listitem, which_httppost_item) != 2) { + PyErr_SetString(PyExc_TypeError, "list or tuple must contain two elements (name, value)"); + goto error; + } + if (PyText_AsStringAndSize(PyListOrTuple_GetItem(listitem, 0, which_httppost_item), + &nstr, &nlen, &nencoded_obj) != 0) { + PyErr_SetString(PyExc_TypeError, "list or tuple must contain a byte string or Unicode string with ASCII code points only as first element"); + goto error; + } + httppost_option = PyListOrTuple_GetItem(listitem, 1, which_httppost_item); + if (PyText_Check(httppost_option)) { + /* Handle strings as second argument for backwards compatibility */ + + if (PyText_AsStringAndSize(httppost_option, &cstr, &clen, &cencoded_obj)) { + PyText_EncodedDecref(nencoded_obj); + create_and_set_error_object(self, CURLE_BAD_FUNCTION_ARGUMENT); + goto error; + } + /* INFO: curl_formadd() internally does memdup() the data, so + * embedded NUL characters _are_ allowed here. */ + res = curl_formadd(&post, &last, + CURLFORM_COPYNAME, nstr, + CURLFORM_NAMELENGTH, (long) nlen, + CURLFORM_COPYCONTENTS, cstr, + CURLFORM_CONTENTSLENGTH, (long) clen, + CURLFORM_END); + PyText_EncodedDecref(cencoded_obj); + if (res != CURLE_OK) { + PyText_EncodedDecref(nencoded_obj); + CURLERROR_SET_RETVAL(); + goto error; + } + } + /* assignment is intended */ + else if ((which_httppost_option = PyListOrTuple_Check(httppost_option))) { + /* Supports content, file and content-type */ + Py_ssize_t tlen = PyListOrTuple_Size(httppost_option, which_httppost_option); + int j, k, l; + struct curl_forms *forms = NULL; + + /* Sanity check that there are at least two tuple items */ + if (tlen < 2) { + PyText_EncodedDecref(nencoded_obj); + PyErr_SetString(PyExc_TypeError, "list or tuple must contain at least one option and one value"); + goto error; + } + + if (tlen % 2 == 1) { + PyText_EncodedDecref(nencoded_obj); + PyErr_SetString(PyExc_TypeError, "list or tuple must contain an even number of items"); + goto error; + } + + /* Allocate enough space to accommodate length options for content or buffers, plus a terminator. */ + forms = PyMem_New(struct curl_forms, (tlen*2) + 1); + if (forms == NULL) { + PyText_EncodedDecref(nencoded_obj); + PyErr_NoMemory(); + goto error; + } + + /* Iterate all the tuple members pairwise */ + for (j = 0, k = 0, l = 0; j < tlen; j += 2, l++) { + char *ostr; + Py_ssize_t olen; + int val; + + if (j == (tlen-1)) { + PyErr_SetString(PyExc_TypeError, "expected value"); + PyMem_Free(forms); + PyText_EncodedDecref(nencoded_obj); + goto error; + } + if (!PyInt_Check(PyListOrTuple_GetItem(httppost_option, j, which_httppost_option))) { + PyErr_SetString(PyExc_TypeError, "option must be an integer"); + PyMem_Free(forms); + PyText_EncodedDecref(nencoded_obj); + goto error; + } + if (!PyText_Check(PyListOrTuple_GetItem(httppost_option, j+1, which_httppost_option))) { + PyErr_SetString(PyExc_TypeError, "value must be a byte string or a Unicode string with ASCII code points only"); + PyMem_Free(forms); + PyText_EncodedDecref(nencoded_obj); + goto error; + } + + val = PyLong_AsLong(PyListOrTuple_GetItem(httppost_option, j, which_httppost_option)); + if (val != CURLFORM_COPYCONTENTS && + val != CURLFORM_FILE && + val != CURLFORM_FILENAME && + val != CURLFORM_CONTENTTYPE && + val != CURLFORM_BUFFER && + val != CURLFORM_BUFFERPTR) + { + PyErr_SetString(PyExc_TypeError, "unsupported option"); + PyMem_Free(forms); + PyText_EncodedDecref(nencoded_obj); + goto error; + } + + if (PyText_AsStringAndSize(PyListOrTuple_GetItem(httppost_option, j+1, which_httppost_option), &ostr, &olen, &oencoded_obj)) { + /* exception should be already set */ + PyMem_Free(forms); + PyText_EncodedDecref(nencoded_obj); + goto error; + } + forms[k].option = val; + forms[k].value = ostr; + ++k; + + if (val == CURLFORM_COPYCONTENTS) { + /* Contents can contain \0 bytes so we specify the length */ + forms[k].option = CURLFORM_CONTENTSLENGTH; + forms[k].value = (const char *)olen; + ++k; + } else if (val == CURLFORM_BUFFERPTR) { + PyObject *obj = NULL; + + if (ref_params == NULL) { + ref_params = PyList_New((Py_ssize_t)0); + if (ref_params == NULL) { + PyText_EncodedDecref(oencoded_obj); + PyMem_Free(forms); + PyText_EncodedDecref(nencoded_obj); + goto error; + } + } + + /* Keep a reference to the object that holds the ostr buffer. */ + if (oencoded_obj == NULL) { + obj = PyListOrTuple_GetItem(httppost_option, j+1, which_httppost_option); + } + else { + obj = oencoded_obj; + } + + /* Ensure that the buffer remains alive until curl_easy_cleanup() */ + if (PyList_Append(ref_params, obj) != 0) { + PyText_EncodedDecref(oencoded_obj); + PyMem_Free(forms); + PyText_EncodedDecref(nencoded_obj); + goto error; + } + + /* As with CURLFORM_COPYCONTENTS, specify the length. */ + forms[k].option = CURLFORM_BUFFERLENGTH; + forms[k].value = (const char *)olen; + ++k; + } + } + forms[k].option = CURLFORM_END; + res = curl_formadd(&post, &last, + CURLFORM_COPYNAME, nstr, + CURLFORM_NAMELENGTH, (long) nlen, + CURLFORM_ARRAY, forms, + CURLFORM_END); + PyText_EncodedDecref(oencoded_obj); + PyMem_Free(forms); + if (res != CURLE_OK) { + PyText_EncodedDecref(nencoded_obj); + CURLERROR_SET_RETVAL(); + goto error; + } + } else { + /* Some other type was given, ignore */ + PyText_EncodedDecref(nencoded_obj); + PyErr_SetString(PyExc_TypeError, "unsupported second type in tuple"); + goto error; + } + PyText_EncodedDecref(nencoded_obj); + } + res = curl_easy_setopt(self->handle, CURLOPT_HTTPPOST, post); + /* Check for errors */ + if (res != CURLE_OK) { + CURLERROR_SET_RETVAL(); + goto error; + } + /* Finally, decref previous httppost object and replace it with a + * new one. */ + util_curlhttppost_update(self, post, ref_params); + + Py_RETURN_NONE; + +error: + curl_formfree(post); + Py_XDECREF(ref_params); + return NULL; +} + + +static PyObject * +do_curl_setopt_list(CurlObject *self, int option, int which, PyObject *obj) +{ + CurlSlistObject **old_slist_obj = NULL; + struct curl_slist *slist = NULL; + Py_ssize_t len; + int res; + + switch (option) { + case CURLOPT_HTTP200ALIASES: + old_slist_obj = &self->http200aliases; + break; + case CURLOPT_HTTPHEADER: + old_slist_obj = &self->httpheader; + break; +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 37, 0) + case CURLOPT_PROXYHEADER: + old_slist_obj = &self->proxyheader; + break; +#endif + case CURLOPT_POSTQUOTE: + old_slist_obj = &self->postquote; + break; + case CURLOPT_PREQUOTE: + old_slist_obj = &self->prequote; + break; + case CURLOPT_QUOTE: + old_slist_obj = &self->quote; + break; + case CURLOPT_TELNETOPTIONS: + old_slist_obj = &self->telnetoptions; + break; +#ifdef HAVE_CURLOPT_RESOLVE + case CURLOPT_RESOLVE: + old_slist_obj = &self->resolve; + break; +#endif +#ifdef HAVE_CURL_7_20_0_OPTS + case CURLOPT_MAIL_RCPT: + old_slist_obj = &self->mail_rcpt; + break; +#endif +#ifdef HAVE_CURLOPT_CONNECT_TO + case CURLOPT_CONNECT_TO: + old_slist_obj = &self->connect_to; + break; +#endif + default: + /* None of the list options were recognized, raise exception */ + PyErr_SetString(PyExc_TypeError, "lists are not supported for this option"); + return NULL; + } + + len = PyListOrTuple_Size(obj, which); + if (len == 0) + Py_RETURN_NONE; + + /* Just to be sure we do not bug off here */ + assert(old_slist_obj != NULL && slist == NULL); + + /* Handle regular list operations on the other options */ + slist = pycurl_list_or_tuple_to_slist(which, obj, len); + if (slist == NULL) { + return NULL; + } + res = curl_easy_setopt(self->handle, (CURLoption)option, slist); + /* Check for errors */ + if (res != CURLE_OK) { + curl_slist_free_all(slist); + CURLERROR_RETVAL(); + } + /* Finally, decref previous slist object and replace it with a + * new one. */ + util_curlslist_update(old_slist_obj, slist); + + Py_RETURN_NONE; +} + + +static PyObject * +do_curl_setopt_callable(CurlObject *self, int option, PyObject *obj) +{ + /* We use function types here to make sure that our callback + * definitions exactly match the interface. + */ + const curl_write_callback w_cb = write_callback; + const curl_write_callback h_cb = header_callback; + const curl_read_callback r_cb = read_callback; + const curl_progress_callback pro_cb = progress_callback; +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 32, 0) + const curl_xferinfo_callback xferinfo_cb = xferinfo_callback; +#endif + const curl_debug_callback debug_cb = debug_callback; + const curl_ioctl_callback ioctl_cb = ioctl_callback; + const curl_opensocket_callback opensocket_cb = opensocket_callback; +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 7) + const curl_closesocket_callback closesocket_cb = closesocket_callback; +#endif + const curl_seek_callback seek_cb = seek_callback; + + switch(option) { + case CURLOPT_WRITEFUNCTION: + Py_INCREF(obj); + Py_CLEAR(self->writedata_fp); + Py_CLEAR(self->w_cb); + self->w_cb = obj; + curl_easy_setopt(self->handle, CURLOPT_WRITEFUNCTION, w_cb); + curl_easy_setopt(self->handle, CURLOPT_WRITEDATA, self); + break; + case CURLOPT_HEADERFUNCTION: + Py_INCREF(obj); + Py_CLEAR(self->writeheader_fp); + Py_CLEAR(self->h_cb); + self->h_cb = obj; + curl_easy_setopt(self->handle, CURLOPT_HEADERFUNCTION, h_cb); + curl_easy_setopt(self->handle, CURLOPT_WRITEHEADER, self); + break; + case CURLOPT_READFUNCTION: + Py_INCREF(obj); + Py_CLEAR(self->readdata_fp); + Py_CLEAR(self->r_cb); + self->r_cb = obj; + curl_easy_setopt(self->handle, CURLOPT_READFUNCTION, r_cb); + curl_easy_setopt(self->handle, CURLOPT_READDATA, self); + break; + case CURLOPT_PROGRESSFUNCTION: + Py_INCREF(obj); + Py_CLEAR(self->pro_cb); + self->pro_cb = obj; + curl_easy_setopt(self->handle, CURLOPT_PROGRESSFUNCTION, pro_cb); + curl_easy_setopt(self->handle, CURLOPT_PROGRESSDATA, self); + break; +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 32, 0) + case CURLOPT_XFERINFOFUNCTION: + Py_INCREF(obj); + Py_CLEAR(self->xferinfo_cb); + self->xferinfo_cb = obj; + curl_easy_setopt(self->handle, CURLOPT_XFERINFOFUNCTION, xferinfo_cb); + curl_easy_setopt(self->handle, CURLOPT_XFERINFODATA, self); + break; +#endif + case CURLOPT_DEBUGFUNCTION: + Py_INCREF(obj); + Py_CLEAR(self->debug_cb); + self->debug_cb = obj; + curl_easy_setopt(self->handle, CURLOPT_DEBUGFUNCTION, debug_cb); + curl_easy_setopt(self->handle, CURLOPT_DEBUGDATA, self); + break; + case CURLOPT_IOCTLFUNCTION: + Py_INCREF(obj); + Py_CLEAR(self->ioctl_cb); + self->ioctl_cb = obj; + curl_easy_setopt(self->handle, CURLOPT_IOCTLFUNCTION, ioctl_cb); + curl_easy_setopt(self->handle, CURLOPT_IOCTLDATA, self); + break; + case CURLOPT_OPENSOCKETFUNCTION: + Py_INCREF(obj); + Py_CLEAR(self->opensocket_cb); + self->opensocket_cb = obj; + curl_easy_setopt(self->handle, CURLOPT_OPENSOCKETFUNCTION, opensocket_cb); + curl_easy_setopt(self->handle, CURLOPT_OPENSOCKETDATA, self); + break; +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 7) + case CURLOPT_CLOSESOCKETFUNCTION: + Py_INCREF(obj); + Py_CLEAR(self->closesocket_cb); + self->closesocket_cb = obj; + curl_easy_setopt(self->handle, CURLOPT_CLOSESOCKETFUNCTION, closesocket_cb); + curl_easy_setopt(self->handle, CURLOPT_CLOSESOCKETDATA, self); + break; +#endif + case CURLOPT_SOCKOPTFUNCTION: + Py_INCREF(obj); + Py_CLEAR(self->sockopt_cb); + self->sockopt_cb = obj; + curl_easy_setopt(self->handle, CURLOPT_SOCKOPTFUNCTION, sockopt_cb); + curl_easy_setopt(self->handle, CURLOPT_SOCKOPTDATA, self); + break; +#ifdef HAVE_CURL_7_19_6_OPTS + case CURLOPT_SSH_KEYFUNCTION: + Py_INCREF(obj); + Py_CLEAR(self->ssh_key_cb); + self->ssh_key_cb = obj; + curl_easy_setopt(self->handle, CURLOPT_SSH_KEYFUNCTION, ssh_key_cb); + curl_easy_setopt(self->handle, CURLOPT_SSH_KEYDATA, self); + break; +#endif + case CURLOPT_SEEKFUNCTION: + Py_INCREF(obj); + Py_CLEAR(self->seek_cb); + self->seek_cb = obj; + curl_easy_setopt(self->handle, CURLOPT_SEEKFUNCTION, seek_cb); + curl_easy_setopt(self->handle, CURLOPT_SEEKDATA, self); + break; + + default: + /* None of the function options were recognized, raise exception */ + PyErr_SetString(PyExc_TypeError, "functions are not supported for this option"); + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject * +do_curl_setopt_share(CurlObject *self, PyObject *obj) +{ + CurlShareObject *share; + int res; + + if (self->share == NULL && (obj == NULL || obj == Py_None)) + Py_RETURN_NONE; + + if (self->share) { + if (obj != Py_None) { + PyErr_SetString(ErrorObject, "Curl object already sharing. Unshare first."); + return NULL; + } + else { + share = self->share; + res = curl_easy_setopt(self->handle, CURLOPT_SHARE, NULL); + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } + self->share = NULL; + Py_DECREF(share); + Py_RETURN_NONE; + } + } + if (Py_TYPE(obj) != p_CurlShare_Type) { + PyErr_SetString(PyExc_TypeError, "invalid arguments to setopt"); + return NULL; + } + share = (CurlShareObject*)obj; + res = curl_easy_setopt(self->handle, CURLOPT_SHARE, share->share_handle); + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } + self->share = share; + Py_INCREF(share); + Py_RETURN_NONE; +} + + +PYCURL_INTERNAL PyObject * +do_curl_setopt_filelike(CurlObject *self, int option, PyObject *obj) +{ + const char *method_name; + PyObject *method; + + if (option == CURLOPT_READDATA) { + method_name = "read"; + } else { + method_name = "write"; + } + method = PyObject_GetAttrString(obj, method_name); + if (method) { + PyObject *arglist; + PyObject *rv; + + switch (option) { + case CURLOPT_READDATA: + option = CURLOPT_READFUNCTION; + break; + case CURLOPT_WRITEDATA: + option = CURLOPT_WRITEFUNCTION; + break; + case CURLOPT_WRITEHEADER: + option = CURLOPT_HEADERFUNCTION; + break; + default: + PyErr_SetString(PyExc_TypeError, "objects are not supported for this option"); + Py_DECREF(method); + return NULL; + } + + arglist = Py_BuildValue("(iO)", option, method); + /* reference is now in arglist */ + Py_DECREF(method); + if (arglist == NULL) { + return NULL; + } + rv = do_curl_setopt(self, arglist); + Py_DECREF(arglist); + return rv; + } else { + if (option == CURLOPT_READDATA) { + PyErr_SetString(PyExc_TypeError, "object given without a read method"); + } else { + PyErr_SetString(PyExc_TypeError, "object given without a write method"); + } + return NULL; + } +} + + +PYCURL_INTERNAL PyObject * +do_curl_setopt(CurlObject *self, PyObject *args) +{ + int option; + PyObject *obj; + int which; + + if (!PyArg_ParseTuple(args, "iO:setopt", &option, &obj)) + return NULL; + if (check_curl_state(self, 1 | 2, "setopt") != 0) + return NULL; + + /* early checks of option value */ + if (option <= 0) + goto error; + if (option >= (int)CURLOPTTYPE_OFF_T + OPTIONS_SIZE) + goto error; + if (option % 10000 >= OPTIONS_SIZE) + goto error; + + /* Handle the case of None as the call of unsetopt() */ + if (obj == Py_None) { + return util_curl_unsetopt(self, option); + } + + /* Handle the case of string arguments */ + if (PyText_Check(obj)) { + return do_curl_setopt_string_impl(self, option, obj); + } + + /* Handle the case of integer arguments */ + if (PyInt_Check(obj)) { + return do_curl_setopt_int(self, option, obj); + } + + /* Handle the case of long arguments (used by *_LARGE options) */ + if (PyLong_Check(obj)) { + return do_curl_setopt_long(self, option, obj); + } + +#if PY_MAJOR_VERSION < 3 && !defined(PYCURL_AVOID_STDIO) + /* Handle the case of file objects */ + if (PyFile_Check(obj)) { + return do_curl_setopt_file_passthrough(self, option, obj); + } +#endif + + /* Handle the case of list or tuple objects */ + which = PyListOrTuple_Check(obj); + if (which) { + if (option == CURLOPT_HTTPPOST) { + return do_curl_setopt_httppost(self, option, which, obj); + } else { + return do_curl_setopt_list(self, option, which, obj); + } + } + + /* Handle the case of function objects for callbacks */ + if (PyFunction_Check(obj) || PyCFunction_Check(obj) || + PyCallable_Check(obj) || PyMethod_Check(obj)) { + return do_curl_setopt_callable(self, option, obj); + } + /* handle the SHARE case */ + if (option == CURLOPT_SHARE) { + return do_curl_setopt_share(self, obj); + } + + /* + Handle the case of file-like objects. + + Given an object with a write method, we will call the write method + from the appropriate callback. + + Files in Python 3 are no longer FILE * instances and therefore cannot + be directly given to curl, therefore this method handles all I/O to + Python objects. + + In Python 2 true file objects are FILE * instances and will be handled + by stdio passthrough code invoked above, and file-like objects will + be handled by this method. + */ + if (option == CURLOPT_READDATA || + option == CURLOPT_WRITEDATA || + option == CURLOPT_WRITEHEADER) + { + return do_curl_setopt_filelike(self, option, obj); + } + + /* Failed to match any of the function signatures -- return error */ +error: + PyErr_SetString(PyExc_TypeError, "invalid arguments to setopt"); + return NULL; +} + + +PYCURL_INTERNAL PyObject * +do_curl_setopt_string(CurlObject *self, PyObject *args) +{ + int option; + PyObject *obj; + + if (!PyArg_ParseTuple(args, "iO:setopt", &option, &obj)) + return NULL; + if (check_curl_state(self, 1 | 2, "setopt") != 0) + return NULL; + + /* Handle the case of string arguments */ + if (PyText_Check(obj)) { + return do_curl_setopt_string_impl(self, option, obj); + } + + /* Failed to match any of the function signatures -- return error */ + PyErr_SetString(PyExc_TypeError, "invalid arguments to setopt_string"); + return NULL; +} + + +#if defined(HAVE_CURL_OPENSSL) +/* load ca certs from string */ +PYCURL_INTERNAL PyObject * +do_curl_set_ca_certs(CurlObject *self, PyObject *args) +{ + PyObject *cadata; + PyObject *encoded_obj; + char *buffer; + Py_ssize_t length; + int res; + + if (!PyArg_ParseTuple(args, "O:cadata", &cadata)) + return NULL; + + // This may result in cadata string being encoded twice, + // not going to worry about it for now + if (!PyText_Check(cadata)) { + PyErr_SetString(PyExc_TypeError, "set_ca_certs argument must be a byte string or a Unicode string with ASCII code points only"); + return NULL; + } + + res = PyText_AsStringAndSize(cadata, &buffer, &length, &encoded_obj); + if (res) { + PyErr_SetString(PyExc_TypeError, "set_ca_certs argument must be a byte string or a Unicode string with ASCII code points only"); + return NULL; + } + + Py_CLEAR(self->ca_certs_obj); + if (encoded_obj) { + self->ca_certs_obj = encoded_obj; + } else { + Py_INCREF(cadata); + self->ca_certs_obj = cadata; + } + + res = curl_easy_setopt(self->handle, CURLOPT_SSL_CTX_FUNCTION, (curl_ssl_ctx_callback) ssl_ctx_callback); + if (res != CURLE_OK) { + Py_CLEAR(self->ca_certs_obj); + CURLERROR_RETVAL(); + } + + res = curl_easy_setopt(self->handle, CURLOPT_SSL_CTX_DATA, self); + if (res != CURLE_OK) { + Py_CLEAR(self->ca_certs_obj); + CURLERROR_RETVAL(); + } + + Py_RETURN_NONE; +} +#endif diff --git a/src/easyperform.c b/src/easyperform.c new file mode 100644 index 0000000..5326df3 --- /dev/null +++ b/src/easyperform.c @@ -0,0 +1,125 @@ +#include "pycurl.h" + + +/* --------------- perform --------------- */ + +PYCURL_INTERNAL PyObject * +do_curl_perform(CurlObject *self) +{ + int res; + + if (check_curl_state(self, 1 | 2, "perform") != 0) { + return NULL; + } + + PYCURL_BEGIN_ALLOW_THREADS + res = curl_easy_perform(self->handle); + PYCURL_END_ALLOW_THREADS + + if (res != CURLE_OK) { + CURLERROR_RETVAL(); + } + Py_RETURN_NONE; +} + + +PYCURL_INTERNAL PyObject * +do_curl_perform_rb(CurlObject *self) +{ + PyObject *v, *io; + + /* NOTE: this tuple is never freed. */ + static PyObject *empty_tuple = NULL; + + if (empty_tuple == NULL) { + empty_tuple = PyTuple_New(0); + if (empty_tuple == NULL) { + return NULL; + } + } + + io = PyObject_Call(bytesio, empty_tuple, NULL); + if (io == NULL) { + return NULL; + } + + v = do_curl_setopt_filelike(self, CURLOPT_WRITEDATA, io); + if (v == NULL) { + Py_DECREF(io); + return NULL; + } + + v = do_curl_perform(self); + if (v == NULL) { + return NULL; + } + + v = PyObject_CallMethod(io, "getvalue", NULL); + Py_DECREF(io); + return v; +} + +#if PY_MAJOR_VERSION >= 3 +PYCURL_INTERNAL PyObject * +do_curl_perform_rs(CurlObject *self) +{ + PyObject *v, *decoded; + + v = do_curl_perform_rb(self); + if (v == NULL) { + return NULL; + } + + decoded = PyUnicode_FromEncodedObject(v, NULL, NULL); + Py_DECREF(v); + return decoded; +} +#endif + + +/* --------------- pause --------------- */ + + +/* curl_easy_pause() can be called from inside a callback or outside */ +PYCURL_INTERNAL PyObject * +do_curl_pause(CurlObject *self, PyObject *args) +{ + int bitmask; + CURLcode res; +#ifdef WITH_THREAD + PyThreadState *saved_state; +#endif + + if (!PyArg_ParseTuple(args, "i:pause", &bitmask)) { + return NULL; + } + if (check_curl_state(self, 1, "pause") != 0) { + return NULL; + } + +#ifdef WITH_THREAD + /* Save handle to current thread (used as context for python callbacks) */ + saved_state = self->state; + PYCURL_BEGIN_ALLOW_THREADS_EASY + + /* We must allow threads here because unpausing a handle can cause + some of its callbacks to be invoked immediately, from inside + curl_easy_pause() */ +#endif + + res = curl_easy_pause(self->handle, bitmask); + +#ifdef WITH_THREAD + PYCURL_END_ALLOW_THREADS_EASY + + /* Restore the thread-state to whatever it was on entry */ + self->state = saved_state; +#endif + + if (res != CURLE_OK) { + CURLERROR_MSG("pause/unpause failed"); + } else { + Py_INCREF(Py_None); + return Py_None; + } +} diff --git a/src/module.c b/src/module.c new file mode 100644 index 0000000..d55d3eb --- /dev/null +++ b/src/module.c @@ -0,0 +1,1569 @@ +#include "pycurl.h" +#include "docstrings.h" + +#if defined(WIN32) +# define PYCURL_STRINGIZE_IMP(x) #x +# define PYCURL_STRINGIZE(x) PYCURL_STRINGIZE_IMP(x) +# define PYCURL_VERSION_STRING PYCURL_STRINGIZE(PYCURL_VERSION) +#else +# define PYCURL_VERSION_STRING PYCURL_VERSION +#endif + +#define PYCURL_VERSION_PREFIX "PycURL/" PYCURL_VERSION_STRING + +/* needed for compatibility with python < 3.10, as suggested at: + * https://docs.python.org/3.10/whatsnew/3.10.html#id2 */ +#if PY_VERSION_HEX < 0x030900A4 +# define Py_SET_TYPE(obj, type) ((Py_TYPE(obj) = (type)), (void)0) +#endif + +PYCURL_INTERNAL char *empty_keywords[] = { NULL }; + +PYCURL_INTERNAL PyObject *bytesio = NULL; +PYCURL_INTERNAL PyObject *stringio = NULL; + +/* Initialized during module init */ +PYCURL_INTERNAL char *g_pycurl_useragent = NULL; + +/* Type objects */ +PYCURL_INTERNAL PyObject *ErrorObject = NULL; +PYCURL_INTERNAL PyTypeObject *p_Curl_Type = NULL; +PYCURL_INTERNAL PyTypeObject *p_CurlSlist_Type = NULL; +PYCURL_INTERNAL PyTypeObject *p_CurlHttppost_Type = NULL; +PYCURL_INTERNAL PyTypeObject *p_CurlMulti_Type = NULL; +PYCURL_INTERNAL PyTypeObject *p_CurlShare_Type = NULL; +#ifdef HAVE_CURL_7_19_6_OPTS +PYCURL_INTERNAL PyObject *khkey_type = NULL; +#endif +PYCURL_INTERNAL PyObject *curl_sockaddr_type = NULL; + +PYCURL_INTERNAL PyObject *curlobject_constants = NULL; +PYCURL_INTERNAL PyObject *curlmultiobject_constants = NULL; +PYCURL_INTERNAL PyObject *curlshareobject_constants = NULL; + + +/* List of functions defined in this module */ +static PyMethodDef curl_methods[] = { + {"global_init", (PyCFunction)do_global_init, METH_VARARGS, pycurl_global_init_doc}, + {"global_cleanup", (PyCFunction)do_global_cleanup, METH_NOARGS, pycurl_global_cleanup_doc}, + {"version_info", (PyCFunction)do_version_info, METH_VARARGS, pycurl_version_info_doc}, + {NULL, NULL, 0, NULL} +}; + + +/************************************************************************* +// module level +// Note that the object constructors (do_curl_new, do_multi_new) +// are module-level functions as well. +**************************************************************************/ + +static int +are_global_init_flags_valid(int flags) +{ +#ifdef CURL_GLOBAL_ACK_EINTR + /* CURL_GLOBAL_ACK_EINTR was introduced in libcurl-7.30.0 */ + return !(flags & ~(CURL_GLOBAL_ALL | CURL_GLOBAL_ACK_EINTR)); +#else + return !(flags & ~(CURL_GLOBAL_ALL)); +#endif +} + +PYCURL_INTERNAL PyObject * +do_global_init(PyObject *dummy, PyObject *args) +{ + int res, option; + + UNUSED(dummy); + if (!PyArg_ParseTuple(args, "i:global_init", &option)) { + return NULL; + } + + if (!are_global_init_flags_valid(option)) { + PyErr_SetString(PyExc_ValueError, "invalid option to global_init"); + return NULL; + } + + res = curl_global_init(option); + if (res != CURLE_OK) { + PyErr_SetString(ErrorObject, "unable to set global option"); + return NULL; + } + + Py_RETURN_NONE; +} + + +PYCURL_INTERNAL PyObject * +do_global_cleanup(PyObject *dummy) +{ + UNUSED(dummy); + curl_global_cleanup(); +#ifdef PYCURL_NEED_SSL_TSL + pycurl_ssl_cleanup(); +#endif + Py_RETURN_NONE; +} + + +static PyObject *vi_str(const char *s) +{ + if (s == NULL) + Py_RETURN_NONE; + while (*s == ' ' || *s == '\t') + s++; + return PyText_FromString(s); +} + +PYCURL_INTERNAL PyObject * +do_version_info(PyObject *dummy, PyObject *args) +{ + const curl_version_info_data *vi; + PyObject *ret = NULL; + PyObject *protocols = NULL; + PyObject *tmp; + Py_ssize_t i; + int stamp = CURLVERSION_NOW; + + UNUSED(dummy); + if (!PyArg_ParseTuple(args, "|i:version_info", &stamp)) { + return NULL; + } + vi = curl_version_info((CURLversion) stamp); + if (vi == NULL) { + PyErr_SetString(ErrorObject, "unable to get version info"); + return NULL; + } + + /* INFO: actually libcurl in lib/version.c does ignore + * the "stamp" parameter, and so do we. */ + + for (i = 0; vi->protocols[i] != NULL; ) + i++; + protocols = PyTuple_New(i); + if (protocols == NULL) + goto error; + for (i = 0; vi->protocols[i] != NULL; i++) { + tmp = vi_str(vi->protocols[i]); + if (tmp == NULL) + goto error; + PyTuple_SET_ITEM(protocols, i, tmp); + } + ret = PyTuple_New((Py_ssize_t)12); + if (ret == NULL) + goto error; + +#define SET(i, v) \ + tmp = (v); if (tmp == NULL) goto error; PyTuple_SET_ITEM(ret, i, tmp) + SET(0, PyInt_FromLong((long) vi->age)); + SET(1, vi_str(vi->version)); + SET(2, PyInt_FromLong(vi->version_num)); + SET(3, vi_str(vi->host)); + SET(4, PyInt_FromLong(vi->features)); + SET(5, vi_str(vi->ssl_version)); + SET(6, PyInt_FromLong(vi->ssl_version_num)); + SET(7, vi_str(vi->libz_version)); + SET(8, protocols); + SET(9, vi_str(vi->ares)); + SET(10, PyInt_FromLong(vi->ares_num)); + SET(11, vi_str(vi->libidn)); +#undef SET + return ret; + +error: + Py_XDECREF(ret); + Py_XDECREF(protocols); + return NULL; +} + + +/* Helper functions for inserting constants into the module namespace */ + +static int +insobj2(PyObject *dict1, PyObject *dict2, char *name, PyObject *value) +{ + /* Insert an object into one or two dicts. Eats a reference to value. + * See also the implementation of PyDict_SetItemString(). */ + PyObject *key = NULL; + + if (dict1 == NULL && dict2 == NULL) + goto error; + if (value == NULL) + goto error; + + key = PyText_FromString(name); + + if (key == NULL) + goto error; +#if 0 + PyString_InternInPlace(&key); /* XXX Should we really? */ +#endif + if (dict1 != NULL) { +#if !defined(NDEBUG) + if (PyDict_GetItem(dict1, key) != NULL) { + fprintf(stderr, "Symbol already defined: %s\n", name); + assert(0); + } +#endif + if (PyDict_SetItem(dict1, key, value) != 0) + goto error; + } + if (dict2 != NULL && dict2 != dict1) { + assert(PyDict_GetItem(dict2, key) == NULL); + if (PyDict_SetItem(dict2, key, value) != 0) + goto error; + } + Py_DECREF(key); + Py_DECREF(value); + return 0; + +error: + Py_XDECREF(key); + return -1; +} + +#define insobj2_modinit(dict1, dict2, name, value) \ + if (insobj2(dict1, dict2, name, value) < 0) \ + goto error + + +static int +insstr(PyObject *d, char *name, char *value) +{ + PyObject *v; + int rv; + + v = PyText_FromString(value); + if (v == NULL) + return -1; + + rv = insobj2(d, NULL, name, v); + if (rv < 0) { + Py_DECREF(v); + } + return rv; +} + +#define insstr_modinit(d, name, value) \ + do { \ + if (insstr(d, name, value) < 0) \ + goto error; \ + } while(0) + +static int +insint_worker(PyObject *d, PyObject *extra, char *name, long value) +{ + PyObject *v = PyInt_FromLong(value); + if (v == NULL) + return -1; + if (insobj2(d, extra, name, v) < 0) { + Py_DECREF(v); + return -1; + } + return 0; +} + +#define insint(d, name, value) \ + do { \ + if (insint_worker(d, NULL, name, value) < 0) \ + goto error; \ + } while(0) + +#define insint_c(d, name, value) \ + do { \ + if (insint_worker(d, curlobject_constants, name, value) < 0) \ + goto error; \ + } while(0) + +#define insint_m(d, name, value) \ + do { \ + if (insint_worker(d, curlmultiobject_constants, name, value) < 0) \ + goto error; \ + } while(0) + +#define insint_s(d, name, value) \ + do { \ + if (insint_worker(d, curlshareobject_constants, name, value) < 0) \ + goto error; \ + } while(0) + + +#if PY_MAJOR_VERSION >= 3 +/* Used in Python 3 only, and even then this function seems to never get + * called. Python 2 has no module cleanup: + * http://stackoverflow.com/questions/20741856/run-a-function-when-a-c-extension-module-is-freed-on-python-2 + */ +static void do_curlmod_free(void *unused) { + PyMem_Free(g_pycurl_useragent); + g_pycurl_useragent = NULL; +} + +static PyModuleDef curlmodule = { + PyModuleDef_HEAD_INIT, + "pycurl", /* m_name */ + pycurl_module_doc, /* m_doc */ + -1, /* m_size */ + curl_methods, /* m_methods */ + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + do_curlmod_free /* m_free */ +}; +#endif + + +#if PY_MAJOR_VERSION >= 3 +#define PYCURL_MODINIT_RETURN_NULL return NULL +PyMODINIT_FUNC PyInit_pycurl(void) +#else +#define PYCURL_MODINIT_RETURN_NULL return +/* Initialization function for the module */ +#if defined(PyMODINIT_FUNC) +PyMODINIT_FUNC +#else +#if defined(__cplusplus) +extern "C" +#endif +DL_EXPORT(void) +#endif +initpycurl(void) +#endif +{ + PyObject *m, *d; + const curl_version_info_data *vi; + const char *libcurl_version; + size_t libcurl_version_len, pycurl_version_len; + PyObject *xio_module = NULL; + PyObject *collections_module = NULL; + PyObject *named_tuple = NULL; + PyObject *arglist = NULL; +#ifdef HAVE_CURL_GLOBAL_SSLSET + const curl_ssl_backend **ssllist = NULL; + CURLsslset sslset; + int i, runtime_supported_backend_found = 0; + char backends[200]; + size_t backends_len = 0; +#else + const char *runtime_ssl_lib; +#endif + + assert(Curl_Type.tp_weaklistoffset > 0); + assert(CurlMulti_Type.tp_weaklistoffset > 0); + assert(CurlShare_Type.tp_weaklistoffset > 0); + + /* Check the version, as this has caused nasty problems in + * some cases. */ + vi = curl_version_info(CURLVERSION_NOW); + if (vi == NULL) { + PyErr_SetString(PyExc_ImportError, "pycurl: curl_version_info() failed"); + goto error; + } + if (vi->version_num < LIBCURL_VERSION_NUM) { + PyErr_Format(PyExc_ImportError, "pycurl: libcurl link-time version (%s) is older than compile-time version (%s)", vi->version, LIBCURL_VERSION); + goto error; + } + + /* Our compiled crypto locks should correspond to runtime ssl library. */ +#ifdef HAVE_CURL_GLOBAL_SSLSET + sslset = curl_global_sslset(-1, COMPILE_SSL_LIB, &ssllist); + if (sslset != CURLSSLSET_OK) { + if (sslset == CURLSSLSET_NO_BACKENDS) { + strcpy(backends, "none"); + } else { + for (i = 0; ssllist[i] != NULL; i++) { + switch (ssllist[i]->id) { + case CURLSSLBACKEND_OPENSSL: + case CURLSSLBACKEND_GNUTLS: + case CURLSSLBACKEND_NSS: + case CURLSSLBACKEND_WOLFSSL: + case CURLSSLBACKEND_MBEDTLS: +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 64, 1) + case CURLSSLBACKEND_SECURETRANSPORT: +#else + case CURLSSLBACKEND_DARWINSSL: +#endif + runtime_supported_backend_found = 1; + break; + default: + break; + } + if (backends_len < sizeof(backends)) { + backends_len += snprintf(backends + backends_len, sizeof(backends) - backends_len, "%s%s", (i > 0) ? ", " : "", ssllist[i]->name); + } + } + } + /* Don't error if both the curl library and pycurl itself is compiled without SSL */ + if (runtime_supported_backend_found || COMPILE_SUPPORTED_SSL_BACKEND_FOUND) { + PyErr_Format(PyExc_ImportError, "pycurl: libcurl link-time ssl backends (%s) do not include compile-time ssl backend (%s)", backends, COMPILE_SSL_LIB); + goto error; + } + } +#else + if (vi->ssl_version == NULL) { + runtime_ssl_lib = "none/other"; + } else if (!strncmp(vi->ssl_version, "OpenSSL/", 8) || !strncmp(vi->ssl_version, "LibreSSL/", 9) || + !strncmp(vi->ssl_version, "BoringSSL", 9)) { + runtime_ssl_lib = "openssl"; + } else if (!strncmp(vi->ssl_version, "wolfSSL/", 8)) { + runtime_ssl_lib = "wolfssl"; + } else if (!strncmp(vi->ssl_version, "GnuTLS/", 7)) { + runtime_ssl_lib = "gnutls"; + } else if (!strncmp(vi->ssl_version, "NSS/", 4)) { + runtime_ssl_lib = "nss"; + } else if (!strncmp(vi->ssl_version, "mbedTLS/", 8)) { + runtime_ssl_lib = "mbedtls"; + } else if (!strncmp(vi->ssl_version, "Secure Transport", 16)) { + runtime_ssl_lib = "secure-transport"; + } else { + runtime_ssl_lib = "none/other"; + } + if (strcmp(runtime_ssl_lib, COMPILE_SSL_LIB)) { + PyErr_Format(PyExc_ImportError, "pycurl: libcurl link-time ssl backend (%s) is different from compile-time ssl backend (%s)", runtime_ssl_lib, COMPILE_SSL_LIB); + goto error; + } +#endif + + /* Initialize the type of the new type objects here; doing it here + * is required for portability to Windows without requiring C++. */ + p_Curl_Type = &Curl_Type; + p_CurlSlist_Type = &CurlSlist_Type; + p_CurlHttppost_Type = &CurlHttppost_Type; + p_CurlMulti_Type = &CurlMulti_Type; + p_CurlShare_Type = &CurlShare_Type; + Py_SET_TYPE(&Curl_Type, &PyType_Type); + Py_SET_TYPE(&CurlSlist_Type, &PyType_Type); + Py_SET_TYPE(&CurlHttppost_Type, &PyType_Type); + Py_SET_TYPE(&CurlMulti_Type, &PyType_Type); + Py_SET_TYPE(&CurlShare_Type, &PyType_Type); + + /* Create the module and add the functions */ + if (PyType_Ready(&Curl_Type) < 0) + goto error; + + if (PyType_Ready(&CurlSlist_Type) < 0) + goto error; + + if (PyType_Ready(&CurlHttppost_Type) < 0) + goto error; + + if (PyType_Ready(&CurlMulti_Type) < 0) + goto error; + + if (PyType_Ready(&CurlShare_Type) < 0) + goto error; + + +#if PY_MAJOR_VERSION >= 3 + m = PyModule_Create(&curlmodule); + if (m == NULL) + goto error; +#else + /* returns a borrowed reference, XDECREFing it crashes the interpreter */ + m = Py_InitModule3("pycurl", curl_methods, pycurl_module_doc); + if (m == NULL || !PyModule_Check(m)) + goto error; +#endif + + /* Add error object to the module */ + d = PyModule_GetDict(m); + assert(d != NULL); + ErrorObject = PyErr_NewException("pycurl.error", NULL, NULL); + if (ErrorObject == NULL) + goto error; + if (PyDict_SetItemString(d, "error", ErrorObject) < 0) { + goto error; + } + + curlobject_constants = PyDict_New(); + if (curlobject_constants == NULL) + goto error; + + curlmultiobject_constants = PyDict_New(); + if (curlmultiobject_constants == NULL) + goto error; + + curlshareobject_constants = PyDict_New(); + if (curlshareobject_constants == NULL) + goto error; + + /* Add version strings to the module */ + libcurl_version = curl_version(); + libcurl_version_len = strlen(libcurl_version); +#define PYCURL_VERSION_PREFIX_SIZE sizeof(PYCURL_VERSION_PREFIX) + /* PYCURL_VERSION_PREFIX_SIZE includes terminating null which will be + * replaced with the space; libcurl_version_len does not include + * terminating null. */ + pycurl_version_len = PYCURL_VERSION_PREFIX_SIZE + libcurl_version_len + 1; + g_pycurl_useragent = PyMem_New(char, pycurl_version_len); + if (g_pycurl_useragent == NULL) + goto error; + memcpy(g_pycurl_useragent, PYCURL_VERSION_PREFIX, PYCURL_VERSION_PREFIX_SIZE); + g_pycurl_useragent[PYCURL_VERSION_PREFIX_SIZE-1] = ' '; + memcpy(g_pycurl_useragent + PYCURL_VERSION_PREFIX_SIZE, + libcurl_version, libcurl_version_len); + g_pycurl_useragent[pycurl_version_len - 1] = 0; +#undef PYCURL_VERSION_PREFIX_SIZE + + insstr_modinit(d, "version", g_pycurl_useragent); + insint(d, "COMPILE_PY_VERSION_HEX", PY_VERSION_HEX); + insint(d, "COMPILE_LIBCURL_VERSION_NUM", LIBCURL_VERSION_NUM); + + /* Types */ + insobj2_modinit(d, NULL, "Curl", (PyObject *) p_Curl_Type); + insobj2_modinit(d, NULL, "CurlMulti", (PyObject *) p_CurlMulti_Type); + insobj2_modinit(d, NULL, "CurlShare", (PyObject *) p_CurlShare_Type); + + /** + ** the order of these constants mostly follows + **/ + + /* Abort curl_read_callback(). */ + insint_c(d, "READFUNC_ABORT", CURL_READFUNC_ABORT); + insint_c(d, "READFUNC_PAUSE", CURL_READFUNC_PAUSE); + + /* Pause curl_write_callback(). */ + insint_c(d, "WRITEFUNC_PAUSE", CURL_WRITEFUNC_PAUSE); + + /* constants for ioctl callback return values */ + insint_c(d, "IOE_OK", CURLIOE_OK); + insint_c(d, "IOE_UNKNOWNCMD", CURLIOE_UNKNOWNCMD); + insint_c(d, "IOE_FAILRESTART", CURLIOE_FAILRESTART); + + /* constants for ioctl callback argument values */ + insint_c(d, "IOCMD_NOP", CURLIOCMD_NOP); + insint_c(d, "IOCMD_RESTARTREAD", CURLIOCMD_RESTARTREAD); + + /* opensocketfunction return value */ + insint_c(d, "SOCKET_BAD", CURL_SOCKET_BAD); + + /* curl_infotype: the kind of data that is passed to information_callback */ +/* XXX do we actually need curl_infotype in pycurl ??? */ + insint_c(d, "INFOTYPE_TEXT", CURLINFO_TEXT); + insint_c(d, "INFOTYPE_HEADER_IN", CURLINFO_HEADER_IN); + insint_c(d, "INFOTYPE_HEADER_OUT", CURLINFO_HEADER_OUT); + insint_c(d, "INFOTYPE_DATA_IN", CURLINFO_DATA_IN); + insint_c(d, "INFOTYPE_DATA_OUT", CURLINFO_DATA_OUT); + insint_c(d, "INFOTYPE_SSL_DATA_IN", CURLINFO_SSL_DATA_IN); + insint_c(d, "INFOTYPE_SSL_DATA_OUT", CURLINFO_SSL_DATA_OUT); + + /* CURLcode: error codes */ + insint_c(d, "E_OK", CURLE_OK); + insint_c(d, "E_AGAIN", CURLE_AGAIN); + insint_c(d, "E_ALREADY_COMPLETE", CURLE_ALREADY_COMPLETE); + insint_c(d, "E_BAD_CALLING_ORDER", CURLE_BAD_CALLING_ORDER); + insint_c(d, "E_BAD_PASSWORD_ENTERED", CURLE_BAD_PASSWORD_ENTERED); + insint_c(d, "E_FTP_BAD_DOWNLOAD_RESUME", CURLE_FTP_BAD_DOWNLOAD_RESUME); + insint_c(d, "E_FTP_COULDNT_SET_TYPE", CURLE_FTP_COULDNT_SET_TYPE); + insint_c(d, "E_FTP_PARTIAL_FILE", CURLE_FTP_PARTIAL_FILE); + insint_c(d, "E_FTP_USER_PASSWORD_INCORRECT", CURLE_FTP_USER_PASSWORD_INCORRECT); + insint_c(d, "E_HTTP_NOT_FOUND", CURLE_HTTP_NOT_FOUND); + insint_c(d, "E_HTTP_PORT_FAILED", CURLE_HTTP_PORT_FAILED); + insint_c(d, "E_MALFORMAT_USER", CURLE_MALFORMAT_USER); + insint_c(d, "E_QUOTE_ERROR", CURLE_QUOTE_ERROR); + insint_c(d, "E_RANGE_ERROR", CURLE_RANGE_ERROR); + insint_c(d, "E_REMOTE_ACCESS_DENIED", CURLE_REMOTE_ACCESS_DENIED); + insint_c(d, "E_REMOTE_DISK_FULL", CURLE_REMOTE_DISK_FULL); + insint_c(d, "E_REMOTE_FILE_EXISTS", CURLE_REMOTE_FILE_EXISTS); + insint_c(d, "E_UPLOAD_FAILED", CURLE_UPLOAD_FAILED); + insint_c(d, "E_URL_MALFORMAT_USER", CURLE_URL_MALFORMAT_USER); + insint_c(d, "E_USE_SSL_FAILED", CURLE_USE_SSL_FAILED); + insint_c(d, "E_UNSUPPORTED_PROTOCOL", CURLE_UNSUPPORTED_PROTOCOL); + insint_c(d, "E_FAILED_INIT", CURLE_FAILED_INIT); + insint_c(d, "E_URL_MALFORMAT", CURLE_URL_MALFORMAT); +#ifdef HAVE_CURL_7_21_5 + insint_c(d, "E_NOT_BUILT_IN", CURLE_NOT_BUILT_IN); +#endif + insint_c(d, "E_COULDNT_RESOLVE_PROXY", CURLE_COULDNT_RESOLVE_PROXY); + insint_c(d, "E_COULDNT_RESOLVE_HOST", CURLE_COULDNT_RESOLVE_HOST); + insint_c(d, "E_COULDNT_CONNECT", CURLE_COULDNT_CONNECT); + insint_c(d, "E_FTP_WEIRD_SERVER_REPLY", CURLE_FTP_WEIRD_SERVER_REPLY); + insint_c(d, "E_FTP_ACCESS_DENIED", CURLE_FTP_ACCESS_DENIED); +#ifdef HAVE_CURL_7_24_0 + insint_c(d, "E_FTP_ACCEPT_FAILED", CURLE_FTP_ACCEPT_FAILED); +#endif + insint_c(d, "E_FTP_WEIRD_PASS_REPLY", CURLE_FTP_WEIRD_PASS_REPLY); + insint_c(d, "E_FTP_WEIRD_USER_REPLY", CURLE_FTP_WEIRD_USER_REPLY); + insint_c(d, "E_FTP_WEIRD_PASV_REPLY", CURLE_FTP_WEIRD_PASV_REPLY); + insint_c(d, "E_FTP_WEIRD_227_FORMAT", CURLE_FTP_WEIRD_227_FORMAT); + insint_c(d, "E_FTP_CANT_GET_HOST", CURLE_FTP_CANT_GET_HOST); + insint_c(d, "E_FTP_CANT_RECONNECT", CURLE_FTP_CANT_RECONNECT); + insint_c(d, "E_FTP_COULDNT_SET_BINARY", CURLE_FTP_COULDNT_SET_BINARY); + insint_c(d, "E_PARTIAL_FILE", CURLE_PARTIAL_FILE); + insint_c(d, "E_FTP_COULDNT_RETR_FILE", CURLE_FTP_COULDNT_RETR_FILE); + insint_c(d, "E_FTP_WRITE_ERROR", CURLE_FTP_WRITE_ERROR); + insint_c(d, "E_FTP_QUOTE_ERROR", CURLE_FTP_QUOTE_ERROR); + insint_c(d, "E_HTTP_RETURNED_ERROR", CURLE_HTTP_RETURNED_ERROR); + insint_c(d, "E_WRITE_ERROR", CURLE_WRITE_ERROR); + insint_c(d, "E_FTP_COULDNT_STOR_FILE", CURLE_FTP_COULDNT_STOR_FILE); + insint_c(d, "E_READ_ERROR", CURLE_READ_ERROR); + insint_c(d, "E_OUT_OF_MEMORY", CURLE_OUT_OF_MEMORY); + insint_c(d, "E_OPERATION_TIMEOUTED", CURLE_OPERATION_TIMEOUTED); + insint_c(d, "E_OPERATION_TIMEDOUT", CURLE_OPERATION_TIMEDOUT); + insint_c(d, "E_FTP_COULDNT_SET_ASCII", CURLE_FTP_COULDNT_SET_ASCII); + insint_c(d, "E_FTP_PORT_FAILED", CURLE_FTP_PORT_FAILED); + insint_c(d, "E_FTP_COULDNT_USE_REST", CURLE_FTP_COULDNT_USE_REST); + insint_c(d, "E_FTP_COULDNT_GET_SIZE", CURLE_FTP_COULDNT_GET_SIZE); + insint_c(d, "E_HTTP_RANGE_ERROR", CURLE_HTTP_RANGE_ERROR); + insint_c(d, "E_HTTP_POST_ERROR", CURLE_HTTP_POST_ERROR); + insint_c(d, "E_SSL_CACERT", CURLE_SSL_CACERT); + insint_c(d, "E_SSL_CACERT_BADFILE", CURLE_SSL_CACERT_BADFILE); + insint_c(d, "E_SSL_CERTPROBLEM", CURLE_SSL_CERTPROBLEM); + insint_c(d, "E_SSL_CIPHER", CURLE_SSL_CIPHER); + insint_c(d, "E_SSL_CONNECT_ERROR", CURLE_SSL_CONNECT_ERROR); + insint_c(d, "E_SSL_CRL_BADFILE", CURLE_SSL_CRL_BADFILE); + insint_c(d, "E_SSL_ENGINE_INITFAILED", CURLE_SSL_ENGINE_INITFAILED); + insint_c(d, "E_SSL_ENGINE_NOTFOUND", CURLE_SSL_ENGINE_NOTFOUND); + insint_c(d, "E_SSL_ENGINE_SETFAILED", CURLE_SSL_ENGINE_SETFAILED); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 41, 0) + insint_c(d, "E_SSL_INVALIDCERTSTATUS", CURLE_SSL_INVALIDCERTSTATUS); +#endif + insint_c(d, "E_SSL_ISSUER_ERROR", CURLE_SSL_ISSUER_ERROR); + insint_c(d, "E_SSL_PEER_CERTIFICATE", CURLE_SSL_PEER_CERTIFICATE); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 39, 0) + insint_c(d, "E_SSL_PINNEDPUBKEYNOTMATCH", CURLE_SSL_PINNEDPUBKEYNOTMATCH); +#endif + insint_c(d, "E_SSL_SHUTDOWN_FAILED", CURLE_SSL_SHUTDOWN_FAILED); + insint_c(d, "E_BAD_DOWNLOAD_RESUME", CURLE_BAD_DOWNLOAD_RESUME); + insint_c(d, "E_FILE_COULDNT_READ_FILE", CURLE_FILE_COULDNT_READ_FILE); + insint_c(d, "E_LDAP_CANNOT_BIND", CURLE_LDAP_CANNOT_BIND); + insint_c(d, "E_LDAP_SEARCH_FAILED", CURLE_LDAP_SEARCH_FAILED); + insint_c(d, "E_LIBRARY_NOT_FOUND", CURLE_LIBRARY_NOT_FOUND); + insint_c(d, "E_FUNCTION_NOT_FOUND", CURLE_FUNCTION_NOT_FOUND); + insint_c(d, "E_ABORTED_BY_CALLBACK", CURLE_ABORTED_BY_CALLBACK); + insint_c(d, "E_BAD_FUNCTION_ARGUMENT", CURLE_BAD_FUNCTION_ARGUMENT); + insint_c(d, "E_INTERFACE_FAILED", CURLE_INTERFACE_FAILED); + insint_c(d, "E_TOO_MANY_REDIRECTS", CURLE_TOO_MANY_REDIRECTS); +#ifdef HAVE_CURL_7_21_5 + insint_c(d, "E_UNKNOWN_OPTION", CURLE_UNKNOWN_OPTION); +#endif + /* same as E_UNKNOWN_OPTION */ + insint_c(d, "E_UNKNOWN_TELNET_OPTION", CURLE_UNKNOWN_TELNET_OPTION); + insint_c(d, "E_TELNET_OPTION_SYNTAX", CURLE_TELNET_OPTION_SYNTAX); + insint_c(d, "E_GOT_NOTHING", CURLE_GOT_NOTHING); + insint_c(d, "E_SEND_ERROR", CURLE_SEND_ERROR); + insint_c(d, "E_RECV_ERROR", CURLE_RECV_ERROR); + insint_c(d, "E_SHARE_IN_USE", CURLE_SHARE_IN_USE); + insint_c(d, "E_BAD_CONTENT_ENCODING", CURLE_BAD_CONTENT_ENCODING); + insint_c(d, "E_LDAP_INVALID_URL", CURLE_LDAP_INVALID_URL); + insint_c(d, "E_FILESIZE_EXCEEDED", CURLE_FILESIZE_EXCEEDED); + insint_c(d, "E_FTP_SSL_FAILED", CURLE_FTP_SSL_FAILED); + insint_c(d, "E_SEND_FAIL_REWIND", CURLE_SEND_FAIL_REWIND); + insint_c(d, "E_LOGIN_DENIED", CURLE_LOGIN_DENIED); + insint_c(d, "E_PEER_FAILED_VERIFICATION", CURLE_PEER_FAILED_VERIFICATION); + insint_c(d, "E_TFTP_NOTFOUND", CURLE_TFTP_NOTFOUND); + insint_c(d, "E_TFTP_PERM", CURLE_TFTP_PERM); + insint_c(d, "E_TFTP_DISKFULL", CURLE_TFTP_DISKFULL); + insint_c(d, "E_TFTP_ILLEGAL", CURLE_TFTP_ILLEGAL); + insint_c(d, "E_TFTP_UNKNOWNID", CURLE_TFTP_UNKNOWNID); + insint_c(d, "E_TFTP_EXISTS", CURLE_TFTP_EXISTS); + insint_c(d, "E_TFTP_NOSUCHUSER", CURLE_TFTP_NOSUCHUSER); + insint_c(d, "E_CONV_FAILED", CURLE_CONV_FAILED); + insint_c(d, "E_CONV_REQD", CURLE_CONV_REQD); + insint_c(d, "E_REMOTE_FILE_NOT_FOUND", CURLE_REMOTE_FILE_NOT_FOUND); + insint_c(d, "E_SSH", CURLE_SSH); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 20, 0) + insint_c(d, "E_FTP_PRET_FAILED", CURLE_FTP_PRET_FAILED); + insint_c(d, "E_RTSP_CSEQ_ERROR", CURLE_RTSP_CSEQ_ERROR); + insint_c(d, "E_RTSP_SESSION_ERROR", CURLE_RTSP_SESSION_ERROR); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 0) + insint_c(d, "E_CHUNK_FAILED", CURLE_CHUNK_FAILED); + insint_c(d, "E_FTP_BAD_FILE_LIST", CURLE_FTP_BAD_FILE_LIST); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 24, 0) + insint_c(d, "E_FTP_ACCEPT_TIMEOUT", CURLE_FTP_ACCEPT_TIMEOUT); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 30, 0) + insint_c(d, "E_NO_CONNECTION_AVAILABLE", CURLE_NO_CONNECTION_AVAILABLE); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 38, 0) + insint_c(d, "E_HTTP2", CURLE_HTTP2); +#endif + + /* curl_proxytype: constants for setopt(PROXYTYPE, x) */ + insint_c(d, "PROXYTYPE_HTTP", CURLPROXY_HTTP); +#ifdef HAVE_CURL_7_19_4_OPTS + insint_c(d, "PROXYTYPE_HTTP_1_0", CURLPROXY_HTTP_1_0); +#endif + insint_c(d, "PROXYTYPE_SOCKS4", CURLPROXY_SOCKS4); + insint_c(d, "PROXYTYPE_SOCKS4A", CURLPROXY_SOCKS4A); + insint_c(d, "PROXYTYPE_SOCKS5", CURLPROXY_SOCKS5); + insint_c(d, "PROXYTYPE_SOCKS5_HOSTNAME", CURLPROXY_SOCKS5_HOSTNAME); + + /* curl_httpauth: constants for setopt(HTTPAUTH, x) */ + insint_c(d, "HTTPAUTH_ANY", CURLAUTH_ANY); + insint_c(d, "HTTPAUTH_ANYSAFE", CURLAUTH_ANYSAFE); + insint_c(d, "HTTPAUTH_BASIC", CURLAUTH_BASIC); + insint_c(d, "HTTPAUTH_DIGEST", CURLAUTH_DIGEST); +#ifdef HAVE_CURLAUTH_DIGEST_IE + insint_c(d, "HTTPAUTH_DIGEST_IE", CURLAUTH_DIGEST_IE); +#endif + insint_c(d, "HTTPAUTH_GSSNEGOTIATE", CURLAUTH_GSSNEGOTIATE); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 38, 0) + insint_c(d, "HTTPAUTH_NEGOTIATE", CURLAUTH_NEGOTIATE); +#endif + insint_c(d, "HTTPAUTH_NTLM", CURLAUTH_NTLM); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 22, 0) + insint_c(d, "HTTPAUTH_NTLM_WB", CURLAUTH_NTLM_WB); +#endif + insint_c(d, "HTTPAUTH_NONE", CURLAUTH_NONE); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 3) + insint_c(d, "HTTPAUTH_ONLY", CURLAUTH_ONLY); +#endif + +#ifdef HAVE_CURL_7_22_0_OPTS + insint_c(d, "GSSAPI_DELEGATION_FLAG", CURLGSSAPI_DELEGATION_FLAG); + insint_c(d, "GSSAPI_DELEGATION_NONE", CURLGSSAPI_DELEGATION_NONE); + insint_c(d, "GSSAPI_DELEGATION_POLICY_FLAG", CURLGSSAPI_DELEGATION_POLICY_FLAG); + + insint_c(d, "GSSAPI_DELEGATION", CURLOPT_GSSAPI_DELEGATION); +#endif + + /* curl_ftpssl: constants for setopt(FTP_SSL, x) */ + insint_c(d, "FTPSSL_NONE", CURLFTPSSL_NONE); + insint_c(d, "FTPSSL_TRY", CURLFTPSSL_TRY); + insint_c(d, "FTPSSL_CONTROL", CURLFTPSSL_CONTROL); + insint_c(d, "FTPSSL_ALL", CURLFTPSSL_ALL); + + /* curl_ftpauth: constants for setopt(FTPSSLAUTH, x) */ + insint_c(d, "FTPAUTH_DEFAULT", CURLFTPAUTH_DEFAULT); + insint_c(d, "FTPAUTH_SSL", CURLFTPAUTH_SSL); + insint_c(d, "FTPAUTH_TLS", CURLFTPAUTH_TLS); + + /* curl_ftpauth: constants for setopt(FTPSSLAUTH, x) */ + insint_c(d, "FORM_BUFFER", CURLFORM_BUFFER); + insint_c(d, "FORM_BUFFERPTR", CURLFORM_BUFFERPTR); + insint_c(d, "FORM_CONTENTS", CURLFORM_COPYCONTENTS); + insint_c(d, "FORM_FILE", CURLFORM_FILE); + insint_c(d, "FORM_CONTENTTYPE", CURLFORM_CONTENTTYPE); + insint_c(d, "FORM_FILENAME", CURLFORM_FILENAME); + + /* FTP_FILEMETHOD options */ + insint_c(d, "FTPMETHOD_DEFAULT", CURLFTPMETHOD_DEFAULT); + insint_c(d, "FTPMETHOD_MULTICWD", CURLFTPMETHOD_MULTICWD); + insint_c(d, "FTPMETHOD_NOCWD", CURLFTPMETHOD_NOCWD); + insint_c(d, "FTPMETHOD_SINGLECWD", CURLFTPMETHOD_SINGLECWD); + + /* CURLoption: symbolic constants for setopt() */ + insint_c(d, "APPEND", CURLOPT_APPEND); + insint_c(d, "COOKIESESSION", CURLOPT_COOKIESESSION); + insint_c(d, "DIRLISTONLY", CURLOPT_DIRLISTONLY); + /* ERRORBUFFER is not supported */ + insint_c(d, "FILE", CURLOPT_WRITEDATA); + insint_c(d, "FTPPORT", CURLOPT_FTPPORT); + insint_c(d, "INFILE", CURLOPT_READDATA); + insint_c(d, "INFILESIZE", CURLOPT_INFILESIZE_LARGE); /* _LARGE ! */ + insint_c(d, "KEYPASSWD", CURLOPT_KEYPASSWD); + insint_c(d, "LOW_SPEED_LIMIT", CURLOPT_LOW_SPEED_LIMIT); + insint_c(d, "LOW_SPEED_TIME", CURLOPT_LOW_SPEED_TIME); + insint_c(d, "PORT", CURLOPT_PORT); + insint_c(d, "POSTFIELDS", CURLOPT_POSTFIELDS); + insint_c(d, "PROXY", CURLOPT_PROXY); +#ifdef HAVE_CURLOPT_PROXYUSERNAME + insint_c(d, "PROXYPASSWORD", CURLOPT_PROXYPASSWORD); + insint_c(d, "PROXYUSERNAME", CURLOPT_PROXYUSERNAME); +#endif + insint_c(d, "PROXYUSERPWD", CURLOPT_PROXYUSERPWD); + insint_c(d, "RANGE", CURLOPT_RANGE); + insint_c(d, "READFUNCTION", CURLOPT_READFUNCTION); + insint_c(d, "REFERER", CURLOPT_REFERER); + insint_c(d, "RESUME_FROM", CURLOPT_RESUME_FROM_LARGE); /* _LARGE ! */ + insint_c(d, "TELNETOPTIONS", CURLOPT_TELNETOPTIONS); + insint_c(d, "TIMEOUT", CURLOPT_TIMEOUT); + insint_c(d, "URL", CURLOPT_URL); + insint_c(d, "USE_SSL", CURLOPT_USE_SSL); + insint_c(d, "USERAGENT", CURLOPT_USERAGENT); + insint_c(d, "USERPWD", CURLOPT_USERPWD); + insint_c(d, "WRITEFUNCTION", CURLOPT_WRITEFUNCTION); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 20, 0) + insint_c(d, "OPT_RTSP_CLIENT_CSEQ", CURLOPT_RTSP_CLIENT_CSEQ); + insint_c(d, "OPT_RTSP_REQUEST", CURLOPT_RTSP_REQUEST); + insint_c(d, "OPT_RTSP_SERVER_CSEQ", CURLOPT_RTSP_SERVER_CSEQ); + insint_c(d, "OPT_RTSP_SESSION_ID", CURLOPT_RTSP_SESSION_ID); + insint_c(d, "OPT_RTSP_STREAM_URI", CURLOPT_RTSP_STREAM_URI); + insint_c(d, "OPT_RTSP_TRANSPORT", CURLOPT_RTSP_TRANSPORT); +#endif +#ifdef HAVE_CURLOPT_USERNAME + insint_c(d, "USERNAME", CURLOPT_USERNAME); + insint_c(d, "PASSWORD", CURLOPT_PASSWORD); +#endif + insint_c(d, "WRITEDATA", CURLOPT_WRITEDATA); + insint_c(d, "READDATA", CURLOPT_READDATA); + insint_c(d, "PROXYPORT", CURLOPT_PROXYPORT); + insint_c(d, "HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL); + insint_c(d, "VERBOSE", CURLOPT_VERBOSE); + insint_c(d, "HEADER", CURLOPT_HEADER); + insint_c(d, "NOPROGRESS", CURLOPT_NOPROGRESS); + insint_c(d, "NOBODY", CURLOPT_NOBODY); + insint_c(d, "FAILONERROR", CURLOPT_FAILONERROR); + insint_c(d, "UPLOAD", CURLOPT_UPLOAD); + insint_c(d, "POST", CURLOPT_POST); + insint_c(d, "FTPLISTONLY", CURLOPT_FTPLISTONLY); + insint_c(d, "FTPAPPEND", CURLOPT_FTPAPPEND); + insint_c(d, "NETRC", CURLOPT_NETRC); + insint_c(d, "FOLLOWLOCATION", CURLOPT_FOLLOWLOCATION); + insint_c(d, "TRANSFERTEXT", CURLOPT_TRANSFERTEXT); + insint_c(d, "PUT", CURLOPT_PUT); + insint_c(d, "POSTFIELDSIZE", CURLOPT_POSTFIELDSIZE_LARGE); /* _LARGE ! */ + insint_c(d, "COOKIE", CURLOPT_COOKIE); + insint_c(d, "HTTPHEADER", CURLOPT_HTTPHEADER); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 37, 0) + insint_c(d, "PROXYHEADER", CURLOPT_PROXYHEADER); + insint_c(d, "HEADEROPT", CURLOPT_HEADEROPT); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 42, 0) + insint_c(d, "PATH_AS_IS", CURLOPT_PATH_AS_IS); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 43, 0) + insint_c(d, "PIPEWAIT", CURLOPT_PIPEWAIT); +#endif + insint_c(d, "HTTPPOST", CURLOPT_HTTPPOST); + insint_c(d, "SSLCERT", CURLOPT_SSLCERT); + insint_c(d, "SSLCERTPASSWD", CURLOPT_SSLCERTPASSWD); + insint_c(d, "CRLF", CURLOPT_CRLF); + insint_c(d, "QUOTE", CURLOPT_QUOTE); + insint_c(d, "POSTQUOTE", CURLOPT_POSTQUOTE); + insint_c(d, "PREQUOTE", CURLOPT_PREQUOTE); + insint_c(d, "WRITEHEADER", CURLOPT_WRITEHEADER); + insint_c(d, "HEADERFUNCTION", CURLOPT_HEADERFUNCTION); + insint_c(d, "SEEKFUNCTION", CURLOPT_SEEKFUNCTION); + insint_c(d, "COOKIEFILE", CURLOPT_COOKIEFILE); + insint_c(d, "SSLVERSION", CURLOPT_SSLVERSION); + insint_c(d, "TIMECONDITION", CURLOPT_TIMECONDITION); + insint_c(d, "TIMEVALUE", CURLOPT_TIMEVALUE); + insint_c(d, "CUSTOMREQUEST", CURLOPT_CUSTOMREQUEST); + insint_c(d, "STDERR", CURLOPT_STDERR); + insint_c(d, "INTERFACE", CURLOPT_INTERFACE); + insint_c(d, "KRB4LEVEL", CURLOPT_KRB4LEVEL); + insint_c(d, "KRBLEVEL", CURLOPT_KRBLEVEL); + insint_c(d, "PROGRESSFUNCTION", CURLOPT_PROGRESSFUNCTION); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 32, 0) + insint_c(d, "XFERINFOFUNCTION", CURLOPT_XFERINFOFUNCTION); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 20, 0) + insint_c(d, "FTP_USE_PRET", CURLOPT_FTP_USE_PRET); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 34, 0) + insint_c(d, "LOGIN_OPTIONS", CURLOPT_LOGIN_OPTIONS); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 31, 0) + insint_c(d, "SASL_IR", CURLOPT_SASL_IR); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 33, 0) + insint_c(d, "XOAUTH2_BEARER", CURLOPT_XOAUTH2_BEARER); +#endif + insint_c(d, "SSL_VERIFYPEER", CURLOPT_SSL_VERIFYPEER); + insint_c(d, "CAPATH", CURLOPT_CAPATH); + insint_c(d, "CAINFO", CURLOPT_CAINFO); + insint_c(d, "OPT_FILETIME", CURLOPT_FILETIME); + insint_c(d, "MAXREDIRS", CURLOPT_MAXREDIRS); + insint_c(d, "MAXCONNECTS", CURLOPT_MAXCONNECTS); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 65, 0) + insint_c(d, "MAXAGE_CONN", CURLOPT_MAXAGE_CONN); +#endif + insint_c(d, "FRESH_CONNECT", CURLOPT_FRESH_CONNECT); + insint_c(d, "FORBID_REUSE", CURLOPT_FORBID_REUSE); + insint_c(d, "RANDOM_FILE", CURLOPT_RANDOM_FILE); + insint_c(d, "EGDSOCKET", CURLOPT_EGDSOCKET); + insint_c(d, "CONNECTTIMEOUT", CURLOPT_CONNECTTIMEOUT); + insint_c(d, "HTTPGET", CURLOPT_HTTPGET); + insint_c(d, "SSL_VERIFYHOST", CURLOPT_SSL_VERIFYHOST); + insint_c(d, "COOKIEJAR", CURLOPT_COOKIEJAR); + insint_c(d, "SSL_CIPHER_LIST", CURLOPT_SSL_CIPHER_LIST); + insint_c(d, "HTTP_VERSION", CURLOPT_HTTP_VERSION); + insint_c(d, "FTP_USE_EPSV", CURLOPT_FTP_USE_EPSV); + insint_c(d, "SSLCERTTYPE", CURLOPT_SSLCERTTYPE); + insint_c(d, "SSLKEY", CURLOPT_SSLKEY); + insint_c(d, "SSLKEYTYPE", CURLOPT_SSLKEYTYPE); + /* same as CURLOPT_KEYPASSWD */ + insint_c(d, "SSLKEYPASSWD", CURLOPT_SSLKEYPASSWD); + insint_c(d, "SSLENGINE", CURLOPT_SSLENGINE); + insint_c(d, "SSLENGINE_DEFAULT", CURLOPT_SSLENGINE_DEFAULT); + insint_c(d, "DNS_CACHE_TIMEOUT", CURLOPT_DNS_CACHE_TIMEOUT); + insint_c(d, "DNS_USE_GLOBAL_CACHE", CURLOPT_DNS_USE_GLOBAL_CACHE); + insint_c(d, "DEBUGFUNCTION", CURLOPT_DEBUGFUNCTION); + insint_c(d, "BUFFERSIZE", CURLOPT_BUFFERSIZE); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 62, 0) + insint_c(d, "UPLOAD_BUFFERSIZE", CURLOPT_UPLOAD_BUFFERSIZE); +#endif + insint_c(d, "NOSIGNAL", CURLOPT_NOSIGNAL); + insint_c(d, "SHARE", CURLOPT_SHARE); + insint_c(d, "PROXYTYPE", CURLOPT_PROXYTYPE); + /* superseded by ACCEPT_ENCODING */ + insint_c(d, "ENCODING", CURLOPT_ENCODING); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 6) + insint_c(d, "ACCEPT_ENCODING", CURLOPT_ACCEPT_ENCODING); + insint_c(d, "TRANSFER_ENCODING", CURLOPT_TRANSFER_ENCODING); +#endif + insint_c(d, "HTTP200ALIASES", CURLOPT_HTTP200ALIASES); + insint_c(d, "UNRESTRICTED_AUTH", CURLOPT_UNRESTRICTED_AUTH); + insint_c(d, "FTP_USE_EPRT", CURLOPT_FTP_USE_EPRT); + insint_c(d, "HTTPAUTH", CURLOPT_HTTPAUTH); + insint_c(d, "FTP_CREATE_MISSING_DIRS", CURLOPT_FTP_CREATE_MISSING_DIRS); + insint_c(d, "PROXYAUTH", CURLOPT_PROXYAUTH); + insint_c(d, "FTP_RESPONSE_TIMEOUT", CURLOPT_FTP_RESPONSE_TIMEOUT); + insint_c(d, "IPRESOLVE", CURLOPT_IPRESOLVE); + insint_c(d, "MAXFILESIZE", CURLOPT_MAXFILESIZE_LARGE); /* _LARGE ! */ + insint_c(d, "INFILESIZE_LARGE", CURLOPT_INFILESIZE_LARGE); + insint_c(d, "RESUME_FROM_LARGE", CURLOPT_RESUME_FROM_LARGE); + insint_c(d, "MAXFILESIZE_LARGE", CURLOPT_MAXFILESIZE_LARGE); + insint_c(d, "NETRC_FILE", CURLOPT_NETRC_FILE); + insint_c(d, "FTP_SSL", CURLOPT_FTP_SSL); + insint_c(d, "POSTFIELDSIZE_LARGE", CURLOPT_POSTFIELDSIZE_LARGE); + insint_c(d, "TCP_NODELAY", CURLOPT_TCP_NODELAY); + insint_c(d, "FTPSSLAUTH", CURLOPT_FTPSSLAUTH); + insint_c(d, "IOCTLFUNCTION", CURLOPT_IOCTLFUNCTION); + insint_c(d, "OPENSOCKETFUNCTION", CURLOPT_OPENSOCKETFUNCTION); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 7) + insint_c(d, "CLOSESOCKETFUNCTION", CURLOPT_CLOSESOCKETFUNCTION); +#endif + insint_c(d, "SOCKOPTFUNCTION", CURLOPT_SOCKOPTFUNCTION); + insint_c(d, "FTP_ACCOUNT", CURLOPT_FTP_ACCOUNT); + insint_c(d, "IGNORE_CONTENT_LENGTH", CURLOPT_IGNORE_CONTENT_LENGTH); + insint_c(d, "COOKIELIST", CURLOPT_COOKIELIST); + insint_c(d, "OPT_COOKIELIST", CURLOPT_COOKIELIST); + insint_c(d, "FTP_SKIP_PASV_IP", CURLOPT_FTP_SKIP_PASV_IP); + insint_c(d, "FTP_FILEMETHOD", CURLOPT_FTP_FILEMETHOD); + insint_c(d, "CONNECT_ONLY", CURLOPT_CONNECT_ONLY); + insint_c(d, "LOCALPORT", CURLOPT_LOCALPORT); + insint_c(d, "LOCALPORTRANGE", CURLOPT_LOCALPORTRANGE); + insint_c(d, "FTP_ALTERNATIVE_TO_USER", CURLOPT_FTP_ALTERNATIVE_TO_USER); + insint_c(d, "MAX_SEND_SPEED_LARGE", CURLOPT_MAX_SEND_SPEED_LARGE); + insint_c(d, "MAX_RECV_SPEED_LARGE", CURLOPT_MAX_RECV_SPEED_LARGE); + insint_c(d, "SSL_SESSIONID_CACHE", CURLOPT_SSL_SESSIONID_CACHE); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 41, 0) + insint_c(d, "SSL_VERIFYSTATUS", CURLOPT_SSL_VERIFYSTATUS); +#endif + insint_c(d, "SSH_AUTH_TYPES", CURLOPT_SSH_AUTH_TYPES); + insint_c(d, "SSH_PUBLIC_KEYFILE", CURLOPT_SSH_PUBLIC_KEYFILE); + insint_c(d, "SSH_PRIVATE_KEYFILE", CURLOPT_SSH_PRIVATE_KEYFILE); +#ifdef HAVE_CURL_7_19_6_OPTS + insint_c(d, "SSH_KNOWNHOSTS", CURLOPT_SSH_KNOWNHOSTS); + insint_c(d, "SSH_KEYFUNCTION", CURLOPT_SSH_KEYFUNCTION); +#endif + insint_c(d, "FTP_SSL_CCC", CURLOPT_FTP_SSL_CCC); + insint_c(d, "TIMEOUT_MS", CURLOPT_TIMEOUT_MS); + insint_c(d, "CONNECTTIMEOUT_MS", CURLOPT_CONNECTTIMEOUT_MS); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 24, 0) + insint_c(d, "ACCEPTTIMEOUT_MS", CURLOPT_ACCEPTTIMEOUT_MS); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 36, 0) + insint_c(d, "EXPECT_100_TIMEOUT_MS", CURLOPT_EXPECT_100_TIMEOUT_MS); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 25, 0) + insint_c(d, "TCP_KEEPALIVE", CURLOPT_TCP_KEEPALIVE); + insint_c(d, "TCP_KEEPIDLE", CURLOPT_TCP_KEEPIDLE); + insint_c(d, "TCP_KEEPINTVL", CURLOPT_TCP_KEEPINTVL); +#endif + insint_c(d, "HTTP_TRANSFER_DECODING", CURLOPT_HTTP_TRANSFER_DECODING); + insint_c(d, "HTTP_CONTENT_DECODING", CURLOPT_HTTP_CONTENT_DECODING); + insint_c(d, "NEW_FILE_PERMS", CURLOPT_NEW_FILE_PERMS); + insint_c(d, "NEW_DIRECTORY_PERMS", CURLOPT_NEW_DIRECTORY_PERMS); + insint_c(d, "POST301", CURLOPT_POST301); + insint_c(d, "PROXY_TRANSFER_MODE", CURLOPT_PROXY_TRANSFER_MODE); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 43, 0) + insint_c(d, "SERVICE_NAME", CURLOPT_SERVICE_NAME); + insint_c(d, "PROXY_SERVICE_NAME", CURLOPT_PROXY_SERVICE_NAME); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 52, 0) + insint_c(d, "PROXY_CAPATH", CURLOPT_PROXY_CAPATH); + insint_c(d, "PROXY_CAINFO", CURLOPT_PROXY_CAINFO); + insint_c(d, "PRE_PROXY", CURLOPT_PRE_PROXY); + insint_c(d, "PROXY_SSLCERT", CURLOPT_PROXY_SSLCERT); + insint_c(d, "PROXY_SSLCERTTYPE", CURLOPT_PROXY_SSLCERTTYPE); + insint_c(d, "PROXY_SSLKEY", CURLOPT_PROXY_SSLKEY); + insint_c(d, "PROXY_SSLKEYTYPE", CURLOPT_PROXY_SSLKEYTYPE); + insint_c(d, "PROXY_SSL_VERIFYPEER", CURLOPT_PROXY_SSL_VERIFYPEER); + insint_c(d, "PROXY_SSL_VERIFYHOST", CURLOPT_PROXY_SSL_VERIFYHOST); +#endif + insint_c(d, "COPYPOSTFIELDS", CURLOPT_COPYPOSTFIELDS); + insint_c(d, "SSH_HOST_PUBLIC_KEY_MD5", CURLOPT_SSH_HOST_PUBLIC_KEY_MD5); + insint_c(d, "AUTOREFERER", CURLOPT_AUTOREFERER); + insint_c(d, "CRLFILE", CURLOPT_CRLFILE); + insint_c(d, "ISSUERCERT", CURLOPT_ISSUERCERT); + insint_c(d, "ADDRESS_SCOPE", CURLOPT_ADDRESS_SCOPE); +#ifdef HAVE_CURLOPT_RESOLVE + insint_c(d, "RESOLVE", CURLOPT_RESOLVE); +#endif +#ifdef HAVE_CURLOPT_CERTINFO + insint_c(d, "OPT_CERTINFO", CURLOPT_CERTINFO); +#endif +#ifdef HAVE_CURLOPT_POSTREDIR + insint_c(d, "POSTREDIR", CURLOPT_POSTREDIR); +#endif +#ifdef HAVE_CURLOPT_NOPROXY + insint_c(d, "NOPROXY", CURLOPT_NOPROXY); +#endif +#ifdef HAVE_CURLOPT_PROTOCOLS + insint_c(d, "PROTOCOLS", CURLOPT_PROTOCOLS); + insint_c(d, "REDIR_PROTOCOLS", CURLOPT_REDIR_PROTOCOLS); + insint_c(d, "PROTO_HTTP", CURLPROTO_HTTP); + insint_c(d, "PROTO_HTTPS", CURLPROTO_HTTPS); + insint_c(d, "PROTO_FTP", CURLPROTO_FTP); + insint_c(d, "PROTO_FTPS", CURLPROTO_FTPS); + insint_c(d, "PROTO_SCP", CURLPROTO_SCP); + insint_c(d, "PROTO_SFTP", CURLPROTO_SFTP); + insint_c(d, "PROTO_TELNET", CURLPROTO_TELNET); + insint_c(d, "PROTO_LDAP", CURLPROTO_LDAP); + insint_c(d, "PROTO_LDAPS", CURLPROTO_LDAPS); + insint_c(d, "PROTO_DICT", CURLPROTO_DICT); + insint_c(d, "PROTO_FILE", CURLPROTO_FILE); + insint_c(d, "PROTO_TFTP", CURLPROTO_TFTP); +#ifdef HAVE_CURL_7_20_0_OPTS + insint_c(d, "PROTO_IMAP", CURLPROTO_IMAP); + insint_c(d, "PROTO_IMAPS", CURLPROTO_IMAPS); + insint_c(d, "PROTO_POP3", CURLPROTO_POP3); + insint_c(d, "PROTO_POP3S", CURLPROTO_POP3S); + insint_c(d, "PROTO_SMTP", CURLPROTO_SMTP); + insint_c(d, "PROTO_SMTPS", CURLPROTO_SMTPS); +#endif +#ifdef HAVE_CURL_7_21_0_OPTS + insint_c(d, "PROTO_RTSP", CURLPROTO_RTSP); + insint_c(d, "PROTO_RTMP", CURLPROTO_RTMP); + insint_c(d, "PROTO_RTMPT", CURLPROTO_RTMPT); + insint_c(d, "PROTO_RTMPE", CURLPROTO_RTMPE); + insint_c(d, "PROTO_RTMPTE", CURLPROTO_RTMPTE); + insint_c(d, "PROTO_RTMPS", CURLPROTO_RTMPS); + insint_c(d, "PROTO_RTMPTS", CURLPROTO_RTMPTS); +#endif +#ifdef HAVE_CURL_7_21_2_OPTS + insint_c(d, "PROTO_GOPHER", CURLPROTO_GOPHER); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 40, 0) + insint_c(d, "PROTO_SMB", CURLPROTO_SMB); + insint_c(d, "PROTO_SMBS", CURLPROTO_SMBS); +#endif + insint_c(d, "PROTO_ALL", CURLPROTO_ALL); +#endif +#ifdef HAVE_CURL_7_19_4_OPTS + insint_c(d, "TFTP_BLKSIZE", CURLOPT_TFTP_BLKSIZE); + insint_c(d, "SOCKS5_GSSAPI_SERVICE", CURLOPT_SOCKS5_GSSAPI_SERVICE); + insint_c(d, "SOCKS5_GSSAPI_NEC", CURLOPT_SOCKS5_GSSAPI_NEC); +#endif +#ifdef HAVE_CURL_7_20_0_OPTS + insint_c(d, "MAIL_FROM", CURLOPT_MAIL_FROM); + insint_c(d, "MAIL_RCPT", CURLOPT_MAIL_RCPT); +#endif +#ifdef HAVE_CURL_7_25_0_OPTS + insint_c(d, "MAIL_AUTH", CURLOPT_MAIL_AUTH); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 39, 0) + insint_c(d, "PINNEDPUBLICKEY", CURLOPT_PINNEDPUBLICKEY); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 0) + insint_c(d, "WILDCARDMATCH", CURLOPT_WILDCARDMATCH); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 40, 0) + insint_c(d, "UNIX_SOCKET_PATH", CURLOPT_UNIX_SOCKET_PATH); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 36, 0) + insint_c(d, "SSL_ENABLE_ALPN", CURLOPT_SSL_ENABLE_ALPN); + insint_c(d, "SSL_ENABLE_NPN", CURLOPT_SSL_ENABLE_NPN); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 42, 0) + insint_c(d, "SSL_FALSESTART", CURLOPT_SSL_FALSESTART); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 25, 0) + insint_c(d, "SSL_OPTIONS", CURLOPT_SSL_OPTIONS); + insint_c(d, "SSLOPT_ALLOW_BEAST", CURLSSLOPT_ALLOW_BEAST); +# if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 44, 0) + insint_c(d, "SSLOPT_NO_REVOKE", CURLSSLOPT_NO_REVOKE); +# endif +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 4) + insint_c(d, "TLSAUTH_TYPE", CURLOPT_TLSAUTH_TYPE); + insint_c(d, "TLSAUTH_USERNAME", CURLOPT_TLSAUTH_USERNAME); + insint_c(d, "TLSAUTH_PASSWORD", CURLOPT_TLSAUTH_PASSWORD); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 45, 0) + insint_c(d, "DEFAULT_PROTOCOL", CURLOPT_DEFAULT_PROTOCOL); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 61, 0) + insint_c(d, "TLS13_CIPHERS", CURLOPT_TLS13_CIPHERS); + insint_c(d, "PROXY_TLS13_CIPHERS", CURLOPT_PROXY_TLS13_CIPHERS); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 62, 0) + insint_c(d, "DOH_URL", CURLOPT_DOH_URL); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 64, 0) + insint_c(d, "HTTP09_ALLOWED", CURLOPT_HTTP09_ALLOWED); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 80, 0) + insint_c(d, "MAXLIFETIME_CONN", CURLOPT_MAXLIFETIME_CONN); +#endif + + insint_m(d, "M_TIMERFUNCTION", CURLMOPT_TIMERFUNCTION); + insint_m(d, "M_SOCKETFUNCTION", CURLMOPT_SOCKETFUNCTION); + insint_m(d, "M_PIPELINING", CURLMOPT_PIPELINING); + insint_m(d, "M_MAXCONNECTS", CURLMOPT_MAXCONNECTS); +#ifdef HAVE_CURL_7_30_0_PIPELINE_OPTS + insint_m(d, "M_MAX_HOST_CONNECTIONS", CURLMOPT_MAX_HOST_CONNECTIONS); + insint_m(d, "M_MAX_TOTAL_CONNECTIONS", CURLMOPT_MAX_TOTAL_CONNECTIONS); + insint_m(d, "M_MAX_PIPELINE_LENGTH", CURLMOPT_MAX_PIPELINE_LENGTH); + insint_m(d, "M_CONTENT_LENGTH_PENALTY_SIZE", CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE); + insint_m(d, "M_CHUNK_LENGTH_PENALTY_SIZE", CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE); + insint_m(d, "M_PIPELINING_SITE_BL", CURLMOPT_PIPELINING_SITE_BL); + insint_m(d, "M_PIPELINING_SERVER_BL", CURLMOPT_PIPELINING_SERVER_BL); +#endif +#ifdef HAVE_CURL_7_67_0_MULTI_STREAMS + insint_m(d, "M_MAX_CONCURRENT_STREAMS", CURLMOPT_MAX_CONCURRENT_STREAMS); +#endif + +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 43, 0) + insint_m(d, "PIPE_NOTHING", CURLPIPE_NOTHING); + insint_m(d, "PIPE_HTTP1", CURLPIPE_HTTP1); + insint_m(d, "PIPE_MULTIPLEX", CURLPIPE_MULTIPLEX); +#endif + + /* constants for setopt(IPRESOLVE, x) */ + insint_c(d, "IPRESOLVE_WHATEVER", CURL_IPRESOLVE_WHATEVER); + insint_c(d, "IPRESOLVE_V4", CURL_IPRESOLVE_V4); + insint_c(d, "IPRESOLVE_V6", CURL_IPRESOLVE_V6); + + /* constants for setopt(HTTP_VERSION, x) */ + insint_c(d, "CURL_HTTP_VERSION_NONE", CURL_HTTP_VERSION_NONE); + insint_c(d, "CURL_HTTP_VERSION_1_0", CURL_HTTP_VERSION_1_0); + insint_c(d, "CURL_HTTP_VERSION_1_1", CURL_HTTP_VERSION_1_1); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 33, 0) + insint_c(d, "CURL_HTTP_VERSION_2_0", CURL_HTTP_VERSION_2_0); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 43, 0) + insint_c(d, "CURL_HTTP_VERSION_2", CURL_HTTP_VERSION_2); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 47, 0) + insint_c(d, "CURL_HTTP_VERSION_2TLS", CURL_HTTP_VERSION_2TLS); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 49, 0) + insint_c(d, "CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE", CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE); + insint_c(d, "TCP_FASTOPEN", CURLOPT_TCP_FASTOPEN); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 66, 0) + insint_c(d, "CURL_HTTP_VERSION_3", CURL_HTTP_VERSION_3); +#endif + insint_c(d, "CURL_HTTP_VERSION_LAST", CURL_HTTP_VERSION_LAST); + + /* CURL_NETRC_OPTION: constants for setopt(NETRC, x) */ + insint_c(d, "NETRC_OPTIONAL", CURL_NETRC_OPTIONAL); + insint_c(d, "NETRC_IGNORED", CURL_NETRC_IGNORED); + insint_c(d, "NETRC_REQUIRED", CURL_NETRC_REQUIRED); + + /* constants for setopt(SSLVERSION, x) */ + insint_c(d, "SSLVERSION_DEFAULT", CURL_SSLVERSION_DEFAULT); + insint_c(d, "SSLVERSION_SSLv2", CURL_SSLVERSION_SSLv2); + insint_c(d, "SSLVERSION_SSLv3", CURL_SSLVERSION_SSLv3); + insint_c(d, "SSLVERSION_TLSv1", CURL_SSLVERSION_TLSv1); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 34, 0) + insint_c(d, "SSLVERSION_TLSv1_0", CURL_SSLVERSION_TLSv1_0); + insint_c(d, "SSLVERSION_TLSv1_1", CURL_SSLVERSION_TLSv1_1); + insint_c(d, "SSLVERSION_TLSv1_2", CURL_SSLVERSION_TLSv1_2); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 52, 0) + insint_c(d, "SSLVERSION_TLSv1_3", CURL_SSLVERSION_TLSv1_3); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 54, 0) + insint_c(d, "SSLVERSION_MAX_DEFAULT", CURL_SSLVERSION_MAX_DEFAULT); + insint_c(d, "SSLVERSION_MAX_TLSv1_0", CURL_SSLVERSION_MAX_TLSv1_0); + insint_c(d, "SSLVERSION_MAX_TLSv1_1", CURL_SSLVERSION_MAX_TLSv1_1); + insint_c(d, "SSLVERSION_MAX_TLSv1_2", CURL_SSLVERSION_MAX_TLSv1_2); + insint_c(d, "SSLVERSION_MAX_TLSv1_3", CURL_SSLVERSION_MAX_TLSv1_3); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 60, 0) + insint_c(d, "HAPROXYPROTOCOL", CURLOPT_HAPROXYPROTOCOL); +#endif + + /* curl_TimeCond: constants for setopt(TIMECONDITION, x) */ + insint_c(d, "TIMECONDITION_NONE", CURL_TIMECOND_NONE); + insint_c(d, "TIMECONDITION_IFMODSINCE", CURL_TIMECOND_IFMODSINCE); + insint_c(d, "TIMECONDITION_IFUNMODSINCE", CURL_TIMECOND_IFUNMODSINCE); + insint_c(d, "TIMECONDITION_LASTMOD", CURL_TIMECOND_LASTMOD); + + /* constants for setopt(CURLOPT_SSH_AUTH_TYPES, x) */ + insint_c(d, "SSH_AUTH_ANY", CURLSSH_AUTH_ANY); + insint_c(d, "SSH_AUTH_NONE", CURLSSH_AUTH_NONE); + insint_c(d, "SSH_AUTH_PUBLICKEY", CURLSSH_AUTH_PUBLICKEY); + insint_c(d, "SSH_AUTH_PASSWORD", CURLSSH_AUTH_PASSWORD); + insint_c(d, "SSH_AUTH_HOST", CURLSSH_AUTH_HOST); + insint_c(d, "SSH_AUTH_KEYBOARD", CURLSSH_AUTH_KEYBOARD); + insint_c(d, "SSH_AUTH_DEFAULT", CURLSSH_AUTH_DEFAULT); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 28, 0) + insint_c(d, "SSH_AUTH_AGENT", CURLSSH_AUTH_AGENT); +#endif + +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 37, 0) + insint_c(d, "HEADER_UNIFIED", CURLHEADER_UNIFIED); + insint_c(d, "HEADER_SEPARATE", CURLHEADER_SEPARATE); +#endif + +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 5) + insint_c(d, "SOCKOPT_ALREADY_CONNECTED", CURL_SOCKOPT_ALREADY_CONNECTED); + insint_c(d, "SOCKOPT_ERROR", CURL_SOCKOPT_ERROR); + insint_c(d, "SOCKOPT_OK", CURL_SOCKOPT_OK); +#endif + +#ifdef HAVE_CURL_7_19_6_OPTS + /* curl_khtype constants */ + insint_c(d, "KHTYPE_UNKNOWN", CURLKHTYPE_UNKNOWN); + insint_c(d, "KHTYPE_RSA1", CURLKHTYPE_RSA1); + insint_c(d, "KHTYPE_RSA", CURLKHTYPE_RSA); + insint_c(d, "KHTYPE_DSS", CURLKHTYPE_DSS); + + /* curl_khmatch constants, passed to sshkeycallback */ + insint_c(d, "KHMATCH_OK", CURLKHMATCH_OK); + insint_c(d, "KHMATCH_MISMATCH", CURLKHMATCH_MISMATCH); + insint_c(d, "KHMATCH_MISSING", CURLKHMATCH_MISSING); + + /* return values for CURLOPT_SSH_KEYFUNCTION */ + insint_c(d, "KHSTAT_FINE_ADD_TO_FILE", CURLKHSTAT_FINE_ADD_TO_FILE); + insint_c(d, "KHSTAT_FINE", CURLKHSTAT_FINE); + insint_c(d, "KHSTAT_REJECT", CURLKHSTAT_REJECT); + insint_c(d, "KHSTAT_DEFER", CURLKHSTAT_DEFER); +#endif + +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 28, 0) + insint_c(d, "SOCKTYPE_ACCEPT", CURLSOCKTYPE_ACCEPT); +#endif + insint_c(d, "SOCKTYPE_IPCXN", CURLSOCKTYPE_IPCXN); + + insint_c(d, "USESSL_NONE", CURLUSESSL_NONE); + insint_c(d, "USESSL_TRY", CURLUSESSL_TRY); + insint_c(d, "USESSL_CONTROL", CURLUSESSL_CONTROL); + insint_c(d, "USESSL_ALL", CURLUSESSL_ALL); + + /* CURLINFO: symbolic constants for getinfo(x) */ + insint_c(d, "EFFECTIVE_URL", CURLINFO_EFFECTIVE_URL); + /* same as CURLINFO_RESPONSE_CODE */ + insint_c(d, "HTTP_CODE", CURLINFO_HTTP_CODE); + insint_c(d, "RESPONSE_CODE", CURLINFO_RESPONSE_CODE); + insint_c(d, "TOTAL_TIME", CURLINFO_TOTAL_TIME); + insint_c(d, "NAMELOOKUP_TIME", CURLINFO_NAMELOOKUP_TIME); + insint_c(d, "CONNECT_TIME", CURLINFO_CONNECT_TIME); + insint_c(d, "APPCONNECT_TIME", CURLINFO_APPCONNECT_TIME); + insint_c(d, "PRETRANSFER_TIME", CURLINFO_PRETRANSFER_TIME); + insint_c(d, "SIZE_UPLOAD", CURLINFO_SIZE_UPLOAD); + insint_c(d, "SIZE_DOWNLOAD", CURLINFO_SIZE_DOWNLOAD); + insint_c(d, "SPEED_DOWNLOAD", CURLINFO_SPEED_DOWNLOAD); + insint_c(d, "SPEED_UPLOAD", CURLINFO_SPEED_UPLOAD); + insint_c(d, "HEADER_SIZE", CURLINFO_HEADER_SIZE); + insint_c(d, "REQUEST_SIZE", CURLINFO_REQUEST_SIZE); + insint_c(d, "SSL_VERIFYRESULT", CURLINFO_SSL_VERIFYRESULT); + insint_c(d, "INFO_FILETIME", CURLINFO_FILETIME); + insint_c(d, "CONTENT_LENGTH_DOWNLOAD", CURLINFO_CONTENT_LENGTH_DOWNLOAD); + insint_c(d, "CONTENT_LENGTH_UPLOAD", CURLINFO_CONTENT_LENGTH_UPLOAD); + insint_c(d, "STARTTRANSFER_TIME", CURLINFO_STARTTRANSFER_TIME); + insint_c(d, "CONTENT_TYPE", CURLINFO_CONTENT_TYPE); + insint_c(d, "REDIRECT_TIME", CURLINFO_REDIRECT_TIME); + insint_c(d, "REDIRECT_COUNT", CURLINFO_REDIRECT_COUNT); + insint_c(d, "REDIRECT_URL", CURLINFO_REDIRECT_URL); + insint_c(d, "PRIMARY_IP", CURLINFO_PRIMARY_IP); +#ifdef HAVE_CURLINFO_PRIMARY_PORT + insint_c(d, "PRIMARY_PORT", CURLINFO_PRIMARY_PORT); +#endif +#ifdef HAVE_CURLINFO_LOCAL_IP + insint_c(d, "LOCAL_IP", CURLINFO_LOCAL_IP); +#endif +#ifdef HAVE_CURLINFO_LOCAL_PORT + insint_c(d, "LOCAL_PORT", CURLINFO_LOCAL_PORT); +#endif + insint_c(d, "HTTP_CONNECTCODE", CURLINFO_HTTP_CONNECTCODE); + insint_c(d, "HTTPAUTH_AVAIL", CURLINFO_HTTPAUTH_AVAIL); + insint_c(d, "PROXYAUTH_AVAIL", CURLINFO_PROXYAUTH_AVAIL); + insint_c(d, "OS_ERRNO", CURLINFO_OS_ERRNO); + insint_c(d, "NUM_CONNECTS", CURLINFO_NUM_CONNECTS); + insint_c(d, "SSL_ENGINES", CURLINFO_SSL_ENGINES); + insint_c(d, "INFO_COOKIELIST", CURLINFO_COOKIELIST); + insint_c(d, "LASTSOCKET", CURLINFO_LASTSOCKET); + insint_c(d, "FTP_ENTRY_PATH", CURLINFO_FTP_ENTRY_PATH); +#ifdef HAVE_CURLOPT_CERTINFO + insint_c(d, "INFO_CERTINFO", CURLINFO_CERTINFO); +#endif +#ifdef HAVE_CURL_7_19_4_OPTS + insint_c(d, "CONDITION_UNMET", CURLINFO_CONDITION_UNMET); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 20, 0) + insint_c(d, "INFO_RTSP_CLIENT_CSEQ", CURLINFO_RTSP_CLIENT_CSEQ); + insint_c(d, "INFO_RTSP_CSEQ_RECV", CURLINFO_RTSP_CSEQ_RECV); + insint_c(d, "INFO_RTSP_SERVER_CSEQ", CURLINFO_RTSP_SERVER_CSEQ); + insint_c(d, "INFO_RTSP_SESSION_ID", CURLINFO_RTSP_SESSION_ID); + insint_c(d, "RTSPREQ_NONE",CURL_RTSPREQ_NONE); + insint_c(d, "RTSPREQ_OPTIONS",CURL_RTSPREQ_OPTIONS); + insint_c(d, "RTSPREQ_DESCRIBE",CURL_RTSPREQ_DESCRIBE); + insint_c(d, "RTSPREQ_ANNOUNCE",CURL_RTSPREQ_ANNOUNCE); + insint_c(d, "RTSPREQ_SETUP",CURL_RTSPREQ_SETUP); + insint_c(d, "RTSPREQ_PLAY",CURL_RTSPREQ_PLAY); + insint_c(d, "RTSPREQ_PAUSE",CURL_RTSPREQ_PAUSE); + insint_c(d, "RTSPREQ_TEARDOWN",CURL_RTSPREQ_TEARDOWN); + insint_c(d, "RTSPREQ_GET_PARAMETER",CURL_RTSPREQ_GET_PARAMETER); + insint_c(d, "RTSPREQ_SET_PARAMETER",CURL_RTSPREQ_SET_PARAMETER); + insint_c(d, "RTSPREQ_RECORD",CURL_RTSPREQ_RECORD); + insint_c(d, "RTSPREQ_RECEIVE",CURL_RTSPREQ_RECEIVE); + insint_c(d, "RTSPREQ_LAST",CURL_RTSPREQ_LAST); +#endif + + /* CURLPAUSE: symbolic constants for pause(bitmask) */ + insint_c(d, "PAUSE_RECV", CURLPAUSE_RECV); + insint_c(d, "PAUSE_SEND", CURLPAUSE_SEND); + insint_c(d, "PAUSE_ALL", CURLPAUSE_ALL); + insint_c(d, "PAUSE_CONT", CURLPAUSE_CONT); + +#ifdef HAVE_CURL_7_19_5_OPTS + /* CURL_SEEKFUNC: return values for seek function */ + insint_c(d, "SEEKFUNC_OK", CURL_SEEKFUNC_OK); + insint_c(d, "SEEKFUNC_FAIL", CURL_SEEKFUNC_FAIL); + insint_c(d, "SEEKFUNC_CANTSEEK", CURL_SEEKFUNC_CANTSEEK); +#endif + +#ifdef HAVE_CURLOPT_DNS_SERVERS + insint_c(d, "DNS_SERVERS", CURLOPT_DNS_SERVERS); +#endif + +#ifdef HAVE_CURLOPT_POSTREDIR + insint_c(d, "REDIR_POST_301", CURL_REDIR_POST_301); + insint_c(d, "REDIR_POST_302", CURL_REDIR_POST_302); +# ifdef HAVE_CURL_REDIR_POST_303 + insint_c(d, "REDIR_POST_303", CURL_REDIR_POST_303); +# endif + insint_c(d, "REDIR_POST_ALL", CURL_REDIR_POST_ALL); +#endif + +#ifdef HAVE_CURLOPT_CONNECT_TO + insint_c(d, "CONNECT_TO", CURLOPT_CONNECT_TO); +#endif + +#ifdef HAVE_CURLINFO_HTTP_VERSION + insint_c(d, "INFO_HTTP_VERSION", CURLINFO_HTTP_VERSION); +#endif + + /* options for global_init() */ + insint(d, "GLOBAL_SSL", CURL_GLOBAL_SSL); + insint(d, "GLOBAL_WIN32", CURL_GLOBAL_WIN32); + insint(d, "GLOBAL_ALL", CURL_GLOBAL_ALL); + insint(d, "GLOBAL_NOTHING", CURL_GLOBAL_NOTHING); + insint(d, "GLOBAL_DEFAULT", CURL_GLOBAL_DEFAULT); +#ifdef CURL_GLOBAL_ACK_EINTR + /* CURL_GLOBAL_ACK_EINTR was introduced in libcurl-7.30.0 */ + insint(d, "GLOBAL_ACK_EINTR", CURL_GLOBAL_ACK_EINTR); +#endif + + + /* constants for curl_multi_socket interface */ + insint(d, "CSELECT_IN", CURL_CSELECT_IN); + insint(d, "CSELECT_OUT", CURL_CSELECT_OUT); + insint(d, "CSELECT_ERR", CURL_CSELECT_ERR); + insint(d, "SOCKET_TIMEOUT", CURL_SOCKET_TIMEOUT); + insint(d, "POLL_NONE", CURL_POLL_NONE); + insint(d, "POLL_IN", CURL_POLL_IN); + insint(d, "POLL_OUT", CURL_POLL_OUT); + insint(d, "POLL_INOUT", CURL_POLL_INOUT); + insint(d, "POLL_REMOVE", CURL_POLL_REMOVE); + + /* curl_lock_data: XXX do we need this in pycurl ??? */ + /* curl_lock_access: XXX do we need this in pycurl ??? */ + /* CURLSHcode: XXX do we need this in pycurl ??? */ + /* CURLSHoption: XXX do we need this in pycurl ??? */ + + /* CURLversion: constants for curl_version_info(x) */ +#if 0 + /* XXX - do we need these ?? */ + insint(d, "VERSION_FIRST", CURLVERSION_FIRST); + insint(d, "VERSION_SECOND", CURLVERSION_SECOND); + insint(d, "VERSION_THIRD", CURLVERSION_THIRD); + insint(d, "VERSION_NOW", CURLVERSION_NOW); +#endif + + /* version features - bitmasks for curl_version_info_data.features */ + insint(d, "VERSION_IPV6", CURL_VERSION_IPV6); + insint(d, "VERSION_KERBEROS4", CURL_VERSION_KERBEROS4); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 40, 0) + insint(d, "VERSION_KERBEROS5", CURL_VERSION_KERBEROS5); +#endif + insint(d, "VERSION_SSL", CURL_VERSION_SSL); + insint(d, "VERSION_LIBZ", CURL_VERSION_LIBZ); + insint(d, "VERSION_NTLM", CURL_VERSION_NTLM); + insint(d, "VERSION_GSSNEGOTIATE", CURL_VERSION_GSSNEGOTIATE); + insint(d, "VERSION_DEBUG", CURL_VERSION_DEBUG); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 19, 6) + insint(d, "VERSION_CURLDEBUG", CURL_VERSION_CURLDEBUG); +#endif + insint(d, "VERSION_ASYNCHDNS", CURL_VERSION_ASYNCHDNS); + insint(d, "VERSION_SPNEGO", CURL_VERSION_SPNEGO); + insint(d, "VERSION_LARGEFILE", CURL_VERSION_LARGEFILE); + insint(d, "VERSION_IDN", CURL_VERSION_IDN); + insint(d, "VERSION_SSPI", CURL_VERSION_SSPI); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 38, 0) + insint(d, "VERSION_GSSAPI", CURL_VERSION_GSSAPI); +#endif + insint(d, "VERSION_CONV", CURL_VERSION_CONV); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 4) + insint(d, "VERSION_TLSAUTH_SRP", CURL_VERSION_TLSAUTH_SRP); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 22, 0) + insint(d, "VERSION_NTLM_WB", CURL_VERSION_NTLM_WB); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 33, 0) + insint(d, "VERSION_HTTP2", CURL_VERSION_HTTP2); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 40, 0) + insint(d, "VERSION_UNIX_SOCKETS", CURL_VERSION_UNIX_SOCKETS); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 47, 0) + insint(d, "VERSION_PSL", CURL_VERSION_PSL); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 52, 0) + insint(d, "CURL_VERSION_HTTPS_PROXY", CURL_VERSION_HTTPS_PROXY); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 56, 0) + insint(d, "CURL_VERSION_MULTI_SSL", CURL_VERSION_MULTI_SSL); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 57, 0) + insint(d, "CURL_VERSION_BROTLI", CURL_VERSION_BROTLI); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 64, 1) + insint(d, "CURL_VERSION_ALTSVC", CURL_VERSION_ALTSVC); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 66, 0) + insint(d, "CURL_VERSION_HTTP3", CURL_VERSION_HTTP3); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 72, 0) + insint(d, "CURL_VERSION_UNICODE", CURL_VERSION_UNICODE); + insint(d, "CURL_VERSION_ZSTD", CURL_VERSION_ZSTD); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 74, 0) + insint(d, "CURL_VERSION_HSTS", CURL_VERSION_HSTS); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 76, 0) + insint(d, "CURL_VERSION_GSASL", CURL_VERSION_GSASL); +#endif + + /** + ** the order of these constants mostly follows + **/ + + /* CURLMcode: multi error codes */ + /* old symbol */ + insint_m(d, "E_CALL_MULTI_PERFORM", CURLM_CALL_MULTI_PERFORM); + /* new symbol for consistency */ + insint_m(d, "E_MULTI_CALL_MULTI_PERFORM", CURLM_CALL_MULTI_PERFORM); + insint_m(d, "E_MULTI_OK", CURLM_OK); + insint_m(d, "E_MULTI_BAD_HANDLE", CURLM_BAD_HANDLE); + insint_m(d, "E_MULTI_BAD_EASY_HANDLE", CURLM_BAD_EASY_HANDLE); + insint_m(d, "E_MULTI_BAD_SOCKET", CURLM_BAD_SOCKET); + insint_m(d, "E_MULTI_CALL_MULTI_SOCKET", CURLM_CALL_MULTI_SOCKET); + insint_m(d, "E_MULTI_OUT_OF_MEMORY", CURLM_OUT_OF_MEMORY); + insint_m(d, "E_MULTI_INTERNAL_ERROR", CURLM_INTERNAL_ERROR); + insint_m(d, "E_MULTI_UNKNOWN_OPTION", CURLM_UNKNOWN_OPTION); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 32, 1) + insint_m(d, "E_MULTI_ADDED_ALREADY", CURLM_ADDED_ALREADY); +#endif + /* curl shared constants */ + insint_s(d, "SH_SHARE", CURLSHOPT_SHARE); + insint_s(d, "SH_UNSHARE", CURLSHOPT_UNSHARE); + + insint_s(d, "LOCK_DATA_COOKIE", CURL_LOCK_DATA_COOKIE); + insint_s(d, "LOCK_DATA_DNS", CURL_LOCK_DATA_DNS); + insint_s(d, "LOCK_DATA_SSL_SESSION", CURL_LOCK_DATA_SSL_SESSION); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 57, 0) + insint_s(d, "LOCK_DATA_CONNECT", CURL_LOCK_DATA_CONNECT); +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 61, 0) + insint_s(d, "LOCK_DATA_PSL", CURL_LOCK_DATA_PSL); +#endif + + /* Initialize callback locks if ssl is enabled */ +#if defined(PYCURL_NEED_SSL_TSL) + if (pycurl_ssl_init() != 0) { + goto error; + } +#endif + +#if PY_MAJOR_VERSION >= 3 + xio_module = PyImport_ImportModule("io"); + if (xio_module == NULL) { + goto error; + } + bytesio = PyObject_GetAttrString(xio_module, "BytesIO"); + if (bytesio == NULL) { + goto error; + } + stringio = PyObject_GetAttrString(xio_module, "StringIO"); + if (stringio == NULL) { + goto error; + } +#else + xio_module = PyImport_ImportModule("cStringIO"); + if (xio_module == NULL) { + PyErr_Clear(); + xio_module = PyImport_ImportModule("StringIO"); + if (xio_module == NULL) { + goto error; + } + } + stringio = PyObject_GetAttrString(xio_module, "StringIO"); + if (stringio == NULL) { + goto error; + } + bytesio = stringio; + Py_INCREF(bytesio); +#endif + + collections_module = PyImport_ImportModule("collections"); + if (collections_module == NULL) { + goto error; + } + named_tuple = PyObject_GetAttrString(collections_module, "namedtuple"); + if (named_tuple == NULL) { + goto error; + } +#ifdef HAVE_CURL_7_19_6_OPTS + arglist = Py_BuildValue("ss", "KhKey", "key keytype"); + if (arglist == NULL) { + goto error; + } + khkey_type = PyObject_Call(named_tuple, arglist, NULL); + if (khkey_type == NULL) { + goto error; + } + Py_DECREF(arglist); + PyDict_SetItemString(d, "KhKey", khkey_type); +#endif + + arglist = Py_BuildValue("ss", "CurlSockAddr", "family socktype protocol addr"); + if (arglist == NULL) { + goto error; + } + curl_sockaddr_type = PyObject_Call(named_tuple, arglist, NULL); + if (curl_sockaddr_type == NULL) { + goto error; + } + Py_DECREF(arglist); + PyDict_SetItemString(d, "CurlSockAddr", curl_sockaddr_type); + +#if defined(WITH_THREAD) && (PY_MAJOR_VERSION < 3 || PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 9) + /* Finally initialize global interpreter lock */ + PyEval_InitThreads(); +#endif + +#if PY_MAJOR_VERSION >= 3 + return m; +#else + PYCURL_MODINIT_RETURN_NULL; +#endif + +error: + Py_XDECREF(curlobject_constants); + Py_XDECREF(curlmultiobject_constants); + Py_XDECREF(curlshareobject_constants); + Py_XDECREF(ErrorObject); + Py_XDECREF(collections_module); + Py_XDECREF(named_tuple); + Py_XDECREF(xio_module); + Py_XDECREF(bytesio); + Py_XDECREF(stringio); + Py_XDECREF(arglist); +#ifdef HAVE_CURL_7_19_6_OPTS + Py_XDECREF(khkey_type); + Py_XDECREF(curl_sockaddr_type); +#endif + PyMem_Free(g_pycurl_useragent); + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_ImportError, "curl module init failed"); + PYCURL_MODINIT_RETURN_NULL; +} diff --git a/src/multi.c b/src/multi.c new file mode 100644 index 0000000..3dbc3fc --- /dev/null +++ b/src/multi.c @@ -0,0 +1,1081 @@ +#include "pycurl.h" +#include "docstrings.h" + +/************************************************************************* +// static utility functions +**************************************************************************/ + + +/* assert some CurlMultiObject invariants */ +static void +assert_multi_state(const CurlMultiObject *self) +{ + assert(self != NULL); + assert(PyObject_IsInstance((PyObject *) self, (PyObject *) p_CurlMulti_Type) == 1); +#ifdef WITH_THREAD + if (self->state != NULL) { + assert(self->multi_handle != NULL); + } +#endif +} + + +static int +check_multi_state(const CurlMultiObject *self, int flags, const char *name) +{ + assert_multi_state(self); + if ((flags & 1) && self->multi_handle == NULL) { + PyErr_Format(ErrorObject, "cannot invoke %s() - no multi handle", name); + return -1; + } +#ifdef WITH_THREAD + if ((flags & 2) && self->state != NULL) { + PyErr_Format(ErrorObject, "cannot invoke %s() - multi_perform() is currently running", name); + return -1; + } +#endif + return 0; +} + + +/************************************************************************* +// CurlMultiObject +**************************************************************************/ + +/* --------------- construct/destruct (i.e. open/close) --------------- */ + +/* constructor */ +PYCURL_INTERNAL CurlMultiObject * +do_multi_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) +{ + CurlMultiObject *self; + int *ptr; + + if (subtype == p_CurlMulti_Type && !PyArg_ParseTupleAndKeywords(args, kwds, "", empty_keywords)) { + return NULL; + } + + /* Allocate python curl-multi object */ + self = (CurlMultiObject *) subtype->tp_alloc(subtype, 0); + if (!self) { + return NULL; + } + + /* tp_alloc is expected to return zeroed memory */ + for (ptr = (int *) &self->dict; + ptr < (int *) (((char *) self) + sizeof(CurlMultiObject)); + ++ptr) + assert(*ptr == 0); + + self->easy_object_dict = PyDict_New(); + if (self->easy_object_dict == NULL) { + Py_DECREF(self); + return NULL; + } + + /* Allocate libcurl multi handle */ + self->multi_handle = curl_multi_init(); + if (self->multi_handle == NULL) { + Py_DECREF(self); + PyErr_SetString(ErrorObject, "initializing curl-multi failed"); + return NULL; + } + return self; +} + +static void +util_multi_close(CurlMultiObject *self) +{ + assert(self != NULL); + +#ifdef WITH_THREAD + self->state = NULL; +#endif + + if (self->multi_handle != NULL) { + CURLM *multi_handle = self->multi_handle; + /* Allow threads because callbacks can be invoked */ + PYCURL_BEGIN_ALLOW_THREADS + curl_multi_cleanup(multi_handle); + PYCURL_END_ALLOW_THREADS + self->multi_handle = NULL; + } +} + + +static void +util_multi_xdecref(CurlMultiObject *self) +{ + Py_CLEAR(self->easy_object_dict); + Py_CLEAR(self->dict); + Py_CLEAR(self->t_cb); + Py_CLEAR(self->s_cb); +} + + +PYCURL_INTERNAL void +do_multi_dealloc(CurlMultiObject *self) +{ + PyObject_GC_UnTrack(self); + CPy_TRASHCAN_BEGIN(self, do_multi_dealloc); + + util_multi_xdecref(self); + util_multi_close(self); + + if (self->weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject *) self); + } + + CurlMulti_Type.tp_free(self); + CPy_TRASHCAN_END(self); +} + + +static PyObject * +do_multi_close(CurlMultiObject *self) +{ + if (check_multi_state(self, 2, "close") != 0) { + return NULL; + } + util_multi_close(self); + Py_RETURN_NONE; +} + + +/* --------------- GC support --------------- */ + +/* Drop references that may have created reference cycles. */ +PYCURL_INTERNAL int +do_multi_clear(CurlMultiObject *self) +{ + util_multi_xdecref(self); + return 0; +} + +PYCURL_INTERNAL int +do_multi_traverse(CurlMultiObject *self, visitproc visit, void *arg) +{ + int err; +#undef VISIT +#define VISIT(v) if ((v) != NULL && ((err = visit(v, arg)) != 0)) return err + + VISIT(self->dict); + VISIT(self->easy_object_dict); + + return 0; +#undef VISIT +} + + +/* --------------- setopt --------------- */ + +static int +multi_socket_callback(CURL *easy, + curl_socket_t s, + int what, + void *userp, + void *socketp) +{ + CurlMultiObject *self; + PyObject *arglist; + PyObject *result = NULL; + PYCURL_DECLARE_THREAD_STATE; + + /* acquire thread */ + self = (CurlMultiObject *)userp; + if (!PYCURL_ACQUIRE_THREAD_MULTI()) { + PyGILState_STATE tmp_warn_state = PyGILState_Ensure(); + PyErr_WarnEx(PyExc_RuntimeWarning, "multi_socket_callback failed to acquire thread", 1); + PyGILState_Release(tmp_warn_state); + return 0; + } + + /* check args */ + if (self->s_cb == NULL) + goto silent_error; + + if (socketp == NULL) { + Py_INCREF(Py_None); + socketp = Py_None; + } + + /* run callback */ + arglist = Py_BuildValue("(iiOO)", what, s, userp, (PyObject *)socketp); + if (arglist == NULL) + goto verbose_error; + result = PyObject_Call(self->s_cb, arglist, NULL); + Py_DECREF(arglist); + if (result == NULL) + goto verbose_error; + + /* return values from socket callbacks should be ignored */ + +silent_error: + Py_XDECREF(result); + PYCURL_RELEASE_THREAD(); + return 0; +verbose_error: + PyErr_Print(); + goto silent_error; + return 0; +} + + +static int +multi_timer_callback(CURLM *multi, + long timeout_ms, + void *userp) +{ + CurlMultiObject *self; + PyObject *arglist; + PyObject *result = NULL; + int ret = 0; /* always success */ + PYCURL_DECLARE_THREAD_STATE; + + UNUSED(multi); + + /* acquire thread */ + self = (CurlMultiObject *)userp; + if (!PYCURL_ACQUIRE_THREAD_MULTI()) { + PyGILState_STATE tmp_warn_state = PyGILState_Ensure(); + PyErr_WarnEx(PyExc_RuntimeWarning, "multi_timer_callback failed to acquire thread", 1); + PyGILState_Release(tmp_warn_state); + return ret; + } + + /* check args */ + if (self->t_cb == NULL) + goto silent_error; + + /* run callback */ + arglist = Py_BuildValue("(i)", timeout_ms); + if (arglist == NULL) + goto verbose_error; + result = PyObject_Call(self->t_cb, arglist, NULL); + Py_DECREF(arglist); + if (result == NULL) + goto verbose_error; + + /* return values from timer callbacks should be ignored */ + +silent_error: + Py_XDECREF(result); + PYCURL_RELEASE_THREAD(); + return ret; +verbose_error: + PyErr_Print(); + goto silent_error; + + return 0; +} + + +static PyObject * +do_multi_setopt_int(CurlMultiObject *self, int option, PyObject *obj) +{ + long d = PyInt_AsLong(obj); + switch(option) { + case CURLMOPT_MAXCONNECTS: + case CURLMOPT_PIPELINING: +#ifdef HAVE_CURL_7_30_0_PIPELINE_OPTS + case CURLMOPT_MAX_HOST_CONNECTIONS: + case CURLMOPT_MAX_TOTAL_CONNECTIONS: + case CURLMOPT_MAX_PIPELINE_LENGTH: + case CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE: + case CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE: +#endif +#ifdef HAVE_CURL_7_67_0_MULTI_STREAMS + case CURLMOPT_MAX_CONCURRENT_STREAMS: +#endif + curl_multi_setopt(self->multi_handle, option, d); + break; + default: + PyErr_SetString(PyExc_TypeError, "integers are not supported for this option"); + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject * +do_multi_setopt_charpp(CurlMultiObject *self, int option, int which, PyObject *obj) +{ + Py_ssize_t len, i; + int res; + static const char *empty_list[] = { NULL }; + char **list = NULL; + PyObject **encoded_objs = NULL; + PyObject *encoded_obj = NULL; + char *encoded_str; + PyObject *rv = NULL; + + len = PyListOrTuple_Size(obj, which); + if (len == 0) { + res = curl_multi_setopt(self->multi_handle, option, empty_list); + if (res != CURLE_OK) { + CURLERROR_RETVAL_MULTI_DONE(); + } + Py_RETURN_NONE; + } + + /* add NULL terminator as the last list item */ + list = PyMem_New(char *, len+1); + if (list == NULL) { + PyErr_NoMemory(); + return NULL; + } + /* no need for the NULL terminator here */ + encoded_objs = PyMem_New(PyObject *, len); + if (encoded_objs == NULL) { + PyErr_NoMemory(); + goto done; + } + memset(encoded_objs, 0, sizeof(PyObject *) * len); + + for (i = 0; i < len; i++) { + PyObject *listitem = PyListOrTuple_GetItem(obj, i, which); + if (!PyText_Check(listitem)) { + PyErr_SetString(ErrorObject, "list/tuple items must be strings"); + goto done; + } + encoded_str = PyText_AsString_NoNUL(listitem, &encoded_obj); + if (encoded_str == NULL) { + goto done; + } + list[i] = encoded_str; + encoded_objs[i] = encoded_obj; + } + list[len] = NULL; + + res = curl_multi_setopt(self->multi_handle, option, list); + if (res != CURLE_OK) { + rv = NULL; + CURLERROR_RETVAL_MULTI_DONE(); + } + + rv = Py_None; +done: + if (encoded_objs) { + for (i = 0; i < len; i++) { + Py_XDECREF(encoded_objs[i]); + } + PyMem_Free(encoded_objs); + } + PyMem_Free(list); + return rv; +} + + +static PyObject * +do_multi_setopt_list(CurlMultiObject *self, int option, int which, PyObject *obj) +{ + switch(option) { +#ifdef HAVE_CURL_7_30_0_PIPELINE_OPTS + case CURLMOPT_PIPELINING_SITE_BL: + case CURLMOPT_PIPELINING_SERVER_BL: +#endif + return do_multi_setopt_charpp(self, option, which, obj); + break; + default: + PyErr_SetString(PyExc_TypeError, "lists/tuples are not supported for this option"); + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject * +do_multi_setopt_callable(CurlMultiObject *self, int option, PyObject *obj) +{ + /* We use function types here to make sure that our callback + * definitions exactly match the interface. + */ + const curl_multi_timer_callback t_cb = multi_timer_callback; + const curl_socket_callback s_cb = multi_socket_callback; + + switch(option) { + case CURLMOPT_SOCKETFUNCTION: + curl_multi_setopt(self->multi_handle, CURLMOPT_SOCKETFUNCTION, s_cb); + curl_multi_setopt(self->multi_handle, CURLMOPT_SOCKETDATA, self); + Py_INCREF(obj); + self->s_cb = obj; + break; + case CURLMOPT_TIMERFUNCTION: + curl_multi_setopt(self->multi_handle, CURLMOPT_TIMERFUNCTION, t_cb); + curl_multi_setopt(self->multi_handle, CURLMOPT_TIMERDATA, self); + Py_INCREF(obj); + self->t_cb = obj; + break; + default: + PyErr_SetString(PyExc_TypeError, "callables are not supported for this option"); + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject * +do_multi_setopt_none(CurlMultiObject *self, int option, PyObject *obj) +{ + switch(option) { +#ifdef HAVE_CURL_7_30_0_PIPELINE_OPTS + case CURLMOPT_PIPELINING_SITE_BL: + case CURLMOPT_PIPELINING_SERVER_BL: + curl_multi_setopt(self->multi_handle, option, NULL); + break; +#endif + case CURLMOPT_SOCKETFUNCTION: + curl_multi_setopt(self->multi_handle, CURLMOPT_SOCKETFUNCTION, NULL); + curl_multi_setopt(self->multi_handle, CURLMOPT_SOCKETDATA, NULL); + Py_CLEAR(self->s_cb); + break; + case CURLMOPT_TIMERFUNCTION: + curl_multi_setopt(self->multi_handle, CURLMOPT_TIMERFUNCTION, NULL); + curl_multi_setopt(self->multi_handle, CURLMOPT_TIMERDATA, NULL); + Py_CLEAR(self->t_cb); + break; + default: + PyErr_SetString(PyExc_TypeError, "unsetting is not supported for this option"); + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject * +do_multi_setopt(CurlMultiObject *self, PyObject *args) +{ + int option, which; + PyObject *obj; + + if (!PyArg_ParseTuple(args, "iO:setopt", &option, &obj)) + return NULL; + if (check_multi_state(self, 1 | 2, "setopt") != 0) + return NULL; + + /* Early checks of option value */ + if (option <= 0) + goto error; + if (option >= (int)CURLOPTTYPE_OFF_T + MOPTIONS_SIZE) + goto error; + if (option % 10000 >= MOPTIONS_SIZE) + goto error; + + /* Handle unsetting of options */ + if (obj == Py_None) { + return do_multi_setopt_none(self, option, obj); + } + + /* Handle the case of integer arguments */ + if (PyInt_Check(obj)) { + return do_multi_setopt_int(self, option, obj); + } + + /* Handle the case of list or tuple objects */ + which = PyListOrTuple_Check(obj); + if (which) { + return do_multi_setopt_list(self, option, which, obj); + } + + if (PyFunction_Check(obj) || PyCFunction_Check(obj) || + PyCallable_Check(obj) || PyMethod_Check(obj)) { + return do_multi_setopt_callable(self, option, obj); + } + + /* Failed to match any of the function signatures -- return error */ +error: + PyErr_SetString(PyExc_TypeError, "invalid arguments to setopt"); + return NULL; +} + + +/* --------------- timeout --------------- */ + +static PyObject * +do_multi_timeout(CurlMultiObject *self) +{ + CURLMcode res; + long timeout; + + if (check_multi_state(self, 1 | 2, "timeout") != 0) { + return NULL; + } + + res = curl_multi_timeout(self->multi_handle, &timeout); + if (res != CURLM_OK) { + CURLERROR_MSG("timeout failed"); + } + + /* Return number of millisecs until timeout */ + return Py_BuildValue("l", timeout); +} + + +/* --------------- assign --------------- */ + +static PyObject * +do_multi_assign(CurlMultiObject *self, PyObject *args) +{ + CURLMcode res; + curl_socket_t socket; + PyObject *obj; + + if (!PyArg_ParseTuple(args, "iO:assign", &socket, &obj)) + return NULL; + if (check_multi_state(self, 1 | 2, "assign") != 0) { + return NULL; + } + Py_INCREF(obj); + + res = curl_multi_assign(self->multi_handle, socket, obj); + if (res != CURLM_OK) { + CURLERROR_MSG("assign failed"); + } + + Py_RETURN_NONE; +} + + +/* --------------- socket_action --------------- */ +static PyObject * +do_multi_socket_action(CurlMultiObject *self, PyObject *args) +{ + CURLMcode res; + curl_socket_t socket; + int ev_bitmask; + int running = -1; + + if (!PyArg_ParseTuple(args, "ii:socket_action", &socket, &ev_bitmask)) + return NULL; + if (check_multi_state(self, 1 | 2, "socket_action") != 0) { + return NULL; + } + + PYCURL_BEGIN_ALLOW_THREADS + res = curl_multi_socket_action(self->multi_handle, socket, ev_bitmask, &running); + PYCURL_END_ALLOW_THREADS + + if (res != CURLM_OK) { + CURLERROR_MSG("multi_socket_action failed"); + } + /* Return a tuple with the result and the number of running handles */ + return Py_BuildValue("(ii)", (int)res, running); +} + +/* --------------- socket_all --------------- */ + +static PyObject * +do_multi_socket_all(CurlMultiObject *self) +{ + CURLMcode res; + int running = -1; + + if (check_multi_state(self, 1 | 2, "socket_all") != 0) { + return NULL; + } + + PYCURL_BEGIN_ALLOW_THREADS + res = curl_multi_socket_all(self->multi_handle, &running); + PYCURL_END_ALLOW_THREADS + + /* We assume these errors are ok, otherwise raise exception */ + if (res != CURLM_OK && res != CURLM_CALL_MULTI_PERFORM) { + CURLERROR_MSG("perform failed"); + } + + /* Return a tuple with the result and the number of running handles */ + return Py_BuildValue("(ii)", (int)res, running); +} + + +/* --------------- perform --------------- */ + +static PyObject * +do_multi_perform(CurlMultiObject *self) +{ + CURLMcode res; + int running = -1; + + if (check_multi_state(self, 1 | 2, "perform") != 0) { + return NULL; + } + + PYCURL_BEGIN_ALLOW_THREADS + res = curl_multi_perform(self->multi_handle, &running); + PYCURL_END_ALLOW_THREADS + + /* We assume these errors are ok, otherwise raise exception */ + if (res != CURLM_OK && res != CURLM_CALL_MULTI_PERFORM) { + CURLERROR_MSG("perform failed"); + } + + /* Return a tuple with the result and the number of running handles */ + return Py_BuildValue("(ii)", (int)res, running); +} + + +/* --------------- add_handle/remove_handle --------------- */ + +/* static utility function */ +static int +check_multi_add_remove(const CurlMultiObject *self, const CurlObject *obj) +{ + /* check CurlMultiObject status */ + assert_multi_state(self); + if (self->multi_handle == NULL) { + PyErr_SetString(ErrorObject, "cannot add/remove handle - multi-stack is closed"); + return -1; + } +#ifdef WITH_THREAD + if (self->state != NULL) { + PyErr_SetString(ErrorObject, "cannot add/remove handle - multi_perform() already running"); + return -1; + } +#endif + /* check CurlObject status */ + assert_curl_state(obj); +#ifdef WITH_THREAD + if (obj->state != NULL) { + PyErr_SetString(ErrorObject, "cannot add/remove handle - perform() of curl object already running"); + return -1; + } +#endif + if (obj->multi_stack != NULL && obj->multi_stack != self) { + PyErr_SetString(ErrorObject, "cannot add/remove handle - curl object already on another multi-stack"); + return -1; + } + return 0; +} + + +static PyObject * +do_multi_add_handle(CurlMultiObject *self, PyObject *args) +{ + CurlObject *obj; + CURLMcode res; + + if (!PyArg_ParseTuple(args, "O!:add_handle", p_Curl_Type, &obj)) { + return NULL; + } + if (check_multi_add_remove(self, obj) != 0) { + return NULL; + } + if (obj->handle == NULL) { + PyErr_SetString(ErrorObject, "curl object already closed"); + return NULL; + } + if (obj->multi_stack == self) { + PyErr_SetString(ErrorObject, "curl object already on this multi-stack"); + return NULL; + } + + PyDict_SetItem(self->easy_object_dict, (PyObject *) obj, Py_True); + + assert(obj->multi_stack == NULL); + /* Allow threads because callbacks can be invoked */ + PYCURL_BEGIN_ALLOW_THREADS + res = curl_multi_add_handle(self->multi_handle, obj->handle); + PYCURL_END_ALLOW_THREADS + if (res != CURLM_OK) { + PyDict_DelItem(self->easy_object_dict, (PyObject *) obj); + CURLERROR_MSG("curl_multi_add_handle() failed due to internal errors"); + } + obj->multi_stack = self; + Py_INCREF(self); + + Py_RETURN_NONE; +} + + +static PyObject * +do_multi_remove_handle(CurlMultiObject *self, PyObject *args) +{ + CurlObject *obj; + CURLMcode res; + + if (!PyArg_ParseTuple(args, "O!:remove_handle", p_Curl_Type, &obj)) { + return NULL; + } + if (check_multi_add_remove(self, obj) != 0) { + return NULL; + } + if (obj->handle == NULL) { + /* CurlObject handle already closed -- ignore */ + if (PyDict_GetItem(self->easy_object_dict, (PyObject *) obj)) { + PyDict_DelItem(self->easy_object_dict, (PyObject *) obj); + } + goto done; + } + if (obj->multi_stack != self) { + PyErr_SetString(ErrorObject, "curl object not on this multi-stack"); + return NULL; + } + /* Allow threads because callbacks can be invoked */ + PYCURL_BEGIN_ALLOW_THREADS + res = curl_multi_remove_handle(self->multi_handle, obj->handle); + PYCURL_END_ALLOW_THREADS + if (res == CURLM_OK) { + PyDict_DelItem(self->easy_object_dict, (PyObject *) obj); + // if PyDict_DelItem fails, remove_handle call will also fail. + // but the dictionary should always have our object in it + // hence this failure shouldn't happen unless something unaccounted + // for went wrong + } else { + CURLERROR_MSG("curl_multi_remove_handle() failed due to internal errors"); + } + assert(obj->multi_stack == self); + obj->multi_stack = NULL; + Py_DECREF(self); +done: + Py_RETURN_NONE; +} + + +/* --------------- fdset ---------------------- */ + +static PyObject * +do_multi_fdset(CurlMultiObject *self) +{ + CURLMcode res; + int max_fd = -1, fd; + PyObject *ret = NULL; + PyObject *read_list = NULL, *write_list = NULL, *except_list = NULL; + PyObject *py_fd = NULL; + + if (check_multi_state(self, 1 | 2, "fdset") != 0) { + return NULL; + } + + /* Clear file descriptor sets */ + FD_ZERO(&self->read_fd_set); + FD_ZERO(&self->write_fd_set); + FD_ZERO(&self->exc_fd_set); + + /* Don't bother releasing the gil as this is just a data structure operation */ + res = curl_multi_fdset(self->multi_handle, &self->read_fd_set, + &self->write_fd_set, &self->exc_fd_set, &max_fd); + if (res != CURLM_OK) { + CURLERROR_MSG("curl_multi_fdset() failed due to internal errors"); + } + + /* Allocate lists. */ + if ((read_list = PyList_New((Py_ssize_t)0)) == NULL) goto error; + if ((write_list = PyList_New((Py_ssize_t)0)) == NULL) goto error; + if ((except_list = PyList_New((Py_ssize_t)0)) == NULL) goto error; + + /* Populate lists */ + for (fd = 0; fd < max_fd + 1; fd++) { + if (FD_ISSET(fd, &self->read_fd_set)) { + if ((py_fd = PyInt_FromLong((long)fd)) == NULL) goto error; + if (PyList_Append(read_list, py_fd) != 0) goto error; + Py_DECREF(py_fd); + py_fd = NULL; + } + if (FD_ISSET(fd, &self->write_fd_set)) { + if ((py_fd = PyInt_FromLong((long)fd)) == NULL) goto error; + if (PyList_Append(write_list, py_fd) != 0) goto error; + Py_DECREF(py_fd); + py_fd = NULL; + } + if (FD_ISSET(fd, &self->exc_fd_set)) { + if ((py_fd = PyInt_FromLong((long)fd)) == NULL) goto error; + if (PyList_Append(except_list, py_fd) != 0) goto error; + Py_DECREF(py_fd); + py_fd = NULL; + } + } + + /* Return a tuple with the 3 lists */ + ret = Py_BuildValue("(OOO)", read_list, write_list, except_list); +error: + Py_XDECREF(py_fd); + Py_XDECREF(except_list); + Py_XDECREF(write_list); + Py_XDECREF(read_list); + return ret; +} + + +/* --------------- info_read --------------- */ + +static PyObject * +do_multi_info_read(CurlMultiObject *self, PyObject *args) +{ + PyObject *ret = NULL; + PyObject *ok_list = NULL, *err_list = NULL; + CURLMsg *msg; + int in_queue = 0, num_results = INT_MAX; + + /* Sanity checks */ + if (!PyArg_ParseTuple(args, "|i:info_read", &num_results)) { + return NULL; + } + if (num_results <= 0) { + PyErr_SetString(ErrorObject, "argument to info_read must be greater than zero"); + return NULL; + } + if (check_multi_state(self, 1 | 2, "info_read") != 0) { + return NULL; + } + + if ((ok_list = PyList_New((Py_ssize_t)0)) == NULL) goto error; + if ((err_list = PyList_New((Py_ssize_t)0)) == NULL) goto error; + + /* Loop through all messages */ + while ((msg = curl_multi_info_read(self->multi_handle, &in_queue)) != NULL) { + CURLcode res; + CurlObject *co = NULL; + + /* Check for termination as specified by the user */ + if (num_results-- <= 0) { + break; + } + + /* Fetch the curl object that corresponds to the curl handle in the message */ + res = curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, (char **) &co); + if (res != CURLE_OK || co == NULL) { + Py_DECREF(err_list); + Py_DECREF(ok_list); + CURLERROR_MSG("Unable to fetch curl handle from curl object"); + } + assert(PyObject_IsInstance((PyObject *) co, (PyObject *) p_Curl_Type) == 1); + if (msg->msg != CURLMSG_DONE) { + /* FIXME: what does this mean ??? */ + } + if (msg->data.result == CURLE_OK) { + /* Append curl object to list of objects which succeeded */ + if (PyList_Append(ok_list, (PyObject *)co) != 0) { + goto error; + } + } + else { + /* Create a result tuple that will get added to err_list. */ + PyObject *error_str = NULL; + PyObject *v; +#if PY_MAJOR_VERSION >= 3 + error_str = PyUnicode_DecodeLocale(co->error, "surrogateescape"); + if (error_str == NULL) { + goto error; + } + v = Py_BuildValue("(OiO)", (PyObject *)co, (int)msg->data.result, error_str); +#else + v = Py_BuildValue("(Ois)", (PyObject *)co, (int)msg->data.result, co->error); +#endif + /* Append curl object to list of objects which failed */ + if (v == NULL || PyList_Append(err_list, v) != 0) { + Py_XDECREF(error_str); + Py_XDECREF(v); + goto error; + } + Py_DECREF(v); + } + } + /* Return (number of queued messages, [ok_objects], [error_objects]) */ + ret = Py_BuildValue("(iOO)", in_queue, ok_list, err_list); +error: + Py_XDECREF(err_list); + Py_XDECREF(ok_list); + return ret; +} + + +/* --------------- select --------------- */ + +static PyObject * +do_multi_select(CurlMultiObject *self, PyObject *args) +{ + int max_fd = -1, n; + double timeout = -1.0; + struct timeval tv, *tvp; + CURLMcode res; + + if (!PyArg_ParseTuple(args, "d:select", &timeout)) { + return NULL; + } + if (check_multi_state(self, 1 | 2, "select") != 0) { + return NULL; + } + + if (timeout < 0 || timeout >= 365 * 24 * 60 * 60) { + PyErr_SetString(PyExc_OverflowError, "invalid timeout period"); + return NULL; + } else { + long seconds = (long)timeout; + timeout = timeout - (double)seconds; + assert(timeout >= 0.0); assert(timeout < 1.0); + tv.tv_sec = seconds; + tv.tv_usec = (long)(timeout*1000000.0); + tvp = &tv; + } + + FD_ZERO(&self->read_fd_set); + FD_ZERO(&self->write_fd_set); + FD_ZERO(&self->exc_fd_set); + + res = curl_multi_fdset(self->multi_handle, &self->read_fd_set, + &self->write_fd_set, &self->exc_fd_set, &max_fd); + if (res != CURLM_OK) { + CURLERROR_MSG("multi_fdset failed"); + } + + if (max_fd < 0) { + n = 0; + } + else { + Py_BEGIN_ALLOW_THREADS + n = select(max_fd + 1, &self->read_fd_set, &self->write_fd_set, &self->exc_fd_set, tvp); + Py_END_ALLOW_THREADS + /* info: like Python's socketmodule.c we do not raise an exception + * if select() fails - we'll leave it to the actual libcurl + * socket code to report any errors. + */ + } + + return PyInt_FromLong(n); +} + + +static PyObject *do_curlmulti_getstate(CurlMultiObject *self) +{ + PyErr_SetString(PyExc_TypeError, "CurlMulti objects do not support serialization"); + return NULL; +} + + +static PyObject *do_curlmulti_setstate(CurlMultiObject *self, PyObject *args) +{ + PyErr_SetString(PyExc_TypeError, "CurlMulti objects do not support deserialization"); + return NULL; +} + + +/************************************************************************* +// type definitions +**************************************************************************/ + +/* --------------- methods --------------- */ + +PYCURL_INTERNAL PyMethodDef curlmultiobject_methods[] = { + {"add_handle", (PyCFunction)do_multi_add_handle, METH_VARARGS, multi_add_handle_doc}, + {"close", (PyCFunction)do_multi_close, METH_NOARGS, multi_close_doc}, + {"fdset", (PyCFunction)do_multi_fdset, METH_NOARGS, multi_fdset_doc}, + {"info_read", (PyCFunction)do_multi_info_read, METH_VARARGS, multi_info_read_doc}, + {"perform", (PyCFunction)do_multi_perform, METH_NOARGS, multi_perform_doc}, + {"socket_action", (PyCFunction)do_multi_socket_action, METH_VARARGS, multi_socket_action_doc}, + {"socket_all", (PyCFunction)do_multi_socket_all, METH_NOARGS, multi_socket_all_doc}, + {"setopt", (PyCFunction)do_multi_setopt, METH_VARARGS, multi_setopt_doc}, + {"timeout", (PyCFunction)do_multi_timeout, METH_NOARGS, multi_timeout_doc}, + {"assign", (PyCFunction)do_multi_assign, METH_VARARGS, multi_assign_doc}, + {"remove_handle", (PyCFunction)do_multi_remove_handle, METH_VARARGS, multi_remove_handle_doc}, + {"select", (PyCFunction)do_multi_select, METH_VARARGS, multi_select_doc}, + {"__getstate__", (PyCFunction)do_curlmulti_getstate, METH_NOARGS, NULL}, + {"__setstate__", (PyCFunction)do_curlmulti_setstate, METH_VARARGS, NULL}, + {NULL, NULL, 0, NULL} +}; + + +/* --------------- setattr/getattr --------------- */ + + +#if PY_MAJOR_VERSION >= 3 + +PYCURL_INTERNAL PyObject * +do_multi_getattro(PyObject *o, PyObject *n) +{ + PyObject *v; + assert_multi_state((CurlMultiObject *)o); + v = PyObject_GenericGetAttr(o, n); + if( !v && PyErr_ExceptionMatches(PyExc_AttributeError) ) + { + PyErr_Clear(); + v = my_getattro(o, n, ((CurlMultiObject *)o)->dict, + curlmultiobject_constants, curlmultiobject_methods); + } + return v; +} + +PYCURL_INTERNAL int +do_multi_setattro(PyObject *o, PyObject *n, PyObject *v) +{ + assert_multi_state((CurlMultiObject *)o); + return my_setattro(&((CurlMultiObject *)o)->dict, n, v); +} + +#else /* PY_MAJOR_VERSION >= 3 */ + +PYCURL_INTERNAL PyObject * +do_multi_getattr(CurlMultiObject *co, char *name) +{ + assert_multi_state(co); + return my_getattr((PyObject *)co, name, co->dict, + curlmultiobject_constants, curlmultiobject_methods); +} + +PYCURL_INTERNAL int +do_multi_setattr(CurlMultiObject *co, char *name, PyObject *v) +{ + assert_multi_state(co); + return my_setattr(&co->dict, name, v); +} + +#endif /* PY_MAJOR_VERSION >= 3 */ + +PYCURL_INTERNAL PyTypeObject CurlMulti_Type = { +#if PY_MAJOR_VERSION >= 3 + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ +#endif + "pycurl.CurlMulti", /* tp_name */ + sizeof(CurlMultiObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)do_multi_dealloc, /* tp_dealloc */ + 0, /* tp_print */ +#if PY_MAJOR_VERSION >= 3 + 0, /* tp_getattr */ + 0, /* tp_setattr */ +#else + (getattrfunc)do_multi_getattr, /* tp_getattr */ + (setattrfunc)do_multi_setattr, /* tp_setattr */ +#endif + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ +#if PY_MAJOR_VERSION >= 3 + (getattrofunc)do_multi_getattro, /* tp_getattro */ + (setattrofunc)do_multi_setattro, /* tp_setattro */ +#else + 0, /* tp_getattro */ + 0, /* tp_setattro */ +#endif + 0, /* tp_as_buffer */ + PYCURL_TYPE_FLAGS, /* tp_flags */ + multi_doc, /* tp_doc */ + (traverseproc)do_multi_traverse, /* tp_traverse */ + (inquiry)do_multi_clear, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(CurlMultiObject, weakreflist), /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + curlmultiobject_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + (newfunc)do_multi_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; + +/* vi:ts=4:et:nowrap + */ diff --git a/src/oscompat.c b/src/oscompat.c new file mode 100644 index 0000000..5fb070c --- /dev/null +++ b/src/oscompat.c @@ -0,0 +1,57 @@ +#include "pycurl.h" + +#if defined(WIN32) +PYCURL_INTERNAL int +dup_winsock(int sock, const struct curl_sockaddr *address) +{ + int rv; + WSAPROTOCOL_INFO pi; + + rv = WSADuplicateSocket(sock, GetCurrentProcessId(), &pi); + if (rv) { + return CURL_SOCKET_BAD; + } + + /* not sure if WSA_FLAG_OVERLAPPED is needed, but it does not seem to hurt */ + return (int) WSASocket(address->family, address->socktype, address->protocol, &pi, 0, WSA_FLAG_OVERLAPPED); +} +#endif + +#if defined(WIN32) && ((_WIN32_WINNT < 0x0600) || (NTDDI_VERSION < NTDDI_VISTA)) +/* + * Only Winsock on Vista+ has inet_ntop(). + */ +PYCURL_INTERNAL const char * +pycurl_inet_ntop (int family, void *addr, char *string, size_t string_size) +{ + SOCKADDR *sa; + int sa_len; + /* both size_t and DWORD should be unsigned ints */ + DWORD string_size_dword = (DWORD) string_size; + + if (family == AF_INET6) { + struct sockaddr_in6 sa6; + memset(&sa6, 0, sizeof(sa6)); + sa6.sin6_family = AF_INET6; + memcpy(&sa6.sin6_addr, addr, sizeof(sa6.sin6_addr)); + sa = (SOCKADDR*) &sa6; + sa_len = sizeof(sa6); + } else if (family == AF_INET) { + struct sockaddr_in sa4; + memset(&sa4, 0, sizeof(sa4)); + sa4.sin_family = AF_INET; + memcpy(&sa4.sin_addr, addr, sizeof(sa4.sin_addr)); + sa = (SOCKADDR*) &sa4; + sa_len = sizeof(sa4); + } else { + errno = EAFNOSUPPORT; + return NULL; + } + if (WSAAddressToString(sa, sa_len, NULL, string, &string_size_dword)) + return NULL; + return string; +} +#endif + +/* vi:ts=4:et:nowrap + */ diff --git a/src/pycurl.h b/src/pycurl.h new file mode 100644 index 0000000..9a97f0b --- /dev/null +++ b/src/pycurl.h @@ -0,0 +1,705 @@ +#if (defined(_WIN32) || defined(__WIN32__)) && !defined(WIN32) +# define WIN32 1 +#endif +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(WIN32) +#include +#include +#include +#include +#endif + +#if defined(WIN32) +/* + * Since setup.py uses a '-WX' in the CFLAGS (treat warnings as errors), + * the below will turn off some warnings when using MS-SDK 8.1+. + * This MUST be defined before including via the libcurl + * headers. + */ +# if !defined(_WINSOCK_DEPRECATED_NO_WARNINGS) +# define _WINSOCK_DEPRECATED_NO_WARNINGS +# endif +#endif + +#include +#include +#include +#undef NDEBUG +#include + +#define MAKE_LIBCURL_VERSION(major, minor, patch) \ + ((major) * 0x10000 + (minor) * 0x100 + (patch)) + +/* spot check */ +#if MAKE_LIBCURL_VERSION(7, 21, 16) != 0x071510 +# error MAKE_LIBCURL_VERSION is not working correctly +#endif + +#if defined(PYCURL_SINGLE_FILE) +# define PYCURL_INTERNAL static +#else +# define PYCURL_INTERNAL +#endif + +#if defined(WIN32) +/* supposedly not present in errno.h provided with VC */ +# if !defined(EAFNOSUPPORT) +# define EAFNOSUPPORT 97 +# endif + +PYCURL_INTERNAL int +dup_winsock(int sock, const struct curl_sockaddr *address); +#endif + +/* The inet_ntop() was added in ws2_32.dll on Windows Vista [1]. Hence the + * Windows SDK targeting lesser OS'es doesn't provide that prototype. + * Maybe we should use the local hidden inet_ntop() for all OS'es thus + * making a pycurl.pyd work across OS'es w/o rebuilding? + * + * [1] http://msdn.microsoft.com/en-us/library/windows/desktop/cc805843(v=vs.85).aspx + */ +#if defined(WIN32) && ((_WIN32_WINNT < 0x0600) || (NTDDI_VERSION < NTDDI_VISTA)) +PYCURL_INTERNAL const char * +pycurl_inet_ntop (int family, void *addr, char *string, size_t string_size); +#define inet_ntop(fam,addr,string,size) pycurl_inet_ntop(fam,addr,string,size) +#endif + +#if !defined(LIBCURL_VERSION_NUM) || (LIBCURL_VERSION_NUM < 0x071300) +# error "Need libcurl version 7.19.0 or greater to compile pycurl." +#endif + +#if LIBCURL_VERSION_NUM >= 0x071301 /* check for 7.19.1 or greater */ +#define HAVE_CURLOPT_USERNAME +#define HAVE_CURLOPT_PROXYUSERNAME +#define HAVE_CURLOPT_CERTINFO +#define HAVE_CURLOPT_POSTREDIR +#endif + +#if LIBCURL_VERSION_NUM >= 0x071303 /* check for 7.19.3 or greater */ +#define HAVE_CURLAUTH_DIGEST_IE +#endif + +#if LIBCURL_VERSION_NUM >= 0x071304 /* check for 7.19.4 or greater */ +#define HAVE_CURLOPT_NOPROXY +#define HAVE_CURLOPT_PROTOCOLS +#define HAVE_CURL_7_19_4_OPTS +#endif + +#if LIBCURL_VERSION_NUM >= 0x071305 /* check for 7.19.5 or greater */ +#define HAVE_CURL_7_19_5_OPTS +#endif + +#if LIBCURL_VERSION_NUM >= 0x071306 /* check for 7.19.6 or greater */ +#define HAVE_CURL_7_19_6_OPTS +#endif + +#if LIBCURL_VERSION_NUM >= 0x071400 /* check for 7.20.0 or greater */ +#define HAVE_CURL_7_20_0_OPTS +#endif + +#if LIBCURL_VERSION_NUM >= 0x071500 /* check for 7.21.0 or greater */ +#define HAVE_CURLINFO_LOCAL_PORT +#define HAVE_CURLINFO_PRIMARY_PORT +#define HAVE_CURLINFO_LOCAL_IP +#define HAVE_CURL_7_21_0_OPTS +#endif + +#if LIBCURL_VERSION_NUM >= 0x071502 /* check for 7.21.2 or greater */ +#define HAVE_CURL_7_21_2_OPTS +#endif + +#if LIBCURL_VERSION_NUM >= 0x071503 /* check for 7.21.3 or greater */ +#define HAVE_CURLOPT_RESOLVE +#endif + +#if LIBCURL_VERSION_NUM >= 0x071505 /* check for 7.21.5 or greater */ +#define HAVE_CURL_7_21_5 +#endif + +#if LIBCURL_VERSION_NUM >= 0x071600 /* check for 7.22.0 or greater */ +#define HAVE_CURL_7_22_0_OPTS +#endif + +#if LIBCURL_VERSION_NUM >= 0x071800 /* check for 7.24.0 or greater */ +#define HAVE_CURLOPT_DNS_SERVERS +#define HAVE_CURL_7_24_0 +#endif + +#if LIBCURL_VERSION_NUM >= 0x071900 /* check for 7.25.0 or greater */ +#define HAVE_CURL_7_25_0_OPTS +#endif + +#if LIBCURL_VERSION_NUM >= 0x071A00 /* check for 7.26.0 or greater */ +#define HAVE_CURL_REDIR_POST_303 +#endif + +#if LIBCURL_VERSION_NUM >= 0x071E00 /* check for 7.30.0 or greater */ +#define HAVE_CURL_7_30_0_PIPELINE_OPTS +#endif + +#if LIBCURL_VERSION_NUM >= 0x073100 /* check for 7.49.0 or greater */ +#define HAVE_CURLOPT_CONNECT_TO +#endif + +#if LIBCURL_VERSION_NUM >= 0x073200 /* check for 7.50.0 or greater */ +#define HAVE_CURLINFO_HTTP_VERSION +#endif + +#if LIBCURL_VERSION_NUM >= 0x073C00 /* check for 7.60.0 or greater */ +#define HAVE_CURLOPT_HAPROXYPROTOCOL +#endif + +/* curl_global_sslset() was added in 7.56.0 but was buggy until 7.63.0 */ +#if LIBCURL_VERSION_NUM >= 0x073F00 /* check for 7.63.0 or greater */ +#define HAVE_CURL_GLOBAL_SSLSET +#endif + +#if LIBCURL_VERSION_NUM >= 0x074300 /* check for 7.67.0 or greater */ +#define HAVE_CURL_7_67_0_MULTI_STREAMS +#endif + +#undef UNUSED +#define UNUSED(var) ((void)&var) + +/* Cruft for thread safe SSL crypto locks, snapped from the PHP curl extension */ +#if defined(HAVE_CURL_SSL) +# if defined(HAVE_CURL_OPENSSL) +# define PYCURL_NEED_SSL_TSL +# define PYCURL_NEED_OPENSSL_TSL +# include +# include +# define COMPILE_SSL_LIB "openssl" +# define COMPILE_SUPPORTED_SSL_BACKEND_FOUND 1 +# elif defined(HAVE_CURL_WOLFSSL) +# include +# if defined(OPENSSL_EXTRA) +# define HAVE_CURL_OPENSSL +# define PYCURL_NEED_SSL_TSL +# define PYCURL_NEED_OPENSSL_TSL +# include +# include +# else +# ifdef _MSC_VER +# pragma message(\ + "libcurl was compiled with wolfSSL, but the library was built without " \ + "--enable-opensslextra; thus no SSL crypto locking callbacks will be set, " \ + "which may cause random crashes on SSL requests") +# else +# warning \ + "libcurl was compiled with wolfSSL, but the library was built without " \ + "--enable-opensslextra; thus no SSL crypto locking callbacks will be set, " \ + "which may cause random crashes on SSL requests" +# endif +# endif +# define COMPILE_SSL_LIB "wolfssl" +# define COMPILE_SUPPORTED_SSL_BACKEND_FOUND 1 +# elif defined(HAVE_CURL_GNUTLS) +# include +# if GNUTLS_VERSION_NUMBER <= 0x020b00 +# define PYCURL_NEED_SSL_TSL +# define PYCURL_NEED_GNUTLS_TSL +# include +# endif +# define COMPILE_SSL_LIB "gnutls" +# define COMPILE_SUPPORTED_SSL_BACKEND_FOUND 1 +# elif defined(HAVE_CURL_NSS) +# define COMPILE_SSL_LIB "nss" +# define COMPILE_SUPPORTED_SSL_BACKEND_FOUND 1 +# elif defined(HAVE_CURL_MBEDTLS) +# include +# define PYCURL_NEED_SSL_TSL +# define PYCURL_NEED_MBEDTLS_TSL +# define COMPILE_SSL_LIB "mbedtls" +# define COMPILE_SUPPORTED_SSL_BACKEND_FOUND 1 +# elif defined(HAVE_CURL_SECTRANSP) +# define COMPILE_SSL_LIB "secure-transport" +# define COMPILE_SUPPORTED_SSL_BACKEND_FOUND 1 +# else +# ifdef _MSC_VER + /* sigh */ +# pragma message(\ + "libcurl was compiled with SSL support, but configure could not determine which " \ + "library was used; thus no SSL crypto locking callbacks will be set, which may " \ + "cause random crashes on SSL requests") +# else +# warning \ + "libcurl was compiled with SSL support, but configure could not determine which " \ + "library was used; thus no SSL crypto locking callbacks will be set, which may " \ + "cause random crashes on SSL requests" +# endif + /* since we have no crypto callbacks for other ssl backends, + * no reason to require users match those */ +# define COMPILE_SSL_LIB "none/other" +# define COMPILE_SUPPORTED_SSL_BACKEND_FOUND 0 +# endif /* HAVE_CURL_OPENSSL || HAVE_CURL_WOLFSSL || HAVE_CURL_GNUTLS || HAVE_CURL_NSS || HAVE_CURL_MBEDTLS || HAVE_CURL_SECTRANSP */ +#else +# define COMPILE_SSL_LIB "none/other" +# define COMPILE_SUPPORTED_SSL_BACKEND_FOUND 0 +#endif /* HAVE_CURL_SSL */ + +#if defined(PYCURL_NEED_SSL_TSL) +PYCURL_INTERNAL int pycurl_ssl_init(void); +PYCURL_INTERNAL void pycurl_ssl_cleanup(void); +#endif + +#ifdef WITH_THREAD +# define PYCURL_DECLARE_THREAD_STATE PyThreadState *tmp_state +# define PYCURL_ACQUIRE_THREAD() pycurl_acquire_thread(self, &tmp_state) +# define PYCURL_ACQUIRE_THREAD_MULTI() pycurl_acquire_thread_multi(self, &tmp_state) +# define PYCURL_RELEASE_THREAD() pycurl_release_thread(tmp_state) +/* Replacement for Py_BEGIN_ALLOW_THREADS/Py_END_ALLOW_THREADS when python + callbacks are expected during blocking i/o operations: self->state will hold + the handle to current thread to be used as context */ +# define PYCURL_BEGIN_ALLOW_THREADS \ + self->state = PyThreadState_Get(); \ + assert(self->state != NULL); \ + Py_BEGIN_ALLOW_THREADS +# define PYCURL_END_ALLOW_THREADS \ + Py_END_ALLOW_THREADS \ + self->state = NULL; +# define PYCURL_BEGIN_ALLOW_THREADS_EASY \ + if (self->multi_stack == NULL) { \ + self->state = PyThreadState_Get(); \ + assert(self->state != NULL); \ + } else { \ + self->multi_stack->state = PyThreadState_Get(); \ + assert(self->multi_stack->state != NULL); \ + } \ + Py_BEGIN_ALLOW_THREADS +# define PYCURL_END_ALLOW_THREADS_EASY \ + PYCURL_END_ALLOW_THREADS \ + if (self->multi_stack != NULL) \ + self->multi_stack->state = NULL; +#else +# define PYCURL_DECLARE_THREAD_STATE +# define PYCURL_ACQUIRE_THREAD() (1) +# define PYCURL_ACQUIRE_THREAD_MULTI() (1) +# define PYCURL_RELEASE_THREAD() +# define PYCURL_BEGIN_ALLOW_THREADS +# define PYCURL_END_ALLOW_THREADS +#endif + +#if PY_MAJOR_VERSION >= 3 + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_FromLong PyLong_FromLong + #define PyInt_AsLong PyLong_AsLong +#endif + +#define PYLISTORTUPLE_LIST 1 +#define PYLISTORTUPLE_TUPLE 2 +#define PYLISTORTUPLE_OTHER 0 + +PYCURL_INTERNAL int +PyListOrTuple_Check(PyObject *v); +PYCURL_INTERNAL Py_ssize_t +PyListOrTuple_Size(PyObject *v, int which); +PYCURL_INTERNAL PyObject * +PyListOrTuple_GetItem(PyObject *v, Py_ssize_t i, int which); + +/************************************************************************* +// python 2/3 compatibility +**************************************************************************/ + +#if PY_MAJOR_VERSION >= 3 +# define PyText_FromFormat(format, str) PyUnicode_FromFormat((format), (str)) +# define PyText_FromString(str) PyUnicode_FromString(str) +# define PyByteStr_FromString(str) PyBytes_FromString(str) +# define PyByteStr_Check(obj) PyBytes_Check(obj) +# define PyByteStr_AsStringAndSize(obj, buffer, length) PyBytes_AsStringAndSize((obj), (buffer), (length)) +#else +# define PyText_FromFormat(format, str) PyString_FromFormat((format), (str)) +# define PyText_FromString(str) PyString_FromString(str) +# define PyByteStr_FromString(str) PyString_FromString(str) +# define PyByteStr_Check(obj) PyString_Check(obj) +# define PyByteStr_AsStringAndSize(obj, buffer, length) PyString_AsStringAndSize((obj), (buffer), (length)) +#endif +#define PyText_EncodedDecref(encoded) Py_XDECREF(encoded) + +PYCURL_INTERNAL int +PyText_AsStringAndSize(PyObject *obj, char **buffer, Py_ssize_t *length, PyObject **encoded_obj); +PYCURL_INTERNAL char * +PyText_AsString_NoNUL(PyObject *obj, PyObject **encoded_obj); +PYCURL_INTERNAL int +PyText_Check(PyObject *o); +PYCURL_INTERNAL PyObject * +PyText_FromString_Ignore(const char *string); + +/* Py_NewRef and Py_XNewRef - not part of Python's C API before 3.10 */ +static inline PyObject* my_Py_NewRef(PyObject *obj) { Py_INCREF(obj); return obj; } +static inline PyObject* my_Py_XNewRef(PyObject *obj) { Py_XINCREF(obj); return obj; } + +struct CurlObject; + +PYCURL_INTERNAL void +create_and_set_error_object(struct CurlObject *self, int code); + + +/* Raise exception based on return value `res' and `self->error' */ +#define CURLERROR_RETVAL() do {\ + create_and_set_error_object((self), (int) (res)); \ + return NULL; \ +} while (0) + +#define CURLERROR_SET_RETVAL() \ + create_and_set_error_object((self), (int) (res)); + +#define CURLERROR_RETVAL_MULTI_DONE() do {\ + PyObject *v; \ + v = Py_BuildValue("(i)", (int) (res)); \ + if (v != NULL) { PyErr_SetObject(ErrorObject, v); Py_DECREF(v); } \ + goto done; \ +} while (0) + +/* Raise exception based on return value `res' and custom message */ +/* msg should be ASCII */ +#define CURLERROR_MSG(msg) do {\ + PyObject *v; const char *m = (msg); \ + v = Py_BuildValue("(is)", (int) (res), (m)); \ + if (v != NULL) { PyErr_SetObject(ErrorObject, v); Py_DECREF(v); } \ + return NULL; \ +} while (0) + + +/* Calculate the number of OBJECTPOINT options we need to store */ +#define OPTIONS_SIZE ((int)CURLOPT_LASTENTRY % 10000) +#define MOPTIONS_SIZE ((int)CURLMOPT_LASTENTRY % 10000) + +/* Memory groups */ +/* Attributes dictionary */ +#define PYCURL_MEMGROUP_ATTRDICT 1 +/* multi_stack */ +#define PYCURL_MEMGROUP_MULTI 2 +/* Python callbacks */ +#define PYCURL_MEMGROUP_CALLBACK 4 +/* Python file objects */ +#define PYCURL_MEMGROUP_FILE 8 +/* Share objects */ +#define PYCURL_MEMGROUP_SHARE 16 +/* httppost buffer references */ +#define PYCURL_MEMGROUP_HTTPPOST 32 +/* Postfields object */ +#define PYCURL_MEMGROUP_POSTFIELDS 64 +/* CA certs object */ +#define PYCURL_MEMGROUP_CACERTS 128 +/* Curl slist objects */ +#define PYCURL_MEMGROUP_SLIST 256 + +#define PYCURL_MEMGROUP_EASY \ + (PYCURL_MEMGROUP_CALLBACK | PYCURL_MEMGROUP_FILE | \ + PYCURL_MEMGROUP_HTTPPOST | PYCURL_MEMGROUP_POSTFIELDS | \ + PYCURL_MEMGROUP_CACERTS | PYCURL_MEMGROUP_SLIST) + +#define PYCURL_MEMGROUP_ALL \ + (PYCURL_MEMGROUP_ATTRDICT | PYCURL_MEMGROUP_EASY | \ + PYCURL_MEMGROUP_MULTI | PYCURL_MEMGROUP_SHARE) + +typedef struct CurlSlistObject { + PyObject_HEAD + struct curl_slist *slist; +} CurlSlistObject; + +typedef struct CurlHttppostObject { + PyObject_HEAD + struct curl_httppost *httppost; + /* List of INC'ed references associated with httppost. */ + PyObject *reflist; +} CurlHttppostObject; + +typedef struct CurlObject { + PyObject_HEAD + PyObject *dict; /* Python attributes dictionary */ + // https://docs.python.org/3/extending/newtypes.html + PyObject *weakreflist; + CURL *handle; +#ifdef WITH_THREAD + PyThreadState *state; +#endif + struct CurlMultiObject *multi_stack; + struct CurlShareObject *share; + struct CurlHttppostObject *httppost; + struct CurlSlistObject *httpheader; +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 37, 0) + struct CurlSlistObject *proxyheader; +#endif + struct CurlSlistObject *http200aliases; + struct CurlSlistObject *quote; + struct CurlSlistObject *postquote; + struct CurlSlistObject *prequote; + struct CurlSlistObject *telnetoptions; +#ifdef HAVE_CURLOPT_RESOLVE + struct CurlSlistObject *resolve; +#endif +#ifdef HAVE_CURL_7_20_0_OPTS + struct CurlSlistObject *mail_rcpt; +#endif +#ifdef HAVE_CURLOPT_CONNECT_TO + struct CurlSlistObject *connect_to; +#endif + /* callbacks */ + PyObject *w_cb; + PyObject *h_cb; + PyObject *r_cb; + PyObject *pro_cb; +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 32, 0) + PyObject *xferinfo_cb; +#endif + PyObject *debug_cb; + PyObject *ioctl_cb; + PyObject *opensocket_cb; +#if LIBCURL_VERSION_NUM >= 0x071507 /* check for 7.21.7 or greater */ + PyObject *closesocket_cb; +#endif + PyObject *seek_cb; + PyObject *sockopt_cb; + PyObject *ssh_key_cb; + /* file objects */ + PyObject *readdata_fp; + PyObject *writedata_fp; + PyObject *writeheader_fp; + /* reference to the object used for CURLOPT_POSTFIELDS */ + PyObject *postfields_obj; + /* reference to the object containing ca certs */ + PyObject *ca_certs_obj; + /* misc */ + char error[CURL_ERROR_SIZE+1]; +} CurlObject; + +typedef struct CurlMultiObject { + PyObject_HEAD + PyObject *dict; /* Python attributes dictionary */ + // https://docs.python.org/3/extending/newtypes.html + PyObject *weakreflist; + CURLM *multi_handle; +#ifdef WITH_THREAD + PyThreadState *state; +#endif + fd_set read_fd_set; + fd_set write_fd_set; + fd_set exc_fd_set; + /* callbacks */ + PyObject *t_cb; + PyObject *s_cb; + + PyObject *easy_object_dict; +} CurlMultiObject; + +typedef struct { + PyThread_type_lock locks[CURL_LOCK_DATA_LAST]; +} ShareLock; + +typedef struct CurlShareObject { + PyObject_HEAD + PyObject *dict; /* Python attributes dictionary */ + // https://docs.python.org/3/extending/newtypes.html + PyObject *weakreflist; + CURLSH *share_handle; +#ifdef WITH_THREAD + ShareLock *lock; /* lock object to implement CURLSHOPT_LOCKFUNC */ +#endif +} CurlShareObject; + +#ifdef WITH_THREAD + +PYCURL_INTERNAL PyThreadState * +pycurl_get_thread_state(const CurlObject *self); +PYCURL_INTERNAL PyThreadState * +pycurl_get_thread_state_multi(const CurlMultiObject *self); +PYCURL_INTERNAL int +pycurl_acquire_thread(const CurlObject *self, PyThreadState **state); +PYCURL_INTERNAL int +pycurl_acquire_thread_multi(const CurlMultiObject *self, PyThreadState **state); +PYCURL_INTERNAL void +pycurl_release_thread(PyThreadState *state); + +PYCURL_INTERNAL void +share_lock_lock(ShareLock *lock, curl_lock_data data); +PYCURL_INTERNAL void +share_lock_unlock(ShareLock *lock, curl_lock_data data); +PYCURL_INTERNAL ShareLock * +share_lock_new(void); +PYCURL_INTERNAL void +share_lock_destroy(ShareLock *lock); +PYCURL_INTERNAL void +share_lock_callback(CURL *handle, curl_lock_data data, curl_lock_access locktype, void *userptr); +PYCURL_INTERNAL void +share_unlock_callback(CURL *handle, curl_lock_data data, void *userptr); + +#endif /* WITH_THREAD */ + +#if PY_MAJOR_VERSION >= 3 +PYCURL_INTERNAL PyObject * +my_getattro(PyObject *co, PyObject *name, PyObject *dict1, PyObject *dict2, PyMethodDef *m); +PYCURL_INTERNAL int +my_setattro(PyObject **dict, PyObject *name, PyObject *v); +#else /* PY_MAJOR_VERSION >= 3 */ +PYCURL_INTERNAL int +my_setattr(PyObject **dict, char *name, PyObject *v); +PYCURL_INTERNAL PyObject * +my_getattr(PyObject *co, char *name, PyObject *dict1, PyObject *dict2, PyMethodDef *m); +#endif /* PY_MAJOR_VERSION >= 3 */ + +/* used by multi object */ +PYCURL_INTERNAL void +assert_curl_state(const CurlObject *self); + +PYCURL_INTERNAL PyObject * +do_global_init(PyObject *dummy, PyObject *args); +PYCURL_INTERNAL PyObject * +do_global_cleanup(PyObject *dummy); +PYCURL_INTERNAL PyObject * +do_version_info(PyObject *dummy, PyObject *args); + +PYCURL_INTERNAL PyObject * +do_curl_setopt(CurlObject *self, PyObject *args); +PYCURL_INTERNAL PyObject * +do_curl_setopt_string(CurlObject *self, PyObject *args); +PYCURL_INTERNAL PyObject * +do_curl_unsetopt(CurlObject *self, PyObject *args); +#if defined(HAVE_CURL_OPENSSL) +PYCURL_INTERNAL PyObject * +do_curl_set_ca_certs(CurlObject *self, PyObject *args); +#endif +PYCURL_INTERNAL PyObject * +do_curl_perform(CurlObject *self); +PYCURL_INTERNAL PyObject * +do_curl_perform_rb(CurlObject *self); +#if PY_MAJOR_VERSION >= 3 +PYCURL_INTERNAL PyObject * +do_curl_perform_rs(CurlObject *self); +#else +# define do_curl_perform_rs do_curl_perform_rb +#endif + +PYCURL_INTERNAL PyObject * +do_curl_pause(CurlObject *self, PyObject *args); + +PYCURL_INTERNAL int +check_curl_state(const CurlObject *self, int flags, const char *name); +PYCURL_INTERNAL void +util_curl_xdecref(CurlObject *self, int flags, CURL *handle); +PYCURL_INTERNAL PyObject * +do_curl_setopt_filelike(CurlObject *self, int option, PyObject *obj); + +PYCURL_INTERNAL void +util_curlslist_update(CurlSlistObject **old, struct curl_slist *slist); +PYCURL_INTERNAL void +util_curlhttppost_update(CurlObject *obj, struct curl_httppost *httppost, PyObject *reflist); + +PYCURL_INTERNAL PyObject * +do_curl_getinfo_raw(CurlObject *self, PyObject *args); +#if PY_MAJOR_VERSION >= 3 +PYCURL_INTERNAL PyObject * +do_curl_getinfo(CurlObject *self, PyObject *args); +#else +# define do_curl_getinfo do_curl_getinfo_raw +#endif +PYCURL_INTERNAL PyObject * +do_curl_errstr(CurlObject *self); +#if PY_MAJOR_VERSION >= 3 +PYCURL_INTERNAL PyObject * +do_curl_errstr_raw(CurlObject *self); +#else +# define do_curl_errstr_raw do_curl_errstr +#endif + +PYCURL_INTERNAL size_t +write_callback(char *ptr, size_t size, size_t nmemb, void *stream); +PYCURL_INTERNAL size_t +header_callback(char *ptr, size_t size, size_t nmemb, void *stream); +PYCURL_INTERNAL curl_socket_t +opensocket_callback(void *clientp, curlsocktype purpose, + struct curl_sockaddr *address); +PYCURL_INTERNAL int +sockopt_cb(void *clientp, curl_socket_t curlfd, curlsocktype purpose); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 21, 7) +PYCURL_INTERNAL int +closesocket_callback(void *clientp, curl_socket_t curlfd); +#endif +#ifdef HAVE_CURL_7_19_6_OPTS +PYCURL_INTERNAL int +ssh_key_cb(CURL *easy, const struct curl_khkey *knownkey, + const struct curl_khkey *foundkey, int khmatch, void *clientp); +#endif +PYCURL_INTERNAL int +seek_callback(void *stream, curl_off_t offset, int origin); +PYCURL_INTERNAL size_t +read_callback(char *ptr, size_t size, size_t nmemb, void *stream); +PYCURL_INTERNAL int +progress_callback(void *stream, + double dltotal, double dlnow, double ultotal, double ulnow); +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 32, 0) +PYCURL_INTERNAL int +xferinfo_callback(void *stream, + curl_off_t dltotal, curl_off_t dlnow, + curl_off_t ultotal, curl_off_t ulnow); +#endif +PYCURL_INTERNAL int +debug_callback(CURL *curlobj, curl_infotype type, + char *buffer, size_t total_size, void *stream); +PYCURL_INTERNAL curlioerr +ioctl_callback(CURL *curlobj, int cmd, void *stream); +#if defined(HAVE_CURL_OPENSSL) +PYCURL_INTERNAL CURLcode +ssl_ctx_callback(CURL *curl, void *ssl_ctx, void *ptr); +#endif + +#if !defined(PYCURL_SINGLE_FILE) +/* Type objects */ +extern PyTypeObject Curl_Type; +extern PyTypeObject CurlSlist_Type; +extern PyTypeObject CurlHttppost_Type; +extern PyTypeObject CurlMulti_Type; +extern PyTypeObject CurlShare_Type; + +extern PyObject *ErrorObject; +extern PyTypeObject *p_Curl_Type; +extern PyTypeObject *p_CurlSlist_Type; +extern PyTypeObject *p_CurlHttppost_Type; +extern PyTypeObject *p_CurlMulti_Type; +extern PyTypeObject *p_CurlShare_Type; +extern PyObject *khkey_type; +extern PyObject *curl_sockaddr_type; + +extern PyObject *curlobject_constants; +extern PyObject *curlmultiobject_constants; +extern PyObject *curlshareobject_constants; + +extern char *g_pycurl_useragent; + +extern PYCURL_INTERNAL char *empty_keywords[]; +extern PYCURL_INTERNAL PyObject *bytesio; +extern PYCURL_INTERNAL PyObject *stringio; + +#if PY_MAJOR_VERSION >= 3 +extern PyMethodDef curlobject_methods[]; +extern PyMethodDef curlshareobject_methods[]; +extern PyMethodDef curlmultiobject_methods[]; +#endif +#endif /* !PYCURL_SINGLE_FILE */ + +#if PY_MAJOR_VERSION >= 3 +# define PYCURL_TYPE_FLAGS Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE +#else +# define PYCURL_TYPE_FLAGS Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_WEAKREFS | Py_TPFLAGS_BASETYPE +#endif + +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 8 +# define CPy_TRASHCAN_BEGIN(op, dealloc) Py_TRASHCAN_BEGIN(op, dealloc) +# define CPy_TRASHCAN_END(op) Py_TRASHCAN_END +#else +# define CPy_TRASHCAN_BEGIN(op, dealloc) Py_TRASHCAN_SAFE_BEGIN(op) +# define CPy_TRASHCAN_END(op) Py_TRASHCAN_SAFE_END(op) +#endif + +/* vi:ts=4:et:nowrap + */ diff --git a/src/pythoncompat.c b/src/pythoncompat.c new file mode 100644 index 0000000..4ef586b --- /dev/null +++ b/src/pythoncompat.c @@ -0,0 +1,128 @@ +#include "pycurl.h" + +#if PY_MAJOR_VERSION >= 3 + +PYCURL_INTERNAL PyObject * +my_getattro(PyObject *co, PyObject *name, PyObject *dict1, PyObject *dict2, PyMethodDef *m) +{ + PyObject *v = NULL; + if( dict1 != NULL ) + v = PyDict_GetItem(dict1, name); + if( v == NULL && dict2 != NULL ) + v = PyDict_GetItem(dict2, name); + if( v != NULL ) + { + Py_INCREF(v); + return v; + } + PyErr_Format(PyExc_AttributeError, "trying to obtain a non-existing attribute: %U", name); + return NULL; +} + +PYCURL_INTERNAL int +my_setattro(PyObject **dict, PyObject *name, PyObject *v) +{ + if( *dict == NULL ) + { + *dict = PyDict_New(); + if( *dict == NULL ) + return -1; + } + if (v != NULL) + return PyDict_SetItem(*dict, name, v); + else { + int v = PyDict_DelItem(*dict, name); + if (v != 0) { + /* need to convert KeyError to AttributeError */ + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Format(PyExc_AttributeError, "trying to delete a non-existing attribute: %U", name); + } + } + return v; + } +} + +#else /* PY_MAJOR_VERSION >= 3 */ + +PYCURL_INTERNAL int +my_setattr(PyObject **dict, char *name, PyObject *v) +{ + if (v == NULL) { + int rv = -1; + if (*dict != NULL) + rv = PyDict_DelItemString(*dict, name); + if (rv < 0) + PyErr_Format(PyExc_AttributeError, "trying to delete a non-existing attribute: %s", name); + return rv; + } + if (*dict == NULL) { + *dict = PyDict_New(); + if (*dict == NULL) + return -1; + } + return PyDict_SetItemString(*dict, name, v); +} + +PYCURL_INTERNAL PyObject * +my_getattr(PyObject *co, char *name, PyObject *dict1, PyObject *dict2, PyMethodDef *m) +{ + PyObject *v = NULL; + if (v == NULL && dict1 != NULL) + v = PyDict_GetItemString(dict1, name); + if (v == NULL && dict2 != NULL) + v = PyDict_GetItemString(dict2, name); + if (v != NULL) { + Py_INCREF(v); + return v; + } + return Py_FindMethod(m, co, name); +} + +#endif /* PY_MAJOR_VERSION >= 3 */ + +PYCURL_INTERNAL int +PyListOrTuple_Check(PyObject *v) +{ + int result; + + if (PyList_Check(v)) { + result = PYLISTORTUPLE_LIST; + } else if (PyTuple_Check(v)) { + result = PYLISTORTUPLE_TUPLE; + } else { + result = PYLISTORTUPLE_OTHER; + } + + return result; +} + +PYCURL_INTERNAL Py_ssize_t +PyListOrTuple_Size(PyObject *v, int which) +{ + switch (which) { + case PYLISTORTUPLE_LIST: + return PyList_Size(v); + case PYLISTORTUPLE_TUPLE: + return PyTuple_Size(v); + default: + assert(0); + return 0; + } +} + +PYCURL_INTERNAL PyObject * +PyListOrTuple_GetItem(PyObject *v, Py_ssize_t i, int which) +{ + switch (which) { + case PYLISTORTUPLE_LIST: + return PyList_GetItem(v, i); + case PYLISTORTUPLE_TUPLE: + return PyTuple_GetItem(v, i); + default: + assert(0); + return NULL; + } +} + +/* vi:ts=4:et:nowrap + */ diff --git a/src/share.c b/src/share.c new file mode 100644 index 0000000..94b25b4 --- /dev/null +++ b/src/share.c @@ -0,0 +1,348 @@ +#include "pycurl.h" +#include "docstrings.h" + +/************************************************************************* +// static utility functions +**************************************************************************/ + + +/* assert some CurlShareObject invariants */ +static void +assert_share_state(const CurlShareObject *self) +{ + assert(self != NULL); + assert(PyObject_IsInstance((PyObject *) self, (PyObject *) p_CurlShare_Type) == 1); +#ifdef WITH_THREAD + assert(self->lock != NULL); +#endif +} + + +static int +check_share_state(const CurlShareObject *self, int flags, const char *name) +{ + assert_share_state(self); + return 0; +} + + +/* constructor */ +PYCURL_INTERNAL CurlShareObject * +do_share_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) +{ + int res; + CurlShareObject *self; +#ifdef WITH_THREAD + const curl_lock_function lock_cb = share_lock_callback; + const curl_unlock_function unlock_cb = share_unlock_callback; +#endif + int *ptr; + + if (subtype == p_CurlShare_Type && !PyArg_ParseTupleAndKeywords(args, kwds, "", empty_keywords)) { + return NULL; + } + + /* Allocate python curl-share object */ + self = (CurlShareObject *) subtype->tp_alloc(subtype, 0); + if (!self) { + return NULL; + } + + /* tp_alloc is expected to return zeroed memory */ + for (ptr = (int *) &self->dict; + ptr < (int *) (((char *) self) + sizeof(CurlShareObject)); + ++ptr) { + assert(*ptr == 0); + } + +#ifdef WITH_THREAD + self->lock = share_lock_new(); + assert(self->lock != NULL); +#endif + + /* Allocate libcurl share handle */ + self->share_handle = curl_share_init(); + if (self->share_handle == NULL) { + Py_DECREF(self); + PyErr_SetString(ErrorObject, "initializing curl-share failed"); + return NULL; + } + +#ifdef WITH_THREAD + /* Set locking functions and data */ + res = curl_share_setopt(self->share_handle, CURLSHOPT_LOCKFUNC, lock_cb); + assert(res == CURLE_OK); + res = curl_share_setopt(self->share_handle, CURLSHOPT_USERDATA, self); + assert(res == CURLE_OK); + res = curl_share_setopt(self->share_handle, CURLSHOPT_UNLOCKFUNC, unlock_cb); + assert(res == CURLE_OK); +#endif + + return self; +} + + +PYCURL_INTERNAL int +do_share_traverse(CurlShareObject *self, visitproc visit, void *arg) +{ + int err; +#undef VISIT +#define VISIT(v) if ((v) != NULL && ((err = visit(v, arg)) != 0)) return err + + VISIT(self->dict); + + return 0; +#undef VISIT +} + + +/* Drop references that may have created reference cycles. */ +PYCURL_INTERNAL int +do_share_clear(CurlShareObject *self) +{ + Py_CLEAR(self->dict); + return 0; +} + + +static void +util_share_close(CurlShareObject *self){ + if (self->share_handle != NULL) { + CURLSH *share_handle = self->share_handle; + self->share_handle = NULL; + curl_share_cleanup(share_handle); + } +} + + +PYCURL_INTERNAL void +do_share_dealloc(CurlShareObject *self) +{ + PyObject_GC_UnTrack(self); + CPy_TRASHCAN_BEGIN(self, do_share_dealloc); + + Py_CLEAR(self->dict); + util_share_close(self); + +#ifdef WITH_THREAD + share_lock_destroy(self->lock); +#endif + + if (self->weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject *) self); + } + + CurlShare_Type.tp_free(self); + CPy_TRASHCAN_END(self); +} + + +static PyObject * +do_share_close(CurlShareObject *self) +{ + if (check_share_state(self, 2, "close") != 0) { + return NULL; + } + util_share_close(self); + Py_RETURN_NONE; +} + + +/* setopt, unsetopt*/ +/* --------------- unsetopt/setopt/getinfo --------------- */ + +static PyObject * +do_curlshare_setopt(CurlShareObject *self, PyObject *args) +{ + int option; + PyObject *obj; + + if (!PyArg_ParseTuple(args, "iO:setopt", &option, &obj)) + return NULL; + if (check_share_state(self, 1 | 2, "sharesetopt") != 0) + return NULL; + + /* early checks of option value */ + if (option <= 0) + goto error; + if (option >= (int)CURLOPTTYPE_OFF_T + OPTIONS_SIZE) + goto error; + if (option % 10000 >= OPTIONS_SIZE) + goto error; + +#if 0 /* XXX - should we ??? */ + /* Handle the case of None */ + if (obj == Py_None) { + return util_curl_unsetopt(self, option); + } +#endif + + /* Handle the case of integer arguments */ + if (PyInt_Check(obj)) { + long d = PyInt_AsLong(obj); + switch(d) { + case CURL_LOCK_DATA_COOKIE: + case CURL_LOCK_DATA_DNS: + case CURL_LOCK_DATA_SSL_SESSION: +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 57, 0) + case CURL_LOCK_DATA_CONNECT: +#endif +#if LIBCURL_VERSION_NUM >= MAKE_LIBCURL_VERSION(7, 61, 0) + case CURL_LOCK_DATA_PSL: +#endif + break; + default: + goto error; + } + switch(option) { + case CURLSHOPT_SHARE: + case CURLSHOPT_UNSHARE: + curl_share_setopt(self->share_handle, option, d); + break; + default: + PyErr_SetString(PyExc_TypeError, "integers are not supported for this option"); + return NULL; + } + Py_RETURN_NONE; + } + /* Failed to match any of the function signatures -- return error */ +error: + PyErr_SetString(PyExc_TypeError, "invalid arguments to setopt"); + return NULL; +} + + +static PyObject *do_curlshare_getstate(CurlShareObject *self) +{ + PyErr_SetString(PyExc_TypeError, "CurlShare objects do not support serialization"); + return NULL; +} + + +static PyObject *do_curlshare_setstate(CurlShareObject *self, PyObject *args) +{ + PyErr_SetString(PyExc_TypeError, "CurlShare objects do not support deserialization"); + return NULL; +} + + +/************************************************************************* +// type definitions +**************************************************************************/ + +/* --------------- methods --------------- */ + +PYCURL_INTERNAL PyMethodDef curlshareobject_methods[] = { + {"close", (PyCFunction)do_share_close, METH_NOARGS, share_close_doc}, + {"setopt", (PyCFunction)do_curlshare_setopt, METH_VARARGS, share_setopt_doc}, + {"__getstate__", (PyCFunction)do_curlshare_getstate, METH_NOARGS, NULL}, + {"__setstate__", (PyCFunction)do_curlshare_setstate, METH_VARARGS, NULL}, + {NULL, NULL, 0, 0} +}; + + +/* --------------- setattr/getattr --------------- */ + + +#if PY_MAJOR_VERSION >= 3 + +PYCURL_INTERNAL PyObject * +do_share_getattro(PyObject *o, PyObject *n) +{ + PyObject *v; + assert_share_state((CurlShareObject *)o); + v = PyObject_GenericGetAttr(o, n); + if( !v && PyErr_ExceptionMatches(PyExc_AttributeError) ) + { + PyErr_Clear(); + v = my_getattro(o, n, ((CurlShareObject *)o)->dict, + curlshareobject_constants, curlshareobject_methods); + } + return v; +} + +PYCURL_INTERNAL int +do_share_setattro(PyObject *o, PyObject *n, PyObject *v) +{ + assert_share_state((CurlShareObject *)o); + return my_setattro(&((CurlShareObject *)o)->dict, n, v); +} + +#else /* PY_MAJOR_VERSION >= 3 */ + +PYCURL_INTERNAL PyObject * +do_share_getattr(CurlShareObject *cso, char *name) +{ + assert_share_state(cso); + return my_getattr((PyObject *)cso, name, cso->dict, + curlshareobject_constants, curlshareobject_methods); +} + +PYCURL_INTERNAL int +do_share_setattr(CurlShareObject *so, char *name, PyObject *v) +{ + assert_share_state(so); + return my_setattr(&so->dict, name, v); +} + +#endif /* PY_MAJOR_VERSION >= 3 */ + +PYCURL_INTERNAL PyTypeObject CurlShare_Type = { +#if PY_MAJOR_VERSION >= 3 + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ +#endif + "pycurl.CurlShare", /* tp_name */ + sizeof(CurlShareObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)do_share_dealloc, /* tp_dealloc */ + 0, /* tp_print */ +#if PY_MAJOR_VERSION >= 3 + 0, /* tp_getattr */ + 0, /* tp_setattr */ +#else + (getattrfunc)do_share_getattr, /* tp_getattr */ + (setattrfunc)do_share_setattr, /* tp_setattr */ +#endif + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ +#if PY_MAJOR_VERSION >= 3 + (getattrofunc)do_share_getattro, /* tp_getattro */ + (setattrofunc)do_share_setattro, /* tp_setattro */ +#else + 0, /* tp_getattro */ + 0, /* tp_setattro */ +#endif + 0, /* tp_as_buffer */ + PYCURL_TYPE_FLAGS, /* tp_flags */ + share_doc, /* tp_doc */ + (traverseproc)do_share_traverse, /* tp_traverse */ + (inquiry)do_share_clear, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(CurlShareObject, weakreflist), /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + curlshareobject_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + (newfunc)do_share_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; + +/* vi:ts=4:et:nowrap + */ diff --git a/src/stringcompat.c b/src/stringcompat.c new file mode 100644 index 0000000..673b370 --- /dev/null +++ b/src/stringcompat.c @@ -0,0 +1,86 @@ +#include "pycurl.h" + +/************************************************************************* +// python utility functions +**************************************************************************/ + +PYCURL_INTERNAL int +PyText_AsStringAndSize(PyObject *obj, char **buffer, Py_ssize_t *length, PyObject **encoded_obj) +{ + if (PyByteStr_Check(obj)) { + *encoded_obj = NULL; + return PyByteStr_AsStringAndSize(obj, buffer, length); + } else { + int rv; + assert(PyUnicode_Check(obj)); + *encoded_obj = PyUnicode_AsEncodedString(obj, "ascii", "strict"); + if (*encoded_obj == NULL) { + return -1; + } + rv = PyByteStr_AsStringAndSize(*encoded_obj, buffer, length); + if (rv != 0) { + /* If we free the object, pointer must be reset to NULL */ + Py_CLEAR(*encoded_obj); + } + return rv; + } +} + + +/* Like PyString_AsString(), but set an exception if the string contains + * embedded NULs. Actually PyString_AsStringAndSize() already does that for + * us if the `len' parameter is NULL - see Objects/stringobject.c. + */ + +PYCURL_INTERNAL char * +PyText_AsString_NoNUL(PyObject *obj, PyObject **encoded_obj) +{ + char *s = NULL; + Py_ssize_t r; + r = PyText_AsStringAndSize(obj, &s, NULL, encoded_obj); + if (r != 0) + return NULL; /* exception already set */ + assert(s != NULL); + return s; +} + + +/* Returns true if the object is of a type that can be given to + * curl_easy_setopt and such - either a byte string or a Unicode string + * with ASCII code points only. + */ +#if PY_MAJOR_VERSION >= 3 +PYCURL_INTERNAL int +PyText_Check(PyObject *o) +{ + return PyUnicode_Check(o) || PyBytes_Check(o); +} +#else +PYCURL_INTERNAL int +PyText_Check(PyObject *o) +{ + return PyUnicode_Check(o) || PyString_Check(o); +} +#endif + +PYCURL_INTERNAL PyObject * +PyText_FromString_Ignore(const char *string) +{ + PyObject *v; + +#if PY_MAJOR_VERSION >= 3 + PyObject *u; + + v = Py_BuildValue("y", string); + if (v == NULL) { + return NULL; + } + + u = PyUnicode_FromEncodedObject(v, NULL, "replace"); + Py_DECREF(v); + return u; +#else + v = Py_BuildValue("s", string); + return v; +#endif +} diff --git a/src/threadsupport.c b/src/threadsupport.c new file mode 100644 index 0000000..b188872 --- /dev/null +++ b/src/threadsupport.c @@ -0,0 +1,365 @@ +#include "pycurl.h" + +#ifdef WITH_THREAD + +PYCURL_INTERNAL PyThreadState * +pycurl_get_thread_state(const CurlObject *self) +{ + /* Get the thread state for callbacks to run in. + * This is either `self->state' when running inside perform() or + * `self->multi_stack->state' when running inside multi_perform(). + * When the result is != NULL we also implicitly assert + * a valid `self->handle'. + */ + if (self == NULL) + return NULL; + assert(PyObject_IsInstance((PyObject *) self, (PyObject *) p_Curl_Type) == 1); + if (self->state != NULL) + { + /* inside perform() */ + assert(self->handle != NULL); + if (self->multi_stack != NULL) { + assert(self->multi_stack->state == NULL); + } + return self->state; + } + if (self->multi_stack != NULL && self->multi_stack->state != NULL) + { + /* inside multi_perform() */ + assert(self->handle != NULL); + assert(self->multi_stack->multi_handle != NULL); + assert(self->state == NULL); + return self->multi_stack->state; + } + return NULL; +} + + +PYCURL_INTERNAL PyThreadState * +pycurl_get_thread_state_multi(const CurlMultiObject *self) +{ + /* Get the thread state for callbacks to run in when given + * multi handles instead of regular handles + */ + if (self == NULL) + return NULL; + assert(PyObject_IsInstance((PyObject *) self, (PyObject *) p_CurlMulti_Type) == 1); + if (self->state != NULL) + { + /* inside multi_perform() */ + assert(self->multi_handle != NULL); + return self->state; + } + return NULL; +} + + +PYCURL_INTERNAL int +pycurl_acquire_thread(const CurlObject *self, PyThreadState **state) +{ + *state = pycurl_get_thread_state(self); + if (*state == NULL) + return 0; + PyEval_AcquireThread(*state); + return 1; +} + + +PYCURL_INTERNAL int +pycurl_acquire_thread_multi(const CurlMultiObject *self, PyThreadState **state) +{ + *state = pycurl_get_thread_state_multi(self); + if (*state == NULL) + return 0; + PyEval_AcquireThread(*state); + return 1; +} + + +PYCURL_INTERNAL void +pycurl_release_thread(PyThreadState *state) +{ + PyEval_ReleaseThread(state); +} + +/************************************************************************* +// SSL TSL +**************************************************************************/ + +#ifdef PYCURL_NEED_OPENSSL_TSL + +#if OPENSSL_VERSION_NUMBER < 0x10100000 +static PyThread_type_lock *pycurl_openssl_tsl = NULL; + +static void +pycurl_ssl_lock(int mode, int n, const char * file, int line) +{ + if (mode & CRYPTO_LOCK) { + PyThread_acquire_lock(pycurl_openssl_tsl[n], 1); + } else { + PyThread_release_lock(pycurl_openssl_tsl[n]); + } +} + +#if OPENSSL_VERSION_NUMBER >= 0x10000000 +/* use new CRYPTO_THREADID API. */ +static void +pycurl_ssl_threadid_callback(CRYPTO_THREADID *id) +{ + CRYPTO_THREADID_set_numeric(id, (unsigned long)PyThread_get_thread_ident()); +} +#else +/* deprecated CRYPTO_set_id_callback() API. */ +static unsigned long +pycurl_ssl_id(void) +{ + return (unsigned long) PyThread_get_thread_ident(); +} +#endif +#endif + +PYCURL_INTERNAL int +pycurl_ssl_init(void) +{ +#if OPENSSL_VERSION_NUMBER < 0x10100000 + int i, c = CRYPTO_num_locks(); + + pycurl_openssl_tsl = PyMem_New(PyThread_type_lock, c); + if (pycurl_openssl_tsl == NULL) { + PyErr_NoMemory(); + return -1; + } + memset(pycurl_openssl_tsl, 0, sizeof(PyThread_type_lock) * c); + + for (i = 0; i < c; ++i) { + pycurl_openssl_tsl[i] = PyThread_allocate_lock(); + if (pycurl_openssl_tsl[i] == NULL) { + for (--i; i >= 0; --i) { + PyThread_free_lock(pycurl_openssl_tsl[i]); + } + PyMem_Free(pycurl_openssl_tsl); + PyErr_NoMemory(); + return -1; + } + } + +#if OPENSSL_VERSION_NUMBER >= 0x10000000 + CRYPTO_THREADID_set_callback(pycurl_ssl_threadid_callback); +#else + CRYPTO_set_id_callback(pycurl_ssl_id); +#endif + CRYPTO_set_locking_callback(pycurl_ssl_lock); +#endif + return 0; +} + +PYCURL_INTERNAL void +pycurl_ssl_cleanup(void) +{ +#if OPENSSL_VERSION_NUMBER < 0x10100000 + if (pycurl_openssl_tsl) { + int i, c = CRYPTO_num_locks(); + +#if OPENSSL_VERSION_NUMBER >= 0x10000000 + CRYPTO_THREADID_set_callback(NULL); +#else + CRYPTO_set_id_callback(NULL); +#endif + CRYPTO_set_locking_callback(NULL); + + for (i = 0; i < c; ++i) { + PyThread_free_lock(pycurl_openssl_tsl[i]); + } + + PyMem_Free(pycurl_openssl_tsl); + pycurl_openssl_tsl = NULL; + } +#endif +} +#endif + +#ifdef PYCURL_NEED_GNUTLS_TSL +static int +pycurl_ssl_mutex_create(void **m) +{ + if ((*((PyThread_type_lock *) m) = PyThread_allocate_lock()) == NULL) { + return -1; + } else { + return 0; + } +} + +static int +pycurl_ssl_mutex_destroy(void **m) +{ + PyThread_free_lock(*((PyThread_type_lock *) m)); + return 0; +} + +static int +pycurl_ssl_mutex_lock(void **m) +{ + return !PyThread_acquire_lock(*((PyThread_type_lock *) m), 1); +} + +static int +pycurl_ssl_mutex_unlock(void **m) +{ + PyThread_release_lock(*((PyThread_type_lock *) m)); + return 0; +} + +static struct gcry_thread_cbs pycurl_gnutls_tsl = { + GCRY_THREAD_OPTION_USER, + NULL, + pycurl_ssl_mutex_create, + pycurl_ssl_mutex_destroy, + pycurl_ssl_mutex_lock, + pycurl_ssl_mutex_unlock +}; + +PYCURL_INTERNAL int +pycurl_ssl_init(void) +{ + gcry_control(GCRYCTL_SET_THREAD_CBS, &pycurl_gnutls_tsl); + return 0; +} + +PYCURL_INTERNAL void +pycurl_ssl_cleanup(void) +{ + return; +} +#endif + +/* mbedTLS */ + +#ifdef PYCURL_NEED_MBEDTLS_TSL +static int +pycurl_ssl_mutex_create(void **m) +{ + if ((*((PyThread_type_lock *) m) = PyThread_allocate_lock()) == NULL) { + return -1; + } else { + return 0; + } +} + +static int +pycurl_ssl_mutex_destroy(void **m) +{ + PyThread_free_lock(*((PyThread_type_lock *) m)); + return 0; +} + +static int +pycurl_ssl_mutex_lock(void **m) +{ + return !PyThread_acquire_lock(*((PyThread_type_lock *) m), 1); +} + +PYCURL_INTERNAL int +pycurl_ssl_init(void) +{ + return 0; +} + +PYCURL_INTERNAL void +pycurl_ssl_cleanup(void) +{ + return; +} +#endif + +/************************************************************************* +// CurlShareObject +**************************************************************************/ + +PYCURL_INTERNAL void +share_lock_lock(ShareLock *lock, curl_lock_data data) +{ + PyThread_acquire_lock(lock->locks[data], 1); +} + +PYCURL_INTERNAL void +share_lock_unlock(ShareLock *lock, curl_lock_data data) +{ + PyThread_release_lock(lock->locks[data]); +} + +PYCURL_INTERNAL ShareLock * +share_lock_new(void) +{ + int i; + ShareLock *lock = PyMem_New(ShareLock, 1); + if (lock == NULL) { + PyErr_NoMemory(); + return NULL; + } + + for (i = 0; i < CURL_LOCK_DATA_LAST; ++i) { + lock->locks[i] = PyThread_allocate_lock(); + if (lock->locks[i] == NULL) { + PyErr_NoMemory(); + goto error; + } + } + return lock; + +error: + for (--i; i >= 0; --i) { + PyThread_free_lock(lock->locks[i]); + lock->locks[i] = NULL; + } + PyMem_Free(lock); + return NULL; +} + +PYCURL_INTERNAL void +share_lock_destroy(ShareLock *lock) +{ + int i; + + assert(lock); + for (i = 0; i < CURL_LOCK_DATA_LAST; ++i){ + assert(lock->locks[i] != NULL); + PyThread_free_lock(lock->locks[i]); + } + PyMem_Free(lock); + lock = NULL; +} + +PYCURL_INTERNAL void +share_lock_callback(CURL *handle, curl_lock_data data, curl_lock_access locktype, void *userptr) +{ + CurlShareObject *share = (CurlShareObject*)userptr; + share_lock_lock(share->lock, data); +} + +PYCURL_INTERNAL void +share_unlock_callback(CURL *handle, curl_lock_data data, void *userptr) +{ + CurlShareObject *share = (CurlShareObject*)userptr; + share_lock_unlock(share->lock, data); +} + +#else /* WITH_THREAD */ + +#if defined(PYCURL_NEED_SSL_TSL) +PYCURL_INTERNAL void +pycurl_ssl_init(void) +{ + return 0; +} + +PYCURL_INTERNAL void +pycurl_ssl_cleanup(void) +{ + return; +} +#endif + +#endif /* WITH_THREAD */ + +/* vi:ts=4:et:nowrap + */ diff --git a/src/util.c b/src/util.c new file mode 100644 index 0000000..f00416f --- /dev/null +++ b/src/util.c @@ -0,0 +1,31 @@ +#include "pycurl.h" + +static PyObject * +create_error_object(CurlObject *self, int code) +{ + PyObject *s, *v; + + s = PyText_FromString_Ignore(self->error); + if (s == NULL) { + return NULL; + } + v = Py_BuildValue("(iO)", code, s); + if (v == NULL) { + Py_DECREF(s); + return NULL; + } + return v; +} + +PYCURL_INTERNAL void +create_and_set_error_object(CurlObject *self, int code) +{ + PyObject *e; + + self->error[sizeof(self->error) - 1] = 0; + e = create_error_object(self, code); + if (e != NULL) { + PyErr_SetObject(ErrorObject, e); + Py_DECREF(e); + } +} diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..663ec2f --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,13 @@ +# On recent windowses there is no localhost entry in hosts file, +# hence localhost resolves fail. https://github.com/c-ares/c-ares/issues/85 +# FTP tests also seem to want the numeric IP address rather than localhost. +localhost = '127.0.0.1' + +def setup_package(): + # import here, not globally, so that running + # python -m tests.appmanager + # to launch the app manager is possible without having pycurl installed + # (as the test app does not depend on pycurl) + import pycurl + + print('Testing %s' % pycurl.version) diff --git a/tests/app.py b/tests/app.py new file mode 100644 index 0000000..62ff574 --- /dev/null +++ b/tests/app.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import time as _time, sys +import bottle +try: + import json +except ImportError: + import simplejson as json + +py3 = sys.version_info[0] == 3 + +app = bottle.Bottle() +app.debug = True + +@app.route('/success') +def ok(): + return 'success' + +@app.route('/short_wait') +def short_wait(): + _time.sleep(0.1) + return 'success' + +@app.route('/status/403') +def forbidden(): + return bottle.HTTPResponse('forbidden', 403) + +@app.route('/status/404') +def not_found(): + return bottle.HTTPResponse('not found', 404) + +@app.route('/postfields', method='get') +@app.route('/postfields', method='post') +def postfields(): + return json.dumps(dict(bottle.request.forms)) + +@app.route('/raw_utf8', method='post') +def raw_utf8(): + data = bottle.request.body.getvalue().decode('utf8') + return json.dumps(data) + +# XXX file is not a bottle FileUpload instance, but FieldStorage? +def xconvert_file(key, file): + return { + 'key': key, + 'name': file.name, + 'raw_filename': file.raw_filename, + 'headers': file.headers, + 'content_type': file.content_type, + 'content_length': file.content_length, + 'data': file.read(), + } + +if hasattr(bottle, 'FileUpload'): + # bottle 0.12 + def convert_file(key, file): + return { + 'name': file.name, + # file.filename lowercases the file name + # https://github.com/defnull/bottle/issues/582 + # raw_filenames is a string on python 3 + 'filename': file.raw_filename, + 'data': file.file.read().decode(), + } +else: + # bottle 0.11 + def convert_file(key, file): + return { + 'name': file.name, + 'filename': file.filename, + 'data': file.file.read().decode(), + } + +@app.route('/files', method='post') +def files(): + files = [convert_file(key, bottle.request.files[key]) for key in bottle.request.files] + return json.dumps(files) + +@app.route('/header') +def header(): + return bottle.request.headers.get(bottle.request.query['h'], '') + +# This is a hacky endpoint to test non-ascii text being given to libcurl +# via headers. +# HTTP RFC requires headers to be latin1-encoded. +# Any string can be decoded as latin1; here we encode the header value +# back into latin1 to obtain original bytestring, then decode it in utf-8. +# Thanks to bdarnell for the idea: https://github.com/pycurl/pycurl/issues/124 +@app.route('/header_utf8') +def header_utf8(): + header_value = bottle.request.headers.get(bottle.request.query['h'], '' if py3 else b'') + if py3: + # header_value is a string, headers are decoded in latin1 + header_value = header_value.encode('latin1').decode('utf8') + else: + # header_value is a binary string, decode in utf-8 directly + header_value = header_value.decode('utf8') + return header_value + +@app.route('/param_utf8_hack', method='post') +def param_utf8_hack(): + param = bottle.request.forms['p'] + if py3: + # python 3 decodes bytes as latin1 perhaps? + # apply the latin1-utf8 hack + param = param.encode('latin').decode('utf8') + return param + +def pause_writer(interval): + yield 'part1' + _time.sleep(interval) + yield 'part2' + +@app.route('/pause') +def pause(): + return pause_writer(0.5) + +@app.route('/long_pause') +def long_pause(): + return pause_writer(1) + +@app.route('/utf8_body') +def utf8_body(): + # bottle encodes the body + return 'Дружба народов' + +@app.route('/invalid_utf8_body') +def invalid_utf8_body(): + # bottle encodes the body + raise bottle.HTTPResponse(b'\xb3\xd2\xda\xcd\xd7', 200) + +@app.route('/set_cookie_invalid_utf8') +def set_cookie_invalid_utf8(): + bottle.response.set_header('Set-Cookie', '\xb3\xd2\xda\xcd\xd7=%96%A6g%9Ay%B0%A5g%A7tm%7C%95%9A') + return 'cookie set' + +@app.route('/content_type_invalid_utf8') +def content_type_invalid_utf8(): + bottle.response.set_header('Content-Type', '\xb3\xd2\xda\xcd\xd7') + return 'content type set' + +@app.route('/status_invalid_utf8') +def status_invalid_utf8(): + raise bottle.HTTPResponse('status set', '555 \xb3\xd2\xda\xcd\xd7') diff --git a/tests/appmanager.py b/tests/appmanager.py new file mode 100644 index 0000000..1857810 --- /dev/null +++ b/tests/appmanager.py @@ -0,0 +1,57 @@ +import sys, time, os + +def noop(*args): + pass + +def setup(*specs): + if os.environ.get('PYCURL_STANDALONE_APP') and os.environ['PYCURL_STANDALONE_APP'].lower() in ['1', 'yes', 'true']: + return (noop, noop) + else: + return perform_setup(*specs) + +def perform_setup(*specs): + from . import runwsgi + + app_specs = [] + for spec in specs: + app_module = __import__(spec[0], globals(), locals(), ['app'], 1) + app = getattr(app_module, 'app') + app_specs.append([app] + list(spec[1:])) + + return runwsgi.app_runner_setup(*app_specs) + +quit = False + +def sigterm_handler(*args): + global quit + quit = True + +def run_standalone(): + import signal + + funcs = [] + + signal.signal(signal.SIGTERM, sigterm_handler) + + funcs.append(setup(('app', 8380))) + funcs.append(setup(('app', 8381))) + funcs.append(setup(('app', 8382))) + funcs.append(setup(('app', 8383, dict(ssl=True)))) + funcs.append(setup(('app', 8384, dict(ssl=True)))) + + for setup_func, teardown_func in funcs: + setup_func(sys.modules[__name__]) + + sys.stdout.write("Running, use SIGTERM or SIGINT to stop\n") + + try: + while not quit: + time.sleep(1) + except KeyboardInterrupt: + pass + + for setup_func, teardown_func in funcs: + teardown_func(sys.modules[__name__]) + +if __name__ == '__main__': + run_standalone() diff --git a/tests/cadata_test.py b/tests/cadata_test.py new file mode 100644 index 0000000..32adc2e --- /dev/null +++ b/tests/cadata_test.py @@ -0,0 +1,43 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import os +import pycurl +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8384, dict(ssl=True))) + +class CaCertsTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurlLocalhost(8384) + + def tearDown(self): + self.curl.close() + + @util.only_ssl_backends('openssl') + def test_request_with_verifypeer(self): + with open(os.path.join(os.path.dirname(__file__), 'certs', 'ca.crt'), 'rb') as stream: + cadata = stream.read().decode('ASCII') + self.curl.setopt(pycurl.URL, 'https://localhost:8384/success') + sio = util.BytesIO() + self.curl.set_ca_certs(cadata) + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + # self signed certificate, but ca cert should be loaded + self.curl.setopt(pycurl.SSL_VERIFYPEER, 1) + self.curl.perform() + assert sio.getvalue().decode() == 'success' + + @util.only_ssl_backends('openssl') + def test_set_ca_certs_bytes(self): + self.curl.set_ca_certs(util.b('hello world\x02\xe0')) + + @util.only_ssl_backends('openssl') + def test_set_ca_certs_bogus_type(self): + try: + self.curl.set_ca_certs(42) + except TypeError as e: + self.assertEqual('set_ca_certs argument must be a byte string or a Unicode string with ASCII code points only', str(e)) diff --git a/tests/certinfo_test.py b/tests/certinfo_test.py new file mode 100644 index 0000000..4662b19 --- /dev/null +++ b/tests/certinfo_test.py @@ -0,0 +1,94 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8383, dict(ssl=True))) + +class CertinfoTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurlLocalhost(8383) + + def tearDown(self): + self.curl.close() + + # CURLOPT_CERTINFO was introduced in libcurl-7.19.1 + @util.min_libcurl(7, 19, 1) + def test_certinfo_option(self): + assert hasattr(pycurl, 'OPT_CERTINFO') + + # CURLOPT_CERTINFO was introduced in libcurl-7.19.1 + @util.min_libcurl(7, 19, 1) + @util.only_ssl + def test_request_without_certinfo(self): + self.curl.setopt(pycurl.URL, 'https://localhost:8383/success') + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + # self signed certificate + self.curl.setopt(pycurl.SSL_VERIFYPEER, 0) + self.curl.perform() + assert sio.getvalue().decode() == 'success' + + certinfo = self.curl.getinfo(pycurl.INFO_CERTINFO) + self.assertEqual([], certinfo) + + # CURLOPT_CERTINFO was introduced in libcurl-7.19.1 + @util.min_libcurl(7, 19, 1) + @util.only_ssl + def test_request_with_certinfo(self): + # CURLOPT_CERTINFO only works with OpenSSL + if 'openssl' not in pycurl.version.lower(): + raise unittest.SkipTest('libcurl does not use openssl') + + self.curl.setopt(pycurl.URL, 'https://localhost:8383/success') + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.setopt(pycurl.OPT_CERTINFO, 1) + # self signed certificate + self.curl.setopt(pycurl.SSL_VERIFYPEER, 0) + self.curl.perform() + assert sio.getvalue().decode() == 'success' + + certinfo = self.curl.getinfo(pycurl.INFO_CERTINFO) + # self signed certificate, one certificate in chain + assert len(certinfo) == 1 + certinfo = certinfo[0] + # convert to a dictionary + certinfo_dict = {} + for entry in certinfo: + certinfo_dict[entry[0]] = entry[1] + assert util.u('Subject') in certinfo_dict + assert util.u('PycURL test suite') in certinfo_dict[util.u('Subject')] + + # CURLOPT_CERTINFO was introduced in libcurl-7.19.1 + @util.min_libcurl(7, 19, 1) + @util.only_ssl + def test_getinfo_raw_certinfo(self): + # CURLOPT_CERTINFO only works with OpenSSL + if 'openssl' not in pycurl.version.lower(): + raise unittest.SkipTest('libcurl does not use openssl') + + self.curl.setopt(pycurl.URL, 'https://localhost:8383/success') + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.setopt(pycurl.OPT_CERTINFO, 1) + # self signed certificate + self.curl.setopt(pycurl.SSL_VERIFYPEER, 0) + self.curl.perform() + assert sio.getvalue().decode() == 'success' + + certinfo = self.curl.getinfo_raw(pycurl.INFO_CERTINFO) + # self signed certificate, one certificate in chain + assert len(certinfo) == 1 + certinfo = certinfo[0] + # convert to a dictionary + certinfo_dict = {} + for entry in certinfo: + certinfo_dict[entry[0]] = entry[1] + assert util.b('Subject') in certinfo_dict + assert util.b('PycURL test suite') in certinfo_dict[util.b('Subject')] diff --git a/tests/certs/ca.crt b/tests/certs/ca.crt new file mode 100644 index 0000000..83cf2a5 --- /dev/null +++ b/tests/certs/ca.crt @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDwzCCAqugAwIBAgIUSxsCNrFED1qO/AQe5iz0sFzgdRowDQYJKoZIhvcNAQEL +BQAwZjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxGjAYBgNVBAoM +EVB5Y1VSTCB0ZXN0IHN1aXRlMRIwEAYDVQQLDAlsb2NhbGhvc3QxEjAQBgNVBAMM +CWxvY2FsaG9zdDAeFw0yMjA1MDYxNTI5MDVaFw00OTA5MjExNTI5MDVaMGYxCzAJ +BgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMRowGAYDVQQKDBFQeWNVUkwg +dGVzdCBzdWl0ZTESMBAGA1UECwwJbG9jYWxob3N0MRIwEAYDVQQDDAlsb2NhbGhv +c3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtrtJgPWnNZCHEmvJg +p/0C4o8l8tqaRypjTrZgFSChIMLHRmMV8xNra7TOobZW3YRO69WQOEEDD/QAXu8s +KFT22MA0pe6FEgrMauoST/A4wXG1tVgbBz5W58Hc0EHd85cWPB7/IA/k39nDj4/c +uSfg4BVEW+lGs2FGCLElRWmrOPPMQsyP5llwuVhaRQ5QN8wQgkd5n2wXF2tsQ2dO +YmJ5fVDjs0P0f0TNCWhS9zxd/orV7UqWIiGWiZt2jdEsAZTNmVaUbZaisXNfXrUT +aFjYUcUh31K6xYc0nEqyY5R6s2/StZh7Png47BdaH/Y4pw1XWErUgUqQQdQ/tQwu +G8BTAgMBAAGjaTBnMB0GA1UdDgQWBBQu1NQH28vT0GEQe3ZIFM2R/ZpGwTAfBgNV +HSMEGDAWgBQu1NQH28vT0GEQe3ZIFM2R/ZpGwTAPBgNVHRMBAf8EBTADAQH/MBQG +A1UdEQQNMAuCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAQOYupzgDcLn3 +dT7lPXDrWSWFQQoNGDD3suu3UPIHuLIBTghADOTgAW9QxmcB5z7EWXj8TLRssAZ7 +6bwPR1g466IgDpR7U+q9YIyBVW98MgYyaSX6TuRyrRxEugH5mzIKQ7Ed9qYiJrVA +5npHkQOXdlcdsjLsjD/itfbk/M13GuCMkixXM3RGcruUbd143aKFGVvGPhl31L14 +AOjQsbC1OQQ6rPsA5FObSqLjm6L+cq53PqOaRIiTRdiaKD2Xj8hXWrs5zivx60qM +GQkUnTfyPuQ7EXf7jQrjwrCYN8lk+KbuE9FKKS89b6ZyfK7iZp6AR3IdYRAVo1FE +H7ZnvduIZQ== +-----END CERTIFICATE----- diff --git a/tests/certs/ca.key b/tests/certs/ca.key new file mode 100644 index 0000000..75a9ba4 --- /dev/null +++ b/tests/certs/ca.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAra7SYD1pzWQhxJryYKf9AuKPJfLamkcqY062YBUgoSDCx0Zj +FfMTa2u0zqG2Vt2ETuvVkDhBAw/0AF7vLChU9tjANKXuhRIKzGrqEk/wOMFxtbVY +Gwc+VufB3NBB3fOXFjwe/yAP5N/Zw4+P3Lkn4OAVRFvpRrNhRgixJUVpqzjzzELM +j+ZZcLlYWkUOUDfMEIJHeZ9sFxdrbENnTmJieX1Q47ND9H9EzQloUvc8Xf6K1e1K +liIhlombdo3RLAGUzZlWlG2WorFzX161E2hY2FHFId9SusWHNJxKsmOUerNv0rWY +ez54OOwXWh/2OKcNV1hK1IFKkEHUP7UMLhvAUwIDAQABAoIBAGLsEpiMAgngwTbo +hao1o+6TubKEiquaYvMi7s702ZvMPAQh++eRhfsF4npaMq9xBZ2pxv6Ye7bRzEi1 +yYWeBx59P6P86khSiWH6dw0tCIZa73fuLJtgWcpHv+wTlaBj0Cby4TiwOz1Bnhc7 +WlX+A0+acaJ4svn4yyuHYdX3ngLNwe0WkP0a6M0KOv+rxRW4FFITrO98Yz9PtPZP +Z4nLvt6dsX6m879WIcFA+wkab4aMrWMI6b80wKN72sKAVC3LNU4iJ7PTYmbyIwJP +YDOD/+UAp4T/VV+CIKOlS6YEozMpOD4kt4SxzEZQu8cuAovtKdKcOn0WhJYBJVGc +LCJpSfkCgYEA2+JC/VCtF91d4/70S+Jswqc3T3ldifZXYRVH+iFeBSGrg++GHeKW +JKRk0+v1Ul+P/6Ygd3ycsw/leKUr5c7+Fi3z2vk8q0i0815yQJuqFJIg+WTbeutv +DiUj3UTycvFQtAreEMyAMB99fWmm4CuAuYVQIHojhm5SI3lQUjhBnu8CgYEAyjXj +0DygA8obXQaB2ljoj75ukDMN+0JCmZ+WqI2+obREAR2f+X7wHtbdwyQiwRBIV+b7 +wCnAXAACXFDEoFboN+0Ex7aFVjAc2BawdHmwilhxk2nYaCTgxQZLYUzEWSNp20WA +BTUU5Tjs4fdDoKT7G6xMEaBqvooaImXxFVmjNN0CgYBeJPZBt3UlLqawo8y9YOjo +Pugzoucl1s96xb3Xnsm+sLfa+YcW7JkUfz6cbf7PkhL5houIHVaKZFf/29h7wLCR +loM+UlBjlfHD8cBBYWTlAdwUa9Z9PqiCCezdJFQaWrAPJkgGMUkBUbpNJBtLB9VJ +mYbBIQps2HdasOpvCZ8vCQKBgQDF9jwxgSimjRZ83AIEYUZMc4KKaXEmqpfJDhPQ +r/QRGwn4jagv+bXae0Bf6uCbYfVxGREd78ICT4AAIJJe5rYxCjnDy0x+NFwIsS3O +2dObnTqTtuvGCVSDjsX9W8pd+e2IXWIXtv/d6Pz/u7LZcqrjTKqsFwBpyYoMYwDC +hh7hgQKBgCI2wr3QrDfabD9vZd/pO8v7jD3mk84fj4pO6D1c8wg4n6IffOqQih/z +1AU/RBIUIPERJweNGeG+YOlA2pE02u/J/0UpH5663vM76GQ7nY/Vr6rd4OcNJsR3 +xLlOz8XMzkqt+BcTsLfzjO4wAFEutUywDrT8DBkQR5nuUqHjVj8f +-----END RSA PRIVATE KEY----- diff --git a/tests/certs/server.crt b/tests/certs/server.crt new file mode 100644 index 0000000..57b6036 --- /dev/null +++ b/tests/certs/server.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDXjCCAkagAwIBAgIUGSiMniv/FqlCAJXut6b+TJx1gzYwDQYJKoZIhvcNAQEL +BQAwZjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxGjAYBgNVBAoM +EVB5Y1VSTCB0ZXN0IHN1aXRlMRIwEAYDVQQLDAlsb2NhbGhvc3QxEjAQBgNVBAMM +CWxvY2FsaG9zdDAeFw0yMjA1MDYyMTU4NTJaFw00OTA5MjEyMTU4NTJaMFIxCzAJ +BgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMRowGAYDVQQKDBFQeWNVUkwg +dGVzdCBzdWl0ZTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEA5icu/uU40MkMpthiEhxV9Y1F2p8aT6EWk8dWLxF8Pj2D +3RYp1K+GYis8K1+h5dVm9KCH61e4NlGK5zO6hGrS44zimQ0xfJBqeUOfkAW2EE2c +NtzgNdvqUFch1RslkH5hemIxC9CyY7RcUwP2dnwJ55m62TEXfhNwMTVgTjIJkiCm +20tfVUbw5UUmiZd5q+t9Dx+OtGkN85+a4zgjtiAKXkjE4BXKEU8Yc2GkTwz98meT +tRnuK4HTxBxrzv9vSG+UZpwvYDBZ/AB3PYlzFV2a0oyF3Tf48yptQ1gPteUcmIrD +7z1Uu/2VkYskQ73aU1GUnTqy7kfNBITJwumIq8McEQIDAQABoxgwFjAUBgNVHREE +DTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQADggEBAC0y9o5FUCzpSEqBK7Ns +1FiAR/cNS6MKmpCKjN4sNvALSThCTdDB4QVBEOe+eTZP/q105oyf8boSktCG/3MO +B6Jwdo5AnBHiE2QGfacMluUkuYRGf1XqWl9oa1AeuqCS+ilGk485akiI0A/z6ZRz +ynGvk/9bqYYhqIPV2ioxFdHaXNlNKT36BV1NFrW3ebZSa3w7nHIrEakZuVmc67vH +dCuxHf8l2Bya/xT1yktq4MaiFzUY9ZZLSnpWPuGzXnika0IeREF7rm9ubozq8mSq +JyTC0KAZvwt7BmbEO98NcfL8gtYAVqDBR/t6gW4TSxKGc0PB3j/+73Nj5hGBasxd +2cc= +-----END CERTIFICATE----- diff --git a/tests/certs/server.key b/tests/certs/server.key new file mode 100644 index 0000000..12103d3 --- /dev/null +++ b/tests/certs/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA5icu/uU40MkMpthiEhxV9Y1F2p8aT6EWk8dWLxF8Pj2D3RYp +1K+GYis8K1+h5dVm9KCH61e4NlGK5zO6hGrS44zimQ0xfJBqeUOfkAW2EE2cNtzg +NdvqUFch1RslkH5hemIxC9CyY7RcUwP2dnwJ55m62TEXfhNwMTVgTjIJkiCm20tf +VUbw5UUmiZd5q+t9Dx+OtGkN85+a4zgjtiAKXkjE4BXKEU8Yc2GkTwz98meTtRnu +K4HTxBxrzv9vSG+UZpwvYDBZ/AB3PYlzFV2a0oyF3Tf48yptQ1gPteUcmIrD7z1U +u/2VkYskQ73aU1GUnTqy7kfNBITJwumIq8McEQIDAQABAoIBAEYzcXxCQsA8cuV5 +XwCTMA0EGGiE2yuqwQ42YS1eMf1yGgSXvA6ps13CPkokk2ddXlgDlzHLwd6fpLS8 +7IlzY/wQfxWcFpoeGrv+Sm9NrqjuY1XArYsAF0qGKUWtUBnw0p7X0IoAEEmlO/v+ +W3DsiMDh/UI+XSIRn8kCtOtlC9JMG0M7HAAOYsQHEpCqxPVp72mNR3zYnmQiWhSm +cecKGr+EpAqRGSe8Yj8CjhVYuaM+qptaOYhZlShr3pv0HWnZTEvVIp1TzSJ7gypg +uUvAIhvL6zSAjwMpDkIXicDUJxb08uwn/xjVOvgeJx5HjzTWDk6FwuUB52PUdVBn +oWPUwaECgYEA9RqI7J4TGmF0abvYPDBx9BVzvA/pqWWPGT5EKfBT557V5LV9XTEW +8Fw0U85XFdxeW+E/0ULUmh0e7yN8T+uy6Hb97HXhqm8JCQSXimvLptyqP+5YCHPD +CcTx4941TLTuEe+KNAZAd1syOrA92m9BwFjJSVcfXbXGVZxpFKwntV0CgYEA8GJ/ +6ndYRkIe3+WGNIqGDJSYHP60Xq4SKyMW4Pxt7z7Nrb6zHg34QLzhsQNZ5I1JDA4V +umuYVNphxgb96xzS7WNK/QmgYnpjkoRm1eHSusSav9g8bvow1z+6KFq/qt1JBRKX +F2BQ2u+QlC1zLJSElVRL5ay2tXboIv97OxFugkUCgYB9uaO8xAUGhjDhv7JWhX8e +dhaMxBjWhLrXdwIeBSH08JvFGnd44yJiHtnUl0ZCd2yLcsp6e+50MzXX8vrkQAHg +jpEHxxv/gb8/ufRF069+IzjNXGQZyc+k5jox6ZyrgS+RUa8xqndNAiGMyzSfJGy0 +zpZJoX/8YK6g4X9hVEF2HQKBgHHxrsKcKZq8Eth8er4C/4GNGgF8dlD+4BvUeS7S +WOXz9hiqcUsIwiklnzGB7iVZF0wAjSodgEqQbZIplEjTE+R0kYIaAw1LCFHWMsyl +S3c+ZEAVpqfQLkCJs5sXUQ0T8V3XLwlknU76CaVDWfnCuIn0ODm5Qa4InAai5W3d +WG2lAoGBAJ0X6zG21dRN2O35Y5HIt0ydD0NZmuOYk+h8eIIkyAZDEDHeuqKOPwF1 +N5tUIBATZ5yHwy2wWOwKn+0+7i+1N8n9aC+qE7tWUsJOpgGLcl2Q0weSuQqIYNcN +/yzGx5WcWbKfMTfn70vOju84f9FuO9DVYiNPg67H7aWJ7roKWzez +-----END RSA PRIVATE KEY----- diff --git a/tests/close_socket_cb_test.py b/tests/close_socket_cb_test.py new file mode 100644 index 0000000..39a3802 --- /dev/null +++ b/tests/close_socket_cb_test.py @@ -0,0 +1,79 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import socket +import unittest +import pycurl + +from . import util +from . import appmanager + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class CloseSocketCbTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + self.curl.setopt(self.curl.URL, 'http://%s:8380/success' % localhost) + self.curl.setopt(pycurl.FORBID_REUSE, True) + + def tearDown(self): + self.curl.close() + + @util.min_libcurl(7, 21, 7) + def test_closesocketfunction_ok(self): + called = {} + + def closesocketfunction(curlfd): + called['called'] = True + # Unix only + #os.close(curlfd) + # Unix & Windows + socket.fromfd(curlfd, socket.AF_INET, socket.SOCK_STREAM).close() + return 0 + + self.curl.setopt(pycurl.CLOSESOCKETFUNCTION, closesocketfunction) + + self.curl.perform() + assert called['called'] + + @util.min_libcurl(7, 21, 7) + def test_closesocketfunction_fail(self): + called = {} + + def closesocketfunction(curlfd): + called['called'] = True + return 1 + + self.curl.setopt(pycurl.CLOSESOCKETFUNCTION, closesocketfunction) + + # no exception on errors, apparently + self.curl.perform() + assert called['called'] + + @util.min_libcurl(7, 21, 7) + def test_closesocketfunction_bogus_return(self): + called = {} + + def closesocketfunction(curlfd): + called['called'] = True + return 'bogus' + + self.curl.setopt(pycurl.CLOSESOCKETFUNCTION, closesocketfunction) + + # no exception on errors, apparently + self.curl.perform() + assert called['called'] + +class CloseSocketCbUnsetTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + @util.min_libcurl(7, 21, 7) + def test_closesocketfunction_none(self): + self.curl.setopt(pycurl.CLOSESOCKETFUNCTION, None) + + @util.min_libcurl(7, 21, 7) + def test_closesocketfunction_unset(self): + self.curl.unsetopt(pycurl.CLOSESOCKETFUNCTION) diff --git a/tests/curl_object_test.py b/tests/curl_object_test.py new file mode 100644 index 0000000..dfe8bc7 --- /dev/null +++ b/tests/curl_object_test.py @@ -0,0 +1,153 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import pytest +import unittest + +class ExplicitConstructionCurlObjectTest(unittest.TestCase): + def test_close(self): + c = pycurl.Curl() + c.close() + + def test_close_twice(self): + c = pycurl.Curl() + c.close() + c.close() + + # positional arguments are rejected + def test_positional_arguments(self): + with pytest.raises(TypeError): + pycurl.Curl(1) + + # keyword arguments are rejected + def test_keyword_arguments(self): + with pytest.raises(TypeError): + pycurl.Curl(a=1) + +class CurlObjectTest(unittest.TestCase): + def setUp(self): + self.curl = pycurl.Curl() + + def tearDown(self): + self.curl.close() + + def test_set_attribute_curl(self): + self.instantiate_and_check(self.check_set_attribute, 'Curl') + + def test_get_attribute_curl(self): + self.instantiate_and_check(self.check_get_attribute, 'Curl') + + def test_get_missing_attribute_curl(self): + self.instantiate_and_check(self.check_get_missing_attribute, 'Curl') + + def test_delete_attribute_curl(self): + self.instantiate_and_check(self.check_delete_attribute, 'Curl') + + def test_delete_missing_attribute_curl(self): + self.instantiate_and_check(self.check_delete_missing_attribute, 'Curl') + + def test_set_attribute_multi(self): + self.instantiate_and_check(self.check_set_attribute, 'CurlMulti') + + def test_get_attribute_multi(self): + self.instantiate_and_check(self.check_get_attribute, 'CurlMulti') + + def test_get_missing_attribute_curl_multi(self): + self.instantiate_and_check(self.check_get_missing_attribute, 'CurlMulti') + + def test_delete_attribute_multi(self): + self.instantiate_and_check(self.check_delete_attribute, 'CurlMulti') + + def test_delete_missing_attribute_curl_multi(self): + self.instantiate_and_check(self.check_delete_missing_attribute, 'CurlMulti') + + def test_set_attribute_share(self): + self.instantiate_and_check(self.check_set_attribute, 'CurlShare') + + def test_get_attribute_share(self): + self.instantiate_and_check(self.check_get_attribute, 'CurlShare') + + def test_get_missing_attribute_curl_share(self): + self.instantiate_and_check(self.check_get_missing_attribute, 'CurlShare') + + def test_delete_attribute_share(self): + self.instantiate_and_check(self.check_delete_attribute, 'CurlShare') + + def test_delete_missing_attribute_curl_share(self): + self.instantiate_and_check(self.check_delete_missing_attribute, 'CurlShare') + + def instantiate_and_check(self, fn, cls_name): + cls = getattr(pycurl, cls_name) + instance = cls() + try: + fn(instance) + finally: + instance.close() + + def check_set_attribute(self, pycurl_obj): + assert not hasattr(pycurl_obj, 'attr') + pycurl_obj.attr = 1 + assert hasattr(pycurl_obj, 'attr') + + def check_get_attribute(self, pycurl_obj): + assert not hasattr(pycurl_obj, 'attr') + pycurl_obj.attr = 1 + self.assertEqual(1, pycurl_obj.attr) + + def check_get_missing_attribute(self, pycurl_obj): + try: + getattr(pycurl_obj, 'doesnotexist') + self.fail('Expected an AttributeError exception to be raised') + except AttributeError: + pass + + def check_delete_attribute(self, pycurl_obj): + assert not hasattr(pycurl_obj, 'attr') + pycurl_obj.attr = 1 + self.assertEqual(1, pycurl_obj.attr) + assert hasattr(pycurl_obj, 'attr') + del pycurl_obj.attr + assert not hasattr(pycurl_obj, 'attr') + + def check_delete_missing_attribute(self, pycurl_obj): + try: + del pycurl_obj.doesnotexist + self.fail('Expected an AttributeError exception to be raised') + except AttributeError: + pass + + def test_modify_attribute_curl(self): + self.check_modify_attribute(pycurl.Curl, 'READFUNC_PAUSE') + + def test_modify_attribute_multi(self): + self.check_modify_attribute(pycurl.CurlMulti, 'E_MULTI_OK') + + def test_modify_attribute_share(self): + self.check_modify_attribute(pycurl.CurlShare, 'SH_SHARE') + + def check_modify_attribute(self, cls, name): + obj1 = cls() + obj2 = cls() + old_value = getattr(obj1, name) + self.assertNotEqual('helloworld', old_value) + # value should be identical to pycurl global + self.assertEqual(old_value, getattr(pycurl, name)) + setattr(obj1, name, 'helloworld') + self.assertEqual('helloworld', getattr(obj1, name)) + + # change does not affect other existing objects + self.assertEqual(old_value, getattr(obj2, name)) + + # change does not affect objects created later + obj3 = cls() + self.assertEqual(old_value, getattr(obj3, name)) + + def test_bogus_attribute_access(self): + with pytest.raises(AttributeError, match='trying to obtain.*'): + self.curl.foo + + def test_bogus_attribute_delete(self): + with pytest.raises(AttributeError, match='trying to delete.*'): + del self.curl.foo diff --git a/tests/debug_test.py b/tests/debug_test.py new file mode 100644 index 0000000..50da3d8 --- /dev/null +++ b/tests/debug_test.py @@ -0,0 +1,66 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class DebugTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + self.debug_entries = [] + + def tearDown(self): + self.curl.close() + + def debug_function(self, t, b): + self.debug_entries.append((t, b)) + + def test_perform_get_with_debug_function(self): + self.curl.setopt(pycurl.VERBOSE, 1) + self.curl.setopt(pycurl.DEBUGFUNCTION, self.debug_function) + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + + # Some checks with no particular intent + self.check(0, util.b('Trying')) + if util.pycurl_version_less_than(7, 24): + self.check(0, util.b('connected')) + else: + self.check(0, util.b('Connected to %s' % localhost)) + self.check(0, util.b('port 8380')) + # request + self.check(2, util.b('GET /success HTTP/1.1')) + # response + self.check(1, util.b('HTTP/1.0 200 OK')) + self.check(1, util.b('Content-Length: 7')) + # result + self.check(3, util.b('success')) + + # test for #210 + def test_debug_unicode(self): + self.curl.setopt(pycurl.VERBOSE, 1) + self.curl.setopt(pycurl.DEBUGFUNCTION, self.debug_function) + self.curl.setopt(pycurl.URL, 'http://%s:8380/utf8_body' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + + # 3 = response body + search = util.b('\xd0\x94\xd1\x80\xd1\x83\xd0\xb6\xd0\xb1\xd0\xb0 \xd0\xbd\xd0\xb0\xd1\x80\xd0\xbe\xd0\xb4\xd0\xbe\xd0\xb2').decode('utf8') + self.check(3, search.encode('utf8')) + + def check(self, wanted_t, wanted_b): + for t, b in self.debug_entries: + if t == wanted_t and wanted_b in b: + return + assert False, "%d: %s not found in debug entries\nEntries are:\n%s" % \ + (wanted_t, repr(wanted_b), repr(self.debug_entries)) diff --git a/tests/default_write_cb_test.py b/tests/default_write_cb_test.py new file mode 100644 index 0000000..ff6478e --- /dev/null +++ b/tests/default_write_cb_test.py @@ -0,0 +1,85 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import unittest +import pycurl +import sys +import tempfile +import os + +from . import appmanager, util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +STDOUT_FD_NUM = 1 + +def try_fsync(fd): + try: + os.fsync(fd) + except OSError: + # On travis: + # OSError: [Errno 22] Invalid argument + # ignore + pass + +class DefaultWriteCbTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_perform_get(self): + # This test performs a GET request without doing anything else. + # Unfortunately, the default curl behavior is to print response + # body to standard output, which spams test output. + # As a result this test is commented out. Uncomment for debugging. + # test_perform_get_with_default_write_function is the test + # which exercises default curl write handler. + + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + self.curl.perform() + # If this flush is not done, stdout output bleeds into the next test + # that is executed (without nose output capture) + sys.stdout.flush() + try_fsync(STDOUT_FD_NUM) + + # I have a really hard time getting this to work with nose output capture + def skip_perform_get_with_default_write_function(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + f = tempfile.NamedTemporaryFile() + try: + #with open('w', 'w+') as f: + # nose output capture plugin replaces sys.stdout with a StringIO + # instance. We want to redirect the underlying file descriptor + # anyway because underlying C code uses it. + # Therefore: + # 1. Use file descriptor 1 rather than sys.stdout.fileno() to + # reference the standard output file descriptor. + # 2. We do not touch sys.stdout. This means anything written to + # sys.stdout will be captured by nose, and not make it to our code. + # But the output we care about happens at libcurl level, below + # nose, therefore this is fine. + saved_stdout_fd = os.dup(STDOUT_FD_NUM) + os.dup2(f.fileno(), STDOUT_FD_NUM) + #os.dup2(1, 100) + #os.dup2(2, 1) + # We also need to flush the output that libcurl wrote to stdout. + # Since sys.stdout might be nose's StringIO instance, open the + # stdout file descriptor manually. + + try: + self.curl.perform() + sys.stdout.flush() + finally: + try_fsync(STDOUT_FD_NUM) + os.dup2(saved_stdout_fd, STDOUT_FD_NUM) + os.close(saved_stdout_fd) + #os.dup2(100, 1) + f.seek(0) + body = f.read() + finally: + f.close() + self.assertEqual('success', body) diff --git a/tests/duphandle_test.py b/tests/duphandle_test.py new file mode 100644 index 0000000..fa30e80 --- /dev/null +++ b/tests/duphandle_test.py @@ -0,0 +1,144 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import unittest +import gc +import weakref +try: + import json +except ImportError: + import simplejson as json + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class DuphandleTest(unittest.TestCase): + def setUp(self): + self.orig = util.DefaultCurl() + + def test_duphandle_attribute_dict(self): + self.orig.orig_attr = 'orig-value' + # attribute dict should be copied - the *object*, not the reference + dup = self.orig.duphandle() + assert dup.orig_attr == 'orig-value' + # cloned dict should be a separate object + dup.dup_attr = 'dup-value' + try: + self.orig.dup_attr == 'does not exist' + except AttributeError as error: + assert 'trying to obtain a non-existing attribute: dup_attr' in str(error.args) + else: + self.fail('should have raised AttributeError') + # dealloc self.orig - original dict is freed from memory + self.orig.close() + del self.orig + # cloned dict should still exist + assert dup.orig_attr == 'orig-value' + assert dup.dup_attr == 'dup-value' + dup.close() + + def slist_check(self, handle, value, persistance=True): + body = util.BytesIO() + handle.setopt(pycurl.WRITEFUNCTION, body.write) + handle.setopt(pycurl.URL, 'http://%s:8380/header_utf8?h=x-test-header' % localhost) + handle.perform() + result = body.getvalue().decode('utf-8') + assert (result == value) == persistance + + def slist_test(self, clear_func, *args): + # new slist object is created with ref count = 1 + self.orig.setopt(pycurl.HTTPHEADER, ['x-test-header: orig-slist']) + # ref is copied and object incref'ed + dup1 = self.orig.duphandle() + # slist object is decref'ed and ref set to null + clear_func(*args) + # null ref is copied - no effect + dup2 = self.orig.duphandle() + # check slist object persistance + self.slist_check(dup1, 'orig-slist', True) + self.slist_check(dup2, 'orig-slist', False) + # check overwriting - orig slist is decref'ed to 0 and finally deallocated + # util_curlslist_update() and util_curlslist_dealloc() + dup1.setopt(pycurl.HTTPHEADER, ['x-test-header: dup-slist']) + self.slist_check(dup1, 'dup-slist', True) + # cleanup + dup1.close() + dup2.close() + self.orig.close() + + def test_duphandle_slist_xdecref(self): + # util_curl_xdecref() + self.slist_test(self.orig.reset) + + def test_duphandle_slist_unsetopt(self): + # util_curl_unsetopt() + self.slist_test(self.orig.unsetopt, pycurl.HTTPHEADER) + + def httppost_check(self, handle, value, persistance=True): + body = util.BytesIO() + handle.setopt(pycurl.WRITEFUNCTION, body.write) + handle.setopt(pycurl.URL, 'http://%s:8380/postfields' % localhost) + handle.perform() + result = json.loads(body.getvalue()) + assert (result == value) == persistance + + def httppost_test(self, clear_func, *args): + self.orig.setopt(pycurl.HTTPPOST, [ + ('field', (pycurl.FORM_CONTENTS, 'orig-httppost')), + ]) + dup1 = self.orig.duphandle() + clear_func(*args) + dup2 = self.orig.duphandle() + self.httppost_check(dup1, {'field': 'orig-httppost'}, True) + self.httppost_check(dup2, {'field': 'orig-httppost'}, False) + # util_curlhttppost_update() and util_curlhttppost_dealloc() + dup1.setopt(pycurl.HTTPPOST, [ + ('field', (pycurl.FORM_CONTENTS, 'dup-httppost')), + ]) + self.httppost_check(dup1, {'field': 'dup-httppost'}, True) + dup1.close() + dup2.close() + self.orig.close() + + def test_duphandle_httppost_xdecref(self): + # util_curl_xdecref() + self.httppost_test(self.orig.reset) + + def test_duphandle_httppost_unsetopt(self): + # util_curl_unsetopt() + self.httppost_test(self.orig.unsetopt, pycurl.HTTPPOST) + + def test_duphandle_references(self): + body = util.BytesIO() + def callback(data): + body.write(data) + callback_ref = weakref.ref(callback) + # preliminary checks of gc and weakref working as expected + assert gc.get_referrers(callback) == [] + assert callback_ref() is not None + # setopt - callback ref is copied and callback incref'ed + self.orig.setopt(pycurl.WRITEFUNCTION, callback) + assert gc.get_referrers(callback) == [self.orig] + # duphandle - callback ref is copied and callback incref'ed + dup = self.orig.duphandle() + assert set(gc.get_referrers(callback)) == {self.orig, dup} + # dealloc self.orig and decref callback + self.orig.close() + del self.orig + assert gc.get_referrers(callback) == [dup] + # decref callback again - back to ref count = 1 + del callback + assert callback_ref() is not None + # check that callback object still exists and is invoked + dup.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + dup.perform() + result = body.getvalue().decode('utf-8') + assert result == 'success' + # final decref - callback is deallocated + dup.close() + assert callback_ref() is None diff --git a/tests/error_constants_test.py b/tests/error_constants_test.py new file mode 100644 index 0000000..d07e17a --- /dev/null +++ b/tests/error_constants_test.py @@ -0,0 +1,25 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import unittest + +from . import util + +class ErrorConstantsTest(unittest.TestCase): + @util.min_libcurl(7, 21, 5) + def test_not_built_in(self): + assert hasattr(pycurl, 'E_NOT_BUILT_IN') + + @util.min_libcurl(7, 24, 0) + def test_ftp_accept_failed(self): + assert hasattr(pycurl, 'E_FTP_ACCEPT_FAILED') + + @util.min_libcurl(7, 21, 5) + def test_unknown_option(self): + assert hasattr(pycurl, 'E_UNKNOWN_OPTION') + + @util.min_libcurl(7, 39, 0) + def test_pinnedpubkeynotmatch(self): + assert hasattr(pycurl, 'E_SSL_PINNEDPUBKEYNOTMATCH') diff --git a/tests/error_test.py b/tests/error_test.py new file mode 100644 index 0000000..f32b693 --- /dev/null +++ b/tests/error_test.py @@ -0,0 +1,85 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import sys +import unittest + +class ErrorTest(unittest.TestCase): + def setUp(self): + self.curl = pycurl.Curl() + + def tearDown(self): + self.curl.close() + + # error originating in libcurl + def test_pycurl_error_libcurl(self): + try: + # perform without a url + self.curl.perform() + except pycurl.error: + exc_type, exc = sys.exc_info()[:2] + assert exc_type == pycurl.error + # pycurl.error's arguments are libcurl errno and message + self.assertEqual(2, len(exc.args)) + self.assertEqual(int, type(exc.args[0])) + self.assertEqual(str, type(exc.args[1])) + # unpack + err, msg = exc.args + self.assertEqual(pycurl.E_URL_MALFORMAT, err) + # possibly fragile + # curl < 7.83.0 has an exclamation mark in this error message + self.assertIn(msg, ['No URL set!', 'No URL set']) + else: + self.fail('Expected pycurl.error to be raised') + + def test_pycurl_errstr_initially_empty(self): + self.assertEqual('', self.curl.errstr()) + + def test_pycurl_errstr_type(self): + self.assertEqual('', self.curl.errstr()) + try: + # perform without a url + self.curl.perform() + except pycurl.error: + # might be fragile + # curl < 7.83.0 has an exclamation mark in this error message + self.assertIn(self.curl.errstr(), ['No URL set!', 'No URL set']) + # repeated checks do not clear value + self.assertIn(self.curl.errstr(), ['No URL set!', 'No URL set']) + # check the type - on all python versions + self.assertEqual(str, type(self.curl.errstr())) + else: + self.fail('no exception') + + # pycurl raises standard library exceptions in some cases + def test_pycurl_error_stdlib(self): + try: + # set an option of the wrong type + self.curl.setopt(pycurl.WRITEFUNCTION, True) + except TypeError: + exc_type, exc = sys.exc_info()[:2] + else: + self.fail('Expected TypeError to be raised') + + # error originating in pycurl + # looks like currently there are none + def xtest_pycurl_error_pycurl(self): + try: + # invalid option combination + self.curl.setopt(pycurl.WRITEFUNCTION, lambda x: x) + f = open(__file__) + try: + self.curl.setopt(pycurl.WRITEHEADER, f) + finally: + f.close() + except pycurl.error: + exc_type, exc = sys.exc_info()[:2] + assert exc_type == pycurl.error + # for non-libcurl errors, arguments are just the error string + self.assertEqual(1, len(exc.args)) + self.assertEqual(str, type(exc.args[0])) + self.assertEqual('cannot combine WRITEHEADER with WRITEFUNCTION.', exc.args[0]) + else: + self.fail('Expected pycurl.error to be raised') diff --git a/tests/ext/test-lib.sh b/tests/ext/test-lib.sh new file mode 100644 index 0000000..0cb9489 --- /dev/null +++ b/tests/ext/test-lib.sh @@ -0,0 +1,69 @@ +# shell test framework based on test framework in rpg: +# https://github.com/rtomayko/rpg +# +# Copyright (c) 2010 Ryan Tomayko +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +: ${VERBOSE:=false} + +unset CDPATH + +#cd "$(dirname $0)" +if test -z "$TESTDIR"; then + TESTDIR=$(realpath $(pwd)) +fi + +test_count=0 +successes=0 +failures=0 + +output="$TESTDIR/$(basename "$0" .sh).out" +trap "rm -f $output" 0 + +succeeds () { + test_count=$(( test_count + 1 )) + echo "\$ ${2:-$1}" > "$output" + eval "( ${2:-$1} )" 1>>"$output" 2>&1 + ec=$? + if test $ec -eq 0 + then successes=$(( successes + 1 )) + printf 'ok %d - %s\n' $test_count "$1" + else failures=$(( failures + 1 )) + printf 'not ok %d - %s [%d]\n' $test_count "$1" "$ec" + fi + + $VERBOSE && dcat $output + return 0 +} + +fails () { + if test $# -eq 1 + then succeeds "! $1" + else succeeds "$1" "! $2" + fi +} + +diag () { echo "$@" | sed 's/^/# /'; } +dcat () { cat "$@" | sed 's/^/# /'; } +desc () { diag "$@"; } + +setup () { + rm -rf "$TESTDIR/trash" + return 0 +} diff --git a/tests/ext/test-suite.sh b/tests/ext/test-suite.sh new file mode 100755 index 0000000..4b1c9a6 --- /dev/null +++ b/tests/ext/test-suite.sh @@ -0,0 +1,27 @@ +# + +dir=$(dirname "$0") + +export PATH="$(pwd)/tests/bin":$PATH + +. "$dir"/test-lib.sh + +setup + +desc 'setup.py without arguments' +fails 'python setup.py' +succeeds 'python setup.py 2>&1 |grep "usage: setup.py"' + +desc 'setup.py --help' +succeeds 'python setup.py --help' +# .* = Unix|Windows +succeeds 'python setup.py --help |grep "PycURL .* options:"' +# distutils help +succeeds 'python setup.py --help |grep "Common commands:"' + +desc 'setup.py --help with bogus --curl-config' +succeeds 'python setup.py --help --curl-config=/dev/null' +succeeds 'python setup.py --help --curl-config=/dev/null |grep "PycURL .* options:"' +# this checks that --curl-config is consumed prior to +# distutils processing --help +fails 'python setup.py --help --curl-config=/dev/null 2>&1 |grep "option .* not recognized"' diff --git a/tests/failonerror_test.py b/tests/failonerror_test.py new file mode 100644 index 0000000..519aed8 --- /dev/null +++ b/tests/failonerror_test.py @@ -0,0 +1,89 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class FailonerrorTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + # not sure what the actual min is but 7.26 is too old + # and does not include status text, only the status code + @util.min_libcurl(7, 38, 0) + # no longer supported by libcurl: https://github.com/curl/curl/issues/6615 + @util.removed_in_libcurl(7, 75, 0) + def test_failonerror(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/status/403' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEDATA, sio) + self.curl.setopt(pycurl.FAILONERROR, True) + #self.curl.setopt(pycurl.VERBOSE, True) + try: + self.curl.perform() + except pycurl.error as e: + self.assertEqual(pycurl.E_HTTP_RETURNED_ERROR, e.args[0]) + self.assertEqual('The requested URL returned error: 403 Forbidden', e.args[1]) + self.assertEqual(util.u('The requested URL returned error: 403 Forbidden'), self.curl.errstr()) + self.assertEqual(util.b('The requested URL returned error: 403 Forbidden'), self.curl.errstr_raw()) + else: + self.fail('Should have raised pycurl.error') + + @util.only_python2 + # not sure what the actual min is but 7.26 is too old + # and does not include status text, only the status code + @util.min_libcurl(7, 38, 0) + # no longer supported by libcurl: https://github.com/curl/curl/issues/6615 + @util.removed_in_libcurl(7, 75, 0) + def test_failonerror_status_line_invalid_utf8_python2(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/status_invalid_utf8' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEDATA, sio) + self.curl.setopt(pycurl.FAILONERROR, True) + #self.curl.setopt(pycurl.VERBOSE, True) + try: + self.curl.perform() + except pycurl.error as e: + self.assertEqual(pycurl.E_HTTP_RETURNED_ERROR, e.args[0]) + self.assertEqual('The requested URL returned error: 555 \xb3\xd2\xda\xcd\xd7', e.args[1]) + self.assertEqual('The requested URL returned error: 555 \xb3\xd2\xda\xcd\xd7', self.curl.errstr()) + self.assertEqual('The requested URL returned error: 555 \xb3\xd2\xda\xcd\xd7', self.curl.errstr_raw()) + else: + self.fail('Should have raised pycurl.error') + + @util.only_python3 + # not sure what the actual min is but 7.26 is too old + # and does not include status text, only the status code + @util.min_libcurl(7, 38, 0) + # no longer supported by libcurl: https://github.com/curl/curl/issues/6615 + @util.removed_in_libcurl(7, 75, 0) + def test_failonerror_status_line_invalid_utf8_python3(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/status_invalid_utf8' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEDATA, sio) + self.curl.setopt(pycurl.FAILONERROR, True) + #self.curl.setopt(pycurl.VERBOSE, True) + try: + self.curl.perform() + except pycurl.error as e: + self.assertEqual(pycurl.E_HTTP_RETURNED_ERROR, e.args[0]) + assert e.args[1].startswith('The requested URL returned error: 555 ') + try: + self.curl.errstr() + except UnicodeDecodeError: + pass + else: + self.fail('Should have raised') + self.assertEqual(util.b('The requested URL returned error: 555 \xb3\xd2\xda\xcd\xd7'), self.curl.errstr_raw()) + else: + self.fail('Should have raised pycurl.error') diff --git a/tests/fake-curl/curl-config-empty b/tests/fake-curl/curl-config-empty new file mode 100755 index 0000000..cf93615 --- /dev/null +++ b/tests/fake-curl/curl-config-empty @@ -0,0 +1,15 @@ +#!/bin/sh + +# A curl-config that returns empty responses as much as possible + +output= + +while test -n "$1"; do + case "$1" in + --libs) + # --libs or --static-libs must succeed and produce output + echo '-lcurl' + ;; + esac + shift +done diff --git a/tests/fake-curl/curl-config-libs-and-static-libs b/tests/fake-curl/curl-config-libs-and-static-libs new file mode 100755 index 0000000..56135e0 --- /dev/null +++ b/tests/fake-curl/curl-config-libs-and-static-libs @@ -0,0 +1,17 @@ +#!/bin/sh + +# A curl-config that returns different libraries in --libs and --static-libs + +output= + +while test -n "$1"; do + case "$1" in + --libs) + echo '-lcurl -lflurby' + ;; + --static-libs) + echo '-lkzzert' + ;; + esac + shift +done diff --git a/tests/fake-curl/curl-config-ssl-feature-only b/tests/fake-curl/curl-config-ssl-feature-only new file mode 100755 index 0000000..3999f11 --- /dev/null +++ b/tests/fake-curl/curl-config-ssl-feature-only @@ -0,0 +1,18 @@ +#!/bin/sh + +# A curl-config that indicates SSL is supported but does not say +# which SSL library is being used + +output= + +while test -n "$1"; do + case "$1" in + --libs) + echo '-lcurl' + ;; + --features) + echo 'SSL' + ;; + esac + shift +done diff --git a/tests/fake-curl/curl-config-ssl-in-libs b/tests/fake-curl/curl-config-ssl-in-libs new file mode 100755 index 0000000..bb47fc3 --- /dev/null +++ b/tests/fake-curl/curl-config-ssl-in-libs @@ -0,0 +1,17 @@ +#!/bin/sh + +# A curl-config that returns -lssl in --libs but not in --static-libs + +output= + +while test -n "$1"; do + case "$1" in + --libs) + echo '-lcurl -lssl' + ;; + --features) + echo 'SSL' + ;; + esac + shift +done diff --git a/tests/fake-curl/curl-config-ssl-in-static-libs b/tests/fake-curl/curl-config-ssl-in-static-libs new file mode 100755 index 0000000..8c93b5f --- /dev/null +++ b/tests/fake-curl/curl-config-ssl-in-static-libs @@ -0,0 +1,20 @@ +#!/bin/sh + +# A curl-config that returns -lssl in --static-libs but not in --libs + +output= + +while test -n "$1"; do + case "$1" in + --libs) + echo '-lcurl' + ;; + --static-libs) + echo '-lssl' + ;; + --features) + echo 'SSL' + ;; + esac + shift +done diff --git a/tests/fake-curl/libcurl/Makefile b/tests/fake-curl/libcurl/Makefile new file mode 100644 index 0000000..b5d0816 --- /dev/null +++ b/tests/fake-curl/libcurl/Makefile @@ -0,0 +1,28 @@ +ALL = \ + with_gnutls.so \ + with_nss.so \ + with_openssl.so \ + with_unknown_ssl.so \ + without_ssl.so + +all: $(ALL) +clean: + rm -f $(ALL) + +.SUFFIXES: .c .so + +CC = `curl-config --cc` +CFLAGS += `curl-config --cflags` +UNAME := $(shell uname -s) +ifeq ($(UNAME),Darwin) + SONAME_FLAG = -install_name +else + SONAME_FLAG = -soname +endif + +.c.so: + $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -shared -fPIC \ + -Wl,$(SONAME_FLAG),$@ -o $@ $< + +show-targets: + ls *c |sed -e 's/.c$$/.so/' | awk '{print $$1 " \\"}' diff --git a/tests/fake-curl/libcurl/with_gnutls.c b/tests/fake-curl/libcurl/with_gnutls.c new file mode 100644 index 0000000..6795b7b --- /dev/null +++ b/tests/fake-curl/libcurl/with_gnutls.c @@ -0,0 +1,29 @@ +#include + +static const char *protocols[] = { +}; + +static curl_version_info_data version_info = { + /* age */ + 3, + /* version */ + "", + /* version_num */ + 0, + /* host */ + "", + /* features */ + 0, + /* ssl_version */ + "GnuTLS/2.11", + /* ssl_version_num */ + 0, + /* libz_version */ + "", + /* protocols */ + protocols +}; + +curl_version_info_data *curl_version_info(CURLversion type) { + return &version_info; +} diff --git a/tests/fake-curl/libcurl/with_nss.c b/tests/fake-curl/libcurl/with_nss.c new file mode 100644 index 0000000..2bf46a5 --- /dev/null +++ b/tests/fake-curl/libcurl/with_nss.c @@ -0,0 +1,29 @@ +#include + +static const char *protocols[] = { +}; + +static curl_version_info_data version_info = { + /* age */ + 3, + /* version */ + "", + /* version_num */ + 0, + /* host */ + "", + /* features */ + 0, + /* ssl_version */ + "NSS/3.0", + /* ssl_version_num */ + 0, + /* libz_version */ + "", + /* protocols */ + protocols +}; + +curl_version_info_data *curl_version_info(CURLversion type) { + return &version_info; +} diff --git a/tests/fake-curl/libcurl/with_openssl.c b/tests/fake-curl/libcurl/with_openssl.c new file mode 100644 index 0000000..e2c7227 --- /dev/null +++ b/tests/fake-curl/libcurl/with_openssl.c @@ -0,0 +1,29 @@ +#include + +static const char *protocols[] = { +}; + +static curl_version_info_data version_info = { + /* age */ + 3, + /* version */ + "", + /* version_num */ + 0, + /* host */ + "", + /* features */ + 0, + /* ssl_version */ + "OpenSSL/1.0.1a", + /* ssl_version_num */ + 0, + /* libz_version */ + "", + /* protocols */ + protocols +}; + +curl_version_info_data *curl_version_info(CURLversion type) { + return &version_info; +} diff --git a/tests/fake-curl/libcurl/with_unknown_ssl.c b/tests/fake-curl/libcurl/with_unknown_ssl.c new file mode 100644 index 0000000..702155e --- /dev/null +++ b/tests/fake-curl/libcurl/with_unknown_ssl.c @@ -0,0 +1,29 @@ +#include + +static const char *protocols[] = { +}; + +static curl_version_info_data version_info = { + /* age */ + 3, + /* version */ + "", + /* version_num */ + 0, + /* host */ + "", + /* features */ + 0, + /* ssl_version */ + "HelloWorldSSL/1.0", + /* ssl_version_num */ + 0, + /* libz_version */ + "", + /* protocols */ + protocols +}; + +curl_version_info_data *curl_version_info(CURLversion type) { + return &version_info; +} diff --git a/tests/fake-curl/libcurl/without_ssl.c b/tests/fake-curl/libcurl/without_ssl.c new file mode 100644 index 0000000..091600c --- /dev/null +++ b/tests/fake-curl/libcurl/without_ssl.c @@ -0,0 +1,29 @@ +#include + +static const char *protocols[] = { +}; + +static curl_version_info_data version_info = { + /* age */ + 3, + /* version */ + "", + /* version_num */ + 0, + /* host */ + "", + /* features */ + 0, + /* ssl_version */ + "", + /* ssl_version_num */ + 0, + /* libz_version */ + "", + /* protocols */ + protocols +}; + +curl_version_info_data *curl_version_info(CURLversion type) { + return &version_info; +} diff --git a/tests/fixtures/form_submission.txt b/tests/fixtures/form_submission.txt new file mode 100644 index 0000000..c9f0304 --- /dev/null +++ b/tests/fixtures/form_submission.txt @@ -0,0 +1 @@ +foo=bar \ No newline at end of file diff --git a/tests/ftp_test.py b/tests/ftp_test.py new file mode 100644 index 0000000..be5c7af --- /dev/null +++ b/tests/ftp_test.py @@ -0,0 +1,53 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +# Note: this test is meant to be run from pycurl project root. + +import pycurl +import unittest + +from . import util +from . import procmgr, localhost + +setup_module, teardown_module = procmgr.vsftpd_setup() + +class FtpTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_get_ftp(self): + self.curl.setopt(pycurl.URL, 'ftp://%s:8321' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + + result = sio.getvalue().decode() + assert 'README.rst' in result + assert 'INSTALL.rst' in result + + # XXX this test needs to be fixed + def test_quote(self): + self.curl.setopt(pycurl.URL, 'ftp://%s:8321' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.setopt(pycurl.QUOTE, ['CWD tests']) + self.curl.perform() + + result = sio.getvalue().decode() + assert 'README.rst' not in result + assert 'ftp_test.py' in result + + def test_epsv(self): + self.curl.setopt(pycurl.URL, 'ftp://%s:8321' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.setopt(pycurl.FTP_USE_EPSV, 1) + self.curl.perform() + + result = sio.getvalue().decode() + assert 'README.rst' in result + assert 'INSTALL.rst' in result diff --git a/tests/getinfo_test.py b/tests/getinfo_test.py new file mode 100644 index 0000000..838b42e --- /dev/null +++ b/tests/getinfo_test.py @@ -0,0 +1,130 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import flaky +import pycurl +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class GetinfoTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + @flaky.flaky(max_runs=3) + def test_getinfo(self): + self.make_request() + + self.assertEqual(200, self.curl.getinfo(pycurl.HTTP_CODE)) + self.assertEqual(200, self.curl.getinfo(pycurl.RESPONSE_CODE)) + assert type(self.curl.getinfo(pycurl.TOTAL_TIME)) is float + assert type(self.curl.getinfo(pycurl.SPEED_DOWNLOAD)) is float + assert self.curl.getinfo(pycurl.SPEED_DOWNLOAD) > 0 + self.assertEqual(7, self.curl.getinfo(pycurl.SIZE_DOWNLOAD)) + self.assertEqual('http://%s:8380/success' % localhost, self.curl.getinfo(pycurl.EFFECTIVE_URL)) + self.assertEqual('text/html; charset=utf-8', self.curl.getinfo(pycurl.CONTENT_TYPE).lower()) + assert type(self.curl.getinfo(pycurl.NAMELOOKUP_TIME)) is float + assert self.curl.getinfo(pycurl.NAMELOOKUP_TIME) > 0 + assert self.curl.getinfo(pycurl.NAMELOOKUP_TIME) < 1 + self.assertEqual(0, self.curl.getinfo(pycurl.REDIRECT_TIME)) + self.assertEqual(0, self.curl.getinfo(pycurl.REDIRECT_COUNT)) + # time not requested + self.assertEqual(-1, self.curl.getinfo(pycurl.INFO_FILETIME)) + + # It seems that times are 0 on appveyor + @util.only_unix + @flaky.flaky(max_runs=3) + def test_getinfo_times(self): + self.make_request() + + self.assertEqual(200, self.curl.getinfo(pycurl.HTTP_CODE)) + self.assertEqual(200, self.curl.getinfo(pycurl.RESPONSE_CODE)) + assert type(self.curl.getinfo(pycurl.TOTAL_TIME)) is float + assert self.curl.getinfo(pycurl.TOTAL_TIME) > 0 + assert self.curl.getinfo(pycurl.TOTAL_TIME) < 1 + + @util.min_libcurl(7, 21, 0) + def test_primary_port_etc(self): + self.make_request() + assert type(self.curl.getinfo(pycurl.PRIMARY_PORT)) is int + assert type(self.curl.getinfo(pycurl.LOCAL_IP)) is str + assert type(self.curl.getinfo(pycurl.LOCAL_PORT)) is int + + def make_request(self, path='/success', expected_body='success'): + self.curl.setopt(pycurl.URL, 'http://%s:8380' % localhost + path) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + self.assertEqual(expected_body, sio.getvalue().decode()) + + @util.only_python2 + def test_getinfo_cookie_invalid_utf8_python2(self): + self.curl.setopt(self.curl.COOKIELIST, '') + self.make_request('/set_cookie_invalid_utf8', 'cookie set') + + self.assertEqual(200, self.curl.getinfo(pycurl.HTTP_CODE)) + expected = "%s" % localhost + "\tFALSE\t/\tFALSE\t0\t\xb3\xd2\xda\xcd\xd7\t%96%A6g%9Ay%B0%A5g%A7tm%7C%95%9A" + self.assertEqual([expected], self.curl.getinfo(pycurl.INFO_COOKIELIST)) + + @util.only_python3 + def test_getinfo_cookie_invalid_utf8_python3(self): + self.curl.setopt(self.curl.COOKIELIST, '') + self.make_request('/set_cookie_invalid_utf8', 'cookie set') + + self.assertEqual(200, self.curl.getinfo(pycurl.HTTP_CODE)) + + info = self.curl.getinfo(pycurl.INFO_COOKIELIST) + domain, incl_subdomains, path, secure, expires, name, value = info[0].split("\t") + self.assertEqual('\xb3\xd2\xda\xcd\xd7', name) + + def test_getinfo_raw_cookie_invalid_utf8(self): + raise unittest.SkipTest('bottle converts to utf-8? try without it') + + self.curl.setopt(self.curl.COOKIELIST, '') + self.make_request('/set_cookie_invalid_utf8', 'cookie set') + + self.assertEqual(200, self.curl.getinfo(pycurl.HTTP_CODE)) + expected = util.b("%s" % localhost + "\tFALSE\t/\tFALSE\t0\t\xb3\xd2\xda\xcd\xd7\t%96%A6g%9Ay%B0%A5g%A7tm%7C%95%9A") + self.assertEqual([expected], self.curl.getinfo_raw(pycurl.INFO_COOKIELIST)) + + @util.only_python2 + def test_getinfo_content_type_invalid_utf8_python2(self): + self.make_request('/content_type_invalid_utf8', 'content type set') + + self.assertEqual(200, self.curl.getinfo(pycurl.HTTP_CODE)) + expected = '\xb3\xd2\xda\xcd\xd7' + self.assertEqual(expected, self.curl.getinfo(pycurl.CONTENT_TYPE)) + + @util.only_python3 + def test_getinfo_content_type_invalid_utf8_python3(self): + self.make_request('/content_type_invalid_utf8', 'content type set') + + self.assertEqual(200, self.curl.getinfo(pycurl.HTTP_CODE)) + + value = self.curl.getinfo(pycurl.CONTENT_TYPE) + self.assertEqual('\xb3\xd2\xda\xcd\xd7', value) + + def test_getinfo_raw_content_type_invalid_utf8(self): + raise unittest.SkipTest('bottle converts to utf-8? try without it') + + self.make_request('/content_type_invalid_utf8', 'content type set') + + self.assertEqual(200, self.curl.getinfo(pycurl.HTTP_CODE)) + expected = util.b('\xb3\xd2\xda\xcd\xd7') + self.assertEqual(expected, self.curl.getinfo_raw(pycurl.CONTENT_TYPE)) + + def test_getinfo_number(self): + self.make_request() + self.assertEqual(7, self.curl.getinfo(pycurl.SIZE_DOWNLOAD)) + + def test_getinfo_raw_number(self): + self.make_request() + self.assertEqual(7, self.curl.getinfo_raw(pycurl.SIZE_DOWNLOAD)) diff --git a/tests/global_init_test.py b/tests/global_init_test.py new file mode 100644 index 0000000..f93393f --- /dev/null +++ b/tests/global_init_test.py @@ -0,0 +1,30 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import pytest +import unittest + +from . import util + +class GlobalInitTest(unittest.TestCase): + def test_global_init_default(self): + # initialize libcurl with DEFAULT flags + pycurl.global_init(pycurl.GLOBAL_DEFAULT) + pycurl.global_cleanup() + + def test_global_init_ack_eintr(self): + # the GLOBAL_ACK_EINTR flag was introduced in libcurl-7.30, but can also + # be backported for older versions of libcurl at the distribution level + if util.pycurl_version_less_than(7, 30) and not hasattr(pycurl, 'GLOBAL_ACK_EINTR'): + raise unittest.SkipTest('libcurl < 7.30.0 or no GLOBAL_ACK_EINTR') + + # initialize libcurl with the GLOBAL_ACK_EINTR flag + pycurl.global_init(pycurl.GLOBAL_ACK_EINTR) + pycurl.global_cleanup() + + def test_global_init_bogus(self): + # initialize libcurl with bogus flags + with pytest.raises(ValueError): + pycurl.global_init(0xffff) diff --git a/tests/header_cb_test.py b/tests/header_cb_test.py new file mode 100644 index 0000000..71f7bc9 --- /dev/null +++ b/tests/header_cb_test.py @@ -0,0 +1,50 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import unittest +import time as _time + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class HeaderCbTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + self.header_lines = [] + + def tearDown(self): + self.curl.close() + + def header_function(self, line): + self.header_lines.append(line.decode()) + + def test_get(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.setopt(pycurl.HEADERFUNCTION, self.header_function) + self.curl.perform() + self.assertEqual('success', sio.getvalue().decode()) + + assert len(self.header_lines) > 0 + self.assertEqual("HTTP/1.0 200 OK\r\n", self.header_lines[0]) + # day of week + # important: must be in utc + todays_day = _time.strftime('%a', _time.gmtime()) + # Date: Sun, 03 Mar 2013 05:38:12 GMT\r\n + self.check('Date: %s' % todays_day) + # Server: WSGIServer/0.1 Python/2.7.3\r\n + self.check('Server: WSGIServer') + self.check('Content-Length: 7') + self.check('Content-Type: text/html') + + def check(self, wanted_text): + for line in self.header_lines: + if wanted_text in line: + return + assert False, "%s not found in header lines" % wanted_text diff --git a/tests/header_test.py b/tests/header_test.py new file mode 100644 index 0000000..548a9b2 --- /dev/null +++ b/tests/header_test.py @@ -0,0 +1,55 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pytest +import pycurl +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +# NB: HTTP RFC requires headers to be latin1 encoded, which we violate. +# See the comments under /header_utf8 route in app.py. + +class HeaderTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_ascii_string_header(self): + self.check('x-test-header: ascii', 'ascii') + + def test_ascii_unicode_header(self): + self.check(util.u('x-test-header: ascii'), 'ascii') + + # on python 2 unicode is accepted in strings because strings are byte strings + @util.only_python3 + def test_unicode_string_header(self): + with pytest.raises(UnicodeEncodeError): + self.check('x-test-header: Москва', 'Москва') + + def test_unicode_unicode_header(self): + with pytest.raises(UnicodeEncodeError): + self.check(util.u('x-test-header: Москва'), util.u('Москва')) + + def test_encoded_unicode_header(self): + self.check(util.u('x-test-header: Москва').encode('utf-8'), util.u('Москва')) + + def check(self, send, expected): + # check as list and as tuple, because they may be handled differently + self.do_check([send], expected) + self.do_check((send,), expected) + + def do_check(self, send, expected): + self.curl.setopt(pycurl.URL, 'http://%s:8380/header_utf8?h=x-test-header' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.setopt(pycurl.HTTPHEADER, send) + self.curl.perform() + self.assertEqual(expected, sio.getvalue().decode('utf-8')) diff --git a/tests/high_level_curl_test.py b/tests/high_level_curl_test.py new file mode 100644 index 0000000..ce3e250 --- /dev/null +++ b/tests/high_level_curl_test.py @@ -0,0 +1,35 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +# uses the high level interface +import curl +import unittest + +from . import appmanager + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class RelativeUrlTest(unittest.TestCase): + def setUp(self): + self.curl = curl.Curl('http://%s:8380/' % localhost) + + def tearDown(self): + self.curl.close() + + def test_get(self): + result = self.curl.get('/success') + self.assertEqual('success', result.decode()) + + def test_head(self): + result = self.curl.head('/success') + self.assertEqual('', result.decode()) + self.assertEqual(200, self.curl.info()['http-code']) + + def test_reuse(self): + result = self.curl.get('/success') + self.assertEqual('success', result.decode()) + + result = self.curl.get('/success') + self.assertEqual('success', result.decode()) diff --git a/tests/info_constants_test.py b/tests/info_constants_test.py new file mode 100644 index 0000000..5f87cea --- /dev/null +++ b/tests/info_constants_test.py @@ -0,0 +1,16 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import unittest + +from . import util + +class InfoConstantsTest(unittest.TestCase): + # CURLINFO_CONDITION_UNMET was introduced in libcurl-7.19.4 + @util.min_libcurl(7, 19, 4) + def test_condition_unmet(self): + curl = pycurl.Curl() + assert hasattr(curl, 'CONDITION_UNMET') + curl.close() diff --git a/tests/info_test.py b/tests/info_test.py new file mode 100644 index 0000000..eeb5ed0 --- /dev/null +++ b/tests/info_test.py @@ -0,0 +1,19 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import unittest + +from . import util + +class InfoTest(unittest.TestCase): + @util.only_ssl + def test_ssl_engines(self): + curl = pycurl.Curl() + engines = curl.getinfo(curl.SSL_ENGINES) + # Typical result: + # - an empty list in some configurations + # - ['rdrand', 'dynamic'] + self.assertEqual(type(engines), list) + curl.close() diff --git a/tests/internals_test.py b/tests/internals_test.py new file mode 100644 index 0000000..d3f4e7c --- /dev/null +++ b/tests/internals_test.py @@ -0,0 +1,225 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import unittest +try: + import cPickle +except ImportError: + cPickle = None +import pickle +import copy + +from . import util + +class InternalsTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + del self.curl + + # /*********************************************************************** + # // test misc + # ************************************************************************/ + + def test_constant_aliasing(self): + assert self.curl.URL is pycurl.URL + + # /*********************************************************************** + # // test handles + # ************************************************************************/ + + def test_remove_invalid_handle(self): + m = pycurl.CurlMulti() + try: + m.remove_handle(self.curl) + except pycurl.error: + pass + else: + assert False, "No exception when trying to remove a handle that is not in CurlMulti" + del m + + def test_remove_invalid_closed_handle(self): + m = pycurl.CurlMulti() + c = util.DefaultCurl() + c.close() + m.remove_handle(c) + del m, c + + def test_add_closed_handle(self): + m = pycurl.CurlMulti() + c = util.DefaultCurl() + c.close() + try: + m.add_handle(c) + except pycurl.error: + pass + else: + assert 0, "No exception when trying to add a close handle to CurlMulti" + m.close() + del m, c + + def test_add_handle_twice(self): + m = pycurl.CurlMulti() + m.add_handle(self.curl) + try: + m.add_handle(self.curl) + except pycurl.error: + pass + else: + assert 0, "No exception when trying to add the same handle twice" + del m + + def test_add_handle_on_multiple_stacks(self): + m1 = pycurl.CurlMulti() + m2 = pycurl.CurlMulti() + m1.add_handle(self.curl) + try: + m2.add_handle(self.curl) + except pycurl.error: + pass + else: + assert 0, "No exception when trying to add the same handle on multiple stacks" + del m1, m2 + + def test_move_handle(self): + m1 = pycurl.CurlMulti() + m2 = pycurl.CurlMulti() + m1.add_handle(self.curl) + m1.remove_handle(self.curl) + m2.add_handle(self.curl) + del m1, m2 + + # /*********************************************************************** + # // test copying and pickling - copying and pickling of + # // instances of Curl and CurlMulti is not allowed + # ************************************************************************/ + + def test_copy_curl(self): + try: + copy.copy(self.curl) + # python 2 raises copy.Error, python 3 raises TypeError + except (copy.Error, TypeError): + pass + else: + assert False, "No exception when trying to copy a Curl handle" + + def test_copy_multi(self): + m = pycurl.CurlMulti() + try: + copy.copy(m) + except (copy.Error, TypeError): + pass + else: + assert False, "No exception when trying to copy a CurlMulti handle" + + def test_copy_share(self): + s = pycurl.CurlShare() + try: + copy.copy(s) + except (copy.Error, TypeError): + pass + else: + assert False, "No exception when trying to copy a CurlShare handle" + + def test_pickle_curl(self): + fp = util.StringIO() + p = pickle.Pickler(fp, 1) + try: + p.dump(self.curl) + # python 2 raises pickle.PicklingError, python 3 raises TypeError + except (pickle.PicklingError, TypeError): + pass + else: + assert 0, "No exception when trying to pickle a Curl handle" + del fp, p + + def test_pickle_multi(self): + m = pycurl.CurlMulti() + fp = util.StringIO() + p = pickle.Pickler(fp, 1) + try: + p.dump(m) + except (pickle.PicklingError, TypeError): + pass + else: + assert 0, "No exception when trying to pickle a CurlMulti handle" + del m, fp, p + + def test_pickle_share(self): + s = pycurl.CurlShare() + fp = util.StringIO() + p = pickle.Pickler(fp, 1) + try: + p.dump(s) + except (pickle.PicklingError, TypeError): + pass + else: + assert 0, "No exception when trying to pickle a CurlShare handle" + del s, fp, p + + def test_pickle_dumps_curl(self): + try: + pickle.dumps(self.curl) + # python 2 raises pickle.PicklingError, python 3 raises TypeError + except (pickle.PicklingError, TypeError): + pass + else: + self.fail("No exception when trying to pickle a Curl handle") + + def test_pickle_dumps_multi(self): + m = pycurl.CurlMulti() + try: + pickle.dumps(m) + except (pickle.PicklingError, TypeError): + pass + else: + self.fail("No exception when trying to pickle a CurlMulti handle") + + def test_pickle_dumps_share(self): + s = pycurl.CurlShare() + try: + pickle.dumps(s) + except (pickle.PicklingError, TypeError): + pass + else: + self.fail("No exception when trying to pickle a CurlShare handle") + + if cPickle is not None: + def test_cpickle_curl(self): + fp = util.StringIO() + p = cPickle.Pickler(fp, 1) + try: + p.dump(self.curl) + except cPickle.PicklingError: + pass + else: + assert 0, "No exception when trying to pickle a Curl handle via cPickle" + del fp, p + + def test_cpickle_multi(self): + m = pycurl.CurlMulti() + fp = util.StringIO() + p = cPickle.Pickler(fp, 1) + try: + p.dump(m) + except cPickle.PicklingError: + pass + else: + assert 0, "No exception when trying to pickle a CurlMulti handle via cPickle" + del m, fp, p + + def test_cpickle_share(self): + s = pycurl.CurlMulti() + fp = util.StringIO() + p = cPickle.Pickler(fp, 1) + try: + p.dump(s) + except cPickle.PicklingError: + pass + else: + assert 0, "No exception when trying to pickle a CurlShare handle via cPickle" + del s, fp, p diff --git a/tests/matrix.py b/tests/matrix.py new file mode 100644 index 0000000..698d289 --- /dev/null +++ b/tests/matrix.py @@ -0,0 +1,165 @@ +import os, os.path, subprocess, shutil + +try: + from urllib.request import urlopen +except ImportError: + from urllib import urlopen + +python_versions = ['2.6.8', '2.7.5', '3.1.5', '3.2.5', '3.3.5', '3.4.1'] +libcurl_versions = ['7.19.0', '7.46.0'] + +libcurl_meta = { + '7.19.0': { + 'patches': [ + 'curl-7.19.0-sslv2-c66b0b32fba-modified.patch', + #'curl-7.19.0-sslv2-2b0e09b0f98.patch', + ], + }, +} + +root = os.path.abspath(os.path.dirname(__file__)) + +class in_dir: + def __init__(self, dir): + self.dir = dir + + def __enter__(self): + self.oldwd = os.getcwd() + os.chdir(self.dir) + + def __exit__(self, type, value, traceback): + os.chdir(self.oldwd) + +def subprocess_check_call(cmd, **kwargs): + try: + subprocess.check_call(cmd, **kwargs) + except OSError as exc: + message = exc.args[0] + message = '%s while trying to execute %s' % (message, str(cmd)) + args = tuple([message] + exc.args[1:]) + raise type(exc)(args) + +def fetch(url, archive=None): + if archive is None: + archive = os.path.basename(url) + if not os.path.exists(archive): + sys.stdout.write("Fetching %s\n" % url) + io = urlopen(url) + with open('.tmp.%s' % archive, 'wb') as f: + while True: + chunk = io.read(65536) + if len(chunk) == 0: + break + f.write(chunk) + os.rename('.tmp.%s' % archive, archive) + +def build(archive, dir, prefix, meta=None): + if not os.path.exists(dir): + sys.stdout.write("Building %s\n" % archive) + subprocess_check_call(['tar', 'xf', archive]) + with in_dir(dir): + if meta and 'patches' in meta: + for patch in meta['patches']: + patch_path = os.path.join(root, 'matrix', patch) + subprocess_check_call(['patch', '-p1', '-i', patch_path]) + subprocess_check_call(['./configure', '--prefix=%s' % prefix]) + if 'post-configure' in meta: + for cmd in meta['post-configure']: + subprocess_check_call(cmd, shell=True) + subprocess_check_call(['make']) + subprocess_check_call(['make', 'install']) + +def run_matrix(python_versions, libcurl_versions): + for python_version in python_versions: + url = 'http://www.python.org/ftp/python/%s/Python-%s.tgz' % (python_version, python_version) + archive = os.path.basename(url) + fetch(url, archive) + + dir = archive.replace('.tgz', '') + prefix = os.path.abspath('i/%s' % dir) + build(archive, dir, prefix) + + for libcurl_version in libcurl_versions: + url = 'https://curl.haxx.se/download/curl-%s.tar.gz' % libcurl_version + archive = os.path.basename(url) + fetch(url, archive) + + dir = archive.replace('.tar.gz', '') + prefix = os.path.abspath('i/%s' % dir) + build(archive, dir, prefix, meta=libcurl_meta.get(libcurl_version)) + + fetch('https://raw.github.com/pypa/virtualenv/1.7/virtualenv.py', 'virtualenv-1.7.py') + fetch('https://raw.github.com/pypa/virtualenv/1.9.1/virtualenv.py', 'virtualenv-1.9.1.py') + + if not os.path.exists('venv'): + os.mkdir('venv') + + for python_version in python_versions: + python_version_pieces = [int(piece) for piece in python_version.split('.')[:2]] + for libcurl_version in libcurl_versions: + python_prefix = os.path.abspath('i/Python-%s' % python_version) + libcurl_prefix = os.path.abspath('i/curl-%s' % libcurl_version) + venv = os.path.abspath('venv/Python-%s-curl-%s' % (python_version, libcurl_version)) + if os.path.exists(venv): + shutil.rmtree(venv) + fetch('https://pypi.python.org/packages/2.6/s/setuptools/setuptools-0.6c11-py2.6.egg') + fetch('https://pypi.python.org/packages/2.7/s/setuptools/setuptools-0.6c11-py2.7.egg') + # I had virtualenv 1.8.2 installed systemwide which + # did not work with python 3.0: + # http://stackoverflow.com/questions/1422361/why-am-i-getting-this-error-related-to-pip-and-easy-install-when-trying-to-set + # so, use known versions everywhere + # md5=89e68df89faf1966bcbd99a0033fbf8e + fetch('https://pypi.python.org/packages/source/d/distribute/distribute-0.6.49.tar.gz') + subprocess_check_call(['python', 'virtualenv-1.9.1.py', venv, '-p', '%s/bin/python%d.%d' % (python_prefix, python_version_pieces[0], python_version_pieces[1]), '--no-site-packages', '--never-download']) + curl_config_path = os.path.join(libcurl_prefix, 'bin/curl-config') + curl_lib_path = os.path.join(libcurl_prefix, 'lib') + with in_dir('pycurl'): + extra_patches = [] + extra_env = [] + deps_cmd = 'pip install -r requirements-dev.txt' + extra_patches = ' && '.join(extra_patches) + extra_env = ' '.join(extra_env) + cmd = ''' + make clean && + . %(venv)s/bin/activate && + %(deps_cmd)s && %(extra_patches)s + python -V && + LD_LIBRARY_PATH=%(curl_lib_path)s PYCURL_CURL_CONFIG=%(curl_config_path)s %(extra_env)s make test + ''' % dict( + venv=venv, + deps_cmd=deps_cmd, + extra_patches=extra_patches, + curl_lib_path=curl_lib_path, + curl_config_path=curl_config_path, + extra_env=extra_env + ) + print(cmd) + subprocess_check_call(cmd, shell=True) + +if __name__ == '__main__': + import sys + + def main(): + import optparse + + parser = optparse.OptionParser() + parser.add_option('-p', '--python', help='Specify python version to test against') + parser.add_option('-c', '--curl', help='Specify libcurl version to test against') + options, args = parser.parse_args() + if options.python: + python_version = options.python + if python_version in python_versions: + chosen_python_versions = [python_version] + else: + chosen_python_versions = [v for v in python_versions if v.startswith(python_version)] + if len(chosen_python_versions) != 1: + raise Exception('Bogus python version requested: %s' % python_version) + else: + chosen_python_versions = python_versions + if options.curl: + chosen_libcurl_versions = [options.curl] + else: + chosen_libcurl_versions = libcurl_versions + run_matrix(chosen_python_versions, chosen_libcurl_versions) + + main() diff --git a/tests/matrix/curl-7.19.0-sslv2-2b0e09b0f98.patch b/tests/matrix/curl-7.19.0-sslv2-2b0e09b0f98.patch new file mode 100644 index 0000000..ae31bf1 --- /dev/null +++ b/tests/matrix/curl-7.19.0-sslv2-2b0e09b0f98.patch @@ -0,0 +1,40 @@ +commit 2b0e09b0f98e0f67417652dd7f4afd59bf895326 +Author: Daniel Stenberg +Date: Tue Dec 6 14:22:45 2011 +0100 + + OpenSSL: check for the SSLv2 function in configure + + If no SSLv2 was detected in OpenSSL by configure, then we enforce the + OPENSSL_NO_SSL2 define as it seems some people report it not being + defined properly in the OpenSSL headers. + +diff --git a/configure.ac b/configure.ac +index 94cdd83..4bf25dc 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -1514,7 +1514,8 @@ if test X"$OPT_SSL" != Xno; then + RAND_egd \ + ENGINE_cleanup \ + CRYPTO_cleanup_all_ex_data \ +- SSL_get_shutdown ) ++ SSL_get_shutdown \ ++ SSLv2_client_method ) + + dnl Make an attempt to detect if this is actually yassl's headers and + dnl OpenSSL emulation layer. We still leave everything else believing +diff --git a/lib/ssluse.c b/lib/ssluse.c +index af70fe0..8deea26 100644 +--- a/lib/ssluse.c ++++ b/lib/ssluse.c +@@ -127,6 +127,11 @@ + #define HAVE_ERR_REMOVE_THREAD_STATE 1 + #endif + ++#ifndef HAVE_SSLV2_CLIENT_METHOD ++#undef OPENSSL_NO_SSL2 /* undef first to avoid compiler warnings */ ++#define OPENSSL_NO_SSL2 ++#endif ++ + /* + * Number of bytes to read from the random number seed file. This must be + * a finite value (because some entropy "files" like /dev/urandom have diff --git a/tests/matrix/curl-7.19.0-sslv2-c66b0b32fba-modified.patch b/tests/matrix/curl-7.19.0-sslv2-c66b0b32fba-modified.patch new file mode 100644 index 0000000..c7f3cb3 --- /dev/null +++ b/tests/matrix/curl-7.19.0-sslv2-c66b0b32fba-modified.patch @@ -0,0 +1,29 @@ +commit c66b0b32fba175d5f096c944d8ec8f9f06299f4a +Author: Daniel Stenberg +Date: Sun Apr 10 19:14:22 2011 +0200 + + OpenSSL: no-sslv2 aware + + Allow openSSL without SSL2 to be used. This fix is inspired by the fix + provided by Cristian Rodríguez. + + Reported by: Cristian Rodríguez + +diff --git a/lib/ssluse.c b/lib/ssluse.c +index 654ffaa..caffdad 100644 +--- a/lib/ssluse.c ++++ b/lib/ssluse.c +@@ -1327,8 +1327,13 @@ ossl_connect_step1(struct connectdata *conn, + req_method = TLSv1_client_method(); + break; + case CURL_SSLVERSION_SSLv2: ++#ifdef OPENSSL_NO_SSL2 ++ failf(data, "OpenSSL was built without SSLv2 support"); ++ return CURLE_UNSUPPORTED_PROTOCOL /* CURLE_NOT_BUILT_IN not defined in 7.19.0 */; ++#else + req_method = SSLv2_client_method(); + break; ++#endif + case CURL_SSLVERSION_SSLv3: + req_method = SSLv3_client_method(); + break; diff --git a/tests/matrix/openssl-1.0.1e-fix_pod_syntax-1.patch b/tests/matrix/openssl-1.0.1e-fix_pod_syntax-1.patch new file mode 100644 index 0000000..ba25afe --- /dev/null +++ b/tests/matrix/openssl-1.0.1e-fix_pod_syntax-1.patch @@ -0,0 +1,393 @@ +Submitted By: Martin Ward +Date: 2013-06-18 +Initial Package Version: 1.0.1e +Upstream Status: Unknown +Origin: self, based on fedora +Description: Fixes install with perl-5.18. + +diff -Naur openssl-1.0.1e.orig/doc/apps/cms.pod openssl-1.0.1e/doc/apps/cms.pod +--- openssl-1.0.1e.orig/doc/apps/cms.pod 2013-06-06 14:35:15.867871879 +0100 ++++ openssl-1.0.1e/doc/apps/cms.pod 2013-06-06 14:35:25.791747119 +0100 +@@ -450,28 +450,28 @@ + + =over 4 + +-=item 0 ++=item C<0> + + the operation was completely successfully. + +-=item 1 ++=item C<1> + + an error occurred parsing the command options. + +-=item 2 ++=item C<2> + + one of the input files could not be read. + +-=item 3 ++=item C<3> + + an error occurred creating the CMS file or when reading the MIME + message. + +-=item 4 ++=item C<4> + + an error occurred decrypting or verifying the message. + +-=item 5 ++=item C<5> + + the message was verified correctly but an error occurred writing out + the signers certificates. +diff -Naur openssl-1.0.1e.orig/doc/apps/smime.pod openssl-1.0.1e/doc/apps/smime.pod +--- openssl-1.0.1e.orig/doc/apps/smime.pod 2013-06-06 14:35:15.867871879 +0100 ++++ openssl-1.0.1e/doc/apps/smime.pod 2013-06-06 14:35:25.794747082 +0100 +@@ -308,28 +308,28 @@ + + =over 4 + +-=item 0 ++=item C<0> + + the operation was completely successfully. + +-=item 1 ++=item C<1> + + an error occurred parsing the command options. + +-=item 2 ++=item C<2> + + one of the input files could not be read. + +-=item 3 ++=item C<3> + + an error occurred creating the PKCS#7 file or when reading the MIME + message. + +-=item 4 ++=item C<4> + + an error occurred decrypting or verifying the message. + +-=item 5 ++=item C<5> + + the message was verified correctly but an error occurred writing out + the signers certificates. +diff -Naur openssl-1.0.1e.orig/doc/crypto/X509_STORE_CTX_get_error.pod openssl-1.0.1e/doc/crypto/X509_STORE_CTX_get_error.pod +--- openssl-1.0.1e.orig/doc/crypto/X509_STORE_CTX_get_error.pod 2013-06-06 14:35:15.874871791 +0100 ++++ openssl-1.0.1e/doc/crypto/X509_STORE_CTX_get_error.pod 2013-06-06 14:37:13.826388940 +0100 +@@ -278,6 +278,8 @@ + an application specific error. This will never be returned unless explicitly + set by an application. + ++=back ++ + =head1 NOTES + + The above functions should be used instead of directly referencing the fields +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_accept.pod openssl-1.0.1e/doc/ssl/SSL_accept.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_accept.pod 2013-06-06 14:35:15.871871829 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_accept.pod 2013-06-06 14:35:25.796747057 +0100 +@@ -44,12 +44,12 @@ + + =over 4 + +-=item 1 ++=item C<1> + + The TLS/SSL handshake was successfully completed, a TLS/SSL connection has been + established. + +-=item 0 ++=item C<0> + + The TLS/SSL handshake was not successful but was shut down controlled and + by the specifications of the TLS/SSL protocol. Call SSL_get_error() with the +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_clear.pod openssl-1.0.1e/doc/ssl/SSL_clear.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_clear.pod 2013-06-06 14:35:15.871871829 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_clear.pod 2013-06-06 14:35:25.803746969 +0100 +@@ -56,12 +56,12 @@ + + =over 4 + +-=item 0 ++=item C<0> + + The SSL_clear() operation could not be performed. Check the error stack to + find out the reason. + +-=item 1 ++=item C<1> + + The SSL_clear() operation was successful. + +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_COMP_add_compression_method.pod openssl-1.0.1e/doc/ssl/SSL_COMP_add_compression_method.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_COMP_add_compression_method.pod 2013-06-06 14:35:15.870871842 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_COMP_add_compression_method.pod 2013-06-06 14:35:25.806746931 +0100 +@@ -53,11 +53,11 @@ + + =over 4 + +-=item 0 ++=item C<0> + + The operation succeeded. + +-=item 1 ++=item C<1> + + The operation failed. Check the error queue to find out the reason. + +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_connect.pod openssl-1.0.1e/doc/ssl/SSL_connect.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_connect.pod 2013-06-06 14:35:15.869871854 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_connect.pod 2013-06-06 14:35:25.808746906 +0100 +@@ -41,12 +41,12 @@ + + =over 4 + +-=item 1 ++=item C<1> + + The TLS/SSL handshake was successfully completed, a TLS/SSL connection has been + established. + +-=item 0 ++=item C<0> + + The TLS/SSL handshake was not successful but was shut down controlled and + by the specifications of the TLS/SSL protocol. Call SSL_get_error() with the +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_CTX_add_session.pod openssl-1.0.1e/doc/ssl/SSL_CTX_add_session.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_CTX_add_session.pod 2013-06-06 14:35:15.871871829 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_CTX_add_session.pod 2013-06-06 14:35:25.816746805 +0100 +@@ -52,13 +52,13 @@ + + =over 4 + +-=item 0 ++=item C<0> + + The operation failed. In case of the add operation, it was tried to add + the same (identical) session twice. In case of the remove operation, the + session was not found in the cache. + +-=item 1 ++=item C<1> + + The operation succeeded. + +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_CTX_load_verify_locations.pod openssl-1.0.1e/doc/ssl/SSL_CTX_load_verify_locations.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_CTX_load_verify_locations.pod 2013-06-06 14:35:15.870871842 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_CTX_load_verify_locations.pod 2013-06-06 14:35:25.818746780 +0100 +@@ -100,13 +100,13 @@ + + =over 4 + +-=item 0 ++=item C<0> + + The operation failed because B and B are NULL or the + processing at one of the locations specified failed. Check the error + stack to find out the reason. + +-=item 1 ++=item C<1> + + The operation succeeded. + +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_CTX_set_client_CA_list.pod openssl-1.0.1e/doc/ssl/SSL_CTX_set_client_CA_list.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_CTX_set_client_CA_list.pod 2013-06-06 14:35:15.871871829 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_CTX_set_client_CA_list.pod 2013-06-06 14:35:25.821746742 +0100 +@@ -66,11 +66,11 @@ + + =over 4 + +-=item 1 ++=item C<1> + + The operation succeeded. + +-=item 0 ++=item C<0> + + A failure while manipulating the STACK_OF(X509_NAME) object occurred or + the X509_NAME could not be extracted from B. Check the error stack +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_CTX_set_session_id_context.pod openssl-1.0.1e/doc/ssl/SSL_CTX_set_session_id_context.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_CTX_set_session_id_context.pod 2013-06-06 14:35:15.871871829 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_CTX_set_session_id_context.pod 2013-06-06 14:35:25.828746654 +0100 +@@ -64,13 +64,13 @@ + + =over 4 + +-=item 0 ++=item C<0> + + The length B of the session id context B exceeded + the maximum allowed length of B. The error + is logged to the error stack. + +-=item 1 ++=item C<1> + + The operation succeeded. + +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_CTX_set_ssl_version.pod openssl-1.0.1e/doc/ssl/SSL_CTX_set_ssl_version.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_CTX_set_ssl_version.pod 2013-06-06 14:35:15.871871829 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_CTX_set_ssl_version.pod 2013-06-06 14:35:25.831746617 +0100 +@@ -42,11 +42,11 @@ + + =over 4 + +-=item 0 ++=item C<0> + + The new choice failed, check the error stack to find out the reason. + +-=item 1 ++=item C<1> + + The operation succeeded. + +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_CTX_use_psk_identity_hint.pod openssl-1.0.1e/doc/ssl/SSL_CTX_use_psk_identity_hint.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_CTX_use_psk_identity_hint.pod 2013-06-06 14:35:15.870871842 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_CTX_use_psk_identity_hint.pod 2013-06-06 14:36:42.456783309 +0100 +@@ -81,6 +81,8 @@ + + Return values from the server callback are interpreted as follows: + ++=over ++ + =item > 0 + + PSK identity was found and the server callback has provided the PSK +@@ -94,9 +96,11 @@ + connection will fail with decryption_error before it will be finished + completely. + +-=item 0 ++=item C<0> + + PSK identity was not found. An "unknown_psk_identity" alert message + will be sent and the connection setup fails. + ++=back ++ + =cut +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_do_handshake.pod openssl-1.0.1e/doc/ssl/SSL_do_handshake.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_do_handshake.pod 2013-06-06 14:35:15.869871854 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_do_handshake.pod 2013-06-06 14:35:25.839746516 +0100 +@@ -45,12 +45,12 @@ + + =over 4 + +-=item 1 ++=item C<1> + + The TLS/SSL handshake was successfully completed, a TLS/SSL connection has been + established. + +-=item 0 ++=item C<0> + + The TLS/SSL handshake was not successful but was shut down controlled and + by the specifications of the TLS/SSL protocol. Call SSL_get_error() with the +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_read.pod openssl-1.0.1e/doc/ssl/SSL_read.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_read.pod 2013-06-06 14:35:15.871871829 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_read.pod 2013-06-06 14:35:25.847746415 +0100 +@@ -86,7 +86,7 @@ + The read operation was successful; the return value is the number of + bytes actually read from the TLS/SSL connection. + +-=item 0 ++=item C<0> + + The read operation was not successful. The reason may either be a clean + shutdown due to a "close notify" alert sent by the peer (in which case +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_session_reused.pod openssl-1.0.1e/doc/ssl/SSL_session_reused.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_session_reused.pod 2013-06-06 14:35:15.871871829 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_session_reused.pod 2013-06-06 14:35:25.849746390 +0100 +@@ -27,11 +27,11 @@ + + =over 4 + +-=item 0 ++=item C<0> + + A new session was negotiated. + +-=item 1 ++=item C<1> + + A session was reused. + +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_set_fd.pod openssl-1.0.1e/doc/ssl/SSL_set_fd.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_set_fd.pod 2013-06-06 14:35:15.869871854 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_set_fd.pod 2013-06-06 14:35:25.852746353 +0100 +@@ -35,11 +35,11 @@ + + =over 4 + +-=item 0 ++=item C<0> + + The operation failed. Check the error stack to find out why. + +-=item 1 ++=item C<1> + + The operation succeeded. + +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_set_session.pod openssl-1.0.1e/doc/ssl/SSL_set_session.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_set_session.pod 2013-06-06 14:35:15.870871842 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_set_session.pod 2013-06-06 14:35:25.855746315 +0100 +@@ -37,11 +37,11 @@ + + =over 4 + +-=item 0 ++=item C<0> + + The operation failed; check the error stack to find out the reason. + +-=item 1 ++=item C<1> + + The operation succeeded. + +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_shutdown.pod openssl-1.0.1e/doc/ssl/SSL_shutdown.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_shutdown.pod 2013-06-06 14:35:15.870871842 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_shutdown.pod 2013-06-06 14:35:25.857746290 +0100 +@@ -92,12 +92,12 @@ + + =over 4 + +-=item 1 ++=item C<1> + + The shutdown was successfully completed. The "close notify" alert was sent + and the peer's "close notify" alert was received. + +-=item 0 ++=item C<0> + + The shutdown is not yet finished. Call SSL_shutdown() for a second time, + if a bidirectional shutdown shall be performed. +diff -Naur openssl-1.0.1e.orig/doc/ssl/SSL_write.pod openssl-1.0.1e/doc/ssl/SSL_write.pod +--- openssl-1.0.1e.orig/doc/ssl/SSL_write.pod 2013-06-06 14:35:15.870871842 +0100 ++++ openssl-1.0.1e/doc/ssl/SSL_write.pod 2013-06-06 14:35:25.865746189 +0100 +@@ -79,7 +79,7 @@ + The write operation was successful, the return value is the number of + bytes actually written to the TLS/SSL connection. + +-=item 0 ++=item C<0> + + The write operation was not successful. Probably the underlying connection + was closed. Call SSL_get_error() with the return value B to find out, diff --git a/tests/memory_mgmt_test.py b/tests/memory_mgmt_test.py new file mode 100644 index 0000000..ec841f2 --- /dev/null +++ b/tests/memory_mgmt_test.py @@ -0,0 +1,380 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import sys +import weakref +import pycurl +import unittest +import gc +import flaky +from . import util + +debug = False + +if sys.platform == 'win32': + devnull = 'NUL' +else: + devnull = '/dev/null' + +@flaky.flaky(max_runs=3) +class MemoryMgmtTest(unittest.TestCase): + def maybe_enable_debug(self): + if debug: + flags = gc.DEBUG_COLLECTABLE | gc.DEBUG_UNCOLLECTABLE + # python 3 has no DEBUG_OBJECTS + if hasattr(gc, 'DEBUG_OBJECTS'): + flags |= gc.DEBUG_OBJECTS + flags |= gc.DEBUG_STATS + gc.set_debug(flags) + gc.collect() + + print("Tracked objects:", len(gc.get_objects())) + + def maybe_print_objects(self): + if debug: + print("Tracked objects:", len(gc.get_objects())) + + def tearDown(self): + gc.set_debug(0) + + def test_multi_collection(self): + gc.collect() + self.maybe_enable_debug() + + multi = pycurl.CurlMulti() + t = [] + searches = [] + for a in range(100): + curl = util.DefaultCurl() + multi.add_handle(curl) + t.append(curl) + + c_id = id(curl) + searches.append(c_id) + m_id = id(multi) + searches.append(m_id) + + self.maybe_print_objects() + + for curl in t: + curl.close() + multi.remove_handle(curl) + + self.maybe_print_objects() + + del curl + del t + del multi + + self.maybe_print_objects() + gc.collect() + self.maybe_print_objects() + + objects = gc.get_objects() + for search in searches: + for object in objects: + assert search != id(object) + + def test_multi_cycle(self): + gc.collect() + self.maybe_enable_debug() + + multi = pycurl.CurlMulti() + t = [] + searches = [] + for a in range(100): + curl = util.DefaultCurl() + multi.add_handle(curl) + t.append(curl) + + c_id = id(curl) + searches.append(c_id) + m_id = id(multi) + searches.append(m_id) + + self.maybe_print_objects() + + del curl + del t + del multi + + self.maybe_print_objects() + gc.collect() + self.maybe_print_objects() + + objects = gc.get_objects() + for search in searches: + for object in objects: + assert search != id(object) + + def test_share_collection(self): + gc.collect() + self.maybe_enable_debug() + + share = pycurl.CurlShare() + t = [] + searches = [] + for a in range(100): + curl = util.DefaultCurl() + curl.setopt(curl.SHARE, share) + t.append(curl) + + c_id = id(curl) + searches.append(c_id) + m_id = id(share) + searches.append(m_id) + + self.maybe_print_objects() + + for curl in t: + curl.unsetopt(curl.SHARE) + curl.close() + + self.maybe_print_objects() + + del curl + del t + del share + + self.maybe_print_objects() + gc.collect() + self.maybe_print_objects() + + objects = gc.get_objects() + for search in searches: + for object in objects: + assert search != id(object) + + def test_share_cycle(self): + gc.collect() + self.maybe_enable_debug() + + share = pycurl.CurlShare() + t = [] + searches = [] + for a in range(100): + curl = util.DefaultCurl() + curl.setopt(curl.SHARE, share) + t.append(curl) + + c_id = id(curl) + searches.append(c_id) + m_id = id(share) + searches.append(m_id) + + self.maybe_print_objects() + + del curl + del t + del share + + self.maybe_print_objects() + gc.collect() + self.maybe_print_objects() + + objects = gc.get_objects() + for search in searches: + for object in objects: + assert search != id(object) + + # basic check of reference counting (use a memory checker like valgrind) + def test_reference_counting(self): + c = util.DefaultCurl() + m = pycurl.CurlMulti() + m.add_handle(c) + del m + m = pycurl.CurlMulti() + c.close() + del m, c + + def test_cyclic_gc(self): + gc.collect() + c = util.DefaultCurl() + c.m = pycurl.CurlMulti() + c.m.add_handle(c) + # create some nasty cyclic references + c.c = c + c.c.c1 = c + c.c.c2 = c + c.c.c3 = c.c + c.c.c4 = c.m + c.m.c = c + c.m.m = c.m + c.m.c = c + # delete + gc.collect() + self.maybe_enable_debug() + ##print gc.get_referrers(c) + ##print gc.get_objects() + #if opts.verbose >= 1: + #print("Tracked objects:", len(gc.get_objects())) + c_id = id(c) + # The `del' below should delete these 4 objects: + # Curl + internal dict, CurlMulti + internal dict + del c + gc.collect() + objects = gc.get_objects() + for object in objects: + assert id(object) != c_id + #if opts.verbose >= 1: + #print("Tracked objects:", len(gc.get_objects())) + + def test_refcounting_bug_in_reset(self): + if sys.platform == 'win32': + iters = 10000 + else: + iters = 100000 + + try: + range_generator = xrange + except NameError: + range_generator = range + # Ensure that the refcounting error in "reset" is fixed: + for i in range_generator(iters): + c = util.DefaultCurl() + c.reset() + c.close() + + def test_writefunction_collection(self): + self.check_callback(pycurl.WRITEFUNCTION) + + def test_headerfunction_collection(self): + self.check_callback(pycurl.HEADERFUNCTION) + + def test_readfunction_collection(self): + self.check_callback(pycurl.READFUNCTION) + + def test_progressfunction_collection(self): + self.check_callback(pycurl.PROGRESSFUNCTION) + + @util.min_libcurl(7, 32, 0) + def test_xferinfofunction_collection(self): + self.check_callback(pycurl.XFERINFOFUNCTION) + + def test_debugfunction_collection(self): + self.check_callback(pycurl.DEBUGFUNCTION) + + def test_ioctlfunction_collection(self): + self.check_callback(pycurl.IOCTLFUNCTION) + + def test_opensocketfunction_collection(self): + self.check_callback(pycurl.OPENSOCKETFUNCTION) + + def test_seekfunction_collection(self): + self.check_callback(pycurl.SEEKFUNCTION) + + # This is failing too much on appveyor + @util.only_unix + def check_callback(self, callback): + # Note: extracting a context manager seems to result in + # everything being garbage collected even if the C code + # does not clear the callback + object_count = 0 + gc.collect() + object_count = len(gc.get_objects()) + + c = util.DefaultCurl() + c.setopt(callback, lambda x: True) + del c + + gc.collect() + new_object_count = len(gc.get_objects()) + # it seems that GC sometimes collects something that existed + # before this test ran, GH issues #273/#274 + self.assertIn(new_object_count, (object_count, object_count-1)) + + def test_postfields_unicode_memory_leak_gh252(self): + # this test passed even before the memory leak was fixed, + # not sure why. + + c = util.DefaultCurl() + gc.collect() + before_object_count = len(gc.get_objects()) + + for i in range(100000): + c.setopt(pycurl.POSTFIELDS, util.u('hello world')) + + gc.collect() + after_object_count = len(gc.get_objects()) + self.assertTrue(after_object_count <= before_object_count + 1000, 'Grew from %d to %d objects' % (before_object_count, after_object_count)) + c.close() + + def test_form_bufferptr_memory_leak_gh267(self): + c = util.DefaultCurl() + gc.collect() + before_object_count = len(gc.get_objects()) + + for i in range(100000): + c.setopt(pycurl.HTTPPOST, [ + # Newer versions of libcurl accept FORM_BUFFERPTR + # without FORM_BUFFER and reproduce the memory leak; + # libcurl 7.19.0 requires FORM_BUFFER to be given before + # FORM_BUFFERPTR. + ("post1", (pycurl.FORM_BUFFER, 'foo.txt', pycurl.FORM_BUFFERPTR, "data1")), + ("post2", (pycurl.FORM_BUFFER, 'bar.txt', pycurl.FORM_BUFFERPTR, "data2")), + ]) + + gc.collect() + after_object_count = len(gc.get_objects()) + self.assertTrue(after_object_count <= before_object_count + 1000, 'Grew from %d to %d objects' % (before_object_count, after_object_count)) + c.close() + + def do_data_refcounting(self, option): + c = util.DefaultCurl() + f = open(devnull, 'a+') + c.setopt(option, f) + ref = weakref.ref(f) + del f + gc.collect() + assert ref() + + for i in range(100): + assert ref() + c.setopt(option, ref()) + gc.collect() + assert ref() + + c.close() + gc.collect() + assert ref() is None + + def test_readdata_refcounting(self): + self.do_data_refcounting(pycurl.READDATA) + + def test_writedata_refcounting(self): + self.do_data_refcounting(pycurl.WRITEDATA) + + def test_writeheader_refcounting(self): + self.do_data_refcounting(pycurl.WRITEHEADER) + + # Python < 3.5 cannot create weak references to functions + @util.min_python(3, 5) + def do_function_refcounting(self, option, method_name): + c = util.DefaultCurl() + f = open(devnull, 'a+') + fn = getattr(f, method_name) + c.setopt(option, fn) + ref = weakref.ref(fn) + del f, fn + gc.collect() + assert ref() + + for i in range(100): + assert ref() + c.setopt(option, ref()) + gc.collect() + assert ref() + + c.close() + gc.collect() + assert ref() is None + + def test_readfunction_refcounting(self): + self.do_function_refcounting(pycurl.READFUNCTION, 'read') + + def test_writefunction_refcounting(self): + self.do_function_refcounting(pycurl.WRITEFUNCTION, 'write') + + def test_headerfunction_refcounting(self): + self.do_function_refcounting(pycurl.HEADERFUNCTION, 'write') diff --git a/tests/multi_callback_test.py b/tests/multi_callback_test.py new file mode 100644 index 0000000..bbb13f4 --- /dev/null +++ b/tests/multi_callback_test.py @@ -0,0 +1,100 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import pytest +import sys +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class MultiCallbackTest(unittest.TestCase): + def setUp(self): + self.easy = util.DefaultCurl() + self.easy.setopt(pycurl.URL, 'http://%s:8380/long_pause' % localhost) + self.multi = pycurl.CurlMulti() + self.multi.setopt(pycurl.M_SOCKETFUNCTION, self.socket_callback) + self.multi.setopt(pycurl.M_TIMERFUNCTION, self.timer_callback) + self.socket_result = None + self.timer_result = None + self.sockets = {} + + def tearDown(self): + self.multi.close() + self.easy.close() + + def socket_callback(self, ev_bitmask, sock_fd, multi, data): + self.socket_result = (sock_fd, ev_bitmask) + if ev_bitmask & pycurl.POLL_REMOVE: + self.sockets.pop(sock_fd) + else: + self.sockets[sock_fd] = ev_bitmask | self.sockets.get(sock_fd, 0) + + def timer_callback(self, timeout_ms): + self.timer_result = timeout_ms + + def partial_transfer(self): + perform = True + def write_callback(data): + nonlocal perform + perform = False + self.easy.setopt(pycurl.WRITEFUNCTION, write_callback) + self.multi.add_handle(self.easy) + self.multi.socket_action(pycurl.SOCKET_TIMEOUT, 0) + while self.sockets and perform: + for socket, action in tuple(self.sockets.items()): + self.multi.socket_action(socket, action) + + # multi.socket_action must call both SOCKETFUNCTION and TIMERFUNCTION at + # various points during the transfer (at least at the start and end) + @pytest.mark.xfail(sys.platform == 'darwin', reason='https://github.com/pycurl/pycurl/issues/729') + def test_multi_socket_action(self): + self.multi.add_handle(self.easy) + self.timer_result = None + self.socket_result = None + self.multi.socket_action(pycurl.SOCKET_TIMEOUT, 0) + assert self.socket_result is not None + assert self.timer_result is not None + + # multi.add_handle must call TIMERFUNCTION to schedule a kick-start + def test_multi_add_handle(self): + self.multi.add_handle(self.easy) + assert self.timer_result is not None + + # (mid-transfer) multi.remove_handle must call SOCKETFUNCTION to remove sockets + @pytest.mark.xfail(sys.platform == 'darwin', reason='https://github.com/pycurl/pycurl/issues/729') + def test_multi_remove_handle(self): + self.multi.add_handle(self.easy) + self.multi.socket_action(pycurl.SOCKET_TIMEOUT, 0) + self.socket_result = None + self.multi.remove_handle(self.easy) + assert self.socket_result is not None + + # (mid-transfer) easy.pause(PAUSE_ALL) must call SOCKETFUNCTION to remove sockets + # (mid-transfer) easy.pause(PAUSE_CONT) must call TIMERFUNCTION to resume + @pytest.mark.xfail(sys.platform == 'darwin', reason='https://github.com/pycurl/pycurl/issues/729') + def test_easy_pause_unpause(self): + self.partial_transfer() + self.socket_result = None + # libcurl will now inform us that we should remove some sockets + self.easy.pause(pycurl.PAUSE_ALL) + assert self.socket_result is not None + self.socket_result = None + self.timer_result = None + # libcurl will now tell us to add those sockets and schedule a kickstart + self.easy.pause(pycurl.PAUSE_CONT) + assert self.socket_result is not None + assert self.timer_result is not None + + # (mid-transfer) easy.close() must call SOCKETFUNCTION to remove sockets + @pytest.mark.xfail(sys.platform == 'darwin', reason='https://github.com/pycurl/pycurl/issues/729') + def test_easy_close(self): + self.partial_transfer() + self.socket_result = None + self.easy.close() + assert self.socket_result is not None diff --git a/tests/multi_memory_mgmt_test.py b/tests/multi_memory_mgmt_test.py new file mode 100644 index 0000000..62d66ed --- /dev/null +++ b/tests/multi_memory_mgmt_test.py @@ -0,0 +1,58 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import unittest +import gc +import flaky +import weakref + +from . import util + +debug = False + +@flaky.flaky(max_runs=3) +class MultiMemoryMgmtTest(unittest.TestCase): + def test_opensocketfunction_collection(self): + self.check_callback(pycurl.M_SOCKETFUNCTION) + + def test_seekfunction_collection(self): + self.check_callback(pycurl.M_TIMERFUNCTION) + + def check_callback(self, callback): + # Note: extracting a context manager seems to result in + # everything being garbage collected even if the C code + # does not clear the callback + object_count = 0 + gc.collect() + # gc.collect() can create new objects... running it again here + # settles tracked object count for the actual test below + gc.collect() + object_count = len(gc.get_objects()) + + c = pycurl.CurlMulti() + c.setopt(callback, lambda x: True) + del c + + gc.collect() + new_object_count = len(gc.get_objects()) + # it seems that GC sometimes collects something that existed + # before this test ran, GH issues #273/#274 + self.assertIn(new_object_count, (object_count, object_count-1)) + + def test_curl_ref(self): + c = util.DefaultCurl() + m = pycurl.CurlMulti() + + ref = weakref.ref(c) + m.add_handle(c) + del c + + assert ref() + gc.collect() + assert ref() + + m.remove_handle(ref()) + gc.collect() + assert ref() is None diff --git a/tests/multi_option_constants_test.py b/tests/multi_option_constants_test.py new file mode 100644 index 0000000..3f5ed0b --- /dev/null +++ b/tests/multi_option_constants_test.py @@ -0,0 +1,89 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import sys +import pycurl +import unittest + +from . import util + +class MultiOptionConstantsTest(unittest.TestCase): + def setUp(self): + super(MultiOptionConstantsTest, self).setUp() + + self.m = pycurl.CurlMulti() + + def tearDown(self): + super(MultiOptionConstantsTest, self).tearDown() + + self.m.close() + + def test_option_constant_on_pycurl(self): + assert hasattr(pycurl, 'M_PIPELINING') + + def test_option_constant_on_curlmulti(self): + assert hasattr(self.m, 'M_PIPELINING') + + @util.min_libcurl(7, 43, 0) + def test_pipe_constants(self): + self.m.setopt(self.m.M_PIPELINING, self.m.PIPE_NOTHING) + self.m.setopt(self.m.M_PIPELINING, self.m.PIPE_HTTP1) + self.m.setopt(self.m.M_PIPELINING, self.m.PIPE_MULTIPLEX) + + @util.min_libcurl(7, 30, 0) + def test_multi_pipeline_opts(self): + assert hasattr(pycurl, 'M_MAX_HOST_CONNECTIONS') + assert hasattr(pycurl, 'M_MAX_PIPELINE_LENGTH') + assert hasattr(pycurl, 'M_CONTENT_LENGTH_PENALTY_SIZE') + assert hasattr(pycurl, 'M_CHUNK_LENGTH_PENALTY_SIZE') + assert hasattr(pycurl, 'M_MAX_TOTAL_CONNECTIONS') + self.m.setopt(pycurl.M_MAX_HOST_CONNECTIONS, 2) + self.m.setopt(pycurl.M_MAX_PIPELINE_LENGTH, 2) + self.m.setopt(pycurl.M_CONTENT_LENGTH_PENALTY_SIZE, 2) + self.m.setopt(pycurl.M_CHUNK_LENGTH_PENALTY_SIZE, 2) + self.m.setopt(pycurl.M_MAX_TOTAL_CONNECTIONS, 2) + + @util.min_libcurl(7, 30, 0) + def test_multi_pipelining_site_bl(self): + self.check_multi_charpp_option(self.m.M_PIPELINING_SITE_BL) + + @util.min_libcurl(7, 30, 0) + def test_multi_pipelining_server_bl(self): + self.check_multi_charpp_option(self.m.M_PIPELINING_SERVER_BL) + + def check_multi_charpp_option(self, option): + input = [util.b('test1'), util.b('test2')] + self.m.setopt(option, input) + input = [util.u('test1'), util.u('test2')] + self.m.setopt(option, input) + self.m.setopt(option, []) + input = (util.b('test1'), util.b('test2')) + self.m.setopt(option, input) + input = (util.u('test1'), util.u('test2')) + self.m.setopt(option, input) + self.m.setopt(option, ()) + self.m.setopt(option, None) + + try: + self.m.setopt(option, 1) + self.fail('expected to raise') + except TypeError: + exc = sys.exc_info()[1] + assert 'integers are not supported for this option' in str(exc) + + def test_multi_callback_opts(self): + def callback(*args, **kwargs): + pass + self.m.setopt(pycurl.M_SOCKETFUNCTION, callback) + self.m.setopt(pycurl.M_TIMERFUNCTION, callback) + self.m.setopt(pycurl.M_SOCKETFUNCTION, None) + self.m.setopt(pycurl.M_TIMERFUNCTION, None) + + def test_multi_unsetopt_unsupported(self): + try: + self.m.setopt(pycurl.M_MAXCONNECTS, None) + self.fail('expected to raise') + except TypeError: + exc = sys.exc_info()[1] + assert 'unsetting is not supported for this option' in str(exc) diff --git a/tests/multi_socket_select_test.py b/tests/multi_socket_select_test.py new file mode 100644 index 0000000..5ce7fe8 --- /dev/null +++ b/tests/multi_socket_select_test.py @@ -0,0 +1,123 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import unittest +import select +import flaky + +from . import appmanager +from . import util + +setup_module_1, teardown_module_1 = appmanager.setup(('app', 8380)) +setup_module_2, teardown_module_2 = appmanager.setup(('app', 8381)) +setup_module_3, teardown_module_3 = appmanager.setup(('app', 8382)) + +def setup_module(mod): + setup_module_1(mod) + setup_module_2(mod) + setup_module_3(mod) + +def teardown_module(mod): + teardown_module_3(mod) + teardown_module_2(mod) + teardown_module_1(mod) + +@flaky.flaky(max_runs=3) +class MultiSocketSelectTest(unittest.TestCase): + def test_multi_socket_select(self): + sockets = set() + timeout = 0 + + urls = [ + # we need libcurl to actually wait on the handles, + # and initiate polling. + # thus use urls that sleep for a bit. + 'http://%s:8380/short_wait' % localhost, + 'http://%s:8381/short_wait' % localhost, + 'http://%s:8382/short_wait' % localhost, + ] + + socket_events = [] + + # socket callback + def socket(event, socket, multi, data): + if event == pycurl.POLL_REMOVE: + #print("Remove Socket %d"%socket) + sockets.remove(socket) + else: + if socket not in sockets: + #print("Add socket %d"%socket) + sockets.add(socket) + socket_events.append((event, multi)) + + # init + m = pycurl.CurlMulti() + m.setopt(pycurl.M_SOCKETFUNCTION, socket) + m.handles = [] + for url in urls: + c = util.DefaultCurl() + # save info in standard Python attributes + c.url = url + c.body = util.BytesIO() + c.http_code = -1 + m.handles.append(c) + # pycurl API calls + c.setopt(c.URL, c.url) + c.setopt(c.WRITEFUNCTION, c.body.write) + m.add_handle(c) + + # get data + #num_handles = len(m.handles) + + while (pycurl.E_CALL_MULTI_PERFORM==m.socket_all()[0]): + pass + + timeout = m.timeout() + + # timeout might be -1, indicating that all work is done + # XXX make sure there is always work to be done here? + while timeout >= 0: + (rr, wr, er) = select.select(sockets,sockets,sockets,timeout/1000.0) + socketSet = set(rr+wr+er) + if socketSet: + for s in socketSet: + while True: + (ret,running) = m.socket_action(s,0) + if ret!=pycurl.E_CALL_MULTI_PERFORM: + break + else: + (ret,running) = m.socket_action(pycurl.SOCKET_TIMEOUT,0) + if running==0: + break + + for c in m.handles: + # save info in standard Python attributes + c.http_code = c.getinfo(c.HTTP_CODE) + + # at least in and remove events per socket + assert len(socket_events) >= 6, 'Less than 6 socket events: %s' % repr(socket_events) + + # print result + for c in m.handles: + self.assertEqual('success', c.body.getvalue().decode()) + self.assertEqual(200, c.http_code) + + # multi, not curl handle + self.check(pycurl.POLL_IN, m, socket_events) + self.check(pycurl.POLL_REMOVE, m, socket_events) + + # close handles + for c in m.handles: + # pycurl API calls + m.remove_handle(c) + c.close() + m.close() + + def check(self, event, multi, socket_events): + for event_, multi_ in socket_events: + if event == event_ and multi == multi_: + return + assert False, '%d %s not found in socket events' % (event, multi) diff --git a/tests/multi_socket_test.py b/tests/multi_socket_test.py new file mode 100644 index 0000000..5bee7af --- /dev/null +++ b/tests/multi_socket_test.py @@ -0,0 +1,98 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import unittest + +from . import appmanager +from . import util + +setup_module_1, teardown_module_1 = appmanager.setup(('app', 8380)) +setup_module_2, teardown_module_2 = appmanager.setup(('app', 8381)) +setup_module_3, teardown_module_3 = appmanager.setup(('app', 8382)) + +def setup_module(mod): + setup_module_1(mod) + setup_module_2(mod) + setup_module_3(mod) + +def teardown_module(mod): + teardown_module_3(mod) + teardown_module_2(mod) + teardown_module_1(mod) + +class MultiSocketTest(unittest.TestCase): + def test_multi_socket(self): + urls = [ + # not sure why requesting /success produces no events. + # see multi_socket_select_test.py for a longer explanation + # why short wait is used there. + 'http://%s:8380/short_wait' % localhost, + 'http://%s:8381/short_wait' % localhost, + 'http://%s:8382/short_wait' % localhost, + ] + + socket_events = [] + + # socket callback + def socket(event, socket, multi, data): + #print(event, socket, multi, data) + socket_events.append((event, multi)) + + # init + m = pycurl.CurlMulti() + m.setopt(pycurl.M_SOCKETFUNCTION, socket) + m.handles = [] + for url in urls: + c = util.DefaultCurl() + # save info in standard Python attributes + c.url = url + c.body = util.BytesIO() + c.http_code = -1 + m.handles.append(c) + # pycurl API calls + c.setopt(c.URL, c.url) + c.setopt(c.WRITEFUNCTION, c.body.write) + m.add_handle(c) + + # get data + num_handles = len(m.handles) + while num_handles: + while 1: + ret, num_handles = m.socket_all() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + # currently no more I/O is pending, could do something in the meantime + # (display a progress bar, etc.) + m.select(0.1) + + for c in m.handles: + # save info in standard Python attributes + c.http_code = c.getinfo(c.HTTP_CODE) + + # at least in and remove events per socket + assert len(socket_events) >= 6 + + # print result + for c in m.handles: + self.assertEqual('success', c.body.getvalue().decode()) + self.assertEqual(200, c.http_code) + + # multi, not curl handle + self.check(pycurl.POLL_IN, m, socket_events) + self.check(pycurl.POLL_REMOVE, m, socket_events) + + # close handles + for c in m.handles: + # pycurl API calls + m.remove_handle(c) + c.close() + m.close() + + def check(self, event, multi, socket_events): + for event_, multi_ in socket_events: + if event == event_ and multi == multi_: + return + assert False, '%d %s not found in socket events' % (event, multi) diff --git a/tests/multi_test.py b/tests/multi_test.py new file mode 100644 index 0000000..18cb5b6 --- /dev/null +++ b/tests/multi_test.py @@ -0,0 +1,380 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import pytest +import unittest +import select + +from . import appmanager +from . import util + +setup_module_1, teardown_module_1 = appmanager.setup(('app', 8380)) +setup_module_2, teardown_module_2 = appmanager.setup(('app', 8381)) +setup_module_3, teardown_module_3 = appmanager.setup(('app', 8382)) + +def setup_module(mod): + setup_module_1(mod) + setup_module_2(mod) + setup_module_3(mod) + +def teardown_module(mod): + teardown_module_3(mod) + teardown_module_2(mod) + teardown_module_1(mod) + +class MultiTest(unittest.TestCase): + def test_multi(self): + io1 = util.BytesIO() + io2 = util.BytesIO() + m = pycurl.CurlMulti() + handles = [] + c1 = util.DefaultCurl() + c2 = util.DefaultCurl() + c1.setopt(c1.URL, 'http://%s:8380/success' % localhost) + c1.setopt(c1.WRITEFUNCTION, io1.write) + c2.setopt(c2.URL, 'http://%s:8381/success' % localhost) + c2.setopt(c1.WRITEFUNCTION, io2.write) + m.add_handle(c1) + m.add_handle(c2) + handles.append(c1) + handles.append(c2) + + num_handles = len(handles) + while num_handles: + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + m.select(1.0) + + m.remove_handle(c2) + m.remove_handle(c1) + m.close() + c1.close() + c2.close() + + self.assertEqual('success', io1.getvalue().decode()) + self.assertEqual('success', io2.getvalue().decode()) + + def test_multi_select_fdset(self): + c1 = util.DefaultCurl() + c2 = util.DefaultCurl() + c3 = util.DefaultCurl() + c1.setopt(c1.URL, "http://%s:8380/success" % localhost) + c2.setopt(c2.URL, "http://%s:8381/success" % localhost) + c3.setopt(c3.URL, "http://%s:8382/success" % localhost) + c1.body = util.BytesIO() + c2.body = util.BytesIO() + c3.body = util.BytesIO() + c1.setopt(c1.WRITEFUNCTION, c1.body.write) + c2.setopt(c2.WRITEFUNCTION, c2.body.write) + c3.setopt(c3.WRITEFUNCTION, c3.body.write) + + m = pycurl.CurlMulti() + m.add_handle(c1) + m.add_handle(c2) + m.add_handle(c3) + + # Number of seconds to wait for a timeout to happen + SELECT_TIMEOUT = 0.1 + + # Stir the state machine into action + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + # Keep going until all the connections have terminated + while num_handles: + select.select(*m.fdset() + (SELECT_TIMEOUT,)) + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + # Cleanup + m.remove_handle(c3) + m.remove_handle(c2) + m.remove_handle(c1) + m.close() + c1.close() + c2.close() + c3.close() + + self.assertEqual('success', c1.body.getvalue().decode()) + self.assertEqual('success', c2.body.getvalue().decode()) + self.assertEqual('success', c3.body.getvalue().decode()) + + def test_multi_status_codes(self): + # init + m = pycurl.CurlMulti() + m.handles = [] + urls = [ + 'http://%s:8380/success' % localhost, + 'http://%s:8381/status/403' % localhost, + 'http://%s:8382/status/404' % localhost, + ] + for url in urls: + c = util.DefaultCurl() + # save info in standard Python attributes + c.url = url.rstrip() + c.body = util.BytesIO() + c.http_code = -1 + m.handles.append(c) + # pycurl API calls + c.setopt(c.URL, c.url) + c.setopt(c.WRITEFUNCTION, c.body.write) + m.add_handle(c) + + # get data + num_handles = len(m.handles) + while num_handles: + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + # currently no more I/O is pending, could do something in the meantime + # (display a progress bar, etc.) + m.select(0.1) + + # close handles + for c in m.handles: + # save info in standard Python attributes + c.http_code = c.getinfo(c.HTTP_CODE) + # pycurl API calls + m.remove_handle(c) + c.close() + m.close() + + # check result + self.assertEqual('success', m.handles[0].body.getvalue().decode()) + self.assertEqual(200, m.handles[0].http_code) + # bottle generated response body + self.assertEqual('forbidden', m.handles[1].body.getvalue().decode()) + self.assertEqual(403, m.handles[1].http_code) + # bottle generated response body + self.assertEqual('not found', m.handles[2].body.getvalue().decode()) + self.assertEqual(404, m.handles[2].http_code) + + def check_adding_closed_handle(self, close_fn): + # init + m = pycurl.CurlMulti() + m.handles = [] + urls = [ + 'http://%s:8380/success' % localhost, + 'http://%s:8381/status/403' % localhost, + 'http://%s:8382/status/404' % localhost, + ] + for url in urls: + c = util.DefaultCurl() + # save info in standard Python attributes + c.url = url + c.body = util.BytesIO() + c.http_code = -1 + c.debug = 0 + m.handles.append(c) + # pycurl API calls + c.setopt(c.URL, c.url) + c.setopt(c.WRITEFUNCTION, c.body.write) + m.add_handle(c) + + # debug - close a handle + c = m.handles[2] + c.debug = 1 + c.close() + + # get data + num_handles = len(m.handles) + while num_handles: + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + # currently no more I/O is pending, could do something in the meantime + # (display a progress bar, etc.) + m.select(0.1) + + # close handles + for c in m.handles: + # save info in standard Python attributes + try: + c.http_code = c.getinfo(c.HTTP_CODE) + except pycurl.error: + # handle already closed - see debug above + assert c.debug + c.http_code = -1 + # pycurl API calls + close_fn(m, c) + m.close() + + # check result + self.assertEqual('success', m.handles[0].body.getvalue().decode()) + self.assertEqual(200, m.handles[0].http_code) + # bottle generated response body + self.assertEqual('forbidden', m.handles[1].body.getvalue().decode()) + self.assertEqual(403, m.handles[1].http_code) + # bottle generated response body + self.assertEqual('', m.handles[2].body.getvalue().decode()) + self.assertEqual(-1, m.handles[2].http_code) + + def _remove_then_close(self, m, c): + m.remove_handle(c) + c.close() + + def _close_then_remove(self, m, c): + # in the C API this is the wrong calling order, but pycurl + # handles this automatically + c.close() + m.remove_handle(c) + + def _close_without_removing(self, m, c): + # actually, remove_handle is called automatically on close + c.close + + def test_adding_closed_handle_remove_then_close(self): + self.check_adding_closed_handle(self._remove_then_close) + + def test_adding_closed_handle_close_then_remove(self): + self.check_adding_closed_handle(self._close_then_remove) + + def test_adding_closed_handle_close_without_removing(self): + self.check_adding_closed_handle(self._close_without_removing) + + def test_multi_select(self): + c1 = util.DefaultCurl() + c2 = util.DefaultCurl() + c3 = util.DefaultCurl() + c1.setopt(c1.URL, "http://%s:8380/success" % localhost) + c2.setopt(c2.URL, "http://%s:8381/success" % localhost) + c3.setopt(c3.URL, "http://%s:8382/success" % localhost) + c1.body = util.BytesIO() + c2.body = util.BytesIO() + c3.body = util.BytesIO() + c1.setopt(c1.WRITEFUNCTION, c1.body.write) + c2.setopt(c2.WRITEFUNCTION, c2.body.write) + c3.setopt(c3.WRITEFUNCTION, c3.body.write) + + m = pycurl.CurlMulti() + m.add_handle(c1) + m.add_handle(c2) + m.add_handle(c3) + + # Number of seconds to wait for a timeout to happen + SELECT_TIMEOUT = 1.0 + + # Stir the state machine into action + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + # Keep going until all the connections have terminated + while num_handles: + # The select method uses fdset internally to determine which file descriptors + # to check. + m.select(SELECT_TIMEOUT) + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + # Cleanup + m.remove_handle(c3) + m.remove_handle(c2) + m.remove_handle(c1) + m.close() + c1.close() + c2.close() + c3.close() + + self.assertEqual('success', c1.body.getvalue().decode()) + self.assertEqual('success', c2.body.getvalue().decode()) + self.assertEqual('success', c3.body.getvalue().decode()) + + def test_multi_info_read(self): + c1 = util.DefaultCurl() + c2 = util.DefaultCurl() + c3 = util.DefaultCurl() + c1.setopt(c1.URL, "http://%s:8380/short_wait" % localhost) + c2.setopt(c2.URL, "http://%s:8381/short_wait" % localhost) + c3.setopt(c3.URL, "http://%s:8382/short_wait" % localhost) + c1.body = util.BytesIO() + c2.body = util.BytesIO() + c3.body = util.BytesIO() + c1.setopt(c1.WRITEFUNCTION, c1.body.write) + c2.setopt(c2.WRITEFUNCTION, c2.body.write) + c3.setopt(c3.WRITEFUNCTION, c3.body.write) + + m = pycurl.CurlMulti() + m.add_handle(c1) + m.add_handle(c2) + m.add_handle(c3) + + # Number of seconds to wait for a timeout to happen + SELECT_TIMEOUT = 1.0 + + # Stir the state machine into action + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + infos = [] + # Keep going until all the connections have terminated + while num_handles: + # The select method uses fdset internally to determine which file descriptors + # to check. + m.select(SELECT_TIMEOUT) + while 1: + ret, num_handles = m.perform() + info = m.info_read() + infos.append(info) + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + all_handles = [] + for info in infos: + handles = info[1] + # last info is an empty array + if handles: + all_handles.extend(handles) + + self.assertEqual(3, len(all_handles)) + assert c1 in all_handles + assert c2 in all_handles + assert c3 in all_handles + + # Cleanup + m.remove_handle(c3) + m.remove_handle(c2) + m.remove_handle(c1) + m.close() + c1.close() + c2.close() + c3.close() + + self.assertEqual('success', c1.body.getvalue().decode()) + self.assertEqual('success', c2.body.getvalue().decode()) + self.assertEqual('success', c3.body.getvalue().decode()) + + def test_multi_close(self): + m = pycurl.CurlMulti() + m.close() + + def test_multi_close_twice(self): + m = pycurl.CurlMulti() + m.close() + m.close() + + # positional arguments are rejected + def test_positional_arguments(self): + with pytest.raises(TypeError): + pycurl.CurlMulti(1) + + # keyword arguments are rejected + def test_keyword_arguments(self): + with pytest.raises(TypeError): + pycurl.CurlMulti(a=1) diff --git a/tests/multi_timer_test.py b/tests/multi_timer_test.py new file mode 100644 index 0000000..109f35e --- /dev/null +++ b/tests/multi_timer_test.py @@ -0,0 +1,91 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import unittest + +from . import appmanager +from . import util + +setup_module_1, teardown_module_1 = appmanager.setup(('app', 8380)) +setup_module_2, teardown_module_2 = appmanager.setup(('app', 8381)) +setup_module_3, teardown_module_3 = appmanager.setup(('app', 8382)) + +def setup_module(mod): + setup_module_1(mod) + setup_module_2(mod) + setup_module_3(mod) + +def teardown_module(mod): + teardown_module_3(mod) + teardown_module_2(mod) + teardown_module_1(mod) + +class MultiSocketTest(unittest.TestCase): + def test_multi_timer(self): + urls = [ + 'http://%s:8380/success' % localhost, + 'http://%s:8381/success' % localhost, + 'http://%s:8382/success' % localhost, + ] + + timers = [] + + # timer callback + def timer(msecs): + #print('Timer callback msecs:', msecs) + timers.append(msecs) + + # init + m = pycurl.CurlMulti() + m.setopt(pycurl.M_TIMERFUNCTION, timer) + m.handles = [] + for url in urls: + c = util.DefaultCurl() + # save info in standard Python attributes + c.url = url + c.body = util.BytesIO() + c.http_code = -1 + m.handles.append(c) + # pycurl API calls + c.setopt(c.URL, c.url) + c.setopt(c.WRITEFUNCTION, c.body.write) + m.add_handle(c) + + # get data + num_handles = len(m.handles) + while num_handles: + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + # currently no more I/O is pending, could do something in the meantime + # (display a progress bar, etc.) + m.select(1.0) + + for c in m.handles: + # save info in standard Python attributes + c.http_code = c.getinfo(c.HTTP_CODE) + + # print result + for c in m.handles: + self.assertEqual('success', c.body.getvalue().decode()) + self.assertEqual(200, c.http_code) + + assert len(timers) > 0 + # libcurl 7.23.0 produces a 0 timer + assert timers[0] >= 0 + # this assertion does not appear to hold on older libcurls + # or apparently on any linuxes, see + # https://github.com/p/pycurl/issues/19 + #if not util.pycurl_version_less_than(7, 24): + # self.assertEqual(-1, timers[-1]) + + # close handles + for c in m.handles: + # pycurl API calls + m.remove_handle(c) + c.close() + m.close() diff --git a/tests/open_socket_cb_test.py b/tests/open_socket_cb_test.py new file mode 100644 index 0000000..b2ddbeb --- /dev/null +++ b/tests/open_socket_cb_test.py @@ -0,0 +1,141 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import socket +import pycurl +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +socket_open_called_ipv4 = False +socket_open_called_ipv6 = False +socket_open_called_unix = False +socket_open_address = None + +def socket_open_ipv4(purpose, curl_address): + family, socktype, protocol, address = curl_address + global socket_open_called_ipv4 + global socket_open_address + socket_open_called_ipv4 = True + socket_open_address = address + + s = socket.socket(family, socktype, protocol) + s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + return s + +def socket_open_ipv6(purpose, curl_address): + family, socktype, protocol, address = curl_address + global socket_open_called_ipv6 + global socket_open_address + socket_open_called_ipv6 = True + socket_open_address = address + + s = socket.socket(family, socktype, protocol) + s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + return s + +def socket_open_unix(purpose, curl_address): + family, socktype, protocol, address = curl_address + global socket_open_called_unix + global socket_open_address + socket_open_called_unix = True + socket_open_address = address + + sockets = socket.socketpair() + sockets[0].close() + return sockets[1] + +def socket_open_bad(purpose, curl_address): + return pycurl.SOCKET_BAD + +class OpenSocketCbTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + # This is failing too much on appveyor + @util.only_unix + def test_socket_open(self): + self.curl.setopt(pycurl.OPENSOCKETFUNCTION, socket_open_ipv4) + self.curl.setopt(self.curl.URL, 'http://%s:8380/success' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + + assert socket_open_called_ipv4 + self.assertEqual(("127.0.0.1", 8380), socket_open_address) + self.assertEqual('success', sio.getvalue().decode()) + + @util.only_ipv6 + def test_socket_open_ipv6(self): + self.curl.setopt(pycurl.OPENSOCKETFUNCTION, socket_open_ipv6) + self.curl.setopt(self.curl.URL, 'http://[::1]:8380/success') + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + try: + # perform fails because we do not listen on ::1 + self.curl.perform() + except pycurl.error: + pass + + assert socket_open_called_ipv6 + + assert len(socket_open_address) == 4 + assert socket_open_address[0] == '::1' + assert socket_open_address[1] == 8380 + assert type(socket_open_address[2]) == int + assert type(socket_open_address[3]) == int + + @util.min_libcurl(7, 40, 0) + @util.only_unix + def test_socket_open_unix(self): + self.curl.setopt(pycurl.OPENSOCKETFUNCTION, socket_open_unix) + self.curl.setopt(self.curl.URL, 'http://%s:8380/success' % localhost) + self.curl.setopt(self.curl.UNIX_SOCKET_PATH, '/tmp/pycurl-test-path.sock') + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + try: + # perform fails because we return a socket that is + # not attached to anything + self.curl.perform() + except pycurl.error: + pass + + assert socket_open_called_unix + if util.py3: + assert isinstance(socket_open_address, bytes) + self.assertEqual(b'/tmp/pycurl-test-path.sock', socket_open_address) + else: + assert isinstance(socket_open_address, str) + self.assertEqual('/tmp/pycurl-test-path.sock', socket_open_address) + + def test_socket_open_none(self): + self.curl.setopt(pycurl.OPENSOCKETFUNCTION, None) + + def test_unset_socket_open(self): + self.curl.unsetopt(pycurl.OPENSOCKETFUNCTION) + + def test_socket_bad(self): + self.assertEqual(-1, pycurl.SOCKET_BAD) + + def test_socket_open_bad(self): + self.curl.setopt(pycurl.OPENSOCKETFUNCTION, socket_open_bad) + self.curl.setopt(self.curl.URL, 'http://%s:8380/success' % localhost) + try: + self.curl.perform() + except pycurl.error as e: + # libcurl 7.38.0 for some reason fails with a timeout + # (and spends 5 minutes on this test) + if pycurl.version_info()[1].split('.') == ['7', '38', '0']: + self.assertEqual(pycurl.E_OPERATION_TIMEDOUT, e.args[0]) + else: + self.assertEqual(pycurl.E_COULDNT_CONNECT, e.args[0]) + else: + self.fail('Should have raised') diff --git a/tests/option_constants_test.py b/tests/option_constants_test.py new file mode 100644 index 0000000..479c4e7 --- /dev/null +++ b/tests/option_constants_test.py @@ -0,0 +1,544 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import pytest +import unittest + +from . import util + +class OptionConstantsTest(unittest.TestCase): + # CURLOPT_USERNAME was introduced in libcurl-7.19.1 + @util.min_libcurl(7, 19, 1) + def test_username(self): + assert hasattr(pycurl, 'USERNAME') + assert hasattr(pycurl, 'PASSWORD') + assert hasattr(pycurl, 'PROXYUSERNAME') + assert hasattr(pycurl, 'PROXYPASSWORD') + + # CURLOPT_DNS_SERVERS was introduced in libcurl-7.24.0 + @util.min_libcurl(7, 24, 0) + def test_dns_servers(self): + assert hasattr(pycurl, 'DNS_SERVERS') + + # Does not work unless libcurl was built against c-ares + #c = pycurl.Curl() + #c.setopt(c.DNS_SERVERS, '1.2.3.4') + #c.close() + + # CURLOPT_POSTREDIR was introduced in libcurl-7.19.1 + @util.min_libcurl(7, 19, 1) + def test_postredir(self): + assert hasattr(pycurl, 'POSTREDIR') + assert hasattr(pycurl, 'REDIR_POST_301') + assert hasattr(pycurl, 'REDIR_POST_302') + assert hasattr(pycurl, 'REDIR_POST_ALL') + + # CURLOPT_POSTREDIR was introduced in libcurl-7.19.1 + @util.min_libcurl(7, 19, 1) + def test_postredir_setopt(self): + curl = pycurl.Curl() + curl.setopt(curl.POSTREDIR, curl.REDIR_POST_301) + curl.close() + + # CURL_REDIR_POST_303 was introduced in libcurl-7.26.0 + @util.min_libcurl(7, 26, 0) + def test_redir_post_303(self): + assert hasattr(pycurl, 'REDIR_POST_303') + + # CURLOPT_POSTREDIR was introduced in libcurl-7.19.1 + @util.min_libcurl(7, 19, 1) + def test_postredir_flags(self): + self.assertEqual(pycurl.REDIR_POST_301, pycurl.REDIR_POST_ALL & pycurl.REDIR_POST_301) + self.assertEqual(pycurl.REDIR_POST_302, pycurl.REDIR_POST_ALL & pycurl.REDIR_POST_302) + + # CURL_REDIR_POST_303 was introduced in libcurl-7.26.0 + @util.min_libcurl(7, 26, 0) + def test_postredir_post_303(self): + self.assertEqual(pycurl.REDIR_POST_303, pycurl.REDIR_POST_ALL & pycurl.REDIR_POST_303) + + # HTTPAUTH_DIGEST_IE was introduced in libcurl-7.19.3 + @util.min_libcurl(7, 19, 3) + def test_httpauth_digest_ie(self): + assert hasattr(pycurl, 'HTTPAUTH_DIGEST_IE') + + # CURLE_OPERATION_TIMEDOUT was introduced in libcurl-7.10.2 + # to replace CURLE_OPERATION_TIMEOUTED + def test_operation_timedout_constant(self): + self.assertEqual(pycurl.E_OPERATION_TIMEDOUT, pycurl.E_OPERATION_TIMEOUTED) + + # CURLOPT_NOPROXY was introduced in libcurl-7.19.4 + @util.min_libcurl(7, 19, 4) + def test_noproxy_setopt(self): + curl = pycurl.Curl() + curl.setopt(curl.NOPROXY, localhost) + curl.close() + + # CURLOPT_PROTOCOLS was introduced in libcurl-7.19.4 + @util.min_libcurl(7, 19, 4) + def test_protocols_setopt(self): + curl = pycurl.Curl() + curl.setopt(curl.PROTOCOLS, curl.PROTO_ALL & ~curl.PROTO_HTTP) + curl.close() + + # CURLOPT_REDIR_PROTOCOLS was introduced in libcurl-7.19.4 + @util.min_libcurl(7, 19, 4) + def test_redir_protocols_setopt(self): + curl = pycurl.Curl() + curl.setopt(curl.PROTOCOLS, curl.PROTO_ALL & ~curl.PROTO_HTTP) + curl.close() + + # CURLOPT_TFTP_BLKSIZE was introduced in libcurl-7.19.4 + @util.min_libcurl(7, 19, 4) + def test_tftp_blksize_setopt(self): + curl = pycurl.Curl() + curl.setopt(curl.TFTP_BLKSIZE, 1024) + curl.close() + + # CURLOPT_SOCKS5_GSSAPI_SERVICE was introduced in libcurl-7.19.4 + @util.min_libcurl(7, 19, 4) + @pytest.mark.gssapi + def test_socks5_gssapi_service_setopt(self): + curl = pycurl.Curl() + curl.setopt(curl.SOCKS5_GSSAPI_SERVICE, 'helloworld') + curl.close() + + # CURLOPT_SOCKS5_GSSAPI_NEC was introduced in libcurl-7.19.4 + @util.min_libcurl(7, 19, 4) + @pytest.mark.gssapi + def test_socks5_gssapi_nec_setopt(self): + curl = pycurl.Curl() + curl.setopt(curl.SOCKS5_GSSAPI_NEC, True) + curl.close() + + # CURLPROXY_HTTP_1_0 was introduced in libcurl-7.19.4 + @util.min_libcurl(7, 19, 4) + def test_curlproxy_http_1_0_setopt(self): + curl = pycurl.Curl() + curl.setopt(curl.PROXYTYPE, curl.PROXYTYPE_HTTP_1_0) + curl.close() + + # CURLOPT_SSH_KNOWNHOSTS was introduced in libcurl-7.19.6 + @util.min_libcurl(7, 19, 6) + @util.guard_unknown_libcurl_option + def test_ssh_knownhosts_setopt(self): + curl = pycurl.Curl() + curl.setopt(curl.SSH_KNOWNHOSTS, '/hello/world') + curl.close() + + # CURLOPT_MAIL_FROM was introduced in libcurl-7.20.0 + @util.min_libcurl(7, 20, 0) + def test_mail_from(self): + curl = pycurl.Curl() + curl.setopt(curl.MAIL_FROM, 'hello@world.com') + curl.close() + + # CURLOPT_MAIL_RCPT was introduced in libcurl-7.20.0 + @util.min_libcurl(7, 20, 0) + def test_mail_rcpt(self): + curl = pycurl.Curl() + curl.setopt(curl.MAIL_RCPT, ['hello@world.com', 'foo@bar.com']) + curl.close() + + # CURLOPT_MAIL_AUTH was introduced in libcurl-7.25.0 + @util.min_libcurl(7, 25, 0) + def test_mail_auth(self): + curl = pycurl.Curl() + curl.setopt(curl.MAIL_AUTH, 'hello@world.com') + curl.close() + + @util.min_libcurl(7, 22, 0) + @pytest.mark.gssapi + def test_gssapi_delegation_options(self): + curl = pycurl.Curl() + curl.setopt(curl.GSSAPI_DELEGATION, curl.GSSAPI_DELEGATION_FLAG) + curl.setopt(curl.GSSAPI_DELEGATION, curl.GSSAPI_DELEGATION_NONE) + curl.setopt(curl.GSSAPI_DELEGATION, curl.GSSAPI_DELEGATION_POLICY_FLAG) + curl.close() + + # SSLVERSION_DEFAULT causes CURLE_UNKNOWN_OPTION without SSL + @util.only_ssl + def test_sslversion_options(self): + curl = pycurl.Curl() + curl.setopt(curl.SSLVERSION, curl.SSLVERSION_DEFAULT) + curl.setopt(curl.SSLVERSION, curl.SSLVERSION_TLSv1) + curl.close() + + # SSLVERSION_SSLv* return CURLE_BAD_FUNCTION_ARGUMENT with curl-7.77.0 + @util.removed_in_libcurl(7, 77, 0) + @util.only_ssl + def test_legacy_sslversion_options(self): + curl = pycurl.Curl() + curl.setopt(curl.SSLVERSION, curl.SSLVERSION_SSLv2) + curl.setopt(curl.SSLVERSION, curl.SSLVERSION_SSLv3) + curl.close() + + @util.min_libcurl(7, 34, 0) + # SSLVERSION_TLSv1_0 causes CURLE_UNKNOWN_OPTION without SSL + @util.only_ssl + def test_sslversion_7_34_0(self): + curl = pycurl.Curl() + curl.setopt(curl.SSLVERSION, curl.SSLVERSION_TLSv1_0) + curl.setopt(curl.SSLVERSION, curl.SSLVERSION_TLSv1_1) + curl.setopt(curl.SSLVERSION, curl.SSLVERSION_TLSv1_2) + curl.close() + + @util.min_libcurl(7, 41, 0) + @util.only_ssl_backends('openssl', 'nss') + def test_ssl_verifystatus(self): + curl = pycurl.Curl() + curl.setopt(curl.SSL_VERIFYSTATUS, True) + curl.close() + + @util.min_libcurl(7, 43, 0) + @pytest.mark.gssapi + def test_proxy_service_name(self): + curl = pycurl.Curl() + curl.setopt(curl.PROXY_SERVICE_NAME, 'fakehttp') + curl.close() + + @util.min_libcurl(7, 43, 0) + @pytest.mark.gssapi + def test_service_name(self): + curl = pycurl.Curl() + curl.setopt(curl.SERVICE_NAME, 'fakehttp') + curl.close() + + @util.min_libcurl(7, 39, 0) + @util.only_ssl + def test_pinnedpublickey(self): + curl = pycurl.Curl() + curl.setopt(curl.PINNEDPUBLICKEY, '/etc/publickey.der') + curl.close() + + @util.min_libcurl(7, 21, 0) + def test_wildcardmatch(self): + curl = pycurl.Curl() + curl.setopt(curl.WILDCARDMATCH, '*') + curl.close() + + @util.only_unix + @util.min_libcurl(7, 40, 0) + def test_unix_socket_path(self): + curl = pycurl.Curl() + curl.setopt(curl.UNIX_SOCKET_PATH, '/tmp/socket.sock') + curl.close() + + @util.min_libcurl(7, 36, 0) + @pytest.mark.http2 + def test_ssl_enable_alpn(self): + curl = pycurl.Curl() + curl.setopt(curl.SSL_ENABLE_ALPN, 1) + curl.close() + + @util.min_libcurl(7, 36, 0) + @pytest.mark.http2 + def test_ssl_enable_npn(self): + curl = pycurl.Curl() + curl.setopt(curl.SSL_ENABLE_NPN, 1) + curl.close() + + @util.min_libcurl(7, 42, 0) + @util.only_ssl_backends('nss', 'secure-transport') + def test_ssl_falsestart(self): + curl = pycurl.Curl() + curl.setopt(curl.SSL_FALSESTART, 1) + curl.close() + + def test_ssl_verifyhost(self): + curl = pycurl.Curl() + curl.setopt(curl.SSL_VERIFYHOST, 2) + curl.close() + + def test_cainfo(self): + curl = pycurl.Curl() + curl.setopt(curl.CAINFO, '/bogus-cainfo') + curl.close() + + @util.only_ssl + def test_issuercert(self): + curl = pycurl.Curl() + curl.setopt(curl.ISSUERCERT, '/bogus-issuercert') + curl.close() + + @util.only_ssl_backends('openssl', 'gnutls', 'nss') + def test_capath(self): + curl = pycurl.Curl() + curl.setopt(curl.CAPATH, '/bogus-capath') + curl.close() + + # CURLOPT_PROXY_CAPATH was introduced in libcurl-7.52.0 + @util.min_libcurl(7, 52, 0) + @util.only_ssl_backends('openssl', 'gnutls', 'nss') + def test_proxy_capath(self): + curl = pycurl.Curl() + curl.setopt(curl.PROXY_CAPATH, '/bogus-capath') + curl.close() + + @util.min_libcurl(7, 52, 0) + @util.only_ssl + def test_proxy_sslcert(self): + curl = pycurl.Curl() + curl.setopt(curl.PROXY_SSLCERT, '/bogus-sslcert') + curl.close() + + @util.min_libcurl(7, 52, 0) + @util.only_ssl + def test_proxy_sslcerttype(self): + curl = pycurl.Curl() + curl.setopt(curl.PROXY_SSLCERTTYPE, 'PEM') + curl.close() + + @util.min_libcurl(7, 52, 0) + @util.only_ssl + def test_proxy_sslkey(self): + curl = pycurl.Curl() + curl.setopt(curl.PROXY_SSLKEY, '/bogus-sslkey') + curl.close() + + @util.min_libcurl(7, 52, 0) + @util.only_ssl + def test_proxy_sslkeytype(self): + curl = pycurl.Curl() + curl.setopt(curl.PROXY_SSLKEYTYPE, 'PEM') + curl.close() + + @util.min_libcurl(7, 52, 0) + @util.only_ssl + def test_proxy_ssl_verifypeer(self): + curl = pycurl.Curl() + curl.setopt(curl.PROXY_SSL_VERIFYPEER, 1) + curl.close() + + @util.min_libcurl(7, 52, 0) + @util.only_ssl + def test_proxy_ssl_verifyhost(self): + curl = pycurl.Curl() + curl.setopt(curl.PROXY_SSL_VERIFYHOST, 2) + curl.close() + + @util.only_ssl + def test_crlfile(self): + curl = pycurl.Curl() + curl.setopt(curl.CRLFILE, '/bogus-crlfile') + curl.close() + + @util.only_ssl + def test_random_file(self): + curl = pycurl.Curl() + curl.setopt(curl.RANDOM_FILE, '/bogus-random') + curl.close() + + @util.only_ssl_backends('openssl', 'gnutls', 'secure-transport') + def test_egdsocket(self): + curl = pycurl.Curl() + curl.setopt(curl.EGDSOCKET, '/bogus-egdsocket') + curl.close() + + @util.only_ssl + def test_ssl_cipher_list(self): + curl = pycurl.Curl() + curl.setopt(curl.SSL_CIPHER_LIST, 'RC4-SHA:SHA1+DES') + curl.close() + + @util.only_ssl + def test_ssl_sessionid_cache(self): + curl = pycurl.Curl() + curl.setopt(curl.SSL_SESSIONID_CACHE, True) + curl.close() + + def test_krblevel(self): + curl = pycurl.Curl() + curl.setopt(curl.KRBLEVEL, 'clear') + curl.close() + + def test_krb4level(self): + curl = pycurl.Curl() + curl.setopt(curl.KRB4LEVEL, 'clear') + curl.close() + + @util.min_libcurl(7, 25, 0) + @util.only_ssl + def test_ssl_options(self): + curl = pycurl.Curl() + curl.setopt(curl.SSL_OPTIONS, curl.SSLOPT_ALLOW_BEAST) + curl.close() + + @util.min_libcurl(7, 44, 0) + @util.only_ssl + def test_ssl_option_no_revoke(self): + curl = pycurl.Curl() + curl.setopt(curl.SSL_OPTIONS, curl.SSLOPT_NO_REVOKE) + curl.close() + + @util.min_libcurl(7, 64, 0) + def test_http09_allowed_option(self): + curl = pycurl.Curl() + curl.setopt(curl.HTTP09_ALLOWED, 1) + curl.close() + + @util.min_libcurl(7, 61, 0) + @util.only_ssl_backends('openssl') + def test_tls13_ciphers(self): + curl = pycurl.Curl() + curl.setopt(curl.TLS13_CIPHERS, 'TLS_CHACHA20_POLY1305_SHA256') + curl.close() + + @util.min_libcurl(7, 61, 0) + @util.only_ssl_backends('openssl') + def test_proxy_tls13_ciphers(self): + curl = pycurl.Curl() + curl.setopt(curl.PROXY_TLS13_CIPHERS, 'TLS_CHACHA20_POLY1305_SHA256') + curl.close() + +class OptionConstantsSettingTest(unittest.TestCase): + def setUp(self): + self.curl = pycurl.Curl() + + def tearDown(self): + self.curl.close() + + def test_append(self): + self.curl.setopt(self.curl.APPEND, True) + + def test_cookiesession(self): + self.curl.setopt(self.curl.COOKIESESSION, True) + + def test_dirlistonly(self): + self.curl.setopt(self.curl.DIRLISTONLY, True) + + @util.only_ssl + def test_keypasswd(self): + self.curl.setopt(self.curl.KEYPASSWD, 'secret') + + @util.only_telnet + def test_telnetoptions(self): + self.curl.setopt(self.curl.TELNETOPTIONS, ('TTYPE=1', 'XDISPLOC=2')) + + @util.only_ssl + def test_use_ssl(self): + self.curl.setopt(self.curl.USE_SSL, self.curl.USESSL_NONE) + self.curl.setopt(self.curl.USE_SSL, self.curl.USESSL_TRY) + self.curl.setopt(self.curl.USE_SSL, self.curl.USESSL_CONTROL) + self.curl.setopt(self.curl.USE_SSL, self.curl.USESSL_ALL) + + def test_encoding(self): + # old name for ACCEPT_ENCODING + self.curl.setopt(self.curl.ENCODING, "") + self.curl.setopt(self.curl.ENCODING, "application/json") + + @util.min_libcurl(7, 21, 6) + def test_accept_encoding(self): + self.curl.setopt(self.curl.ACCEPT_ENCODING, "") + self.curl.setopt(self.curl.ACCEPT_ENCODING, "application/json") + + @util.min_libcurl(7, 21, 6) + def test_transfer_encoding(self): + self.curl.setopt(self.curl.TRANSFER_ENCODING, True) + + @util.min_libcurl(7, 24, 0) + def test_accepttimeout_ms(self): + self.curl.setopt(self.curl.ACCEPTTIMEOUT_MS, 1000) + + @util.min_libcurl(7, 25, 0) + def test_tcp_keepalive(self): + self.curl.setopt(self.curl.TCP_KEEPALIVE, True) + + @util.min_libcurl(7, 25, 0) + def test_tcp_keepidle(self): + self.curl.setopt(self.curl.TCP_KEEPIDLE, 100) + + @util.min_libcurl(7, 25, 0) + def test_tcp_keepintvl(self): + self.curl.setopt(self.curl.TCP_KEEPINTVL, 100) + + @util.min_libcurl(7, 36, 0) + def test_expect_100_timeout_ms(self): + self.curl.setopt(self.curl.EXPECT_100_TIMEOUT_MS, 100) + + @util.min_libcurl(7, 37, 0) + def test_headeropt(self): + self.curl.setopt(self.curl.HEADEROPT, self.curl.HEADER_UNIFIED) + self.curl.setopt(self.curl.HEADEROPT, self.curl.HEADER_SEPARATE) + + @util.min_libcurl(7, 42, 0) + def test_path_as_is(self): + self.curl.setopt(self.curl.PATH_AS_IS, True) + + @util.min_libcurl(7, 43, 0) + def test_pipewait(self): + self.curl.setopt(self.curl.PIPEWAIT, True) + + def test_http_version(self): + self.curl.setopt(self.curl.HTTP_VERSION, self.curl.CURL_HTTP_VERSION_NONE) + self.curl.setopt(self.curl.HTTP_VERSION, self.curl.CURL_HTTP_VERSION_1_0) + self.curl.setopt(self.curl.HTTP_VERSION, self.curl.CURL_HTTP_VERSION_1_1) + + @util.min_libcurl(7, 33, 0) + @pytest.mark.http2 + def test_http_version_2_0(self): + self.curl.setopt(self.curl.HTTP_VERSION, self.curl.CURL_HTTP_VERSION_2_0) + + @util.min_libcurl(7, 43, 0) + @pytest.mark.http2 + def test_http_version_2(self): + self.curl.setopt(self.curl.HTTP_VERSION, self.curl.CURL_HTTP_VERSION_2) + + @util.min_libcurl(7, 47, 0) + @pytest.mark.http2 + def test_http_version_2tls(self): + self.curl.setopt(self.curl.HTTP_VERSION, self.curl.CURL_HTTP_VERSION_2TLS) + + @util.min_libcurl(7, 49, 0) + @pytest.mark.http2 + def test_http_version_2prior_knowledge(self): + self.curl.setopt(self.curl.HTTP_VERSION, self.curl.CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE) + + @util.min_libcurl(7, 66, 0) + def test_http_version_3(self): + self.curl.setopt(self.curl.HTTP_VERSION, self.curl.CURL_HTTP_VERSION_3) + + @util.min_libcurl(7, 21, 5) + def test_sockopt_constants(self): + assert self.curl.SOCKOPT_OK is not None + assert self.curl.SOCKOPT_ERROR is not None + assert self.curl.SOCKOPT_ALREADY_CONNECTED is not None + + @util.min_libcurl(7, 40, 0) + def test_proto_smb(self): + assert self.curl.PROTO_SMB is not None + assert self.curl.PROTO_SMBS is not None + + # Apparently TLSAUTH_TYPE=SRP is an unknown option on appveyor + @util.only_unix + @util.min_libcurl(7, 21, 4) + @util.only_ssl_backends('openssl', 'gnutls') + def test_tlsauth(self): + self.curl.setopt(self.curl.TLSAUTH_TYPE, "SRP") + self.curl.setopt(self.curl.TLSAUTH_USERNAME, "test") + self.curl.setopt(self.curl.TLSAUTH_PASSWORD, "test") + + @util.min_libcurl(7, 45, 0) + def test_default_protocol(self): + self.curl.setopt(self.curl.DEFAULT_PROTOCOL, "http") + + @util.min_libcurl(7, 20, 0) + def test_ftp_use_pret(self): + self.curl.setopt(self.curl.FTP_USE_PRET, True) + + @util.min_libcurl(7, 34, 0) + def test_login_options(self): + self.curl.setopt(self.curl.LOGIN_OPTIONS, 'AUTH=NTLM') + + @util.min_libcurl(7, 31, 0) + def test_sasl_ir(self): + self.curl.setopt(self.curl.SASL_IR, True) + + @util.min_libcurl(7, 33, 0) + def test_xauth_bearer(self): + self.curl.setopt(self.curl.XOAUTH2_BEARER, 'test') + + def test_cookielist_constants(self): + self.assertEqual(pycurl.OPT_COOKIELIST, pycurl.COOKIELIST) diff --git a/tests/pause_test.py b/tests/pause_test.py new file mode 100644 index 0000000..af24b61 --- /dev/null +++ b/tests/pause_test.py @@ -0,0 +1,97 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import flaky +import pycurl +import unittest, signal +import time as _time + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +@flaky.flaky(max_runs=3) +class PauseTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_pause_via_call(self): + self.check_pause(True) + + def test_pause_via_return(self): + self.check_pause(False) + + @util.only_unix + def check_pause(self, call): + # the app sleeps for 0.5 seconds + self.curl.setopt(pycurl.URL, 'http://%s:8380/pause' % localhost) + sio = util.BytesIO() + state = dict(paused=False, resumed=False) + if call: + def writefunc(data): + rv = sio.write(data) + if not state['paused']: + self.curl.pause(pycurl.PAUSE_ALL) + state['paused'] = True + return rv + else: + def writefunc(data): + if not state['paused']: + # cannot write to sio here, because + # curl takes pause return value to mean that + # nothing was written + state['paused'] = True + return pycurl.READFUNC_PAUSE + else: + return sio.write(data) + def resume(*args): + state['resumed'] = True + self.curl.pause(pycurl.PAUSE_CONT) + signal.signal(signal.SIGALRM, resume) + # alarm for 1 second which is 0.5 seconds more than the server side + # should sleep for + signal.alarm(1) + start = _time.time() + self.curl.setopt(pycurl.WRITEFUNCTION, writefunc) + + m = pycurl.CurlMulti() + m.add_handle(self.curl) + + # Number of seconds to wait for a timeout to happen + SELECT_TIMEOUT = 1.0 + + # Stir the state machine into action + while 1: + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + # Keep going until all the connections have terminated + while num_handles: + # The select method uses fdset internally to determine which file descriptors + # to check. + m.select(SELECT_TIMEOUT) + while 1: + if _time.time() - start > 2: + # test is taking too long, fail + assert False, 'Test is taking too long' + ret, num_handles = m.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + # Cleanup + m.remove_handle(self.curl) + m.close() + + self.assertEqual('part1part2', sio.getvalue().decode()) + end = _time.time() + # check that client side waited + self.assertTrue(end-start > 1) + + assert state['resumed'] diff --git a/tests/perform_test.py b/tests/perform_test.py new file mode 100644 index 0000000..ccda941 --- /dev/null +++ b/tests/perform_test.py @@ -0,0 +1,66 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +try: + import unittest2 as unittest +except ImportError: + import unittest +import pycurl + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class PerformTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_perform_rb(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + body = self.curl.perform_rb() + self.assertEqual(util.b('success'), body) + + def test_perform_rs(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + body = self.curl.perform_rs() + self.assertEqual(util.u('success'), body) + + def test_perform_rb_utf8(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/utf8_body' % localhost) + body = self.curl.perform_rb() + if util.py3: + self.assertEqual('Дружба народов'.encode('utf8'), body) + else: + self.assertEqual('Дружба народов', body) + + def test_perform_rs_utf8(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/utf8_body' % localhost) + body = self.curl.perform_rs() + self.assertEqual('Дружба народов', body) + + def test_perform_rb_invalid_utf8(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/invalid_utf8_body' % localhost) + body = self.curl.perform_rb() + self.assertEqual(util.b('\xb3\xd2\xda\xcd\xd7'), body) + + @util.only_python2 + def test_perform_rs_invalid_utf8_python2(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/invalid_utf8_body' % localhost) + body = self.curl.perform_rs() + self.assertEqual('\xb3\xd2\xda\xcd\xd7', body) + + @util.only_python3 + def test_perform_rs_invalid_utf8_python3(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/invalid_utf8_body' % localhost) + try: + self.curl.perform_rs() + except UnicodeDecodeError: + pass + else: + self.fail('Should have raised') diff --git a/tests/post_test.py b/tests/post_test.py new file mode 100644 index 0000000..81ef701 --- /dev/null +++ b/tests/post_test.py @@ -0,0 +1,200 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import flaky +import os.path +import pycurl +import unittest +try: + import json +except ImportError: + import simplejson as json +try: + import urllib.parse as urllib_parse +except ImportError: + import urllib as urllib_parse + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +@flaky.flaky(max_runs=3) +class PostTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_post_single_field(self): + pf = {'field1': 'value1'} + self.urlencode_and_check(pf) + + def test_post_multiple_fields(self): + pf = {'field1':'value1', 'field2':'value2 with blanks', 'field3':'value3'} + self.urlencode_and_check(pf) + + def test_post_fields_with_ampersand(self): + pf = {'field1':'value1', 'field2':'value2 with blanks and & chars', + 'field3':'value3'} + self.urlencode_and_check(pf) + + def urlencode_and_check(self, pf): + self.curl.setopt(pycurl.URL, 'http://%s:8380/postfields' % localhost) + postfields = urllib_parse.urlencode(pf) + self.curl.setopt(pycurl.POSTFIELDS, postfields) + + # But directly passing urlencode result into setopt call: + #self.curl.setopt(pycurl.POSTFIELDS, urllib_parse.urlencode(pf)) + # produces: + # {'\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00': ''} + # Traceback (most recent call last): + # File "/usr/local/bin/bottle.py", line 744, in _handle + # return route.call(**args) + # File "/usr/local/bin/bottle.py", line 1479, in wrapper + # rv = callback(*a, **ka) + # File "/home/pie/apps/pycurl/tests/app.py", line 21, in postfields + # return json.dumps(dict(bottle.request.forms)) + # File "/usr/local/lib/python2.7/json/__init__.py", line 231, in dumps + # return _default_encoder.encode(obj) + # File "/usr/local/lib/python2.7/json/encoder.py", line 201, in encode + # chunks = self.iterencode(o, _one_shot=True) + # File "/usr/local/lib/python2.7/json/encoder.py", line 264, in iterencode + # return _iterencode(o, 0) + # UnicodeDecodeError: 'utf8' codec can't decode byte 0x80 in position 4: invalid start byte + + #self.curl.setopt(pycurl.VERBOSE, 1) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + self.assertEqual(200, self.curl.getinfo(pycurl.HTTP_CODE)) + body = sio.getvalue().decode() + returned_fields = json.loads(body) + self.assertEqual(pf, returned_fields) + + def test_post_with_null_byte(self): + send = [ + ('field3', (pycurl.FORM_CONTENTS, 'this is wei\000rd, but null-bytes are okay')) + ] + expect = { + 'field3': 'this is wei\000rd, but null-bytes are okay', + } + self.check_post(send, expect, 'http://%s:8380/postfields' % localhost) + + def test_post_file(self): + path = os.path.join(os.path.dirname(__file__), '..', 'README.rst') + f = open(path, newline='') + try: + contents = f.read() + finally: + f.close() + send = [ + #('field2', (pycurl.FORM_FILE, 'test_post.py', pycurl.FORM_FILE, 'test_post2.py')), + ('field2', (pycurl.FORM_FILE, path)), + ] + expect = [{ + 'name': 'field2', + 'filename': 'README.rst', + 'data': contents, + }] + self.check_post(send, expect, 'http://%s:8380/files' % localhost) + + def test_post_byte_buffer(self): + contents = util.b('hello, world!') + send = [ + ('field2', (pycurl.FORM_BUFFER, 'uploaded.file', pycurl.FORM_BUFFERPTR, contents)), + ] + expect = [{ + 'name': 'field2', + 'filename': 'uploaded.file', + 'data': 'hello, world!', + }] + self.check_post(send, expect, 'http://%s:8380/files' % localhost) + + def test_post_unicode_buffer(self): + contents = util.u('hello, world!') + send = [ + ('field2', (pycurl.FORM_BUFFER, 'uploaded.file', pycurl.FORM_BUFFERPTR, contents)), + ] + expect = [{ + 'name': 'field2', + 'filename': 'uploaded.file', + 'data': 'hello, world!', + }] + self.check_post(send, expect, 'http://%s:8380/files' % localhost) + + def test_post_tuple_of_tuples_of_tuples(self): + contents = util.u('hello, world!') + send = ( + ('field2', (pycurl.FORM_BUFFER, 'uploaded.file', pycurl.FORM_BUFFERPTR, contents)), + ) + expect = [{ + 'name': 'field2', + 'filename': 'uploaded.file', + 'data': 'hello, world!', + }] + self.check_post(send, expect, 'http://%s:8380/files' % localhost) + + def test_post_tuple_of_lists_of_tuples(self): + contents = util.u('hello, world!') + send = ( + ['field2', (pycurl.FORM_BUFFER, 'uploaded.file', pycurl.FORM_BUFFERPTR, contents)], + ) + expect = [{ + 'name': 'field2', + 'filename': 'uploaded.file', + 'data': 'hello, world!', + }] + self.check_post(send, expect, 'http://%s:8380/files' % localhost) + + def test_post_tuple_of_tuple_of_lists(self): + contents = util.u('hello, world!') + send = ( + ('field2', [pycurl.FORM_BUFFER, 'uploaded.file', pycurl.FORM_BUFFERPTR, contents]), + ) + expect = [{ + 'name': 'field2', + 'filename': 'uploaded.file', + 'data': 'hello, world!', + }] + self.check_post(send, expect, 'http://%s:8380/files' % localhost) + + def test_post_list_of_tuple_of_tuples(self): + contents = util.u('hello, world!') + send = [ + ('field2', (pycurl.FORM_BUFFER, 'uploaded.file', pycurl.FORM_BUFFERPTR, contents)), + ] + expect = [{ + 'name': 'field2', + 'filename': 'uploaded.file', + 'data': 'hello, world!', + }] + self.check_post(send, expect, 'http://%s:8380/files' % localhost) + + def test_post_list_of_list_of_lists(self): + contents = util.u('hello, world!') + send = [ + ['field2', [pycurl.FORM_BUFFER, 'uploaded.file', pycurl.FORM_BUFFERPTR, contents]], + ] + expect = [{ + 'name': 'field2', + 'filename': 'uploaded.file', + 'data': 'hello, world!', + }] + self.check_post(send, expect, 'http://%s:8380/files' % localhost) + + # XXX this test takes about a second to run, check keep-alives? + def check_post(self, send, expect, endpoint): + self.curl.setopt(pycurl.URL, endpoint) + self.curl.setopt(pycurl.HTTPPOST, send) + #self.curl.setopt(pycurl.VERBOSE, 1) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + self.assertEqual(200, self.curl.getinfo(pycurl.HTTP_CODE)) + body = sio.getvalue().decode() + returned_fields = json.loads(body) + self.assertEqual(expect, returned_fields) diff --git a/tests/procmgr.py b/tests/procmgr.py new file mode 100644 index 0000000..a7460d2 --- /dev/null +++ b/tests/procmgr.py @@ -0,0 +1,103 @@ +import threading +import subprocess +import os +import sys +import signal +import unittest + +from . import util, localhost + +class ProcessManager(object): + def __init__(self, cmd): + self.cmd = cmd + self.running = False + + def start(self): + self.process = subprocess.Popen(self.cmd) + self.running = True + + self.thread = threading.Thread(target=self.run) + self.thread.daemon = True + self.thread.start() + + def run(self): + self.process.communicate() + + def stop(self): + try: + os.kill(self.process.pid, signal.SIGTERM) + except OSError: + pass + self.running = False + +managers = {} + +def start(cmd): + if str(cmd) in managers and managers[str(cmd)].running: + # already started + return + + manager = ProcessManager(cmd) + managers[str(cmd)] = manager + manager.start() + +def start_setup(cmd): + def do_start(): + start(cmd) + return do_start + +# Example on FreeBSD: +# PYCURL_VSFTPD_PATH=/usr/local/libexec/vsftpd pytest + +if 'PYCURL_VSFTPD_PATH' in os.environ: + vsftpd_path = os.environ['PYCURL_VSFTPD_PATH'] +else: + vsftpd_path = None + +try: + # python 2 + exception_base = StandardError +except NameError: + # python 3 + exception_base = Exception +class VsftpdNotConfigured(exception_base): + pass + +def vsftpd_setup(): + config_file_path = os.path.join(os.path.dirname(__file__), 'vsftpd.conf') + root_path = os.path.join(os.path.dirname(__file__), '..') + cmd = [ + vsftpd_path, + config_file_path, + '-oanon_root=%s' % root_path, + ] + if os.environ.get('CI') and os.environ.get('TRAVIS'): + cmd.append('-oftp_username=travis') + setup_module = start_setup(cmd) + def do_setup_module(): + if vsftpd_path is None: + raise unittest.SkipTest('PYCURL_VSFTPD_PATH environment variable not set') + try: + setup_module() + except OSError: + import errno + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + msg = "Tried to execute `%s`\nTry specifying path to vsftpd via PYCURL_VSFTPD_PATH environment variable\n" % vsftpd_path + raise OSError(e.errno, e.strerror + "\n" + msg) + else: + raise + ok = util.wait_for_network_service((localhost, 8321), 0.1, 10) + if not ok: + import warnings + warnings.warn('vsftpd did not start after 1 second') + + def teardown_module(): + try: + manager = managers[str(cmd)] + except KeyError: + pass + else: + manager.stop() + + return do_setup_module, teardown_module diff --git a/tests/protocol_constants_test.py b/tests/protocol_constants_test.py new file mode 100644 index 0000000..c97889d --- /dev/null +++ b/tests/protocol_constants_test.py @@ -0,0 +1,48 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import unittest + +from . import util + +class ProtocolConstantsTest(unittest.TestCase): + @util.min_libcurl(7, 19, 4) + def test_7_19_4_protocols(self): + assert hasattr(pycurl, 'PROTO_ALL') + assert hasattr(pycurl, 'PROTO_DICT') + assert hasattr(pycurl, 'PROTO_FILE') + assert hasattr(pycurl, 'PROTO_FTP') + assert hasattr(pycurl, 'PROTO_FTPS') + assert hasattr(pycurl, 'PROTO_HTTP') + assert hasattr(pycurl, 'PROTO_HTTPS') + assert hasattr(pycurl, 'PROTO_LDAP') + assert hasattr(pycurl, 'PROTO_LDAPS') + assert hasattr(pycurl, 'PROTO_SCP') + assert hasattr(pycurl, 'PROTO_SFTP') + assert hasattr(pycurl, 'PROTO_TELNET') + assert hasattr(pycurl, 'PROTO_TFTP') + + @util.min_libcurl(7, 20, 0) + def test_7_20_0_protocols(self): + assert hasattr(pycurl, 'PROTO_IMAP') + assert hasattr(pycurl, 'PROTO_IMAPS') + assert hasattr(pycurl, 'PROTO_POP3') + assert hasattr(pycurl, 'PROTO_POP3S') + assert hasattr(pycurl, 'PROTO_RTSP') + assert hasattr(pycurl, 'PROTO_SMTP') + assert hasattr(pycurl, 'PROTO_SMTPS') + + @util.min_libcurl(7, 21, 0) + def test_7_21_0_protocols(self): + assert hasattr(pycurl, 'PROTO_RTMP') + assert hasattr(pycurl, 'PROTO_RTMPE') + assert hasattr(pycurl, 'PROTO_RTMPS') + assert hasattr(pycurl, 'PROTO_RTMPT') + assert hasattr(pycurl, 'PROTO_RTMPTE') + assert hasattr(pycurl, 'PROTO_RTMPTS') + + @util.min_libcurl(7, 21, 2) + def test_7_21_2_protocols(self): + assert hasattr(pycurl, 'PROTO_GOPHER') diff --git a/tests/read_cb_test.py b/tests/read_cb_test.py new file mode 100644 index 0000000..68d4888 --- /dev/null +++ b/tests/read_cb_test.py @@ -0,0 +1,127 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import unittest +import sys +try: + import json +except ImportError: + import simplejson as json +try: + import urllib.parse as urllib_parse +except ImportError: + import urllib as urllib_parse + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +POSTFIELDS = { + 'field1':'value1', + 'field2':'value2 with blanks', + 'field3':'value3', +} +POSTSTRING = urllib_parse.urlencode(POSTFIELDS) + +class DataProvider(object): + def __init__(self, data): + self.data = data + self.finished = False + + def read_cb(self, size): + assert len(self.data) <= size + if not self.finished: + self.finished = True + return self.data + else: + # Nothing more to read + return "" + +class ReadCbTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_post_with_read_callback(self): + d = DataProvider(POSTSTRING) + self.curl.setopt(self.curl.URL, 'http://%s:8380/postfields' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.POSTFIELDSIZE, len(POSTSTRING)) + self.curl.setopt(self.curl.READFUNCTION, d.read_cb) + #self.curl.setopt(self.curl.VERBOSE, 1) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + + actual = json.loads(sio.getvalue().decode()) + self.assertEqual(POSTFIELDS, actual) + + def test_post_with_read_callback_returning_bytes(self): + self.check_bytes('world') + + def test_post_with_read_callback_returning_bytes_with_nulls(self): + self.check_bytes("wor\0ld") + + def test_post_with_read_callback_returning_bytes_with_multibyte(self): + self.check_bytes(util.u("Пушкин")) + + def check_bytes(self, poststring): + data = poststring.encode('utf8') + assert type(data) == util.binary_type + d = DataProvider(data) + + self.curl.setopt(self.curl.URL, 'http://%s:8380/raw_utf8' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.HTTPHEADER, ['Content-Type: application/octet-stream']) + # length of bytes + self.curl.setopt(self.curl.POSTFIELDSIZE, len(data)) + self.curl.setopt(self.curl.READFUNCTION, d.read_cb) + #self.curl.setopt(self.curl.VERBOSE, 1) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + + # json should be ascii + actual = json.loads(sio.getvalue().decode('ascii')) + self.assertEqual(poststring, actual) + + def test_post_with_read_callback_returning_unicode(self): + self.check_unicode(util.u('world')) + + def test_post_with_read_callback_returning_unicode_with_nulls(self): + self.check_unicode(util.u("wor\0ld")) + + def test_post_with_read_callback_returning_unicode_with_multibyte(self): + try: + self.check_unicode(util.u("Пушкин")) + # prints: + # UnicodeEncodeError: 'ascii' codec can't encode characters in position 6-11: ordinal not in range(128) + except pycurl.error: + err, msg = sys.exc_info()[1].args + # we expect pycurl.E_WRITE_ERROR as the response + self.assertEqual(pycurl.E_ABORTED_BY_CALLBACK, err) + self.assertEqual('operation aborted by callback', msg) + + def check_unicode(self, poststring): + assert type(poststring) == util.text_type + d = DataProvider(poststring) + + self.curl.setopt(self.curl.URL, 'http://%s:8380/raw_utf8' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.HTTPHEADER, ['Content-Type: application/octet-stream']) + self.curl.setopt(self.curl.POSTFIELDSIZE, len(poststring)) + self.curl.setopt(self.curl.READFUNCTION, d.read_cb) + #self.curl.setopt(self.curl.VERBOSE, 1) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + + # json should be ascii + actual = json.loads(sio.getvalue().decode('ascii')) + self.assertEqual(poststring, actual) diff --git a/tests/readdata_test.py b/tests/readdata_test.py new file mode 100644 index 0000000..5ce6904 --- /dev/null +++ b/tests/readdata_test.py @@ -0,0 +1,253 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +try: + import unittest2 as unittest +except ImportError: + import unittest +import sys +import os.path +try: + import json +except ImportError: + import simplejson as json +try: + import urllib.parse as urllib_parse +except ImportError: + import urllib as urllib_parse + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +POSTFIELDS = { + 'field1':'value1', + 'field2':'value2 with blanks', + 'field3':'value3', +} +POSTSTRING = urllib_parse.urlencode(POSTFIELDS) + +class DataProvider(object): + def __init__(self, data): + self.data = data + self.finished = False + + def read(self, size): + assert len(self.data) <= size + if not self.finished: + self.finished = True + return self.data + else: + # Nothing more to read + return "" + +FORM_SUBMISSION_PATH = os.path.join(os.path.dirname(__file__), 'fixtures', 'form_submission.txt') + +class ReaddataTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_readdata_object(self): + d = DataProvider(POSTSTRING) + self.curl.setopt(self.curl.URL, 'http://%s:8380/postfields' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.POSTFIELDSIZE, len(POSTSTRING)) + self.curl.setopt(self.curl.READDATA, d) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + + actual = json.loads(sio.getvalue().decode()) + self.assertEqual(POSTFIELDS, actual) + + def test_post_with_read_returning_bytes(self): + self.check_bytes('world') + + def test_post_with_read_returning_bytes_with_nulls(self): + self.check_bytes("wor\0ld") + + def test_post_with_read_returning_bytes_with_multibyte(self): + self.check_bytes(util.u("Пушкин")) + + def check_bytes(self, poststring): + data = poststring.encode('utf8') + assert type(data) == util.binary_type + d = DataProvider(data) + + self.curl.setopt(self.curl.URL, 'http://%s:8380/raw_utf8' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.HTTPHEADER, ['Content-Type: application/octet-stream']) + # length of bytes + self.curl.setopt(self.curl.POSTFIELDSIZE, len(data)) + self.curl.setopt(self.curl.READDATA, d) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + + # json should be ascii + actual = json.loads(sio.getvalue().decode('ascii')) + self.assertEqual(poststring, actual) + + def test_post_with_read_callback_returning_unicode(self): + self.check_unicode(util.u('world')) + + def test_post_with_read_callback_returning_unicode_with_nulls(self): + self.check_unicode(util.u("wor\0ld")) + + def test_post_with_read_callback_returning_unicode_with_multibyte(self): + try: + self.check_unicode(util.u("Пушкин")) + # prints: + # UnicodeEncodeError: 'ascii' codec can't encode characters in position 6-11: ordinal not in range(128) + except pycurl.error: + err, msg = sys.exc_info()[1].args + # we expect pycurl.E_WRITE_ERROR as the response + self.assertEqual(pycurl.E_ABORTED_BY_CALLBACK, err) + self.assertEqual('operation aborted by callback', msg) + + def check_unicode(self, poststring): + assert type(poststring) == util.text_type + d = DataProvider(poststring) + + self.curl.setopt(self.curl.URL, 'http://%s:8380/raw_utf8' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.HTTPHEADER, ['Content-Type: application/octet-stream']) + self.curl.setopt(self.curl.POSTFIELDSIZE, len(poststring)) + self.curl.setopt(self.curl.READDATA, d) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + + # json should be ascii + actual = json.loads(sio.getvalue().decode('ascii')) + self.assertEqual(poststring, actual) + + def test_readdata_file_binary(self): + # file opened in binary mode + f = open(FORM_SUBMISSION_PATH, 'rb') + try: + self.curl.setopt(self.curl.URL, 'http://%s:8380/postfields' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.POSTFIELDSIZE, os.stat(FORM_SUBMISSION_PATH).st_size) + self.curl.setopt(self.curl.READDATA, f) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEDATA, sio) + self.curl.perform() + + actual = json.loads(sio.getvalue().decode()) + self.assertEqual({'foo': 'bar'}, actual) + finally: + f.close() + + def test_readdata_file_text(self): + # file opened in text mode + f = open(FORM_SUBMISSION_PATH, 'rt') + try: + self.curl.setopt(self.curl.URL, 'http://%s:8380/postfields' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.POSTFIELDSIZE, os.stat(FORM_SUBMISSION_PATH).st_size) + self.curl.setopt(self.curl.READDATA, f) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEDATA, sio) + self.curl.perform() + + actual = json.loads(sio.getvalue().decode()) + self.assertEqual({'foo': 'bar'}, actual) + finally: + f.close() + + def test_readdata_file_like(self): + data = 'hello=world' + data_provider = DataProvider(data) + self.curl.setopt(self.curl.URL, 'http://%s:8380/postfields' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.POSTFIELDSIZE, len(data)) + self.curl.setopt(self.curl.READDATA, data_provider) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEDATA, sio) + self.curl.perform() + + actual = json.loads(sio.getvalue().decode()) + self.assertEqual({'hello': 'world'}, actual) + + def test_readdata_and_readfunction_file_like(self): + data = 'hello=world' + data_provider = DataProvider(data) + # data must be the same length + function_provider = DataProvider('aaaaa=bbbbb') + self.curl.setopt(self.curl.URL, 'http://%s:8380/postfields' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.POSTFIELDSIZE, len(data)) + self.curl.setopt(self.curl.READDATA, data_provider) + self.curl.setopt(self.curl.READFUNCTION, function_provider.read) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEDATA, sio) + self.curl.perform() + + actual = json.loads(sio.getvalue().decode()) + self.assertEqual({'aaaaa': 'bbbbb'}, actual) + + def test_readfunction_and_readdata_file_like(self): + data = 'hello=world' + data_provider = DataProvider(data) + # data must be the same length + function_provider = DataProvider('aaaaa=bbbbb') + self.curl.setopt(self.curl.URL, 'http://%s:8380/postfields' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.POSTFIELDSIZE, len(data)) + self.curl.setopt(self.curl.READFUNCTION, function_provider.read) + self.curl.setopt(self.curl.READDATA, data_provider) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEDATA, sio) + self.curl.perform() + + actual = json.loads(sio.getvalue().decode()) + self.assertEqual({'hello': 'world'}, actual) + + def test_readdata_and_readfunction_real_file(self): + # data must be the same length + with open(FORM_SUBMISSION_PATH) as f: + function_provider = DataProvider('aaa=bbb') + self.curl.setopt(self.curl.URL, 'http://%s:8380/postfields' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.POSTFIELDSIZE, os.stat(FORM_SUBMISSION_PATH).st_size) + self.curl.setopt(self.curl.READDATA, f) + self.curl.setopt(self.curl.READFUNCTION, function_provider.read) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEDATA, sio) + self.curl.perform() + + actual = json.loads(sio.getvalue().decode()) + self.assertEqual({'aaa': 'bbb'}, actual) + + def test_readfunction_and_readdata_real_file(self): + # data must be the same length + with open(FORM_SUBMISSION_PATH) as f: + function_provider = DataProvider('aaa=bbb') + self.curl.setopt(self.curl.URL, 'http://%s:8380/postfields' % localhost) + self.curl.setopt(self.curl.POST, 1) + self.curl.setopt(self.curl.POSTFIELDSIZE, os.stat(FORM_SUBMISSION_PATH).st_size) + self.curl.setopt(self.curl.READFUNCTION, function_provider.read) + self.curl.setopt(self.curl.READDATA, f) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEDATA, sio) + self.curl.perform() + + actual = json.loads(sio.getvalue().decode()) + self.assertEqual({'foo': 'bar'}, actual) + + def test_readdata_not_file_like(self): + not_file_like = object() + try: + self.curl.setopt(self.curl.READDATA, not_file_like) + except TypeError as exc: + self.assertIn('object given without a read method', str(exc)) + else: + self.fail('TypeError not raised') diff --git a/tests/relative_url_test.py b/tests/relative_url_test.py new file mode 100644 index 0000000..a267255 --- /dev/null +++ b/tests/relative_url_test.py @@ -0,0 +1,23 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +# uses the high level interface +import curl +import unittest + +from . import appmanager + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class RelativeUrlTest(unittest.TestCase): + def setUp(self): + self.curl = curl.Curl('http://%s:8380/' % localhost) + + def tearDown(self): + self.curl.close() + + def test_get_relative(self): + self.curl.get('/success') + self.assertEqual('success', self.curl.body().decode()) diff --git a/tests/reload_test.py b/tests/reload_test.py new file mode 100644 index 0000000..e6c1fbc --- /dev/null +++ b/tests/reload_test.py @@ -0,0 +1,19 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import pytest +import unittest + +class ReloadTest(unittest.TestCase): + @pytest.mark.standalone + def test_reloading(self): + try: + # python 2 + reload_fn = reload + except NameError: + # python 3 + import importlib + reload_fn = importlib.reload + reload_fn(pycurl) diff --git a/tests/reset_test.py b/tests/reset_test.py new file mode 100644 index 0000000..c9938f3 --- /dev/null +++ b/tests/reset_test.py @@ -0,0 +1,85 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class ResetTest(unittest.TestCase): + def test_reset(self): + c = util.DefaultCurl() + c.setopt(pycurl.USERAGENT, 'Phony/42') + c.setopt(pycurl.URL, 'http://%s:8380/header?h=user-agent' % localhost) + sio = util.BytesIO() + c.setopt(pycurl.WRITEFUNCTION, sio.write) + c.perform() + user_agent = sio.getvalue().decode() + assert user_agent == 'Phony/42' + + c.reset() + c.setopt(pycurl.URL, 'http://%s:8380/header?h=user-agent' % localhost) + sio = util.BytesIO() + c.setopt(pycurl.WRITEFUNCTION, sio.write) + c.perform() + user_agent = sio.getvalue().decode() + # we also check that the request succeeded after curl + # object has been reset + assert user_agent.startswith('PycURL') + + # XXX this test was broken when it was test_reset.py + def skip_reset_with_multi(self): + outf = util.BytesIO() + cm = pycurl.CurlMulti() + + eh = util.DefaultCurl() + + for x in range(1, 20): + eh.setopt(pycurl.WRITEFUNCTION, outf.write) + eh.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + cm.add_handle(eh) + + while 1: + ret, active_handles = cm.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + while active_handles: + ret = cm.select(1.0) + if ret == -1: + continue + while 1: + ret, active_handles = cm.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + count, good, bad = cm.info_read() + + for h, en, em in bad: + print("Transfer to %s failed with %d, %s\n" % \ + (h.getinfo(pycurl.EFFECTIVE_URL), en, em)) + raise RuntimeError + + for h in good: + httpcode = h.getinfo(pycurl.RESPONSE_CODE) + if httpcode != 200: + print("Transfer to %s failed with code %d\n" %\ + (h.getinfo(pycurl.EFFECTIVE_URL), httpcode)) + raise RuntimeError + + else: + print("Recd %d bytes from %s" % \ + (h.getinfo(pycurl.SIZE_DOWNLOAD), + h.getinfo(pycurl.EFFECTIVE_URL))) + + cm.remove_handle(eh) + eh.reset() + + eh.close() + cm.close() + outf.close() diff --git a/tests/resolve_test.py b/tests/resolve_test.py new file mode 100644 index 0000000..d49c396 --- /dev/null +++ b/tests/resolve_test.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +import pycurl +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class ResolveTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_resolve(self): + if util.pycurl_version_less_than(7, 21, 3) and not hasattr(pycurl, 'RESOLVE'): + raise unittest.SkipTest('libcurl < 7.21.3 or no RESOLVE') + + self.curl.setopt(pycurl.URL, 'http://p.localhost:8380/success') + self.curl.setopt(pycurl.RESOLVE, ['p.localhost:8380:127.0.0.1']) + self.curl.perform() + self.assertEqual(200, self.curl.getinfo(pycurl.RESPONSE_CODE)) diff --git a/tests/run-quickstart.sh b/tests/run-quickstart.sh new file mode 100755 index 0000000..103df36 --- /dev/null +++ b/tests/run-quickstart.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +set -e + +export PYTHONSUFFIX=$(python -V 2>&1 |awk '{print $2}' |awk -F. '{print $1 "." $2}') +export PYTHONPATH="`pwd`"/$(ls -d build/lib.*$PYTHONSUFFIX):$PYTHONPATH + +tmpdir=`mktemp -d` + +finish() { + rm -rf "$tmpdir" +} + +trap finish EXIT + +for file in "`pwd`"/examples/quickstart/*.py; do \ + # skip Python 2-only examples on Python 3 + if echo "$file" |grep -q python2 && + python -V 2>&1 |grep -q 'Python 3' + then + continue + fi + + set +e + (cd "$tmpdir" && python "$file" >output) + rv=$? + set -e + if test "$rv" != 0; then + echo "$file failed, standard error contents (if any) is above" + if test -n "`cat "$tmpdir"/output`"; then + echo "Standard output contents:" + cat "$tmpdir"/output + fi + exit $rv + fi +done + +echo 'All ok' diff --git a/tests/run.sh b/tests/run.sh new file mode 100755 index 0000000..48c29c3 --- /dev/null +++ b/tests/run.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -e +set -x + +test -n "$PYTHON" || PYTHON=python +test -n "$PYTEST" || PYTEST=pytest + +mkdir -p tests/tmp +export PYTHONMAJOR=$($PYTHON -V 2>&1 |awk '{print $2}' |awk -F. '{print $1}') +export PYTHONMINOR=$($PYTHON -V 2>&1 |awk '{print $2}' |awk -F. '{print $2}') +export PYTHONPATH=$(ls -d build/lib.*$PYTHONMAJOR*$PYTHONMINOR):$PYTHONPATH + +extra_attrs= +if test "$CI" = true; then + if test -n "$USECURL" && echo "$USECURL" |grep -q gssapi; then + : + else + extra_attrs="$extra_attrs",\!gssapi + fi + if test -n "$USECURL" && echo "$USECURL" |grep -q libssh2; then + : + else + extra_attrs="$extra_attrs",\!ssh + fi +fi + +$PYTHON -c 'import pycurl; print(pycurl.version)' +$PYTEST -v diff --git a/tests/runwsgi.py b/tests/runwsgi.py new file mode 100644 index 0000000..ff00b8c --- /dev/null +++ b/tests/runwsgi.py @@ -0,0 +1,119 @@ +# Run a WSGI application in a daemon thread + +import bottle +import threading +import os.path + +from . import util + +global_stop = False + +class Server(bottle.WSGIRefServer): + def run(self, handler): # pragma: no cover + self.srv = self.make_server(handler) + self.serve() + + def make_server(self, handler): + from wsgiref.simple_server import make_server, WSGIRequestHandler + if self.quiet: + base = self.options.get('handler_class', WSGIRequestHandler) + class QuietHandler(base): + def log_request(*args, **kw): + pass + self.options['handler_class'] = QuietHandler + srv = make_server(self.host, self.port, handler, **self.options) + return srv + + def serve(self): + self.srv.serve_forever(poll_interval=0.1) + +# http://www.socouldanyone.com/2014/01/bottle-with-ssl.html +# https://github.com/mfm24/miscpython/blob/master/bottle_ssl.py +class SslServer(Server): + def run(self, handler): # pragma: no cover + self.srv = self.make_server(handler) + + import ssl + cert_dir = os.path.join(os.path.dirname(__file__), 'certs') + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain( + os.path.join(cert_dir, 'server.crt'), + keyfile=os.path.join(cert_dir, 'server.key')) + self.srv.socket = context.wrap_socket( + self.srv.socket, + server_side=True) + + self.serve() + +def start_bottle_server(app, port, server, **kwargs): + server_thread = ServerThread(app, port, server, kwargs) + server_thread.daemon = True + server_thread.start() + + ok = util.wait_for_network_service(('127.0.0.1', port), 0.1, 10) + if not ok: + import warnings + warnings.warn('Server did not start after 1 second') + + return server_thread.server + +class ServerThread(threading.Thread): + def __init__(self, app, port, server, server_kwargs): + threading.Thread.__init__(self) + self.app = app + self.port = port + self.server_kwargs = server_kwargs + self.server = server(host='127.0.0.1', port=self.port, **self.server_kwargs) + + def run(self): + bottle.run(self.app, server=self.server, quiet=True) + +started_servers = {} + +def app_runner_setup(*specs): + '''Returns setup and teardown methods for running a list of WSGI + applications in a daemon thread. + + Each argument is an (app, port) pair. + + Return value is a (setup, teardown) function pair. + + The setup and teardown functions expect to be called with an argument + on which server state will be stored. + + Example usage with nose: + + >>> setup_module, teardown_module = \ + runwsgi.app_runner_setup((app_module.app, 8050)) + ''' + + def setup(self): + self.servers = [] + for spec in specs: + if len(spec) == 2: + app, port = spec + kwargs = {} + else: + app, port, kwargs = spec + if port in started_servers: + assert started_servers[port] == (app, kwargs) + else: + server = Server + if 'server' in kwargs: + server = kwargs['server'] + del kwargs['server'] + elif 'ssl' in kwargs: + if kwargs['ssl']: + server = SslServer + del kwargs['ssl'] + self.servers.append(start_bottle_server(app, port, server, **kwargs)) + started_servers[port] = (app, kwargs) + + def teardown(self): + return + for server in self.servers: + # if no tests from module were run, there is no server to shut down + if hasattr(server, 'srv'): + server.srv.shutdown() + + return [setup, teardown] diff --git a/tests/seek_cb_constants_test.py b/tests/seek_cb_constants_test.py new file mode 100644 index 0000000..a3f2f8a --- /dev/null +++ b/tests/seek_cb_constants_test.py @@ -0,0 +1,31 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import unittest + +from . import util + +class SeekCbConstantsTest(unittest.TestCase): + # numeric value is understood by older libcurls but + # the constant is only defined in 7.19.5+ + @util.min_libcurl(7, 19, 5) + def test_ok(self): + curl = pycurl.Curl() + self.assertEqual(0, curl.SEEKFUNC_OK) + curl.close() + + # numeric value is understood by older libcurls but + # the constant is only defined in 7.19.5+ + @util.min_libcurl(7, 19, 5) + def test_fail(self): + curl = pycurl.Curl() + self.assertEqual(1, curl.SEEKFUNC_FAIL) + curl.close() + + @util.min_libcurl(7, 19, 5) + def test_cantseek(self): + curl = pycurl.Curl() + self.assertEqual(2, curl.SEEKFUNC_CANTSEEK) + curl.close() diff --git a/tests/seek_cb_test.py b/tests/seek_cb_test.py new file mode 100644 index 0000000..4cbf862 --- /dev/null +++ b/tests/seek_cb_test.py @@ -0,0 +1,77 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +# Note: this test is meant to be run from pycurl project root. + +import pycurl +import unittest +import os.path + +from . import procmgr, localhost, util + +setup_module, teardown_module = procmgr.vsftpd_setup() + +class PartialFileSource: + def __init__(self): + self.__buf = '1234567890.1234567890' + self.__maxread = None + self.__bufptr = 0 + + def read(self, size): + p = self.__bufptr + end = p+size + if self.__maxread: + end = min(self.__maxread, end) + ret = self.__buf[p:end] + self.__bufptr+= len(ret) + #print 20*">>>", "read(%s) ==> %s" % (size, len(ret)) + return ret + + def seek(self, offset, origin): + #print 20*">>>", "seek(%s, %s)" % (offset, origin) + self.__bufptr = offset + + def set_maxread(self, maxread): + self.__maxread = maxread + +class SeekCbTest(unittest.TestCase): + def test_seek_function(self): + c = util.DefaultCurl() + c.setopt(pycurl.UPLOAD, 1) + c.setopt(pycurl.URL, "ftp://%s:8321/tests/tmp/upload.txt" % localhost) + c.setopt(pycurl.RESUME_FROM, 0) + #c.setopt(pycurl.VERBOSE, 1) + upload_file = PartialFileSource() + c.setopt(pycurl.READFUNCTION, upload_file.read) + upload_file.set_maxread(10) + c.perform() + + f = open(os.path.join(os.path.dirname(__file__), 'tmp', 'upload.txt')) + try: + content = f.read() + finally: + f.close() + self.assertEqual('1234567890', content) + + c.close() + del c + del upload_file + + c = util.DefaultCurl() + c.setopt(pycurl.URL, "ftp://%s:8321/tests/tmp/upload.txt" % localhost) + c.setopt(pycurl.RESUME_FROM, -1) + c.setopt(pycurl.UPLOAD, 1) + #c.setopt(pycurl.VERBOSE, 1) + upload_file = PartialFileSource() + c.setopt(pycurl.READFUNCTION, upload_file.read) + c.setopt(pycurl.SEEKFUNCTION, upload_file.seek) + c.perform() + c.close() + + f = open(os.path.join(os.path.dirname(__file__), 'tmp', 'upload.txt')) + try: + content = f.read() + finally: + f.close() + self.assertEqual('1234567890.1234567890', content) diff --git a/tests/setopt_lifecycle_test.py b/tests/setopt_lifecycle_test.py new file mode 100644 index 0000000..eacedc8 --- /dev/null +++ b/tests/setopt_lifecycle_test.py @@ -0,0 +1,59 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import gc +import pycurl +import unittest +try: + import json +except ImportError: + import simplejson as json + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class TestString(str): + __test__ = False + def __del__(self): + self.replace('1', '2') + #print self + #print 'd' + +class SetoptLifecycleTest(unittest.TestCase): + # separate method to permit pf to go out of scope and be + # garbage collected before perform call + def do_setopt(self, curl, index): + pf = TestString('&'.join(50*['field=value%d' % (index,)])) + curl.setopt(pycurl.URL, 'http://%s:8380/postfields' % localhost) + curl.setopt(pycurl.POSTFIELDS, pf) + + # This test takes 6+ seconds to run. + # It seems to pass with broken pycurl code when run by itself, + # but fails when run as part of the entire test suite. + def test_postfields_lifecycle(self): + requests = [] + for i in range(1000): + curl = util.DefaultCurl() + self.do_setopt(curl, i) + gc.collect() + requests.append(curl) + + # send requests here to permit maximum garbage recycling + for i in range(100): + curl = requests[i] + #self.curl.setopt(pycurl.VERBOSE, 1) + sio = util.BytesIO() + curl.setopt(pycurl.WRITEFUNCTION, sio.write) + curl.perform() + self.assertEqual(200, curl.getinfo(pycurl.HTTP_CODE)) + body = sio.getvalue().decode() + returned_fields = json.loads(body) + self.assertEqual(dict(field='value%d' % i), returned_fields) + + for i in range(100): + curl = requests[i] + curl.close() diff --git a/tests/setopt_string_test.py b/tests/setopt_string_test.py new file mode 100644 index 0000000..043b569 --- /dev/null +++ b/tests/setopt_string_test.py @@ -0,0 +1,31 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import pytest +from . import localhost +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class SetoptTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_setopt_string(self): + self.curl.setopt_string(pycurl.URL, 'http://%s:8380/success' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + self.assertEqual('success', sio.getvalue().decode()) + + def test_setopt_string_integer(self): + with pytest.raises(TypeError): + self.curl.setopt_string(pycurl.VERBOSE, True) diff --git a/tests/setopt_test.py b/tests/setopt_test.py new file mode 100644 index 0000000..472cf34 --- /dev/null +++ b/tests/setopt_test.py @@ -0,0 +1,113 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import pytest +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class SetoptTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_boolean_value(self): + # expect no exceptions raised + self.curl.setopt(pycurl.VERBOSE, True) + + def test_integer_value(self): + # expect no exceptions raised + self.curl.setopt(pycurl.VERBOSE, 1) + + def test_string_value_for_integer_option(self): + with pytest.raises(TypeError): + self.curl.setopt(pycurl.VERBOSE, "Hello, world!") + + def test_string_value(self): + # expect no exceptions raised + self.curl.setopt(pycurl.URL, 'http://hello.world') + + def test_integer_value_for_string_option(self): + with pytest.raises(TypeError): + self.curl.setopt(pycurl.URL, 1) + + def test_float_value_for_integer_option(self): + with pytest.raises(TypeError): + self.curl.setopt(pycurl.VERBOSE, 1.0) + + def test_httpheader_list(self): + self.curl.setopt(self.curl.HTTPHEADER, ['Accept:']) + + def test_httpheader_tuple(self): + self.curl.setopt(self.curl.HTTPHEADER, ('Accept:',)) + + def test_httpheader_unicode(self): + self.curl.setopt(self.curl.HTTPHEADER, (util.u('Accept:'),)) + + def test_unset_httpheader(self): + self.curl.setopt(self.curl.HTTPHEADER, ('x-test: foo',)) + self.curl.setopt(self.curl.URL, 'http://%s:8380/header?h=x-test' % localhost) + io = util.BytesIO() + self.curl.setopt(self.curl.WRITEDATA, io) + self.curl.perform() + self.assertEqual(util.b('foo'), io.getvalue()) + + self.curl.unsetopt(self.curl.HTTPHEADER) + io = util.BytesIO() + self.curl.setopt(self.curl.WRITEDATA, io) + self.curl.perform() + self.assertEqual(util.b(''), io.getvalue()) + + def test_set_httpheader_none(self): + self.curl.setopt(self.curl.HTTPHEADER, ('x-test: foo',)) + self.curl.setopt(self.curl.URL, 'http://%s:8380/header?h=x-test' % localhost) + io = util.BytesIO() + self.curl.setopt(self.curl.WRITEDATA, io) + self.curl.perform() + self.assertEqual(util.b('foo'), io.getvalue()) + + self.curl.setopt(self.curl.HTTPHEADER, None) + io = util.BytesIO() + self.curl.setopt(self.curl.WRITEDATA, io) + self.curl.perform() + self.assertEqual(util.b(''), io.getvalue()) + + @util.min_libcurl(7, 37, 0) + def test_proxyheader_list(self): + self.curl.setopt(self.curl.PROXYHEADER, ['Accept:']) + + @util.min_libcurl(7, 37, 0) + def test_proxyheader_tuple(self): + self.curl.setopt(self.curl.PROXYHEADER, ('Accept:',)) + + @util.min_libcurl(7, 37, 0) + def test_proxyheader_unicode(self): + self.curl.setopt(self.curl.PROXYHEADER, (util.u('Accept:'),)) + + @util.min_libcurl(7, 37, 0) + def test_unset_proxyheader(self): + self.curl.unsetopt(self.curl.PROXYHEADER) + + @util.min_libcurl(7, 37, 0) + def test_set_proxyheader_none(self): + self.curl.setopt(self.curl.PROXYHEADER, None) + + def test_unset_encoding(self): + self.curl.unsetopt(self.curl.ENCODING) + + # github issue #405 + def test_large_options(self): + self.curl.setopt(self.curl.INFILESIZE, 3333858173) + self.curl.setopt(self.curl.MAX_RECV_SPEED_LARGE, 3333858173) + self.curl.setopt(self.curl.MAX_SEND_SPEED_LARGE, 3333858173) + self.curl.setopt(self.curl.MAXFILESIZE, 3333858173) + self.curl.setopt(self.curl.POSTFIELDSIZE, 3333858173) + self.curl.setopt(self.curl.RESUME_FROM, 3333858173) diff --git a/tests/setopt_unicode_test.py b/tests/setopt_unicode_test.py new file mode 100644 index 0000000..7ce4656 --- /dev/null +++ b/tests/setopt_unicode_test.py @@ -0,0 +1,38 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import pytest +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class SetoptUnicodeTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_ascii_string(self): + self.check('p=test', 'test') + + def test_unicode_string(self): + with pytest.raises(UnicodeEncodeError): + self.check(util.u('p=Москва'), util.u('Москва')) + + def test_unicode_encoded(self): + self.check(util.u('p=Москва').encode('utf8'), util.u('Москва')) + + def check(self, send, expected): + self.curl.setopt(pycurl.URL, 'http://%s:8380/param_utf8_hack' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.setopt(pycurl.POSTFIELDS, send) + self.curl.perform() + self.assertEqual(expected, sio.getvalue().decode('utf-8')) diff --git a/tests/setup_test.py b/tests/setup_test.py new file mode 100644 index 0000000..50f94f7 --- /dev/null +++ b/tests/setup_test.py @@ -0,0 +1,285 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import util +import setup as pycurl_setup +import unittest +import os, os.path, sys +import functools +try: + # Python 2 + from StringIO import StringIO +except ImportError: + # Python 3 + from io import StringIO + +def set_env(key, new_value): + old_value = os.environ.get(key) + if new_value is not None: + os.environ[key] = new_value + elif old_value is not None: + del os.environ[key] + else: + # new and old values are None which mean the variable is not set + pass + return old_value + +def reset_env(key, old_value): + # empty string means environment variable was empty + # None means it was not set + if old_value is not None: + os.environ[key] = old_value + elif key in os.environ: + del os.environ[key] + +def using_curl_config(path, ssl_library=None): + path = os.path.join(os.path.dirname(__file__), 'fake-curl', path) + def decorator(fn): + @functools.wraps(fn) + def decorated(*args, **kwargs): + old_path = set_env('PYCURL_CURL_CONFIG', path) + old_ssl_library = set_env('PYCURL_SSL_LIBRARY', ssl_library) + try: + return fn(*args, **kwargs) + finally: + reset_env('PYCURL_CURL_CONFIG', old_path) + reset_env('PYCURL_SSL_LIBRARY', old_ssl_library) + return decorated + return decorator + +def min_python_version(*spec): + def decorator(fn): + @functools.wraps(fn) + def decorated(*args, **kwargs): + if sys.version_info < spec: + raise unittest.SkipTest('Minimum Python version %s required' % spec.join('.')) + + return fn(*args, **kwargs) + return decorated + return decorator + +class SetupTest(unittest.TestCase): + + @util.only_unix + def test_sanity_check(self): + config = pycurl_setup.ExtensionConfiguration() + # we should link against libcurl, one would expect + assert 'curl' in config.libraries + + @util.only_unix + def test_valid_option_consumes_argv(self): + argv = ['', '--with-nss'] + pycurl_setup.ExtensionConfiguration(argv) + self.assertEqual([''], argv) + + @util.only_unix + def test_invalid_option_not_consumed(self): + argv = ['', '--bogus'] + pycurl_setup.ExtensionConfiguration(argv) + self.assertEqual(['', '--bogus'], argv) + + @util.only_unix + def test_invalid_option_suffix_not_consumed(self): + argv = ['', '--with-nss-bogus'] + pycurl_setup.ExtensionConfiguration(argv) + self.assertEqual(['', '--with-nss-bogus'], argv) + + @util.only_unix + @using_curl_config('curl-config-empty') + def test_no_ssl(self): + config = pycurl_setup.ExtensionConfiguration() + # do not expect anything to do with ssl + assert 'crypto' not in config.libraries + + @util.only_unix + @using_curl_config('curl-config-libs-and-static-libs') + def test_does_not_use_static_libs(self): + config = pycurl_setup.ExtensionConfiguration() + # should not link against any libraries from --static-libs if + # --libs succeeded + assert 'flurby' in config.libraries + assert 'kzzert' not in config.libraries + + @util.only_unix + @using_curl_config('curl-config-ssl-in-libs') + def test_ssl_in_libs(self): + config = pycurl_setup.ExtensionConfiguration() + # should link against openssl + assert 'crypto' in config.libraries + + @util.only_unix + @using_curl_config('curl-config-ssl-in-static-libs') + def test_ssl_in_static_libs(self): + config = pycurl_setup.ExtensionConfiguration() + # should link against openssl + assert 'crypto' in config.libraries + + @util.only_unix + @using_curl_config('curl-config-empty') + def test_no_ssl_define(self): + config = pycurl_setup.ExtensionConfiguration() + # ssl define should be off + assert 'HAVE_CURL_SSL' not in config.define_symbols + + @util.only_unix + @using_curl_config('curl-config-ssl-in-libs') + def test_ssl_in_libs_sets_ssl_define(self): + config = pycurl_setup.ExtensionConfiguration() + # ssl define should be on + assert 'HAVE_CURL_SSL' in config.define_symbols + + @util.only_unix + @using_curl_config('curl-config-ssl-in-static-libs') + def test_ssl_in_static_libs_sets_ssl_define(self): + config = pycurl_setup.ExtensionConfiguration() + # ssl define should be on + assert 'HAVE_CURL_SSL' in config.define_symbols + + @util.only_unix + @using_curl_config('curl-config-ssl-in-libs') + def test_ssl_feature_sets_ssl_define(self): + config = pycurl_setup.ExtensionConfiguration() + # ssl define should be on + assert 'HAVE_CURL_SSL' in config.define_symbols + + @util.only_unix + @using_curl_config('curl-config-ssl-feature-only') + def test_ssl_feature_only(self): + saved_stderr = sys.stderr + sys.stderr = captured_stderr = StringIO() + try: + config = pycurl_setup.ExtensionConfiguration() + finally: + sys.stderr = saved_stderr + # ssl define should be on + assert 'HAVE_CURL_SSL' in config.define_symbols + # and a warning message + assert 'Warning: libcurl is configured to use SSL, but we have \ +not been able to determine which SSL backend it is using.' in captured_stderr.getvalue() + + @util.only_unix + @using_curl_config('curl-config-ssl-feature-only') + def test_libcurl_ssl_openssl(self): + config = pycurl_setup.ExtensionConfiguration(['', + '--libcurl-dll=tests/fake-curl/libcurl/with_openssl.so']) + # openssl should be detected + assert 'HAVE_CURL_SSL' in config.define_symbols + assert 'HAVE_CURL_OPENSSL' in config.define_symbols + assert 'crypto' in config.libraries + + assert 'HAVE_CURL_GNUTLS' not in config.define_symbols + assert 'HAVE_CURL_NSS' not in config.define_symbols + + @util.only_unix + @using_curl_config('curl-config-ssl-feature-only') + def test_libcurl_ssl_gnutls(self): + config = pycurl_setup.ExtensionConfiguration(['', + '--libcurl-dll=tests/fake-curl/libcurl/with_gnutls.so']) + # gnutls should be detected + assert 'HAVE_CURL_SSL' in config.define_symbols + assert 'HAVE_CURL_GNUTLS' in config.define_symbols + assert 'gnutls' in config.libraries + + assert 'HAVE_CURL_OPENSSL' not in config.define_symbols + assert 'HAVE_CURL_NSS' not in config.define_symbols + + @util.only_unix + @using_curl_config('curl-config-ssl-feature-only') + def test_libcurl_ssl_nss(self): + config = pycurl_setup.ExtensionConfiguration(['', + '--libcurl-dll=tests/fake-curl/libcurl/with_nss.so']) + # nss should be detected + assert 'HAVE_CURL_SSL' in config.define_symbols + assert 'HAVE_CURL_NSS' in config.define_symbols + assert 'ssl3' in config.libraries + + assert 'HAVE_CURL_OPENSSL' not in config.define_symbols + assert 'HAVE_CURL_GNUTLS' not in config.define_symbols + + @util.only_unix + @using_curl_config('curl-config-empty') + def test_libcurl_ssl_unrecognized(self): + config = pycurl_setup.ExtensionConfiguration(['', + '--libcurl-dll=tests/fake-curl/libcurl/with_unknown_ssl.so']) + assert 'HAVE_CURL_SSL' not in config.define_symbols + assert 'HAVE_CURL_OPENSSL' not in config.define_symbols + assert 'HAVE_CURL_GNUTLS' not in config.define_symbols + assert 'HAVE_CURL_NSS' not in config.define_symbols + + @util.only_unix + @using_curl_config('curl-config-ssl-feature-only') + def test_with_ssl_library(self): + config = pycurl_setup.ExtensionConfiguration(['', + '--with-ssl']) + assert 'HAVE_CURL_SSL' in config.define_symbols + assert 'HAVE_CURL_OPENSSL' in config.define_symbols + assert 'crypto' in config.libraries + + assert 'HAVE_CURL_GNUTLS' not in config.define_symbols + assert 'HAVE_CURL_NSS' not in config.define_symbols + + @util.only_unix + @using_curl_config('curl-config-ssl-feature-only') + def test_with_openssl_library(self): + config = pycurl_setup.ExtensionConfiguration(['', + '--with-openssl']) + assert 'HAVE_CURL_SSL' in config.define_symbols + assert 'HAVE_CURL_OPENSSL' in config.define_symbols + assert 'crypto' in config.libraries + + assert 'HAVE_CURL_GNUTLS' not in config.define_symbols + assert 'HAVE_CURL_NSS' not in config.define_symbols + + @util.only_unix + @using_curl_config('curl-config-ssl-feature-only') + def test_with_gnutls_library(self): + config = pycurl_setup.ExtensionConfiguration(['', + '--with-gnutls']) + assert 'HAVE_CURL_SSL' in config.define_symbols + assert 'HAVE_CURL_GNUTLS' in config.define_symbols + assert 'gnutls' in config.libraries + + assert 'HAVE_CURL_OPENSSL' not in config.define_symbols + assert 'HAVE_CURL_NSS' not in config.define_symbols + + @util.only_unix + @using_curl_config('curl-config-ssl-feature-only') + def test_with_nss_library(self): + config = pycurl_setup.ExtensionConfiguration(['', + '--with-nss']) + assert 'HAVE_CURL_SSL' in config.define_symbols + assert 'HAVE_CURL_NSS' in config.define_symbols + assert 'ssl3' in config.libraries + + assert 'HAVE_CURL_OPENSSL' not in config.define_symbols + assert 'HAVE_CURL_GNUTLS' not in config.define_symbols + + @util.only_unix + @using_curl_config('curl-config-empty') + def test_no_ssl_feature_with_libcurl_dll(self): + config = pycurl_setup.ExtensionConfiguration(['', + '--libcurl-dll=tests/fake-curl/libcurl/with_openssl.so']) + # openssl should not be detected + assert 'HAVE_CURL_SSL' not in config.define_symbols + assert 'HAVE_CURL_OPENSSL' not in config.define_symbols + assert 'crypto' not in config.libraries + + @util.only_unix + @using_curl_config('curl-config-empty') + def test_no_ssl_feature_with_ssl(self): + old_stderr = sys.stderr + sys.stderr = captured_stderr = StringIO() + + try: + config = pycurl_setup.ExtensionConfiguration(['', + '--with-ssl']) + # openssl should not be detected + assert 'HAVE_CURL_SSL' not in config.define_symbols + assert 'HAVE_CURL_OPENSSL' not in config.define_symbols + assert 'crypto' not in config.libraries + finally: + sys.stderr = old_stderr + + self.assertEqual("Warning: SSL backend specified manually but libcurl does not use SSL", + captured_stderr.getvalue().strip()) diff --git a/tests/share_test.py b/tests/share_test.py new file mode 100644 index 0000000..342b2fa --- /dev/null +++ b/tests/share_test.py @@ -0,0 +1,68 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import threading +import pycurl +import pytest +import unittest + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class WorkerThread(threading.Thread): + + def __init__(self, share): + threading.Thread.__init__(self) + self.curl = util.DefaultCurl() + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + self.curl.setopt(pycurl.SHARE, share) + self.sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, self.sio.write) + + def run(self): + self.curl.perform() + self.curl.close() + +class ShareTest(unittest.TestCase): + def test_share(self): + s = pycurl.CurlShare() + s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE) + s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS) + s.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_SSL_SESSION) + + t1 = WorkerThread(s) + t2 = WorkerThread(s) + + t1.start() + t2.start() + + t1.join() + t2.join() + + del s + + self.assertEqual('success', t1.sio.getvalue().decode()) + self.assertEqual('success', t2.sio.getvalue().decode()) + + def test_share_close(self): + s = pycurl.CurlShare() + s.close() + + def test_share_close_twice(self): + s = pycurl.CurlShare() + s.close() + s.close() + + # positional arguments are rejected + def test_positional_arguments(self): + with pytest.raises(TypeError): + pycurl.CurlShare(1) + + # keyword arguments are rejected + def test_keyword_arguments(self): + with pytest.raises(TypeError): + pycurl.CurlShare(a=1) diff --git a/tests/sockopt_cb_test.py b/tests/sockopt_cb_test.py new file mode 100644 index 0000000..c926460 --- /dev/null +++ b/tests/sockopt_cb_test.py @@ -0,0 +1,85 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import unittest +import pycurl + +from . import util +from . import appmanager + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class SockoptCbTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + self.curl.setopt(self.curl.URL, 'http://%s:8380/success' % localhost) + + def tearDown(self): + self.curl.close() + + def test_sockoptfunction_ok(self): + called = {} + + def sockoptfunction(curlfd, purpose): + called['called'] = True + return 0 + + self.curl.setopt(pycurl.SOCKOPTFUNCTION, sockoptfunction) + + self.curl.perform() + assert called['called'] + + def test_sockoptfunction_fail(self): + called = {} + + def sockoptfunction(curlfd, purpose): + called['called'] = True + return 1 + + self.curl.setopt(pycurl.SOCKOPTFUNCTION, sockoptfunction) + + try: + self.curl.perform() + self.fail('should have raised') + except pycurl.error as e: + assert e.args[0] in [pycurl.E_ABORTED_BY_CALLBACK, pycurl.E_COULDNT_CONNECT], \ + 'Unexpected pycurl error code %s' % e.args[0] + assert called['called'] + + def test_sockoptfunction_bogus_return(self): + called = {} + + def sockoptfunction(curlfd, purpose): + called['called'] = True + return 'bogus' + + self.curl.setopt(pycurl.SOCKOPTFUNCTION, sockoptfunction) + + try: + self.curl.perform() + self.fail('should have raised') + except pycurl.error as e: + assert e.args[0] in [pycurl.E_ABORTED_BY_CALLBACK, pycurl.E_COULDNT_CONNECT], \ + 'Unexpected pycurl error code %s' % e.args[0] + assert called['called'] + + @util.min_libcurl(7, 28, 0) + def test_socktype_accept(self): + assert hasattr(pycurl, 'SOCKTYPE_ACCEPT') + assert hasattr(self.curl, 'SOCKTYPE_ACCEPT') + + def test_socktype_ipcxn(self): + assert hasattr(pycurl, 'SOCKTYPE_IPCXN') + assert hasattr(self.curl, 'SOCKTYPE_IPCXN') + +class SockoptCbUnsetTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def test_sockoptfunction_none(self): + self.curl.setopt(pycurl.SOCKOPTFUNCTION, None) + + def test_sockoptfunction_unset(self): + self.curl.unsetopt(pycurl.SOCKOPTFUNCTION) diff --git a/tests/ssh_key_cb_test.py b/tests/ssh_key_cb_test.py new file mode 100644 index 0000000..f317988 --- /dev/null +++ b/tests/ssh_key_cb_test.py @@ -0,0 +1,90 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import unittest +import pycurl +import pytest + +from . import util + +sftp_server = 'sftp://web.sourceforge.net' + +@pytest.mark.online +@pytest.mark.ssh +class SshKeyCbTest(unittest.TestCase): + '''This test requires Internet access.''' + + def setUp(self): + self.curl = util.DefaultCurl() + self.curl.setopt(pycurl.URL, sftp_server) + self.curl.setopt(pycurl.VERBOSE, True) + + def tearDown(self): + self.curl.close() + + @util.min_libcurl(7, 19, 6) + # curl compiled with libssh doesn't support + # CURLOPT_SSH_KNOWNHOSTS and CURLOPT_SSH_KEYFUNCTION + @util.guard_unknown_libcurl_option + def test_keyfunction(self): + # with keyfunction returning ok + + def keyfunction(known_key, found_key, match): + return pycurl.KHSTAT_FINE + + self.curl.setopt(pycurl.SSH_KNOWNHOSTS, '.known_hosts') + self.curl.setopt(pycurl.SSH_KEYFUNCTION, keyfunction) + + try: + self.curl.perform() + self.fail('should have raised') + except pycurl.error as e: + self.assertEqual(pycurl.E_LOGIN_DENIED, e.args[0]) + + # with keyfunction returning not ok + + def keyfunction(known_key, found_key, match): + return pycurl.KHSTAT_REJECT + + self.curl.setopt(pycurl.SSH_KNOWNHOSTS, '.known_hosts') + self.curl.setopt(pycurl.SSH_KEYFUNCTION, keyfunction) + + try: + self.curl.perform() + self.fail('should have raised') + except pycurl.error as e: + self.assertEqual(pycurl.E_PEER_FAILED_VERIFICATION, e.args[0]) + + @util.min_libcurl(7, 19, 6) + @util.guard_unknown_libcurl_option + def test_keyfunction_bogus_return(self): + def keyfunction(known_key, found_key, match): + return 'bogus' + + self.curl.setopt(pycurl.SSH_KNOWNHOSTS, '.known_hosts') + self.curl.setopt(pycurl.SSH_KEYFUNCTION, keyfunction) + + try: + self.curl.perform() + self.fail('should have raised') + except pycurl.error as e: + self.assertEqual(pycurl.E_PEER_FAILED_VERIFICATION, e.args[0]) + + +@pytest.mark.ssh +class SshKeyCbUnsetTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + self.curl.setopt(pycurl.URL, sftp_server) + self.curl.setopt(pycurl.VERBOSE, True) + + @util.min_libcurl(7, 19, 6) + @util.guard_unknown_libcurl_option + def test_keyfunction_none(self): + self.curl.setopt(pycurl.SSH_KEYFUNCTION, None) + + @util.min_libcurl(7, 19, 6) + @util.guard_unknown_libcurl_option + def test_keyfunction_unset(self): + self.curl.unsetopt(pycurl.SSH_KEYFUNCTION) diff --git a/tests/subclass_test.py b/tests/subclass_test.py new file mode 100644 index 0000000..fafc272 --- /dev/null +++ b/tests/subclass_test.py @@ -0,0 +1,88 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +try: + import unittest2 as unittest +except ImportError: + import unittest +import pycurl + +CLASSES = (pycurl.Curl, pycurl.CurlMulti, pycurl.CurlShare) + +class SubclassTest(unittest.TestCase): + def test_baseclass_init(self): + # base classes do not accept any arguments on initialization + for baseclass in CLASSES: + try: + baseclass(0) + except TypeError: + pass + else: + raise AssertionError('Base class accepted invalid args') + try: + baseclass(a=1) + except TypeError: + pass + else: + raise AssertionError('Base class accepted invalid kwargs') + + def test_subclass_create(self): + for baseclass in CLASSES: + # test creation of a subclass + class MyCurlClass(baseclass): + pass + # test creation of its object + obj = MyCurlClass() + # must be of type subclass, but also an instance of base class + assert type(obj) == MyCurlClass + assert isinstance(obj, baseclass) + + def test_subclass_init(self): + for baseclass in CLASSES: + class MyCurlClass(baseclass): + def __init__(self, x, y=4): + self.x = x + self.y = y + # subclass __init__ must be able to accept args and kwargs + obj = MyCurlClass(3) + assert obj.x == 3 + assert obj.y == 4 + obj = MyCurlClass(5, y=6) + assert obj.x == 5 + assert obj.y == 6 + # and it must throw TypeError if arguments don't match + try: + MyCurlClass(1, 2, 3, kwarg=4) + except TypeError: + pass + else: + raise AssertionError('Subclass accepted invalid arguments') + + def test_subclass_method(self): + for baseclass in CLASSES: + class MyCurlClass(baseclass): + def my_method(self, x): + return x + 1 + obj = MyCurlClass() + # methods must be able to accept arguments and return a value + assert obj.my_method(1) == 2 + + def test_subclass_method_override(self): + # setopt args for each base class + args = { + pycurl.Curl: (pycurl.VERBOSE, 1), + pycurl.CurlMulti: (pycurl.M_MAXCONNECTS, 3), + pycurl.CurlShare: (pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE), + } + for baseclass in CLASSES: + class MyCurlClass(baseclass): + def setopt(self, option, value): + # base method must not be overwritten + assert super().setopt != self.setopt + # base method mut be callable, setopt must return None + assert super().setopt(option, value) is None + # return something else + return 'my setopt' + obj = MyCurlClass() + assert obj.setopt(*args[baseclass]) == 'my setopt' diff --git a/tests/unset_range_test.py b/tests/unset_range_test.py new file mode 100644 index 0000000..3167680 --- /dev/null +++ b/tests/unset_range_test.py @@ -0,0 +1,52 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import os.path +import pycurl +import unittest +import urllib.request + +class UnsetRangeTest(unittest.TestCase): + def setUp(self): + self.curl = pycurl.Curl() + + def tearDown(self): + self.curl.close() + + def test_unset_range(self): + def write_cb(data): + self.read += len(data) + return None + + # download bytes 0-9 of the script itself through the file:// protocol + self.read = 0 + url = 'file:' + urllib.request.pathname2url(os.path.abspath(__file__)) + self.curl.setopt(pycurl.URL, url) + self.curl.setopt(pycurl.WRITEFUNCTION, write_cb) + self.curl.setopt(pycurl.RANGE, '0-9') + self.curl.perform() + assert 10 == self.read + + # the RANGE setting should be preserved from the previous transfer + self.read = 0 + self.curl.perform() + assert 10 == self.read + + # drop the RANGE setting using unsetopt() and download entire script + self.read = 0 + self.curl.unsetopt(pycurl.RANGE) + self.curl.perform() + assert 10 < self.read + + # now set the RANGE again and check that pycurl takes it into account + self.read = 0 + self.curl.setopt(pycurl.RANGE, '0-9') + self.curl.perform() + assert 10 == self.read + + # now drop the RANGE setting using setopt(..., None) + self.read = 0 + self.curl.setopt(pycurl.RANGE, None) + self.curl.perform() + assert 10 < self.read diff --git a/tests/user_agent_string_test.py b/tests/user_agent_string_test.py new file mode 100644 index 0000000..e89023e --- /dev/null +++ b/tests/user_agent_string_test.py @@ -0,0 +1,28 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import unittest +import pycurl + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class UserAgentStringTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_pycurl_user_agent_string(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/header?h=user-agent' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + user_agent = sio.getvalue().decode() + assert user_agent.startswith('PycURL/') + assert 'libcurl/' in user_agent, 'User agent did not include libcurl/: %s' % user_agent diff --git a/tests/util.py b/tests/util.py new file mode 100644 index 0000000..43c3682 --- /dev/null +++ b/tests/util.py @@ -0,0 +1,283 @@ +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import tempfile +import sys, socket +import time as _time +import functools +import unittest + +py3 = sys.version_info[0] == 3 + +# python 2/3 compatibility +if py3: + from io import StringIO, BytesIO + + # borrowed from six + def b(s): + '''Byte literal''' + return s.encode("latin-1") + def u(s): + '''Text literal''' + return s + text_type = str + binary_type = bytes + + long_int = int +else: + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO + BytesIO = StringIO + + # pyflakes workaround + # https://github.com/kevinw/pyflakes/issues/13 + # https://bugs.launchpad.net/pyflakes/+bug/1308508/comments/3 + if False: + unicode = object + + # borrowed from six + def b(s): + '''Byte literal''' + return s + # Workaround for standalone backslash + def u(s): + '''Text literal''' + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + text_type = unicode + binary_type = str + + if False: + # pacify pyflakes + long = int + long_int = long + +def version_less_than_spec(version_tuple, spec_tuple): + # spec_tuple may have 2 elements, expect version_tuple to have 3 elements + assert len(version_tuple) >= len(spec_tuple) + for i in range(len(spec_tuple)): + if version_tuple[i] < spec_tuple[i]: + return True + if version_tuple[i] > spec_tuple[i]: + return False + return False + +def pycurl_version_less_than(*spec): + import pycurl + + version = [int(part) for part in pycurl.version_info()[1].split('-')[0].split('.')] + return version_less_than_spec(version, spec) + +def only_python2(fn): + @functools.wraps(fn) + def decorated(*args, **kwargs): + if sys.version_info[0] >= 3: + raise unittest.SkipTest('python >= 3') + + return fn(*args, **kwargs) + + return decorated + +def only_python3(fn): + @functools.wraps(fn) + def decorated(*args, **kwargs): + if sys.version_info[0] < 3: + raise unittest.SkipTest('python < 3') + + return fn(*args, **kwargs) + + return decorated + +def min_python(major, minor): + def decorator(fn): + @functools.wraps(fn) + def decorated(*args, **kwargs): + if sys.version_info[0:2] < (major, minor): + raise unittest.SkipTest('python < %d.%d' % (major, minor)) + + return fn(*args, **kwargs) + + return decorated + + return decorator + +def min_libcurl(major, minor, patch): + def decorator(fn): + @functools.wraps(fn) + def decorated(*args, **kwargs): + if pycurl_version_less_than(major, minor, patch): + raise unittest.SkipTest('libcurl < %d.%d.%d' % (major, minor, patch)) + + return fn(*args, **kwargs) + + return decorated + + return decorator + +def removed_in_libcurl(major, minor, patch): + def decorator(fn): + @functools.wraps(fn) + def decorated(*args, **kwargs): + if not pycurl_version_less_than(major, minor, patch): + raise unittest.SkipTest('libcurl >= %d.%d.%d' % (major, minor, patch)) + + return fn(*args, **kwargs) + + return decorated + + return decorator + +def only_ssl(fn): + import pycurl + + @functools.wraps(fn) + def decorated(*args, **kwargs): + # easier to check that pycurl supports https, although + # theoretically it is not the same test. + # pycurl.version_info()[8] is a tuple of protocols supported by libcurl + if 'https' not in pycurl.version_info()[8]: + raise unittest.SkipTest('libcurl does not support ssl') + + return fn(*args, **kwargs) + + return decorated + +def only_telnet(fn): + import pycurl + + @functools.wraps(fn) + def decorated(*args, **kwargs): + # pycurl.version_info()[8] is a tuple of protocols supported by libcurl + if 'telnet' not in pycurl.version_info()[8]: + raise unittest.SkipTest('libcurl does not support telnet') + + return fn(*args, **kwargs) + + return decorated + +def only_ssl_backends(*backends): + def decorator(fn): + import pycurl + + @functools.wraps(fn) + def decorated(*args, **kwargs): + # easier to check that pycurl supports https, although + # theoretically it is not the same test. + # pycurl.version_info()[8] is a tuple of protocols supported by libcurl + if 'https' not in pycurl.version_info()[8]: + raise unittest.SkipTest('libcurl does not support ssl') + + # XXX move to pycurl library + if 'OpenSSL/' in pycurl.version: + current_backend = 'openssl' + elif 'GnuTLS/' in pycurl.version: + current_backend = 'gnutls' + elif 'NSS/' in pycurl.version: + current_backend = 'nss' + elif 'SecureTransport' in pycurl.version: + current_backend = 'secure-transport' + else: + current_backend = 'none' + if current_backend not in backends: + raise unittest.SkipTest('SSL backend is %s' % current_backend) + + return fn(*args, **kwargs) + + return decorated + return decorator + +def only_ipv6(fn): + import pycurl + + @functools.wraps(fn) + def decorated(*args, **kwargs): + if not pycurl.version_info()[4] & pycurl.VERSION_IPV6: + raise unittest.SkipTest('libcurl does not support ipv6') + + return fn(*args, **kwargs) + + return decorated + +def only_unix(fn): + @functools.wraps(fn) + def decorated(*args, **kwargs): + if sys.platform == 'win32': + raise unittest.SkipTest('Unix only') + + return fn(*args, **kwargs) + + return decorated + +def guard_unknown_libcurl_option(fn): + '''Converts curl error 48, CURLE_UNKNOWN_OPTION, into a SkipTest + exception. This is meant to be used with tests exercising libcurl + features that depend on external libraries, such as libssh2/gssapi, + where libcurl does not provide a way of detecting whether the + required libraries were compiled against.''' + + import pycurl + + @functools.wraps(fn) + def decorated(*args, **kwargs): + try: + return fn(*args, **kwargs) + except pycurl.error: + exc = sys.exc_info()[1] + # E_UNKNOWN_OPTION is available as of libcurl 7.21.5 + if hasattr(pycurl, 'E_UNKNOWN_OPTION') and exc.args[0] == pycurl.E_UNKNOWN_OPTION: + raise unittest.SkipTest('CURLE_UNKNOWN_OPTION, skipping test') + + return decorated + +try: + create_connection = socket.create_connection +except AttributeError: + # python 2.5 + def create_connection(netloc, timeout=None): + # XXX ipv4 only + s = socket.socket() + if timeout is not None: + s.settimeout(timeout) + s.connect(netloc) + return s + +def wait_for_network_service(netloc, check_interval, num_attempts): + ok = False + for i in range(num_attempts): + try: + conn = create_connection(netloc, check_interval) + except socket.error: + #e = sys.exc_info()[1] + _time.sleep(check_interval) + else: + conn.close() + ok = True + break + return ok + +def DefaultCurl(): + import pycurl + + curl = pycurl.Curl() + curl.setopt(curl.FORBID_REUSE, True) + return curl + +def DefaultCurlLocalhost(port): + '''This is a default curl with localhost -> 127.0.0.1 name mapping + on windows systems, because they don't have it in the hosts file. + ''' + + curl = DefaultCurl() + + if sys.platform == 'win32': + curl.setopt(curl.RESOLVE, ['localhost:%d:127.0.0.1' % port]) + + return curl + +def with_real_write_file(fn): + @functools.wraps(fn) + def wrapper(*args): + with tempfile.NamedTemporaryFile() as f: + return fn(*(list(args) + [f.file])) + return wrapper diff --git a/tests/version_comparison_test.py b/tests/version_comparison_test.py new file mode 100644 index 0000000..f13a53c --- /dev/null +++ b/tests/version_comparison_test.py @@ -0,0 +1,15 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import unittest + +from . import util + +class VersionComparisonTest(unittest.TestCase): + def test_comparison(self): + assert util.version_less_than_spec((7, 22, 0), (7, 23, 0)) + assert util.version_less_than_spec((7, 22, 0), (7, 23)) + assert util.version_less_than_spec((7, 22, 0), (7, 22, 1)) + assert not util.version_less_than_spec((7, 22, 0), (7, 22, 0)) + assert not util.version_less_than_spec((7, 22, 0), (7, 22)) diff --git a/tests/version_constants_test.py b/tests/version_constants_test.py new file mode 100644 index 0000000..a2b9009 --- /dev/null +++ b/tests/version_constants_test.py @@ -0,0 +1,81 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import pycurl +import unittest + +from . import util + +class VersionConstantsTest(unittest.TestCase): + def test_ipv6(self): + assert hasattr(pycurl, 'VERSION_IPV6') + + def test_kerberos4(self): + assert hasattr(pycurl, 'VERSION_KERBEROS4') + + @util.min_libcurl(7, 40, 0) + def test_kerberos5(self): + assert hasattr(pycurl, 'VERSION_KERBEROS5') + + def test_ssl(self): + assert hasattr(pycurl, 'VERSION_SSL') + + def test_libz(self): + assert hasattr(pycurl, 'VERSION_LIBZ') + + def test_ntlm(self): + assert hasattr(pycurl, 'VERSION_NTLM') + + def test_gssnegotiate(self): + assert hasattr(pycurl, 'VERSION_GSSNEGOTIATE') + + def test_debug(self): + assert hasattr(pycurl, 'VERSION_DEBUG') + + @util.min_libcurl(7, 19, 6) + def test_curldebug(self): + assert hasattr(pycurl, 'VERSION_CURLDEBUG') + + def test_asynchdns(self): + assert hasattr(pycurl, 'VERSION_ASYNCHDNS') + + def test_spnego(self): + assert hasattr(pycurl, 'VERSION_SPNEGO') + + def test_largefile(self): + assert hasattr(pycurl, 'VERSION_LARGEFILE') + + def test_idn(self): + assert hasattr(pycurl, 'VERSION_IDN') + + def test_sspi(self): + assert hasattr(pycurl, 'VERSION_SSPI') + + @util.min_libcurl(7, 38, 0) + def test_gssapi(self): + assert hasattr(pycurl, 'VERSION_GSSAPI') + + def test_conv(self): + assert hasattr(pycurl, 'VERSION_CONV') + + @util.min_libcurl(7, 21, 4) + def test_tlsauth_srp(self): + assert hasattr(pycurl, 'VERSION_TLSAUTH_SRP') + + @util.min_libcurl(7, 22, 0) + def test_ntlm_wb(self): + assert hasattr(pycurl, 'VERSION_NTLM_WB') + + @util.min_libcurl(7, 33, 0) + def test_http2(self): + assert hasattr(pycurl, 'VERSION_HTTP2') + + @util.min_libcurl(7, 40, 0) + def test_unix_sockets(self): + assert hasattr(pycurl, 'VERSION_UNIX_SOCKETS') + + @util.min_libcurl(7, 47, 0) + def test_psl(self): + assert hasattr(pycurl, 'VERSION_PSL') + diff --git a/tests/version_test.py b/tests/version_test.py new file mode 100644 index 0000000..a021a49 --- /dev/null +++ b/tests/version_test.py @@ -0,0 +1,13 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import unittest +import pycurl + +class VersionTest(unittest.TestCase): + def test_pycurl_presence_and_case(self): + assert pycurl.version.startswith('PycURL/') + + def test_libcurl_presence(self): + assert 'libcurl/' in pycurl.version diff --git a/tests/vsftpd.conf b/tests/vsftpd.conf new file mode 100644 index 0000000..787da0e --- /dev/null +++ b/tests/vsftpd.conf @@ -0,0 +1,13 @@ +anon_world_readable_only=yes +anonymous_enable=yes +background=no +# currently we only list files +download_enable=no +listen=yes +run_as_launching_user=yes +write_enable=yes +anon_upload_enable=yes +anon_other_write_enable=yes +listen_port=8321 +# should be supplied on command line +anon_root=/var/empty diff --git a/tests/weakref_test.py b/tests/weakref_test.py new file mode 100644 index 0000000..117ba94 --- /dev/null +++ b/tests/weakref_test.py @@ -0,0 +1,23 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import unittest +import weakref +import pycurl + +class WeakrefTest(unittest.TestCase): + def test_easy(self): + c = pycurl.Curl() + weakref.ref(c) + c.close() + + def test_multi(self): + m = pycurl.CurlMulti() + weakref.ref(m) + m.close() + + def test_share(self): + s = pycurl.CurlShare() + weakref.ref(s) + s.close() diff --git a/tests/write_abort_test.py b/tests/write_abort_test.py new file mode 100644 index 0000000..6969af2 --- /dev/null +++ b/tests/write_abort_test.py @@ -0,0 +1,42 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import os.path +import pycurl +import sys +import unittest +import urllib.request + +class WriteAbortTest(unittest.TestCase): + def setUp(self): + self.curl = pycurl.Curl() + + def tearDown(self): + self.curl.close() + + def test_write_abort(self): + def write_cb(_): + # this should cause pycurl.WRITEFUNCTION (without any range errors) + return -1 + + try: + # set when running full test suite if any earlier tests + # failed in Python code called from C + del sys.last_value + except AttributeError: + pass + + # download the script itself through the file:// protocol into write_cb + url = 'file:' + urllib.request.pathname2url(os.path.abspath(__file__)) + self.curl.setopt(pycurl.URL, url) + self.curl.setopt(pycurl.WRITEFUNCTION, write_cb) + try: + self.curl.perform() + except pycurl.error: + err, msg = sys.exc_info()[1].args + # we expect pycurl.E_WRITE_ERROR as the response + assert pycurl.E_WRITE_ERROR == err + + # no additional errors should be reported + assert not hasattr(sys, 'last_value') diff --git a/tests/write_cb_bogus_test.py b/tests/write_cb_bogus_test.py new file mode 100644 index 0000000..a0d44eb --- /dev/null +++ b/tests/write_cb_bogus_test.py @@ -0,0 +1,48 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +import os.path +import pycurl +import sys +import unittest +import urllib.request + +class WriteAbortTest(unittest.TestCase): + def setUp(self): + self.curl = pycurl.Curl() + + def tearDown(self): + self.curl.close() + + def write_cb_returning_string(self, data): + return 'foo' + + def write_cb_returning_float(self, data): + return 0.5 + + def test_write_cb_returning_string(self): + self.check(self.write_cb_returning_string) + + def test_write_cb_returning_float(self): + self.check(self.write_cb_returning_float) + + def check(self, write_cb): + # download the script itself through the file:// protocol into write_cb + url = 'file:' + urllib.request.pathname2url(os.path.abspath(__file__)) + self.curl.setopt(pycurl.URL, url) + self.curl.setopt(pycurl.WRITEFUNCTION, write_cb) + try: + self.curl.perform() + + self.fail('Should not get here') + except pycurl.error: + err, msg = sys.exc_info()[1].args + # we expect pycurl.E_WRITE_ERROR as the response + assert pycurl.E_WRITE_ERROR == err + + # actual error + assert hasattr(sys, 'last_type') + self.assertEqual(pycurl.error, sys.last_type) + assert hasattr(sys, 'last_value') + self.assertEqual('write callback must return int or None', str(sys.last_value)) diff --git a/tests/write_test.py b/tests/write_test.py new file mode 100644 index 0000000..09d3c60 --- /dev/null +++ b/tests/write_test.py @@ -0,0 +1,257 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +try: + import unittest2 as unittest +except ImportError: + import unittest +import pycurl +import tempfile +import shutil +import os.path + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class Acceptor(object): + def __init__(self): + self.buffer = '' + + def write(self, chunk): + self.buffer += chunk.decode() + +class WriteTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_write_to_tempfile_via_function(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + f = tempfile.NamedTemporaryFile() + try: + self.curl.setopt(pycurl.WRITEFUNCTION, f.write) + self.curl.perform() + f.seek(0) + body = f.read() + finally: + f.close() + self.assertEqual('success', body.decode()) + + def test_write_to_tempfile_via_object(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + f = tempfile.NamedTemporaryFile() + try: + self.curl.setopt(pycurl.WRITEDATA, f) + self.curl.perform() + f.seek(0) + body = f.read() + finally: + f.close() + self.assertEqual('success', body.decode()) + + def test_write_to_file_via_function(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + dir = tempfile.mkdtemp() + try: + path = os.path.join(dir, 'pycurltest') + f = open(path, 'wb+') + try: + self.curl.setopt(pycurl.WRITEFUNCTION, f.write) + self.curl.perform() + f.seek(0) + body = f.read() + finally: + f.close() + finally: + shutil.rmtree(dir) + self.assertEqual('success', body.decode()) + + def test_write_to_file_via_object(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + dir = tempfile.mkdtemp() + try: + path = os.path.join(dir, 'pycurltest') + f = open(path, 'wb+') + try: + self.curl.setopt(pycurl.WRITEDATA, f) + self.curl.perform() + f.seek(0) + body = f.read() + finally: + f.close() + finally: + shutil.rmtree(dir) + self.assertEqual('success', body.decode()) + + def test_write_to_file_like(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + acceptor = Acceptor() + self.curl.setopt(pycurl.WRITEDATA, acceptor) + self.curl.perform() + self.assertEqual('success', acceptor.buffer) + + @util.with_real_write_file + def test_write_to_file_like_then_real_file(self, real_f): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + acceptor = Acceptor() + self.curl.setopt(pycurl.WRITEDATA, acceptor) + self.curl.perform() + self.assertEqual('success', acceptor.buffer) + + self.curl.setopt(pycurl.WRITEDATA, real_f) + self.curl.perform() + real_f.seek(0) + body = real_f.read() + self.assertEqual('success', body.decode()) + + def test_headerfunction_and_writefunction(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + header_acceptor = Acceptor() + body_acceptor = Acceptor() + self.curl.setopt(pycurl.HEADERFUNCTION, header_acceptor.write) + self.curl.setopt(pycurl.WRITEFUNCTION, body_acceptor.write) + self.curl.perform() + self.assertEqual('success', body_acceptor.buffer) + self.assertIn('content-type', header_acceptor.buffer.lower()) + + def test_writeheader_and_writedata_file_like(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + header_acceptor = Acceptor() + body_acceptor = Acceptor() + self.curl.setopt(pycurl.WRITEHEADER, header_acceptor) + self.curl.setopt(pycurl.WRITEDATA, body_acceptor) + self.curl.perform() + self.assertEqual('success', body_acceptor.buffer) + self.assertIn('content-type', header_acceptor.buffer.lower()) + + @util.with_real_write_file + @util.with_real_write_file + def test_writeheader_and_writedata_real_file(self, real_f_header, real_f_data): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + self.curl.setopt(pycurl.WRITEHEADER, real_f_header) + self.curl.setopt(pycurl.WRITEDATA, real_f_data) + self.curl.perform() + real_f_header.seek(0) + real_f_data.seek(0) + self.assertEqual('success', real_f_data.read().decode()) + self.assertIn('content-type', real_f_header.read().decode().lower()) + + def test_writedata_and_writefunction_file_like(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + data_acceptor = Acceptor() + function_acceptor = Acceptor() + self.curl.setopt(pycurl.WRITEDATA, data_acceptor) + self.curl.setopt(pycurl.WRITEFUNCTION, function_acceptor.write) + self.curl.perform() + self.assertEqual('', data_acceptor.buffer) + self.assertEqual('success', function_acceptor.buffer) + + @util.with_real_write_file + def test_writedata_and_writefunction_real_file(self, real_f): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + function_acceptor = Acceptor() + self.curl.setopt(pycurl.WRITEDATA, real_f) + self.curl.setopt(pycurl.WRITEFUNCTION, function_acceptor.write) + self.curl.perform() + real_f.seek(0) + self.assertEqual('', real_f.read().decode().lower()) + self.assertEqual('success', function_acceptor.buffer) + + def test_writefunction_and_writedata_file_like(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + data_acceptor = Acceptor() + function_acceptor = Acceptor() + self.curl.setopt(pycurl.WRITEFUNCTION, function_acceptor.write) + self.curl.setopt(pycurl.WRITEDATA, data_acceptor) + self.curl.perform() + self.assertEqual('success', data_acceptor.buffer) + self.assertEqual('', function_acceptor.buffer) + + @util.with_real_write_file + def test_writefunction_and_writedata_real_file(self, real_f): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + function_acceptor = Acceptor() + self.curl.setopt(pycurl.WRITEFUNCTION, function_acceptor.write) + self.curl.setopt(pycurl.WRITEDATA, real_f) + self.curl.perform() + real_f.seek(0) + self.assertEqual('success', real_f.read().decode().lower()) + self.assertEqual('', function_acceptor.buffer) + + def test_writeheader_and_headerfunction_file_like(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + data_acceptor = Acceptor() + function_acceptor = Acceptor() + body_acceptor = Acceptor() + self.curl.setopt(pycurl.WRITEHEADER, data_acceptor) + self.curl.setopt(pycurl.HEADERFUNCTION, function_acceptor.write) + # silence output + self.curl.setopt(pycurl.WRITEDATA, body_acceptor) + self.curl.perform() + self.assertEqual('', data_acceptor.buffer) + self.assertIn('content-type', function_acceptor.buffer.lower()) + + @util.with_real_write_file + def test_writeheader_and_headerfunction_real_file(self, real_f): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + function_acceptor = Acceptor() + body_acceptor = Acceptor() + self.curl.setopt(pycurl.WRITEHEADER, real_f) + self.curl.setopt(pycurl.HEADERFUNCTION, function_acceptor.write) + # silence output + self.curl.setopt(pycurl.WRITEDATA, body_acceptor) + self.curl.perform() + real_f.seek(0) + self.assertEqual('', real_f.read().decode().lower()) + self.assertIn('content-type', function_acceptor.buffer.lower()) + + def test_headerfunction_and_writeheader_file_like(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + data_acceptor = Acceptor() + function_acceptor = Acceptor() + body_acceptor = Acceptor() + self.curl.setopt(pycurl.HEADERFUNCTION, function_acceptor.write) + self.curl.setopt(pycurl.WRITEHEADER, data_acceptor) + # silence output + self.curl.setopt(pycurl.WRITEDATA, body_acceptor) + self.curl.perform() + self.assertIn('content-type', data_acceptor.buffer.lower()) + self.assertEqual('', function_acceptor.buffer) + + @util.with_real_write_file + def test_headerfunction_and_writeheader_real_file(self, real_f): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + function_acceptor = Acceptor() + body_acceptor = Acceptor() + self.curl.setopt(pycurl.HEADERFUNCTION, function_acceptor.write) + self.curl.setopt(pycurl.WRITEHEADER, real_f) + # silence output + self.curl.setopt(pycurl.WRITEDATA, body_acceptor) + self.curl.perform() + real_f.seek(0) + self.assertIn('content-type', real_f.read().decode().lower()) + self.assertEqual('', function_acceptor.buffer) + + def test_writedata_not_file_like(self): + not_file_like = object() + try: + self.curl.setopt(self.curl.WRITEDATA, not_file_like) + except TypeError as exc: + self.assertIn('object given without a write method', str(exc)) + else: + self.fail('TypeError not raised') + + def test_writeheader_not_file_like(self): + not_file_like = object() + try: + self.curl.setopt(self.curl.WRITEHEADER, not_file_like) + except TypeError as exc: + self.assertIn('object given without a write method', str(exc)) + else: + self.fail('TypeError not raised') diff --git a/tests/write_to_stringio_test.py b/tests/write_to_stringio_test.py new file mode 100644 index 0000000..50f3f77 --- /dev/null +++ b/tests/write_to_stringio_test.py @@ -0,0 +1,42 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import pycurl +import unittest +import sys + +from . import appmanager +from . import util + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class WriteToStringioTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + + def tearDown(self): + self.curl.close() + + def test_write_to_bytesio(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + sio = util.BytesIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + self.curl.perform() + self.assertEqual('success', sio.getvalue().decode()) + + @util.only_python3 + def test_write_to_stringio(self): + self.curl.setopt(pycurl.URL, 'http://%s:8380/success' % localhost) + # stringio in python 3 + sio = util.StringIO() + self.curl.setopt(pycurl.WRITEFUNCTION, sio.write) + try: + self.curl.perform() + + self.fail('Should have received a write error') + except pycurl.error: + err, msg = sys.exc_info()[1].args + # we expect pycurl.E_WRITE_ERROR as the response + assert pycurl.E_WRITE_ERROR == err diff --git a/tests/xferinfo_cb_test.py b/tests/xferinfo_cb_test.py new file mode 100644 index 0000000..ada5e2d --- /dev/null +++ b/tests/xferinfo_cb_test.py @@ -0,0 +1,75 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vi:ts=4:et + +from . import localhost +import unittest +import pycurl + +from . import util +from . import appmanager + +setup_module, teardown_module = appmanager.setup(('app', 8380)) + +class XferinfoCbTest(unittest.TestCase): + def setUp(self): + self.curl = util.DefaultCurl() + self.curl.setopt(self.curl.URL, 'http://%s:8380/long_pause' % localhost) + + def tearDown(self): + self.curl.close() + + @util.min_libcurl(7, 32, 0) + def test_xferinfo_cb(self): + all_args = [] + + def xferinfofunction(*args): + all_args.append(args) + + self.curl.setopt(pycurl.XFERINFOFUNCTION, xferinfofunction) + self.curl.setopt(pycurl.NOPROGRESS, False) + + self.curl.perform() + assert len(all_args) > 0 + for args in all_args: + assert len(args) == 4 + for arg in args: + assert isinstance(arg, util.long_int) + + @util.min_libcurl(7, 32, 0) + def test_sockoptfunction_fail(self): + called = {} + + def xferinfofunction(*args): + called['called'] = True + return -1 + + self.curl.setopt(pycurl.XFERINFOFUNCTION, xferinfofunction) + self.curl.setopt(pycurl.NOPROGRESS, False) + + try: + self.curl.perform() + self.fail('should have raised') + except pycurl.error as e: + assert e.args[0] in [pycurl.E_ABORTED_BY_CALLBACK], \ + 'Unexpected pycurl error code %s' % e.args[0] + assert called['called'] + + @util.min_libcurl(7, 32, 0) + def test_sockoptfunction_exception(self): + called = {} + + def xferinfofunction(*args): + called['called'] = True + raise ValueError + + self.curl.setopt(pycurl.XFERINFOFUNCTION, xferinfofunction) + self.curl.setopt(pycurl.NOPROGRESS, False) + + try: + self.curl.perform() + self.fail('should have raised') + except pycurl.error as e: + assert e.args[0] in [pycurl.E_ABORTED_BY_CALLBACK], \ + 'Unexpected pycurl error code %s' % e.args[0] + assert called['called'] diff --git a/winbuild.py b/winbuild.py new file mode 100644 index 0000000..0a1af62 --- /dev/null +++ b/winbuild.py @@ -0,0 +1,412 @@ +# This file builds official Windows binaries of PycURL and all of its dependencies. +# +# It is written to be run on a system dedicated to building pycurl, but can be configured +# for any system that has the required tools installed. +# +# Generally, the workflow of building pycurl binaries is as follows: +# 1. Install git for windows. Use it to check out pycurl repository on the build system. +# 2. There must be a python installation already present on the build system +# in order to execute this file at all. It doesn't matter what the python +# version of the bootstrap python is. The first step is to install some +# version of python. It saves effort to install one of the versions that will be used +# to build pycurl later, however if this is done the target path should be +# in line with where all other pythons are going to be installed (i.e. c:/dev/{32,64}/pythonXY by default). +# Try these binaries: +# https://www.python.org/ftp/python/3.8.0/python-3.8.0.exe +# https://www.python.org/ftp/python/3.8.0/python-3.8.0-amd64.exe +# Then execute: +# c:\dev\python-3.8.0.exe /norestart /passive InstallAllUsers=1 Include_test=0 Include_doc=0 Include_launcher=0 Include_tcltk=0 TargetDir=c:\dev\32\python38 +# 3. Define python versions to build for in the configuration below, then +# run `python winbuild.py download` and `python winbuild.py installpy` to install them. +# 4. Download and install visual studio. Any edition of 2015 or newer should work; +# 2019 in particular (including community edition) provides batch files to set up a 2015 build environment, +# such that there is no reason to get an older version. +# 5. You may need to install platform sdk/windows sdk, especially if you installed community edition of +# visual studio as opposed to a fuller edition. Try https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk. +# 6. You may also need to install windows 8.1 sdk for building nghttp2 with cmake. +# See https://developer.microsoft.com/en-us/windows/downloads/sdk-archive. +# 7. Download and install perl. This script is tested with activestate perl, although +# other distributions may also work. activestate perl can be downloaded at http://www.activestate.com/activeperl/downloads, +# although it now requires registration to download thus using a third party download site may be preferable. +# 8. Download and install nasm: https://www.nasm.us/pub/nasm/releasebuilds/?C=M;O=D +# (homepage: http://www.nasm.us/) +# 9a. Not needed since nghttp2 is currently built using gmake: download and install cmake: https://cmake.org/download/ +# 9b. Download and install gmake: http://gnuwin32.sourceforge.net/packages/make.htm +# 10. Run `python winbuild.py builddeps` to compile all dependencies for all environments (32/64 bit and python versions). +# 11. Optional: run `python winbuild.py assembledeps` to assemble all dependencies into archives suitable for use in appveyor. +# 12. Run `python winbuild.py installvirtualenv` to install virtualenv in all python interpreters. +# 13. Run `python winbuild.py createvirtualenvs` to create virtualenvs used for pycurl compilation. +# 14. Run `python winbuild.py` to compile pycurl in all defined configurations. +# 15. Optional: run `python winbuild.py assemble` to assemble all built versions of pycurl in the current directory. + +class Config: + '''User-adjustable configuration. + + This class contains version numbers for dependencies, + which dependencies to use, + and where various binaries, headers and libraries are located in the filesystem. + ''' + + # work directory for downloading dependencies and building everything + root = 'c:/dev/build-pycurl' + # where msysgit is installed + git_root = 'c:/program files/git' + msysgit_bin_paths = [ + "c:\\Program Files\\Git\\bin", + "c:\\Program Files\\Git\\usr\\bin", + #"c:\\Program Files\\Git\\mingw64\\bin", + ] + # where NASM is installed, for building OpenSSL + nasm_path = ('c:/dev/nasm', 'c:/program files/nasm', 'c:/program files (x86)/nasm') + cmake_path = r"c:\Program Files\CMake\bin\cmake.exe" + gmake_path = r"c:\Program Files (x86)\GnuWin32\bin\make.exe" + # where ActiveState Perl is installed, for building 64-bit OpenSSL + activestate_perl_path = ('c:/perl64', r'c:\dev\perl64') + # which versions of python to build against + #python_versions = ['2.7.10', '3.2.5', '3.3.5', '3.4.3', '3.5.4', '3.6.2'] + # these require only vc9 and vc14 + python_versions = ['3.5.4', '3.6.8', '3.7.6', '3.8.1'] + # where pythons are installed + python_path_template = 'c:/dev/%(bitness)s/python%(python_release)s/python' + # overrides only, defaults are given in default_vc_paths below + vc_paths = { + # where msvc 9/vs 2008 is installed, for python 2.6 through 3.2 + 'vc9': None, + # where msvc 10/vs 2010 is installed, for python 3.3 through 3.4 + 'vc10': None, + # where msvc 14/vs 2015 is installed, for python 3.5 through 3.8 + 'vc14': None, + } + # whether to link libcurl against zlib + use_zlib = True + # which version of zlib to use, will be downloaded from internet + zlib_version = '1.2.11' + # whether to use openssl instead of winssl + use_openssl = True + # which version of openssl to use, will be downloaded from internet + openssl_version = '1.1.1d' + # whether to use c-ares + use_cares = True + cares_version = '1.15.0' + # whether to use libssh2 + use_libssh2 = True + libssh2_version = '1.9.0' + use_nghttp2 = True + nghttp2_version = '1.40.0' + use_libidn = False + libiconv_version = '1.16' + libidn_version = '1.35' + # which version of libcurl to use, will be downloaded from internet + libcurl_version = '7.68.0' + # virtualenv version + virtualenv_version = '15.1.0' + # whether to build binary wheels + build_wheels = True + # pycurl version to build, we should know this ourselves + pycurl_version = '7.45.2' + + # Sometimes vc14 does not include windows sdk path in vcvars which breaks stuff. + # another application for this is to supply normaliz.lib for vc9 + # which has an older version that doesn't have the symbols we need + windows_sdk_path = 'c:\\program files (x86)\\microsoft sdks\\windows\\v7.1a' + + # See the note below about VCTargetsPath and + # https://stackoverflow.com/questions/16092169/why-does-msbuild-look-in-c-for-microsoft-cpp-default-props-instead-of-c-progr. + # Since we are targeting vc14, use the v140 path. + vc_targets_path = "c:\\Program Files (x86)\\MSBuild\\Microsoft.Cpp\\v4.0\\v140" + #vc_targets_path = "c:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\MSBuild\\Current" + + # Where the msbuild that is part of visual studio lives + msbuild_bin_path = "c:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\MSBuild\\Current\\Bin" + +# *** +# No user-serviceable parts beyond this point. +# *** + +# OpenSSL build resources including 64-bit builds: +# http://stackoverflow.com/questions/158232/how-do-you-compile-openssl-for-x64 +# https://wiki.openssl.org/index.php/Compilation_and_Installation +# http://developer.covenanteyes.com/building-openssl-for-visual-studio/ + +import os, os.path, sys, subprocess, shutil, contextlib, zipfile, re +from winbuild.utils import * +from winbuild.config import * +from winbuild.builder import * +from winbuild.nghttp_gmake import * +from winbuild.tools import * +from winbuild.zlib import * +from winbuild.openssl import * +from winbuild.cares import * +from winbuild.ssh import * +from winbuild.curl import * +from winbuild.pycurl import * + +user_config = {} +for attr in dir(Config): + if attr.startswith('_'): + continue + user_config[attr] = getattr(Config, attr) + +# This must be at top level as __file__ can be a relative path +# and changing current directory will break it +DIR_HERE = os.path.abspath(os.path.dirname(__file__)) + +def fetch_to_archives(url): + mkdir_p(config.archives_path) + path = os.path.join(config.archives_path, os.path.basename(url)) + fetch(url, path) + +@contextlib.contextmanager +def step(step_fn, args, target_dir): + #step = step_fn.__name__ + state_tag = target_dir + mkdir_p(config.state_path) + state_file_path = os.path.join(config.state_path, state_tag) + if not os.path.exists(state_file_path) or not os.path.exists(target_dir): + step_fn(*args) + with open(state_file_path, 'w'): + pass + +def dep_builders(bconf): + builders = [] + if config.use_zlib: + builders.append(ZlibBuilder) + if config.use_openssl: + builders.append(OpensslBuilder) + if config.use_cares: + builders.append(CaresBuilder) + if config.use_libssh2: + builders.append(Libssh2Builder) + if config.use_nghttp2: + builders.append(Nghttp2Builder) + if config.use_libidn: + builders.append(LibiconvBuilder) + builders.append(LibidnBuilder) + builders.append(LibcurlBuilder) + builders = [ + cls(bconf=bconf) + for cls in builders + ] + return builders + +def build_dependencies(config): + if config.use_libssh2: + if not config.use_zlib: + # technically we can build libssh2 without zlib but I don't want to bother + raise ValueError('use_zlib must be true if use_libssh2 is true') + if not config.use_openssl: + raise ValueError('use_openssl must be true if use_libssh2 is true') + + if config.git_bin_path: + os.environ['PATH'] += ";%s" % config.git_bin_path + mkdir_p(config.archives_path) + with in_dir(config.archives_path): + for bconf in buildconfigs(): + if opts.verbose: + print('Builddep for %s, %s-bit' % (bconf.vc_version, bconf.bitness)) + for builder in dep_builders(bconf): + step(builder.build, (), builder.state_tag) + +def build(config): + # note: adds git_bin_path to PATH if necessary, and creates archives_path + build_dependencies(config) + with in_dir(config.archives_path): + for bitness in config.bitnesses: + for python_release in config.python_releases: + targets = ['bdist', 'bdist_wininst', 'bdist_msi'] + vc_version = PYTHON_VC_VERSIONS[python_release] + bconf = BuildConfig(config, bitness=bitness, vc_version=vc_version) + builder = PycurlBuilder(bconf=bconf, python_release=python_release) + builder.prepare_tree() + builder.build(targets) + +def assemble(config): + rm_rf(config, 'dist') + mkdir_p('dist') + for bitness in config.bitnesses: + for python_release in config.python_releases: + vc_version = PYTHON_VC_VERSIONS[python_release] + bconf = BuildConfig(config, bitness=bitness, vc_version=vc_version) + builder = PycurlBuilder(bconf=bconf, python_release=python_release) + print(builder.build_dir_name) + sys.stdout.flush() + src = os.path.join(config.archives_path, builder.build_dir_name, 'dist') + cp_r(config, src, '.') + +def python_metas(): + metas = [] + for version in config.python_versions: + parts = [int(part) for part in version.split('.')] + if parts[0] >= 3 and parts[1] >= 5: + ext = 'exe' + amd64_suffix = '-amd64' + else: + ext = 'msi' + amd64_suffix = '.amd64' + url_32 = 'https://www.python.org/ftp/python/%s/python-%s.%s' % (version, version, ext) + url_64 = 'https://www.python.org/ftp/python/%s/python-%s%s.%s' % (version, version, amd64_suffix, ext) + meta = dict( + version=version, ext=ext, amd64_suffix=amd64_suffix, + url_32=url_32, url_64=url_64, + installed_path_32 = 'c:\\dev\\32\\python%d%d' % (parts[0], parts[1]), + installed_path_64 = 'c:\\dev\\64\\python%d%d' % (parts[0], parts[1]), + ) + metas.append(meta) + return metas + +def download_pythons(config): + for meta in python_metas(): + for bitness in config.bitnesses: + fetch_to_archives(meta['url_%d' % bitness]) + +def install_pythons(config): + for meta in python_metas(): + for bitness in config.bitnesses: + if not os.path.exists(meta['installed_path_%d' % bitness]): + install_python(config, meta, bitness) + +# http://eddiejackson.net/wp/?p=10276 +def install_python(config, meta, bitness): + archive_path = fix_slashes(os.path.join(config.archives_path, os.path.basename(meta['url_%d' % bitness]))) + if meta['ext'] == 'exe': + cmd = [archive_path] + else: + cmd = ['msiexec', '/i', archive_path, '/norestart'] + cmd += ['/passive', 'InstallAllUsers=1', + 'Include_test=0', 'Include_doc=0', 'Include_launcher=0', + 'Include_tcltk=0', + 'TargetDir=%s' % meta['installed_path_%d' % bitness], + ] + sys.stdout.write('Installing python %s (%d bit)\n' % (meta['version'], bitness)) + print(' '.join(cmd)) + sys.stdout.flush() + check_call(cmd) + +def download_bootstrap_python(config): + version = config.python_versions[-2] + url = 'https://www.python.org/ftp/python/%s/python-%s.msi' % (version, version) + fetch(url) + +def install_virtualenv(config): + with in_dir(config.archives_path): + #fetch('https://pypi.python.org/packages/source/v/virtualenv/virtualenv-%s.tar.gz' % virtualenv_version) + fetch('https://pypi.python.org/packages/d4/0c/9840c08189e030873387a73b90ada981885010dd9aea134d6de30cd24cb8/virtualenv-15.1.0.tar.gz') + for bitness in config.bitnesses: + for python_release in config.python_releases: + print('Installing virtualenv %s for Python %s (%s bit)' % (config.virtualenv_version, python_release, bitness)) + sys.stdout.flush() + untar(config, 'virtualenv-%s' % config.virtualenv_version) + with in_dir('virtualenv-%s' % config.virtualenv_version): + python_binary = PythonBinary(python_release, bitness) + cmd = [python_binary.executable_path(config), 'setup.py', 'install'] + check_call(cmd) + +def create_virtualenvs(config): + for bitness in config.bitnesses: + for python_release in config.python_releases: + print('Creating a virtualenv for Python %s (%s bit)' % (python_release, bitness)) + sys.stdout.flush() + with in_dir(config.archives_path): + python_binary = PythonBinary(python_release, bitness) + venv_basename = 'venv-%s-%s' % (python_release, bitness) + cmd = [python_binary.executable_path(config), '-m', 'virtualenv', venv_basename] + check_call(cmd) + +def assemble_deps(config): + rm_rf(config, 'deps') + os.mkdir('deps') + for bconf in buildconfigs(): + print(bconf.vc_tag) + sys.stdout.flush() + dest = os.path.join('deps', bconf.vc_tag) + os.mkdir(dest) + for builder in dep_builders(bconf): + cp_r(config, builder.include_path, dest) + cp_r(config, builder.lib_path, dest) + with zipfile.ZipFile(os.path.join('deps', bconf.vc_tag + '.zip'), 'w', zipfile.ZIP_DEFLATED) as zip: + for root, dirs, files in os.walk(dest): + for file in files: + path = os.path.join(root, file) + zip_name = path[len(dest)+1:] + zip.write(path, zip_name) + +def get_deps(): + import struct + + python_release = sys.version_info[:2] + vc_version = PYTHON_VC_VERSIONS['.'.join(map(str, python_release))] + bitness = struct.calcsize('P') * 8 + vc_tag = '%s-%d' % (vc_version, bitness) + fetch('https://dl.bintray.com/pycurl/deps/%s.zip' % vc_tag) + check_call(['unzip', '-d', 'deps', vc_tag + '.zip']) + +import optparse + +parser = optparse.OptionParser() +parser.add_option('-b', '--bitness', help='Bitnesses build for, comma separated') +parser.add_option('-p', '--python', help='Python versions to build for, comma separated') +parser.add_option('-v', '--verbose', help='Print what is being done', action='store_true') +opts, args = parser.parse_args() + +if opts.bitness: + chosen_bitnesses = [int(bitness) for bitness in opts.bitness.split(',')] + for bitness in chosen_bitnesses: + if bitness not in BITNESSES: + print('Invalid bitness %d' % bitness) + exit(2) +else: + chosen_bitnesses = BITNESSES + +if opts.python: + chosen_pythons = opts.python.split(',') + chosen_python_versions = [] + for python in chosen_pythons: + python = python.replace('.', '') + python = python[0] + '.' + python[1] + '.' + ok = False + for python_version in Config.python_versions: + if python_version.startswith(python): + chosen_python_versions.append(python_version) + ok = True + if not ok: + print('Invalid python %s' % python) + exit(2) +else: + chosen_python_versions = Config.python_versions + +config = ExtendedConfig(user_config, + bitnesses=chosen_bitnesses, + python_versions=chosen_python_versions, + winbuild_root=DIR_HERE, +) + +def buildconfigs(): + return [BuildConfig(config, bitness=bitness, vc_version=vc_version) + for bitness in config.bitnesses + for vc_version in needed_vc_versions(config, config.python_versions) + ] + +if len(args) > 0: + if args[0] == 'download': + download_pythons(config) + elif args[0] == 'bootstrap': + download_bootstrap_python(config) + elif args[0] == 'installpy': + install_pythons(config) + elif args[0] == 'builddeps': + build_dependencies(config) + elif args[0] == 'installvirtualenv': + install_virtualenv(config) + elif args[0] == 'createvirtualenvs': + create_virtualenvs(config) + elif args[0] == 'assembledeps': + assemble_deps(config) + elif args[0] == 'assemble': + assemble(config) + elif args[0] == 'getdeps': + get_deps() + else: + print('Unknown command: %s' % args[0]) + exit(2) +else: + build(config) diff --git a/winbuild/__init__.py b/winbuild/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/winbuild/builder.py b/winbuild/builder.py new file mode 100644 index 0000000..1e60b63 --- /dev/null +++ b/winbuild/builder.py @@ -0,0 +1,146 @@ +import os, os.path, shutil, sys, subprocess +from .utils import * +from .config import * + +class Batch(object): + def __init__(self, bconf): + self.bconf = bconf + self.commands = [] + + self.add(self.vcvars_cmd) + self.add('echo on') + if self.bconf.vc_version == 'vc14': + # I don't know why vcvars doesn't configure this under vc14 + self.add('set include=%s\\include;%%include%%' % self.bconf.windows_sdk_path) + if self.bconf.bitness == 32: + self.add('set lib=%s\\lib;%%lib%%' % self.bconf.windows_sdk_path) + self.add('set path=%s\\bin;%%path%%' % self.bconf.windows_sdk_path) + else: + self.add('set lib=%s\\lib\\x64;%%lib%%' % self.bconf.windows_sdk_path) + self.add('set path=%s\\bin\\x64;%%path%%' % self.bconf.windows_sdk_path) + self.add(self.nasm_cmd) + + self.add('set path=%s;%%path%%' % self.bconf.extra_bin_paths[self.bconf.bitness]['rc_bin']) + + def add(self, cmd): + self.commands.append(cmd) + + # if patch fails to apply hunks, it exits with nonzero code. + # if patch doesn't find the patch file to apply, it exits with a zero code! + ERROR_CHECK = 'IF %ERRORLEVEL% NEQ 0 exit %errorlevel%' + + def batch_text(self): + return ("\n" + self.ERROR_CHECK + "\n").join(self.commands) + + @property + def vcvars_bitness_parameter(self): + params = { + 32: 'x86', + 64: 'amd64', + } + return params[self.bconf.bitness] + + @property + def vcvars_relative_path(self): + return 'vc/vcvarsall.bat' + + @property + def vc_path(self): + if self.bconf.vc_version in self.bconf.vc_paths and self.bconf.vc_paths[self.bconf.vc_version]: + path = self.bconf.vc_paths[self.bconf.vc_version] + if not os.path.join(path, self.vcvars_relative_path): + raise Exception('vcvars not found in specified path') + return path + else: + for path in self.bconf.default_vc_paths[self.bconf.vc_version]: + if os.path.exists(os.path.join(path, self.vcvars_relative_path)): + return path + raise Exception('No usable vc path found') + + @property + def vcvars_path(self): + return os.path.join(self.vc_path, self.vcvars_relative_path) + + @property + def vcvars_cmd(self): + # https://msdn.microsoft.com/en-us/library/x4d2c09s.aspx + return "call \"%s\" %s" % ( + self.vcvars_path, + self.vcvars_bitness_parameter, + ) + + @property + def nasm_cmd(self): + return "set path=%s;%%path%%\n" % self.bconf.nasm_path + +class Builder(object): + def __init__(self, **kwargs): + self.bconf = kwargs.pop('bconf') + self.use_dlls = False + + @contextlib.contextmanager + def execute_batch(self): + batch = Batch(self.bconf) + yield batch + with open('doit.bat', 'w') as f: + f.write(batch.batch_text()) + if False: + print("Executing:") + with open('doit.bat', 'r') as f: + print(f.read()) + sys.stdout.flush() + rv = subprocess.call(['doit.bat']) + if rv != 0: + print("\nFailed to execute the following commands:\n") + with open('doit.bat', 'r') as f: + print(f.read()) + sys.stdout.flush() + exit(3) + +class StandardBuilder(Builder): + @property + def state_tag(self): + return self.output_dir_path + + @property + def bin_path(self): + return os.path.join(self.bconf.archives_path, self.output_dir_path, 'dist', 'bin') + + @property + def include_path(self): + return os.path.join(self.bconf.archives_path, self.output_dir_path, 'dist', 'include') + + @property + def lib_path(self): + return os.path.join(self.bconf.archives_path, self.output_dir_path, 'dist', 'lib') + + @property + def dll_paths(self): + raise NotImplementedError + + @property + def builder_name(self): + return self.__class__.__name__.replace('Builder', '').lower() + + @property + def my_version(self): + return getattr(self.bconf, '%s_version' % self.builder_name) + + @property + def output_dir_path(self): + return '%s-%s-%s' % (self.builder_name, self.my_version, self.bconf.vc_tag) + + def standard_fetch_extract(self, url_template): + url = url_template % dict( + my_version=self.my_version, + ) + fetch(url) + archive_basename = os.path.basename(url) + archive_name = archive_basename.replace('.tar.gz', '') + untar(self.bconf, archive_name) + + suffixed_dir = self.output_dir_path + if os.path.exists(suffixed_dir): + shutil.rmtree(suffixed_dir) + os.rename(archive_name, suffixed_dir) + return suffixed_dir diff --git a/winbuild/c-ares-vs2015.patch b/winbuild/c-ares-vs2015.patch new file mode 100644 index 0000000..4a810a8 --- /dev/null +++ b/winbuild/c-ares-vs2015.patch @@ -0,0 +1,13 @@ +--- a/Makefile.msvc 2015-12-02 22:40:45 ++++ b/Makefile.msvc 2015-12-02 22:46:39 +@@ -125,6 +125,12 @@ + CC_VERS_NUM = 110 + !ELSEIF "$(_NMAKE_VER)" == "11.00.60315.1" + CC_VERS_NUM = 110 ++!ELSEIF "$(_NMAKE_VER)" == "11.00.61030.0" ++CC_VERS_NUM = 110 ++!ELSEIF "$(_NMAKE_VER)" == "12.00.21005.1" ++CC_VERS_NUM = 120 ++!ELSEIF "$(_NMAKE_VER)" == "14.00.23026.0" ++CC_VERS_NUM = 140 + !ELSE diff --git a/winbuild/cares.py b/winbuild/cares.py new file mode 100644 index 0000000..1c9929b --- /dev/null +++ b/winbuild/cares.py @@ -0,0 +1,27 @@ +from .utils import * +from .builder import * + +class CaresBuilder(StandardBuilder): + def build(self): + cares_dir = self.standard_fetch_extract( + 'http://c-ares.haxx.se/download/c-ares-%(my_version)s.tar.gz') + if self.bconf.cares_version == '1.12.0': + # msvc_ver.inc is missing in c-ares-1.12.0.tar.gz + # https://github.com/c-ares/c-ares/issues/69 + fetch('https://raw.githubusercontent.com/c-ares/c-ares/cares-1_12_0/msvc_ver.inc', + archive='cares-1.12.0/msvc_ver.inc') + with in_dir(cares_dir): + with self.execute_batch() as b: + if self.bconf.cares_version == '1.10.0': + b.add("patch -p1 < %s" % + require_file_exists(os.path.join(config.winbuild_patch_root, 'c-ares-vs2015.patch'))) + b.add("nmake -f Makefile.msvc") + + # assemble dist + b.add('mkdir dist dist\\include dist\\lib') + if self.bconf.cares_version_tuple < (1, 14, 0): + subdir = 'ms%s0' % self.bconf.vc_version + else: + subdir = 'msvc' + b.add('cp %s/cares/lib-release/*.lib dist/lib' % subdir) + b.add('cp *.h dist/include') diff --git a/winbuild/config.py b/winbuild/config.py new file mode 100644 index 0000000..c8fe558 --- /dev/null +++ b/winbuild/config.py @@ -0,0 +1,155 @@ +import os +from .utils import * +from .pythons import * + +class ExtendedConfig: + '''Global configuration that specifies what the entire process will do. + + Unlike Config, this class contains also various derived properties + for convenience. + ''' + + def __init__(self, user_config, **kwargs): + for k in user_config: + self.__dict__[k] = user_config[k] + for k in kwargs: + setattr(self, k, kwargs[k]) + + # These are defaults, overrides can be specified as vc_paths in Config above + default_vc_paths = { + # where msvc 9 is installed, for python 2.6-3.2 + 'vc9': [ + 'c:/program files (x86)/microsoft visual studio 9.0', + 'c:/program files/microsoft visual studio 9.0', + ], + # where msvc 10 is installed, for python 3.3-3.4 + 'vc10': [ + 'c:/program files (x86)/microsoft visual studio 10.0', + 'c:/program files/microsoft visual studio 10.0', + ], + # where msvc 14 is installed, for python 3.5-3.9 + 'vc14': [ + 'c:/program files (x86)/microsoft visual studio 14.0', + 'c:/program files/microsoft visual studio 14.0', + ], + } + + @property + def nasm_path(self): + return select_existing_path(self.__dict__['nasm_path']) + + @property + def activestate_perl_path(self): + return select_existing_path(self.__dict__['activestate_perl_path']) + + @property + def archives_path(self): + return os.path.join(self.root, 'archives') + + @property + def state_path(self): + return os.path.join(self.root, 'state') + + @property + def git_bin_path(self): + #git_bin_path = os.path.join(git_root, 'bin') + return '' + + @property + def git_path(self): + return os.path.join(self.git_bin_path, 'git') + + @property + def rm_path(self): + return find_in_paths('rm', self.msysgit_bin_paths) + + @property + def cp_path(self): + return find_in_paths('cp', self.msysgit_bin_paths) + + @property + def sed_path(self): + return find_in_paths('sed', self.msysgit_bin_paths) + + @property + def tar_path(self): + return find_in_paths('tar', self.msysgit_bin_paths) + + @property + def activestate_perl_bin_path(self): + return os.path.join(self.activestate_perl_path, 'bin') + + @property + def winbuild_patch_root(self): + return os.path.join(self.winbuild_root, 'winbuild') + + @property + def openssl_version_tuple(self): + return tuple( + int(part) if part < 'a' else part + for part in re.sub(r'([a-z])', r'.\1', self.openssl_version).split('.') + ) + + @property + def libssh2_version_tuple(self): + return tuple(int(part) for part in self.libssh2_version.split('.')) + + @property + def cares_version_tuple(self): + return tuple(int(part) for part in self.cares_version.split('.')) + + @property + def libcurl_version_tuple(self): + return tuple(int(part) for part in self.libcurl_version.split('.')) + + @property + def python_releases(self): + return [PythonRelease('.'.join(version.split('.')[:2])) + for version in self.python_versions] + + @property + def extra_bin_paths(self): + paths = {32: {}, 64: {}} + + # When using visual studio 2019 community, rc.exe is not in path for whatever reason - handle this manually. + paths[32]['rc_bin'] = os.path.dirname(glob_first('c:/{program files,program files (x86)}/windows kits/*/bin/*/x86/rc.exe')) + paths[64]['rc_bin'] = os.path.dirname(glob_first('c:/{program files,program files (x86)}/windows kits/*/bin/*/x64/rc.exe')) + + return paths + +BITNESSES = (32, 64) + +PYTHON_VC_VERSIONS = { + '2.6': 'vc9', + '2.7': 'vc9', + '3.2': 'vc9', + '3.3': 'vc10', + '3.4': 'vc10', + '3.5': 'vc14', + '3.6': 'vc14', + '3.7': 'vc14', + '3.8': 'vc14', + '3.9': 'vc14', +} + +class BuildConfig: + '''Parameters for a particular build configuration. + + Unlike ExtendedConfig, this class fixes bitness and Python version. + ''' + + def __init__(self, ext_config, **kwargs): + for k in dir(ext_config): + if k.startswith('_'): + continue + self.__dict__[k] = getattr(ext_config, k) + for k in kwargs: + setattr(self, k, kwargs[k]) + + assert self.bitness + assert self.bitness in (32, 64) + assert self.vc_version + + @property + def vc_tag(self): + return '%s-%s' % (self.vc_version, self.bitness) diff --git a/winbuild/curl.py b/winbuild/curl.py new file mode 100644 index 0000000..0b40acf --- /dev/null +++ b/winbuild/curl.py @@ -0,0 +1,110 @@ +import os.path, shutil, os +from .utils import * +from .builder import * +from .zlib import * +from .openssl import * +from .cares import * +from .ssh import * +from .nghttp_gmake import * + +class LibcurlBuilder(StandardBuilder): + def build(self): + curl_dir = self.standard_fetch_extract( + 'https://curl.haxx.se/download/curl-%(my_version)s.tar.gz') + + with in_dir(os.path.join(curl_dir, 'winbuild')): + if self.bconf.vc_version == 'vc9': + # normaliz.lib in vc9 does not have the symbols libcurl + # needs for winidn. + # Handily we have a working normaliz.lib in vc14. + # Let's take the working one and copy it locally. + os.mkdir('support') + if self.bconf.bitness == 32: + shutil.copy(os.path.join(self.bconf.windows_sdk_path, 'lib', 'normaliz.lib'), + os.path.join('support', 'normaliz.lib')) + else: + shutil.copy(os.path.join(self.bconf.windows_sdk_path, 'lib', 'x64', 'normaliz.lib'), + os.path.join('support', 'normaliz.lib')) + + with self.execute_batch() as b: + b.add("patch -p1 < %s" % + require_file_exists(os.path.join(self.bconf.winbuild_patch_root, 'libcurl-fix-zlib-references.patch'))) + if self.use_dlls: + dll_or_static = 'dll' + else: + dll_or_static = 'static' + extra_options = ' mode=%s' % dll_or_static + if self.bconf.vc_version == 'vc9': + # use normaliz.lib from msvc14/more recent windows sdk + b.add("set lib=%s;%%lib%%" % os.path.abspath('support')) + if self.bconf.use_zlib: + zlib_builder = ZlibBuilder(bconf=self.bconf) + b.add("set include=%%include%%;%s" % zlib_builder.include_path) + b.add("set lib=%%lib%%;%s" % zlib_builder.lib_path) + extra_options += ' WITH_ZLIB=%s' % dll_or_static + if self.bconf.use_openssl: + openssl_builder = OpensslBuilder(bconf=self.bconf) + b.add("set include=%%include%%;%s" % openssl_builder.include_path) + b.add("set lib=%%lib%%;%s" % openssl_builder.lib_path) + extra_options += ' WITH_SSL=%s' % dll_or_static + if self.bconf.use_cares: + cares_builder = CaresBuilder(bconf=self.bconf) + b.add("set include=%%include%%;%s" % cares_builder.include_path) + b.add("set lib=%%lib%%;%s" % cares_builder.lib_path) + extra_options += ' WITH_CARES=%s' % dll_or_static + if self.bconf.use_libssh2: + libssh2_builder = Libssh2Builder(bconf=self.bconf) + b.add("set include=%%include%%;%s" % libssh2_builder.include_path) + b.add("set lib=%%lib%%;%s" % libssh2_builder.lib_path) + extra_options += ' WITH_SSH2=%s' % dll_or_static + if self.bconf.use_nghttp2: + nghttp2_builder = Nghttp2Builder(bconf=self.bconf) + b.add("set include=%%include%%;%s" % nghttp2_builder.include_path) + b.add("set lib=%%lib%%;%s" % nghttp2_builder.lib_path) + extra_options += ' WITH_NGHTTP2=%s NGHTTP2_STATICLIB=1' % dll_or_static + if self.bconf.use_libidn: + libidn_builder = LibidnBuilder(bconf=self.bconf) + b.add("set include=%%include%%;%s" % libidn_builder.include_path) + b.add("set lib=%%lib%%;%s" % libidn_builder.lib_path) + extra_options += ' WITH_LIBIDN=%s' % dll_or_static + if self.bconf.openssl_version_tuple >= (1, 1): + # openssl 1.1.0 + # https://curl.haxx.se/mail/lib-2016-08/0104.html + # https://github.com/curl/curl/issues/984 + # crypt32.lib: http://stackoverflow.com/questions/37522654/linking-with-openssl-lib-statically + extra_options += ' MAKE="NMAKE /e" SSL_LIBS="libssl.lib libcrypto.lib crypt32.lib"' + # https://github.com/curl/curl/issues/1863 + extra_options += ' VC=%s' % self.bconf.vc_version[2:] + + # curl uses winidn APIs that do not exist in msvc9: + # https://github.com/curl/curl/issues/1863 + # We work around the msvc9 deficiency by using + # msvc14 normaliz.lib on vc9. + extra_options += ' ENABLE_IDN=yes' + + b.add("nmake /f Makefile.vc %s" % extra_options) + + # assemble dist - figure out where libcurl put its files + # and move them to a more reasonable location + with in_dir(curl_dir): + subdirs = sorted(os.listdir('builds')) + if len(subdirs) != 3: + raise Exception('Should be 3 directories here') + expected_dir = subdirs.pop(0) + for dir in subdirs: + if not dir.startswith(expected_dir): + raise Exception('%s does not start with %s' % (dir, expected_dir)) + + os.rename(os.path.join('builds', expected_dir), 'dist') + if self.bconf.vc_version == 'vc9': + # need this normaliz.lib to build pycurl later on + shutil.copy('winbuild/support/normaliz.lib', 'dist/lib/normaliz.lib') + + # need libcurl.lib to build pycurl with --curl-dir argument + shutil.copy('dist/lib/libcurl_a.lib', 'dist/lib/libcurl.lib') + + @property + def dll_paths(self): + return [ + os.path.join(self.bin_path, 'libcurl.dll'), + ] diff --git a/winbuild/iconv.py b/winbuild/iconv.py new file mode 100644 index 0000000..8a25fd9 --- /dev/null +++ b/winbuild/iconv.py @@ -0,0 +1,11 @@ +from .utils import * +from .builder import * + +class LibiconvBuilder(StandardBuilder): + def build(self): + libiconv_dir = self.standard_fetch_extract( + 'https://ftp.gnu.org/pub/gnu/libiconv/libiconv-%(my_version)s.tar.gz') + with in_dir(libiconv_dir): + with self.execute_batch() as b: + b.add("env LD=link bash ./configure") + b.add(config.gmake_path) diff --git a/winbuild/idn.py b/winbuild/idn.py new file mode 100644 index 0000000..e7820b8 --- /dev/null +++ b/winbuild/idn.py @@ -0,0 +1,10 @@ +from .utils import * +from .builder import * + +class LibidnBuilder(StandardBuilder): + def build(self): + libidn_dir = self.standard_fetch_extract( + 'https://ftp.gnu.org/gnu/libidn/libidn-%(my_version)s.tar.gz') + with in_dir(libidn_dir): + with self.execute_batch() as b: + b.add("env LD=link bash ./configure") diff --git a/winbuild/libcurl-fix-zlib-references.patch b/winbuild/libcurl-fix-zlib-references.patch new file mode 100644 index 0000000..20e06ef --- /dev/null +++ b/winbuild/libcurl-fix-zlib-references.patch @@ -0,0 +1,11 @@ +--- winbuild/MakefileBuild.vc.orig 2015-11-27 07:00:14.000000000 -0800 ++++ winbuild/MakefileBuild.vc 2016-01-01 21:33:44.263840800 -0800 +@@ -238,7 +238,7 @@ + + # Runtime library configuration + !IF "$(RTLIBCFG)"=="static" +-RTLIB = /MT ++RTLIB = /MD + RTLIB_DEBUG = /MTd + !ELSE + RTLIB = /MD diff --git a/winbuild/libssh2-vs2015.patch b/winbuild/libssh2-vs2015.patch new file mode 100644 index 0000000..b859ced --- /dev/null +++ b/winbuild/libssh2-vs2015.patch @@ -0,0 +1,10 @@ +--- win32/libssh2_config.h.orig 2014-12-04 13:43:57.000000000 -0800 ++++ win32/libssh2_config.h 2016-01-02 21:56:50.468363200 -0800 +@@ -24,7 +24,6 @@ + #define HAVE_SELECT + + #ifdef _MSC_VER +-#define snprintf _snprintf + #if _MSC_VER < 1500 + #define vsnprintf _vsnprintf + #endif diff --git a/winbuild/nghttp_cmake.py b/winbuild/nghttp_cmake.py new file mode 100644 index 0000000..fe57934 --- /dev/null +++ b/winbuild/nghttp_cmake.py @@ -0,0 +1,121 @@ +import shutil +from .builder import * + +class Nghttp2Builder(StandardBuilder): + CMAKE_GENERATORS = { + # Thanks cmake for requiring both version number and year, + # necessitating this additional map + 'vc9': 'Visual Studio 9 2008', + 'vc14': 'Visual Studio 14 2015', + } + + def build(self): + nghttp2_dir = self.standard_fetch_extract( + 'https://github.com/nghttp2/nghttp2/releases/download/v%(my_version)s/nghttp2-%(my_version)s.tar.gz') + + # nghttp2 uses stdint.h which msvc9 does not ship. + # Amazingly, nghttp2 can seemingly build successfully without this + # file existing, but libcurl build subsequently fails + # when it tries to include stdint.h. + # Well, the reason why nghttp2 builds correctly is because it is built + # with the wrong compiler - msvc14 when 9 and 14 are both installed. + # nghttp2 build with msvc9 does fail without stdint.h existing. + if self.bconf.vc_version == 'vc9': + # https://stackoverflow.com/questions/126279/c99-stdint-h-header-and-ms-visual-studio + fetch('https://raw.githubusercontent.com/mattn/gntp-send/master/include/msinttypes/stdint.h') + with in_dir(nghttp2_dir): + shutil.copy('../stdint.h', 'lib/includes/stdint.h') + + with in_dir(nghttp2_dir): + generator = self.CMAKE_GENERATORS[self.bconf.vc_version] + with self.execute_batch() as b: + # Workaround for VCTargetsPath issue that looks like this: + # C:\dev\build-pycurl\archives\nghttp2-1.40.0-vc14-32\CMakeFiles\3.16.3\VCTargetsPath.vcxproj(14,2): error MSB4019: The imported project "C:\Microsoft.Cpp.Default.props" was not found. Confirm that the path in the declaration is correct, and that the file exists on disk. + # + # Many solutions proposed on SO, including: + # https://stackoverflow.com/questions/41695251/c-microsoft-cpp-default-props-was-not-found + # https://stackoverflow.com/questions/16092169/why-does-msbuild-look-in-c-for-microsoft-cpp-default-props-instead-of-c-progr + if not os.path.exists(self.bconf.vc_targets_path): + raise ValueError("VCTargetsPath does not exist: %s" % self.bconf.vc_targets_path) + b.add('SET VCTargetsPath=%s' % self.bconf.vc_targets_path) + + # The msbuild.exe in path could be v4.0 from .net sdk, whereas the + # vctargetspath ends up referencing the msbuild from visual studio... + # Put the visual studio msbuild into the path first. + if self.bconf.bitness == 64: + msbuild_bin_path = os.path.join(self.bconf.msbuild_bin_path, 'amd64') + else: + msbuild_bin_path = self.bconf.msbuild_bin_path + b.add("set path=%s;%%path%%" % msbuild_bin_path) + + # When performing 64-bit build, ucrtd.lib is not in library path for whatever reason. Sigh. + # Superseded by https://stackoverflow.com/questions/56145118/cmake-cannot-open-ucrtd-lib instructions below. + if self.bconf.bitness == 64 and False: + windows_sdk_lib_path = glob_first("c:\\Program Files (x86)\\Windows Kits\\10\\Lib\\*\\ucrt\\x64") + b.add('set lib=%s;%%lib%%' % windows_sdk_lib_path) + + parts = [ + '"%s"' % self.bconf.cmake_path, + # I don't know if this does anything, build type/config + # must be specified with --build option below. + '-DCMAKE_BUILD_TYPE=Release', + # This configures libnghttp2 only which is what we want. + # However, configure step still complains about all of the + # missing dependencies for nghttp2 server. + # And there is no indication whatsoever from configure step + # that this option is enabled, or that the missing + # dependency complaints can be ignored. + '-DENABLE_LIB_ONLY=1', + # This is required to get a static library built. + # However, even with this turned on there is still a DLL + # built - without an import library for it. + '-DENABLE_STATIC_LIB=1', + # And cmake ignores all visual studio environment variables + # and uses the newest compiler by default, which is great + # if one doesn't care what compiler their code is compiled with. + # https://stackoverflow.com/questions/6430251/what-is-the-default-generator-for-cmake-in-windows + '-G', '"%s"' % generator, + ] + + # Cmake also couldn't care less about the bitness I have configured in the + # environment since it ignores the environment entirely. + # Educate it on the required bitness by hand. + # https://stackoverflow.com/questions/28350214/how-to-build-x86-and-or-x64-on-windows-from-command-line-with-cmake#28370892 + # + # New strategy: + # https://cmake.org/cmake/help/v3.14/generator/Visual%20Studio%2014%202015.html + if self.bconf.bitness == 64 and False: + parts += ['-A', 'x64'] + + # And it does its own windows sdk selection, apparently, and botches it. + # https://stackoverflow.com/questions/56145118/cmake-cannot-open-ucrtd-lib + # TODO figure out which version is needed here, 8.1 or 10.0 or 10.0.10240.0 + parts.append('-DCMAKE_SYSTEM_VERSION=8.1') + + b.add('%s .' % ' '.join(parts)) + b.add(' '.join([ + '"%s"' % self.bconf.cmake_path, + '--build', '.', + # this is what produces a release build + '--config', 'Release', + # this builds the static library. + # without this option cmake configures itself to be capable + # of building a static library but sometimes builds a DLL + # and sometimes builds a static library + # depending on compiler in use (vc9/vc14) or, possibly, + # phase of the moon. + '--target', 'nghttp2_static', + ])) + + # assemble dist + b.add('mkdir dist dist\\include dist\\include\\nghttp2 dist\\lib') + b.add('cp lib/Release/*.lib dist/lib') + b.add('cp lib/includes/nghttp2/*.h dist/include/nghttp2') + if self.bconf.vc_version == 'vc9': + # stdint.h + b.add('cp lib/includes/*.h dist/include') + + # libcurl expects nghttp2_static.lib apparently, and depending on nghttp2 version/configuration(?) + # the library name is sometimes nghttp2.lib + if not os.path.exists('lib/Release/nghttp2_static.lib'): + shutil.copy('lib/Release/nghttp2.lib', 'lib/Release/nghttp2_static.lib') diff --git a/winbuild/nghttp_gmake.py b/winbuild/nghttp_gmake.py new file mode 100644 index 0000000..5dba20c --- /dev/null +++ b/winbuild/nghttp_gmake.py @@ -0,0 +1,22 @@ +import shutil +from .builder import * + +class Nghttp2Builder(StandardBuilder): + def build(self): + nghttp2_dir = self.standard_fetch_extract( + 'https://github.com/nghttp2/nghttp2/releases/download/v%(my_version)s/nghttp2-%(my_version)s.tar.gz') + + with in_dir(os.path.join(nghttp2_dir, 'lib')): + with self.execute_batch() as b: + + b.add('"%s" -f Makefile.msvc' % self.bconf.gmake_path) + + # assemble dist + b.add('mkdir ..\\dist ..\\dist\\include ..\\dist\\include\\nghttp2 ..\\dist\\lib') + b.add('cp msvc_obj/*.lib ../dist/lib') + b.add('cp includes/nghttp2/*.h ../dist/include/nghttp2') + + # libcurl expects nghttp2_static.lib apparently, the makefile + # gives a different name to the static library + if not os.path.exists('../dist/lib/nghttp2_static.lib'): + shutil.copy('../dist/lib/nghttp2-static.lib', '../dist/lib/nghttp2_static.lib') diff --git a/winbuild/openssl-fix-crt-1.0.2.patch b/winbuild/openssl-fix-crt-1.0.2.patch new file mode 100644 index 0000000..d0ce166 --- /dev/null +++ b/winbuild/openssl-fix-crt-1.0.2.patch @@ -0,0 +1,36 @@ +--- util/pl/VC-32.pl.orig 2015-12-03 06:04:23.000000000 -0800 ++++ util/pl/VC-32.pl 2016-01-01 23:56:32.542632200 -0800 +@@ -45,7 +45,7 @@ + # considered safe to ignore. + # + $base_cflags= " $mf_cflag"; +- my $f = $shlib || $fips ?' /MD':' /MT'; ++ my $f = $shlib || $fips ?' /MD':' /MD'; + $opt_cflags=$f.' /Ox'; + $dbg_cflags=$f.'d /Od -DDEBUG -D_DEBUG'; + $lflags="/nologo /subsystem:console /opt:ref"; +@@ -119,7 +119,7 @@ + $base_cflags.=' -I$(WCECOMPAT)/include' if (defined($ENV{'WCECOMPAT'})); + $base_cflags.=' -I$(PORTSDK_LIBPATH)/../../include' if (defined($ENV{'PORTSDK_LIBPATH'})); + if (`$cc 2>&1` =~ /Version ([0-9]+)\./ && $1>=14) { +- $base_cflags.=$shlib?' /MD':' /MT'; ++ $base_cflags.=$shlib?' /MD':' /MD'; + } else { + $base_cflags.=' /MC'; + } +@@ -130,13 +130,13 @@ + else # Win32 + { + $base_cflags= " $mf_cflag"; +- my $f = $shlib || $fips ?' /MD':' /MT'; ++ my $f = $shlib || $fips ?' /MD':' /MD'; + $ff = "/fixed"; + $opt_cflags=$f.' /Ox /O2 /Ob2'; + $dbg_cflags=$f.'d /Od -DDEBUG -D_DEBUG'; + $lflags="/nologo /subsystem:console /opt:ref"; + } +-$lib_cflag='/Zl' if (!$shlib); # remove /DEFAULTLIBs from static lib ++#$lib_cflag='/Zl' if (!$shlib); # remove /DEFAULTLIBs from static lib + $mlflags=''; + + $out_def ="out32"; $out_def.="dll" if ($shlib); diff --git a/winbuild/openssl-fix-crt-1.1.0.patch b/winbuild/openssl-fix-crt-1.1.0.patch new file mode 100644 index 0000000..d04b15d --- /dev/null +++ b/winbuild/openssl-fix-crt-1.1.0.patch @@ -0,0 +1,29 @@ +--- Configurations/10-main.conf.orig 2016-11-10 06:03:43.000000000 -0800 ++++ Configurations/10-main.conf 2016-12-15 20:18:47.576426000 -0800 +@@ -1291,7 +1291,7 @@ + ($disabled{shared} ? "" : "/MD") + ." /O2"; + })), +- lib_cflags => add(sub { $disabled{shared} ? "/MT /Zl" : () }), ++ lib_cflags => add(sub { $disabled{shared} ? "/MD" : () }), + # Following might/should appears controversial, i.e. defining + # /MDd without evaluating $disabled{shared}. It works in + # non-shared build because static library is compiled with /Zl +@@ -1304,7 +1304,7 @@ + # prefer [non-debug] openssl.exe to be free from Micorosoft RTL + # redistributable. + bin_cflags => add(picker(debug => "/MDd", +- release => sub { $disabled{shared} ? "/MT" : () }, ++ release => sub { $disabled{shared} ? "/MD" : () }, + )), + bin_lflags => add("/subsystem:console /opt:ref"), + ex_libs => add(sub { +@@ -1385,7 +1385,7 @@ + sub { defined($ENV{'PORTSDK_LIBPATH'}) + ? '-I$(PORTSDK_LIBPATH)/../../include' : (); }, + sub { `cl 2>&1` =~ /Version ([0-9]+)\./ && $1>=14 +- ? ($disabled{shared} ? " /MT" : " /MD") ++ ? ($disabled{shared} ? " /MD" : " /MD") + : " /MC"; }), + debug => "/Od -DDEBUG -D_DEBUG", + release => "/O1i"), diff --git a/winbuild/openssl-fix-crt-1.1.1.patch b/winbuild/openssl-fix-crt-1.1.1.patch new file mode 100644 index 0000000..a1db4d9 --- /dev/null +++ b/winbuild/openssl-fix-crt-1.1.1.patch @@ -0,0 +1,29 @@ +--- Configurations/10-main.conf.orig 2019-09-10 09:13:07.000000000 -0400 ++++ Configurations/10-main.conf 2020-01-27 13:16:41.992273600 -0500 +@@ -1252,7 +1252,7 @@ + })), + defines => add(picker(default => [], # works as type cast + debug => [ "DEBUG", "_DEBUG" ])), +- lib_cflags => add(sub { $disabled{shared} ? "/MT /Zl" : () }), ++ lib_cflags => add(sub { $disabled{shared} ? "/MD" : () }), + # Following might/should appears controversial, i.e. defining + # /MDd without evaluating $disabled{shared}. It works in + # non-shared build because static library is compiled with /Zl +@@ -1265,7 +1265,7 @@ + # prefer [non-debug] openssl.exe to be free from Micorosoft RTL + # redistributable. + bin_cflags => add(picker(debug => "/MDd", +- release => sub { $disabled{shared} ? "/MT" : () }, ++ release => sub { $disabled{shared} ? "/MD" : () }, + )), + bin_lflags => add("/subsystem:console /opt:ref"), + ex_libs => add(sub { +@@ -1335,7 +1335,7 @@ + combine('/GF /Gy', + sub { vc_wince_info()->{cflags}; }, + sub { `cl 2>&1` =~ /Version ([0-9]+)\./ && $1>=14 +- ? ($disabled{shared} ? " /MT" : " /MD") ++ ? ($disabled{shared} ? " /MD" : " /MD") + : " /MC"; }), + cppflags => sub { vc_wince_info()->{cppflags}; }, + lib_defines => add("NO_CHMOD", "OPENSSL_SMALL_FOOTPRINT"), diff --git a/winbuild/openssl.py b/winbuild/openssl.py new file mode 100644 index 0000000..cc8bba4 --- /dev/null +++ b/winbuild/openssl.py @@ -0,0 +1,76 @@ +import os.path +from .utils import * +from .builder import * + +class OpensslBuilder(StandardBuilder): + def build(self): + # another openssl gem: + # nasm output is redirected to NUL which ends up creating a file named NUL. + # however being a reserved file name this file is not deletable by + # ordinary tools. + nul_file = "openssl-%s-%s\\NUL" % (self.bconf.openssl_version, self.bconf.vc_tag) + check_call(['rm', '-f', nul_file]) + openssl_dir = self.standard_fetch_extract( + 'https://www.openssl.org/source/openssl-%(my_version)s.tar.gz') + with in_dir(openssl_dir): + with self.execute_batch() as b: + if self.bconf.openssl_version_tuple < (1, 1): + # openssl 1.0.2 + b.add("patch -p0 < %s" % + require_file_exists(os.path.join(config.winbuild_patch_root, 'openssl-fix-crt-1.0.2.patch'))) + elif self.bconf.openssl_version_tuple < (1, 1, 1): + # openssl 1.1.0 + b.add("patch -p0 < %s" % + require_file_exists(os.path.join(config.winbuild_patch_root, 'openssl-fix-crt-1.1.0.patch'))) + else: + # openssl 1.1.1 + b.add("patch -p0 < %s" % + require_file_exists(os.path.join(config.winbuild_patch_root, 'openssl-fix-crt-1.1.1.patch'))) + if self.bconf.bitness == 64: + target = 'VC-WIN64A' + batch_file = 'do_win64a' + else: + target = 'VC-WIN32' + batch_file = 'do_nasm' + + # msysgit perl has trouble with backslashes used in + # win64 assembly things in openssl 1.0.2 + # and in x86 assembly as well in openssl 1.1.0; + # use ActiveState Perl + if not os.path.exists(config.activestate_perl_bin_path): + raise ValueError('activestate_perl_bin_path refers to a nonexisting path') + if not os.path.exists(os.path.join(config.activestate_perl_bin_path, 'perl.exe')): + raise ValueError('No perl binary in activestate_perl_bin_path') + b.add("set path=%s;%%path%%" % config.activestate_perl_bin_path) + b.add("perl -v") + + openssl_prefix = os.path.join(os.path.realpath('.'), 'build') + # Do not want compression: + # https://en.wikipedia.org/wiki/CRIME + extras = ['no-comp', 'no-unit-test', 'no-tests', 'no-external-tests'] + if config.openssl_version_tuple >= (1, 1): + # openssl 1.1.0 + # in 1.1.0 the static/shared selection is handled by + # invoking the right makefile + extras += ['no-shared'] + + # looks like openssl 1.1.0c does not derive + # --openssldir from --prefix, like its Configure claims, + # and like 1.0.2 does; provide a relative openssl dir + # manually + extras += ['--openssldir=ssl'] + b.add("perl Configure %s %s --prefix=%s" % (target, ' '.join(extras), openssl_prefix)) + + if config.openssl_version_tuple < (1, 1): + # openssl 1.0.2 + b.add("call ms\\%s" % batch_file) + b.add("nmake -f ms\\nt.mak") + b.add("nmake -f ms\\nt.mak install") + else: + # openssl 1.1.0 + b.add("nmake") + b.add("nmake install") + + # assemble dist + b.add('mkdir dist') + b.add('cp -r build/include build/lib dist') diff --git a/winbuild/pycurl.py b/winbuild/pycurl.py new file mode 100644 index 0000000..975de77 --- /dev/null +++ b/winbuild/pycurl.py @@ -0,0 +1,136 @@ +import os.path, shutil, zipfile +from .builder import * +from .utils import * +from .curl import * + +class PycurlBuilder(Builder): + def __init__(self, **kwargs): + self.python_release = kwargs.pop('python_release') + super(PycurlBuilder, self).__init__(**kwargs) + # vc_version is specified externally for bconf/BuildConfig + assert self.bconf.vc_version == PYTHON_VC_VERSIONS[self.python_release] + + @property + def python_path(self): + if self.bconf.build_wheels: + python_path = os.path.join(self.bconf.archives_path, 'venv-%s-%s' % (self.python_release, self.bconf.bitness), 'scripts', 'python') + else: + python_path = PythonBinary(self.python_release, self.bconf.bitness).executable_path + return python_path + + @property + def platform_indicator(self): + platform_indicators = {32: 'win32', 64: 'win-amd64'} + return platform_indicators[self.bconf.bitness] + + def build(self, targets): + libcurl_builder = LibcurlBuilder(bconf=self.bconf) + libcurl_dir = os.path.join(os.path.abspath(libcurl_builder.output_dir_path), 'dist') + dll_paths = libcurl_builder.dll_paths + if self.bconf.use_zlib: + zlib_builder = ZlibBuilder(bconf=self.bconf) + dll_paths += zlib_builder.dll_paths + dll_paths = [os.path.abspath(dll_path) for dll_path in dll_paths] + with in_dir(self.build_dir_name): + dest_lib_path = 'build/lib.%s-%s' % (self.platform_indicator, + self.python_release) + # exists for building additional targets for the same python version + mkdir_p(dest_lib_path) + if self.use_dlls: + for dll_path in dll_paths: + shutil.copy(dll_path, dest_lib_path) + with self.execute_batch() as b: + b.add("%s setup.py docstrings" % (self.python_path,)) + if self.use_dlls: + libcurl_arg = '--use-libcurl-dll' + else: + libcurl_arg = '--libcurl-lib-name=libcurl_a.lib' + if self.bconf.use_openssl: + libcurl_arg += ' --with-openssl' + if self.bconf.openssl_version_tuple >= (1, 1): + libcurl_arg += ' --openssl-lib-name=""' + openssl_builder = OpensslBuilder(bconf=self.bconf) + b.add("set include=%%include%%;%s" % openssl_builder.include_path) + b.add("set lib=%%lib%%;%s" % openssl_builder.lib_path) + #if build_wheels: + #b.add("call %s" % os.path.join('..', 'venv-%s-%s' % (self.python_release, self.bconf.bitness), 'Scripts', 'activate')) + if self.bconf.build_wheels: + targets = targets + ['bdist_wheel'] + if self.bconf.libcurl_version_tuple >= (7, 60, 0): + # As of 7.60.0 libcurl does not include its dependencies into + # its static libraries. + # libcurl_a.lib in 7.59.0 is 30 mb. + # libcurl_a.lib in 7.60.0 is 2 mb. + # https://github.com/curl/curl/pull/2474 is most likely culprit. + # As a result we need to specify all of the libraries that + # libcurl depends on here, plus the library paths, + # plus even windows standard libraries for good measure. + if self.bconf.use_zlib: + zlib_builder = ZlibBuilder(bconf=self.bconf) + libcurl_arg += ' --link-arg=/LIBPATH:%s' % zlib_builder.lib_path + libcurl_arg += ' --link-arg=zlib.lib' + if self.bconf.use_openssl: + openssl_builder = OpensslBuilder(bconf=self.bconf) + libcurl_arg += ' --link-arg=/LIBPATH:%s' % openssl_builder.lib_path + # openssl 1.1 + libcurl_arg += ' --link-arg=libcrypto.lib' + libcurl_arg += ' --link-arg=libssl.lib' + libcurl_arg += ' --link-arg=crypt32.lib' + libcurl_arg += ' --link-arg=advapi32.lib' + if self.bconf.use_cares: + cares_builder = CaresBuilder(bconf=self.bconf) + libcurl_arg += ' --link-arg=/LIBPATH:%s' % cares_builder.lib_path + libcurl_arg += ' --link-arg=libcares.lib' + if self.bconf.use_libssh2: + libssh2_builder = Libssh2Builder(bconf=self.bconf) + libcurl_arg += ' --link-arg=/LIBPATH:%s' % libssh2_builder.lib_path + libcurl_arg += ' --link-arg=libssh2.lib' + if self.bconf.use_nghttp2: + nghttp2_builder = Nghttp2Builder(bconf=self.bconf) + libcurl_arg += ' --link-arg=/LIBPATH:%s' % nghttp2_builder.lib_path + libcurl_arg += ' --link-arg=nghttp2_static.lib' + if self.bconf.vc_version == 'vc9': + # this is for normaliz.lib + libcurl_builder = LibcurlBuilder(bconf=self.bconf) + libcurl_arg += ' --link-arg=/LIBPATH:%s' % libcurl_builder.lib_path + # We always use normaliz.lib, but it may come from + # "standard" msvc location or from libcurl's lib dir for msvc9 + libcurl_arg += ' --link-arg=normaliz.lib' + libcurl_arg += ' --link-arg=user32.lib' + b.add("%s setup.py %s --curl-dir=%s %s" % ( + self.python_path, ' '.join(targets), libcurl_dir, libcurl_arg)) + # Fixing of bizarre paths in created zip archives, + # no longer relevant because we only keep wheels + if False and 'bdist' in targets: + zip_basename_orig = 'pycurl-%s.%s.zip' % ( + self.bconf.pycurl_version, self.platform_indicator) + zip_basename_new = 'pycurl-%s.%s-py%s.zip' % ( + self.bconf.pycurl_version, self.platform_indicator, self.python_release) + with zipfile.ZipFile('dist/%s' % zip_basename_orig, 'r') as src_zip: + with zipfile.ZipFile('dist/%s' % zip_basename_new, 'w') as dest_zip: + for name in src_zip.namelist(): + parts = name.split('/') + while True: + popped = parts.pop(0) + if popped == 'python%s' % self.python_release.dotless or popped.startswith('venv-'): + break + assert len(parts) > 0 + new_name = '/'.join(parts) + print('Recompressing %s -> %s' % (name, new_name)) + + member = src_zip.open(name) + dest_zip.writestr(new_name, member.read(), zipfile.ZIP_DEFLATED) + + @property + def build_dir_name(self): + return 'pycurl-%s-py%s-%s' % (self.bconf.pycurl_version, self.python_release, self.bconf.vc_tag) + + def prepare_tree(self): + #fetch('https://dl.bintray.com/pycurl/pycurl/pycurl-%s.tar.gz' % pycurl_version) + if os.path.exists(self.build_dir_name): + # shutil.rmtree is incapable of removing .git directory because it contains + # files marked read-only (tested on python 2.7 and 3.6) + #shutil.rmtree('pycurl-%s' % config.pycurl_version) + rm_rf(self.bconf, self.build_dir_name) + #check_call([tar_path, 'xf', 'pycurl-%s.tar.gz' % pycurl_version]) + shutil.copytree('c:/dev/pycurl', self.build_dir_name) diff --git a/winbuild/pythons.py b/winbuild/pythons.py new file mode 100644 index 0000000..1533f27 --- /dev/null +++ b/winbuild/pythons.py @@ -0,0 +1,20 @@ + +class PythonRelease(str): + @property + def dotless(self): + return self.replace('.', '') + +class PythonVersion(str): + @property + def release(self): + return PythonRelease('.'.join(self.split('.')[:2])) + +class PythonBinary(object): + def __init__(self, python_release, bitness): + self.python_release = python_release + self.bitness = bitness + + def executable_path(self, config): + return config.python_path_template % dict( + python_release=self.python_release.dotless, + bitness=self.bitness) diff --git a/winbuild/ssh.py b/winbuild/ssh.py new file mode 100644 index 0000000..64d46e6 --- /dev/null +++ b/winbuild/ssh.py @@ -0,0 +1,40 @@ +from .utils import * +from .builder import * + +class Libssh2Builder(StandardBuilder): + def build(self): + libssh2_dir = self.standard_fetch_extract( + 'http://www.libssh2.org/download/libssh2-%(my_version)s.tar.gz') + with in_dir(libssh2_dir): + with self.execute_batch() as b: + if self.bconf.libssh2_version_tuple < (1, 8, 0) and self.bconf.vc_version == 'vc14': + b.add("patch -p0 < %s" % + require_file_exists(os.path.join(config.winbuild_patch_root, 'libssh2-vs2015.patch'))) + zlib_builder = ZlibBuilder(bconf=self.bconf) + openssl_builder = OpensslBuilder(bconf=self.bconf) + vars = ''' +OPENSSLINC=%(openssl_include_path)s +OPENSSLLIB=%(openssl_lib_path)s +ZLIBINC=%(zlib_include_path)s +ZLIBLIB=%(zlib_lib_path)s +WITH_ZLIB=1 +BUILD_STATIC_LIB=1 + ''' % dict( + openssl_include_path=openssl_builder.include_path, + openssl_lib_path=openssl_builder.lib_path, + zlib_include_path=zlib_builder.include_path, + zlib_lib_path=zlib_builder.lib_path, + ) + with open('win32/config.mk', 'r+') as cf: + contents = cf.read() + cf.seek(0) + cf.write(vars) + cf.write(contents) + b.add("nmake -f NMakefile") + # libcurl loves its _a suffixes on static library names + b.add("cp Release\\src\\libssh2.lib Release\\src\\libssh2_a.lib") + + # assemble dist + b.add('mkdir dist dist\\include dist\\lib') + b.add('cp Release/src/*.lib dist/lib') + b.add('cp -r include dist') diff --git a/winbuild/tools.py b/winbuild/tools.py new file mode 100644 index 0000000..90c7921 --- /dev/null +++ b/winbuild/tools.py @@ -0,0 +1,11 @@ +from .config import * + +def short_python_versions(python_versions): + return ['.'.join(python_version.split('.')[:2]) + for python_version in python_versions] + +def needed_vc_versions(config, python_versions): + return [vc_version for vc_version in config.vc_paths.keys() + if vc_version in [ + PYTHON_VC_VERSIONS[short_python_version] + for short_python_version in short_python_versions(python_versions)]] diff --git a/winbuild/utils.py b/winbuild/utils.py new file mode 100644 index 0000000..0e043d8 --- /dev/null +++ b/winbuild/utils.py @@ -0,0 +1,114 @@ +import os.path, subprocess, sys, os, glob, re, contextlib, shutil +try: + from urllib.request import urlopen +except ImportError: + from urllib import urlopen + +# https://stackoverflow.com/questions/35569042/python-3-ssl-certificate-verify-failed +import ssl +try: + ssl._create_default_https_context = ssl._create_unverified_context +except AttributeError: + pass + +# Given a list of paths, return the first path that exists. +def select_existing_path(paths): + if isinstance(paths, list) or isinstance(paths, tuple): + for path in paths: + if os.path.exists(path): + return path + return paths[0] + else: + return paths + +# Find the given binary by its short name in the specified +# list of directories. +def find_in_paths(binary, paths): + for path in paths: + if os.path.exists(os.path.join(path, binary)) or os.path.exists(os.path.join(path, binary + '.exe')): + return os.path.join(path, binary) + raise Exception('Could not find %s' % binary) + +# Executes the specified command, raising an exception if execution failed. +def check_call(cmd): + try: + subprocess.check_call(cmd) + except Exception as e: + raise Exception('Failed to execute ' + str(cmd) + ': ' + str(type(e)) + ': ' +str(e)) + +def mkdir_p(path): + if not os.path.exists(path): + os.makedirs(path) + +def rm_rf(config, path): + check_call([config.rm_path, '-rf', path]) + +def cp_r(config, src, dest): + check_call([config.cp_path, '-r', src, dest]) + +# Retrieves the file at the given url, saving it in the specified local filesystem path. +# Does nothing if the local path already exists. +def fetch(url, archive=None): + if archive is None: + archive = os.path.basename(url) + if not os.path.exists(archive): + sys.stdout.write("Fetching %s\n" % url) + sys.stdout.flush() + io = urlopen(url) + tmp_path = os.path.join(os.path.dirname(archive), + '.%s.part' % os.path.basename(archive)) + with open(tmp_path, 'wb') as f: + while True: + chunk = io.read(65536) + if len(chunk) == 0: + break + f.write(chunk) + os.rename(tmp_path, archive) + +# Verifies that provided path exists, and returns it. +def require_file_exists(path): + if not os.path.exists(path): + raise Exception('Path %s does not exist!' % path) + return path + +# Converts forward slashes to backslashes. +def fix_slashes(path): + return path.replace('/', '\\') + +# Returns the first path matching the pattern, where pattern is anything the +# standard library glob module recognizes plus {a,b,c} alterations. +# Raises an exception if no paths matched the pattern. +def glob_first(pattern, selector=None): + # python's glob does not support {} + final_patterns = [] + pattern_queue = [pattern] + while pattern_queue: + pattern = pattern_queue.pop() + if re.search(r'\{.*}', pattern): + match = re.match(r'(.*){(.*?)}(.*)', pattern, re.S) + for variant in match.group(2).split(','): + pattern_queue.append(match.group(1) + variant + match.group(3)) + else: + final_patterns.append(pattern) + for pattern in final_patterns: + paths = glob.glob(pattern) + if paths: + if selector: + return selector(paths) + else: + return paths[0] + raise Exception("Not found: %s" % pattern) + +@contextlib.contextmanager +def in_dir(dir): + old_cwd = os.getcwd() + try: + os.chdir(dir) + yield + finally: + os.chdir(old_cwd) + +def untar(config, basename): + if os.path.exists(basename): + shutil.rmtree(basename) + check_call([config.tar_path, 'xf', '%s.tar.gz' % basename]) diff --git a/winbuild/vcvars-vc14-32.sh b/winbuild/vcvars-vc14-32.sh new file mode 100644 index 0000000..6831b4f --- /dev/null +++ b/winbuild/vcvars-vc14-32.sh @@ -0,0 +1,25 @@ +# Courtesy of libiconv 1.15 + +# Set environment variables for using MSVC 14, +# for creating native 32-bit Windows executables. + +# Windows C library headers and libraries. +WindowsCrtIncludeDir='C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt' +WindowsCrtLibDir='C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\' +INCLUDE="${WindowsCrtIncludeDir};$INCLUDE" +LIB="${WindowsCrtLibDir}x86;$LIB" + +# Windows API headers and libraries. +WindowsSdkIncludeDir='C:\Program Files (x86)\Windows Kits\8.1\Include\' +WindowsSdkLibDir='C:\Program Files (x86)\Windows Kits\8.1\Lib\winv6.3\um\' +INCLUDE="${WindowsSdkIncludeDir}um;${WindowsSdkIncludeDir}shared;$INCLUDE" +LIB="${WindowsSdkLibDir}x86;$LIB" + +# Visual C++ tools, headers and libraries. +VSINSTALLDIR='C:\Program Files (x86)\Microsoft Visual Studio 14.0' +VCINSTALLDIR="${VSINSTALLDIR}"'\VC' +PATH=`cygpath -u "${VCINSTALLDIR}"`/bin:"$PATH" +INCLUDE="${VCINSTALLDIR}"'\include;'"${INCLUDE}" +LIB="${VCINSTALLDIR}"'\lib;'"${LIB}" + +export INCLUDE LIB diff --git a/winbuild/vcvars-vc14-64.sh b/winbuild/vcvars-vc14-64.sh new file mode 100644 index 0000000..502fa76 --- /dev/null +++ b/winbuild/vcvars-vc14-64.sh @@ -0,0 +1,25 @@ +# Courtesy of libiconv 1.15 + +# Set environment variables for using MSVC 14, +# for creating native 64-bit Windows executables. + +# Windows C library headers and libraries. +WindowsCrtIncludeDir='C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt' +WindowsCrtLibDir='C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\' +INCLUDE="${WindowsCrtIncludeDir};$INCLUDE" +LIB="${WindowsCrtLibDir}x64;$LIB" + +# Windows API headers and libraries. +WindowsSdkIncludeDir='C:\Program Files (x86)\Windows Kits\8.1\Include\' +WindowsSdkLibDir='C:\Program Files (x86)\Windows Kits\8.1\Lib\winv6.3\um\' +INCLUDE="${WindowsSdkIncludeDir}um;${WindowsSdkIncludeDir}shared;$INCLUDE" +LIB="${WindowsSdkLibDir}x64;$LIB" + +# Visual C++ tools, headers and libraries. +VSINSTALLDIR='C:\Program Files (x86)\Microsoft Visual Studio 14.0' +VCINSTALLDIR="${VSINSTALLDIR}"'\VC' +PATH=`cygpath -u "${VCINSTALLDIR}"`/bin/amd64:"$PATH" +INCLUDE="${VCINSTALLDIR}"'\include;'"${INCLUDE}" +LIB="${VCINSTALLDIR}"'\lib\amd64;'"${LIB}" + +export INCLUDE LIB \ No newline at end of file diff --git a/winbuild/zlib.py b/winbuild/zlib.py new file mode 100644 index 0000000..299a4c3 --- /dev/null +++ b/winbuild/zlib.py @@ -0,0 +1,25 @@ +import os.path +from .utils import * +from .builder import * + +class ZlibBuilder(StandardBuilder): + def build(self): + zlib_dir = self.standard_fetch_extract( + 'http://downloads.sourceforge.net/project/libpng/zlib/%(my_version)s/zlib-%(my_version)s.tar.gz') + with in_dir(zlib_dir): + with self.execute_batch() as b: + b.add("nmake /f win32/Makefile.msc") + # libcurl loves its _a suffixes on static library names + b.add("cp zlib.lib zlib_a.lib") + + # assemble dist + b.add('mkdir dist dist\\include dist\\lib dist\\bin') + b.add('cp *.lib *.exp dist/lib') + b.add('cp *.dll dist/bin') + b.add('cp *.h dist/include') + + @property + def dll_paths(self): + return [ + os.path.join(self.bin_path, 'zlib1.dll'), + ] -- 2.34.1