From 1a3cef77d465ea1a53e5c80f869908f359e98c5e Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Mon, 20 Jul 2020 05:48:57 +0800 Subject: [PATCH] python3: Update to 3.8.4, refresh/rework patches, backport patches This version includes fixes for: * CVE-2020-14422: Hash collisions in IPv4Interface and IPv6Interface * CVE-2020-15523: Python uses invalid DLL path after calling Py_SetPath on Windows This version also includes support for OpenSSL 1.1.x builds that use 'no-deprecated' and '--api=1.1.0'[1], and so this removes the previous OpenSSL-related patches. This also backports fixes for security issues, including: * CVE-2019-20907: Infinite loop in the tarfile module This also updates the setuptools and pip packages to 47.1.0 and 20.1.1, respectively. [1]: https://github.com/python/cpython/pull/20566 Signed-off-by: Jeffery To --- lang/python/python3-version.mk | 10 +- lang/python/python3/Makefile | 6 +- .../patches-pip/001-pep517-pyc-fix.patch | 25 +- .../patches-setuptools/001-reproducible.patch | 2 +- .../002-sorted-requires.patch | 8 +- .../003-PKG-INFO-output-reproducible.patch | 2 +- .../patches-setuptools/004-site-patch.patch | 2 +- .../020-ssl-module-emulate-tls-methods.patch | 218 ------------------ .../patches/021-openssl-deprecated.patch | 63 ----- ...npickling-invalid-NEWOBJ_EX-GH-21458.patch | 111 +++++++++ ...the-tarfile-module-GH-21454-GH-21483.patch | 62 +++++ ...r-injection-in-http-methods-GH-18485.patch | 99 ++++++++ 12 files changed, 304 insertions(+), 304 deletions(-) delete mode 100644 lang/python/python3/patches/020-ssl-module-emulate-tls-methods.patch delete mode 100644 lang/python/python3/patches/021-openssl-deprecated.patch create mode 100644 lang/python/python3/patches/025-bpo-41288-Fix-a-crash-in-unpickling-invalid-NEWOBJ_EX-GH-21458.patch create mode 100644 lang/python/python3/patches/026-bpo-39017-Avoid-infinite-loop-in-the-tarfile-module-GH-21454-GH-21483.patch create mode 100644 lang/python/python3/patches/027-bpo-39603-Prevent-header-injection-in-http-methods-GH-18485.patch diff --git a/lang/python/python3-version.mk b/lang/python/python3-version.mk index e8f8fba052..e85effed49 100644 --- a/lang/python/python3-version.mk +++ b/lang/python/python3-version.mk @@ -8,12 +8,12 @@ # Note: keep in sync with setuptools & pip PYTHON3_VERSION_MAJOR:=3 PYTHON3_VERSION_MINOR:=8 -PYTHON3_VERSION_MICRO:=3 +PYTHON3_VERSION_MICRO:=4 PYTHON3_VERSION:=$(PYTHON3_VERSION_MAJOR).$(PYTHON3_VERSION_MINOR) -PYTHON3_SETUPTOOLS_PKG_RELEASE:=3 -PYTHON3_PIP_PKG_RELEASE:=4 +PYTHON3_SETUPTOOLS_PKG_RELEASE:=1 +PYTHON3_PIP_PKG_RELEASE:=1 -PYTHON3_SETUPTOOLS_VERSION:=41.2.0 -PYTHON3_PIP_VERSION:=19.2.3 +PYTHON3_SETUPTOOLS_VERSION:=47.1.0 +PYTHON3_PIP_VERSION:=20.1.1 diff --git a/lang/python/python3/Makefile b/lang/python/python3/Makefile index e3769e5018..4b853d27f3 100644 --- a/lang/python/python3/Makefile +++ b/lang/python/python3/Makefile @@ -11,12 +11,12 @@ include $(TOPDIR)/rules.mk include ../python3-version.mk PKG_NAME:=python3 -PKG_RELEASE:=3 +PKG_RELEASE:=1 PKG_VERSION:=$(PYTHON3_VERSION).$(PYTHON3_VERSION_MICRO) PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION) -PKG_HASH:=dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864 +PKG_HASH:=5f41968a95afe9bc12192d7e6861aab31e80a46c46fa59d3d837def6a4cd4d37 PKG_MAINTAINER:=Alexandru Ardelean , Jeffery To PKG_LICENSE:=Python/2.0 @@ -175,7 +175,7 @@ define Build/Compile/python3-setuptools --ignore-installed \ --root=$(PKG_BUILD_DIR)/install-setuptools \ --prefix=/usr \ - $(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/setuptools-$(PYTHON3_SETUPTOOLS_VERSION)-py2.py3-none-any.whl + $(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/setuptools-$(PYTHON3_SETUPTOOLS_VERSION)-py3-none-any.whl $(call PatchDir,$(PKG_BUILD_DIR)/install-setuptools/usr/lib/python$(PYTHON3_VERSION)/site-packages,./patches-setuptools,) endef endif # CONFIG_PACKAGE_python3-setuptools diff --git a/lang/python/python3/patches-pip/001-pep517-pyc-fix.patch b/lang/python/python3/patches-pip/001-pep517-pyc-fix.patch index c284f01113..b07cdd19ec 100644 --- a/lang/python/python3/patches-pip/001-pep517-pyc-fix.patch +++ b/lang/python/python3/patches-pip/001-pep517-pyc-fix.patch @@ -1,13 +1,22 @@ diff -Nurp a/pip/_vendor/pep517/wrappers.py b/pip/_vendor/pep517/wrappers.py ---- a/pip/_vendor/pep517/wrappers.py 2019-07-30 20:02:13.000000000 +0800 -+++ b/pip/_vendor/pep517/wrappers.py 2020-04-24 17:23:35.764905235 +0800 -@@ -10,6 +10,9 @@ from . import compat +--- a/pip/_vendor/pep517/wrappers.py 2020-05-19 10:39:38.000000000 +0800 ++++ b/pip/_vendor/pep517/wrappers.py 2020-06-30 20:19:05.495033208 +0800 +@@ -14,11 +14,16 @@ try: + import importlib.resources as resources - _in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.py') + def _in_proc_script_path(): +- return resources.path(__package__, '_in_process.py') ++ if resources.is_resource(__package__, '_in_process.py'): ++ return resources.path(__package__, '_in_process.py') ++ return resources.path(__package__, '_in_process.pyc') + except ImportError: + @contextmanager + def _in_proc_script_path(): +- yield pjoin(dirname(abspath(__file__)), '_in_process.py') ++ _in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.py') ++ if not os.path.isfile(_in_proc_script): ++ _in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.pyc') ++ yield _in_proc_script -+if not os.path.isfile(_in_proc_script): -+ _in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.pyc') -+ @contextmanager - def tempdir(): diff --git a/lang/python/python3/patches-setuptools/001-reproducible.patch b/lang/python/python3/patches-setuptools/001-reproducible.patch index 32edc568d1..a4e80103f8 100644 --- a/lang/python/python3/patches-setuptools/001-reproducible.patch +++ b/lang/python/python3/patches-setuptools/001-reproducible.patch @@ -5,7 +5,7 @@ Index: b/setuptools/command/easy_install.py =================================================================== --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py -@@ -436,7 +436,7 @@ consider to install to another location, +@@ -423,7 +423,7 @@ class easy_install(Command): for spec in self.args: self.easy_install(spec, not self.no_deps) if self.record: diff --git a/lang/python/python3/patches-setuptools/002-sorted-requires.patch b/lang/python/python3/patches-setuptools/002-sorted-requires.patch index 2ad4795b36..9e48f523ef 100644 --- a/lang/python/python3/patches-setuptools/002-sorted-requires.patch +++ b/lang/python/python3/patches-setuptools/002-sorted-requires.patch @@ -5,10 +5,10 @@ Index: b/setuptools/command/egg_info.py =================================================================== --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py -@@ -621,7 +621,7 @@ def warn_depends_obsolete(cmd, basename, - def _write_requirements(stream, reqs): - lines = yield_lines(reqs or ()) - append_cr = lambda line: line + '\n' +@@ -641,7 +641,7 @@ def _write_requirements(stream, reqs): + + def append_cr(line): + return line + '\n' - lines = map(append_cr, lines) + lines = map(append_cr, sorted(lines)) stream.writelines(lines) diff --git a/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch b/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch index 15f34dcdb8..f1b6ca169a 100644 --- a/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch +++ b/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch @@ -3,7 +3,7 @@ https://sources.debian.org/patches/python-setuptools/40.8.0-1/PKG-INFO-output-re --- a/setuptools/dist.py +++ b/setuptools/dist.py -@@ -191,7 +191,7 @@ def write_pkg_file(self, file): +@@ -193,7 +193,7 @@ def write_pkg_file(self, file): self.long_description_content_type ) if self.provides_extras: diff --git a/lang/python/python3/patches-setuptools/004-site-patch.patch b/lang/python/python3/patches-setuptools/004-site-patch.patch index 39b8c90268..279c35284b 100644 --- a/lang/python/python3/patches-setuptools/004-site-patch.patch +++ b/lang/python/python3/patches-setuptools/004-site-patch.patch @@ -1,6 +1,6 @@ --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py -@@ -1315,7 +1315,10 @@ class easy_install(Command): +@@ -1324,7 +1324,10 @@ class easy_install(Command): return # already did it, or don't need to sitepy = os.path.join(self.install_dir, "site.py") diff --git a/lang/python/python3/patches/020-ssl-module-emulate-tls-methods.patch b/lang/python/python3/patches/020-ssl-module-emulate-tls-methods.patch deleted file mode 100644 index 5cd1b94df1..0000000000 --- a/lang/python/python3/patches/020-ssl-module-emulate-tls-methods.patch +++ /dev/null @@ -1,218 +0,0 @@ -From 991f0176e188227647bf4c993d8da81cf794b3ae Mon Sep 17 00:00:00 2001 -From: Christian Heimes -Date: Sun, 25 Feb 2018 20:03:07 +0100 -Subject: [PATCH] bpo-30008: SSL module: emulate tls methods - -OpenSSL 1.1 compatility: emulate version specific TLS methods with -SSL_CTX_set_min/max_proto_version(). ---- - .../2018-02-25-20-05-51.bpo-30008.6Bmyhr.rst | 4 + - Modules/_ssl.c | 134 ++++++++++++++---- - 2 files changed, 108 insertions(+), 30 deletions(-) - create mode 100644 Misc/NEWS.d/next/Library/2018-02-25-20-05-51.bpo-30008.6Bmyhr.rst - ---- /dev/null -+++ b/Misc/NEWS.d/next/Library/2018-02-25-20-05-51.bpo-30008.6Bmyhr.rst -@@ -0,0 +1,4 @@ -+The ssl module no longer uses function that are deprecated since OpenSSL -+1.1.0. The version specific TLS methods are emulated with TLS_method() plus -+SSL_CTX_set_min/max_proto_version(). Pseudo random numbers are generated -+with RAND_bytes(). ---- a/Modules/_ssl.c -+++ b/Modules/_ssl.c -@@ -45,14 +45,6 @@ static PySocketModule_APIObject PySocket - #include - #endif - --/* Don't warn about deprecated functions */ --#ifdef __GNUC__ --#pragma GCC diagnostic ignored "-Wdeprecated-declarations" --#endif --#ifdef __clang__ --#pragma clang diagnostic ignored "-Wdeprecated-declarations" --#endif -- - /* Include OpenSSL header files */ - #include "openssl/rsa.h" - #include "openssl/crypto.h" -@@ -205,6 +197,7 @@ static void _PySSLFixErrno(void) { - #ifndef PY_OPENSSL_1_1_API - /* OpenSSL 1.1 API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7.0 */ - -+#define ASN1_STRING_get0_data ASN1_STRING_data - #define TLS_method SSLv23_method - #define TLS_client_method SSLv23_client_method - #define TLS_server_method SSLv23_server_method -@@ -896,7 +889,7 @@ _ssl_configure_hostname(PySSLSocket *sel - goto error; - } - } else { -- if (!X509_VERIFY_PARAM_set1_ip(param, ASN1_STRING_data(ip), -+ if (!X509_VERIFY_PARAM_set1_ip(param, ASN1_STRING_get0_data(ip), - ASN1_STRING_length(ip))) { - _setSSLError(NULL, 0, __FILE__, __LINE__); - goto error; -@@ -1372,8 +1365,9 @@ _get_peer_alt_names (X509 *certificate) - goto fail; - } - PyTuple_SET_ITEM(t, 0, v); -- v = PyUnicode_FromStringAndSize((char *)ASN1_STRING_data(as), -- ASN1_STRING_length(as)); -+ v = PyUnicode_FromStringAndSize( -+ (char *)ASN1_STRING_get0_data(as), -+ ASN1_STRING_length(as)); - if (v == NULL) { - Py_DECREF(t); - goto fail; -@@ -3078,44 +3072,124 @@ _ssl__SSLContext_impl(PyTypeObject *type - long options; - SSL_CTX *ctx = NULL; - X509_VERIFY_PARAM *params; -- int result; -+ int result = 0; - #if defined(SSL_MODE_RELEASE_BUFFERS) - unsigned long libver; - #endif - - PySSL_BEGIN_ALLOW_THREADS -- if (proto_version == PY_SSL_VERSION_TLS1) -+ switch (proto_version) { -+#if OPENSSL_VERSION_NUMBER <= 0x10100000L -+ /* OpenSSL < 1.1.0 or not LibreSSL -+ * Use old-style methods for OpenSSL 1.0.2 -+ */ -+#if defined(SSL2_VERSION) && !defined(OPENSSL_NO_SSL2) -+ case PY_SSL_VERSION_SSL2: -+ ctx = SSL_CTX_new(SSLv2_method()); -+ break; -+#endif -+#if defined(SSL3_VERSION) && !defined(OPENSSL_NO_SSL3) -+ case PY_SSL_VERSION_SSL3: -+ ctx = SSL_CTX_new(SSLv3_method()); -+ break; -+#endif -+#if defined(TLS1_VERSION) && !defined(OPENSSL_NO_TLS1) -+ case PY_SSL_VERSION_TLS1: - ctx = SSL_CTX_new(TLSv1_method()); --#if HAVE_TLSv1_2 -- else if (proto_version == PY_SSL_VERSION_TLS1_1) -+ break; -+#endif -+#if defined(TLS1_1_VERSION) && !defined(OPENSSL_NO_TLS1_1) -+ case PY_SSL_VERSION_TLS1_1: - ctx = SSL_CTX_new(TLSv1_1_method()); -- else if (proto_version == PY_SSL_VERSION_TLS1_2) -+ break; -+#endif -+#if defined(TLS1_2_VERSION) && !defined(OPENSSL_NO_TLS1_2) -+ case PY_SSL_VERSION_TLS1_2: - ctx = SSL_CTX_new(TLSv1_2_method()); -+ break; - #endif --#ifndef OPENSSL_NO_SSL3 -- else if (proto_version == PY_SSL_VERSION_SSL3) -- ctx = SSL_CTX_new(SSLv3_method()); -+#else -+ /* OpenSSL >= 1.1 or LibreSSL -+ * create context with TLS_method for all protocols -+ * no SSLv2_method in OpenSSL 1.1. -+ */ -+#if defined(SSL3_VERSION) && !defined(OPENSSL_NO_SSL3) -+ case PY_SSL_VERSION_SSL3: -+ ctx = SSL_CTX_new(TLS_method()); -+ if (ctx != NULL) { -+ /* OpenSSL 1.1.0 sets SSL_OP_NO_SSLv3 for TLS_method by default */ -+ SSL_CTX_clear_options(ctx, SSL_OP_NO_SSLv3); -+ if (!SSL_CTX_set_min_proto_version(ctx, SSL3_VERSION)) -+ result = -2; -+ if (!SSL_CTX_set_max_proto_version(ctx, SSL3_VERSION)) -+ result = -2; -+ } -+ break; - #endif --#ifndef OPENSSL_NO_SSL2 -- else if (proto_version == PY_SSL_VERSION_SSL2) -- ctx = SSL_CTX_new(SSLv2_method()); -+#if defined(TLS1_VERSION) && !defined(OPENSSL_NO_TLS1) -+ case PY_SSL_VERSION_TLS1: -+ ctx = SSL_CTX_new(TLS_method()); -+ if (ctx != NULL) { -+ SSL_CTX_clear_options(ctx, SSL_OP_NO_TLSv1); -+ if (!SSL_CTX_set_min_proto_version(ctx, TLS1_VERSION)) -+ result = -2; -+ if (!SSL_CTX_set_max_proto_version(ctx, TLS1_VERSION)) -+ result = -2; -+ } -+ break; -+#endif -+#if defined(TLS1_1_VERSION) && !defined(OPENSSL_NO_TLS1_1) -+ case PY_SSL_VERSION_TLS1_1: -+ ctx = SSL_CTX_new(TLS_method()); -+ if (ctx != NULL) { -+ SSL_CTX_clear_options(ctx, SSL_OP_NO_TLSv1_1); -+ if (!SSL_CTX_set_min_proto_version(ctx, TLS1_1_VERSION)) -+ result = -2; -+ if (!SSL_CTX_set_max_proto_version(ctx, TLS1_1_VERSION)) -+ result = -2; -+ } -+ break; -+#endif -+#if defined(TLS1_2_VERSION) && !defined(OPENSSL_NO_TLS1_2) -+ case PY_SSL_VERSION_TLS1_2: -+ ctx = SSL_CTX_new(TLS_method()); -+ if (ctx != NULL) { -+ SSL_CTX_clear_options(ctx, SSL_OP_NO_TLSv1_2); -+ if (!SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION)) -+ result = -2; -+ if (!SSL_CTX_set_max_proto_version(ctx, TLS1_2_VERSION)) -+ result = -2; -+ } -+ break; - #endif -- else if (proto_version == PY_SSL_VERSION_TLS) /* SSLv23 */ -+#endif /* OpenSSL >= 1.1 */ -+ case PY_SSL_VERSION_TLS: -+ /* SSLv23 */ - ctx = SSL_CTX_new(TLS_method()); -- else if (proto_version == PY_SSL_VERSION_TLS_CLIENT) -+ break; -+ case PY_SSL_VERSION_TLS_CLIENT: - ctx = SSL_CTX_new(TLS_client_method()); -- else if (proto_version == PY_SSL_VERSION_TLS_SERVER) -+ break; -+ case PY_SSL_VERSION_TLS_SERVER: - ctx = SSL_CTX_new(TLS_server_method()); -- else -- proto_version = -1; -+ break; -+ default: -+ result = -1; -+ break; -+ } - PySSL_END_ALLOW_THREADS - -- if (proto_version == -1) { -+ if (result == -1) { - PyErr_SetString(PyExc_ValueError, - "invalid protocol version"); - return NULL; - } -- if (ctx == NULL) { -+ else if (result == -2) { -+ PyErr_SetString(PyExc_ValueError, -+ "protocol configuration error"); -+ return NULL; -+ } -+ else if (ctx == NULL) { - _setSSLError(NULL, 0, __FILE__, __LINE__); - return NULL; - } -@@ -5288,7 +5362,7 @@ PySSL_RAND(int len, int pseudo) - if (bytes == NULL) - return NULL; - if (pseudo) { -- ok = RAND_pseudo_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); -+ ok = (_PyOS_URandom((unsigned char*)PyBytes_AS_STRING(bytes), len) == 0 ? 1 : 0); - if (ok == 0 || ok == 1) - return Py_BuildValue("NO", bytes, ok == 1 ? Py_True : Py_False); - } diff --git a/lang/python/python3/patches/021-openssl-deprecated.patch b/lang/python/python3/patches/021-openssl-deprecated.patch deleted file mode 100644 index 4dd33f6236..0000000000 --- a/lang/python/python3/patches/021-openssl-deprecated.patch +++ /dev/null @@ -1,63 +0,0 @@ ---- a/Modules/_ssl.c -+++ b/Modules/_ssl.c -@@ -201,6 +201,11 @@ static void _PySSLFixErrno(void) { - #define TLS_method SSLv23_method - #define TLS_client_method SSLv23_client_method - #define TLS_server_method SSLv23_server_method -+#define X509_getm_notBefore X509_get_notBefore -+#define X509_getm_notAfter X509_get_notAfter -+#define OpenSSL_version_num SSLeay -+#define OpenSSL_version SSLeay_version -+#define OPENSSL_VERSION SSLEAY_VERSION - - static int X509_NAME_ENTRY_set(const X509_NAME_ENTRY *ne) - { -@@ -1724,7 +1729,7 @@ _decode_certificate(X509 *certificate) { - Py_DECREF(sn_obj); - - (void) BIO_reset(biobuf); -- notBefore = X509_get_notBefore(certificate); -+ notBefore = X509_getm_notBefore(certificate); - ASN1_TIME_print(biobuf, notBefore); - len = BIO_gets(biobuf, buf, sizeof(buf)-1); - if (len < 0) { -@@ -1741,7 +1746,7 @@ _decode_certificate(X509 *certificate) { - Py_DECREF(pnotBefore); - - (void) BIO_reset(biobuf); -- notAfter = X509_get_notAfter(certificate); -+ notAfter = X509_getm_notAfter(certificate); - ASN1_TIME_print(biobuf, notAfter); - len = BIO_gets(biobuf, buf, sizeof(buf)-1); - if (len < 0) { -@@ -3282,7 +3287,7 @@ _ssl__SSLContext_impl(PyTypeObject *type - conservative and assume it wasn't fixed until release. We do this check - at runtime to avoid problems from the dynamic linker. - See #25672 for more on this. */ -- libver = SSLeay(); -+ libver = OpenSSL_version_num(); - if (!(libver >= 0x10001000UL && libver < 0x1000108fUL) && - !(libver >= 0x10000000UL && libver < 0x100000dfUL)) { - SSL_CTX_set_mode(self->ctx, SSL_MODE_RELEASE_BUFFERS); -@@ -6450,10 +6455,10 @@ PyInit__ssl(void) - return NULL; - - /* OpenSSL version */ -- /* SSLeay() gives us the version of the library linked against, -+ /* OpenSSL_version_num() gives us the version of the library linked against, - which could be different from the headers version. - */ -- libver = SSLeay(); -+ libver = OpenSSL_version_num(); - r = PyLong_FromUnsignedLong(libver); - if (r == NULL) - return NULL; -@@ -6463,7 +6468,7 @@ PyInit__ssl(void) - r = Py_BuildValue("IIIII", major, minor, fix, patch, status); - if (r == NULL || PyModule_AddObject(m, "OPENSSL_VERSION_INFO", r)) - return NULL; -- r = PyUnicode_FromString(SSLeay_version(SSLEAY_VERSION)); -+ r = PyUnicode_FromString(OpenSSL_version(OPENSSL_VERSION)); - if (r == NULL || PyModule_AddObject(m, "OPENSSL_VERSION", r)) - return NULL; - diff --git a/lang/python/python3/patches/025-bpo-41288-Fix-a-crash-in-unpickling-invalid-NEWOBJ_EX-GH-21458.patch b/lang/python/python3/patches/025-bpo-41288-Fix-a-crash-in-unpickling-invalid-NEWOBJ_EX-GH-21458.patch new file mode 100644 index 0000000000..c4c736c2d0 --- /dev/null +++ b/lang/python/python3/patches/025-bpo-41288-Fix-a-crash-in-unpickling-invalid-NEWOBJ_EX-GH-21458.patch @@ -0,0 +1,111 @@ +From f56c75ed53dcad4d59dff4377ae463d6b96acd3e Mon Sep 17 00:00:00 2001 +From: "Miss Islington (bot)" + <31488909+miss-islington@users.noreply.github.com> +Date: Mon, 13 Jul 2020 06:05:44 -0700 +Subject: [PATCH] bpo-41288: Fix a crash in unpickling invalid NEWOBJ_EX. + (GH-21458) + +Automerge-Triggered-By: @tiran +(cherry picked from commit 4f309abf55f0e6f8950ac13d6ec83c22b8d47bf8) + +Co-authored-by: Serhiy Storchaka +--- + Lib/test/pickletester.py | 18 ++++++++++++ + .../2020-07-13-15-06-35.bpo-41288.8mn5P-.rst | 2 ++ + Modules/_pickle.c | 29 ++++++++++++++----- + 3 files changed, 41 insertions(+), 8 deletions(-) + create mode 100644 Misc/NEWS.d/next/Library/2020-07-13-15-06-35.bpo-41288.8mn5P-.rst + +diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py +index 9401043d78d18..ff7bbb0c8a9bf 100644 +--- a/Lib/test/pickletester.py ++++ b/Lib/test/pickletester.py +@@ -1170,6 +1170,24 @@ def test_compat_unpickle(self): + self.assertIs(type(unpickled), collections.UserDict) + self.assertEqual(unpickled, collections.UserDict({1: 2})) + ++ def test_bad_reduce(self): ++ self.assertEqual(self.loads(b'cbuiltins\nint\n)R.'), 0) ++ self.check_unpickling_error(TypeError, b'N)R.') ++ self.check_unpickling_error(TypeError, b'cbuiltins\nint\nNR.') ++ ++ def test_bad_newobj(self): ++ error = (pickle.UnpicklingError, TypeError) ++ self.assertEqual(self.loads(b'cbuiltins\nint\n)\x81.'), 0) ++ self.check_unpickling_error(error, b'cbuiltins\nlen\n)\x81.') ++ self.check_unpickling_error(error, b'cbuiltins\nint\nN\x81.') ++ ++ def test_bad_newobj_ex(self): ++ error = (pickle.UnpicklingError, TypeError) ++ self.assertEqual(self.loads(b'cbuiltins\nint\n)}\x92.'), 0) ++ self.check_unpickling_error(error, b'cbuiltins\nlen\n)}\x92.') ++ self.check_unpickling_error(error, b'cbuiltins\nint\nN}\x92.') ++ self.check_unpickling_error(error, b'cbuiltins\nint\n)N\x92.') ++ + def test_bad_stack(self): + badpickles = [ + b'.', # STOP +diff --git a/Misc/NEWS.d/next/Library/2020-07-13-15-06-35.bpo-41288.8mn5P-.rst b/Misc/NEWS.d/next/Library/2020-07-13-15-06-35.bpo-41288.8mn5P-.rst +new file mode 100644 +index 0000000000000..3c3adbabf16ff +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2020-07-13-15-06-35.bpo-41288.8mn5P-.rst +@@ -0,0 +1,2 @@ ++Unpickling invalid NEWOBJ_EX opcode with the C implementation raises now ++UnpicklingError instead of crashing. +diff --git a/Modules/_pickle.c b/Modules/_pickle.c +index 55affb2c7c479..42ce62fc7cdf4 100644 +--- a/Modules/_pickle.c ++++ b/Modules/_pickle.c +@@ -5988,23 +5988,30 @@ load_newobj_ex(UnpicklerObject *self) + } + + if (!PyType_Check(cls)) { +- Py_DECREF(kwargs); +- Py_DECREF(args); + PyErr_Format(st->UnpicklingError, + "NEWOBJ_EX class argument must be a type, not %.200s", + Py_TYPE(cls)->tp_name); +- Py_DECREF(cls); +- return -1; ++ goto error; + } + + if (((PyTypeObject *)cls)->tp_new == NULL) { +- Py_DECREF(kwargs); +- Py_DECREF(args); +- Py_DECREF(cls); + PyErr_SetString(st->UnpicklingError, + "NEWOBJ_EX class argument doesn't have __new__"); +- return -1; ++ goto error; ++ } ++ if (!PyTuple_Check(args)) { ++ PyErr_Format(st->UnpicklingError, ++ "NEWOBJ_EX args argument must be a tuple, not %.200s", ++ Py_TYPE(args)->tp_name); ++ goto error; ++ } ++ if (!PyDict_Check(kwargs)) { ++ PyErr_Format(st->UnpicklingError, ++ "NEWOBJ_EX kwargs argument must be a dict, not %.200s", ++ Py_TYPE(kwargs)->tp_name); ++ goto error; + } ++ + obj = ((PyTypeObject *)cls)->tp_new((PyTypeObject *)cls, args, kwargs); + Py_DECREF(kwargs); + Py_DECREF(args); +@@ -6014,6 +6021,12 @@ load_newobj_ex(UnpicklerObject *self) + } + PDATA_PUSH(self->stack, obj, -1); + return 0; ++ ++error: ++ Py_DECREF(kwargs); ++ Py_DECREF(args); ++ Py_DECREF(cls); ++ return -1; + } + + static int diff --git a/lang/python/python3/patches/026-bpo-39017-Avoid-infinite-loop-in-the-tarfile-module-GH-21454-GH-21483.patch b/lang/python/python3/patches/026-bpo-39017-Avoid-infinite-loop-in-the-tarfile-module-GH-21454-GH-21483.patch new file mode 100644 index 0000000000..e954eb685e --- /dev/null +++ b/lang/python/python3/patches/026-bpo-39017-Avoid-infinite-loop-in-the-tarfile-module-GH-21454-GH-21483.patch @@ -0,0 +1,62 @@ +From c55479556db015f48fc8bbca17f64d3e65598559 Mon Sep 17 00:00:00 2001 +From: "Miss Islington (bot)" + <31488909+miss-islington@users.noreply.github.com> +Date: Wed, 15 Jul 2020 05:30:53 -0700 +Subject: [PATCH] [3.8] bpo-39017: Avoid infinite loop in the tarfile module + (GH-21454) (GH-21483) + +Avoid infinite loop when reading specially crafted TAR files using the tarfile module +(CVE-2019-20907). +(cherry picked from commit 5a8d121a1f3ef5ad7c105ee378cc79a3eac0c7d4) + + +Co-authored-by: Rishi + +Automerge-Triggered-By: @encukou +--- + Lib/tarfile.py | 2 ++ + Lib/test/recursion.tar | Bin 0 -> 516 bytes + Lib/test/test_tarfile.py | 7 +++++++ + .../2020-07-12-22-16-58.bpo-39017.x3Cg-9.rst | 1 + + 4 files changed, 10 insertions(+) + create mode 100644 Lib/test/recursion.tar + create mode 100644 Misc/NEWS.d/next/Library/2020-07-12-22-16-58.bpo-39017.x3Cg-9.rst + +diff --git a/Lib/tarfile.py b/Lib/tarfile.py +index d31b9cbb51d65..7a69e1b1aa544 100755 +--- a/Lib/tarfile.py ++++ b/Lib/tarfile.py +@@ -1241,6 +1241,8 @@ def _proc_pax(self, tarfile): + + length, keyword = match.groups() + length = int(length) ++ if length == 0: ++ raise InvalidHeaderError("invalid header") + value = buf[match.end(2) + 1:match.start(1) + length - 1] + + # Normally, we could just use "utf-8" as the encoding and "strict" +diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py +index 15324a4e48819..b512168d6ea87 100644 +--- a/Lib/test/test_tarfile.py ++++ b/Lib/test/test_tarfile.py +@@ -397,6 +397,13 @@ def test_premature_end_of_archive(self): + with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"): + tar.extractfile(t).read() + ++ def test_length_zero_header(self): ++ # bpo-39017 (CVE-2019-20907): reading a zero-length header should fail ++ # with an exception ++ with self.assertRaisesRegex(tarfile.ReadError, "file could not be opened successfully"): ++ with tarfile.open(support.findfile('recursion.tar')) as tar: ++ pass ++ + class MiscReadTestBase(CommonReadTest): + def requires_name_attribute(self): + pass +diff --git a/Misc/NEWS.d/next/Library/2020-07-12-22-16-58.bpo-39017.x3Cg-9.rst b/Misc/NEWS.d/next/Library/2020-07-12-22-16-58.bpo-39017.x3Cg-9.rst +new file mode 100644 +index 0000000000000..ad26676f8b856 +--- /dev/null ++++ b/Misc/NEWS.d/next/Library/2020-07-12-22-16-58.bpo-39017.x3Cg-9.rst +@@ -0,0 +1 @@ ++Avoid infinite loop when reading specially crafted TAR files using the tarfile module (CVE-2019-20907). diff --git a/lang/python/python3/patches/027-bpo-39603-Prevent-header-injection-in-http-methods-GH-18485.patch b/lang/python/python3/patches/027-bpo-39603-Prevent-header-injection-in-http-methods-GH-18485.patch new file mode 100644 index 0000000000..89fc64fc6f --- /dev/null +++ b/lang/python/python3/patches/027-bpo-39603-Prevent-header-injection-in-http-methods-GH-18485.patch @@ -0,0 +1,99 @@ +From 668d321476d974c4f51476b33aaca870272523bf Mon Sep 17 00:00:00 2001 +From: "Miss Islington (bot)" + <31488909+miss-islington@users.noreply.github.com> +Date: Sat, 18 Jul 2020 13:39:12 -0700 +Subject: [PATCH] bpo-39603: Prevent header injection in http methods + (GH-18485) + +reject control chars in http method in http.client.putrequest to prevent http header injection +(cherry picked from commit 8ca8a2e8fb068863c1138f07e3098478ef8be12e) + +Co-authored-by: AMIR <31338382+amiremohamadi@users.noreply.github.com> +--- + Lib/http/client.py | 15 +++++++++++++ + Lib/test/test_httplib.py | 22 +++++++++++++++++++ + .../2020-02-12-14-17-39.bpo-39603.Gt3RSg.rst | 2 ++ + 3 files changed, 39 insertions(+) + create mode 100644 Misc/NEWS.d/next/Security/2020-02-12-14-17-39.bpo-39603.Gt3RSg.rst + +diff --git a/Lib/http/client.py b/Lib/http/client.py +index 019380a720318..c2ad0471bfee5 100644 +--- a/Lib/http/client.py ++++ b/Lib/http/client.py +@@ -147,6 +147,10 @@ + # _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$") + # We are more lenient for assumed real world compatibility purposes. + ++# These characters are not allowed within HTTP method names ++# to prevent http header injection. ++_contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]') ++ + # We always set the Content-Length header for these methods because some + # servers will otherwise respond with a 411 + _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} +@@ -1087,6 +1091,8 @@ def putrequest(self, method, url, skip_host=False, + else: + raise CannotSendRequest(self.__state) + ++ self._validate_method(method) ++ + # Save the method for use later in the response phase + self._method = method + +@@ -1177,6 +1183,15 @@ def _encode_request(self, request): + # ASCII also helps prevent CVE-2019-9740. + return request.encode('ascii') + ++ def _validate_method(self, method): ++ """Validate a method name for putrequest.""" ++ # prevent http header injection ++ match = _contains_disallowed_method_pchar_re.search(method) ++ if match: ++ raise ValueError( ++ f"method can't contain control characters. {method!r} " ++ f"(found at least {match.group()!r})") ++ + def _validate_path(self, url): + """Validate a url for putrequest.""" + # Prevent CVE-2019-9740. +diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py +index 8f0e27a1fb836..5a5fcecbc9c15 100644 +--- a/Lib/test/test_httplib.py ++++ b/Lib/test/test_httplib.py +@@ -364,6 +364,28 @@ def test_headers_debuglevel(self): + self.assertEqual(lines[3], "header: Second: val2") + + ++class HttpMethodTests(TestCase): ++ def test_invalid_method_names(self): ++ methods = ( ++ 'GET\r', ++ 'POST\n', ++ 'PUT\n\r', ++ 'POST\nValue', ++ 'POST\nHOST:abc', ++ 'GET\nrHost:abc\n', ++ 'POST\rRemainder:\r', ++ 'GET\rHOST:\n', ++ '\nPUT' ++ ) ++ ++ for method in methods: ++ with self.assertRaisesRegex( ++ ValueError, "method can't contain control characters"): ++ conn = client.HTTPConnection('example.com') ++ conn.sock = FakeSocket(None) ++ conn.request(method=method, url="/") ++ ++ + class TransferEncodingTest(TestCase): + expected_body = b"It's just a flesh wound" + +diff --git a/Misc/NEWS.d/next/Security/2020-02-12-14-17-39.bpo-39603.Gt3RSg.rst b/Misc/NEWS.d/next/Security/2020-02-12-14-17-39.bpo-39603.Gt3RSg.rst +new file mode 100644 +index 0000000000000..990affc3edd9d +--- /dev/null ++++ b/Misc/NEWS.d/next/Security/2020-02-12-14-17-39.bpo-39603.Gt3RSg.rst +@@ -0,0 +1,2 @@ ++Prevent http header injection by rejecting control characters in ++http.client.putrequest(...). -- 2.30.2