From: Alexandru Ardelean Date: Fri, 18 Oct 2019 06:25:38 +0000 (+0300) Subject: python3: bump to version 3.8 X-Git-Url: http://git.lede-project.org./?a=commitdiff_plain;h=48277ec9158151763239461c6f60808e38a99c2f;p=feed%2Fpackages.git python3: bump to version 3.8 This required a bit work to get working, compared to other versions. So, some things have changed a bit more significantly. Some highlights: * there is no longer a pgen executable, seems this is now part of libpython; let's see what this means for us in the future * blake2 hash (from OpenSSL) detection needs some fixing; will upstream added patch 002-fix-blake2-detection.patch * removed all bpo patches; those should be fixed in upstream * some needed to be manually re-applied as stuff changed: - 001-enable-zlib.patch - file changed - 004-do-not-write-bytes-codes.patch - file changed - 015-abort-on-failed-modules.patch - variable was renamed cross_compiling -> CROSS_COMPILING * 017_lib2to3_fix_pyc_search.patch - the code changed, it does not seem to have the original problem with respect to file-extension, as there does not seem to be any special extension logic anymore there * 006-remove-multi-arch-and-local-paths.patch - dropped patch; I can't remember the full-details of this issue; it was something with Debian/Ubuntu's multi-arch stuff; it was probably added maybe due to some overzealous (on my part) thingy caused by some weird reports, that I could never solve; let's have this patch dropped and see * make package/python3/refresh to reduce fuzz for the rest Signed-off-by: Alexandru Ardelean --- diff --git a/lang/python/python3-version.mk b/lang/python/python3-version.mk index 93c01e9908..6a4e5e3bba 100644 --- a/lang/python/python3-version.mk +++ b/lang/python/python3-version.mk @@ -7,13 +7,13 @@ # Note: keep in sync with setuptools & pip PYTHON3_VERSION_MAJOR:=3 -PYTHON3_VERSION_MINOR:=7 -PYTHON3_VERSION_MICRO:=4 +PYTHON3_VERSION_MINOR:=8 +PYTHON3_VERSION_MICRO:=0 PYTHON3_VERSION:=$(PYTHON3_VERSION_MAJOR).$(PYTHON3_VERSION_MINOR) PYTHON3_SETUPTOOLS_PKG_RELEASE:=1 PYTHON3_PIP_PKG_RELEASE:=1 -PYTHON3_SETUPTOOLS_VERSION:=40.8.0 -PYTHON3_PIP_VERSION:=19.0.3 +PYTHON3_SETUPTOOLS_VERSION:=41.2.0 +PYTHON3_PIP_VERSION:=19.2.3 diff --git a/lang/python/python3/Makefile b/lang/python/python3/Makefile index 10842f12c1..cfa1c00c0a 100644 --- a/lang/python/python3/Makefile +++ b/lang/python/python3/Makefile @@ -14,12 +14,12 @@ PYTHON_VERSION:=$(PYTHON3_VERSION) PYTHON_VERSION_MICRO:=$(PYTHON3_VERSION_MICRO) PKG_NAME:=python3 -PKG_RELEASE:=5 +PKG_RELEASE:=1 PKG_VERSION:=$(PYTHON_VERSION).$(PYTHON_VERSION_MICRO) PKG_SOURCE:=Python-$(PKG_VERSION).tar.xz PKG_SOURCE_URL:=https://www.python.org/ftp/python/$(PKG_VERSION) -PKG_HASH:=fb799134b868199930b75f26678f18932214042639cd52b16da7fd134cd9b13f +PKG_HASH:=b356244e13fb5491da890b35b13b2118c3122977c2cd825e3eb6e7d462030d84 PKG_MAINTAINER:=Alexandru Ardelean , Jeffery To PKG_LICENSE:=Python/2.0 @@ -129,8 +129,7 @@ endef MAKE_FLAGS+=\ CROSS_COMPILE=yes \ - LD="$(TARGET_CC)" \ - PGEN=pgen3 + LD="$(TARGET_CC)" EXTRA_CFLAGS+= \ -DNDEBUG -fno-inline @@ -322,7 +321,7 @@ define Host/Configure endef define Host/Compile - +$(HOST_MAKE_VARS) $(MAKE) $(HOST_JOBS) -C $(HOST_BUILD_DIR) python Parser/pgen + +$(HOST_MAKE_VARS) $(MAKE) $(HOST_JOBS) -C $(HOST_BUILD_DIR) python +$(HOST_MAKE_VARS) $(MAKE) $(HOST_JOBS) -C $(HOST_BUILD_DIR) sharedmods endef @@ -344,8 +343,6 @@ define Host/Install $(HOST_PYTHON3_PKG_DIR)/.pip_installed_* ) $(MAKE) -C $(HOST_BUILD_DIR) install - $(INSTALL_DIR) $(HOST_PYTHON3_DIR)/bin/ - $(INSTALL_BIN) $(HOST_BUILD_DIR)/Parser/pgen $(HOST_PYTHON3_DIR)/bin/pgen3 $(if $(wildcard $(HOST_PYTHON3_PKG_DIR)/.setuptools_installed_$(PYTHON3_SETUPTOOLS_VERSION)-$(PYTHON3_SETUPTOOLS_PKG_RELEASE)),, $(call HostPatchDir,$(HOST_PYTHON3_PKG_DIR),./patches-setuptools,) touch $(HOST_PYTHON3_PKG_DIR)/.setuptools_installed_$(PYTHON3_SETUPTOOLS_VERSION)-$(PYTHON3_SETUPTOOLS_PKG_RELEASE) diff --git a/lang/python/python3/patches/001-enable-zlib.patch b/lang/python/python3/patches/001-enable-zlib.patch index 287b147d14..9a93eb263c 100644 --- a/lang/python/python3/patches/001-enable-zlib.patch +++ b/lang/python/python3/patches/001-enable-zlib.patch @@ -1,15 +1,6 @@ -From 6eeab87bc852481e599325549c854b701bf2e39f Mon Sep 17 00:00:00 2001 -From: Alexandru Ardelean -Date: Thu, 25 Sep 2014 18:18:29 +0300 -Subject: [PATCH] enable zlib - ---- - Modules/Setup.dist | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - ---- a/Modules/Setup.dist -+++ b/Modules/Setup.dist -@@ -335,7 +335,7 @@ _symtable symtablemodule.c +--- a/Modules/Setup ++++ b/Modules/Setup +@@ -334,7 +334,7 @@ _symtable symtablemodule.c # Andrew Kuchling's zlib module. # This require zlib 1.1.3 (or later). # See http://www.gzip.org/zlib/ diff --git a/lang/python/python3/patches/002-fix-blake2-detection.patch b/lang/python/python3/patches/002-fix-blake2-detection.patch new file mode 100644 index 0000000000..72b6ea219f --- /dev/null +++ b/lang/python/python3/patches/002-fix-blake2-detection.patch @@ -0,0 +1,11 @@ +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -42,7 +42,7 @@ + #define PY_OPENSSL_HAS_SHAKE 1 + #endif + +-#ifdef NID_blake2b512 ++#ifndef OPENSSL_NO_BLAKE2 + #define PY_OPENSSL_HAS_BLAKE2 1 + #endif + diff --git a/lang/python/python3/patches/003-do-not-run-distutils-tests.patch b/lang/python/python3/patches/003-do-not-run-distutils-tests.patch index bfd415d773..4784ba22d4 100644 --- a/lang/python/python3/patches/003-do-not-run-distutils-tests.patch +++ b/lang/python/python3/patches/003-do-not-run-distutils-tests.patch @@ -1,6 +1,6 @@ --- a/Makefile.pre.in +++ b/Makefile.pre.in -@@ -1391,6 +1391,7 @@ libinstall: build_all $(srcdir)/Modules/ +@@ -1451,6 +1451,7 @@ libinstall: build_all $(srcdir)/Modules/ $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py \ $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt @@ -8,11 +8,11 @@ if test -d $(DESTDIR)$(LIBDEST)/distutils/tests; then \ $(INSTALL_DATA) $(srcdir)/Modules/xxmodule.c \ $(DESTDIR)$(LIBDEST)/distutils/tests ; \ -@@ -1426,6 +1427,7 @@ libinstall: build_all $(srcdir)/Modules/ +@@ -1486,6 +1487,7 @@ libinstall: build_all $(srcdir)/Modules/ $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/Grammar.txt -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/PatternGrammar.txt +endif - python-config: $(srcdir)/Misc/python-config.in Misc/python-config.sh - @ # Substitution happens here, as the completely-expanded BINDIR + # bpo-21536: Misc/python-config.sh is generated in the build directory + # from $(srcdir)Misc/python-config.sh.in. diff --git a/lang/python/python3/patches/004-do-not-write-bytes-codes.patch b/lang/python/python3/patches/004-do-not-write-bytes-codes.patch index eb66443aa8..a67f55332a 100644 --- a/lang/python/python3/patches/004-do-not-write-bytes-codes.patch +++ b/lang/python/python3/patches/004-do-not-write-bytes-codes.patch @@ -1,5 +1,5 @@ ---- a/Python/pylifecycle.c -+++ b/Python/pylifecycle.c +--- a/Python/initconfig.c ++++ b/Python/initconfig.c @@ -120,7 +120,7 @@ int Py_NoSiteFlag = 0; /* Suppress 'impo int Py_BytesWarningFlag = 0; /* Warn on str(bytes) and str(buffer) */ int Py_FrozenFlag = 0; /* Needed by getpath.c */ diff --git a/lang/python/python3/patches/006-remove-multi-arch-and-local-paths.patch b/lang/python/python3/patches/006-remove-multi-arch-and-local-paths.patch deleted file mode 100644 index c4914f2787..0000000000 --- a/lang/python/python3/patches/006-remove-multi-arch-and-local-paths.patch +++ /dev/null @@ -1,19 +0,0 @@ ---- a/setup.py -+++ b/setup.py -@@ -581,16 +581,9 @@ class PyBuildExt(build_ext): - os.unlink(tmpfile) - - def detect_modules(self): -- # Ensure that /usr/local is always used, but the local build -- # directories (i.e. '.' and 'Include') must be first. See issue -- # 10520. -- if not cross_compiling: -- add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib') -- add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') - # only change this for cross builds for 3.3, issues on Mageia - if cross_compiling: - self.add_gcc_paths() -- self.add_multiarch_paths() - - # Add paths specified in the environment variables LDFLAGS and - # CPPFLAGS for header and library files. diff --git a/lang/python/python3/patches/008-distutils-use-python-sysroot.patch b/lang/python/python3/patches/008-distutils-use-python-sysroot.patch index f218c46d54..751f9d7979 100644 --- a/lang/python/python3/patches/008-distutils-use-python-sysroot.patch +++ b/lang/python/python3/patches/008-distutils-use-python-sysroot.patch @@ -36,8 +36,8 @@ Signed-off-by: Thomas Petazzoni self.library_dirs.append('.') --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py -@@ -17,10 +17,17 @@ import sys - from .errors import DistutilsPlatformError +@@ -18,10 +18,17 @@ from .errors import DistutilsPlatformErr + from .util import get_platform, get_host_platform # These are needed in a couple of spots, so just compute them once. -PREFIX = os.path.normpath(sys.prefix) diff --git a/lang/python/python3/patches/010-do-not-add-rt-lib-dirs-when-cross-compiling.patch b/lang/python/python3/patches/010-do-not-add-rt-lib-dirs-when-cross-compiling.patch index 12544b2d67..b3da28a69c 100644 --- a/lang/python/python3/patches/010-do-not-add-rt-lib-dirs-when-cross-compiling.patch +++ b/lang/python/python3/patches/010-do-not-add-rt-lib-dirs-when-cross-compiling.patch @@ -1,10 +1,10 @@ --- a/setup.py +++ b/setup.py -@@ -591,8 +591,9 @@ class PyBuildExt(build_ext): +@@ -631,8 +631,9 @@ class PyBuildExt(build_ext): # directly since an inconsistently reproducible issue comes up where # the environment variable is not set even though the value were passed # into configure and stored in the Makefile (issue found on OS X 10.3). -+ rt_lib_dirs = [] if cross_compiling else self.compiler.runtime_library_dirs ++ rt_lib_dirs = [] if CROSS_COMPILING else self.compiler.runtime_library_dirs for env_var, arg_name, dir_list in ( - ('LDFLAGS', '-R', self.compiler.runtime_library_dirs), + ('LDFLAGS', '-R', rt_lib_dirs), diff --git a/lang/python/python3/patches/013-getbuildinfo-date-time-source-date-epoch.patch b/lang/python/python3/patches/013-getbuildinfo-date-time-source-date-epoch.patch index d37ed8aa5d..503159d669 100644 --- a/lang/python/python3/patches/013-getbuildinfo-date-time-source-date-epoch.patch +++ b/lang/python/python3/patches/013-getbuildinfo-date-time-source-date-epoch.patch @@ -1,6 +1,6 @@ --- a/Makefile.pre.in +++ b/Makefile.pre.in -@@ -753,6 +753,16 @@ regen-all: regen-opcode regen-opcode-tar +@@ -735,6 +735,16 @@ regen-all: regen-opcode regen-opcode-tar ############################################################################ # Special rules for object files @@ -17,8 +17,8 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ $(OBJECT_OBJS) \ $(PYTHON_OBJS) \ -@@ -760,6 +770,8 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ - $(MODOBJS) \ +@@ -743,6 +753,8 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ + $(DTRACE_OBJS) \ $(srcdir)/Modules/getbuildinfo.c $(CC) -c $(PY_CORE_CFLAGS) \ + -DDATE="\"$(BUILD_DATE)\"" \ diff --git a/lang/python/python3/patches/014-remove-platform-so-suffix.patch b/lang/python/python3/patches/014-remove-platform-so-suffix.patch index c87086b8f0..cc6a8511b3 100644 --- a/lang/python/python3/patches/014-remove-platform-so-suffix.patch +++ b/lang/python/python3/patches/014-remove-platform-so-suffix.patch @@ -1,6 +1,6 @@ --- a/configure +++ b/configure -@@ -15273,7 +15273,7 @@ $as_echo_n "checking ABIFLAGS... " >&6; +@@ -15142,7 +15142,7 @@ $as_echo_n "checking ABIFLAGS... " >&6; $as_echo "$ABIFLAGS" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: checking SOABI" >&5 $as_echo_n "checking SOABI... " >&6; } @@ -11,7 +11,7 @@ --- a/configure.ac +++ b/configure.ac -@@ -4736,7 +4736,7 @@ AC_SUBST(SOABI) +@@ -4633,7 +4633,7 @@ AC_SUBST(SOABI) AC_MSG_CHECKING(ABIFLAGS) AC_MSG_RESULT($ABIFLAGS) AC_MSG_CHECKING(SOABI) @@ -19,4 +19,4 @@ +SOABI='cpython-'`echo $VERSION | tr -d .` AC_MSG_RESULT($SOABI) - AC_SUBST(EXT_SUFFIX) + # Release and debug (Py_DEBUG) ABI are compatible, but not Py_TRACE_REFS ABI diff --git a/lang/python/python3/patches/015-abort-on-failed-modules.patch b/lang/python/python3/patches/015-abort-on-failed-modules.patch index effa0b1d31..234ef6c52a 100644 --- a/lang/python/python3/patches/015-abort-on-failed-modules.patch +++ b/lang/python/python3/patches/015-abort-on-failed-modules.patch @@ -1,10 +1,10 @@ --- a/setup.py +++ b/setup.py -@@ -398,6 +398,7 @@ class PyBuildExt(build_ext): +@@ -441,6 +441,7 @@ class PyBuildExt(build_ext): print("Failed to build these modules:") print_three_column(failed) print() -+ if cross_compiling: sys.exit(1) ++ if CROSS_COMPILING: sys.exit(1) if self.failed_on_import: failed = self.failed_on_import[:] diff --git a/lang/python/python3/patches/016-adjust-config-paths.patch b/lang/python/python3/patches/016-adjust-config-paths.patch index c9e1599dc4..b8ea6ccfcd 100644 --- a/lang/python/python3/patches/016-adjust-config-paths.patch +++ b/lang/python/python3/patches/016-adjust-config-paths.patch @@ -1,6 +1,6 @@ --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py -@@ -446,6 +446,7 @@ def _init_posix(): +@@ -445,6 +445,7 @@ def _init_posix(): platform=sys.platform, multiarch=getattr(sys.implementation, '_multiarch', ''), )) @@ -20,7 +20,7 @@ abi=sys.abiflags, --- a/Makefile.pre.in +++ b/Makefile.pre.in -@@ -1400,7 +1400,7 @@ libinstall: build_all $(srcdir)/Modules/ +@@ -1460,7 +1460,7 @@ libinstall: build_all $(srcdir)/Modules/ esac; \ done; \ done @@ -29,7 +29,7 @@ $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt ifeq (@COMPILE_ALL_TESTS@,yes) -@@ -1545,7 +1545,7 @@ sharedinstall: sharedmods +@@ -1618,7 +1618,7 @@ sharedinstall: sharedmods --install-scripts=$(BINDIR) \ --install-platlib=$(DESTSHARED) \ --root=$(DESTDIR)/ @@ -40,8 +40,8 @@ # Here are a couple of targets for MacOSX again, to install a full --- a/configure +++ b/configure -@@ -15292,7 +15292,7 @@ LDVERSION='$(VERSION)$(ABIFLAGS)' - $as_echo "$LDVERSION" >&6; } +@@ -15181,7 +15181,7 @@ else + fi -if test x$PLATFORM_TRIPLET = x; then @@ -51,7 +51,7 @@ LIBPL='$(prefix)'"/lib/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}" --- a/configure.ac +++ b/configure.ac -@@ -4753,7 +4753,7 @@ AC_MSG_RESULT($LDVERSION) +@@ -4667,7 +4667,7 @@ fi dnl define LIBPL after ABIFLAGS and LDVERSION is defined. AC_SUBST(PY_ENABLE_SHARED) diff --git a/lang/python/python3/patches/017_lib2to3_fix_pyc_search.patch b/lang/python/python3/patches/017_lib2to3_fix_pyc_search.patch deleted file mode 100644 index ab55f39660..0000000000 --- a/lang/python/python3/patches/017_lib2to3_fix_pyc_search.patch +++ /dev/null @@ -1,15 +0,0 @@ ---- a/Lib/lib2to3/refactor.py -+++ b/Lib/lib2to3/refactor.py -@@ -37,6 +37,12 @@ def get_all_fix_names(fixer_pkg, remove_ - if remove_prefix: - name = name[4:] - fix_names.append(name[:-3]) -+ if name.startswith("fix_") and name.endswith(".pyc"): -+ if remove_prefix: -+ name = name[4:] -+ name = name[:-4] -+ if name not in fix_names: -+ fix_names.append(name) - return fix_names - - diff --git a/lang/python/python3/patches/025-bpo-37461-Fix-infinite-loop-in-parsing-of-specially-.patch b/lang/python/python3/patches/025-bpo-37461-Fix-infinite-loop-in-parsing-of-specially-.patch deleted file mode 100644 index c61ba8b04a..0000000000 --- a/lang/python/python3/patches/025-bpo-37461-Fix-infinite-loop-in-parsing-of-specially-.patch +++ /dev/null @@ -1,56 +0,0 @@ -From 391511ccaaf0050970dfbe95bf2df1bcf6c33440 Mon Sep 17 00:00:00 2001 -From: "Miss Islington (bot)" - <31488909+miss-islington@users.noreply.github.com> -Date: Wed, 17 Jul 2019 10:02:05 -0700 -Subject: [PATCH] bpo-37461: Fix infinite loop in parsing of specially crafted - email headers (GH-14794) - -* bpo-37461: Fix infinite loop in parsing of specially crafted email headers. - -Some crafted email header would cause the get_parameter method to run in an -infinite loop causing a DoS attack surface when parsing those headers. This -patch fixes that by making sure the DQUOTE character is handled to prevent -going into an infinite loop. -(cherry picked from commit a4a994bd3e619cbaff97610a1cee8ffa87c672f5) - -Co-authored-by: Abhilash Raj ---- - Lib/email/_header_value_parser.py | 3 +++ - Lib/test/test_email/test__header_value_parser.py | 7 +++++++ - .../next/Security/2019-07-16-08-11-00.bpo-37461.1Ahz7O.rst | 2 ++ - 3 files changed, 12 insertions(+) - create mode 100644 Misc/NEWS.d/next/Security/2019-07-16-08-11-00.bpo-37461.1Ahz7O.rst - ---- a/Lib/email/_header_value_parser.py -+++ b/Lib/email/_header_value_parser.py -@@ -2387,6 +2387,9 @@ def get_parameter(value): - while value: - if value[0] in WSP: - token, value = get_fws(value) -+ elif value[0] == '"': -+ token = ValueTerminal('"', 'DQUOTE') -+ value = value[1:] - else: - token, value = get_qcontent(value) - v.append(token) ---- a/Lib/test/test_email/test__header_value_parser.py -+++ b/Lib/test/test_email/test__header_value_parser.py -@@ -2621,6 +2621,13 @@ class Test_parse_mime_parameters(TestPar - # Defects are apparent missing *0*, and two 'out of sequence'. - [errors.InvalidHeaderDefect]*3), - -+ # bpo-37461: Check that we don't go into an infinite loop. -+ 'extra_dquote': ( -+ 'r*="\'a\'\\"', -+ ' r="\\""', -+ 'r*=\'a\'"', -+ [('r', '"')], -+ [errors.InvalidHeaderDefect]*2), - } - - @parameterize ---- /dev/null -+++ b/Misc/NEWS.d/next/Security/2019-07-16-08-11-00.bpo-37461.1Ahz7O.rst -@@ -0,0 +1,2 @@ -+Fix an inifite loop when parsing specially crafted email headers. Patch by -+Abhilash Raj. diff --git a/lang/python/python3/patches/026-3.7-bpo-37764-Fix-infinite-loop-when-parsing-unstruc.patch b/lang/python/python3/patches/026-3.7-bpo-37764-Fix-infinite-loop-when-parsing-unstruc.patch deleted file mode 100644 index 6903ac4e37..0000000000 --- a/lang/python/python3/patches/026-3.7-bpo-37764-Fix-infinite-loop-when-parsing-unstruc.patch +++ /dev/null @@ -1,167 +0,0 @@ -From ea21389dda401457198fb214aa2c981a45ed9528 Mon Sep 17 00:00:00 2001 -From: Ashwin Ramaswami -Date: Tue, 3 Sep 2019 09:42:53 -0700 -Subject: [PATCH] [3.7] bpo-37764: Fix infinite loop when parsing unstructured - email headers. (GH-15239) (GH-15654) -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -…aders. (GH-15239) - -Fixes a case in which email._header_value_parser.get_unstructured hangs the system for some invalid headers. This covers the cases in which the header contains either: -- a case without trailing whitespace -- an invalid encoded word - -https://bugs.python.org/issue37764 - -This fix should also be backported to 3.7 and 3.8 - -https://bugs.python.org/issue37764 -(cherry picked from commit c5b242f87f31286ad38991bc3868cf4cfbf2b681) - -Co-authored-by: Ashwin Ramaswami - - - - - -https://bugs.python.org/issue37764 ---- - Lib/email/_header_value_parser.py | 19 ++++++++++++++--- - .../test_email/test__header_value_parser.py | 16 ++++++++++++++ - Lib/test/test_email/test_email.py | 21 +++++++++++++++++++ - Misc/ACKS | 1 + - .../2019-08-27-01-13-05.bpo-37764.qv67PQ.rst | 1 + - 5 files changed, 55 insertions(+), 3 deletions(-) - create mode 100644 Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst - ---- a/Lib/email/_header_value_parser.py -+++ b/Lib/email/_header_value_parser.py -@@ -931,6 +931,10 @@ class EWWhiteSpaceTerminal(WhiteSpaceTer - return '' - - -+class _InvalidEwError(errors.HeaderParseError): -+ """Invalid encoded word found while parsing headers.""" -+ -+ - # XXX these need to become classes and used as instances so - # that a program can't change them in a parse tree and screw - # up other parse trees. Maybe should have tests for that, too. -@@ -1035,7 +1039,10 @@ def get_encoded_word(value): - raise errors.HeaderParseError( - "expected encoded word but found {}".format(value)) - remstr = ''.join(remainder) -- if len(remstr) > 1 and remstr[0] in hexdigits and remstr[1] in hexdigits: -+ if (len(remstr) > 1 and -+ remstr[0] in hexdigits and -+ remstr[1] in hexdigits and -+ tok.count('?') < 2): - # The ? after the CTE was followed by an encoded word escape (=XX). - rest, *remainder = remstr.split('?=', 1) - tok = tok + '?=' + rest -@@ -1047,7 +1054,7 @@ def get_encoded_word(value): - try: - text, charset, lang, defects = _ew.decode('=?' + tok + '?=') - except ValueError: -- raise errors.HeaderParseError( -+ raise _InvalidEwError( - "encoded word format invalid: '{}'".format(ew.cte)) - ew.charset = charset - ew.lang = lang -@@ -1097,9 +1104,12 @@ def get_unstructured(value): - token, value = get_fws(value) - unstructured.append(token) - continue -+ valid_ew = True - if value.startswith('=?'): - try: - token, value = get_encoded_word(value) -+ except _InvalidEwError: -+ valid_ew = False - except errors.HeaderParseError: - # XXX: Need to figure out how to register defects when - # appropriate here. -@@ -1121,7 +1131,10 @@ def get_unstructured(value): - # Split in the middle of an atom if there is a rfc2047 encoded word - # which does not have WSP on both sides. The defect will be registered - # the next time through the loop. -- if rfc2047_matcher.search(tok): -+ # This needs to only be performed when the encoded word is valid; -+ # otherwise, performing it on an invalid encoded word can cause -+ # the parser to go in an infinite loop. -+ if valid_ew and rfc2047_matcher.search(tok): - tok, *remainder = value.partition('=?') - vtext = ValueTerminal(tok, 'vtext') - _validate_xtext(vtext) ---- a/Lib/test/test_email/test__header_value_parser.py -+++ b/Lib/test/test_email/test__header_value_parser.py -@@ -383,6 +383,22 @@ class TestParser(TestParserMixin, TestEm - [errors.InvalidHeaderDefect], - '') - -+ def test_get_unstructured_without_trailing_whitespace_hang_case(self): -+ self._test_get_x(self._get_unst, -+ '=?utf-8?q?somevalue?=aa', -+ 'somevalueaa', -+ 'somevalueaa', -+ [errors.InvalidHeaderDefect], -+ '') -+ -+ def test_get_unstructured_invalid_ew(self): -+ self._test_get_x(self._get_unst, -+ '=?utf-8?q?=somevalue?=', -+ '=?utf-8?q?=somevalue?=', -+ '=?utf-8?q?=somevalue?=', -+ [], -+ '') -+ - # get_qp_ctext - - def test_get_qp_ctext_only(self): ---- a/Lib/test/test_email/test_email.py -+++ b/Lib/test/test_email/test_email.py -@@ -5367,6 +5367,27 @@ Content-Type: application/x-foo; - eq(language, 'en-us') - eq(s, 'My Document For You') - -+ def test_should_not_hang_on_invalid_ew_messages(self): -+ messages = ["""From: user@host.com -+To: user@host.com -+Bad-Header: -+ =?us-ascii?Q?LCSwrV11+IB0rSbSker+M9vWR7wEDSuGqmHD89Gt=ea0nJFSaiz4vX3XMJPT4vrE?= -+ =?us-ascii?Q?xGUZeOnp0o22pLBB7CYLH74Js=wOlK6Tfru2U47qR?= -+ =?us-ascii?Q?72OfyEY2p2=2FrA9xNFyvH+fBTCmazxwzF8nGkK6D?= -+ -+Hello! -+""", """From: ����� �������� -+To: "xxx" -+Subject: ��� ���������� ����� ����� � ��������� �� ���� -+MIME-Version: 1.0 -+Content-Type: text/plain; charset="windows-1251"; -+Content-Transfer-Encoding: 8bit -+ -+�� ����� � ���� ������ ��� �������� -+"""] -+ for m in messages: -+ with self.subTest(m=m): -+ msg = email.message_from_string(m) - - - # Tests to ensure that signed parts of an email are completely preserved, as ---- a/Misc/ACKS -+++ b/Misc/ACKS -@@ -1305,6 +1305,7 @@ Burton Radons - Abhilash Raj - Shorya Raj - Dhushyanth Ramasamy -+Ashwin Ramaswami - Jeff Ramnani - Bayard Randel - Varpu Rantala ---- /dev/null -+++ b/Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst -@@ -0,0 +1 @@ -+Fixes email._header_value_parser.get_unstructured going into an infinite loop for a specific case in which the email header does not have trailing whitespace, and the case in which it contains an invalid encoded word. Patch by Ashwin Ramaswami. -\ No newline at end of file diff --git a/lang/python/python3/patches/027-bpo-38243-xmlrpc.server-Escape-the-server_title-GH-1.patch b/lang/python/python3/patches/027-bpo-38243-xmlrpc.server-Escape-the-server_title-GH-1.patch deleted file mode 100644 index 52598466a2..0000000000 --- a/lang/python/python3/patches/027-bpo-38243-xmlrpc.server-Escape-the-server_title-GH-1.patch +++ /dev/null @@ -1,74 +0,0 @@ -From 39a0c7555530e31c6941a78da19b6a5b61170687 Mon Sep 17 00:00:00 2001 -From: "Miss Islington (bot)" - <31488909+miss-islington@users.noreply.github.com> -Date: Fri, 27 Sep 2019 13:18:14 -0700 -Subject: [PATCH] bpo-38243, xmlrpc.server: Escape the server_title (GH-16373) - -Escape the server title of xmlrpc.server.DocXMLRPCServer -when rendering the document page as HTML. -(cherry picked from commit e8650a4f8c7fb76f570d4ca9c1fbe44e91c8dfaa) - -Co-authored-by: Dong-hee Na ---- - Lib/test/test_docxmlrpc.py | 16 ++++++++++++++++ - Lib/xmlrpc/server.py | 3 ++- - .../2019-09-25-13-21-09.bpo-38243.1pfz24.rst | 3 +++ - 3 files changed, 21 insertions(+), 1 deletion(-) - create mode 100644 Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst - ---- a/Lib/test/test_docxmlrpc.py -+++ b/Lib/test/test_docxmlrpc.py -@@ -1,5 +1,6 @@ - from xmlrpc.server import DocXMLRPCServer - import http.client -+import re - import sys - import threading - from test import support -@@ -193,6 +194,21 @@ class DocXMLRPCHTTPGETServer(unittest.Te - b'method_annotation(x: bytes)'), - response.read()) - -+ def test_server_title_escape(self): -+ # bpo-38243: Ensure that the server title and documentation -+ # are escaped for HTML. -+ self.serv.set_server_title('test_title