python-setuptools: Add reproducibility patches from Debian

This adds the ability to patch setuptools (and pip), and adds 3
reproducibility patches from Debian[1].
(003-PKG-INFO-output-reproducible.patch addresses the issue identified
in #9039.)

The patching is not perfect, in that the patches are applied to
setuptools and pip after they have been installed, since they are
installed from wheels which are already "precompiled".

Also, patching for the host install cannot be updated in place, for
example if a patch is added or removed.

[1]: https://sources.debian.org/patches/python-setuptools/40.8.0-1/

Signed-off-by: Jeffery To <jeffery.to@gmail.com>

diff --git a/lang/python/python-version.mk b/lang/python/python-version.mk
index 4182176232bab7c3213798ed41e0ed84f4da3e92..84b8e4f0535ae8c7871a17d54181a5504fe21b58 100644 (file)
--- a/lang/python/python-version.mk
+++ b/lang/python/python-version.mk
@@ -8,7 +8,7 @@
 PYTHON_VERSION:=2.7
 PYTHON_VERSION_MICRO:=16
 
-PYTHON_SETUPTOOLS_PKG_RELEASE:=2
+PYTHON_SETUPTOOLS_PKG_RELEASE:=3
 PYTHON_PIP_PKG_RELEASE:=2
 
 PYTHON_SETUPTOOLS_VERSION:=40.6.2

diff --git a/lang/python/python/Makefile b/lang/python/python/Makefile
index df825bc0dcd885bdfaef695a03382a104eb182c8..979ccc7714871ebbf58fb8af9c4b2000265c37a0 100644 (file)
--- a/lang/python/python/Makefile
+++ b/lang/python/python/Makefile
@@ -174,6 +174,7 @@ define Build/Compile/python-setuptools
                --ignore-installed \
                --root=$(PKG_BUILD_DIR)/install-setuptools --prefix=. \
                $(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/setuptools-$(PYTHON_SETUPTOOLS_VERSION)-py2.py3-none-any.whl
+       $(call PatchDir,$(PKG_BUILD_DIR)/install-setuptools/lib/python$(PYTHON_VERSION)/site-packages,./patches-setuptools,)
 endef
 endif # CONFIG_PACKAGE_python-setuptools
 
@@ -183,6 +184,7 @@ define Build/Compile/python-pip
                --ignore-installed \
                --root=$(PKG_BUILD_DIR)/install-pip --prefix=. \
                $(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/pip-$(PYTHON_PIP_VERSION)-py2.py3-none-any.whl
+       $(call PatchDir,$(PKG_BUILD_DIR)/install-pip/lib/python$(PYTHON_VERSION)/site-packages,./patches-pip,)
 endef
 endif # CONFIG_PACKAGE_python-pip
 
@@ -304,6 +306,14 @@ define Host/Install
        $(MAKE) -C $(HOST_BUILD_DIR) install
        $(INSTALL_DIR) $(HOST_PYTHON_DIR)/bin/
        $(INSTALL_BIN) $(HOST_BUILD_DIR)/Parser/pgen $(HOST_PYTHON_DIR)/bin/pgen2
+  ifeq ($(wildcard $(HOST_PYTHON_PKG_DIR)/.setuptools-patched),)
+       $(call HostPatchDir,$(HOST_PYTHON_PKG_DIR),./patches-setuptools,)
+       touch $(HOST_PYTHON_PKG_DIR)/.setuptools-patched
+  endif
+  ifeq ($(wildcard $(HOST_PYTHON_PKG_DIR)/.pip-patched),)
+       $(call HostPatchDir,$(HOST_PYTHON_PKG_DIR),./patches-pip,)
+       touch $(HOST_PYTHON_PKG_DIR)/.pip-patched
+  endif
 endef
 
 $(eval $(call HostBuild))

diff --git a/lang/python/python/patches-setuptools/001-reproducible.patch b/lang/python/python/patches-setuptools/001-reproducible.patch
new file mode 100644 (file)
index 0000000..32edc56
--- /dev/null
+++ b/lang/python/python/patches-setuptools/001-reproducible.patch
@@ -0,0 +1,16 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=848136
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/reproducible.diff/
+
+Index: b/setuptools/command/easy_install.py
+===================================================================
+--- a/setuptools/command/easy_install.py
++++ b/setuptools/command/easy_install.py
+@@ -436,7 +436,7 @@ consider to install to another location,
+             for spec in self.args:
+                 self.easy_install(spec, not self.no_deps)
+             if self.record:
+-                outputs = self.outputs
++                outputs = list(sorted(self.outputs))
+                 if self.root:  # strip any package prefix
+                     root_len = len(self.root)
+                     for counter in range(len(outputs)):

diff --git a/lang/python/python/patches-setuptools/002-sorted-requires.patch b/lang/python/python/patches-setuptools/002-sorted-requires.patch
new file mode 100644 (file)
index 0000000..2ad4795
--- /dev/null
+++ b/lang/python/python/patches-setuptools/002-sorted-requires.patch
@@ -0,0 +1,16 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=804249
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/sorted-requires.diff/
+
+Index: b/setuptools/command/egg_info.py
+===================================================================
+--- a/setuptools/command/egg_info.py
++++ b/setuptools/command/egg_info.py
+@@ -621,7 +621,7 @@ def warn_depends_obsolete(cmd, basename,
+ def _write_requirements(stream, reqs):
+     lines = yield_lines(reqs or ())
+     append_cr = lambda line: line + '\n'
+-    lines = map(append_cr, lines)
++    lines = map(append_cr, sorted(lines))
+     stream.writelines(lines)
+ 
+ 

diff --git a/lang/python/python/patches-setuptools/003-PKG-INFO-output-reproducible.patch b/lang/python/python/patches-setuptools/003-PKG-INFO-output-reproducible.patch
new file mode 100644 (file)
index 0000000..15f34dc
--- /dev/null
+++ b/lang/python/python/patches-setuptools/003-PKG-INFO-output-reproducible.patch
@@ -0,0 +1,14 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=894215
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/PKG-INFO-output-reproducible.diff/
+
+--- a/setuptools/dist.py
++++ b/setuptools/dist.py
+@@ -191,7 +191,7 @@ def write_pkg_file(self, file):
+             self.long_description_content_type
+         )
+     if self.provides_extras:
+-        for extra in self.provides_extras:
++        for extra in sorted(self.provides_extras):
+             write_field('Provides-Extra', extra)
+ 
+ 

diff --git a/lang/python/python3-version.mk b/lang/python/python3-version.mk
index 9eebeb26012ed0e36c0990a28976117464fb0dda..784ec1778d89dc8d46ca2f520c7f3bd7f0011bf6 100644 (file)
--- a/lang/python/python3-version.mk
+++ b/lang/python/python3-version.mk
@@ -12,7 +12,7 @@ PYTHON3_VERSION_MICRO:=2
 
 PYTHON3_VERSION:=$(PYTHON3_VERSION_MAJOR).$(PYTHON3_VERSION_MINOR)
 
-PYTHON3_SETUPTOOLS_PKG_RELEASE:=2
+PYTHON3_SETUPTOOLS_PKG_RELEASE:=3
 PYTHON3_PIP_PKG_RELEASE:=2
 
 PYTHON3_SETUPTOOLS_VERSION:=40.6.2

diff --git a/lang/python/python3/Makefile b/lang/python/python3/Makefile
index 4053ff0c9699e9240149d33b2a247e3be03adb72..d151459ed891259131038d2791a5e8d19180a395 100644 (file)
--- a/lang/python/python3/Makefile
+++ b/lang/python/python3/Makefile
@@ -178,6 +178,7 @@ define Build/Compile/python3-setuptools
                --ignore-installed \
                --root=$(PKG_BUILD_DIR)/install-setuptools --prefix=. \
                $(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/setuptools-$(PYTHON3_SETUPTOOLS_VERSION)-py2.py3-none-any.whl
+       $(call PatchDir,$(PKG_BUILD_DIR)/install-setuptools/lib/python$(PYTHON3_VERSION)/site-packages,./patches-setuptools,)
 endef
 endif # CONFIG_PACKAGE_python3-setuptools
 
@@ -187,6 +188,7 @@ define Build/Compile/python3-pip
                --ignore-installed \
                --root=$(PKG_BUILD_DIR)/install-pip --prefix=. \
                $(PKG_BUILD_DIR)/Lib/ensurepip/_bundled/pip-$(PYTHON3_PIP_VERSION)-py2.py3-none-any.whl
+       $(call PatchDir,$(PKG_BUILD_DIR)/install-pip/lib/python$(PYTHON3_VERSION)/site-packages,./patches-pip,)
 endef
 endif # CONFIG_PACKAGE_python3-pip
 
@@ -299,6 +301,14 @@ define Host/Install
        $(MAKE) -C $(HOST_BUILD_DIR) install
        $(INSTALL_DIR) $(HOST_PYTHON3_DIR)/bin/
        $(INSTALL_BIN) $(HOST_BUILD_DIR)/Parser/pgen $(HOST_PYTHON3_DIR)/bin/pgen3
+  ifeq ($(wildcard $(HOST_PYTHON3_PKG_DIR)/.setuptools-patched),)
+       $(call HostPatchDir,$(HOST_PYTHON3_PKG_DIR),./patches-setuptools,)
+       touch $(HOST_PYTHON3_PKG_DIR)/.setuptools-patched
+  endif
+  ifeq ($(wildcard $(HOST_PYTHON3_PKG_DIR)/.pip-patched),)
+       $(call HostPatchDir,$(HOST_PYTHON3_PKG_DIR),./patches-pip,)
+       touch $(HOST_PYTHON3_PKG_DIR)/.pip-patched
+  endif
 endef
 
 $(eval $(call HostBuild))

diff --git a/lang/python/python3/patches-setuptools/001-reproducible.patch b/lang/python/python3/patches-setuptools/001-reproducible.patch
new file mode 100644 (file)
index 0000000..32edc56
--- /dev/null
+++ b/lang/python/python3/patches-setuptools/001-reproducible.patch
@@ -0,0 +1,16 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=848136
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/reproducible.diff/
+
+Index: b/setuptools/command/easy_install.py
+===================================================================
+--- a/setuptools/command/easy_install.py
++++ b/setuptools/command/easy_install.py
+@@ -436,7 +436,7 @@ consider to install to another location,
+             for spec in self.args:
+                 self.easy_install(spec, not self.no_deps)
+             if self.record:
+-                outputs = self.outputs
++                outputs = list(sorted(self.outputs))
+                 if self.root:  # strip any package prefix
+                     root_len = len(self.root)
+                     for counter in range(len(outputs)):

diff --git a/lang/python/python3/patches-setuptools/002-sorted-requires.patch b/lang/python/python3/patches-setuptools/002-sorted-requires.patch
new file mode 100644 (file)
index 0000000..2ad4795
--- /dev/null
+++ b/lang/python/python3/patches-setuptools/002-sorted-requires.patch
@@ -0,0 +1,16 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=804249
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/sorted-requires.diff/
+
+Index: b/setuptools/command/egg_info.py
+===================================================================
+--- a/setuptools/command/egg_info.py
++++ b/setuptools/command/egg_info.py
+@@ -621,7 +621,7 @@ def warn_depends_obsolete(cmd, basename,
+ def _write_requirements(stream, reqs):
+     lines = yield_lines(reqs or ())
+     append_cr = lambda line: line + '\n'
+-    lines = map(append_cr, lines)
++    lines = map(append_cr, sorted(lines))
+     stream.writelines(lines)
+ 
+ 

diff --git a/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch b/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch
new file mode 100644 (file)
index 0000000..15f34dc
--- /dev/null
+++ b/lang/python/python3/patches-setuptools/003-PKG-INFO-output-reproducible.patch
@@ -0,0 +1,14 @@
+https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=894215
+https://sources.debian.org/patches/python-setuptools/40.8.0-1/PKG-INFO-output-reproducible.diff/
+
+--- a/setuptools/dist.py
++++ b/setuptools/dist.py
+@@ -191,7 +191,7 @@ def write_pkg_file(self, file):
+             self.long_description_content_type
+         )
+     if self.provides_extras:
+-        for extra in self.provides_extras:
++        for extra in sorted(self.provides_extras):
+             write_field('Provides-Extra', extra)
+ 
+