python: Add pyproject.toml-based builds for host Python packages

Using pip to install host packages with pyproject.toml-based (PEP 517)
builds is problematic:

* If build isolation is used, pip will create an isolated build
  environment, install any build dependencies for the requested package,
  then build the requested package.

  It does not appear currently possible to have pip install the build
  dependencies with hash-checking mode enabled[1].

* If build isolation is not used, any build dependencies must be
  installed in the build environment before invoking pip to build the
  requested package[2].

  This would require creating a package dependency resolution system to
  install build dependencies, and any dependencies of dependencies, in
  the correct order.

* It is very difficult to patch the packages installed by pip.

This adds a new include file (python3-host-build.mk) with recipes to
install host Python packages with pyproject.toml-based builds. This is
backwards-compatible with packages that require running setup.py.

Besides addressing the above issues (the OpenWrt build system already
resolves dependencies between packages, checks all source downloads
against known hashes, and supports patching packages), host packages
also:

* Capture package licensing and maintainer information
* Enable uscan checking for package updates/CVEs
* Are a known concept for OpenWrt packagers/developers

The existing functionality of using host pip to install packages will
remain for now, but should be considered deprecated and expected to be
removed in the future.

This also updates Py3Build/CheckHostPipVersionMatch for the case where
the host-pip-requirements directory does not exist or is empty.

[1]: https://pip.pypa.io/en/stable/user_guide/#changes-to-the-pip-dependency-resolver-in-20-3-2020
[2]: https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-no-build-isolation

Signed-off-by: Jeffery To <jeffery.to@gmail.com>
(cherry picked from commit fe78c07a316b1722b8d35b63772a7067dfd87c5e)

diff --git a/lang/python/python3-host-build.mk b/lang/python/python3-host-build.mk
new file mode 100644 (file)
index 0000000..738d157
--- /dev/null
+++ b/lang/python/python3-host-build.mk
@@ -0,0 +1,99 @@
+#
+# Copyright (C) 2023 Jeffery To
+#
+# This is free software, licensed under the GNU General Public License v2.
+# See /LICENSE for more information.
+#
+
+python3_mk_path:=$(dir $(lastword $(MAKEFILE_LIST)))
+include $(python3_mk_path)python3-host.mk
+
+PYTHON3_HOST_BUILD?=1
+
+PYTHON3_HOST_BUILD_CONFIG_SETTINGS?=
+PYTHON3_HOST_BUILD_VARS?=
+PYTHON3_HOST_BUILD_ARGS?=
+PYTHON3_HOST_BUILD_PATH?=
+
+PYTHON3_HOST_INSTALL_VARS?=
+
+PYTHON3_HOST_WHEEL_NAME?=$(subst -,_,$(if $(PYPI_SOURCE_NAME),$(PYPI_SOURCE_NAME),$(PKG_NAME)))
+PYTHON3_HOST_WHEEL_VERSION?=$(PKG_VERSION)
+
+PYTHON3_HOST_BUILD_DIR?=$(HOST_BUILD_DIR)/$(PYTHON3_HOST_BUILD_PATH)
+
+
+PYTHON3_HOST_DIR_NAME:=$(lastword $(subst /,$(space),$(CURDIR)))
+PYTHON3_HOST_STAGING_DIR:=$(TMP_DIR)/host-stage-$(PYTHON3_HOST_DIR_NAME)
+PYTHON3_HOST_STAGING_FILES_LIST_DIR:=$(HOST_BUILD_PREFIX)/stamp
+PYTHON3_HOST_STAGING_FILES_LIST:=$(PYTHON3_HOST_STAGING_FILES_LIST_DIR)/$(PYTHON3_HOST_DIR_NAME).list
+
+define Py3Host/Compile/Bootstrap
+       $(call HostPython3/Run, \
+               $(HOST_BUILD_DIR), \
+               -m flit_core.wheel \
+                       --outdir "$(PYTHON3_HOST_BUILD_DIR)"/openwrt-build \
+                       "$(PYTHON3_HOST_BUILD_DIR)" \
+       )
+endef
+
+define Py3Host/Compile
+       $(call HostPython3/Run, \
+               $(HOST_BUILD_DIR), \
+               -m build \
+                       --no-isolation \
+                       --outdir "$(PYTHON3_HOST_BUILD_DIR)"/openwrt-build \
+                       --wheel \
+                       $(foreach setting,$(PYTHON3_HOST_BUILD_CONFIG_SETTINGS),--config-setting=$(setting)) \
+                       $(PYTHON3_HOST_BUILD_ARGS) \
+                       "$(PYTHON3_HOST_BUILD_DIR)" \
+                       , \
+               $(PYTHON3_HOST_BUILD_VARS) \
+       )
+endef
+
+define Py3Host/Install/Installer
+       $(call HostPython3/Run, \
+               $(HOST_BUILD_DIR), \
+               -m installer \
+                       --destdir "$(1)" \
+                       --prefix "" \
+                       "$(PYTHON3_HOST_BUILD_DIR)"/openwrt-build/$(PYTHON3_HOST_WHEEL_NAME)-$(PYTHON3_HOST_WHEEL_VERSION)-*.whl \
+                       , \
+               $(PYTHON3_HOST_INSTALL_VARS) \
+       )
+endef
+
+define Py3Host/Install
+       rm -rf "$(PYTHON3_HOST_STAGING_DIR)"
+       mkdir -p "$(PYTHON3_HOST_STAGING_DIR)" "$(PYTHON3_HOST_STAGING_FILES_LIST_DIR)"
+
+       $(call Py3Host/Install/Installer,$(PYTHON3_HOST_STAGING_DIR))
+
+       $(call Py3Host/Uninstall,$(1))
+
+       cd "$(PYTHON3_HOST_STAGING_DIR)" && find ./ > "$(PYTHON3_HOST_STAGING_DIR).files"
+
+       $(call locked, \
+               mv "$(PYTHON3_HOST_STAGING_DIR).files" "$(PYTHON3_HOST_STAGING_FILES_LIST)" && \
+               $(CP) "$(PYTHON3_HOST_STAGING_DIR)"/* "$(1)/", \
+               host-staging-dir \
+       )
+
+       rm -rf "$(PYTHON3_HOST_STAGING_DIR)"
+endef
+
+define Py3Host/Uninstall
+       if [ -f "$(PYTHON3_HOST_STAGING_FILES_LIST)" ]; then \
+               "$(SCRIPT_DIR)/clean-package.sh" \
+                       "$(PYTHON3_HOST_STAGING_FILES_LIST)" \
+                       "$(1)" ; \
+               rm -f "$(PYTHON3_HOST_STAGING_FILES_LIST)" ; \
+       fi
+endef
+
+ifeq ($(strip $(PYTHON3_HOST_BUILD)),1)
+  Host/Compile=$(Py3Host/Compile)
+  Host/Install=$(Py3Host/Install)
+  Host/Uninstall=$(call Py3Host/Uninstall,$(HOST_BUILD_PREFIX))
+endif

diff --git a/lang/python/python3-host.mk b/lang/python/python3-host.mk
index fdf6658fd85ee8e00664c4f776722ef0d2db7abb..8dc5e732fda3ba3d379b1a895f67a92d426e475a 100644 (file)
--- a/lang/python/python3-host.mk
+++ b/lang/python/python3-host.mk
@@ -118,14 +118,3 @@ define HostPython3/PipInstall
                pip \
        )
 endef
-
-# $(1) => build subdir
-# $(2) => additional arguments to setup.py
-# $(3) => additional variables
-define HostPython3/ModSetup
-       $(call SetupPyShim,$(HOST_BUILD_DIR)/$(strip $(1)))
-       $(call HostPython3/Run, \
-               $(HOST_BUILD_DIR)/$(strip $(1)), \
-               setup.py $(2), \
-               $(3) PY_PKG_VERSION=$(PKG_VERSION))
-endef

diff --git a/lang/python/python3-package.mk b/lang/python/python3-package.mk
index 8477d8c75d02c019ac502395d375a4f4155a3e65..f2336b8fcf1569c26a12bbfde188c3b390c4af04 100644 (file)
--- a/lang/python/python3-package.mk
+++ b/lang/python/python3-package.mk
@@ -206,10 +206,13 @@ endef
 
 ifneq ($(strip $(PYPI_NAME)),)
 define Py3Build/CheckHostPipVersionMatch
-       if grep -q "$(PYPI_NAME)==" $(python3_mk_path)host-pip-requirements/*.txt ; then \
-               if ! grep -q "$(PYPI_NAME)==$(PKG_VERSION)" $(python3_mk_path)host-pip-requirements/*.txt ; then \
-                       printf "\nPlease update version of $(PYPI_NAME) to $(PKG_VERSION) in 'host-pip-requirements'/\n\n" ; \
-                       exit 1 ; \
+       if [ -d "$(python3_mk_path)host-pip-requirements" ] && \
+                       [ -n "$$$$($(FIND) $(python3_mk_path)host-pip-requirements -maxdepth 1 -mindepth 1 -name '*.txt' -print -quit 2>/dev/null)" ]; then \
+               if grep -q "$(PYPI_NAME)==" $(python3_mk_path)host-pip-requirements/*.txt ; then \
+                       if ! grep -q "$(PYPI_NAME)==$(PKG_VERSION)" $(python3_mk_path)host-pip-requirements/*.txt ; then \
+                               printf "\nPlease update version of $(PYPI_NAME) to $(PKG_VERSION) in 'host-pip-requirements'/\n\n" ; \
+                               exit 1 ; \
+                       fi \
                fi \
        fi
 endef