python-packages: initial version 1.0

Signed-off-by: Yousong Zhou <yszhou4tech@gmail.com>

diff --git a/lang/python-packages/Makefile b/lang/python-packages/Makefile
new file mode 100644 (file)
index 0000000..f12b78e
--- /dev/null
+++ b/lang/python-packages/Makefile
@@ -0,0 +1,131 @@
+#
+# Copyright (C) 2016 Yousong Zhou <yszhou4tech@gmail.com>
+#
+# This is free software, licensed under the GNU General Public License v2.
+# See /LICENSE for more information.
+#
+
+include $(TOPDIR)/rules.mk
+
+PKG_NAME:=python-packages
+PKG_VERSION:=1.0
+PKG_RELEASE:=1
+
+PKG_MAINTAINER:=Yousong Zhou <yszhou4tech@gmail.com>
+
+#
+# NOTE: move the host module installation to Host/Compile when
+# HOST_CONFIG_DEPENDS is supported
+#
+# NOTE: PKG_CONFIG_DEPENDS cannot correctly track changes of string type config
+# options, so you may want to do manual cleanup on config change.
+#
+PKG_CONFIG_DEPENDS:= \
+       CONFIG_PACKAGE_python-packages-list-host \
+       CONFIG_PACKAGE_python-packages-list \
+       CONFIG_PACKAGE_python-packages-list-cleanup \
+       CONFIG_PACKAGE_python-packages-envs \
+       CONFIG_PACKAGE_python-packages-extra-deps \
+       CONFIG_PACKAGE_python-packages-index-url \
+       CONFIG_PACKAGE_python-packages-pip-opts \
+
+PKG_BUILD_DEPENDS:=python python-pip/host
+
+include $(INCLUDE_DIR)/package.mk
+$(call include_mk, python-host.mk)
+$(call include_mk, python-package.mk)
+
+define Package/python-packages
+  SUBMENU:=Python
+  SECTION:=lang
+  CATEGORY:=Languages
+  TITLE:=A dummy package for packaging python modules with pip
+  DEPENDS:=@DEVEL +python
+endef
+
+define Package/python-packages/config
+if PACKAGE_python-packages
+config PACKAGE_python-packages-list-host
+       string "List of python packages to install on host"
+config PACKAGE_python-packages-list
+       string "List of python packages to install on target"
+config PACKAGE_python-packages-list-cleanup
+       string "List of python packages to cleanup to avoid clash with existing packages"
+config PACKAGE_python-packages-envs
+       string "Extra environment variables to pass on to pip and its children on target build"
+config PACKAGE_python-packages-extra-deps
+       string "List of deps fulfilled but not tracked by the build system"
+config PACKAGE_python-packages-index-url
+       string "Index URL passed to pip with --index-url"
+config PACKAGE_python-packages-pip-opts
+       string "Additional arguments to pip command line"
+endif
+endef
+
+CONFIG_PACKAGE_python-packages-list-host:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list-host))
+CONFIG_PACKAGE_python-packages-list:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list))
+CONFIG_PACKAGE_python-packages-list-cleanup:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list-cleanup))
+CONFIG_PACKAGE_python-packages-envs:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-envs))
+CONFIG_PACKAGE_python-packages-extra-deps:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-extra-deps))
+CONFIG_PACKAGE_python-packages-pip-opts:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-pip-opts))
+
+HOST_PYTHON_PIP:=$(STAGING_DIR)/host/bin/pip$(PYTHON_VERSION)
+
+decr=$(word $(1),0 1 2 3 4 5 6 7 8 9 10)
+recur=$(if $(subst 0,,$(2)),$(call recur,$(1),$(call decr,$(2)),$(call $(1)$(2),$(3))),$(3))
+_req2dir1=$(subst >,gt,$(1))
+_req2dir2=$(subst <,lt,$(1))
+_req2dir3=$(subst >=,geq,$(1))
+_req2dir4=$(subst <=,leq,$(1))
+_req2dir5=$(subst ://,:::,$(1))
+_req2dir6=$(subst *,_,$(1))
+_req2dir7=$(subst ?,_,$(1))
+req2dir=$(call recur,_req2dir,7,$(1))
+
+# --ignore-installed, it may happen that host pip will ignore target install if
+#  it was already installed as host module, e.g. cffi deps of cryptograph
+HOST_PYTHON_PIP_INSTALL=$(HOST_PYTHON_PIP) install \
+       --root=$(1) \
+       --prefix=$(2) \
+       --ignore-installed \
+       --no-compile \
+       $(if $(CONFIG_PACKAGE_python-packages-index-url), --index-url $(CONFIG_PACKAGE_python-packages-index-url)) \
+       $(if $(CONFIG_PACKAGE_python-packages-pip-opts), $(CONFIG_PACKAGE_python-packages-pip-opts)) \
+
+HOST_PYTHON_PIP_INSTALL_HOST:=$(call HOST_PYTHON_PIP_INSTALL,$(STAGING_DIR)/host,"")
+HOST_PYTHON_PIP_INSTALL_TARGET=$(call HOST_PYTHON_PIP_INSTALL,$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg)),/usr)
+HOST_PYTHON_PIP_INSTALL_CLEANUP:=$(call HOST_PYTHON_PIP_INSTALL,$(PKG_INSTALL_DIR)/_cleanup,/usr)
+
+define Build/Compile
+       $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list-host),
+               $(call Build/Compile/HostPyRunHost,,$(HOST_PYTHON_PIP_INSTALL_HOST) $(pkg))
+       )
+       $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list),
+               $(call Build/Compile/HostPyRunTarget,,$(call HOST_PYTHON_PIP_INSTALL_TARGET,$(pkg)) $(pkg),$(CONFIG_PACKAGE_python-packages-envs))
+       )
+       $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list-cleanup),
+               $(call Build/Compile/HostPyRunTarget,,$(HOST_PYTHON_PIP_INSTALL_CLEANUP) $(pkg),$(CONFIG_PACKAGE_python-packages-envs))
+       )
+endef
+
+define Package/python-packages/install
+       $(foreach pkg,$(CONFIG_PACKAGE_python-packages-list),
+               $(CP) "$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg))"/* $(1)
+       )
+
+       find "$(PKG_INSTALL_DIR)/_cleanup" -mindepth 1 -depth | while read sf; do \
+               tf="$$$${sf#$(PKG_INSTALL_DIR)/_cleanup/}"; \
+               tf="$(1)/$$$$tf"; \
+               if [ -f "$$$$tf" -o -L "$$$$tf" ]; then \
+                       rm -vf "$$$$tf"; \
+               elif [ -d "$$$$tf" ]; then \
+                       rmdir -v -p "$$$$tf" || true; \
+               fi \
+       done
+endef
+
+define Package/python-packages/extra_provides
+       echo $(CONFIG_PACKAGE_python-packages-extra-deps) | tr ' ' '\n'
+endef
+
+$(eval $(call BuildPackage,python-packages))

diff --git a/lang/python-packages/README.md b/lang/python-packages/README.md
new file mode 100644 (file)
index 0000000..3c09d8e
--- /dev/null
+++ b/lang/python-packages/README.md
@@ -0,0 +1,72 @@
+This package allows users to package python modules without creating package
+Makefiles for each individual module and their dependencies.  It provides a
+way making packaging python packages faster and may also facilitate the process
+of developing Makefiles for new python packages
+
+This is a raw DEVEL only package.  Using it may entail a lot of implementation
+details and you may need to resolve target dependencies and package details on
+your own
+
+- Third party python packages may depend on features not included in e.g.
+  python-light
+- Some python modules may require host install of another module to progress,
+  e.g. target cryptography requires host cffi
+- Some python modules have external C library dependencies, e.g. pyOpenSSL
+  requires openssl libs
+- Some packages may have an autoconf configure script whose arguments we
+  cannot control with pip and has to be passed on (hacked) by overriding some
+  environment variables
+
+## How it works
+
+1. Install host modules required for building target modules
+2. Install each target module to separate directories
+3. Install another copy of modules for cleanup purposes to make list of
+   installed files to be removed from target modules installed in step 2
+
+Why should it be so
+
+1. Installing target cryptography requires host installation of cffi module
+2. cryptography requires setuptools and pip will install its own copy with
+   --ignore-installed.  When PACKAGE_python-setuptools is also selected, opkg
+   will complain of data file clashes if it was not removed here.
+
+Pip will handle dependency requirements of python modules, but external
+dependencies like c libraries has to be prepared by the build system.  The
+issue is that there is currently no way to express such dependencies, thus may
+cause build failure, e.g. pycrypto requires the presence of libgmp to build
+successfully.
+
+## Tips
+
+If something goes wrong, we can add additional arguments to pip command
+line to check the detailed build process.  Some useful arguments may be
+
+- -v, for verbose output.  Repeat this option if the current level of
+  verbosity is not enough
+- --no-clean, for preserving pip build dir on build failure
+
+## Examples
+
+tornado (python-only module)
+
+       CONFIG_PACKAGE_python-packages=y
+       CONFIG_PACKAGE_python-packages-list="tornado==4.4.2"
+
+cryptography (requires installation of host modules and cleanup on target modules)
+
+       CONFIG_PACKAGE_python-packages=y
+       CONFIG_PACKAGE_python-packages-list-host="cffi"
+       CONFIG_PACKAGE_python-packages-list="cryptography"
+       CONFIG_PACKAGE_python-packages-list-cleanup="setuptools"
+
+pycrypto 2.7a1 (python module with autoconf configure script; depends on
+libgmp; broken wmmintrin.h).  2.6.1 does not work because of a flaw in
+the setup.py hardcoding host include directory
+
+       CONFIG_PACKAGE_libgmp=y
+       CONFIG_PACKAGE_python-packages=y
+       CONFIG_PACKAGE_python-packages-list="https://github.com/dlitz/pycrypto/archive/v2.7a1.tar.gz"
+       CONFIG_PACKAGE_python-packages-envs="ac_cv_header_wmmintrin_h=no build_alias=$(GNU_HOST_NAME) host_alias=$(GNU_TARGET_NAME) target_alias=$(GNU_TARGET_NAME)"
+       CONFIG_PACKAGE_python-packages-extra-deps="libgmp.so.10"
+