From 9390bd0262ad9f0775b76984d1c8f29b9a35584e Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Tue, 30 May 2023 03:24:54 +0800 Subject: [PATCH] micropython-lib: Update to master, split target package The package has been reworked to install the same files that are available to be downloaded/installed by mip, the package manager new to MicroPython 1.20.0. This also splits the original target package into four: * micropython-lib * Includes packages common to all MicroPython ports (python-stdlib, python-ecosys, micropython) * Contains mpy bytecode files * micropython-lib-src * Includes packages common to all MicroPython ports (python-stdlib, python-ecosys, micropython) * Contains py source files * micropython-lib-unix * Includes packages specific to the MicroPython Unix port (unix-ffi) * Contains mpy bytecode files * Installs a specific launcher (micropython-unix) that adds the Unix package directory into MicroPython's library path * micropython-lib-unix-src * Includes packages specific to the MicroPython Unix port (unix-ffi) * Contains py source files This also updates the package license, title, and description. Patches: * 001-build-unix-ffi.patch This enables the repo build script to also build the Unix-specific packages. Not sure if upstream is open to accepting this since the Unix-specific packages don't fit well into the existing package distribution mechanism. * 002-add-unix-ffi-os-path.patch and 003-add-unix-ffi-uu.patch These fix instances where the unix-ffi version of the os package is overridden by the python-stdlib version. These have been submitted to upstream: https://github.com/micropython/micropython-lib/pull/672 Signed-off-by: Jeffery To --- lang/python/micropython-lib/Makefile | 115 +++++++++++++++--- .../micropython-lib/files/micropython-unix | 3 + lang/python/micropython-lib/install.py | 62 ++++++++++ .../patches/001-build-unix-ffi.patch | 37 ++++++ .../patches/002-add-unix-ffi-os-path.patch | 53 ++++++++ .../patches/003-add-unix-ffi-uu.patch | 37 ++++++ 6 files changed, 291 insertions(+), 16 deletions(-) create mode 100644 lang/python/micropython-lib/files/micropython-unix create mode 100644 lang/python/micropython-lib/install.py create mode 100644 lang/python/micropython-lib/patches/001-build-unix-ffi.patch create mode 100644 lang/python/micropython-lib/patches/002-add-unix-ffi-os-path.patch create mode 100644 lang/python/micropython-lib/patches/003-add-unix-ffi-uu.patch diff --git a/lang/python/micropython-lib/Makefile b/lang/python/micropython-lib/Makefile index 1eb2bea73a..7bd167898a 100644 --- a/lang/python/micropython-lib/Makefile +++ b/lang/python/micropython-lib/Makefile @@ -12,42 +12,125 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/micropython/micropython-lib.git -PKG_SOURCE_VERSION:=db4c739863e49fc874bdaae8aa8c316c7ed4276a -PKG_SOURCE_DATE:=20220506 -PKG_MIRROR_HASH:=ec4be91755fcd4d4fa61e1e7eadc748377ba63b82b7bb4254864473fafcd3173 +PKG_SOURCE_VERSION:=7128d423c2e7c0309ac17a1e6ba873b909b24fcc +PKG_SOURCE_DATE:=20230522 +PKG_MIRROR_HASH:=1f094aac257d2094ee91b457164f845f6461df1cf1d0ed7ee556c98f273f5afb PKG_MAINTAINER:=Jeffery To -PKG_LICENSE:=MIT PSF-2.0 +PKG_LICENSE:=MIT Python-2.0.1 PKG_LICENSE_FILES:=LICENSE +PKG_BUILD_DEPENDS:=python3/host PKG_BUILD_PARALLEL:=1 +# keep in sync with micropython +MP_VERSION:=1.20.0 +MP_MPY_FILE_VERSION:=6 + include $(INCLUDE_DIR)/package.mk -define Package/micropython-lib +define Package/micropython-lib/Default SUBMENU:=Python SECTION:=lang CATEGORY:=Languages - TITLE:=micropython-lib + TITLE:=MicroPython package repository URL:=https://github.com/micropython/micropython-lib +endef + +define Package/micropython-lib +$(call Package/micropython-lib/Default) DEPENDS:=+micropython endef +define Package/micropython-lib-src +$(call Package/micropython-lib/Default) + TITLE+= (sources) +endef + +define Package/micropython-lib-unix +$(call Package/micropython-lib/Default) + TITLE+= - Unix port packages + DEPENDS:=+micropython +libpcre +librt +libsqlite3 +endef + +define Package/micropython-lib-unix-src +$(call Package/micropython-lib/Default) + TITLE+= - Unix port packages (sources) +endef + +define Package/micropython-lib/Default/description +This is a repository of packages designed to be useful for writing +MicroPython applications. +endef + define Package/micropython-lib/description - This is a repository of libraries designed to be useful for writing - MicroPython applications. +$(call Package/micropython-lib/Default/description) + +This contains packages common to all MicroPython ports. +endef + +define Package/micropython-lib-src/description +$(call Package/micropython-lib/Default/description) + +This contains source files for packages common to all MicroPython ports. +endef + +define Package/micropython-lib-unix/description +$(call Package/micropython-lib/Default/description) + +This contains packages specific to the MicroPython Unix port. +endef + +define Package/micropython-lib-unix-src/description +$(call Package/micropython-lib/Default/description) + +This contains source files for packages specific to the MicroPython Unix +port. +endef + +MP_INSTALLDEV_PATH:=$(STAGING_DIR)/host/lib/micropython-$(MP_VERSION) + +define MicroPythonLib/Compile + cd "$(PKG_BUILD_DIR)" && python3 tools/build.py \ + --hash-prefix 64 \ + --micropython "$(MP_INSTALLDEV_PATH)" \ + --mpy-cross "$(MP_INSTALLDEV_PATH)/mpy-cross/build/mpy-cross" \ + --output "$(PKG_BUILD_DIR)/$(strip $(1))" \ + $(2) +endef + +define MicroPythonLib/Install + python3 install.py \ + --input "$(PKG_BUILD_DIR)/$(strip $(1))" \ + --output "$(strip $(3))" \ + --version "$(strip $(2))" endef -Build/Compile:=: +define Build/Compile + $(call MicroPythonLib/Compile) + $(call MicroPythonLib/Compile,unix-ffi-index,--unix-ffi) +endef define Package/micropython-lib/install - for dir in micropython python-ecosys python-stdlib unix-ffi; do \ - $(INSTALL_DIR) $(1)/usr/lib/micropython/$$$$dir ; \ - $(CP) $(PKG_BUILD_DIR)/$$$$dir/* $(1)/usr/lib/micropython/$$$$dir/ ; \ - done - $(FIND) $(1)/usr/lib/micropython \ - -not -type d \( -not -name '*.py' -o -name 'test_*' -o -name 'setup.py' \) -delete - $(FIND) $(1)/usr/lib/micropython -mindepth 1 -empty -type d -delete + $(call MicroPythonLib/Install,,$(MP_MPY_FILE_VERSION),$(1)/usr/lib/micropython) +endef + +define Package/micropython-lib-src/install + $(call MicroPythonLib/Install,,py,$(1)/usr/lib/micropython) +endef + +define Package/micropython-lib-unix/install + $(call MicroPythonLib/Install,unix-ffi-index,$(MP_MPY_FILE_VERSION),$(1)/usr/lib/micropython/unix) + + $(INSTALL_DIR) $(1)/usr/bin + $(INSTALL_BIN) ./files/micropython-unix $(1)/usr/bin/ +endef + +define Package/micropython-lib-unix-src/install + $(call MicroPythonLib/Install,unix-ffi-index,py,$(1)/usr/lib/micropython/unix) endef $(eval $(call BuildPackage,micropython-lib)) +$(eval $(call BuildPackage,micropython-lib-src)) +$(eval $(call BuildPackage,micropython-lib-unix)) +$(eval $(call BuildPackage,micropython-lib-unix-src)) diff --git a/lang/python/micropython-lib/files/micropython-unix b/lang/python/micropython-lib/files/micropython-unix new file mode 100644 index 0000000000..a0c372cf42 --- /dev/null +++ b/lang/python/micropython-lib/files/micropython-unix @@ -0,0 +1,3 @@ +#!/bin/sh +export MICROPYPATH=".frozen:~/.micropython/lib:/usr/lib/micropython/unix:/usr/lib/micropython" +exec micropython "$@" diff --git a/lang/python/micropython-lib/install.py b/lang/python/micropython-lib/install.py new file mode 100644 index 0000000000..f107072324 --- /dev/null +++ b/lang/python/micropython-lib/install.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 Jeffery To +# +# This is free software, licensed under the GNU General Public License v2. +# See /LICENSE for more information. +# + +import json +import os +import re +import shutil +import sys + + +def install(input_path, mpy_version, output_path): + index_json_path = os.path.join(input_path, "index.json") + files = {} + + with open(index_json_path) as f: + index_json = json.load(f) + + for p in index_json["packages"]: + package_name = p["name"] + package_json_path = os.path.join(input_path, "package", mpy_version, package_name, "latest.json") + + with open(package_json_path) as f: + package_json = json.load(f) + + for file_name, file_hash in package_json["hashes"]: + if file_name in files: + if file_hash != files[file_name]: + print("File name/hash collision:", package_name, file=sys.stderr) + print(" File: ", file_name, file=sys.stderr) + print(" Curent hash: ", file_hash, file=sys.stderr) + print(" Previous hash: ", files[file_name], file=sys.stderr) + sys.exit(1) + else: + files[file_name] = file_hash + + for file_name, file_hash in files.items(): + in_file_path = os.path.join(input_path, "file", file_hash[:2], file_hash) + out_file_path = os.path.join(output_path, file_name) + + os.makedirs(os.path.dirname(out_file_path), exist_ok=True) + shutil.copy2(in_file_path, out_file_path) + + +def main(): + import argparse + + cmd_parser = argparse.ArgumentParser(description="Install compiled micropython-lib packages.") + cmd_parser.add_argument("--input", required=True, help="input directory") + cmd_parser.add_argument("--version", required=True, help="mpy version to install") + cmd_parser.add_argument("--output", required=True, help="output directory") + args = cmd_parser.parse_args() + + install(args.input, args.version, args.output) + + +if __name__ == "__main__": + main() diff --git a/lang/python/micropython-lib/patches/001-build-unix-ffi.patch b/lang/python/micropython-lib/patches/001-build-unix-ffi.patch new file mode 100644 index 0000000000..4dfaca6f91 --- /dev/null +++ b/lang/python/micropython-lib/patches/001-build-unix-ffi.patch @@ -0,0 +1,37 @@ +--- a/tools/build.py ++++ b/tools/build.py +@@ -284,7 +284,7 @@ def _update_index_package_metadata(index + index_package_json["versions"][v].append(metadata.version) + + +-def build(output_path, hash_prefix_len, mpy_cross_path): ++def build(output_path, unix_ffi, hash_prefix_len, mpy_cross_path): + import manifestfile + import mpy_cross + +@@ -310,7 +310,7 @@ def build(output_path, hash_prefix_len, + + # For now, don't process unix-ffi. In the future this can be extended to + # allow a way to request unix-ffi packages via mip. +- lib_dirs = ["micropython", "python-stdlib", "python-ecosys"] ++ lib_dirs = ["unix-ffi"] if unix_ffi else ["micropython", "python-stdlib", "python-ecosys"] + + mpy_version, _mpy_sub_version = mpy_cross.mpy_version(mpy_cross=mpy_cross_path) + mpy_version = str(mpy_version) +@@ -438,6 +438,7 @@ def main(): + + cmd_parser = argparse.ArgumentParser(description="Compile micropython-lib for serving to mip.") + cmd_parser.add_argument("--output", required=True, help="output directory") ++ cmd_parser.add_argument("--unix-ffi", action="store_true", help="process unix-ffi packages") + cmd_parser.add_argument("--hash-prefix", default=8, type=int, help="hash prefix length") + cmd_parser.add_argument("--mpy-cross", default=None, help="optional path to mpy-cross binary") + cmd_parser.add_argument("--micropython", default=None, help="path to micropython repo") +@@ -447,7 +448,7 @@ def main(): + sys.path.append(os.path.join(args.micropython, "tools")) # for manifestfile + sys.path.append(os.path.join(args.micropython, "mpy-cross")) # for mpy_cross + +- build(args.output, hash_prefix_len=max(4, args.hash_prefix), mpy_cross_path=args.mpy_cross) ++ build(args.output, args.unix_ffi, hash_prefix_len=max(4, args.hash_prefix), mpy_cross_path=args.mpy_cross) + + + if __name__ == "__main__": diff --git a/lang/python/micropython-lib/patches/002-add-unix-ffi-os-path.patch b/lang/python/micropython-lib/patches/002-add-unix-ffi-os-path.patch new file mode 100644 index 0000000000..5ebc6820ec --- /dev/null +++ b/lang/python/micropython-lib/patches/002-add-unix-ffi-os-path.patch @@ -0,0 +1,53 @@ +From dcce62dd525cf0f8e572e56a8990aea7ec2f0ade Mon Sep 17 00:00:00 2001 +From: Jeffery To +Date: Tue, 30 May 2023 23:47:59 +0800 +Subject: [PATCH] unix-ffi/os-path: Add unix-ffi version of `os-path` package. + +This package reuses the code from the python-stdlib version of `os-path` +but requires the unix-ffi version of `os`. + +This also updates `glob` to require this version of `os-path`. + +Signed-off-by: Jeffery To +--- + unix-ffi/glob/manifest.py | 2 +- + unix-ffi/os-path/manifest.py | 6 ++++++ + unix-ffi/os/os/__init__.py | 6 ++++++ + 3 files changed, 13 insertions(+), 1 deletion(-) + create mode 100644 unix-ffi/os-path/manifest.py + +--- a/unix-ffi/glob/manifest.py ++++ b/unix-ffi/glob/manifest.py +@@ -1,7 +1,7 @@ + metadata(version="0.5.2") + + require("os", unix_ffi=True) +-require("os-path") ++require("os-path", unix_ffi=True) + require("re", unix_ffi=True) + require("fnmatch") + +--- /dev/null ++++ b/unix-ffi/os-path/manifest.py +@@ -0,0 +1,6 @@ ++metadata(version="0.1.4") ++ ++# Originally written by Paul Sokolovsky. ++ ++require("os", unix_ffi=True) ++package("os", base_path="../../python-stdlib/os-path") +--- a/unix-ffi/os/os/__init__.py ++++ b/unix-ffi/os/os/__init__.py +@@ -5,6 +5,12 @@ import stat as stat_ + import ffilib + import uos + ++# Provide optional dependencies (which may be installed separately). ++try: ++ from . import path ++except ImportError: ++ pass ++ + R_OK = const(4) + W_OK = const(2) + X_OK = const(1) diff --git a/lang/python/micropython-lib/patches/003-add-unix-ffi-uu.patch b/lang/python/micropython-lib/patches/003-add-unix-ffi-uu.patch new file mode 100644 index 0000000000..363fae394a --- /dev/null +++ b/lang/python/micropython-lib/patches/003-add-unix-ffi-uu.patch @@ -0,0 +1,37 @@ +From 2e7bfd08a306bd9e80b22097ef8fe66e1dd85054 Mon Sep 17 00:00:00 2001 +From: Jeffery To +Date: Wed, 31 May 2023 00:00:11 +0800 +Subject: [PATCH] unix-ffi/uu: Add unix-ffi version of `uu` package. + +This package reuses the code from the python-stdlib version of `uu` but +requires the unix-ffi version of `os-path`. + +This also updates `email.message` to require this version of `uu`. + +Signed-off-by: Jeffery To +--- + unix-ffi/email.message/manifest.py | 2 +- + unix-ffi/uu/manifest.py | 6 ++++++ + 2 files changed, 7 insertions(+), 1 deletion(-) + create mode 100644 unix-ffi/uu/manifest.py + +--- a/unix-ffi/email.message/manifest.py ++++ b/unix-ffi/email.message/manifest.py +@@ -1,7 +1,7 @@ + metadata(version="0.5.3") + + require("re", unix_ffi=True) +-require("uu") ++require("uu", unix_ffi=True) + require("base64") + require("binascii") + require("email.utils", unix_ffi=True) +--- /dev/null ++++ b/unix-ffi/uu/manifest.py +@@ -0,0 +1,6 @@ ++metadata(version="0.5.1") ++ ++require("binascii") ++require("os-path", unix_ffi=True) ++ ++module("uu.py", base_path="../../python-stdlib/uu") -- 2.30.2