micropython-lib: Update to master, split target package

The package has been reworked to install the same files that are
available to be downloaded/installed by mip, the package manager new to
MicroPython 1.20.0.

This also splits the original target package into four:

* micropython-lib
  * Includes packages common to all MicroPython ports (python-stdlib,
    python-ecosys, micropython)
  * Contains mpy bytecode files

* micropython-lib-src
  * Includes packages common to all MicroPython ports (python-stdlib,
    python-ecosys, micropython)
  * Contains py source files

* micropython-lib-unix
  * Includes packages specific to the MicroPython Unix port (unix-ffi)
  * Contains mpy bytecode files
  * Installs a specific launcher (micropython-unix) that adds the Unix
    package directory into MicroPython's library path

* micropython-lib-unix-src
  * Includes packages specific to the MicroPython Unix port (unix-ffi)
  * Contains py source files

This also updates the package license, title, and description.

Patches:

* 001-build-unix-ffi.patch

  This enables the repo build script to also build the Unix-specific
  packages. Not sure if upstream is open to accepting this since the
  Unix-specific packages don't fit well into the existing package
  distribution mechanism.

* 002-add-unix-ffi-os-path.patch and 003-add-unix-ffi-uu.patch

  These fix instances where the unix-ffi version of the os package is
  overridden by the python-stdlib version. These have been submitted to
  upstream: https://github.com/micropython/micropython-lib/pull/672

Signed-off-by: Jeffery To <jeffery.to@gmail.com>

diff --git a/lang/python/micropython-lib/Makefile b/lang/python/micropython-lib/Makefile
index 1eb2bea73a055edeba59b7f3e324f88c337db520..7bd167898a871edee62cba3593449ba041286d96 100644 (file)
--- a/lang/python/micropython-lib/Makefile
+++ b/lang/python/micropython-lib/Makefile
@@ -12,42 +12,125 @@ PKG_RELEASE:=1
 
 PKG_SOURCE_PROTO:=git
 PKG_SOURCE_URL:=https://github.com/micropython/micropython-lib.git
-PKG_SOURCE_VERSION:=db4c739863e49fc874bdaae8aa8c316c7ed4276a
-PKG_SOURCE_DATE:=20220506
-PKG_MIRROR_HASH:=ec4be91755fcd4d4fa61e1e7eadc748377ba63b82b7bb4254864473fafcd3173
+PKG_SOURCE_VERSION:=7128d423c2e7c0309ac17a1e6ba873b909b24fcc
+PKG_SOURCE_DATE:=20230522
+PKG_MIRROR_HASH:=1f094aac257d2094ee91b457164f845f6461df1cf1d0ed7ee556c98f273f5afb
 
 PKG_MAINTAINER:=Jeffery To <jeffery.to@gmail.com>
-PKG_LICENSE:=MIT PSF-2.0
+PKG_LICENSE:=MIT Python-2.0.1
 PKG_LICENSE_FILES:=LICENSE
 
+PKG_BUILD_DEPENDS:=python3/host
 PKG_BUILD_PARALLEL:=1
 
+# keep in sync with micropython
+MP_VERSION:=1.20.0
+MP_MPY_FILE_VERSION:=6
+
 include $(INCLUDE_DIR)/package.mk
 
-define Package/micropython-lib
+define Package/micropython-lib/Default
   SUBMENU:=Python
   SECTION:=lang
   CATEGORY:=Languages
-  TITLE:=micropython-lib
+  TITLE:=MicroPython package repository
   URL:=https://github.com/micropython/micropython-lib
+endef
+
+define Package/micropython-lib
+$(call Package/micropython-lib/Default)
   DEPENDS:=+micropython
 endef
 
+define Package/micropython-lib-src
+$(call Package/micropython-lib/Default)
+  TITLE+= (sources)
+endef
+
+define Package/micropython-lib-unix
+$(call Package/micropython-lib/Default)
+  TITLE+= - Unix port packages
+  DEPENDS:=+micropython +libpcre +librt +libsqlite3
+endef
+
+define Package/micropython-lib-unix-src
+$(call Package/micropython-lib/Default)
+  TITLE+= - Unix port packages (sources)
+endef
+
+define Package/micropython-lib/Default/description
+This is a repository of packages designed to be useful for writing
+MicroPython applications.
+endef
+
 define Package/micropython-lib/description
-  This is a repository of libraries designed to be useful for writing
-  MicroPython applications.
+$(call Package/micropython-lib/Default/description)
+
+This contains packages common to all MicroPython ports.
+endef
+
+define Package/micropython-lib-src/description
+$(call Package/micropython-lib/Default/description)
+
+This contains source files for packages common to all MicroPython ports.
+endef
+
+define Package/micropython-lib-unix/description
+$(call Package/micropython-lib/Default/description)
+
+This contains packages specific to the MicroPython Unix port.
+endef
+
+define Package/micropython-lib-unix-src/description
+$(call Package/micropython-lib/Default/description)
+
+This contains source files for packages specific to the MicroPython Unix
+port.
+endef
+
+MP_INSTALLDEV_PATH:=$(STAGING_DIR)/host/lib/micropython-$(MP_VERSION)
+
+define MicroPythonLib/Compile
+       cd "$(PKG_BUILD_DIR)" && python3 tools/build.py \
+               --hash-prefix 64 \
+               --micropython "$(MP_INSTALLDEV_PATH)" \
+               --mpy-cross "$(MP_INSTALLDEV_PATH)/mpy-cross/build/mpy-cross" \
+               --output "$(PKG_BUILD_DIR)/$(strip $(1))" \
+               $(2)
+endef
+
+define MicroPythonLib/Install
+       python3 install.py \
+               --input "$(PKG_BUILD_DIR)/$(strip $(1))" \
+               --output "$(strip $(3))" \
+               --version "$(strip $(2))"
 endef
 
-Build/Compile:=:
+define Build/Compile
+       $(call MicroPythonLib/Compile)
+       $(call MicroPythonLib/Compile,unix-ffi-index,--unix-ffi)
+endef
 
 define Package/micropython-lib/install
-       for dir in micropython python-ecosys python-stdlib unix-ffi; do \
-               $(INSTALL_DIR) $(1)/usr/lib/micropython/$$$$dir ; \
-               $(CP) $(PKG_BUILD_DIR)/$$$$dir/* $(1)/usr/lib/micropython/$$$$dir/ ; \
-       done
-       $(FIND) $(1)/usr/lib/micropython \
-               -not -type d \( -not -name '*.py' -o -name 'test_*' -o -name 'setup.py' \) -delete
-       $(FIND) $(1)/usr/lib/micropython -mindepth 1 -empty -type d -delete
+       $(call MicroPythonLib/Install,,$(MP_MPY_FILE_VERSION),$(1)/usr/lib/micropython)
+endef
+
+define Package/micropython-lib-src/install
+       $(call MicroPythonLib/Install,,py,$(1)/usr/lib/micropython)
+endef
+
+define Package/micropython-lib-unix/install
+       $(call MicroPythonLib/Install,unix-ffi-index,$(MP_MPY_FILE_VERSION),$(1)/usr/lib/micropython/unix)
+
+       $(INSTALL_DIR) $(1)/usr/bin
+       $(INSTALL_BIN) ./files/micropython-unix $(1)/usr/bin/
+endef
+
+define Package/micropython-lib-unix-src/install
+       $(call MicroPythonLib/Install,unix-ffi-index,py,$(1)/usr/lib/micropython/unix)
 endef
 
 $(eval $(call BuildPackage,micropython-lib))
+$(eval $(call BuildPackage,micropython-lib-src))
+$(eval $(call BuildPackage,micropython-lib-unix))
+$(eval $(call BuildPackage,micropython-lib-unix-src))

diff --git a/lang/python/micropython-lib/files/micropython-unix b/lang/python/micropython-lib/files/micropython-unix
new file mode 100644 (file)
index 0000000..a0c372c
--- /dev/null
+++ b/lang/python/micropython-lib/files/micropython-unix
@@ -0,0 +1,3 @@
+#!/bin/sh
+export MICROPYPATH=".frozen:~/.micropython/lib:/usr/lib/micropython/unix:/usr/lib/micropython"
+exec micropython "$@"

diff --git a/lang/python/micropython-lib/install.py b/lang/python/micropython-lib/install.py
new file mode 100644 (file)
index 0000000..f107072
--- /dev/null
+++ b/lang/python/micropython-lib/install.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 Jeffery To
+#
+# This is free software, licensed under the GNU General Public License v2.
+# See /LICENSE for more information.
+#
+
+import json
+import os
+import re
+import shutil
+import sys
+
+
+def install(input_path, mpy_version, output_path):
+    index_json_path = os.path.join(input_path, "index.json")
+    files = {}
+
+    with open(index_json_path) as f:
+        index_json = json.load(f)
+
+    for p in index_json["packages"]:
+        package_name = p["name"]
+        package_json_path = os.path.join(input_path, "package", mpy_version, package_name, "latest.json")
+
+        with open(package_json_path) as f:
+            package_json = json.load(f)
+
+        for file_name, file_hash in package_json["hashes"]:
+            if file_name in files:
+                if file_hash != files[file_name]:
+                    print("File name/hash collision:", package_name, file=sys.stderr)
+                    print("  File:                  ", file_name, file=sys.stderr)
+                    print("  Curent hash:           ", file_hash, file=sys.stderr)
+                    print("  Previous hash:         ", files[file_name], file=sys.stderr)
+                    sys.exit(1)
+            else:
+                files[file_name] = file_hash
+
+    for file_name, file_hash in files.items():
+        in_file_path = os.path.join(input_path, "file", file_hash[:2], file_hash)
+        out_file_path = os.path.join(output_path, file_name)
+
+        os.makedirs(os.path.dirname(out_file_path), exist_ok=True)
+        shutil.copy2(in_file_path, out_file_path)
+
+
+def main():
+    import argparse
+
+    cmd_parser = argparse.ArgumentParser(description="Install compiled micropython-lib packages.")
+    cmd_parser.add_argument("--input", required=True, help="input directory")
+    cmd_parser.add_argument("--version", required=True, help="mpy version to install")
+    cmd_parser.add_argument("--output", required=True, help="output directory")
+    args = cmd_parser.parse_args()
+
+    install(args.input, args.version, args.output)
+
+
+if __name__ == "__main__":
+    main()

diff --git a/lang/python/micropython-lib/patches/001-build-unix-ffi.patch b/lang/python/micropython-lib/patches/001-build-unix-ffi.patch
new file mode 100644 (file)
index 0000000..4dfaca6
--- /dev/null
+++ b/lang/python/micropython-lib/patches/001-build-unix-ffi.patch
@@ -0,0 +1,37 @@
+--- a/tools/build.py
++++ b/tools/build.py
+@@ -284,7 +284,7 @@ def _update_index_package_metadata(index
+                 index_package_json["versions"][v].append(metadata.version)
+ 
+ 
+-def build(output_path, hash_prefix_len, mpy_cross_path):
++def build(output_path, unix_ffi, hash_prefix_len, mpy_cross_path):
+     import manifestfile
+     import mpy_cross
+ 
+@@ -310,7 +310,7 @@ def build(output_path, hash_prefix_len,
+ 
+     # For now, don't process unix-ffi. In the future this can be extended to
+     # allow a way to request unix-ffi packages via mip.
+-    lib_dirs = ["micropython", "python-stdlib", "python-ecosys"]
++    lib_dirs = ["unix-ffi"] if unix_ffi else ["micropython", "python-stdlib", "python-ecosys"]
+ 
+     mpy_version, _mpy_sub_version = mpy_cross.mpy_version(mpy_cross=mpy_cross_path)
+     mpy_version = str(mpy_version)
+@@ -438,6 +438,7 @@ def main():
+ 
+     cmd_parser = argparse.ArgumentParser(description="Compile micropython-lib for serving to mip.")
+     cmd_parser.add_argument("--output", required=True, help="output directory")
++    cmd_parser.add_argument("--unix-ffi", action="store_true", help="process unix-ffi packages")
+     cmd_parser.add_argument("--hash-prefix", default=8, type=int, help="hash prefix length")
+     cmd_parser.add_argument("--mpy-cross", default=None, help="optional path to mpy-cross binary")
+     cmd_parser.add_argument("--micropython", default=None, help="path to micropython repo")
+@@ -447,7 +448,7 @@ def main():
+         sys.path.append(os.path.join(args.micropython, "tools"))  # for manifestfile
+         sys.path.append(os.path.join(args.micropython, "mpy-cross"))  # for mpy_cross
+ 
+-    build(args.output, hash_prefix_len=max(4, args.hash_prefix), mpy_cross_path=args.mpy_cross)
++    build(args.output, args.unix_ffi, hash_prefix_len=max(4, args.hash_prefix), mpy_cross_path=args.mpy_cross)
+ 
+ 
+ if __name__ == "__main__":

diff --git a/lang/python/micropython-lib/patches/002-add-unix-ffi-os-path.patch b/lang/python/micropython-lib/patches/002-add-unix-ffi-os-path.patch
new file mode 100644 (file)
index 0000000..5ebc682
--- /dev/null
+++ b/lang/python/micropython-lib/patches/002-add-unix-ffi-os-path.patch
@@ -0,0 +1,53 @@
+From dcce62dd525cf0f8e572e56a8990aea7ec2f0ade Mon Sep 17 00:00:00 2001
+From: Jeffery To <jeffery.to@gmail.com>
+Date: Tue, 30 May 2023 23:47:59 +0800
+Subject: [PATCH] unix-ffi/os-path: Add unix-ffi version of `os-path` package.
+
+This package reuses the code from the python-stdlib version of `os-path`
+but requires the unix-ffi version of `os`.
+
+This also updates `glob` to require this version of `os-path`.
+
+Signed-off-by: Jeffery To <jeffery.to@gmail.com>
+---
+ unix-ffi/glob/manifest.py    | 2 +-
+ unix-ffi/os-path/manifest.py | 6 ++++++
+ unix-ffi/os/os/__init__.py   | 6 ++++++
+ 3 files changed, 13 insertions(+), 1 deletion(-)
+ create mode 100644 unix-ffi/os-path/manifest.py
+
+--- a/unix-ffi/glob/manifest.py
++++ b/unix-ffi/glob/manifest.py
+@@ -1,7 +1,7 @@
+ metadata(version="0.5.2")
+ 
+ require("os", unix_ffi=True)
+-require("os-path")
++require("os-path", unix_ffi=True)
+ require("re", unix_ffi=True)
+ require("fnmatch")
+ 
+--- /dev/null
++++ b/unix-ffi/os-path/manifest.py
+@@ -0,0 +1,6 @@
++metadata(version="0.1.4")
++
++# Originally written by Paul Sokolovsky.
++
++require("os", unix_ffi=True)
++package("os", base_path="../../python-stdlib/os-path")
+--- a/unix-ffi/os/os/__init__.py
++++ b/unix-ffi/os/os/__init__.py
+@@ -5,6 +5,12 @@ import stat as stat_
+ import ffilib
+ import uos
+ 
++# Provide optional dependencies (which may be installed separately).
++try:
++    from . import path
++except ImportError:
++    pass
++
+ R_OK = const(4)
+ W_OK = const(2)
+ X_OK = const(1)

diff --git a/lang/python/micropython-lib/patches/003-add-unix-ffi-uu.patch b/lang/python/micropython-lib/patches/003-add-unix-ffi-uu.patch
new file mode 100644 (file)
index 0000000..363fae3
--- /dev/null
+++ b/lang/python/micropython-lib/patches/003-add-unix-ffi-uu.patch
@@ -0,0 +1,37 @@
+From 2e7bfd08a306bd9e80b22097ef8fe66e1dd85054 Mon Sep 17 00:00:00 2001
+From: Jeffery To <jeffery.to@gmail.com>
+Date: Wed, 31 May 2023 00:00:11 +0800
+Subject: [PATCH] unix-ffi/uu: Add unix-ffi version of `uu` package.
+
+This package reuses the code from the python-stdlib version of `uu` but
+requires the unix-ffi version of `os-path`.
+
+This also updates `email.message` to require this version of `uu`.
+
+Signed-off-by: Jeffery To <jeffery.to@gmail.com>
+---
+ unix-ffi/email.message/manifest.py | 2 +-
+ unix-ffi/uu/manifest.py            | 6 ++++++
+ 2 files changed, 7 insertions(+), 1 deletion(-)
+ create mode 100644 unix-ffi/uu/manifest.py
+
+--- a/unix-ffi/email.message/manifest.py
++++ b/unix-ffi/email.message/manifest.py
+@@ -1,7 +1,7 @@
+ metadata(version="0.5.3")
+ 
+ require("re", unix_ffi=True)
+-require("uu")
++require("uu", unix_ffi=True)
+ require("base64")
+ require("binascii")
+ require("email.utils", unix_ffi=True)
+--- /dev/null
++++ b/unix-ffi/uu/manifest.py
+@@ -0,0 +1,6 @@
++metadata(version="0.5.1")
++
++require("binascii")
++require("os-path", unix_ffi=True)
++
++module("uu.py", base_path="../../python-stdlib/uu")