build: make find_md5 reproducible with AUTOREMOVE

While experimenting with the AUTOREMOVE option in search of a way to use
prebuilt host tools in different buildroot, it was discovered that the
md5 generated by find_md5 in depends.mk is not reproducible.

Currently the hash is generated by the path of the file in addition to
the file mod time. Out of confusion, probably, there was an idea that
such command was used on the package build_dir. Reality is that this
command is run on the package files. (Makefile, patches, src)

This is problematic because the package Makefile (for example) change at
each git clone and base the hash on the Makefile mtime doesn't really
reflect if the Makefile actually changes across a buildroot or not.

A better approach is to generate an hash of each file and then generate
an hash on the sort hash list. This way we remove the problem of git
clone setting a wrong mtime while keeping the integrity of checking if a
file changed for the package as any change will result in a different
hash.

Introduce a new kind of find_md5 function, find_md5_reproducible that
apply this new logic and limit it only with AUTOREMOVE option set to
prevent any kind of slowdown due to additional hash generation.

Signed-off-by: Christian Marangi <ansuelsmth@gmail.com>

diff --git a/include/depends.mk b/include/depends.mk
index 073fb03632ba309f42f90fc5388068eb33fd08e3..a858d4b033eb47e7b83564d578f78596914b79ce 100644 (file)
--- a/include/depends.mk
+++ b/include/depends.mk
@@ -12,6 +12,7 @@
 DEP_FINDPARAMS := -x "*/.svn*" -x ".*" -x "*:*" -x "*\!*" -x "* *" -x "*\\\#*" -x "*/.*_check" -x "*/.*.swp" -x "*/.pkgdir*"
 
 find_md5=find $(wildcard $(1)) -type f $(patsubst -x,-and -not -path,$(DEP_FINDPARAMS) $(2)) -printf "%p%T@\n" | sort | $(MKHASH) md5
+find_md5_reproducible=find $(wildcard $(1)) -type f $(patsubst -x,-and -not -path,$(DEP_FINDPARAMS) $(2)) -print0 | xargs -0 $(MKHASH) md5 | sort | $(MKHASH) md5
 
 define rdep
   .PRECIOUS: $(2)

diff --git a/include/host-build.mk b/include/host-build.mk
index f06dd802301327cbedda4c43a043114ae9cbd8b8..22fcc31f15c7586a52f208509a8ae2949f33b71a 100644 (file)
--- a/include/host-build.mk
+++ b/include/host-build.mk
@@ -21,7 +21,7 @@ include $(INCLUDE_DIR)/depends.mk
 include $(INCLUDE_DIR)/quilt.mk
 
 BUILD_TYPES += host
-HOST_STAMP_PREPARED=$(HOST_BUILD_DIR)/.prepared$(if $(HOST_QUILT)$(DUMP),,$(shell $(call find_md5,${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(HOST_PREPARED_DEPENDS)))
+HOST_STAMP_PREPARED=$(HOST_BUILD_DIR)/.prepared$(if $(HOST_QUILT)$(DUMP),,$(shell $(call $(if $(CONFIG_AUTOREMOVE),find_md5_reproducible,find_md5),${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(HOST_PREPARED_DEPENDS)))
 HOST_STAMP_CONFIGURED:=$(HOST_BUILD_DIR)/.configured
 HOST_STAMP_BUILT:=$(HOST_BUILD_DIR)/.built
 HOST_BUILD_PREFIX?=$(if $(IS_PACKAGE_BUILD),$(STAGING_DIR_HOSTPKG),$(STAGING_DIR_HOST))

diff --git a/include/kernel-build.mk b/include/kernel-build.mk
index cc651f29b112a8b2232dd6f9dfaffb1ab3055e4f..5b332258f8b8c5e9ae3fc481d5d637ab03694380 100644 (file)
--- a/include/kernel-build.mk
+++ b/include/kernel-build.mk
@@ -10,7 +10,7 @@ ifneq ($(DUMP),1)
 endif
 
 KERNEL_FILE_DEPENDS=$(GENERIC_BACKPORT_DIR) $(GENERIC_PATCH_DIR) $(GENERIC_HACK_DIR) $(PATCH_DIR) $(GENERIC_FILES_DIR) $(FILES_DIR)
-STAMP_PREPARED=$(LINUX_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call find_md5,$(KERNEL_FILE_DEPENDS),)))
+STAMP_PREPARED=$(LINUX_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call $(if $(CONFIG_AUTOREMOVE),find_md5_reproducible,find_md5),$(KERNEL_FILE_DEPENDS),)))
 STAMP_CONFIGURED:=$(LINUX_DIR)/.configured
 include $(INCLUDE_DIR)/download.mk
 include $(INCLUDE_DIR)/quilt.mk

diff --git a/include/package.mk b/include/package.mk
index 5861533bf504627f3880c7b2f5475c968dadcbcc..368bf0d7cac1eff4b8ab33d62aa6b7f1e3c8bbc8 100644 (file)
--- a/include/package.mk
+++ b/include/package.mk
@@ -85,7 +85,7 @@ ifneq ($(PREV_STAMP_PREPARED),)
   STAMP_PREPARED:=$(PREV_STAMP_PREPARED)
   CONFIG_AUTOREBUILD:=
 else
-  STAMP_PREPARED=$(PKG_BUILD_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call find_md5,${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(PKG_PREPARED_DEPENDS)))
+  STAMP_PREPARED=$(PKG_BUILD_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call $(if $(CONFIG_AUTOREMOVE),find_md5_reproducible,find_md5),${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(PKG_PREPARED_DEPENDS)))
 endif
 STAMP_CONFIGURED=$(PKG_BUILD_DIR)/.configured$(if $(DUMP),,_$(call confvar,$(PKG_CONFIG_DEPENDS)))
 STAMP_CONFIGURED_WILDCARD=$(PKG_BUILD_DIR)/.configured_*