download: improve handling of invalid local files

4e19cbc5533: [download: handle possibly invalid local tarballs] added a
FORCE rule to downloaded files, so that they will be always checked by
download.pl.

As a side-effect, check-compile will fail, forcing unnecessary package
rebuilds.
The check-compile.txt log shows (for libxml2 for example):
  Considering target file '.../dl/libxml2-2.9.12.tar.gz'.
    ...
    prerequisite 'FORCE' of target '.../dl/libxml2-2.9.12.tar.gz' does
    not exist.
    Must remake target '.../dl/libxml2-2.9.12.tar.gz'.
    ...
   Giving up on target file '...libxml2-2.9.12/.prepared_...'.
   Giving up on target file '...libxml2-2.9.12/.configured_...'.
   Giving up on target file '...libxml2-2.9.12/.built'.
   Giving up on target file '...stamp/.libxml2_installed'.
  Giving up on target file '.compile'.

Then the package is rebuilt even if it is not otherwise needed.

To fix this, instead of always forcing the download target to be remade,
check its hash first: if it matches, then the FORCE is not added.

Signed-off-by: Eneas U de Queiroz <cotequeiroz@gmail.com>

diff --git a/include/download.mk b/include/download.mk
index 609956b004ad9255eb06f4fb7e7e7b3d46082cc1..76bd374cf7e349f31196401725c170df9262cf88 100644 (file)
--- a/include/download.mk
+++ b/include/download.mk
@@ -59,6 +59,21 @@ define dl_tar_pack
                $$$${TAR_TIMESTAMP:+--mtime="$$$$TAR_TIMESTAMP"} -c $(2) | $(call dl_pack,$(1))
 endef
 
+gen_sha256sum = $(shell $(MKHASH) sha256 $(DL_DIR)/$(1))
+
+# Used in Build/CoreTargets and HostBuild/Core as an integrity check for
+# downloaded files.  It will add a FORCE rule if the sha256 hash does not
+# match, so that the download can be more thoroughly handled by download.pl.
+define check_download_integrity
+  expected_hash:=$(strip $(if $(filter-out x,$(HASH)),$(HASH),$(MIRROR_HASH)))
+  $$(if $$(and $(FILE),$$(wildcard $(DL_DIR)/$(FILE)), \
+              $$(filter undefined,$$(flavor DownloadChecked/$(FILE)))), \
+    $$(eval DownloadChecked/$(FILE):=1) \
+    $$(if $$(filter-out $$(call gen_sha256sum,$(FILE)),$$(expected_hash)), \
+      $(DL_DIR)/$(FILE): FORCE) \
+  )
+endef
+
 ifdef CHECK
 check_escape=$(subst ','\'',$(1))
 #')
@@ -74,8 +89,6 @@ else
   check_warn = $(if $(filter-out undefined,$(origin F_$(1))),$(filter ,$(shell $(call F_$(1),$(2),$(3),$(4)) >&2)),$(check_warn_nofix))
 endif
 
-gen_sha256sum = $(shell $(MKHASH) sha256 $(DL_DIR)/$(1))
-
 ifdef FIXUP
 F_hash_deprecated = $(SCRIPT_DIR)/fixup-makefile.pl $(CURDIR)/Makefile fix-hash $(3) $(call gen_sha256sum,$(1)) $(2)
 F_hash_mismatch = $(F_hash_deprecated)

diff --git a/include/host-build.mk b/include/host-build.mk
index e4a5c48e72f6f0a91acb558e94eec265b0bb0d5d..cfa29419aa91bd3caf170b2ba72d9525d9d5045b 100644 (file)
--- a/include/host-build.mk
+++ b/include/host-build.mk
@@ -180,7 +180,7 @@ ifndef DUMP
     clean-build: host-clean-build
   endif
 
-  $(DL_DIR)/$(FILE): FORCE
+  $(call check_download_integrity)
 
   $(_host_target)host-prepare: $(HOST_STAMP_PREPARED)
   $(_host_target)host-configure: $(HOST_STAMP_CONFIGURED)

diff --git a/include/package.mk b/include/package.mk
index db0a869dab62b9b914a069796d1d8dc69091e77e..55d9352072e5493469948f4efbe19bbf38bb3fd5 100644 (file)
--- a/include/package.mk
+++ b/include/package.mk
@@ -183,7 +183,7 @@ define Build/CoreTargets
   $(call Build/Autoclean)
   $(call DefaultTargets)
 
-  $(DL_DIR)/$(FILE): FORCE
+  $(call check_download_integrity)
 
   download:
        $(foreach hook,$(Hooks/Download),