Refactor downloading code into download.mk

Support multiple file downloads
Support svn downloads

SVN-Revision: 9057

diff --git a/include/download.mk b/include/download.mk
new file mode 100644 (file)
index 0000000..1f227e2
--- /dev/null
+++ b/include/download.mk
@@ -0,0 +1,90 @@
+# 
+# Copyright (C) 2007 OpenWrt.org
+#
+# This is free software, licensed under the GNU General Public License v2.
+# See /LICENSE for more information.
+#
+
+DOWNLOAD_RDEP:=$(STAMP_PREPARED)
+
+# Try to guess the download method from the URL
+define dl_method 
+$(strip \
+  $(if $(2),$(2), \
+    $(if $(filter @GNU/% @KERNEL/% @SF/% ftp://% http://%,$(1)),default, \
+      $(if $(filter git://%,$(1)),git, \
+        $(if $(filter svn://%,$(1)),svn, \
+          unknown \
+        ) \
+      ) \
+    ) \
+  ) \
+)
+endef
+
+# code for creating tarballs from svn/git checkouts - useful for mirror support
+dl_pack/bz2=tar cfj $(1) $(2)
+dl_pack/gz=tar cfz $(1) $(2)
+dl_pack/unknown=echo "ERROR: Unknown pack format for file $(1)"; false
+define dl_pack
+       $(if $(dl_pack/$(call ext,$(1))),$(dl_pack/$(call ext,$(1))),$(dl_pack/unknown))
+endef
+
+define DownloadMethod/unknown
+       @echo "ERROR: No download method available"; false
+endef
+
+define DownloadMethod/default
+       $(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(FILE)" "$(MD5SUM)" $(URL)
+endef
+
+define wrap_mirror
+       @$(if $(CONFIG_LOCALMIRROR),$(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(FILE)" "x" || ) \
+       ( $(1) ) \
+       $(if $(CONFIG_LOCALMIRROR),, || $(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(FILE)" "x")
+endef
+
+define DownloadMethod/svn
+       $(call wrap_mirror, \
+               echo "Checking out files from svn repository..."; \
+               mkdir -p $(TMP_DIR)/dl && \
+               cd $(TMP_DIR)/dl && \
+               rm -rf $(SUBDIR) && \
+               [ \! -d $(SUBDIR) ] && \
+               svn co -r$(VERSION) $(URL) $(SUBDIR) && \
+               find $(SUBDIR) -name .svn | xargs rm -rf && \
+               echo "Packing checkout..." && \
+               $(call dl_pack,$(TMP_DIR)/dl/$(FILE),$(SUBDIR)) && \
+               mv $(TMP_DIR)/dl/$(FILE) $(DL_DIR)/; \
+       )
+endef
+
+Validate/svn=VERSION SUBDIR
+#Validate/git=VERSION SUBDIR
+
+define Download/Defaults
+  URL:=
+  FILE:=
+  PROTO:=
+  MD5SUM:=
+  SUBDIR:=
+  VERSION:=
+endef
+
+define Download
+  $(eval $(Download/Defaults))
+  $(eval $(Download/$(1)))
+  $(foreach FIELD,URL FILE $(Validate/$(call dl_method,$(URL),$(PROTO))),
+    ifeq ($($(FIELD)),)
+      $$(error Download/$(1) is missing the $(FIELD) field.)
+    endif
+  )
+
+  $(if $(DOWNLOAD_RDEP),$(DOWNLOAD_RDEP): $(DL_DIR)/$(FILE))
+  download: $(DL_DIR)/$(FILE)
+
+  $(DL_DIR)/$(FILE):
+       mkdir -p $(DL_DIR)
+       $(if $(DownloadMethod/$(call dl_method,$(URL),$(PROTO))),$(DownloadMethod/$(call dl_method,$(URL),$(PROTO))),$(DownloadMethod/unknown))
+
+endef

diff --git a/include/host-build.mk b/include/host-build.mk
index 88129aad079cd83f39ecc4fc7a5ce7be8df185f0..c0d62148e881a79af31044df8caddfc1cdb834bc 100644 (file)
--- a/include/host-build.mk
+++ b/include/host-build.mk
@@ -19,6 +19,7 @@ STAMP_INSTALLED:=$(STAGING_DIR_HOST)/stamp/.$(PKG_NAME)_installed
 
 override MAKEFLAGS=
 
+include $(INCLUDE_DIR)/download.mk
 include $(INCLUDE_DIR)/quilt.mk
 
 Build/Patch:=$(Build/Patch/Default)
@@ -69,17 +70,6 @@ define Build/Compile
   $(call Build/Compile/Default)
 endef
 
-               
-ifneq ($(strip $(PKG_SOURCE)),)
-  download: $(DL_DIR)/$(PKG_SOURCE)
-
-  $(DL_DIR)/$(PKG_SOURCE):
-       mkdir -p $(DL_DIR)
-       $(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(PKG_SOURCE)" "$(PKG_MD5SUM)" $(PKG_SOURCE_URL)
-
-  $(STAMP_PREPARED): $(DL_DIR)/$(PKG_SOURCE)
-endif
-
 ifneq ($(if $(QUILT),,$(CONFIG_AUTOREBUILD)),)
   define HostBuild/Autoclean
     $(call rdep,${CURDIR} $(PKG_FILE_DEPEND),$(STAMP_PREPARED))
@@ -87,10 +77,17 @@ ifneq ($(if $(QUILT),,$(CONFIG_AUTOREBUILD)),)
   endef
 endif
 
+define Download/default
+  FILE:=$(PKG_SOURCE)
+  URL:=$(PKG_SOURCE_URL)
+  PROTO:=$(PKG_SOURCE_PROTO)
+  VERSION:=$(PKG_SOURCE_VERSION)
+  MD5SUM:=$(PKG_MD5SUM)
+endef
+
 define HostBuild
-  ifeq ($(DUMP),)
-    $(call HostBuild/Autoclean)
-  endif
+  $(if $(strip $(PKG_SOURCE_URL)),$(call Download,default))
+  $(if $(DUMP),,$(call HostBuild/Autoclean))
   
   $(STAMP_PREPARED):
        @-rm -rf $(PKG_BUILD_DIR)

diff --git a/include/kernel-build.mk b/include/kernel-build.mk
index 3b767c382bba22458e94d1087313ff5a214eb09d..82b0bcff27af7e023a11be3a49bf2e73fe82ab72 100644 (file)
--- a/include/kernel-build.mk
+++ b/include/kernel-build.mk
@@ -17,6 +17,7 @@ endif
 
 STAMP_PREPARED:=$(LINUX_DIR)/.prepared
 STAMP_CONFIGURED:=$(LINUX_DIR)/.configured
+include $(INCLUDE_DIR)/download.mk
 include $(INCLUDE_DIR)/quilt.mk
 include $(INCLUDE_DIR)/kernel-defaults.mk
 
@@ -40,12 +41,14 @@ define Kernel/Clean
        $(call Kernel/Clean/Default)
 endef
 
+define Download/kernel
+  URL:=$(LINUX_SITE)
+  FILE:=$(LINUX_SOURCE)
+  MD5SUM:=$(LINUX_KERNEL_MD5SUM)
+endef
+
 define BuildKernel
-  ifneq ($(LINUX_SITE),)
-    $(DL_DIR)/$(LINUX_SOURCE):
-               -mkdir -p $(DL_DIR)
-               $(SCRIPT_DIR)/download.pl $(DL_DIR) $(LINUX_SOURCE) $(LINUX_KERNEL_MD5SUM) $(LINUX_SITE)
-  endif
+  $(if $(LINUX_SITE),$(call Download,kernel))
 
   $(STAMP_PREPARED): $(DL_DIR)/$(LINUX_SOURCE)
        -rm -rf $(KERNEL_BUILD_DIR)

diff --git a/include/package.mk b/include/package.mk
index 2b93972f24360cc29fb9acd5257883cac5301eed..bf82c604582dfdd3db00cdabc0b254724728c404 100644 (file)
--- a/include/package.mk
+++ b/include/package.mk
@@ -20,6 +20,7 @@ STAMP_PREPARED=$(PKG_BUILD_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call f
 STAMP_CONFIGURED:=$(PKG_BUILD_DIR)/.configured
 STAMP_BUILT:=$(PKG_BUILD_DIR)/.built
 
+include $(INCLUDE_DIR)/download.mk
 include $(INCLUDE_DIR)/quilt.mk
 include $(INCLUDE_DIR)/package-defaults.mk
 include $(INCLUDE_DIR)/package-dumpinfo.mk
@@ -38,18 +39,16 @@ ifeq ($(DUMP)$(filter prereq clean refresh update,$(MAKECMDGOALS)),)
   endif
 endif
 
+define Download/default
+  FILE:=$(PKG_SOURCE)
+  URL:=$(PKG_SOURCE_URL)
+  PROTO:=$(PKG_SOURCE_PROTO)
+  VERSION:=$(PKG_SOURCE_VERSION)
+  MD5SUM:=$(PKG_MD5SUM)
+endef
 
 define Build/DefaultTargets
-  ifneq ($(strip $(PKG_SOURCE_URL)),)
-    download: $(DL_DIR)/$(PKG_SOURCE)
-
-    $(DL_DIR)/$(PKG_SOURCE):
-       mkdir -p $(DL_DIR)
-       $(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(PKG_SOURCE)" "$(PKG_MD5SUM)" $(PKG_SOURCE_URL)
-
-    $(STAMP_PREPARED): $(DL_DIR)/$(PKG_SOURCE)
-  endif
-
+  $(if $(strip $(PKG_SOURCE_URL)),$(call Download,default))
   $(call Build/Autoclean)
 
   $(STAMP_PREPARED):

diff --git a/include/unpack.mk b/include/unpack.mk
index 500aada4620901b03dde1b09397addca3aee4b55..9b68e5dea39d1e97b9722f1735c4f40d736054e1 100644 (file)
--- a/include/unpack.mk
+++ b/include/unpack.mk
@@ -1,4 +1,9 @@
-ext=$(word $(words $(subst ., ,$(1))),$(subst ., ,$(1)))
+# 
+# Copyright (C) 2006-2007 OpenWrt.org
+#
+# This is free software, licensed under the GNU General Public License v2.
+# See /LICENSE for more information.
+#
 
 # unpacking files with +s may break on some platforms. this typically emits error code 2
 ifneq ($(HOST_OS),Linux)

diff --git a/rules.mk b/rules.mk
index 9d5fcd72b408d060c74135bb79da48cdb9521e38..9f1c4c474970705c978b5a612bdd98c413f98662 100644 (file)
--- a/rules.mk
+++ b/rules.mk
@@ -1,5 +1,5 @@
 # 
-# Copyright (C) 2006 OpenWrt.org
+# Copyright (C) 2006-2007 OpenWrt.org
 #
 # This is free software, licensed under the GNU General Public License v2.
 # See /LICENSE for more information.
@@ -132,6 +132,9 @@ $(call shvar,$(1))=$$(call $(1))
 export $(call shvar,$(1))
 endef
 
+# file extension
+ext=$(word $(words $(subst ., ,$(1))),$(subst ., ,$(1)))
+
 all:
 FORCE: ;
 .PHONY: FORCE

diff --git a/scripts/download.pl b/scripts/download.pl
index 66eaca8b36497062aedf496ba6bb9802350d99df..161ae7f64b992aa6e1e50374214dccbe0638ec83 100755 (executable)
--- a/scripts/download.pl
+++ b/scripts/download.pl
@@ -10,16 +10,15 @@ use strict;
 use warnings;
 use File::Basename;
 
+@ARGV > 2 or die "Syntax: $0 <target dir> <filename> <md5sum> [<mirror> ...]\n";
+
 my $target = shift @ARGV;
 my $filename = shift @ARGV;
 my $md5sum = shift @ARGV;
 my $scriptdir = dirname($0);
 my @mirrors;
-
 my $ok;
 
-@ARGV > 0 or die "Syntax: $0 <target dir> <filename> <md5sum> <mirror> [<mirror> ...]\n";
-
 sub localmirrors {
     my @mlist;
     open LM, "$scriptdir/localmirrors" and do {