From 863186ba6dd33f1d078de1a3124f0bdf18566d59 Mon Sep 17 00:00:00 2001 From: John Crispin Date: Wed, 6 Aug 2014 18:18:47 +0000 Subject: [PATCH] build: introduce per feed repository support This changeset implements a new menuconfig option to generate separate repositories for each enabled package feed instead of one monolithic one. Signed-off-by: Jo-Philipp Wich Backport of r42002 SVN-Revision: 42016 --- include/feeds.mk | 23 ++++++++++++++++++++ include/package-dumpinfo.mk | 1 + include/package-ipkg.mk | 10 ++++++--- include/scan.mk | 8 +++++-- include/toplevel.mk | 2 ++ package/Makefile | 21 ++++++++++++++---- package/base-files/image-config.in | 11 ++++++++++ package/system/opkg/Makefile | 8 +++++++ package/system/opkg/files/opkg-smime.conf | 1 - package/system/opkg/files/opkg.conf | 1 - scripts/feeds | 26 ++++++++++++++++++++++- scripts/metadata.pl | 12 +++++++++++ scripts/metadata.pm | 1 + target/imagebuilder/files/Makefile | 4 ++-- 14 files changed, 115 insertions(+), 14 deletions(-) create mode 100644 include/feeds.mk diff --git a/include/feeds.mk b/include/feeds.mk new file mode 100644 index 0000000000..17ce64f83a --- /dev/null +++ b/include/feeds.mk @@ -0,0 +1,23 @@ +# +# Copyright (C) 2014 OpenWrt.org +# +# This is free software, licensed under the GNU General Public License v2. +# See /LICENSE for more information. +# + +-include $(TMP_DIR)/.packagefeeds + +FEEDS_AVAILABLE:=$(shell $(SCRIPT_DIR)/feeds list -n) +FEEDS_INSTALLED:=$(patsubst %.index,%,$(notdir $(wildcard $(TOPDIR)/feeds/*.index))) +FEEDS_ENABLED:=$(foreach feed,$(FEEDS_INSTALLED),$(if $(CONFIG_FEED_$(feed)),$(feed))) + +PKG_CONFIG_DEPENDS += \ + CONFIG_PER_FEED_REPO \ + $(foreach feed,$(FEEDS_INSTALLED),CONFIG_FEED_$(feed)) + +# 1: package name +define FeedPackageDir +$(strip $(if $(CONFIG_PER_FEED_REPO), \ + $(abspath $(PACKAGE_DIR)/$(if $(Package/$(1)/feed),$(Package/$(1)/feed),base)), \ + $(PACKAGE_DIR))) +endef diff --git a/include/package-dumpinfo.mk b/include/package-dumpinfo.mk index 2bf78c3f18..9dc847dc1d 100644 --- a/include/package-dumpinfo.mk +++ b/include/package-dumpinfo.mk @@ -46,6 +46,7 @@ Type: $(if $(Package/$(1)/targets),$(Package/$(1)/targets),$(if $(PKG_TARGETS),$ $(if $(KCONFIG),Kernel-Config: $(KCONFIG) )$(if $(BUILDONLY),Build-Only: $(BUILDONLY) )$(if $(HIDDEN),Hidden: $(HIDDEN) +)$(if $(FEED),Feed: $(FEED) )Description: $(if $(Package/$(1)/description),$(Package/$(1)/description),$(TITLE)) $(if $(URL),$(URL) )$(MAINTAINER) diff --git a/include/package-ipkg.mk b/include/package-ipkg.mk index 9783a63254..be47526467 100644 --- a/include/package-ipkg.mk +++ b/include/package-ipkg.mk @@ -5,6 +5,8 @@ # See /LICENSE for more information. # +include $(INCLUDE_DIR)/feeds.mk + # invoke ipkg-build with some default options IPKG_BUILD:= \ ipkg-build -c -o 0 -g 0 @@ -74,7 +76,8 @@ endif ifeq ($(DUMP),) define BuildTarget/ipkg - IPKG_$(1):=$(PACKAGE_DIR)/$(1)_$(VERSION)_$(PKGARCH).ipk + PDIR_$(1):=$(call FeedPackageDir,$(1)) + IPKG_$(1):=$$(PDIR_$(1))/$(1)_$(VERSION)_$(PKGARCH).ipk IDIR_$(1):=$(PKG_BUILD_DIR)/ipkg-$(PKGARCH)/$(1) KEEP_$(1):=$(strip $(call Package/$(1)/conffiles)) @@ -187,11 +190,12 @@ ifeq ($(DUMP),) ) endif - $(IPKG_BUILD) $$(IDIR_$(1)) $(PACKAGE_DIR) + $(INSTALL_DIR) $$(PDIR_$(1)) + $(IPKG_BUILD) $$(IDIR_$(1)) $$(PDIR_$(1)) @[ -f $$(IPKG_$(1)) ] $(1)-clean: - rm -f $(PACKAGE_DIR)/$(1)_* + rm -f $$(PDIR_$(1))/$(1)_* clean: $(1)-clean diff --git a/include/scan.mk b/include/scan.mk index 4d08ee8f0e..0998333439 100644 --- a/include/scan.mk +++ b/include/scan.mk @@ -21,15 +21,19 @@ else endef endif +define feedname +$(if $(patsubst feeds/%,,$(1)),,$(word 2,$(subst /, ,$(1)))) +endef + define PackageDir $(TMP_DIR)/.$(SCAN_TARGET): $(TMP_DIR)/info/.$(SCAN_TARGET)-$(1) $(TMP_DIR)/info/.$(SCAN_TARGET)-$(1): $(SCAN_DIR)/$(2)/Makefile $(SCAN_STAMP) $(foreach DEP,$(DEPS_$(SCAN_DIR)/$(2)/Makefile) $(SCAN_DEPS),$(wildcard $(if $(filter /%,$(DEP)),$(DEP),$(SCAN_DIR)/$(2)/$(DEP)))) { \ $$(call progress,Collecting $(SCAN_NAME) info: $(SCAN_DIR)/$(2)) \ echo Source-Makefile: $(SCAN_DIR)/$(2)/Makefile; \ - $(NO_TRACE_MAKE) --no-print-dir -r DUMP=1 -C $(SCAN_DIR)/$(2) $(SCAN_MAKEOPTS) 2>/dev/null || { \ + $(NO_TRACE_MAKE) --no-print-dir -r DUMP=1 FEED="$(call feedname,$(2))" -C $(SCAN_DIR)/$(2) $(SCAN_MAKEOPTS) 2>/dev/null || { \ mkdir -p "$(TOPDIR)/logs/$(SCAN_DIR)/$(2)"; \ - $(NO_TRACE_MAKE) --no-print-dir -r DUMP=1 -C $(SCAN_DIR)/$(2) $(SCAN_MAKEOPTS) > $(TOPDIR)/logs/$(SCAN_DIR)/$(2)/dump.txt 2>&1; \ + $(NO_TRACE_MAKE) --no-print-dir -r DUMP=1 FEED="$(call feedname,$(2))" -C $(SCAN_DIR)/$(2) $(SCAN_MAKEOPTS) > $(TOPDIR)/logs/$(SCAN_DIR)/$(2)/dump.txt 2>&1; \ $$(call progress,ERROR: please fix $(SCAN_DIR)/$(2)/Makefile - see logs/$(SCAN_DIR)/$(2)/dump.txt for details\n) \ rm -f $$@; \ }; \ diff --git a/include/toplevel.mk b/include/toplevel.mk index 10b81049c5..6949be38a3 100644 --- a/include/toplevel.mk +++ b/include/toplevel.mk @@ -71,7 +71,9 @@ prepare-tmpinfo: FORCE f=tmp/.$${type}info; t=tmp/.config-$${type}.in; \ [ "$$t" -nt "$$f" ] || ./scripts/metadata.pl $${type}_config "$$f" > "$$t" || { rm -f "$$t"; echo "Failed to build $$t"; false; break; }; \ done + [ tmp/.config-feeds.in -nt tmp/.packagefeeds ] || ./scripts/feeds feed_config > tmp/.config-feeds.in ./scripts/metadata.pl package_mk tmp/.packageinfo > tmp/.packagedeps || { rm -f tmp/.packagedeps; false; } + ./scripts/metadata.pl package_feeds tmp/.packageinfo > tmp/.packagefeeds || { rm -f tmp/.packagefeeds; false; } touch $(TOPDIR)/tmp/.build .config: ./scripts/config/conf $(if $(CONFIG_HAVE_DOT_CONFIG),,prepare-tmpinfo) diff --git a/package/Makefile b/package/Makefile index 1930d27cf8..bf1b16fc9a 100644 --- a/package/Makefile +++ b/package/Makefile @@ -7,6 +7,8 @@ curdir:=package +include $(INCLUDE_DIR)/feeds.mk + -include $(TMP_DIR)/.packagedeps $(curdir)/builddirs:=$(sort $(package-) $(package-y) $(package-m)) $(curdir)/builddirs-install:=. @@ -127,11 +129,19 @@ ifndef CONFIG_OPKGSMIME_PASSPHRASE endif endif +PACKAGE_SUBDIRS=. +ifneq ($(CONFIG_PER_FEED_REPO),) + PACKAGE_SUBDIRS=base $(FEEDS_ENABLED) +endif + $(curdir)/index: FORCE @echo Generating package index... - @(cd $(PACKAGE_DIR); \ + @for d in $(PACKAGE_SUBDIRS); do ( \ + [ -d $(PACKAGE_DIR)/$$d ] && \ + cd $(PACKAGE_DIR)/$$d || continue; \ $(SCRIPT_DIR)/ipkg-make-index.sh . 2>&1 > Packages && \ - gzip -9c Packages > Packages.gz ) + gzip -9c Packages > Packages.gz; \ + ); done ifeq ($(call qstrip,$(CONFIG_OPKGSMIME_KEY)),) @echo Signing key has not been configured else @@ -139,12 +149,15 @@ ifeq ($(call qstrip,$(CONFIG_OPKGSMIME_CERT)),) @echo Certificate has not been configured else @echo Signing package index... - @(cd $(PACKAGE_DIR); \ + @for d in $(PACKAGE_SUBDIRS); do ( \ + [ -d $(PACKAGE_DIR)/$$d ] && \ + cd $(PACKAGE_DIR)/$$d || continue; \ openssl smime -binary -in Packages.gz \ -out Packages.sig -outform PEM -sign \ -signer $(CONFIG_OPKGSMIME_CERT) \ -inkey $(CONFIG_OPKGSMIME_KEY) \ - $(PASSOPT) $(PASSARG) ) + $(PASSOPT) $(PASSARG); \ + ); done endif endif diff --git a/package/base-files/image-config.in b/package/base-files/image-config.in index 66134aad3c..d745369a2c 100644 --- a/package/base-files/image-config.in +++ b/package/base-files/image-config.in @@ -208,6 +208,17 @@ if VERSIONOPT Useful for OEMs building OpenWrt based firmware endif + +menuconfig PER_FEED_REPO + bool "Separate feed repositories" if IMAGEOPT + default n + help + If set, a separate repository is generated within bin/*/packages/ + for the core packages and each enabled feed. + +source "tmp/.config-feeds.in" + + menuconfig SMIMEOPT bool "Package signing options" if IMAGEOPT default n diff --git a/package/system/opkg/Makefile b/package/system/opkg/Makefile index 857fe561d8..5dfabc2ca2 100644 --- a/package/system/opkg/Makefile +++ b/package/system/opkg/Makefile @@ -7,6 +7,7 @@ include $(TOPDIR)/rules.mk include $(INCLUDE_DIR)/kernel.mk include $(INCLUDE_DIR)/version.mk +include $(INCLUDE_DIR)/feeds.mk PKG_NAME:=opkg PKG_REV:=9c97d5ecd795709c8584e972bfdf3aee3a5b846d @@ -104,6 +105,13 @@ define Package/opkg/Default/install $(INSTALL_DIR) $(1)/bin $(INSTALL_DIR) $(1)/etc $(INSTALL_DATA) ./files/opkg$(2).conf $(1)/etc/opkg.conf + ifeq ($(CONFIG_PER_FEED_REPO),) + echo "src/gz %n %U" >> $(1)/etc/opkg.conf + else + for d in base $(FEEDS_ENABLED); do \ + echo "src/gz %n_$$$$d %U/$$$$d" >> $(1)/etc/opkg.conf; \ + done + endif $(VERSION_SED) $(1)/etc/opkg.conf $(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/bin/opkg-cl $(1)/bin/opkg endef diff --git a/package/system/opkg/files/opkg-smime.conf b/package/system/opkg/files/opkg-smime.conf index 849bb65b20..fd2cade1f8 100644 --- a/package/system/opkg/files/opkg-smime.conf +++ b/package/system/opkg/files/opkg-smime.conf @@ -1,4 +1,3 @@ -src/gz %n %U dest root / dest ram /tmp lists_dir ext /var/opkg-lists diff --git a/package/system/opkg/files/opkg.conf b/package/system/opkg/files/opkg.conf index 6fb42b7fa4..d8d3a2d693 100644 --- a/package/system/opkg/files/opkg.conf +++ b/package/system/opkg/files/opkg.conf @@ -1,4 +1,3 @@ -src/gz %n %U dest root / dest ram /tmp lists_dir ext /var/opkg-lists diff --git a/scripts/feeds b/scripts/feeds index be07e59613..26c2de0945 100755 --- a/scripts/feeds +++ b/scripts/feeds @@ -262,11 +262,17 @@ sub list_feed { sub list { my %opts; - getopts('r:d:sh', \%opts); + getopts('r:d:nsh', \%opts); if ($opts{h}) { usage(); return 0; } + if ($opts{n}) { + foreach my $feed (@feeds) { + printf "%s\n", $feed->[1]; + } + return 0; + } if ($opts{s}) { foreach my $feed (@feeds) { my $localpath = "./feeds/$feed->[1]"; @@ -598,6 +604,22 @@ sub update { return 0; } +sub feed_config() { + foreach my $feed (@feeds) { + my $installed = (-f "feeds/$feed->[1].index"); + + printf "\tconfig FEED_%s\n", $feed->[1]; + printf "\t\tbool \"Enable feed %s\"\n", $feed->[1]; + printf "\t\tdepends on PER_FEED_REPO\n"; + printf "\t\tdefault y\n" if $installed; + printf "\t\thelp\n"; + printf "\t\t Enable the \\\"%s\\\" feed at %s.\n", $feed->[1], $feed->[2][0]; + printf "\n"; + } + + return 0; +} + sub usage() { print < [options] @@ -605,6 +627,7 @@ Usage: $0 [options] Commands: list [options]: List feeds, their content and revisions (if installed) Options: + -n : List of feed names. -s : List of feed names and their URL. -r : List packages of specified feed. -d : Use specified delimiter to distinguish rows (default: spaces) @@ -640,6 +663,7 @@ my %commands = ( 'install' => \&install, 'search' => \&search, 'uninstall' => \&uninstall, + 'feed_config' => \&feed_config, 'clean' => sub { system("rm -rf feeds"); } diff --git a/scripts/metadata.pl b/scripts/metadata.pl index e408beb507..79f930c5ea 100755 --- a/scripts/metadata.pl +++ b/scripts/metadata.pl @@ -848,6 +848,16 @@ sub gen_package_source() { } } +sub gen_package_feeds() { + parse_package_metadata($ARGV[0]) or exit 1; + foreach my $name (sort {uc($a) cmp uc($b)} keys %package) { + my $pkg = $package{$name}; + if ($pkg->{name} && $pkg->{feed}) { + print "Package/$name/feed = $pkg->{feed}\n"; + } + } +} + sub parse_command() { my $cmd = shift @ARGV; for ($cmd) { @@ -856,6 +866,7 @@ sub parse_command() { /^package_config$/ and return gen_package_config(); /^kconfig/ and return gen_kconfig_overrides(); /^package_source$/ and return gen_package_source(); + /^package_feeds$/ and return gen_package_feeds(); } print <{builddepends} = [ split /\s+/, $1 ]; /^Build-Depends\/(\w+): \s*(.+)\s*$/ and $pkg->{"builddepends/$1"} = [ split /\s+/, $2 ]; /^Build-Types:\s*(.+)\s*$/ and $pkg->{buildtypes} = [ split /\s+/, $1 ]; + /^Feed:\s*(.+?)\s*$/ and $pkg->{feed} = $1; /^Category: \s*(.+)\s*$/ and do { $pkg->{category} = $1; defined $category{$1} or $category{$1} = {}; diff --git a/target/imagebuilder/files/Makefile b/target/imagebuilder/files/Makefile index ecaf5ccda3..222dac43cc 100644 --- a/target/imagebuilder/files/Makefile +++ b/target/imagebuilder/files/Makefile @@ -126,8 +126,8 @@ package_index: FORCE package_install: FORCE @echo @echo Installing packages... - $(OPKG) install $(PACKAGE_DIR)/libc_*.ipk - $(OPKG) install $(PACKAGE_DIR)/kernel_*.ipk + $(OPKG) install $(firstword $(wildcard $(PACKAGE_DIR)/libc_*.ipk $(PACKAGE_DIR)/base/libc_*.ipk)) + $(OPKG) install $(firstword $(wildcard $(PACKAGE_DIR)/kernel_*.ipk $(PACKAGE_DIR)/base/kernel_*.ipk)) $(OPKG) install $(BUILD_PACKAGES) rm -f $(TARGET_DIR)/usr/lib/opkg/lists/* -- 2.30.2