From: Jo-Philipp Wich Date: Wed, 20 Jun 2018 14:43:59 +0000 (+0200) Subject: phase1: add kmod archiving and cleanup settings X-Git-Tag: v1~158 X-Git-Url: http://git.lede-project.org./?a=commitdiff_plain;h=8d61fac605bc14b13955bb8c87e842cc36dadedd;p=buildbot.git phase1: add kmod archiving and cleanup settings Signed-off-by: Jo-Philipp Wich --- diff --git a/phase1/master.cfg b/phase1/master.cfg index af07afb..f3eae70 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -130,6 +130,8 @@ if ini.has_option("gpg", "comment"): if ini.has_option("gpg", "passfile"): gpg_passfile = ini.get("gpg", "passfile") +enable_kmod_archive = True + # find targets targets = [ ] @@ -233,6 +235,13 @@ def IsNoTaggingRequested(step): def IsNoMasterBuild(step): return repo_branch != "master" +def IsCleanupConfigured(step): + slave = step.getProperty("slavename") + if slave and slave in do_cleanup: + return do_cleanup[slave] > 0 + else: + return False + def GetBaseVersion(props): if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch): return repo_branch.split('-')[1] @@ -632,6 +641,15 @@ for target in targets: haltOnFailure = True )) + # find kernel version + factory.addStep(SetProperty( + name = "kernelversion", + property = "kernelversion", + description = "Finding the effective Kernel version", + command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s'", + env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) } + )) + factory.addStep(ShellCommand( name = "pkgclean", description = "Cleaning up package build", @@ -664,6 +682,56 @@ for target in targets: haltOnFailure = True )) + if enable_kmod_archive: + factory.addStep(ShellCommand( + name = "kmoddir", + description = "Creating kmod directory", + command=["mkdir", "-p", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s" %(ts[0], ts[1]))], + haltOnFailure = True + )) + + factory.addStep(ShellCommand( + name = "kmodprepare", + description = "Preparing kmod archive", + command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va", + WithProperties("bin/targets/%s/%s%%(libc)s/packages/" %(ts[0], ts[1])), + WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))], + haltOnFailure = True + )) + + factory.addStep(ShellCommand( + name = "kmodindex", + description = "Indexing kmod archive", + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s", + WithProperties("PACKAGE_SUBDIRS=bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))], + env = MakeEnv(), + haltOnFailure = True + )) + + # find rootfs staging directory + factory.addStep(SetProperty( + name = "stageroot", + property = "stageroot", + description = "Finding the rootfs staging directory", + command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"], + env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) } + )) + + factory.addStep(ShellCommand( + name = "filesdir", + description = "Creating file overlay directory", + command=["mkdir", "-p", "files/etc/opkg"], + haltOnFailure = True + )) + + factory.addStep(ShellCommand( + name = "kmodconfig", + description = "Embedding kmod repository configuration", + command=WithProperties("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(kernelversion)s#' " + + "%(stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"), + haltOnFailure = True + )) + #factory.addStep(IfBuiltinShellCommand( factory.addStep(ShellCommand( name = "images", @@ -701,7 +769,7 @@ for target in targets: factory.addStep(ShellCommand( name = "signpack", description = "Packing files to sign", - command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])), + command = WithProperties("find bin/targets/%s/%s%%(libc)s/ bin/targets/%s/%s%%(libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1], ts[0], ts[1])), haltOnFailure = True )) @@ -748,6 +816,14 @@ for target in targets: haltOnFailure = True )) + if enable_kmod_archive: + factory.addStep(ShellCommand( + name = "kmoddirprepare", + description = "Preparing kmod archive upload directory", + command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s" %(ts[0], ts[1]), prefix=GetVersionPrefix)], + haltOnFailure = True + )) + factory.addStep(ShellCommand( name = "dirupload", description = "Uploading directory structure", @@ -760,7 +836,7 @@ for target in targets: factory.addStep(ShellCommand( name = "targetupload", description = "Uploading target files", - command=["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), + command=["rsync", "-4", "--exclude=/kmods/", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, @@ -768,6 +844,18 @@ for target in targets: logEnviron = False )) + if enable_kmod_archive: + factory.addStep(ShellCommand( + name = "kmodupload", + description = "Uploading kmod archive", + command=["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), + "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])), + WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], + env={'RSYNC_PASSWORD': rsync_bin_key}, + haltOnFailure = True, + logEnviron = False + )) + if rsync_src_url is not None: factory.addStep(ShellCommand( name = "sourceupload", @@ -801,6 +889,15 @@ for target in targets: logEnviron = False )) + factory.addStep(ShellCommand( + name = "df", + description = "Reporting disk usage", + command=["df", "-h", "."], + env={'LC_ALL': 'C'}, + haltOnFailure = False, + alwaysRun = True + )) + from buildbot.config import BuilderConfig c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))