phase1: add kmod archiving and cleanup settings

Signed-off-by: Jo-Philipp Wich <jo@mein.io>

diff --git a/phase1/master.cfg b/phase1/master.cfg
index af07afbfd268bcef70a1eba0e26e10a292afc315..f3eae70a2a483a46bb81c30158e504a2864f07cc 100644 (file)
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -130,6 +130,8 @@ if ini.has_option("gpg", "comment"):
 if ini.has_option("gpg", "passfile"):
        gpg_passfile = ini.get("gpg", "passfile")
 
+enable_kmod_archive = True
+
 
 # find targets
 targets = [ ]
@@ -233,6 +235,13 @@ def IsNoTaggingRequested(step):
 def IsNoMasterBuild(step):
        return repo_branch != "master"
 
+def IsCleanupConfigured(step):
+       slave = step.getProperty("slavename")
+       if slave and slave in do_cleanup:
+               return do_cleanup[slave] > 0
+       else:
+               return False
+
 def GetBaseVersion(props):
        if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
                return repo_branch.split('-')[1]
@@ -632,6 +641,15 @@ for target in targets:
                haltOnFailure = True
        ))
 
+       # find kernel version
+       factory.addStep(SetProperty(
+               name = "kernelversion",
+               property = "kernelversion",
+               description = "Finding the effective Kernel version",
+               command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s'",
+               env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) }
+       ))
+
        factory.addStep(ShellCommand(
                name = "pkgclean",
                description = "Cleaning up package build",
@@ -664,6 +682,56 @@ for target in targets:
                haltOnFailure = True
        ))
 
+       if enable_kmod_archive:
+               factory.addStep(ShellCommand(
+                       name = "kmoddir",
+                       description = "Creating kmod directory",
+                       command=["mkdir", "-p", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s" %(ts[0], ts[1]))],
+                       haltOnFailure = True
+               ))
+
+               factory.addStep(ShellCommand(
+                       name = "kmodprepare",
+                       description = "Preparing kmod archive",
+                       command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
+                                WithProperties("bin/targets/%s/%s%%(libc)s/packages/" %(ts[0], ts[1])),
+                                WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))],
+                       haltOnFailure = True
+               ))
+
+               factory.addStep(ShellCommand(
+                       name = "kmodindex",
+                       description = "Indexing kmod archive",
+                       command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s",
+                                WithProperties("PACKAGE_SUBDIRS=bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))],
+                       env = MakeEnv(),
+                       haltOnFailure = True
+               ))
+
+               # find rootfs staging directory
+               factory.addStep(SetProperty(
+                       name = "stageroot",
+                       property = "stageroot",
+                       description = "Finding the rootfs staging directory",
+                       command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
+                       env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) }
+               ))
+
+               factory.addStep(ShellCommand(
+                       name = "filesdir",
+                       description = "Creating file overlay directory",
+                       command=["mkdir", "-p", "files/etc/opkg"],
+                       haltOnFailure = True
+               ))
+
+               factory.addStep(ShellCommand(
+                       name = "kmodconfig",
+                       description = "Embedding kmod repository configuration",
+                       command=WithProperties("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(kernelversion)s#' " +
+                                              "%(stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
+                       haltOnFailure = True
+               ))
+
        #factory.addStep(IfBuiltinShellCommand(
        factory.addStep(ShellCommand(
                name = "images",
@@ -701,7 +769,7 @@ for target in targets:
                factory.addStep(ShellCommand(
                        name = "signpack",
                        description = "Packing files to sign",
-                       command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
+                       command = WithProperties("find bin/targets/%s/%s%%(libc)s/ bin/targets/%s/%s%%(libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1], ts[0], ts[1])),
                        haltOnFailure = True
                ))
 
@@ -748,6 +816,14 @@ for target in targets:
                haltOnFailure = True
        ))
 
+       if enable_kmod_archive:
+               factory.addStep(ShellCommand(
+                       name = "kmoddirprepare",
+                       description = "Preparing kmod archive upload directory",
+                       command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
+                       haltOnFailure = True
+               ))
+
        factory.addStep(ShellCommand(
                name = "dirupload",
                description = "Uploading directory structure",
@@ -760,7 +836,7 @@ for target in targets:
        factory.addStep(ShellCommand(
                name = "targetupload",
                description = "Uploading target files",
-               command=["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
+               command=["rsync", "-4", "--exclude=/kmods/", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
                         "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
                         WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
                env={'RSYNC_PASSWORD': rsync_bin_key},
@@ -768,6 +844,18 @@ for target in targets:
                logEnviron = False
        ))
 
+       if enable_kmod_archive:
+               factory.addStep(ShellCommand(
+                       name = "kmodupload",
+                       description = "Uploading kmod archive",
+                       command=["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
+                                "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])),
+                                WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
+                       env={'RSYNC_PASSWORD': rsync_bin_key},
+                       haltOnFailure = True,
+                       logEnviron = False
+               ))
+
        if rsync_src_url is not None:
                factory.addStep(ShellCommand(
                        name = "sourceupload",
@@ -801,6 +889,15 @@ for target in targets:
                        logEnviron = False
                ))
 
+       factory.addStep(ShellCommand(
+               name = "df",
+               description = "Reporting disk usage",
+               command=["df", "-h", "."],
+               env={'LC_ALL': 'C'},
+               haltOnFailure = False,
+               alwaysRun = True
+       ))
+
        from buildbot.config import BuilderConfig
 
        c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))