From d1a92bab36ff9818f56d60d173ff438db5bafe47 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Thibaut=20VAR=C3=88NE?= Date: Mon, 25 Jun 2018 12:27:00 +0200 Subject: [PATCH] phase1: use Interpolate instead of WithProperties MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit WithProperties is marked as deprecated since buildbot 0.8.7. Trying to make the config future-proof, this patch replaces every instance of WithProperties() by the corresponding Interpolate() equivalent. As of buildbot 0.8.9 it appears that Interpolate() doesn't correctly handle integer format with parameters that call a function, so make GetNumJobs() return a string: File "phase1/master.cfg", line 672, in command = ["make", Interpolate("-j%(kw:jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"], fmtstring % dd exceptions.TypeError: %d format: a number is required, not str Configuration Errors: error while parsing config file: %d format: a number is required, not str (traceback in logfile) Signed-off-by: Thibaut VARÈNE --- phase1/master.cfg | 76 +++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index 62675f1..697f6fb 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -209,7 +209,7 @@ from buildbot.steps.shell import SetPropertyFromCommand from buildbot.steps.transfer import FileUpload from buildbot.steps.transfer import FileDownload from buildbot.steps.master import MasterShellCommand -from buildbot.process.properties import WithProperties +from buildbot.process.properties import Interpolate from buildbot.process import properties @@ -271,14 +271,15 @@ def IsNoTaggingRequested(step): def IsNoMasterBuild(step): return repo_branch != "master" -def GetBaseVersion(props): +def GetBaseVersion(): if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch): return repo_branch.split('-')[1] else: return "master" +@properties.renderer def GetVersionPrefix(props): - basever = GetBaseVersion(props) + basever = GetBaseVersion() if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]): return "%s/" % props["tag"] elif basever != "master": @@ -286,24 +287,28 @@ def GetVersionPrefix(props): else: return "" +@properties.renderer def GetNumJobs(props): if props.hasProperty("max_builds") and props.hasProperty("nproc"): - return (int(props["nproc"]) / (props["max_builds"] + other_builds)) + return str(int(props["nproc"]) / (props["max_builds"] + other_builds)) else: - return 1 + return "1" +@properties.renderer def GetCC(props): if props.hasProperty("cc_command"): return props["cc_command"] else: return "gcc" +@properties.renderer def GetCXX(props): if props.hasProperty("cxx_command"): return props["cxx_command"] else: return "g++" +@properties.renderer def GetCwd(props): if props.hasProperty("builddir"): return props["builddir"] @@ -312,6 +317,7 @@ def GetCwd(props): else: return "/" +@properties.renderer def GetCCache(props): if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]: return props["ccache_command"] + " " @@ -492,7 +498,7 @@ for target in targets: factory.addStep(ShellCommand( name = "cleanold", description = "Cleaning previous builds", - command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"], + command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:slavename)s"), Interpolate("%(prop:buildername)s"), "full"], workdir = ".", haltOnFailure = True, doStepIf = IsCleanupRequested, @@ -501,7 +507,7 @@ for target in targets: factory.addStep(ShellCommand( name = "cleanup", description = "Cleaning work area", - command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"], + command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:slavename)s"), Interpolate("%(prop:buildername)s"), "single"], workdir = ".", haltOnFailure = True, doStepIf = IsCleanupRequested, @@ -556,7 +562,7 @@ for target in targets: factory.addStep(ShellCommand( name = "switchtag", description = "Checking out Git tag", - command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")], + command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")], haltOnFailure = True, doStepIf = IsTaggingRequested )) @@ -670,7 +676,7 @@ for target in targets: factory.addStep(ShellCommand( name = "dltar", description = "Building and installing GNU tar", - command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"], + command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/tar/compile", "V=s"], env = MakeEnv(tryccache=True), haltOnFailure = True )) @@ -679,7 +685,7 @@ for target in targets: factory.addStep(ShellCommand( name = "dlrun", description = "Populating dl/", - command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"], + command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "download", "V=s"], env = MakeEnv(), logEnviron = False, locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]), @@ -695,7 +701,7 @@ for target in targets: factory.addStep(ShellCommand( name = "tools", description = "Building and installing tools", - command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"], + command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/install", "V=s"], env = MakeEnv(tryccache=True), haltOnFailure = True )) @@ -703,7 +709,7 @@ for target in targets: factory.addStep(ShellCommand( name = "toolchain", description = "Building and installing toolchain", - command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"], + command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "toolchain/install", "V=s"], env = MakeEnv(), haltOnFailure = True )) @@ -711,7 +717,7 @@ for target in targets: factory.addStep(ShellCommand( name = "kmods", description = "Building kmods", - command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], + command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], env = MakeEnv(), #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])}, haltOnFailure = True @@ -723,7 +729,7 @@ for target in targets: property = "kernelversion", description = "Finding the effective Kernel version", command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'", - env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) } + env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) } )) factory.addStep(ShellCommand( @@ -735,7 +741,7 @@ for target in targets: factory.addStep(ShellCommand( name = "pkgbuild", description = "Building packages", - command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], + command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], env = MakeEnv(), #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])}, haltOnFailure = True @@ -745,7 +751,7 @@ for target in targets: factory.addStep(ShellCommand( name = "pkginstall", description = "Installing packages", - command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"], + command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/install", "V=s"], env = MakeEnv(), haltOnFailure = True )) @@ -753,7 +759,7 @@ for target in targets: factory.addStep(ShellCommand( name = "pkgindex", description = "Indexing packages", - command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"], + command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s"], env = MakeEnv(), haltOnFailure = True )) @@ -762,7 +768,7 @@ for target in targets: factory.addStep(ShellCommand( name = "kmoddir", description = "Creating kmod directory", - command=["mkdir", "-p", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s" %(ts[0], ts[1]))], + command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])], haltOnFailure = True )) @@ -770,16 +776,16 @@ for target in targets: name = "kmodprepare", description = "Preparing kmod archive", command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va", - WithProperties("bin/targets/%s/%s%%(libc)s/packages/" %(ts[0], ts[1])), - WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))], + Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]), + Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])], haltOnFailure = True )) factory.addStep(ShellCommand( name = "kmodindex", description = "Indexing kmod archive", - command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s", - WithProperties("PACKAGE_SUBDIRS=bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))], + command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", + Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])], env = MakeEnv(), haltOnFailure = True )) @@ -790,7 +796,7 @@ for target in targets: property = "stageroot", description = "Finding the rootfs staging directory", command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"], - env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) } + env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) } )) factory.addStep(ShellCommand( @@ -803,8 +809,8 @@ for target in targets: factory.addStep(ShellCommand( name = "kmodconfig", description = "Embedding kmod repository configuration", - command=WithProperties("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(kernelversion)s#' " + - "%(stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"), + command=Interpolate("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(prop:kernelversion)s#' " + + "%(prop:stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"), haltOnFailure = True )) @@ -812,7 +818,7 @@ for target in targets: factory.addStep(ShellCommand( name = "images", description = "Building and installing images", - command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"], + command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/install", "V=s"], env = MakeEnv(), haltOnFailure = True )) @@ -845,7 +851,7 @@ for target in targets: factory.addStep(ShellCommand( name = "signpack", description = "Packing files to sign", - command = WithProperties("find bin/targets/%s/%s%%(libc)s/ bin/targets/%s/%s%%(libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1], ts[0], ts[1])), + command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]), haltOnFailure = True )) @@ -880,14 +886,14 @@ for target in targets: factory.addStep(ShellCommand( name = "dirprepare", description = "Preparing upload directory structure", - command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)], + command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], haltOnFailure = True )) factory.addStep(ShellCommand( name = "linkprepare", description = "Preparing repository symlink", - command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)], + command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=GetBaseVersion()), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)], doStepIf = IsNoMasterBuild, haltOnFailure = True )) @@ -896,7 +902,7 @@ for target in targets: factory.addStep(ShellCommand( name = "kmoddirprepare", description = "Preparing kmod archive upload directory", - command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s" %(ts[0], ts[1]), prefix=GetVersionPrefix)], + command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], haltOnFailure = True )) @@ -919,8 +925,8 @@ for target in targets: name = "targetupload", description = "Uploading target files", command=["../rsync.sh", "-4", "--info=name", "--exclude=/kmods/", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), - "-a", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), - WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], + "-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]), + Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False, @@ -932,8 +938,8 @@ for target in targets: name = "kmodupload", description = "Uploading kmod archive", command=["../rsync.sh", "-4", "--info=name", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), - "-a", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])), - WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], + "-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]), + Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False, @@ -945,7 +951,7 @@ for target in targets: name = "sourceupload", description = "Uploading source archives", command=["../rsync.sh", "-4", "--info=name", "--size-only", "--delay-updates", - WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-a", "dl/", "%s/" %(rsync_src_url)], + Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:slavename)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)], env={'RSYNC_PASSWORD': rsync_src_key}, haltOnFailure = True, logEnviron = False, -- 2.30.2