From: Thibaut VARÈNE Date: Thu, 7 Jun 2018 17:12:38 +0000 (+0200) Subject: phase1: provide upload/download locking X-Git-Tag: v1~157 X-Git-Url: http://git.lede-project.org./?a=commitdiff_plain;h=0edb49341869527f89c2b446614be6e86c7413dd;p=buildbot.git phase1: provide upload/download locking This patch offers an optional locking mechanism to ensure that specific build slaves do not perform concurrent network operations Each slave definition can feature two additional options: ul_lock = dl_lock = In the scenario where a group of build slaves share the same physical network link, these variables can be used as follows: If the link is full duplex capable, each slave from the target group would be configured with e.g.: dl_lock = slavegroup1_dl ul_lock = slavegroup1_ul This enables separate locks for uplink and downlink. If the link is not full duplex capable, then each slave from the target group would be configured with e.g.: dl_lock = slavegroup1 ul_lock = slavegroup1 Effectively making uplink and downlink share the same lock If neither option is set, no lock is enforced and the patch is a no-op. In this patch the locks are only applied to steps that cause significant network traffic. Signed-off-by: Thibaut VARÈNE --- diff --git a/phase1/master.cfg b/phase1/master.cfg index f3eae70..9643d28 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -50,10 +50,12 @@ if ini.has_option("general", "port"): c['slaves'] = [] max_builds = dict() do_cleanup = dict() +NetLocks = dict() for section in ini.sections(): if section.startswith("slave "): if ini.has_option(section, "name") and ini.has_option(section, "password"): + sl_props = { 'dl_lock':None, 'ul_lock':None, } name = ini.get(section, "name") password = ini.get(section, "password") max_builds[name] = 1 @@ -62,7 +64,17 @@ for section in ini.sections(): max_builds[name] = ini.getint(section, "builds") if ini.has_option(section, "cleanup"): do_cleanup[name] = ini.getboolean(section, "cleanup") - c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name])) + if ini.has_option(section, "dl_lock"): + lockname = ini.get(section, "dl_lock") + sl_props['dl_lock'] = lockname + if lockname not in NetLocks: + NetLocks[lockname] = locks.MasterLock(lockname) + if ini.has_option(section, "ul_lock"): + lockname = ini.get(section, "dl_lock") + sl_props['ul_lock'] = lockname + if lockname not in NetLocks: + NetLocks[lockname] = locks.MasterLock(lockname) + c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name], properties = sl_props)) # 'slavePortnum' defines the TCP port to listen on for connections from slaves. # This must match the value configured into the buildslaves (with their @@ -195,6 +207,7 @@ from buildbot.steps.transfer import FileUpload from buildbot.steps.transfer import FileDownload from buildbot.steps.master import MasterShellCommand from buildbot.process.properties import WithProperties +from buildbot.process import properties CleanTargetMap = [ @@ -299,6 +312,25 @@ def MakeEnv(overrides=None): env.update(overrides) return env +@properties.renderer +def NetLockDl(props): + lock = None + if props.hasProperty("dl_lock"): + lock = NetLocks[props["dl_lock"]] + if lock is not None: + return [lock.access('exclusive')] + else: + return [] + +@properties.renderer +def NetLockUl(props): + lock = None + if props.hasProperty("ul_lock"): + lock = NetLocks[props["ul_lock"]] + if lock is not None: + return [lock.access('exclusive')] + else: + return [] c['builders'] = [] @@ -451,7 +483,8 @@ for target in targets: description = "Checking out Git branch", command = "if [ -d .git ]; then git fetch && git checkout '%s'; else exit 0; fi" % repo_branch, haltOnFailure = True, - doStepIf = IsNoTaggingRequested + doStepIf = IsNoTaggingRequested, + locks = NetLockDl, )) # check out the source @@ -459,7 +492,9 @@ for target in targets: repourl = repo_url, branch = repo_branch, mode = 'incremental', - method = 'clean')) + method = 'clean', + locks = NetLockDl, + )) # update remote refs factory.addStep(ShellCommand( @@ -475,7 +510,8 @@ for target in targets: description = "Fetching Git tags", command = ["git", "fetch", "--tags", "--", repo_url], haltOnFailure = True, - doStepIf = IsTaggingRequested + doStepIf = IsTaggingRequested, + locks = NetLockDl, )) # switch to tag @@ -509,7 +545,9 @@ for target in targets: name = "updatefeeds", description = "Updating feeds", command=["./scripts/feeds", "update"], - env = MakeEnv())) + env = MakeEnv(), + locks = NetLockDl, + )) # feed factory.addStep(ShellCommand( @@ -606,7 +644,7 @@ for target in targets: command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"], env = MakeEnv(), logEnviron = False, - locks = [dlLock.access('exclusive')] + locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]), )) factory.addStep(ShellCommand( @@ -830,7 +868,8 @@ for target in targets: command = ["rsync", "-4", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, - logEnviron = False + logEnviron = False, + locks = NetLockUl, )) factory.addStep(ShellCommand( @@ -841,7 +880,8 @@ for target in targets: WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, - logEnviron = False + logEnviron = False, + locks = NetLockUl, )) if enable_kmod_archive: @@ -853,7 +893,8 @@ for target in targets: WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, - logEnviron = False + logEnviron = False, + locks = NetLockUl, )) if rsync_src_url is not None: @@ -864,7 +905,8 @@ for target in targets: WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-avz", "dl/", "%s/" %(rsync_src_url)], env={'RSYNC_PASSWORD': rsync_src_key}, haltOnFailure = True, - logEnviron = False + logEnviron = False, + locks = NetLockUl, )) if False: @@ -874,7 +916,8 @@ for target in targets: command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = False, - logEnviron = False + logEnviron = False, + locks = NetLockUl, )) # logs @@ -886,7 +929,8 @@ for target in targets: env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = False, alwaysRun = True, - logEnviron = False + logEnviron = False, + locks = NetLockUl, )) factory.addStep(ShellCommand(