From: Jo-Philipp Wich Date: Thu, 15 Dec 2016 16:14:32 +0000 (+0000) Subject: global: calculate suitable number of jobs X-Git-Tag: v1~202 X-Git-Url: http://git.lede-project.org./?a=commitdiff_plain;h=5a1e485dd9e24c7025ff9cf00ef13c73b749614b;p=buildbot.git global: calculate suitable number of jobs Use the formula (n_cpus / n_builds) + 1 to calculate a suitable value for -j. Also introduce a new general configuration option "other_builds" which is added to n_builds. This is needed in cases where multiple build masters run on the same machine. Signed-off-by: Jo-Philipp Wich --- diff --git a/phase1/master.cfg b/phase1/master.cfg index b9bbf12..554fe4c 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -31,16 +31,17 @@ if ini.has_option("general", "port"): slave_port = ini.getint("general", "port") c['slaves'] = [] +max_builds = dict() for section in ini.sections(): if section.startswith("slave "): if ini.has_option(section, "name") and ini.has_option(section, "password"): name = ini.get(section, "name") password = ini.get(section, "password") - max_builds = 1 + max_builds[name] = 1 if ini.has_option(section, "builds"): - max_builds = ini.getint(section, "builds") - c['slaves'].append(BuildSlave(name, password, max_builds = max_builds)) + max_builds[name] = ini.getint(section, "builds") + c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name])) # 'slavePortnum' defines the TCP port to listen on for connections from slaves. # This must match the value configured into the buildslaves (with their @@ -54,10 +55,14 @@ c['mergeRequests'] = True home_dir = os.path.abspath(ini.get("general", "homedir")) tree_expire = 0 +other_builds = 0 if ini.has_option("general", "expire"): tree_expire = ini.getint("general", "expire") +if ini.has_option("general", "other_builds"): + other_builds = ini.getint("general", "other_builds") + repo_url = ini.get("repo", "url") repo_branch = "master" @@ -202,6 +207,12 @@ def GetVersionPrefix(props): else: return "" +def GetNumJobs(props): + if props.hasProperty("slavename") and props.hasProperty("nproc"): + return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1) + else: + return 1 + c['builders'] = [] @@ -446,7 +457,7 @@ for target in targets: factory.addStep(ShellCommand( name = "dltar", description = "Building GNU tar", - command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"], + command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/install", "V=s"], haltOnFailure = True )) @@ -454,7 +465,7 @@ for target in targets: factory.addStep(ShellCommand( name = "dlrun", description = "Populating dl/", - command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"], + command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"], logEnviron = False, locks = [dlLock.access('exclusive')] )) @@ -469,21 +480,21 @@ for target in targets: factory.addStep(ShellCommand( name = "tools", description = "Building tools", - command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"], + command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"], haltOnFailure = True )) factory.addStep(ShellCommand( name = "toolchain", description = "Building toolchain", - command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"], + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"], haltOnFailure = True )) factory.addStep(ShellCommand( name = "kmods", description = "Building kmods", - command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])}, haltOnFailure = True )) @@ -491,7 +502,7 @@ for target in targets: factory.addStep(ShellCommand( name = "pkgbuild", description = "Building packages", - command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])}, haltOnFailure = True )) @@ -500,14 +511,14 @@ for target in targets: factory.addStep(ShellCommand( name = "pkginstall", description = "Installing packages", - command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"], + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"], haltOnFailure = True )) factory.addStep(ShellCommand( name = "pkgindex", description = "Indexing packages", - command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"], + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"], haltOnFailure = True )) @@ -515,7 +526,7 @@ for target in targets: factory.addStep(ShellCommand( name = "images", description = "Building images", - command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"], + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"], haltOnFailure = True )) diff --git a/phase2/master.cfg b/phase2/master.cfg index 06e1b59..c711b09 100644 --- a/phase2/master.cfg +++ b/phase2/master.cfg @@ -29,6 +29,7 @@ from buildbot.buildslave import BuildSlave slave_port = 9990 persistent = False +other_builds = 0 if ini.has_option("general", "port"): slave_port = ini.getint("general", "port") @@ -36,17 +37,21 @@ if ini.has_option("general", "port"): if ini.has_option("general", "persistent"): persistent = ini.getboolean("general", "persistent") +if ini.has_option("general", "other_builds"): + other_builds = ini.getint("general", "other_builds") + c['slaves'] = [] +max_builds = dict() for section in ini.sections(): if section.startswith("slave "): if ini.has_option(section, "name") and ini.has_option(section, "password"): name = ini.get(section, "name") password = ini.get(section, "password") - max_builds = 1 + max_builds[name] = 1 if ini.has_option(section, "builds"): - max_builds = ini.getint(section, "builds") - c['slaves'].append(BuildSlave(name, password, max_builds = max_builds)) + max_builds[name] = ini.getint(section, "builds") + c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name])) # 'slavePortnum' defines the TCP port to listen on for connections from slaves. # This must match the value configured into the buildslaves (with their @@ -179,6 +184,12 @@ def GetDirectorySuffix(props): else: return "" +def GetNumJobs(props): + if props.hasProperty("slavename") and props.hasProperty("nproc"): + return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1) + else: + return 1 + c['builders'] = [] @@ -269,7 +280,7 @@ for arch in arches: name = "compile", description = "Building packages", workdir = "build/sdk", - command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"])) + command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"])) if gpg_keyid is not None: factory.addStep(MasterShellCommand(