Determine number of cores dynamically

Signed-off-by: Jo-Philipp Wich <jo@mein.io>

diff --git a/phase1/master.cfg b/phase1/master.cfg
index fa3161de53b4c1b8dda33811099be6d5b36e4fbf..9018c5e0625c342cd9a5f86fbe2aeb058e062675 100644 (file)
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -106,7 +106,9 @@ c['schedulers'].append(ForceScheduler(
 from buildbot.process.factory import BuildFactory
 from buildbot.steps.source import Git
 from buildbot.steps.shell import ShellCommand
+from buildbot.steps.shell import SetProperty
 from buildbot.steps.transfer import FileDownload
+from buildbot.process.properties import WithProperties
 
 
 MakeTargetMap = {
@@ -239,6 +241,12 @@ for target in targets:
 
        factory = BuildFactory()
 
+       # find number of cores
+       factory.addStep(SetProperty(
+               name = "nproc",
+               description = "Finding number of CPUs",
+               command = ["nproc"]))
+
        # check out the source
        factory.addStep(Git(repourl=repo_url, mode='update'))
 
@@ -280,7 +288,7 @@ CONFIG_DEVEL=y
 CONFIG_CCACHE=y
 CONFIG_SIGNED_PACKAGES=y
 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
-CONFIG_KERNEL_KALLSYMS=y 
+CONFIG_KERNEL_KALLSYMS=y
 CONFIG_COLLECT_KERNEL_DEBUG=y
 EOT''' %(ts[0], ts[0], ts[1]) ))
 
@@ -334,7 +342,7 @@ EOT''' %(ts[0], ts[0], ts[1]) ))
        factory.addStep(ShellCommand(
                name = "dlrun",
                description = "Populating dl/",
-               command = ["make", "-j4", "download", "V=s"],
+               command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
                logEnviron = False,
                locks = [dlLock.access('exclusive')]
        ))
@@ -357,21 +365,21 @@ EOT''' %(ts[0], ts[0], ts[1]) ))
        factory.addStep(ShellCommand(
                name = "tools",
                description = "Building tools",
-               command = ["make", "-j4", "tools/install", "V=s"],
+               command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
                haltOnFailure = True
        ))
 
        factory.addStep(ShellCommand(
                name = "toolchain",
                description = "Building toolchain",
-               command=["make", "-j4", "toolchain/install", "V=s"],
+               command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
                haltOnFailure = True
        ))
 
        factory.addStep(ShellCommand(
                name = "kmods",
                description = "Building kmods",
-               command=["make", "-j4", "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+               command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
                #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
                haltOnFailure = True
        ))
@@ -379,7 +387,7 @@ EOT''' %(ts[0], ts[0], ts[1]) ))
        factory.addStep(ShellCommand(
                name = "pkgbuild",
                description = "Building packages",
-               command=["make", "-j4", "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+               command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
                #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
                haltOnFailure = True
        ))
@@ -388,7 +396,7 @@ EOT''' %(ts[0], ts[0], ts[1]) ))
        factory.addStep(ShellCommand(
                name = "pkginstall",
                description = "Installing packages",
-               command=["make", "-j4", "package/install", "V=s"],
+               command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
                doStepIf = isChangeBuiltin,
                haltOnFailure = True
        ))
@@ -396,7 +404,7 @@ EOT''' %(ts[0], ts[0], ts[1]) ))
        factory.addStep(ShellCommand(
                name = "pkgindex",
                description = "Indexing packages",
-               command=["make", "-j4", "package/index", "V=s"],
+               command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
                haltOnFailure = True
        ))
 
@@ -508,4 +516,3 @@ c['db'] = {
        # this at its default for all but the largest installations.
        'db_url' : "sqlite:///state.sqlite",
 }
-

diff --git a/phase2/master.cfg b/phase2/master.cfg
index 21862d4ffce4716c3eaaaf3dfc4544b218e98965..39e36e50d857da5364160b83b5af77919cff917e 100644 (file)
--- a/phase2/master.cfg
+++ b/phase2/master.cfg
@@ -112,6 +112,7 @@ c['schedulers'].append(ForceScheduler(
 from buildbot.process.factory import BuildFactory
 from buildbot.steps.source import Git
 from buildbot.steps.shell import ShellCommand
+from buildbot.steps.shell import SetProperty
 from buildbot.steps.transfer import FileDownload
 from buildbot.process.properties import WithProperties
 
@@ -129,6 +130,12 @@ for arch in arches:
 
        factory = BuildFactory()
 
+       # find number of cores
+       factory.addStep(SetProperty(
+               name = "nproc",
+               description = "Finding number of CPUs",
+               command = ["nproc"]))
+
        # prepare workspace
        factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 
@@ -188,7 +195,7 @@ for arch in arches:
                name = "compile",
                description = "Building packages",
                workdir = "build/sdk",
-               command = ["make", "-j4", "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
+               command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 
        factory.addStep(ShellCommand(
                name = "uploadprepare",
@@ -304,4 +311,3 @@ c['db'] = {
        # this at its default for all but the largest installations.
        'db_url' : "sqlite:///state.sqlite",
 }
-