phase1: add support for per-slave cleanup

Signed-off-by: Jo-Philipp Wich <jo@mein.io>

diff --git a/phase1/cleanup.sh b/phase1/cleanup.sh
new file mode 100755 (executable)
index 0000000..f2d0bcc
--- /dev/null
+++ b/phase1/cleanup.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+export LC_ALL=C
+
+master_url="$1"
+current_slave="$2"
+current_builder="$3"
+current_mode="$4"
+
+running_builders="$(wget -qO- "${master_url%/}/json/slaves/$current_slave?as_text=1" | sed -ne 's,^.*"builderName": "\(.*\)".*$,\1,p')"
+
+is_running() {
+       local running_builder
+       for running_builder in $running_builders; do
+               if [ "${running_builder//\//_}" = "${1//\//_}" ]; then
+                               return 0
+               fi
+       done
+       return 1
+}
+
+do_cleanup() {
+       printf "Cleaning up '$current_builder' work directory"
+
+       if [ -d .git ]; then
+               echo " using git"
+               git reset --hard HEAD
+               git clean -f -d -x
+       else
+               find . -mindepth 1 -maxdepth 1 | while read entry; do
+                       rm -vrf "$entry" | while read entry2; do
+                               case "$entry2" in *directory:*)
+                                       printf "."
+                               esac
+                       done
+               done
+       fi
+
+       echo ""
+}
+
+#
+# Sanity check, current builder should be in running builders list
+#
+
+if ! is_running "$current_builder"; then
+       echo "Current builder '$current_builder' not found in current builders list, aborting cleanup."
+       exit 1
+fi
+
+
+#
+# Clean up leftovers
+#
+
+if [ "$current_mode" = full ]; then
+(
+       if ! flock -x -w 2700 200; then
+               echo "Unable to obtain exclusive lock, aborting cleanup."
+               exit 1
+       fi
+
+       for build_dir in ../../*; do
+
+               build_dir="$(readlink -f "$build_dir")"
+
+               if [ -z "$build_dir" ] || [ ! -d "$build_dir/build/build_dir" ]; then
+                       continue
+               fi
+
+               current_builder="${build_dir##*/}"
+
+               if is_running "$current_builder"; then
+                       echo "Skipping currently active '$current_builder' work directory."
+                       continue
+               fi
+
+               (
+                       cd "$build_dir/build"
+
+                       if [ -d build_dir ]; then
+                               do_cleanup
+                       else
+                               echo "Skipping clean '$current_builder' work directory."
+                       fi
+               )
+       done
+
+) 200>../../cleanup.lock
+
+#
+# Clean up current build
+#
+
+else
+       do_cleanup
+fi
+
+exit 0

diff --git a/phase1/config.ini.example b/phase1/config.ini.example
index bfbe837465dd0992d98715a741867f704951732c..47e454d78bfa6519c55908181fc8a2d170459429 100644 (file)
--- a/phase1/config.ini.example
+++ b/phase1/config.ini.example
@@ -44,3 +44,4 @@ builds = 3
 name = example-slave-2
 password = example2
 builds = 1
+cleanup = 1

diff --git a/phase1/master.cfg b/phase1/master.cfg
index a68d0671fccff3d4df30b584cfbcc085bcdde6ae..8f53f9de0e586969b2058e4be5d1564f52244835 100644 (file)
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -18,6 +18,23 @@ ini.read("./config.ini")
 # a shorter alias to save typing.
 c = BuildmasterConfig = {}
 
+####### PROJECT IDENTITY
+
+# the 'title' string will appear at the top of this buildbot
+# installation's html.WebStatus home page (linked to the
+# 'titleURL') and is embedded in the title of the waterfall HTML page.
+
+c['title'] = ini.get("general", "title")
+c['titleURL'] = ini.get("general", "title_url")
+
+# the 'buildbotURL' string should point to the location where the buildbot's
+# internal web server (usually the html.WebStatus page) is visible. This
+# typically uses the port number set in the Waterfall 'status' entry, but
+# with an externally-visible host name which the buildbot cannot figure out
+# without some help.
+
+c['buildbotURL'] = ini.get("general", "buildbot_url")
+
 ####### BUILDSLAVES
 
 # The 'slaves' list defines the set of recognized buildslaves. Each element is
@@ -32,6 +49,7 @@ if ini.has_option("general", "port"):
 
 c['slaves'] = []
 max_builds = dict()
+do_cleanup = dict()
 
 for section in ini.sections():
        if section.startswith("slave "):
@@ -39,8 +57,11 @@ for section in ini.sections():
                        name = ini.get(section, "name")
                        password = ini.get(section, "password")
                        max_builds[name] = 1
+                       do_cleanup[name] = False
                        if ini.has_option(section, "builds"):
                                max_builds[name] = ini.getint(section, "builds")
+                       if ini.has_option(section, "cleanup"):
+                               do_cleanup[name] = ini.getboolean(section, "cleanup")
                        c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
 
 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
@@ -182,7 +203,7 @@ CleanTargetMap = [
        [ "dist",       "distclean"                             ]
 ]
 
-def IsCleanRequested(pattern):
+def IsMakeCleanRequested(pattern):
        def CheckCleanProperty(step):
                val = step.getProperty("clean")
                if val and re.match(pattern, val):
@@ -192,6 +213,13 @@ def IsCleanRequested(pattern):
 
        return CheckCleanProperty
 
+def IsCleanupRequested(step):
+       val = step.getProperty("slavename")
+       if val and do_cleanup[val]:
+               return True
+       else:
+               return False
+
 def IsTaggingRequested(step):
        val = step.getProperty("tag")
        if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
@@ -373,6 +401,29 @@ for target in targets:
                        haltOnFailure = True,
                        timeout = 2400))
 
+       # cleanup.sh if needed
+       factory.addStep(FileDownload(
+               mastersrc = "cleanup.sh",
+               slavedest = "cleanup.sh",
+               mode = 0755,
+               doStepIf = IsCleanupRequested))
+
+       factory.addStep(ShellCommand(
+               name = "cleanold",
+               description = "Cleaning previous builds",
+               command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
+               haltOnFailure = True,
+               doStepIf = IsCleanupRequested,
+               timeout = 2400))
+
+       factory.addStep(ShellCommand(
+               name = "cleanup",
+               description = "Cleaning work area",
+               command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
+               haltOnFailure = True,
+               doStepIf = IsCleanupRequested,
+               timeout = 2400))
+
        # user-requested clean targets
        for tuple in CleanTargetMap:
                factory.addStep(ShellCommand(
@@ -380,7 +431,7 @@ for target in targets:
                        description = 'User-requested "make %s"' % tuple[1],
                        command = ["make", tuple[1], "V=s"],
                        env = MakeEnv(),
-                       doStepIf = IsCleanRequested(tuple[0])
+                       doStepIf = IsMakeCleanRequested(tuple[0])
                ))
 
        # switch to branch
@@ -808,24 +859,6 @@ if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.h
 
        c['status'].append(irc)
 
-
-####### PROJECT IDENTITY
-
-# the 'title' string will appear at the top of this buildbot
-# installation's html.WebStatus home page (linked to the
-# 'titleURL') and is embedded in the title of the waterfall HTML page.
-
-c['title'] = ini.get("general", "title")
-c['titleURL'] = ini.get("general", "title_url")
-
-# the 'buildbotURL' string should point to the location where the buildbot's
-# internal web server (usually the html.WebStatus page) is visible. This
-# typically uses the port number set in the Waterfall 'status' entry, but
-# with an externally-visible host name which the buildbot cannot figure out
-# without some help.
-
-c['buildbotURL'] = ini.get("general", "buildbot_url")
-
 ####### DB URL
 
 c['db'] = {