From: Jo-Philipp Wich Date: Wed, 6 Dec 2017 15:25:21 +0000 (+0100) Subject: phase1: add support for per-slave cleanup X-Git-Tag: v1~163 X-Git-Url: http://git.lede-project.org./?a=commitdiff_plain;h=0339b05397f5199aa0992aa84871ef8d3c1cb4b4;p=buildbot.git phase1: add support for per-slave cleanup Signed-off-by: Jo-Philipp Wich --- diff --git a/phase1/cleanup.sh b/phase1/cleanup.sh new file mode 100755 index 0000000..f2d0bcc --- /dev/null +++ b/phase1/cleanup.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +export LC_ALL=C + +master_url="$1" +current_slave="$2" +current_builder="$3" +current_mode="$4" + +running_builders="$(wget -qO- "${master_url%/}/json/slaves/$current_slave?as_text=1" | sed -ne 's,^.*"builderName": "\(.*\)".*$,\1,p')" + +is_running() { + local running_builder + for running_builder in $running_builders; do + if [ "${running_builder//\//_}" = "${1//\//_}" ]; then + return 0 + fi + done + return 1 +} + +do_cleanup() { + printf "Cleaning up '$current_builder' work directory" + + if [ -d .git ]; then + echo " using git" + git reset --hard HEAD + git clean -f -d -x + else + find . -mindepth 1 -maxdepth 1 | while read entry; do + rm -vrf "$entry" | while read entry2; do + case "$entry2" in *directory:*) + printf "." + esac + done + done + fi + + echo "" +} + +# +# Sanity check, current builder should be in running builders list +# + +if ! is_running "$current_builder"; then + echo "Current builder '$current_builder' not found in current builders list, aborting cleanup." + exit 1 +fi + + +# +# Clean up leftovers +# + +if [ "$current_mode" = full ]; then +( + if ! flock -x -w 2700 200; then + echo "Unable to obtain exclusive lock, aborting cleanup." + exit 1 + fi + + for build_dir in ../../*; do + + build_dir="$(readlink -f "$build_dir")" + + if [ -z "$build_dir" ] || [ ! -d "$build_dir/build/build_dir" ]; then + continue + fi + + current_builder="${build_dir##*/}" + + if is_running "$current_builder"; then + echo "Skipping currently active '$current_builder' work directory." + continue + fi + + ( + cd "$build_dir/build" + + if [ -d build_dir ]; then + do_cleanup + else + echo "Skipping clean '$current_builder' work directory." + fi + ) + done + +) 200>../../cleanup.lock + +# +# Clean up current build +# + +else + do_cleanup +fi + +exit 0 diff --git a/phase1/config.ini.example b/phase1/config.ini.example index bfbe837..47e454d 100644 --- a/phase1/config.ini.example +++ b/phase1/config.ini.example @@ -44,3 +44,4 @@ builds = 3 name = example-slave-2 password = example2 builds = 1 +cleanup = 1 diff --git a/phase1/master.cfg b/phase1/master.cfg index a68d067..8f53f9d 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -18,6 +18,23 @@ ini.read("./config.ini") # a shorter alias to save typing. c = BuildmasterConfig = {} +####### PROJECT IDENTITY + +# the 'title' string will appear at the top of this buildbot +# installation's html.WebStatus home page (linked to the +# 'titleURL') and is embedded in the title of the waterfall HTML page. + +c['title'] = ini.get("general", "title") +c['titleURL'] = ini.get("general", "title_url") + +# the 'buildbotURL' string should point to the location where the buildbot's +# internal web server (usually the html.WebStatus page) is visible. This +# typically uses the port number set in the Waterfall 'status' entry, but +# with an externally-visible host name which the buildbot cannot figure out +# without some help. + +c['buildbotURL'] = ini.get("general", "buildbot_url") + ####### BUILDSLAVES # The 'slaves' list defines the set of recognized buildslaves. Each element is @@ -32,6 +49,7 @@ if ini.has_option("general", "port"): c['slaves'] = [] max_builds = dict() +do_cleanup = dict() for section in ini.sections(): if section.startswith("slave "): @@ -39,8 +57,11 @@ for section in ini.sections(): name = ini.get(section, "name") password = ini.get(section, "password") max_builds[name] = 1 + do_cleanup[name] = False if ini.has_option(section, "builds"): max_builds[name] = ini.getint(section, "builds") + if ini.has_option(section, "cleanup"): + do_cleanup[name] = ini.getboolean(section, "cleanup") c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name])) # 'slavePortnum' defines the TCP port to listen on for connections from slaves. @@ -182,7 +203,7 @@ CleanTargetMap = [ [ "dist", "distclean" ] ] -def IsCleanRequested(pattern): +def IsMakeCleanRequested(pattern): def CheckCleanProperty(step): val = step.getProperty("clean") if val and re.match(pattern, val): @@ -192,6 +213,13 @@ def IsCleanRequested(pattern): return CheckCleanProperty +def IsCleanupRequested(step): + val = step.getProperty("slavename") + if val and do_cleanup[val]: + return True + else: + return False + def IsTaggingRequested(step): val = step.getProperty("tag") if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val): @@ -373,6 +401,29 @@ for target in targets: haltOnFailure = True, timeout = 2400)) + # cleanup.sh if needed + factory.addStep(FileDownload( + mastersrc = "cleanup.sh", + slavedest = "cleanup.sh", + mode = 0755, + doStepIf = IsCleanupRequested)) + + factory.addStep(ShellCommand( + name = "cleanold", + description = "Cleaning previous builds", + command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"], + haltOnFailure = True, + doStepIf = IsCleanupRequested, + timeout = 2400)) + + factory.addStep(ShellCommand( + name = "cleanup", + description = "Cleaning work area", + command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"], + haltOnFailure = True, + doStepIf = IsCleanupRequested, + timeout = 2400)) + # user-requested clean targets for tuple in CleanTargetMap: factory.addStep(ShellCommand( @@ -380,7 +431,7 @@ for target in targets: description = 'User-requested "make %s"' % tuple[1], command = ["make", tuple[1], "V=s"], env = MakeEnv(), - doStepIf = IsCleanRequested(tuple[0]) + doStepIf = IsMakeCleanRequested(tuple[0]) )) # switch to branch @@ -808,24 +859,6 @@ if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.h c['status'].append(irc) - -####### PROJECT IDENTITY - -# the 'title' string will appear at the top of this buildbot -# installation's html.WebStatus home page (linked to the -# 'titleURL') and is embedded in the title of the waterfall HTML page. - -c['title'] = ini.get("general", "title") -c['titleURL'] = ini.get("general", "title_url") - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server (usually the html.WebStatus page) is visible. This -# typically uses the port number set in the Waterfall 'status' entry, but -# with an externally-visible host name which the buildbot cannot figure out -# without some help. - -c['buildbotURL'] = ini.get("general", "buildbot_url") - ####### DB URL c['db'] = {