phase1: reduce verbosity of rsync

The purpose of this patch is to reduce uplink bandwidth contention when
rsync uploads are running.

This patch:
- replaces '-v' and '--progress' by '--info=name' in rsync's parameter
  list (assuming it's necessary to output a list of updated files)
- introduces a wrapper script 'rsync.sh' which checks for the presence
  and executability of 'pv' and pipes the output of rsync through
  'pv -t -i 60 -f' if possible, and falls back to plain rsync otherwise.

'--info=name' prints one line to stdout for each file that is updated on
the remote side.
'pv -t -i 60 -f' prints a 1mn-resolution timer to stderr, with the goal
of preventing the current buildbot step to timeout due to lack of process
output.

It's worth noting that '--progress' did not prevent some timeouts (most
notably on the 'sourceupload' step) since --progress only emits output
for files that are updated on the remote side.

Signed-off-by: Thibaut VARÈNE <hacks@slashdirt.org>

diff --git a/phase1/master.cfg b/phase1/master.cfg
index f7c1ae89cee6663538197eda5220ea83ac30cd6f..8ebe6c9dfe85b95cafcc5046be01f093ad976237 100644 (file)
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -865,18 +865,23 @@ for target in targets:
        factory.addStep(ShellCommand(
                name = "dirupload",
                description = "Uploading directory structure",
-               command = ["rsync", "-4", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
+               command = ["rsync", "-4", "--info=name", "-az", "tmp/upload/", "%s/" %(rsync_bin_url)],
                env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
                logEnviron = False,
                locks = NetLockUl,
        ))
 
+       factory.addStep(FileDownload(
+                       mastersrc = "rsync.sh",
+                       slavedest = "../rsync.sh",
+                       mode = 0755))
+
        factory.addStep(ShellCommand(
                name = "targetupload",
                description = "Uploading target files",
-               command=["rsync", "-4", "--exclude=/kmods/", "--progress", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
-                        "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
+               command=["../rsync.sh", "-4", "--info=name", "--exclude=/kmods/", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
+                        "-az", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
                         WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
                env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
@@ -888,8 +893,8 @@ for target in targets:
                factory.addStep(ShellCommand(
                        name = "kmodupload",
                        description = "Uploading kmod archive",
-                       command=["rsync", "-4", "--progress", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
-                                "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])),
+                       command=["../rsync.sh", "-4", "--info=name", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
+                                "-az", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])),
                                 WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
                        env={'RSYNC_PASSWORD': rsync_bin_key},
                        haltOnFailure = True,
@@ -901,8 +906,8 @@ for target in targets:
                factory.addStep(ShellCommand(
                        name = "sourceupload",
                        description = "Uploading source archives",
-                       command=["rsync", "-4", "--progress", "--size-only", "--delay-updates",
-                                WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-avz", "dl/", "%s/" %(rsync_src_url)],
+                       command=["../rsync.sh", "-4", "--info=name", "--size-only", "--delay-updates",
+                                WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-az", "dl/", "%s/" %(rsync_src_url)],
                        env={'RSYNC_PASSWORD': rsync_src_key},
                        haltOnFailure = True,
                        logEnviron = False,
@@ -913,7 +918,7 @@ for target in targets:
                factory.addStep(ShellCommand(
                        name = "packageupload",
                        description = "Uploading package files",
-                       command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
+                       command=["../rsync.sh", "-4", "--info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
                        env={'RSYNC_PASSWORD': rsync_bin_key},
                        haltOnFailure = False,
                        logEnviron = False,
@@ -925,7 +930,7 @@ for target in targets:
                factory.addStep(ShellCommand(
                        name = "upload",
                        description = "Uploading logs",
-                       command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
+                       command=["../rsync.sh", "-4", "-info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
                        env={'RSYNC_PASSWORD': rsync_bin_key},
                        haltOnFailure = False,
                        alwaysRun = True,

diff --git a/phase1/rsync.sh b/phase1/rsync.sh
new file mode 100755 (executable)
index 0000000..64f9576
--- /dev/null
+++ b/phase1/rsync.sh
@@ -0,0 +1,12 @@
+#!/bin/bash -x
+
+export LC_ALL=C
+
+PV=`which pv`
+RSYNC=rsync
+
+if [[ -x $PV ]]; then
+       $RSYNC "$@" | $PV -t -i 60 -f
+else
+       $RSYNC "$@"
+fi