From: Thibaut VARÈNE <hacks@slashdirt.org>
Date: Wed, 20 Jun 2018 18:37:59 +0000 (+0200)
Subject: phase1: reduce verbosity of rsync
X-Git-Tag: v1~155
X-Git-Url: http://git.lede-project.org./?a=commitdiff_plain;h=e55b76fd2f8188b34a85157fb119722dcee97117;p=buildbot.git

phase1: reduce verbosity of rsync

The purpose of this patch is to reduce uplink bandwidth contention when
rsync uploads are running.

This patch:
- replaces '-v' and '--progress' by '--info=name' in rsync's parameter
  list (assuming it's necessary to output a list of updated files)
- introduces a wrapper script 'rsync.sh' which checks for the presence
  and executability of 'pv' and pipes the output of rsync through
  'pv -t -i 60 -f' if possible, and falls back to plain rsync otherwise.

'--info=name' prints one line to stdout for each file that is updated on
the remote side.
'pv -t -i 60 -f' prints a 1mn-resolution timer to stderr, with the goal
of preventing the current buildbot step to timeout due to lack of process
output.

It's worth noting that '--progress' did not prevent some timeouts (most
notably on the 'sourceupload' step) since --progress only emits output
for files that are updated on the remote side.

Signed-off-by: Thibaut VARÈNE <hacks@slashdirt.org>
---

diff --git a/phase1/master.cfg b/phase1/master.cfg
index f7c1ae8..8ebe6c9 100644
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -865,18 +865,23 @@ for target in targets:
 	factory.addStep(ShellCommand(
 		name = "dirupload",
 		description = "Uploading directory structure",
-		command = ["rsync", "-4", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
+		command = ["rsync", "-4", "--info=name", "-az", "tmp/upload/", "%s/" %(rsync_bin_url)],
 		env={'RSYNC_PASSWORD': rsync_bin_key},
 		haltOnFailure = True,
 		logEnviron = False,
 		locks = NetLockUl,
 	))
 
+	factory.addStep(FileDownload(
+			mastersrc = "rsync.sh",
+			slavedest = "../rsync.sh",
+			mode = 0755))
+
 	factory.addStep(ShellCommand(
 		name = "targetupload",
 		description = "Uploading target files",
-		command=["rsync", "-4", "--exclude=/kmods/", "--progress", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
-		         "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
+		command=["../rsync.sh", "-4", "--info=name", "--exclude=/kmods/", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
+		         "-az", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 		         WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 		env={'RSYNC_PASSWORD': rsync_bin_key},
 		haltOnFailure = True,
@@ -888,8 +893,8 @@ for target in targets:
 		factory.addStep(ShellCommand(
 			name = "kmodupload",
 			description = "Uploading kmod archive",
-			command=["rsync", "-4", "--progress", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
-			         "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])),
+			command=["../rsync.sh", "-4", "--info=name", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
+			         "-az", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])),
 			         WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 			env={'RSYNC_PASSWORD': rsync_bin_key},
 			haltOnFailure = True,
@@ -901,8 +906,8 @@ for target in targets:
 		factory.addStep(ShellCommand(
 			name = "sourceupload",
 			description = "Uploading source archives",
-			command=["rsync", "-4", "--progress", "--size-only", "--delay-updates",
-			         WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-avz", "dl/", "%s/" %(rsync_src_url)],
+			command=["../rsync.sh", "-4", "--info=name", "--size-only", "--delay-updates",
+			         WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-az", "dl/", "%s/" %(rsync_src_url)],
 			env={'RSYNC_PASSWORD': rsync_src_key},
 			haltOnFailure = True,
 			logEnviron = False,
@@ -913,7 +918,7 @@ for target in targets:
 		factory.addStep(ShellCommand(
 			name = "packageupload",
 			description = "Uploading package files",
-			command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
+			command=["../rsync.sh", "-4", "--info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 			env={'RSYNC_PASSWORD': rsync_bin_key},
 			haltOnFailure = False,
 			logEnviron = False,
@@ -925,7 +930,7 @@ for target in targets:
 		factory.addStep(ShellCommand(
 			name = "upload",
 			description = "Uploading logs",
-			command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
+			command=["../rsync.sh", "-4", "-info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 			env={'RSYNC_PASSWORD': rsync_bin_key},
 			haltOnFailure = False,
 			alwaysRun = True,
diff --git a/phase1/rsync.sh b/phase1/rsync.sh
new file mode 100755
index 0000000..64f9576
--- /dev/null
+++ b/phase1/rsync.sh
@@ -0,0 +1,12 @@
+#!/bin/bash -x
+
+export LC_ALL=C
+
+PV=`which pv`
+RSYNC=rsync
+
+if [[ -x $PV ]]; then
+	$RSYNC "$@" | $PV -t -i 60 -f
+else
+	$RSYNC "$@"
+fi