From e55b76fd2f8188b34a85157fb119722dcee97117 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Thibaut=20VAR=C3=88NE?= Date: Wed, 20 Jun 2018 20:37:59 +0200 Subject: [PATCH] phase1: reduce verbosity of rsync MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit The purpose of this patch is to reduce uplink bandwidth contention when rsync uploads are running. This patch: - replaces '-v' and '--progress' by '--info=name' in rsync's parameter list (assuming it's necessary to output a list of updated files) - introduces a wrapper script 'rsync.sh' which checks for the presence and executability of 'pv' and pipes the output of rsync through 'pv -t -i 60 -f' if possible, and falls back to plain rsync otherwise. '--info=name' prints one line to stdout for each file that is updated on the remote side. 'pv -t -i 60 -f' prints a 1mn-resolution timer to stderr, with the goal of preventing the current buildbot step to timeout due to lack of process output. It's worth noting that '--progress' did not prevent some timeouts (most notably on the 'sourceupload' step) since --progress only emits output for files that are updated on the remote side. Signed-off-by: Thibaut VARÈNE --- phase1/master.cfg | 23 ++++++++++++++--------- phase1/rsync.sh | 12 ++++++++++++ 2 files changed, 26 insertions(+), 9 deletions(-) create mode 100755 phase1/rsync.sh diff --git a/phase1/master.cfg b/phase1/master.cfg index f7c1ae8..8ebe6c9 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -865,18 +865,23 @@ for target in targets: factory.addStep(ShellCommand( name = "dirupload", description = "Uploading directory structure", - command = ["rsync", "-4", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)], + command = ["rsync", "-4", "--info=name", "-az", "tmp/upload/", "%s/" %(rsync_bin_url)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False, locks = NetLockUl, )) + factory.addStep(FileDownload( + mastersrc = "rsync.sh", + slavedest = "../rsync.sh", + mode = 0755)) + factory.addStep(ShellCommand( name = "targetupload", description = "Uploading target files", - command=["rsync", "-4", "--exclude=/kmods/", "--progress", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), - "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), + command=["../rsync.sh", "-4", "--info=name", "--exclude=/kmods/", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), + "-az", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, @@ -888,8 +893,8 @@ for target in targets: factory.addStep(ShellCommand( name = "kmodupload", description = "Uploading kmod archive", - command=["rsync", "-4", "--progress", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), - "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])), + command=["../rsync.sh", "-4", "--info=name", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), + "-az", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])), WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, @@ -901,8 +906,8 @@ for target in targets: factory.addStep(ShellCommand( name = "sourceupload", description = "Uploading source archives", - command=["rsync", "-4", "--progress", "--size-only", "--delay-updates", - WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-avz", "dl/", "%s/" %(rsync_src_url)], + command=["../rsync.sh", "-4", "--info=name", "--size-only", "--delay-updates", + WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-az", "dl/", "%s/" %(rsync_src_url)], env={'RSYNC_PASSWORD': rsync_src_key}, haltOnFailure = True, logEnviron = False, @@ -913,7 +918,7 @@ for target in targets: factory.addStep(ShellCommand( name = "packageupload", description = "Uploading package files", - command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)], + command=["../rsync.sh", "-4", "--info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "bin/packages/", "%s/packages/" %(rsync_bin_url)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = False, logEnviron = False, @@ -925,7 +930,7 @@ for target in targets: factory.addStep(ShellCommand( name = "upload", description = "Uploading logs", - command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])], + command=["../rsync.sh", "-4", "-info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = False, alwaysRun = True, diff --git a/phase1/rsync.sh b/phase1/rsync.sh new file mode 100755 index 0000000..64f9576 --- /dev/null +++ b/phase1/rsync.sh @@ -0,0 +1,12 @@ +#!/bin/bash -x + +export LC_ALL=C + +PV=`which pv` +RSYNC=rsync + +if [[ -x $PV ]]; then + $RSYNC "$@" | $PV -t -i 60 -f +else + $RSYNC "$@" +fi -- 2.30.2