From: Thibaut VARÈNE Date: Wed, 20 Jun 2018 15:36:02 +0000 (+0200) Subject: phase1: rsync: use --size-only instead of --checksum X-Git-Tag: v1~156 X-Git-Url: http://git.lede-project.org./?a=commitdiff_plain;h=62a01a1;p=buildbot.git phase1: rsync: use --size-only instead of --checksum '--checksum' puts heavy I/O and memory load on both the sender and the receiver and dramatically reduces rsync's speedup. Furthermore, if multiple slaves are uploading at once the receiver may become overloaded, slowing down the whole build farm. From rsync manpage: Generating the checksums means that both sides will expend a lot of disk I/O reading all the data in the files in the transfer (and this is prior to any reading that will be done to transfer changed files), so this can slow things down significantly. This slowdown causes timeouts for some build slaves, noticeably in the 'sourceupload' step. '--size-only' matches files by size only. Compared to '--checksum', it is faster, and does not aggravate the collision/error risk (if two files of the same name and size have different checksums, we don't have a heuristic to determine which is "correct" and which is "invalid" anyway) Signed-off-by: Thibaut VARÈNE --- diff --git a/phase1/master.cfg b/phase1/master.cfg index 9643d28..f7c1ae8 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -875,7 +875,7 @@ for target in targets: factory.addStep(ShellCommand( name = "targetupload", description = "Uploading target files", - command=["rsync", "-4", "--exclude=/kmods/", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), + command=["rsync", "-4", "--exclude=/kmods/", "--progress", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, @@ -888,7 +888,7 @@ for target in targets: factory.addStep(ShellCommand( name = "kmodupload", description = "Uploading kmod archive", - command=["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), + command=["rsync", "-4", "--progress", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])), WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, @@ -901,7 +901,7 @@ for target in targets: factory.addStep(ShellCommand( name = "sourceupload", description = "Uploading source archives", - command=["rsync", "-4", "--progress", "--checksum", "--delay-updates", + command=["rsync", "-4", "--progress", "--size-only", "--delay-updates", WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-avz", "dl/", "%s/" %(rsync_src_url)], env={'RSYNC_PASSWORD': rsync_src_key}, haltOnFailure = True,