From cf7b9baa818a2e183db9738669bcafd06409ddde Mon Sep 17 00:00:00 2001 From: =?utf8?q?Thibaut=20VAR=C3=88NE?= Date: Thu, 21 Jun 2018 13:51:26 +0200 Subject: [PATCH] phase1: don't enable rsync compression where unnecessary MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Enabling rsync compression for already compressed files adds CPU and memory overhead that negatively impacts upload speed on data that cannot be further compressed. By default rsync ignores '-z' on a number of files it knows to be already compressed (based on filename suffix), but the list is not exhaustive and doesn't include e.g. '.ipk'. Instead of trying to maintain a list of all known compressed suffixes uploaded by the build system, it's simpler and less error prone to simply disable the option for specific steps where all or nearly all of the transferred files are known to be already compressed. Signed-off-by: Thibaut VARÈNE --- phase1/master.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index 8ebe6c9..f6027ff 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -881,7 +881,7 @@ for target in targets: name = "targetupload", description = "Uploading target files", command=["../rsync.sh", "-4", "--info=name", "--exclude=/kmods/", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), - "-az", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), + "-a", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, @@ -894,7 +894,7 @@ for target in targets: name = "kmodupload", description = "Uploading kmod archive", command=["../rsync.sh", "-4", "--info=name", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), - "-az", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])), + "-a", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])), WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, @@ -907,7 +907,7 @@ for target in targets: name = "sourceupload", description = "Uploading source archives", command=["../rsync.sh", "-4", "--info=name", "--size-only", "--delay-updates", - WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-az", "dl/", "%s/" %(rsync_src_url)], + WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-a", "dl/", "%s/" %(rsync_src_url)], env={'RSYNC_PASSWORD': rsync_src_key}, haltOnFailure = True, logEnviron = False, @@ -918,7 +918,7 @@ for target in targets: factory.addStep(ShellCommand( name = "packageupload", description = "Uploading package files", - command=["../rsync.sh", "-4", "--info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "bin/packages/", "%s/packages/" %(rsync_bin_url)], + command=["../rsync.sh", "-4", "--info=name", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a", "bin/packages/", "%s/packages/" %(rsync_bin_url)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = False, logEnviron = False, -- 2.30.2