From 62a01a12129999d7e201786d9392af010f43aaf9 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Thibaut=20VAR=C3=88NE?= Date: Wed, 20 Jun 2018 17:36:02 +0200 Subject: [PATCH] phase1: rsync: use --size-only instead of --checksum MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit '--checksum' puts heavy I/O and memory load on both the sender and the receiver and dramatically reduces rsync's speedup. Furthermore, if multiple slaves are uploading at once the receiver may become overloaded, slowing down the whole build farm. From rsync manpage: Generating the checksums means that both sides will expend a lot of disk I/O reading all the data in the files in the transfer (and this is prior to any reading that will be done to transfer changed files), so this can slow things down significantly. This slowdown causes timeouts for some build slaves, noticeably in the 'sourceupload' step. '--size-only' matches files by size only. Compared to '--checksum', it is faster, and does not aggravate the collision/error risk (if two files of the same name and size have different checksums, we don't have a heuristic to determine which is "correct" and which is "invalid" anyway) Signed-off-by: Thibaut VARÈNE --- phase1/master.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index 9643d28..f7c1ae8 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -875,7 +875,7 @@ for target in targets: factory.addStep(ShellCommand( name = "targetupload", description = "Uploading target files", - command=["rsync", "-4", "--exclude=/kmods/", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), + command=["rsync", "-4", "--exclude=/kmods/", "--progress", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, @@ -888,7 +888,7 @@ for target in targets: factory.addStep(ShellCommand( name = "kmodupload", description = "Uploading kmod archive", - command=["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), + command=["rsync", "-4", "--progress", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])), WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, @@ -901,7 +901,7 @@ for target in targets: factory.addStep(ShellCommand( name = "sourceupload", description = "Uploading source archives", - command=["rsync", "-4", "--progress", "--checksum", "--delay-updates", + command=["rsync", "-4", "--progress", "--size-only", "--delay-updates", WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-avz", "dl/", "%s/" %(rsync_src_url)], env={'RSYNC_PASSWORD': rsync_src_key}, haltOnFailure = True, -- 2.30.2