From 54e80d5cedcd61455d2e297ef52db71e470ddf5f Mon Sep 17 00:00:00 2001 From: =?utf8?q?Thibaut=20VAR=C3=88NE?= Date: Tue, 15 Nov 2022 10:44:56 +0100 Subject: [PATCH] phase1: make 'rsync -4' worker-configurable MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit We set '-4' in rsync_defopts, asking rsync to "prefer" ipv4 whenever possible on _all_ workers. This was historically done because some workers had flaky ipv6 connectivity, however in the future the reverse may be true, with worker having worse networking over ipv4 than ipv6. This change introduces an 'rsync_ipv4' worker configuration option: when set to a true value, the old rsync behavior (adding '-4') is used, when unset or set to false, the extra rsync argument is not used. Signed-off-by: Thibaut VARÈNE --- phase1/config.ini.example | 2 ++ phase1/master.cfg | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/phase1/config.ini.example b/phase1/config.ini.example index 5ae45fa..d7262bd 100644 --- a/phase1/config.ini.example +++ b/phase1/config.ini.example @@ -68,3 +68,5 @@ ul_lock = host1 dl_lock = host1 # tag_only workers will only build forced tag buildrequests (i.e. release builds) tag_only = yes +# if rsync operations must prefer ipv4 ('rsync -4'), set the following +rsync_ipv4 = yes diff --git a/phase1/master.cfg b/phase1/master.cfg index 15112ce..f511e03 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -62,7 +62,7 @@ scripts_dir = os.path.abspath("../scripts") repo_url = ini['repo'].get("url") -rsync_defopts = ["-v", "-4", "--timeout=120"] +rsync_defopts = ["-v", "--timeout=120"] #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0: # rsync_bin_defopts += ["--contimeout=20"] @@ -136,6 +136,7 @@ def ini_parse_workers(section): password = section.get("password") phase = section.getint("phase") tagonly = section.getboolean("tag_only") + rsyncipv4 = section.getboolean("rsync_ipv4") if not name or not password or not phase == 1: log.msg("invalid worker configuration ignored: {}".format(repr(section))) @@ -152,6 +153,8 @@ def ini_parse_workers(section): sl_props['ul_lock'] = lockname if lockname not in NetLocks: NetLocks[lockname] = locks.MasterLock(lockname) + if rsyncipv4: + sl_props['rsync_ipv4'] = True # only set prop if required, we use '+' Interpolate substitution log.msg("Configured worker: {}".format(name)) # NB: phase1 build factory requires workers to be single-build only @@ -1120,7 +1123,7 @@ for target in targets: name = "dirupload", description = "Uploading directory structure", descriptionDone = "Directory structure uploaded", - command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))], + command = ["rsync", "-az"] + rsync_defopts + [Interpolate("%(prop:rsync_ipv4:+-4)s"), "tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))], env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) }, haltOnFailure = True, logEnviron = False, @@ -1133,7 +1136,7 @@ for target in targets: name = "target-sha256sums", description = "Fetching remote sha256sums for target", descriptionDone = "Remote sha256sums for target fetched", - command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"], + command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"], env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) }, logEnviron = False, haltOnFailure = False, @@ -1171,7 +1174,7 @@ for target in targets: description = "Uploading target files", descriptionDone = "Target files uploaded", command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts + - ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]), + ["-a", Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]), Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) }, haltOnFailure = True, @@ -1185,7 +1188,7 @@ for target in targets: description = "Pruning target files", descriptionDone = "Target files pruned", command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts + - ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]), + ["-a", Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]), Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) }, haltOnFailure = True, @@ -1199,7 +1202,7 @@ for target in targets: description = "Uploading kmod archive", descriptionDone = "Kmod archive uploaded", command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts + - ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]), + ["-a", Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]), Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) }, haltOnFailure = True, @@ -1220,8 +1223,8 @@ for target in targets: name = "sourceupload", description = "Uploading source archives", descriptionDone = "Source archives uploaded", - command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts + - [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))], + command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts + [Interpolate("%(prop:rsync_ipv4:+-4)s"), + Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))], env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) }, haltOnFailure = True, logEnviron = False, -- 2.30.2