phase1: make 'rsync -4' worker-configurable

We set '-4' in rsync_defopts, asking rsync to "prefer" ipv4 whenever
possible on _all_ workers. This was historically done because some
workers had flaky ipv6 connectivity, however in the future the reverse
may be true, with worker having worse networking over ipv4 than ipv6.

This change introduces an 'rsync_ipv4' worker configuration option:
when set to a true value, the old rsync behavior (adding '-4') is used,
when unset or set to false, the extra rsync argument is not used.

Signed-off-by: Thibaut VARÈNE <hacks@slashdirt.org>

diff --git a/phase1/config.ini.example b/phase1/config.ini.example
index 5ae45fa1d1aab93df2feb4b6a5e400d2eed1aa92..d7262bd0bbd400610360427fec2a42290f88c171 100644 (file)
--- a/phase1/config.ini.example
+++ b/phase1/config.ini.example
@@ -68,3 +68,5 @@ ul_lock = host1
 dl_lock = host1
 # tag_only workers will only build forced tag buildrequests (i.e. release builds)
 tag_only = yes
+# if rsync operations must prefer ipv4 ('rsync -4'), set the following
+rsync_ipv4 = yes

diff --git a/phase1/master.cfg b/phase1/master.cfg
index 15112ce921b3884abe6b6a91ae34323a31f24f87..f511e03a5391b184073ce5453cbe2cbc0b584e48 100644 (file)
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -62,7 +62,7 @@ scripts_dir = os.path.abspath("../scripts")
 
 repo_url = ini['repo'].get("url")
 
-rsync_defopts = ["-v", "-4", "--timeout=120"]
+rsync_defopts = ["-v", "--timeout=120"]
 
 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
 #      rsync_bin_defopts += ["--contimeout=20"]
@@ -136,6 +136,7 @@ def ini_parse_workers(section):
        password = section.get("password")
        phase = section.getint("phase")
        tagonly = section.getboolean("tag_only")
+       rsyncipv4 = section.getboolean("rsync_ipv4")
 
        if not name or not password or not phase == 1:
                log.msg("invalid worker configuration ignored: {}".format(repr(section)))
@@ -152,6 +153,8 @@ def ini_parse_workers(section):
                sl_props['ul_lock'] = lockname
                if lockname not in NetLocks:
                        NetLocks[lockname] = locks.MasterLock(lockname)
+       if rsyncipv4:
+               sl_props['rsync_ipv4'] = True   # only set prop if required, we use '+' Interpolate substitution
 
        log.msg("Configured worker: {}".format(name))
        # NB: phase1 build factory requires workers to be single-build only
@@ -1120,7 +1123,7 @@ for target in targets:
                name = "dirupload",
                description = "Uploading directory structure",
                descriptionDone = "Directory structure uploaded",
-               command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
+               command = ["rsync", "-az"] + rsync_defopts + [Interpolate("%(prop:rsync_ipv4:+-4)s"), "tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
                env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
                haltOnFailure = True,
                logEnviron = False,
@@ -1133,7 +1136,7 @@ for target in targets:
                name = "target-sha256sums",
                description = "Fetching remote sha256sums for target",
                descriptionDone = "Remote sha256sums for target fetched",
-               command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
+               command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
                env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
                logEnviron = False,
                haltOnFailure = False,
@@ -1171,7 +1174,7 @@ for target in targets:
                description = "Uploading target files",
                descriptionDone = "Target files uploaded",
                command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
-                       ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
+                       ["-a", Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
                        Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
                env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
                haltOnFailure = True,
@@ -1185,7 +1188,7 @@ for target in targets:
                description = "Pruning target files",
                descriptionDone = "Target files pruned",
                command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
-                       ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
+                       ["-a", Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
                        Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
                env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
                haltOnFailure = True,
@@ -1199,7 +1202,7 @@ for target in targets:
                description = "Uploading kmod archive",
                descriptionDone = "Kmod archive uploaded",
                command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
-                       ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
+                       ["-a", Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
                        Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
                env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
                haltOnFailure = True,
@@ -1220,8 +1223,8 @@ for target in targets:
                name = "sourceupload",
                description = "Uploading source archives",
                descriptionDone = "Source archives uploaded",
-               command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
-                       [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
+               command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts + [Interpolate("%(prop:rsync_ipv4:+-4)s"),
+                       Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
                env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
                haltOnFailure = True,
                logEnviron = False,