phase1: use sha2rsync.pl for 'targetupload'

This patch uses sha2rsync.pl to build a list of files to upload
to the remote server via rsync.

Because --files-from is taken as a literal list of files to consider by
rsync (the file list generation is skipped), it will not delete files on
the receiver.

To work around that without implementing a broken client-side upload
queue management system, 'targetupload' is split in two: the first step
uploads and updates new and existing files using rsync standard
mechanism, and the second step does nothing but delete files that are
present on the receiver but not on the sender.

As a side effect, this implements a '--delete-after' deletion style where
files are only deleted after uploads/updates finished successfully.

Signed-off-by: Thibaut VARÈNE <hacks@slashdirt.org>

diff --git a/phase1/master.cfg b/phase1/master.cfg
index 5bf8d90642eecff3e8ef39f6b5cfdc911b1b5aee..54a576148e58ac7163b2b3e1e8e86fbf932043e4 100644 (file)
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -907,6 +907,33 @@ for target in targets:
                logEnviron = False,
        ))
 
+       # download remote sha256sums to 'target-sha256sums'
+       factory.addStep(ShellCommand(
+               name = "target-sha256sums",
+               description = "Fetching remote sha256sums for target",
+               command = ["rsync", "-4", "-vz", Interpolate("%(kw:rsyncbinurl)s/targets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1]), "target-sha256sums"],
+               env={'RSYNC_PASSWORD': rsync_bin_key},
+               logEnviron = False,
+               haltOnFailure = False,
+               flunkOnFailure = False,
+               warnOnFailure = False,
+       ))
+
+       # build list of files to upload
+       factory.addStep(FileDownload(
+               name = "dlsha2rsyncpl",
+               mastersrc = "sha2rsync.pl",
+               slavedest = "../sha2rsync.pl",
+               mode = 0755,
+       ))
+
+       factory.addStep(ShellCommand(
+               name = "buildlist",
+               description = "Building list of files to upload",
+               command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
+               haltOnFailure = True,
+       ))
+
        factory.addStep(FileDownload(
                name = "dlrsync.sh",
                mastersrc = "rsync.sh",
@@ -914,10 +941,23 @@ for target in targets:
                mode = 0755
        ))
 
+       # upload new files and update existing ones
        factory.addStep(ShellCommand(
                name = "targetupload",
                description = "Uploading target files",
-               command=["../rsync.sh", "-4", "--exclude=/kmods/", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
+               command=["../rsync.sh", "-4", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
+                        "-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
+                        Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+               env={'RSYNC_PASSWORD': rsync_bin_key},
+               haltOnFailure = True,
+               logEnviron = False,
+       ))
+
+       # delete files which don't exist locally
+       factory.addStep(ShellCommand(
+               name = "targetprune",
+               description = "Pruning target files",
+               command=["../rsync.sh", "-4", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
                         "-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
                         Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
                env={'RSYNC_PASSWORD': rsync_bin_key},