From: Thibaut VARÈNE <hacks@slashdirt.org>
Date: Wed, 27 Jun 2018 16:19:12 +0000 (+0200)
Subject: phase1: use sha2rsync.pl for 'targetupload'
X-Git-Tag: v1~128
X-Git-Url: http://git.lede-project.org./?a=commitdiff_plain;h=3246628;p=buildbot.git

phase1: use sha2rsync.pl for 'targetupload'

This patch uses sha2rsync.pl to build a list of files to upload
to the remote server via rsync.

Because --files-from is taken as a literal list of files to consider by
rsync (the file list generation is skipped), it will not delete files on
the receiver.

To work around that without implementing a broken client-side upload
queue management system, 'targetupload' is split in two: the first step
uploads and updates new and existing files using rsync standard
mechanism, and the second step does nothing but delete files that are
present on the receiver but not on the sender.

As a side effect, this implements a '--delete-after' deletion style where
files are only deleted after uploads/updates finished successfully.

Signed-off-by: Thibaut VARÈNE <hacks@slashdirt.org>
---

diff --git a/phase1/master.cfg b/phase1/master.cfg
index 5bf8d90..54a5761 100644
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -907,6 +907,33 @@ for target in targets:
 		logEnviron = False,
 	))
 
+	# download remote sha256sums to 'target-sha256sums'
+	factory.addStep(ShellCommand(
+		name = "target-sha256sums",
+		description = "Fetching remote sha256sums for target",
+		command = ["rsync", "-4", "-vz", Interpolate("%(kw:rsyncbinurl)s/targets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1]), "target-sha256sums"],
+		env={'RSYNC_PASSWORD': rsync_bin_key},
+		logEnviron = False,
+		haltOnFailure = False,
+		flunkOnFailure = False,
+		warnOnFailure = False,
+	))
+
+	# build list of files to upload
+	factory.addStep(FileDownload(
+		name = "dlsha2rsyncpl",
+		mastersrc = "sha2rsync.pl",
+		slavedest = "../sha2rsync.pl",
+		mode = 0755,
+	))
+
+	factory.addStep(ShellCommand(
+		name = "buildlist",
+		description = "Building list of files to upload",
+		command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
+		haltOnFailure = True,
+	))
+
 	factory.addStep(FileDownload(
 		name = "dlrsync.sh",
 		mastersrc = "rsync.sh",
@@ -914,10 +941,23 @@ for target in targets:
 		mode = 0755
 	))
 
+	# upload new files and update existing ones
 	factory.addStep(ShellCommand(
 		name = "targetupload",
 		description = "Uploading target files",
-		command=["../rsync.sh", "-4", "--exclude=/kmods/", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
+		command=["../rsync.sh", "-4", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
+		         "-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
+		         Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+		env={'RSYNC_PASSWORD': rsync_bin_key},
+		haltOnFailure = True,
+		logEnviron = False,
+	))
+
+	# delete files which don't exist locally
+	factory.addStep(ShellCommand(
+		name = "targetprune",
+		description = "Pruning target files",
+		command=["../rsync.sh", "-4", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 		         "-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
 		         Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 		env={'RSYNC_PASSWORD': rsync_bin_key},