ini = configparser.ConfigParser()
ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
-if "general" not in ini or "phase1" not in ini or "rsync" not in ini:
+if "general" not in ini or "phase1" not in ini:
raise ValueError("Fix your configuration")
inip1 = ini['phase1']
# only take place on one worker.
def IsTaggingRequested(step):
- val = step.getProperty("tag")
- if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
- return True
- else:
- return False
+ tag = step.getProperty("tag")
+ return tag and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", tag)
def IsNoMasterBuild(step):
return step.getProperty("branch") != "master"
+def IsUsignEnabled(step):
+ branch = step.getProperty("branch")
+ return branch and branches[branch].get("usign_key")
+
+def IsSignEnabled(step):
+ branch = step.getProperty("branch")
+ return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
+
+def IsKmodArchiveEnabled(step):
+ branch = step.getProperty("branch")
+ return branch and branches[branch].get("kmod_archive")
+
def GetBaseVersion(branch):
if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
return branch.split('-')[1]
else:
return ""
+@util.renderer
+def GetConfigSeed(props):
+ branch = props.getProperty("branch")
+ return branch and branches[branch].get("config_seed") or ""
+
+@util.renderer
+def GetRsyncParams(props, srcorbin, urlorkey):
+ # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
+ branch = props.getProperty("branch")
+ opt = srcorbin + "_" + urlorkey
+ return branch and branches[branch].get(opt)
+
+@util.renderer
+def GetUsignKey(props):
+ branch = props.getProperty("branch")
+ return branch and branches[branch].get("usign_key")
+
def GetNextBuild(builder, requests):
for r in requests:
if r.properties and r.properties.hasProperty("tag"):
return CheckTargetProperty
-def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
+@util.renderer
+def UsignSec2Pub(props):
+ branch = props.getProperty("branch")
try:
+ comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
+ seckey = branches[branch].get("usign_key")
seckey = base64.b64decode(seckey)
except:
return None
))
# seed config
- if config_seed is not None:
- factory.addStep(StringDownload(
- name = "dlconfigseed",
- s = config_seed + '\n',
- workerdest = ".config",
- mode = 0o644
- ))
+ factory.addStep(StringDownload(
+ name = "dlconfigseed",
+ s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
+ workerdest = ".config",
+ mode = 0o644
+ ))
# configure
factory.addStep(ShellCommand(
name = "newconfig",
description = "Seeding .config",
- command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\nCONFIG_SIGNED_PACKAGES=%s\\n' >> .config" %(ts[0], ts[0], ts[1], 'y' if usign_key is not None else 'n')
+ command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey)
))
factory.addStep(ShellCommand(
command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
# install build key
- if usign_key is not None:
- factory.addStep(StringDownload(
- name = "dlkeybuildpub",
- s = UsignSec2Pub(usign_key, usign_comment),
- workerdest = "key-build.pub",
- mode = 0o600,
- ))
-
- factory.addStep(StringDownload(
- name = "dlkeybuild",
- s = "# fake private key",
- workerdest = "key-build",
- mode = 0o600,
- ))
-
- factory.addStep(StringDownload(
- name = "dlkeybuilducert",
- s = "# fake certificate",
- workerdest = "key-build.ucert",
- mode = 0o600,
- ))
+ factory.addStep(StringDownload(
+ name = "dlkeybuildpub",
+ s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
+ workerdest = "key-build.pub",
+ mode = 0o600,
+ doStepIf = IsUsignEnabled,
+ ))
+
+ factory.addStep(StringDownload(
+ name = "dlkeybuild",
+ s = "# fake private key",
+ workerdest = "key-build",
+ mode = 0o600,
+ doStepIf = IsUsignEnabled,
+ ))
+
+ factory.addStep(StringDownload(
+ name = "dlkeybuilducert",
+ s = "# fake certificate",
+ workerdest = "key-build.ucert",
+ mode = 0o600,
+ doStepIf = IsUsignEnabled,
+ ))
# prepare dl
factory.addStep(ShellCommand(
haltOnFailure = True
))
- if enable_kmod_archive:
- factory.addStep(ShellCommand(
- name = "kmoddir",
- description = "Creating kmod directory",
- command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
- haltOnFailure = True
- ))
-
- factory.addStep(ShellCommand(
- name = "kmodprepare",
- description = "Preparing kmod archive",
- command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
- Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
- Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
- haltOnFailure = True
- ))
-
- factory.addStep(ShellCommand(
- name = "kmodindex",
- description = "Indexing kmod archive",
- command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
- Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
- env = MakeEnv(),
- haltOnFailure = True
- ))
+ factory.addStep(ShellCommand(
+ name = "kmoddir",
+ description = "Creating kmod directory",
+ command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
+ haltOnFailure = True,
+ doStepIf = IsKmodArchiveEnabled,
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "kmodprepare",
+ description = "Preparing kmod archive",
+ command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
+ Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
+ Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
+ haltOnFailure = True,
+ doStepIf = IsKmodArchiveEnabled,
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "kmodindex",
+ description = "Indexing kmod archive",
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
+ Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
+ env = MakeEnv(),
+ haltOnFailure = True,
+ doStepIf = IsKmodArchiveEnabled,
+ ))
# sign
- if ini.has_option("gpg", "key") or usign_key is not None:
- factory.addStep(MasterShellCommand(
- name = "signprepare",
- description = "Preparing temporary signing directory",
- command = ["mkdir", "-p", "%s/signing" %(work_dir)],
- haltOnFailure = True
- ))
-
- factory.addStep(ShellCommand(
- name = "signpack",
- description = "Packing files to sign",
- command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
- haltOnFailure = True
- ))
-
- factory.addStep(FileUpload(
- workersrc = "sign.tar.gz",
- masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
- haltOnFailure = True
- ))
-
- factory.addStep(MasterShellCommand(
- name = "signfiles",
- description = "Signing files",
- command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])],
- env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
- haltOnFailure = True
- ))
-
- factory.addStep(FileDownload(
- name = "dlsigntargz",
- mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
- workerdest = "sign.tar.gz",
- haltOnFailure = True
- ))
-
- factory.addStep(ShellCommand(
- name = "signunpack",
- description = "Unpacking signed files",
- command = ["tar", "-xzf", "sign.tar.gz"],
- haltOnFailure = True
- ))
+ factory.addStep(MasterShellCommand(
+ name = "signprepare",
+ description = "Preparing temporary signing directory",
+ command = ["mkdir", "-p", "%s/signing" %(work_dir)],
+ haltOnFailure = True,
+ doStepIf = IsSignEnabled,
+
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "signpack",
+ description = "Packing files to sign",
+ command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
+ haltOnFailure = True,
+ doStepIf = IsSignEnabled,
+ ))
+
+ factory.addStep(FileUpload(
+ workersrc = "sign.tar.gz",
+ masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
+ haltOnFailure = True,
+ doStepIf = IsSignEnabled,
+ ))
+
+ factory.addStep(MasterShellCommand(
+ name = "signfiles",
+ description = "Signing files",
+ command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])],
+ env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
+ haltOnFailure = True,
+ doStepIf = IsSignEnabled,
+ ))
+
+ factory.addStep(FileDownload(
+ name = "dlsigntargz",
+ mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
+ workerdest = "sign.tar.gz",
+ haltOnFailure = True,
+ doStepIf = IsSignEnabled,
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "signunpack",
+ description = "Unpacking signed files",
+ command = ["tar", "-xzf", "sign.tar.gz"],
+ haltOnFailure = True,
+ doStepIf = IsSignEnabled,
+ ))
# upload
factory.addStep(ShellCommand(
haltOnFailure = True
))
- if enable_kmod_archive:
- factory.addStep(ShellCommand(
- name = "kmoddirprepare",
- description = "Preparing kmod archive upload directory",
- command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- haltOnFailure = True
- ))
+ factory.addStep(ShellCommand(
+ name = "kmoddirprepare",
+ description = "Preparing kmod archive upload directory",
+ command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ haltOnFailure = True,
+ doStepIf = IsKmodArchiveEnabled,
+ ))
factory.addStep(ShellCommand(
name = "dirupload",
description = "Uploading directory structure",
- command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", "%s/" %(rsync_bin_url)],
- env={'RSYNC_PASSWORD': rsync_bin_key},
+ command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
+ env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
logEnviron = False,
locks = NetLockUl,
factory.addStep(ShellCommand(
name = "target-sha256sums",
description = "Fetching remote sha256sums for target",
- command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
- env={'RSYNC_PASSWORD': rsync_bin_key},
+ command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
+ env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
logEnviron = False,
haltOnFailure = False,
flunkOnFailure = False,
description = "Uploading target files",
command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
- Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- env={'RSYNC_PASSWORD': rsync_bin_key},
+ Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
logEnviron = False,
))
description = "Pruning target files",
command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
- Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- env={'RSYNC_PASSWORD': rsync_bin_key},
+ Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
logEnviron = False,
locks = NetLockUl,
))
- if enable_kmod_archive:
- factory.addStep(ShellCommand(
- name = "kmodupload",
- description = "Uploading kmod archive",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
- ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
- Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- env={'RSYNC_PASSWORD': rsync_bin_key},
- haltOnFailure = True,
- logEnviron = False,
- locks = NetLockUl,
- ))
+ factory.addStep(ShellCommand(
+ name = "kmodupload",
+ description = "Uploading kmod archive",
+ command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
+ ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
+ Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
+ haltOnFailure = True,
+ logEnviron = False,
+ locks = NetLockUl,
+ doStepIf = IsKmodArchiveEnabled,
+ ))
factory.addStep(ShellCommand(
name = "sourcelist",
name = "sourceupload",
description = "Uploading source archives",
command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts +
- [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
- env={'RSYNC_PASSWORD': rsync_src_key},
+ [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
+ env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
haltOnFailure = True,
logEnviron = False,
locks = NetLockUl,
projects / buildbot.git / commitdiff