From 7aafe3cb6e14537574e51853e44e3371321ef0e4 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Petr=20=C5=A0tetiar?= Date: Fri, 12 Mar 2021 14:20:46 +0100 Subject: [PATCH] treewide: use worker term MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit [Placeholder for your politically correct commit description] Suggested-by: Paul Spooren Signed-off-by: Petr Å tetiar --- .gitlab-ci.yml | 2 +- .gitlab/docker/README.md | 2 +- .../{buildslave => buildworker}/gitlab.yml | 12 +++--- build-docker-images.sh | 4 +- docker/buildslave/files/start.sh | 33 --------------- docker/{buildslave => buildworker}/Dockerfile | 10 ++--- .../files/entry.sh | 0 docker/buildworker/files/start.sh | 33 +++++++++++++++ docker/config.ini | 8 ++-- docker/docker-compose.yml | 32 +++++++------- phase1/config.ini.example | 8 ++-- phase1/master.cfg | 42 +++++++++---------- phase2/config.ini.example | 8 ++-- phase2/master.cfg | 34 +++++++-------- scripts/cleanup.sh | 4 +- 15 files changed, 116 insertions(+), 116 deletions(-) rename .gitlab/docker/{buildslave => buildworker}/gitlab.yml (67%) delete mode 100644 docker/buildslave/files/start.sh rename docker/{buildslave => buildworker}/Dockerfile (78%) rename docker/{buildslave => buildworker}/files/entry.sh (100%) create mode 100644 docker/buildworker/files/start.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fdc3237..4551da1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ variables: include: - local: .gitlab/docker.yml - local: .gitlab/docker/buildmaster/gitlab.yml - - local: .gitlab/docker/buildslave/gitlab.yml + - local: .gitlab/docker/buildworker/gitlab.yml stages: - docker diff --git a/.gitlab/docker/README.md b/.gitlab/docker/README.md index a8aef0a..36bebea 100644 --- a/.gitlab/docker/README.md +++ b/.gitlab/docker/README.md @@ -3,4 +3,4 @@ This directory contains bits for Docker images used on the GitLab CI. ## buildmaster -## buildslave +## buildworker diff --git a/.gitlab/docker/buildslave/gitlab.yml b/.gitlab/docker/buildworker/gitlab.yml similarity index 67% rename from .gitlab/docker/buildslave/gitlab.yml rename to .gitlab/docker/buildworker/gitlab.yml index 2469795..4658f53 100644 --- a/.gitlab/docker/buildslave/gitlab.yml +++ b/.gitlab/docker/buildworker/gitlab.yml @@ -1,21 +1,21 @@ -build Docker image buildslave: +build Docker image buildworker: stage: docker extends: .build Docker image -test Docker image buildslave: +test Docker image buildworker: stage: docker test extends: .docker in docker - needs: ["build Docker image buildslave"] + needs: ["build Docker image buildworker"] script: - export IMAGE_NAME="$(echo $CI_JOB_NAME | sed 's/test Docker image \(.*\)/\1/')" - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME-$BUILDBOT_VERSION:$CI_COMMIT_REF_SLUG" - > - docker run --rm --env BUILDSLAVE_NAME=foo --env BUILDSLAVE_PASSWORD=XXX + docker run --rm --env BUILDWORKER_NAME=foo --env BUILDWORKER_PASSWORD=XXX "$CI_REGISTRY_IMAGE/$IMAGE_NAME-$BUILDBOT_VERSION:$CI_COMMIT_REF_SLUG" | grep "worker configured in /builder" -deploy Docker image buildslave: +deploy Docker image buildworker: stage: docker deploy extends: .deploy Docker image - needs: ["test Docker image buildslave"] + needs: ["test Docker image buildworker"] diff --git a/build-docker-images.sh b/build-docker-images.sh index 18932c4..f2f264e 100755 --- a/build-docker-images.sh +++ b/build-docker-images.sh @@ -3,5 +3,5 @@ docker build -t openwrtorg/buildmaster -f docker/buildmaster/Dockerfile . docker push openwrtorg/buildmaster -docker build -t openwrtorg/buildslave -f docker/buildslave/Dockerfile . -docker push openwrtorg/buildslave +docker build -t openwrtorg/buildworker -f docker/buildworker/Dockerfile . +docker push openwrtorg/buildworker diff --git a/docker/buildslave/files/start.sh b/docker/buildslave/files/start.sh deleted file mode 100644 index 01e888e..0000000 --- a/docker/buildslave/files/start.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -[ -n "$BUILDSLAVE_NAME" ] || { - echo "Please supply a name via --env BUILDSLAVE_NAME=XXX" >&2 - exit 1 -} - -[ -n "$BUILDSLAVE_PASSWORD" ] || { - echo "Please supply a password via --env BUILDSLAVE_PASSWORD=XXX" >&2 - exit 2 -} - -rm -f /builder/buildbot.tac - -use_tls="" -[ "$BUILDSLAVE_TLS" = 1 ] && use_tls="--use-tls" -/usr/local/bin/buildbot-worker create-worker --force --umask="0o22" $use_tls /builder \ - "$BUILDSLAVE_MASTER" "$BUILDSLAVE_NAME" "$BUILDSLAVE_PASSWORD" - -if [ "$BUILDSLAVE_TLS" = 1 ]; then - sed -i \ - -e 's#(buildmaster_host, port, #(None, None, #' \ - -e 's#allow_shutdown=allow_shutdown#&, connection_string="SSL:%s:%d" %(buildmaster_host, port)#' \ - /builder/buildbot.tac -fi - -echo "$BUILDSLAVE_ADMIN" > /builder/info/admin -echo "$BUILDSLAVE_DESCRIPTION" > /builder/info/host - -unset BUILDSLAVE_ADMIN BUILDSLAVE_DESCRIPTION BUILDSLAVE_MASTER BUILDSLAVE_NAME BUILDSLAVE_PASSWORD - -rm -f /builder/twistd.pid -exec /usr/local/bin/buildbot-worker start --nodaemon /builder diff --git a/docker/buildslave/Dockerfile b/docker/buildworker/Dockerfile similarity index 78% rename from docker/buildslave/Dockerfile rename to docker/buildworker/Dockerfile index 31eb720..d568ebb 100644 --- a/docker/buildslave/Dockerfile +++ b/docker/buildworker/Dockerfile @@ -4,9 +4,9 @@ MAINTAINER OpenWrt Maintainers ARG DEBIAN_FRONTEND=noninteractive ARG BUILDBOT_VERSION=2.10.1 -ENV BUILDSLAVE_MASTER builds.openwrt.org:9990 -ENV BUILDSLAVE_ADMIN contact@openwrt.org -ENV BUILDSLAVE_DESCRIPTION Buildslave Docker Instance +ENV BUILDWORKER_MASTER builds.openwrt.org:9990 +ENV BUILDWORKER_ADMIN contact@openwrt.org +ENV BUILDWORKER_DESCRIPTION Buildworker Docker Instance USER root @@ -42,8 +42,8 @@ RUN pip3 install \ ENV LANG=en_US.utf8 -COPY docker/buildslave/files/entry.sh /entry.sh -COPY docker/buildslave/files/start.sh /start.sh +COPY docker/buildworker/files/entry.sh /entry.sh +COPY docker/buildworker/files/start.sh /start.sh RUN \ groupadd buildbot && \ diff --git a/docker/buildslave/files/entry.sh b/docker/buildworker/files/entry.sh similarity index 100% rename from docker/buildslave/files/entry.sh rename to docker/buildworker/files/entry.sh diff --git a/docker/buildworker/files/start.sh b/docker/buildworker/files/start.sh new file mode 100644 index 0000000..81e2671 --- /dev/null +++ b/docker/buildworker/files/start.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +[ -n "$BUILDWORKER_NAME" ] || { + echo "Please supply a name via --env BUILDWORKER_NAME=XXX" >&2 + exit 1 +} + +[ -n "$BUILDWORKER_PASSWORD" ] || { + echo "Please supply a password via --env BUILDWORKER_PASSWORD=XXX" >&2 + exit 2 +} + +rm -f /builder/buildbot.tac + +use_tls="" +[ "$BUILDWORKER_TLS" = 1 ] && use_tls="--use-tls" +/usr/local/bin/buildbot-worker create-worker --force --umask="0o22" $use_tls /builder \ + "$BUILDWORKER_MASTER" "$BUILDWORKER_NAME" "$BUILDWORKER_PASSWORD" + +if [ "$BUILDWORKER_TLS" = 1 ]; then + sed -i \ + -e 's#(buildmaster_host, port, #(None, None, #' \ + -e 's#allow_shutdown=allow_shutdown#&, connection_string="SSL:%s:%d" %(buildmaster_host, port)#' \ + /builder/buildbot.tac +fi + +echo "$BUILDWORKER_ADMIN" > /builder/info/admin +echo "$BUILDWORKER_DESCRIPTION" > /builder/info/host + +unset BUILDWORKER_ADMIN BUILDWORKER_DESCRIPTION BUILDWORKER_MASTER BUILDWORKER_NAME BUILDWORKER_PASSWORD + +rm -f /builder/twistd.pid +exec /usr/local/bin/buildbot-worker start --nodaemon /builder diff --git a/docker/config.ini b/docker/config.ini index 79c3e6b..da868ea 100644 --- a/docker/config.ini +++ b/docker/config.ini @@ -131,16 +131,16 @@ comment = Example GPG key key = RWRCSwAAAADUvtjCkFEF4bWWxpPBo9o8R5FK6Rz5aPUsaZONLu8kxIjud9Fd+Mgu7J2fFJDVyKFAXNH6pKS+AuBW3v+TQT5m1J0W/JYTjqzIrgAZhRtm5v3vSKRl3HUD2zEEbG5j3tg= comment = Example usign key -[slave 1] +[worker 1] phase = 1 -name = buildslave-phase1 +name = buildworker-phase1 password = secret builds = 1 cleanup = 1 -[slave 2] +[worker 2] phase = 2 -name = buildslave-phase2 +name = buildworker-phase2 password = secret builds = 1 cleanup = 1 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 28f0b9e..80623b8 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -52,40 +52,40 @@ services: - './certs/buildmaster-phase2.key:/certs/master.key' - './build/master-phase2:/master' - buildslave-phase1: + buildworker-phase1: build: context: ../ - dockerfile: docker/buildslave/Dockerfile - image: buildslave:latest + dockerfile: docker/buildworker/Dockerfile + image: buildworker:latest restart: always cpuset: 1-3 environment: - BUILDSLAVE_MASTER: buildmaster-phase1:9989 - BUILDSLAVE_NAME: buildslave-phase1 - BUILDSLAVE_PASSWORD: secret - BUILDSLAVE_TLS: '1' + BUILDWORKER_MASTER: buildmaster-phase1:9989 + BUILDWORKER_NAME: buildworker-phase1 + BUILDWORKER_PASSWORD: secret + BUILDWORKER_TLS: '1' links: - 'rsync-server' - 'buildmaster-phase1' volumes: - './certs/ca.crt:/certs/ca.pem' - - './build/slave-phase1:/builder' + - './build/worker-phase1:/builder' - buildslave-phase2: + buildworker-phase2: build: context: ../ - dockerfile: docker/buildslave/Dockerfile - image: buildslave:latest + dockerfile: docker/buildworker/Dockerfile + image: buildworker:latest restart: always cpuset: 1-3 environment: - BUILDSLAVE_MASTER: buildmaster-phase2:9990 - BUILDSLAVE_NAME: buildslave-phase2 - BUILDSLAVE_PASSWORD: secret - BUILDSLAVE_TLS: '1' + BUILDWORKER_MASTER: buildmaster-phase2:9990 + BUILDWORKER_NAME: buildworker-phase2 + BUILDWORKER_PASSWORD: secret + BUILDWORKER_TLS: '1' links: - 'rsync-server' - 'buildmaster-phase2' volumes: - './certs/ca.crt:/certs/ca.pem' - - './build/slave-phase2:/builder' + - './build/worker-phase2:/builder' diff --git a/phase1/config.ini.example b/phase1/config.ini.example index eacf7a4..ccc7d31 100644 --- a/phase1/config.ini.example +++ b/phase1/config.ini.example @@ -57,15 +57,15 @@ comment = Unattended build signature key = RWRCSwAAA...OihABfuLvGRVfVaJ6wLf0= comment = Unattended build signature -[slave 1] +[worker 1] phase = 1 -name = example-slave-1 +name = example-worker-1 password = example builds = 3 -[slave 2] +[worker 2] phase = 1 -name = example-slave-2 +name = example-worker-2 password = example2 builds = 1 cleanup = 1 diff --git a/phase1/master.cfg b/phase1/master.cfg index 5773257..e57b666 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -62,22 +62,22 @@ c['titleURL'] = ini.get("general", "title_url") c['buildbotURL'] = ini.get("phase1", "buildbot_url") -####### BUILDSLAVES +####### BUILDWORKERS -# The 'workers' list defines the set of recognized buildslaves. Each element is -# a Worker object, specifying a unique slave name and password. The same -# slave name and password must be configured on the slave. +# The 'workers' list defines the set of recognized buildworkers. Each element is +# a Worker object, specifying a unique worker name and password. The same +# worker name and password must be configured on the worker. -slave_port = 9989 +worker_port = 9989 if ini.has_option("phase1", "port"): - slave_port = ini.get("phase1", "port") + worker_port = ini.get("phase1", "port") c['workers'] = [] NetLocks = dict() for section in ini.sections(): - if section.startswith("slave "): + if section.startswith("worker "): if ini.has_option(section, "name") and ini.has_option(section, "password") and \ (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1): sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False } @@ -108,10 +108,10 @@ for section in ini.sections(): raise ValueError('max_builds must be 1 with shared workdir!') c['workers'].append(Worker(name, password, max_builds = max_builds, properties = sl_props)) -# 'slavePortnum' defines the TCP port to listen on for connections from workers. -# This must match the value configured into the buildslaves (with their +# 'workerPortnum' defines the TCP port to listen on for connections from workers. +# This must match the value configured into the buildworkers (with their # --master option) -c['protocols'] = {'pb': {'port': slave_port}} +c['protocols'] = {'pb': {'port': worker_port}} # coalesce builds c['collapseRequests'] = True @@ -337,7 +337,7 @@ c['schedulers'].append(ForceScheduler( # The 'builders' list defines the Builders, which tell Buildbot how to perform a build: # what steps, and which workers can execute them. Note that any particular build will -# only take place on one slave. +# only take place on one worker. CleanTargetMap = [ [ "tools", "tools/clean" ], @@ -529,7 +529,7 @@ def UsignSec2Pub(seckey, comment="untrusted comment: secret key"): c['builders'] = [] -dlLock = locks.WorkerLock("slave_dl") +dlLock = locks.WorkerLock("worker_dl") checkBuiltin = re.sub('[\t\n ]+', ' ', """ checkBuiltin() { @@ -571,29 +571,29 @@ class IfBuiltinShellCommand(ShellCommand): ] def setupEnvironment(self, cmd): - slaveEnv = self.slaveEnvironment - if slaveEnv is None: - slaveEnv = { } + workerEnv = self.workerEnvironment + if workerEnv is None: + workerEnv = { } changedFiles = { } for request in self.build.requests: for source in request.sources: for change in source.changes: for file in change.files: changedFiles[file] = True - fullSlaveEnv = slaveEnv.copy() + fullSlaveEnv = workerEnv.copy() fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys()) cmd.args['env'] = fullSlaveEnv -slaveNames = [ ] +workerNames = [ ] -for slave in c['workers']: - slaveNames.append(slave.workername) +for worker in c['workers']: + workerNames.append(worker.workername) force_factory = BuildFactory() c['builders'].append(BuilderConfig( name = "00_force_build", - workernames = slaveNames, + workernames = workerNames, factory = force_factory)) for target in targets: @@ -1310,7 +1310,7 @@ for target in targets: alwaysRun = True, )) - c['builders'].append(BuilderConfig(name=target, workernames=slaveNames, factory=factory, nextBuild=GetNextBuild)) + c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild)) c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ])) force_factory.addStep(steps.Trigger( diff --git a/phase2/config.ini.example b/phase2/config.ini.example index d01ecd9..ba8ff76 100644 --- a/phase2/config.ini.example +++ b/phase2/config.ini.example @@ -46,15 +46,15 @@ comment = Unattended build signature key = RWRCSwAAA...OihABfuLvGRVfVaJ6wLf0= comment = Unattended build signature -[slave 1] +[worker 1] phase = 2 -name = slave-example-1 +name = worker-example-1 password = example builds = 1 -[slave 2] +[worker 2] phase = 2 -name = slave-example-2 +name = worker-example-2 password = example2 builds = 3 diff --git a/phase2/master.cfg b/phase2/master.cfg index d1513c3..1e1bafa 100644 --- a/phase2/master.cfg +++ b/phase2/master.cfg @@ -42,13 +42,13 @@ buildbot_url = ini.get("phase2", "buildbot_url") # a shorter alias to save typing. c = BuildmasterConfig = {} -####### BUILDSLAVES +####### BUILDWORKERS -# The 'workers' list defines the set of recognized buildslaves. Each element is -# a Worker object, specifying a unique slave name and password. The same -# slave name and password must be configured on the slave. +# The 'workers' list defines the set of recognized buildworkers. Each element is +# a Worker object, specifying a unique worker name and password. The same +# worker name and password must be configured on the worker. -slave_port = 9990 +worker_port = 9990 persistent = False other_builds = 0 tree_expire = 0 @@ -56,7 +56,7 @@ git_ssh = False git_ssh_key = None if ini.has_option("phase2", "port"): - slave_port = ini.get("phase2", "port") + worker_port = ini.get("phase2", "port") if ini.has_option("phase2", "persistent"): persistent = ini.getboolean("phase2", "persistent") @@ -79,7 +79,7 @@ c['workers'] = [] max_builds = dict() for section in ini.sections(): - if section.startswith("slave "): + if section.startswith("worker "): if ini.has_option(section, "name") and ini.has_option(section, "password") and \ ini.has_option(section, "phase") and ini.getint(section, "phase") == 2: name = ini.get(section, "name") @@ -100,10 +100,10 @@ for section in ini.sections(): c['workers'].append(Worker(name, password, max_builds = max_builds[name], properties = sl_props)) -# 'slavePortnum' defines the TCP port to listen on for connections from workers. -# This must match the value configured into the buildslaves (with their +# 'workerPortnum' defines the TCP port to listen on for connections from workers. +# This must match the value configured into the buildworkers (with their # --master option) -c['protocols'] = {'pb': {'port': slave_port}} +c['protocols'] = {'pb': {'port': worker_port}} # coalesce builds c['collapseRequests'] = True @@ -267,7 +267,7 @@ c['schedulers'].append(ForceScheduler( # The 'builders' list defines the Builders, which tell Buildbot how to perform a build: # what steps, and which workers can execute them. Note that any particular build will -# only take place on one slave. +# only take place on one worker. def GetDirectorySuffix(props): verpat = re.compile(r'^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$') @@ -321,18 +321,18 @@ def IsSharedWorkdir(step): c['builders'] = [] -dlLock = locks.WorkerLock("slave_dl") +dlLock = locks.WorkerLock("worker_dl") -slaveNames = [ ] +workerNames = [ ] -for slave in c['workers']: - slaveNames.append(slave.workername) +for worker in c['workers']: + workerNames.append(worker.workername) force_factory = BuildFactory() c['builders'].append(BuilderConfig( name = "00_force_build", - workernames = slaveNames, + workernames = workerNames, factory = force_factory)) for arch in arches: @@ -679,7 +679,7 @@ for arch in arches: alwaysRun = True )) - c['builders'].append(BuilderConfig(name=arch[0], workernames=slaveNames, factory=factory)) + c['builders'].append(BuilderConfig(name=arch[0], workernames=workerNames, factory=factory)) c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % arch[0], builderNames=[ arch[0] ])) force_factory.addStep(steps.Trigger( diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index a328f02..090e573 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -3,11 +3,11 @@ export LC_ALL=C master_url="$1" -current_slave="$2" +current_worker="$2" current_builder="$3" current_mode="$4" -worker_id="$(wget -qO- "${master_url%/}/api/v2/workers/$current_slave" | sed -rne 's#^ +"workerid": ([0-9]+),?$#\1#p')" +worker_id="$(wget -qO- "${master_url%/}/api/v2/workers/$current_worker" | sed -rne 's#^ +"workerid": ([0-9]+),?$#\1#p')" active_builder_ids="$(wget -qO- "${master_url%/}/api/v2/workers/$worker_id/builds" | sed -rne '/"builderid"/ { s/^.+: ([0-9]+),$/\1/; h }; /"state_string"/ { s/^.+: "([^"]*)".*$/\1/; H; x; s/\n/ /; p }' | sed -ne 's/ building$//p')" find /tmp/ -maxdepth 1 -mtime +1 '(' -name 'npm-*' -or -name 'jsmake-*' ')' -print0 | xargs -0 -r rm -vr -- 2.30.2