From 3fdb667d19ee21453f82e31a23442ab01e68f981 Mon Sep 17 00:00:00 2001 From: Paul Spooren Date: Wed, 1 Jul 2020 19:24:27 -1000 Subject: [PATCH] CI: Use CI for snapshot builds This commits adds a GitHub actions CI workflow file which builds all targets every 24 hours. To do so it imitates most steps of the buildbot.git[0] and uploads created binaries and kmods to a storage server. As the OpenWrt core developers clearly stated their interest in moving away from GitHub and towards a self-hosted GitLab instance, this is mostly intended as a proof of concept and to show the buildbot steps in easier understandable fashion than looking through the buildbot script. The script is easily adaptable to work with the GitLab CI, however as GitHub currently offers more parallel builds, it was preferred. Using a CI instead of buildbot for snapshot builds introduces multiple advanced which are described below: * The current setup is (mostly) handled and maintained by a single person. Moving to a "in-repo" based CI configuration enables a bigger audience to follow the setup and introduce fixes and improvements. * Changes have to be applied very carefully or all builds may fail. Using a CI allows to automatically test changes and deploy (build & upload all ~50 targets/subtargets) only in on success. * GitHub currently offers up to 60 parallel jobs for organizations. That means 60 targets can be build in parallel, each taking about 90 minutes. This would greatly reduce current server load and allow more resources for release building and thereby lower maintenance. * The CI interface is (obviously) better integrated in the GitHub (and GitLab) website and thereby allows developers to easier see what breaks, instead of waiting for bug buildbot fail reports in IRC elsewhere. * Worker setup and maintenance should ideally be easier. Both GitHub and GitLab offer to host your own workers. While uninteresting for GitHub due to the temporary (if at all) usage, GitLab workers build in defined Docker containers and communicate with the main instance over HTTPS. No SSH tunneling or similar features required. Due to the nature of Docker containers the workspace cleaning is also drastically simplified. Updating the build containers can be tested locally and then rolled out do all workers at once, instead of maintaining multiple servers with varying OS versions. [0]: https://git.openwrt.org/?p=buildbot.git Signed-off-by: Paul Spooren --- .ci/default.config | 0 .ci/gcc10.config | 3 + .ci/nls.config | 1 + .ci/selinux.config | 2 + .github/workflows/build.yml | 213 ++++++++++++++++++++++++++++ package/system/procd/files/procd.sh | 2 +- 6 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 .ci/default.config create mode 100644 .ci/gcc10.config create mode 100644 .ci/nls.config create mode 100644 .ci/selinux.config create mode 100644 .github/workflows/build.yml diff --git a/.ci/default.config b/.ci/default.config new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.ci/gcc10.config b/.ci/gcc10.config new file mode 100644 index 0000000000..4142be806b --- /dev/null +++ b/.ci/gcc10.config @@ -0,0 +1,3 @@ +CONFIG_GCC_USE_VERSION_10=y +CONFIG_DEVEL=y +CONFIG_TOOLCHAINOPTS=y diff --git a/.ci/nls.config b/.ci/nls.config new file mode 100644 index 0000000000..adbc4ca394 --- /dev/null +++ b/.ci/nls.config @@ -0,0 +1 @@ +CONFIG_BUILD_NLS=y diff --git a/.ci/selinux.config b/.ci/selinux.config new file mode 100644 index 0000000000..8520705df7 --- /dev/null +++ b/.ci/selinux.config @@ -0,0 +1,2 @@ +CONFIG_KERNEL_SECURITY_SELINUX_DEVELOP=y +CONFIG_SELINUX=y diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..6110398b94 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,213 @@ +name: Build OpenWrt staging + +on: [push] + +jobs: + determine_targets: + name: Find available targets + runs-on: ubuntu-20.04 + outputs: + targets: ${{ steps.find_targets.outputs.targets }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Find targets + id: find_targets + run: | + TARGETS="$(perl ./scripts/dump-target-info.pl targets 2>/dev/null | awk '{ print $1 }')" + JSON='{"targets":[' + FIRST=1 + for TARGET in $TARGETS; do + [[ $FIRST -ne 1 ]] && JSON="$JSON"',' + JSON="$JSON"'"'"${TARGET}"'"' + FIRST=0 + done + JSON="$JSON"']}' + + echo -e "\n---- targets ----\n" + echo "$JSON" + echo -e "\n---- targets ----\n" + + echo "targets=$JSON" >> "$GITHUB_ENV" + + build: + name: Build ${{ matrix.config }}/${{ matrix.targets }} + needs: determine_targets + runs-on: ubuntu-latest + strategy: + fail-fast: False + matrix: + targets: + - "x86/64" + - "mediatek/mt7622" + config: + - "default" + - "selinux" + - "nls" + - "gcc10" + + #matrix: ${{fromJson(needs.determine_targets.outputs.targets)}} + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Cache sources + uses: actions/cache@v2 + with: + path: dl/ + key: ${{ matrix.targets }} + + - name: Initialization environment + env: + DEBIAN_FRONTEND: noninteractive + run: | + sudo apt-get -y install libncurses-dev + TARGET=$(echo ${{ matrix.targets }} | cut -d "/" -f 1) + SUBTARGET=$(echo ${{ matrix.targets }} | cut -d "/" -f 2) + echo "TARGET=$TARGET" >> "$GITHUB_ENV" + echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV" + + - name: Update & Install feeds + run: | + ./scripts/feeds update -a + ./scripts/feeds install -a + + - name: Set configuration + run: | + curl "https://downloads.openwrt.org/snapshots/targets/${{ matrix.targets }}/config.buildinfo" > .config + cat ".ci/${{ matrix.config }}.config" >> .config + + echo -e "\n---- config input ----\n" + cat .config + echo -e "\n---- config input ----\n" + + make defconfig + + echo -e "\n---- config post-defconfig ----\n" + cat .config + echo -e "\n---- config post-defconfig ----\n" + + - name: Download package + run: | + make download -j$(nproc) + + - name: Build tools + run: | + make tools/install -j$(nproc) || \ + make tools/install V=s || exit 1 + + - name: Build toolchain + run: | + make toolchain/install -j$(nproc) || \ + make toolchain/install V=s || exit 1 + + - name: Build target + run: | + make target/compile -j$(nproc) IGNORE_ERRORS='n m' || \ + make target/compile IGNORE_ERRORS='n m' V=s || exit 1 + + - name: Build packages + run: | + make package/compile -j$(nproc) IGNORE_ERRORS='n m' || \ + make package/compile IGNORE_ERRORS='n m' V=s || exit 1 + + make package/install -j$(nproc) || \ + make package/install V=s || exit 1 + + make package/index CONFIG_SIGNED_PACKAGES= V=s + + - name: Build firmware + run: | + make target/install EXTRA_IMAGE_NAME=${{ matrix.config }} -j$(nproc) || \ + make target/install V=s || exit 1 + + - name: Buildinfo + run: | + make buildinfo V=s + + - name: JSON overview + run: | + make json_overview_image_info V=s + continue-on-error: true + + - name: Checksum + run: | + make checksum V=s + + - name: Sanitize target + run: echo "target_sani=$(echo ${{ matrix.targets }} | tr '/' '-')" >> "$GITHUB_ENV" + + - name: Upload images + uses: actions/upload-artifact@v2 + with: + name: ${{ env.target_sani }}-${{ matrix.config }}-images + path: bin/targets/${{ matrix.targets }}/openwrt-${{ matrix.config }}-${{ env.TARGET }}-* + + - name: Upload packages + uses: actions/upload-artifact@v2 + with: + name: ${{ env.target_sani }}-${{ matrix.config }}-packages + path: | + bin/**/*.ipk + !bin/targets/${{ matrix.targets }}/packages/kmod-*.ipk + + - name: Upload kmods + uses: actions/upload-artifact@v2 + with: + name: ${{ env.target_sani }}-${{ matrix.config }}-kmods + path: bin/targets/${{ matrix.targets }}/packages/kmod-*.ipk + + - name: Upload supplementary + uses: actions/upload-artifact@v2 + with: + name: ${{ env.target_sani }}-${{ matrix.config }}-supplementary + path: | + bin/targets/${{ matrix.targets }}/*.buildinfo + bin/targets/${{ matrix.targets }}/*.json + bin/targets/${{ matrix.targets }}/*.manifest + bin/targets/${{ matrix.targets }}/kernel-debug.tar.zst + bin/targets/${{ matrix.targets }}/openwrt-imagebuilder* + bin/targets/${{ matrix.targets }}/openwrt-sdk* + bin/targets/${{ matrix.targets }}/sha256sums* + + - name: Upload logs + uses: actions/upload-artifact@v2 + with: + name: ${{ env.target_sani }}-${{ matrix.config }}-logs + path: logs/ + + - name: Test ImageBuilder + run: | + cd bin/targets/${{ matrix.targets }}/ + tar xf openwrt-imagebuilder-* + cd openwrt-imagebuilder-*/ + mkdir packages + cp "$GITHUB_WORKSPACE"/bin/targets/${{ matrix.targets }}/packages/kmod-*.ipk packages/ + echo "src imagebuilder file:packages" >> repositories.conf + make image + ls ./bin/targets/${{ matrix.targets }}/openwrt-* | grep squashfs + + - name: Test SDK + run: | + cd bin/targets/${{ matrix.targets }}/ + tar xf openwrt-sdk-* + cd openwrt-sdk-*/ + touch .config + make prepare-tmpinfo scripts/config/conf + ./scripts/config/conf --defconfig=.config Config.in + make prereq + rm .config + make defconfig + ./scripts/feeds update base + ./scripts/feeds install busybox + sync + sleep 1 + make package/busybox/compile V=s + sync + sleep 1 + ls ./bin/packages/*/base/busybox* diff --git a/package/system/procd/files/procd.sh b/package/system/procd/files/procd.sh index d86b7219da..1fa2da1487 100644 --- a/package/system/procd/files/procd.sh +++ b/package/system/procd/files/procd.sh @@ -262,7 +262,7 @@ _procd_set_param() { json_add_int "$type" $(kill -l "$1") ;; pidfile|user|group|seccomp|capabilities|facility|\ - extroot|overlaydir|tmpoverlaysize) + extroot|overlaydir|tmpoverlaysize|context) json_add_string "$type" "$1" ;; stdout|stderr|no_new_privs) -- 2.30.2