From eecc6e48117be26c2eefd9257cceb9d9b1e842f2 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Mon, 22 May 2023 16:47:08 +0200 Subject: [PATCH] CI: rework build workflow to have split target and subtarget directly Instead of referring to a redundant job and ENV variables, rework build workflow to accept and require split target and subtarget and use them directly from inputs. Rework each user and pass a JSON of tuple to matrix include with each target/subtarget combination to test. Special notice this doesn't use the github actions matrix combination feature but reference each specific tuple of target and subtarget to test. Just a cleanup no behaviour change intended. Signed-off-by: Christian Marangi --- .github/workflows/build.yml | 32 ++++++++++------------ .github/workflows/check-kernel-patches.yml | 26 ++++++++---------- .github/workflows/coverity.yml | 3 +- .github/workflows/kernel.yml | 12 +++++--- .github/workflows/packages.yml | 7 +++-- .github/workflows/toolchain.yml | 6 ++-- 6 files changed, 44 insertions(+), 42 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8744bc7737..52fba8e106 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,9 @@ on: target: required: true type: string + subtarget: + required: true + type: string testing: type: boolean build_toolchain: @@ -50,7 +53,7 @@ permissions: jobs: setup_build: - name: Setup build ${{ inputs.target }} + name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }} runs-on: ubuntu-latest outputs: owner_lc: ${{ steps.lower_owner.outputs.owner_lc }} @@ -109,7 +112,7 @@ jobs: echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT build: - name: Build ${{ inputs.target }} + name: Build ${{ inputs.target }}/${{ inputs.subtarget }} needs: setup_build runs-on: ubuntu-latest @@ -157,13 +160,6 @@ jobs: run: | chown -R buildbot:buildbot openwrt - - name: Initialization environment - run: | - TARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 1) - SUBTARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 2) - echo "TARGET=$TARGET" >> "$GITHUB_ENV" - echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV" - - name: Prepare prebuilt tools shell: su buildbot -c "sh -e {0}" working-directory: openwrt @@ -213,7 +209,7 @@ jobs: fi fi - SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/sha256sums" + SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums" if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")" TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p') @@ -235,16 +231,16 @@ jobs: uses: actions/cache@v3 with: path: openwrt/.ccache - key: ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-${{ needs.setup_build.outputs.ccache_hash }} + key: ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-${{ needs.setup_build.outputs.ccache_hash }} restore-keys: | - ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}- + ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}- - name: Download external toolchain/sdk if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal' shell: su buildbot -c "sh -e {0}" working-directory: openwrt run: | - wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \ + wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \ | tar --xz -xf - - name: Configure testing kernel @@ -289,7 +285,7 @@ jobs: ./scripts/ext-toolchain.sh \ --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \ --overwrite-config \ - --config ${{ env.TARGET }}/${{ env.SUBTARGET }} + --config ${{ inputs.target }}/${{ inputs.subtarget }} - name: Adapt external sdk to external toolchain format if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk' @@ -331,7 +327,7 @@ jobs: ./scripts/ext-toolchain.sh \ --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \ --overwrite-config \ - --config ${{ env.TARGET }}/${{ env.SUBTARGET }} + --config ${{ inputs.target }}/${{ inputs.subtarget }} - name: Configure internal toolchain if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal' @@ -342,8 +338,8 @@ jobs: echo CONFIG_AUTOREMOVE=y >> .config echo CONFIG_CCACHE=y >> .config - echo "CONFIG_TARGET_${{ env.TARGET }}=y" >> .config - echo "CONFIG_TARGET_${{ env.TARGET }}_${{ env.SUBTARGET }}=y" >> .config + echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config + echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config make defconfig @@ -435,5 +431,5 @@ jobs: if: failure() uses: actions/upload-artifact@v3 with: - name: ${{ env.TARGET }}-${{ env.SUBTARGET }}-logs + name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs path: "openwrt/logs" diff --git a/.github/workflows/check-kernel-patches.yml b/.github/workflows/check-kernel-patches.yml index 274bae367d..1ab4a361f5 100644 --- a/.github/workflows/check-kernel-patches.yml +++ b/.github/workflows/check-kernel-patches.yml @@ -6,6 +6,9 @@ on: target: required: true type: string + subtarget: + required: true + type: string testing: type: boolean use_openwrt_container: @@ -85,13 +88,6 @@ jobs: run: | chown -R buildbot:buildbot openwrt - - name: Initialization environment - run: | - TARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 1) - SUBTARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 2) - echo "TARGET=$TARGET" >> "$GITHUB_ENV" - echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV" - - name: Prepare prebuilt tools shell: su buildbot -c "sh -e {0}" working-directory: openwrt @@ -118,8 +114,8 @@ jobs: echo CONFIG_AUTOREMOVE=y >> .config echo CONFIG_CCACHE=y >> .config - echo "CONFIG_TARGET_${{ env.TARGET }}=y" >> .config - echo "CONFIG_TARGET_${{ env.TARGET }}_${{ env.SUBTARGET }}=y" >> .config + echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config + echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config make defconfig @@ -140,13 +136,13 @@ jobs: . .github/workflows/scripts/ci_helpers.sh if git diff --name-only --exit-code; then - success "Kernel patches for ${{ env.TARGET }}/${{ env.SUBTARGET }} seems ok" + success "Kernel patches for ${{ inputs.target }}/${{ inputs.subtarget }} seems ok" else - err "Kernel patches for ${{ env.TARGET }}/${{ env.SUBTARGET }} require refresh. (run 'make target/linux/refresh' and force push this pr)" + err "Kernel patches for ${{ inputs.target }}/${{ inputs.subtarget }} require refresh. (run 'make target/linux/refresh' and force push this pr)" err "You can also check the provided artifacts with the refreshed patch from this CI run." - mkdir ${{ env.TARGET }}-${{ env.SUBTARGET }}-refreshed + mkdir ${{ inputs.target }}-${{ inputs.subtarget }}-refreshed for f in $(git diff --name-only); do - cp --parents $f ${{ env.TARGET }}-${{ env.SUBTARGET }}-refreshed/ + cp --parents $f ${{ inputs.target }}-${{ inputs.subtarget }}-refreshed/ done exit 1 fi @@ -155,5 +151,5 @@ jobs: if: failure() uses: actions/upload-artifact@v3 with: - name: ${{ env.TARGET }}-${{ env.SUBTARGET }}-refreshed - path: openwrt/${{ env.TARGET }}-${{ env.SUBTARGET }}-refreshed + name: ${{ inputs.target }}-${{ inputs.subtarget }}-refreshed + path: openwrt/${{ inputs.target }}-${{ inputs.subtarget }}-refreshed diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index db628d05ee..db59ef8ca6 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -17,7 +17,8 @@ jobs: packages: read uses: ./.github/workflows/build.yml with: - target: x86/64 + target: x86 + subtarget: 64 build_full: true include_feeds: true coverity_compiler_template_list: >- diff --git a/.github/workflows/kernel.yml b/.github/workflows/kernel.yml index a25829a4c0..02aee8b27c 100644 --- a/.github/workflows/kernel.yml +++ b/.github/workflows/kernel.yml @@ -64,8 +64,9 @@ jobs: if echo "$CHANGED_FILES" | grep -v -q target/linux || echo "$CHANGED_FILES" | grep -q target/linux/generic || echo "$CHANGED_FILES" | grep -q $(echo $TARGET | cut -d "/" -f 1); then + TUPLE='{"target":"'"$(echo $TARGET | cut -d "/" -f 1)"'","subtarget":"'"$(echo $TARGET | cut -d "/" -f 2)"'"}' [[ $FIRST -ne 1 ]] && JSON_TARGETS_SUBTARGETS="$JSON_TARGETS_SUBTARGETS"',' - JSON_TARGETS_SUBTARGETS="$JSON_TARGETS_SUBTARGETS"'"'"${TARGET}"'"' + JSON_TARGETS_SUBTARGETS="$JSON_TARGETS_SUBTARGETS""$TUPLE" FIRST=0 fi done @@ -77,8 +78,9 @@ jobs: if echo "$CHANGED_FILES" | grep -v -q target/linux || echo "$CHANGED_FILES" | grep -q target/linux/generic || echo "$CHANGED_FILES" | grep -q $(echo $TARGET | cut -d "/" -f 1); then + TUPLE='{"target":"'"$(echo $TARGET | cut -d "/" -f 1)"'","subtarget":"'"$(echo $TARGET | cut -d "/" -f 2)"'"}' [[ $FIRST -ne 1 ]] && JSON_TARGETS="$JSON_TARGETS"',' - JSON_TARGETS="$JSON_TARGETS"'"'"${TARGET}"'"' + JSON_TARGETS="$JSON_TARGETS""$TUPLE" FIRST=0 fi done @@ -104,10 +106,11 @@ jobs: strategy: fail-fast: False matrix: - target: ${{fromJson(needs.determine_targets.outputs.targets_subtargets)}} + include: ${{fromJson(needs.determine_targets.outputs.targets_subtargets)}} uses: ./.github/workflows/build.yml with: target: ${{ matrix.target }} + subtarget: ${{ matrix.subtarget }} build_kernel: true build_all_kmods: true @@ -120,8 +123,9 @@ jobs: strategy: fail-fast: False matrix: - target: ${{fromJson(needs.determine_targets.outputs.targets)}} + include: ${{fromJson(needs.determine_targets.outputs.targets)}} uses: ./.github/workflows/check-kernel-patches.yml with: target: ${{ matrix.target }} + subtarget: ${{ matrix.subtarget }} diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 340ee0c204..e2f932b1ba 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -37,11 +37,14 @@ jobs: fail-fast: False matrix: include: - - target: malta/be - - target: x86/64 + - target: malta + subtarget: be + - target: x86 + subtarget: 64 uses: ./.github/workflows/build.yml with: target: ${{ matrix.target }} + subtarget: ${{ matrix.subtarget }} build_kernel: true build_all_kmods: true build_all_modules: true diff --git a/.github/workflows/toolchain.yml b/.github/workflows/toolchain.yml index 2a24d82e30..5755ca25b9 100644 --- a/.github/workflows/toolchain.yml +++ b/.github/workflows/toolchain.yml @@ -40,8 +40,9 @@ jobs: JSON='[' FIRST=1 for TARGET in $TARGETS; do + TUPLE='{"target":"'"$(echo $TARGET | cut -d "/" -f 1)"'","subtarget":"'"$(echo $TARGET | cut -d "/" -f 2)"'"}' [[ $FIRST -ne 1 ]] && JSON="$JSON"',' - JSON="$JSON"'"'"${TARGET}"'"' + JSON="$JSON""$TUPLE" FIRST=0 done JSON="$JSON"']' @@ -61,8 +62,9 @@ jobs: strategy: fail-fast: False matrix: - target: ${{fromJson(needs.determine_targets.outputs.target)}} + include: ${{fromJson(needs.determine_targets.outputs.target)}} uses: ./.github/workflows/build.yml with: target: ${{ matrix.target }} + subtarget: ${{ matrix.subtarget }} build_toolchain: true -- 2.30.2