From ebbc806d30502ff003ae7a19098c6afaaf1295a5 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Tue, 30 May 2023 18:57:42 +0200 Subject: [PATCH] CI: add support for getting ccache cache from S3 Add support for getting ccache cache from S3. ccache is archieved in a tar and downloaded from S3 Cloud Storage. For push events, ccache is then uplodaed back to S3 to refresh and have a ccache cache always fresh. An additional workflow is added to upload files to an S3 Cloud Storage from artifacts uplodaed to github. The minio tool is used to upload files to S3. If the ccache can't be downloaded from s3, we fallback to github cache system. Also limit s3 upload to the openwrt repository since external fork won't have (obviously) the required secrtes to upload data to the S3 Cloud Storage. Signed-off-by: Christian Marangi --- .github/workflows/build.yml | 40 ++++++++++++++++++++++-- .github/workflows/kernel.yml | 18 +++++++++++ .github/workflows/packages.yml | 23 ++++++++++++++ .github/workflows/upload-file-s3.yml | 46 ++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/upload-file-s3.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 95d19f4c4e0b..efaf7594033f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,6 +60,8 @@ on: ccache_type: type: string default: kernel + upload_ccache_cache: + type: boolean permissions: contents: read @@ -232,6 +234,20 @@ jobs: echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV" echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV" + - name: Download and extract ccache cache from s3 + id: restore-ccache-cache-s3 + if: inputs.use_ccache_cache == true + working-directory: openwrt + run: | + ENDPOINT=https://storage.googleapis.com + BUCKET=openwrt-ci-cache + CCACHE_TAR=ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar + + if curl -o /dev/null -s --head --fail $ENDPOINT/$BUCKET/$CCACHE_TAR; then + wget -O - $ENDPOINT/$BUCKET/$CCACHE_TAR | tar -xf - + echo "cache-hit=true" >> $GITHUB_OUTPUT + fi + - name: Fix permission run: | chown -R buildbot:buildbot openwrt @@ -256,7 +272,7 @@ jobs: - name: Restore ccache cache id: restore-ccache-cache - if: inputs.use_ccache_cache == true + if: inputs.use_ccache_cache == true && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true' uses: actions/cache/restore@v3 with: path: openwrt/.ccache @@ -498,7 +514,8 @@ jobs: path: "openwrt/logs" - name: Delete already present ccache cache - if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true && github.event_name == 'push' + if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true && + github.event_name == 'push' && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true' uses: octokit/request-action@v2.x with: route: DELETE /repos/{repository}/actions/caches?key={key} @@ -508,12 +525,29 @@ jobs: INPUT_KEY: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }} - name: Save ccache cache - if: inputs.use_ccache_cache == true && github.event_name == 'push' + if: inputs.use_ccache_cache == true && github.event_name == 'push' && + steps.restore-ccache-cache-s3.outputs.cache-hit != 'true' uses: actions/cache/save@v3 with: path: openwrt/.ccache key: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }} + - name: Archive ccache + if: inputs.use_ccache_cache == true && github.event_name == 'push' && + inputs.upload_ccache_cache == true + shell: su buildbot -c "sh -e {0}" + working-directory: openwrt + run: tar -cf ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar .ccache + + - name: Upload ccache cache + if: inputs.use_ccache_cache == true && github.event_name == 'push' && + inputs.upload_ccache_cache == true + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.target }}-${{ inputs.subtarget }}-ccache-cache + path: openwrt/ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar + retention-days: 1 + - name: Find external toolchain name id: get-toolchain-name if: inputs.upload_external_toolchain == true diff --git a/.github/workflows/kernel.yml b/.github/workflows/kernel.yml index d9ab8581c9b4..5f18a39c5532 100644 --- a/.github/workflows/kernel.yml +++ b/.github/workflows/kernel.yml @@ -117,6 +117,7 @@ jobs: subtarget: ${{ matrix.subtarget }} build_kernel: true build_all_kmods: true + upload_ccache_cache: ${{ github.repository_owner == 'openwrt' }} check-kernel-patches: name: Check Kernel patches @@ -133,3 +134,20 @@ jobs: target: ${{ matrix.target }} subtarget: ${{ matrix.subtarget }} + upload-ccache-cache-in-s3: + if: github.event_name == 'push' && github.repository_owner == 'openwrt' + name: Upload ccache cache to s3 + needs: [determine_targets, build] + strategy: + fail-fast: False + matrix: + include: ${{fromJson(needs.determine_targets.outputs.targets_subtargets)}} + secrets: + s3_access_key: ${{ secrets.GCS_S3_ACCESS_KEY }} + s3_secret_key: ${{ secrets.GCS_S3_SECRET_KEY }} + uses: ./.github/workflows/upload-file-s3.yml + with: + endpoint: https://storage.googleapis.com + bucket: openwrt-ci-cache + download_id: ${{ matrix.target }}-${{ matrix.subtarget }}-ccache-cache + filename: ccache-kernel-${{ matrix.target }}-${{ matrix.subtarget }}.tar diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 58bbcd013112..fe2898cacdb8 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -54,4 +54,27 @@ jobs: build_all_modules: true build_full: true ccache_type: packages + upload_ccache_cache: ${{ github.repository_owner == 'openwrt' }} + + upload-ccache-cache-in-s3: + if: github.event_name == 'push' && github.repository_owner == 'openwrt' + name: Upload ccache cache to s3 + needs: build + strategy: + fail-fast: False + matrix: + include: + - target: malta + subtarget: be + - target: x86 + subtarget: 64 + secrets: + s3_access_key: ${{ secrets.GCS_S3_ACCESS_KEY }} + s3_secret_key: ${{ secrets.GCS_S3_SECRET_KEY }} + uses: ./.github/workflows/upload-file-s3.yml + with: + endpoint: https://storage.googleapis.com + bucket: openwrt-ci-cache + download_id: ${{ matrix.target }}-${{ matrix.subtarget }}-ccache-cache + filename: ccache-packages-${{ matrix.target }}-${{ matrix.subtarget }}.tar diff --git a/.github/workflows/upload-file-s3.yml b/.github/workflows/upload-file-s3.yml new file mode 100644 index 000000000000..6bcb172468da --- /dev/null +++ b/.github/workflows/upload-file-s3.yml @@ -0,0 +1,46 @@ +name: Upload File to S3 + +on: + workflow_call: + secrets: + s3_access_key: + s3_secret_key: + inputs: + endpoint: + required: true + type: string + bucket: + required: true + type: string + download_id: + required: true + type: string + filename: + required: true + type: string + +jobs: + upload-file-in-s3: + name: Upload file in S3 + runs-on: ubuntu-latest + + steps: + - name: Install minio + run: | + curl https://dl.min.io/client/mc/release/linux-amd64/mc \ + --create-dirs \ + -o $GITHUB_WORKSPACE/minio-binaries/mc + + chmod +x $GITHUB_WORKSPACE/minio-binaries/mc + echo $GITHUB_WORKSPACE/minio-binaries/ >> $GITHUB_PATH + + - name: Setup minio + run: mc alias set s3 ${{ inputs.endpoint }} ${{ secrets.s3_access_key }} ${{ secrets.s3_secret_key }} + + - name: Download file + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.download_id }} + + - name: Upload file to s3 + run: mc cp ${{ inputs.filename }} s3/${{ inputs.bucket }}/ -- 2.30.2