CI: add support for getting ccache cache from S3

Add support for getting ccache cache from S3.
ccache is archieved in a tar and downloaded from S3 Cloud Storage.

For push events, ccache is then uplodaed back to S3 to refresh and have
a ccache cache always fresh.

An additional workflow is added to upload files to an S3 Cloud Storage
from artifacts uplodaed to github. The minio tool is used to upload
files to S3.

If the ccache can't be downloaded from s3, we fallback to github cache
system.

Also limit s3 upload to the openwrt repository since external fork won't
have (obviously) the required secrtes to upload data to the S3 Cloud
Storage.

Signed-off-by: Christian Marangi <ansuelsmth@gmail.com>

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 95d19f4c4e0b6e80730c498a52ca424c170b1fa2..efaf7594033f1b64736dc5fe11ab9a5a76b12b3e 100644 (file)
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -60,6 +60,8 @@ on:
       ccache_type:
         type: string
         default: kernel
+      upload_ccache_cache:
+        type: boolean
 
 permissions:
   contents: read
@@ -232,6 +234,20 @@ jobs:
           echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
           echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
 
+      - name: Download and extract ccache cache from s3
+        id: restore-ccache-cache-s3
+        if: inputs.use_ccache_cache == true
+        working-directory: openwrt
+        run: |
+          ENDPOINT=https://storage.googleapis.com
+          BUCKET=openwrt-ci-cache
+          CCACHE_TAR=ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar
+
+          if curl -o /dev/null -s --head --fail $ENDPOINT/$BUCKET/$CCACHE_TAR; then
+            wget -O - $ENDPOINT/$BUCKET/$CCACHE_TAR | tar -xf -
+            echo "cache-hit=true" >> $GITHUB_OUTPUT
+          fi
+
       - name: Fix permission
         run: |
           chown -R buildbot:buildbot openwrt
@@ -256,7 +272,7 @@ jobs:
 
       - name: Restore ccache cache
         id: restore-ccache-cache
-        if: inputs.use_ccache_cache == true
+        if: inputs.use_ccache_cache == true && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true'
         uses: actions/cache/restore@v3
         with:
           path: openwrt/.ccache
@@ -498,7 +514,8 @@ jobs:
           path: "openwrt/logs"
 
       - name: Delete already present ccache cache
-        if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true  && github.event_name == 'push'
+        if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true  &&
+          github.event_name == 'push' && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true'
         uses: octokit/request-action@v2.x
         with:
           route: DELETE /repos/{repository}/actions/caches?key={key}
@@ -508,12 +525,29 @@ jobs:
           INPUT_KEY: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
 
       - name: Save ccache cache
-        if: inputs.use_ccache_cache == true && github.event_name == 'push'
+        if: inputs.use_ccache_cache == true && github.event_name == 'push' &&
+          steps.restore-ccache-cache-s3.outputs.cache-hit != 'true'
         uses: actions/cache/save@v3
         with:
           path: openwrt/.ccache
           key: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
 
+      - name: Archive ccache
+        if: inputs.use_ccache_cache == true  && github.event_name == 'push' &&
+          inputs.upload_ccache_cache == true
+        shell: su buildbot -c "sh -e {0}"
+        working-directory: openwrt
+        run: tar -cf ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar .ccache
+
+      - name: Upload ccache cache
+        if: inputs.use_ccache_cache == true && github.event_name == 'push' &&
+          inputs.upload_ccache_cache == true
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.target }}-${{ inputs.subtarget }}-ccache-cache
+          path: openwrt/ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar
+          retention-days: 1
+
       - name: Find external toolchain name
         id: get-toolchain-name
         if: inputs.upload_external_toolchain == true

diff --git a/.github/workflows/kernel.yml b/.github/workflows/kernel.yml
index d9ab8581c9b455968ce36731b9a8b7335e149b07..5f18a39c55322271d6e5291a9c210965946d4ed9 100644 (file)
--- a/.github/workflows/kernel.yml
+++ b/.github/workflows/kernel.yml
@@ -117,6 +117,7 @@ jobs:
       subtarget: ${{ matrix.subtarget }}
       build_kernel: true
       build_all_kmods: true
+      upload_ccache_cache: ${{ github.repository_owner == 'openwrt' }}
 
   check-kernel-patches:
     name: Check Kernel patches
@@ -133,3 +134,20 @@ jobs:
       target: ${{ matrix.target }}
       subtarget: ${{ matrix.subtarget }}
 
+  upload-ccache-cache-in-s3:
+    if: github.event_name == 'push' && github.repository_owner == 'openwrt'
+    name: Upload ccache cache to s3
+    needs: [determine_targets, build]
+    strategy:
+       fail-fast: False
+       matrix:
+         include: ${{fromJson(needs.determine_targets.outputs.targets_subtargets)}}
+    secrets:
+      s3_access_key: ${{ secrets.GCS_S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.GCS_S3_SECRET_KEY }}
+    uses: ./.github/workflows/upload-file-s3.yml
+    with:
+      endpoint: https://storage.googleapis.com
+      bucket: openwrt-ci-cache
+      download_id: ${{ matrix.target }}-${{ matrix.subtarget }}-ccache-cache
+      filename: ccache-kernel-${{ matrix.target }}-${{ matrix.subtarget }}.tar

diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml
index 58bbcd01311275b5c89ed92271abc82498de7caa..fe2898cacdb8cb0a5ddf2caf433ecba641c336bb 100644 (file)
--- a/.github/workflows/packages.yml
+++ b/.github/workflows/packages.yml
@@ -54,4 +54,27 @@ jobs:
       build_all_modules: true
       build_full: true
       ccache_type: packages
+      upload_ccache_cache: ${{ github.repository_owner == 'openwrt' }}
+
+  upload-ccache-cache-in-s3:
+    if: github.event_name == 'push' && github.repository_owner == 'openwrt'
+    name: Upload ccache cache to s3
+    needs: build
+    strategy:
+       fail-fast: False
+       matrix:
+        include:
+          - target: malta
+            subtarget: be
+          - target: x86
+            subtarget: 64
+    secrets:
+      s3_access_key: ${{ secrets.GCS_S3_ACCESS_KEY }}
+      s3_secret_key: ${{ secrets.GCS_S3_SECRET_KEY }}
+    uses: ./.github/workflows/upload-file-s3.yml
+    with:
+      endpoint: https://storage.googleapis.com
+      bucket: openwrt-ci-cache
+      download_id: ${{ matrix.target }}-${{ matrix.subtarget }}-ccache-cache
+      filename: ccache-packages-${{ matrix.target }}-${{ matrix.subtarget }}.tar
 

diff --git a/.github/workflows/upload-file-s3.yml b/.github/workflows/upload-file-s3.yml
new file mode 100644 (file)
index 0000000..6bcb172
--- /dev/null
+++ b/.github/workflows/upload-file-s3.yml
@@ -0,0 +1,46 @@
+name: Upload File to S3
+
+on:
+  workflow_call:
+    secrets:
+      s3_access_key:
+      s3_secret_key:
+    inputs:
+      endpoint:
+        required: true
+        type: string
+      bucket:
+        required: true
+        type: string
+      download_id:
+        required: true
+        type: string
+      filename:
+        required: true
+        type: string
+
+jobs:
+  upload-file-in-s3:
+    name: Upload file in S3
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Install minio
+        run: |
+          curl https://dl.min.io/client/mc/release/linux-amd64/mc \
+            --create-dirs \
+            -o $GITHUB_WORKSPACE/minio-binaries/mc
+
+          chmod +x $GITHUB_WORKSPACE/minio-binaries/mc
+          echo $GITHUB_WORKSPACE/minio-binaries/ >> $GITHUB_PATH
+
+      - name: Setup minio
+        run: mc alias set s3 ${{ inputs.endpoint }} ${{ secrets.s3_access_key }} ${{ secrets.s3_secret_key }}
+
+      - name: Download file
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ inputs.download_id }}
+
+      - name: Upload file to s3
+        run: mc cp ${{ inputs.filename }} s3/${{ inputs.bucket }}/