.github/workflows/build.yml

   1 name: Build sub target
   2
   3 on:
   4   workflow_call:
   5     inputs:
   6       target:
   7         required: true
   8         type: string
   9       testing:
  10         type: boolean
  11       build_toolchain:
  12         type: boolean
  13       include_feeds:
  14         type: boolean
  15       build_full:
  16         type: boolean
  17       build_kernel:
  18         type: boolean
  19       build_all_modules:
  20         type: boolean
  21       build_all_kmods:
  22         type: boolean
  23       build_all_boards:
  24         type: boolean
  25
  26 permissions:
  27   contents: read
  28
  29 jobs:
  30   setup_build:
  31     name: Setup build ${{ inputs.target }}
  32     runs-on: ubuntu-latest
  33     outputs:
  34       owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
  35       ccache_hash: ${{ steps.ccache_hash.outputs.ccache_hash }}
  36       container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
  37
  38     steps:
  39       - name: Checkout
  40         uses: actions/checkout@v3
  41
  42       - name: Set lower case owner name
  43         id: lower_owner
  44         run: |
  45           OWNER_LC=$(echo "${{ github.repository_owner }}" \
  46             | tr '[:upper:]' '[:lower:]')
  47           echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
  48
  49       - name: Generate ccache hash
  50         id: ccache_hash
  51         run: |
  52           CCACHE_HASH=$(md5sum include/kernel-* | awk '{ print $1 }' \
  53            | md5sum | awk '{ print $1 }')
  54           echo "ccache_hash=$CCACHE_HASH" >> $GITHUB_OUTPUT
  55
  56       # Per branch tools container tag
  57       # By default stick to latest
  58       # For official test targetting openwrt stable branch
  59       # Get the branch or parse the tag and push dedicated tools containers
  60       # For local test to use the correct container for stable release testing
  61       # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
  62       - name: Determine tools container tag
  63         id: determine_tools_container
  64         run: |
  65           CONTAINER_TAG=latest
  66           if [ -n "${{ github.base_ref }}" ]; then
  67             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
  68               CONTAINER_TAG="${{ github.base_ref }}"
  69             fi
  70           elif [ ${{ github.ref_type }} == "branch" ]; then
  71             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
  72               CONTAINER_TAG=${{ github.ref_name }}
  73             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
  74               CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
  75             fi
  76           elif [ ${{ github.ref_type }} == "tag" ]; then
  77             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
  78               CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
  79             fi
  80           fi
  81           echo "Tools container to use tools:$CONTAINER_TAG"
  82           echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
  83
  84   build:
  85     name: Build ${{ inputs.target }}
  86     needs: setup_build
  87     runs-on: ubuntu-latest
  88
  89     container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/tools:${{ needs.setup_build.outputs.container_tag }}
  90
  91     permissions:
  92       contents: read
  93       packages: read
  94
  95     steps:
  96       - name: Checkout master directory
  97         uses: actions/checkout@v3
  98         with:
  99           path: openwrt
 100
 101       - name: Checkout packages feed
 102         if: inputs.include_feeds == true
 103         uses: actions/checkout@v3
 104         with:
 105           repository: openwrt/packages
 106           path: openwrt/feeds/packages
 107
 108       - name: Checkout luci feed
 109         if: inputs.include_feeds == true
 110         uses: actions/checkout@v3
 111         with:
 112           repository: openwrt/luci
 113           path: openwrt/feeds/luci
 114
 115       - name: Checkout routing feed
 116         if: inputs.include_feeds == true
 117         uses: actions/checkout@v3
 118         with:
 119           repository: openwrt/routing
 120           path: openwrt/feeds/routing
 121
 122       - name: Checkout telephony feed
 123         if: inputs.include_feeds == true
 124         uses: actions/checkout@v3
 125         with:
 126           repository: openwrt/telephony
 127           path: openwrt/feeds/telephony
 128
 129       - name: Fix permission
 130         run: |
 131           chown -R buildbot:buildbot openwrt
 132
 133       - name: Initialization environment
 134         run: |
 135           TARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 1)
 136           SUBTARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 2)
 137           echo "TARGET=$TARGET" >> "$GITHUB_ENV"
 138           echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV"
 139
 140       - name: Update & Install feeds
 141         if: inputs.include_feeds == true
 142         shell: su buildbot -c "sh -e {0}"
 143         working-directory: openwrt
 144         run: |
 145           ./scripts/feeds update -a
 146           ./scripts/feeds install -a
 147
 148       - name: Parse toolchain file
 149         if: inputs.build_toolchain == false
 150         id: parse-toolchain
 151         working-directory: openwrt
 152         run: |
 153           TOOLCHAIN_PATH=snapshots
 154
 155           if [ -n "${{ github.base_ref }}" ]; then
 156             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 157               major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
 158             fi
 159           elif [ "${{ github.ref_type }}" = "branch" ]; then
 160             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 161               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
 162             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 163               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
 164             fi
 165           elif [ "${{ github.ref_type }}" = "tag" ]; then
 166             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 167               major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 168             fi
 169           fi
 170
 171           if [ -n "$major_ver" ]; then
 172             git fetch --tags -f
 173             latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
 174             if [ -n "$latest_tag" ]; then
 175               TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
 176             fi
 177           fi
 178
 179           SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/sha256sums"
 180           if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
 181             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
 182             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
 183             TOOLCHAIN_SHA256=$(echo "$TOOLCHAIN_STRING" | cut -d ' ' -f 1)
 184
 185             echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
 186           elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
 187             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
 188             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
 189             TOOLCHAIN_SHA256=$(echo "$TOOLCHAIN_STRING" | cut -d ' ' -f 1)
 190
 191             echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
 192           fi
 193
 194           echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
 195           echo "TOOLCHAIN_SHA256=$TOOLCHAIN_SHA256" >> "$GITHUB_ENV"
 196           echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
 197
 198       - name: Cache external toolchain/sdk
 199         if: inputs.build_toolchain == false
 200         id: cache-external-toolchain
 201         uses: actions/cache@v3
 202         with:
 203           path: openwrt/${{ env.TOOLCHAIN_FILE }}
 204           key: ${{ env.TOOLCHAIN_FILE }}-${{ steps.parse-toolchain.outputs.toolchain-type }}-${{ env.TOOLCHAIN_SHA256 }}
 205
 206       - name: Cache ccache
 207         uses: actions/cache@v3
 208         with:
 209           path: openwrt/.ccache
 210           key: ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-${{ needs.setup_build.outputs.ccache_hash }}
 211           restore-keys: |
 212             ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-
 213
 214       - name: Download external toolchain/sdk
 215         if: inputs.build_toolchain == false && steps.cache-external-toolchain.outputs.cache-hit != 'true'
 216         shell: su buildbot -c "sh -e {0}"
 217         working-directory: openwrt
 218         run: |
 219           wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
 220             | tar --xz -xf -
 221
 222       - name: Extract prebuilt tools
 223         shell: su buildbot -c "sh -e {0}"
 224         working-directory: openwrt
 225         run: ./scripts/ext-tools.sh --tools /tools.tar
 226
 227       - name: Configure testing kernel
 228         if: inputs.testing == true
 229         shell: su buildbot -c "sh -e {0}"
 230         working-directory: openwrt
 231         run: |
 232           echo CONFIG_TESTING_KERNEL=y >> .config
 233
 234       - name: Configure all kernel modules
 235         if: inputs.build_all_kmods == true
 236         shell: su buildbot -c "sh -e {0}"
 237         working-directory: openwrt
 238         run: |
 239           echo CONFIG_ALL_KMODS=y >> .config
 240
 241       - name: Configure all modules
 242         if: inputs.build_all_modules == true
 243         shell: su buildbot -c "sh -e {0}"
 244         working-directory: openwrt
 245         run: |
 246           echo CONFIG_ALL=y >> .config
 247
 248       - name: Configure all boards
 249         if: inputs.build_all_boards == true
 250         shell: su buildbot -c "sh -e {0}"
 251         working-directory: openwrt
 252         run: |
 253           echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
 254           echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
 255           echo CONFIG_TARGET_ALL_PROFILES=y >> .config
 256
 257       - name: Configure external toolchain
 258         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
 259         shell: su buildbot -c "sh -e {0}"
 260         working-directory: openwrt
 261         run: |
 262           echo CONFIG_DEVEL=y >> .config
 263           echo CONFIG_AUTOREMOVE=y >> .config
 264           echo CONFIG_CCACHE=y >> .config
 265
 266           ./scripts/ext-toolchain.sh \
 267             --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
 268             --overwrite-config \
 269             --config ${{ env.TARGET }}/${{ env.SUBTARGET }}
 270
 271       - name: Adapt external sdk to external toolchain format
 272         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk' && steps.cache-external-toolchain.outputs.cache-hit != 'true'
 273         shell: su buildbot -c "sh -e {0}"
 274         working-directory: openwrt
 275         run: |
 276           TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
 277           TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
 278           OPENWRT_DIR=$(pwd)
 279
 280           # Find target name from toolchain info.mk
 281           GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
 282
 283           cd $TOOLCHAIN_BIN
 284
 285           # Revert sdk wrapper scripts applied to all the bins
 286           for app in $(find . -name "*.bin"); do
 287             TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
 288             rm $TARGET_APP
 289             mv .$TARGET_APP.bin $TARGET_APP
 290           done
 291
 292           # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
 293           cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
 294           for app in cc gcc g++ c++ cpp ld as ; do
 295             [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
 296             ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
 297           done
 298
 299       - name: Configure external toolchain with sdk
 300         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 301         shell: su buildbot -c "sh -e {0}"
 302         working-directory: openwrt
 303         run: |
 304           echo CONFIG_DEVEL=y >> .config
 305           echo CONFIG_AUTOREMOVE=y >> .config
 306           echo CONFIG_CCACHE=y >> .config
 307
 308           ./scripts/ext-toolchain.sh \
 309             --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
 310             --overwrite-config \
 311             --config ${{ env.TARGET }}/${{ env.SUBTARGET }}
 312
 313       - name: Configure internal toolchain
 314         if: inputs.build_toolchain == true
 315         shell: su buildbot -c "sh -e {0}"
 316         working-directory: openwrt
 317         run: |
 318           echo CONFIG_DEVEL=y >> .config
 319           echo CONFIG_AUTOREMOVE=y >> .config
 320           echo CONFIG_CCACHE=y >> .config
 321
 322           echo "CONFIG_TARGET_${{ env.TARGET }}=y" >> .config
 323           echo "CONFIG_TARGET_${{ env.TARGET }}_${{ env.SUBTARGET }}=y" >> .config
 324
 325           make defconfig
 326
 327       - name: Show configuration
 328         shell: su buildbot -c "sh -e {0}"
 329         working-directory: openwrt
 330         run: ./scripts/diffconfig.sh
 331
 332       - name: Build tools
 333         shell: su buildbot -c "sh -e {0}"
 334         working-directory: openwrt
 335         run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 336
 337       - name: Build toolchain
 338         shell: su buildbot -c "sh -e {0}"
 339         working-directory: openwrt
 340         run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 341
 342       - name: Build Kernel
 343         if: inputs.build_kernel == true
 344         shell: su buildbot -c "sh -e {0}"
 345         working-directory: openwrt
 346         run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 347
 348       - name: Build Kernel Kmods
 349         if: inputs.build_kernel == true
 350         shell: su buildbot -c "sh -e {0}"
 351         working-directory: openwrt
 352         run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 353
 354       - name: Build everything
 355         if: inputs.build_full == true
 356         shell: su buildbot -c "sh -e {0}"
 357         working-directory: openwrt
 358         run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 359
 360       - name: Upload logs
 361         if: failure()
 362         uses: actions/upload-artifact@v3
 363         with:
 364           name: ${{ env.TARGET }}-${{ env.SUBTARGET }}-logs
 365           path: "openwrt/logs"