.github/workflows/build.yml

   1 name: Build sub target
   2
   3 on:
   4   workflow_call:
   5     secrets:
   6       coverity_api_token:
   7     inputs:
   8       container_name:
   9         type: string
  10         default: tools
  11       target:
  12         required: true
  13         type: string
  14       subtarget:
  15         required: true
  16         type: string
  17       testing:
  18         type: boolean
  19       build_toolchain:
  20         type: boolean
  21       include_feeds:
  22         type: boolean
  23       build_full:
  24         type: boolean
  25       build_kernel:
  26         type: boolean
  27       build_all_modules:
  28         type: boolean
  29       build_all_kmods:
  30         type: boolean
  31       build_all_boards:
  32         type: boolean
  33       use_openwrt_container:
  34         type: boolean
  35         default: true
  36       coverity_project_name:
  37         type: string
  38         default: OpenWrt
  39       coverity_check_packages:
  40         type: string
  41       coverity_compiler_template_list:
  42         type: string
  43         default: >-
  44           arm-openwrt-linux-gcc
  45       coverity_force_compile_packages:
  46         type: string
  47         default: >-
  48           curl
  49           libnl
  50           mbedtls
  51           wolfssl
  52           openssl
  53       build_external_toolchain:
  54         type: boolean
  55       upload_external_toolchain:
  56         type: boolean
  57
  58 permissions:
  59   contents: read
  60
  61 jobs:
  62   setup_build:
  63     name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }}
  64     runs-on: ubuntu-latest
  65     outputs:
  66       owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
  67       ccache_hash: ${{ steps.ccache_hash.outputs.ccache_hash }}
  68       container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
  69       container_name: ${{ steps.determine_tools_container.outputs.container_name }}
  70
  71     steps:
  72       - name: Checkout
  73         uses: actions/checkout@v3
  74
  75       - name: Set lower case owner name
  76         id: lower_owner
  77         run: |
  78           OWNER_LC=$(echo "${{ github.repository_owner }}" \
  79             | tr '[:upper:]' '[:lower:]')
  80
  81           if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
  82             OWNER_LC=openwrt
  83           fi
  84
  85           echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
  86
  87       - name: Generate ccache hash
  88         id: ccache_hash
  89         run: |
  90           CCACHE_HASH=$(md5sum include/kernel-* | awk '{ print $1 }' \
  91            | md5sum | awk '{ print $1 }')
  92           echo "ccache_hash=$CCACHE_HASH" >> $GITHUB_OUTPUT
  93
  94       # Per branch tools container tag
  95       # By default stick to latest
  96       # For official test targetting openwrt stable branch
  97       # Get the branch or parse the tag and push dedicated tools containers
  98       # For local test to use the correct container for stable release testing
  99       # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
 100       - name: Determine tools container tag
 101         id: determine_tools_container
 102         run: |
 103           CONTAINER_NAME=${{ inputs.container_name }}
 104           CONTAINER_TAG=latest
 105           if [ -n "${{ github.base_ref }}" ]; then
 106             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 107               CONTAINER_TAG="${{ github.base_ref }}"
 108             fi
 109           elif [ ${{ github.ref_type }} == "branch" ]; then
 110             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 111               CONTAINER_TAG=${{ github.ref_name }}
 112             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 113               CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
 114             fi
 115           elif [ ${{ github.ref_type }} == "tag" ]; then
 116             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 117               CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 118             fi
 119           fi
 120
 121           if [ "$CONTAINER_NAME" = "toolchain" ]; then
 122             GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64)
 123             GHCR_HEADER="Authorization: Bearer ${GHCR_TOKEN}"
 124             GHCR_MANIFEST_LINK=https://ghcr.io/v2/${{ steps.lower_owner.outputs.owner_lc }}/${{ inputs.container_name }}/manifests/${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
 125             # Check if container exist
 126             if [ $(curl -s -o /dev/null -w "%{http_code}" -H "$GHCR_HEADER" -I "$GHCR_MANIFEST_LINK") = 200 ]; then
 127               CONTAINER_TAG=${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
 128             else
 129               CONTAINER_NAME=tools
 130             fi
 131           fi
 132
 133           echo "Tools container to use $CONTAINER_NAME:$CONTAINER_TAG"
 134           echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
 135           echo "container_name=$CONTAINER_NAME" >> $GITHUB_OUTPUT
 136
 137   build:
 138     name: Build ${{ inputs.target }}/${{ inputs.subtarget }}
 139     needs: setup_build
 140     runs-on: ubuntu-latest
 141
 142     container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/${{ needs.setup_build.outputs.container_name }}:${{ needs.setup_build.outputs.container_tag }}
 143
 144     permissions:
 145       contents: read
 146       packages: read
 147
 148     steps:
 149       - name: Checkout master directory
 150         uses: actions/checkout@v3
 151         with:
 152           path: openwrt
 153
 154       - name: Checkout packages feed
 155         if: inputs.include_feeds == true
 156         uses: actions/checkout@v3
 157         with:
 158           repository: openwrt/packages
 159           path: openwrt/feeds/packages
 160
 161       - name: Checkout luci feed
 162         if: inputs.include_feeds == true
 163         uses: actions/checkout@v3
 164         with:
 165           repository: openwrt/luci
 166           path: openwrt/feeds/luci
 167
 168       - name: Checkout routing feed
 169         if: inputs.include_feeds == true
 170         uses: actions/checkout@v3
 171         with:
 172           repository: openwrt/routing
 173           path: openwrt/feeds/routing
 174
 175       - name: Checkout telephony feed
 176         if: inputs.include_feeds == true
 177         uses: actions/checkout@v3
 178         with:
 179           repository: openwrt/telephony
 180           path: openwrt/feeds/telephony
 181
 182       - name: Fix permission
 183         run: |
 184           chown -R buildbot:buildbot openwrt
 185
 186       - name: Prepare prebuilt tools
 187         shell: su buildbot -c "sh -e {0}"
 188         working-directory: openwrt
 189         run: |
 190           mkdir -p staging_dir build_dir
 191           ln -s /prebuilt_tools/staging_dir/host staging_dir/host
 192           ln -s /prebuilt_tools/build_dir/host build_dir/host
 193
 194           ./scripts/ext-tools.sh --refresh
 195
 196       - name: Update & Install feeds
 197         if: inputs.include_feeds == true
 198         shell: su buildbot -c "sh -e {0}"
 199         working-directory: openwrt
 200         run: |
 201           ./scripts/feeds update -a
 202           ./scripts/feeds install -a
 203
 204       - name: Parse toolchain file
 205         if: inputs.build_toolchain == false
 206         id: parse-toolchain
 207         working-directory: openwrt
 208         run: |
 209           if [ -d /external-toolchain/ ]; then
 210             echo "toolchain-type=external_container" >> $GITHUB_OUTPUT
 211             exit 0
 212           fi
 213
 214           TOOLCHAIN_PATH=snapshots
 215
 216           if [ -n "${{ github.base_ref }}" ]; then
 217             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 218               major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
 219             fi
 220           elif [ "${{ github.ref_type }}" = "branch" ]; then
 221             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 222               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
 223             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 224               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
 225             fi
 226           elif [ "${{ github.ref_type }}" = "tag" ]; then
 227             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 228               major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 229             fi
 230           fi
 231
 232           if [ -n "$major_ver" ]; then
 233             git fetch --tags -f
 234             latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
 235             if [ -n "$latest_tag" ]; then
 236               TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
 237             fi
 238           fi
 239
 240           SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums"
 241           if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
 242             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
 243             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
 244
 245             echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
 246           elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
 247             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
 248             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
 249
 250             echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
 251           else
 252             echo "toolchain-type=internal" >> $GITHUB_OUTPUT
 253           fi
 254
 255           echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
 256           echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
 257
 258       - name: Cache ccache
 259         uses: actions/cache@v3
 260         with:
 261           path: openwrt/.ccache
 262           key: ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-${{ needs.setup_build.outputs.ccache_hash }}
 263           restore-keys: |
 264             ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-
 265
 266       - name: Download external toolchain/sdk
 267         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal' && steps.parse-toolchain.outputs.toolchain-type != 'external_container'
 268         shell: su buildbot -c "sh -e {0}"
 269         working-directory: openwrt
 270         run: |
 271           wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
 272             | tar --xz -xf -
 273
 274       - name: Configure testing kernel
 275         if: inputs.testing == true
 276         shell: su buildbot -c "sh -e {0}"
 277         working-directory: openwrt
 278         run: |
 279           echo CONFIG_TESTING_KERNEL=y >> .config
 280
 281       - name: Configure all kernel modules
 282         if: inputs.build_all_kmods == true
 283         shell: su buildbot -c "sh -e {0}"
 284         working-directory: openwrt
 285         run: |
 286           echo CONFIG_ALL_KMODS=y >> .config
 287
 288       - name: Configure all modules
 289         if: inputs.build_all_modules == true
 290         shell: su buildbot -c "sh -e {0}"
 291         working-directory: openwrt
 292         run: |
 293           echo CONFIG_ALL=y >> .config
 294
 295       - name: Configure all boards
 296         if: inputs.build_all_boards == true
 297         shell: su buildbot -c "sh -e {0}"
 298         working-directory: openwrt
 299         run: |
 300           echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
 301           echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
 302           echo CONFIG_TARGET_ALL_PROFILES=y >> .config
 303
 304       - name: Configure external toolchain in container
 305         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_container'
 306         shell: su buildbot -c "sh -e {0}"
 307         working-directory: openwrt
 308         run: |
 309           echo CONFIG_DEVEL=y >> .config
 310           echo CONFIG_AUTOREMOVE=y >> .config
 311           echo CONFIG_CCACHE=y >> .config
 312
 313           ./scripts/ext-toolchain.sh \
 314             --toolchain /external-toolchain/$(ls /external-toolchain/ | grep openwrt-toolchain)/toolchain-* \
 315             --overwrite-config \
 316             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 317
 318       - name: Configure external toolchain
 319         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
 320         shell: su buildbot -c "sh -e {0}"
 321         working-directory: openwrt
 322         run: |
 323           echo CONFIG_DEVEL=y >> .config
 324           echo CONFIG_AUTOREMOVE=y >> .config
 325           echo CONFIG_CCACHE=y >> .config
 326
 327           ./scripts/ext-toolchain.sh \
 328             --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
 329             --overwrite-config \
 330             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 331
 332       - name: Adapt external sdk to external toolchain format
 333         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 334         shell: su buildbot -c "sh -e {0}"
 335         working-directory: openwrt
 336         run: |
 337           TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
 338           TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
 339           OPENWRT_DIR=$(pwd)
 340
 341           # Find target name from toolchain info.mk
 342           GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
 343
 344           cd $TOOLCHAIN_BIN
 345
 346           # Revert sdk wrapper scripts applied to all the bins
 347           for app in $(find . -name "*.bin"); do
 348             TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
 349             rm $TARGET_APP
 350             mv .$TARGET_APP.bin $TARGET_APP
 351           done
 352
 353           # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
 354           cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
 355           for app in cc gcc g++ c++ cpp ld as ; do
 356             [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
 357             ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
 358           done
 359
 360       - name: Configure external toolchain with sdk
 361         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 362         shell: su buildbot -c "sh -e {0}"
 363         working-directory: openwrt
 364         run: |
 365           echo CONFIG_DEVEL=y >> .config
 366           echo CONFIG_AUTOREMOVE=y >> .config
 367           echo CONFIG_CCACHE=y >> .config
 368
 369           ./scripts/ext-toolchain.sh \
 370             --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
 371             --overwrite-config \
 372             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 373
 374       - name: Configure internal toolchain
 375         if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
 376         shell: su buildbot -c "sh -e {0}"
 377         working-directory: openwrt
 378         run: |
 379           echo CONFIG_DEVEL=y >> .config
 380           echo CONFIG_AUTOREMOVE=y >> .config
 381           echo CONFIG_CCACHE=y >> .config
 382
 383           echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config
 384           echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config
 385
 386           make defconfig
 387
 388       - name: Show configuration
 389         shell: su buildbot -c "sh -e {0}"
 390         working-directory: openwrt
 391         run: ./scripts/diffconfig.sh
 392
 393       - name: Build tools
 394         shell: su buildbot -c "sh -e {0}"
 395         working-directory: openwrt
 396         run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 397
 398       - name: Build toolchain
 399         shell: su buildbot -c "sh -e {0}"
 400         working-directory: openwrt
 401         run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 402
 403       - name: Build Kernel
 404         if: inputs.build_kernel == true
 405         shell: su buildbot -c "sh -e {0}"
 406         working-directory: openwrt
 407         run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 408
 409       - name: Build Kernel Kmods
 410         if: inputs.build_kernel == true
 411         shell: su buildbot -c "sh -e {0}"
 412         working-directory: openwrt
 413         run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 414
 415       - name: Build everything
 416         if: inputs.build_full == true
 417         shell: su buildbot -c "sh -e {0}"
 418         working-directory: openwrt
 419         run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 420
 421       - name: Build external toolchain
 422         if: inputs.build_external_toolchain == true
 423         shell: su buildbot -c "sh -e {0}"
 424         working-directory: openwrt
 425         run: make target/toolchain/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 426
 427       - name: Coverity prepare toolchain
 428         if: inputs.coverity_check_packages != ''
 429         shell: su buildbot -c "sh -e {0}"
 430         working-directory: openwrt
 431         run: |
 432           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz
 433           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5
 434           echo ' coverity.tar.gz' >> coverity.tar.gz.md5
 435           md5sum -c coverity.tar.gz.md5
 436
 437           mkdir cov-analysis-linux64
 438           tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64
 439           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 440
 441           for template in ${{ inputs.coverity_compiler_template_list }}; do
 442             cov-configure --template --comptype gcc --compiler "$template"
 443           done
 444
 445       - name: Clean and recompile packages with Coverity toolchain
 446         if: inputs.coverity_check_packages != ''
 447         shell: su buildbot -c "bash {0}"
 448         working-directory: openwrt
 449         run: |
 450           set -o pipefail -o errexit
 451
 452           coverity_check_packages=(${{ inputs.coverity_check_packages }})
 453           printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}"
 454           make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 455
 456           coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }})
 457           printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}"
 458           make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 459
 460           printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}"
 461           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 462           cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 463
 464       - name: Upload build to Coverity for analysis
 465         if: inputs.coverity_check_packages != ''
 466         shell: su buildbot -c "sh -e {0}"
 467         working-directory: openwrt
 468         run: |
 469           tar czf cov-int.tar.gz ./cov-int
 470           curl \
 471             --form token="${{ secrets.coverity_api_token }}" \
 472             --form email="contact@openwrt.org" \
 473             --form file=@cov-int.tar.gz \
 474             --form version="${{ github.ref_name }}-${{ github.sha }}" \
 475             --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \
 476             "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}"
 477
 478       - name: Upload logs
 479         if: failure()
 480         uses: actions/upload-artifact@v3
 481         with:
 482           name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs
 483           path: "openwrt/logs"
 484
 485       - name: Find external toolchain name
 486         id: get-toolchain-name
 487         if: inputs.upload_external_toolchain == true
 488         working-directory: openwrt
 489         run: |
 490           TOOLCHAIN_NAME=$(ls bin/targets/${{inputs.target }}/${{ inputs.subtarget }} | grep toolchain)
 491           echo "toolchain-name=$TOOLCHAIN_NAME" >> $GITHUB_OUTPUT
 492
 493       - name: Upload prebuilt toolchain
 494         if: inputs.upload_external_toolchain == true
 495         uses: actions/upload-artifact@v3
 496         with:
 497           name: ${{ inputs.target }}-${{ inputs.subtarget }}-external-toolchain
 498           path: openwrt/bin/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ steps.get-toolchain-name.outputs.toolchain-name }}
 499           retention-days: 1