.github/workflows/build.yml

   1 name: Build sub target
   2
   3 on:
   4   workflow_call:
   5     secrets:
   6       coverity_api_token:
   7     inputs:
   8       container_name:
   9         type: string
  10         default: tools
  11       target:
  12         required: true
  13         type: string
  14       subtarget:
  15         required: true
  16         type: string
  17       testing:
  18         type: boolean
  19       build_toolchain:
  20         type: boolean
  21       include_feeds:
  22         type: boolean
  23       build_full:
  24         type: boolean
  25       build_kernel:
  26         type: boolean
  27       build_all_modules:
  28         type: boolean
  29       build_all_kmods:
  30         type: boolean
  31       build_all_boards:
  32         type: boolean
  33       use_openwrt_container:
  34         type: boolean
  35         default: true
  36       coverity_project_name:
  37         type: string
  38         default: OpenWrt
  39       coverity_check_packages:
  40         type: string
  41       coverity_compiler_template_list:
  42         type: string
  43         default: >-
  44           arm-openwrt-linux-gcc
  45       coverity_force_compile_packages:
  46         type: string
  47         default: >-
  48           curl
  49           libnl
  50           mbedtls
  51           wolfssl
  52           openssl
  53       build_external_toolchain:
  54         type: boolean
  55       upload_external_toolchain:
  56         type: boolean
  57
  58 permissions:
  59   contents: read
  60
  61 jobs:
  62   setup_build:
  63     name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }}
  64     runs-on: ubuntu-latest
  65     outputs:
  66       owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
  67       container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
  68       container_name: ${{ steps.determine_tools_container.outputs.container_name }}
  69
  70     steps:
  71       - name: Checkout
  72         uses: actions/checkout@v3
  73
  74       - name: Set lower case owner name
  75         id: lower_owner
  76         run: |
  77           OWNER_LC=$(echo "${{ github.repository_owner }}" \
  78             | tr '[:upper:]' '[:lower:]')
  79
  80           if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
  81             OWNER_LC=openwrt
  82           fi
  83
  84           echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
  85
  86       # Per branch tools container tag
  87       # By default stick to latest
  88       # For official test targetting openwrt stable branch
  89       # Get the branch or parse the tag and push dedicated tools containers
  90       # For local test to use the correct container for stable release testing
  91       # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
  92       - name: Determine tools container tag
  93         id: determine_tools_container
  94         run: |
  95           CONTAINER_NAME=${{ inputs.container_name }}
  96           CONTAINER_TAG=latest
  97           if [ -n "${{ github.base_ref }}" ]; then
  98             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
  99               CONTAINER_TAG="${{ github.base_ref }}"
 100             fi
 101           elif [ ${{ github.ref_type }} == "branch" ]; then
 102             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 103               CONTAINER_TAG=${{ github.ref_name }}
 104             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 105               CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
 106             fi
 107           elif [ ${{ github.ref_type }} == "tag" ]; then
 108             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 109               CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 110             fi
 111           fi
 112
 113           if [ "$CONTAINER_NAME" = "toolchain" ]; then
 114             GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64)
 115             GHCR_HEADER="Authorization: Bearer ${GHCR_TOKEN}"
 116             GHCR_MANIFEST_LINK=https://ghcr.io/v2/${{ steps.lower_owner.outputs.owner_lc }}/${{ inputs.container_name }}/manifests/${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
 117             # Check if container exist
 118             if [ $(curl -s -o /dev/null -w "%{http_code}" -H "$GHCR_HEADER" -I "$GHCR_MANIFEST_LINK") = 200 ]; then
 119               CONTAINER_TAG=${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
 120             else
 121               CONTAINER_NAME=tools
 122             fi
 123           fi
 124
 125           echo "Tools container to use $CONTAINER_NAME:$CONTAINER_TAG"
 126           echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
 127           echo "container_name=$CONTAINER_NAME" >> $GITHUB_OUTPUT
 128
 129   build:
 130     name: Build ${{ inputs.target }}/${{ inputs.subtarget }}
 131     needs: setup_build
 132     runs-on: ubuntu-latest
 133
 134     container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/${{ needs.setup_build.outputs.container_name }}:${{ needs.setup_build.outputs.container_tag }}
 135
 136     permissions:
 137       contents: read
 138       packages: read
 139
 140     steps:
 141       - name: Checkout master directory
 142         uses: actions/checkout@v3
 143         with:
 144           path: openwrt
 145
 146       - name: Checkout packages feed
 147         if: inputs.include_feeds == true
 148         uses: actions/checkout@v3
 149         with:
 150           repository: openwrt/packages
 151           path: openwrt/feeds/packages
 152
 153       - name: Checkout luci feed
 154         if: inputs.include_feeds == true
 155         uses: actions/checkout@v3
 156         with:
 157           repository: openwrt/luci
 158           path: openwrt/feeds/luci
 159
 160       - name: Checkout routing feed
 161         if: inputs.include_feeds == true
 162         uses: actions/checkout@v3
 163         with:
 164           repository: openwrt/routing
 165           path: openwrt/feeds/routing
 166
 167       - name: Checkout telephony feed
 168         if: inputs.include_feeds == true
 169         uses: actions/checkout@v3
 170         with:
 171           repository: openwrt/telephony
 172           path: openwrt/feeds/telephony
 173
 174       - name: Parse toolchain file
 175         if: inputs.build_toolchain == false
 176         id: parse-toolchain
 177         working-directory: openwrt
 178         run: |
 179           if [ -d /external-toolchain/ ]; then
 180             echo "toolchain-type=external_container" >> $GITHUB_OUTPUT
 181             exit 0
 182           fi
 183
 184           TOOLCHAIN_PATH=snapshots
 185
 186           if [ -n "${{ github.base_ref }}" ]; then
 187             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 188               major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
 189             fi
 190           elif [ "${{ github.ref_type }}" = "branch" ]; then
 191             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 192               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
 193             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 194               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
 195             fi
 196           elif [ "${{ github.ref_type }}" = "tag" ]; then
 197             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 198               major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 199             fi
 200           fi
 201
 202           if [ -n "$major_ver" ]; then
 203             git fetch --tags -f
 204             latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
 205             if [ -n "$latest_tag" ]; then
 206               TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
 207             fi
 208           fi
 209
 210           SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums"
 211           if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
 212             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
 213             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
 214
 215             echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
 216           elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
 217             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
 218             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
 219
 220             echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
 221           else
 222             echo "toolchain-type=internal" >> $GITHUB_OUTPUT
 223           fi
 224
 225           echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
 226           echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
 227
 228       - name: Fix permission
 229         run: |
 230           chown -R buildbot:buildbot openwrt
 231
 232       - name: Prepare prebuilt tools
 233         shell: su buildbot -c "sh -e {0}"
 234         working-directory: openwrt
 235         run: |
 236           mkdir -p staging_dir build_dir
 237           ln -s /prebuilt_tools/staging_dir/host staging_dir/host
 238           ln -s /prebuilt_tools/build_dir/host build_dir/host
 239
 240           ./scripts/ext-tools.sh --refresh
 241
 242       - name: Update & Install feeds
 243         if: inputs.include_feeds == true
 244         shell: su buildbot -c "sh -e {0}"
 245         working-directory: openwrt
 246         run: |
 247           ./scripts/feeds update -a
 248           ./scripts/feeds install -a
 249
 250       - name: Restore ccache cache
 251         id: restore-ccache-cache
 252         uses: actions/cache/restore@v3
 253         with:
 254           path: openwrt/.ccache
 255           key: ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-${{ hashFiles('openwrt/include/kernel-**') }}
 256           restore-keys: |
 257             ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-
 258
 259       - name: Download external toolchain/sdk
 260         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal' && steps.parse-toolchain.outputs.toolchain-type != 'external_container'
 261         shell: su buildbot -c "sh -e {0}"
 262         working-directory: openwrt
 263         run: |
 264           wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
 265             | tar --xz -xf -
 266
 267       - name: Configure testing kernel
 268         if: inputs.testing == true
 269         shell: su buildbot -c "sh -e {0}"
 270         working-directory: openwrt
 271         run: |
 272           echo CONFIG_TESTING_KERNEL=y >> .config
 273
 274       - name: Configure all kernel modules
 275         if: inputs.build_all_kmods == true
 276         shell: su buildbot -c "sh -e {0}"
 277         working-directory: openwrt
 278         run: |
 279           echo CONFIG_ALL_KMODS=y >> .config
 280
 281       - name: Configure all modules
 282         if: inputs.build_all_modules == true
 283         shell: su buildbot -c "sh -e {0}"
 284         working-directory: openwrt
 285         run: |
 286           echo CONFIG_ALL=y >> .config
 287
 288       - name: Configure all boards
 289         if: inputs.build_all_boards == true
 290         shell: su buildbot -c "sh -e {0}"
 291         working-directory: openwrt
 292         run: |
 293           echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
 294           echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
 295           echo CONFIG_TARGET_ALL_PROFILES=y >> .config
 296
 297       - name: Configure external toolchain in container
 298         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_container'
 299         shell: su buildbot -c "sh -e {0}"
 300         working-directory: openwrt
 301         run: |
 302           echo CONFIG_DEVEL=y >> .config
 303           echo CONFIG_AUTOREMOVE=y >> .config
 304           echo CONFIG_CCACHE=y >> .config
 305
 306           ./scripts/ext-toolchain.sh \
 307             --toolchain /external-toolchain/$(ls /external-toolchain/ | grep openwrt-toolchain)/toolchain-* \
 308             --overwrite-config \
 309             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 310
 311       - name: Configure external toolchain
 312         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
 313         shell: su buildbot -c "sh -e {0}"
 314         working-directory: openwrt
 315         run: |
 316           echo CONFIG_DEVEL=y >> .config
 317           echo CONFIG_AUTOREMOVE=y >> .config
 318           echo CONFIG_CCACHE=y >> .config
 319
 320           ./scripts/ext-toolchain.sh \
 321             --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
 322             --overwrite-config \
 323             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 324
 325       - name: Adapt external sdk to external toolchain format
 326         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 327         shell: su buildbot -c "sh -e {0}"
 328         working-directory: openwrt
 329         run: |
 330           TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
 331           TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
 332           OPENWRT_DIR=$(pwd)
 333
 334           # Find target name from toolchain info.mk
 335           GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
 336
 337           cd $TOOLCHAIN_BIN
 338
 339           # Revert sdk wrapper scripts applied to all the bins
 340           for app in $(find . -name "*.bin"); do
 341             TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
 342             rm $TARGET_APP
 343             mv .$TARGET_APP.bin $TARGET_APP
 344           done
 345
 346           # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
 347           cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
 348           for app in cc gcc g++ c++ cpp ld as ; do
 349             [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
 350             ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
 351           done
 352
 353       - name: Configure external toolchain with sdk
 354         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 355         shell: su buildbot -c "sh -e {0}"
 356         working-directory: openwrt
 357         run: |
 358           echo CONFIG_DEVEL=y >> .config
 359           echo CONFIG_AUTOREMOVE=y >> .config
 360           echo CONFIG_CCACHE=y >> .config
 361
 362           ./scripts/ext-toolchain.sh \
 363             --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
 364             --overwrite-config \
 365             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 366
 367       - name: Configure internal toolchain
 368         if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
 369         shell: su buildbot -c "sh -e {0}"
 370         working-directory: openwrt
 371         run: |
 372           echo CONFIG_DEVEL=y >> .config
 373           echo CONFIG_AUTOREMOVE=y >> .config
 374           echo CONFIG_CCACHE=y >> .config
 375
 376           echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config
 377           echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config
 378
 379           make defconfig
 380
 381       - name: Show configuration
 382         shell: su buildbot -c "sh -e {0}"
 383         working-directory: openwrt
 384         run: ./scripts/diffconfig.sh
 385
 386       - name: Build tools
 387         shell: su buildbot -c "sh -e {0}"
 388         working-directory: openwrt
 389         run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 390
 391       - name: Build toolchain
 392         shell: su buildbot -c "sh -e {0}"
 393         working-directory: openwrt
 394         run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 395
 396       - name: Build Kernel
 397         if: inputs.build_kernel == true
 398         shell: su buildbot -c "sh -e {0}"
 399         working-directory: openwrt
 400         run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 401
 402       - name: Build Kernel Kmods
 403         if: inputs.build_kernel == true
 404         shell: su buildbot -c "sh -e {0}"
 405         working-directory: openwrt
 406         run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 407
 408       - name: Build everything
 409         if: inputs.build_full == true
 410         shell: su buildbot -c "sh -e {0}"
 411         working-directory: openwrt
 412         run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 413
 414       - name: Build external toolchain
 415         if: inputs.build_external_toolchain == true
 416         shell: su buildbot -c "sh -e {0}"
 417         working-directory: openwrt
 418         run: make target/toolchain/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 419
 420       - name: Coverity prepare toolchain
 421         if: inputs.coverity_check_packages != ''
 422         shell: su buildbot -c "sh -e {0}"
 423         working-directory: openwrt
 424         run: |
 425           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz
 426           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5
 427           echo ' coverity.tar.gz' >> coverity.tar.gz.md5
 428           md5sum -c coverity.tar.gz.md5
 429
 430           mkdir cov-analysis-linux64
 431           tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64
 432           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 433
 434           for template in ${{ inputs.coverity_compiler_template_list }}; do
 435             cov-configure --template --comptype gcc --compiler "$template"
 436           done
 437
 438       - name: Clean and recompile packages with Coverity toolchain
 439         if: inputs.coverity_check_packages != ''
 440         shell: su buildbot -c "bash {0}"
 441         working-directory: openwrt
 442         run: |
 443           set -o pipefail -o errexit
 444
 445           coverity_check_packages=(${{ inputs.coverity_check_packages }})
 446           printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}"
 447           make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 448
 449           coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }})
 450           printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}"
 451           make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 452
 453           printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}"
 454           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 455           cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 456
 457       - name: Upload build to Coverity for analysis
 458         if: inputs.coverity_check_packages != ''
 459         shell: su buildbot -c "sh -e {0}"
 460         working-directory: openwrt
 461         run: |
 462           tar czf cov-int.tar.gz ./cov-int
 463           curl \
 464             --form token="${{ secrets.coverity_api_token }}" \
 465             --form email="contact@openwrt.org" \
 466             --form file=@cov-int.tar.gz \
 467             --form version="${{ github.ref_name }}-${{ github.sha }}" \
 468             --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \
 469             "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}"
 470
 471       - name: Upload logs
 472         if: failure()
 473         uses: actions/upload-artifact@v3
 474         with:
 475           name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs
 476           path: "openwrt/logs"
 477
 478       - name: Save ccache cache
 479         uses: actions/cache/save@v3
 480         with:
 481           path: openwrt/.ccache
 482           key: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
 483
 484       - name: Find external toolchain name
 485         id: get-toolchain-name
 486         if: inputs.upload_external_toolchain == true
 487         working-directory: openwrt
 488         run: |
 489           TOOLCHAIN_NAME=$(ls bin/targets/${{inputs.target }}/${{ inputs.subtarget }} | grep toolchain)
 490           echo "toolchain-name=$TOOLCHAIN_NAME" >> $GITHUB_OUTPUT
 491
 492       - name: Upload prebuilt toolchain
 493         if: inputs.upload_external_toolchain == true
 494         uses: actions/upload-artifact@v3
 495         with:
 496           name: ${{ inputs.target }}-${{ inputs.subtarget }}-external-toolchain
 497           path: openwrt/bin/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ steps.get-toolchain-name.outputs.toolchain-name }}
 498           retention-days: 1