.github/workflows/build.yml

   1 name: Build sub target
   2
   3 on:
   4   workflow_call:
   5     secrets:
   6       coverity_api_token:
   7     inputs:
   8       target:
   9         required: true
  10         type: string
  11       testing:
  12         type: boolean
  13       build_toolchain:
  14         type: boolean
  15       include_feeds:
  16         type: boolean
  17       build_full:
  18         type: boolean
  19       build_kernel:
  20         type: boolean
  21       build_all_modules:
  22         type: boolean
  23       build_all_kmods:
  24         type: boolean
  25       build_all_boards:
  26         type: boolean
  27       use_openwrt_container:
  28         type: boolean
  29         default: true
  30       coverity_project_name:
  31         type: string
  32         default: OpenWrt
  33       coverity_check_packages:
  34         type: string
  35       coverity_compiler_template_list:
  36         type: string
  37         default: >-
  38           arm-openwrt-linux-gcc
  39       coverity_force_compile_packages:
  40         type: string
  41         default: >-
  42           curl
  43           libnl
  44           mbedtls
  45           wolfssl
  46           openssl
  47
  48 permissions:
  49   contents: read
  50
  51 jobs:
  52   setup_build:
  53     name: Setup build ${{ inputs.target }}
  54     runs-on: ubuntu-latest
  55     outputs:
  56       owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
  57       ccache_hash: ${{ steps.ccache_hash.outputs.ccache_hash }}
  58       container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
  59
  60     steps:
  61       - name: Checkout
  62         uses: actions/checkout@v3
  63
  64       - name: Set lower case owner name
  65         id: lower_owner
  66         run: |
  67           OWNER_LC=$(echo "${{ github.repository_owner }}" \
  68             | tr '[:upper:]' '[:lower:]')
  69
  70           if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
  71             OWNER_LC=openwrt
  72           fi
  73
  74           echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
  75
  76       - name: Generate ccache hash
  77         id: ccache_hash
  78         run: |
  79           CCACHE_HASH=$(md5sum include/kernel-* | awk '{ print $1 }' \
  80            | md5sum | awk '{ print $1 }')
  81           echo "ccache_hash=$CCACHE_HASH" >> $GITHUB_OUTPUT
  82
  83       # Per branch tools container tag
  84       # By default stick to latest
  85       # For official test targetting openwrt stable branch
  86       # Get the branch or parse the tag and push dedicated tools containers
  87       # For local test to use the correct container for stable release testing
  88       # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
  89       - name: Determine tools container tag
  90         id: determine_tools_container
  91         run: |
  92           CONTAINER_TAG=latest
  93           if [ -n "${{ github.base_ref }}" ]; then
  94             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
  95               CONTAINER_TAG="${{ github.base_ref }}"
  96             fi
  97           elif [ ${{ github.ref_type }} == "branch" ]; then
  98             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
  99               CONTAINER_TAG=${{ github.ref_name }}
 100             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 101               CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
 102             fi
 103           elif [ ${{ github.ref_type }} == "tag" ]; then
 104             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 105               CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 106             fi
 107           fi
 108           echo "Tools container to use tools:$CONTAINER_TAG"
 109           echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
 110
 111   build:
 112     name: Build ${{ inputs.target }}
 113     needs: setup_build
 114     runs-on: ubuntu-latest
 115
 116     container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/tools:${{ needs.setup_build.outputs.container_tag }}
 117
 118     permissions:
 119       contents: read
 120       packages: read
 121
 122     steps:
 123       - name: Checkout master directory
 124         uses: actions/checkout@v3
 125         with:
 126           path: openwrt
 127
 128       - name: Checkout packages feed
 129         if: inputs.include_feeds == true
 130         uses: actions/checkout@v3
 131         with:
 132           repository: openwrt/packages
 133           path: openwrt/feeds/packages
 134
 135       - name: Checkout luci feed
 136         if: inputs.include_feeds == true
 137         uses: actions/checkout@v3
 138         with:
 139           repository: openwrt/luci
 140           path: openwrt/feeds/luci
 141
 142       - name: Checkout routing feed
 143         if: inputs.include_feeds == true
 144         uses: actions/checkout@v3
 145         with:
 146           repository: openwrt/routing
 147           path: openwrt/feeds/routing
 148
 149       - name: Checkout telephony feed
 150         if: inputs.include_feeds == true
 151         uses: actions/checkout@v3
 152         with:
 153           repository: openwrt/telephony
 154           path: openwrt/feeds/telephony
 155
 156       - name: Fix permission
 157         run: |
 158           chown -R buildbot:buildbot openwrt
 159
 160       - name: Initialization environment
 161         run: |
 162           TARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 1)
 163           SUBTARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 2)
 164           echo "TARGET=$TARGET" >> "$GITHUB_ENV"
 165           echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV"
 166
 167       - name: Prepare prebuilt tools
 168         shell: su buildbot -c "sh -e {0}"
 169         working-directory: openwrt
 170         run: |
 171           mkdir -p staging_dir build_dir
 172           ln -s /prebuilt_tools/staging_dir/host staging_dir/host
 173           ln -s /prebuilt_tools/build_dir/host build_dir/host
 174
 175           ./scripts/ext-tools.sh --refresh
 176
 177       - name: Update & Install feeds
 178         if: inputs.include_feeds == true
 179         shell: su buildbot -c "sh -e {0}"
 180         working-directory: openwrt
 181         run: |
 182           ./scripts/feeds update -a
 183           ./scripts/feeds install -a
 184
 185       - name: Parse toolchain file
 186         if: inputs.build_toolchain == false
 187         id: parse-toolchain
 188         working-directory: openwrt
 189         run: |
 190           TOOLCHAIN_PATH=snapshots
 191
 192           if [ -n "${{ github.base_ref }}" ]; then
 193             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 194               major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
 195             fi
 196           elif [ "${{ github.ref_type }}" = "branch" ]; then
 197             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 198               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
 199             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 200               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
 201             fi
 202           elif [ "${{ github.ref_type }}" = "tag" ]; then
 203             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 204               major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 205             fi
 206           fi
 207
 208           if [ -n "$major_ver" ]; then
 209             git fetch --tags -f
 210             latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
 211             if [ -n "$latest_tag" ]; then
 212               TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
 213             fi
 214           fi
 215
 216           SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/sha256sums"
 217           if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
 218             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
 219             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
 220
 221             echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
 222           elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
 223             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
 224             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
 225
 226             echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
 227           else
 228             echo "toolchain-type=internal" >> $GITHUB_OUTPUT
 229           fi
 230
 231           echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
 232           echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
 233
 234       - name: Cache ccache
 235         uses: actions/cache@v3
 236         with:
 237           path: openwrt/.ccache
 238           key: ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-${{ needs.setup_build.outputs.ccache_hash }}
 239           restore-keys: |
 240             ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-
 241
 242       - name: Download external toolchain/sdk
 243         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal'
 244         shell: su buildbot -c "sh -e {0}"
 245         working-directory: openwrt
 246         run: |
 247           wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
 248             | tar --xz -xf -
 249
 250       - name: Configure testing kernel
 251         if: inputs.testing == true
 252         shell: su buildbot -c "sh -e {0}"
 253         working-directory: openwrt
 254         run: |
 255           echo CONFIG_TESTING_KERNEL=y >> .config
 256
 257       - name: Configure all kernel modules
 258         if: inputs.build_all_kmods == true
 259         shell: su buildbot -c "sh -e {0}"
 260         working-directory: openwrt
 261         run: |
 262           echo CONFIG_ALL_KMODS=y >> .config
 263
 264       - name: Configure all modules
 265         if: inputs.build_all_modules == true
 266         shell: su buildbot -c "sh -e {0}"
 267         working-directory: openwrt
 268         run: |
 269           echo CONFIG_ALL=y >> .config
 270
 271       - name: Configure all boards
 272         if: inputs.build_all_boards == true
 273         shell: su buildbot -c "sh -e {0}"
 274         working-directory: openwrt
 275         run: |
 276           echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
 277           echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
 278           echo CONFIG_TARGET_ALL_PROFILES=y >> .config
 279
 280       - name: Configure external toolchain
 281         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
 282         shell: su buildbot -c "sh -e {0}"
 283         working-directory: openwrt
 284         run: |
 285           echo CONFIG_DEVEL=y >> .config
 286           echo CONFIG_AUTOREMOVE=y >> .config
 287           echo CONFIG_CCACHE=y >> .config
 288
 289           ./scripts/ext-toolchain.sh \
 290             --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
 291             --overwrite-config \
 292             --config ${{ env.TARGET }}/${{ env.SUBTARGET }}
 293
 294       - name: Adapt external sdk to external toolchain format
 295         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 296         shell: su buildbot -c "sh -e {0}"
 297         working-directory: openwrt
 298         run: |
 299           TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
 300           TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
 301           OPENWRT_DIR=$(pwd)
 302
 303           # Find target name from toolchain info.mk
 304           GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
 305
 306           cd $TOOLCHAIN_BIN
 307
 308           # Revert sdk wrapper scripts applied to all the bins
 309           for app in $(find . -name "*.bin"); do
 310             TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
 311             rm $TARGET_APP
 312             mv .$TARGET_APP.bin $TARGET_APP
 313           done
 314
 315           # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
 316           cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
 317           for app in cc gcc g++ c++ cpp ld as ; do
 318             [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
 319             ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
 320           done
 321
 322       - name: Configure external toolchain with sdk
 323         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 324         shell: su buildbot -c "sh -e {0}"
 325         working-directory: openwrt
 326         run: |
 327           echo CONFIG_DEVEL=y >> .config
 328           echo CONFIG_AUTOREMOVE=y >> .config
 329           echo CONFIG_CCACHE=y >> .config
 330
 331           ./scripts/ext-toolchain.sh \
 332             --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
 333             --overwrite-config \
 334             --config ${{ env.TARGET }}/${{ env.SUBTARGET }}
 335
 336       - name: Configure internal toolchain
 337         if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
 338         shell: su buildbot -c "sh -e {0}"
 339         working-directory: openwrt
 340         run: |
 341           echo CONFIG_DEVEL=y >> .config
 342           echo CONFIG_AUTOREMOVE=y >> .config
 343           echo CONFIG_CCACHE=y >> .config
 344
 345           echo "CONFIG_TARGET_${{ env.TARGET }}=y" >> .config
 346           echo "CONFIG_TARGET_${{ env.TARGET }}_${{ env.SUBTARGET }}=y" >> .config
 347
 348           make defconfig
 349
 350       - name: Show configuration
 351         shell: su buildbot -c "sh -e {0}"
 352         working-directory: openwrt
 353         run: ./scripts/diffconfig.sh
 354
 355       - name: Build tools
 356         shell: su buildbot -c "sh -e {0}"
 357         working-directory: openwrt
 358         run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 359
 360       - name: Build toolchain
 361         shell: su buildbot -c "sh -e {0}"
 362         working-directory: openwrt
 363         run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 364
 365       - name: Build Kernel
 366         if: inputs.build_kernel == true
 367         shell: su buildbot -c "sh -e {0}"
 368         working-directory: openwrt
 369         run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 370
 371       - name: Build Kernel Kmods
 372         if: inputs.build_kernel == true
 373         shell: su buildbot -c "sh -e {0}"
 374         working-directory: openwrt
 375         run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 376
 377       - name: Build everything
 378         if: inputs.build_full == true
 379         shell: su buildbot -c "sh -e {0}"
 380         working-directory: openwrt
 381         run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 382
 383       - name: Coverity prepare toolchain
 384         if: inputs.coverity_check_packages != ''
 385         shell: su buildbot -c "sh -e {0}"
 386         working-directory: openwrt
 387         run: |
 388           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz
 389           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5
 390           echo ' coverity.tar.gz' >> coverity.tar.gz.md5
 391           md5sum -c coverity.tar.gz.md5
 392
 393           mkdir cov-analysis-linux64
 394           tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64
 395           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 396
 397           for template in ${{ inputs.coverity_compiler_template_list }}; do
 398             cov-configure --template --comptype gcc --compiler "$template"
 399           done
 400
 401       - name: Clean and recompile packages with Coverity toolchain
 402         if: inputs.coverity_check_packages != ''
 403         shell: su buildbot -c "bash {0}"
 404         working-directory: openwrt
 405         run: |
 406           set -o pipefail -o errexit
 407
 408           coverity_check_packages=(${{ inputs.coverity_check_packages }})
 409           printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}"
 410           make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 411
 412           coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }})
 413           printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}"
 414           make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 415
 416           printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}"
 417           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 418           cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 419
 420       - name: Upload build to Coverity for analysis
 421         if: inputs.coverity_check_packages != ''
 422         shell: su buildbot -c "sh -e {0}"
 423         working-directory: openwrt
 424         run: |
 425           tar czf cov-int.tar.gz ./cov-int
 426           curl \
 427             --form token="${{ secrets.coverity_api_token }}" \
 428             --form email="contact@openwrt.org" \
 429             --form file=@cov-int.tar.gz \
 430             --form version="${{ github.ref_name }}-${{ github.sha }}" \
 431             --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \
 432             "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}"
 433
 434       - name: Upload logs
 435         if: failure()
 436         uses: actions/upload-artifact@v3
 437         with:
 438           name: ${{ env.TARGET }}-${{ env.SUBTARGET }}-logs
 439           path: "openwrt/logs"