.github/workflows/build.yml

   1 name: Build sub target
   2
   3 on:
   4   workflow_call:
   5     secrets:
   6       coverity_api_token:
   7     inputs:
   8       target:
   9         required: true
  10         type: string
  11       subtarget:
  12         required: true
  13         type: string
  14       testing:
  15         type: boolean
  16       build_toolchain:
  17         type: boolean
  18       include_feeds:
  19         type: boolean
  20       build_full:
  21         type: boolean
  22       build_kernel:
  23         type: boolean
  24       build_all_modules:
  25         type: boolean
  26       build_all_kmods:
  27         type: boolean
  28       build_all_boards:
  29         type: boolean
  30       use_openwrt_container:
  31         type: boolean
  32         default: true
  33       coverity_project_name:
  34         type: string
  35         default: OpenWrt
  36       coverity_check_packages:
  37         type: string
  38       coverity_compiler_template_list:
  39         type: string
  40         default: >-
  41           arm-openwrt-linux-gcc
  42       coverity_force_compile_packages:
  43         type: string
  44         default: >-
  45           curl
  46           libnl
  47           mbedtls
  48           wolfssl
  49           openssl
  50
  51 permissions:
  52   contents: read
  53
  54 jobs:
  55   setup_build:
  56     name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }}
  57     runs-on: ubuntu-latest
  58     outputs:
  59       owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
  60       ccache_hash: ${{ steps.ccache_hash.outputs.ccache_hash }}
  61       container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
  62
  63     steps:
  64       - name: Checkout
  65         uses: actions/checkout@v3
  66
  67       - name: Set lower case owner name
  68         id: lower_owner
  69         run: |
  70           OWNER_LC=$(echo "${{ github.repository_owner }}" \
  71             | tr '[:upper:]' '[:lower:]')
  72
  73           if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
  74             OWNER_LC=openwrt
  75           fi
  76
  77           echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
  78
  79       - name: Generate ccache hash
  80         id: ccache_hash
  81         run: |
  82           CCACHE_HASH=$(md5sum include/kernel-* | awk '{ print $1 }' \
  83            | md5sum | awk '{ print $1 }')
  84           echo "ccache_hash=$CCACHE_HASH" >> $GITHUB_OUTPUT
  85
  86       # Per branch tools container tag
  87       # By default stick to latest
  88       # For official test targetting openwrt stable branch
  89       # Get the branch or parse the tag and push dedicated tools containers
  90       # For local test to use the correct container for stable release testing
  91       # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
  92       - name: Determine tools container tag
  93         id: determine_tools_container
  94         run: |
  95           CONTAINER_TAG=latest
  96           if [ -n "${{ github.base_ref }}" ]; then
  97             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
  98               CONTAINER_TAG="${{ github.base_ref }}"
  99             fi
 100           elif [ ${{ github.ref_type }} == "branch" ]; then
 101             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 102               CONTAINER_TAG=${{ github.ref_name }}
 103             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 104               CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
 105             fi
 106           elif [ ${{ github.ref_type }} == "tag" ]; then
 107             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 108               CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 109             fi
 110           fi
 111           echo "Tools container to use tools:$CONTAINER_TAG"
 112           echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
 113
 114   build:
 115     name: Build ${{ inputs.target }}/${{ inputs.subtarget }}
 116     needs: setup_build
 117     runs-on: ubuntu-latest
 118
 119     container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/tools:${{ needs.setup_build.outputs.container_tag }}
 120
 121     permissions:
 122       contents: read
 123       packages: read
 124
 125     steps:
 126       - name: Checkout master directory
 127         uses: actions/checkout@v3
 128         with:
 129           path: openwrt
 130
 131       - name: Checkout packages feed
 132         if: inputs.include_feeds == true
 133         uses: actions/checkout@v3
 134         with:
 135           repository: openwrt/packages
 136           path: openwrt/feeds/packages
 137
 138       - name: Checkout luci feed
 139         if: inputs.include_feeds == true
 140         uses: actions/checkout@v3
 141         with:
 142           repository: openwrt/luci
 143           path: openwrt/feeds/luci
 144
 145       - name: Checkout routing feed
 146         if: inputs.include_feeds == true
 147         uses: actions/checkout@v3
 148         with:
 149           repository: openwrt/routing
 150           path: openwrt/feeds/routing
 151
 152       - name: Checkout telephony feed
 153         if: inputs.include_feeds == true
 154         uses: actions/checkout@v3
 155         with:
 156           repository: openwrt/telephony
 157           path: openwrt/feeds/telephony
 158
 159       - name: Fix permission
 160         run: |
 161           chown -R buildbot:buildbot openwrt
 162
 163       - name: Prepare prebuilt tools
 164         shell: su buildbot -c "sh -e {0}"
 165         working-directory: openwrt
 166         run: |
 167           mkdir -p staging_dir build_dir
 168           ln -s /prebuilt_tools/staging_dir/host staging_dir/host
 169           ln -s /prebuilt_tools/build_dir/host build_dir/host
 170
 171           ./scripts/ext-tools.sh --refresh
 172
 173       - name: Update & Install feeds
 174         if: inputs.include_feeds == true
 175         shell: su buildbot -c "sh -e {0}"
 176         working-directory: openwrt
 177         run: |
 178           ./scripts/feeds update -a
 179           ./scripts/feeds install -a
 180
 181       - name: Parse toolchain file
 182         if: inputs.build_toolchain == false
 183         id: parse-toolchain
 184         working-directory: openwrt
 185         run: |
 186           TOOLCHAIN_PATH=snapshots
 187
 188           if [ -n "${{ github.base_ref }}" ]; then
 189             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 190               major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
 191             fi
 192           elif [ "${{ github.ref_type }}" = "branch" ]; then
 193             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 194               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
 195             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 196               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
 197             fi
 198           elif [ "${{ github.ref_type }}" = "tag" ]; then
 199             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 200               major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 201             fi
 202           fi
 203
 204           if [ -n "$major_ver" ]; then
 205             git fetch --tags -f
 206             latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
 207             if [ -n "$latest_tag" ]; then
 208               TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
 209             fi
 210           fi
 211
 212           SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums"
 213           if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
 214             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
 215             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
 216
 217             echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
 218           elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
 219             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
 220             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
 221
 222             echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
 223           else
 224             echo "toolchain-type=internal" >> $GITHUB_OUTPUT
 225           fi
 226
 227           echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
 228           echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
 229
 230       - name: Cache ccache
 231         uses: actions/cache@v3
 232         with:
 233           path: openwrt/.ccache
 234           key: ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-${{ needs.setup_build.outputs.ccache_hash }}
 235           restore-keys: |
 236             ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-
 237
 238       - name: Download external toolchain/sdk
 239         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal'
 240         shell: su buildbot -c "sh -e {0}"
 241         working-directory: openwrt
 242         run: |
 243           wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
 244             | tar --xz -xf -
 245
 246       - name: Configure testing kernel
 247         if: inputs.testing == true
 248         shell: su buildbot -c "sh -e {0}"
 249         working-directory: openwrt
 250         run: |
 251           echo CONFIG_TESTING_KERNEL=y >> .config
 252
 253       - name: Configure all kernel modules
 254         if: inputs.build_all_kmods == true
 255         shell: su buildbot -c "sh -e {0}"
 256         working-directory: openwrt
 257         run: |
 258           echo CONFIG_ALL_KMODS=y >> .config
 259
 260       - name: Configure all modules
 261         if: inputs.build_all_modules == true
 262         shell: su buildbot -c "sh -e {0}"
 263         working-directory: openwrt
 264         run: |
 265           echo CONFIG_ALL=y >> .config
 266
 267       - name: Configure all boards
 268         if: inputs.build_all_boards == true
 269         shell: su buildbot -c "sh -e {0}"
 270         working-directory: openwrt
 271         run: |
 272           echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
 273           echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
 274           echo CONFIG_TARGET_ALL_PROFILES=y >> .config
 275
 276       - name: Configure external toolchain
 277         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
 278         shell: su buildbot -c "sh -e {0}"
 279         working-directory: openwrt
 280         run: |
 281           echo CONFIG_DEVEL=y >> .config
 282           echo CONFIG_AUTOREMOVE=y >> .config
 283           echo CONFIG_CCACHE=y >> .config
 284
 285           ./scripts/ext-toolchain.sh \
 286             --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
 287             --overwrite-config \
 288             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 289
 290       - name: Adapt external sdk to external toolchain format
 291         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 292         shell: su buildbot -c "sh -e {0}"
 293         working-directory: openwrt
 294         run: |
 295           TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
 296           TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
 297           OPENWRT_DIR=$(pwd)
 298
 299           # Find target name from toolchain info.mk
 300           GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
 301
 302           cd $TOOLCHAIN_BIN
 303
 304           # Revert sdk wrapper scripts applied to all the bins
 305           for app in $(find . -name "*.bin"); do
 306             TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
 307             rm $TARGET_APP
 308             mv .$TARGET_APP.bin $TARGET_APP
 309           done
 310
 311           # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
 312           cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
 313           for app in cc gcc g++ c++ cpp ld as ; do
 314             [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
 315             ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
 316           done
 317
 318       - name: Configure external toolchain with sdk
 319         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 320         shell: su buildbot -c "sh -e {0}"
 321         working-directory: openwrt
 322         run: |
 323           echo CONFIG_DEVEL=y >> .config
 324           echo CONFIG_AUTOREMOVE=y >> .config
 325           echo CONFIG_CCACHE=y >> .config
 326
 327           ./scripts/ext-toolchain.sh \
 328             --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
 329             --overwrite-config \
 330             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 331
 332       - name: Configure internal toolchain
 333         if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
 334         shell: su buildbot -c "sh -e {0}"
 335         working-directory: openwrt
 336         run: |
 337           echo CONFIG_DEVEL=y >> .config
 338           echo CONFIG_AUTOREMOVE=y >> .config
 339           echo CONFIG_CCACHE=y >> .config
 340
 341           echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config
 342           echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config
 343
 344           make defconfig
 345
 346       - name: Show configuration
 347         shell: su buildbot -c "sh -e {0}"
 348         working-directory: openwrt
 349         run: ./scripts/diffconfig.sh
 350
 351       - name: Build tools
 352         shell: su buildbot -c "sh -e {0}"
 353         working-directory: openwrt
 354         run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 355
 356       - name: Build toolchain
 357         shell: su buildbot -c "sh -e {0}"
 358         working-directory: openwrt
 359         run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 360
 361       - name: Build Kernel
 362         if: inputs.build_kernel == true
 363         shell: su buildbot -c "sh -e {0}"
 364         working-directory: openwrt
 365         run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 366
 367       - name: Build Kernel Kmods
 368         if: inputs.build_kernel == true
 369         shell: su buildbot -c "sh -e {0}"
 370         working-directory: openwrt
 371         run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 372
 373       - name: Build everything
 374         if: inputs.build_full == true
 375         shell: su buildbot -c "sh -e {0}"
 376         working-directory: openwrt
 377         run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 378
 379       - name: Coverity prepare toolchain
 380         if: inputs.coverity_check_packages != ''
 381         shell: su buildbot -c "sh -e {0}"
 382         working-directory: openwrt
 383         run: |
 384           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz
 385           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5
 386           echo ' coverity.tar.gz' >> coverity.tar.gz.md5
 387           md5sum -c coverity.tar.gz.md5
 388
 389           mkdir cov-analysis-linux64
 390           tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64
 391           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 392
 393           for template in ${{ inputs.coverity_compiler_template_list }}; do
 394             cov-configure --template --comptype gcc --compiler "$template"
 395           done
 396
 397       - name: Clean and recompile packages with Coverity toolchain
 398         if: inputs.coverity_check_packages != ''
 399         shell: su buildbot -c "bash {0}"
 400         working-directory: openwrt
 401         run: |
 402           set -o pipefail -o errexit
 403
 404           coverity_check_packages=(${{ inputs.coverity_check_packages }})
 405           printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}"
 406           make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 407
 408           coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }})
 409           printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}"
 410           make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 411
 412           printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}"
 413           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 414           cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 415
 416       - name: Upload build to Coverity for analysis
 417         if: inputs.coverity_check_packages != ''
 418         shell: su buildbot -c "sh -e {0}"
 419         working-directory: openwrt
 420         run: |
 421           tar czf cov-int.tar.gz ./cov-int
 422           curl \
 423             --form token="${{ secrets.coverity_api_token }}" \
 424             --form email="contact@openwrt.org" \
 425             --form file=@cov-int.tar.gz \
 426             --form version="${{ github.ref_name }}-${{ github.sha }}" \
 427             --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \
 428             "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}"
 429
 430       - name: Upload logs
 431         if: failure()
 432         uses: actions/upload-artifact@v3
 433         with:
 434           name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs
 435           path: "openwrt/logs"