7ce8a7c034e70d8f3f9448ba4f2ff88d3660837b
[openwrt/openwrt.git] / .github / workflows / build.yml
1 name: Build sub target
2
3 on:
4 workflow_call:
5 secrets:
6 coverity_api_token:
7 inputs:
8 container_name:
9 type: string
10 default: tools
11 target:
12 required: true
13 type: string
14 subtarget:
15 required: true
16 type: string
17 testing:
18 type: boolean
19 build_toolchain:
20 type: boolean
21 include_feeds:
22 type: boolean
23 build_full:
24 type: boolean
25 build_kernel:
26 type: boolean
27 build_all_modules:
28 type: boolean
29 build_all_kmods:
30 type: boolean
31 build_all_boards:
32 type: boolean
33 use_openwrt_container:
34 type: boolean
35 default: true
36 coverity_project_name:
37 type: string
38 default: OpenWrt
39 coverity_check_packages:
40 type: string
41 coverity_compiler_template_list:
42 type: string
43 default: >-
44 arm-openwrt-linux-gcc
45 coverity_force_compile_packages:
46 type: string
47 default: >-
48 curl
49 libnl
50 mbedtls
51 wolfssl
52 openssl
53 build_external_toolchain:
54 type: boolean
55 upload_external_toolchain:
56 type: boolean
57
58 permissions:
59 contents: read
60
61 jobs:
62 setup_build:
63 name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }}
64 runs-on: ubuntu-latest
65 outputs:
66 owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
67 container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
68 container_name: ${{ steps.determine_tools_container.outputs.container_name }}
69
70 steps:
71 - name: Checkout
72 uses: actions/checkout@v3
73
74 - name: Set lower case owner name
75 id: lower_owner
76 run: |
77 OWNER_LC=$(echo "${{ github.repository_owner }}" \
78 | tr '[:upper:]' '[:lower:]')
79
80 if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
81 OWNER_LC=openwrt
82 fi
83
84 echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
85
86 # Per branch tools container tag
87 # By default stick to latest
88 # For official test targetting openwrt stable branch
89 # Get the branch or parse the tag and push dedicated tools containers
90 # For local test to use the correct container for stable release testing
91 # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
92 - name: Determine tools container tag
93 id: determine_tools_container
94 run: |
95 CONTAINER_NAME=${{ inputs.container_name }}
96 CONTAINER_TAG=latest
97 if [ -n "${{ github.base_ref }}" ]; then
98 if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
99 CONTAINER_TAG="${{ github.base_ref }}"
100 fi
101 elif [ ${{ github.ref_type }} == "branch" ]; then
102 if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
103 CONTAINER_TAG=${{ github.ref_name }}
104 elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
105 CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
106 fi
107 elif [ ${{ github.ref_type }} == "tag" ]; then
108 if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
109 CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
110 fi
111 fi
112
113 if [ "$CONTAINER_NAME" = "toolchain" ]; then
114 GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64)
115 GHCR_HEADER="Authorization: Bearer ${GHCR_TOKEN}"
116 GHCR_MANIFEST_LINK=https://ghcr.io/v2/${{ steps.lower_owner.outputs.owner_lc }}/${{ inputs.container_name }}/manifests/${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
117 # Check if container exist
118 if [ $(curl -s -o /dev/null -w "%{http_code}" -H "$GHCR_HEADER" -I "$GHCR_MANIFEST_LINK") = 200 ]; then
119 CONTAINER_TAG=${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
120 else
121 CONTAINER_NAME=tools
122 fi
123 fi
124
125 echo "Tools container to use $CONTAINER_NAME:$CONTAINER_TAG"
126 echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
127 echo "container_name=$CONTAINER_NAME" >> $GITHUB_OUTPUT
128
129 build:
130 name: Build ${{ inputs.target }}/${{ inputs.subtarget }}
131 needs: setup_build
132 runs-on: ubuntu-latest
133
134 container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/${{ needs.setup_build.outputs.container_name }}:${{ needs.setup_build.outputs.container_tag }}
135
136 permissions:
137 contents: read
138 packages: read
139 actions: write
140
141 steps:
142 - name: Checkout master directory
143 uses: actions/checkout@v3
144 with:
145 path: openwrt
146
147 - name: Checkout packages feed
148 if: inputs.include_feeds == true
149 uses: actions/checkout@v3
150 with:
151 repository: openwrt/packages
152 path: openwrt/feeds/packages
153
154 - name: Checkout luci feed
155 if: inputs.include_feeds == true
156 uses: actions/checkout@v3
157 with:
158 repository: openwrt/luci
159 path: openwrt/feeds/luci
160
161 - name: Checkout routing feed
162 if: inputs.include_feeds == true
163 uses: actions/checkout@v3
164 with:
165 repository: openwrt/routing
166 path: openwrt/feeds/routing
167
168 - name: Checkout telephony feed
169 if: inputs.include_feeds == true
170 uses: actions/checkout@v3
171 with:
172 repository: openwrt/telephony
173 path: openwrt/feeds/telephony
174
175 - name: Parse toolchain file
176 if: inputs.build_toolchain == false
177 id: parse-toolchain
178 working-directory: openwrt
179 run: |
180 if [ -d /external-toolchain/ ]; then
181 echo "toolchain-type=external_container" >> $GITHUB_OUTPUT
182 exit 0
183 fi
184
185 TOOLCHAIN_PATH=snapshots
186
187 if [ -n "${{ github.base_ref }}" ]; then
188 if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
189 major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
190 fi
191 elif [ "${{ github.ref_type }}" = "branch" ]; then
192 if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
193 major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
194 elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
195 major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
196 fi
197 elif [ "${{ github.ref_type }}" = "tag" ]; then
198 if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
199 major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
200 fi
201 fi
202
203 if [ -n "$major_ver" ]; then
204 git fetch --tags -f
205 latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
206 if [ -n "$latest_tag" ]; then
207 TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
208 fi
209 fi
210
211 SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums"
212 if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
213 TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
214 TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
215
216 echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
217 elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
218 TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
219 TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
220
221 echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
222 else
223 echo "toolchain-type=internal" >> $GITHUB_OUTPUT
224 fi
225
226 echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
227 echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
228
229 - name: Fix permission
230 run: |
231 chown -R buildbot:buildbot openwrt
232
233 - name: Prepare prebuilt tools
234 shell: su buildbot -c "sh -e {0}"
235 working-directory: openwrt
236 run: |
237 mkdir -p staging_dir build_dir
238 ln -s /prebuilt_tools/staging_dir/host staging_dir/host
239 ln -s /prebuilt_tools/build_dir/host build_dir/host
240
241 ./scripts/ext-tools.sh --refresh
242
243 - name: Update & Install feeds
244 if: inputs.include_feeds == true
245 shell: su buildbot -c "sh -e {0}"
246 working-directory: openwrt
247 run: |
248 ./scripts/feeds update -a
249 ./scripts/feeds install -a
250
251 - name: Restore ccache cache
252 id: restore-ccache-cache
253 uses: actions/cache/restore@v3
254 with:
255 path: openwrt/.ccache
256 key: ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-${{ hashFiles('openwrt/include/kernel-**') }}
257 restore-keys: |
258 ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-
259
260 - name: Download external toolchain/sdk
261 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal' && steps.parse-toolchain.outputs.toolchain-type != 'external_container'
262 shell: su buildbot -c "sh -e {0}"
263 working-directory: openwrt
264 run: |
265 wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
266 | tar --xz -xf -
267
268 - name: Configure testing kernel
269 if: inputs.testing == true
270 shell: su buildbot -c "sh -e {0}"
271 working-directory: openwrt
272 run: |
273 echo CONFIG_TESTING_KERNEL=y >> .config
274
275 - name: Configure all kernel modules
276 if: inputs.build_all_kmods == true
277 shell: su buildbot -c "sh -e {0}"
278 working-directory: openwrt
279 run: |
280 echo CONFIG_ALL_KMODS=y >> .config
281
282 - name: Configure all modules
283 if: inputs.build_all_modules == true
284 shell: su buildbot -c "sh -e {0}"
285 working-directory: openwrt
286 run: |
287 echo CONFIG_ALL=y >> .config
288
289 - name: Configure all boards
290 if: inputs.build_all_boards == true
291 shell: su buildbot -c "sh -e {0}"
292 working-directory: openwrt
293 run: |
294 echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
295 echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
296 echo CONFIG_TARGET_ALL_PROFILES=y >> .config
297
298 - name: Configure external toolchain in container
299 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_container'
300 shell: su buildbot -c "sh -e {0}"
301 working-directory: openwrt
302 run: |
303 echo CONFIG_DEVEL=y >> .config
304 echo CONFIG_AUTOREMOVE=y >> .config
305 echo CONFIG_CCACHE=y >> .config
306
307 ./scripts/ext-toolchain.sh \
308 --toolchain /external-toolchain/$(ls /external-toolchain/ | grep openwrt-toolchain)/toolchain-* \
309 --overwrite-config \
310 --config ${{ inputs.target }}/${{ inputs.subtarget }}
311
312 - name: Configure external toolchain
313 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
314 shell: su buildbot -c "sh -e {0}"
315 working-directory: openwrt
316 run: |
317 echo CONFIG_DEVEL=y >> .config
318 echo CONFIG_AUTOREMOVE=y >> .config
319 echo CONFIG_CCACHE=y >> .config
320
321 ./scripts/ext-toolchain.sh \
322 --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
323 --overwrite-config \
324 --config ${{ inputs.target }}/${{ inputs.subtarget }}
325
326 - name: Adapt external sdk to external toolchain format
327 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
328 shell: su buildbot -c "sh -e {0}"
329 working-directory: openwrt
330 run: |
331 TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
332 TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
333 OPENWRT_DIR=$(pwd)
334
335 # Find target name from toolchain info.mk
336 GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
337
338 cd $TOOLCHAIN_BIN
339
340 # Revert sdk wrapper scripts applied to all the bins
341 for app in $(find . -name "*.bin"); do
342 TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
343 rm $TARGET_APP
344 mv .$TARGET_APP.bin $TARGET_APP
345 done
346
347 # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
348 cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
349 for app in cc gcc g++ c++ cpp ld as ; do
350 [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
351 ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
352 done
353
354 - name: Configure external toolchain with sdk
355 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
356 shell: su buildbot -c "sh -e {0}"
357 working-directory: openwrt
358 run: |
359 echo CONFIG_DEVEL=y >> .config
360 echo CONFIG_AUTOREMOVE=y >> .config
361 echo CONFIG_CCACHE=y >> .config
362
363 ./scripts/ext-toolchain.sh \
364 --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
365 --overwrite-config \
366 --config ${{ inputs.target }}/${{ inputs.subtarget }}
367
368 - name: Configure internal toolchain
369 if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
370 shell: su buildbot -c "sh -e {0}"
371 working-directory: openwrt
372 run: |
373 echo CONFIG_DEVEL=y >> .config
374 echo CONFIG_AUTOREMOVE=y >> .config
375 echo CONFIG_CCACHE=y >> .config
376
377 echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config
378 echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config
379
380 make defconfig
381
382 - name: Show configuration
383 shell: su buildbot -c "sh -e {0}"
384 working-directory: openwrt
385 run: ./scripts/diffconfig.sh
386
387 - name: Build tools
388 shell: su buildbot -c "sh -e {0}"
389 working-directory: openwrt
390 run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
391
392 - name: Build toolchain
393 shell: su buildbot -c "sh -e {0}"
394 working-directory: openwrt
395 run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
396
397 - name: Build Kernel
398 if: inputs.build_kernel == true
399 shell: su buildbot -c "sh -e {0}"
400 working-directory: openwrt
401 run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
402
403 - name: Build Kernel Kmods
404 if: inputs.build_kernel == true
405 shell: su buildbot -c "sh -e {0}"
406 working-directory: openwrt
407 run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
408
409 - name: Build everything
410 if: inputs.build_full == true
411 shell: su buildbot -c "sh -e {0}"
412 working-directory: openwrt
413 run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
414
415 - name: Build external toolchain
416 if: inputs.build_external_toolchain == true
417 shell: su buildbot -c "sh -e {0}"
418 working-directory: openwrt
419 run: make target/toolchain/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
420
421 - name: Coverity prepare toolchain
422 if: inputs.coverity_check_packages != ''
423 shell: su buildbot -c "sh -e {0}"
424 working-directory: openwrt
425 run: |
426 wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz
427 wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5
428 echo ' coverity.tar.gz' >> coverity.tar.gz.md5
429 md5sum -c coverity.tar.gz.md5
430
431 mkdir cov-analysis-linux64
432 tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64
433 export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
434
435 for template in ${{ inputs.coverity_compiler_template_list }}; do
436 cov-configure --template --comptype gcc --compiler "$template"
437 done
438
439 - name: Clean and recompile packages with Coverity toolchain
440 if: inputs.coverity_check_packages != ''
441 shell: su buildbot -c "bash {0}"
442 working-directory: openwrt
443 run: |
444 set -o pipefail -o errexit
445
446 coverity_check_packages=(${{ inputs.coverity_check_packages }})
447 printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}"
448 make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
449
450 coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }})
451 printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}"
452 make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
453
454 printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}"
455 export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
456 cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
457
458 - name: Upload build to Coverity for analysis
459 if: inputs.coverity_check_packages != ''
460 shell: su buildbot -c "sh -e {0}"
461 working-directory: openwrt
462 run: |
463 tar czf cov-int.tar.gz ./cov-int
464 curl \
465 --form token="${{ secrets.coverity_api_token }}" \
466 --form email="contact@openwrt.org" \
467 --form file=@cov-int.tar.gz \
468 --form version="${{ github.ref_name }}-${{ github.sha }}" \
469 --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \
470 "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}"
471
472 - name: Upload logs
473 if: failure()
474 uses: actions/upload-artifact@v3
475 with:
476 name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs
477 path: "openwrt/logs"
478
479 - name: Delete already present ccache cache
480 if: steps.restore-ccache-cache.outputs.cache-hit == 'true'
481 uses: octokit/request-action@v2.x
482 with:
483 route: DELETE /repos/{repository}/actions/caches?key={key}
484 env:
485 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
486 INPUT_REPOSITORY: ${{ github.repository }}
487 INPUT_KEY: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
488
489 - name: Save ccache cache
490 uses: actions/cache/save@v3
491 with:
492 path: openwrt/.ccache
493 key: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
494
495 - name: Find external toolchain name
496 id: get-toolchain-name
497 if: inputs.upload_external_toolchain == true
498 working-directory: openwrt
499 run: |
500 TOOLCHAIN_NAME=$(ls bin/targets/${{inputs.target }}/${{ inputs.subtarget }} | grep toolchain)
501 echo "toolchain-name=$TOOLCHAIN_NAME" >> $GITHUB_OUTPUT
502
503 - name: Upload prebuilt toolchain
504 if: inputs.upload_external_toolchain == true
505 uses: actions/upload-artifact@v3
506 with:
507 name: ${{ inputs.target }}-${{ inputs.subtarget }}-external-toolchain
508 path: openwrt/bin/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ steps.get-toolchain-name.outputs.toolchain-name }}
509 retention-days: 1