CI: add manual staging build

Signed-off-by: Paul Spooren <mail@aparcar.org>

diff --git a/.github/workflows/Dockerfile b/.github/workflows/Dockerfile
new file mode 100644 (file)
index 0000000..5d110df
--- /dev/null
+++ b/.github/workflows/Dockerfile
@@ -0,0 +1,45 @@
+FROM debian:10
+
+RUN apt-get update -qq &&\
+    apt-get install -y \
+        build-essential \
+        ccache \
+        clang \
+        curl \
+        file \
+        g++-multilib \
+        gawk \
+        gcc-multilib \
+        gettext \
+        git \
+        libdw-dev \
+        libelf-dev \
+        libncurses5-dev \
+        locales \
+        pv \
+        pwgen \
+        python \
+        python3 \
+        python3-pip \
+        qemu-utils \
+        rsync \
+        signify-openbsd \
+        subversion \
+        sudo \
+        swig \
+        unzip \
+        wget \
+        zlib1g-dev \
+        && apt-get -y autoremove \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/*
+
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+RUN useradd -c "OpenWrt Builder" -m -d /home/build -G sudo -s /bin/bash build
+
+COPY --chown=build:build . /home/build/openwrt/
+RUN chown build:build /home/build/openwrt/
+
+USER build
+ENV HOME /home/build
+WORKDIR /home/build/openwrt/
\ No newline at end of file

diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml
new file mode 100644 (file)
index 0000000..47cf62d
--- /dev/null
+++ b/.github/workflows/build-staging.yml
@@ -0,0 +1,362 @@
+name: Build ApkWrt snapshot
+
+on:
+  workflow_dispatch:
+    inputs:
+      target:
+        description: "Targets to build (x86/64 ath79/genric mediatek/mt7622)"
+        required: false
+        default: "x86/64 ath79/generic mediatek/mt7622"
+      config:
+        description: "Extra lines to append to the config"
+        required: false
+        default: ""
+
+jobs:
+  determine_targets:
+    name: Set targets
+    runs-on: ubuntu-latest
+    outputs:
+      target: ${{ steps.find_targets.outputs.target }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set targets
+        id: find_targets
+        run: |
+          if [ "${{ github.event.inputs.target }}" = "all" ]; then
+            export TARGETS="$(perl ./scripts/dump-target-info.pl targets 2>/dev/null | awk '{ print $1 }')"
+          else
+            if [ -n "${{ github.event.inputs.target }}" ]; then
+              export TARGETS="${{ github.event.inputs.target }}"
+            else
+              export TARGETS="x86/64 ath79/generic mediatek/mt7622"
+            fi
+          fi
+
+          JSON='['
+          FIRST=1
+          for TARGET in $TARGETS; do
+            [[ $FIRST -ne 1 ]] && JSON="$JSON"','
+            JSON="$JSON"'"'"${TARGET}"'"'
+            FIRST=0
+          done
+          JSON="$JSON"']'
+
+          echo -e "\n---- targets ----\n"
+          echo "$JSON"
+          echo -e "\n---- targets ----\n"
+
+          echo "::set-output name=target::$JSON"
+
+  build:
+    name: Build ${{ matrix.os }}/${{ matrix.target }}
+    needs: determine_targets
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: False
+      matrix:
+        os: [ubuntu-latest]
+        target: ${{fromJson(needs.determine_targets.outputs.target)}}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          path: openwrt
+
+      - name: Setup MacOS
+        if: ${{ matrix.os == 'macos-latest' }}
+        run: |
+          echo "WORKPATH=/Volumes/OpenWrt/openwrt/" >> "$GITHUB_ENV"
+          hdiutil create -size 20g -type SPARSE -fs "Case-sensitive HFS+" -volname OpenWrt OpenWrt.sparseimage
+          hdiutil attach OpenWrt.sparseimage
+          mv "$GITHUB_WORKSPACE/openwrt" /Volumes/OpenWrt/
+          cd "$WORKPATH"
+
+          brew install \
+            autoconf \
+            automake \
+            coreutils \
+            clang-12 \
+            diffutils \
+            findutils \
+            gawk \
+            gettext \
+            git-extras \
+            gmp \
+            gnu-getopt \
+            gnu-sed \
+            gnu-tar \
+            grep \
+            libidn2 \
+            libunistring \
+            m4 \
+            make \
+            mpfr \
+            ncurses \
+            openssl@1.1 \
+            pcre \
+            pkg-config \
+            quilt \
+            readline \
+            wget \
+            zstd
+
+            echo "/bin" >> "$GITHUB_PATH"
+            echo "/sbin/Library/Apple/usr/bin" >> "$GITHUB_PATH"
+            echo "/usr/bin" >> "$GITHUB_PATH"
+            echo "/usr/local/bin" >> "$GITHUB_PATH"
+            echo "/usr/local/opt/coreutils/bin" >> "$GITHUB_PATH"
+            echo "/usr/local/opt/findutils/libexec/gnubin" >> "$GITHUB_PATH"
+            echo "/usr/local/opt/gettext/bin" >> "$GITHUB_PATH"
+            echo "/usr/local/opt/gnu-getopt/bin" >> "$GITHUB_PATH"
+            echo "/usr/local/opt/make/libexec/gnubin" >> "$GITHUB_PATH"
+            echo "/usr/local/opt/make/libexec/gnubin" >> "$GITHUB_PATH"
+            echo "/usr/sbin" >> "$GITHUB_PATH"
+            pwd
+
+      - name: Setup Ubuntu
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        env:
+          DEBIAN_FRONTEND: noninteractive
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install libncurses-dev qemu-utils clang-12 \
+            build-essential ccache ecj fastjar file g++ gawk \
+            gettext git java-propose-classpath libelf-dev libncurses5-dev \
+            libncursesw5-dev libssl-dev python python2.7-dev python3 unzip wget \
+            python3-distutils python3-setuptools python3-dev rsync subversion \
+            swig time xsltproc zlib1g-dev mkisofs
+          echo "WORKPATH=$GITHUB_WORKSPACE/openwrt/" >> "$GITHUB_ENV"
+          cd "$WORKPATH"
+          pwd
+
+      - name: Make prereq
+        run: |
+          cd "$WORKPATH"
+          pwd
+          make defconfig
+
+      - name: Initialization environment
+        run: |
+          TARGET=$(echo ${{ matrix.target }} | cut -d "/" -f 1)
+          SUBTARGET=$(echo ${{ matrix.target }} | cut -d "/" -f 2)
+          echo "TARGET=$TARGET" >> "$GITHUB_ENV"
+          echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV"
+
+      - name: Update & Install feeds
+        run: |
+          cd "$WORKPATH"
+          ./scripts/feeds update -a
+          ./scripts/feeds install -a -p base -d m
+          ./scripts/feeds install -a
+
+      - name: Set configuration
+        run: |
+          cd "$WORKPATH"
+          curl "https://downloads.openwrt.org/snapshots/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/config.buildinfo" > .config
+          for CONFIG in ${{ github.event.inputs.config }}; do
+            echo "CONFIG_$CONFIG" >> .config
+          done
+
+          echo "CONFIG_BPF_TOOLCHAIN_HOST=y" >> .config
+          echo "CONFIG_PACKAGE_uxc=m" >> .config
+          echo "CONFIG_PACKAGE_uvol=m" >> .config
+          echo "CONFIG_PACKAGE_strace=m" >> .config
+          echo "CONFIG_PACKAGE_tmate=m" >> .config
+          echo "CONFIG_PACKAGE_luci=m" >> .config
+          echo "CONFIG_PACKAGE_freeswitch=m" >> .config
+
+          echo -e "\n---- config input ----\n"
+          cat .config
+          echo -e "\n---- config input ----\n"
+
+          make defconfig
+
+          echo -e "\n---- config post-defconfig ----\n"
+          cat .config
+          echo -e "\n---- config post-defconfig ----\n"
+
+          echo "${{ secrets.PUBLIC_KEY_PEM }}" > public-key.pem
+          echo "${{ secrets.PRIVATE_KEY_PEM }}" > private-key.pem
+
+      - name: Build tools
+        run: |
+          cd "$WORKPATH"
+          make tools/install -j$(nproc) || \
+            make tools/install V=s
+
+      - name: Build toolchain
+        run: |
+          cd "$WORKPATH"
+          make toolchain/install -j$(nproc) || \
+            make toolchain/install V=s
+
+      - name: Build target
+        run: |
+          cd "$WORKPATH"
+          make target/compile -j$(nproc) IGNORE_ERRORS='n m' || \
+            make target/compile IGNORE_ERRORS='n m' V=s
+
+      - name: Build packages
+        run: |
+          cd "$WORKPATH"
+          make package/apk/host/compile -j$(nproc) || \
+            make package/apk/host/compile V=s
+
+          make package/compile -j$(nproc) IGNORE_ERRORS='n m' || \
+            make package/compile IGNORE_ERRORS='n m' V=s
+
+          make package/index V=s
+
+      - name: Determine Kernel version
+        run: |
+          cd "$WORKPATH"
+          export TOPDIR=$(pwd)
+          KERNEL_VERSION="$(make --no-print-directory -C target/linux \
+            val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | \
+            tr '\n' '-' | head -c -1)"
+
+          echo "KERNEL_VERSION=$KERNEL_VERSION" >> "$GITHUB_ENV"
+
+      - name: Determine architecutre
+        run: |
+          cd "$WORKPATH"
+          echo "ARCH=$(make val.ARCH_PACKAGES)" >> "$GITHUB_ENV"
+
+      - name: Upload packages
+        uses: jakejarvis/s3-sync-action@master
+        with:
+          args: --acl public-read --follow-symlinks --delete
+        env:
+          AWS_S3_BUCKET: openwrt-ci
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_S3_ENDPOINT: https://s3.aparcar.org
+          SOURCE_DIR: openwrt/bin/packages/${{ env.ARCH }}/
+          DEST_DIR: apkwrt/packages/${{ env.ARCH }}/
+
+      - name: Prepare upload of kmods
+        continue-on-error: true
+        run: |
+          cd "$WORKPATH"
+
+          KMOD_DIR="bin/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/kmods/${{ env.KERNEL_VERSION }}/"
+          mkdir -p "$KMOD_DIR"
+          cp bin/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}//packages/kmod-*.apk "$KMOD_DIR"
+          make package/index V=s PACKAGE_SUBDIRS="$KMOD_DIR"
+
+      - name: Upload kmods
+        continue-on-error: true
+        uses: jakejarvis/s3-sync-action@master
+        with:
+          args: --acl public-read --follow-symlinks --delete
+        env:
+          AWS_S3_BUCKET: openwrt-ci
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_S3_ENDPOINT: https://s3.aparcar.org
+          SOURCE_DIR: openwrt/bin/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/kmods/${{ env.KERNEL_VERSION }}/
+          DEST_DIR: apkwrt/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/kmods/${{ env.KERNEL_VERSION }}/
+
+      - name: Install packages
+        run: |
+          cd "$WORKPATH"
+          make package/install -j$(nproc) || \
+            make package/install V=s
+
+      - name: Build firmware
+        run: |
+          cd "$WORKPATH"
+          make target/install -j$(nproc) || \
+            make target/install V=s
+
+      - name: Buildinfo
+        run: |
+          cd "$WORKPATH"
+          make buildinfo V=s
+
+      - name: JSON overview
+        run: |
+          cd "$WORKPATH"
+          make json_overview_image_info V=s
+
+      - name: Checksum
+        run: |
+          cd "$WORKPATH"
+          make checksum V=s
+
+      - name: Clean build
+        run: |
+          cd "$WORKPATH"
+          rm -rf staging_dir/ build_dir
+
+      - name: Move everything back in place
+        if: ${{ matrix.os == 'macos-latest' }}
+        run: mv "$WORKPATH" "$GITHUB_WORKSPACE"
+
+      - name: Upload artifacts
+        if:  always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.TARGET }}-${{ env.SUBTARGET }}-artifacts
+          path: openwrt/bin/
+
+      - name: Upload target
+        uses: jakejarvis/s3-sync-action@master
+        with:
+          args: --acl public-read --follow-symlinks --delete
+        env:
+          AWS_S3_BUCKET: openwrt-ci
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_S3_ENDPOINT: https://s3.asu.aparcar.org
+          SOURCE_DIR: openwrt/bin/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/
+          DEST_DIR: apkwrt/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Prepare Docker SDK
+        run: |
+          cd "$WORKPATH"
+          rm -rf bin/packages # free some space
+          mkdir -p sdk/
+          tar xf bin/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/openwrt-sdk-*.tar.xz
+          mv openwrt-sdk-*/ sdk/
+
+      - name: Build and push SDK
+        uses: docker/build-push-action@v2
+        with:
+          context: "openwrt/sdk/"
+          file: "openwrt/.github/workflows/Dockerfile"
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/sdk:${{ env.ARCH }}
+            ghcr.io/${{ github.repository_owner }}/sdk:${{ env.TARGET }}-${{ env.SUBTARGET }}
+
+      - name: Prepare Docker ImageBuilder
+        run: |
+          cd "$WORKPATH"
+          rm -rf sdk/ # free some space
+          mkdir -p imagebuilder/
+          tar xf bin/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/openwrt-imagebuilder-*.tar.xz
+          mv openwrt-imagebuilder-*/ imagebuilder/
+
+      - name: Build and push ImageBuilder
+        uses: docker/build-push-action@v2
+        with:
+          context: "openwrt/imagebuilder/"
+          file: "openwrt/.github/workflows/Dockerfile"
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/imagebuilder:${{ env.ARCH }}
+            ghcr.io/${{ github.repository_owner }}/imagebuilder:${{ env.TARGET }}-${{ env.SUBTARGET }}

diff --git a/include/version.mk b/include/version.mk
index f39e35cd099a7df15029cd25001dfe1e169b309b..f62af65f53f2a44db9d19e5e1ab7b9a52025ac80 100644 (file)
--- a/include/version.mk
+++ b/include/version.mk
@@ -29,7 +29,7 @@ VERSION_CODE:=$(call qstrip,$(CONFIG_VERSION_CODE))
 VERSION_CODE:=$(if $(VERSION_CODE),$(VERSION_CODE),$(REVISION))
 
 VERSION_REPO:=$(call qstrip,$(CONFIG_VERSION_REPO))
-VERSION_REPO:=$(if $(VERSION_REPO),$(VERSION_REPO),https://downloads.openwrt.org/snapshots)
+VERSION_REPO:=$(if $(VERSION_REPO),$(VERSION_REPO),https://s3.aparcar.org/openwrt-ci/apkwrt)
 
 VERSION_DIST:=$(call qstrip,$(CONFIG_VERSION_DIST))
 VERSION_DIST:=$(if $(VERSION_DIST),$(VERSION_DIST),OpenWrt)

diff --git a/package/base-files/image-config.in b/package/base-files/image-config.in
index ac406bc456bd91c17e5ffef7da01314a5477bdc5..4dfc0a80e7a8aaabb1374770cc0c238fc4cb1153 100644 (file)
--- a/package/base-files/image-config.in
+++ b/package/base-files/image-config.in
@@ -183,7 +183,7 @@ if VERSIONOPT
        config VERSION_REPO
                string
                prompt "Release repository"
-               default "https://downloads.openwrt.org/snapshots"
+               default "https://s3.aparcar.org/openwrt-ci/apkwrt"
                help
                        This is the repository address embedded in the image, it defaults
                        to the trunk snapshot repo; the url may contain the following placeholders: