.travis_do.sh

   1 #!/bin/bash
   2 #
   3 # MIT Alexander Couzens <lynxis@fe80.eu>
   4
   5 set -e
   6
   7 SDK_HOME="$HOME/sdk"
   8 SDK_PATH=https://downloads.openwrt.org/snapshots/targets/ar71xx/generic/
   9 SDK=-sdk-ar71xx-generic_
  10 PACKAGES_DIR="$PWD"
  11
  12 echo_red()   { printf "\033[1;31m$*\033[m\n"; }
  13 echo_green() { printf "\033[1;32m$*\033[m\n"; }
  14 echo_blue()  { printf "\033[1;34m$*\033[m\n"; }
  15
  16 exec_status() {
  17         PATTERN="$1"
  18         shift
  19         while :;do sleep 590;echo "still running (please don't kill me Travis)";done &
  20         ("$@" 2>&1) | tee logoutput
  21         R=${PIPESTATUS[0]}
  22         kill $! && wait $! 2>/dev/null
  23         if [ $R -ne 0 ]; then
  24                 echo_red   "=> '$*' failed (return code $R)"
  25                 return 1
  26         fi
  27         if grep -qE "$PATTERN" logoutput; then
  28                 echo_red   "=> '$*' failed (log matched '$PATTERN')"
  29                 return 1
  30         fi
  31
  32         echo_green "=> '$*' successful"
  33         return 0
  34 }
  35
  36 get_sdk_file() {
  37         if [ -e "$SDK_HOME/sha256sums" ] ; then
  38                 grep -- "$SDK" "$SDK_HOME/sha256sums" | awk '{print $2}' | sed 's/*//g'
  39         else
  40                 false
  41         fi
  42 }
  43
  44 # download will run on the `before_script` step
  45 # The travis cache will be used (all files under $HOME/sdk/). Meaning
  46 # We don't have to download the file again
  47 download_sdk() {
  48         mkdir -p "$SDK_HOME"
  49         cd "$SDK_HOME"
  50
  51         echo_blue "=== download SDK"
  52         wget "$SDK_PATH/sha256sums" -O sha256sums
  53         wget "$SDK_PATH/sha256sums.gpg" -O sha256sums.asc
  54
  55         # LEDE Build System (LEDE GnuPG key for unattended build jobs)
  56         gpg --import $PACKAGES_DIR/.keys/626471F1.asc
  57         echo '54CC74307A2C6DC9CE618269CD84BCED626471F1:6:' | gpg --import-ownertrust
  58         # LEDE Release Builder (17.01 "Reboot" Signing Key)
  59         gpg --import $PACKAGES_DIR/.keys/D52BBB6B.asc
  60         echo 'B09BE781AE8A0CD4702FDCD3833C6010D52BBB6B:6:' | gpg --import-ownertrust
  61
  62         echo_blue "=== Verifying sha256sums signature"
  63         gpg --verify sha256sums.asc
  64         echo_blue "=== Verified sha256sums signature."
  65         if ! grep -- "$SDK" sha256sums > sha256sums.small ; then
  66                 echo_red "=== Can not find $SDK file in sha256sums."
  67                 echo_red "=== Is \$SDK out of date?"
  68                 false
  69         fi
  70
  71         # if missing, outdated or invalid, download again
  72         if ! sha256sum -c ./sha256sums.small ; then
  73                 local sdk_file
  74                 sdk_file="$(get_sdk_file)"
  75                 echo_blue "=== sha256 doesn't match or SDK file wasn't downloaded yet."
  76                 echo_blue "=== Downloading a fresh version"
  77                 wget "$SDK_PATH/$sdk_file" -O "$sdk_file"
  78         fi
  79
  80         # check again and fail here if the file is still bad
  81         echo_blue "Checking sha256sum a second time"
  82         if ! sha256sum -c ./sha256sums.small ; then
  83                 echo_red "=== SDK can not be verified!"
  84                 false
  85         fi
  86         echo_blue "=== SDK is up-to-date"
  87 }
  88
  89 # test_package will run on the `script` step.
  90 # test_package call make download check for very new/modified package
  91 test_packages2() {
  92         local commit_range=$TRAVIS_COMMIT_RANGE
  93         if [ -z "$TRAVIS_PULL_REQUEST_SHA" ]; then
  94                 echo_blue "Using only the latest commit, since we're not in a Pull Request"
  95                 commit_range=HEAD~1
  96         fi
  97
  98         # search for new or modified packages. PKGS will hold a list of package like 'admin/muninlite admin/monit ...'
  99         PKGS=$(git diff --diff-filter=d --name-only "$commit_range" | grep 'Makefile$' | grep -v '/files/' | awk -F'/Makefile' '{ print $1 }')
 100
 101         if [ -z "$PKGS" ] ; then
 102                 echo_blue "No new or modified packages found!"
 103                 return 0
 104         fi
 105
 106         echo_blue "=== Found new/modified packages:"
 107         for pkg in $PKGS ; do
 108                 echo "===+ $pkg"
 109         done
 110
 111         echo_blue "=== Setting up SDK"
 112         tmp_path=$(mktemp -d)
 113         cd "$tmp_path"
 114         tar Jxf "$SDK_HOME/$(get_sdk_file)" --strip=1
 115
 116         # use github mirrors to spare lede servers
 117         cat > feeds.conf <<EOF
 118 src-git base https://github.com/lede-project/source.git
 119 src-link packages $PACKAGES_DIR
 120 src-git luci https://github.com/openwrt/luci.git
 121 EOF
 122
 123         # enable BUILD_LOG
 124         sed -i '1s/^/config BUILD_LOG\n\tbool\n\tdefault y\n\n/' Config-build.in
 125
 126         ./scripts/feeds update -a > /dev/null
 127         ./scripts/feeds install -a > /dev/null
 128         make defconfig > /dev/null
 129         echo_blue "=== Setting up SDK done"
 130
 131         RET=0
 132         # E.g: pkg_dir => admin/muninlite
 133         # pkg_name => muninlite
 134         for pkg_dir in $PKGS ; do
 135                 pkg_name=$(echo "$pkg_dir" | awk -F/ '{ print $NF }')
 136                 echo_blue "=== $pkg_name: Starting quick tests"
 137
 138                 exec_status '^ERROR' make "package/$pkg_name/download" V=s || RET=1
 139                 badhash_msg_regex="HASH does not match "
 140                 badhash_msg_regex="$badhash_msg_regex|HASH uses deprecated hash,"
 141                 badhash_msg_regex="$badhash_msg_regex|HASH is missing,"
 142                 exec_status '^ERROR'"|$badhash_msg_regex" make "package/$pkg_name/check" V=s || RET=1
 143
 144                 echo_blue "=== $pkg_name: quick tests done"
 145         done
 146
 147         [ $RET -ne 0 ] && return $RET
 148
 149         for pkg_dir in $PKGS ; do
 150                 pkg_name=$(echo "$pkg_dir" | awk -F/ '{ print $NF }')
 151                 echo_blue "=== $pkg_name: Starting compile test"
 152
 153                 # we can't enable verbose built else we often hit Travis limits
 154                 # on log size and the job get killed
 155                 exec_status '^ERROR' make "package/$pkg_name/compile" -j3 || RET=1
 156
 157                 echo_blue "=== $pkg_name: compile test done"
 158
 159                 echo_blue "=== $pkg_name: begin compile logs"
 160                 for f in $(find logs/package/feeds/packages/$pkg_name/ -type f); do
 161                         echo_blue "Printing last 200 lines of $f"
 162                         tail -n200 "$f"
 163                 done
 164                 echo_blue "=== $pkg_name: end compile logs"
 165
 166                 echo_blue "=== $pkg_name: begin packages sizes"
 167                 du -ba bin/
 168                 echo_blue "=== $pkg_name: end packages sizes"
 169         done
 170
 171         return $RET
 172 }
 173
 174 test_commits() {
 175         RET=0
 176         if [ -z "$TRAVIS_PULL_REQUEST_SHA" ]; then
 177                 echo_blue "Skipping commits tests (not in a Pull Request)"
 178                 return 0
 179         fi
 180         for commit in $(git rev-list ${TRAVIS_COMMIT_RANGE/.../..}); do
 181                 echo_blue "=== Checking commit '$commit'"
 182                 if git show --format='%P' -s $commit | grep -qF ' '; then
 183                         echo_red "Pull request should not include merge commits"
 184                         RET=1
 185                 fi
 186
 187                 author="$(git show -s --format=%aN $commit)"
 188                 if echo $author | grep -q '\S\+\s\+\S\+'; then
 189                         echo_green "Author name ($author) seems ok"
 190                 else
 191                         echo_red "Author name ($author) need to be your real name 'firstname lastname'"
 192                         RET=1
 193                 fi
 194
 195                 subject="$(git show -s --format=%s $commit)"
 196                 if echo "$subject" | grep -q -e '^[0-9A-Za-z,/_-]\+: ' -e '^Revert '; then
 197                         echo_green "Commit subject line seems ok ($subject)"
 198                 else
 199                         echo_red "Commit subject line MUST start with '<package name>: ' ($subject)"
 200                         RET=1
 201                 fi
 202
 203                 body="$(git show -s --format=%b $commit)"
 204                 sob="$(git show -s --format='Signed-off-by: %aN <%aE>' $commit)"
 205                 if echo "$body" | grep -qF "$sob"; then
 206                         echo_green "Signed-off-by match author"
 207                 else
 208                         echo_red "Signed-off-by is missing or doesn't match author (should be '$sob')"
 209                         RET=1
 210                 fi
 211         done
 212
 213         return $RET
 214 }
 215
 216 test_packages() {
 217         test_commits && test_packages2 || return 1
 218 }
 219
 220 echo_blue "=== Travis ENV"
 221 env
 222 echo_blue "=== Travis ENV"
 223
 224 if [ -n "$TRAVIS_PULL_REQUEST_SHA" ]; then
 225         while true; do
 226                 # if clone depth is too small, git rev-list / diff return incorrect or empty results
 227                 C="$(git rev-list ${TRAVIS_COMMIT_RANGE/.../..} | tail -n1)" 2>/dev/null
 228                 [ -n "$C" -a "$C" != "a22de9b74cf9579d1ce7e6cf1845b4afa4277b00" ] && break
 229                 echo_blue "Fetching 50 commits more"
 230                 git fetch origin --deepen=50
 231         done
 232 fi
 233
 234 if [ $# -ne 1 ] ; then
 235         cat <<EOF
 236 Usage: $0 (download_sdk|test_packages)
 237
 238 download_sdk - download the SDK to $HOME/sdk.tar.xz
 239 test_packages - do a make check on the package
 240 EOF
 241         exit 1
 242 fi
 243
 244 $@