run_tests.sh

   1 #!/usr/bin/env bash
   2
   3 line='........................................'
   4 uenv='{ "REQUIRE_SEARCH_PATH": [ "/usr/local/lib/ucode/*.so", "/usr/lib/ucode/*.so", "./tests/*.uc", "./root/usr/share/ucode/*.uc" ] }'
   5
   6 extract_sections() {
   7         local file=$1
   8         local dir=$2
   9         local count=0
  10         local tag line outfile
  11
  12         while IFS= read -r line; do
  13                 case "$line" in
  14                         "-- Testcase --")
  15                                 tag="test"
  16                                 count=$((count + 1))
  17                                 outfile=$(printf "%s/%03d.in" "$dir" $count)
  18                                 printf "" > "$outfile"
  19                         ;;
  20                         "-- Environment --")
  21                                 tag="env"
  22                                 count=$((count + 1))
  23                                 outfile=$(printf "%s/%03d.env" "$dir" $count)
  24                                 printf "" > "$outfile"
  25                         ;;
  26                         "-- Expect stdout --"|"-- Expect stderr --"|"-- Expect exitcode --")
  27                                 tag="${line#-- Expect }"
  28                                 tag="${tag% --}"
  29                                 count=$((count + 1))
  30                                 outfile=$(printf "%s/%03d.%s" "$dir" $count "$tag")
  31                                 printf "" > "$outfile"
  32                         ;;
  33                         "-- End --")
  34                                 tag=""
  35                                 outfile=""
  36                         ;;
  37                         *)
  38                                 if [ -n "$tag" ]; then
  39                                         printf "%s\\n" "$line" >> "$outfile"
  40                                 fi
  41                         ;;
  42                 esac
  43         done < "$file"
  44
  45         return $(ls -l "$dir/"*.in 2>/dev/null | wc -l)
  46 }
  47
  48 run_testcase() {
  49         local num=$1
  50         local dir=$2
  51         local in=$3
  52         local env=$4
  53         local out=$5
  54         local err=$6
  55         local code=$7
  56         local fail=0
  57
  58         ucode ${uenv:+-e "$uenv"} ${env:+-e "$(cat "$env")"} -i - <"$in" >"$dir/res.out" 2>"$dir/res.err"
  59
  60         printf "%d\n" $? > "$dir/res.code"
  61
  62         touch "$dir/empty"
  63
  64         if ! cmp -s "$dir/res.err" "${err:-$dir/empty}"; then
  65                 [ $fail = 0 ] && printf "!\n"
  66                 printf "Testcase #%d: Expected stderr did not match:\n" $num
  67                 diff -u --color=always --label="Expected stderr" --label="Resulting stderr" "${err:-$dir/empty}" "$dir/res.err"
  68                 printf -- "---\n"
  69                 fail=1
  70         fi
  71
  72         if ! cmp -s "$dir/res.out" "${out:-$dir/empty}"; then
  73                 [ $fail = 0 ] && printf "!\n"
  74                 printf "Testcase #%d: Expected stdout did not match:\n" $num
  75                 diff -u --color=always --label="Expected stdout" --label="Resulting stdout" "${out:-$dir/empty}" "$dir/res.out"
  76                 printf -- "---\n"
  77                 fail=1
  78         fi
  79
  80         if [ -n "$code" ] && ! cmp -s "$dir/res.code" "$code"; then
  81                 [ $fail = 0 ] && printf "!\n"
  82                 printf "Testcase #%d: Expected exit code did not match:\n" $num
  83                 diff -u --color=always --label="Expected code" --label="Resulting code" "$code" "$dir/res.code"
  84                 printf -- "---\n"
  85                 fail=1
  86         fi
  87
  88         return $fail
  89 }
  90
  91 run_test() {
  92         local file=$1
  93         local name=${file##*/}
  94         local res ecode eout eerr ein eenv tests
  95         local testcase_first=0 failed=0 count=0
  96
  97         printf "%s %s " "$name" "${line:${#name}}"
  98
  99         mkdir "/tmp/test.$$"
 100
 101         extract_sections "$file" "/tmp/test.$$"
 102         tests=$?
 103
 104         [ -f "/tmp/test.$$/001.in" ] && testcase_first=1
 105
 106         for res in "/tmp/test.$$/"[0-9]*; do
 107                 case "$res" in
 108                         *.in)
 109                                 count=$((count + 1))
 110
 111                                 if [ $testcase_first = 1 ]; then
 112                                         # Flush previous test
 113                                         if [ -n "$ein" ]; then
 114                                                 run_testcase $count "/tmp/test.$$" "$ein" "$eenv" "$eout" "$eerr" "$ecode" || failed=$((failed + 1))
 115
 116                                                 eout=""
 117                                                 eerr=""
 118                                                 ecode=""
 119                                                 eenv=""
 120                                         fi
 121
 122                                         ein=$res
 123                                 else
 124                                         run_testcase $count "/tmp/test.$$" "$res" "$eenv" "$eout" "$eerr" "$ecode" || failed=$((failed + 1))
 125
 126                                         eout=""
 127                                         eerr=""
 128                                         ecode=""
 129                                         eenv=""
 130                                 fi
 131
 132                         ;;
 133                         *.env) eenv=$res ;;
 134                         *.stdout) eout=$res ;;
 135                         *.stderr) eerr=$res ;;
 136                         *.exitcode) ecode=$res ;;
 137                 esac
 138         done
 139
 140         # Flush last test
 141         if [ $testcase_first = 1 ] && [ -n "$eout$eerr$ecode" ]; then
 142                 run_testcase $count "/tmp/test.$$" "$ein" "$eenv" "$eout" "$eerr" "$ecode" || failed=$((failed + 1))
 143         fi
 144
 145         rm -r "/tmp/test.$$"
 146
 147         if [ $failed = 0 ]; then
 148                 printf "OK\n"
 149         else
 150                 printf "%s %s FAILED (%d/%d)\n" "$name" "${line:${#name}}" $failed $tests
 151         fi
 152
 153         return $failed
 154 }
 155
 156
 157 n_tests=0
 158 n_fails=0
 159
 160 for catdir in tests/[0-9][0-9]_*; do
 161         [ -d "$catdir" ] || continue
 162
 163         printf "\n##\n## Running %s tests\n##\n\n" "${catdir##*/[0-9][0-9]_}"
 164
 165         for testfile in "$catdir/"[0-9][0-9]_*; do
 166                 [ -f "$testfile" ] || continue
 167
 168                 n_tests=$((n_tests + 1))
 169                 run_test "$testfile" || n_fails=$((n_fails + 1))
 170         done
 171 done
 172
 173 printf "\nRan %d tests, %d okay, %d failures\n" $n_tests $((n_tests - n_fails)) $n_fails