run-tests.sh 9.54 KB
Newer Older
Daniel Barlow's avatar
Daniel Barlow committed
1 2
#!/bin/sh

3
# do_tests {lisp invocation} {scripts-regex}
4 5 6 7
# - read lisp forms one at a time from standard input
# - quit with exit status 0 on getting eof
# - quit with exit status >0 if an unhandled error occurs

8 9 10 11 12 13
usage () {
    echo "$0 [lisp invocation] [scripts-regex]"
    echo " - read lisp forms one at a time from matching scripts"
    echo " - quit with exit status 0 on getting eof"
    echo " - quit with exit status >0 if an unhandled error occurs"
    echo " you need to supply the .script in the second argument"
14
    echo " lisps include abcl, allegro, allegromodern, ccl (clozure),"
15
    echo "  clisp, cmucl, ecl, gcl, gclcvs, sbcl, scl and xcl."
16 17
    echo "OPTIONS:"
    echo "    -d -- debug mode"
18 19
    echo "    -h -- show this message."
    echo "    -u -- upgrade tests."
20 21
}

22
unset DEBUG_ASDF_TEST upgrade clean_load
Robert P. Goldman's avatar
 
Robert P. Goldman committed
23

24
while getopts "duhc" OPTION
Robert P. Goldman's avatar
 
Robert P. Goldman committed
25
do
26
    case $OPTION in
Robert P. Goldman's avatar
 
Robert P. Goldman committed
27
        d)
28
            export DEBUG_ASDF_TEST=t
Robert P. Goldman's avatar
 
Robert P. Goldman committed
29 30
            ;;
        u)
31
            upgrade=t
Robert P. Goldman's avatar
 
Robert P. Goldman committed
32
            ;;
33 34 35
        c)
            clean_load=t
            ;;
Robert P. Goldman's avatar
 
Robert P. Goldman committed
36 37 38 39 40 41 42
        h)
            usage
            exit 1
            ;;
    esac
done
shift $(($OPTIND - 1))
43

44
if [ x"$1" = "xhelp" ]; then
Robert P. Goldman's avatar
 
Robert P. Goldman committed
45 46
    usage
    exit 1
47
fi
48
lisp=${1:-sbcl} ; shift
49

50
if [ -z "$*" ]; then
51 52
    scripts="*.script"
else
53
    scripts="$*"
54 55
fi

56 57
sok=1

Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
58 59
DO () { ( set -x ; "$@" ); }

60
do_tests() {
61
  command="$1" eval="$2"
62
  env | grep -i asdf
63
  rm -f ~/.cache/common-lisp/"`pwd`"/* || true
64
  ( cd .. && DO $command $eval '(or #.(load "test/script-support.lisp") #.(asdf-test::compile-asdf-script))' )
65 66 67 68
  if [ $? -ne 0 ] ; then
    echo "Compilation FAILED" >&2
  else
    echo "Compiled OK" >&2
69 70 71 72
    test_count=0
    test_pass=0
    test_fail=0
    failed_list=""
73 74
    for i in $scripts ;
    do
75 76
      echo "Testing: $i" >&2
      test_count=`expr "$test_count" + 1`
77
      rm -f ~/.cache/common-lisp/"`pwd`"/* || true
Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
78
      if DO $command $eval "(load \"$i\")" ; then
79
        echo "Using $command, $i passed" >&2
80 81
	test_pass=`expr "$test_pass" + 1`
      else
82
        echo "Using $command, $i failed" >&2
83 84 85 86
	test_fail=`expr "$test_fail" + 1`
	failed_list="$failed_list $i"
        sok=0
      fi
87 88
      echo >&2
      echo >&2
89 90
    done
    echo >&2
Gary King's avatar
Gary King committed
91
    echo "-#---------------------------------------" >&2
92
    echo "Using $command" >&2
93 94 95 96
    echo "Ran $test_count tests: " >&2
    echo "  $test_pass passing and $test_fail failing" >&2
    if [ $test_fail -eq 0 ] ; then
	echo "all tests apparently successful" >&2
97
        echo success > ../build/results/status
98 99 100
    else
	echo "failing test(s): $failed_list" >&2
    fi
Gary King's avatar
Gary King committed
101
    echo "-#---------------------------------------" >&2
102
    echo >&2
103
  fi
104 105
}

106
# terminate on error
107 108
set -e

109
command= flags= nodebug= eval=
110
case "$lisp" in
111 112
  abcl)
    command="${ABCL:-abcl}"
113
    flags="--noinit --nosystem --noinform"
114
    eval="--eval" ;;
115
  allegro)
Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
116
    command="${ALLEGRO:-alisp}"
117
    #flags="-q"
118 119
    nodebug="-batch"
    eval="-e" ;;
120
  allegromodern)
Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
121
    command="${ALLEGROMODERN:-mlisp}"
122 123 124
    flags="-q"
    nodebug="-batch"
    eval="-e" ;;
125
  ccl)
Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
126
    command="${CCL:-ccl}"
127 128 129
    flags="--no-init --quiet"
    nodebug="--batch"
    eval="--eval" ;;
130 131
  clisp)
    command="${CLISP:-clisp}"
132
    flags="-norc --silent -ansi -I "
133 134
    nodebug="-on-error exit"
    eval="-x" ;;
135
  cmucl)
136 137 138
    # cmucl likes to have its executable called lisp, but so does scl
    # Please use a symlink or an exec ... "$@" trampoline script.
    command="${CMUCL:-cmucl}"
139 140 141
    flags="-noinit"
    nodebug="-batch"
    eval="-eval" ;;
142
  ecl)
Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
143
    command="${ECL:-ecl}"
144 145
    flags="-norc -load sys:cmp"
    eval="-eval" ;;
146
  ecl_bytecodes)
147 148
    command="${ECL:-ecl}"
    flags="-norc -eval (ext::install-bytecodes-compiler)"
149
    eval="-eval" ;;
150 151 152 153 154
  gcl)
    export GCL_ANSI=t
    command="${GCL:-gcl}"
    flags="-batch"
    eval="-eval" ;;
155 156
  gclcvs)
    export GCL_ANSI=t
157
    command="${GCLCVS:-gclcvs}"
158 159
    flags="-batch"
    eval="-eval" ;;
160
  lispworks)
Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
161
    command="${LISPWORKS:-lispworks}"
162 163 164 165
    # If you have a licensed copy of lispworks,
    # you can obtain the "lispworks" binary with, e.g.
    # echo '(hcl:save-image "/lispworks" :environment nil)' > /tmp/build.lisp ;
    # ./lispworks-6-0-0-x86-linux -siteinit - -init - -build /tmp/build.lisp
166 167
    flags="-siteinit - -init -"
    eval="-eval" ;;
168 169 170 171
  mkcl)
    command="${MKCL:-mkcl}"
    flags="-norc"
    eval="-eval" ;;
172 173
  sbcl)
    command="${SBCL:-sbcl}"
174
    flags="--noinform --userinit /dev/null --sysinit /dev/null" # --eval (require'asdf)
175
    nodebug="--disable-debugger"
176
    eval="--eval" ;;
177 178 179 180 181
  scl)
    command="${SCL:-scl}"
    flags="-noinit"
    nodebug="-batch"
    eval="-eval" ;;
182 183 184 185
  xcl)
    command="${XCL:-xcl}"
    flags="--no-userinit --no-siteinit"
    eval="--eval" ;;
186 187 188 189
  *)
    echo "Unsupported lisp: $1" >&2
    echo "Please add support to run-tests.sh" >&2
    exit 42 ;;
190
esac
191

192
if ! type "$command" > /dev/null ; then
193
    echo "lisp implementation not found: $command" >&2
194 195 196
    exit 43
fi

197 198
ASDFDIR="$(cd .. ; /bin/pwd)"
export CL_SOURCE_REGISTRY="${ASDFDIR}"
199
export ASDF_OUTPUT_TRANSLATIONS="(:output-translations (\"${ASDFDIR}\" (\"${ASDFDIR}/build/fasls\" :implementation)) :ignore-inherited-configuration)"
Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
200 201

command="$command $flags"
202 203 204 205
if [ -z "${DEBUG_ASDF_TEST}" ] ; then
  command="$command $nodebug"
fi

Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
206

207
create_config () {
208
    mkdir -p ../build/test-source-registry-conf.d ../build/test-asdf-output-translations-conf.d
209
}
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
upgrade_tags () {
    if [ -n "$TEST_ASDF_TAGS" ] ; then
        echo $TEST_ASDF_TAGS ; return
    fi
    # 1.37 is the last release by Daniel Barlow
    # 1.97 is the last release before Gary King takes over
    # 1.369 is the last release by Gary King
    # 2.000 to 2.019 and 2.20 to 2.27 and beyond are Faré's "stable" releases
    echo 1.37 1.97 1.369
    git tag -l '2.0??'
    git tag -l '2.??'
}
extract_tagged_asdf () {
    ver=$1
    file=build/asdf-${tag}.lisp ;
    if [ ! -f $file ] ; then
        case $ver in
            1.*|2.0*|2.2[0-6])
                git show ${tag}:asdf.lisp > $file ;;
            *)
                echo "Don't know how to extract asdf.lisp for version $tag"
                exit 55
                ;;
        esac
    fi
}
run_upgrade_tests () {
    su=test/script-support.lisp
    lu="(load\"$su\")"
    lv="$command $eval $lu $eval" ;
    for tag in `upgrade_tags` ; do
        for x in load-system load-lisp load-lisp-compile-load-fasl load-fasl just-load-fasl ; do
            lo="(asdf-test::load-asdf-lisp \"${tag}\")" ;
            echo "Testing upgrade from ASDF ${tag} using method $x" ;
            extract_tagged_asdf $tag
            case ${lisp}:$tag:$x in
                abcl:2.0[01][1-9]:*|abcl:2.2[1-2]:*)
                    : Skip, because it is so damn slow ;;
                ccl:1.*|ccl:2.0[01]*)
                    : Skip, because ccl broke old asdf ;;
Francois-Rene Rideau's avatar
Francois-Rene Rideau committed
250 251
                clisp:1.*|clisp:2.0[01]*)
                    : Skip, because ccl broke old asdf ;;
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
                cmucl:1.*|cmucl:2.00*|cmucl:2.01[0-4]:*)
                    : Skip, CMUCL has problems before 2.014.7 due to source-registry upgrade ;;
                ecl*:1.*|ecl*:2.0[01]*|ecl*:2.20:*)
                    : Skip, because of various ASDF issues ;;
                gcl:1.*|gcl:2.0*|gcl:2.2[0-6]*) : Skip old versions that do not support GCL 2.6 ;;
                mkcl:1.*|mkcl:2.0[01]*|mkcl:2.2[0-3]:*)
                    : Skip, because MKCL is only supported starting with 2.24 ;;
                xcl:1.*|xcl:2.00*|xcl:2.01[0-4]:*|xcl:*)
                    : XCL support starts with ASDF 2.014.2 - It also hangs badly during upgrade. ;;
                *) (set -x ; case $x in
                            load-system) l="$lo (asdf-test::load-asdf-system)" ;;
                            load-lisp) l="$lo (asdf-test::load-asdf-lisp)" ;;
                            load-lisp-compile-load-fasl) l="$lo (asdf-test::compile-load-asdf)" ;;
                            load-fasl) l="$lo (asdf-test::load-asdf-fasl)" ;;
                            just-load-fasl) l="(asdf-test::load-asdf-fasl)" ;;
                            *) echo "WTF?" ; exit 2 ;; esac ;
                        $lv "(asdf-test::test-asdf $l)" ) ||
                    { echo "upgrade FAILED" ; exit 1 ;} ;; esac ;
        done ; done 2>&1 | tee build/results/${lisp}-upgrade.text
}
run_tests () {
  create_config
  mkdir -p ../build/results
  echo failure > ../build/results/status
    thedate=`date "+%Y-%m-%d"`
    do_tests "$command" "$eval" 2>&1 | \
	tee "../build/results/${lisp}.text" "../build/results/${lisp}-${thedate}.save"
    read a < ../build/results/status
  clean_up
281 282 283 284 285 286 287
  if [ success = "$a" ] ; then ## exit code
      return 0
  else
     echo "To view full results and failures, try the following command:" >&2
     echo "     less -p ABORTED build/results/${lisp}.text" >&2
     return 1
  fi
288
}
289
clean_up () {
290
    rm -rf ../build/test-source-registry-conf.d ../build/test-asdf-output-translations-conf.d
291
}
292
test_clean_load () {
293
    case $lisp in
294
        gcl|cmucl) return 0 ;; # These are hopeless
295
    esac
296 297 298 299 300 301 302 303 304 305 306 307 308 309
    nop=build/results/${lisp}-nop.text
    load=build/results/${lisp}-load.text
    ${command} ${eval} \
      '(or #.(setf *load-verbose* nil) #.(load "test/script-support.lisp") #.(asdf-test::exit-lisp 0))' \
        > $nop 2>&1
    ${command} ${eval} \
      '(or #.(setf *load-verbose* nil) #.(load "build/asdf.lisp") #.(asdf/image:quit 0))' \
        > $load 2>&1
    if diff $nop $load ; then
      echo "GOOD: Loading ASDF on $lisp produces no message" >&2 ; return 0
    else
      echo "BAD: Loading ASDF on $lisp produces messages" >&2 ; return 1
    fi
}
310

311
if [ -z "$command" ] ; then
Gary King's avatar
Gary King committed
312
    echo "Error: cannot find or do not know how to run Lisp named $lisp"
313 314
elif [ -n "$clean_load" ] ; then
    test_clean_load
315 316
elif [ -n "$upgrade" ] ; then
    run_upgrade_tests
317
else
318
    run_tests
319
fi