run-tests.sh 9.72 KB
Newer Older
Daniel Barlow's avatar
Daniel Barlow committed
1
2
#!/bin/sh

3
# do_tests {lisp invocation} {scripts-regex}
4
5
6
7
# - read lisp forms one at a time from standard input
# - quit with exit status 0 on getting eof
# - quit with exit status >0 if an unhandled error occurs

8
9
10
11
12
13
usage () {
    echo "$0 [lisp invocation] [scripts-regex]"
    echo " - read lisp forms one at a time from matching scripts"
    echo " - quit with exit status 0 on getting eof"
    echo " - quit with exit status >0 if an unhandled error occurs"
    echo " you need to supply the .script in the second argument"
14
    echo " lisps include abcl, allegro, allegromodern, ccl (clozure),"
15
    echo "  clisp, cmucl, ecl, gcl, gclcvs, sbcl, scl and xcl."
16
17
    echo "OPTIONS:"
    echo "    -d -- debug mode"
18
19
    echo "    -h -- show this message."
    echo "    -u -- upgrade tests."
20
21
}

22
unset DEBUG_ASDF_TEST upgrade clean_load
Robert P. Goldman's avatar
   
Robert P. Goldman committed
23

24
while getopts "duhc" OPTION
Robert P. Goldman's avatar
   
Robert P. Goldman committed
25
do
26
    case $OPTION in
Robert P. Goldman's avatar
   
Robert P. Goldman committed
27
        d)
28
            export DEBUG_ASDF_TEST=t
Robert P. Goldman's avatar
   
Robert P. Goldman committed
29
30
            ;;
        u)
31
            upgrade=t
Robert P. Goldman's avatar
   
Robert P. Goldman committed
32
            ;;
33
34
35
        c)
            clean_load=t
            ;;
Robert P. Goldman's avatar
   
Robert P. Goldman committed
36
37
38
39
40
41
42
        h)
            usage
            exit 1
            ;;
    esac
done
shift $(($OPTIND - 1))
43

44
if [ x"$1" = "xhelp" ]; then
Robert P. Goldman's avatar
   
Robert P. Goldman committed
45
46
    usage
    exit 1
47
fi
48
lisp=${1:-sbcl} ; shift
49

50
if [ -z "$*" ]; then
51
52
    scripts="*.script"
else
53
    scripts="$*"
54
55
fi

56
57
sok=1

Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
58
59
DO () { ( set -x ; "$@" ); }

60
do_tests() {
61
  command="$1" eval="$2"
62
  env | grep -i asdf
63
  rm -f ~/.cache/common-lisp/"`pwd`"/* || true
64
  ( cd .. && DO $command $eval '(or #.(load "test/script-support.lisp") #.(asdf-test::compile-asdf-script))' )
65
66
67
68
  if [ $? -ne 0 ] ; then
    echo "Compilation FAILED" >&2
  else
    echo "Compiled OK" >&2
69
70
71
72
    test_count=0
    test_pass=0
    test_fail=0
    failed_list=""
73
74
    for i in $scripts ;
    do
75
76
      echo "Testing: $i" >&2
      test_count=`expr "$test_count" + 1`
77
      rm -f ~/.cache/common-lisp/"`pwd`"/* || true
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
78
      if DO $command $eval "(load \"$i\")" ; then
79
        echo "Using $command, $i passed" >&2
80
81
	test_pass=`expr "$test_pass" + 1`
      else
82
        echo "Using $command, $i failed" >&2
83
84
85
86
	test_fail=`expr "$test_fail" + 1`
	failed_list="$failed_list $i"
        sok=0
      fi
87
88
      echo >&2
      echo >&2
89
90
    done
    echo >&2
Gary King's avatar
Gary King committed
91
    echo "-#---------------------------------------" >&2
92
    echo "Using $command" >&2
93
94
95
96
    echo "Ran $test_count tests: " >&2
    echo "  $test_pass passing and $test_fail failing" >&2
    if [ $test_fail -eq 0 ] ; then
	echo "all tests apparently successful" >&2
97
        echo success > ../build/results/status
98
99
100
    else
	echo "failing test(s): $failed_list" >&2
    fi
Gary King's avatar
Gary King committed
101
    echo "-#---------------------------------------" >&2
102
    echo >&2
103
  fi
104
105
}

106
# terminate on error
107
108
set -e

109
command= flags= nodebug= eval=
110
case "$lisp" in
111
112
  abcl)
    command="${ABCL:-abcl}"
113
    flags="--noinit --nosystem --noinform"
114
    eval="--eval" ;;
115
  allegro)
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
116
    command="${ALLEGRO:-alisp}"
117
    #flags="-q"
118
119
    nodebug="-batch"
    eval="-e" ;;
120
  allegromodern)
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
121
    command="${ALLEGROMODERN:-mlisp}"
122
123
124
    flags="-q"
    nodebug="-batch"
    eval="-e" ;;
125
  ccl)
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
126
    command="${CCL:-ccl}"
127
128
129
    flags="--no-init --quiet"
    nodebug="--batch"
    eval="--eval" ;;
130
131
  clisp)
    command="${CLISP:-clisp}"
132
    flags="-norc --silent -ansi -I "
133
134
    nodebug="-on-error exit"
    eval="-x" ;;
135
  cmucl)
136
137
138
    # cmucl likes to have its executable called lisp, but so does scl
    # Please use a symlink or an exec ... "$@" trampoline script.
    command="${CMUCL:-cmucl}"
139
140
141
    flags="-noinit"
    nodebug="-batch"
    eval="-eval" ;;
142
  ecl)
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
143
    command="${ECL:-ecl}"
144
145
    flags="-norc -load sys:cmp"
    eval="-eval" ;;
146
  ecl_bytecodes)
147
148
    command="${ECL:-ecl}"
    flags="-norc -eval (ext::install-bytecodes-compiler)"
149
    eval="-eval" ;;
150
151
152
153
154
  gcl)
    export GCL_ANSI=t
    command="${GCL:-gcl}"
    flags="-batch"
    eval="-eval" ;;
155
156
  gclcvs)
    export GCL_ANSI=t
157
    command="${GCLCVS:-gclcvs}"
158
159
    flags="-batch"
    eval="-eval" ;;
160
  lispworks)
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
161
    command="${LISPWORKS:-lispworks}"
162
163
164
165
    # If you have a licensed copy of lispworks,
    # you can obtain the "lispworks" binary with, e.g.
    # echo '(hcl:save-image "/lispworks" :environment nil)' > /tmp/build.lisp ;
    # ./lispworks-6-0-0-x86-linux -siteinit - -init - -build /tmp/build.lisp
166
167
    flags="-siteinit - -init -"
    eval="-eval" ;;
168
169
170
171
  mkcl)
    command="${MKCL:-mkcl}"
    flags="-norc"
    eval="-eval" ;;
172
173
  sbcl)
    command="${SBCL:-sbcl}"
174
    flags="--noinform --userinit /dev/null --sysinit /dev/null" # --eval (require'asdf)
175
    nodebug="--disable-debugger"
176
    eval="--eval" ;;
177
178
179
180
181
  scl)
    command="${SCL:-scl}"
    flags="-noinit"
    nodebug="-batch"
    eval="-eval" ;;
182
183
184
185
  xcl)
    command="${XCL:-xcl}"
    flags="--no-userinit --no-siteinit"
    eval="--eval" ;;
186
187
188
189
  *)
    echo "Unsupported lisp: $1" >&2
    echo "Please add support to run-tests.sh" >&2
    exit 42 ;;
190
esac
191

192
if ! type "$command" > /dev/null ; then
193
    echo "lisp implementation not found: $command" >&2
194
195
196
    exit 43
fi

197
ASDFDIR="$(cd $(dirname $0)/.. ; /bin/pwd)"
198
export CL_SOURCE_REGISTRY="${ASDFDIR}"
199
export ASDF_OUTPUT_TRANSLATIONS="(:output-translations (\"${ASDFDIR}\" (\"${ASDFDIR}/build/fasls\" :implementation)) :ignore-inherited-configuration)"
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
200
201

command="$command $flags"
202
203
204
205
if [ -z "${DEBUG_ASDF_TEST}" ] ; then
  command="$command $nodebug"
fi

Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
206

207
create_config () {
208
    mkdir -p ../build/test-source-registry-conf.d ../build/test-asdf-output-translations-conf.d
209
}
210
211
212
213
upgrade_tags () {
    if [ -n "$TEST_ASDF_TAGS" ] ; then
        echo $TEST_ASDF_TAGS ; return
    fi
214
    # REQUIRE is a magic tag meaning whatever your implementation provides
215
216
217
218
    # 1.37 is the last release by Daniel Barlow
    # 1.97 is the last release before Gary King takes over
    # 1.369 is the last release by Gary King
    # 2.000 to 2.019 and 2.20 to 2.27 and beyond are Faré's "stable" releases
219
220
    # 2.26.61 is the last single-package ASDF.
    echo REQUIRE 1.37 1.97 1.369 2.26.61
221
222
223
    git tag -l '2.0??'
    git tag -l '2.??'
}
224
225
226
227
228
229
230
231
232
233
234
upgrade_methods () {
    if [ -n "$TEST_ASDF_METHODS" ] ; then
        echo $TEST_ASDF_METHODS ; return
    fi
    cat <<EOF
'load-asdf-lisp'load-asdf-system
'load-asdf-lisp'compile-load-asdf
'load-asdf-lisp'load-asdf-fasl
()'load-asdf-fasl
EOF
}
235
236
extract_tagged_asdf () {
    ver=$1
237
    if [ REQUIRE = "$ver" ] ; then return 0 ; fi
238
239
240
    file=build/asdf-${tag}.lisp ;
    if [ ! -f $file ] ; then
        case $ver in
241
            1.*|2.0*|2.2[0-6]|2.26.61)
242
243
244
245
246
247
248
249
                git show ${tag}:asdf.lisp > $file ;;
            *)
                echo "Don't know how to extract asdf.lisp for version $tag"
                exit 55
                ;;
        esac
    fi
}
250
251
252
253
254
255
256
257
258
259
valid_upgrade_test_p () {
    case "${1}:${2}:${3}" in
        abcl:2.0[01][1-9]:*|abcl:2.2[1-2]:*)
            : Skip, because it is so damn slow ;;
        ccl:1.*|ccl:2.0[01]*)
            : Skip, because ccl broke old asdf ;;
        clisp:1.??*|clisp:2.00[0-7]:*)
            # my old ubuntu clisp 2.44.1 is wired in with an antique ASDF 1 from CLC that can't be downgraded.
            # 2.00[0-7] use UID, which fails on that CLISP and was removed afterwards.
            # Note that for the longest time, CLISP has included 2.011 in its distribution.
260
            : ;;
261
262
263
264
265
266
267
268
269
270
271
272
        cmucl:1.*|cmucl:2.00*|cmucl:2.01[0-4]:*)
            : Skip, CMUCL has problems before 2.014.7 due to source-registry upgrade ;;
        ecl*:1.*|ecl*:2.0[01]*|ecl*:2.20:*)
            : Skip, because of various ASDF issues ;;
        gcl:1.*|gcl:2.0*|gcl:2.2[0-6]*) : Skip old versions that do not support GCL 2.6 ;;
        mkcl:1.*|mkcl:2.0[01]*|mkcl:2.2[0-3]:*)
            : Skip, because MKCL is only supported starting with 2.24 ;;
        xcl:1.*|xcl:2.00*|xcl:2.01[0-4]:*|xcl:*)
            : XCL support starts with ASDF 2.014.2 - It also hangs badly during upgrade. ;;
        *) return 0 ;;
   esac
   return 1
273
}
274
run_upgrade_tests () {
275
    cd ${ASDFDIR}
276
277
    su=test/script-support.lisp
    for tag in `upgrade_tags` ; do
278
279
280
281
282
283
284
        for method in `upgrade_methods` ; do
            if valid_upgrade_test_p $lisp $tag $method ; then
                echo "Testing ASDF upgrade from ${tag} using method $method"
                extract_tagged_asdf $tag
                $command $eval \
                "'(#.(load\"$su\")#.(in-package :asdf-test)#.(test-upgrade $method \"$tag\"))" ||
                { echo "upgrade FAILED for $lisp from $tag using method $method" ; exit 1 ;}
285
    fi ; done ; done 2>&1 | tee build/results/${lisp}-upgrade.text
286
287
}
run_tests () {
288
  cd ${ASDFDIR}/test
289
290
291
292
293
294
295
296
  create_config
  mkdir -p ../build/results
  echo failure > ../build/results/status
    thedate=`date "+%Y-%m-%d"`
    do_tests "$command" "$eval" 2>&1 | \
	tee "../build/results/${lisp}.text" "../build/results/${lisp}-${thedate}.save"
    read a < ../build/results/status
  clean_up
297
298
299
300
301
302
303
  if [ success = "$a" ] ; then ## exit code
      return 0
  else
     echo "To view full results and failures, try the following command:" >&2
     echo "     less -p ABORTED build/results/${lisp}.text" >&2
     return 1
  fi
304
}
305
clean_up () {
306
    rm -rf ../build/test-source-registry-conf.d ../build/test-asdf-output-translations-conf.d
307
}
308
test_clean_load () {
309
    case $lisp in
310
        gcl|cmucl) return 0 ;; # These are hopeless
311
    esac
312
313
314
315
316
317
318
319
320
321
322
323
324
325
    nop=build/results/${lisp}-nop.text
    load=build/results/${lisp}-load.text
    ${command} ${eval} \
      '(or #.(setf *load-verbose* nil) #.(load "test/script-support.lisp") #.(asdf-test::exit-lisp 0))' \
        > $nop 2>&1
    ${command} ${eval} \
      '(or #.(setf *load-verbose* nil) #.(load "build/asdf.lisp") #.(asdf/image:quit 0))' \
        > $load 2>&1
    if diff $nop $load ; then
      echo "GOOD: Loading ASDF on $lisp produces no message" >&2 ; return 0
    else
      echo "BAD: Loading ASDF on $lisp produces messages" >&2 ; return 1
    fi
}
326

327
if [ -z "$command" ] ; then
Gary King's avatar
Gary King committed
328
    echo "Error: cannot find or do not know how to run Lisp named $lisp"
329
330
elif [ -n "$clean_load" ] ; then
    test_clean_load
331
332
elif [ -n "$upgrade" ] ; then
    run_upgrade_tests
333
else
334
    run_tests
335
fi