run-tests.sh 9.44 KB
Newer Older
Daniel Barlow's avatar
Daniel Barlow committed
1
2
#!/bin/sh

3
# do_tests {lisp invocation} {scripts-regex}
4
5
6
7
# - read lisp forms one at a time from standard input
# - quit with exit status 0 on getting eof
# - quit with exit status >0 if an unhandled error occurs

8
9
10
11
12
13
usage () {
    echo "$0 [lisp invocation] [scripts-regex]"
    echo " - read lisp forms one at a time from matching scripts"
    echo " - quit with exit status 0 on getting eof"
    echo " - quit with exit status >0 if an unhandled error occurs"
    echo " you need to supply the .script in the second argument"
14
    echo " lisps include abcl, allegro, allegromodern, ccl (clozure),"
15
    echo "  clisp, cmucl, ecl, gcl, gclcvs, sbcl, scl and xcl."
16
17
    echo "OPTIONS:"
    echo "    -d -- debug mode"
18
19
    echo "    -h -- show this message."
    echo "    -u -- upgrade tests."
20
21
}

22
unset DEBUG_ASDF_TEST upgrade clean_load
Robert P. Goldman's avatar
   
Robert P. Goldman committed
23

24
while getopts "duhc" OPTION
Robert P. Goldman's avatar
   
Robert P. Goldman committed
25
do
26
    case $OPTION in
Robert P. Goldman's avatar
   
Robert P. Goldman committed
27
        d)
28
            export DEBUG_ASDF_TEST=t
Robert P. Goldman's avatar
   
Robert P. Goldman committed
29
30
            ;;
        u)
31
            upgrade=t
Robert P. Goldman's avatar
   
Robert P. Goldman committed
32
            ;;
33
34
35
        c)
            clean_load=t
            ;;
Robert P. Goldman's avatar
   
Robert P. Goldman committed
36
37
38
39
40
41
42
        h)
            usage
            exit 1
            ;;
    esac
done
shift $(($OPTIND - 1))
43

44
if [ x"$1" = "xhelp" ]; then
Robert P. Goldman's avatar
   
Robert P. Goldman committed
45
46
    usage
    exit 1
47
fi
48
lisp=${1:-sbcl} ; shift
49

50
if [ -z "$*" ]; then
51
52
    scripts="*.script"
else
53
    scripts="$*"
54
55
fi

56
57
sok=1

Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
58
59
DO () { ( set -x ; "$@" ); }

60
do_tests() {
61
  command="$1" eval="$2"
62
  env | grep -i asdf
63
  rm -f ~/.cache/common-lisp/"`pwd`"/* || true
64
  ( cd .. && DO $command $eval '(or #.(load "test/script-support.lisp") #.(asdf-test::compile-asdf-script))' )
65
66
67
68
  if [ $? -ne 0 ] ; then
    echo "Compilation FAILED" >&2
  else
    echo "Compiled OK" >&2
69
70
71
72
    test_count=0
    test_pass=0
    test_fail=0
    failed_list=""
73
74
    for i in $scripts ;
    do
75
76
      echo "Testing: $i" >&2
      test_count=`expr "$test_count" + 1`
77
      rm -f ~/.cache/common-lisp/"`pwd`"/* || true
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
78
      if DO $command $eval "(load \"$i\")" ; then
79
        echo "Using $command, $i passed" >&2
80
81
	test_pass=`expr "$test_pass" + 1`
      else
82
        echo "Using $command, $i failed" >&2
83
84
85
86
	test_fail=`expr "$test_fail" + 1`
	failed_list="$failed_list $i"
        sok=0
      fi
87
88
      echo >&2
      echo >&2
89
90
    done
    echo >&2
Gary King's avatar
Gary King committed
91
    echo "-#---------------------------------------" >&2
92
    echo "Using $command" >&2
93
94
95
96
    echo "Ran $test_count tests: " >&2
    echo "  $test_pass passing and $test_fail failing" >&2
    if [ $test_fail -eq 0 ] ; then
	echo "all tests apparently successful" >&2
97
        echo success > ../build/results/status
98
99
100
    else
	echo "failing test(s): $failed_list" >&2
    fi
Gary King's avatar
Gary King committed
101
    echo "-#---------------------------------------" >&2
102
    echo >&2
103
  fi
104
105
}

106
# terminate on error
107
108
set -e

109
command= flags= nodebug= eval=
110
case "$lisp" in
111
112
  abcl)
    command="${ABCL:-abcl}"
113
    flags="--noinit --nosystem --noinform"
114
    eval="--eval" ;;
115
  allegro)
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
116
    command="${ALLEGRO:-alisp}"
117
118
119
    flags="-q"
    nodebug="-batch"
    eval="-e" ;;
120
  allegromodern)
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
121
    command="${ALLEGROMODERN:-mlisp}"
122
123
124
    flags="-q"
    nodebug="-batch"
    eval="-e" ;;
125
  ccl)
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
126
    command="${CCL:-ccl}"
127
128
129
    flags="--no-init --quiet"
    nodebug="--batch"
    eval="--eval" ;;
130
131
  clisp)
    command="${CLISP:-clisp}"
132
    flags="-norc --silent -ansi -I "
133
134
    nodebug="-on-error exit"
    eval="-x" ;;
135
  cmucl)
136
137
138
    # cmucl likes to have its executable called lisp, but so does scl
    # Please use a symlink or an exec ... "$@" trampoline script.
    command="${CMUCL:-cmucl}"
139
140
141
    flags="-noinit"
    nodebug="-batch"
    eval="-eval" ;;
142
  ecl)
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
143
    command="${ECL:-ecl}"
144
145
    flags="-norc -load sys:cmp"
    eval="-eval" ;;
146
  ecl_bytecodes)
147
148
    command="${ECL:-ecl}"
    flags="-norc -eval (ext::install-bytecodes-compiler)"
149
    eval="-eval" ;;
150
151
152
153
154
  gcl)
    export GCL_ANSI=t
    command="${GCL:-gcl}"
    flags="-batch"
    eval="-eval" ;;
155
156
  gclcvs)
    export GCL_ANSI=t
157
    command="${GCLCVS:-gclcvs}"
158
159
    flags="-batch"
    eval="-eval" ;;
160
  lispworks)
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
161
    command="${LISPWORKS:-lispworks}"
162
163
164
165
    # If you have a licensed copy of lispworks,
    # you can obtain the "lispworks" binary with, e.g.
    # echo '(hcl:save-image "/lispworks" :environment nil)' > /tmp/build.lisp ;
    # ./lispworks-6-0-0-x86-linux -siteinit - -init - -build /tmp/build.lisp
166
167
    flags="-siteinit - -init -"
    eval="-eval" ;;
168
169
170
171
  mkcl)
    command="${MKCL:-mkcl}"
    flags="-norc"
    eval="-eval" ;;
172
173
  sbcl)
    command="${SBCL:-sbcl}"
174
    flags="--noinform --userinit /dev/null --sysinit /dev/null" # --eval (require'asdf)
175
    nodebug="--disable-debugger"
176
    eval="--eval" ;;
177
178
179
180
181
  scl)
    command="${SCL:-scl}"
    flags="-noinit"
    nodebug="-batch"
    eval="-eval" ;;
182
183
184
185
  xcl)
    command="${XCL:-xcl}"
    flags="--no-userinit --no-siteinit"
    eval="--eval" ;;
186
187
188
189
  *)
    echo "Unsupported lisp: $1" >&2
    echo "Please add support to run-tests.sh" >&2
    exit 42 ;;
190
esac
191

192
if ! type "$command" > /dev/null ; then
193
    echo "lisp implementation not found: $command" >&2
194
195
196
    exit 43
fi

197
198
ASDFDIR="$(cd .. ; /bin/pwd)"
export CL_SOURCE_REGISTRY="${ASDFDIR}"
199
export ASDF_OUTPUT_TRANSLATIONS="(:output-translations (\"${ASDFDIR}\" (\"${ASDFDIR}/build/fasls\" :implementation)) :ignore-inherited-configuration)"
Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
200
201

command="$command $flags"
202
203
204
205
if [ -z "${DEBUG_ASDF_TEST}" ] ; then
  command="$command $nodebug"
fi

Francois-Rene Rideau's avatar
1.647:    
Francois-Rene Rideau committed
206

207
create_config () {
208
    mkdir -p ../build/test-source-registry-conf.d ../build/test-asdf-output-translations-conf.d
209
}
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
upgrade_tags () {
    if [ -n "$TEST_ASDF_TAGS" ] ; then
        echo $TEST_ASDF_TAGS ; return
    fi
    # 1.37 is the last release by Daniel Barlow
    # 1.97 is the last release before Gary King takes over
    # 1.369 is the last release by Gary King
    # 2.000 to 2.019 and 2.20 to 2.27 and beyond are Faré's "stable" releases
    echo 1.37 1.97 1.369
    git tag -l '2.0??'
    git tag -l '2.??'
}
extract_tagged_asdf () {
    ver=$1
    file=build/asdf-${tag}.lisp ;
    if [ ! -f $file ] ; then
        case $ver in
            1.*|2.0*|2.2[0-6])
                git show ${tag}:asdf.lisp > $file ;;
            *)
                echo "Don't know how to extract asdf.lisp for version $tag"
                exit 55
                ;;
        esac
    fi
}
run_upgrade_tests () {
    su=test/script-support.lisp
    lu="(load\"$su\")"
    lv="$command $eval $lu $eval" ;
    for tag in `upgrade_tags` ; do
        for x in load-system load-lisp load-lisp-compile-load-fasl load-fasl just-load-fasl ; do
            lo="(asdf-test::load-asdf-lisp \"${tag}\")" ;
            echo "Testing upgrade from ASDF ${tag} using method $x" ;
            extract_tagged_asdf $tag
            case ${lisp}:$tag:$x in
                abcl:2.0[01][1-9]:*|abcl:2.2[1-2]:*)
                    : Skip, because it is so damn slow ;;
                ccl:1.*|ccl:2.0[01]*)
                    : Skip, because ccl broke old asdf ;;
                cmucl:1.*|cmucl:2.00*|cmucl:2.01[0-4]:*)
                    : Skip, CMUCL has problems before 2.014.7 due to source-registry upgrade ;;
                ecl*:1.*|ecl*:2.0[01]*|ecl*:2.20:*)
                    : Skip, because of various ASDF issues ;;
                gcl:1.*|gcl:2.0*|gcl:2.2[0-6]*) : Skip old versions that do not support GCL 2.6 ;;
                mkcl:1.*|mkcl:2.0[01]*|mkcl:2.2[0-3]:*)
                    : Skip, because MKCL is only supported starting with 2.24 ;;
                xcl:1.*|xcl:2.00*|xcl:2.01[0-4]:*|xcl:*)
                    : XCL support starts with ASDF 2.014.2 - It also hangs badly during upgrade. ;;
                *) (set -x ; case $x in
                            load-system) l="$lo (asdf-test::load-asdf-system)" ;;
                            load-lisp) l="$lo (asdf-test::load-asdf-lisp)" ;;
                            load-lisp-compile-load-fasl) l="$lo (asdf-test::compile-load-asdf)" ;;
                            load-fasl) l="$lo (asdf-test::load-asdf-fasl)" ;;
                            just-load-fasl) l="(asdf-test::load-asdf-fasl)" ;;
                            *) echo "WTF?" ; exit 2 ;; esac ;
                        $lv "(asdf-test::test-asdf $l)" ) ||
                    { echo "upgrade FAILED" ; exit 1 ;} ;; esac ;
        done ; done 2>&1 | tee build/results/${lisp}-upgrade.text
}
run_tests () {
  create_config
  mkdir -p ../build/results
  echo failure > ../build/results/status
    thedate=`date "+%Y-%m-%d"`
    do_tests "$command" "$eval" 2>&1 | \
	tee "../build/results/${lisp}.text" "../build/results/${lisp}-${thedate}.save"
    read a < ../build/results/status
  clean_up
279
280
281
282
283
284
285
  if [ success = "$a" ] ; then ## exit code
      return 0
  else
     echo "To view full results and failures, try the following command:" >&2
     echo "     less -p ABORTED build/results/${lisp}.text" >&2
     return 1
  fi
286
}
287
clean_up () {
288
    rm -rf ../build/test-source-registry-conf.d ../build/test-asdf-output-translations-conf.d
289
}
290
test_clean_load () {
291
292
293
    case $lisp in
        gcl) return 0 ;; # GCL 2.6 is hopeless
    esac
294
295
296
297
298
299
300
301
302
303
304
305
306
307
    nop=build/results/${lisp}-nop.text
    load=build/results/${lisp}-load.text
    ${command} ${eval} \
      '(or #.(setf *load-verbose* nil) #.(load "test/script-support.lisp") #.(asdf-test::exit-lisp 0))' \
        > $nop 2>&1
    ${command} ${eval} \
      '(or #.(setf *load-verbose* nil) #.(load "build/asdf.lisp") #.(asdf/image:quit 0))' \
        > $load 2>&1
    if diff $nop $load ; then
      echo "GOOD: Loading ASDF on $lisp produces no message" >&2 ; return 0
    else
      echo "BAD: Loading ASDF on $lisp produces messages" >&2 ; return 1
    fi
}
308

309
if [ -z "$command" ] ; then
Gary King's avatar
Gary King committed
310
    echo "Error: cannot find or do not know how to run Lisp named $lisp"
311
312
elif [ -n "$clean_load" ] ; then
    test_clean_load
313
314
elif [ -n "$upgrade" ] ; then
    run_upgrade_tests
315
else
316
    run_tests
317
fi