run-tests.sh 15.6 KB
Newer Older
Daniel Barlow's avatar
Daniel Barlow committed
1 2
#!/bin/sh

3
# run-tests {lisp invocation} {scripts-regex}
4 5 6 7
# - read lisp forms one at a time from standard input
# - quit with exit status 0 on getting eof
# - quit with exit status >0 if an unhandled error occurs

8 9 10 11 12 13
usage () {
    echo "$0 [lisp invocation] [scripts-regex]"
    echo " - read lisp forms one at a time from matching scripts"
    echo " - quit with exit status 0 on getting eof"
    echo " - quit with exit status >0 if an unhandled error occurs"
    echo " you need to supply the .script in the second argument"
14
    echo " lisps include abcl, allegro, allegromodern, ccl (clozure),"
15
    echo "  clisp, cmucl, ecl, gcl, gclcvs, sbcl, scl and xcl."
16 17
    echo "OPTIONS:"
    echo "    -d -- debug mode"
18 19
    echo "    -h -- show this message."
    echo "    -u -- upgrade tests."
20 21
    echo "    -c -- clean load test"
    echo "    -l -- load systems tests"
22
    echo "    -t -- test interactively"
23
    echo "    -H -- extract all asdf versions to upgrade from"
24 25
}

26
unset DEBUG_ASDF_TEST upgrade clean_load load_systems test_interactively extract_all
27
SHELL=/bin/sh
28
export SHELL DEBUG_ASDF_TEST GCL_ANSI ASDF_OUTPUT_TRANSLATIONS
Robert P. Goldman's avatar
 
Robert P. Goldman committed
29

30
while getopts "cdtHulhu" OPTION
Robert P. Goldman's avatar
 
Robert P. Goldman committed
31
do
32
    case $OPTION in
Robert P. Goldman's avatar
 
Robert P. Goldman committed
33
        d)
34
            DEBUG_ASDF_TEST=t
Robert P. Goldman's avatar
 
Robert P. Goldman committed
35 36
            ;;
        u)
37
            upgrade=t
Robert P. Goldman's avatar
 
Robert P. Goldman committed
38
            ;;
39 40 41
        c)
            clean_load=t
            ;;
42 43 44
        l)
            load_systems=t
            ;;
45 46 47
        t)
            test_interactively=t
            ;;
48 49 50
        H)
            extract_all=t
            ;;
Robert P. Goldman's avatar
 
Robert P. Goldman committed
51 52 53 54 55 56 57
        h)
            usage
            exit 1
            ;;
    esac
done
shift $(($OPTIND - 1))
58

59
if [ x"$1" = "xhelp" ]; then
Robert P. Goldman's avatar
 
Robert P. Goldman committed
60 61
    usage
    exit 1
62
fi
63
lisp=${1:-sbcl} ; shift
64

65

66 67 68 69 70 71 72 73 74 75 76 77 78
ECHO () { printf '%s\n' "$*" ;}
ECHOn () { printf '%s' "$*" ;}
DBG () { ECHO "$*" >& 2 ;}
simple_term_p () {
  case "$1" in *[!a-zA-Z0-9-+_,.:=%/]*) return 1 ;; *) return 0 ;; esac
}
kwote0 () { ECHOn "$1" | sed -e "s/\([\\\\\"\$\`]\)/\\\\\\1/g" ;}
kwote1 () { if simple_term_p "$1" ; then ECHOn "$1"
  else ECHOn "\"$(kwote0 "$1")\"" ; fi ;}
kwote () { ( set +x
  k="" ; for i ; do ECHOn "$k" ; kwote1 "$i" ; k=" " ; done ; echo
) }
DO () { kwote "$@" ; "$@" ; }
Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
79

80
do_tests () {
81 82 83 84 85
  if [ -z "$*" ]; then
       scripts="*.script"
  else
       scripts="$*"
  fi
86
  env | grep -i asdf
87
  rm -f ~/.cache/common-lisp/"`pwd`"/* || true
88 89 90
  ## We go through great lengths to avoid " in the command line,
  ## the quoting of which some Windows implementations get wrong.
  ## While we're at it, we also avoid spaces and backslashes.
91
  ( cd .. && DO $bcmd $eval '(or #.(load(string`|test/script-support.lisp|))#.(asdf-test::compile-asdf-script))' )
92 93
  if [ $? -ne 0 ] ; then
    echo "Compilation FAILED" >&2
94 95 96
    echo "you can retry compilation with:" >&2
    echo ./test/run-tests.sh $lisp >&2
    echo "or more interactively (and maybe with rlwrap or in emacs), start with:" >&2
97
    echo "$icmd" >&2
98 99
    echo "then copy/paste:" >&2
    echo '(load "test/script-support.lisp") (asdf-test::compile-asdf-script)' >&2
100 101
  else
    echo "Compiled OK" >&2
102 103 104 105
    test_count=0
    test_pass=0
    test_fail=0
    failed_list=""
106 107
    for i in $scripts ;
    do
108 109
      echo "Testing: $i" >&2
      test_count=`expr "$test_count" + 1`
110
      rm -f ~/.cache/common-lisp/"`pwd`"/* || true
111
      if DO $bcmd $eval "'(#.(load(string'|script-support.lisp|))#.(asdf-test::load-asdf)#.(asdf-test::frob-packages)#.(asdf-test::with-test()(load(string'|$i|))))" ; then
112
        echo "Using $command, $i passed" >&2
113 114
	test_pass=`expr "$test_pass" + 1`
      else
115
        echo "Using $command, $i failed" >&2
116 117
	test_fail=`expr "$test_fail" + 1`
	failed_list="$failed_list $i"
118 119 120
        echo "you can retry compilation with:" >&2
        echo ./test/run-tests.sh $lisp $i >&2
        echo "or more interactively (and maybe with rlwrap or in emacs), start with:" >&2
121
        echo "(cd test ; $icmd )" >&2
122
        echo "then copy/paste:" >&2
123
        echo "'(#.(load \"script-support.lisp\") #.(asdf-test::da) #.(load-asdf) #.(frob-packages) #.(load \"$i\"))" >&2
124
      fi
125 126
      echo >&2
      echo >&2
127 128
    done
    echo >&2
Gary King's avatar
Gary King committed
129
    echo "-#---------------------------------------" >&2
130
    echo "Using $command" >&2
131 132 133 134
    echo "Ran $test_count tests: " >&2
    echo "  $test_pass passing and $test_fail failing" >&2
    if [ $test_fail -eq 0 ] ; then
	echo "all tests apparently successful" >&2
135
        echo success > ../build/results/status
136 137 138
    else
	echo "failing test(s): $failed_list" >&2
    fi
Gary King's avatar
Gary King committed
139
    echo "-#---------------------------------------" >&2
140
    echo >&2
141
  fi
142 143
}

144 145 146
#
# not used currently but leave here for future reference.
#
147 148
case $(uname) in
    CYGWIN*) os=windows ;;
149 150 151
    Darwin) os=macos ;;
    Linux) os=linux ;;
    *) os=unknown ;;
152 153
esac

154 155


156
# terminate on error
157 158
set -e

159
command= flags= nodebug= eval= bcmd= icmd=
160
case "$lisp" in
161 162
  abcl)
    command="${ABCL:-abcl}"
163
    flags="--noinit --nosystem --noinform"
164 165
    eval="--eval"
    ;;
166 167 168 169 170 171 172
  allegro|allegro8|allegromodern|allegromodern8)
    case "$lisp" in
      allegro) command="${ALLEGRO:-alisp}" ;;
      allegro8) command="${ALLEGRO8:-alisp8}" ;;
      allegromodern) command="${ALLEGROMODERN:-mlisp}" ;;
      allegromodern8) command="${ALLEGROMODERN8:-mlisp8}" ;;
    esac
173 174
    flags="-q"
    nodebug="-batch"
175
    if [ "$os" = windows ] && [ -z "$ALLEGRO_NOISY" ] ; then bcmd="$command +c $flags" ; fi
176
    eval="-e" ;;
177
  ccl)
Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
178
    command="${CCL:-ccl}"
179 180 181
    flags="--no-init --quiet"
    nodebug="--batch"
    eval="--eval" ;;
182 183
  clisp)
    command="${CLISP:-clisp}"
184
    flags="-norc --silent -ansi -I "
185 186
    nodebug="-on-error exit"
    eval="-x" ;;
187
  cmucl)
188 189 190
    # cmucl likes to have its executable called lisp, but so does scl
    # Please use a symlink or an exec ... "$@" trampoline script.
    command="${CMUCL:-cmucl}"
191 192 193
    flags="-noinit"
    nodebug="-batch"
    eval="-eval" ;;
194
  ecl)
Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
195
    command="${ECL:-ecl}"
196 197
    flags="-norc -load sys:cmp"
    eval="-eval" ;;
198
  ecl_bytecodes)
199 200
    command="${ECL:-ecl}"
    flags="-norc -eval (ext::install-bytecodes-compiler)"
201
    eval="-eval" ;;
202
  gcl)
203
    GCL_ANSI=t
204 205 206
    command="${GCL:-gcl}"
    flags="-batch"
    eval="-eval" ;;
207
  gclcvs)
208
    GCL_ANSI=t
209
    command="${GCLCVS:-gclcvs}"
210 211
    flags="-batch"
    eval="-eval" ;;
212
  lispworks)
Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
213
    command="${LISPWORKS:-lispworks}"
214 215 216 217
    # If you have a licensed copy of lispworks,
    # you can obtain the "lispworks" binary with, e.g.
    # echo '(hcl:save-image "/lispworks" :environment nil)' > /tmp/build.lisp ;
    # ./lispworks-6-0-0-x86-linux -siteinit - -init - -build /tmp/build.lisp
218 219
    flags="-siteinit - -init -"
    eval="-eval" ;;
220 221 222 223
  mkcl)
    command="${MKCL:-mkcl}"
    flags="-norc"
    eval="-eval" ;;
224 225
  sbcl)
    command="${SBCL:-sbcl}"
226
    flags="--noinform --no-userinit --no-sysinit"
227
    nodebug="--disable-debugger"
228
    eval="--eval" ;;
229 230 231 232 233
  scl)
    command="${SCL:-scl}"
    flags="-noinit"
    nodebug="-batch"
    eval="-eval" ;;
234 235
  xcl)
    command="${XCL:-xcl}"
236
    flags="--no-userinit --no-siteinit --noinform"
237
    eval="--eval" ;;
238 239 240 241
  *)
    echo "Unsupported lisp: $1" >&2
    echo "Please add support to run-tests.sh" >&2
    exit 42 ;;
242
esac
243

244
if ! type "$command" > /dev/null ; then
245
    echo "lisp implementation not found: $command" >&2
246 247 248
    exit 43
fi

249
ASDFDIR="$(cd $(dirname $0)/.. ; /bin/pwd)"
250
: ${bcmd:=$command $flags} ${icmd:=$command $flags} # batch and interactive
251
if [ -z "${DEBUG_ASDF_TEST}" ] ; then
252
  bcmd="$bcmd $nodebug"
253 254
fi

Francois-Rene Rideau's avatar
1.647:  
Francois-Rene Rideau committed
255

256
create_config () {
257 258
    cd ${ASDFDIR}
    mkdir -p build/results/ build/test-source-registry-conf.d build/test-asdf-output-translations-conf.d
259
}
260
upgrade_tags () {
261 262
    if [ -n "$ASDF_UPGRADE_TEST_TAGS" ] ; then
        echo $ASDF_UPGRADE_TEST_TAGS ; return
263
    fi
264 265 266 267 268 269 270
    # REQUIRE is a magic tag meaning whatever your implementation provides, if anything
    #
    # 1.85 (2004-05-16) is the last release by Daniel Barlow (not 1.37, which is the README revision!)
    # 1.97 (2006-05-14) is the last release before Gary King takes over
    # 1.369 (2009-10-27) is the last release by Gary King
    #
    # 2.000 to 2.019 and 2.20 to 2.26 and beyond are Faré's "stable" ASDF 2 releases
271
    # 2.26.61 is the last single-file, single-package ASDF.
272 273 274 275 276 277 278 279 280 281 282 283
    # 2.27 and beyond are Faré's "stable" ASDF 3 pre-releases
    #
    # 2.000 (2010-05-31) was the first ASDF 2 release
    # 2.008 (2010-09-10) was a somewhat stable ASDF 2 release
    # 2.011 (2010-11-28) was used by CLISP 2.49, Debian squeeze, Ubuntu 10.04 LTS
    # 2.014.6 (2011-04-06) was used by Quicklisp in 2011
    # 2.019 (2011-11-27) was stable
    # 2.20 (2012-01-18) was in CCL 1.8, Ubuntu 12.04 LTS
    # 2.22 (2012-06-12) was used by debian wheezy
    # 2.26 (2012-10-30) was used by Quicklisp
    # 2.27 (2013-02-01) is the first ASDF 3 pre-release
    # 2.32 (2013-03-05) is the first really stable ASDF 3 pre-release
284
    # 3.0.1 (2013-05-16) is the first stable ASDF 3 release
285
    echo REQUIRE 1.85 1.97 1.369
286 287 288
    # git tag -l '2.0??'
    # git tag -l '2.??'
    echo 2.000 2.008 2.011 2.014.6 2.019 2.20 2.22 2.26
289 290
    echo 2.27 2.32
    git tag -l '3.0.[1-9]'
291
}
292
upgrade_methods () {
293 294
    if [ -n "$ASDF_UPGRADE_TEST_METHODS" ] ; then
        echo $ASDF_UPGRADE_TEST_METHODS ; return
295 296
    fi
    cat <<EOF
297
'load-asdf-lisp'load-asdf-lisp-clean
298
'load-asdf-lisp'load-asdf-system
299
'load-asdf-lisp'compile-load-asdf-upgrade
300 301
'load-asdf-lisp'load-asdf-fasl
()'load-asdf-fasl
302
'load-asdf-lisp-and-test-uiop'load-asdf-fasl
303 304
EOF
}
305
extract_tagged_asdf () {
306 307
    cd ${ASDFDIR}
    mkdir -p build/
308 309
    tag=$1
    if [ REQUIRE = "$tag" ] ; then return 0 ; fi
310 311
    file=build/asdf-${tag}.lisp ;
    if [ ! -f $file ] ; then
312
        case $tag in
313
            1.*|2.0*|2.2[0-6]|2.26.61)
314
                git show ${tag}:asdf.lisp > $file ;;
315
            2.2[7-9]*|2.[3-9]*|3.*)
316
                mkdir -p build/old/build
317
                git archive ${tag} | (cd build/old/ ; tar xf -)
318
                make -C build/old
319
                mv build/old/build/asdf.lisp build/asdf-${tag}.lisp
320 321
                rm -rf build/old ;;
             *)
322 323 324 325 326 327
                echo "Don't know how to extract asdf.lisp for version $tag"
                exit 55
                ;;
        esac
    fi
}
328 329 330 331 332
extract_all_tagged_asdf () {
    for i in `upgrade_tags` ; do
      extract_tagged_asdf $i
    done
}
333 334
valid_upgrade_test_p () {
    case "${1}:${2}:${3}" in
335 336 337
        # It's damn slow. Also, for some reason, we punt on anything earlier than 2.25,
        # and only need to test it once, below for 2.24.
        abcl:1.*|abcl:2.00[0-9]:*|abcl:201[0-9]:*|abcl:2.2[0-3]:*) : ;;
338 339
        # Skip allegro modern on 1.x -- fails for rpgoldman on his mac (!)
        allegromodern:1.*) : ;;
340 341
        # ccl fasl numbering broke loading of old asdf 2.0
        ccl:2.0[01]*) : ;;
342 343
        # my old ubuntu clisp 2.44.1 is wired in
        # with an antique ASDF 1.374 from CLC that can't be downgraded.
344 345
        # 2.00[0-7] use UID, which fails on that CLISP and was removed afterwards.
        # Note that for the longest time, CLISP has included 2.011 in its distribution.
346 347
        # Since we punt on the upgrade, let's only do the test once, for 2.26.
        clisp:2.00[0-7]:*|clisp:1.*|clisp:2.0[01]*|clisp:2.2[0-5]:*) : ;;
348
        # Skip, CMUCL has problems before 2.014.7 due to source-registry upgrade.
349 350 351 352 353 354 355 356 357
        # Weird unidentified problems before 2.018, so we punt equally for everything before,
        # and only need to test it once: above, for 2.017.
        cmucl:1.*|cmucl:2.00*|cmucl:2.01[0-6]:*) : ;;
        # Skip many ECL tests, for various ASDF issues
        ecl*:1.*|ecl*:2.0[01]*|ecl*:2.20:*) : ;;
        # GCL 2.6 is only supported with ASDF 2.27, so skip earlier versions
        gcl:1.*|gcl:2.0*|gcl:2.2[0-6]*) : ;;
        # MKCL is only supported starting with 2.24, so skip earlier versions
        mkcl:1.*|mkcl:2.0[01]*|mkcl:2.2[0-3]:*) : ;;
358 359
        # XCL support starts with ASDF 2.014.2
        # — It also dies during upgrade trying to show the backtrace.
360
        xcl:1.*|xcl:2.00*|xcl:2.01[0-4]:*|xcl:*) : ;;
361 362 363
        *) return 0 ;;
   esac
   return 1
364
}
365
run_upgrade_tests () {
366
    cd ${ASDFDIR}
367
    mkdir -p build/results/
368
    rm -f build/*.*f* uiop/*.*f* test/*.*f* ## Remove stale FASLs from ASDF 1.x, especially when different implementations have same name
369
    ASDF_OUTPUT_TRANSLATIONS="(:output-translations (\"${ASDFDIR}\" (\"${ASDFDIR}/build/fasls/\" :implementation \"asdf/\")) (t (\"${ASDFDIR}/build/fasls/\" :implementation \"root/\")) :ignore-inherited-configuration)"
370 371
    su=test/script-support.lisp
    for tag in `upgrade_tags` ; do
372 373 374 375
        for method in `upgrade_methods` ; do
            if valid_upgrade_test_p $lisp $tag $method ; then
                echo "Testing ASDF upgrade from ${tag} using method $method"
                extract_tagged_asdf $tag
376
                $bcmd $eval \
377
                "'(#.(load(string'|$su|))#.#.\`(in-package,:asdf-test)#.(test-upgrade$method\`|$tag|))" ||
378 379
                { echo "upgrade FAILED for $lisp from $tag using method $method" ;
                  echo "you can retry just that test with:" ;
380
                  echo ASDF_UPGRADE_TEST_TAGS=\"$tag\" ASDF_UPGRADE_TEST_METHODS=\"$method\" ./test/run-tests.sh -u $lisp ;
381
                  echo "or more interactively (and maybe with rlwrap or in emacs), start with:"
382
                  echo "$icmd"
383
                  echo "then copy/paste:"
384
                  echo "(load \"$su\") (asdf-test::da) (test-upgrade $method \"$tag\")"
385
                  exit 1 ;}
386
    fi ; done ; done 2>&1 | tee build/results/${lisp}-upgrade.text
387 388 389
}
run_tests () {
  create_config
390
  cd ./test/
391 392
  echo failure > ../build/results/status
    thedate=`date "+%Y-%m-%d"`
393
    rm -f "../build/results/${lisp}.text" || :
394
    do_tests "$@" 2>&1 | \
395 396 397
	tee "../build/results/${lisp}.text" "../build/results/${lisp}-${thedate}.save"
    read a < ../build/results/status
  clean_up
398 399 400 401 402 403 404
  if [ success = "$a" ] ; then ## exit code
      return 0
  else
     echo "To view full results and failures, try the following command:" >&2
     echo "     less -p ABORTED build/results/${lisp}.text" >&2
     return 1
  fi
405
}
406
clean_up () {
407
    rm -rf ../build/test-source-registry-conf.d ../build/test-asdf-output-translations-conf.d
408
}
409
test_clean_load () {
410
    case $lisp in
411
        gcl|cmucl) return 0 ;; # These are hopeless
412
    esac
413 414
    cd ${ASDFDIR}
    mkdir -p build/results/
415 416
    nop=build/results/${lisp}-nop.text
    load=build/results/${lisp}-load.text
417
    $bcmd $eval \
418
      '(or`#.(load(string`|test/script-support.lisp|)`:verbose():print())#.(asdf-test::exit-lisp`0))' \
419
        > $nop 2>&1
420
    $bcmd $eval \
421
      '(or`#.(load(string`|test/script-support.lisp|)`:verbose`():print`())#.(asdf-test::verbose())#.(load(string`|build/asdf.lisp|):verbose())#.(asdf/image:quit`0))' \
422 423 424 425 426 427 428
        > $load 2>&1
    if diff $nop $load ; then
      echo "GOOD: Loading ASDF on $lisp produces no message" >&2 ; return 0
    else
      echo "BAD: Loading ASDF on $lisp produces messages" >&2 ; return 1
    fi
}
429 430 431 432
test_load_systems () {
    case $lisp in
        gcl) return 0 ;; # This one is hopeless
    esac
433 434
    cd ${ASDFDIR}
    mkdir -p build/results/
435
    echo "Loading all these systems: $*"
436
    $bcmd $eval \
437
      "(or #.(load(string'|test/script-support.lisp|))#.(asdf-test::with-test()(asdf-test::test-load-systems $*)))" \
438 439
        2>&1 | tee build/results/${lisp}-systems.text
}
440
test_interactively () {
441 442
    cd ${ASDFDIR}
    mkdir -p build/results/
443
    rlwrap $icmd $eval "(or'#.(load(string'|test/script-support.lisp|))#.(asdf-test::interactive-test'($*)))"
444
}
445

446
if [ -z "$command" ] ; then
Gary King's avatar
Gary King committed
447
    echo "Error: cannot find or do not know how to run Lisp named $lisp"
448 449
elif [ -n "$test_interactively" ] ; then
    test_interactively "$@"
450 451
elif [ -n "$clean_load" ] ; then
    test_clean_load
452
elif [ -n "$load_systems" ] ; then
453
    test_load_systems "$@"
454 455
elif [ -n "$upgrade" ] ; then
    run_upgrade_tests
456 457
elif [ -n "$extract_all" ] ; then
    extract_all_tagged_asdf
458
else
459
    run_tests "$@"
460
fi ; exit # NB: "; exit" makes it robust wrt the script being modified while running.