test-all.lisp 3.01 KB
Newer Older
1 2
(in-package :asdf-tools)

3 4 5 6 7 8 9 10
(defun call-with-all-lisps (thunk &key (lisps *test-lisps*) (fail-fast t))
  (let ((thunks
          (loop :for lisp :in (get-lisps lisps)
                :collect (lambda () (with-failure-context ((format nil "using ~(~A~)" lisp))
                                      (funcall thunk lisp))))))
    (if fail-fast
        (progn (map () 'funcall thunks) (success))
        (apply 'call-without-stopping thunks))))
11

12 13
(defmacro with-all-lisps ((lisp-var lisps &key (fail-fast t)) &body body)
  `(call-with-all-lisps (lambda (,lisp-var) ,@body) :lisps ,lisps :fail-fast ,fail-fast))
14

15
(deftestcmd test-all-clean-load (lisps)
16
  "test-clean-load on all lisps"
17
  (with-all-lisps (l lisps) (test-clean-load l)))
18

19
(deftestcmd test-all-scripts (lisps)
20
  "test-scripts on all lisps"
21
  (with-all-lisps (l lisps) (test-scripts l)))
22 23

(deftestcmd test-all-no-upgrade ()
24
  "test-basic, and test-all-script"
25
  (test-basic) (test-all-scripts))
26

27
(deftestcmd test-all-upgrade (upgrade-lisps)
28
  "test-upgrade on all lisps"
29
  (with-all-lisps (l upgrade-lisps) (test-upgrade l)))
30

31 32
(deftestcmd test-all ()
  "all tests"
33
  (test-all-no-upgrade) (test-all-upgrade))
34

35
(deftestcmd test-all-scripts-no-stop (lisps)
36
  "test-scripts on all lisps, no stop"
37
  (with-all-lisps (l lisps :fail-fast nil) (test-scripts l)))
38

39
(deftestcmd test-all-upgrade-no-stop (upgrade-lisps)
40
  "test-upgrade on all lisps, no stop"
41
  (with-all-lisps (l upgrade-lisps :fail-fast nil) (test-upgrade l)))
42 43

(deftestcmd test-all-no-upgrade-no-stop ()
44
  "all tests but upgrade on all lisps, no stop"
45 46 47 48 49
  (without-stopping ()
    (doc) (test-load-systems)
    (test-all-clean-load)
    (test-all-scripts-no-stop)
    (check-all-scripts-results)))
50

51
(deftestcmd test-all-no-stop () ;; TODO: pass arguments!
52
  "all tests on all lisps, no stop"
53 54 55 56 57 58
  (without-stopping ()
    (doc) (test-load-systems)
    (test-all-clean-load)
    (test-all-scripts-no-stop)
    (test-all-upgrade-no-stop)
    (check-all-results)))
59

60
(deftestcmd check-all-scripts-results ()
61 62
  "were there errors in test scripts?"
  (with-asdf-dir ()
63 64 65
    (let ((bad-lisps
            (run/lines
             `(grep "-L" "[5-9][0-9] passing and 0 failing"
66 67
                    ,@(mapcar (lambda (l) (format nil "build/results/~(~A~)-test.text" l))
                              *test-lisps*)))))
68 69
    (failure-if bad-lisps
                "Unexpected test failures on these implementations:~%~{~A~%~}" bad-lisps))))
70 71

(deftestcmd check-all-upgrade-results ()
72
  "were there upgrade test failures?"
73
  (with-asdf-dir ()
74 75 76
    (let ((bad-lisps
            (run/lines
             `(grep "-L" "Upgrade test succeeded for "
77 78
                    ,@(mapcar (lambda (l) (format nil "build/results/~(~A~)-upgrade.text" l))
                              *upgrade-test-lisps*)))))
79 80
      (failure-if bad-lisps
                  "Unexpected upgrade failures on these implementations:~%~{~A~%~}~%" bad-lisps))))
81 82

(deftestcmd check-all-results ()
83
  "were there failures in any scripts or upgrade?"
84 85
  (without-stopping ()
    (check-all-scripts-results) (check-all-upgrade-results)))