test-all.lisp 3.1 KB
Newer Older
1 2
(in-package :asdf-tools)

3
(defun call-with-all-lisps (thunk &key (lisps *test-lisps*) muffle-failures)
4 5
  (let ((thunks
          (loop :for lisp :in (get-lisps lisps)
6
                :collect (let ((l lisp))
7
                           (lambda () (with-failure-context (:name (format nil "using ~(~A~)" l))
8
                                        (funcall thunk l)))))))
9 10 11
    (if muffle-failures
        (call-without-stopping thunks)
        (progn (map () 'funcall thunks) (success)))))
12

13 14
(defmacro with-all-lisps ((lisp-var lisps &key muffle-failures) &body body)
  `(call-with-all-lisps (lambda (,lisp-var) ,@body) :lisps ,lisps :muffle-failures ,muffle-failures))
15

16
(deftestcmd test-all-clean-load (lisps)
17
  "test-clean-load on all lisps"
18
  (with-all-lisps (l lisps) (test-clean-load l)))
19

20
(deftestcmd test-all-scripts (lisps)
21
  "test-scripts on all lisps"
22
  (with-all-lisps (l lisps) (test-scripts l)))
23 24

(deftestcmd test-all-no-upgrade ()
25
  "test-basic, and test-all-script"
26
  (test-basic) (test-all-scripts))
27

28
(deftestcmd test-all-upgrade (upgrade-lisps)
29
  "test-upgrade on all lisps"
30
  (with-all-lisps (l upgrade-lisps) (test-upgrade l)))
31

32 33
(deftestcmd test-all ()
  "all tests"
34
  (test-all-no-upgrade) (test-all-upgrade))
35

36
(deftestcmd test-all-scripts-no-stop (lisps)
37
  "test-scripts on all lisps, no stop"
38
  (with-all-lisps (l lisps :muffle-failures t) (test-scripts l)))
39

40
(deftestcmd test-all-upgrade-no-stop (upgrade-lisps)
41
  "test-upgrade on all lisps, no stop"
42
  (with-all-lisps (l upgrade-lisps :muffle-failures t) (test-upgrade l)))
43 44

(deftestcmd test-all-no-upgrade-no-stop ()
45
  "all tests but upgrade on all lisps, no stop"
46 47 48 49 50
  (without-stopping ()
    (doc) (test-load-systems)
    (test-all-clean-load)
    (test-all-scripts-no-stop)
    (check-all-scripts-results)))
51

52
(deftestcmd test-all-no-stop () ;; TODO: pass arguments!
53
  "all tests on all lisps, no stop"
54 55 56 57 58 59
  (without-stopping ()
    (doc) (test-load-systems)
    (test-all-clean-load)
    (test-all-scripts-no-stop)
    (test-all-upgrade-no-stop)
    (check-all-results)))
60

61
(deftestcmd check-all-scripts-results ()
62 63
  "were there errors in test scripts?"
  (with-asdf-dir ()
64 65
    (let ((bad-lisps
            (run/lines
66
             `(grep "-L" "All tests apparently successful."
67
                    ,@(mapcar (lambda (l) (format nil "build/results/~(~A~)-test.text" l))
68
                              (get-lisps))) :on-error nil)))
69 70
    (failure-if bad-lisps
                "Unexpected test failures on these implementations:~%~{~A~%~}" bad-lisps))))
71 72

(deftestcmd check-all-upgrade-results ()
73
  "were there upgrade test failures?"
74
  (with-asdf-dir ()
75 76 77
    (let ((bad-lisps
            (run/lines
             `(grep "-L" "Upgrade test succeeded for "
78
                    ,@(mapcar (lambda (l) (format nil "build/results/~(~A~)-upgrade.text" l))
79
                              (get-upgrade-lisps))) :on-error nil)))
80 81
      (failure-if bad-lisps
                  "Unexpected upgrade failures on these implementations:~%~{~A~%~}~%" bad-lisps))))
82 83

(deftestcmd check-all-results ()
84
  "were there failures in any scripts or upgrade?"
85 86
  (without-stopping ()
    (check-all-scripts-results) (check-all-upgrade-results)))