test-all.lisp 3.41 KB
Newer Older
1 2
(in-package :asdf-tools)

3
(defun call-with-all-lisps (thunk &key (lisps *test-lisps*) muffle-failures)
4 5
  (let ((thunks
          (loop :for lisp :in (get-lisps lisps)
6
                :collect (let ((l lisp))
7
                           (lambda () (with-failure-context (:name (format nil "using ~(~A~)" l))
8
                                        (funcall thunk l)))))))
9 10 11
    (if muffle-failures
        (call-without-stopping thunks)
        (progn (map () 'funcall thunks) (success)))))
12

13 14
(defmacro with-all-lisps ((lisp-var lisps &key muffle-failures) &body body)
  `(call-with-all-lisps (lambda (,lisp-var) ,@body) :lisps ,lisps :muffle-failures ,muffle-failures))
15

16 17 18 19 20 21 22 23 24 25 26 27 28
(deftestcmd test-all-basic (lisps systems)
  "basic test: doc, clean-load, load-systems"
  (without-stopping ()
    (show-version)
    (doc)
    (test-ascii)
    (test-all-clean-load lisps)
    (test-all-load-systems lisps systems)))

(deftestcmd test-all-load-systems (lisps systems)
  (with-systems-test (systems)
    (with-all-lisps (l lisps) (test-load-systems l systems))))

29
(deftestcmd test-all-clean-load (lisps)
30
  "test-clean-load on all lisps"
31
  (with-all-lisps (l lisps) (test-clean-load l)))
32

33
(deftestcmd test-all-scripts (lisps)
34
  "test-scripts on all lisps"
35
  (with-all-lisps (l lisps) (test-scripts l)))
36 37

(deftestcmd test-all-no-upgrade ()
38 39
  "test-all-basic, and test-all-script"
  (test-all-basic) (test-all-scripts))
40

41
(deftestcmd test-all-upgrade (upgrade-lisps)
42
  "test-upgrade on all lisps"
43
  (with-all-lisps (l upgrade-lisps) (test-upgrade l)))
44

45 46
(deftestcmd test-all ()
  "all tests"
47
  (test-all-no-upgrade) (test-all-upgrade))
48

49
(deftestcmd test-all-scripts-no-stop (lisps)
50
  "test-scripts on all lisps, no stop"
51
  (with-all-lisps (l lisps :muffle-failures t) (test-scripts l)))
52

53
(deftestcmd test-all-upgrade-no-stop (upgrade-lisps)
54
  "test-upgrade on all lisps, no stop"
55
  (with-all-lisps (l upgrade-lisps :muffle-failures t) (test-upgrade l)))
56 57

(deftestcmd test-all-no-upgrade-no-stop ()
58
  "all tests but upgrade on all lisps, no stop"
59
  (without-stopping ()
60
    (test-all-basic)
61 62
    (test-all-scripts-no-stop)
    (check-all-scripts-results)))
63

64
(deftestcmd test-all-no-stop () ;; TODO: pass arguments!
65
  "all tests on all lisps, no stop"
66
  (without-stopping ()
67
    (test-all-basic)
68 69 70
    (test-all-scripts-no-stop)
    (test-all-upgrade-no-stop)
    (check-all-results)))
71

72
(deftestcmd check-all-scripts-results ()
73 74
  "were there errors in test scripts?"
  (with-asdf-dir ()
75 76
    (let ((bad-lisps
            (run/lines
77
             `(grep "-L" "All tests apparently successful."
78
                    ,@(mapcar (lambda (l) (format nil "build/results/~(~A~)-test.text" l))
79
                              (get-lisps))) :on-error nil)))
80 81
    (failure-if bad-lisps
                "Unexpected test failures on these implementations:~%~{~A~%~}" bad-lisps))))
82 83

(deftestcmd check-all-upgrade-results ()
84
  "were there upgrade test failures?"
85
  (with-asdf-dir ()
86 87 88
    (let ((bad-lisps
            (run/lines
             `(grep "-L" "Upgrade test succeeded for "
89
                    ,@(mapcar (lambda (l) (format nil "build/results/~(~A~)-upgrade.text" l))
90
                              (get-upgrade-lisps))) :on-error nil)))
91 92
      (failure-if bad-lisps
                  "Unexpected upgrade failures on these implementations:~%~{~A~%~}~%" bad-lisps))))
93 94

(deftestcmd check-all-results ()
95
  "were there failures in any scripts or upgrade?"
96 97
  (without-stopping ()
    (check-all-scripts-results) (check-all-upgrade-results)))