test-all.lisp 2.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
(in-package :asdf-tools)

(defun call-with-all-lisps (thunk &optional (lisps *test-lisps*))
  (apply 'all-pass
         (loop :for lisp :in lisps
               :collect (let ((*test-lisps* (list lisp))) (funcall thunk)))))

(defmacro with-all-lisps ((&rest maybe-lisps) &body body)
  `(call-with-all-lisps (lambda () ,@body) ,@maybe-lisps))

(deftestcmd test-all-clean-load ()
12
  "test-clean-load on all lisps"
13 14
  (with-all-lisps () (test-clean-load)))

15 16
(deftestcmd test-all-scripts ()
  "test-scripts on all lisps"
17 18 19
  (with-all-lisps () (test-scripts)))

(deftestcmd test-all-no-upgrade ()
20 21
  "test-basic, and test-all-script"
  (all-pass (test-basic) (test-all-scripts)))
22 23

(deftestcmd test-all-upgrade ()
24
  "test-upgrade on all lisps"
25 26
  (with-all-lisps (*upgrade-test-lisps*) (test-upgrade)))

27 28
(deftestcmd test-all ()
  "all tests"
29 30
  (all-pass (test-all-no-upgrade) (test-all-upgrade)))

31
(deftestcmd test-all-scripts-no-stop ()
32
  "test-scripts on all lisps, no stop"
33 34 35 36
  (with-all-lisps ()
    (ignore-errors (test-scripts))))

(deftestcmd test-all-upgrade-no-stop ()
37
  "test-upgrade on all lisps, no stop"
38 39 40 41
  (with-all-lisps (*upgrade-test-lisps*)
    (ignore-errors (test-upgrade))))

(deftestcmd test-all-no-upgrade-no-stop ()
42
  "all tests but upgrade on all lisps, no stop"
43
  (all-pass
44
   (doc) (test-load-systems)
45
   (test-all-clean-load)
46 47
   (test-all-scripts-no-stop)
   (check-all-scripts-results)))
48

49
(deftestcmd test-all-no-stop () ;; TODO: pass arguments!
50
  "all tests on all lisps, no stop"
51
  (all-pass
52
   (doc) (test-load-systems)
53
   (test-all-clean-load)
54
   (test-all-scripts-no-stop)
55 56 57
   (test-all-upgrade-no-stop)
   (check-all-results)))

58
(deftestcmd check-all-scripts-results ()
59 60 61 62 63 64 65 66 67 68 69 70
  "were there errors in test scripts?"
  (with-asdf-dir ()
    (let ((a (run/lines
              `(grep "-L" "[5-9][0-9] passing and 0 failing"
                     ,(mapcar (lambda (l) (format nil "build/results/~(~A~)-test.text" l))
                              *test-lisps*)))))
      (or (null a)
          (progn
            (format! *error-output* "Unexpected test failures on these implementations:~%~{~A~%~}" a)
            nil)))))

(deftestcmd check-all-upgrade-results ()
71
  "were there upgrade test failures?"
72 73 74 75 76 77 78 79 80 81 82
  (with-asdf-dir ()
    (let ((a (run/lines
              `(grep "-L" "Upgrade test succeeded for "
                     ,(mapcar (lambda (l) (format nil "build/results/~(~A~)-upgrade.text" l))
                              *upgrade-test-lisps*)))))
      (or (null a)
          (progn
            (format t "Unexpected upgrade failures on these implementations:~%~{~A~%~}~%" a)
            nil)))))

(deftestcmd check-all-results ()
83
  "were there failures in any scripts or upgrade?"
84
  (all-pass (check-all-scripts-results) (check-all-upgrade-results)))
85