test-all.lisp 3.18 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
(in-package :asdf-tools)

(defun call-with-all-lisps (thunk &optional (lisps *test-lisps*))
  (apply 'all-pass
         (loop :for lisp :in lisps
               :collect (let ((*test-lisps* (list lisp))) (funcall thunk)))))

(defmacro with-all-lisps ((&rest maybe-lisps) &body body)
  `(call-with-all-lisps (lambda () ,@body) ,@maybe-lisps))

(deftestcmd test-all-clean-load ()
  "test that all lisp implementations can load asdf cleanly without any output message"
  (with-all-lisps () (test-clean-load)))

(deftestcmd test-all-lisp ()
  "test that all lisp implementations pass all asdf test scripts"
  (with-all-lisps () (test-scripts)))

(deftestcmd test-all-no-upgrade ()
  "test that all lisp implementations pass all normal asdf; also test-basic"
  (all-pass (test-basic) (test-all-lisp)))

(deftestcmd test-all-upgrade ()
  "test that all lisp implementations pass all asdf upgrade tests"
  (with-all-lisps (*upgrade-test-lisps*) (test-upgrade)))

27
(deftestcmd test-all-scripts ()
28 29
  "test that all lisp implementations pass all asdf tests"
  (all-pass (test-all-no-upgrade) (test-all-upgrade)))
30
(defalias test-all test-all-scripts)
31

32
(deftestcmd test-all-scripts-no-stop ()
33 34 35 36 37 38 39 40 41 42 43 44 45 46
  "test that all lisp implementations pass all asdf test scripts, but don't stop on error"
  (with-all-lisps ()
    (ignore-errors (test-scripts))))

(deftestcmd test-all-upgrade-no-stop ()
  "test that all lisp implementations pass all asdf upgrade tests, but don't stop on error"
  (with-all-lisps (*upgrade-test-lisps*)
    (ignore-errors (test-upgrade))))

(deftestcmd test-all-no-upgrade-no-stop ()
  "test that all lisp implementations pass all normal asdf tests (no upgrade), but don't stop on error."
  (all-pass
   (test-basic)
   (test-all-clean-load)
47 48
   (test-all-scripts-no-stop)
   (check-all-scripts-results)))
49

50
(deftestcmd test-all-no-stop () ;; TODO: pass arguments!
51 52 53 54
  "test that all lisp implementations pass all asdf tests (including upgrade), but don't stop on error."
  (all-pass
   (test-basic)
   (test-all-clean-load)
55
   (test-all-scripts-no-stop)
56 57 58
   (test-all-upgrade-no-stop)
   (check-all-results)))

59
(deftestcmd check-all-scripts-results ()
60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
  "were there errors in test scripts?"
  (with-asdf-dir ()
    (let ((a (run/lines
              `(grep "-L" "[5-9][0-9] passing and 0 failing"
                     ,(mapcar (lambda (l) (format nil "build/results/~(~A~)-test.text" l))
                              *test-lisps*)))))
      (or (null a)
          (progn
            (format! *error-output* "Unexpected test failures on these implementations:~%~{~A~%~}" a)
            nil)))))

(deftestcmd check-all-upgrade-results ()
  "were there upgrade tests failures?"
  (with-asdf-dir ()
    (let ((a (run/lines
              `(grep "-L" "Upgrade test succeeded for "
                     ,(mapcar (lambda (l) (format nil "build/results/~(~A~)-upgrade.text" l))
                              *upgrade-test-lisps*)))))
      (or (null a)
          (progn
            (format t "Unexpected upgrade failures on these implementations:~%~{~A~%~}~%" a)
            nil)))))

(deftestcmd check-all-results ()
  "check that there were no errors in either test scripts or upgrade tests"
85
  (all-pass (check-all-scripts-results) (check-all-upgrade-results)))
86