rt.lsp 17 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
;-*-syntax:COMMON-LISP;Package:(RT :use "COMMON-LISP" :colon-mode :external)-*-

#|----------------------------------------------------------------------------|
 | Copyright 1990 by the Massachusetts Institute of Technology, Cambridge MA. |
 |                                                                            |
 | Permission  to  use,  copy, modify, and distribute this software  and  its |
 | documentation for any purpose  and without fee is hereby granted, provided |
 | that this copyright  and  permission  notice  appear  in  all  copies  and |
 | supporting  documentation,  and  that  the  name  of M.I.T. not be used in |
 | advertising or  publicity  pertaining  to  distribution  of  the  software |
 | without   specific,   written   prior   permission.      M.I.T.  makes  no |
 | representations  about  the  suitability of this software for any purpose. |
 | It is provided "as is" without express or implied warranty.                |
 |                                                                            |
 |  M.I.T. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,  INCLUDING  |
 |  ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL  |
 |  M.I.T. BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL  DAMAGES  OR  |
 |  ANY  DAMAGES  WHATSOEVER  RESULTING  FROM  LOSS OF USE, DATA OR PROFITS,  |
 |  WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER  TORTIOUS  ACTION,  |
 |  ARISING  OUT  OF  OR  IN  CONNECTION WITH THE USE OR PERFORMANCE OF THIS  |
 |  SOFTWARE.                                                                 |
 |----------------------------------------------------------------------------|#

24
25
;This was the December 19, 1990 version of the regression tester, but
;has since been modified.
26

27
(in-package :regression-test)
28

29
30
31
32
33
(declaim (ftype (function (t) t) get-entry expanded-eval do-entries))
(declaim (type list *entries*))
(declaim (ftype (function (t &rest t) t) report-error))
(declaim (ftype (function (t &optional t) t) do-entry))

34
35
(defvar *test* nil "Current test name")
(defvar *do-tests-when-defined* nil)
36
(defvar *entries* (list nil) "Test database.  Has a leading dummy cell that does not contain an entry.")
37
38
39
(defvar *entries-tail* *entries* "Tail of the *entries* list")
(defvar *entries-table* (make-hash-table :test #'equal)
    "Map the names of entries to the cons cell in *entries* that precedes the one whose car is the entry.")
40
41
42
(defvar *in-test* nil "Used by TEST")
(defvar *debug* nil "For debugging")
(defvar *catch-errors* t "When true, causes errors in a test to be caught.")
43
44
(defvar *print-circle-on-failure* nil
  "Failure reports are printed with *PRINT-CIRCLE* bound to this value.")
45

46
47
(defvar *compile-tests* nil "When true, compile the tests before running them.")
(defvar *expanded-eval* nil "When true, convert the tests into a form that is less likely to have compiler optimizations.")
48
49
(defvar *optimization-settings* '((safety 3)))

50
51
52
(defvar *failed-tests* nil "After DO-TESTS, becomes the list of names of tests that have failed")
(defvar *passed-tests* nil "After DO-TESTS, becomes the list of names of tests that have passed")

53
54
55
(defvar *expected-failures* nil
  "A list of test names that are expected to fail.")

56
57
58
59
60
61
(defvar *unexpected-successes* nil
  "A list of tests that passed but were expected to fail.")

(defvar *unexpected-failures* nil
  "A list of tests that failed but were not expected to fail.")

62
63
(defvar *notes* (make-hash-table :test 'equal)
  "A mapping from names of notes to note objects.")
sds's avatar
sds committed
64

65
66
(defstruct (entry (:conc-name nil))
  pend name props form vals)
67

68
69
70
71
72
73
;;; Note objects are used to attach information to tests.
;;; A typical use is to mark tests that depend on a particular
;;; part of a set of requirements, or a particular interpretation
;;; of the requirements.

(defstruct note
sds's avatar
sds committed
74
  name
75
76
77
78
  contents
  disabled ;; When true, tests with this note are considered inactive
  )

79
;; (defmacro vals (entry) `(cdddr ,entry))
80

81
82
83
84
(defmacro defn (entry)
  (let ((var (gensym)))
    `(let ((,var ,entry))
       (list* (name ,var) (form ,var) (vals ,var)))))
85

86
87
(defun entry-notes (entry)
  (let* ((props (props entry))
sds's avatar
sds committed
88
         (notes (getf props :notes)))
89
    (if (listp notes)
sds's avatar
sds committed
90
        notes
91
92
93
94
95
      (list notes))))

(defun has-disabled-note (entry)
  (let ((notes (entry-notes entry)))
    (loop for n in notes
sds's avatar
sds committed
96
97
98
          for note = (if (note-p n) n
                       (gethash n *notes*))
          thereis (and note (note-disabled note)))))
99

100
101
102
103
104
105
(defun has-note (entry note)
  (unless (note-p note)
    (let ((new-note (gethash note *notes*)))
      (setf note new-note)))
  (and note (not (not (member note (entry-notes entry))))))

106
(defun pending-tests ()
107
  (loop for entry in (cdr *entries*)
sds's avatar
sds committed
108
109
        when (and (pend entry) (not (has-disabled-note entry)))
        collect (name entry)))
110
111
112

(defun rem-all-tests ()
  (setq *entries* (list nil))
113
114
  (setq *entries-tail* *entries*)
  (clrhash *entries-table*)
115
116
117
  nil)

(defun rem-test (&optional (name *test*))
118
119
  (let ((pred (gethash name *entries-table*)))
    (when pred
pfdietz's avatar
pfdietz committed
120
      (if (null (cddr pred))
sds's avatar
sds committed
121
122
          (setq *entries-tail* pred)
        (setf (gethash (name (caddr pred)) *entries-table*) pred))
123
124
125
      (setf (cdr pred) (cddr pred))
      (remhash name *entries-table*)
      name)))
126

127
128
129
130
(defun get-test (&optional (name *test*))
  (defn (get-entry name)))

(defun get-entry (name)
131
  (let ((entry ;; (find name (the list (cdr *entries*))
sds's avatar
sds committed
132
133
134
               ;;     :key #'name :test #'equal)
         (cadr (gethash name *entries-table*))
         ))
135
136
137
    (when (null entry)
      (report-error t
        "~%No test with name ~:@(~S~)."
sds's avatar
sds committed
138
        name))
139
140
    entry))

141
142
(defmacro deftest (name &rest body)
  (let* ((p body)
sds's avatar
sds committed
143
144
145
146
147
148
149
150
         (properties
          (loop while (keywordp (first p))
                unless (cadr p)
                do (error "Poorly formed deftest: ~A~%"
                          (list* 'deftest name body))
                append (list (pop p) (pop p))))
         (form (pop p))
         (vals p))
151
    `(add-entry (make-entry :pend t
sds's avatar
sds committed
152
153
154
155
                            :name ',name
                            :props ',properties
                            :form ',form
                            :vals ',vals))))
156
157

(defun add-entry (entry)
158
  (setq entry (copy-entry entry))
159
160
161
162
  (let* ((pred (gethash (name entry) *entries-table*)))
    (cond
     (pred
      (setf (cadr pred) entry)
163
164
      (report-error nil
        "Redefining test ~:@(~S~)"
165
166
167
168
169
170
        (name entry)))
     (t
      (setf (gethash (name entry) *entries-table*) *entries-tail*)
      (setf (cdr *entries-tail*) (cons entry nil))
      (setf *entries-tail* (cdr *entries-tail*))
      )))
171
172
173
174
175
  (when *do-tests-when-defined*
    (do-entry entry))
  (setq *test* (name entry)))

(defun report-error (error? &rest args)
sds's avatar
sds committed
176
177
178
179
180
  (cond (*debug*
         (apply #'format t args)
         (if error? (throw '*debug* nil)))
        (error? (apply #'error args))
        (t (apply #'warn args)))
181
  nil)
182

183
184
(defun do-test (&optional (name *test*) &rest key-args)
  (flet ((%parse-key-args
sds's avatar
sds committed
185
186
187
188
          (&key
           ((:catch-errors *catch-errors*) *catch-errors*)
           ((:compile *compile-tests*) *compile-tests*))
          (do-entry (get-entry name))))
189
    (apply #'%parse-key-args key-args)))
190

191
192
193
194
195
196
(defun my-aref (a &rest args)
  (apply #'aref a args))

(defun my-row-major-aref (a index)
  (row-major-aref a index))

197
(defun equalp-with-case (x y)
198
  "Like EQUALP, but doesn't do case conversion of characters.
199
   Currently doesn't work on arrays of dimension > 2."
200
  (cond
201
   ((eq x y) t)
202
203
   ((consp x)
    (and (consp y)
sds's avatar
sds committed
204
205
         (equalp-with-case (car x) (car y))
         (equalp-with-case (cdr x) (cdr y))))
206
   ((and (typep x 'array)
sds's avatar
sds committed
207
         (= (array-rank x) 0))
208
    (equalp-with-case (my-aref x) (my-aref y)))
209
210
   ((typep x 'vector)
    (and (typep y 'vector)
sds's avatar
sds committed
211
212
213
214
215
216
217
218
         (let ((x-len (length x))
               (y-len (length y)))
           (and (eql x-len y-len)
                (loop
                 for i from 0 below x-len
                 for e1 = (my-aref x i)
                 for e2 = (my-aref y i)
                 always (equalp-with-case e1 e2))))))
219
   ((and (typep x 'array)
sds's avatar
sds committed
220
221
222
         (typep y 'array)
         (not (equal (array-dimensions x)
                     (array-dimensions y))))
223
    nil)
224
225

   ((typep x 'array)
226
    (and (typep y 'array)
sds's avatar
sds committed
227
228
229
230
         (let ((size (array-total-size x)))
           (loop for i from 0 below size
                 always (equalp-with-case (my-row-major-aref x i)
                                          (my-row-major-aref y i))))))
pfdietz's avatar
pfdietz committed
231
232
   ((typep x 'pathname)
    (equal x y))
233
234
   (t (eql x y))))

235
(defun do-entry (entry &optional
sds's avatar
sds committed
236
                       (s *standard-output*))
237
238
239
240
  (catch '*in-test*
    (setq *test* (name entry))
    (setf (pend entry) t)
    (let* ((*in-test* t)
sds's avatar
sds committed
241
242
243
           ;; (*break-on-warnings* t)
           (aborted nil)
           r)
244
245
      ;; (declare (special *break-on-warnings*))

246
      (block aborted
sds's avatar
sds committed
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
        (setf r
              (flet ((%do ()
                          (handler-bind
                           #-sbcl nil
                           #+sbcl ((sb-ext:code-deletion-note #'(lambda (c)
                                                                  (if (has-note entry :do-not-muffle)
                                                                      nil
                                                                    (muffle-warning c)))))
                           (cond
                            (*compile-tests*
                             (multiple-value-list
                              (funcall (compile
                                        nil
                                        `(lambda ()
                                           (declare
                                            (optimize ,@*optimization-settings*))
                                           ,(form entry))))))
                            (*expanded-eval*
                             (multiple-value-list
                              (expanded-eval (form entry))))
                            (t
                             (multiple-value-list
                              (eval (form entry))))))))
                (if *catch-errors*
                    (handler-bind
                     (#-ecl (style-warning #'(lambda (c) (if (has-note entry :do-not-muffle-warnings)
                                                             c
                                                           (muffle-warning c))))
                            (error #'(lambda (c)
                                       (setf aborted t)
                                       (setf r (list c))
                                       (return-from aborted nil))))
                     (%do))
                  (%do)))))
281

282
      (setf (pend entry)
sds's avatar
sds committed
283
284
285
            (or aborted
                (not (equalp-with-case r (vals entry)))))

286
      (when (pend entry)
sds's avatar
sds committed
287
        (let ((*print-circle* *print-circle-on-failure*))
sds's avatar
sds committed
288
289
290
          (format s "~&Test ~:@(~S~) failed~%Form: ~S~%Expected value~P:~%"
                  *test* (form entry) (length (vals entry)))
          (dolist (v (vals entry)) (format s "~10t~S~%" v))
sds's avatar
sds committed
291
          (handler-case
sds's avatar
sds committed
292
293
294
295
296
297
              (progn
                (format s "Actual value~P:~%" (length r))
                (dolist (v r)
                  (format s "~10t~S~:[~; [~2:*~A]~]~%"
                          v (typep v 'condition))))
            (error () (format s "Actual value: #<error during printing>~%")))
sds's avatar
sds committed
298
          (finish-output s)))))
299
  (when (not (pend entry)) *test*))
300

301
302
303
304
305
306
307
308
309
(defun expanded-eval (form)
  "Split off top level of a form and eval separately.  This reduces the chance that
   compiler optimizations will fold away runtime computation."
  (if (not (consp form))
      (eval form)
   (let ((op (car form)))
     (cond
      ((eq op 'let)
       (let* ((bindings (loop for b in (cadr form)
sds's avatar
sds committed
310
311
312
313
314
315
316
                              collect (if (consp b) b (list b nil))))
              (vars (mapcar #'car bindings))
              (binding-forms (mapcar #'cadr bindings)))
         (apply
          (the function
            (eval `(lambda ,vars ,@(cddr form))))
          (mapcar #'eval binding-forms))))
317
318
      ((and (eq op 'let*) (cadr form))
       (let* ((bindings (loop for b in (cadr form)
sds's avatar
sds committed
319
320
321
322
323
324
325
                              collect (if (consp b) b (list b nil))))
              (vars (mapcar #'car bindings))
              (binding-forms (mapcar #'cadr bindings)))
         (funcall
          (the function
            (eval `(lambda (,(car vars) &aux ,@(cdr bindings)) ,@(cddr form))))
          (eval (car binding-forms)))))
326
327
      ((eq op 'progn)
       (loop for e on (cdr form)
sds's avatar
sds committed
328
329
             do (if (null (cdr e)) (return (eval (car e)))
                  (eval (car e)))))
330
      ((and (symbolp op) (fboundp op)
sds's avatar
sds committed
331
332
            (not (macro-function op))
            (not (special-operator-p op)))
333
       (apply (symbol-function op)
sds's avatar
sds committed
334
              (mapcar #'eval (cdr form))))
335
336
      (t (eval form))))))

337
338
339
340
(defun continue-testing ()
  (if *in-test*
      (throw '*in-test* nil)
      (do-entries *standard-output*)))
341

342
(defun do-tests (&key (out *standard-output*)
sds's avatar
sds committed
343
344
                      ((:catch-errors *catch-errors*) *catch-errors*)
                      ((:compile *compile-tests*) *compile-tests*))
345
  (setq *failed-tests* nil
sds's avatar
sds committed
346
        *passed-tests* nil)
347
348
349
350
  (dolist (entry (cdr *entries*))
    (setf (pend entry) t))
  (if (streamp out)
      (do-entries out)
sds's avatar
sds committed
351
352
353
      (with-open-file
          (stream out :direction :output)
        (do-entries stream))))
354

355
(defun do-entries (s)
356
357
  (format s "~&Doing ~A pending test~:P ~
             of ~A tests total.~%"
358
          (count t (the list (cdr *entries*)) :key #'pend)
sds's avatar
sds committed
359
          (length (cdr *entries*)))
360
  (finish-output s)
361
  (dolist (entry (cdr *entries*))
362
    (when (and (pend entry)
sds's avatar
sds committed
363
               (not (has-disabled-note entry)))
364
      (let ((success? (do-entry entry s)))
sds's avatar
sds committed
365
366
367
368
        (if success?
          (push (name entry) *passed-tests*)
          (push (name entry) *failed-tests*))
        (format s "~@[~<~%~:; ~:@(~S~)~>~]" success?))
369
370
      (finish-output s)
      ))
371
  (let ((pending (pending-tests))
sds's avatar
sds committed
372
        (expected-table (make-hash-table :test #'equal)))
373
374
375
    (dolist (ex *expected-failures*)
      (setf (gethash ex expected-table) t))
    (let ((new-failures
sds's avatar
sds committed
376
377
378
           (loop for pend in pending
                 unless (gethash pend expected-table)
                 collect pend)))
379
      (if (null pending)
sds's avatar
sds committed
380
381
382
383
384
385
386
387
388
          (format s "~&No tests failed.")
        (progn
          (format t "~&~A out of ~A total tests failed: ~%(~{~a~^~%~})"
                  (length pending)
                  (length (cdr *entries*))
                  pending)
          (if (null new-failures)
              (format s "~&No unexpected failures.")
            (when *expected-failures*
389
              (setf *unexpected-failures* new-failures)
sds's avatar
sds committed
390
              (format s "~&~A unexpected failures: ~
391
392
                   ~:@(~{~<~%   ~1:;~S~>~
                         ~^, ~}~)."
sds's avatar
sds committed
393
394
                    (length new-failures)
                    new-failures)))
395
396
397
398
399
400
401
402
          (when *expected-failures*
            (let ((pending-table (make-hash-table :test #'equal)))
              (dolist (ex pending)
                (setf (gethash ex pending-table) t))
              (let ((unexpected-successes
                     (loop :for ex :in *expected-failures*
                       :unless (gethash ex pending-table) :collect ex)))
                (if unexpected-successes
403
404
405
                    (progn
                      (setf *unexpected-successes* unexpected-successes)
                      (format t "~&~:D unexpected successes: ~
406
407
                   ~:@(~{~<~%   ~1:;~S~>~
                         ~^, ~}~)."
408
409
                              (length unexpected-successes)
                              unexpected-successes))
410
                    (format t "~&No unexpected successes.")))))
sds's avatar
sds committed
411
          ))
412
      (finish-output s)
413
      (null pending))))
414
415
416

;;; Note handling functions and macros

417
(defmacro defnote (name contents &optional disabled)
418
  `(eval-when (:load-toplevel :execute)
419
     (let ((note (make-note :name ',name
sds's avatar
sds committed
420
421
                            :contents ',contents
                            :disabled ',disabled)))
422
       (setf (gethash (note-name note) *notes*) note)
423
424
       note)))

425
426
(defun disable-note (n)
  (let ((note (if (note-p n) n
sds's avatar
sds committed
427
                (setf n (gethash n *notes*)))))
428
429
430
431
432
433
    (unless note (error "~A is not a note or note name." n))
    (setf (note-disabled note) t)
    note))

(defun enable-note (n)
  (let ((note (if (note-p n) n
sds's avatar
sds committed
434
                (setf n (gethash n *notes*)))))
435
436
437
    (unless note (error "~A is not a note or note name." n))
    (setf (note-disabled note) nil)
    note))
438
439
440
441

;;; Extended random regression

(defun do-extended-tests (&key (tests *passed-tests*) (count nil)
sds's avatar
sds committed
442
443
                               ((:catch-errors *catch-errors*) *catch-errors*)
                               ((:compile *compile-tests*) *compile-tests*))
444
445
446
447
448
449
  "Execute randomly chosen tests from TESTS until one fails or until
   COUNT is an integer and that many tests have been executed."
  (let ((test-vector (coerce tests 'simple-vector)))
    (let ((n (length test-vector)))
      (when (= n 0) (error "Must provide at least one test."))
      (loop for i from 0
sds's avatar
sds committed
450
451
452
453
            for name = (svref test-vector (random n))
            until (eql i count)
            do (print name)
            unless (do-test name) return (values name (1+ i))))))