plan.lisp 20 KB
Newer Older
1 2 3
;;;; -------------------------------------------------------------------------
;;;; Plan

4 5
(asdf/package:define-package :asdf/plan
  (:recycle :asdf/plan :asdf)
6
  (:use :asdf/common-lisp :asdf/driver :asdf/upgrade
7
   :asdf/component :asdf/operation :asdf/system
8
   :asdf/cache :asdf/find-system :asdf/find-component
9 10 11
   :asdf/operation :asdf/action)
  (:export
   #:component-operation-time #:mark-operation-done
12
   #:plan-traversal #:sequential-plan #:*default-plan-class*
13 14 15
   #:planned-action-status #:plan-action-status #:action-already-done-p
   #:circular-dependency #:circular-dependency-actions
   #:node-for #:needed-in-image-p
16
   #:action-index #:action-planned-p #:action-valid-p
17 18
   #:plan-record-dependency #:visiting-action-p
   #:normalize-forced-systems #:action-forced-p #:action-forced-not-p
19
   #:map-direct-dependencies #:reduce-direct-dependencies #:direct-dependencies
20
   #:visit-dependencies #:compute-action-stamp #:traverse-action
21
   #:circular-dependency #:circular-dependency-actions
22
   #:call-while-visiting-action #:while-visiting-action
23 24 25
   #:traverse #:plan-actions #:perform-plan #:plan-operates-on-p
   #:planned-p #:index #:forced #:forced-not #:total-action-count
   #:planned-action-count #:planned-output-action-count #:visited-actions
26
   #:visiting-action-set #:visiting-action-list #:plan-actions-r
27
   #:required-components #:filtered-sequential-plan
28
   #:plan-system
29 30
   #:plan-action-filter #:plan-component-type #:plan-keep-operation #:plan-keep-component
   #:traverse-actions #:traverse-sub-actions))
31
(in-package :asdf/plan)
32

33 34 35 36 37 38 39 40 41 42 43 44 45 46
;;;; Generic plan traversal class

(defclass plan-traversal ()
  ((system :initform nil :initarg :system :accessor plan-system)
   (forced :initform nil :initarg :force :accessor plan-forced)
   (forced-not :initform nil :initarg :force-not :accessor plan-forced-not)
   (total-action-count :initform 0 :accessor plan-total-action-count)
   (planned-action-count :initform 0 :accessor plan-planned-action-count)
   (planned-output-action-count :initform 0 :accessor plan-planned-output-action-count)
   (visited-actions :initform (make-hash-table :test 'equal) :accessor plan-visited-actions)
   (visiting-action-set :initform (make-hash-table :test 'equal) :accessor plan-visiting-action-set)
   (visiting-action-list :initform () :accessor plan-visiting-action-list)))


47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
;;;; Planned action status

(defgeneric* plan-action-status (plan operation component)
  (:documentation "Returns the ACTION-STATUS associated to
the action of OPERATION on COMPONENT in the PLAN"))

(defgeneric* (setf plan-action-status) (new-status plan operation component)
  (:documentation "Sets the ACTION-STATUS associated to
the action of OPERATION on COMPONENT in the PLAN"))

(defclass planned-action-status (action-status)
  ((planned-p
    :initarg :planned-p :reader action-planned-p
    :documentation "a boolean, true iff the action was included in the plan.")
   (index
    :initarg :index :reader action-index
    :documentation "an integer, counting all traversed actions in traversal order."))
  (:documentation "Status of an action in a plan"))

(defmethod print-object ((status planned-action-status) stream)
  (print-unreadable-object (status stream :type t)
    (with-slots (stamp done-p planned-p) status
      (format stream "~@{~S~^ ~}" :stamp stamp :done-p done-p :planned-p planned-p))))

71 72 73 74
(defmethod action-planned-p (action-status)
  (declare (ignorable action-status)) ; default method for non planned-action-status objects
  t)

75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106
;; TODO: eliminate NODE-FOR, use CONS.
;; Supposes cleaner protocol for operation initargs passed to MAKE-OPERATION.
;; However, see also component-operation-time and mark-operation-done
(defun* node-for (o c) (cons (type-of o) c))

(defun* action-already-done-p (plan operation component)
  (action-done-p (plan-action-status plan operation component)))

(defmethod plan-action-status ((plan null) (o operation) (c component))
  (declare (ignorable plan))
  (multiple-value-bind (stamp done-p) (component-operation-time o c)
    (make-instance 'action-status :stamp stamp :done-p done-p)))

(defmethod (setf plan-action-status) (new-status (plan null) (o operation) (c component))
  (declare (ignorable plan))
  (let ((to (type-of o))
        (times (component-operation-times c)))
    (if (action-done-p new-status)
        (remhash to times)
        (setf (gethash to times) (action-stamp new-status))))
  new-status)


;;;; forcing

(defgeneric* action-forced-p (plan operation component))
(defgeneric* action-forced-not-p (plan operation component))

(defun* normalize-forced-systems (x system)
  (etypecase x
    ((member nil :all) x)
    (cons (list-to-hash-set (mapcar #'coerce-name x)))
107
    ((eql t) (when system (list-to-hash-set (list (coerce-name system)))))))
108 109 110 111 112 113 114 115 116 117

(defun* action-override-p (plan operation component override-accessor)
  (declare (ignorable operation))
  (let* ((override (funcall override-accessor plan)))
    (and override
         (if (typep override 'hash-table)
             (gethash (coerce-name (component-system (find-component () component))) override)
             t))))

(defmethod action-forced-p (plan operation component)
118 119 120 121 122 123 124 125 126
  (and
   ;; Did the user ask us to re-perform the action?
   (action-override-p plan operation component 'plan-forced)
   ;; You really can't force a builtin system and :all doesn't apply to it,
   ;; except it it's the specifically the system currently being built.
   (not (let ((system (component-system component)))
          (and (builtin-system-p system)
               (not (eq system (plan-system plan))))))))

127
(defmethod action-forced-not-p (plan operation component)
128 129 130 131 132 133
  (and
   ;; Did the user ask us to not re-perform the action?
   (action-override-p plan operation component 'plan-forced-not)
   ;; Force takes precedence over force-not
   (not (action-forced-p plan operation component))))

134 135 136
(defmethod action-forced-p ((plan null) operation component)
  (declare (ignorable plan operation component))
  nil)
137

138 139 140
(defmethod action-forced-not-p ((plan null) operation component)
  (declare (ignorable plan operation component))
  nil)
141 142 143 144 145 146 147 148


;;;; action-valid-p

(defgeneric action-valid-p (plan operation component)
  (:documentation "Is this action valid to include amongst dependencies?"))
(defmethod action-valid-p (plan operation (c component))
  (declare (ignorable plan operation))
149
  (if-let (it (component-if-feature c)) (featurep it) t))
150 151
(defmethod action-valid-p (plan (o null) c) (declare (ignorable plan o c)) nil)
(defmethod action-valid-p (plan o (c null)) (declare (ignorable plan o c)) nil)
152 153 154
(defmethod action-valid-p ((plan null) operation component)
  (declare (ignorable plan operation component))
  (and operation component t))
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172


;;;; Is the action needed in this image?

(defgeneric* needed-in-image-p (operation component)
  (:documentation "Is the action of OPERATION on COMPONENT needed in the current image to be meaningful,
    or could it just as well have been done in another Lisp image?"))

(defmethod needed-in-image-p ((o operation) (c component))
  ;; We presume that actions that modify the filesystem don't need be run
  ;; in the current image if they have already been done in another,
  ;; and can be run in another process (e.g. a fork),
  ;; whereas those that don't are meant to side-effect the current image and can't.
  (not (output-files o c)))


;;;; Visiting dependencies of an action and computing action stamps

173
(defun* map-direct-dependencies (operation component fun)
174 175 176 177 178 179
  (loop* :for (dep-o-spec . dep-c-specs) :in (component-depends-on operation component)
     :unless (eq dep-o-spec 'feature) ;; avoid the "FEATURE" misfeature
        :do (loop :with dep-o = (find-operation operation dep-o-spec)
                  :for dep-c-spec :in dep-c-specs
                  :for dep-c = (resolve-dependency-spec component dep-c-spec)
                  :do (funcall fun dep-o dep-c))))
180 181 182 183

(defun* reduce-direct-dependencies (operation component combinator seed)
  (map-direct-dependencies
   operation component
184 185
   #'(lambda (dep-o dep-c)
       (setf seed (funcall combinator dep-o dep-c seed))))
186 187 188 189 190 191 192 193
  seed)

(defun* direct-dependencies (operation component)
  (reduce-direct-dependencies operation component #'acons nil))

(defun* visit-dependencies (plan operation component dependency-stamper &aux stamp)
  (map-direct-dependencies
   operation component
194 195 196
   #'(lambda (dep-o dep-c)
       (when (action-valid-p plan dep-o dep-c)
         (latest-stamp-f stamp (funcall dependency-stamper dep-o dep-c)))))
197 198 199
  stamp)

(defmethod compute-action-stamp (plan (o operation) (c component) &key just-done)
200
  ;; In a distant future, get-file-stamp and component-operation-time
201
  ;; shall also be parametrized by the plan, or by a second model object.
202
  (let* ((stamp-lookup #'(lambda (o c)
203
                           (if-let (it (plan-action-status plan o c)) (action-stamp it) t)))
204 205 206 207 208 209 210 211 212 213 214
         (out-files (output-files o c))
         (in-files (input-files o c))
         ;; Three kinds of actions:
         (out-op (and out-files t)) ; those that create files on the filesystem
         ;(image-op (and in-files (null out-files))) ; those that load stuff into the image
         ;(null-op (and (null out-files) (null in-files))) ; dependency placeholders that do nothing
         ;; When was the thing last actually done? (Now, or ask.)
         (op-time (or just-done (component-operation-time o c)))
         ;; Accumulated timestamp from dependencies (or T if forced or out-of-date)
         (dep-stamp (visit-dependencies plan o c stamp-lookup))
         ;; Time stamps from the files at hand, and whether any is missing
215 216
         (out-stamps (mapcar (if just-done 'register-file-stamp 'get-file-stamp) out-files))
         (in-stamps (mapcar #'get-file-stamp in-files))
217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
         (missing-in
           (loop :for f :in in-files :for s :in in-stamps :unless s :collect f))
         (missing-out
           (loop :for f :in out-files :for s :in out-stamps :unless s :collect f))
         (all-present (not (or missing-in missing-out)))
         ;; Has any input changed since we last generated the files?
         (earliest-out (stamps-earliest out-stamps))
         (latest-in (stamps-latest (cons dep-stamp in-stamps)))
         (up-to-date-p (stamp<= latest-in earliest-out))
         ;; If everything is up to date, the latest of inputs and outputs is our stamp
         (done-stamp (stamps-latest (cons latest-in out-stamps))))
    ;; Warn if some files are missing:
    ;; either our model is wrong or some other process is messing with our files.
    (when (and just-done (not all-present))
      (warn "~A completed without ~:[~*~;~*its input file~:p~2:*~{ ~S~}~*~]~
             ~:[~; or ~]~:[~*~;~*its output file~:p~2:*~{ ~S~}~*~]"
233
            (action-description o c)
234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
            missing-in (length missing-in) (and missing-in missing-out)
            missing-out (length missing-out)))
    ;; Note that we use stamp<= instead of stamp< to play nice with generated files.
    ;; Any race condition is intrinsic to the limited timestamp resolution.
    (if (or just-done ;; The done-stamp is valid: if we're just done, or
            ;; if all filesystem effects are up-to-date and there's no invalidating reason.
            (and all-present up-to-date-p (operation-done-p o c) (not (action-forced-p plan o c))))
        (values done-stamp ;; return the hard-earned timestamp
                (or just-done
                    (or out-op ;; a file-creating op is done when all files are up to date
                        ;; a image-effecting a placeholder op is done when it was actually run,
                        (and op-time (eql op-time done-stamp))))) ;; with the matching stamp
        ;; done-stamp invalid: return a timestamp in an indefinite future, action not done yet
        (values t nil))))

;;;; Generic support for plan-traversal

(defgeneric* plan-record-dependency (plan operation component))

(defgeneric call-while-visiting-action (plan operation component function)
  (:documentation "Detect circular dependencies"))

(defmethod initialize-instance :after ((plan plan-traversal)
257 258
                                       &key (force () fp) (force-not () fnp) system
                                       &allow-other-keys)
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
  (with-slots (forced forced-not) plan
    (when fp (setf forced (normalize-forced-systems force system)))
    (when fnp (setf forced-not (normalize-forced-systems force-not system)))))

(defmethod (setf plan-action-status) (new-status (plan plan-traversal) (o operation) (c component))
  (setf (gethash (node-for o c) (plan-visited-actions plan)) new-status))

(defmethod plan-action-status ((plan plan-traversal) (o operation) (c component))
  (or (and (action-forced-not-p plan o c) (plan-action-status nil o c))
      (values (gethash (node-for o c) (plan-visited-actions plan)))))

(defmethod action-valid-p ((plan plan-traversal) (o operation) (s system))
  (and (not (action-forced-not-p plan o s)) (call-next-method)))

(defmethod call-while-visiting-action ((plan plan-traversal) operation component fun)
  (with-accessors ((action-set plan-visiting-action-set)
                   (action-list plan-visiting-action-list)) plan
    (let ((action (cons operation component)))
277 278 279 280 281 282 283 284 285
      (when (gethash action action-set)
        (error 'circular-dependency :actions
               (member action (reverse action-list) :test 'equal)))
      (setf (gethash action action-set) t)
      (push action action-list)
      (unwind-protect
           (funcall fun)
        (pop action-list)
        (setf (gethash action action-set) nil)))))
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341


;;;; Actual traversal: traverse-action

(define-condition circular-dependency (system-definition-error)
  ((actions :initarg :actions :reader circular-dependency-actions))
  (:report (lambda (c s)
             (format s (compatfmt "~@<Circular dependency: ~3i~_~S~@:>")
                     (circular-dependency-actions c)))))

(defmacro while-visiting-action ((p o c) &body body)
  `(call-while-visiting-action ,p ,o ,c #'(lambda () ,@body)))

(defgeneric* traverse-action (plan operation component needed-in-image-p))

(defmethod traverse-action (plan operation component needed-in-image-p)
  (block nil
    (unless (action-valid-p plan operation component) (return nil))
    (plan-record-dependency plan operation component)
    (let* ((aniip (needed-in-image-p operation component))
           (eniip (and aniip needed-in-image-p))
           (status (plan-action-status plan operation component)))
      (when (and status (or (action-done-p status) (action-planned-p status) (not eniip)))
        ;; Already visited with sufficient need-in-image level: just return the stamp.
        (return (action-stamp status)))
      (labels ((visit-action (niip)
                 (visit-dependencies plan operation component
                                     #'(lambda (o c) (traverse-action plan o c niip)))
                 (multiple-value-bind (stamp done-p)
                     (compute-action-stamp plan operation component)
                   (let ((add-to-plan-p (or (eql stamp t) (and niip (not done-p)))))
                     (cond
                       ((and add-to-plan-p (not niip)) ;; if we need to do it,
                        (visit-action t)) ;; then we need to do it in the image!
                       (t
                        (setf (plan-action-status plan operation component)
                              (make-instance
                               'planned-action-status
                               :stamp stamp
                               :done-p (and done-p (not add-to-plan-p))
                               :planned-p add-to-plan-p
                               :index (if status (action-index status) (incf (plan-total-action-count plan)))))
                        (when add-to-plan-p
                          (incf (plan-planned-action-count plan))
                          (unless aniip
                            (incf (plan-planned-output-action-count plan))))
                        stamp))))))
        (while-visiting-action (plan operation component) ; maintain context, handle circularity.
          (visit-action eniip))))))


;;;; Sequential plans (the default)

(defclass sequential-plan (plan-traversal)
  ((actions-r :initform nil :accessor plan-actions-r)))

342 343 344 345
(defgeneric* plan-actions (plan))
(defmethod plan-actions ((plan sequential-plan))
  (reverse (plan-actions-r plan)))

346 347 348 349 350 351 352 353 354 355 356 357 358
(defmethod plan-record-dependency ((plan sequential-plan)
                                   (operation operation) (component component))
  (declare (ignorable plan operation component))
  (values))

(defmethod (setf plan-action-status) :after
    (new-status (p sequential-plan) (o operation) (c component))
  (when (action-planned-p new-status)
    (push (cons o c) (plan-actions-r p))))


;;;; high-level interface: traverse, perform-plan, plan-operates-on-p

359
(defgeneric* (traverse) (operation component &key &allow-other-keys)
360 361 362 363 364 365
  (:documentation
"Generate and return a plan for performing OPERATION on COMPONENT.

The plan returned is a list of dotted-pairs. Each pair is the CONS
of ASDF operation object and a COMPONENT object. The pairs will be
processed in order by OPERATE."))
366 367
(define-convenience-action-methods traverse (operation component &key))

368 369 370
(defgeneric* perform-plan (plan &key))
(defgeneric* plan-operates-on-p (plan component))

371
(defparameter *default-plan-class* 'sequential-plan)
372

373 374
(defmethod traverse ((o operation) (c component) &rest keys &key plan-class &allow-other-keys)
  (let ((plan (apply 'make-instance
375
                     (or plan-class *default-plan-class*)
376
                     :system (component-system c) (remove-plist-key :plan-class keys))))
377 378
    (traverse-action plan o c t)
    (plan-actions plan)))
379

380 381
(defmethod perform-plan :around (plan &key)
  (declare (ignorable plan))
382 383
  (let ((*package* *package*)
        (*readtable* *readtable*))
384 385 386 387 388 389
    (with-compilation-unit () ;; backward-compatibility.
      (call-next-method))))   ;; Going forward, see deferred-warning support in lisp-build.

(defmethod perform-plan ((steps list) &key)
  (loop* :for (op . component) :in steps :do
         (perform-with-restarts op component)))
390 391 392 393 394

(defmethod plan-operates-on-p ((plan list) (component-path list))
  (find component-path (mapcar 'cdr plan)
        :test 'equal :key 'component-find-path))

395 396 397 398 399 400 401 402 403 404 405

;;;; Incidental traversals 

(defclass filtered-sequential-plan (sequential-plan)
  ((action-filter :initform t :initarg :action-filter :reader plan-action-filter)
   (component-type :initform t :initarg :component-type :reader plan-component-type)
   (keep-operation :initform t :initarg :keep-operation :reader plan-keep-operation)
   (keep-component :initform t :initarg :keep-component :reader plan-keep-component)))

(defmethod initialize-instance :after ((plan filtered-sequential-plan)
                                       &key (force () fp) (force-not () fnp)
406
                                         other-systems)
407
  (declare (ignore force force-not))
408
  (with-slots (forced forced-not action-filter system) plan
409 410 411 412 413 414 415 416 417
    (unless fp (setf forced (normalize-forced-systems (if other-systems :all t) system)))
    (unless fnp (setf forced-not (normalize-forced-systems (if other-systems nil :all) system)))
    (setf action-filter (ensure-function action-filter))))

(defmethod action-valid-p ((plan filtered-sequential-plan) o c)
  (and (funcall (plan-action-filter plan) o c)
       (typep c (plan-component-type plan))
       (call-next-method)))

418
(defmethod traverse-actions (actions &rest keys &key plan-class &allow-other-keys)
419
  (let ((plan (apply 'make-instance (or plan-class 'filtered-sequential-plan) keys)))
420
    (loop* :for (o . c) :in actions :do
421 422 423
      (traverse-action plan o c t))
    (plan-actions plan)))

424
(defmethod traverse-sub-actions (operation component &rest keys &key &allow-other-keys)
425 426 427 428 429
  (apply 'traverse-actions (direct-dependencies operation component)
         :system (component-system component) keys))

(defmethod plan-actions ((plan filtered-sequential-plan))
  (with-slots (keep-operation keep-component) plan
430 431 432 433
  (loop* :for (o . c) :in (call-next-method)
         :when (and (typep o keep-operation)
                    (typep c keep-component))
           :collect (cons o c))))
434

435
(defmethod required-components (system &rest keys &key (goal-operation 'load-op) &allow-other-keys)
436 437 438 439
  (remove-duplicates
   (mapcar 'cdr (apply 'traverse-sub-actions (make-operation goal-operation) system keys))
   :from-end t))