Commit 40b86923 authored by Francois-Rene Rideau's avatar Francois-Rene Rideau

Correctly track status of actions across phases

Do not perform an action twice in a session. Introduce three status bits
that every action has in a given session, in addition to the stamp:
KEEP-P (from previous session), DONE-P (and up-to-date, in this image),
NEED-P (needed in this image as part of this session).

BACKWARD INCOMPATIBILITY: Invert the meaning of NIL and T for STAMP,
so that NIL means out-of-date. This makes our code conceptually nicer,
and no one in Quicklisp uses our stamps, anyway. Still, beware.
parent 08ff74b1
......@@ -17,7 +17,7 @@
#:action-operation #:action-component #:make-action
#:component-operation-time #:mark-operation-done #:compute-action-stamp
#:perform #:perform-with-restarts #:retry #:accept
#:action-path #:find-action #:stamp #:done-p
#:action-path #:find-action
#:operation-definition-warning #:operation-definition-error ;; condition
#:action-valid-p
#:circular-dependency #:circular-dependency-actions
......@@ -435,10 +435,13 @@ Returns two values:
(gethash o (component-operation-times c)))
(defmethod (setf component-operation-time) (stamp (o operation) (c component))
(assert stamp () "invalid null stamp for ~A" (action-description o c))
(setf (gethash o (component-operation-times c)) stamp))
(defmethod mark-operation-done ((o operation) (c component))
(setf (component-operation-time o c) (compute-action-stamp nil o c :just-done t))))
(let ((stamp (compute-action-stamp nil o c :just-done t)))
(assert stamp () "Failed to compute a stamp for completed action ~A" (action-description o c))1
(setf (component-operation-time o c) stamp))))
;;;; Perform
......
......@@ -12,57 +12,70 @@
;; or if it has just been done, return the time that it has.
;; Returns two values:
;; 1- the TIMESTAMP of the action if it has already been done and is up to date,
;; or T is either hasn't been done or is out of date.
;; or NIL is either hasn't been done or is out of date.
;; (An ASDF extension could use a cryptographic digest instead.)
;; 2- the DONE-IN-IMAGE-P boolean flag that is T if the action has already been done
;; in the current image, or NIL if it hasn't.
;; Note that if e.g. LOAD-OP only depends on up-to-date files, but
;; hasn't been done in the current image yet, then it can have a non-T timestamp,
;; hasn't been done in the current image yet, then it can have a non-NIL timestamp,
;; yet a NIL done-in-image-p flag: we can predict what timestamp it will have once loaded,
;; i.e. that of the input-files.
;; If just-done is NIL, these values return are the notional fields of
;; a KEEP, REDO or TODO status (VOID is possible, but probably an error).
;; If just-done is T, they are the notional fields of DONE status
;; (or, if something went wrong, TODO).
(nest
(block ())
(let ((dep-stamp ; collect timestamp from dependencies (or T if forced or out-of-date)
(let* ((dep-status ; collect timestamp from dependencies (or T if forced or out-of-date)
(reduce-direct-dependencies
o c
#'(lambda (o c stamp)
(if-let (it (action-status plan o c))
(latest-stamp stamp (action-stamp it))
t))
nil)))
;; out-of-date dependency: don't bother expensively querying the filesystem
(when (and (eq dep-stamp t) (not just-done))
(let ((outdated-deps
(reduce-direct-dependencies
o c
#'(lambda (o c deps)
(if (if-let (it (action-status plan o c))
(not (eq t (action-stamp it))))
deps
(cons (cons o c) deps)))
nil)))
(DBG "compute-action-stamp: forced by out of date dependency"
(cons o c)
outdated-deps))
(return (values t nil))))
;; collect timestamps from inputs, and exit early if any is missing
(let* ((in-files (input-files o c))
#'(lambda (do dc status)
;; out-of-date dependency: don't bother looking further
(let ((action-status (action-status plan do dc)))
(cond
((and action-status (or (status-keep-p action-status)
(and just-done (status-stamp action-status))))
(merge-action-status action-status status))
(just-done
;; It's OK to lose some ASDF action stamps during self-upgrade
(unless (equal "asdf" (primary-system-name dc))
(warn "Computing just-done stamp in plan ~S for action ~S, but dependency ~S wasn't done yet!"
plan
(action-path (make-action o c))
(action-path (make-action do dc))))
status)
(t
(DBG "compute-action-stamp: forced by out of date dependency" (action-path (make-action o c)) action-status)
(return (values nil nil))))))
+status-good+))
(dep-stamp (status-stamp dep-status)))
;; DBG
(when (null dep-stamp)
(let* ((action-path (action-path (cons o c)))
(dep-statuses (loop for action in (direct-dependencies o c) for (o . c) = action collect
(list (action-path action) (action-status plan o c)))))
(DBG :nds action-path dep-statuses))))
(let* (;; collect timestamps from inputs, and exit early if any is missing
(in-files (input-files o c))
(in-stamps (mapcar #'get-file-stamp in-files))
(missing-in (loop :for f :in in-files :for s :in in-stamps :unless s :collect f))
(latest-in (stamps-latest (cons dep-stamp in-stamps))))
(when (and missing-in (not just-done))
(DBG "compute-action-stamp: missing inputs"
(cons o c)
missing-in)
(return (values t nil))))
;; collect timestamps from outputs, and exit early if any is missing
(let* ((out-files (remove-if 'null (output-files o c)))
(DBG "compute-action-stamp: missing inputs" (cons o c) missing-in)
(return (values nil nil))))
(let* (;; collect timestamps from outputs, and exit early if any is missing
(out-files (remove-if 'null (output-files o c)))
(out-stamps (mapcar (if just-done 'register-file-stamp 'get-file-stamp) out-files))
(missing-out (loop :for f :in out-files :for s :in out-stamps :unless s :collect f))
(earliest-out (stamps-earliest out-stamps)))
(when (and missing-out (not just-done)) (return (values t nil))))
(when (and missing-out (not just-done))
(DBG "compute-action-stamp: missing outputs" (cons o c) missing-out)
(return (values nil nil))))
(let (;; Time stamps from the files at hand, and whether any is missing
(all-present (not (or missing-in missing-out)))
;; Has any input changed since we last generated the files?
;; Note that we use stamp<= instead of stamp< to play nice with generated files.
;; Any race condition is intrinsic to the limited timestamp resolution.
(up-to-date-p (stamp<= latest-in earliest-out))
;; If everything is up to date, the latest of inputs and outputs is our stamp
(done-stamp (stamps-latest (cons latest-in out-stamps))))
......@@ -81,17 +94,18 @@
;;(null-op (and (null out-files) (null in-files))) ; placeholders that do nothing
))
;; Status of the action as previously performed in the image
(progn ;; DBG
(multiple-value-bind (perform-stamp perform-done-p)
(if just-done
(values done-stamp t)
(component-operation-time o c)))
;; Note that we use stamp<= instead of stamp< to play nice with generated files.
;; Any race condition is intrinsic to the limited timestamp resolution.
(let ((already-done-p (and all-present up-to-date-p (operation-done-p o c)
(not (action-forced-p (forcing (or plan *asdf-session*)) o c))))
(matching-stamp-p (and perform-done-p (eql perform-stamp done-stamp))))
(component-operation-time o c))
(let* ((forced (action-forced-p (forcing (or plan *asdf-session*)) o c))
(already-done-p (and all-present up-to-date-p (operation-done-p o c)
(not forced)))
(matching-stamp-p (and perform-done-p (eql perform-stamp done-stamp)))
(action-path (action-path (cons o c))))
(DBG "compute-action-stamp"
(cons o c)
action-path
in-files
in-stamps
dep-stamp
......@@ -104,20 +118,24 @@
perform-done-p
done-stamp
all-present
forced
already-done-p
just-done
out-op
matching-stamp-p))
;; Note that we use stamp<= instead of stamp< to play nice with generated files.
;; Any race condition is intrinsic to the limited timestamp resolution.
matching-stamp-p))))
(if (or just-done ;; The done-stamp is valid: if we're just done, or
;; if all filesystem effects are up-to-date and there's no invalidating reason.
(and all-present up-to-date-p (operation-done-p o c)
(and all-present ;; if all filesystem effects are up-to-date
up-to-date-p
(operation-done-p o c) ;; and there's no invalidating reason.
(not (action-forced-p (forcing (or plan *asdf-session*)) o c))))
(values done-stamp ;; return the hard-earned timestamp
(or just-done
out-op ;; a file-creating op is done when all files are up to date
;; a image-effecting a placeholder op is done when it was actually run,
(and perform-done-p (equal perform-stamp done-stamp)))) ;; with the matching stamp
out-op ;; A file-creating op is done when all files are up to date.
;; An image-effecting operation is done when
(and (status-done-p dep-status) ;; all the dependencies were done, and
(multiple-value-bind (perform-stamp perform-done-p)
(component-operation-time o c)
(and perform-done-p ;; the op was actually run,
(equal perform-stamp done-stamp)))))) ;; with a matching stamp.
;; done-stamp invalid: return a timestamp in an indefinite future, action not done yet
(values t nil)))))
(values nil nil)))))
......@@ -30,4 +30,7 @@ use a wild pathname instead." module)))
(compute-children-by-name self)
(values)))
(defmethod input-files ((o compile-op) (c wild-module)) ())
(defmethod input-files ((o load-op) (c wild-module)) ())
(export 'wild-module)
......@@ -8,7 +8,7 @@
:asdf/find-component :asdf/system-registry :asdf/plan :asdf/operate)
(:export
#:find-system #:locate-system #:load-asd #:define-op
#:load-system-definition-error #:error-name #:error-pathname #:error-condition #:system-out-of-date))
#:load-system-definition-error #:error-name #:error-pathname #:error-condition))
(in-package :asdf/find-system)
(with-upgradability ()
......@@ -20,13 +20,6 @@
(format s (compatfmt "~@<Error while trying to load definition for system ~A from pathname ~A: ~3i~_~A~@:>")
(error-name c) (error-pathname c) (error-condition c)))))
(define-condition system-out-of-date (condition)
;; NB: not an error, not a warning, but a normal expected condition
((name :initarg :name :reader component-name))
(:documentation "condition signaled when a system is detected as being out of date")
(:report (lambda (c s)
(format s "system ~A is out of date" (component-name c)))))
;;; Methods for find-system
......@@ -130,7 +123,7 @@ Do NOT try to load a .asd file directly with CL:LOAD. Always use ASDF:LOAD-ASD."
(progn
;; We already determine this to be obsolete ---
;; or should we move some tests from find-system to check for up-to-date-ness here?
(setf (component-operation-time operation system) nil
(setf (component-operation-time operation system) t
(definition-dependency-list system) nil
(definition-dependency-set system) (list-to-hash-set nil))
(do-it operation system))
......@@ -221,11 +214,28 @@ PREVIOUS-TIME when not null is the time at which the PREVIOUS system was loaded.
;; reconcile the finding (if any) with any previous definition (in a previous session,
;; preloaded, with a previous configuration, or before filesystem changes), and
;; load a found .asd if appropriate. Finally, update registration table and return results.
(defun definition-dependencies-up-to-date-p (system)
(check-type system system)
(assert (primary-system-p system))
(handler-case
(loop :with plan = (make-instance *plan-class*)
:for action :in (definition-dependency-list system)
:always (action-up-to-date-p
plan (action-operation action) (action-component action))
:finally
(let ((o (make-operation 'define-op)))
(multiple-value-bind (stamp done-p)
(compute-action-stamp plan o system)
(return (and (stamp<= stamp (component-operation-time o system))
done-p)))))
(system-out-of-date () nil)))
(defmethod find-system ((name string) &optional (error-p t))
(nest
(with-asdf-session (:key `(find-system ,name)))
(let* ((name-primary-p (primary-system-p name))
(primary-system (unless name-primary-p (find-system (primary-system-name name) nil)))))
(let ((name-primary-p (primary-system-p name)))
(unless name-primary-p (find-system (primary-system-name name) nil)))
(or (and *immutable-systems* (gethash name *immutable-systems*) (registered-system name)))
(multiple-value-bind (foundp found-system pathname previous previous-time)
(locate-system name)
......@@ -248,22 +258,8 @@ PREVIOUS-TIME when not null is the time at which the PREVIOUS system was loaded.
;; TODO: check that all dependencies are up-to-date.
;; This necessitates traversing them without triggering
;; the adding of nodes to the plan.
(loop :with plan = (make-instance *plan-class*)
:for action :in (definition-dependency-list previous)
:always (handler-bind
((system-out-of-date
(lambda (c) (declare (ignore c)) (return nil))))
(action-up-to-date-p plan
(action-operation action)
(action-component action)))
:finally
(let ((o (make-operation 'define-op))
(s (if name-primary-p previous primary-system)))
(when s
(multiple-value-bind (stamp done-p)
(compute-action-stamp plan o s)
(return (and (stamp<= stamp (component-operation-time o s))
done-p)))))))))
(or (not name-primary-p)
(definition-dependencies-up-to-date-p previous)))))
(unless up-to-date-p
(restart-case
(signal 'system-out-of-date :name name)
......@@ -280,8 +276,8 @@ PREVIOUS-TIME when not null is the time at which the PREVIOUS system was loaded.
;; Recurse to children, so asdf/plan will hopefully be happy.
(map () 'mark-component-preloaded (component-children component))
;; Mark the timestamps of the common lisp-action operations as 0.
(let ((times (component-operation-times component)))
(let ((cot (component-operation-times component)))
(dolist (o `(,@(when (primary-system-p component) '(define-op))
prepare-op compile-op load-op))
(setf (gethash (make-operation o) times) 0))))))
(setf (gethash (make-operation o) cot) 0))))))
......@@ -163,7 +163,7 @@ defaults to LOAD-OP, to load it in current image."
"Has the given COMPONENT been successfully loaded in the current image (yet)?
Note that this returns true even if the component is not up to date."
(if-let ((component (find-component component () :registered t)))
(action-already-done-p nil (make-operation 'load-op) component)))
(nth-value 1 (component-operation-time (make-operation 'load-op) component))))
(defun already-loaded-systems ()
"return a list of the names of the systems that have been successfully loaded so far"
......
This diff is collapsed.
......@@ -148,7 +148,7 @@
;; Compute the file stamp for a normalized namestring
(defun compute-file-stamp (normalized-namestring)
(with-pathname-defaults ()
(safe-file-write-date normalized-namestring)))
(or (safe-file-write-date normalized-namestring) t)))
;; Override the time STAMP associated to a given FILE in the session cache.
;; If no STAMP is specified, recompute a new one from the filesystem.
......
......@@ -6,9 +6,21 @@
(assert (not (component-loaded-p 'test-asdf/force)))
(defun force-plan (&rest forcing-options)
(asdf::plan-actions
(asdf::make-plan nil 'load-op 'test-asdf/force
:forcing (apply 'asdf::make-forcing :system 'test-asdf/force forcing-options))))
:forcing (apply 'asdf::make-forcing :system 'test-asdf/force forcing-options)))
(defvar *plan* nil)
(defun component-out-of-date-p (component)
(not
(asdf::action-up-to-date-p
*plan*
(make-operation 'load-op)
(find-component component nil :registered t))))
(defun plan-loads-p (component)
(member (asdf::make-action (make-operation 'load-op) (find-component () component :registered t))
(asdf::plan-actions *plan*) :test 'equal))
(with-asdf-session (:override t :override-forcing t)
(load-system 'test-asdf/force))
......@@ -28,50 +40,61 @@
(reset-session-visited)
(defparameter *plan* (force-plan :force t))
(DBG "Check that :force t forces the current system but not its dependencies" *plan*)
(assert (asdf::plan-operates-on-p *plan* '("test-asdf/force" "file4")))
(assert (not (asdf::plan-operates-on-p *plan* '("file3-only" "file3"))))
(assert (not (asdf::plan-operates-on-p *plan* '("test-asdf/force1" "file1"))))
(assert (component-out-of-date-p '("test-asdf/force" "file4")))
(assert (not (component-out-of-date-p '("file3-only" "file3"))))
(assert (not (component-out-of-date-p '("test-asdf/force1" "file1"))))
(reset-session-visited)
(defparameter *plan* (force-plan :force :all))
(DBG "Check that :force :all forces the current system and its dependencies" *plan*)
(assert *plan*)
(assert (asdf::plan-operates-on-p *plan* '("test-asdf/force" "file4")))
(assert (asdf::plan-operates-on-p *plan* '("file3-only" "file3")))
(assert (asdf::plan-operates-on-p *plan* '("test-asdf/force1" "file1")))
(assert (asdf::plan-actions *plan*))
(assert (component-out-of-date-p '("test-asdf/force" "file4")))
(assert (component-out-of-date-p '("file3-only" "file3")))
(assert (component-out-of-date-p '("test-asdf/force1" "file1")))
(reset-session-visited)
(defparameter *plan* (force-plan :force :all :force-not t))
(DBG "Check that :force-not takes precedence over :force, with t means \"all but current system\"" *plan*)
(assert *plan*)
(assert (asdf::plan-operates-on-p *plan* '("test-asdf/force" "file4")))
(assert (not (asdf::plan-operates-on-p *plan* '("file3-only" "file3"))))
(assert (not (asdf::plan-operates-on-p *plan* '("test-asdf/force1" "file1"))))
(assert (asdf::plan-actions *plan*))
(assert (component-out-of-date-p '("test-asdf/force" "file4")))
(assert (not (component-out-of-date-p '("file3-only" "file3"))))
(assert (not (component-out-of-date-p '("test-asdf/force1" "file1"))))
(reset-session-visited)
(defparameter *plan* (force-plan :force-not :all))
(DBG "Check that :force-not :all means \"all systems\"" *plan*)
(assert (null *plan*))
(assert (null (asdf::plan-actions *plan*)))
(reset-session-visited)
(defparameter *plan* (force-plan :force :all :force-not :all))
(DBG "Check that :force-not :all takes precedence over :force" *plan*)
(assert (null *plan*))
(assert (null (asdf::plan-actions *plan*)))
(reset-session-visited)
(defparameter *plan* (force-plan :force :all :force-not '(:test-asdf/force :test-asdf/force1)))
(defparameter *plan* (force-plan :force :all :force-not '(:test-asdf/force)))
(DBG "Check that :force-not with a list takes precedence over :force" *plan*)
(assert (null *plan*))
(assert (null (asdf::plan-actions *plan*)))
(reset-session-visited)
(defparameter *plan* (force-plan :force :all :force-not '(:test-asdf/force1)))
(DBG "Check that :force-not with a list takes precedence over :force, 2" *plan*)
(assert (asdf::plan-actions *plan*))
(assert (component-out-of-date-p '("test-asdf/force" "file4")))
;; transitively included via test-asdf/force1 test-asdf/only, which blocks the traversal
(assert (not (plan-loads-p '("file3-only" "file3"))))
(assert (component-out-of-date-p '("file3-only" "file3")))
(assert (not (component-out-of-date-p '("test-asdf/force1" "file1"))))
(reset-session-visited)
(let* ((asdf::*immutable-systems* (list-to-hash-set '("test-asdf/force1")))
(plan (force-plan :force :all)))
(DBG "Check that immutable-systems will block forcing" plan)
(assert plan)
(assert (asdf::plan-operates-on-p plan '("test-asdf/force" "file4")))
(*plan* (force-plan :force :all)))
(DBG "Check that immutable-systems will block forcing" *plan*)
(assert (asdf::plan-actions *plan*))
(assert (component-out-of-date-p '("test-asdf/force" "file4")))
;; transitively included via force1 test-asdf/only, which blocks the traversal
(assert (not (asdf::plan-operates-on-p plan '("file3-only" "file3"))))
(assert (not (asdf::plan-operates-on-p plan '("test-asdf/force1" "file1")))))
(assert (not (plan-loads-p '("file3-only" "file3"))))
(assert (component-out-of-date-p '("file3-only" "file3")))
(assert (not (component-out-of-date-p '("test-asdf/force1" "file1")))))
;; unforced, date should stay same
(touch-file "test-asdf.asd" :timestamp date1)
......
......@@ -209,8 +209,6 @@
'(;; slots names
asdf/action:accept
asdf/action:action
asdf/action:done-p
asdf/action:stamp
asdf/bundle:prologue-code
asdf/bundle:epilogue-code
asdf/component:absolute-pathname
......@@ -240,8 +238,6 @@
asdf/component:sideway-dependencies
asdf/component:version
asdf/operation:feature
asdf/plan:index
asdf/plan:planned-p
asdf/system:bug-tracker
asdf/system:build-pathname
asdf/system:system-defsystem-depends-on
......
......@@ -375,26 +375,26 @@ If optional ERROR argument is NIL, return NIL instead of an error when the symbo
(string (standard-case-symbol-name package-designator)))
error)))
;;; stamps: a REAL or a boolean where NIL=-infinity, T=+infinity
;;; stamps: a REAL or a boolean where T=-infinity, NIL=+infinity
(eval-when (#-lispworks :compile-toplevel :load-toplevel :execute)
(deftype stamp () '(or real boolean)))
(with-upgradability ()
(defun stamp< (x y)
(etypecase x
(null (and y t))
((eql t) nil)
((eql t) (not (eql y t)))
(real (etypecase y
(null nil)
((eql t) t)
(real (< x y))))))
((eql t) nil)
(real (< x y))
(null t)))
(null nil)))
(defun stamps< (list) (loop :for y :in list :for x = nil :then y :always (stamp< x y)))
(defun stamp*< (&rest list) (stamps< list))
(defun stamp<= (x y) (not (stamp< y x)))
(defun earlier-stamp (x y) (if (stamp< x y) x y))
(defun stamps-earliest (list) (reduce 'earlier-stamp list :initial-value t))
(defun stamps-earliest (list) (reduce 'earlier-stamp list :initial-value nil))
(defun earliest-stamp (&rest list) (stamps-earliest list))
(defun later-stamp (x y) (if (stamp< x y) y x))
(defun stamps-latest (list) (reduce 'later-stamp list :initial-value nil))
(defun stamps-latest (list) (reduce 'later-stamp list :initial-value t))
(defun latest-stamp (&rest list) (stamps-latest list))
(define-modify-macro latest-stamp-f (&rest stamps) latest-stamp))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment