From 846b7fba5e5be59b682b3bf46934edf95e280624 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Tue, 11 Mar 2014 21:37:01 -0400 Subject: [PATCH 01/40] Load and compile CL code using a predictable, standard syntax. No more leakage from past files or to future files. --- find-system.lisp | 2 +- lisp-action.lisp | 13 ++++++++++--- uiop/stream.lisp | 5 +++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/find-system.lisp b/find-system.lisp index 564d1350..a5f6880d 100644 --- a/find-system.lisp +++ b/find-system.lisp @@ -277,7 +277,7 @@ Going forward, we recommend new users should be using the source-registry. (defmacro with-system-definitions ((&optional) &body body) `(call-with-system-definitions #'(lambda () ,@body))) - (defun load-asd (pathname &key name (external-format (encoding-external-format (detect-encoding pathname))) &aux (readtable *readtable*) (print-pprint-dispatch *print-pprint-dispatch*)) + (defun load-asd (pathname &key name (external-format (encoding-external-format (detect-encoding pathname))) &aux (readtable *standard-readtable*) (print-pprint-dispatch *standard-print-pprint-dispatch*)) ;; Tries to load system definition with canonical NAME from PATHNAME. (with-system-definitions () (with-standard-io-syntax diff --git a/lisp-action.lisp b/lisp-action.lisp index 0622c42e..c2b9ffb9 100644 --- a/lisp-action.lisp +++ b/lisp-action.lisp @@ -33,7 +33,14 @@ (defclass basic-load-op (operation) ()) (defclass basic-compile-op (operation) ((proclamations :initarg :proclamations :accessor compile-op-proclamations :initform nil) - (flags :initarg :flags :accessor compile-op-flags :initform nil)))) + (flags :initarg :flags :accessor compile-op-flags :initform nil))) + + (defclass cl-reading-op (operation) ())) ;; operations that read CL source + (defmethod perform :around ((o cl-reading-op) (c cl-source-file)) + (with-standard-io-syntax + (let ((*package* (find-package :asdf-user)) + (*print-readably* nil)) + (call-next-method)))) ;;; Our default operations: loading into the current lisp image (with-upgradability () @@ -44,12 +51,12 @@ ;; NB: even though compile-op depends on prepare-op it is not needed-in-image-p, ;; so we need to directly depend on prepare-op for its side-effects in the current image. ((selfward-operation :initform '(prepare-op compile-op) :allocation :class))) - (defclass compile-op (basic-compile-op downward-operation selfward-operation) + (defclass compile-op (basic-compile-op cl-reading-op downward-operation selfward-operation) ((selfward-operation :initform 'prepare-op :allocation :class))) (defclass prepare-source-op (upward-operation sideway-operation) ((sideway-operation :initform 'load-source-op :allocation :class))) - (defclass load-source-op (basic-load-op downward-operation selfward-operation) + (defclass load-source-op (basic-load-op cl-reading-op downward-operation selfward-operation) ((selfward-operation :initform 'prepare-source-op :allocation :class))) (defclass test-op (selfward-operation) diff --git a/uiop/stream.lisp b/uiop/stream.lisp index 464e933e..bd620e14 100644 --- a/uiop/stream.lisp +++ b/uiop/stream.lisp @@ -11,6 +11,7 @@ #:detect-encoding #:*encoding-detection-hook* #:always-default-encoding #:encoding-external-format #:*encoding-external-format-hook* #:default-encoding-external-format #:*default-encoding* #:*utf-8-external-format* + #:*standard-readtable* #:*standard-print-pprint-dispatch* #:with-safe-io-syntax #:call-with-safe-io-syntax #:safe-read-from-string #:with-output #:output-string #:with-input #:with-input-file #:call-with-input-file #:with-output-file #:call-with-output-file @@ -146,6 +147,10 @@ going through all the proper hooks." "The standard readtable, implementing the syntax specified by the CLHS. It must never be modified, though only good implementations will even enforce that.") + (defvar *standard-print-pprint-dispatch* (with-standard-io-syntax *print-pprint-dispatch*) + "The standard pprint dispatch table, implementing the syntax specified by the CLHS. +It must never be modified, though only good implementations will even enforce that.") + (defmacro with-safe-io-syntax ((&key (package :cl)) &body body) "Establish safe CL reader options around the evaluation of BODY" `(call-with-safe-io-syntax #'(lambda () (let ((*package* (find-package ,package))) ,@body)))) -- GitLab From 2c11074e55b298346f05c30255a7fe7db59709fc Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Sat, 15 Mar 2014 22:31:08 -0400 Subject: [PATCH 02/40] Introduce a new (call-)with-action-context protocol and a system-variables protocol, and use them to implement per-system syntax tables (*readtable*, *print-pprint-dispatch*). Introduce a basic-prepare-op that initializes the system variables. Fix some bundle hierarchy. UIOP: introduce ensure-variable and variable-value, accept NIL as package designator for :CL. --- action.lisp | 14 ++++++- bundle.lisp | 4 +- find-system.lisp | 17 +++++++-- lisp-action.lisp | 17 +++++---- operate.lisp | 9 +++-- system.lisp | 95 +++++++++++++++++++++++++++++++++++++++++++++-- uiop/stream.lisp | 34 ++++++++++++++--- uiop/utility.lisp | 4 +- 8 files changed, 165 insertions(+), 29 deletions(-) diff --git a/action.lisp b/action.lisp index 59810e77..eb279050 100644 --- a/action.lisp +++ b/action.lisp @@ -15,6 +15,7 @@ #:action-status #:action-stamp #:action-done-p #:component-operation-time #:mark-operation-done #:compute-action-stamp #:perform #:perform-with-restarts #:retry #:accept + #:with-action-context #:call-with-action-context #:traverse-actions #:traverse-sub-actions #:required-components ;; in plan #:action-path #:find-action #:stamp #:done-p #:operation-definition-warning #:operation-definition-error ;; condition @@ -66,7 +67,7 @@ `(apply ',function ,@prefix ,o ,c ,@suffix ,rest) `(,function ,@prefix ,o ,c ,@suffix)))) `(progn - (defmethod ,function (,@prefix (,operation symbol) component ,@suffix ,@more-args) + (defmethod ,function (,@prefix (,operation symbol) ,component ,@suffix ,@more-args) (if ,operation ,(next-method (if operation-initargs ;backward-compatibility with ASDF1's operate. Yuck. @@ -378,11 +379,20 @@ in some previous image, or T if it needs to be done.") (compatfmt "~@") 'perform (cons o c)))) + (defgeneric call-with-action-context (operation component thunk)) + (define-convenience-action-methods call-with-action-context (operation component thunk)) + (defmethod call-with-action-context ((o operation) (c component) thunk) + (with-updated-system-variables ((component-system c)) + (funcall thunk))) + (defmacro with-action-context ((operation component) &body body) + `(call-with-action-context ,operation ,component #'(lambda () ,@body))) + (defmethod perform-with-restarts (operation component) ;; TOO verbose, especially as the default. Add your own :before method ;; to perform-with-restart or perform if you want that: #|(explain operation component)|# - (perform operation component)) + (with-action-context (operation component) + (perform operation component))) (defmethod perform-with-restarts :around (operation component) (loop (restart-case diff --git a/bundle.lisp b/bundle.lisp index fe020bf3..d7ff2895 100644 --- a/bundle.lisp +++ b/bundle.lisp @@ -76,7 +76,7 @@ itself.")) ;; operation on a system and its dependencies (defclass basic-fasl-op (bundle-op) ((bundle-type :initform :fasl))) - (defclass prepare-fasl-op (sideway-operation) + (defclass prepare-fasl-op (basic-prepare-op sideway-operation) ((sideway-operation :initform #+ecl 'load-fasl-op #-ecl 'load-op :allocation :class))) (defclass lib-op (link-op gather-op non-propagating-operation) @@ -87,7 +87,7 @@ itself.")) ;; operation on a system and its dependencies ((selfward-operation :initform '(prepare-fasl-op #+ecl lib-op) :allocation :class))) (defclass load-fasl-op (basic-load-op selfward-operation) - ((selfward-operation :initform '(prepare-op fasl-op) :allocation :class))) + ((selfward-operation :initform '(prepare-fasl-op fasl-op) :allocation :class))) ;; NB: since the monolithic-op's can't be sideway-operation's, ;; if we wanted lib-op, dll-op, deliver-asd-op to be sideway-operation's, diff --git a/find-system.lisp b/find-system.lisp index a5f6880d..5ea3bd8c 100644 --- a/find-system.lisp +++ b/find-system.lisp @@ -9,6 +9,7 @@ #:remove-entry-from-registry #:coerce-entry-to-directory #:coerce-name #:primary-system-name #:coerce-filename #:find-system #:locate-system #:load-asd #:with-system-definitions + #:call-with-asdf-syntax #:with-asdf-syntax #:system-registered-p #:register-system #:registered-systems #:clear-system #:map-systems #:missing-component #:missing-requires #:missing-parent #:formatted-system-definition-error #:format-control #:format-arguments #:sysdef-error @@ -277,17 +278,25 @@ Going forward, we recommend new users should be using the source-registry. (defmacro with-system-definitions ((&optional) &body body) `(call-with-system-definitions #'(lambda () ,@body))) - (defun load-asd (pathname &key name (external-format (encoding-external-format (detect-encoding pathname))) &aux (readtable *standard-readtable*) (print-pprint-dispatch *standard-print-pprint-dispatch*)) + (defun call-with-asdf-syntax (thunk) + (with-standard-io-syntax + (let ((*package* (find-package :asdf-user)) + (*print-readably* nil)) + (funcall thunk)))) + (defmacro with-asdf-syntax ((&optional) &body body) + `(call-with-asdf-syntax #'(lambda () ,@body))) + + (defun load-asd (pathname &key name (external-format (encoding-external-format (detect-encoding pathname)))) ;; Tries to load system definition with canonical NAME from PATHNAME. (with-system-definitions () - (with-standard-io-syntax + (with-asdf-syntax () (let ((*package* (find-package :asdf-user)) ;; Note that our backward-compatible *readtable* is ;; a global readtable that gets globally side-effected. Ouch. ;; Same for the *print-pprint-dispatch* table. ;; We should do something about that for ASDF3 if possible, or else ASDF4. - (*readtable* readtable) - (*print-pprint-dispatch* print-pprint-dispatch) + (*readtable* (copy-readtable nil)) + (*print-pprint-dispatch* (copy-pprint-dispatch nil)) (*print-readably* nil) (*default-pathname-defaults* ;; resolve logical-pathnames so they won't wreak havoc in parsing namestrings. diff --git a/lisp-action.lisp b/lisp-action.lisp index c2b9ffb9..e0005971 100644 --- a/lisp-action.lisp +++ b/lisp-action.lisp @@ -11,6 +11,7 @@ #:try-recompiling #:cl-source-file #:cl-source-file.cl #:cl-source-file.lsp #:basic-load-op #:basic-compile-op #:compile-op-flags #:compile-op-proclamations + #:basic-prepare-op #:cl-reading-op #:load-op #:prepare-op #:compile-op #:test-op #:load-source-op #:prepare-source-op #:call-with-around-compile-hook #:perform-lisp-compilation #:perform-lisp-load-fasl #:perform-lisp-load-source @@ -35,16 +36,18 @@ ((proclamations :initarg :proclamations :accessor compile-op-proclamations :initform nil) (flags :initarg :flags :accessor compile-op-flags :initform nil))) + (defclass basic-prepare-op (operation) ()) + (defmethod perform ((operation basic-prepare-op) (system system)) + (initialize-system-variables system)) + (defclass cl-reading-op (operation) ())) ;; operations that read CL source - (defmethod perform :around ((o cl-reading-op) (c cl-source-file)) - (with-standard-io-syntax - (let ((*package* (find-package :asdf-user)) - (*print-readably* nil)) - (call-next-method)))) + (defmethod call-with-action-context ((o cl-reading-op) (c cl-source-file) thunk) + (call-next-method + o c #'(lambda () (call-with-asdf-syntax thunk)))) ;;; Our default operations: loading into the current lisp image (with-upgradability () - (defclass prepare-op (upward-operation sideway-operation) + (defclass prepare-op (basic-prepare-op upward-operation sideway-operation) ((sideway-operation :initform 'load-op :allocation :class)) (:documentation "Load dependencies necessary for COMPILE-OP or LOAD-OP of a given COMPONENT.")) (defclass load-op (basic-load-op downward-operation selfward-operation) @@ -54,7 +57,7 @@ (defclass compile-op (basic-compile-op cl-reading-op downward-operation selfward-operation) ((selfward-operation :initform 'prepare-op :allocation :class))) - (defclass prepare-source-op (upward-operation sideway-operation) + (defclass prepare-source-op (basic-prepare-op upward-operation sideway-operation) ((sideway-operation :initform 'load-source-op :allocation :class))) (defclass load-source-op (basic-load-op cl-reading-op downward-operation selfward-operation) ((selfward-operation :initform 'prepare-source-op :allocation :class))) diff --git a/operate.lisp b/operate.lisp index 79ac3f8e..0a91f0c3 100644 --- a/operate.lisp +++ b/operate.lisp @@ -76,10 +76,11 @@ The :FORCE or :FORCE-NOT argument to OPERATE can be: component-path keys)))) ;; Setup proper bindings around any operate call. (with-system-definitions () - (let* ((*verbose-out* (and verbose *standard-output*)) - (*compile-file-warnings-behaviour* on-warnings) - (*compile-file-failure-behaviour* on-failure)) - (call-next-method))))) + (with-asdf-syntax () + (let* ((*verbose-out* (and verbose *standard-output*)) + (*compile-file-warnings-behaviour* on-warnings) + (*compile-file-failure-behaviour* on-failure)) + (call-next-method)))))) (defmethod operate :before ((operation operation) (component component) &key version &allow-other-keys) diff --git a/system.lisp b/system.lisp index 3faf2dc0..8255362f 100644 --- a/system.lisp +++ b/system.lisp @@ -11,6 +11,10 @@ #:system-description #:system-long-description #:system-author #:system-maintainer #:system-licence #:system-license #:system-defsystem-depends-on #:system-depends-on #:system-weakly-depends-on + #:compute-system-variables #:configure-system-variables #:configure-system-variable + #:with-updated-system-variables + #:use-system-variables #:initialize-system-variables #:update-system-variables + #:system-variable-specs #:system-variable-names #:system-variable-values #:system-variable-initializers #:component-build-pathname #:build-pathname #:component-entry-point #:entry-point #:homepage #:system-homepage @@ -31,9 +35,7 @@ (defmethod component-entry-point ((c component)) nil)) - ;;;; The system class - (with-upgradability () (defclass proto-system () ; slots to keep when resetting a system ;; To preserve identity for all objects, we'd need keep the components slots @@ -66,7 +68,94 @@ :initform nil) ;; these two are specially set in parse-component-form, so have no :INITARGs. (depends-on :reader system-depends-on :initform nil) - (weakly-depends-on :reader system-weakly-depends-on :initform nil))) + (weakly-depends-on :reader system-weakly-depends-on :initform nil) + ;; System variables + (variable-specs :initform nil :initarg :variables :reader system-variable-specs) + (variable-names :accessor system-variable-names) + (variable-values :accessor system-variable-values) + (variable-initializers :accessor system-variable-initializers))) + + (defgeneric configure-system-variable (system variable &key)) + (defmethod configure-system-variable ((system system) variable &key (initializer nil initp)) + ;; You should configure system variables only during the call to compute-system-variables. + ;; During configuration, the initializer hash-table ensures each variable appears only once, + ;; modulo the fact that you can designate a variable using multiple names, in which case you lose. + ;; The heuristics should be simple, though: if it's a CL or ASDF variable, just use it; + ;; otherwise, use a string that has : if and only if the variable is exported, + ;; which it should be if you're not the author of the system. + ;; Beware that initializers will be called in the order the variables were declared, + ;; which, if you override a variable's initializer, + ;; will be earlier than other variables you're initializing. + (let ((value (variable-value variable :package :asdf :when-undefined nil)) + (previousp (nth-value 1 (gethash variable (system-variable-initializers system))))) + (unless previousp + (push variable (system-variable-names system)) + (push value (system-variable-values system))) + (when (or initp (not previousp)) + (setf (gethash variable (system-variable-initializers system)) + (if initp initializer (constantly value))))) + nil) + + (defgeneric configure-system-variables (system variable-specs)) + (defmethod configure-system-variables ((system system) variable-specs) + (dolist (spec variable-specs) + (multiple-value-bind (variable keys) + (if (consp spec) + (values (first spec) (when (consp (cdr spec)) (list :initializer (second spec)))) + (values spec nil)) + (apply 'configure-system-variable system variable keys)))) + + (defgeneric use-system-variables (system)) + (defmethod use-system-variables ((system system)) + (loop :for variable :in (system-variable-names system) + :for value :in (system-variable-values system) :do + (setf (variable-value variable :package :asdf :when-undefined nil) value))) + + (defgeneric update-system-variables (system)) + (defmethod update-system-variables ((system system)) + (setf (system-variable-values system) + (loop :for variable :in (system-variable-names system) + :collect (variable-value variable :package :asdf :when-undefined nil)))) + + (defgeneric initialize-system-variables (system)) + (defmethod initialize-system-variables ((system system)) + (loop :for variable :in (system-variable-names system) + :for initializer = (gethash variable (system-variable-initializers system)) + :for var = (ensure-variable variable :package :asdf :when-undefined nil) + :when var :do (setf (symbol-value var) (call-function initializer)))) + + (defmacro with-updated-system-variables ((component) &body body) + `(call-with-updated-system-variables ,component #'(lambda () ,@body))) + + (defun call-with-updated-system-variables (component thunk) + (loop :with system = (component-system component) + :for name :in (system-variable-names system) + :for value :in (system-variable-values system) + :for variable = (ensure-variable name :package :asdf :when-undefined nil) + :when variable + :collect variable :into vars + :and :collect value :into vals + :finally + (progv vars vals + (prog1 + (funcall thunk) + (update-system-variables system))))) + + (defgeneric compute-system-variables (system)) + (defmethod compute-system-variables ((system system)) + (configure-system-variables system '(*readtable* *print-pprint-dispatch*)) + (configure-system-variables system (system-variable-specs system)) + ;; TODO: insert an out-of-band system configuration facility , or in an :after method. + nil) + + (defmethod shared-initialize :after ((system system) slot-names &key) + (declare (ignore slot-names)) + (setf (system-variable-names system) nil + (system-variable-values system) nil + (system-variable-initializers system) (make-hash-table :test 'equal)) + (compute-system-variables system) + (setf (system-variable-names system) (reverse (system-variable-names system)) + (system-variable-values system) (reverse (system-variable-values system)))) (defun reset-system (system &rest keys &key &allow-other-keys) (change-class (change-class system 'proto-system) 'system) diff --git a/uiop/stream.lisp b/uiop/stream.lisp index bd620e14..147a303d 100644 --- a/uiop/stream.lisp +++ b/uiop/stream.lisp @@ -24,6 +24,7 @@ #:read-file-string #:read-file-line #:read-file-lines #:safe-read-file-line #:read-file-forms #:read-file-form #:safe-read-file-form #:eval-input #:eval-thunk #:standard-eval-thunk + #:ensure-variable #:variable-value #:println #:writeln ;; Temporary files #:*temporary-directory* #:temporary-directory #:default-temporary-directory @@ -151,19 +152,19 @@ It must never be modified, though only good implementations will even enforce th "The standard pprint dispatch table, implementing the syntax specified by the CLHS. It must never be modified, though only good implementations will even enforce that.") - (defmacro with-safe-io-syntax ((&key (package :cl)) &body body) + (defmacro with-safe-io-syntax ((&key package) &body body) "Establish safe CL reader options around the evaluation of BODY" - `(call-with-safe-io-syntax #'(lambda () (let ((*package* (find-package ,package))) ,@body)))) + `(call-with-safe-io-syntax #'(lambda () ,@body) :package ,package)) - (defun call-with-safe-io-syntax (thunk &key (package :cl)) + (defun call-with-safe-io-syntax (thunk &key package) (with-standard-io-syntax - (let ((*package* (find-package package)) + (let ((*package* (find-package (or package :common-lisp))) (*read-default-float-format* 'double-float) (*print-readably* nil) (*read-eval* nil)) (funcall thunk)))) - (defun safe-read-from-string (string &key (package :cl) (eof-error-p t) eof-value (start 0) end preserve-whitespace) + (defun safe-read-from-string (string &key package (eof-error-p t) eof-value (start 0) end preserve-whitespace) "Read from STRING using a safe syntax, as per WITH-SAFE-IO-SYNTAX" (with-safe-io-syntax (:package package) (read-from-string string eof-error-p eof-value :start start :end end :preserve-whitespace preserve-whitespace)))) @@ -513,6 +514,29 @@ If a string, repeatedly read and evaluate from it, returning the last values." (let ((*read-eval* t)) (eval-thunk thunk)))))) +;;; Late-binding variables +(with-upgradability () + (defun ensure-variable (name &key package (when-undefined 'error)) + (etypecase name + (symbol name) + (string (or (ignore-errors + (let ((s (safe-read-from-string name :package package))) + (and (symbolp s) s))) + (call-function when-undefined + "Cannot read non-nil symbol from ~S" name))))) + + (defun variable-value (name &key package (when-undefined 'error)) + (let ((var (ensure-variable name :package package :when-undefined when-undefined))) + (if (boundp var) + (symbol-value var) + (call-function when-undefined "Symbol ~S unbound" name)))) + + (defun (setf variable-value) (value name &key package (when-undefined 'error)) + (if-let (var (ensure-variable name :package package :when-undefined when-undefined)) + (setf (symbol-value var) value) + value))) + +;;; Printers (with-upgradability () (defun println (x &optional (stream *standard-output*)) "Variant of PRINC that also calls TERPRI afterwards" diff --git a/uiop/utility.lisp b/uiop/utility.lisp index 230c0c76..3ba4653b 100644 --- a/uiop/utility.lisp +++ b/uiop/utility.lisp @@ -365,7 +365,7 @@ the two results passed to STRCAT always reconstitute the original string" ;;; Function designators (with-upgradability () - (defun ensure-function (fun &key (package :cl)) + (defun ensure-function (fun &key package) "Coerce the object FUN into a function. If FUN is a FUNCTION, return it. @@ -386,7 +386,7 @@ and EVAL that in a (FUNCTION ...) context." (eval fun) #'(lambda (&rest args) (apply (car fun) (append (cdr fun) args))))) (string (eval `(function ,(with-standard-io-syntax - (let ((*package* (find-package package))) + (let ((*package* (find-package (or package :common-lisp)))) (read-from-string fun)))))))) (defun access-at (object at) -- GitLab From 3618de9f9ac28c92a5723578c6c0b96b41b4b2e0 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Mon, 24 Mar 2014 15:21:36 -0400 Subject: [PATCH 03/40] Implement shared syntax tables for ASDF systems. --- Makefile | 2 +- TODO | 78 ++++++++++++++++++++++++++++++++++++++++++++++++ asdf.asd | 3 +- find-system.lisp | 23 ++------------ lisp-action.lisp | 2 +- syntax.lisp | 38 +++++++++++++++++++++++ system.lisp | 1 - 7 files changed, 123 insertions(+), 24 deletions(-) create mode 100644 syntax.lisp diff --git a/Makefile b/Makefile index c9287d06..bbcc78ba 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ XCL ?= xcl header_lisp := header.lisp driver_lisp := uiop/package.lisp uiop/common-lisp.lisp uiop/utility.lisp uiop/os.lisp uiop/pathname.lisp uiop/filesystem.lisp uiop/stream.lisp uiop/image.lisp uiop/run-program.lisp uiop/lisp-build.lisp uiop/configuration.lisp uiop/backward-driver.lisp uiop/driver.lisp -defsystem_lisp := upgrade.lisp component.lisp system.lisp cache.lisp find-system.lisp find-component.lisp operation.lisp action.lisp lisp-action.lisp plan.lisp operate.lisp output-translations.lisp source-registry.lisp backward-internals.lisp parse-defsystem.lisp bundle.lisp concatenate-source.lisp backward-interface.lisp package-system.lisp interface.lisp user.lisp footer.lisp +defsystem_lisp := upgrade.lisp component.lisp system.lisp syntax.lisp cache.lisp find-system.lisp find-component.lisp operation.lisp action.lisp lisp-action.lisp plan.lisp operate.lisp output-translations.lisp source-registry.lisp backward-internals.lisp parse-defsystem.lisp bundle.lisp concatenate-source.lisp backward-interface.lisp package-system.lisp interface.lisp user.lisp footer.lisp all_lisp := $(header_lisp) $(driver_lisp) $(defsystem_lisp) # Making ASDF itself should be our first, default, target: diff --git a/TODO b/TODO index a1e37163..00290192 100644 --- a/TODO +++ b/TODO @@ -378,3 +378,81 @@ It looks like SWANK can be fixed soon, though, so we'll see. after the package created if it doesn't exist yet (!) *** There again, a check that a forward-package is not backward would be very nice. + +* Syntax Control: implement and document this plan +1- ASDF maintains an asdf:*shared-readtable*, which is the *readtable* + object at the time it was loaded. + +2- This *shared-readtable* is subject to the current restrictions: + A- no modifying any standard character, + B- no two dependencies assigning different meaning to the same +non-standard character. + C- libraries need to document any change to the readtable + D- free software libraries will register these changes on the page on cliki. + +3- Unhappily, there is no cheap way to enforce these restrictions, but +that's no regression with respect to the current situation. + +4- ASDF wraps any compile-op and load-source-op in this +asdf:*shared-readtable*, but probably not load-op, to preserve +combine-fasl linking semantics. + +5- Systems that want to do crazier things with the readtable that may +violate (2) must arrange to use their own private readtable, but can +otherwise do it safely. It is an error (unhappily not enforceable) to +modify the current readtable in these ways. + +6A- ASDF binds *read-table* to the *shared-readtable* at the start of +every system's compilation (and loading?), and around the entire +asdf:operate, leaving the *readtable* unchanged at the end. + +This easily supports systems that "modify the current readtable data structure". + +However, that doesn't systems that "bind *readtable* to a new value", +because the changes they make will shadow the changes that other +systems following this style make and depend on. To allow such, an +idiom, we must also do the following: + +6B- ASDF binds *read-table* to a proper "entry readtable" at the start +of every system's compilation, and record an "exit readtable" at the +end of the system's loading. + +7- maintain a partial order on these readtable objects, assuming that +each system's exit readtable supersedes the entry readtable. The least +readtable is the *shared-readtable*. It's enough to store for each new +exit readtable, identified by the name of system that created it, the +set of its inferior readtables, as a list or eq-hash-table, or an +equal hash-table, with each readtable being identified by the name of +the system that created it. + +8- before a system is compiled or loaded, compute the maximum +readtable of all the exit readtables of its dependencies. If this +maximum is unique, then it will be the entry readtable of the system. +If there is not a unique readtable that is more than all the other +ones, that's an error, and we refuse to load the system. + +9- after a system is loaded, check its exit readtable, if it already +exists, check that this doesn't create a cycle or issue an error; it +it doesn't already exist, add it to the set of all known exit +readtables. + +10- ASDF either + A- binds the *read-table* to the *shared-readtable* around the entire +asdf:operate, leaving the *readtable* unchanged at the end, or + B- always side-effects the *read-table* to correspond to the exit +readtable of the loaded system, or + C- operate does the binding around thing, but load-system does the +side-effect after it's done operate'ing. + +Does that strike you as complex? Because it is. That's the price of +*safely* supporting this "systems can bind a new value to *readtable*" +style. Unhappily some of the constraints are not enforceable (2A and +2B), but that's the very same as now. + +So my next question is: do you want to safely support these +conventions? Do your systems modify the current *readtable* structure, +or do they bind *readtable* to a new value? + +PS: thank you for making me come up with better solutions. I care +about enough hygiene to use readtables safely, and I also care about +supporting legacy systems if possible. diff --git a/asdf.asd b/asdf.asd index 3c07987a..8cdfa523 100644 --- a/asdf.asd +++ b/asdf.asd @@ -46,8 +46,9 @@ ((:file "upgrade") (:file "component" :depends-on ("upgrade")) (:file "system" :depends-on ("component")) + (:file "syntax" :depends-on ("system")) (:file "cache" :depends-on ("upgrade")) - (:file "find-system" :depends-on ("system" "cache")) + (:file "find-system" :depends-on ("syntax" "cache")) (:file "find-component" :depends-on ("find-system")) (:file "operation" :depends-on ("upgrade")) (:file "action" :depends-on ("find-component" "operation")) diff --git a/find-system.lisp b/find-system.lisp index a0122a1d..81fae125 100644 --- a/find-system.lisp +++ b/find-system.lisp @@ -4,12 +4,11 @@ (uiop/package:define-package :asdf/find-system (:recycle :asdf/find-system :asdf) (:use :uiop/common-lisp :uiop :asdf/upgrade - :asdf/cache :asdf/component :asdf/system) + :asdf/cache :asdf/component :asdf/system :asdf/syntax) (:export #:remove-entry-from-registry #:coerce-entry-to-directory #:coerce-name #:primary-system-name #:coerce-filename #:find-system #:locate-system #:load-asd - #:call-with-asdf-syntax #:with-asdf-syntax #:system-registered-p #:register-system #:registered-systems #:clear-system #:map-systems #:missing-component #:missing-requires #:missing-parent #:formatted-system-definition-error #:format-control #:format-arguments #:sysdef-error @@ -262,27 +261,11 @@ Going forward, we recommend new users should be using the source-registry. ;; notable side effect: mark the system as being defined, to avoid infinite loops (first (gethash `(find-system ,(coerce-name name)) *asdf-cache*))) - (defun call-with-asdf-syntax (thunk &key package) - (with-standard-io-syntax - (let ((*package* (find-package (or package :asdf-user))) - (*print-readably* nil)) - (funcall thunk)))) - (defmacro with-asdf-syntax ((&key package) &body body) - `(call-with-asdf-syntax #'(lambda () ,@body) :package ,package)) - (defun load-asd (pathname &key name (external-format (encoding-external-format (detect-encoding pathname)))) ;; Tries to load system definition with canonical NAME from PATHNAME. (with-asdf-cache () - (with-asdf-syntax () - (let ((*package* (find-package :asdf-user)) - ;; Note that our backward-compatible *readtable* is - ;; a global readtable that gets globally side-effected. Ouch. - ;; Same for the *print-pprint-dispatch* table. - ;; We should do something about that for ASDF3 if possible, or else ASDF4. - (*readtable* (copy-readtable nil)) - (*print-pprint-dispatch* (copy-pprint-dispatch nil)) - (*print-readably* nil) - (*default-pathname-defaults* + (with-asdf-syntax (:package :asdf-user) + (let ((*default-pathname-defaults* ;; resolve logical-pathnames so they won't wreak havoc in parsing namestrings. (pathname-directory-pathname (physicalize-pathname pathname)))) (handler-bind diff --git a/lisp-action.lisp b/lisp-action.lisp index e0005971..5ac02977 100644 --- a/lisp-action.lisp +++ b/lisp-action.lisp @@ -5,7 +5,7 @@ (:recycle :asdf/lisp-action :asdf) (:intern #:proclamations #:flags) (:use :uiop/common-lisp :uiop :asdf/upgrade :asdf/cache - :asdf/component :asdf/system :asdf/find-component :asdf/find-system + :asdf/component :asdf/system :asdf/syntax :asdf/find-component :asdf/find-system :asdf/operation :asdf/action) (:export #:try-recompiling diff --git a/syntax.lisp b/syntax.lisp new file mode 100644 index 00000000..9577285b --- /dev/null +++ b/syntax.lisp @@ -0,0 +1,38 @@ +;;;; ------------------------------------------------------------------------- +;;;; Finding systems + +(uiop/package:define-package :asdf/syntax + (:recycle :asdf/syntax :asdf) + (:use :uiop/common-lisp :uiop :asdf/upgrade :asdf/component :asdf/system) + (:export + #:*shared-readtable* #:*shared-pprint-dispatch* + #:call-with-asdf-syntax #:with-asdf-syntax)) +(in-package :asdf/syntax) + +(with-upgradability () + (defvar *shared-readtable* (copy-readtable nil) + "This shared readtable allows legacy applications to keep modifying a global shared readtable + while maintaining some hygiene for those who want to use their own readtable. + It is subject to the following restrictions, which always existed but were previously implicit: + A- no modifying any standard character, + B- no two dependencies assigning different meaning to the same non-standard character. + Using any non-standard character while expecting the implementation to treat some way + counts as such an assignment of meaning. + C- libraries need to document these assignments of meaning to non-standard characters. + D- free software libraries will register these changes on: + http://www.cliki.net/Macro%20Characters +") + (defvar *shared-pprint-dispatch* (copy-pprint-dispatch nil) + "*print-pprint-dispatch* table shared by all ASDF systems. +It should match the extensions of *shared-readtable* -- see the latter variable's documentation.") + + (defun call-with-asdf-syntax (thunk &key package) + (with-standard-io-syntax + (let ((*readtable* *shared-readtable*) + (*print-pprint-dispatch* *shared-pprint-dispatch*) + (*package* (find-package (or package :asdf-user))) + (*print-readably* nil)) + (funcall thunk)))) + (defmacro with-asdf-syntax ((&key package) &body body) + `(call-with-asdf-syntax #'(lambda () ,@body) :package ,package))) + diff --git a/system.lisp b/system.lisp index 8255362f..5b0f0b18 100644 --- a/system.lisp +++ b/system.lisp @@ -143,7 +143,6 @@ (defgeneric compute-system-variables (system)) (defmethod compute-system-variables ((system system)) - (configure-system-variables system '(*readtable* *print-pprint-dispatch*)) (configure-system-variables system (system-variable-specs system)) ;; TODO: insert an out-of-band system configuration facility , or in an :after method. nil) -- GitLab From ae42c7d162651657e87b74e5a5fed22240007540 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Mon, 24 Mar 2014 15:48:35 -0400 Subject: [PATCH 04/40] Trivial fixes to previous branch work. --- action.lisp | 2 +- bundle.lisp | 4 +++ test/test-operation-classes.script | 4 ++- uiop/utility.lisp | 56 +++++++++++++++--------------- 4 files changed, 36 insertions(+), 30 deletions(-) diff --git a/action.lisp b/action.lisp index de47b69e..99679f3b 100644 --- a/action.lisp +++ b/action.lisp @@ -69,7 +69,7 @@ `(progn (defmethod ,function (,@prefix (,operation string) ,component ,@suffix ,@more-args) (let ((,component (find-component () ,component))) ;; do it first, for defsystem-depends-on - ,(next-method `(safe-read-from-string ,operation :package :asdf/interface) ,component))) + ,(next-method `(safe-read-from-string ,operation :package :asdf/interface) component))) (defmethod ,function (,@prefix (,operation symbol) ,component ,@suffix ,@more-args) (if ,operation ,(next-method diff --git a/bundle.lisp b/bundle.lisp index 4c5c6ea9..7c6204cf 100644 --- a/bundle.lisp +++ b/bundle.lisp @@ -127,6 +127,10 @@ itself.")) ;; operation on a system and its dependencies ((gather-op :initform #+(or ecl mkcl) 'lib-op #-(or ecl mkcl) 'compile-bundle-op :allocation :class)) (:documentation "Create a single fasl for the system and its dependencies.")) + (defclass monolithic-load-bundle-op (monolithic-bundle-op load-bundle-op) + ((selfward-operation :initform 'monolithic-compile-bundle-op :allocation :class)) + (:documentation "Load a single fasl for the system and its dependencies.")) + (defclass monolithic-lib-op (monolithic-bundle-op lib-op non-propagating-operation) () (:documentation "Create a single linkable library for the system and its dependencies.")) diff --git a/test/test-operation-classes.script b/test/test-operation-classes.script index 8482596e..90816d20 100644 --- a/test/test-operation-classes.script +++ b/test/test-operation-classes.script @@ -11,6 +11,7 @@ (defparameter *good-classes* '(build-op + compile-bundle-op compile-concatenated-source-op compile-op concatenate-source-op @@ -19,6 +20,7 @@ fasl-op image-op lib-op + load-bundle-op load-compiled-concatenated-source-op load-concatenated-source-op load-fasl-op @@ -32,7 +34,7 @@ monolithic-lib-op monolithic-load-compiled-concatenated-source-op monolithic-load-concatenated-source-op - prepare-fasl-op + prepare-bundle-op prepare-op prepare-source-op program-op diff --git a/uiop/utility.lisp b/uiop/utility.lisp index d8387b51..a1e26874 100644 --- a/uiop/utility.lisp +++ b/uiop/utility.lisp @@ -331,34 +331,6 @@ the two results passed to STRCAT always reconstitute the original string" (when x (c +crlf+) (c +lf+) (c +cr+) (values x nil))))) -;;; CLOS -(with-upgradability () - (defun coerce-class (class &key (package :cl) (super t) (error 'error)) - "Coerce CLASS to a class that is subclass of SUPER if specified, -or invoke ERROR handler as per CALL-FUNCTION. - -A keyword designates the name a symbol, which when found in PACKAGE, designates a class. -A string is read as a symbol while in PACKAGE, the symbol designates a class. - -A class object designates itself. -NIL designates itself (no class). -A symbol otherwise designates a class by name." - (let* ((normalized - (typecase class - (keyword (find-symbol* class package nil)) - (string (symbol-call :uiop :safe-read-from-string class :package package)) - (t class))) - (found - (etypecase normalized - ((or standard-class built-in-class) normalized) - ((or null keyword) nil) - (symbol (find-class normalized nil nil))))) - (or (and found - (or (eq super t) (#-cormanlisp subtypep #+cormanlisp cl::subclassp found super)) - found) - (call-function error "Can't coerce ~S to a ~@[class~;subclass of ~:*~S]" class super))))) - - ;;; stamps: a REAL or a boolean where NIL=-infinity, T=+infinity (eval-when (#-lispworks :compile-toplevel :load-toplevel :execute) (deftype stamp () '(or real boolean))) @@ -457,6 +429,34 @@ When CALL-NOW-P is true, also call the function immediately." (when call-now-p (call-function hook)))) +;;; CLOS +(with-upgradability () + (defun coerce-class (class &key (package :cl) (super t) (error 'error)) + "Coerce CLASS to a class that is subclass of SUPER if specified, +or invoke ERROR handler as per CALL-FUNCTION. + +A keyword designates the name a symbol, which when found in PACKAGE, designates a class. +A string is read as a symbol while in PACKAGE, the symbol designates a class. + +A class object designates itself. +NIL designates itself (no class). +A symbol otherwise designates a class by name." + (let* ((normalized + (typecase class + (keyword (find-symbol* class package nil)) + (string (symbol-call :uiop :safe-read-from-string class :package package)) + (t class))) + (found + (etypecase normalized + ((or standard-class built-in-class) normalized) + ((or null keyword) nil) + (symbol (find-class normalized nil nil))))) + (or (and found + (or (eq super t) (#-cormanlisp subtypep #+cormanlisp cl::subclassp found super)) + found) + (call-function error "Can't coerce ~S to a ~@[class~;subclass of ~:*~S]" class super))))) + + ;;; Hash-tables (with-upgradability () (defun ensure-gethash (key table default) -- GitLab From 12a6f3ec70fcde93825e0a7ff8be32adbfb1ecfa Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Mon, 24 Mar 2014 15:52:40 -0400 Subject: [PATCH 05/40] Accept *package* as well as package in coerce-class, for backward compatibility with e.g. ironclad. --- uiop/utility.lisp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uiop/utility.lisp b/uiop/utility.lisp index a1e26874..28743cfa 100644 --- a/uiop/utility.lisp +++ b/uiop/utility.lisp @@ -435,7 +435,8 @@ When CALL-NOW-P is true, also call the function immediately." "Coerce CLASS to a class that is subclass of SUPER if specified, or invoke ERROR handler as per CALL-FUNCTION. -A keyword designates the name a symbol, which when found in PACKAGE, designates a class. +A keyword designates the name a symbol, which when found in either PACKAGE, designates a class. +-- for backward compatibility, *PACKAGE* is also accepted for now, but this may go in the future. A string is read as a symbol while in PACKAGE, the symbol designates a class. A class object designates itself. @@ -443,7 +444,8 @@ NIL designates itself (no class). A symbol otherwise designates a class by name." (let* ((normalized (typecase class - (keyword (find-symbol* class package nil)) + (keyword (or (find-symbol* class package nil) + (find-symbol* class *package* nil))) (string (symbol-call :uiop :safe-read-from-string class :package package)) (t class))) (found -- GitLab From 66c4d3ceca975fbba68cd556ae43e499f3dc2f43 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Mon, 24 Mar 2014 16:00:32 -0400 Subject: [PATCH 06/40] For more backward compatibility, set *shared-readtbale* from *readtable*. Export it. --- interface.lisp | 2 ++ syntax.lisp | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/interface.lisp b/interface.lisp index 02690227..7dffaee2 100644 --- a/interface.lisp +++ b/interface.lisp @@ -136,6 +136,8 @@ #:*default-encoding* #:*utf-8-external-format* + #:*shared-readtable* #:*shared-pprint-dispatch* + #:clear-configuration #:*output-translations-parameter* #:initialize-output-translations diff --git a/syntax.lisp b/syntax.lisp index 9577285b..3c899187 100644 --- a/syntax.lisp +++ b/syntax.lisp @@ -10,7 +10,7 @@ (in-package :asdf/syntax) (with-upgradability () - (defvar *shared-readtable* (copy-readtable nil) + (defvar *shared-readtable* *readtable* "This shared readtable allows legacy applications to keep modifying a global shared readtable while maintaining some hygiene for those who want to use their own readtable. It is subject to the following restrictions, which always existed but were previously implicit: @@ -22,7 +22,7 @@ D- free software libraries will register these changes on: http://www.cliki.net/Macro%20Characters ") - (defvar *shared-pprint-dispatch* (copy-pprint-dispatch nil) + (defvar *shared-pprint-dispatch* *print-pprint-dispatch* "*print-pprint-dispatch* table shared by all ASDF systems. It should match the extensions of *shared-readtable* -- see the latter variable's documentation.") -- GitLab From 7d65fa66220066c53ea878a07cb9f2a2378a8260 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Mon, 24 Mar 2014 16:27:29 -0400 Subject: [PATCH 07/40] Package :asdf uses :asdf/syntax --- interface.lisp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface.lisp b/interface.lisp index 7dffaee2..749c5da7 100644 --- a/interface.lisp +++ b/interface.lisp @@ -8,7 +8,7 @@ #:loaded-systems ; makes for annoying SLIME completion #:output-files-for-system-and-operation) ; ASDF-BINARY-LOCATION function we use to detect ABL (:use :uiop/common-lisp :uiop :asdf/upgrade :asdf/cache - :asdf/component :asdf/system :asdf/find-system :asdf/find-component + :asdf/component :asdf/system :asdf/syntax :asdf/find-system :asdf/find-component :asdf/operation :asdf/action :asdf/lisp-action :asdf/output-translations :asdf/source-registry :asdf/plan :asdf/operate :asdf/parse-defsystem :asdf/bundle :asdf/concatenate-source -- GitLab From a3a6e8daec22999025ead17bccb0f39ee02dc798 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Mon, 24 Mar 2014 16:35:01 -0400 Subject: [PATCH 08/40] Better debug plan issues in test-asdf. --- test/test-asdf.script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-asdf.script b/test/test-asdf.script index 5c64500c..52d4e043 100644 --- a/test/test-asdf.script +++ b/test/test-asdf.script @@ -1,7 +1,7 @@ (load-system "test-asdf/test-module-depend") (defparameter *plan* (make-plan () 'load-op "test-asdf/test-module-depend")) -(DBG :foo *plan* +(DBG :foo (asdf::plan-actions *plan*) (component-depends-on 'compile-op '("test-asdf/test-module-depend" "quux" "file3mod" "file3")) (asdf::component-if-feature (find-component "test-asdf/test-module-depend" '("quux" "file3mod" "file3")))) -- GitLab From e09d4c27ac40590e66136a465879ad2414dec2e2 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Wed, 26 Mar 2014 17:00:39 -0400 Subject: [PATCH 09/40] Bind *read-default-float-format* to 'double-float in with-asdf-syntax. Makes femlisp compile, and is a more useful default than single-float from with-standard-io-syntax. But that raises the question: should we try to propagate *read-default-float-format* and other such syntax variables within a system? From one system to another? That's a can of worms. --- syntax.lisp | 1 + 1 file changed, 1 insertion(+) diff --git a/syntax.lisp b/syntax.lisp index 3c899187..9a235c09 100644 --- a/syntax.lisp +++ b/syntax.lisp @@ -31,6 +31,7 @@ It should match the extensions of *shared-readtable* -- see the latter variable' (let ((*readtable* *shared-readtable*) (*print-pprint-dispatch* *shared-pprint-dispatch*) (*package* (find-package (or package :asdf-user))) + (*read-default-float-format* 'double-float) (*print-readably* nil)) (funcall thunk)))) (defmacro with-asdf-syntax ((&key package) &body body) -- GitLab From d32afa0c6bde3827dfae979b7463dc82514bc26a Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Fri, 28 Mar 2014 01:42:10 -0400 Subject: [PATCH 10/40] Set the initial *read-default-float-format* to the newly enforced with-asdf-syntax default. Considering the discussion on asdf-devel, it looks like this will be reversed in favor of 'single-float, at which point we should grep for double-float in header, syntax, uiop/stream. --- header.lisp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/header.lisp b/header.lisp index 4b392fc4..1498aa5d 100644 --- a/header.lisp +++ b/header.lisp @@ -60,8 +60,13 @@ (second (third (sys::arglist 'directory)))) (push :abcl-bundle-op-supported *features*))) -;; Punt on hard package upgrade: from ASDF1 always, and even from ASDF2 on most implementations. (eval-when (:load-toplevel :compile-toplevel :execute) + ;; New default since ASDF 3.1: enforced via with-asdf-syntax for programs, + ;; it is set here so that the REPL matches this expectation by default. + ;; Before that, programs just couldn't trust it having any value, + ;; and the initial value 'single-float was relatively lossy. + (setf *read-default-float-format* 'double-float) + ;; Punt on hard package upgrade: from ASDF1 always, and even from ASDF2 on most implementations. (unless (member :asdf3 *features*) (let* ((existing-version (when (find-package :asdf) -- GitLab From 631659b7e49099790651f45d59a2f954ec971020 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Sun, 30 Mar 2014 16:25:49 -0400 Subject: [PATCH 11/40] Checkpointing some changes in the syntax-control branch: * no more trying to bind *read-default-float-format* to 'double-float, there is consensus against that on the mailing-list. * move syntax and evaluation functions to uiop/eval * refactor the system variables support * introduce fwrap to combine function wrappers. --- Makefile | 4 +- asdf.asd | 3 +- find-system.lisp | 14 +++++- header.lisp | 5 -- interface.lisp | 4 +- lisp-action.lisp | 2 +- syntax.lisp | 39 ---------------- system.lisp | 63 ++++++++++++++----------- uiop/driver.lisp | 2 +- uiop/eval.lisp | 117 ++++++++++++++++++++++++++++++++++++++++++++++ uiop/image.lisp | 6 +-- uiop/stream.lisp | 73 ++--------------------------- uiop/uiop.asd | 9 ++-- uiop/utility.lisp | 61 ++++++++++++++++++++++-- 14 files changed, 243 insertions(+), 159 deletions(-) delete mode 100644 syntax.lisp create mode 100644 uiop/eval.lisp diff --git a/Makefile b/Makefile index bbcc78ba..6cc354b2 100644 --- a/Makefile +++ b/Makefile @@ -53,8 +53,8 @@ SCL ?= scl XCL ?= xcl header_lisp := header.lisp -driver_lisp := uiop/package.lisp uiop/common-lisp.lisp uiop/utility.lisp uiop/os.lisp uiop/pathname.lisp uiop/filesystem.lisp uiop/stream.lisp uiop/image.lisp uiop/run-program.lisp uiop/lisp-build.lisp uiop/configuration.lisp uiop/backward-driver.lisp uiop/driver.lisp -defsystem_lisp := upgrade.lisp component.lisp system.lisp syntax.lisp cache.lisp find-system.lisp find-component.lisp operation.lisp action.lisp lisp-action.lisp plan.lisp operate.lisp output-translations.lisp source-registry.lisp backward-internals.lisp parse-defsystem.lisp bundle.lisp concatenate-source.lisp backward-interface.lisp package-system.lisp interface.lisp user.lisp footer.lisp +driver_lisp := uiop/package.lisp uiop/common-lisp.lisp uiop/utility.lisp uiop/os.lisp uiop/pathname.lisp uiop/filesystem.lisp uiop/stream.lisp uiop/eval.lisp uiop/image.lisp uiop/run-program.lisp uiop/lisp-build.lisp uiop/configuration.lisp uiop/backward-driver.lisp uiop/driver.lisp +defsystem_lisp := upgrade.lisp component.lisp system.lisp cache.lisp find-system.lisp find-component.lisp operation.lisp action.lisp lisp-action.lisp plan.lisp operate.lisp output-translations.lisp source-registry.lisp backward-internals.lisp parse-defsystem.lisp bundle.lisp concatenate-source.lisp backward-interface.lisp package-system.lisp interface.lisp user.lisp footer.lisp all_lisp := $(header_lisp) $(driver_lisp) $(defsystem_lisp) # Making ASDF itself should be our first, default, target: diff --git a/asdf.asd b/asdf.asd index 0012d9f7..f79df388 100644 --- a/asdf.asd +++ b/asdf.asd @@ -46,9 +46,8 @@ ((:file "upgrade") (:file "component" :depends-on ("upgrade")) (:file "system" :depends-on ("component")) - (:file "syntax" :depends-on ("system")) (:file "cache" :depends-on ("upgrade")) - (:file "find-system" :depends-on ("syntax" "cache")) + (:file "find-system" :depends-on ("system" "cache")) (:file "find-component" :depends-on ("find-system")) (:file "operation" :depends-on ("find-system")) (:file "action" :depends-on ("find-component" "operation")) diff --git a/find-system.lisp b/find-system.lisp index 81fae125..d70db1bb 100644 --- a/find-system.lisp +++ b/find-system.lisp @@ -4,11 +4,12 @@ (uiop/package:define-package :asdf/find-system (:recycle :asdf/find-system :asdf) (:use :uiop/common-lisp :uiop :asdf/upgrade - :asdf/cache :asdf/component :asdf/system :asdf/syntax) + :asdf/cache :asdf/component :asdf/system) (:export #:remove-entry-from-registry #:coerce-entry-to-directory #:coerce-name #:primary-system-name #:coerce-filename #:find-system #:locate-system #:load-asd + #:call-with-asdf-syntax #:with-asdf-syntax #:system-registered-p #:register-system #:registered-systems #:clear-system #:map-systems #:missing-component #:missing-requires #:missing-parent #:formatted-system-definition-error #:format-control #:format-arguments #:sysdef-error @@ -261,6 +262,17 @@ Going forward, we recommend new users should be using the source-registry. ;; notable side effect: mark the system as being defined, to avoid infinite loops (first (gethash `(find-system ,(coerce-name name)) *asdf-cache*))) + (defun call-with-asdf-syntax (function &key package) + (with-standard-io-syntax + (let ((*readtable* *shared-readtable*) + (*print-pprint-dispatch* *shared-print-pprint-dispatch*) + (*package* (find-package (or package :asdf-user))) + (*print-readably* nil)) + (call-function function)))) + + (defmacro with-asdf-syntax ((&key package) &body body) + `(call-with-asdf-syntax #'(lambda () ,@body) :package ,package)) + (defun load-asd (pathname &key name (external-format (encoding-external-format (detect-encoding pathname)))) ;; Tries to load system definition with canonical NAME from PATHNAME. (with-asdf-cache () diff --git a/header.lisp b/header.lisp index 1498aa5d..0a93a68a 100644 --- a/header.lisp +++ b/header.lisp @@ -61,11 +61,6 @@ (push :abcl-bundle-op-supported *features*))) (eval-when (:load-toplevel :compile-toplevel :execute) - ;; New default since ASDF 3.1: enforced via with-asdf-syntax for programs, - ;; it is set here so that the REPL matches this expectation by default. - ;; Before that, programs just couldn't trust it having any value, - ;; and the initial value 'single-float was relatively lossy. - (setf *read-default-float-format* 'double-float) ;; Punt on hard package upgrade: from ASDF1 always, and even from ASDF2 on most implementations. (unless (member :asdf3 *features*) (let* ((existing-version diff --git a/interface.lisp b/interface.lisp index dcd0d840..d6fa1368 100644 --- a/interface.lisp +++ b/interface.lisp @@ -8,7 +8,7 @@ #:loaded-systems ; makes for annoying SLIME completion #:output-files-for-system-and-operation) ; ASDF-BINARY-LOCATION function we use to detect ABL (:use :uiop/common-lisp :uiop :asdf/upgrade :asdf/cache - :asdf/component :asdf/system :asdf/syntax :asdf/find-system :asdf/find-component + :asdf/component :asdf/system :asdf/find-system :asdf/find-component :asdf/operation :asdf/action :asdf/lisp-action :asdf/output-translations :asdf/source-registry :asdf/plan :asdf/operate :asdf/parse-defsystem :asdf/bundle :asdf/concatenate-source @@ -136,7 +136,7 @@ #:*default-encoding* #:*utf-8-external-format* - #:*shared-readtable* #:*shared-pprint-dispatch* + #:call-with-asdf-syntax #:with-asdf-syntax #:clear-configuration #:*output-translations-parameter* diff --git a/lisp-action.lisp b/lisp-action.lisp index 5ac02977..e0005971 100644 --- a/lisp-action.lisp +++ b/lisp-action.lisp @@ -5,7 +5,7 @@ (:recycle :asdf/lisp-action :asdf) (:intern #:proclamations #:flags) (:use :uiop/common-lisp :uiop :asdf/upgrade :asdf/cache - :asdf/component :asdf/system :asdf/syntax :asdf/find-component :asdf/find-system + :asdf/component :asdf/system :asdf/find-component :asdf/find-system :asdf/operation :asdf/action) (:export #:try-recompiling diff --git a/syntax.lisp b/syntax.lisp deleted file mode 100644 index 9a235c09..00000000 --- a/syntax.lisp +++ /dev/null @@ -1,39 +0,0 @@ -;;;; ------------------------------------------------------------------------- -;;;; Finding systems - -(uiop/package:define-package :asdf/syntax - (:recycle :asdf/syntax :asdf) - (:use :uiop/common-lisp :uiop :asdf/upgrade :asdf/component :asdf/system) - (:export - #:*shared-readtable* #:*shared-pprint-dispatch* - #:call-with-asdf-syntax #:with-asdf-syntax)) -(in-package :asdf/syntax) - -(with-upgradability () - (defvar *shared-readtable* *readtable* - "This shared readtable allows legacy applications to keep modifying a global shared readtable - while maintaining some hygiene for those who want to use their own readtable. - It is subject to the following restrictions, which always existed but were previously implicit: - A- no modifying any standard character, - B- no two dependencies assigning different meaning to the same non-standard character. - Using any non-standard character while expecting the implementation to treat some way - counts as such an assignment of meaning. - C- libraries need to document these assignments of meaning to non-standard characters. - D- free software libraries will register these changes on: - http://www.cliki.net/Macro%20Characters -") - (defvar *shared-pprint-dispatch* *print-pprint-dispatch* - "*print-pprint-dispatch* table shared by all ASDF systems. -It should match the extensions of *shared-readtable* -- see the latter variable's documentation.") - - (defun call-with-asdf-syntax (thunk &key package) - (with-standard-io-syntax - (let ((*readtable* *shared-readtable*) - (*print-pprint-dispatch* *shared-pprint-dispatch*) - (*package* (find-package (or package :asdf-user))) - (*read-default-float-format* 'double-float) - (*print-readably* nil)) - (funcall thunk)))) - (defmacro with-asdf-syntax ((&key package) &body body) - `(call-with-asdf-syntax #'(lambda () ,@body) :package ,package))) - diff --git a/system.lisp b/system.lisp index 5b0f0b18..a66fdf4a 100644 --- a/system.lisp +++ b/system.lisp @@ -11,9 +11,11 @@ #:system-description #:system-long-description #:system-author #:system-maintainer #:system-licence #:system-license #:system-defsystem-depends-on #:system-depends-on #:system-weakly-depends-on + #:*default-system-variables* #:*asdf-syntax-variables* #:compute-system-variables #:configure-system-variables #:configure-system-variable #:with-updated-system-variables - #:use-system-variables #:initialize-system-variables #:update-system-variables + #:use-initial-system-variables #:use-current-system-variables + #:initialize-system-variables #:update-system-variables #:system-variable-specs #:system-variable-names #:system-variable-values #:system-variable-initializers #:component-build-pathname #:build-pathname #:component-entry-point #:entry-point @@ -75,6 +77,17 @@ (variable-values :accessor system-variable-values) (variable-initializers :accessor system-variable-initializers))) + (defun ensure-system-variable (variable &key when-undefined) + (ensure-variable variable :package :asdf :when-undefined when-undefined)) + + (defvar *asdf-syntax-variables* + `(,@*standard-syntax-variables* + (*readtable* . ,*shared-readtable*) + (*print-pprint-dispatch* . ,*shared-print-pprint-dispatch*) + (*package* . (find-package :asdf-user)))) + + (defvar *default-system-variables* '()) ;; make it *asdf-syntax-variables* for safety + (defgeneric configure-system-variable (system variable &key)) (defmethod configure-system-variable ((system system) variable &key (initializer nil initp)) ;; You should configure system variables only during the call to compute-system-variables. @@ -105,41 +118,34 @@ (values spec nil)) (apply 'configure-system-variable system variable keys)))) - (defgeneric use-system-variables (system)) - (defmethod use-system-variables ((system system)) - (loop :for variable :in (system-variable-names system) - :for value :in (system-variable-values system) :do - (setf (variable-value variable :package :asdf :when-undefined nil) value))) - (defgeneric update-system-variables (system)) (defmethod update-system-variables ((system system)) (setf (system-variable-values system) (loop :for variable :in (system-variable-names system) :collect (variable-value variable :package :asdf :when-undefined nil)))) - (defgeneric initialize-system-variables (system)) - (defmethod initialize-system-variables ((system system)) - (loop :for variable :in (system-variable-names system) - :for initializer = (gethash variable (system-variable-initializers system)) - :for var = (ensure-variable variable :package :asdf :when-undefined nil) - :when var :do (setf (symbol-value var) (call-function initializer)))) + (defgeneric system-variable-initializer-list (system)) + (defmethod system-variable-initializer-list ((system system)) + (loop :for var :in (system-variable-names system) + :collect (gethash var (system-variable-initializers system)))) + + (defun use-initial-system-variables (system) + (set-variables (system-variable-names system) (system-variable-initializer-list system) + :name #'ensure-system-variable :value #'call-function)) + + (defun use-current-system-variables (system) + (set-variables (system-variable-names system) (system-variable-values system) + :name #'ensure-system-variable)) (defmacro with-updated-system-variables ((component) &body body) `(call-with-updated-system-variables ,component #'(lambda () ,@body))) (defun call-with-updated-system-variables (component thunk) - (loop :with system = (component-system component) - :for name :in (system-variable-names system) - :for value :in (system-variable-values system) - :for variable = (ensure-variable name :package :asdf :when-undefined nil) - :when variable - :collect variable :into vars - :and :collect value :into vals - :finally - (progv vars vals - (prog1 - (funcall thunk) - (update-system-variables system))))) + (let ((system (component-system component))) + (with-variables ((system-variable-names system) (system-variable-values system) + :name #'ensure-system-variable) + (multiple-value-prog1 (call-function thunk) + (update-system-variables system))))) (defgeneric compute-system-variables (system)) (defmethod compute-system-variables ((system system)) @@ -147,8 +153,7 @@ ;; TODO: insert an out-of-band system configuration facility , or in an :after method. nil) - (defmethod shared-initialize :after ((system system) slot-names &key) - (declare (ignore slot-names)) + (defun initialize-system-variables (system) (setf (system-variable-names system) nil (system-variable-values system) nil (system-variable-initializers system) (make-hash-table :test 'equal)) @@ -156,6 +161,10 @@ (setf (system-variable-names system) (reverse (system-variable-names system)) (system-variable-values system) (reverse (system-variable-values system)))) + (defmethod shared-initialize :after ((system system) slot-names &key) + (declare (ignore slot-names)) + (initialize-system-variables system)) + (defun reset-system (system &rest keys &key &allow-other-keys) (change-class (change-class system 'proto-system) 'system) (apply 'reinitialize-instance system keys))) diff --git a/uiop/driver.lisp b/uiop/driver.lisp index f36e02df..eacbc722 100644 --- a/uiop/driver.lisp +++ b/uiop/driver.lisp @@ -10,7 +10,7 @@ ;; or :use (closer-common-lisp uiop), etc. (:use-reexport :uiop/package :uiop/utility - :uiop/os :uiop/pathname :uiop/stream :uiop/filesystem :uiop/image + :uiop/os :uiop/pathname :uiop/stream :uiop/filesystem :uiop/eval :uiop/image :uiop/run-program :uiop/lisp-build :uiop/configuration :uiop/backward-driver)) diff --git a/uiop/eval.lisp b/uiop/eval.lisp new file mode 100644 index 00000000..aa546e20 --- /dev/null +++ b/uiop/eval.lisp @@ -0,0 +1,117 @@ +;;;; ------------------------------------------------------------------------- +;;;; Lisp Evaluation + +(uiop/package:define-package :uiop/eval + (:recycle :uiop/eval :asdf) + (:use :uiop/common-lisp :uiop/utility :uiop/stream) + (:export + #:*standard-readtable* #:*standard-print-pprint-dispatch* + #:*shared-readtable* #:*shared-print-pprint-dispatch* + #:*standard-syntax-variables* + #:eval-input #:eval-thunk #:standard-eval-thunk + #:ensure-variable #:variable-value)) +(in-package :uiop/eval) + +;;; Safe syntax +(with-upgradability () + (defvar *standard-readtable* (with-standard-io-syntax *readtable*) + "The standard readtable, implementing the syntax specified by the CLHS. +It must never be modified, though only good implementations will even enforce that.") + + (defvar *standard-print-pprint-dispatch* (with-standard-io-syntax *print-pprint-dispatch*) + "The standard pprint dispatch table, implementing the syntax specified by the CLHS. +It must never be modified, though only good implementations will even enforce that.") + + (defvar *shared-readtable* *readtable* + "This shared readtable allows legacy applications to keep modifying a global shared readtable + while maintaining some hygiene for those who want to use their own readtable. + It is subject to the following restrictions, which always existed but were previously implicit: + A- no modifying any standard character, + B- no two dependencies assigning different meaning to the same non-standard character. + Using any non-standard character while expecting the implementation to treat some way + counts as such an assignment of meaning. + C- libraries need to document these assignments of meaning to non-standard characters. + D- free software libraries will register these changes on: + http://www.cliki.net/Macro%20Characters +") + (defvar *shared-print-pprint-dispatch* *print-pprint-dispatch* + "*print-pprint-dispatch* table shared by all ASDF systems. +It should match the extensions of *shared-readtable* -- see the latter variable's documentation.") + + (defvar *standard-syntax-variables* + `((*package* . (find-package :cl-user)) + (*print-array* . t) + (*print-base* . 10) + (*print-case* . :upcase) + (*print-circle* . nil) + (*print-escape* . t) + (*print-gensym* . t) + (*print-length* . nil) + (*print-level* . nil) + (*print-lines* . nil) + (*print-miser-width* . nil) + (*print-pprint-dispatch* . ,*standard-print-pprint-dispatch*) + (*print-pretty* . nil) + (*print-radix* . nil) + (*print-readably* . t) + (*print-right-margin* . nil) + (*read-base* . 10) + (*read-default-float-format* . single-float) + (*read-eval* . t) + (*read-suppress* . nil) + (*readtable* . ,*standard-readtable*)))) + +(with-upgradability () + (defun call-with-standard-io-syntax (function) + (with-standard-io-syntax (call-function function))) + + (defun eval-input (input) + "Portably read and evaluate forms from INPUT, return the last values." + (with-input (input) + (loop :with results :with eof ='#:eof + :for form = (read input nil eof) + :until (eq form eof) + :do (setf results (multiple-value-list (eval form))) + :finally (return (apply 'values results))))) + + (defun eval-thunk (thunk) + "Evaluate a THUNK of code: +If a function, FUNCALL it without arguments. +If a constant literal and not a sequence, return it. +If a cons or a symbol, EVAL it. +If a string, repeatedly read and evaluate from it, returning the last values." + (etypecase thunk + ((or boolean keyword number character pathname) thunk) + ((or cons symbol) (eval thunk)) + (function (funcall thunk)) + (string (eval-input thunk)))) + + (defun standard-eval-thunk (thunk &key (package :cl)) + "Like EVAL-THUNK, but in a more standardized evaluation context." + ;; Note: it's "standard-" not "safe-", because evaluation is never safe. + (when thunk + (with-safe-io-syntax (:package package) + (let ((*read-eval* t)) + (eval-thunk thunk)))))) + +;;; Late-binding variables +(with-upgradability () + (defun ensure-variable (name &key package (when-undefined 'error)) + (etypecase name + (symbol name) + (string (or (ignore-errors + (let ((s (safe-read-from-string name :package package))) + (and (symbolp s) s))) + (call-function when-undefined + "Cannot read non-nil symbol from ~S" name))))) + + (defun variable-value (name &key package (when-undefined 'error)) + (let ((var (ensure-variable name :package package :when-undefined when-undefined))) + (if (boundp var) + (symbol-value var) + (call-function when-undefined "Symbol ~S unbound" name)))) + + (defun (setf variable-value) (value name &key package (when-undefined 'error)) + (if-let (var (ensure-variable name :package package :when-undefined when-undefined)) + (setf (symbol-value var) value) + value))) diff --git a/uiop/image.lisp b/uiop/image.lisp index cb562c43..3a1b180c 100644 --- a/uiop/image.lisp +++ b/uiop/image.lisp @@ -4,7 +4,8 @@ (uiop/package:define-package :uiop/image (:nicknames :asdf/image) (:recycle :uiop/image :asdf/image :xcvb-driver) - (:use :uiop/common-lisp :uiop/package :uiop/utility :uiop/pathname :uiop/stream :uiop/os) + (:use :uiop/common-lisp :uiop/package :uiop/utility + :uiop/os :uiop/pathname :uiop/stream :uiop/eval) (:export #:*image-dumped-p* #:raw-command-line-arguments #:*command-line-arguments* #:command-line-arguments #:raw-command-line-arguments #:setup-command-line-arguments #:argv0 @@ -17,8 +18,7 @@ #:shell-boolean-exit #:register-image-restore-hook #:register-image-dump-hook #:call-image-restore-hook #:call-image-dump-hook - #:restore-image #:dump-image #:create-image -)) + #:restore-image #:dump-image #:create-image)) (in-package :uiop/image) (with-upgradability () diff --git a/uiop/stream.lisp b/uiop/stream.lisp index 05fd2553..036ccfe0 100644 --- a/uiop/stream.lisp +++ b/uiop/stream.lisp @@ -11,7 +11,6 @@ #:detect-encoding #:*encoding-detection-hook* #:always-default-encoding #:encoding-external-format #:*encoding-external-format-hook* #:default-encoding-external-format #:*default-encoding* #:*utf-8-external-format* - #:*standard-readtable* #:*standard-print-pprint-dispatch* #:with-safe-io-syntax #:call-with-safe-io-syntax #:safe-read-from-string #:with-output #:output-string #:with-input #:with-input-file #:call-with-input-file #:with-output-file #:call-with-output-file @@ -23,8 +22,6 @@ #:slurp-stream-forms #:slurp-stream-form #:read-file-string #:read-file-line #:read-file-lines #:safe-read-file-line #:read-file-forms #:read-file-form #:safe-read-file-form - #:eval-input #:eval-thunk #:standard-eval-thunk - #:ensure-variable #:variable-value #:println #:writeln ;; Temporary files #:*temporary-directory* #:temporary-directory #:default-temporary-directory @@ -141,28 +138,18 @@ from non-default encodings to and implementation-defined external-format's") going through all the proper hooks." (funcall *encoding-external-format-hook* (or encoding *default-encoding*)))) - -;;; Safe syntax +;;; Safe I/O (with-upgradability () - (defvar *standard-readtable* (with-standard-io-syntax *readtable*) - "The standard readtable, implementing the syntax specified by the CLHS. -It must never be modified, though only good implementations will even enforce that.") - - (defvar *standard-print-pprint-dispatch* (with-standard-io-syntax *print-pprint-dispatch*) - "The standard pprint dispatch table, implementing the syntax specified by the CLHS. -It must never be modified, though only good implementations will even enforce that.") - (defmacro with-safe-io-syntax ((&key package) &body body) "Establish safe CL reader options around the evaluation of BODY" `(call-with-safe-io-syntax #'(lambda () ,@body) :package ,package)) - (defun call-with-safe-io-syntax (thunk &key package) + (defun call-with-safe-io-syntax (function &key package) (with-standard-io-syntax (let ((*package* (find-package (or package :common-lisp))) - (*read-default-float-format* 'double-float) (*print-readably* nil) (*read-eval* nil)) - (funcall thunk)))) + (call-function function)))) (defun safe-read-from-string (string &key package (eof-error-p t) eof-value (start 0) end preserve-whitespace) "Read from STRING using a safe syntax, as per WITH-SAFE-IO-SYNTAX" @@ -483,58 +470,8 @@ within an WITH-SAFE-IO-SYNTAX using the specified PACKAGE." Extracts the form using READ-FILE-FORM, within an WITH-SAFE-IO-SYNTAX using the specified PACKAGE." (with-safe-io-syntax (:package package) - (apply 'read-file-form pathname (remove-plist-key :package keys)))) - - (defun eval-input (input) - "Portably read and evaluate forms from INPUT, return the last values." - (with-input (input) - (loop :with results :with eof ='#:eof - :for form = (read input nil eof) - :until (eq form eof) - :do (setf results (multiple-value-list (eval form))) - :finally (return (apply 'values results))))) - - (defun eval-thunk (thunk) - "Evaluate a THUNK of code: -If a function, FUNCALL it without arguments. -If a constant literal and not a sequence, return it. -If a cons or a symbol, EVAL it. -If a string, repeatedly read and evaluate from it, returning the last values." - (etypecase thunk - ((or boolean keyword number character pathname) thunk) - ((or cons symbol) (eval thunk)) - (function (funcall thunk)) - (string (eval-input thunk)))) - - (defun standard-eval-thunk (thunk &key (package :cl)) - "Like EVAL-THUNK, but in a more standardized evaluation context." - ;; Note: it's "standard-" not "safe-", because evaluation is never safe. - (when thunk - (with-safe-io-syntax (:package package) - (let ((*read-eval* t)) - (eval-thunk thunk)))))) - -;;; Late-binding variables -(with-upgradability () - (defun ensure-variable (name &key package (when-undefined 'error)) - (etypecase name - (symbol name) - (string (or (ignore-errors - (let ((s (safe-read-from-string name :package package))) - (and (symbolp s) s))) - (call-function when-undefined - "Cannot read non-nil symbol from ~S" name))))) - - (defun variable-value (name &key package (when-undefined 'error)) - (let ((var (ensure-variable name :package package :when-undefined when-undefined))) - (if (boundp var) - (symbol-value var) - (call-function when-undefined "Symbol ~S unbound" name)))) - - (defun (setf variable-value) (value name &key package (when-undefined 'error)) - (if-let (var (ensure-variable name :package package :when-undefined when-undefined)) - (setf (symbol-value var) value) - value))) + (apply 'read-file-form pathname (remove-plist-key :package keys))))) + ;;; Printers (with-upgradability () diff --git a/uiop/uiop.asd b/uiop/uiop.asd index 0b3aa580..eb59184d 100644 --- a/uiop/uiop.asd +++ b/uiop/uiop.asd @@ -33,11 +33,12 @@ being transcluded into asdf.lisp together with ASDF/DEFSYSTEM." (:file "common-lisp" :depends-on ("package")) (:file "utility" :depends-on ("common-lisp")) (:file "os" :depends-on ("utility")) - (:file "pathname" :depends-on ("utility" "os")) - (:file "filesystem" :depends-on ("os" "pathname")) + (:file "pathname" :depends-on ("os")) + (:file "filesystem" :depends-on ("pathname")) (:file "stream" :depends-on ("filesystem")) - (:file "image" :depends-on ("stream")) - (:file "run-program" :depends-on ("stream")) + (:file "eval" :depends-on ("stream")) + (:file "image" :depends-on ("eval")) + (:file "run-program" :depends-on ("eval")) (:file "lisp-build" :depends-on ("image")) (:file "configuration" :depends-on ("image")) (:file "backward-driver" :depends-on ("lisp-build" "run-program" "configuration")) diff --git a/uiop/utility.lisp b/uiop/utility.lisp index 28743cfa..37bedb63 100644 --- a/uiop/utility.lisp +++ b/uiop/utility.lisp @@ -28,13 +28,15 @@ #:stamp< #:stamps< #:stamp*< #:stamp<= ;; stamps #:earlier-stamp #:stamps-earliest #:earliest-stamp #:later-stamp #:stamps-latest #:latest-stamp #:latest-stamp-f - #:list-to-hash-set #:ensure-gethash ;; hash-table + #:list-to-hash-set #:ensure-gethash #:table-alist ;; hash-table #:ensure-function #:access-at #:access-at-count ;; functions - #:call-function #:call-functions #:register-hook-function + #:call-function #:call-functions #:fwrap #:register-hook-function #:match-condition-p #:match-any-condition-p ;; conditions #:call-with-muffled-conditions #:with-muffled-conditions #:lexicographic< #:lexicographic<= - #:parse-version #:unparse-version #:version< #:version<= #:version-compatible-p)) ;; version + #:parse-version #:unparse-version #:version< #:version<= #:version-compatible-p ;; version + #:call-with-variables #:with-variables #:set-variables ;; variables + #:call-with-alist-variables #:with-alist-variables #:set-alist-variables)) (in-package :uiop/utility) ;;;; Defining functions in a way compatible with hot-upgrade: @@ -422,6 +424,13 @@ with the given ARGUMENTS" "For each function in the list FUNCTION-SPECS, in order, call the function as per CALL-FUNCTION" (map () 'call-function function-specs)) + (defun fwrap (&rest functions) + (when functions + (destructuring-bind (first . rest) functions + (if rest + (call-function first #'(lambda () (apply 'fwrap rest))) + (call-function first))))) + (defun register-hook-function (variable hook &optional call-now-p) "Push the HOOK function (a designator as per ENSURE-FUNCTION) onto the hook VARIABLE. When CALL-NOW-P is true, also call the function immediately." @@ -476,7 +485,14 @@ Return two values: the entry after its optional computation, and whether it was (defun list-to-hash-set (list &aux (h (make-hash-table :test 'equal))) "Convert a LIST into hash-table that has the same elements when viewed as a set, up to the given equality TEST" - (dolist (x list h) (setf (gethash x h) t)))) + (dolist (x list h) (setf (gethash x h) t))) + + (defgeneric table-alist (table)) + (defmethod table-alist ((table hash-table)) + (loop :for k :being :the :hash-keys :of table :using (:hash-value v) + :collect (cons k v)))) + (defmethod table-alist ((table cons)) + table) ;;; Version handling @@ -584,3 +600,40 @@ or a string describing the format-control of a simple-condition." "Shorthand syntax for CALL-WITH-MUFFLED-CONDITIONS" `(call-with-muffled-conditions #'(lambda () ,@body) ,conditions))) +;;; Variables + +(with-upgradability () + (defun call-with-variables (variables bindings thunk &key (name #'identity) (value #'identity)) + (loop :for variable :in variables + :for binding :in bindings + :for var = (call-function name variable) + :when var + :collect var :into vars + :and :collect (call-function value binding) :into vals + :finally (progv vars vals (call-function thunk)))) + + (defmacro with-variables ((variables initializers &rest keys &key name value) &body body) + (declare (ignore name value)) + `(call-with-variables ,variables ,initializers #'(lambda () ,@body) ,@keys)) + + (defun set-variables (variables bindings &key (name #'identity) (value #'identity)) + (loop :for variable :in variables + :for binding :in bindings + :for var = (call-function name variable) + :when var + :collect var :into vars + :and :collect (call-function value binding) :into vals + :finally (map () #'set vars vals))) + + (defun call-with-alist-variables (alist thunk &rest keys &key name value) + (declare (ignore name value)) + (apply 'call-with-variables (mapcar #'car alist) (mapcar #'cdr alist) thunk keys)) + + (defmacro with-alist-variables ((alist &rest keys &key name value) &body body) + (declare (ignore name value)) + `(call-with-alist-variables ,alist #'(lambda () ,@body) ,@keys)) + + (defun set-alist-variables (alist &rest keys &key name value) + (declare (ignore name value)) + (apply 'set-variables (mapcar #'car alist) (mapcar #'cdr alist) keys))) + -- GitLab From 9b1d0a3c79386dcf91a48e901f25deaef1d80b0c Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Mon, 31 Mar 2014 17:11:25 -0400 Subject: [PATCH 12/40] Make the default context handler bind only the *READTABLE* to the shared one. --- action.lisp | 2 +- interface.lisp | 2 ++ lisp-action.lisp | 15 +++++++++++---- operate.lisp | 6 +++--- uiop/eval.lisp | 4 ++++ 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/action.lisp b/action.lisp index 25d038c8..e6dbcb34 100644 --- a/action.lisp +++ b/action.lisp @@ -382,7 +382,7 @@ in some previous image, or T if it needs to be done.") (define-convenience-action-methods call-with-action-context (operation component thunk)) (defmethod call-with-action-context ((o operation) (c component) thunk) (with-updated-system-variables ((component-system c)) - (funcall thunk))) + (call-function thunk))) (defmacro with-action-context ((operation component) &body body) `(call-with-action-context ,operation ,component #'(lambda () ,@body))) diff --git a/interface.lisp b/interface.lisp index d6fa1368..61b3f3ad 100644 --- a/interface.lisp +++ b/interface.lisp @@ -26,8 +26,10 @@ #:upward-operation #:downward-operation #:sideway-operation #:selfward-operation #:non-propagating-operation #:build-op #:make + #:basic-load-op #:basic-prepare-op #:basic-compile-op #:load-op #:prepare-op #:compile-op #:prepare-source-op #:load-source-op #:test-op + #:cl-reading-op #:*cl-reading-hook* #:feature #:version #:version-satisfies #:upgrade-asdf #:implementation-identifier #:implementation-type #:hostname #:input-files #:output-files #:output-file #:perform #:perform-with-restarts diff --git a/lisp-action.lisp b/lisp-action.lisp index e0005971..c0e11a8b 100644 --- a/lisp-action.lisp +++ b/lisp-action.lisp @@ -11,7 +11,7 @@ #:try-recompiling #:cl-source-file #:cl-source-file.cl #:cl-source-file.lsp #:basic-load-op #:basic-compile-op #:compile-op-flags #:compile-op-proclamations - #:basic-prepare-op #:cl-reading-op + #:basic-prepare-op #:cl-reading-op #:*cl-reading-hook* #:load-op #:prepare-op #:compile-op #:test-op #:load-source-op #:prepare-source-op #:call-with-around-compile-hook #:perform-lisp-compilation #:perform-lisp-load-fasl #:perform-lisp-load-source @@ -38,12 +38,19 @@ (defclass basic-prepare-op (operation) ()) (defmethod perform ((operation basic-prepare-op) (system system)) - (initialize-system-variables system)) + (initialize-system-variables system) + (use-initial-system-variables system)) + + (defvar *cl-reading-hook* 'call-function + "Hook to call around operations that read CL code (since ASDF 3.1). +To make it a NOP, set it to UIOP:CALL-FUNCTION. +To make it bind just *READTABLE*, set it to UIOP:CALL-WITH-SHARED-READTABLE (the default). +To bind all syntax variables to predictable portable values, set it to ASDF:CALL-WITH-ASDF-SYNTAX. +For stricter values, set it to UIOP:CALL-WITH-STANDARD-IO-SYNTAX.") (defclass cl-reading-op (operation) ())) ;; operations that read CL source (defmethod call-with-action-context ((o cl-reading-op) (c cl-source-file) thunk) - (call-next-method - o c #'(lambda () (call-with-asdf-syntax thunk)))) + (call-next-method o c #'(lambda () (call-function *cl-reading-hook* thunk)))) ;;; Our default operations: loading into the current lisp image (with-upgradability () diff --git a/operate.lisp b/operate.lisp index 74aa5c53..18bf5e51 100644 --- a/operate.lisp +++ b/operate.lisp @@ -74,9 +74,9 @@ The :FORCE or :FORCE-NOT argument to OPERATE can be: (apply 'operate (funcall operation-remaker) component-path keys)))) ;; Setup proper bindings around any operate call. (with-asdf-cache () - (let* ((*verbose-out* (and verbose *standard-output*)) - (*compile-file-warnings-behaviour* on-warnings) - (*compile-file-failure-behaviour* on-failure)) + (let ((*verbose-out* (and verbose *standard-output*)) + (*compile-file-warnings-behaviour* on-warnings) + (*compile-file-failure-behaviour* on-failure)) (call-next-method))))) (defmethod operate :before ((operation operation) (component component) diff --git a/uiop/eval.lisp b/uiop/eval.lisp index aa546e20..0262d487 100644 --- a/uiop/eval.lisp +++ b/uiop/eval.lisp @@ -8,6 +8,7 @@ #:*standard-readtable* #:*standard-print-pprint-dispatch* #:*shared-readtable* #:*shared-print-pprint-dispatch* #:*standard-syntax-variables* + #:call-with-shared-readtable #:call-with-standard-io-syntax #:eval-input #:eval-thunk #:standard-eval-thunk #:ensure-variable #:variable-value)) (in-package :uiop/eval) @@ -65,6 +66,9 @@ It should match the extensions of *shared-readtable* -- see the latter variable' (defun call-with-standard-io-syntax (function) (with-standard-io-syntax (call-function function))) + (defun call-with-shared-readtable (thunk) + (let ((*readtable* *shared-readtable*)) (call-function thunk))) + (defun eval-input (input) "Portably read and evaluate forms from INPUT, return the last values." (with-input (input) -- GitLab From b993293ca3e4f2816b37fa1ec1c586fbe96216c7 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Tue, 8 Apr 2014 11:43:24 -0400 Subject: [PATCH 13/40] Enforce asdf syntax for in asdf/package-system. --- package-system.lisp | 1 + 1 file changed, 1 insertion(+) diff --git a/package-system.lisp b/package-system.lisp index 0133ed51..2b530a0e 100644 --- a/package-system.lisp +++ b/package-system.lisp @@ -120,6 +120,7 @@ otherwise return a default system name computed from PACKAGE-NAME." :source-file nil :pathname ,dir :depends-on ,dependencies + :around-compile call-with-asdf-syntax :components ((cl-source-file "lisp" :pathname ,sub))))))))))))))) (with-upgradability () -- GitLab From a0a52bde9c71e2a32d857b3358502dbea6905179 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Wed, 9 Apr 2014 02:42:00 -0400 Subject: [PATCH 14/40] package-system: parsing also should use with-asdf-syntax. --- package-system.lisp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package-system.lisp b/package-system.lisp index 2b530a0e..4053d96d 100644 --- a/package-system.lisp +++ b/package-system.lisp @@ -31,8 +31,9 @@ (member (car form) *defpackage-forms*))) (defun stream-defpackage-form (stream) - (loop :for form = (read stream nil nil) :while form - :when (defpackage-form-p form) :return form)) + (with-asdf-syntax () + (loop :for form = (read stream nil nil) :while form + :when (defpackage-form-p form) :return form))) (defun file-defpackage-form (file) "Return the first DEFPACKAGE form in FILE." -- GitLab From f5714a0ac442a354e4767ae4ed25690da784ad2b Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Wed, 9 Apr 2014 02:47:41 -0400 Subject: [PATCH 15/40] uiop/lisp-build depends on uiop/eval ignore more files. --- .gitignore | 1 + uiop/lisp-build.lisp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7982ff7a..d014532c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ doc/asdf.pdf doc/asdf.pg doc/asdf.toc doc/asdf.tp +doc/asdf.tps doc/asdf.vr doc/asdf.vrs doc/asdf.t2d/ diff --git a/uiop/lisp-build.lisp b/uiop/lisp-build.lisp index 6b105a44..6b22f4b0 100644 --- a/uiop/lisp-build.lisp +++ b/uiop/lisp-build.lisp @@ -5,7 +5,7 @@ (:nicknames :asdf/lisp-build) (:recycle :uiop/lisp-build :asdf/lisp-build :asdf) (:use :uiop/common-lisp :uiop/package :uiop/utility - :uiop/os :uiop/pathname :uiop/filesystem :uiop/stream :uiop/image) + :uiop/os :uiop/pathname :uiop/filesystem :uiop/stream :uiop/eval :uiop/image) (:export ;; Variables #:*compile-file-warnings-behaviour* #:*compile-file-failure-behaviour* -- GitLab From e9976a9270da9caad740bbb32ef9c1869a34d91d Mon Sep 17 00:00:00 2001 From: "Robert P. Goldman" Date: Sun, 1 Jun 2014 17:22:33 -0500 Subject: [PATCH 16/40] New syntax control document. Extracted from the ASDF todo, this is intended to document design and objectives of the syntax-control branch. --- doc/syntax-control.txt | 121 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 doc/syntax-control.txt diff --git a/doc/syntax-control.txt b/doc/syntax-control.txt new file mode 100644 index 00000000..7f1756bc --- /dev/null +++ b/doc/syntax-control.txt @@ -0,0 +1,121 @@ +* Syntax control: objective + +What is the problem that we are trying to fix? + +It seems like we are trying to fix uncontrolled, unintentional leaking of +readtable side effects, without removing /intended/ leaking of readtable side +effects. + +* Syntax Control: implement and document this plan + +1. ASDF maintains an =asdf:*shared-readtable*=, which is the =*readtable*= + object at the time it was loaded. + + [RPG: What is the intended function of the *shared-readtable* as distinct from + the default readtable?] + +2. This *shared-readtable* is subject to the current restrictions: [RPG: what do + you mean by "current" in this sentence? I mean, in the current CL world, the + readtable is a mess; there are few, if any restrictions. Is this a just a + typo for "the following restrictions"?] + + A. no modifying any standard character, + B. no two dependencies assigning different meaning to the same non-standard + character. [RPG: what is a "dependency" in this context? Do you mean "no + system should be loaded on top of systems that assign different meanings to a + character in asdf:*shared-readtable*?] + C. libraries need to document any change to the readtable [RPG: should "the + readtable" be "asdf:*shared-readtable*"?] + D. free software libraries will register these changes on the page on cliki. + +3. Unhappily, there is no cheap way to enforce these restrictions, but that's no + regression with respect to the current situation. + +4. ASDF wraps any compile-op and load-source-op in this asdf:*shared-readtable* + [RPG: to be painfully explicit: "ASDF binds =*READTABLE*= to + =ASDF:*SHARED-READTABLE*= around any =PERFORM COMPILE-OP= and =PERFORM + LOAD-SOURCE-OP="? Is that a correct reading?], but probably not =load-op=, + to preserve combine-fasl linking semantics. + +5. Systems that want to do crazier things with the readtable that may violate + (2) must arrange to use their own private readtable, but can otherwise do it + safely. It is an error (unhappily not enforceable) to modify the current + readtable in these ways. + + [RPG: I'm not at all sure I know what "the readtable" refers to in the + paragraph above. Is it "the current value of =*READTABLE*=," "the value of + =ASDF:*SHARED-READTABLE*=," or something else?] + +6A. ASDF binds =*readtable*= to the =*shared-readtable*= at the start of every +system's compilation (and loading?), and around the entire =ASDF:OPERATE=, leaving +the *readtable* unchanged at the end. [RPG: Please clarify -- I don't see how we +can wrap around the entire =OPERATE= while at the same time /not/ doing loading +as well as compilation. What's the precise scope of the binding of +=*READTABLE*=?] + +This easily supports systems that "modify the current readtable data structure". + +However, that doesn't handle systems that "bind *readtable* to a new value", +because the changes they make will shadow the changes that other systems +following this style make and depend on. [RPG: We need to clarify the preceding +sentence. In particular, given the way CL works today, we can't actually /bind/ +=*READTABLE*= in a useful way: we always end up setting it. A simple example +would help.] To allow such an idiom, we must also do the following: + +6B- ASDF binds =*readtable*= to a proper "entry readtable" at the start +of every system's compilation, and records an "exit readtable" at the +end of the system's loading. [RPG: 6B seems to directly contradict 4.] + +7. Maintain a partial order on these readtable objects, assuming that +each system's exit readtable supersedes the entry readtable. The least +readtable is the *shared-readtable*. It's enough to store for each new +exit readtable, identified by the name of system that created it, the +set of its inferior readtables, as a list or eq-hash-table, or an +equal hash-table, with each readtable being identified by the name of +the system that created it. + +8. before a system is compiled or loaded, compute the maximum readtable of all + the exit readtables of its dependencies. If this maximum is unique, then it + will be the entry readtable of the system. If there is not a unique readtable + that is more than all the other ones, that's an error, and we refuse to load + the system. + + [RPG: I /think/ I follow this: the idea is to compute what the actual + readtable dependencies are, as a means of ensuring repeatability, as build + operations may be interleaved in different situations.] + + [RPG: Another point -- we aren't actually computing the maximum readtable as + a readtable object, are we? Instead we are computing a dependency graph, + right? And the nodes are readtable /designators/, rather than readtables, + right?] + +9. after a system is loaded, check its exit readtable, if it already exists, + check that this doesn't create a cycle or issue an error. If the exit + readtable doesn't already exist, add it to the set of all known exit + readtables. + +10. ASDF either + a. binds the *readtable* to the *shared-readtable* around the entire + asdf:operate, leaving the *readtable* unchanged at the end, or + + b. always side-effects the *readtable* to correspond to the exit readtable of + the loaded system, or + + c. operate does the binding around thing, but load-system does the side-effect + after it's done operate'ing. + +Does that strike you as complex? Because it is. That's the price of +*safely* supporting this "systems can bind a new value to *readtable*" +style. Unhappily some of the constraints are not enforceable (2A and +2B), but that's the very same as now. + +So my next question is: do you want to safely support these +conventions? Do your systems modify the current *readtable* structure, +or do they bind *readtable* to a new value? + +[RPG: So what is implemented in the syntax-control branch?] + + +# Local Variables: +# mode: org +# End: -- GitLab From 4abba9c35e0b73c019a61e731b653db436e14502 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Mon, 2 Jun 2014 08:09:00 -0400 Subject: [PATCH 17/40] Inline replies from FRR to RPG in the syntax-control document. --- doc/syntax-control.txt | 108 ++++++++++++++++++++++++++++++++--------- 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/doc/syntax-control.txt b/doc/syntax-control.txt index 7f1756bc..7bae47df 100644 --- a/doc/syntax-control.txt +++ b/doc/syntax-control.txt @@ -2,9 +2,10 @@ What is the problem that we are trying to fix? -It seems like we are trying to fix uncontrolled, unintentional leaking of -readtable side effects, without removing /intended/ leaking of readtable side -effects. +We are trying to fix uncontrolled, unintentional leaking of +readtable side effects to systems that depend on such effects not happening, +without removing /intended/ leaking of readtable side effects +to systems that do depend on such effects happening. * Syntax Control: implement and document this plan @@ -13,19 +14,42 @@ effects. [RPG: What is the intended function of the *shared-readtable* as distinct from the default readtable?] - -2. This *shared-readtable* is subject to the current restrictions: [RPG: what do - you mean by "current" in this sentence? I mean, in the current CL world, the - readtable is a mess; there are few, if any restrictions. Is this a just a - typo for "the following restrictions"?] + [FRR: this *is* the default readtable. There is no other way to get to the + /initial readtable/ as opposed to (a copy of) the /standard readtable/, + which could be different and contain extensions, though since it *initially* + shouldn't contain extensions, I suppose we could use (copy-readtable nil)] + +2. This *shared-readtable* is subject to the current restrictions: + [RPG: what do you mean by "current" in this sentence? + I mean, in the current CL world, the readtable is a mess; there are few, + if any restrictions. Is this a just a typo for "the following restrictions"?] + [FRR: that means restrictions implicit in the CL standard, unconsciously + followed by the CL community and now explicit in the asdf manual, section + "How do I work with readtables?", which are indeed as follows.] A. no modifying any standard character, B. no two dependencies assigning different meaning to the same non-standard - character. [RPG: what is a "dependency" in this context? Do you mean "no - system should be loaded on top of systems that assign different meanings to a - character in asdf:*shared-readtable*?] - C. libraries need to document any change to the readtable [RPG: should "the - readtable" be "asdf:*shared-readtable*"?] + character. + [RPG: what is a "dependency" in this context? + Do you mean "no system should be loaded on top of systems + that assign different meanings to a character in asdf:*shared-readtable*?] + [FRR: yes, though I would formulate it as: + no two systems loaded in the same image may assign different meanings + to a same character in the current readtable as bound to uiop:*shared-readtable*?] + C. libraries need to document any change to the readtable + [RPG: should "the readtable" be "asdf:*shared-readtable*"?] + [FRR: same difference: that's what is bound initially when your files are + loaded or compiled, and since it's bound by LOAD and COMPILE-FILE, + you cannot change that. Only ASDF gets to decide whether it controls it, + or whether the REPL gets to override it catastrophically — + it is by construction IMPOSSIBLE for the REPL to do anything but a catastrophe + by rebinding the readtable at the REPL; no system can possibly know what other + systems will or won't be loaded, and thus no system can create a readtable + that will have exactly what will be loaded and nothing else independently + from the set of systems that will be loaded, which they can't control. + They could use (copy-readtable t), and still this wouldn't be safe against + one of the other systems making new changes to the previous copy of the + readtable and expecting them to take effects.] D. free software libraries will register these changes on the page on cliki. 3. Unhappily, there is no cheap way to enforce these restrictions, but that's no @@ -36,6 +60,7 @@ effects. =ASDF:*SHARED-READTABLE*= around any =PERFORM COMPILE-OP= and =PERFORM LOAD-SOURCE-OP="? Is that a correct reading?], but probably not =load-op=, to preserve combine-fasl linking semantics. + [FRR: correct.] 5. Systems that want to do crazier things with the readtable that may violate (2) must arrange to use their own private readtable, but can otherwise do it @@ -45,26 +70,54 @@ effects. [RPG: I'm not at all sure I know what "the readtable" refers to in the paragraph above. Is it "the current value of =*READTABLE*=," "the value of =ASDF:*SHARED-READTABLE*=," or something else?] + [FRR: same difference. I suppose we need a good way to distinguish + "side effects to the data structure to which *readtable* is currently bound" + from "side effects to the value of the variable *readtable*". + Both are problematic, although the CLHS says the latter don't escape + LOAD or COMPILE-FILE, though they may critically escape from the REPL, + and that's why we need the syntax-control feature in ASDF.] 6A. ASDF binds =*readtable*= to the =*shared-readtable*= at the start of every -system's compilation (and loading?), and around the entire =ASDF:OPERATE=, leaving -the *readtable* unchanged at the end. [RPG: Please clarify -- I don't see how we -can wrap around the entire =OPERATE= while at the same time /not/ doing loading -as well as compilation. What's the precise scope of the binding of -=*READTABLE*=?] + system's compilation (and loading?), and around the entire =ASDF:OPERATE=, + leaving the *readtable* unchanged at the end. + [RPG: Please clarify -- I don't see how we can wrap around the entire + =OPERATE= while at the same time /not/ doing loading as well as compilation. + What's the precise scope of the binding of =*READTABLE*=?] + [FRR: we don't bind around individual LOAD-OP operations, so that + when loading a fasl as opposed to a lisp source file, load-time + side-effects to the *readtable* variable won't be cancelled around each + file by binding *readtable* to *shared-readtable*, though a strict reading + of the CLHS says this implicitly happens when loading, irrespective of + the loading being .fasl or .lisp. In case it isn't, though, we don't try, + because that doesn't happen when we concatenate fasls, and we want to + be as compatible with that as possible.] This easily supports systems that "modify the current readtable data structure". However, that doesn't handle systems that "bind *readtable* to a new value", because the changes they make will shadow the changes that other systems -following this style make and depend on. [RPG: We need to clarify the preceding -sentence. In particular, given the way CL works today, we can't actually /bind/ -=*READTABLE*= in a useful way: we always end up setting it. A simple example -would help.] To allow such an idiom, we must also do the following: +following this style make and depend on. +[RPG: We need to clarify the preceding sentence. In particular, given the way +CL works today, we can't actually /bind/ =*READTABLE*= in a useful way: +we always end up setting it. A simple example would help.] +[FRR: no system can currently do it in a regular LOAD-OP or COMPILE-OP, +because the CLHS mandates the bind around LOAD and COMPILE-FILE; +but they can conceivably do it in a perform method, and can indeed +do it in an initialization function that the user calls with or without +understanding the *READTABLE* change that occurs and its implications.] +To allow such an idiom, we must also do the following: 6B- ASDF binds =*readtable*= to a proper "entry readtable" at the start of every system's compilation, and records an "exit readtable" at the end of the system's loading. [RPG: 6B seems to directly contradict 4.] +[FRR: yes, 6B, 7, 8, 9, 10 constitute a more elaborate *alternate plan* +that wasn't implemented, and that I wrote before I realized the CLHS +mandated that *READTABLE* be bound by LOAD and COMPILE-FILE, which +make this plan quite overkill unless someone modifies the *READTABLE* +variable in a magic perform method on prepare-op (and prepare-source-op) +without restoring it at the end of compile-op (and load-source-op) +and we want to support that. I suppose we should just do away with +that alternate plan.] 7. Maintain a partial order on these readtable objects, assuming that each system's exit readtable supersedes the entry readtable. The least @@ -89,6 +142,15 @@ the system that created it. right? And the nodes are readtable /designators/, rather than readtables, right?] + [FRR: yes, in that alternate plan, we don't (can't) merge objects, + but assume that the programs somehow do use copy-readtable followed by + modification of what then constitutes a tree of readtable objects, + and we ensure that *READTABLE* is bound to the correct readtable object + when loading a system. Once again, this is overkill + since LOAD and COMPILE-FILE do the readtable protection, + although if we wanted to extend that to other variables not protected + by the CLHS, notably the pprint-dispatch-table, then that would make more sense] + 9. after a system is loaded, check its exit readtable, if it already exists, check that this doesn't create a cycle or issue an error. If the exit readtable doesn't already exist, add it to the set of all known exit @@ -114,6 +176,8 @@ conventions? Do your systems modify the current *readtable* structure, or do they bind *readtable* to a new value? [RPG: So what is implemented in the syntax-control branch?] +[FRR: The first simpler plan, 1-5, 6A is implemented. +The more complex plan 1-5, 6B, 7-10 isn't.] # Local Variables: -- GitLab From f0ed85dbe5ca1dd86e38f8c166eb2704bd5e9b2b Mon Sep 17 00:00:00 2001 From: "Robert P. Goldman" Date: Mon, 2 Jun 2014 13:24:40 -0500 Subject: [PATCH 18/40] Editing pass. Merged some of Fare's corrections, eliminating the discussion in favor of its conclusions. Renumbered a bunch to try to capture the structure of Fare's argument. --- doc/syntax-control.txt | 228 +++++++++++++++++++---------------------- 1 file changed, 108 insertions(+), 120 deletions(-) diff --git a/doc/syntax-control.txt b/doc/syntax-control.txt index 7bae47df..09f972e7 100644 --- a/doc/syntax-control.txt +++ b/doc/syntax-control.txt @@ -18,66 +18,51 @@ to systems that do depend on such effects happening. /initial readtable/ as opposed to (a copy of) the /standard readtable/, which could be different and contain extensions, though since it *initially* shouldn't contain extensions, I suppose we could use (copy-readtable nil)] + [RPG: I think if it's sanitation we are hoping for, then + =(copy-readtable nil)= is The Right Thing. It eliminates the potential for + getting unpredictable results by delaying load of ASDF.] + + [RPG: Proposed answer to my own question: the =*SHARED-READTABLE*= is + provided as a convenience for libraries that + -2. This *shared-readtable* is subject to the current restrictions: - [RPG: what do you mean by "current" in this sentence? - I mean, in the current CL world, the readtable is a mess; there are few, - if any restrictions. Is this a just a typo for "the following restrictions"?] - [FRR: that means restrictions implicit in the CL standard, unconsciously - followed by the CL community and now explicit in the asdf manual, section - "How do I work with readtables?", which are indeed as follows.] +2. This *shared-readtable* is subject to the the following restrictions (also + discussed in the ASDF manual, in the section "How do I work with + readtables?"): A. no modifying any standard character, - B. no two dependencies assigning different meaning to the same non-standard - character. - [RPG: what is a "dependency" in this context? - Do you mean "no system should be loaded on top of systems - that assign different meanings to a character in asdf:*shared-readtable*?] - [FRR: yes, though I would formulate it as: - no two systems loaded in the same image may assign different meanings - to a same character in the current readtable as bound to uiop:*shared-readtable*?] - C. libraries need to document any change to the readtable - [RPG: should "the readtable" be "asdf:*shared-readtable*"?] - [FRR: same difference: that's what is bound initially when your files are - loaded or compiled, and since it's bound by LOAD and COMPILE-FILE, - you cannot change that. Only ASDF gets to decide whether it controls it, - or whether the REPL gets to override it catastrophically — - it is by construction IMPOSSIBLE for the REPL to do anything but a catastrophe - by rebinding the readtable at the REPL; no system can possibly know what other - systems will or won't be loaded, and thus no system can create a readtable - that will have exactly what will be loaded and nothing else independently - from the set of systems that will be loaded, which they can't control. - They could use (copy-readtable t), and still this wouldn't be safe against - one of the other systems making new changes to the previous copy of the - readtable and expecting them to take effects.] + B. No two systems loaded in the same image may assign different meanings + to a same character in the =uiop:*shared-readtable*=. + [RPG: Spliced in FRR's admirably clear phrasing. Pulled out the hedge "as + bound to," which I beleive to be unnecessary.] + [RPG: Follow-up question: is this condition checkable? I am not up to speed + on portable facilities for access to READTABLEs. If ASDF doesn't check + this, I think it will be hard for the programmer to check it.] + C. libraries need to document any change to =*SHARED-READTABLE*=. Note that + unqualified readtable settings in program code will modify =*SHARED-READTABLE*=. D. free software libraries will register these changes on the page on cliki. + + It is an error (unhappily not enforceable) to modify the =*SHARED-READTABLE*= + in ways that violate these restrictions. 3. Unhappily, there is no cheap way to enforce these restrictions, but that's no - regression with respect to the current situation. - -4. ASDF wraps any compile-op and load-source-op in this asdf:*shared-readtable* - [RPG: to be painfully explicit: "ASDF binds =*READTABLE*= to - =ASDF:*SHARED-READTABLE*= around any =PERFORM COMPILE-OP= and =PERFORM - LOAD-SOURCE-OP="? Is that a correct reading?], but probably not =load-op=, - to preserve combine-fasl linking semantics. - [FRR: correct.] - -5. Systems that want to do crazier things with the readtable that may violate - (2) must arrange to use their own private readtable, but can otherwise do it - safely. It is an error (unhappily not enforceable) to modify the current - readtable in these ways. + regression with respect to the current situation. + + [RPG: See above: can't we check 2B, at least?] + +4. ASDF binds =*READTABLE*= to =ASDF:*SHARED-READTABLE*= around any =PERFORM + COMPILE-OP= and =PERFORM LOAD-SOURCE-OP= + +5. Systems that want to modify a persistent, shared readtable in ways that + violate (2) must arrange to use their own private readtable, but can + otherwise do it safely. - [RPG: I'm not at all sure I know what "the readtable" refers to in the - paragraph above. Is it "the current value of =*READTABLE*=," "the value of - =ASDF:*SHARED-READTABLE*=," or something else?] - [FRR: same difference. I suppose we need a good way to distinguish - "side effects to the data structure to which *readtable* is currently bound" - from "side effects to the value of the variable *readtable*". - Both are problematic, although the CLHS says the latter don't escape - LOAD or COMPILE-FILE, though they may critically escape from the REPL, - and that's why we need the syntax-control feature in ASDF.] - -6A. ASDF binds =*readtable*= to the =*shared-readtable*= at the start of every + Systems that wish to make a modified readtable available should export an API + function to set the value of =*READTABLE*= appropriately. + +6. Implementation: + + A. ASDF binds =*readtable*= to the =*shared-readtable*= at the start of every system's compilation (and loading?), and around the entire =ASDF:OPERATE=, leaving the *readtable* unchanged at the end. [RPG: Please clarify -- I don't see how we can wrap around the entire @@ -92,79 +77,82 @@ to systems that do depend on such effects happening. because that doesn't happen when we concatenate fasls, and we want to be as compatible with that as possible.] -This easily supports systems that "modify the current readtable data structure". - -However, that doesn't handle systems that "bind *readtable* to a new value", -because the changes they make will shadow the changes that other systems -following this style make and depend on. -[RPG: We need to clarify the preceding sentence. In particular, given the way -CL works today, we can't actually /bind/ =*READTABLE*= in a useful way: -we always end up setting it. A simple example would help.] -[FRR: no system can currently do it in a regular LOAD-OP or COMPILE-OP, -because the CLHS mandates the bind around LOAD and COMPILE-FILE; -but they can conceivably do it in a perform method, and can indeed -do it in an initialization function that the user calls with or without -understanding the *READTABLE* change that occurs and its implications.] -To allow such an idiom, we must also do the following: - -6B- ASDF binds =*readtable*= to a proper "entry readtable" at the start -of every system's compilation, and records an "exit readtable" at the -end of the system's loading. [RPG: 6B seems to directly contradict 4.] -[FRR: yes, 6B, 7, 8, 9, 10 constitute a more elaborate *alternate plan* -that wasn't implemented, and that I wrote before I realized the CLHS -mandated that *READTABLE* be bound by LOAD and COMPILE-FILE, which -make this plan quite overkill unless someone modifies the *READTABLE* -variable in a magic perform method on prepare-op (and prepare-source-op) -without restoring it at the end of compile-op (and load-source-op) -and we want to support that. I suppose we should just do away with -that alternate plan.] - -7. Maintain a partial order on these readtable objects, assuming that -each system's exit readtable supersedes the entry readtable. The least -readtable is the *shared-readtable*. It's enough to store for each new -exit readtable, identified by the name of system that created it, the -set of its inferior readtables, as a list or eq-hash-table, or an -equal hash-table, with each readtable being identified by the name of -the system that created it. - -8. before a system is compiled or loaded, compute the maximum readtable of all - the exit readtables of its dependencies. If this maximum is unique, then it - will be the entry readtable of the system. If there is not a unique readtable - that is more than all the other ones, that's an error, and we refuse to load - the system. - - [RPG: I /think/ I follow this: the idea is to compute what the actual - readtable dependencies are, as a means of ensuring repeatability, as build - operations may be interleaved in different situations.] + This easily supports systems that "modify the current readtable data + structure". + + However, that doesn't handle systems that "bind *readtable* to a new value", + because the changes they make will shadow the changes that other systems + following this style make and depend on. + + [RPG: We need to clarify the preceding sentence. In particular, given the way + CL works today, we can't actually /bind/ =*READTABLE*= in a useful way: + we always end up setting it. A simple example would help.] + [FRR: no system can currently do it in a regular LOAD-OP or COMPILE-OP, + because the CLHS mandates the bind around LOAD and COMPILE-FILE; + but they can conceivably do it in a perform method, and can indeed + do it in an initialization function that the user calls with or without + understanding the *READTABLE* change that occurs and its implications.] + + To allow such an idiom, we must also do the following: - [RPG: Another point -- we aren't actually computing the maximum readtable as - a readtable object, are we? Instead we are computing a dependency graph, - right? And the nodes are readtable /designators/, rather than readtables, - right?] - - [FRR: yes, in that alternate plan, we don't (can't) merge objects, - but assume that the programs somehow do use copy-readtable followed by - modification of what then constitutes a tree of readtable objects, - and we ensure that *READTABLE* is bound to the correct readtable object - when loading a system. Once again, this is overkill - since LOAD and COMPILE-FILE do the readtable protection, - although if we wanted to extend that to other variables not protected - by the CLHS, notably the pprint-dispatch-table, then that would make more sense] - -9. after a system is loaded, check its exit readtable, if it already exists, + B. ASDF binds =*readtable*= to a proper "entry readtable" at the start of + every system's compilation, and records an "exit readtable" at the end of the + system's loading. [RPG: 6B seems to directly contradict 4.] + + [FRR: yes, 5B and the following constitute a more elaborate *alternate plan* that + wasn't implemented, and that I wrote before I realized the CLHS mandated that + *READTABLE* be bound by LOAD and COMPILE-FILE, which make this plan quite + overkill unless someone modifies the *READTABLE* variable in a magic perform + method on prepare-op (and prepare-source-op) without restoring it at the end + of compile-op (and load-source-op) and we want to support that. I suppose we + should just do away with that alternate plan.] + + 1. Maintain a partial order on these readtable objects, assuming that each + system's exit readtable supersedes the entry readtable. The least readtable + is the *shared-readtable*. It's enough to store for each new exit readtable, + identified by the name of system that created it, the set of its inferior + readtables, as a list or eq-hash-table, or an equal hash-table, with each + readtable being identified by the name of the system that created it. + + 2. Before a system is compiled or loaded, compute the maximum readtable of + all the exit readtables of its dependencies. If this maximum is unique, + then it will be the entry readtable of the system. If there is not a + unique readtable that is more than all the other ones, that's an error, + and we refuse to load the system. + + [RPG: I /think/ I follow this: the idea is to compute what the actual + readtable dependencies are, as a means of ensuring repeatability, as build + operations may be interleaved in different situations.] + + [RPG: Another point -- we aren't actually computing the maximum readtable + as a readtable object, are we? Instead we are computing a dependency + graph, right? And the nodes are readtable /designators/, rather than + readtables, right?] + + [FRR: yes, in that alternate plan, we don't (can't) merge objects, but + assume that the programs somehow do use copy-readtable followed by + modification of what then constitutes a tree of readtable objects, + and we ensure that *READTABLE* is bound to the correct readtable object + when loading a system. Once again, this is overkill since =LOAD= and + =COMPILE-FILE= do the readtable protection, although if we wanted to + extend that to other variables not protected by the CLHS, notably the + pprint-dispatch-table, then that would make more sense.] + + 3. After a system is loaded, check its exit readtable, if it already exists, check that this doesn't create a cycle or issue an error. If the exit readtable doesn't already exist, add it to the set of all known exit readtables. -10. ASDF either - a. binds the *readtable* to the *shared-readtable* around the entire - asdf:operate, leaving the *readtable* unchanged at the end, or +7. ASDF either + + 1. binds the *readtable* to the *shared-readtable* around the entire + asdf:operate, leaving the *readtable* unchanged at the end, or - b. always side-effects the *readtable* to correspond to the exit readtable of - the loaded system, or + 2. always side-effects the *readtable* to correspond to the exit readtable of + the loaded system, or - c. operate does the binding around thing, but load-system does the side-effect - after it's done operate'ing. + 3. operate does the binding around thing, but load-system does the side-effect + after it's done operate'ing. Does that strike you as complex? Because it is. That's the price of *safely* supporting this "systems can bind a new value to *readtable*" @@ -177,7 +165,7 @@ or do they bind *readtable* to a new value? [RPG: So what is implemented in the syntax-control branch?] [FRR: The first simpler plan, 1-5, 6A is implemented. -The more complex plan 1-5, 6B, 7-10 isn't.] +The more complex plan 1-5, 6B isn't.] # Local Variables: -- GitLab From 64cee35323b016453fbf52bec29b824d3054dfc4 Mon Sep 17 00:00:00 2001 From: "Robert P. Goldman" Date: Mon, 2 Jun 2014 13:28:50 -0500 Subject: [PATCH 19/40] New question. --- doc/syntax-control.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/syntax-control.txt b/doc/syntax-control.txt index 09f972e7..5a35b2ce 100644 --- a/doc/syntax-control.txt +++ b/doc/syntax-control.txt @@ -166,6 +166,7 @@ or do they bind *readtable* to a new value? [RPG: So what is implemented in the syntax-control branch?] [FRR: The first simpler plan, 1-5, 6A is implemented. The more complex plan 1-5, 6B isn't.] +[RPG: Got it. And do we do 7.1, 7.2, or 7.3?] # Local Variables: -- GitLab From a5032b67cbd85ff1c9453d9f13c0f2c8cc41d6e5 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Mon, 2 Jun 2014 22:42:22 -0400 Subject: [PATCH 20/40] More replies to RPG regarding the syntax-control branch. --- doc/syntax-control.txt | 155 ++++++++++++++++++++++++----------------- 1 file changed, 91 insertions(+), 64 deletions(-) diff --git a/doc/syntax-control.txt b/doc/syntax-control.txt index 5a35b2ce..a387e2b4 100644 --- a/doc/syntax-control.txt +++ b/doc/syntax-control.txt @@ -9,107 +9,132 @@ to systems that do depend on such effects happening. * Syntax Control: implement and document this plan -1. ASDF maintains an =asdf:*shared-readtable*=, which is the =*readtable*= - object at the time it was loaded. - - [RPG: What is the intended function of the *shared-readtable* as distinct from - the default readtable?] - [FRR: this *is* the default readtable. There is no other way to get to the - /initial readtable/ as opposed to (a copy of) the /standard readtable/, - which could be different and contain extensions, though since it *initially* - shouldn't contain extensions, I suppose we could use (copy-readtable nil)] - [RPG: I think if it's sanitation we are hoping for, then +1. ASDF (actually UIOP) maintains a =uiop:*shared-readtable*=, which is bound + to the value of =cl:*readtable*= at the time ASDF was loaded, + which is the /initial readtable/. + The variable =*shared-readtable*= is not intended to be rebound + after having been initialized; its value is an object of type =readtable=, + hereafter known as the /shared readtable/. + + [RPG: What is the intended function of the *shared-readtable* as distinct from + the default readtable?] + [FRR: this *is* the default readtable. There is no other way to get to the + /initial readtable/ as opposed to (a copy of) the /standard readtable/, + which could be different and contain extensions, though since it *initially* + shouldn't contain extensions, I suppose we could use =(copy-readtable nil)=] + [RPG: I think if it's sanitation we are hoping for, then =(copy-readtable nil)= is The Right Thing. It eliminates the potential for getting unpredictable results by delaying load of ASDF.] - - [RPG: Proposed answer to my own question: the =*SHARED-READTABLE*= is - provided as a convenience for libraries that + [FRR: I would have liked to agree, but my understanding is that several + implementations provide extensions in the /initial readtable/ that are *not* + part of the /standard readtable/, such as a #! ignorer sometimes, or + importantly in CCL the =#$= and =#_= readers, etc. + I understand that a strict + reading of the CLHS might make this non-conformant, but I suspect that + many systems rely on it, as I believe the ITA codebase for QRes did.] + [FRR: maybe the variable should be named =*initial-readtable*= then?] + [RPG: Proposed answer to my own question: the =*shared-readtable*= is + provided as a convenience for libraries that ...] + [FRR: missing text above] + +2. Modification of the readtable object to which =*shared-readtable*= is bound + is subject to the the following restrictions (also discussed in + the ASDF manual, in the section "How do I work with readtables?"): + + [FRR: be sure to update the manual to match this document after it's settled.] -2. This *shared-readtable* is subject to the the following restrictions (also - discussed in the ASDF manual, in the section "How do I work with - readtables?"): - A. no modifying any standard character, B. No two systems loaded in the same image may assign different meanings - to a same character in the =uiop:*shared-readtable*=. + to a same character in the =*shared-readtable*=. [RPG: Spliced in FRR's admirably clear phrasing. Pulled out the hedge "as bound to," which I beleive to be unnecessary.] [RPG: Follow-up question: is this condition checkable? I am not up to speed - on portable facilities for access to READTABLEs. If ASDF doesn't check + on portable facilities for access to =readtable='s. If ASDF doesn't check this, I think it will be hard for the programmer to check it.] - C. libraries need to document any change to =*SHARED-READTABLE*=. Note that - unqualified readtable settings in program code will modify =*SHARED-READTABLE*=. - D. free software libraries will register these changes on the page on cliki. - - It is an error (unhappily not enforceable) to modify the =*SHARED-READTABLE*= + [FRR: I don't think this is checkable without implementation support, + unless maybe you're ready to pay the price of walking 1 million records + and compare your two readtables character by character, and potentially + square that because of =dispatch-macro-character=] + C. libraries need to document any change to =*shared-readtable*=. Note that + unqualified readtable modifications in the program will modify the object + to which =*readtable*= is bound, which unless rebound will also be object + to which =*shared-readtable*= is bound. + D. free software libraries will register these changes on the page on cliki.net + + It is an error (unhappily not enforceable) to modify the =*shared-readtable*= in ways that violate these restrictions. 3. Unhappily, there is no cheap way to enforce these restrictions, but that's no - regression with respect to the current situation. - + regression with respect to the current situation. + [RPG: See above: can't we check 2B, at least?] -4. ASDF binds =*READTABLE*= to =ASDF:*SHARED-READTABLE*= around any =PERFORM - COMPILE-OP= and =PERFORM LOAD-SOURCE-OP= +4. ASDF binds =*readtable*= to =*shared-readtable*= around any + =perform compile-op= and =perform load-source-op= 5. Systems that want to modify a persistent, shared readtable in ways that violate (2) must arrange to use their own private readtable, but can - otherwise do it safely. - + otherwise do it safely. + Systems that wish to make a modified readtable available should export an API - function to set the value of =*READTABLE*= appropriately. + function to set the value of =*readtable*= appropriately. 6. Implementation: - + A. ASDF binds =*readtable*= to the =*shared-readtable*= at the start of every - system's compilation (and loading?), and around the entire =ASDF:OPERATE=, - leaving the *readtable* unchanged at the end. + system's compilation (and loading?), and around the entire =asdf:operate=, + leaving the =*readtable*= unchanged at the end. [RPG: Please clarify -- I don't see how we can wrap around the entire - =OPERATE= while at the same time /not/ doing loading as well as compilation. - What's the precise scope of the binding of =*READTABLE*=?] - [FRR: we don't bind around individual LOAD-OP operations, so that + =operate= while at the same time /not/ doing loading as well as compilation. + What's the precise scope of the binding of =*readtable*=?] + [FRR: we wrap around both the entire =operate= and individual actions, + so that even if some action manages to modify the value of =*readtable*=, + still that won't pollute the next compilation action, yet things that happen + in-between are also protected, although at a more global level.] + + [FRR: we don't bind around individual =load-op= operations, so that when loading a fasl as opposed to a lisp source file, load-time - side-effects to the *readtable* variable won't be cancelled around each - file by binding *readtable* to *shared-readtable*, though a strict reading + side-effects to the =*readtable*= variable won't be cancelled around each + file by binding =*readtable*= to =*shared-readtable*=, though a strict reading of the CLHS says this implicitly happens when loading, irrespective of - the loading being .fasl or .lisp. In case it isn't, though, we don't try, + the loading being =.fasl= or =.lisp=. In case it isn't, though, we don't try, because that doesn't happen when we concatenate fasls, and we want to be as compatible with that as possible.] This easily supports systems that "modify the current readtable data structure". - However, that doesn't handle systems that "bind *readtable* to a new value", + However, that doesn't handle systems that "bind =*readtable*= to a new value", because the changes they make will shadow the changes that other systems following this style make and depend on. [RPG: We need to clarify the preceding sentence. In particular, given the way - CL works today, we can't actually /bind/ =*READTABLE*= in a useful way: + CL works today, we can't actually /bind/ =*readtable*= in a useful way: we always end up setting it. A simple example would help.] [FRR: no system can currently do it in a regular LOAD-OP or COMPILE-OP, because the CLHS mandates the bind around LOAD and COMPILE-FILE; but they can conceivably do it in a perform method, and can indeed do it in an initialization function that the user calls with or without - understanding the *READTABLE* change that occurs and its implications.] + understanding the =*readtable*= change that occurs and its implications.] To allow such an idiom, we must also do the following: - + B. ASDF binds =*readtable*= to a proper "entry readtable" at the start of every system's compilation, and records an "exit readtable" at the end of the system's loading. [RPG: 6B seems to directly contradict 4.] [FRR: yes, 5B and the following constitute a more elaborate *alternate plan* that wasn't implemented, and that I wrote before I realized the CLHS mandated that - *READTABLE* be bound by LOAD and COMPILE-FILE, which make this plan quite - overkill unless someone modifies the *READTABLE* variable in a magic perform - method on prepare-op (and prepare-source-op) without restoring it at the end - of compile-op (and load-source-op) and we want to support that. I suppose we - should just do away with that alternate plan.] + =*readtable*= be bound by =load= and =compile-file=, which make this plan quite + overkill unless someone modifies the =*readtable*= variable in a magic =perform= + method on =prepare-op= (and =prepare-source-op=) without restoring it at the end + of =compile-op= (and =load-source-op=) and we want to support that. + I suppose we should just do away with that alternate plan.] 1. Maintain a partial order on these readtable objects, assuming that each system's exit readtable supersedes the entry readtable. The least readtable - is the *shared-readtable*. It's enough to store for each new exit readtable, + is the =*shared-readtable*=. It's enough to store for each new exit readtable, identified by the name of system that created it, the set of its inferior readtables, as a list or eq-hash-table, or an equal hash-table, with each readtable being identified by the name of the system that created it. @@ -131,43 +156,45 @@ to systems that do depend on such effects happening. [FRR: yes, in that alternate plan, we don't (can't) merge objects, but assume that the programs somehow do use copy-readtable followed by - modification of what then constitutes a tree of readtable objects, - and we ensure that *READTABLE* is bound to the correct readtable object - when loading a system. Once again, this is overkill since =LOAD= and - =COMPILE-FILE= do the readtable protection, although if we wanted to + modification of what then constitutes a tree of readtable objects, + and we ensure that =*readtable*= is bound to the correct readtable object + when loading a system. Once again, this is overkill since =load= and + =compile-file= do the readtable protection, although if we wanted to extend that to other variables not protected by the CLHS, notably the - pprint-dispatch-table, then that would make more sense.] + =*print-pprint-dispatch*=, then that would make more sense.] 3. After a system is loaded, check its exit readtable, if it already exists, check that this doesn't create a cycle or issue an error. If the exit readtable doesn't already exist, add it to the set of all known exit readtables. - + 7. ASDF either - 1. binds the *readtable* to the *shared-readtable* around the entire - asdf:operate, leaving the *readtable* unchanged at the end, or + 1. binds the =*readtable*= to the =*shared-readtable*= around the entire + =asdf:operate=, leaving the =*readtable*= unchanged at the end, or - 2. always side-effects the *readtable* to correspond to the exit readtable of + 2. always side-effects the =*readtable*= to correspond to the exit readtable of the loaded system, or - 3. operate does the binding around thing, but load-system does the side-effect + 3. =operate= does the binding around thing, but =load-system= does the side-effect after it's done operate'ing. Does that strike you as complex? Because it is. That's the price of -*safely* supporting this "systems can bind a new value to *readtable*" +*safely* supporting this "systems can bind a new value to =*readtable*=" style. Unhappily some of the constraints are not enforceable (2A and 2B), but that's the very same as now. So my next question is: do you want to safely support these -conventions? Do your systems modify the current *readtable* structure, -or do they bind *readtable* to a new value? +conventions? Do your systems modify the current =*readtable*= structure, +or do they bind =*readtable*= to a new value? [RPG: So what is implemented in the syntax-control branch?] [FRR: The first simpler plan, 1-5, 6A is implemented. The more complex plan 1-5, 6B isn't.] [RPG: Got it. And do we do 7.1, 7.2, or 7.3?] - +[FRR: None, since 7 is part of plan B, which we don't implement, +amd which seems overkill now that it's established that +the CLHS mandates that =load= and =compile-file= bind =*readtable*=] # Local Variables: # mode: org -- GitLab From e81002f2543aecccc7a9dba0c2d03e2af03f624b Mon Sep 17 00:00:00 2001 From: "Robert P. Goldman" Date: Tue, 3 Jun 2014 08:36:25 -0500 Subject: [PATCH 21/40] Pull out the alternative plan and add pointers to the hyperspec. --- doc/syntax-control.txt | 124 +++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 74 deletions(-) diff --git a/doc/syntax-control.txt b/doc/syntax-control.txt index a387e2b4..ef48ffea 100644 --- a/doc/syntax-control.txt +++ b/doc/syntax-control.txt @@ -105,70 +105,59 @@ to systems that do depend on such effects happening. This easily supports systems that "modify the current readtable data structure". - However, that doesn't handle systems that "bind =*readtable*= to a new value", - because the changes they make will shadow the changes that other systems - following this style make and depend on. - - [RPG: We need to clarify the preceding sentence. In particular, given the way - CL works today, we can't actually /bind/ =*readtable*= in a useful way: - we always end up setting it. A simple example would help.] - [FRR: no system can currently do it in a regular LOAD-OP or COMPILE-OP, - because the CLHS mandates the bind around LOAD and COMPILE-FILE; - but they can conceivably do it in a perform method, and can indeed - do it in an initialization function that the user calls with or without - understanding the =*readtable*= change that occurs and its implications.] - - To allow such an idiom, we must also do the following: - - B. ASDF binds =*readtable*= to a proper "entry readtable" at the start of - every system's compilation, and records an "exit readtable" at the end of the - system's loading. [RPG: 6B seems to directly contradict 4.] - - [FRR: yes, 5B and the following constitute a more elaborate *alternate plan* that - wasn't implemented, and that I wrote before I realized the CLHS mandated that - =*readtable*= be bound by =load= and =compile-file=, which make this plan quite - overkill unless someone modifies the =*readtable*= variable in a magic =perform= - method on =prepare-op= (and =prepare-source-op=) without restoring it at the end - of =compile-op= (and =load-source-op=) and we want to support that. - I suppose we should just do away with that alternate plan.] - - 1. Maintain a partial order on these readtable objects, assuming that each - system's exit readtable supersedes the entry readtable. The least readtable - is the =*shared-readtable*=. It's enough to store for each new exit readtable, - identified by the name of system that created it, the set of its inferior - readtables, as a list or eq-hash-table, or an equal hash-table, with each - readtable being identified by the name of the system that created it. - - 2. Before a system is compiled or loaded, compute the maximum readtable of - all the exit readtables of its dependencies. If this maximum is unique, - then it will be the entry readtable of the system. If there is not a - unique readtable that is more than all the other ones, that's an error, - and we refuse to load the system. - - [RPG: I /think/ I follow this: the idea is to compute what the actual - readtable dependencies are, as a means of ensuring repeatability, as build - operations may be interleaved in different situations.] - - [RPG: Another point -- we aren't actually computing the maximum readtable - as a readtable object, are we? Instead we are computing a dependency - graph, right? And the nodes are readtable /designators/, rather than - readtables, right?] - - [FRR: yes, in that alternate plan, we don't (can't) merge objects, but - assume that the programs somehow do use copy-readtable followed by - modification of what then constitutes a tree of readtable objects, - and we ensure that =*readtable*= is bound to the correct readtable object - when loading a system. Once again, this is overkill since =load= and - =compile-file= do the readtable protection, although if we wanted to - extend that to other variables not protected by the CLHS, notably the - =*print-pprint-dispatch*=, then that would make more sense.] - - 3. After a system is loaded, check its exit readtable, if it already exists, +* ANSI spec description of binding of readtable ++ [[http://www.lispworks.com/documentation/HyperSpec/Issues/iss196_w.htm][ANSI spec issue "IN-SYNTAX"]] ++ [[http://www.lispworks.com/documentation/HyperSpec/Body/f_cmp_fi.htm]["compile-file binds =*readtable*= and =*package*= to the values they held before processing the file."]] ++ [[http://www.lispworks.com/documentation/HyperSpec/Body/f_load.htm]["load binds =*readtable*= and =*package*= to the values they held before /loading/ the file."]] + + +* Alternative Plan + +The following describes an alternative approach, more ambitious and complicated, +which was /not/ implemented. It is provided here for comparison purposes and +because of the light it sheds on design rationale. + +The preceding, simpler approach does not handle systems that "bind =*readtable*= +to a new value", because the changes they make will shadow the changes that +other systems following this style make and depend on. +No ASDF system can currently do this in a regular =LOAD-OP= or =COMPILE-OP=, +because the CLHS mandates binding =*readtable*= around LOAD and COMPILE-FILE +(see [[http://www.lispworks.com/documentation/HyperSpec/Issues/iss196_w.htm][ANSI spec issue "IN-SYNTAX"]]). However, ASDF systems can conceivably do it +in a =perform= method, and can indeed do it in an initialization function that +the user calls with or without understanding the =*readtable*= change that +occurs and its implications. + +To allow such an idiom, ASDF could perform a much more ambitious computation, +tracking the readtable through ASDF operations. In the event, this is overkill +since =load= and =compile-file= do the readtable protection +(see [[*ANSI spec description of binding of readtable]]) although if +we wanted to extend that to other variables not protected by the ANSI spec, +notably the =*print-pprint-dispatch*=, that would make more sense. + +1. ASDF binds =*readtable*= to a proper "entry readtable" at the start of every + system's compilation, and records an "exit readtable" at the end of the + system's loading. + +2. Maintain a partial order on these readtable objects, assuming that each + system's exit readtable supersedes the entry readtable. The least readtable + is the =*shared-readtable*=. It's enough to store for each new exit + readtable, identified by the name of system that created it, the set of its + inferior readtables, as a list or eq-hash-table, or an equal hash-table, with + each readtable being identified by the name of the system that created it. + +3. Before a system is compiled or loaded, compute the a dependency graph of + readtables. We effectively compute the maximum readtable of all the exit + readtables of the system's dependencies. If this maximum is unique, then it + will be the entry readtable of the system. If there is not a unique readtable + that is more than all the other ones, that's an error, and we refuse to load + the system. + +4. After a system is loaded, check its exit readtable, if it already exists, check that this doesn't create a cycle or issue an error. If the exit readtable doesn't already exist, add it to the set of all known exit readtables. -7. ASDF either +5. ASDF either 1. binds the =*readtable*= to the =*shared-readtable*= around the entire =asdf:operate=, leaving the =*readtable*= unchanged at the end, or @@ -181,20 +170,7 @@ to systems that do depend on such effects happening. Does that strike you as complex? Because it is. That's the price of *safely* supporting this "systems can bind a new value to =*readtable*=" -style. Unhappily some of the constraints are not enforceable (2A and -2B), but that's the very same as now. - -So my next question is: do you want to safely support these -conventions? Do your systems modify the current =*readtable*= structure, -or do they bind =*readtable*= to a new value? - -[RPG: So what is implemented in the syntax-control branch?] -[FRR: The first simpler plan, 1-5, 6A is implemented. -The more complex plan 1-5, 6B isn't.] -[RPG: Got it. And do we do 7.1, 7.2, or 7.3?] -[FRR: None, since 7 is part of plan B, which we don't implement, -amd which seems overkill now that it's established that -the CLHS mandates that =load= and =compile-file= bind =*readtable*=] +style. # Local Variables: # mode: org -- GitLab From 75ae2bae2a455abd67d16887dbda53f636bf30cb Mon Sep 17 00:00:00 2001 From: "Robert P. Goldman" Date: Tue, 3 Jun 2014 14:17:48 -0500 Subject: [PATCH 22/40] More cleanup; new question about conditions on *shared-readtable* use. --- doc/syntax-control.txt | 68 +++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/doc/syntax-control.txt b/doc/syntax-control.txt index ef48ffea..c9776951 100644 --- a/doc/syntax-control.txt +++ b/doc/syntax-control.txt @@ -15,61 +15,69 @@ to systems that do depend on such effects happening. The variable =*shared-readtable*= is not intended to be rebound after having been initialized; its value is an object of type =readtable=, hereafter known as the /shared readtable/. - - [RPG: What is the intended function of the *shared-readtable* as distinct from - the default readtable?] - [FRR: this *is* the default readtable. There is no other way to get to the - /initial readtable/ as opposed to (a copy of) the /standard readtable/, - which could be different and contain extensions, though since it *initially* - shouldn't contain extensions, I suppose we could use =(copy-readtable nil)=] - [RPG: I think if it's sanitation we are hoping for, then - =(copy-readtable nil)= is The Right Thing. It eliminates the potential for - getting unpredictable results by delaying load of ASDF.] - [FRR: I would have liked to agree, but my understanding is that several - implementations provide extensions in the /initial readtable/ that are *not* - part of the /standard readtable/, such as a #! ignorer sometimes, or - importantly in CCL the =#$= and =#_= readers, etc. - I understand that a strict - reading of the CLHS might make this non-conformant, but I suspect that - many systems rely on it, as I believe the ITA codebase for QRes did.] - [FRR: maybe the variable should be named =*initial-readtable*= then?] - - [RPG: Proposed answer to my own question: the =*shared-readtable*= is - provided as a convenience for libraries that ...] - [FRR: missing text above] + + Points that need to be settled: + + Should =*shared-readtable*= be bound as the value of =cl:*readtable*= at + the time ASDF was loaded, or as the return value of =(copy-readtable nil)=? + On the one hand, the latter is portable and clean. However, Fare reports + "my understanding is that several implementations provide extensions in the + /initial readtable/ that are *not* part of the /standard readtable/, such + as a #! ignorer sometimes, or importantly in CCL the =#$= and =#_= readers, + etc. I understand that a strict reading of the CLHS might make this + non-conformant, but I suspect that many systems rely on it, as I believe + the ITA codebase for QRes did." + + This seems to me (rpg) to be an interesting, unresolved ASDF issue: should + an ASDF user be able to specify that his/her system is intended either to + be portable, or to be tied to a specific CL implementation (or set of + implementations)? It seems like the above question could have more than + one answer: if you want portable code, you want =(copy-readtable nil)=; if + you want to exploit your implementation's extensions, you want the value of + =cl:*readtable*= at ASDF load time. + + Follow-up question: if the implementation bundles ASDF, is "the time ASDF + was loaded" well-defined? + + + Should the variable should be named =*initial-readtable*=, instead? I + (rpg) don't believe so. Its value is the initial readtable, but it will + then be modified, so I think "shared" better covers the intended purpose. 2. Modification of the readtable object to which =*shared-readtable*= is bound is subject to the the following restrictions (also discussed in the ASDF manual, in the section "How do I work with readtables?"): - [FRR: be sure to update the manual to match this document after it's settled.] + *NOTE:* be sure to update the manual to match this document after it's + settled. A. no modifying any standard character, B. No two systems loaded in the same image may assign different meanings to a same character in the =*shared-readtable*=. - [RPG: Spliced in FRR's admirably clear phrasing. Pulled out the hedge "as - bound to," which I beleive to be unnecessary.] [RPG: Follow-up question: is this condition checkable? I am not up to speed on portable facilities for access to =readtable='s. If ASDF doesn't check this, I think it will be hard for the programmer to check it.] [FRR: I don't think this is checkable without implementation support, unless maybe you're ready to pay the price of walking 1 million records and compare your two readtables character by character, and potentially - square that because of =dispatch-macro-character=] + square that because of =dispatch-macro-character= ]. C. libraries need to document any change to =*shared-readtable*=. Note that unqualified readtable modifications in the program will modify the object to which =*readtable*= is bound, which unless rebound will also be object to which =*shared-readtable*= is bound. D. free software libraries will register these changes on the page on cliki.net - It is an error (unhappily not enforceable) to modify the =*shared-readtable*= - in ways that violate these restrictions. + It is an error to modify the =*shared-readtable*= in ways that violate these + restrictions. + + [RPG: Question is the above error claim too restrictive? Let's say I am + developing system C. C depends on both A and B. I /know/ that A and B both + modify the =*shared-readtable*= entry for character x. I can't /fix/ this, + since I don't own either A or B. But I can enforce an ordering that loads B + after A, giving me the value for character x that I want. If I do this, why + should ASDF claim that system C is erroneous?] 3. Unhappily, there is no cheap way to enforce these restrictions, but that's no regression with respect to the current situation. - [RPG: See above: can't we check 2B, at least?] - 4. ASDF binds =*readtable*= to =*shared-readtable*= around any =perform compile-op= and =perform load-source-op= -- GitLab From 7057b5be444ba53560879a584fb9f0fc4ffae27a Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Fri, 6 Jun 2014 15:06:27 -0400 Subject: [PATCH 23/40] Document incremental build scenarios whereby a readtable handler conflict breaks the build. --- doc/syntax-control.txt | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/doc/syntax-control.txt b/doc/syntax-control.txt index c9776951..1d080fdb 100644 --- a/doc/syntax-control.txt +++ b/doc/syntax-control.txt @@ -9,13 +9,13 @@ to systems that do depend on such effects happening. * Syntax Control: implement and document this plan -1. ASDF (actually UIOP) maintains a =uiop:*shared-readtable*=, which is bound - to the value of =cl:*readtable*= at the time ASDF was loaded, +1. ASDF (actually UIOP) maintains a =uiop:*shared-readtable*=, + which is bound to the value of =cl:*readtable*= at the time ASDF was loaded, which is the /initial readtable/. The variable =*shared-readtable*= is not intended to be rebound after having been initialized; its value is an object of type =readtable=, hereafter known as the /shared readtable/. - + Points that need to be settled: + Should =*shared-readtable*= be bound as the value of =cl:*readtable*= at the time ASDF was loaded, or as the return value of =(copy-readtable nil)=? @@ -26,7 +26,7 @@ to systems that do depend on such effects happening. etc. I understand that a strict reading of the CLHS might make this non-conformant, but I suspect that many systems rely on it, as I believe the ITA codebase for QRes did." - + This seems to me (rpg) to be an interesting, unresolved ASDF issue: should an ASDF user be able to specify that his/her system is intended either to be portable, or to be tied to a specific CL implementation (or set of @@ -34,7 +34,7 @@ to systems that do depend on such effects happening. one answer: if you want portable code, you want =(copy-readtable nil)=; if you want to exploit your implementation's extensions, you want the value of =cl:*readtable*= at ASDF load time. - + Follow-up question: if the implementation bundles ASDF, is "the time ASDF was loaded" well-defined? @@ -67,7 +67,7 @@ to systems that do depend on such effects happening. It is an error to modify the =*shared-readtable*= in ways that violate these restrictions. - + [RPG: Question is the above error claim too restrictive? Let's say I am developing system C. C depends on both A and B. I /know/ that A and B both modify the =*shared-readtable*= entry for character x. I can't /fix/ this, @@ -75,6 +75,18 @@ to systems that do depend on such effects happening. after A, giving me the value for character x that I want. If I do this, why should ASDF claim that system C is erroneous?] + [FRR: if you list B after A, then it will be loaded second... but only on ASDF 3 + (on ASDF 1 and ASDF 2 (up to 2.26?), the order is reverse(!)), and only when + you build from clean. If you do an incremental build and A is modified (or one + of its dependencies), that won't trigger rebuild of B, but that will trigger + the rebuild of the rest of your system, that will now see the x handler from A, + and fail to rebuild correctly. On the other hand, even if B depends on A, + if C depends on A's handler, which B overrides in an incompatible way, then + if C is modified but not A, A will not be reloaded, and C will be loaded with + the wrong x handler. Unless the character x is never actually used, there will + be incremental build scenarios that break. One "solution" of course is to + only ever build from clean, like we long did at ITA.] + 3. Unhappily, there is no cheap way to enforce these restrictions, but that's no regression with respect to the current situation. @@ -137,8 +149,8 @@ occurs and its implications. To allow such an idiom, ASDF could perform a much more ambitious computation, tracking the readtable through ASDF operations. In the event, this is overkill -since =load= and =compile-file= do the readtable protection -(see [[*ANSI spec description of binding of readtable]]) although if +since =load= and =compile-file= do the readtable protection +(see [[*ANSI spec description of binding of readtable]]) although if we wanted to extend that to other variables not protected by the ANSI spec, notably the =*print-pprint-dispatch*=, that would make more sense. -- GitLab From 40e74ca53542cf6a14276a03ac5e74eb135f4bfd Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Fri, 13 Oct 2017 00:19:34 -0400 Subject: [PATCH 24/40] utility: remove fwrap, table-alist These utilities are not currently used; they can be reinstated later if needed. fwrap was for composing a list of wrapper functions, except that the last was the special case function to be wrapped. Better to just (define and) use compose, with funcall (or call-function) as the base case. table-alist was as the name implies, and was intended to help go between internal table representations and external alist representations of per-system variables. --- uiop/utility.lisp | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/uiop/utility.lisp b/uiop/utility.lisp index 5fbfc1b5..92340920 100644 --- a/uiop/utility.lisp +++ b/uiop/utility.lisp @@ -28,9 +28,9 @@ #:stamp< #:stamps< #:stamp*< #:stamp<= ;; stamps #:earlier-stamp #:stamps-earliest #:earliest-stamp #:later-stamp #:stamps-latest #:latest-stamp #:latest-stamp-f - #:list-to-hash-set #:ensure-gethash #:table-alist ;; hash-table + #:list-to-hash-set #:ensure-gethash ;; hash-table #:ensure-function #:access-at #:access-at-count ;; functions - #:call-function #:call-functions #:fwrap #:register-hook-function + #:call-function #:call-functions #:register-hook-function #:lexicographic< #:lexicographic<= ;; version #:simple-style-warning #:style-warn ;; simple style warnings #:match-condition-p #:match-any-condition-p ;; conditions @@ -468,13 +468,6 @@ with the given ARGUMENTS" "For each function in the list FUNCTION-SPECS, in order, call the function as per CALL-FUNCTION" (map () 'call-function function-specs)) - (defun fwrap (&rest functions) - (when functions - (destructuring-bind (first . rest) functions - (if rest - (call-function first #'(lambda () (apply 'fwrap rest))) - (call-function first))))) - (defun register-hook-function (variable hook &optional call-now-p) "Push the HOOK function (a designator as per ENSURE-FUNCTION) onto the hook VARIABLE. When CALL-NOW-P is true, also call the function immediately." @@ -535,14 +528,7 @@ Return two values: the entry after its optional computation, and whether it was (defun list-to-hash-set (list &aux (h (make-hash-table :test 'equal))) "Convert a LIST into hash-table that has the same elements when viewed as a set, up to the given equality TEST" - (dolist (x list h) (setf (gethash x h) t))) - - (defgeneric table-alist (table)) - (defmethod table-alist ((table hash-table)) - (loop :for k :being :the :hash-keys :of table :using (:hash-value v) - :collect (cons k v))) - (defmethod table-alist ((table cons)) - table)) + (dolist (x list h) (setf (gethash x h) t)))) ;;; Lexicographic comparison of lists of numbers (with-upgradability () -- GitLab From 1bbc7a5a5870e9b9ca49ce58ad07e25ebb868d97 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Fri, 13 Oct 2017 00:24:26 -0400 Subject: [PATCH 25/40] Move with-asdf-syntax from find-system to lisp-action --- find-system.lisp | 12 ------------ lisp-action.lisp | 13 +++++++++++++ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/find-system.lisp b/find-system.lisp index 3eb01e5b..880680ab 100644 --- a/find-system.lisp +++ b/find-system.lisp @@ -8,7 +8,6 @@ :asdf/find-component :asdf/system-registry :asdf/plan :asdf/operate) (:import-from #:asdf/component #:%additional-input-files) (:export - #:call-with-asdf-syntax #:with-asdf-syntax #:find-system #:locate-system #:load-asd #:define-op #:load-system-definition-error #:error-name #:error-pathname #:error-condition)) (in-package :asdf/find-system) @@ -44,17 +43,6 @@ ;; the precise protocol between the two functions may change in the future (or not). (first (gethash `(find-system ,(coerce-name name)) (asdf-cache)))) - (defun call-with-asdf-syntax (function &key package) - (with-standard-io-syntax - (let ((*readtable* *shared-readtable*) - (*print-pprint-dispatch* *shared-print-pprint-dispatch*) - (*package* (find-package (or package :asdf-user))) - (*print-readably* nil)) - (call-function function)))) - - (defmacro with-asdf-syntax ((&key package) &body body) - `(call-with-asdf-syntax #'(lambda () ,@body) :package ,package)) - (defclass define-op (non-propagating-operation) () (:documentation "An operation to record dependencies on loading a .asd file.")) diff --git a/lisp-action.lisp b/lisp-action.lisp index 1569568a..06547ecc 100644 --- a/lisp-action.lisp +++ b/lisp-action.lisp @@ -6,6 +6,7 @@ (:use :uiop/common-lisp :uiop :asdf/upgrade :asdf/session :asdf/component :asdf/system :asdf/operation :asdf/action) (:export + #:call-with-asdf-syntax #:with-asdf-syntax #:try-recompiling #:cl-source-file #:cl-source-file.cl #:cl-source-file.lsp #:basic-load-op #:basic-compile-op @@ -16,6 +17,18 @@ #:lisp-compilation-output-files)) (in-package :asdf/lisp-action) +;;;; Syntax control +(with-upgradability () + (defun call-with-asdf-syntax (function &key package) + (with-standard-io-syntax + (let ((*readtable* *shared-readtable*) + (*print-pprint-dispatch* *shared-print-pprint-dispatch*) + (*package* (find-package (or package :asdf-user))) + (*print-readably* nil)) + (call-function function)))) + + (defmacro with-asdf-syntax ((&key package) &body body) + `(call-with-asdf-syntax #'(lambda () ,@body) :package ,package))) ;;;; Component classes (with-upgradability () -- GitLab From 8c034f6800a99f5cfca7cb1cc4d732dd3d65e747 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Fri, 13 Oct 2017 16:19:31 -0400 Subject: [PATCH 26/40] Rewrite the syntax-control document --- TODO | 51 ----- doc/syntax-control.md | 447 +++++++++++++++++++++++++++++++++++++++++ doc/syntax-control.txt | 197 ------------------ 3 files changed, 447 insertions(+), 248 deletions(-) create mode 100644 doc/syntax-control.md delete mode 100644 doc/syntax-control.txt diff --git a/TODO b/TODO index 4d8dc262..bb0303d6 100644 --- a/TODO +++ b/TODO @@ -432,57 +432,6 @@ a reified context for featurep checks, etc. *** There again, a check that a forward-package is not backward would be very nice. -<<<<<<< HEAD -* Syntax Control: implement and document this plan -** ASDF maintains an asdf:*shared-readtable* -*** This table is initialized as the *readtable* object at the time ASDF was loaded. -** This *shared-readtable* is subject to the current restrictions: -*** No modifying any standard character, -*** No two dependencies assigning different meaning to the same non-standard character. -*** Libraries need to document any change to the readtable -*** Free software libraries will register these changes on the page on cliki.net -** Unhappily, there is no cheap way to enforce these restrictions -*** No enforcement is no no regression with respect to the current situation. -*** Expensive and/or non-portable enforcement is possible -**** Just intercept the API calls made to the implementation (without-package-lock) -** ASDF wraps any compile-op and load-source-op in this asdf:*shared-readtable* -*** However, no wrapping for load-op, to preserve combine-fasl linking semantics. -** Systems that want to bypass the above restrictions must: -*** arrange to use their own private readtable, but can otherwise do it safely. -*** It is an error (unhappily not enforceable) to modify the current readtable in these ways. -** Option A: ASDF binds *readtable* to the *shared-readtable* -*** at the start of every system's compilation (and loading?), and -*** around the entire asdf:operate, leaving the *readtable* unchanged at the end. -** This easily supports systems that "modify the current readtable data structure". -*** However, that doesn't systems that "bind *readtable* to a new value" -because the changes they make will shadow the changes that other -systems following this style make and depend on. To allow such, an -idiom, we must also do the following: -** Option B: ASDF binds *readtable* to a proper "entry readtable" at the start -of every system's compilation, and record an "exit readtable" at the -end of the system's loading. -*** Maintain a partial order on these readtable objects, assuming that each system's exit readtable supersedes the entry readtable. -*** The least readtable is the *shared-readtable*. -*** It's enough to store for each new exit readtable, as identified by: -**** The name of system that created it, and -**** the set of its inferior readtables, as a list or eq-hash-table, or an equal hash-table, -**** with each readtable being identified by the name of the system that created it. -*** Before a system is compiled or loaded, compute its entry readtable -**** That entry readtable will be the maximum of all the exit readtables of its dependencies. -**** If this maximum is unique, then it will be the entry readtable of the system. -**** If there is not a unique readtable that is more than all the other ones, that's an error, and we refuse to load the system. -*** After a system is loaded, check its exit readtable -**** if it already exists, check that this doesn't create a cycle or issue an error; -**** it it doesn't already exist, add it to the set of all known exit readtables. -*** ASDF either -**** A- binds the *read-table* to the *shared-readtable* around the entire asdf:operate, leaving the *readtable* unchanged at the end, or -**** B- always side-effects the *read-table* to correspond to the exit readtable of the loaded system, or -**** C- operate does the binding around thing, but load-system does the side-effect after it's done operate'ing. -** Does that strike you as complex? Because it is. That's the price of *safely* supporting this "systems can bind a new value to *readtable*" style. Unhappily some of the constraints are not enforceable, but that's the very same as now. -** So my next question is: do you want to safely support these -conventions? Do your systems modify the current *readtable* structure, -or do they bind *readtable* to a new value? - * Faster source-registry: ** In addition and/or as a substitute to the .cl-source-registry.cache, that is meant to be semi-automatically managed, there could be diff --git a/doc/syntax-control.md b/doc/syntax-control.md new file mode 100644 index 00000000..3b3aa088 --- /dev/null +++ b/doc/syntax-control.md @@ -0,0 +1,447 @@ +# Syntax Control + +This document describes an issue with using reader macros in Common Lisp, +and makes proposals as to how to handle this issue in ASDF. + +https://bugs.launchpad.net/asdf/+bug/1293325 +http://thread.gmane.org/gmane.lisp.asdf.devel/3883 +http://thread.gmane.org/gmane.lisp.asdf.devel/4033 + + +## When Syntax Gets Out of Control + +Currently ASDF makes no attempt to control what value `*readtable*` is bound to. +As a result, systems with uncontrolled side-effects to the current `*readtable*` +may appear to build properly when built from scratch by themselves, +yet may break the build in subtle way when the build is incremental and/or +when part of a larger build that contains other systems +with conflicting side-effects. +(NB: The first variants of this document were written in 2014 while working on +ASDF 3.1, but its contents remain current as of ASDF 3.3.0 in October 2017.) + +For instance, some CL libraries directly modify the current `*readtable*` +by calling e.g. `set-dispatch-macro-character` without specifying a readtable. +But most CL programs also expect to be read by the standard CL readtable. +This can lead to "interesting" issues: + + * If two systems none of which depends on the other (or an implementation + and a system not tied to it) both define reader macros for the same + character or pair of characters (e.g. `#u`), then a build that includes + both these systems never guarantees which of the two systems will have been + loaded last and whose side-effect will win at the time any particular + subsequent system that depends on either of the above is loaded. + It may depend on which of the libraries was modified last (or last had + a modified dependency) and therefore was or wasn't recompiled during the + incremental build; it may depend on the order in which dependencies appear + in the larger build; it may depend on which strategy is used by the + particular version of ASDF; it may depend on the use of POIU; etc. + + * Worse, even if some system plays nice with everyone else, + it can still mess up or be messed up by an interactive build. + A system can play nice by only using readtables through `named-readtables`, + whereby it only modifies and uses well-defined private readtables, + never modifying a random underspecified "current readtable". + Yet, if ASDF is called while such a readtable is active as the `*readtable*` + then all hell can break loose: + If some system is built that doesn't play nice and modifies the readtable, + then the readtable becomes corrupted and may cause further builds to fail. + If some unrelated system is built that didn't expect the modified readtable, + it may be negatively affected by it and corrupt the build. + At this point, your fasl cache is corrupted and you need to clean it + and re-build everything from scratch in a new image. + +While changes to the readtable will cause the most spectacular failures, +other syntax-controlling variables are at stake, too. + + * Note that because `load` and `compile-file` locally rebinds `*readtable*` + to the same value as in the surrounding context, + changes of *binding* to the `*readtable*` variable, + unlike modifications to the *object* currently bound to it, + do not escape the evaluation of a given file. + See ANSI spec description of binding of readtable: + [ANSI spec issue "IN-SYNTAX"](http://clhs.lisp.se/Issues/iss196_w.htm). + ["compile-file binds `*readtable*` and `*package*` to the values they held before processing the file."](http://clhs.lisp.se/Body/f_cmp_fi.htm), + ["load binds `*readtable*` and `*package*` to the values they held before /loading/ the file."](http://clhs.lisp.se/Body/f_load.htm) + + * However, modifications to *other* syntax variables, + such as `*read-base*`, or `*read-default-float-format*` or `*read-eval*` + or any of the `*print-FOO*` variables, do escape this evaluation. + + * Furthermore, any syntax extension whether using reader macros or regular + macros, if it depends on the bindings of any variable whatsoever, + causes changes to that variable to similarly escape in the environment. + The set of syntax variables is therefore potentially unbounded. + + * Because of the possibility of an incremental build, there can never be + a guarantee in advance about what will or won't be rebuilt incrementally, + and which changes to syntax variables may or may not have been effected + when any system is compiled. + Therefore, it is not allowed to ever let modifications to any such variable + to escape a given system. + Thus, it is not acceptable to use + `(setf *read-default-float-format* 'double-float)` + and not restore the initial value before the end of a given system's build; + if your code depends on such changes, you must either hook into ASDF + to make such bindings happen, or you must only use ASDF as wrapped through + build scripts that enforce the correct values when building any given file. + + +## Some examples of breakage + +### Conflict with macro-characters + +If two systems define the same macro-character or dispatch-macro-character, +such as `#"` being defined by both `closure-common` and `string-escape`, +then these systems, if they are part of the same build, +may cause non-determinism as to which definition will be active at any time +during an incremental build. +Depending on which system was last modified, then only that system may be +re-built, at which point its modifications to the readtable will happen last. +No system can use `#"` syntax and trust which of the above last won. + +Other conflicts found by grepping Quicklisp (as of October 2017) for +`'^(set-dispatch-macro-character '` include: +`#T` defined by both `closure-html` and `racer`, or +`#{` defined by both `cl-quickcheck` and `folio`, or +`#~` defined by both `metatilities` and `perlre`, or +`#t` and `#f` defined by both `paiprolog-html` and `racer`, or +`#_` defined by both `software-evolution` and CCL or MCL. +There may or may not be further conflicts not detected through this simple grep +for direct toplevel modification of the shared `*readtable*`. + + +### Building with the wrong `*readtable*` + +Currently, all components compiled with ASDF promiscuously share +whatever readtable is active at the time `asdf:operate` is called. +This means that if some component side-effects the `*readtable*` variable +and/or the state of the data structure it's bound to, +the effects will be seen by all components compiled with ASDF, +*including components that do not "depends-on" support functions* that the reader output produces. +This causes catastrophic failure, when at the REPL you e.g. + +``` +(asdf:load-system :fare-quasiquote) +(named-readtables:in-readtable :fare-quasiquote) +;; Now modify a file in any system depends on, e.g. ASDF itself. +(uiop:run-program "touch somefile-that-in-the-system-below.lisp") +(asdf:load-system :some-modified-system-that-doesnt-use-fare-quasiquote) ;; example: ASDF +``` + +Then your system will be recompiled using `fare-quasiquote` for its backquote implementation, +except that since it doesn't depends-on `fare-quasiquote`, +the `fare-quasiquote` runtime will not be there next time you load the file from scratch. +Your fasl cache is therefore corrupted, and +there is nothing `fare-quasiquote` can do to protect users. +This is not specific to `fare-quasiquote`, but applies to anyone who uses any private readtable +with any discrepancy from the shared readtable concerning any macro character. + +To protect users who use anything but the shared readtable at the REPL, +ASDF should make sure this shared readtable is used whenever building software. + + +## Statu quo: Readtable discipline + +The _de facto_ though tacit requirements for all Common Lisp software currently +is therefore to follow a double discipline, +without which software may fail to build in mysterious ways: + + * Developers must *never* introduce conflicts with *any* known extension when + modifying the `*readtable*` as inherited from the initial environment. + This means that no one may ever modify `#u` except `puri`, + or else become incompatible with `puri` and prevent the two systems + from ever being used together. Similarly, no one may modify `#$` except CCL + or else become incompatible with CCL. + Unhappily, there is no global registry of who claims what extension, + which makes defining any extension a gamble that jeopardizes the build. + There is thus a chilling effect whereby + prudent programmers never define extensions to the current `*readtable*`; + and there is thus a counter-selective effect whereby + only reckless programmers do define such extensions. + + * Developers must *never* call ASDF while the current `*readtable*` is bound + to any readtable but the initial, shared readtable. + If they do, their readtable may either be corrupted, + or cause the build to be corrupted, or both. + But there is currently no good means to access for certain this readtable, + so no program may call ASDF at runtime while such readtable might be active; + and no program may change the `*readtable*` while calling systems + that might themselves call ASDF. + This once again is a chilling effect against any use readtables other than + the initial `*readtable*` and against any use of ASDF at runtime, + and a counter-selection in who will use those features. + + +## Proposal 1: Minimal Syntax Control + +This first proposal is for immediate implementation in the next release of ASDF +(whichever it may be, which would be ASDF 3.3.1 if done in October 2017). +It would offer a sane way for disciplined programmers to define reader macros +either as part of the shared readtable or as part of private readtables: + + * Start by merely documenting the restrictions that apply to modifying + the implicitly shared initial readtable, and mandating that + modifications be themselves documented: + + 1. No one is allowed to modify any standard character in this readtable, + in any way whatsoever. + + 2. No two systems loaded in the same image may assign different meanings + to a same character in the *shared readtable*. + (This is not presently checkable, but see Proposal 4 below). + + 3. Systems need to document any change to the *shared readtable*. + Note that unqualified readtable modifications in the program will modify + the object to which `*readtable*` is bound, which unless rebound + will also be the *shared readtable*. + + 4. Free software libraries will register these changes in + a suitable page of cliki.net: < http://cliki.net/Macro%20Characters > + + * Optionally, declare a moratorium on defining any further such extension, + except as part of a [CDR](https://common-lisp.net/project/cdr/); + let's call that option 1.1.1 (and no such option 1.1.0). + + * Have ASDF itself enforce that the same readtable will be used + whenever building Lisp software, by defining a `uiop:*shared-readtable*` + that will be shared by all CL code by binding it around `asdf:operate`, thus + allowing developers who want to keep using the above shared modifications + and developers who prefer to use private readtables + to work without interfering with each other anymore. + + * Optionally, have a similar mechanism for the `*print-pprint-dispatch*`, + for all variables modified by `with-standard-io-syntax`, and/or + for a user-extensible set of variables; + let's call these options 1.2.1, 1.2.2 and 1.2.3 respectively + (and adopting none of them 1.2.0). + +With this minimal change in place, there is a viable story going forward +for how to use reader macros in Common Lisp, and +one that maintains backward compatibility. +Note that for a minimal change, the bindings should only go around `operate`, +at which point changes to syntax variables can still backward-compatibly escape +from one system to the other, and it remains the responsibility of the user +to make sure that it happens in a deterministic fashion. + +My vote is for next version of ASDF (i.e. ASAP) to implement this proposal 1, +with options 1.1.1 and 1.2.2 (and maybe later 1.2.3, if the code is ready). + + +### Some hacks still supported under Proposal 1 + +One notably way to modify the syntax in a deterministic way that works with the +current ASDF 3.3.0 and would keep working is to serialize all dependencies +through a syntax-modification system: a system called e.g. `my-modified-syntax` +depends on all the regular-syntaxed software dependencies in the project, +before it itself modifies syntax variables; +all other systems in the project will then depend-on `my-modified-syntax`, +forcing the order of evaluation of the syntactic side-effects. +The above setup ensures that all regular CL files are build with regular syntax, +and that all files that depend on the modified syntax see that modified syntax. +Proposal 1 ensures that the latter modifications do not cause non-deterministic +build issues when one of the regular systems is modified +while the latter modifications are active. +Note that this approach is only possible in a closed system, +as opposed to when publishing free software libraries. +But this approach is indeed possible, and we may want to keep supporting it, +at least until we have better alternatives to offer. + + +### Details of the proposed implementation of Proposal 1 + +UIOP (and hence ASDF) remembers the `uiop:*initial-readtable*` that was +the current value of `cl:*readtable*` when UIOP was first loaded. +This readtable is presumably the +[initial readtable](http://clhs.lisp.se/Body/26_glo_i.htm#initial_readtable). +This readtable comes with all the restrictions described above. +UIOP also remembers a read-only `uiop:*standard-readtable*` +with only the standard characters. +The two above variables are not mean to be re-bound after initialization. + +Last but not least, UIOP maintains a `uiop:*shared-readtable*`. +Its default value is `uiop:*initial-readtable*` which is backward compatible. +The variable can instead be re-bound to `uiop:*standard-readtable*` +for enforcement against uncontrolled readtable modifications, +or to a value of `(copy-readtable nil)` +(which should be a modifiable copy of the standard-readtable, +for code that modifies the current readtable yet +doesn't trust what was "initially" current when UIOP was loaded +and/or doesn't want to use implementation-dependent modifications), +or something else, so users may decide what shared readtable enforcement +they do or don't want in their builds. +Several implementations provide extensions in the *initial readtable* +that are *not* part of the *standard readtable*, such as +a `#!` ignorer sometimes, or importantly in CCL the `#$` and `#_` readers, etc. +A strict reading of the CLHS might make this non-conformant, but +many systems might rely on it, as the ITA codebase for QRes used to. + +Systems that want to modify a persistent readtable +in ways not permitted with respect to the initial readtable +*must* arrange to use their own private readtable, e.g. via `named-readtables`. +Thanks to the re-binding of `*readtable*` by ASDF under Proposal 1 (and later), +they can now do it safely and not cause a build corruption if that readtable +is active while ASDF is called. + + +## Proposal 2: Per-System Syntax + +The second proposal is builds on top of proposal 1, but +instead of only binding syntax variables around the entire build, +it allows for binding those variables +around the build of every system and/or every component. +This proposal adds significant complexity, as it requires +defining and tracking a set variables around every system. +In exchange for this complexity, however, you can achieve +both determinism and flexibility at the same time. + +I (Faré) had code to do a lot of this proposal in the `syntax-control` branch, +but I believe it is not actually ready for release yet, +so I'm removing it. +I do not recommend this approach at this point, but I do describe it here +for comparison purposes and because of the light it sheds on the issue +and the rationale for solving it even in a minimal way. + +With per-system syntax variables, it is possible for the `*readtable*` +as well as all other variables (such as `*print-pprint-dispatch*`) +to be specified as in a way better defined +than with using the promiscuous `*shared-readtable*`. +This makes it possible for systems to fully take advantage of syntax extension, +without either interfering with other systems +or suffering from interference by these other systems, and without +painfully doing a `(named-readtables:in-readtable :foo)` in every file. + + * ASDF tracks a set of "syntax variables". + + * For each variable in the set, and for each system, + ASDF, tracks the "entry value" and "exit value" of the variable + at the beginning and end of building the system. + + * ASDF maintains a partial order on these systems based on dependencies. + For every considered system, identify the set of "maximal" dependencies, + such that there isn't any intermediate system in the partial order + between them and the considered system. + + * For every system and every variable, either the system must specify + an explicit entry value for the variable, or all the maximal dependencies + of that system must agree on their exit value (or else raise an error), + at which point that value becomes the system's entry value of the variable. + This applies to the readtable as to other values of syntax variables. + (Alternatively (which I don't recommend) always inherit exit values + from the last (or the first) of the explicit dependencies as entry values + for the system.) + + * After a system is loaded, record all the exit values + of all its syntax variables. + + * For the purpose of specifying the entry value of `*readtable*`, + each readtable can be identified either + by a string that designates the least system that has it as exit value, + or by a symbol that designates a `named-readtables` readtable. + + * Since `*readtable*` is bound by `load` and `compile-file`, + there needs be a separate mechanism for specifying + exit values of readtable, for instance using `named-readtables` symbols. + + * A given readtable may be modified by further systems, according to rules + essentially similar to those that apply to the initial readtable: + no conflict is allowed. + An extension that allows for read-only readtables would be welcome, too, + so these user-defined readtables too can be protected from corruption. + + * After the end of a call to `operate`, ASDF may either: + + * restore the syntax variable values from before ASDF was called, or + + * side-effect the syntax variables to reflect exit values of the system it just built. + + * Conceivably, `operate` could do the former, and `load-system` the latter. + Or the two options could otherwise both be available depending on a flag. + + +Does that strike you as complex? Because it is. +That's the price of *safely* supporting arbitrary modifications +to syntax variables in a deterministic yet general way. + + +## Proposal 3: Strict Enforcement. + +This third proposal forbids any modification to the current `*readtable*`. +I made this proposal in earnest in 2014, but it was justifiedly met with horror +and a big pushback from the community: on the one hand, it is not +backward compatible and causes a lot of legacy software to fail to build; +on the other hand, most Common Lisp implementations offer little protection +in case this proposal is in place and someone makes a mistake. + +Under this proposal, all Lisp libraries are read and compiled while +the current `*readtable*` is bound to the *standard* readtable, +as made available via `with-standard-io-syntax` +(see [CLHS 2.1.1.2](http://clhs.lisp.se/Body/02_aab.htm) +and [CLHS glossary for "standard readtable"](http://clhs.lisp.se/Body/26_glo_s.htm#standard_readtable)). +This readtable is notionally read-only as per the standard; +however only SBCL is actually enforcing that as of 2017, +such that you will have a nice clean error immediately, +instead of horrible silent system corruption later. +But this SBCL behavior is enough to detect systems that cause this issue and fix them. + +This change however is clearly backward-incompatible, +and will break libraries that currently modify the current `*readtable*`. +These include unmaintained libraries this feature of which is actually used +by other libraries, as can be found in Quicklisp. +Fixing all these libraries will take time. +Yet without all those fixes, this proposal is a non-starter. +And even after those fixes, commercial users with non-visible code +may legitimately object to this change. + +Using proposal 1, you can easily opt in this proposal 3 by setting +the `uiop:*shared-readtable*` to `uiop:*standard-readtable*` +just after you load ASDF and before you load anything else. +Even without proposal 1, you can achieve this effect +by having you entire build inside a `with-standard-io-syntax`, +or by inserting a `(setf *readtable* (with-standard-io-syntax *readtable*))` +before your build (or at an appropriate point in your build). + +I (Faré) still think it is a good idea to mandate that at least public libraries +in Quicklisp should be fixed to never side-effect the current `*readtable*`. +But I acknowledge that the Common Lisp ecosystem is not ready for such a change +at this point; and since I'm not a paid Common Lisp professional at this point, +I don't have the energy to push for such a change by getting all libraries fixed. + + +## Proposal 4: Strict Enforcement modulo Declaration. + +Proposal 4 is more elaborate, and would require modifications +of a least some implementations' support for readtables: +systems would declare as part of their `defsystem` declarations +which system defines (or uses?) which macro characters. +The implementation for readtables (or some semi-portable wrapper around them) +could then enforce those declarations. +I am not going to push for this proposal at this point, +but in case someone ever wants to implement a complete solution, here is a sketch. + + * Unhappily, there is no cheap way to enforce these restrictions, + but they can be achieved by writing suitable wrappers + on top of the implementation's native readtables. + To hook these wrappers onto the standard symbols used by regular libraries, + the implementation's package-locks, if any, + may have to be temporarily disabled. + On implementations where such hooks can't be used, + we can fall back to mandating restrictions without enforcement, + which isn't a regression compared to what is available today. + + * Readtables would detect what characters have or haven't been + either used or modified so far. + These flags can be harvested, individually or globally set or reset, + to detect which systems have used or modified which characters. + When defining a new macro character, an error is issued + if the character was already used before and/or modified before. + + * A system may declare that it modifies some characters + in the shared readtable. That it does actually modify those characters + and only those characters can now be enforced via the above mechanism. + If and when there are conflicts, they can be detected early, + and a meaningful error message can be issued. + Developers do not have to rely on a out-of-date manually curated cliki page + to check that there are no conflicts; + the build system can automatically check it for them, + and the cliki page can be automatically generated from information + automatically gathered by building all of Quicklisp. diff --git a/doc/syntax-control.txt b/doc/syntax-control.txt deleted file mode 100644 index 1d080fdb..00000000 --- a/doc/syntax-control.txt +++ /dev/null @@ -1,197 +0,0 @@ -* Syntax control: objective - -What is the problem that we are trying to fix? - -We are trying to fix uncontrolled, unintentional leaking of -readtable side effects to systems that depend on such effects not happening, -without removing /intended/ leaking of readtable side effects -to systems that do depend on such effects happening. - -* Syntax Control: implement and document this plan - -1. ASDF (actually UIOP) maintains a =uiop:*shared-readtable*=, - which is bound to the value of =cl:*readtable*= at the time ASDF was loaded, - which is the /initial readtable/. - The variable =*shared-readtable*= is not intended to be rebound - after having been initialized; its value is an object of type =readtable=, - hereafter known as the /shared readtable/. - - Points that need to be settled: - + Should =*shared-readtable*= be bound as the value of =cl:*readtable*= at - the time ASDF was loaded, or as the return value of =(copy-readtable nil)=? - On the one hand, the latter is portable and clean. However, Fare reports - "my understanding is that several implementations provide extensions in the - /initial readtable/ that are *not* part of the /standard readtable/, such - as a #! ignorer sometimes, or importantly in CCL the =#$= and =#_= readers, - etc. I understand that a strict reading of the CLHS might make this - non-conformant, but I suspect that many systems rely on it, as I believe - the ITA codebase for QRes did." - - This seems to me (rpg) to be an interesting, unresolved ASDF issue: should - an ASDF user be able to specify that his/her system is intended either to - be portable, or to be tied to a specific CL implementation (or set of - implementations)? It seems like the above question could have more than - one answer: if you want portable code, you want =(copy-readtable nil)=; if - you want to exploit your implementation's extensions, you want the value of - =cl:*readtable*= at ASDF load time. - - Follow-up question: if the implementation bundles ASDF, is "the time ASDF - was loaded" well-defined? - - + Should the variable should be named =*initial-readtable*=, instead? I - (rpg) don't believe so. Its value is the initial readtable, but it will - then be modified, so I think "shared" better covers the intended purpose. - -2. Modification of the readtable object to which =*shared-readtable*= is bound - is subject to the the following restrictions (also discussed in - the ASDF manual, in the section "How do I work with readtables?"): - - *NOTE:* be sure to update the manual to match this document after it's - settled. - - A. no modifying any standard character, - B. No two systems loaded in the same image may assign different meanings - to a same character in the =*shared-readtable*=. - [RPG: Follow-up question: is this condition checkable? I am not up to speed - on portable facilities for access to =readtable='s. If ASDF doesn't check - this, I think it will be hard for the programmer to check it.] - [FRR: I don't think this is checkable without implementation support, - unless maybe you're ready to pay the price of walking 1 million records - and compare your two readtables character by character, and potentially - square that because of =dispatch-macro-character= ]. - C. libraries need to document any change to =*shared-readtable*=. Note that - unqualified readtable modifications in the program will modify the object - to which =*readtable*= is bound, which unless rebound will also be object - to which =*shared-readtable*= is bound. - D. free software libraries will register these changes on the page on cliki.net - - It is an error to modify the =*shared-readtable*= in ways that violate these - restrictions. - - [RPG: Question is the above error claim too restrictive? Let's say I am - developing system C. C depends on both A and B. I /know/ that A and B both - modify the =*shared-readtable*= entry for character x. I can't /fix/ this, - since I don't own either A or B. But I can enforce an ordering that loads B - after A, giving me the value for character x that I want. If I do this, why - should ASDF claim that system C is erroneous?] - - [FRR: if you list B after A, then it will be loaded second... but only on ASDF 3 - (on ASDF 1 and ASDF 2 (up to 2.26?), the order is reverse(!)), and only when - you build from clean. If you do an incremental build and A is modified (or one - of its dependencies), that won't trigger rebuild of B, but that will trigger - the rebuild of the rest of your system, that will now see the x handler from A, - and fail to rebuild correctly. On the other hand, even if B depends on A, - if C depends on A's handler, which B overrides in an incompatible way, then - if C is modified but not A, A will not be reloaded, and C will be loaded with - the wrong x handler. Unless the character x is never actually used, there will - be incremental build scenarios that break. One "solution" of course is to - only ever build from clean, like we long did at ITA.] - -3. Unhappily, there is no cheap way to enforce these restrictions, but that's no - regression with respect to the current situation. - -4. ASDF binds =*readtable*= to =*shared-readtable*= around any - =perform compile-op= and =perform load-source-op= - -5. Systems that want to modify a persistent, shared readtable in ways that - violate (2) must arrange to use their own private readtable, but can - otherwise do it safely. - - Systems that wish to make a modified readtable available should export an API - function to set the value of =*readtable*= appropriately. - -6. Implementation: - - A. ASDF binds =*readtable*= to the =*shared-readtable*= at the start of every - system's compilation (and loading?), and around the entire =asdf:operate=, - leaving the =*readtable*= unchanged at the end. - [RPG: Please clarify -- I don't see how we can wrap around the entire - =operate= while at the same time /not/ doing loading as well as compilation. - What's the precise scope of the binding of =*readtable*=?] - [FRR: we wrap around both the entire =operate= and individual actions, - so that even if some action manages to modify the value of =*readtable*=, - still that won't pollute the next compilation action, yet things that happen - in-between are also protected, although at a more global level.] - - [FRR: we don't bind around individual =load-op= operations, so that - when loading a fasl as opposed to a lisp source file, load-time - side-effects to the =*readtable*= variable won't be cancelled around each - file by binding =*readtable*= to =*shared-readtable*=, though a strict reading - of the CLHS says this implicitly happens when loading, irrespective of - the loading being =.fasl= or =.lisp=. In case it isn't, though, we don't try, - because that doesn't happen when we concatenate fasls, and we want to - be as compatible with that as possible.] - - This easily supports systems that "modify the current readtable data - structure". - -* ANSI spec description of binding of readtable -+ [[http://www.lispworks.com/documentation/HyperSpec/Issues/iss196_w.htm][ANSI spec issue "IN-SYNTAX"]] -+ [[http://www.lispworks.com/documentation/HyperSpec/Body/f_cmp_fi.htm]["compile-file binds =*readtable*= and =*package*= to the values they held before processing the file."]] -+ [[http://www.lispworks.com/documentation/HyperSpec/Body/f_load.htm]["load binds =*readtable*= and =*package*= to the values they held before /loading/ the file."]] - - -* Alternative Plan - -The following describes an alternative approach, more ambitious and complicated, -which was /not/ implemented. It is provided here for comparison purposes and -because of the light it sheds on design rationale. - -The preceding, simpler approach does not handle systems that "bind =*readtable*= -to a new value", because the changes they make will shadow the changes that -other systems following this style make and depend on. -No ASDF system can currently do this in a regular =LOAD-OP= or =COMPILE-OP=, -because the CLHS mandates binding =*readtable*= around LOAD and COMPILE-FILE -(see [[http://www.lispworks.com/documentation/HyperSpec/Issues/iss196_w.htm][ANSI spec issue "IN-SYNTAX"]]). However, ASDF systems can conceivably do it -in a =perform= method, and can indeed do it in an initialization function that -the user calls with or without understanding the =*readtable*= change that -occurs and its implications. - -To allow such an idiom, ASDF could perform a much more ambitious computation, -tracking the readtable through ASDF operations. In the event, this is overkill -since =load= and =compile-file= do the readtable protection -(see [[*ANSI spec description of binding of readtable]]) although if -we wanted to extend that to other variables not protected by the ANSI spec, -notably the =*print-pprint-dispatch*=, that would make more sense. - -1. ASDF binds =*readtable*= to a proper "entry readtable" at the start of every - system's compilation, and records an "exit readtable" at the end of the - system's loading. - -2. Maintain a partial order on these readtable objects, assuming that each - system's exit readtable supersedes the entry readtable. The least readtable - is the =*shared-readtable*=. It's enough to store for each new exit - readtable, identified by the name of system that created it, the set of its - inferior readtables, as a list or eq-hash-table, or an equal hash-table, with - each readtable being identified by the name of the system that created it. - -3. Before a system is compiled or loaded, compute the a dependency graph of - readtables. We effectively compute the maximum readtable of all the exit - readtables of the system's dependencies. If this maximum is unique, then it - will be the entry readtable of the system. If there is not a unique readtable - that is more than all the other ones, that's an error, and we refuse to load - the system. - -4. After a system is loaded, check its exit readtable, if it already exists, - check that this doesn't create a cycle or issue an error. If the exit - readtable doesn't already exist, add it to the set of all known exit - readtables. - -5. ASDF either - - 1. binds the =*readtable*= to the =*shared-readtable*= around the entire - =asdf:operate=, leaving the =*readtable*= unchanged at the end, or - - 2. always side-effects the =*readtable*= to correspond to the exit readtable of - the loaded system, or - - 3. =operate= does the binding around thing, but =load-system= does the side-effect - after it's done operate'ing. - -Does that strike you as complex? Because it is. That's the price of -*safely* supporting this "systems can bind a new value to =*readtable*=" -style. - -# Local Variables: -# mode: org -# End: -- GitLab From 400aa5cf58182217f5969df7735c0fb1a7052cae Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Fri, 13 Oct 2017 16:20:20 -0400 Subject: [PATCH 27/40] Move with-asdf-syntax to with-shared-syntax. From lisp-action.lisp to uiop/eval.lisp. Also replace standard-eval-thunk by a new shared-eval-thunk. --- find-system.lisp | 2 +- interface.lisp | 1 - lisp-action.lisp | 14 ------ package-inferred-system.lisp | 4 +- uiop/eval.lisp | 85 ++++++++++++++++++++++++------------ uiop/image.lisp | 4 +- 6 files changed, 63 insertions(+), 47 deletions(-) diff --git a/find-system.lisp b/find-system.lisp index 880680ab..da22a708 100644 --- a/find-system.lisp +++ b/find-system.lisp @@ -89,7 +89,7 @@ (assert (equal (coerce-name s) (primary-system-name s))) (nest (if-let ((pathname (first (input-files o s))))) - (with-asdf-syntax (:package :asdf-user)) + (with-shared-syntax (:package :asdf-user)) (let ((*default-pathname-defaults* ;; resolve logical-pathnames so they won't wreak havoc in parsing namestrings. (pathname-directory-pathname (physicalize-pathname pathname))))) diff --git a/interface.lisp b/interface.lisp index 99803854..a6d836b2 100644 --- a/interface.lisp +++ b/interface.lisp @@ -123,7 +123,6 @@ #:*encoding-external-format-hook* #:*default-encoding* #:*utf-8-external-format* - #:call-with-asdf-syntax #:with-asdf-syntax #:clear-configuration #:*output-translations-parameter* #:initialize-output-translations diff --git a/lisp-action.lisp b/lisp-action.lisp index 06547ecc..60f289b1 100644 --- a/lisp-action.lisp +++ b/lisp-action.lisp @@ -6,7 +6,6 @@ (:use :uiop/common-lisp :uiop :asdf/upgrade :asdf/session :asdf/component :asdf/system :asdf/operation :asdf/action) (:export - #:call-with-asdf-syntax #:with-asdf-syntax #:try-recompiling #:cl-source-file #:cl-source-file.cl #:cl-source-file.lsp #:basic-load-op #:basic-compile-op @@ -17,19 +16,6 @@ #:lisp-compilation-output-files)) (in-package :asdf/lisp-action) -;;;; Syntax control -(with-upgradability () - (defun call-with-asdf-syntax (function &key package) - (with-standard-io-syntax - (let ((*readtable* *shared-readtable*) - (*print-pprint-dispatch* *shared-print-pprint-dispatch*) - (*package* (find-package (or package :asdf-user))) - (*print-readably* nil)) - (call-function function)))) - - (defmacro with-asdf-syntax ((&key package) &body body) - `(call-with-asdf-syntax #'(lambda () ,@body) :package ,package))) - ;;;; Component classes (with-upgradability () (defclass cl-source-file (source-file) diff --git a/package-inferred-system.lisp b/package-inferred-system.lisp index 9b2b952c..f4879d2d 100644 --- a/package-inferred-system.lisp +++ b/package-inferred-system.lisp @@ -6,7 +6,7 @@ (:use :uiop/common-lisp :uiop :asdf/upgrade :asdf/session :asdf/component :asdf/system :asdf/system-registry :asdf/lisp-action - :asdf/find-system :asdf/parse-defsystem) + :asdf/parse-defsystem) (:export #:package-inferred-system #:sysdef-package-inferred-system-search #:package-system ;; backward compatibility only. To be removed. @@ -46,7 +46,7 @@ every such file")) ;; Find the first defpackage form in a stream, if any (defun stream-defpackage-form (stream) - (with-asdf-syntax () + (with-shared-syntax (:package :asdf-user) (loop :for form = (read stream nil nil) :while form :when (defpackage-form-p form) :return form))) diff --git a/uiop/eval.lisp b/uiop/eval.lisp index 0262d487..45178864 100644 --- a/uiop/eval.lisp +++ b/uiop/eval.lisp @@ -5,37 +5,63 @@ (:recycle :uiop/eval :asdf) (:use :uiop/common-lisp :uiop/utility :uiop/stream) (:export + #:*initial-readtable* #:*initial-print-pprint-dispatch* #:*standard-readtable* #:*standard-print-pprint-dispatch* #:*shared-readtable* #:*shared-print-pprint-dispatch* #:*standard-syntax-variables* - #:call-with-shared-readtable #:call-with-standard-io-syntax - #:eval-input #:eval-thunk #:standard-eval-thunk + #:call-with-shared-syntax #:with-shared-syntax + #:eval-input #:eval-thunk #:shared-eval-thunk #:ensure-variable #:variable-value)) (in-package :uiop/eval) ;;; Safe syntax (with-upgradability () + (defvar *initial-readtable* *readtable* + "The initial readtable, as inherited from the environment at the time UIOP was first loaded. +It initially implements the syntax specified by the CLHS, but may also have implementation-specific +extensions, and may grow user-defined extensions. You MUST ONLY make conservative extensions to it: + A- No one may at any point modify any standard character in any way. + B- No two dependencies may assign different meaning to the same non-standard character. + Using any non-standard character while expecting the implementation to treat some way + counts as such an assignment of meaning (e.g. using CCL's #$ syntax for FFI). + C- Libraries need to document these assignments of meaning to non-standard characters. + D- Free software libraries will register these changes on: + http://www.cliki.net/Macro%20Characters +The above restrictions have always existed but were previously implicit. +There is unhappily no enforcement against non-conservative extensions, so tread carefully. +Do not re-bind this variable.") + + (defvar *initial-print-pprint-dispatch* (with-standard-io-syntax *print-pprint-dispatch*) + "The initial readtable, as inherited from the environment at the time UIOP was first loaded. +It initially implements the syntax specified by the CLHS, but may also have implementation-specific +extensions, and may grow user-defined extensions. You MUST ONLY make conservative extensions to it, +i.e. you may not modify standard behavior, or conflict with anyone else's previous extensions. +There is unhappily no enforcement against non-conservative extensions, so tread carefully. +Do not re-bind this variable.") + (defvar *standard-readtable* (with-standard-io-syntax *readtable*) "The standard readtable, implementing the syntax specified by the CLHS. -It must never be modified, though only good implementations will even enforce that.") +It must never be modified in any way whatsoever. +However, only SBCL is known to enforce this prohibition. +Do not re-bind this variable.") (defvar *standard-print-pprint-dispatch* (with-standard-io-syntax *print-pprint-dispatch*) "The standard pprint dispatch table, implementing the syntax specified by the CLHS. -It must never be modified, though only good implementations will even enforce that.") +It must never be modified in any way whatsoever. +However, only SBCL is known to enforce this prohibition. +Do not re-bind this variable.") - (defvar *shared-readtable* *readtable* - "This shared readtable allows legacy applications to keep modifying a global shared readtable - while maintaining some hygiene for those who want to use their own readtable. - It is subject to the following restrictions, which always existed but were previously implicit: - A- no modifying any standard character, - B- no two dependencies assigning different meaning to the same non-standard character. - Using any non-standard character while expecting the implementation to treat some way - counts as such an assignment of meaning. - C- libraries need to document these assignments of meaning to non-standard characters. - D- free software libraries will register these changes on: - http://www.cliki.net/Macro%20Characters -") - (defvar *shared-print-pprint-dispatch* *print-pprint-dispatch* + (defvar *shared-readtable* *initial-readtable* + "The shared readtable used by all Lisp software while building with ASDF. +This shared readtable is usually the *initial-readtable*, in which case +conservative readtable extensions are permitted and such extensions only; +see the documentation of that variable for more details. +For even stricter enforcement against modifications at all, this readtable may be bound to +the *standard-readtable* (at least on SBCL); this helps enforce good behavior from all libraries, +but is not compatible with some older libraries. +This readtable can also be bound to other readtables to implement various instrumentations +by Lisp-to-Lisp compilers when (re)building software, presumably with a different fasl cache.") + (defvar *shared-print-pprint-dispatch* *initial-print-pprint-dispatch* "*print-pprint-dispatch* table shared by all ASDF systems. It should match the extensions of *shared-readtable* -- see the latter variable's documentation.") @@ -62,12 +88,18 @@ It should match the extensions of *shared-readtable* -- see the latter variable' (*read-suppress* . nil) (*readtable* . ,*standard-readtable*)))) +;;;; Syntax control (with-upgradability () - (defun call-with-standard-io-syntax (function) - (with-standard-io-syntax (call-function function))) + (defun call-with-shared-syntax (function &key package readably) + (with-standard-io-syntax + (let ((*readtable* *shared-readtable*) + (*print-pprint-dispatch* *shared-print-pprint-dispatch*) + (*package* (find-package (or package :cl-user))) + (*print-readably* readably)) + (call-function function)))) - (defun call-with-shared-readtable (thunk) - (let ((*readtable* *shared-readtable*)) (call-function thunk))) + (defmacro with-shared-syntax ((&rest keys &key &allow-other-keys) &body body) + `(call-with-shared-syntax #'(lambda () ,@body) ,@keys)) (defun eval-input (input) "Portably read and evaluate forms from INPUT, return the last values." @@ -90,13 +122,12 @@ If a string, repeatedly read and evaluate from it, returning the last values." (function (funcall thunk)) (string (eval-input thunk)))) - (defun standard-eval-thunk (thunk &key (package :cl)) - "Like EVAL-THUNK, but in a more standardized evaluation context." - ;; Note: it's "standard-" not "safe-", because evaluation is never safe. + (defun shared-eval-thunk (thunk &key (package :cl)) + "Like EVAL-THUNK, but in the shared evaluation context." + ;; Note: it's "shared-" not "safe-", because evaluation is never safe. (when thunk - (with-safe-io-syntax (:package package) - (let ((*read-eval* t)) - (eval-thunk thunk)))))) + (with-shared-syntax (:package package) + (eval-thunk thunk))))) ;;; Late-binding variables (with-upgradability () diff --git a/uiop/image.lisp b/uiop/image.lisp index 1cf74188..20321595 100644 --- a/uiop/image.lisp +++ b/uiop/image.lisp @@ -312,7 +312,7 @@ of the function will be returned rather than interpreted as a boolean designatin (setf *image-prelude* prelude) (setf *image-restored-p* :in-progress) (call-image-restore-hook) - (standard-eval-thunk prelude) + (shared-eval-thunk prelude) (setf *image-restored-p* t) (let ((results (multiple-value-list (if entry-point @@ -347,7 +347,7 @@ or COMPRESSION on SBCL, and APPLICATION-TYPE on SBCL/Windows." (setf *image-dumped-p* (if executable :executable t)) (setf *image-restored-p* :in-regress) (setf *image-postlude* postlude) - (standard-eval-thunk *image-postlude*) + (shared-eval-thunk *image-postlude*) (setf *image-dump-hook* dump-hook) (call-image-dump-hook) (setf *image-restored-p* nil) -- GitLab From 8b3f67f70cf8c9007498fb1f38f847a8372316bf Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Sat, 14 Oct 2017 20:36:40 -0400 Subject: [PATCH 28/40] Fix stylo in eval.lisp --- uiop/eval.lisp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uiop/eval.lisp b/uiop/eval.lisp index 45178864..b00ce5e4 100644 --- a/uiop/eval.lisp +++ b/uiop/eval.lisp @@ -104,7 +104,7 @@ It should match the extensions of *shared-readtable* -- see the latter variable' (defun eval-input (input) "Portably read and evaluate forms from INPUT, return the last values." (with-input (input) - (loop :with results :with eof ='#:eof + (loop :with results :with eof = '#:eof :for form = (read input nil eof) :until (eq form eof) :do (setf results (multiple-value-list (eval form))) -- GitLab From e5243a9f8c3e19f23335427d053f7cb4dd98d9a6 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Sun, 15 Oct 2017 09:55:46 -0400 Subject: [PATCH 29/40] REMOVE support for per-system special bindings Undo all changes that implement per-system special bindings, whereby a set of syntax variables are set to known values around actions concerning a system. This reversal is made in view of making a minimal change towards syntax control, and may be reversed again in some future. A rollback of this commit is expected at some indefinite point in the future. --- action.lisp | 16 +------- lisp-action.lisp | 3 -- system.lisp | 101 +--------------------------------------------- uiop/eval.lisp | 24 +---------- uiop/stream.lisp | 1 + uiop/utility.lisp | 39 +----------------- 6 files changed, 6 insertions(+), 178 deletions(-) diff --git a/action.lisp b/action.lisp index 6f1cf0f4..6d68ffb7 100644 --- a/action.lisp +++ b/action.lisp @@ -4,8 +4,7 @@ (uiop/package:define-package :asdf/action (:nicknames :asdf-action) (:recycle :asdf/action :asdf/plan :asdf) - (:use :uiop/common-lisp :uiop :asdf/upgrade :asdf/session - :asdf/component :asdf/system :asdf/operation) + (:use :uiop/common-lisp :uiop :asdf/upgrade :asdf/session :asdf/component :asdf/operation) (:import-from :asdf/operation #:check-operation-constructor) (:import-from :asdf/component #:%additional-input-files) (:export @@ -18,7 +17,6 @@ #:action-operation #:action-component #:make-action #:component-operation-time #:mark-operation-done #:compute-action-stamp #:perform #:perform-with-restarts #:retry #:accept - #:with-action-context #:call-with-action-context #:action-path #:find-action #:operation-definition-warning #:operation-definition-error ;; condition #:action-valid-p @@ -464,22 +462,12 @@ Returns two values: ;;;; Perform (with-upgradability () - (defgeneric call-with-action-context (operation component thunk)) - (define-convenience-action-methods call-with-action-context (operation component thunk)) - (defmethod call-with-action-context ((o operation) (c component) thunk) - (with-updated-system-variables ((component-system c)) - (call-function thunk))) - (defmacro with-action-context ((operation component) &body body) - `(call-with-action-context ,operation ,component #'(lambda () ,@body))) - (defgeneric perform (operation component) (:documentation "PERFORM an action, consuming its input-files and building its output-files")) (define-convenience-action-methods perform (operation component)) (defmethod perform :around ((o operation) (c component)) - (while-visiting-action (o c) - (with-action-context (o c) - (call-next-method)))) + (while-visiting-action (o c) (call-next-method))) (defmethod perform :before ((o operation) (c component)) (ensure-all-directories-exist (output-files o c))) (defmethod perform :after ((o operation) (c component)) diff --git a/lisp-action.lisp b/lisp-action.lisp index 60f289b1..987a2759 100644 --- a/lisp-action.lisp +++ b/lisp-action.lisp @@ -37,9 +37,6 @@ (:documentation "Base class for operations that apply the compile-time effects of a file")) (defclass basic-prepare-op (operation) ()) - (defmethod perform ((operation basic-prepare-op) (system system)) - (initialize-system-variables system) - (use-initial-system-variables system)) (defvar *cl-reading-hook* 'call-function "Hook to call around operations that read CL code (since ASDF 3.1). diff --git a/system.lisp b/system.lisp index 124c397e..9bd8fb73 100644 --- a/system.lisp +++ b/system.lisp @@ -9,12 +9,6 @@ #:system-source-file #:system-source-directory #:system-relative-pathname #:system-description #:system-long-description #:system-author #:system-maintainer #:system-licence #:system-license - #:*default-system-variables* #:*asdf-syntax-variables* - #:compute-system-variables #:configure-system-variables #:configure-system-variable - #:with-updated-system-variables - #:use-initial-system-variables #:use-current-system-variables - #:initialize-system-variables #:update-system-variables - #:system-variable-specs #:system-variable-names #:system-variable-values #:system-variable-initializers #:definition-dependency-list #:definition-dependency-set #:system-defsystem-depends-on #:system-depends-on #:system-weakly-depends-on #:component-build-pathname #:build-pathname @@ -102,103 +96,10 @@ a SYSTEM is redefined and its class is modified.")) :initform nil) ;; these two are specially set in parse-component-form, so have no :INITARGs. (depends-on :reader system-depends-on :initform nil) - (weakly-depends-on :reader system-weakly-depends-on :initform nil) - ;; System variables - (variable-specs :initform nil :initarg :variables :reader system-variable-specs) - (variable-names :accessor system-variable-names) - (variable-values :accessor system-variable-values) - (variable-initializers :accessor system-variable-initializers)) + (weakly-depends-on :reader system-weakly-depends-on :initform nil)) (:documentation "SYSTEM is the base class for top-level components that users may request ASDF to build.")) - (defun ensure-system-variable (variable &key when-undefined) - (ensure-variable variable :package :asdf :when-undefined when-undefined)) - - (defvar *asdf-syntax-variables* - `(,@*standard-syntax-variables* - (*readtable* . ,*shared-readtable*) - (*print-pprint-dispatch* . ,*shared-print-pprint-dispatch*) - (*package* . (find-package :asdf-user)))) - - (defvar *default-system-variables* '()) ;; make it *asdf-syntax-variables* for safety - - (defgeneric configure-system-variable (system variable &key)) - (defmethod configure-system-variable ((system system) variable &key (initializer nil initp)) - ;; You should configure system variables only during the call to compute-system-variables. - ;; During configuration, the initializer hash-table ensures each variable appears only once, - ;; modulo the fact that you can designate a variable using multiple names, in which case you lose. - ;; The heuristics should be simple, though: if it's a CL or ASDF variable, just use it; - ;; otherwise, use a string that has : if and only if the variable is exported, - ;; which it should be if you're not the author of the system. - ;; Beware that initializers will be called in the order the variables were declared, - ;; which, if you override a variable's initializer, - ;; will be earlier than other variables you're initializing. - (let ((value (variable-value variable :package :asdf :when-undefined nil)) - (previousp (nth-value 1 (gethash variable (system-variable-initializers system))))) - (unless previousp - (push variable (system-variable-names system)) - (push value (system-variable-values system))) - (when (or initp (not previousp)) - (setf (gethash variable (system-variable-initializers system)) - (if initp initializer (constantly value))))) - nil) - - (defgeneric configure-system-variables (system variable-specs)) - (defmethod configure-system-variables ((system system) variable-specs) - (dolist (spec variable-specs) - (multiple-value-bind (variable keys) - (if (consp spec) - (values (first spec) (when (consp (cdr spec)) (list :initializer (second spec)))) - (values spec nil)) - (apply 'configure-system-variable system variable keys)))) - - (defgeneric update-system-variables (system)) - (defmethod update-system-variables ((system system)) - (setf (system-variable-values system) - (loop :for variable :in (system-variable-names system) - :collect (variable-value variable :package :asdf :when-undefined nil)))) - - (defgeneric system-variable-initializer-list (system)) - (defmethod system-variable-initializer-list ((system system)) - (loop :for var :in (system-variable-names system) - :collect (gethash var (system-variable-initializers system)))) - - (defun use-initial-system-variables (system) - (set-variables (system-variable-names system) (system-variable-initializer-list system) - :name #'ensure-system-variable :value #'call-function)) - - (defun use-current-system-variables (system) - (set-variables (system-variable-names system) (system-variable-values system) - :name #'ensure-system-variable)) - - (defmacro with-updated-system-variables ((component) &body body) - `(call-with-updated-system-variables ,component #'(lambda () ,@body))) - - (defun call-with-updated-system-variables (component thunk) - (let ((system (component-system component))) - (with-variables ((system-variable-names system) (system-variable-values system) - :name #'ensure-system-variable) - (multiple-value-prog1 (call-function thunk) - (update-system-variables system))))) - - (defgeneric compute-system-variables (system)) - (defmethod compute-system-variables ((system system)) - (configure-system-variables system (system-variable-specs system)) - ;; TODO: insert an out-of-band system configuration facility , or in an :after method. - nil) - - (defun initialize-system-variables (system) - (setf (system-variable-names system) nil - (system-variable-values system) nil - (system-variable-initializers system) (make-hash-table :test 'equal)) - (compute-system-variables system) - (setf (system-variable-names system) (reverse (system-variable-names system)) - (system-variable-values system) (reverse (system-variable-values system)))) - - (defmethod shared-initialize :after ((system system) slot-names &key) - (declare (ignore slot-names)) - (initialize-system-variables system)) - (defclass undefined-system (system) () (:documentation "System that was not defined yet.")) diff --git a/uiop/eval.lisp b/uiop/eval.lisp index b00ce5e4..0325d417 100644 --- a/uiop/eval.lisp +++ b/uiop/eval.lisp @@ -10,8 +10,7 @@ #:*shared-readtable* #:*shared-print-pprint-dispatch* #:*standard-syntax-variables* #:call-with-shared-syntax #:with-shared-syntax - #:eval-input #:eval-thunk #:shared-eval-thunk - #:ensure-variable #:variable-value)) + #:eval-input #:eval-thunk #:shared-eval-thunk)) (in-package :uiop/eval) ;;; Safe syntax @@ -129,24 +128,3 @@ If a string, repeatedly read and evaluate from it, returning the last values." (with-shared-syntax (:package package) (eval-thunk thunk))))) -;;; Late-binding variables -(with-upgradability () - (defun ensure-variable (name &key package (when-undefined 'error)) - (etypecase name - (symbol name) - (string (or (ignore-errors - (let ((s (safe-read-from-string name :package package))) - (and (symbolp s) s))) - (call-function when-undefined - "Cannot read non-nil symbol from ~S" name))))) - - (defun variable-value (name &key package (when-undefined 'error)) - (let ((var (ensure-variable name :package package :when-undefined when-undefined))) - (if (boundp var) - (symbol-value var) - (call-function when-undefined "Symbol ~S unbound" name)))) - - (defun (setf variable-value) (value name &key package (when-undefined 'error)) - (if-let (var (ensure-variable name :package package :when-undefined when-undefined)) - (setf (symbol-value var) value) - value))) diff --git a/uiop/stream.lisp b/uiop/stream.lisp index 745b4ece..9853abbe 100644 --- a/uiop/stream.lisp +++ b/uiop/stream.lisp @@ -146,6 +146,7 @@ going through all the proper hooks." (defun call-with-safe-io-syntax (function &key package) (with-standard-io-syntax (let ((*package* (find-package (or package :common-lisp))) + (*read-default-float-format* 'double-float) (*print-readably* nil) (*read-eval* nil)) (call-function function)))) diff --git a/uiop/utility.lisp b/uiop/utility.lisp index 92340920..6e69efa2 100644 --- a/uiop/utility.lisp +++ b/uiop/utility.lisp @@ -35,9 +35,7 @@ #:simple-style-warning #:style-warn ;; simple style warnings #:match-condition-p #:match-any-condition-p ;; conditions #:call-with-muffled-conditions #:with-muffled-conditions - #:not-implemented-error #:parameter-error - #:call-with-variables #:with-variables #:set-variables ;; variables - #:call-with-alist-variables #:with-alist-variables #:set-alist-variables)) + #:not-implemented-error #:parameter-error)) (in-package :uiop/utility) ;;;; Defining functions in a way compatible with hot-upgrade: @@ -647,38 +645,3 @@ message, that takes the functionality as its first argument (that can be skipped :format-control format-control :format-arguments format-arguments))) -;;; Variables -(with-upgradability () - (defun call-with-variables (variables bindings thunk &key (name #'identity) (value #'identity)) - (loop :for variable :in variables - :for binding :in bindings - :for var = (call-function name variable) - :when var - :collect var :into vars - :and :collect (call-function value binding) :into vals - :finally (progv vars vals (call-function thunk)))) - - (defmacro with-variables ((variables initializers &rest keys &key name value) &body body) - (declare (ignore name value)) - `(call-with-variables ,variables ,initializers #'(lambda () ,@body) ,@keys)) - - (defun set-variables (variables bindings &key (name #'identity) (value #'identity)) - (loop :for variable :in variables - :for binding :in bindings - :for var = (call-function name variable) - :when var - :collect var :into vars - :and :collect (call-function value binding) :into vals - :finally (map () #'set vars vals))) - - (defun call-with-alist-variables (alist thunk &rest keys &key name value) - (declare (ignore name value)) - (apply 'call-with-variables (mapcar #'car alist) (mapcar #'cdr alist) thunk keys)) - - (defmacro with-alist-variables ((alist &rest keys &key name value) &body body) - (declare (ignore name value)) - `(call-with-alist-variables ,alist #'(lambda () ,@body) ,@keys)) - - (defun set-alist-variables (alist &rest keys &key name value) - (declare (ignore name value)) - (apply 'set-variables (mapcar #'car alist) (mapcar #'cdr alist) keys))) -- GitLab From 72c626da4185d36aa085088b6825eb5cfabbe092 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Sun, 15 Oct 2017 10:28:56 -0400 Subject: [PATCH 30/40] REMOVE support for basic-prepare-op, cl-reading-op Undo all changes that implement basic-prepare-op, cl-reading-op, and binding of syntax around perform'ance of cl-reading-op. Note that these changes depend on with-action-context, which was reverted in the previous commit regarding per-system variable bindings. These reversals are made in view of making a minimal change towards syntax control, and may be reversed again in some future. A rollback of this commit is expected at some indefinite point in the future. --- interface.lisp | 2 -- lisp-action.lisp | 26 +++++++------------------- system.lisp | 2 ++ 3 files changed, 9 insertions(+), 21 deletions(-) diff --git a/interface.lisp b/interface.lisp index a6d836b2..a1a79e2d 100644 --- a/interface.lisp +++ b/interface.lisp @@ -27,10 +27,8 @@ #:upward-operation #:downward-operation #:sideway-operation #:selfward-operation #:non-propagating-operation #:build-op #:make - #:basic-load-op #:basic-prepare-op #:basic-compile-op #:load-op #:prepare-op #:compile-op #:prepare-source-op #:load-source-op #:test-op #:define-op - #:cl-reading-op #:*cl-reading-hook* #:feature #:version #:version-satisfies #:upgrade-asdf #:implementation-identifier #:implementation-type #:hostname #:component-depends-on ; backward-compatible name rather than action-depends-on diff --git a/lisp-action.lisp b/lisp-action.lisp index 987a2759..8bdeb3bf 100644 --- a/lisp-action.lisp +++ b/lisp-action.lisp @@ -9,13 +9,13 @@ #:try-recompiling #:cl-source-file #:cl-source-file.cl #:cl-source-file.lsp #:basic-load-op #:basic-compile-op - #:basic-prepare-op #:cl-reading-op #:*cl-reading-hook* #:load-op #:prepare-op #:compile-op #:test-op #:load-source-op #:prepare-source-op #:call-with-around-compile-hook #:perform-lisp-compilation #:perform-lisp-load-fasl #:perform-lisp-load-source #:lisp-compilation-output-files)) (in-package :asdf/lisp-action) + ;;;; Component classes (with-upgradability () (defclass cl-source-file (source-file) @@ -34,25 +34,12 @@ (defclass basic-load-op (operation) () (:documentation "Base class for operations that apply the load-time effects of a file")) (defclass basic-compile-op (operation) () - (:documentation "Base class for operations that apply the compile-time effects of a file")) - - (defclass basic-prepare-op (operation) ()) - - (defvar *cl-reading-hook* 'call-function - "Hook to call around operations that read CL code (since ASDF 3.1). -To make it a NOP, set it to UIOP:CALL-FUNCTION. -To make it bind just *READTABLE*, set it to UIOP:CALL-WITH-SHARED-READTABLE (the default). -To bind all syntax variables to predictable portable values, set it to ASDF:CALL-WITH-ASDF-SYNTAX. -For stricter values, set it to UIOP:CALL-WITH-STANDARD-IO-SYNTAX.") + (:documentation "Base class for operations that apply the compile-time effects of a file"))) - (defclass cl-reading-op (operation) ()) ;; operations that read CL source - - (defmethod call-with-action-context ((o cl-reading-op) (c cl-source-file) thunk) - (call-next-method o c #'(lambda () (call-function *cl-reading-hook* thunk))))) ;;; Our default operations: loading into the current lisp image (with-upgradability () - (defclass prepare-op (basic-prepare-op upward-operation sideway-operation) + (defclass prepare-op (upward-operation sideway-operation) ((sideway-operation :initform 'load-op :allocation :class)) (:documentation "Load the dependencies for the COMPILE-OP or LOAD-OP of a given COMPONENT.")) (defclass load-op (basic-load-op downward-operation selfward-operation) @@ -60,14 +47,15 @@ For stricter values, set it to UIOP:CALL-WITH-STANDARD-IO-SYNTAX.") ;; so we need to directly depend on prepare-op for its side-effects in the current image. ((selfward-operation :initform '(prepare-op compile-op) :allocation :class)) (:documentation "Operation for loading the compiled FASL for a Lisp file")) - (defclass compile-op (basic-compile-op cl-reading-op downward-operation selfward-operation) + (defclass compile-op (basic-compile-op downward-operation selfward-operation) ((selfward-operation :initform 'prepare-op :allocation :class)) (:documentation "Operation for compiling a Lisp file to a FASL")) - (defclass prepare-source-op (basic-prepare-op upward-operation sideway-operation) + + (defclass prepare-source-op (upward-operation sideway-operation) ((sideway-operation :initform 'load-source-op :allocation :class)) (:documentation "Operation for loading the dependencies of a Lisp file as source.")) - (defclass load-source-op (basic-load-op cl-reading-op downward-operation selfward-operation) + (defclass load-source-op (basic-load-op downward-operation selfward-operation) ((selfward-operation :initform 'prepare-source-op :allocation :class)) (:documentation "Operation for loading a Lisp file as source.")) diff --git a/system.lisp b/system.lisp index 9bd8fb73..3325ee46 100644 --- a/system.lisp +++ b/system.lisp @@ -53,7 +53,9 @@ NB: This interface is subject to change. Please contact ASDF maintainers if you (defmethod component-entry-point ((c component)) nil)) + ;;;; The system class + (with-upgradability () (defclass proto-system () ; slots to keep when resetting a system ;; To preserve identity for all objects, we'd need keep the components slots -- GitLab From 4f763571419f4593197fb285ac2f5a4eaea5ad8f Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Mon, 16 Oct 2017 16:42:18 -0400 Subject: [PATCH 31/40] Add regression test for syntax-control --- test/syntax-control/test-quasiquote.asd | 1 + test/syntax-control/test-quasiquote.lisp | 14 ++++++++++++ test/test-syntax-control.script | 29 ++++++++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 test/syntax-control/test-quasiquote.asd create mode 100644 test/syntax-control/test-quasiquote.lisp create mode 100644 test/test-syntax-control.script diff --git a/test/syntax-control/test-quasiquote.asd b/test/syntax-control/test-quasiquote.asd new file mode 100644 index 00000000..0e5708d7 --- /dev/null +++ b/test/syntax-control/test-quasiquote.asd @@ -0,0 +1 @@ +(defsystem "test-quasiquote" :components ((:file "test-quasiquote"))) diff --git a/test/syntax-control/test-quasiquote.lisp b/test/syntax-control/test-quasiquote.lisp new file mode 100644 index 00000000..1bd047d7 --- /dev/null +++ b/test/syntax-control/test-quasiquote.lisp @@ -0,0 +1,14 @@ +(in-package :cl-user) + +(defparameter *quine* + '((LAMBDA (X) (LIST X `',X)) '(LAMBDA (X) (LIST X `',X)))) + +;; Interestingly, at least on SBCL, this assertion does not hold: +#-(or sbcl) (assert (equal *quine* (eval *quine*))) + +(assert (equal (write-to-string *quine*) + (write-to-string (eval *quine*)))) + +(assert (equal (eval *quine*) + (eval (eval *quine*)))) + diff --git a/test/test-syntax-control.script b/test/test-syntax-control.script new file mode 100644 index 00000000..1c2e54c2 --- /dev/null +++ b/test/test-syntax-control.script @@ -0,0 +1,29 @@ +;;-*- Lisp -*- +;; Testing that ASDF builds with a controlled syntax +;; when invoked with readtable different from the initial/shared readtable. + +(in-package :asdf-test) + +;; Simple stuff +(setf *central-registry* (list (subpathname *test-directory* "syntax-control/"))) + +;; Try to load lisp-invocation from ext/ or from the user environment. +(initialize-source-registry + `(:source-registry + (:directory ,*asdf-directory*) + (:directory (,*asdf-directory* "uiop/")) + (:tree (,*asdf-directory* "ext/")) + :inherit-configuration)) + +(DBG "Run test-quasiquote. Should be using the system's quasiquote.") +(asdf:load-system :test-quasiquote :force t) + +(DBG "Run test-quasiquote. Should STILL be using the system's quasiquote.") +(load-system :fare-quasiquote-readtable) +(named-readtables:in-readtable :fare-quasiquote) +(asdf:load-system :test-quasiquote :force t) + +(DBG "Run the previously compiled test-quasiquote. If it used fare-quasiquote, it will fail.") +(asdf::clear-registered-systems) +(delete-package :fare-quasiquote) +(asdf:load-system :test-quasiquote) -- GitLab From e0e95a26c9424aef8f97ee53b7572d69814a83c8 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Mon, 16 Oct 2017 17:28:52 -0400 Subject: [PATCH 32/40] Implement Proposal 1 for syntax control --- operate.lisp | 1 + 1 file changed, 1 insertion(+) diff --git a/operate.lisp b/operate.lisp index ccd66561..c613a469 100644 --- a/operate.lisp +++ b/operate.lisp @@ -83,6 +83,7 @@ But do NOT depend on it, for this is deprecated behavior.")) (let* ((*verbose-out* (and verbose *standard-output*)) (*compile-file-warnings-behaviour* on-warnings) (*compile-file-failure-behaviour* on-failure))) + (with-shared-syntax (:package :asdf-user)) (unwind-protect (progn (incf (operate-level)) -- GitLab From a636e55b291a63e54201b3e41ac7c43473e56627 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Sat, 21 Oct 2017 03:12:33 +0000 Subject: [PATCH 33/40] Disable test-syntax-control on CMUCL and MKCL. The test depends on named-readtables (via fare-quasiquote-readtables), but named-readtables fails to run on either CMUCL or MKCL at this time. Disabling the test on these two implementations, rather than to rewrite it. --- test/test-syntax-control.script | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test-syntax-control.script b/test/test-syntax-control.script index 1c2e54c2..dd5ba817 100644 --- a/test/test-syntax-control.script +++ b/test/test-syntax-control.script @@ -4,6 +4,9 @@ (in-package :asdf-test) +#+(or cmucl mkcl) +(leave-test "CMUCL and MKCL have trouble with named-readtables." 0) + ;; Simple stuff (setf *central-registry* (list (subpathname *test-directory* "syntax-control/"))) -- GitLab From e29d6744665ce060334632fab589b23b1cfab619 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Sat, 21 Oct 2017 13:43:42 -0400 Subject: [PATCH 34/40] Fix definition of *initial-print-pprint-dispatch* Remove the (with-standard-io-syntax ...), improperly kept from a copy-paste. --- uiop/eval.lisp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uiop/eval.lisp b/uiop/eval.lisp index 0325d417..7082031e 100644 --- a/uiop/eval.lisp +++ b/uiop/eval.lisp @@ -30,7 +30,7 @@ The above restrictions have always existed but were previously implicit. There is unhappily no enforcement against non-conservative extensions, so tread carefully. Do not re-bind this variable.") - (defvar *initial-print-pprint-dispatch* (with-standard-io-syntax *print-pprint-dispatch*) + (defvar *initial-print-pprint-dispatch* *print-pprint-dispatch* "The initial readtable, as inherited from the environment at the time UIOP was first loaded. It initially implements the syntax specified by the CLHS, but may also have implementation-specific extensions, and may grow user-defined extensions. You MUST ONLY make conservative extensions to it, -- GitLab From 11a2c8b520cadea0d91ed03091dba4584b5ba01e Mon Sep 17 00:00:00 2001 From: "Robert P. Goldman" Date: Tue, 5 Dec 2017 13:54:20 -0600 Subject: [PATCH 35/40] =?UTF-8?q?Comments=20with=20questions=20for=20Far?= =?UTF-8?q?=C3=A9.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comments so far limited to proposal 1. --- doc/syntax-control.md | 55 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/doc/syntax-control.md b/doc/syntax-control.md index 3b3aa088..12626888 100644 --- a/doc/syntax-control.md +++ b/doc/syntax-control.md @@ -118,6 +118,12 @@ This means that if some component side-effects the `*readtable*` variable and/or the state of the data structure it's bound to, the effects will be seen by all components compiled with ASDF, *including components that do not "depends-on" support functions* that the reader output produces. + + + + + + This causes catastrophic failure, when at the REPL you e.g. ``` @@ -198,9 +204,24 @@ either as part of the shared readtable or as part of private readtables: 4. Free software libraries will register these changes in a suitable page of cliki.net: < http://cliki.net/Macro%20Characters > + + + + + + + + + + + * Optionally, declare a moratorium on defining any further such extension, except as part of a [CDR](https://common-lisp.net/project/cdr/); let's call that option 1.1.1 (and no such option 1.1.0). + + + + * Have ASDF itself enforce that the same readtable will be used whenever building Lisp software, by defining a `uiop:*shared-readtable*` @@ -208,6 +229,15 @@ either as part of the shared readtable or as part of private readtables: allowing developers who want to keep using the above shared modifications and developers who prefer to use private readtables to work without interfering with each other anymore. + * Optionally, have a similar mechanism for the `*print-pprint-dispatch*`, for all variables modified by `with-standard-io-syntax`, and/or @@ -223,6 +253,12 @@ at which point changes to syntax variables can still backward-compatibly escape from one system to the other, and it remains the responsibility of the user to make sure that it happens in a deterministic fashion. + + + + + + My vote is for next version of ASDF (i.e. ASAP) to implement this proposal 1, with options 1.1.1 and 1.2.2 (and maybe later 1.2.3, if the code is ready). @@ -246,6 +282,10 @@ as opposed to when publishing free software libraries. But this approach is indeed possible, and we may want to keep supporting it, at least until we have better alternatives to offer. + + + + ### Details of the proposed implementation of Proposal 1 @@ -256,7 +296,18 @@ This readtable is presumably the This readtable comes with all the restrictions described above. UIOP also remembers a read-only `uiop:*standard-readtable*` with only the standard characters. -The two above variables are not mean to be re-bound after initialization. +The two above variables are not mean to be re-bound after +initialization. + + + + + + + + + + Last but not least, UIOP maintains a `uiop:*shared-readtable*`. Its default value is `uiop:*initial-readtable*` which is backward compatible. @@ -275,6 +326,8 @@ a `#!` ignorer sometimes, or importantly in CCL the `#$` and `#_` readers, etc. A strict reading of the CLHS might make this non-conformant, but many systems might rely on it, as the ITA codebase for QRes used to. + + Systems that want to modify a persistent readtable in ways not permitted with respect to the initial readtable *must* arrange to use their own private readtable, e.g. via `named-readtables`. -- GitLab From f44e8130c3259a54e2f786dcb212c23521ad0459 Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Wed, 6 Dec 2017 03:09:20 -0500 Subject: [PATCH 36/40] Another pass on syntax-control.md Clarifying and expanding the document based on feedback by RPG. --- doc/syntax-control.md | 585 ++++++++++++++++++++++++++++-------------- 1 file changed, 394 insertions(+), 191 deletions(-) diff --git a/doc/syntax-control.md b/doc/syntax-control.md index 12626888..a3dd9c7e 100644 --- a/doc/syntax-control.md +++ b/doc/syntax-control.md @@ -1,7 +1,7 @@ # Syntax Control This document describes an issue with using reader macros in Common Lisp, -and makes proposals as to how to handle this issue in ASDF. +and makes proposals as to how to handle this issue when using ASDF. https://bugs.launchpad.net/asdf/+bug/1293325 http://thread.gmane.org/gmane.lisp.asdf.devel/3883 @@ -17,7 +17,7 @@ yet may break the build in subtle way when the build is incremental and/or when part of a larger build that contains other systems with conflicting side-effects. (NB: The first variants of this document were written in 2014 while working on -ASDF 3.1, but its contents remain current as of ASDF 3.3.0 in October 2017.) +ASDF 3.1, but its contents remain current as of ASDF 3.3.1 in December 2017.) For instance, some CL libraries directly modify the current `*readtable*` by calling e.g. `set-dispatch-macro-character` without specifying a readtable. @@ -115,32 +115,39 @@ for direct toplevel modification of the shared `*readtable*`. Currently, all components compiled with ASDF promiscuously share whatever readtable is active at the time `asdf:operate` is called. This means that if some component side-effects the `*readtable*` variable -and/or the state of the data structure it's bound to, -the effects will be seen by all components compiled with ASDF, -*including components that do not "depends-on" support functions* that the reader output produces. - - - - - - -This causes catastrophic failure, when at the REPL you e.g. +and/or the state of the underlying data structure that variable is bound to, +the effects will be seen by all components compiled with ASDF. +Worse, if some reader macro expands into uses of symbols, functions and other entities, +these entities will now be used in the fasls for components +that do not "depends-on" the systems in which those entities are defined. +All further attempts to load those fasls in a fresh image will then be catastrophic failures, +with no easy solution beside clearing the fasl cache +(for instance with `rm -rf ~/.cache/common-lisp/' from a shell outside Lisp). + +As a case that happened to me, I was using my `fare-quasiquote` syntax so I may portably +use quasiquotation in match expressions (with `fare-matcher`, `optima` or now `trivia`). +Then for some reason I had to modify some software that didn't use `fare-quasiquote`. ``` (asdf:load-system :fare-quasiquote) (named-readtables:in-readtable :fare-quasiquote) -;; Now modify a file in any system depends on, e.g. ASDF itself. +;; modify a file in any system that does not depend on fare-quasiquote, e.g. ASDF itself. (uiop:run-program "touch somefile-that-in-the-system-below.lisp") (asdf:load-system :some-modified-system-that-doesnt-use-fare-quasiquote) ;; example: ASDF ``` -Then your system will be recompiled using `fare-quasiquote` for its backquote implementation, -except that since it doesn't depends-on `fare-quasiquote`, -the `fare-quasiquote` runtime will not be there next time you load the file from scratch. -Your fasl cache is therefore corrupted, and -there is nothing `fare-quasiquote` can do to protect users. -This is not specific to `fare-quasiquote`, but applies to anyone who uses any private readtable -with any discrepancy from the shared readtable concerning any macro character. +My system was recompiled using `fare-quasiquote` for its backquote implementation. +Backquotes were thus expanded into uses of +`fare-quasiquote::list`, `fare-quasiquote::cons`, `fare-quasiquote::append`, `fare-quasiquote::quote` +and other such symbols that shadow the equivalent in package `common-lisp`. +However, since my system didn't depends-on `fare-quasiquote`, +the `fare-quasiquote` runtime will not be there next time you load the file in a fresh Lisp image. +(And in the case of ASDF itself being recompiled this way, +there is nothing left working to fix the mess from Lisp.) +Your fasl cache is therefore corrupted. +What more, there is nothing `fare-quasiquote` and its users can do to protect end-users. +And this is not specific to `fare-quasiquote`, but applies to anyone who uses any private readtable +with any discrepancy whatsoever from the shared readtable concerning any macro character. To protect users who use anything but the shared readtable at the REPL, ASDF should make sure this shared readtable is used whenever building software. @@ -149,143 +156,307 @@ ASDF should make sure this shared readtable is used whenever building software. ## Statu quo: Readtable discipline The _de facto_ though tacit requirements for all Common Lisp software currently -is therefore to follow a double discipline, +is to follow a double discipline, without which software may fail to build in mysterious ways: - - * Developers must *never* introduce conflicts with *any* known extension when - modifying the `*readtable*` as inherited from the initial environment. - This means that no one may ever modify `#u` except `puri`, - or else become incompatible with `puri` and prevent the two systems - from ever being used together. Similarly, no one may modify `#$` except CCL - or else become incompatible with CCL. - Unhappily, there is no global registry of who claims what extension, - which makes defining any extension a gamble that jeopardizes the build. - There is thus a chilling effect whereby - prudent programmers never define extensions to the current `*readtable*`; - and there is thus a counter-selective effect whereby - only reckless programmers do define such extensions. - - * Developers must *never* call ASDF while the current `*readtable*` is bound - to any readtable but the initial, shared readtable. - If they do, their readtable may either be corrupted, - or cause the build to be corrupted, or both. - But there is currently no good means to access for certain this readtable, - so no program may call ASDF at runtime while such readtable might be active; - and no program may change the `*readtable*` while calling systems - that might themselves call ASDF. - This once again is a chilling effect against any use readtables other than - the initial `*readtable*` and against any use of ASDF at runtime, - and a counter-selection in who will use those features. +First, users must only make monotonic modifications to the shared readtable, and +use private readtables for any modification that cannot be guaranteed monotonic. +Second, users must manually ensure that whenever ASDF is invoked, +or might otherwise be invoked by a library, +then the current `*readtable*` is bound to the shared `*readtable*`. +These requirements are detailed in the respective subsections below. + +Once again, these requirements already exist. +This essay only documents them when they were previously unsurfaced; +this might be another case of Lispers discovering a law of computing, +where designers and practitioners of other languages are often +under the delusion that they are in control of arbitrarily "making" the laws. +Interestingly though, these requirements apply to syntax extension through +reader-macros in a way that they don't apply to syntax extension through +regular macros: because regular macros are named by *symbols*, +that already have their own naming and packaging mechanisms and conventions +to avoid conflicts, in a way that is more common and more familiar +to Lispers as well as to practitioners of other languages. + + +### Shared Readtable Monotonicity + +The first discipline to follow is that in their Lisp source code, +users must *never* introduce conflict with *any* known extension when +modifying the `*readtable*` as inherited from the initial environment. +They may only add new behaviors that do not conflict +on characters so far undefined, +and may never override a previous behavior defined or used +by either the implementation or any program or extension. +Indeed, whatever previous behavior that a previous program may have relied upon, + +it will rely upon again when their are updated and rebuilt; +and any conflict means that the build will then fail. +Therefore: + + 1. No one is allowed to modify any standard character (as listed in the CLHS) + in the *shared readtable* they inherited as current `*readtable*` + at the start of compiling or loading a file. + + 2. No two systems loaded in the same image may assign different meanings + to a same character in this *shared readtable*. + (This is not presently checkable, but see Proposal 4 below). + + 3. Using a non-standard character defined by the implementation + such as `#$` in CCL, or any unquoted non-ASCII character in an symbol, + while permitted, counts as "assigning meaning" to that character, + at which point no other system in this image is allowed + to override this meaning with a conflicting definition. + + 4. All readtable modifications that do not, cannot or will not abide by + the above restrictions must happen in a *private readtable*: + a new readtable object, initially obtained via `copy-readtable`, + saved in its own variable, to which `*readtable*` is dynamically bound + around evaluations that use its incompatible syntax. + The shared readtable must be restored outside of these evaluations, + or be itself dynamically bound around sub-evaluations + that use the shared syntax. + +These restrictions are largely undocumented as a theoretical requirement, +yet mostly respected as a practical necessity: +users who try to make or use some piece of software that fails to respect them +soon enough discover the hard way that that software fails to interoperate +nicely with other software, at which point they fix it, or they work around it, +but in any case they avoid the conflict. +Yet, as of 2017, no implementation can nor will help enforce monotonicity; +users are wholly responsible for manually ensuring this monotonicity. + +Thus for instance, no one may ever modify `#u` except `puri`, +or else they become incompatible with `puri` and +the two systems from ever being used together. +Ideally, `puri` and all its clients should be amended +to use a private readtable instead — but that's already beyond the status quo. +Similarly, no one may modify `#$` except CCL or else +become incompatible with CCL and systems that rely on that CCL extension. + +Unhappily, there is no global registry of who claims what extension, +which makes defining any extension a gamble that jeopardizes the build. +There is thus a chilling effect whereby +prudent programmers never define extensions to the current `*readtable*`; +and there is thus a counter-selective effect whereby +only reckless programmers do define such extensions. + + +### Shared Reatable Restoration + +The second discipline that users must follow as they invoke ASDF is +to *never* call ASDF while the current `*readtable*` is bound +to any readtable but the initial, shared readtable. +If they do, either their current readtable may be corrupted, +or it cause some other part of the build to be corrupted, or both. + +However, there is currently no good means to access for certain +this priveleged "shared initial readtable", except by supposing that +it is the readtable that was current at the start of the file, +and by extending to authors of subsequent files the courtesy +of passing by the same readtable. +In the case of the readtable, this convention is happily facilitated +by the fact that `cl:load` and `cl:compile-file` +will both bind `*readtable*` around the execution of their bodies. + +Now, this facilitation is not available in the case of other syntax variables +that may disrupt the build, such as `*read-base*`, `*read-default-float-format*` +or `*read-eval*`, that users must also take care to bind to their default value +around the invocation of ASDF, as part of the same discipline. + +All in all, no program may call ASDF at runtime while any readtable but +the shared readtable might be active. +No program may even indirectly call any other program that might, +unless it or that other program ensures that is the case. +The only current way to safely invoke ASDF when the `*readtable*` is or may be +bound to anything but the initial readtable is to must explicitly +save that initial readtable to some variable and bind `*readtable*` back to it +around invocations of ASDF. + +This restriction is also valid at the REPL. If for any reason +you change the `*readtable*` or any other syntax variable while at your REPL, +then you must not invoke ASDF or any program that might invoke ASDF +unless you first restore the default bindings for this +and all other modified syntax variables. + +This once again is a chilling effect against any use readtables other than +the initial `*readtable*` and against any use of ASDF at runtime; +it is also a counter-selection in who will use those features. ## Proposal 1: Minimal Syntax Control -This first proposal is for immediate implementation in the next release of ASDF -(whichever it may be, which would be ASDF 3.3.1 if done in October 2017). -It would offer a sane way for disciplined programmers to define reader macros -either as part of the shared readtable or as part of private readtables: - - * Start by merely documenting the restrictions that apply to modifying - the implicitly shared initial readtable, and mandating that - modifications be themselves documented: - - 1. No one is allowed to modify any standard character in this readtable, - in any way whatsoever. - - 2. No two systems loaded in the same image may assign different meanings - to a same character in the *shared readtable*. - (This is not presently checkable, but see Proposal 4 below). - - 3. Systems need to document any change to the *shared readtable*. - Note that unqualified readtable modifications in the program will modify - the object to which `*readtable*` is bound, which unless rebound - will also be the *shared readtable*. - - 4. Free software libraries will register these changes in - a suitable page of cliki.net: < http://cliki.net/Macro%20Characters > - - - - - - - - - - - +This first proposal is to make some minimal changes that would +maximally improve the situation given the small size of the changes. +This proposal is meant be adopted immediately in the next release of ASDF +(whichever it may be, which would be ASDF 3.3.2 if done in early 2018). +It independently amends each of the above two requirements: +The first requirement is amended by making monotonicity an official policy, +and by requiring that users should in turn explicitly document +the ways that their programs define or use syntax extensions. +The second requirement is amended by having ASDF itself +save the initial readtable and bind the `*readtable*` to it around evaluation, +so users of private readtables don't have to worry about it. + + +### Documenting Monotonicity + +From now on, monotonicity of syntax extensions +is not just a de facto requirement, but an official policy. +Users *must* document what extensions their software makes, +and what extensions it uses. +Enforcing monotonicity remains a matter of discipline, +not enforced by software but by social convention. +But at least it is well-documented: + + * Systems need to document any change to the *shared readtable* + that they either make or rely upon. + Note that unqualified readtable modifications in the program + will modify the object to which `*readtable*` is bound, + which unless explicitly re-bound will also be the *shared readtable*. + + * Free software libraries will register these changes in + a suitable page of cliki.net: < http://cliki.net/Macro%20Characters > * Optionally, declare a moratorium on defining any further such extension, except as part of a [CDR](https://common-lisp.net/project/cdr/); let's call that option 1.1.1 (and no such option 1.1.0). - - - - - - * Have ASDF itself enforce that the same readtable will be used - whenever building Lisp software, by defining a `uiop:*shared-readtable*` - that will be shared by all CL code by binding it around `asdf:operate`, thus - allowing developers who want to keep using the above shared modifications - and developers who prefer to use private readtables - to work without interfering with each other anymore. - - - * Optionally, have a similar mechanism for the `*print-pprint-dispatch*`, - for all variables modified by `with-standard-io-syntax`, and/or - for a user-extensible set of variables; - let's call these options 1.2.1, 1.2.2 and 1.2.3 respectively - (and adopting none of them 1.2.0). - -With this minimal change in place, there is a viable story going forward -for how to use reader macros in Common Lisp, and -one that maintains backward compatibility. -Note that for a minimal change, the bindings should only go around `operate`, + This means there must be stronger social control as to what is or isn't + acceptable in terms of syntax extension to the shared readtable. + However, since there currently exist no means of technical enforcement, + this social control may prove inefficient or ineffective; + and if the above declaration isn't backed by a strong consensus + and accompanied by actual social control, then indeed the declaration + will only have brought confusion, dissension, and + a false sense of security in those who believe it is more than it really is. + Then again, the battle can't be won if it is not fought. + This option should probably be pursued as an add-on + that depends on Proposal 1 but that Proposal 1 as such does not depend on. + + * Systems must also document any use of private readtables + that they require from their clients or provide to their clients. + +To compensate the heightened burden that this social solution imposes +on users or the shared syntax, the second part of this proposals +alleviates the burden on use of private syntax. + + +### Supporting Private Syntax + +ASDF can itself handle shared readtable restoration that so far +every direct and indirect ASDF user has to worry about, +replacing a lot of social convention with a litte bit of code. +This change will enable safe interoperation of ASDF with +code that uses private readtables without placing the high burden +that currently lies upon them. +This change will crucially support users of such private readtables. + +As before, users of incompatible syntax must locally bind the `*readtable*` +to their private readtable object. For that, they may use e.g. +`named-readtables:in-readtable` or `cl-syntax:use-syntax`. +Furthermore, their build files must be such that any non-standard binding +of the `*readtable*` and other syntax variables +(such as `*read-default-float-format*`) shall only apply +to the dynamic scope of building relevant files +(e.g. via `:around-compile-hook`, or else `:around` methods to perform). +Note that `*readtable*` is locally bound by `cl:load` already, +so you may safely use `named-readtables:in-readtable` within a file; +such however is not the case for other syntax variables, +so they still require special care. + +Now, ASDF itself will enforce the binding of `*readtable*` +to a canonical shared readtable object around its building of Lisp software: +UIOP will define a variable `uiop:*shared-readtable*` that will at any moment +be bound to this special readtable object. +Then, ASDF will bind `*readtable*` to the value of `uiop:*shared-readtable*` +in an `:around` method to `asdf:operate`, so that all code built by ASDF +will reliably be using this shared readtable object +— except of course that code that itself explicitly and locally binds +`*readtable*` to a private readtable object, which it still very much supported. +Thus, developers use monotonic modifications to "the" shared readtable, and +developers who use private readtables (whether by preference or necessity) +can keep working together without interfering with each other anymore. + +Importantly, as long as they keep abiding by the restrictions of the statu quo, +developers do not have to be aware of this change, +or of the existence of `uiop:*shared-readtable*`: +things "just work", as ASDF takes care of business. +However, users who explicitly *want* to juggle with multiple syntaxes +can now rely on `uiop:*shared-readtable*` instead of current situation: +currently, each of these users must reinvent their own variable +to stash "the" initial syntax; and they don't even have a documented convention +or any technical or social guarantee that this will work. + +Of course, ASDF could *further* manage a larger set of syntax variables, +and bind them to their default value around the building of Lisp code. +Optionally, the same mechanism of binding syntax variables in `asdf:operate` +could be extended beyond `*readtable*` only (which would be option 1.2.0), +to `*print-pprint-dispatch*` (option 1.2.1), +to all variables bound by `with-standard-io-syntax` (option 1.2.2), and/or +to a user-extensible set of variables (option 1.2.3). +Also, instead of binding variables only around `asdf:operate` (option 1.3.0), +the variables could be bound by UIOP around all invocations of +`uiop:compile-file*` and `uiop:load*` (option 1.3.1); +one problem with this latter option, though, is that it might be incompatible +with how some systems currently override these variables in `:around` methods, +so requires some migration, and is not probably applicable as an immediate change. +The `:around-compile` mechanism of `uiop:compile-file*`, +that might have to be adapted for `uiop:load*`, +is still available for such overrides — but we must make sure that users use them. +As these options may make this change not minimal, +they should probably be deferred as part of Proposal 2 below, if adopted at all. + +### Proposal 1, NOW. + +With the minimal changes of Proposal 1 in place, there will be a viable story +going forward for how to define and use reader macros in Common Lisp; +furthermore that story will maintain backward compatibility. + +For a truly minimal change, +the bindings should only go around `operate` (option 1.3.0), at which point changes to syntax variables can still backward-compatibly escape from one system to the other, and it remains the responsibility of the user -to make sure that it happens in a deterministic fashion. - - - - - - +to make sure that it happens in a deterministic fashion, +e.g. through use of proper dependencies and +`perform` methods on `prepare-op` for some systems. -My vote is for next version of ASDF (i.e. ASAP) to implement this proposal 1, -with options 1.1.1 and 1.2.2 (and maybe later 1.2.3, if the code is ready). +My vote is for next version of ASDF (i.e. ASAP) to immediately implement +this proposal 1, with options 1.1.0, 1.2.2 (or else 1.2.0) and 1.3.0. +I further propose that option 1.1.1 be promoted in a separate campaign. +And option 1.2.3 or maybe even Proposal 2 below should be strongly considered +for a future release of ASDF (e.g. 3.5 or such). ### Some hacks still supported under Proposal 1 -One notably way to modify the syntax in a deterministic way that works with the -current ASDF 3.3.0 and would keep working is to serialize all dependencies -through a syntax-modification system: a system called e.g. `my-modified-syntax` +One notably way to modify the syntax in a deterministic way +that works with the current ASDF 3.3.1 and would keep working under Proposal 1 +is to serialize all dependencies through a syntax-modification system: +a system called e.g. `my-modified-syntax` depends on all the regular-syntaxed software dependencies in the project, before it itself modifies syntax variables; all other systems in the project will then depend-on `my-modified-syntax`, forcing the order of evaluation of the syntactic side-effects. The above setup ensures that all regular CL files are build with regular syntax, and that all files that depend on the modified syntax see that modified syntax. + Proposal 1 ensures that the latter modifications do not cause non-deterministic build issues when one of the regular systems is modified while the latter modifications are active. -Note that this approach is only possible in a closed system, +Note that "modifying syntax variables" must be done in a robust way, +i.e. in a `perform :before` method, or it may be undone when +`*readtable*` is bound again. +At that point, though, you might as well use a `:around` method instead, +and make the modifications local. + +In any case, this approach is only possible in a closed system, as opposed to when publishing free software libraries. +For instance, it applies if you write a proprietary software application +that isn't meant to be depended on by anything outside your own ecosystem. But this approach is indeed possible, and we may want to keep supporting it, at least until we have better alternatives to offer. +Or we may want to deprecate it, and +document how users may do things cleanly instead. - - - - ### Details of the proposed implementation of Proposal 1 @@ -294,42 +465,48 @@ the current value of `cl:*readtable*` when UIOP was first loaded. This readtable is presumably the [initial readtable](http://clhs.lisp.se/Body/26_glo_i.htm#initial_readtable). This readtable comes with all the restrictions described above. -UIOP also remembers a read-only `uiop:*standard-readtable*` -with only the standard characters. -The two above variables are not mean to be re-bound after -initialization. - - - - - - - - - - - -Last but not least, UIOP maintains a `uiop:*shared-readtable*`. -Its default value is `uiop:*initial-readtable*` which is backward compatible. -The variable can instead be re-bound to `uiop:*standard-readtable*` -for enforcement against uncontrolled readtable modifications, -or to a value of `(copy-readtable nil)` -(which should be a modifiable copy of the standard-readtable, +UIOP also remembers a `uiop:*standard-readtable*`, +with only the standard characters, that must be treated as read-only +(though that is only enforced on SBCL, ECL, Allegro, CMUCL as of December 2017; +see Proposal 3 below). +Neither of these two variables is meant to be re-bound after initialization. + +Last but not least, UIOP maintains a `uiop:*shared-readtable*`, +that will be used when compiling using ASDF. +Its default value is the same as that of `uiop:*initial-readtable*`, +for backward compatibility: +Using the `uiop:*standard-readtable*` would break backward compatibility +with the statu quo because that table is read-only +whether that restriction is enforced by the implementation or not +(at which point *bad things* can happen). +Even using a mutable copy of the standard readtable with `(copy-readtable nil)` +is not backwards compatible, because several implementations provide extensions +in their *initial readtable* that are *not* part of the *standard readtable*, +such as a `#!` ignorer sometimes, or notably in CCL the `#$` and `#_` readers. +A strict reading of the CLHS might make this non-conformant, but +some widely used systems in Quicklisp do rely on it. +These systems may have to be fixed before any change in default happens. +Desirable or not and welcome by the community or not, +a change that breaks backward compatibility defeats +the immediate applicability of Proposal 1. + +Now, though that must not be the default, +the `*shared-readtable*` variable can be usefully re-bound +to `uiop:*standard-readtable*` for enforcement +against uncontrolled readtable modifications. +It can be re-bound to a value of `(copy-readtable nil)`, +which should be a modifiable copy of the standard-readtable, for code that modifies the current readtable yet doesn't trust what was "initially" current when UIOP was loaded -and/or doesn't want to use implementation-dependent modifications), -or something else, so users may decide what shared readtable enforcement +and/or doesn't want to use implementation-dependent modifications. +It can be re-bound to some readtable that somehow provides an instrumented +Common Lisp in Common Lisp implementation. +Or it can be re-bound to something else, so +users may decide what shared readtable enforcement they do or don't want in their builds. -Several implementations provide extensions in the *initial readtable* -that are *not* part of the *standard readtable*, such as -a `#!` ignorer sometimes, or importantly in CCL the `#$` and `#_` readers, etc. -A strict reading of the CLHS might make this non-conformant, but -many systems might rely on it, as the ITA codebase for QRes used to. - - -Systems that want to modify a persistent readtable -in ways not permitted with respect to the initial readtable +All in all, systems that want to modify a persistent readtable +in ways not permitted with respect to the initial shared readtable *must* arrange to use their own private readtable, e.g. via `named-readtables`. Thanks to the re-binding of `*readtable*` by ASDF under Proposal 1 (and later), they can now do it safely and not cause a build corruption if that readtable @@ -348,16 +525,15 @@ In exchange for this complexity, however, you can achieve both determinism and flexibility at the same time. I (Faré) had code to do a lot of this proposal in the `syntax-control` branch, -but I believe it is not actually ready for release yet, -so I'm removing it. +but I believe it was not actually ready for release yet, so I removed it. I do not recommend this approach at this point, but I do describe it here for comparison purposes and because of the light it sheds on the issue and the rationale for solving it even in a minimal way. With per-system syntax variables, it is possible for the `*readtable*` as well as all other variables (such as `*print-pprint-dispatch*`) -to be specified as in a way better defined -than with using the promiscuous `*shared-readtable*`. +to be specified in a way better defined +than by using the promiscuous `*shared-readtable*`. This makes it possible for systems to fully take advantage of syntax extension, without either interfering with other systems or suffering from interference by these other systems, and without @@ -366,8 +542,10 @@ painfully doing a `(named-readtables:in-readtable :foo)` in every file. * ASDF tracks a set of "syntax variables". * For each variable in the set, and for each system, - ASDF, tracks the "entry value" and "exit value" of the variable + ASDF tracks the "entry value" and "exit value" of the variable at the beginning and end of building the system. + In this, Proposal 2 differs from Proposal 1 option 1.2.3, + that always binds each variable to the same constant global default value. * ASDF maintains a partial order on these systems based on dependencies. For every considered system, identify the set of "maximal" dependencies, @@ -378,9 +556,9 @@ painfully doing a `(named-readtables:in-readtable :foo)` in every file. an explicit entry value for the variable, or all the maximal dependencies of that system must agree on their exit value (or else raise an error), at which point that value becomes the system's entry value of the variable. - This applies to the readtable as to other values of syntax variables. + This applies to the `*readtable*` as well as to all other syntax variables. (Alternatively (which I don't recommend) always inherit exit values - from the last (or the first) of the explicit dependencies as entry values + from the last (or the first) of the *explicit* dependencies as entry values for the system.) * After a system is loaded, record all the exit values @@ -392,12 +570,15 @@ painfully doing a `(named-readtables:in-readtable :foo)` in every file. or by a symbol that designates a `named-readtables` readtable. * Since `*readtable*` is bound by `load` and `compile-file`, - there needs be a separate mechanism for specifying - exit values of readtable, for instance using `named-readtables` symbols. + there needs be a separate mechanism for specifying the exit values + of `*readtable*`, for instance in `perform` methods, + or using the special variable `*shared-readtable*` + as the communication channel. * A given readtable may be modified by further systems, according to rules essentially similar to those that apply to the initial readtable: no conflict is allowed. + Except it's only conflict between users of that private readtable. An extension that allows for read-only readtables would be welcome, too, so these user-defined readtables too can be protected from corruption. @@ -410,7 +591,6 @@ painfully doing a `(named-readtables:in-readtable :foo)` in every file. * Conceivably, `operate` could do the former, and `load-system` the latter. Or the two options could otherwise both be available depending on a flag. - Does that strike you as complex? Because it is. That's the price of *safely* supporting arbitrary modifications to syntax variables in a deterministic yet general way. @@ -420,9 +600,10 @@ to syntax variables in a deterministic yet general way. This third proposal forbids any modification to the current `*readtable*`. I made this proposal in earnest in 2014, but it was justifiedly met with horror -and a big pushback from the community: on the one hand, it is not -backward compatible and causes a lot of legacy software to fail to build; -on the other hand, most Common Lisp implementations offer little protection +and a big pushback from a significant part of the community: +on the one hand, it is not backward compatible and +causes a number of legacy systems to fail to build; +on the other hand, most Common Lisp implementations currently offer little protection in case this proposal is in place and someone makes a mistake. Under this proposal, all Lisp libraries are read and compiled while @@ -430,34 +611,55 @@ the current `*readtable*` is bound to the *standard* readtable, as made available via `with-standard-io-syntax` (see [CLHS 2.1.1.2](http://clhs.lisp.se/Body/02_aab.htm) and [CLHS glossary for "standard readtable"](http://clhs.lisp.se/Body/26_glo_s.htm#standard_readtable)). -This readtable is notionally read-only as per the standard; -however only SBCL is actually enforcing that as of 2017, -such that you will have a nice clean error immediately, -instead of horrible silent system corruption later. -But this SBCL behavior is enough to detect systems that cause this issue and fix them. +This readtable is notionally read-only as per the standard. +However, as of December 2017, only SBCL, Allegro, ECL and CMUCL +actually reject modifications to the standard readtable and +give you a nice clean error immediately. +On at least CCL, CLISP, ABCL, Lispworks and MKCL, +instead will silently let you corrupt the standard readtable, +which may result in horrible failures later +when you use `with-standard-io-syntax` and it doesn't do what you expect. +Happily, those implementations that do enforce read-only behavior +may be enough to detect systems that cause this issue and fix them; +but that will still be painful. + +You may test your current implementation +by evaluating the following form in a throw-away image: +``` +(with-standard-io-syntax (set-macro-character #\$ (constantly 42)) (read-from-string "$")) +``` +If it returns `42`, +then your implementation just let you corrupt the standard readtable +(so don't do it in an image that matters). +If it returns an error object, +then your implementation did enforce the standard readtable being read-only. -This change however is clearly backward-incompatible, +Now this Proposal 3 is clearly backward-incompatible, and will break libraries that currently modify the current `*readtable*`. -These include unmaintained libraries this feature of which is actually used -by other libraries, as can be found in Quicklisp. +These actually include unmaintained libraries that will be a pain to fix, +yet are themselves used by other libraries, as can be found in Quicklisp. Fixing all these libraries will take time. Yet without all those fixes, this proposal is a non-starter. And even after those fixes, commercial users with non-visible code may legitimately object to this change. -Using proposal 1, you can easily opt in this proposal 3 by setting +Using Proposal 1, you can easily opt in this proposal 3 by setting the `uiop:*shared-readtable*` to `uiop:*standard-readtable*` just after you load ASDF and before you load anything else. -Even without proposal 1, you can achieve this effect -by having you entire build inside a `with-standard-io-syntax`, +Even without Proposal 1, you can achieve this effect +by having your entire build inside a `with-standard-io-syntax`, or by inserting a `(setf *readtable* (with-standard-io-syntax *readtable*))` before your build (or at an appropriate point in your build). -I (Faré) still think it is a good idea to mandate that at least public libraries -in Quicklisp should be fixed to never side-effect the current `*readtable*`. -But I acknowledge that the Common Lisp ecosystem is not ready for such a change -at this point; and since I'm not a paid Common Lisp professional at this point, -I don't have the energy to push for such a change by getting all libraries fixed. +I (Faré) still think it is a good idea to mandate that +at least public libraries in Quicklisp should be fixed +to never side-effect the current `*readtable*`. +But I acknowledge that the Common Lisp community +is not ready for such a change at this point; +and since I haven't myself been a paid Common Lisp professional for years, +and have jumped ship to Gerbil Scheme, I don't persaonally have the energy +to push for such a change by getting all libraries fixed. +But maybe some future maintainer will read this and make it their mission. ## Proposal 4: Strict Enforcement modulo Declaration. @@ -465,11 +667,12 @@ I don't have the energy to push for such a change by getting all libraries fixed Proposal 4 is more elaborate, and would require modifications of a least some implementations' support for readtables: systems would declare as part of their `defsystem` declarations -which system defines (or uses?) which macro characters. +what reader macros they use or define. The implementation for readtables (or some semi-portable wrapper around them) could then enforce those declarations. I am not going to push for this proposal at this point, -but in case someone ever wants to implement a complete solution, here is a sketch. +but in case someone ever wants to implement a complete solution, +here is a sketch. * Unhappily, there is no cheap way to enforce these restrictions, but they can be achieved by writing suitable wrappers -- GitLab From 3aba519f33f4ccd83c765b4cbd4290425b217cfa Mon Sep 17 00:00:00 2001 From: "Robert P. Goldman" Date: Fri, 5 Jan 2018 18:45:31 -0600 Subject: [PATCH 37/40] Add "Terminology" subsection and fix some minor typos. Tried to capture the different kinds of syntax we might refer to in the rest of the document. --- doc/syntax-control.md | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/doc/syntax-control.md b/doc/syntax-control.md index a3dd9c7e..7d53994b 100644 --- a/doc/syntax-control.md +++ b/doc/syntax-control.md @@ -153,7 +153,7 @@ To protect users who use anything but the shared readtable at the REPL, ASDF should make sure this shared readtable is used whenever building software. -## Statu quo: Readtable discipline +## Status quo: Readtable discipline The _de facto_ though tacit requirements for all Common Lisp software currently is to follow a double discipline, @@ -241,7 +241,7 @@ and there is thus a counter-selective effect whereby only reckless programmers do define such extensions. -### Shared Reatable Restoration +### Shared Readtable Restoration The second discipline that users must follow as they invoke ASDF is to *never* call ASDF while the current `*readtable*` is bound @@ -297,6 +297,25 @@ The second requirement is amended by having ASDF itself save the initial readtable and bind the `*readtable*` to it around evaluation, so users of private readtables don't have to worry about it. +### Terminology + +* Standard readtable/standard syntax: defined by the ANSI spec as + follows: "standard syntax n. the syntax represented by the standard + readtable and used as a reference syntax throughout this document." + +* Initial syntax: the syntax in place when UIOP and ASDF are loaded. + This *may* be the same as the standard syntax, but is not + necessarily. + +* Private syntax: a syntax intended for use by a system, or a coherent + set of systems. Note that while there is a single standard syntax, + initial syntax, and shared syntax, there may be multiple private + syntaxes. + +* Shared syntax: the syntax that will be shared by ASDF across the + systems it is loading. The shared syntax will be initialized to the + initial syntax, rather than standard syntax, in order to preserve + backwards compatibility. ### Documenting Monotonicity @@ -376,7 +395,7 @@ Thus, developers use monotonic modifications to "the" shared readtable, and developers who use private readtables (whether by preference or necessity) can keep working together without interfering with each other anymore. -Importantly, as long as they keep abiding by the restrictions of the statu quo, +Importantly, as long as they keep abiding by the restrictions of the status quo, developers do not have to be aware of this change, or of the existence of `uiop:*shared-readtable*`: things "just work", as ASDF takes care of business. @@ -428,15 +447,16 @@ for a future release of ASDF (e.g. 3.5 or such). ### Some hacks still supported under Proposal 1 -One notably way to modify the syntax in a deterministic way -that works with the current ASDF 3.3.1 and would keep working under Proposal 1 +One way to modify the syntax in a deterministic way +that works with the current ASDF 3.3.1 and will keep working under Proposal 1 is to serialize all dependencies through a syntax-modification system: a system called e.g. `my-modified-syntax` -depends on all the regular-syntaxed software dependencies in the project, -before it itself modifies syntax variables; -all other systems in the project will then depend-on `my-modified-syntax`, +depends on all the software dependencies in the project that must be +built with standard syntax, +before it itself modifies syntax variables. +All the systems in the project that use the modified syntax will then depend-on `my-modified-syntax`, forcing the order of evaluation of the syntactic side-effects. -The above setup ensures that all regular CL files are build with regular syntax, +The above setup ensures that all regular CL files are built with standard syntax, and that all files that depend on the modified syntax see that modified syntax. Proposal 1 ensures that the latter modifications do not cause non-deterministic @@ -476,7 +496,7 @@ that will be used when compiling using ASDF. Its default value is the same as that of `uiop:*initial-readtable*`, for backward compatibility: Using the `uiop:*standard-readtable*` would break backward compatibility -with the statu quo because that table is read-only +with the status quo because that table is read-only whether that restriction is enforced by the implementation or not (at which point *bad things* can happen). Even using a mutable copy of the standard readtable with `(copy-readtable nil)` -- GitLab From d3958f1d95718be0f6a6478e31d769fb8c779acd Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Sat, 6 Jan 2018 04:07:42 -0500 Subject: [PATCH 38/40] More clarifications for syntax-control. --- doc/syntax-control.md | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/doc/syntax-control.md b/doc/syntax-control.md index 7d53994b..e368eb07 100644 --- a/doc/syntax-control.md +++ b/doc/syntax-control.md @@ -302,20 +302,31 @@ so users of private readtables don't have to worry about it. * Standard readtable/standard syntax: defined by the ANSI spec as follows: "standard syntax n. the syntax represented by the standard readtable and used as a reference syntax throughout this document." + The standard way to access the standard syntax is with + `with-standard-io-syntax`. It is *always* an error to try to modify + that readtable, but, as of January 2018, only a few implementations + (SBCL, ECL, Allegro, CMUCL) will protect you and make the table read-only. + You can safely copy it with `copy-readtable` and modify a copy. * Initial syntax: the syntax in place when UIOP and ASDF are loaded. - This *may* be the same as the standard syntax, but is not - necessarily. + This *may* be the same as a copy of the standard syntax, and cannot + conflict with it; but on some implementations (e.g. CCL) it legitimately + contains some extensions. It is always safe to modify it, but + only in monotonic ways that never interfere with previous behavior. * Private syntax: a syntax intended for use by a system, or a coherent - set of systems. Note that while there is a single standard syntax, - initial syntax, and shared syntax, there may be multiple private - syntaxes. + set of systems. Note that while there is a single (constant, read-only) + standard syntax, and single (constant identity, modifiable) initial syntax, + and a single shared syntax (specially bound, within constraints), + there may be multiple private syntaxes (e.g. using `named-readtables`). * Shared syntax: the syntax that will be shared by ASDF across the systems it is loading. The shared syntax will be initialized to the initial syntax, rather than standard syntax, in order to preserve - backwards compatibility. + backwards compatibility. System integrators may achieve interesting effects + by binding it to their own syntax, but regular libraries must avoid + changing it to remain modular and play nice with other libraries. + ### Documenting Monotonicity @@ -453,12 +464,17 @@ is to serialize all dependencies through a syntax-modification system: a system called e.g. `my-modified-syntax` depends on all the software dependencies in the project that must be built with standard syntax, -before it itself modifies syntax variables. +before it itself modifies syntax variables in non-monotonic ways. All the systems in the project that use the modified syntax will then depend-on `my-modified-syntax`, forcing the order of evaluation of the syntactic side-effects. The above setup ensures that all regular CL files are built with standard syntax, and that all files that depend on the modified syntax see that modified syntax. +Note that if there are any non-standard changes in the modified syntax +that would interfere with building the base libraries (e.g. use of `fare-quasiquote`), +then you *must not* ever rebuild the base libraries from an image with the modified syntax, +but instead rebuild them from scratch. + Proposal 1 ensures that the latter modifications do not cause non-deterministic build issues when one of the regular systems is modified while the latter modifications are active. -- GitLab From 7cd5e0f57edc450a78d087637d2dd8388b84f94d Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Fri, 4 May 2018 15:42:47 -0400 Subject: [PATCH 39/40] Defer deprecation warnings to 3.5. --- backward-interface.lisp | 4 ++-- backward-internals.lisp | 2 +- uiop/backward-driver.lisp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backward-interface.lisp b/backward-interface.lisp index 7eb94fd7..97929192 100644 --- a/backward-interface.lisp +++ b/backward-interface.lisp @@ -23,7 +23,7 @@ ;; NB: the warning status of these functions may have to be distinguished later, ;; as some get removed faster than the others in client code. -(with-asdf-deprecation (:style-warning "3.2" :warning "3.4") +(with-asdf-deprecation (:style-warning "3.2" :warning "3.5") ;; These conditions from ASDF 1 and 2 are used by many packages in Quicklisp; ;; but ASDF3 replaced them with somewhat different variants of uiop:compile-condition @@ -211,7 +211,7 @@ DEPRECATED. Use ASDF:ACTION-DESCRIPTION and/or ASDF::FORMAT-ACTION instead.")) (defmethod explain ((o operation) (c component)) (asdf-message (compatfmt "~&~@<; ~@;~A~:>~%") (action-description o c)))) -(with-asdf-deprecation (:style-warning "3.3") +(with-asdf-deprecation (:style-warning "3.3" :warning "3.5") (defun system-registered-p (name) "DEPRECATED. Return a generalized boolean that is true if a system of given NAME was registered already. NAME is a system designator, to be normalized by COERCE-NAME. diff --git a/backward-internals.lisp b/backward-internals.lisp index b5960c0e..6400ef97 100644 --- a/backward-internals.lisp +++ b/backward-internals.lisp @@ -7,7 +7,7 @@ (:export #:load-sysdef)) (in-package :asdf/backward-internals) -(with-asdf-deprecation (:style-warning "3.2" :warning "3.4") +(with-asdf-deprecation (:style-warning "3.2" :warning "3.5") (defun load-sysdef (name pathname) (declare (ignore name pathname)) ;; Needed for backward compatibility with swank-asdf from SLIME 2015-12-01 or older. diff --git a/uiop/backward-driver.lisp b/uiop/backward-driver.lisp index 05f32614..a7a8720a 100644 --- a/uiop/backward-driver.lisp +++ b/uiop/backward-driver.lisp @@ -14,7 +14,7 @@ (in-package :uiop/backward-driver) (eval-when (:compile-toplevel :load-toplevel :execute) -(with-deprecation ((version-deprecation *uiop-version* :style-warning "3.2" :warning "3.4")) +(with-deprecation ((version-deprecation *uiop-version* :style-warning "3.2" :warning "3.5")) ;; Backward compatibility with ASDF 2.000 to 2.26 ;; For backward-compatibility only, for people using internals -- GitLab From 7c556f26638d75dc7a030465aab518d43199ebef Mon Sep 17 00:00:00 2001 From: Francois-Rene Rideau Date: Wed, 27 May 2020 03:18:20 -0400 Subject: [PATCH 40/40] Update syntax-control document. --- doc/syntax-control.md | 45 ++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/doc/syntax-control.md b/doc/syntax-control.md index e368eb07..9985e5b0 100644 --- a/doc/syntax-control.md +++ b/doc/syntax-control.md @@ -17,7 +17,7 @@ yet may break the build in subtle way when the build is incremental and/or when part of a larger build that contains other systems with conflicting side-effects. (NB: The first variants of this document were written in 2014 while working on -ASDF 3.1, but its contents remain current as of ASDF 3.3.1 in December 2017.) +ASDF 3.1, but its contents remain current as of ASDF 3.3.4.5 in May 2020.) For instance, some CL libraries directly modify the current `*readtable*` by calling e.g. `set-dispatch-macro-character` without specifying a readtable. @@ -91,7 +91,7 @@ other syntax-controlling variables are at stake, too. ### Conflict with macro-characters If two systems define the same macro-character or dispatch-macro-character, -such as `#"` being defined by both `closure-common` and `string-escape`, +such as `#"` being defined by `closure-common`, `string-escape` and `common-lisp-jupyter`, then these systems, if they are part of the same build, may cause non-determinism as to which definition will be active at any time during an incremental build. @@ -99,8 +99,10 @@ Depending on which system was last modified, then only that system may be re-built, at which point its modifications to the readtable will happen last. No system can use `#"` syntax and trust which of the above last won. -Other conflicts found by grepping Quicklisp (as of October 2017) for +Other conflicts found by grepping Quicklisp (as of May 2020) for `'^(set-dispatch-macro-character '` include: +`#!` defined by `cl-i18n`, `cl-parallel`, `mcclim`, `racer` and `cl-launch`, or +`#&` defined by `hemlock` and `racer`, or `#T` defined by both `closure-html` and `racer`, or `#{` defined by both `cl-quickcheck` and `folio`, or `#~` defined by both `metatilities` and `perlre`, or @@ -109,6 +111,20 @@ Other conflicts found by grepping Quicklisp (as of October 2017) for There may or may not be further conflicts not detected through this simple grep for direct toplevel modification of the shared `*readtable*`. +Other: +* `ccl`: `#_` +* `cl-i18n`: `#§` +* `cl-m4`, `evol`, `quickutil`, `testbild`: `#>` (probably not a conflict, all cl-heredoc?) +* `clon`, `declt`, `focus`, `tfm`: `#i` (probably not a conflict, all didierverna) +* `closure-common`: `#"` `#/` +* `closure-html`: `#T` +* `cmu-infix`, `femlisp`: `#I` (possibly not a conflict?) +* `datum-comment`: `#;` +* `deferred`: `#^` +* `folio2`: `#[` +* `hemlock`: `#k` +* `paiprolog`: `#d` +* `puri`: `#u` ### Building with the wrong `*readtable*` @@ -119,10 +135,10 @@ and/or the state of the underlying data structure that variable is bound to, the effects will be seen by all components compiled with ASDF. Worse, if some reader macro expands into uses of symbols, functions and other entities, these entities will now be used in the fasls for components -that do not "depends-on" the systems in which those entities are defined. +that do not `depends-on:` the systems in which those entities are defined. All further attempts to load those fasls in a fresh image will then be catastrophic failures, with no easy solution beside clearing the fasl cache -(for instance with `rm -rf ~/.cache/common-lisp/' from a shell outside Lisp). +(for instance with `rm -rf ~/.cache/common-lisp/` from a shell outside Lisp). As a case that happened to me, I was using my `fare-quasiquote` syntax so I may portably use quasiquotation in match expressions (with `fare-matcher`, `optima` or now `trivia`). @@ -188,7 +204,6 @@ on characters so far undefined, and may never override a previous behavior defined or used by either the implementation or any program or extension. Indeed, whatever previous behavior that a previous program may have relied upon, - it will rely upon again when their are updated and rebuilt; and any conflict means that the build will then fail. Therefore: @@ -221,8 +236,8 @@ yet mostly respected as a practical necessity: users who try to make or use some piece of software that fails to respect them soon enough discover the hard way that that software fails to interoperate nicely with other software, at which point they fix it, or they work around it, -but in any case they avoid the conflict. -Yet, as of 2017, no implementation can nor will help enforce monotonicity; +but in any case they avoid the conflict—or other people will avoid using it. +Yet, as of 2020, no implementation can nor will help enforce monotonicity; users are wholly responsible for manually ensuring this monotonicity. Thus for instance, no one may ever modify `#u` except `puri`, @@ -247,7 +262,7 @@ The second discipline that users must follow as they invoke ASDF is to *never* call ASDF while the current `*readtable*` is bound to any readtable but the initial, shared readtable. If they do, either their current readtable may be corrupted, -or it cause some other part of the build to be corrupted, or both. +or it may cause some other part of the build to be corrupted, or both. However, there is currently no good means to access for certain this priveleged "shared initial readtable", except by supposing that @@ -288,7 +303,7 @@ it is also a counter-selection in who will use those features. This first proposal is to make some minimal changes that would maximally improve the situation given the small size of the changes. This proposal is meant be adopted immediately in the next release of ASDF -(whichever it may be, which would be ASDF 3.3.2 if done in early 2018). +(whichever it may be, which would be ASDF 3.3.5 or 3.4 if done in mid 2020). It independently amends each of the above two requirements: The first requirement is amended by making monotonicity an official policy, and by requiring that users should in turn explicitly document @@ -315,7 +330,7 @@ so users of private readtables don't have to worry about it. only in monotonic ways that never interfere with previous behavior. * Private syntax: a syntax intended for use by a system, or a coherent - set of systems. Note that while there is a single (constant, read-only) + set of systems. Note that while there is a single (constant, read-only) standard syntax, and single (constant identity, modifiable) initial syntax, and a single shared syntax (specially bound, within constraints), there may be multiple private syntaxes (e.g. using `named-readtables`). @@ -459,7 +474,7 @@ for a future release of ASDF (e.g. 3.5 or such). ### Some hacks still supported under Proposal 1 One way to modify the syntax in a deterministic way -that works with the current ASDF 3.3.1 and will keep working under Proposal 1 +that works with the current ASDF 3.3.4 and will keep working under Proposal 1 is to serialize all dependencies through a syntax-modification system: a system called e.g. `my-modified-syntax` depends on all the software dependencies in the project that must be @@ -651,8 +666,8 @@ This readtable is notionally read-only as per the standard. However, as of December 2017, only SBCL, Allegro, ECL and CMUCL actually reject modifications to the standard readtable and give you a nice clean error immediately. -On at least CCL, CLISP, ABCL, Lispworks and MKCL, -instead will silently let you corrupt the standard readtable, +By contrast, on at least CCL, CLISP, ABCL, Lispworks and MKCL, +the implementation will silently let you corrupt the standard readtable, which may result in horrible failures later when you use `with-standard-io-syntax` and it doesn't do what you expect. Happily, those implementations that do enforce read-only behavior @@ -693,7 +708,7 @@ to never side-effect the current `*readtable*`. But I acknowledge that the Common Lisp community is not ready for such a change at this point; and since I haven't myself been a paid Common Lisp professional for years, -and have jumped ship to Gerbil Scheme, I don't persaonally have the energy +and have jumped ship to Gerbil Scheme, I don't personally have the energy to push for such a change by getting all libraries fixed. But maybe some future maintainer will read this and make it their mission. -- GitLab