From 4c3e9ced564a7ef3e228f92d0d71511fe83c5ed4 Mon Sep 17 00:00:00 2001 From: Raymond Toy Date: Fri, 23 Sep 2022 13:23:14 -0700 Subject: [PATCH 1/5] Address #139: set default external format to :utf8 This changes the default external format to utf8 instead of iso8859-1. * fd-stream.lisp: Create `*stdin*`, `*stdout*`, `*stderr*`, and `*tty*` streams using the `:default` external format. Since `*default-external-format*` is `:utf-8`, these streams will then use `:utf-8` for the format. * extfmts.lisp: * Change `*default-external-format*` to `:utf-8` instead of `:iso8859-1`. * Also change `%find-external-format` to treat the `:default` format to mean `:utf-8` instead of `:iso8859-1` Manually tested this by building cmucl and doing `(print (code-char 945))`. This should print a lower case Greek alpha character by default instead of `#\?`, which is what used to happen when the external format was `:iso8859-1`. --- src/code/extfmts.lisp | 9 +++++---- src/code/fd-stream.lisp | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/code/extfmts.lisp b/src/code/extfmts.lisp index 9efcb0a6f..16c847627 100644 --- a/src/code/extfmts.lisp +++ b/src/code/extfmts.lisp @@ -22,7 +22,8 @@ describe-external-format)) (defvar *default-external-format* - :iso8859-1 + #-unicode :iso8859-1 + #+unicode :utf-8 "The default external format to use if no other external format is specified") @@ -460,11 +461,11 @@ (defun %find-external-format (name) ;; avoid loading files, etc., early in the boot sequence - (when (or (eq name :iso8859-1) - (and (eq name :default) (eq *default-external-format* :iso8859-1))) + (when (eq name :iso8859-1) (return-from %find-external-format (gethash :iso8859-1 *external-formats*))) - (when (eq name :utf-8) + (when (or (eq name :utf-8) + (and (eq name :default) (eq *default-external-format* :utf-8))) (return-from %find-external-format (gethash :utf-8 *external-formats*))) diff --git a/src/code/fd-stream.lisp b/src/code/fd-stream.lisp index 470dfe23e..f1b63c7d2 100644 --- a/src/code/fd-stream.lisp +++ b/src/code/fd-stream.lisp @@ -2397,20 +2397,20 @@ (setf *available-buffers* nil) (setf *stdin* (make-fd-stream 0 :name "Standard Input" :input t :buffering :line - :external-format :iso8859-1)) + :external-format :default)) (setf *stdout* (make-fd-stream 1 :name "Standard Output" :output t :buffering :line - :external-format :iso8859-1)) + :external-format :default)) (setf *stderr* (make-fd-stream 2 :name "Standard Error" :output t :buffering :line - :external-format :iso8859-1)) + :external-format :default)) (let ((tty (and (not *batch-mode*) (unix:unix-open "/dev/tty" unix:o_rdwr #o666)))) (setf *tty* (if tty (make-fd-stream tty :name "the Terminal" :input t :output t :buffering :line :auto-close t - :external-format :iso8859-1) + :external-format :default) (make-two-way-stream *stdin* *stdout*)))) nil) -- GitLab From 344b948245ca3b33154c767d6225db9ee7280ab2 Mon Sep 17 00:00:00 2001 From: Raymond Toy Date: Fri, 23 Sep 2022 15:14:21 -0700 Subject: [PATCH 2/5] Add test for default external format and system streams Verify that the default external format is :utf-8 and that all the system streams use :utf-8. --- tests/issues.lisp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/issues.lisp b/tests/issues.lisp index 3ad494bb1..16ddb9bc5 100644 --- a/tests/issues.lisp +++ b/tests/issues.lisp @@ -579,3 +579,20 @@ with user-info = (unix:unix-getpwuid uid) while user-info finally (assert-false user-info))) + +(define-test issue-139.1 + (:tag :issues) + ;; Verify the value of the default external format and that system streams use :utf-8. + (assert-eql :utf-8 stream:*default-external-format*) + (assert-eql :utf-8 (stream-external-format sys:*stdin*)) + (assert-eql :utf-8 (stream-external-format sys:*stdout*)) + (assert-eql :utf-8 (stream-external-format sys:*stderr*)) + (assert-eql :utf-8 (stream-external-format sys:*tty*)) + ;; Check that printing to *standard-output* is correctly encoded. + (dribble "test-format.txt") + ;; Print a Greek lower-case alpha character + (princ #\u+3b1) + (dribble) + (with-open-file (s "test-format.txt" :direction :input) + (let ((c (read-char s))) + (assert-equal #\u+3b1 c)))) -- GitLab From c5f31c3cd0628d668831d285e629da0a526d1678 Mon Sep 17 00:00:00 2001 From: Raymond Toy Date: Fri, 23 Sep 2022 16:06:01 -0700 Subject: [PATCH 3/5] Explicitly set format to :utf-8 instead of :default --- src/code/fd-stream.lisp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/code/fd-stream.lisp b/src/code/fd-stream.lisp index f1b63c7d2..1d9099b0d 100644 --- a/src/code/fd-stream.lisp +++ b/src/code/fd-stream.lisp @@ -2397,20 +2397,20 @@ (setf *available-buffers* nil) (setf *stdin* (make-fd-stream 0 :name "Standard Input" :input t :buffering :line - :external-format :default)) + :external-format :utf-8)) (setf *stdout* (make-fd-stream 1 :name "Standard Output" :output t :buffering :line - :external-format :default)) + :external-format :utf-8)) (setf *stderr* (make-fd-stream 2 :name "Standard Error" :output t :buffering :line - :external-format :default)) + :external-format :utf-8)) (let ((tty (and (not *batch-mode*) (unix:unix-open "/dev/tty" unix:o_rdwr #o666)))) (setf *tty* (if tty (make-fd-stream tty :name "the Terminal" :input t :output t :buffering :line :auto-close t - :external-format :default) + :external-format :utf-8) (make-two-way-stream *stdin* *stdout*)))) nil) -- GitLab From 86b4483e870dc7f5bd512fdcd7002d7844c4d186 Mon Sep 17 00:00:00 2001 From: Raymond Toy Date: Fri, 23 Sep 2022 16:46:25 -0700 Subject: [PATCH 4/5] *tty* format can be either :utf-8 or :default `*tty*` can either be an fd-stream of a two-way-stream. If the former, the format is `:utf-8`; if the latter, `:default`. Update test accordingly. --- tests/issues.lisp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/issues.lisp b/tests/issues.lisp index 16ddb9bc5..eb257e8d1 100644 --- a/tests/issues.lisp +++ b/tests/issues.lisp @@ -587,7 +587,11 @@ (assert-eql :utf-8 (stream-external-format sys:*stdin*)) (assert-eql :utf-8 (stream-external-format sys:*stdout*)) (assert-eql :utf-8 (stream-external-format sys:*stderr*)) - (assert-eql :utf-8 (stream-external-format sys:*tty*)) + ;; *tty* can either be an fd-stream, in which case the format is + ;; utf8, or a two-way-stream, in which case it is :default. + (if (typep sys:*tty* 'two-way-stream) + (assert-eql :default (stream-external-format sys:*tty*)) + (assert-eql :utf-8 (stream-external-format sys:*tty*))) ;; Check that printing to *standard-output* is correctly encoded. (dribble "test-format.txt") ;; Print a Greek lower-case alpha character -- GitLab From 64fd3b43038e1c2891cf1434e4732a5058127556 Mon Sep 17 00:00:00 2001 From: Raymond Toy Date: Sat, 24 Sep 2022 17:39:55 -0700 Subject: [PATCH 5/5] Address #139: Set default filename encoding to utf8 * unix.lisp Add `*default-filename-encoding*` to hold the default encoding if `*filename-encoding*` is `NIL`. * lispinit.lisp In `reinit`, set the filename encoding to the default if it's not already set. * extfmts.lisp In `set-system-external-format`, set `*default-filename-encoding*` to the specified encoding. This will be used to reinit `*filename-encoding*` if needed. --- src/code/extfmts.lisp | 3 ++- src/code/lispinit.lisp | 4 +++- src/code/unix.lisp | 3 +++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/code/extfmts.lisp b/src/code/extfmts.lisp index 16c847627..f3f19c29c 100644 --- a/src/code/extfmts.lisp +++ b/src/code/extfmts.lisp @@ -1155,7 +1155,8 @@ character and illegal outputs are replaced by a question mark.") (not (eq unix::*filename-encoding* filenames))) (cerror (intl:gettext "Change it anyway.") (intl:gettext "The external-format for encoding filenames is already set."))) - (setq unix::*filename-encoding* filenames)) + (setq unix::*filename-encoding* filenames) + (setq unix::*default-filename-encoding* filenames)) t) diff --git a/src/code/lispinit.lisp b/src/code/lispinit.lisp index 01d7f33c2..42cf56889 100644 --- a/src/code/lispinit.lisp +++ b/src/code/lispinit.lisp @@ -482,7 +482,9 @@ (set-floating-point-modes :traps '(:overflow :invalid :divide-by-zero)) ;; Clear pseudo atomic in case this core wasn't compiled with support. - #+(or x86 amd64) (setf lisp::*pseudo-atomic-atomic* 0)))) + #+(or x86 amd64) (setf lisp::*pseudo-atomic-atomic* 0) + (unless unix::*filename-encoding* + (setf unix::*filename-encoding* unix::*default-filename-encoding*))))) ;;;; Miscellaneous external functions: diff --git a/src/code/unix.lisp b/src/code/unix.lisp index 9dd6db03d..fef496437 100644 --- a/src/code/unix.lisp +++ b/src/code/unix.lisp @@ -26,6 +26,9 @@ ;; non-Latin-1 characters "mojibake", but otherwise they'll be inaccessible. ;; Must be set to NIL initially to enable building Lisp! (defvar *filename-encoding* nil) +(defvar *default-filename-encoding* + #-unicode nil + #+unicode :utf-8) (eval-when (:compile-toplevel :load-toplevel :execute) (defmacro %name->file (string) -- GitLab