Commit 31a76ff7 authored by Raymond Toy's avatar Raymond Toy
Browse files

Include BOM in octet count for UTF-16 and UTF-32

`string-to-octets` includes the BOM when encoding strings.  To be
consistent, update `string-octet-count` to include the BOM when
computing the number of octets.

This is only needed for :UTF-16 and :UTF-32 formats.  The other utf-16
and utf-32 formats don't include the BOM.

Enable tests for these two formats too.
parent aa6f0565
Loading
Loading
Loading
Loading
Loading
+10 −10
Original line number Diff line number Diff line
@@ -158,16 +158,16 @@ Unicode replacement character.")
    ;; The state is list. Copy it
	      `(copy-list ,state))
  (octet-count (code state error)
    `(progn
       #+nil
    `(let ((bom-count 0))
       (unless ,state
	 ;; Output BOM
	 (output #xFEFF)
	 (setf bom-count 2)
	 (setf ,state t))
       (cond ((< ,code #x10000)
	      2)
	     ((< ,code #x110000)
	      4)
	     (t
	      ;; Replacement character is 2 octets
	      2)))))
       (+ bom-count
	  (cond ((< ,code #x10000)
		 2)
		((< ,code #x110000)
		 4)
		(t
		 ;; Replacement character is 2 octets
		 2))))))
+4 −6
Original line number Diff line number Diff line
@@ -116,11 +116,9 @@ Unicode replacement character.")
    ;; The state is either NIL or T, so we can just return that.
    `(progn ,state))
  (octet-count (code state error)
    `(progn
       ;; Should we count the BOM?
       #+nil
    `(let ((bom-count 0))
       (unless ,state
	 (out #xFEFF)
	 (setf bom-count 4)
	 (setf ,state t))
       (cond ((lisp::surrogatep ,code)
	      (if ,error
@@ -130,6 +128,6 @@ Unicode replacement character.")
		    (funcall ,error "Surrogate code #x~4,'0X is illegal for UTF32 output"
			     ,code))
		  ;; Replacement character is 2 octets
		  2))
		  (+ 2 bom-count)))
	     (t
	      4)))))
	      (+ 4 bom-count))))))
+0 −2
Original line number Diff line number Diff line
@@ -36,7 +36,6 @@
    (:tag :octet-count)
  (test-octet-count *test-unicode* :utf-8))

#+nil
(define-test octet-count.utf-16
    (:tag :octet-count)
  (test-octet-count *test-unicode* :utf-16))
@@ -49,7 +48,6 @@
    (:tag :octet-count)
  (test-octet-count *test-unicode* :utf-16-le))

#+nil
(define-test octet-count.utf-32
    (:tag :octet-count)
  (test-octet-count *test-unicode* :utf-32))