diff --git a/src/i18n/tests/norm-test.lisp b/src/i18n/tests/norm-test.lisp deleted file mode 100644 index 11fdfdd053a8b40dd20924971410838138425601..0000000000000000000000000000000000000000 --- a/src/i18n/tests/norm-test.lisp +++ /dev/null @@ -1,177 +0,0 @@ -;; Simple test routine to read and parse the normalization test file -;; and run the individual tests mentioned therein. - -(in-package "CL-USER") -(defvar *normalization-test* "target:i18n/tests/NormalizationTest.txt") - -(defun parse-norm-line (line) - (declare (string line)) - (cond ((or (char= (aref line 0) #\@) - (char= (aref line 0) #\#)) - ;; Delete lines starting with # or @ - nil) - (t - ;; Split the line into columns. There are 5 columns. Column 6 is a comment. - (let* ((p1 (position #\; line)) - (p2 (position #\; line :start (1+ p1))) - (p3 (position #\; line :start (1+ p2))) - (p4 (position #\; line :start (1+ p3))) - (p5 (position #\; line :start (1+ p4)))) - (labels - ((convert (codes) - (let ((result nil)) - (map nil #'(lambda (x) - (multiple-value-bind (hi lo) - (lisp::surrogates x) - (push hi result) - (when lo (push lo result)))) - codes) - (coerce (nreverse result) 'simple-string))) - (read-values (string start end) - (with-input-from-string (s string :start start :end end) - (let ((*read-base* 16)) - (convert - (loop for x = (read s nil nil) while x collect x)))))) - (values (read-values line 0 p1) - (read-values line (1+ p1) p2) - (read-values line (1+ p2) p3) - (read-values line (1+ p3) p4) - (read-values line (1+ p4) p5))))))) - - -(defun run-nfd-test (&optional (file *normalization-test*)) - (format t "~&NFD test~%") - (with-open-file (testfile file :direction :input :external-format :iso8859-1) - (let ((total 0) - (failures 0)) - (loop for line = (read-line testfile nil nil) while line do - (multiple-value-bind (c1 c2 c3 c4 c5) - (parse-norm-line line) - (when c1 - (incf total) - (let ((test1 (lisp::string-to-nfd c1)) - (test2 (lisp::string-to-nfd c2)) - (test3 (lisp::string-to-nfd c3)) - (test4 (lisp::string-to-nfd c4)) - (test5 (lisp::string-to-nfd c5))) - (unless (and (string= c3 test1) - (string= c3 test2) - (string= c3 test3) - (string= c5 test4) - (string= c5 test5)) - (incf failures) - (format t "**FAILURE**~%") - (format t "line: ~A~%" line) - (format t "c3 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code c3)) - (format t "test1 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test1)) - (format t "test2 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test2)) - (format t "test3 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test3)) - (format t "c5 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code c5)) - (format t "test4 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test4)) - (format t "test5 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test5))))))) - (format t "~A failures from ~A tests (~A%)~%" failures total - (* 100.0 (/ failures total)))))) - -(defun run-nfkd-test (&optional (file *normalization-test*)) - (format t "~&NFKD test~%") - (with-open-file (testfile file :direction :input :external-format :utf-8) - (let ((total 0) - (failures 0)) - (loop for line = (read-line testfile nil nil) while line do - ;;(format t "line = ~S~%" line) - (multiple-value-bind (c1 c2 c3 c4 c5) - (parse-norm-line line) - (when c1 - (incf total) - (let ((test1 (lisp::string-to-nfkd c1)) - (test2 (lisp::string-to-nfkd c2)) - (test3 (lisp::string-to-nfkd c3)) - (test4 (lisp::string-to-nfkd c4)) - (test5 (lisp::string-to-nfkd c5))) - (unless (and (string= c5 test1) - (string= c5 test2) - (string= c5 test3) - (string= c5 test4) - (string= c5 test5)) - (incf failures) - (format t "**FAILURE**~%") - (format t "line: ~A~%" line) - (format t "c5 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code c5)) - (format t "test1 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test1)) - (format t "test2 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test2)) - (format t "test3 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test3)) - (format t "test4 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test4)) - (format t "test5 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test5))))))) - (format t "~A failures from ~A tests (~A%)~%" failures total - (* 100.0 (/ failures total)))))) - -(defun run-nfc-test (&optional (file *normalization-test*)) - (format t "~&NFC test~%") - (with-open-file (testfile file :direction :input :external-format :utf-8) - (let ((total 0) - (failures 0)) - (loop for line = (read-line testfile nil nil) while line do - (multiple-value-bind (c1 c2 c3 c4 c5) - (parse-norm-line line) - (when c1 - (incf total) - (let ((test1 (lisp::string-to-nfc c1)) - (test2 (lisp::string-to-nfc c2)) - (test3 (lisp::string-to-nfc c3)) - (test4 (lisp::string-to-nfc c4)) - (test5 (lisp::string-to-nfc c5))) - (unless (and (string= c2 test1) - (string= c2 test2) - (string= c2 test3) - (string= c4 test4) - (string= c4 test5)) - (incf failures) - (format t "**FAILURE**~%") - (format t "line: ~A~%" line) - (format t "c2 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code c2)) - (format t "test1 = ~{~4,'0X~^ ~} (~S -> ~S)~%" (map 'list #'char-code test1) c1 test1) - (format t "test2 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test2)) - (format t "test3 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test3)) - (format t "c4 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code c4)) - (format t "test4 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test4)) - (format t "test5 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test5))))))) - (format t "~A failures from ~A tests (~A%)~%" failures total - (* 100.0 (/ failures total)))))) - -(defun run-nfkc-test (&optional (file *normalization-test*)) - (format t "~&NFKC test~%") - (with-open-file (testfile file :direction :input :external-format :utf-8) - (let ((total 0) - (failures 0)) - (loop for line = (read-line testfile nil nil) while line do - (multiple-value-bind (c1 c2 c3 c4 c5) - (parse-norm-line line) - (when c1 - (incf total) - (let ((test1 (lisp::string-to-nfkc c1)) - (test2 (lisp::string-to-nfkc c2)) - (test3 (lisp::string-to-nfkc c3)) - (test4 (lisp::string-to-nfkc c4)) - (test5 (lisp::string-to-nfkc c5))) - (unless (and (string= c4 test1) - (string= c4 test2) - (string= c4 test3) - (string= c4 test4) - (string= c4 test5)) - (incf failures) - (format t "**FAILURE**~%") - (format t "line: ~A~%" line) - (format t "c4 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code c4)) - (format t "test1 = ~{~4,'0X~^ ~} (~S -> ~S)~%" (map 'list #'char-code test1) c1 test1) - (format t "test2 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test2)) - (format t "test3 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test3)) - (format t "test4 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test4)) - (format t "test5 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test5))))))) - (format t "~A failures from ~A tests (~A%)~%" failures total - (* 100.0 (/ failures total)))))) - -(defun run-normalization-tests (&optional (file *normalization-test*)) - (run-nfd-test file) - (run-nfkd-test file) - (run-nfc-test file) - (run-nfkc-test file)) diff --git a/src/i18n/tests/word-break-test.lisp b/src/i18n/tests/word-break-test.lisp deleted file mode 100644 index 899b5a25a364956cbb51e9726361500e52a84780..0000000000000000000000000000000000000000 --- a/src/i18n/tests/word-break-test.lisp +++ /dev/null @@ -1,94 +0,0 @@ -;; Simple test routine to read and parse the word-break test file and -;; run the individual tests mentioned therein. - -(in-package "CL-USER") -(defvar *word-break-test* "target:i18n/tests/WordBreakTest.txt") - - -(defun parse-word-break-line (line) - (let* ((eos (or (position #\# line) - (length line)))) - ;; See WordBreakTest.txt for the format of the file. Basically - ;; each test is on one line and consists of a marker and a - ;; codepoint. The marker indicates whether a break is allowed (a - ;; division sign for a break, a multiplication sign for no break). - ;; The test ends with a marker to indicate if a division is - ;; allowed at the end of the test. After this marker is a comment - ;; for the test. - (when (plusp eos) - ;; For each non-empty line, read the codepints into a string and - ;; the markers into a second array. The second array contains - ;; indices into the string where a break is allowed. - (let ((*read-base* 16) - (count 0) - (breaks (make-array 10 :fill-pointer 0)) - (string (make-array 10 :fill-pointer 0 :element-type 'character))) - (with-input-from-string (s line :end eos) - (flet ((handle-break (c) - (when (eql #\division_sign (char (string c) 0)) - (vector-push-extend count breaks)))) - (handle-break (read s)) - (incf count) - (loop - (let ((c (read s nil nil))) - (unless c - (return)) - ;; Handle codepoints outside the BMP carefully. - (if (> c #xffff) - (let ((s (lisp::codepoints-string (list c)))) - ;; Need to increment the count because of our - ;; UTF-16 encoding of strings. - (incf count) - (vector-push-extend (aref s 0) string) - (vector-push-extend (aref s 1) string)) - (vector-push-extend (code-char c) string)) - (let ((c (read s))) - (handle-break c)) - (incf count))))) - (values string breaks))))) - -(defun do-test (string breaks) - (let ((posn -1) - (count 0) - (string (coerce string 'simple-string))) - (loop - (let ((end (lisp::string-next-word-break string posn))) - (when (= end posn) - (return)) - (unless (= end (aref breaks count)) - (format t "Test failed!~%") - (format t " String = ~S~%" (map 'list #'char-name string)) - (format t " Breaks = ~S~%" breaks) - (format t " Expected ~S but got ~S for break #~D~%" (aref breaks count) end count) - (finish-output) - (return-from do-test nil)) - (setf posn end) - (incf count))) - t)) - - -(defun run-word-break-test (&optional (file *word-break-test*)) - (let ((count 0) - (failed 0)) - (format t "Run WordBreakTest~%") - (with-open-file (s file :direction :input :external-format :utf) - (loop for line = (read-line s nil nil) - while line - do - (progn - (multiple-value-bind (s b) - (parse-word-break-line line) - (when s - (incf count) - (incf failed (if (do-test s b) 0 1))))))) - (format t "~D out of ~D tests failed. (~,2F% success)~%" - failed count (* 100.0 (- 1 (/ failed count)))) - ;; Here is one additional test to see if we are properly handling - ;; surrogate pairs. The string is "A?B cd4", where ? is the - ;; codepoint U+1000B: "LINEAR B SYLLABLE B046 JE". This has word - ;; break property Aletter, so string-capitalize should not put a - ;; break there. The result should be "A?b Cd4". - (let ((s (map 'string #'code-char - '(97 #xd800 #xdc0b 66 32 99 100 52)))) - (assert (string= (string-capitalize s :unicode-word-break t) - (map 'string #'code-char '(65 55296 56331 98 32 67 100 52)))))))