Commit 7aa8a23e authored by rtoy's avatar rtoy
Browse files

Update to Unicode 6.0.0.


code/unidata.lisp:
o Update unicode version to 6.0.0
o Add pointer to build-unidata.lisp.
tools/build-unidata.lisp:
o Update unicode version to 6.0.0
o Print out directory path so we can see where we're getting the data
  from.


i18n/CaseFolding.txt
i18n/CompositionExclusions.txt
i18n/DerivedNormalizationProps.txt
i18n/NameAliases.txt
i18n/NormalizationCorrections.txt
i18n/SpecialCasing.txt
i18n/UnicodeData.txt
i18n/WordBreakProperty.txt
i18n/tests/NormalizationTest.txt
i18n/tests/WordBreakTest.txt:
o Update with new files from unicode.org.
parent b948683f
......@@ -4,11 +4,14 @@
;;; This code was written by Paul Foley and has been placed in the public
;;; domain.
;;;
(ext:file-comment "$Header: /Volumes/share2/src/cmucl/cvs2git/cvsroot/src/code/unidata.lisp,v 1.27 2011/06/10 17:38:27 rtoy Exp $")
(ext:file-comment "$Header: /Volumes/share2/src/cmucl/cvs2git/cvsroot/src/code/unidata.lisp,v 1.28 2011/06/27 15:11:29 rtoy Exp $")
;;;
;;; **********************************************************************
;;;
;;; Unicode Database access
;;;
;;; See tools/build-unidata.lisp for the instructions and code to
;;; create the unicode database.
(in-package "LISP")
(intl:textdomain "cmucl")
......@@ -19,7 +22,7 @@
(defvar *unidata-path* "ext-formats:unidata.bin")
(defvar *unidata-version* "$Revision: 1.27 $")
(defvar *unidata-version* "$Revision: 1.28 $")
(defstruct unidata
range
......@@ -57,8 +60,8 @@
;; The expected Unicode version. This needs to be synced with
;; build-unidata.lisp.
(defconstant +unicode-major-version+ 5)
(defconstant +unicode-minor-version+ 2)
(defconstant +unicode-major-version+ 6)
(defconstant +unicode-minor-version+ 0)
(defconstant +unicode-update-version+ 0)
;;; These need to be synched with tools/build-unidata.lisp
......
# CaseFolding-5.2.0.txt
# Date: 2009-05-28, 23:02:34 GMT [MD]
# CaseFolding-6.0.0.txt
# Date: 2010-05-18, 00:48:57 GMT [MD]
#
# Unicode Character Database
# Copyright (c) 1991-2009 Unicode, Inc.
# Copyright (c) 1991-2010 Unicode, Inc.
# For terms of use, see http://www.unicode.org/terms_of_use.html
# For documentation, see http://www.unicode.org/reports/tr44/
#
......@@ -496,6 +496,7 @@
0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK
0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK
0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER
0526; C; 0527; # CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER
0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
......@@ -1057,6 +1058,7 @@ A658; C; A659; # CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS
A65A; C; A65B; # CYRILLIC CAPITAL LETTER BLENDED YUS
A65C; C; A65D; # CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS
A65E; C; A65F; # CYRILLIC CAPITAL LETTER YN
A660; C; A661; # CYRILLIC CAPITAL LETTER REVERSED TSE
A662; C; A663; # CYRILLIC CAPITAL LETTER SOFT DE
A664; C; A665; # CYRILLIC CAPITAL LETTER SOFT EL
A666; C; A667; # CYRILLIC CAPITAL LETTER SOFT EM
......@@ -1122,6 +1124,13 @@ A782; C; A783; # LATIN CAPITAL LETTER INSULAR R
A784; C; A785; # LATIN CAPITAL LETTER INSULAR S
A786; C; A787; # LATIN CAPITAL LETTER INSULAR T
A78B; C; A78C; # LATIN CAPITAL LETTER SALTILLO
A78D; C; 0265; # LATIN CAPITAL LETTER TURNED H
A790; C; A791; # LATIN CAPITAL LETTER N WITH DESCENDER
A7A0; C; A7A1; # LATIN CAPITAL LETTER G WITH OBLIQUE STROKE
A7A2; C; A7A3; # LATIN CAPITAL LETTER K WITH OBLIQUE STROKE
A7A4; C; A7A5; # LATIN CAPITAL LETTER N WITH OBLIQUE STROKE
A7A6; C; A7A7; # LATIN CAPITAL LETTER R WITH OBLIQUE STROKE
A7A8; C; A7A9; # LATIN CAPITAL LETTER S WITH OBLIQUE STROKE
FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF
FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI
FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL
......
# CompositionExclusions-5.2.0.txt
# Date: 2009-05-22, 12:52:00 PDT [KW]
# CompositionExclusions-6.0.0.txt
# Date: 2010-06-25, 14:34:00 PDT [KW]
#
# This file lists the characters for the Composition Exclusion Table
# defined in UAX #15, Unicode Normalization Forms.
......@@ -7,11 +7,11 @@
# This file is a normative contributory data file in the
# Unicode Character Database.
#
# Copyright (c) 1991-2009 Unicode, Inc.
# Copyright (c) 1991-2010 Unicode, Inc.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
# For more information, see
# http://www.unicode.org/unicode/reports/tr15/#Primary Exclusion List Table
# http://www.unicode.org/unicode/reports/tr15/#Primary_Exclusion_List_Table
#
# For a full derivation of composition exclusions, see the derived property
# Full_Composition_Exclusion in DerivedNormalizationProps.txt
......@@ -126,8 +126,8 @@ FB4E # HEBREW LETTER PE WITH RAFE
# (3) Singleton Decompositions
#
# These characters can be derived from the UnicodeData.txt file
# by including all characters whose canonical decomposition
# consists of a single character.
# by including all canonically decomposable characters whose
# canonical decomposition consists of a single character.
#
# These characters are simply quoted here for reference.
# See also Full_Composition_Exclusion in DerivedNormalizationProps.txt
......@@ -180,9 +180,18 @@ FB4E # HEBREW LETTER PE WITH RAFE
# (4) Non-Starter Decompositions
#
# These characters can be derived from the UnicodeData file
# by including all characters whose canonical decomposition consists
# of a sequence of characters, the first of which has a non-zero
# combining class.
# by including each expanding canonical decomposition
# (i.e., those which canonically decompose to a sequence
# of characters instead of a single character), such that:
#
# A. The character is not a Starter.
#
# OR (inclusive)
#
# B. The character's canonical decomposition begins
# with a character that is not a Starter.
#
# Note that a "Starter" is any character with a zero combining class.
#
# These characters are simply quoted here for reference.
# See also Full_Composition_Exclusion in DerivedNormalizationProps.txt
......
This diff is collapsed.
# NameAliases-5.2.0.txt
# Date: 2009-05-22, 13:05:00 PDT [KW]
# NameAliases-6.0.0.txt
# Date: 2010-05-10, 11:58:00 PDT [KW]
#
# This file is a normative contributory data file in the
# Unicode Character Database.
#
# Copyright (c) 2005-2009 Unicode, Inc.
# Copyright (c) 2005-2010 Unicode, Inc.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
# This file defines the formal name aliases for Unicode characters.
......
# NormalizationCorrections-5.2.0.txt
# Date: 2009-05-22, 13:54:00 PDT [KW]
# NormalizationCorrections-6.0.0.txt
# Date: 2010-05-19, 11:21:00 PDT [KW]
#
# This file is a normative contributory data file in the
# Unicode Character Database.
#
# Copyright (c) 1991-2009 Unicode, Inc.
# Copyright (c) 1991-2010 Unicode, Inc.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
# The normalization stabilization policy of the Unicode
......
# SpecialCasing-5.2.0.txt
# Date: 2009-09-22, 23:25:59 GMT [MD]
# SpecialCasing-6.0.0.txt
# Date: 2010-05-18, 00:49:39 GMT [MD]
#
# Unicode Character Database
# Copyright (c) 1991-2009 Unicode, Inc.
# Copyright (c) 1991-2010 Unicode, Inc.
# For terms of use, see http://www.unicode.org/terms_of_use.html
# For documentation, see http://www.unicode.org/reports/tr44/
#
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
# WordBreakTest-5.2.0.txt
# Date: 2009-09-19, 00:42:16 GMT [MD]
# WordBreakTest-6.0.0.txt
# Date: 2010-08-19, 01:19:54 GMT [MD]
#
# Unicode Character Database
# Copyright (c) 1991-2009 Unicode, Inc.
# Copyright (c) 1991-2010 Unicode, Inc.
# For terms of use, see http://www.unicode.org/terms_of_use.html
# For documentation, see http://www.unicode.org/reports/tr44/
#
......@@ -998,4 +998,4 @@
÷ 2060 ÷ 0061 × 2060 × 0062 × 2060 × 00AD × 2060 × 0062 × 2060 × 0079 × 2060 × 2060 ÷ # ÷ [0.2] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER B (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] SOFT HYPHEN (Format_FE) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER B (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER Y (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]
÷ 2060 ÷ 0061 × 2060 ÷ 0024 × 2060 ÷ 002D × 2060 ÷ 0033 × 2060 × 0034 × 2060 × 002C × 2060 × 0035 × 2060 × 0036 × 2060 × 0037 × 2060 × 002E × 2060 × 0031 × 2060 × 0034 × 2060 ÷ 0025 × 2060 ÷ 0062 × 2060 × 2060 ÷ # ÷ [0.2] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] DOLLAR SIGN (Other) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] HYPHEN-MINUS (Other) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] DIGIT THREE (Numeric) × [4.0] WORD JOINER (Format_FE) × [8.0] DIGIT FOUR (Numeric) × [4.0] WORD JOINER (Format_FE) × [12.0] COMMA (MidNum) × [4.0] WORD JOINER (Format_FE) × [11.0] DIGIT FIVE (Numeric) × [4.0] WORD JOINER (Format_FE) × [8.0] DIGIT SIX (Numeric) × [4.0] WORD JOINER (Format_FE) × [8.0] DIGIT SEVEN (Numeric) × [4.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [11.0] DIGIT ONE (Numeric) × [4.0] WORD JOINER (Format_FE) × [8.0] DIGIT FOUR (Numeric) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] PERCENT SIGN (Other) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER B (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]
÷ 2060 ÷ 0033 × 2060 × 0061 × 2060 × 2060 ÷ # ÷ [0.2] WORD JOINER (Format_FE) ÷ [999.0] DIGIT THREE (Numeric) × [4.0] WORD JOINER (Format_FE) × [10.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]
# Lines: 968
# Lines: 978
......@@ -4,7 +4,7 @@
;;; This code was written by Paul Foley and has been placed in the public
;;; domain.
;;;
(ext:file-comment "$Header: /Volumes/share2/src/cmucl/cvs2git/cvsroot/src/tools/build-unidata.lisp,v 1.8 2010/09/19 03:31:12 rtoy Rel $")
(ext:file-comment "$Header: /Volumes/share2/src/cmucl/cvs2git/cvsroot/src/tools/build-unidata.lisp,v 1.9 2011/06/27 15:11:30 rtoy Exp $")
;;;
;;; **********************************************************************
;;;
......@@ -53,8 +53,8 @@
(defconstant +unicode-magic-number+ #x2A554344)
;; The expected Unicode version
(defconstant +unicode-major-version+ 5)
(defconstant +unicode-minor-version+ 2)
(defconstant +unicode-major-version+ 6)
(defconstant +unicode-minor-version+ 0)
(defconstant +unicode-update-version+ 0)
;;; These need to be synched with code/unidata.lisp
......@@ -947,7 +947,7 @@
;; ucd-directory should be the directory where UnicodeData.txt is
;; located.
(defun build-unidata (&optional (ucd-directory "target:i18n/"))
(format t "~&Reading data~%")
(format t "~&Reading data from ~S~%" (probe-file ucd-directory))
(multiple-value-bind (ucd range) (read-data ucd-directory)
(setf (unidata-range *unicode-data*) range)
(format t "~&Building character name tables~%")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment