Commit ce4cc95e authored by marijnh's avatar marijnh
Browse files

initial import 

darcs-hash:ad4c86436ab1e8e6fccfb7f4c29f1cd6f9055bf9
parents

doc/index.html

0 → 100644
+89 −0
Original line number Diff line number Diff line 
<?xml version="1.0"?>

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"

    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">

<head>

  <title>Trivial UTF-8</title>

  <link rel="stylesheet" type="text/css" href="style.css"/>

  <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"/>

</head>



<body>

 <div class="header">

   <h1>Trivial UTF-8</h1>

 </div>



<p>Trivial UTF-8 is a small library for doing UTF-8-based in- and

output <em>on a Lisp implementation that already supports Unicode</em>

-- meaning <tt>char-code</tt> and <tt>code-char</tt> deal with Unicode

character codes.</p>



<p>The rationale for the existence of this library is that while

Unicode-enabled implementations usually do provide some kind of

interface to dealing with character encodings, these are typically not

flexible or efficient enough. Specifically, SBCL's

<tt>sb-ext:octets-to-string</tt> and <tt>string-to-octets</tt> are 10

times slower than the equivalents in this library (and not easily

optimized because of the way they are defined -- the

<tt>use-value</tt> restart in particular), and do not provide a way to

directly read or write UTF-8 from or to a stream.</p>



<h2>Download and installation</h2>



<p>The latest release of trivial-utf-8 can be downloaded from <a

href="http://common-lisp.net/project/trivial-utf-8/trivial-utf-8.tgz">http://common-lisp.net/project/trivial-utf-8/trivial-utf-8.tgz</a>,

or installed with <a

href="http://www.cliki.net/ASDF-Install">asdf-install</a>.</p>



<h2>Support and mailing lists</h2>



<p>The <a href="http://common-lisp.net/mailman/listinfo/trivial-utf-8-devel">trivial-utf-8-devel</a> mailing list can be used for any questions, discussion, bug-reports, patches, or anything else relating to this library.</p>



<h2>Reference</h2>



<p class="def">function <tt>string-to-utf-8-bytes</tt> (string) => array of (unsigned-byte 8)</p>



<p class="desc">Convert a string into an array of unsigned bytes containing its

utf-8 representation.</p>



<p class="def">function <tt>utf-8-bytes-to-string</tt> (bytes) => string</p>



<p class="desc">Convert a byte array containing utf-8 encoded characters into the

string it encodes.</p>



<p class="def">function <tt>write-utf-8-bytes</tt> (string)</p>



<p class="desc">Write a string to a byte-stream, encoding it as utf-8.</p>



<p class="def">function <tt>read-utf-8-string</tt> (input &amp;key null-terminated stop-at-eof char-length byte-length)</p>



<p class="desc">Read utf-8 encoded data from a byte stream and construct a string

with the characters found. When null-terminated is given it will stop

reading at a null character, stop-at-eof tells it to stop at the end

of file without raising an error, and the char-length and byte-length

parameters can be used to specify the max amount of characters or

bytes to read.</p>



<p class="def">function <tt>utf-8-byte-length</tt> (string) => integer</p>



<p class="desc">Calculate the amount of bytes needed to encode a string.</p>



<p class="def">function <tt>utf-8-group-size</tt> (byte) => integer</p>



<p class="desc">Determine the amount of bytes that are part of the character

starting with a given byte.</p>



<p class="def">condition <tt>utf-8-decoding-error</tt></p>



<p class="desc">A condition of this type is raised whenever an incorrectly encoded

character is encountered.</p>



<hr/>



<p>Back to <a href="http://common-lisp.net/">Common-lisp.net</a>.</p>



 <div class="check">

   <a href="http://validator.w3.org/check/referer">Valid XHTML 1.0 Strict</a>

 </div>

</body>

</html>

doc/style.css

0 → 100644
+72 −0
Original line number Diff line number Diff line 


.header {

	font-size: medium;

	background-color:#336699;

	color:#ffffff;

	border-style:solid;

	border-width: 5px;

	border-color:#002244;

	padding: 1mm 1mm 1mm 5mm;

}



.footer {

	font-size: small;

	font-style: italic;

	text-align: right;

	background-color:#336699;

	color:#ffffff;

	border-style:solid;

	border-width: 2px;

	border-color:#002244;

	padding: 1mm 1mm 1mm 1mm;

}



.footer a:link {

	font-weight:bold;

	color:#ffffff;

	text-decoration:underline;

}



.footer a:visited {

	font-weight:bold;

	color:#ffffff; 

	text-decoration:underline;

}



.footer a:hover { 

	font-weight:bold; 

	color:#002244; 

	text-decoration:underline; }



.check {font-size: x-small;

	text-align:right;}



.check a:link { font-weight:bold;

		color:#a0a0ff;

		text-decoration:underline; }



.check a:visited { font-weight:bold;

		   color:#a0a0ff;

		   text-decoration:underline; }



.check a:hover { font-weight:bold;

		 color:#000000;

		 text-decoration:underline; }



tt {

  font-size: 1.3em;

  font-weight: bold;

}



.def {

  margin-top: 1.5em;

  font-family: tahoma, arial, sans-serif;

}



.desc {

  padding-left: .6em;

}



h2 {

  font-size: 14pt;

}

test.txt

0 → 100644
+8 −0
Original line number Diff line number Diff line 
                                                                      ▉

  ╔══╦══╗  ┌──┬──┐  ╭──┬──╮  ╭──┬──╮  ┏━━┳━━┓  ┎┒┏┑   ╷  ╻ ┏┯┓ ┌┰┐    ▊ ╱╲╱╲╳╳╳

  ║┌─╨─┐║  │╔═╧═╗│  │╒═╪═╕│  │╓─╁─╖│  ┃┌─╂─┐┃  ┗╃╄┙  ╶┼╴╺╋╸┠┼┨ ┝╋┥    ▋ ╲╱╲╱╳╳╳

  ║│╲ ╱│║  │║   ║│  ││ │ ││  │║ ┃ ║│  ┃│ ╿ │┃  ┍╅╆┓   ╵  ╹ ┗┷┛ └┸┘    ▌ ╱╲╱╲╳╳╳

  ╠╡ ╳ ╞╣  ├╢   ╟┤  ├┼─┼─┼┤  ├╫─╂─╫┤  ┣┿╾┼╼┿┫  ┕┛┖┚     ┌┄┄┐ ╎ ┏┅┅┓ ┋ ▍ ╲╱╲╱╳╳╳

  ║│╱ ╲│║  │║   ║│  ││ │ ││  │║ ┃ ║│  ┃│ ╽ │┃  ░░▒▒▓▓██ ┊  ┆ ╎ ╏  ┇ ┋ ▎

  ║└─╥─┘║  │╚═╤═╝│  │╘═╪═╛│  │╙─╀─╜│  ┃└─╂─┘┃  ░░▒▒▓▓██ ┊  ┆ ╎ ╏  ┇ ┋ ▏

  ╚══╩══╝  └──┴──┘  ╰──┴──╯  ╰──┴──╯  ┗━━┻━━┛           └╌╌┘ ╎ ┗╍╍┛ ┋  ▁▂▃▄▅▆▇█

tests.lisp

0 → 100644
+31 −0
Original line number Diff line number Diff line 
(defpackage :trivial-utf-8-tests

  (:use :common-lisp :trivial-utf-8))



(in-package :trivial-utf-8-tests)



;; TODO this could be a lot nicer and more exhaustive.



(defparameter *my-path* (asdf:component-pathname (asdf:find-component nil :trivial-utf-8)))

(defparameter *test-file* (merge-pathnames #P"test.txt" *my-path*))

(defparameter *output-file* (merge-pathnames #P"test-out.txt" *my-path*))



;; Read the content of a file.

(let ((content (with-open-file (input *test-file* :direction :input

				      :element-type '(unsigned-byte 8))

		 (read-utf-8-string input :stop-at-eof t))))

  ;; Is the amount of characters correct?

  (assert (= (length content) 616))

  ;; See if encoding and decoding the string leaves it intact.

  (assert (string= (utf-8-bytes-to-string (string-to-utf-8-bytes content)) content))

  (with-open-file (output *output-file* :direction :output :element-type '(unsigned-byte 8)

			  :if-exists :supersede)

    ;; Write it to another file.

    (write-utf-8-bytes content output)))



;; Check whether the files are really the same.

(with-open-file (original *test-file* :direction :input :element-type '(unsigned-byte 8))

  (with-open-file (new *output-file* :direction :input :element-type '(unsigned-byte 8))

    (loop :for byte1 = (read-byte original nil :eof)

	  :for byte2 = (read-byte new nil :eof)

	  :do (assert (equal byte1 byte2))

	  :while (not (eq byte1 :eof)))))

trivial-utf-8.asd

0 → 100644
+10 −0
Original line number Diff line number Diff line 
(defpackage :trivial-utf-8-system

  (:use :common-lisp :asdf))

(in-package :trivial-utf-8-system)



(defsystem :trivial-utf-8

  :components ((:file "trivial-utf-8")))



(defsystem :trivial-utf-8-tests

  :depends-on (:trivial-utf-8)

  :components ((:file "tests")))