Commit 59c778a3 authored by Robert Strandh's avatar Robert Strandh

Whitespace is no longer considered a syntax element, and is skipped

over instead.  This makes the incremental lexer even faster.
parent 15b54e87
......@@ -48,9 +48,6 @@ macro characters that start more complex expressions."))
()
(:documentation "dummy entry before all the others."))
(defclass whitespace-entry (terminal-entry)
())
(defclass token-entry (terminal-entry)
()
(:documentation "the syntactic class of tokens."))
......@@ -233,12 +230,7 @@ macro characters that start more complex expressions."))
do (fo))
(make-entry 'character-entry))))
(t (make-entry 'error-entry)))))
(t (cond ((whitespacep object)
(loop until (end-of-buffer-p scan)
while (whitespacep (object-after scan))
do (fo))
(make-entry 'whitespace-entry))
((constituentp object)
(t (cond ((constituentp object)
(loop until (end-of-buffer-p scan)
while (constituentp (object-after scan))
do (fo))
......@@ -271,11 +263,13 @@ macro characters that start more complex expressions."))
0
(end-offset (element* elements (1- guess-pos))))))
;; scan
(unless (end-of-buffer-p scan)
(loop with start-mark = nil
until (if (end-of-buffer-p high-mark)
(end-of-buffer-p scan)
(mark> scan high-mark))
do (setf start-mark (clone-mark scan))
(insert* elements guess-pos (next-entry scan))
(incf guess-pos)))))))
(loop with start-mark = nil
do (loop until (end-of-buffer-p scan)
while (whitespacep (object-after scan))
do (forward-object scan))
until (if (end-of-buffer-p high-mark)
(end-of-buffer-p scan)
(mark> scan high-mark))
do (setf start-mark (clone-mark scan))
(insert* elements guess-pos (next-entry scan))
(incf guess-pos))))))
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment