Index: /trunk/example.lisp
===================================================================
--- /trunk/example.lisp	(revision 8)
+++ /trunk/example.lisp	(revision 8)
@@ -0,0 +1,30 @@
+(require '#:cl-ppcre)
+(require '#:dso-lex)
+
+(use-package '(#:cl-ppcre #:dso-lex))
+
+
+
+(defun snip (s) (subseq s 1 (1- (length s))))
+
+(defun un-squote (s) (regex-replace-all "''" (snip s) "'"))
+
+(defun un-dquote (s) (regex-replace-all "\"\"" (snip s) "\""))
+
+(deflexer scan-csv
+  ("," comma)
+  ("[^\"',]+" value)
+  ("'(?:[^']|'')*'" value un-squote)
+  ("\"(?:[^\"]|\"\")*\"" value un-dquote))
+
+(defun scan-all (input)
+  (labels ((scan (input tokens)
+	     (if (> (length input) 0)
+		 (multiple-value-bind (class literal remainder)
+		     (scan-csv input)
+		   (when class
+		     (scan remainder (cons (cons class literal) tokens))))
+		 (reverse tokens))))
+    (scan input '())))
+
+(scan-all "no quotes,'a ''quote''',\"another \"\"quote\"\"\"")
Index: /trunk/lex.lisp
===================================================================
--- /trunk/lex.lisp	(revision 7)
+++ /trunk/lex.lisp	(revision 8)
@@ -1,3 +1,4 @@
 (defpackage #:dso-lex
+    (:documentation "Allows the definition of lexers.  See DEFLEXER.")
   (:use #:cl #:cl-ppcre)
   (:export #:deflexer))
@@ -17,4 +18,10 @@
 
 (defmacro deflexer (name &body body)
+  "Defines lexers, called as a function of the given NAME.  The body
+consists of token-class definitions, each being a list of a regular
+expression, the name of the class, and an optional filter.
+
+Currently, matching is done using *only* priority (first match wins),
+and does not look for the longest match."
   (let ((regex (combine (mapcar #'first body)))
 	(classes (map 'vector #'second body))