Changeset 16 for trunk/lex.lisp

Timestamp:

12/01/2007 10:52:59 PM (19 years ago)

Author:

dsowen

Message:

Additional interface to create lexers.

File:

• trunk/lex.lisp (modified) (2 diffs)

trunk/lex.lisp

@@ -22 +22 @@
     (:documentation "Allows the definition of lexers.  See DEFLEXER.")
   (:use #:cl #:cl-ppcre)
-  (:export #:deflexer))
+  (:export #:deflexer #:make-lexer))
 
 (in-package #:dso-lex)
@@ -35 +35 @@
     `(:sequence (:flags :single-line-mode-p) :start-anchor ,@mapped)))
 
-(defmacro deflexer (name &body body)
-  "Defines lexers, called as a function of the given NAME.  The body
+(defun lexer-form (input-var start-var defs)
+  (let ((regex (combine (mapcar #'first defs)))
+        (classes (map 'vector #'second defs))
+        (filters (map 'vector #'third defs)))
+    `(let ((parts (nth-value 3 (scan (quote ,regex) ,input-var
+                                     :start ,start-var))))
+       (let ((idx (position-if #'identity parts)))
+         (when idx
+           (let ((end (aref parts idx)))
+             (let ((image (make-array (- end ,start-var)
+                                      :element-type 'character
+                                      :displaced-to ,input-var
+                                      :displaced-index-offset ,start-var))
+                   (filter (aref ,filters idx)))
+               (values (aref ,classes idx)
+                       (if filter (funcall filter image) image)
+                       end))))))))
+
+
+
+(defun make-lexer (defs)
+  "Returns a lexer function.  The DEFS consists of token-class
+definitions, each being a list of a regular expression, the name of
+the class, and an optional filter.  The returned function takes as
+arguments an input sequence and an optional start position.
+
+Currently, matching is done using *only* priority (first match wins),
+and does not look for the longest match.
+
+Example:
+
+(let ((lexer (make-lexer '((\"[0-9]+\" number parse-integer)
+                           (\"[a-zA-Z]\" letter)))))
+  (funcall lexer \"2pi\" 1))"
+  (eval `(lambda (input &optional (start 0))
+           ,(lexer-form 'input 'start defs))))
+
+(defmacro deflexer (name &body defs)
+  "Defines a lexer, called as a function of the given NAME.  The body
 consists of token-class definitions, each being a list of a regular
 expression, the name of the class, and an optional filter.
 
 Currently, matching is done using *only* priority (first match wins),
-and does not look for the longest match."
-  (let ((regex (combine (mapcar #'first body)))
-        (classes (map 'vector #'second body))
-        (filters (map 'vector #'third body)))
-    `(defun ,name (input &optional (start 0))
-      (let ((parts (nth-value 3 (scan (quote ,regex) input :start start))))
-        (let ((idx (position-if #'identity parts)))
-          (when idx
-            (let ((end (aref parts idx)))
-              (let ((image (make-array (- end start)
-                                       :element-type 'character
-                                       :displaced-to input
-                                       :displaced-index-offset start))
-                    (filter (aref ,filters idx)))
-                (values (aref ,classes idx)
-                        (if filter (funcall filter image) image)
-                        end)))))))))
+and does not look for the longest match.
+
+Example:
+
+(deflexer lexer
+  (\"[0-9]+\" number parse-integer)
+  (\"[a-zA-Z]\" letter))
+
+(lexer \"2pi\" 1)"
+  `(defun ,name (input &optional (start 0))
+     ,(lexer-form 'input 'start defs)))