[elpa] externals/parser-generator edfb7b4 131/434: Moved lex-analyzer to separate file

[ELPA Syncer]

branch: externals/parser-generator
commit edfb7b451a08d87cf4b6e103dfb32b2e6e0f3d35
Author: Christian Johansson <christian@cvj.se>
Commit: Christian Johansson <christian@cvj.se>

    Moved lex-analyzer to separate file
---
 Makefile                                   |  8 ++-
 parser-generator-lex-analyzer.el           | 68 ++++++++++++++++++++++++
 parser-generator.el                        | 20 +------
 test/parser-generator-lex-analyzer-test.el | 84 ++++++++++++++++++++++++++++++
 test/parser-generator-lr-test.el           |  2 +
 test/parser-generator-test.el              |  2 +
 6 files changed, 164 insertions(+), 20 deletions(-)

diff --git a/Makefile b/Makefile
index 0cd694e..999e63b 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ ifdef emacs
 endif
 EMACS_CMD := $(EMACS) -Q -batch -L . -L test/
 
-EL  := parser-generator.el parser-generator-lr.el 
test/parser-generator-test.el test/parser-generator-lr-test.el
+EL  := parser-generator.el parser-generator-lex-analyzer.el 
parser-generator-lr.el test/parser-generator-test.el 
test/parser-generator-lex-analyzer-test.el test/parser-generator-lr-test.el
 ELC := $(EL:.el=.elc)
 
 .PHONY: clean
@@ -19,9 +19,13 @@ compile:
 test:
        $(EMACS_CMD) -l test/parser-generator-test.el -f "parser-generator-test"
 
+.PHONY: test-lex-analyzer
+test-lex-analyzer:
+       $(EMACS_CMD) -l test/parser-generator-lex-analyzer-test.el -f 
"parser-generator-lex-analyzer-test"
+
 .PHONY: test-lr
 test-lr:
        $(EMACS_CMD) -l test/parser-generator-lr-test.el -f 
"parser-generator-lr-test"
 
 .PHONY: tests
-tests: test test-lr
+tests: test test-lex-analyzer test-lr
diff --git a/parser-generator-lex-analyzer.el b/parser-generator-lex-analyzer.el
new file mode 100644
index 0000000..dcd0db2
--- /dev/null
+++ b/parser-generator-lex-analyzer.el
@@ -0,0 +1,68 @@
+;;; parser-generator-lex-analyzer.el --- Lex-analyzer library -*- 
lexical-binding: t -*-
+
+
+;;; Commentary:
+
+
+;;; Code:
+
+
+(require 'parser-generator)
+
+
+;;; Variables:
+
+
+(defvar parser-generator-lex-analyzer--function
+  nil
+  "Function used as lex-analyzer.")
+
+(defvar parser-generator-lex-analyzer--index
+  nil
+  "Index in lex-analyzer.")
+
+(defvar parser-generator-lex-analyzer--reset-function
+  nil
+  "Function used when resetting lex-analyzer.")
+
+
+;; Functions
+
+
+(defun parser-generator-lex-analyzer--peek-next-look-ahead ()
+  "Peek next look-ahead number of tokens via lex-analyzer."
+  (unless parser-generator-lex-analyzer--index
+    (error "Missing lex-analyzer index!"))
+  (unless parser-generator-lex-analyzer--function
+    (error "Missing lex-analyzer function!"))
+  (unless parser-generator--look-ahead-number
+    (error "Missing look-ahead number!"))
+  (funcall
+   parser-generator-lex-analyzer--function
+   parser-generator-lex-analyzer--index
+   parser-generator--look-ahead-number))
+
+(defun parser-generator-lex-analyzer--pop-token ()
+  "Pop next token via lex-analyzer."
+  (unless parser-generator-lex-analyzer--index
+    (error "Missing lex-analyzer index!"))
+  (unless parser-generator-lex-analyzer--function
+    (error "Missing lex-analyzer function!"))
+  (let ((token (funcall
+                parser-generator-lex-analyzer--function
+                parser-generator-lex-analyzer--index
+                1)))
+    (setq parser-generator-lex-analyzer--index
+          (1+ parser-generator-lex-analyzer--index))
+    token))
+
+(defun parser-generator-lex-analyzer--reset ()
+  "Reset lex-analyzer."
+  (setq parser-generator-lex-analyzer--index 0)
+  (when parser-generator-lex-analyzer--reset-function
+    (funcall parser-generator-lex-analyzer--reset-function)))
+
+
+(provide 'parser-generator-lex-analyzer)
+
+;;; parser-generator-lex-analyzer.el ends here
diff --git a/parser-generator.el b/parser-generator.el
index 3467008..fd29fc1 100644
--- a/parser-generator.el
+++ b/parser-generator.el
@@ -9,6 +9,7 @@
 
 ;;; Variables:
 
+
 (defvar parser-generator--allow-e-productions
   nil
   "Flag whether e-productions is allowed or not.")
@@ -33,10 +34,6 @@
   nil
   "Generated e-free F-sets for grammar.")
 
-(defvar parser-generator--lex-analyzer-function
-  nil
-  "Function used as lex-analyzer.")
-
 (defvar parser-generator--look-ahead-number
   nil
   "Current look-ahead number used.")
@@ -293,20 +290,6 @@
   (parser-generator--clear-cache)
   (parser-generator--load-symbols))
 
-(defun parser-generator--load-next-look-ahead ()
-  "Load next look-ahead number of tokens via lex-analyzer."
-  (unless parser-generator--lex-analyzer-function
-    (error "Missing lex-analyzer function!"))
-  (let ((left parser-generator--look-ahead-number)
-        (look-ahead))
-    (while (> left 0)
-      (let ((token (funcall parser-generator--lex-analyzer-function)))
-        (if token
-            (push token look-ahead)
-          (push parser-generator--e-identifier look-ahead)))
-      (setq left (1- left)))
-    look-ahead))
-
 (defun parser-generator--sort-list (a b)
   "Return non-nil if a element in A is greater than a element in B in 
lexicographic order."
   (let ((length (min (length a) (length b)))
@@ -860,6 +843,7 @@
       (setq follow-set (parser-generator--distinct follow-set)))
     follow-set))
 
+
 (provide 'parser-generator)
 
 ;;; parser-generator.el ends here
diff --git a/test/parser-generator-lex-analyzer-test.el 
b/test/parser-generator-lex-analyzer-test.el
new file mode 100644
index 0000000..0638e47
--- /dev/null
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -0,0 +1,84 @@
+;;; parser-generator-lex-analyzer-test.el --- Tests for lex-analyzer -*- 
lexical-binding: t -*-
+
+
+;;; Commentary:
+
+
+;;; Code:
+
+
+(require 'parser-generator-lex-analyzer)
+(require 'ert)
+
+(defun parser-generator-lex-analyzer-test--peek-next-look-ahead ()
+  "Test `parser-generator-lex-analyzer--peek-next-look-ahead'."
+  (message "Starting tests for 
(parser-generator-lex-analyzer--peek-next-look-ahead)")
+
+  (setq parser-generator-lex-analyzer--index nil)
+  (setq parser-generator-lex-analyzer--function nil)
+  (setq parser-generator--look-ahead-number nil)
+
+  (should-error
+   (parser-generator-lex-analyzer--peek-next-look-ahead))
+  (setq parser-generator-lex-analyzer--function (lambda (index length) 
(substring "abcdefghijklmnopqrstuvxz" index (+ index length))))
+  (should-error
+   (parser-generator-lex-analyzer--peek-next-look-ahead))
+  (parser-generator-lex-analyzer--reset)
+  (should-error
+   (parser-generator-lex-analyzer--peek-next-look-ahead))
+
+  (setq parser-generator--look-ahead-number 1)
+  (should
+   (equal
+    "a"
+    (parser-generator-lex-analyzer--peek-next-look-ahead)))
+
+  (message "Ended tests for 
(parser-generator-lex-analyzer--peek-next-look-ahead)"))
+
+(defun parser-generator-lex-analyzer-test--pop-token ()
+  "Test `parser-generator-lex-analyzer--pop-token'."
+  (message "Starting tests for (parser-generator-lex-analyzer--pop-token)")
+
+  (setq parser-generator-lex-analyzer--index nil)
+  (setq parser-generator-lex-analyzer--function nil)
+  (setq parser-generator--look-ahead-number nil)
+
+  (should-error
+   (parser-generator-lex-analyzer--pop-token))
+  (setq
+   parser-generator-lex-analyzer--function
+   (lambda (index length)
+     (let* ((string "ab")
+            (string-length (length string)))
+       (when (> string-length index)
+         (if (>= string-length (+ index length))
+             (substring string index (+ index length))
+           (substring string index (1- string-length)))))))
+  (should-error
+   (parser-generator-lex-analyzer--pop-token))
+  (parser-generator-lex-analyzer--reset)
+
+  (should
+   (equal
+    "a"
+    (parser-generator-lex-analyzer--pop-token)))
+  (should
+   (equal
+    "b"
+    (parser-generator-lex-analyzer--pop-token)))
+  (should
+   (equal
+    nil
+    (parser-generator-lex-analyzer--pop-token)))
+
+  (message "Ended tests for (parser-generator-lex-analyzer--pop-token)"))
+
+(defun parser-generator-lex-analyzer-test ()
+  "Run test."
+  (parser-generator-lex-analyzer-test--peek-next-look-ahead)
+  (parser-generator-lex-analyzer-test--pop-token))
+
+
+(provide 'parser-generator-lex-analyzer-test)
+
+;;; parser-generator-lex-analyzer-test.el ends here
diff --git a/test/parser-generator-lr-test.el b/test/parser-generator-lr-test.el
index ac3d441..4252bbf 100644
--- a/test/parser-generator-lr-test.el
+++ b/test/parser-generator-lr-test.el
@@ -6,6 +6,7 @@
 
 ;;; Code:
 
+
 (require 'parser-generator-lr)
 (require 'ert)
 
@@ -244,6 +245,7 @@
   (parser-generator-lr-test--generate-action-tables)
   (parser-generator-lr-test--parse))
 
+
 (provide 'parser-generator-lr-test)
 
 ;;; parser-generator-lr-test.el ends here
diff --git a/test/parser-generator-test.el b/test/parser-generator-test.el
index 824bced..554ed82 100644
--- a/test/parser-generator-test.el
+++ b/test/parser-generator-test.el
@@ -6,6 +6,7 @@
 
 ;;; Code:
 
+
 (require 'parser-generator)
 (require 'ert)
 
@@ -538,6 +539,7 @@
   (parser-generator-test--e-free-first)
   (parser-generator-test--follow))
 
+
 (provide 'parser-generator-test)
 
 ;;; parser-generator-test.el ends here