[elpa] externals/parser-generator b2fd896 136/434: Added support for indexed tokens

[ELPA Syncer]

branch: externals/parser-generator
commit b2fd8964074ac859e728dd8186971048df548977
Author: Christian Johansson <christian@cvj.se>
Commit: Christian Johansson <christian@cvj.se>

    Added support for indexed tokens
---
 parser-generator-lex-analyzer.el           |  4 ++--
 parser-generator-lr.el                     | 12 ++++++++++--
 test/parser-generator-lex-analyzer-test.el | 14 +++++++-------
 test/parser-generator-lr-test.el           |  8 ++++++--
 4 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/parser-generator-lex-analyzer.el b/parser-generator-lex-analyzer.el
index dcd0db2..b7f6274 100644
--- a/parser-generator-lex-analyzer.el
+++ b/parser-generator-lex-analyzer.el
@@ -48,10 +48,10 @@
     (error "Missing lex-analyzer index!"))
   (unless parser-generator-lex-analyzer--function
     (error "Missing lex-analyzer function!"))
-  (let ((token (funcall
+  (let ((token (car (funcall
                 parser-generator-lex-analyzer--function
                 parser-generator-lex-analyzer--index
-                1)))
+                1))))
     (setq parser-generator-lex-analyzer--index
           (1+ parser-generator-lex-analyzer--index))
     token))
diff --git a/parser-generator-lr.el b/parser-generator-lr.el
index a7c7ae0..7b79eed 100644
--- a/parser-generator-lr.el
+++ b/parser-generator-lr.el
@@ -514,7 +514,8 @@
 
       ;; (1) The lookahead string u, consisting of the next k input symbols, 
is determined.
       (let ((look-ahead (parser-generator-lex-analyzer--peek-next-look-ahead))
-            (look-ahead-length 0))
+            (look-ahead-length 0)
+            (look-ahead-full))
 
         (setq look-ahead-length (length look-ahead))
 
@@ -523,7 +524,14 @@
           (push parser-generator--e-identifier look-ahead)
           (setq look-ahead-length (1+ look-ahead-length)))
 
-        (setq look-ahead (nreverse look-ahead))
+        ;; Save token stream indexes in separate variable if needed later
+        (setq look-ahead-full (nreverse look-ahead))
+
+        (setq look-ahead nil)
+        (dolist (look-ahead-item look-ahead-full)
+          (if (listp look-ahead-item)
+              (push (car look-ahead-item) look-ahead)
+            (push look-ahead-item look-ahead)))
 
         (let ((table-index (car pushdown-list)))
           (let ((action-table (gethash table-index 
parser-generator-lr--action-tables)))
diff --git a/test/parser-generator-lex-analyzer-test.el 
b/test/parser-generator-lex-analyzer-test.el
index f68722c..03d4a05 100644
--- a/test/parser-generator-lex-analyzer-test.el
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -23,7 +23,7 @@
   (setq
    parser-generator-lex-analyzer--function
    (lambda (index length)
-     (let* ((string '(a b c d))
+     (let* ((string '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5)))
             (string-length (length string))
             (max-index (+ index length))
             (tokens))
@@ -42,19 +42,19 @@
   (setq parser-generator--look-ahead-number 1)
   (should
    (equal
-    '(a)
+    '(("a" 1 . 2))
     (parser-generator-lex-analyzer--peek-next-look-ahead)))
 
   (setq parser-generator--look-ahead-number 2)
   (should
    (equal
-    '(a b)
+    '(("a" 1 . 2) ("b" 2 . 3))
     (parser-generator-lex-analyzer--peek-next-look-ahead)))
 
   (setq parser-generator--look-ahead-number 10)
   (should
    (equal
-    '(a b c d)
+    '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5))
     (parser-generator-lex-analyzer--peek-next-look-ahead)))
 
   (message "Ended tests for 
(parser-generator-lex-analyzer--peek-next-look-ahead)"))
@@ -72,7 +72,7 @@
   (setq
    parser-generator-lex-analyzer--function
    (lambda (index length)
-     (let* ((string '(a b))
+     (let* ((string '(("a" 1 . 2) ("b" 2 . 3)))
             (string-length (length string))
             (max-index (+ index length))
             (tokens))
@@ -88,11 +88,11 @@
 
   (should
    (equal
-    '(a)
+    '("a" 1 . 2)
     (parser-generator-lex-analyzer--pop-token)))
   (should
    (equal
-    '(b)
+    '("b" 2 . 3)
     (parser-generator-lex-analyzer--pop-token)))
   (should
    (equal
diff --git a/test/parser-generator-lr-test.el b/test/parser-generator-lr-test.el
index 222d986..df52e7f 100644
--- a/test/parser-generator-lr-test.el
+++ b/test/parser-generator-lr-test.el
@@ -225,11 +225,12 @@
   (parser-generator--set-grammar '((Sp S) (a b) ((Sp S) (S (S a S b)) (S e)) 
Sp))
   (parser-generator--set-look-ahead-number 1)
   (parser-generator--process-grammar)
+  (parser-generator-lr--reset)
 
   (setq
    parser-generator-lex-analyzer--function
    (lambda (index length)
-     (let* ((string '(a a b b))
+     (let* ((string '((a 1 . 2) (a 2 . 3) (b 3 . 4) (b 4 . 5)))
             (string-length (length string))
             (max-index (+ index length))
             (tokens))
@@ -248,7 +249,7 @@
   (setq
    parser-generator-lex-analyzer--function
    (lambda (index length)
-     (let* ((string '(a a b b b))
+     (let* ((string '((a 1 . 2) (a 2 . 3) (b 3 . 4) (b 4 . 5) (b 5 . 6)))
             (string-length (length string))
             (max-index (+ index length))
             (tokens))
@@ -262,6 +263,9 @@
   (should-error
    (parser-generator-lr--parse))
 
+
+  ;; TODO Test with terminals as strings here
+
   (message "Passed tests for (parser-generator-lr--parse)"))
 
 (defun parser-generator-lr-test ()