[elpa] externals/phps-mode f46b98f 25/96: More work on integrating parser

[Christian Johansson]

branch: externals/phps-mode
commit f46b98f250fff29a468bcbaca1bb348c264723a8
Author: Christian Johansson <christian@cvj.se>
Commit: Christian Johansson <christian@cvj.se>

    More work on integrating parser
---
 phps-mode-lex-analyzer.el           | 179 ++++++++++++------------------------
 test/phps-mode-test-lex-analyzer.el |   4 +-
 2 files changed, 63 insertions(+), 120 deletions(-)

diff --git a/phps-mode-lex-analyzer.el b/phps-mode-lex-analyzer.el
index 337bd38..09a7673 100644
--- a/phps-mode-lex-analyzer.el
+++ b/phps-mode-lex-analyzer.el
@@ -385,9 +385,9 @@
 
         ;; Setup lexer state
         (setq-local
-         phps-mode-lexer--generated-tokens
+         phps-mode-lexer--states
          (if before-lexer-state
-             (nth 6 before-lexer-state)
+             (nth 0 before-lexer-state)
            nil))
         (setq-local
          phps-mode-lexer--state
@@ -396,11 +396,6 @@
              (nth 1 before-lexer-state)
            'ST_INITIAL))
         (setq-local
-         phps-mode-lexer--states
-         (if before-lexer-state
-             (nth 0 before-lexer-state)
-           nil))
-        (setq-local
          phps-mode-lexer--state-stack
          (if before-lexer-state
              (nth 2 before-lexer-state)
@@ -418,7 +413,12 @@
         (setq
          phps-mode-lexer--nest-location-stack
          (if before-lexer-state
-             (nth 4 before-lexer-state)
+             (nth 5 before-lexer-state)
+           nil))
+        (setq-local
+         phps-mode-lexer--generated-tokens
+         (if before-lexer-state
+             (nth 6 before-lexer-state)
            nil))
 
         ;; Catch errors to kill generated buffer
@@ -471,12 +471,13 @@
           ;; Copy lexer variables outside of buffer
           (let ((after-lexer-state
                  (list
+                  phps-mode-lexer--states
                   phps-mode-lexer--state
                   phps-mode-lexer--state-stack
-                  phps-mode-lexer--states
-                  (nreverse phps-mode-lexer--generated-tokens)
                   phps-mode-lexer--heredoc-label
-                  phps-mode-lexer--heredoc-label-stack)))
+                  phps-mode-lexer--heredoc-label-stack
+                  phps-mode-lexer--nest-location-stack
+                  (nreverse phps-mode-lexer--generated-tokens))))
             (kill-buffer)
             (list
              after-lexer-state
@@ -500,13 +501,17 @@
      buffer-name
 
      (lambda()
-       (let* ((lex-result
-               (phps-mode-lex-analyzer--lex-string buffer-contents))
+       (let* ((parser-results
+               (phps-mode-lex-analyzer--parse-string
+                buffer-contents))
+              (lex-result (nth 0 parser-results))
               (processed-result
                (phps-mode-lex-analyzer--process-tokens-in-string
-                (nth 0 lex-result)
+                (nth 6 lex-result)
                 buffer-contents)))
-         (list lex-result processed-result)))
+         (list
+          lex-result
+          processed-result)))
 
      (lambda(result)
        (when (get-buffer buffer-name)
@@ -515,13 +520,13 @@
                  (processed-result (nth 1 result)))
 
              ;; Move variables into this buffers local variables
-             (setq phps-mode-lex-analyzer--tokens (nth 0 lex-result))
-             (setq phps-mode-lex-analyzer--states (nth 1 lex-result))
-             (setq phps-mode-lex-analyzer--state (nth 2 lex-result))
-             (setq phps-mode-lex-analyzer--state-stack (nth 3 lex-result))
-             (setq phps-mode-lex-analyzer--heredoc-label (nth 4 lex-result))
-             (setq phps-mode-lex-analyzer--heredoc-label-stack (nth 5 
lex-result))
-             (setq phps-mode-lex-analyzer--nest-location-stack (nth 6 
lex-result))
+             (setq phps-mode-lex-analyzer--states (nth 0 lex-result))
+             (setq phps-mode-lex-analyzer--state (nth 1 lex-result))
+             (setq phps-mode-lex-analyzer--state-stack (nth 2 lex-result))
+             (setq phps-mode-lex-analyzer--heredoc-label (nth 3 lex-result))
+             (setq phps-mode-lex-analyzer--heredoc-label-stack (nth 4 
lex-result))
+             (setq phps-mode-lex-analyzer--nest-location-stack (nth 5 
lex-result))
+             (setq phps-mode-lex-analyzer--tokens (nth 6 lex-result))
 
              ;; Save processed result
              (setq phps-mode-lex-analyzer--processed-buffer-p t)
@@ -584,7 +589,18 @@
   (let ((async (and (boundp 'phps-mode-async-process)
                     phps-mode-async-process))
         (async-by-process (and (boundp 'phps-mode-async-process-using-async-el)
-                               phps-mode-async-process-using-async-el)))
+                               phps-mode-async-process-using-async-el))
+        (before-lexer-state
+         (list
+          head-states
+          incremental-state
+          incremental-state-stack
+          incremental-heredoc-label
+          incremental-heredoc-label-stack
+          incremental-nest-location-stack
+          head-tokens))
+        (before-parser-state))
+    ;; TODO Build before-parser-state
     (when force-synchronous
       (setq async nil))
     (phps-mode-serial-commands
@@ -592,23 +608,23 @@
      buffer-name
 
      (lambda()
-       (let* ((lex-result
-               (phps-mode-lex-analyzer--lex-string
+       (let* ((parser-results
+               (phps-mode-lex-analyzer--parse-string
                 buffer-contents
                 incremental-start-new-buffer
-                point-max
-                head-states
-                incremental-state
-                incremental-state-stack
-                incremental-heredoc-label
-                incremental-heredoc-label-stack
-                incremental-nest-location-stack
-                head-tokens))
+                before-lexer-state
+                before-parser-state))
+              (lex-result
+               (nth 0 parser-results))
               (processed-result
                (phps-mode-lex-analyzer--process-tokens-in-string
-                (nth 0 lex-result)
+                (nth 6 lex-result)
                 buffer-contents)))
-         (list lex-result processed-result)))
+
+         ;; TODO Add Parser AST as return as well
+         (list
+          lex-result
+          processed-result)))
 
      (lambda(result)
        (when (get-buffer buffer-name)
@@ -619,13 +635,13 @@
              (phps-mode-debug-message
               (message "Incrementally-lexed-string: %s" result))
 
-             (setq phps-mode-lex-analyzer--tokens (nth 0 lex-result))
-             (setq phps-mode-lex-analyzer--states (nth 1 lex-result))
-             (setq phps-mode-lex-analyzer--state (nth 2 lex-result))
-             (setq phps-mode-lex-analyzer--state-stack (nth 3 lex-result))
-             (setq phps-mode-lex-analyzer--heredoc-label (nth 4 lex-result))
-             (setq phps-mode-lex-analyzer--heredoc-label-stack (nth 5 
lex-result))
-             (setq phps-mode-lex-analyzer--nest-location-stack (nth 6 
lex-result))
+             (setq phps-mode-lex-analyzer--states (nth 0 lex-result))
+             (setq phps-mode-lex-analyzer--state (nth 1 lex-result))
+             (setq phps-mode-lex-analyzer--state-stack (nth 2 lex-result))
+             (setq phps-mode-lex-analyzer--heredoc-label (nth 3 lex-result))
+             (setq phps-mode-lex-analyzer--heredoc-label-stack (nth 4 
lex-result))
+             (setq phps-mode-lex-analyzer--nest-location-stack (nth 5 
lex-result))
+             (setq phps-mode-lex-analyzer--tokens (nth 6 lex-result))
 
              ;; Save processed result
              (setq phps-mode-lex-analyzer--processed-buffer-p t)
@@ -1180,8 +1196,9 @@ SQUARE-BRACKET-LEVEL and ROUND-BRACKET-LEVEL."
       (progn
         (phps-mode-debug-message
          (message
-          "\nCalculation indentation and imenu for all lines in buffer:\n\n%s"
-          string))
+          "\nCalculation indentation and imenu for all lines in 
buffer:\n\n%S\n\nTokens:\n\n%S\n\n"
+          string
+          tokens))
         (let ((in-heredoc nil)
               (in-heredoc-started-this-line nil)
               (in-heredoc-ended-this-line nil)
@@ -3244,80 +3261,6 @@ SQUARE-BRACKET-LEVEL and ROUND-BRACKET-LEVEL."
   (unless phps-mode-lex-analyzer--state
     (setq phps-mode-lex-analyzer--state 'ST_INITIAL)))
 
-(defun phps-mode-lex-analyzer--lex-string (contents &optional start end states 
state state-stack heredoc-label heredoc-label-stack nest-location-stack tokens)
-  "Run lexer on CONTENTS."
-  ;; Create a separate buffer, run lexer inside of it, catch errors and return 
them
-  ;; to enable nice presentation
-  (require 'phps-mode-macros)
-  (let ((buffer (generate-new-buffer "*PHPs Lexer*")))
-
-    ;; Create temporary buffer and run lexer in it
-    (when (get-buffer buffer)
-      (with-current-buffer buffer
-        (insert contents)
-
-        (if tokens
-            (setq
-             phps-mode-lexer--generated-tokens
-             (nreverse tokens))
-          (setq
-           phps-mode-lexer--generated-tokens
-           nil))
-        (if state
-            (setq
-             phps-mode-lexer--state state)
-          (setq
-           phps-mode-lexer--state
-           'ST_INITIAL))
-
-        (setq
-         phps-mode-lexer--states
-         states)
-        (setq
-         phps-mode-lexer--state-stack
-         state-stack)
-        (setq
-         phps-mode-lexer--heredoc-label
-         heredoc-label)
-        (setq
-         phps-mode-lexer--heredoc-label-stack
-         heredoc-label-stack)
-        (setq
-         phps-mode-lexer--nest-location-stack
-         nest-location-stack)
-
-        ;; Setup lexer settings
-        (when (boundp 'phps-mode-syntax-table)
-          (setq semantic-lex-syntax-table phps-mode-syntax-table))
-        (setq semantic-lex-analyzer #'phps-mode-lex-analyzer--re2c-lex)
-
-        ;; Catch errors to kill generated buffer
-        (let ((got-error t))
-          (unwind-protect
-              ;; Run lexer or incremental lexer
-              (progn
-                (if (and start end)
-                    (let ((incremental-tokens (semantic-lex start end)))
-                      (setq
-                       phps-mode-lex-analyzer--tokens
-                       (append tokens incremental-tokens)))
-                  (setq
-                   phps-mode-lex-analyzer--tokens
-                   (semantic-lex-buffer)))
-                (setq got-error nil))
-            (when got-error
-              (kill-buffer))))
-
-        ;; Copy variables outside of buffer
-        (setq state phps-mode-lexer--state)
-        (setq state-stack phps-mode-lexer--state-stack)
-        (setq states phps-mode-lexer--states)
-        (setq tokens (nreverse phps-mode-lexer--generated-tokens))
-        (setq heredoc-label phps-mode-lexer--heredoc-label)
-        (setq heredoc-label-stack phps-mode-lexer--heredoc-label-stack)
-        (kill-buffer))))
-  (list tokens states state state-stack heredoc-label heredoc-label-stack))
-
 (provide 'phps-mode-lex-analyzer)
 
 ;;; phps-mode-lex-analyzer.el ends here
diff --git a/test/phps-mode-test-lex-analyzer.el 
b/test/phps-mode-test-lex-analyzer.el
index e673430..3a1275f 100644
--- a/test/phps-mode-test-lex-analyzer.el
+++ b/test/phps-mode-test-lex-analyzer.el
@@ -1415,11 +1415,11 @@
             (list (list " id $var" 1) (list (list 8 12) 1) (list (list 27 31) 
1) (list (list 72 77) 0)))))
 
   (phps-mode-test--with-buffer
-   "<?php\n\n$var2 = 4;\n\nfunction myFunction($var)\n{\n    $var3 = 3\n    if 
($var) {\n        echo 'Hit';\n    }\n    if ($var2) {\n        echo 'Miss';\n  
  }\n    if ($var3) {\n        echo 'Hit';\n    }\n}\n\nfunction 
myFunction2($abc)\n{\n    if ($var) {\n        echo 'Miss';\n    }\n    if 
($abc) {\n        echo 'Hit';\n    }\n}\n\nif ($var) {\n    echo 'Miss';\n}\nif 
($var2) {\n    echo 'Hit';\n}"
+   "<?php\n\n$var2 = 4;\n\nfunction myFunction($var)\n{\n    $var3 = 3;\n    
if ($var) {\n        echo 'Hit';\n    }\n    if ($var2) {\n        echo 
'Miss';\n    }\n    if ($var3) {\n        echo 'Hit';\n    }\n}\n\nfunction 
myFunction2($abc)\n{\n    if ($var) {\n        echo 'Miss';\n    }\n    if 
($abc) {\n        echo 'Hit';\n    }\n}\n\nif ($var) {\n    echo 'Miss';\n}\nif 
($var2) {\n    echo 'Hit';\n}"
    "Bookkeeping in function level with variable assignments."
    (should (equal
             (phps-mode-test--hash-to-list 
(phps-mode-lex-analyzer--get-bookkeeping) t)
-            (list (list " id $var2" 1) (list (list 8 13) 1) (list " function 
myFunction id $var" 1) (list (list 40 44) 1) (list " function myFunction id 
$var3" 1) (list (list 52 57) 1) (list (list 70 74) 1) (list (list 112 117) 0) 
(list (list 156 161) 1) (list " function myFunction2 id $abc" 1) (list (list 
215 219) 1) (list (list 231 235) 0) (list (list 274 278) 1) (list (list 315 
319) 0) (list (list 346 351) 1)))))
+            '((" id $var2" 1) ((8 13) 1) (" function myFunction id $var" 1) 
((40 44) 1) (" function myFunction id $var3" 1) ((52 57) 1) ((71 75) 1) ((113 
118) 0) ((157 162) 1) (" function myFunction2 id $abc" 1) ((216 220) 1) ((232 
236) 0) ((275 279) 1) ((316 320) 0) ((347 352) 1)))))
 
   (phps-mode-test--with-buffer
    "<?php\n\n// Super-globals\n\nif ($_GET) {\n    echo 'Hit';\n}\nif ($_POST) 
{\n    echo 'Hit';\n}\nif ($_COOKIE) {\n    echo 'Hit';\n}\nif ($_SESSION) {\n  
  echo 'Hit';\n}\nif ($_REQUEST) {\n    echo 'Hit';\n}\nif ($GLOBALS) {\n    
echo 'Hit';\n}\nif ($_SERVER) {\n    echo 'Hit';\n}\nif ($_FILES) {\n    echo 
'Hit';\n}\nif ($_ENV) {\n    echo 'Hit';\n}\nif ($argc) {\n    echo 
'Hit';\n}\nif ($argv) {\n    echo 'Hit';\n}\nif ($http_​response_​header) {\n   
 echo 'Hit';\n}"