[elpa] externals/phps-mode 09d0bca 18/62: Passing old unit tests for lexer simple tokens

[Christian Johansson]

branch: externals/phps-mode
commit 09d0bcab7d642810244a07beeb99cd85d07df106
Author: Christian Johansson <christian@mediastrategi.se>
Commit: Christian Johansson <christian@mediastrategi.se>

    Passing old unit tests for lexer simple tokens
---
 phps-mode-lexer.el           |  78 +++++++++++++++++++-------
 test/phps-mode-test-lexer.el | 130 ++++++++++++++++++++++++++++---------------
 2 files changed, 145 insertions(+), 63 deletions(-)

diff --git a/phps-mode-lexer.el b/phps-mode-lexer.el
index a1dcfcd..fc3cf7a 100644
--- a/phps-mode-lexer.el
+++ b/phps-mode-lexer.el
@@ -114,10 +114,6 @@
 ;; VARIABLES
 
 
-;; TODO Figure out way to use this
-(defvar-local phps-mode-lexer--expected nil
-  "Flag whether something is expected or not.")
-
 (defvar-local phps-mode-lexer--generated-tokens nil
   "List of current generated tokens.")
 
@@ -154,18 +150,35 @@
 ;; HELPER FUNCTIONS
 
 
+(defun phps-mode-lexer--parser-mode ()
+  "Return whether we have some expected value or not."
+  nil)
+
 (defun phps-mode-lexer--begin (state)
   "Begin STATE."
+  (phps-mode-debug-message
+   (message "Begin state: %s" state))
   (setq phps-mode-lexer--state state))
 
 (defun phps-mode-lexer--yy-push-state (state)
   "Add STATE to stack and then begin state."
-  (push phps-mode-lexer--state phps-mode-lexer--state-stack)
+  (push
+   phps-mode-lexer--state
+   phps-mode-lexer--state-stack)
+  (phps-mode-debug-message
+   (message
+    "Pushed state: %s"
+    phps-mode-lexer--state))
   (phps-mode-lexer--begin state))
 
 (defun phps-mode-lexer--yy-pop-state ()
   "Pop current state from stack."
   (let ((old-state (pop phps-mode-lexer--state-stack)))
+    (phps-mode-debug-message
+     (message
+      "Popped state: %s"
+      old-state))
+
     ;; (message "Going back to poppped state %s" old-state)
     (if old-state
         (phps-mode-lexer--begin old-state)
@@ -200,6 +213,10 @@
      (buffer-substring-no-properties
       (match-beginning 0)
       (match-end 0))))
+  (phps-mode-debug-message
+   (message
+    "Entered nesting '%s'"
+    opening))
   (push
    opening
    phps-mode-lexer--nest-location-stack))
@@ -221,7 +238,7 @@
     (signal
      'phps-lexer-error
      (list
-      (format "Unmatched '%s' at '%d'" closing (point))
+      (format "Unmatched '%s' at point %d" closing (point))
       (point))))
   (let ((opening
          (car
@@ -239,6 +256,10 @@
        (list
         (format "Bad nesting '%s' vs '%s' at %d'" opening closing (point))
         (point))))
+    (phps-mode-debug-message
+     (message
+      "Exited nesting '%s'"
+      closing))
     (pop phps-mode-lexer--nest-location-stack)
     t))
 
@@ -249,7 +270,10 @@
 
   (phps-mode-debug-message
    (message
-    "Emitted token %s"
+    "Emitted token '%s' -> %s"
+    (buffer-substring-no-properties
+     start
+     end)
     `(,token ,start . ,end)))
   
   ;; Push token start, end, lexer state and state stack to variable
@@ -281,13 +305,13 @@
             (setq escaped nil)))))
     pos))
 
-;; TODO Figure out what this does
 (defun phps-mode-lexer--skip-token (_token &optional start end)
   "Skip TOKEN to list with START and END."
   (unless start
     (setq start (match-beginning 0)))
   (unless end
-    (setq end (match-end 0))))
+    (setq end (match-end 0)))
+  (setq semantic-lex-end-point end))
 
 (defmacro phps-mode-lexer--match-macro (conditions &rest body)
   "Check if CONDITIONS hold, if so execute BODY."
@@ -327,7 +351,7 @@
   "Return end token."
   (if (and
        (phps-mode-lexer--check-nesting-at-end)
-       (phps-mode-parser-grammar-macro-CG 'parser-mode))
+       (phps-mode-lexer--parser-mode))
       (phps-mode-lexer--return-token 'T_ERROR)
     (phps-mode-lexer--return-token 'END)))
 
@@ -370,7 +394,7 @@
       end)))
   (if (and
        (phps-mode-lexer--exit-nesting token)
-       (phps-mode-parser-grammar-macro-CG 'parser-mode))
+       (phps-mode-lexer--parser-mode))
       (phps-mode-lexer--return-token 'T_ERROR)
     (phps-mode-lexer--return-token token start end)))
 
@@ -384,7 +408,8 @@
 
 (defun phps-mode-lexer--return-or-skip-token (&optional token start end)
   "Return TOKEN with START and END but only in parse-mode."
-  (when (phps-mode-parser-grammar-macro-CG 'parser-mode)
+  (if (phps-mode-lexer--parser-mode)
+      (phps-mode-lexer--skip-token token start end)
     (phps-mode-lexer--return-token token start end)))
 
 
@@ -431,9 +456,21 @@
   (setq phps-mode-lexer--restart-flag nil)
   (let ((old-start (point)))
     (phps-mode-debug-message
+     (let ((start (point))
+           (end (+ (point) 5))
+           (lookahead))
+       (when (> end (point-max))
+         (setq end (point-max)))
+       (setq
+        lookahead
+        (buffer-substring-no-properties
+         start
+         end))
      (message
-      "Running lexer from %s"
-      old-start))
+      "\nRunning lexer from point %s, state: %s, lookahead: '%s'.."
+      old-start
+      phps-mode-lexer--state
+      lookahead)))
     (phps-mode-lexer--reset-match-data)
     
     (let ((SHEBANG (equal phps-mode-lexer--state 'SHEBANG))
@@ -713,7 +750,7 @@
               "\\(real\\)"
               phps-mode-lexer--tabs-and-spaces
               ")")))
-       (when (phps-mode-parser-grammar-macro-CG 'parser-mode)
+       (when (phps-mode-lexer--parser-mode)
          (signal
           'phps-lexer-error
           (list
@@ -1152,7 +1189,7 @@
       (phps-mode-lexer--match-macro
        (and ST_INITIAL (looking-at "<\\?="))
        (phps-mode-lexer--begin 'ST_IN_SCRIPTING)
-       (when (phps-mode-parser-grammar-macro-CG 'parser-mode)
+       (when (phps-mode-lexer--parser-mode)
          (phps-mode-lexer--return-token-with-indent 'T_ECHO))
        (phps-mode-lexer--return-token 'T_OPEN_TAG_WITH_ECHO))
 
@@ -1402,7 +1439,7 @@
          (when (= (- end start) 3)
            (setq end (1- end)))
          (phps-mode-lexer--begin 'ST_INITIAL)
-         (when (phps-mode-parser-grammar-macro-CG 'parser-mode)
+         (when (phps-mode-lexer--parser-mode)
            (phps-mode-lexer--return-token
             ";"
             start
@@ -1581,11 +1618,14 @@
          (phps-mode-lexer--return-token 'T_END_HEREDOC start end)))
 
       (phps-mode-lexer--match-macro
-       (and (or ST_DOUBLE_QUOTES ST_BACKQUOTE ST_HEREDOC) (looking-at (concat 
"{\\$")))
+       (and (or ST_DOUBLE_QUOTES ST_BACKQUOTE ST_HEREDOC) (looking-at "{\\$"))
        (phps-mode-lexer--yy-push-state 'ST_IN_SCRIPTING)
        (phps-mode-lexer--yyless 1)
        (phps-mode-lexer--enter-nesting "{")
-       (phps-mode-lexer--return-token 'T_CURLY_OPEN (match-beginning 0) (- 
(match-end 0) 1)))
+       (phps-mode-lexer--return-token
+        'T_CURLY_OPEN
+        (match-beginning 0)
+        (- (match-end 0) 1)))
 
       (phps-mode-lexer--match-macro
        (and ST_DOUBLE_QUOTES (looking-at "[\"]"))
diff --git a/test/phps-mode-test-lexer.el b/test/phps-mode-test-lexer.el
index 7b9b077..f74ad53 100644
--- a/test/phps-mode-test-lexer.el
+++ b/test/phps-mode-test-lexer.el
@@ -35,34 +35,42 @@
   (phps-mode-test--with-buffer
    "<?php\t$öar=1; exit $var;\t?>"
    "Simple PHP with two expression"
-   (should (equal
-            phps-mode-lex-analyzer--tokens
-            '((T_OPEN_TAG 1 . 7) (T_VARIABLE 7 . 11) ("=" 11 . 12) (T_LNUMBER 
12 . 13) (";" 13 . 14) (T_EXIT 15 . 19) (T_VARIABLE 20 . 24) (";" 24 . 25) (";" 
26 . 28) (T_CLOSE_TAG 26 . 28)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_VARIABLE 7 . 11) ("=" 11 . 12) (T_LNUMBER 12 . 
13) (";" 13 . 14) (T_EXIT 15 . 19) (T_VARIABLE 20 . 24) (";" 24 . 25) 
(T_CLOSE_TAG 26 . 28)))))
 
   (phps-mode-test--with-buffer
    "<?php\nexit;\n?>"
    "Minimal PHP expression"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_EXIT 7 . 11) (";" 11 . 12) (";" 13 . 
15) (T_CLOSE_TAG 13 . 15)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_EXIT 7 . 11) (";" 11 . 12) (T_CLOSE_TAG 13 . 
15)))))
 
   (phps-mode-test--with-buffer
    "<?php exit; ?>"
    "Small PHP file"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_EXIT 7 . 11) (";" 11 . 12) (";" 13 . 
15) (T_CLOSE_TAG 13 . 15)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_EXIT 7 . 11) (";" 11 . 12) (T_CLOSE_TAG 13 . 
15)))))
 
   (phps-mode-test--with-buffer
    "<html><head>blabla</head<body>\n\n \t<?php\nexit;\n?>\n\n</body></html>"
    "Mixed inline HTML and PHP"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_INLINE_HTML 1 . 35) (T_OPEN_TAG 35 . 41) (T_EXIT 41 . 
45) (";" 45 . 46) (";" 47 . 49) (T_CLOSE_TAG 47 . 49) (T_INLINE_HTML 49 . 
65)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_INLINE_HTML 1 . 35) (T_OPEN_TAG 35 . 41) (T_EXIT 41 . 45) (";" 45 . 
46) (T_CLOSE_TAG 47 . 49) (T_INLINE_HTML 49 . 65)))))
 
   (phps-mode-test--with-buffer
    "\n\n \t<html><title>echo 
\"Blaha\";</title><?php\n\n\nexit?>\n\n<html><random /></html><?php exit ?>"
    "Another mixed inline HTML and PHP"
-   (should (equal
-            phps-mode-lex-analyzer--tokens
-            '((T_INLINE_HTML 1 . 39) (T_OPEN_TAG 39 . 45) (T_EXIT 47 . 51) 
(";" 51 . 53) (T_CLOSE_TAG 51 . 53) (T_INLINE_HTML 53 . 78) (T_OPEN_TAG 78 . 
84) (T_EXIT 84 . 88) (";" 89 . 91) (T_CLOSE_TAG 89 . 91)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_INLINE_HTML 1 . 39) (T_OPEN_TAG 39 . 45) (T_EXIT 47 . 51) 
(T_CLOSE_TAG 51 . 53) (T_INLINE_HTML 53 . 78) (T_OPEN_TAG 78 . 84) (T_EXIT 84 . 
88) (T_CLOSE_TAG 89 . 91)))))
 
   (phps-mode-test--with-buffer
    "<?php\n\n$k = 'key';\n\necho \"\\$a['{$k}']\";"
@@ -70,7 +78,7 @@
    ;; (message "Tokens: %s" phps-mode-lex-analyzer--tokens)
    (should (equal
             phps-mode-lex-analyzer--tokens
-            '((T_OPEN_TAG 1 . 7) (T_VARIABLE 8 . 10) ("=" 11 . 12) 
(T_CONSTANT_ENCAPSED_STRING 13 . 18) (";" 18 . 19) (T_ECHO 21 . 25) ("\"" 26 . 
27) (T_ENCAPSED_AND_WHITESPACE 27 . 32) (T_CURLY_OPEN 32 . 33) (T_VARIABLE 33 . 
35) ("}" 35 . 36) (T_CONSTANT_ENCAPSED_STRING 36 . 38) ("\"" 38 . 39) (";" 39 . 
40)))))
+            '((T_OPEN_TAG 1 . 7) (T_VARIABLE 8 . 10) ("=" 11 . 12) 
(T_CONSTANT_ENCAPSED_STRING 13 . 18) (";" 18 . 19) (T_ECHO 21 . 25) ("\"" 26 . 
27) (T_CONSTANT_ENCAPSED_STRING 27 . 32) (T_CURLY_OPEN 32 . 33) (T_VARIABLE 33 
. 35) ("}" 35 . 36) (T_CONSTANT_ENCAPSED_STRING 36 . 38) ("\"" 38 . 39) (";" 39 
. 40)))))
 
   )
 
@@ -88,103 +96,135 @@
   (phps-mode-test--with-buffer
    "<?php if 
(empty($parameters[self::PARAMETER_CONFIGURATION_INTERNAL_FILENAME])) { 
$parameters[self::PARAMETER_CONFIGURATION_INTERNAL_FILENAME] = ''; }"
    "Complex PHP with conditional"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_IF 7 . 9) ("(" 10 . 11) (T_EMPTY 11 
. 16) ("(" 16 . 17) (T_VARIABLE 17 . 28) ("[" 28 . 29) (T_STRING 29 . 33) 
(T_PAAMAYIM_NEKUDOTAYIM 33 . 35) (T_STRING 35 . 76) ("]" 76 . 77) (")" 77 . 78) 
(")" 78 . 79) ("{" 80 . 81) (T_VARIABLE 82 . 93) ("[" 93 . 94) (T_STRING 94 . 
98) (T_PAAMAYIM_NEKUDOTAYIM 98 . 100) (T_STRING 100 . 141) ("]" 141 . 142) ("=" 
143 . 144) (T_CONSTANT_ENCAPSED_STRING 145 . 147) (";" 147 . 148) ("}" 149 . 
150)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_IF 7 . 9) ("(" 10 . 11) (T_EMPTY 11 . 16) ("(" 16 
. 17) (T_VARIABLE 17 . 28) ("[" 28 . 29) (T_STRING 29 . 33) 
(T_PAAMAYIM_NEKUDOTAYIM 33 . 35) (T_STRING 35 . 76) ("]" 76 . 77) (")" 77 . 78) 
(")" 78 . 79) ("{" 80 . 81) (T_VARIABLE 82 . 93) ("[" 93 . 94) (T_STRING 94 . 
98) (T_PAAMAYIM_NEKUDOTAYIM 98 . 100) (T_STRING 100 . 141) ("]" 141 . 142) ("=" 
143 . 144) (T_CONSTANT_ENCAPSED_STRING 145 . 147) (";" 147 . 148) ("}" 149 . 
150)))))
 
   (phps-mode-test--with-buffer
    "<?php echo $var = array(\"\");"
    "Simple PHP with empty array assignment"
-   (should (equal phps-mode-lex-analyzer--tokens
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
                   '((T_OPEN_TAG 1 . 7) (T_ECHO 7 . 11) (T_VARIABLE 12 . 16) 
("=" 17 . 18) (T_ARRAY 19 . 24) ("(" 24 . 25) (T_CONSTANT_ENCAPSED_STRING 25 . 
27) (")" 27 . 28) (";" 28 . 29)))))
 
   (phps-mode-test--with-buffer
    "<?php echo $var = array('abc' => '123');"
    "Simple PHP with associative array assignment"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_ECHO 7 . 11) (T_VARIABLE 12 . 16) 
("=" 17 . 18) (T_ARRAY 19 . 24) ("(" 24 . 25) (T_CONSTANT_ENCAPSED_STRING 25 . 
30) (T_DOUBLE_ARROW 31 . 33) (T_CONSTANT_ENCAPSED_STRING 34 . 39) (")" 39 . 40) 
(";" 40 . 41)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_ECHO 7 . 11) (T_VARIABLE 12 . 16) ("=" 17 . 18) 
(T_ARRAY 19 . 24) ("(" 24 . 25) (T_CONSTANT_ENCAPSED_STRING 25 . 30) 
(T_DOUBLE_ARROW 31 . 33) (T_CONSTANT_ENCAPSED_STRING 34 . 39) (")" 39 . 40) 
(";" 40 . 41)))))
 
   (phps-mode-test--with-buffer
    "<?php $var = []; "
    "PHP with short-handed array declaration assignment"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_VARIABLE 7 . 11) ("=" 12 . 13) ("[" 
14 . 15) ("]" 15 . 16) (";" 16 . 17)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_VARIABLE 7 . 11) ("=" 12 . 13) ("[" 14 . 15) ("]" 
15 . 16) (";" 16 . 17)))))
 
   (phps-mode-test--with-buffer
    "<?php $var = ''; $var = 'abc'; "
    "PHP with string assignments"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_VARIABLE 7 . 11) ("=" 12 . 13) 
(T_CONSTANT_ENCAPSED_STRING 14 . 16) (";" 16 . 17) (T_VARIABLE 18 . 22) ("=" 23 
. 24) (T_CONSTANT_ENCAPSED_STRING 25 . 30) (";" 30 . 31)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_VARIABLE 7 . 11) ("=" 12 . 13) 
(T_CONSTANT_ENCAPSED_STRING 14 . 16) (";" 16 . 17) (T_VARIABLE 18 . 22) ("=" 23 
. 24) (T_CONSTANT_ENCAPSED_STRING 25 . 30) (";" 30 . 31)))))
 
   (phps-mode-test--with-buffer
    "<?php\nswitch (myRandomCondition()) {\ncase 'Something here':\necho 
'Something else here';\n}\n"
    "Switch case PHP"
    ;; (message "Tokens: %s" phps-mode-lex-analyzer--tokens)
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_SWITCH 7 . 13) ("(" 14 . 15) 
(T_STRING 15 . 32) ("(" 32 . 33) (")" 33 . 34) (")" 34 . 35) ("{" 36 . 37) 
(T_CASE 38 . 42) (T_CONSTANT_ENCAPSED_STRING 43 . 59) (":" 59 . 60) (T_ECHO 61 
. 65) (T_CONSTANT_ENCAPSED_STRING 66 . 87) (";" 87 . 88) ("}" 89 . 90)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_SWITCH 7 . 13) ("(" 14 . 15) (T_STRING 15 . 32) 
("(" 32 . 33) (")" 33 . 34) (")" 34 . 35) ("{" 36 . 37) (T_CASE 38 . 42) 
(T_CONSTANT_ENCAPSED_STRING 43 . 59) (":" 59 . 60) (T_ECHO 61 . 65) 
(T_CONSTANT_ENCAPSED_STRING 66 . 87) (";" 87 . 88) ("}" 89 . 90)))))
 
   (phps-mode-test--with-buffer
    "<?php $var = \"\"; $var = \"abc\"; $var = \"abc\\def\\ghj\";"
    "PHP with three string assignments"
    ;; (message "Tokens: %s" phps-mode-lex-analyzer--tokens)
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_VARIABLE 7 . 11) ("=" 12 . 13) 
(T_CONSTANT_ENCAPSED_STRING 14 . 16) (";" 16 . 17) (T_VARIABLE 18 . 22) ("=" 23 
. 24) (T_CONSTANT_ENCAPSED_STRING 25 . 30) (";" 30 . 31) (T_VARIABLE 32 . 36) 
("=" 37 . 38) (T_CONSTANT_ENCAPSED_STRING 39 . 52) (";" 52 . 53)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_VARIABLE 7 . 11) ("=" 12 . 13) 
(T_CONSTANT_ENCAPSED_STRING 14 . 16) (";" 16 . 17) (T_VARIABLE 18 . 22) ("=" 23 
. 24) (T_CONSTANT_ENCAPSED_STRING 25 . 30) (";" 30 . 31) (T_VARIABLE 32 . 36) 
("=" 37 . 38) (T_CONSTANT_ENCAPSED_STRING 39 . 52) (";" 52 . 53)))))
 
   (phps-mode-test--with-buffer
    "<?php echo isset($backtrace[1]['file']) ? 'yes' : 'no'; "
    "PHP with short-handed conditional echo"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_ECHO 7 . 11) (T_ISSET 12 . 17) ("(" 
17 . 18) (T_VARIABLE 18 . 28) ("[" 28 . 29) (T_LNUMBER 29 . 30) ("]" 30 . 31) 
("[" 31 . 32) (T_CONSTANT_ENCAPSED_STRING 32 . 38) ("]" 38 . 39) (")" 39 . 40) 
("?" 41 . 42) (T_CONSTANT_ENCAPSED_STRING 43 . 48) (":" 49 . 50) 
(T_CONSTANT_ENCAPSED_STRING 51 . 55) (";" 55 . 56)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_ECHO 7 . 11) (T_ISSET 12 . 17) ("(" 17 . 18) 
(T_VARIABLE 18 . 28) ("[" 28 . 29) (T_LNUMBER 29 . 30) ("]" 30 . 31) ("[" 31 . 
32) (T_CONSTANT_ENCAPSED_STRING 32 . 38) ("]" 38 . 39) (")" 39 . 40) ("?" 41 . 
42) (T_CONSTANT_ENCAPSED_STRING 43 . 48) (":" 49 . 50) 
(T_CONSTANT_ENCAPSED_STRING 51 . 55) (";" 55 . 56)))))
 
   (phps-mode-test--with-buffer
    "<?php myFunction(); "
    "A single function call"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_STRING 7 . 17) ("(" 17 . 18) (")" 18 
. 19) (";" 19 . 20)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_STRING 7 . 17) ("(" 17 . 18) (")" 18 . 19) (";" 
19 . 20)))))
 
   (phps-mode-test--with-buffer
    "<?php // echo 'random';?><!--</div>-->"
    "Commented out PHP expression and inline-html"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_COMMENT 7 . 24) (";" 24 . 26) 
(T_CLOSE_TAG 24 . 26) (T_INLINE_HTML 26 . 39)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_COMMENT 7 . 24) (T_CLOSE_TAG 24 . 26) 
(T_INLINE_HTML 26 . 39)))))
 
   (phps-mode-test--with-buffer
    "<?php //echo $contact_position;?><!--</div>-->"
    "Commented out PHP expression and inline-html 2"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_COMMENT 7 . 32) (";" 32 . 34) 
(T_CLOSE_TAG 32 . 34) (T_INLINE_HTML 34 . 47)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_COMMENT 7 . 32) (T_CLOSE_TAG 32 . 34) 
(T_INLINE_HTML 34 . 47)))))
 
   (phps-mode-test--with-buffer
    "<?php echo isset($backtrace[1]['file']) ? 'yes' : 'no';\n//a random 
comment\n// another random comment\n/**\n * More comments\n* More\n **/\necho 
$backtrace; ?>"
    "Conditional echo, comment and doc-comment block"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_ECHO 7 . 11) (T_ISSET 12 . 17) ("(" 
17 . 18) (T_VARIABLE 18 . 28) ("[" 28 . 29) (T_LNUMBER 29 . 30) ("]" 30 . 31) 
("[" 31 . 32) (T_CONSTANT_ENCAPSED_STRING 32 . 38) ("]" 38 . 39) (")" 39 . 40) 
("?" 41 . 42) (T_CONSTANT_ENCAPSED_STRING 43 . 48) (":" 49 . 50) 
(T_CONSTANT_ENCAPSED_STRING 51 . 55) (";" 55 . 56) (T_COMMENT 57 . 75) 
(T_COMMENT 76 . 101) (T_DOC_COMMENT 102 . 134) (T_ECHO 135 . 139) (T_VARIABLE 
140 . 150) (";" 150 . 151) (";" 152 . 154)  [...]
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_ECHO 7 . 11) (T_ISSET 12 . 17) ("(" 17 . 18) 
(T_VARIABLE 18 . 28) ("[" 28 . 29) (T_LNUMBER 29 . 30) ("]" 30 . 31) ("[" 31 . 
32) (T_CONSTANT_ENCAPSED_STRING 32 . 38) ("]" 38 . 39) (")" 39 . 40) ("?" 41 . 
42) (T_CONSTANT_ENCAPSED_STRING 43 . 48) (":" 49 . 50) 
(T_CONSTANT_ENCAPSED_STRING 51 . 55) (";" 55 . 56) (T_COMMENT 57 . 75) 
(T_COMMENT 76 . 101) (T_DOC_COMMENT 102 . 134) (T_ECHO 135 . 139) (T_VARIABLE 
140 . 150) (";" 150 . 151) (T_CLOSE_TAG 152 . 154)))))
 
   (phps-mode-test--with-buffer
    "<?php forgerarray($arg1, $arg2)"
    "A function call containing keywords in its name"
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_STRING 7 . 18) ("(" 18 . 19) 
(T_VARIABLE 19 . 24) ("," 24 . 25) (T_VARIABLE 26 . 31) (")" 31 . 32)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_STRING 7 . 18) ("(" 18 . 19) (T_VARIABLE 19 . 24) 
("," 24 . 25) (T_VARIABLE 26 . 31) (")" 31 . 32)))))
 
   (phps-mode-test--with-buffer
    "<?php\n$username = $_GET['user'] ?? 
'nobody';\n$this->request->data['comments']['user_id'] ??= 'value';\n"
    "Coalescing comparison operator and coalescing assignment operator"
    ;; (message "Tokens: %s" phps-mode-lex-analyzer--tokens)
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_VARIABLE 7 . 16) ("=" 17 . 18) 
(T_VARIABLE 19 . 24) ("[" 24 . 25) (T_CONSTANT_ENCAPSED_STRING 25 . 31) ("]" 31 
. 32) (T_COALESCE 33 . 35) (T_CONSTANT_ENCAPSED_STRING 36 . 44) (";" 44 . 45) 
(T_VARIABLE 46 . 51) (T_OBJECT_OPERATOR 51 . 53) (T_STRING 53 . 60) 
(T_OBJECT_OPERATOR 60 . 62) (T_STRING 62 . 66) ("[" 66 . 67) 
(T_CONSTANT_ENCAPSED_STRING 67 . 77) ("]" 77 . 78) ("[" 78 . 79) 
(T_CONSTANT_ENCAPSED_STRING 79 . 88) ("]" 88 . 89) (T_COALESCE_EQUA [...]
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_VARIABLE 7 . 16) ("=" 17 . 18) (T_VARIABLE 19 . 
24) ("[" 24 . 25) (T_CONSTANT_ENCAPSED_STRING 25 . 31) ("]" 31 . 32) 
(T_COALESCE 33 . 35) (T_CONSTANT_ENCAPSED_STRING 36 . 44) (";" 44 . 45) 
(T_VARIABLE 46 . 51) (T_OBJECT_OPERATOR 51 . 53) (T_STRING 53 . 60) 
(T_OBJECT_OPERATOR 60 . 62) (T_STRING 62 . 66) ("[" 66 . 67) 
(T_CONSTANT_ENCAPSED_STRING 67 . 77) ("]" 77 . 78) ("[" 78 . 79) 
(T_CONSTANT_ENCAPSED_STRING 79 . 88) ("]" 88 . 89) (T_COALESCE_EQUAL 90 . 93) 
(T [...]
 
   (phps-mode-test--with-buffer
    "<?php\necho $array['abc'];\necho \"My $array[12] random statement\";\n"
    "Long inside array offset"
    ;; (message "Tokens: %s" phps-mode-lex-analyzer--tokens)
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_ECHO 7 . 11) (T_VARIABLE 12 . 18) 
("[" 18 . 19) (T_CONSTANT_ENCAPSED_STRING 19 . 24) ("]" 24 . 25) (";" 25 . 26) 
(T_ECHO 27 . 31) ("\"" 32 . 33) (T_ENCAPSED_AND_WHITESPACE 33 . 36) (T_VARIABLE 
36 . 42) ("[" 42 . 43) (T_NUM_STRING 43 . 45) ("]" 45 . 46) 
(T_CONSTANT_ENCAPSED_STRING 46 . 63) ("\"" 63 . 64) (";" 64 . 65)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_ECHO 7 . 11) (T_VARIABLE 12 . 18) ("[" 18 . 19) 
(T_CONSTANT_ENCAPSED_STRING 19 . 24) ("]" 24 . 25) (";" 25 . 26) (T_ECHO 27 . 
31) ("\"" 32 . 33) (T_CONSTANT_ENCAPSED_STRING 33 . 36) (T_VARIABLE 36 . 42) 
("[" 42 . 43) (T_NUM_STRING 43 . 45) ("]" 45 . 46) (T_CONSTANT_ENCAPSED_STRING 
46 . 63) ("\"" 63 . 64) (";" 64 . 65)))))
 
   (phps-mode-test--with-buffer
    "<?php\n/*my comment */\n/** my doc comment */"
    "Comment vs doc-comment"
    ;; (message "Tokens: %s" phps-mode-lex-analyzer--tokens)
-   (should (equal phps-mode-lex-analyzer--tokens
-                  '((T_OPEN_TAG 1 . 7) (T_COMMENT 7 . 22) (T_DOC_COMMENT 23 . 
44)))))
+   (should
+    (equal
+     phps-mode-lex-analyzer--tokens
+     '((T_OPEN_TAG 1 . 7) (T_COMMENT 7 . 22) (T_DOC_COMMENT 23 . 44)))))
 
   ;; (phps-mode-test--with-buffer
   ;;  "<?php ??= $var EXIT die function return yield from yield try catch 
finally throw if elseif endif else while endwhile do for endfor foreach 
endforeach declare enddeclare instanceof as switch endswitch case default break 
continue goto echo print class interface trait extends implements :: \\ ... ?? 
new clone var (int) (integer) (real) (double) (float) (string) (binary) (array) 
(object) (boolean) (bool) (unset) eval include include_once require 
require_once namespace use insteadof gl [...]
@@ -193,6 +233,8 @@
   ;;  (should (equal phps-mode-lex-analyzer--tokens
   ;;                 '((T_OPEN_TAG 1 . 7) (T_COALESCE_EQUAL 7 . 10) 
(T_VARIABLE 11 . 15) (T_EXIT 16 . 20) (T_DIE 21 . 24) (T_FUNCTION 25 . 33) 
(T_RETURN 34 . 40) (T_YIELD_FROM 41 . 52) (T_YIELD 52 . 57) (T_TRY 58 . 61) 
(T_CATCH 62 . 67) (T_FINALLY 68 . 75) (T_THROW 76 . 81) (T_IF 82 . 84) 
(T_ELSEIF 85 . 91) (T_ENDIF 92 . 97) (T_ELSE 98 . 102) (T_WHILE 103 . 108) 
(T_ENDWHILE 109 . 117) (T_DO 118 . 120) (T_FOR 121 . 124) (T_ENDFOR 125 . 131) 
(T_FOREACH 132 . 139) (T_ENDFOREACH 140 . 150) ( [...]
 
+  ;; TODO Add PHP 8 new tokens here
+
 
   )