[nongnu] elpa/rust-mode ee7043b 289/486: Use syntax-propertize-function, not font-lock-syntactic-keywords

[ELPA Syncer]

branch: elpa/rust-mode
commit ee7043b56d232b1d5fa5601d34bd15652614cea0
Author: Tom Tromey <tom@tromey.com>
Commit: Tom Tromey <tom@tromey.com>

    Use syntax-propertize-function, not font-lock-syntactic-keywords
    
    font-lock-syntactic-keywords have been obsolete since Emacs 24.1, the
    earliest version supported by rust-mode.  Instead, modes are
    encouraged to use syntax-propertize-function.
    
    syntax-propertize-function provides a generally better experience.
    Syntax propertization is not tied to font lock,so angle bracket
    matching will still work when font-lock is disabled.  There's no
    longer a need to call font-lock-fontify-buffer or font-lock-ensure
    (you can see this in the tests).  The resulting code is also shorter.
    
    I removed a few tests:
    
    * font-lock-raw-string-constant was written assuming font-lock based
      syntax propertization, but that's no longer the case.
    * font-lock-extend-region-in-string and
      rust-test-revert-hook-preserves-point both called functions that no
      longer exist.
    
    There's a fix for a hidden bug in rust-is-in-expression-context.  This
    previously could signal in some situations, but the signal was hidden
    by font-lock.  It was visible now when running tests, hence the new
    call to condition-case.
    
    I suspect this might fix #192, but I haven't tried it.
---
 rust-mode-tests.el |  63 ----------
 rust-mode.el       | 329 +++++++++++++++++------------------------------------
 2 files changed, 107 insertions(+), 285 deletions(-)

diff --git a/rust-mode-tests.el b/rust-mode-tests.el
index 1e637d6..303bb51 100644
--- a/rust-mode-tests.el
+++ b/rust-mode-tests.el
@@ -309,9 +309,6 @@ very very very long string
      deindented
      1
      (lambda ()
-       ;; The indentation will fail in some cases if the syntax properties are
-       ;; not set.  This only happens when font-lock fontifies the buffer.
-       (font-lock-fontify-buffer)
        (indent-region 1 (+ 1 (buffer-size))))
      indented)))
 
@@ -960,7 +957,6 @@ INIT-POS, FINAL-POS are position symbols found in 
`rust-test-positions-alist'."
   (with-temp-buffer
     (rust-mode)
     (insert source-code)
-    (font-lock-fontify-buffer)
     (goto-char (rust-get-buffer-pos init-pos))
     (apply manip-func args)
     (should (equal (point) (rust-get-buffer-pos final-pos)))))
@@ -974,7 +970,6 @@ All positions are position symbols found in 
`rust-test-positions-alist'."
   (with-temp-buffer
     (rust-mode)
     (insert source-code)
-    (font-lock-fontify-buffer)
     (goto-char (rust-get-buffer-pos init-pos))
     (apply manip-func args)
     (should (equal (list (region-beginning) (region-end))
@@ -1377,24 +1372,6 @@ this_is_not_a_string();)"
      "r\" this is a comment\n" font-lock-comment-face
      "\"this is a string\"" font-lock-string-face)))
 
-(ert-deftest font-lock-raw-string-constant ()
-  ;; There was an issue in which a multi-line raw string would be fontified
-  ;; correctly if inserted, but then incorrectly if one of the lines was then
-  ;; edited.  This test replicates how font-lock responds when text in the
-  ;; buffer is modified in order to reproduce it.
-  (with-temp-buffer
-    (rust-mode)
-    (font-lock-fontify-buffer)
-    (insert "const BOO:&str = r#\"\nBOO\"#;")
-    (beginning-of-buffer)
-    (insert " ")
-    (font-lock-after-change-function 1 2 0)
-
-    (should (equal 'font-lock-string-face (get-text-property 19 'face))) ;; 
Opening "r" of raw string
-    (should (equal 'font-lock-string-face (get-text-property 27 'face))) ;; 
Closing "#" of raw string
-    (should (equal nil (get-text-property 28 'face))) ;; Semicolon--should not 
be part of the string
-    ))
-
 (ert-deftest font-lock-runaway-raw-string ()
   (rust-test-font-lock
    "const Z = r#\"my raw string\";\n// oops this is still in the string"
@@ -2552,34 +2529,6 @@ type Foo<T> where T: Copy = Box<T>;
      '(7 9))))
 
 
-(ert-deftest font-lock-extend-region-in-string ()
-  
-  (with-temp-buffer
-    (rust-mode)
-    (insert "
-fn foo() {
-    let x = r\"
-Fontification needs to include this whole string or none of it.
-             \"
-}")
-    (font-lock-fontify-buffer)
-    (let ((font-lock-beg 13)
-          (font-lock-end 42))
-      (rust-font-lock-extend-region)
-      (should (<= font-lock-beg 13))
-      (should (>= font-lock-end 106))
-      )
-    (let ((font-lock-beg 42)
-          (font-lock-end 108))
-      (rust-font-lock-extend-region)
-      (should (<= font-lock-beg 25))
-      (should (>= font-lock-end 108)))
-    (let ((font-lock-beg 1)
-          (font-lock-end 12))
-      (rust-font-lock-extend-region)
-      (should (<= font-lock-beg 1))
-      (should (>= font-lock-end 12)))))
-
 (ert-deftest redo-syntax-after-change-far-from-point ()  
   (let*
       ((tmp-file-name (make-temp-file "rust-mdoe-test-issue104"))
@@ -2603,18 +2552,6 @@ Fontification needs to include this whole string or none 
of it.
     )
   )
 
-(ert-deftest rust-test-revert-hook-preserves-point ()
-  (with-temp-buffer
-    ;; Insert some code, and put point in the middle.
-    (insert "fn foo() {}\n")
-    (insert "fn bar() {}\n")
-    (insert "fn baz() {}\n")
-    (goto-char (point-min))
-    (forward-line 1)
-    (let ((initial-point (point)))
-      (rust--after-revert-hook)
-      (should (equal initial-point (point))))))
-
 (defun test-imenu (code expected-items)
   (with-temp-buffer
     (rust-mode)
diff --git a/rust-mode.el b/rust-mode.el
index dfe3e9a..47c6661 100644
--- a/rust-mode.el
+++ b/rust-mode.el
@@ -35,67 +35,6 @@
 (defconst rust-re-unsafe "unsafe")
 (defconst rust-re-extern "extern")
 
-(defconst rust-re-non-standard-string
-  (rx
-   (or
-    ;; Raw string: if it matches, it ends up with the starting character
-    ;; of the string as group 1, any ending backslashes as group 4, and
-    ;; the ending character as either group 5 or group 6.
-    (seq
-     ;; The "r" starts the raw string.  Capture it as group 1 to mark it as 
such syntactically:
-     (group "r")
-
-     ;; Then either:
-     (or
-      ;; a sequence at least one "#" (followed by quote).  Capture all
-      ;; but the last "#" as group 2 for this case.
-      (seq (group (* "#")) "#\"")
-
-      ;; ...or a quote without any "#".  Capture it as group 3. This is
-      ;; used later to match the opposite quote only if this capture
-      ;; occurred
-      (group "\""))
-
-     ;; The contents of the string:
-     (*? anything)
-
-     ;; If there are any backslashes at the end of the string, capture
-     ;; them as group 4 so we can suppress the normal escape syntax
-     ;; parsing:
-     (group (* "\\"))
-
-     ;; Then the end of the string--the backreferences ensure that we
-     ;; only match the kind of ending that corresponds to the beginning
-     ;; we had:
-     (or
-      ;; There were "#"s - capture the last one as group 5 to mark it as
-      ;; the end of the string:
-      (seq "\"" (backref 2) (group "#"))
-
-      ;; No "#"s - capture the ending quote (using a backref to group 3,
-      ;; so that we can't match a quote if we had "#"s) as group 6
-      (group (backref 3))
-
-      ;; If the raw string wasn't actually closed, go all the way to the end
-      string-end))
-
-    ;; Character literal: match the beginning ' of a character literal
-    ;; as group 7, and the ending one as group 8
-    (seq
-     (group "'")
-     (or
-      (seq
-       "\\"
-       (or
-        (: "u{" (** 1 6 xdigit) "}")
-        (: "x" (= 2 xdigit))
-        (any "'nrt0\"\\")))
-      (not (any "'\\"))
-      )
-     (group "'"))
-    )
-   ))
-
 (defun rust-looking-back-str (str)
   "Like `looking-back' but for fixed strings rather than regexps (so that it's 
not so slow)"
   (let ((len (length str)))
@@ -133,8 +72,8 @@
     (modify-syntax-entry ?\" "\"" table)
     (modify-syntax-entry ?\\ "\\" table)
 
-    ;; Angle brackets.  We suppress this with syntactic fontification when
-    ;; needed
+    ;; Angle brackets.  We suppress this with syntactic propertization
+    ;; when needed
     (modify-syntax-entry ?< "(>" table)
     (modify-syntax-entry ?> ")<" table)
 
@@ -643,94 +582,6 @@ the desired identifiers), but does not match type 
annotations \"foo::<\"."
              ("use" . font-lock-constant-face)
              ("fn" . font-lock-function-name-face)))))
 
-(defvar font-lock-beg)
-(defvar font-lock-end)
-
-(defun rust-font-lock-extend-region ()
-  "Extend the region given by `font-lock-beg' and `font-lock-end'
-  to include the beginning of a string or comment if it includes
-  part of it.  Adjusts to include the r[#] of a raw string as
-  well."
-
-  (save-excursion
-    (let ((orig-beg font-lock-beg)
-          (orig-end font-lock-end))
-
-      (let*
-          ;; It's safe to call `syntax-ppss' here on positions that are
-          ;; already syntactically fontified
-          ((beg-ppss (syntax-ppss font-lock-beg))
-           (beg-in-cmnt (and beg-ppss (nth 4 beg-ppss) (nth 8 beg-ppss)))
-           (beg-in-str (and beg-ppss (nth 3 beg-ppss) (nth 8 beg-ppss))))
-
-        (when (and beg-in-str (>= font-lock-beg beg-in-str))
-          (setq font-lock-beg (nth 8 beg-ppss))
-          (while (equal ?# (char-before font-lock-beg))
-            (setq font-lock-beg (1- font-lock-beg)))
-          (when (equal ?r (char-before font-lock-beg))
-            (setq font-lock-beg (1- font-lock-beg))))
-
-        (when (and beg-in-cmnt (> font-lock-beg beg-in-cmnt))
-          (setq font-lock-beg beg-in-cmnt)))
-
-      ;; We need to make sure that if the region ends inside a raw string, we
-      ;; extend it out past the end of it.  But we can't use `syntax-ppss' to
-      ;; detect that, becaue that depends on font-lock already being done, and 
we
-      ;; are trying to figure out how much to font-lock before that.  So we use
-      ;; the regexp directly.
-      (save-match-data
-        (goto-char font-lock-beg)
-        (while (and (< (point) font-lock-end)
-                    (re-search-forward rust-re-non-standard-string (buffer-end 
1) t)
-                    (<= (match-beginning 0) font-lock-end))
-          (setq font-lock-end (max font-lock-end (match-end 0)))
-          (goto-char (1+ (match-beginning 0)))))
-
-      (or (/= font-lock-beg orig-beg)
-          (/= font-lock-end orig-end))
-      )))
-
-(defun rust-conditional-re-search-forward (regexp bound condition)
-  ;; Search forward for regexp (with bound).  If found, call condition and 
return the found
-  ;; match only if it returns true.
-  (let* (found
-         found-ret-list
-         (ret-list (save-excursion
-                     (while (and (not found) (re-search-forward regexp bound 
t))
-                       (setq
-                        found-ret-list (list (point) (match-data))
-                        found (save-match-data (save-excursion (ignore-errors 
(funcall condition)))))
-                       ;; If the condition filters out a match, need to search
-                       ;; again just after its beginning.  This will allow
-                       ;; cases such as:
-                       ;;    "bar" r"foo"
-                       ;; where the filtered out search (r" r") should not
-                       ;; prevent finding another one that begins in the middle
-                       ;; of it (r"foo")
-                       (when (not found)
-                         (goto-char (1+ (match-beginning 0))))
-                       )
-                     (when found found-ret-list))))
-    (when ret-list
-      (goto-char (nth 0 ret-list))
-      (set-match-data (nth 1 ret-list))
-      (nth 0 ret-list))))
-
-(defun rust-look-for-non-standard-string (bound)
-  ;; Find a raw string or character literal, but only if it's not in the middle
-  ;; of another string or a comment.
-
-  (rust-conditional-re-search-forward
-   rust-re-non-standard-string
-   bound
-   (lambda ()
-     (let ((pstate (syntax-ppss (match-beginning 0))))
-       (not
-        (or
-         (nth 4 pstate) ;; Skip if in a comment
-         (and (nth 3 pstate) (wholenump (nth 8 pstate)) (< (nth 8 pstate) 
(match-beginning 0))) ;; Skip if in a string that isn't starting here
-         ))))))
-
 (defun rust-syntax-class-before-point ()
   (when (> (point) 1)
     (syntax-class (syntax-after (1- (point))))))
@@ -960,8 +811,11 @@ the desired identifiers), but does not match type 
annotations \"foo::<\"."
        ((or
          (equal 4 (rust-syntax-class-before-point))
          (rust-looking-back-str ","))
-        (backward-up-list)
-        (rust-is-in-expression-context 'open-brace))
+       (condition-case nil
+           (progn
+             (backward-up-list)
+             (rust-is-in-expression-context 'open-brace))
+         (scan-error nil)))
 
        ;; A => introduces an expression
        ((rust-looking-back-str "=>") t)
@@ -1037,56 +891,42 @@ the desired identifiers), but does not match type 
annotations \"foo::<\"."
        (rust-is-lt-char-operator)))
    (funcall (default-value 'electric-pair-inhibit-predicate) char)))
 
-(defun rust-look-for-non-angle-bracket-lt-gt (bound)
-  "Find an angle bracket (\"<\" or \">\") that should be part of
-  a matched pair Relies on the fact that when it finds a < or >,
-  we have already decided which previous ones are angle brackets
-  and which ones are not.  So this only really works as a
-  font-lock-syntactic-keywords matcher--it won't work at
-  arbitrary positions without the earlier parts of the buffer
-  having already been covered."
-
-  (rust-conditional-re-search-forward
-   "[<>]" bound
-   (lambda ()
-     (goto-char (match-beginning 0))
-     (cond
-      ;; If matching is turned off suppress all of them
-      ((not rust-match-angle-brackets) t)
-
-      ;; We don't take < or > in strings or comments to be angle brackets
-      ((rust-in-str-or-cmnt) t)
-
-      ;; Inside a macro we don't really know the syntax.  Any < or > may be an
-      ;; angle bracket or it may not.  But we know that the other braces have
-      ;; to balance regardless of the < and >, so if we don't treat any < or >
-      ;; as angle brackets it won't mess up any paren balancing.
-      ((rust-in-macro) t)
-      
-      ((looking-at "<")
-       (rust-is-lt-char-operator))
+(defun rust-ordinary-lt-gt-p ()
+  "Test whether the `<' or `>' at point is an ordinary operator of some kind.
 
-      ((looking-at ">")
-       (cond
-        ;; Don't treat the > in -> or => as an angle bracket
-        ((member (char-before (point)) '(?- ?=)) t)
-
-        ;; If we are at top level and not in any list, it can't be a closing
-        ;; angle bracket
-        ((>= 0 (rust-paren-level)) t)
-
-        ;; Otherwise, treat the > as a closing angle bracket if it would
-        ;; match an opening one
-        ((save-excursion
-           (backward-up-list)
-           (not (looking-at "<"))))))))))
-
-(defvar rust-mode-font-lock-syntactic-keywords
-  (append
-   ;; Handle raw strings and character literals:
-   `((rust-look-for-non-standard-string (1 "|" nil t) (4 "_" nil t) (5 "|" nil 
t) (6 "|" nil t) (7 "\"" nil t) (8 "\"" nil t)))
-   ;; Find where < and > characters represent operators rather than angle 
brackets:
-   '((rust-look-for-non-angle-bracket-lt-gt (0 "." t)))))
+This returns t if the `<' or `>' is an ordinary operator (like
+less-than) or part of one (like `->'); and nil if the character
+should be considered a paired angle bracket."
+  (cond
+   ;; If matching is turned off suppress all of them
+   ((not rust-match-angle-brackets) t)
+
+   ;; We don't take < or > in strings or comments to be angle brackets
+   ((rust-in-str-or-cmnt) t)
+
+   ;; Inside a macro we don't really know the syntax.  Any < or > may be an
+   ;; angle bracket or it may not.  But we know that the other braces have
+   ;; to balance regardless of the < and >, so if we don't treat any < or >
+   ;; as angle brackets it won't mess up any paren balancing.
+   ((rust-in-macro) t)
+   
+   ((looking-at "<")
+    (rust-is-lt-char-operator))
+
+   ((looking-at ">")
+    (cond
+     ;; Don't treat the > in -> or => as an angle bracket
+     ((member (char-before (point)) '(?- ?=)) t)
+
+     ;; If we are at top level and not in any list, it can't be a closing
+     ;; angle bracket
+     ((>= 0 (rust-paren-level)) t)
+
+     ;; Otherwise, treat the > as a closing angle bracket if it would
+     ;; match an opening one
+     ((save-excursion
+       (backward-up-list)
+       (not (looking-at "<"))))))))
 
 (defun rust-mode-syntactic-face-function (state)
   "Syntactic face function to distinguish doc comments from other comments."
@@ -1098,6 +938,68 @@ the desired identifiers), but does not match type 
annotations \"foo::<\"."
         'font-lock-comment-face
     ))))
 
+(eval-and-compile
+  (defconst rust--char-literal-rx
+    (rx (seq
+        (group "'")
+        (or
+         (seq
+          "\\"
+          (or
+           (: "u{" (** 1 6 xdigit) "}")
+           (: "x" (= 2 xdigit))
+           (any "'nrt0\"\\")))
+         (not (any "'\\"))
+         )
+        (group "'")))
+    "A regular expression matching a character literal."))
+
+(defun rust--syntax-propertize-raw-string (end)
+  "A helper for rust-syntax-propertize.
+
+If point is already in a raw string, this will apply the
+appropriate string syntax to the character up to the end of the
+raw string, or to `end', whichever comes first."
+  (let ((str-start (nth 8 (syntax-ppss))))
+    (when str-start
+      (when (save-excursion
+             (goto-char str-start)
+             (looking-at "r\\(#*\\)\\(\"\\)"))
+       ;; In a raw string, so try to find the end.
+       (let ((hashes (match-string 1)))
+         ;; Match \ characters at the end of the string to suppress
+         ;; their normal character-quote syntax.
+         (when (re-search-forward (concat "\\(\\\\*\\)\\(\"" hashes "\\)") end 
t)
+           (put-text-property (match-beginning 1) (match-end 1)
+                              'syntax-table (string-to-syntax "_"))
+           (put-text-property (1- (match-end 2)) (match-end 2)
+                              'syntax-table (string-to-syntax "|"))
+           (goto-char (match-end 0))))))))
+
+(defun rust-syntax-propertize (start end)
+  "A `syntax-propertize-function' for `rust-mode'."
+  (goto-char start)
+  (rust--syntax-propertize-raw-string end)
+  (funcall
+   (syntax-propertize-rules
+    ;; Character literals.
+    (rust--char-literal-rx (1 "\"") (2 "\""))
+    ;; Raw strings.
+    ("\\(r\\)#*\""
+     (1 (prog1 "|"
+         (goto-char (match-end 0))
+         (rust--syntax-propertize-raw-string end))))
+    ("[<>]"
+     (0 (ignore
+        (when (save-match-data
+                (save-excursion
+                  (goto-char (match-beginning 0))
+                  (rust-ordinary-lt-gt-p)))
+          (put-text-property (match-beginning 0) (match-end 0)
+                             'syntax-table (string-to-syntax "."))
+          (goto-char (match-end 0)))))))
+   (point) end))
+
 (defun rust-fill-prefix-for-comment-start (line-start)
   "Determine what to use for `fill-prefix' based on what is at the beginning 
of a line."
   (let ((result
@@ -1410,11 +1312,6 @@ This is written mainly to be used as 
`end-of-defun-function' for Rust."
         (set-window-start window start)
         (set-window-point window pos))))
 
-  ;; Issue #127: Running this on a buffer acts like a revert, and could cause
-  ;; the fontification to get out of sync.  Call the same hook to ensure it is
-  ;; restored.
-  (rust--after-revert-hook)
-
   (message "Formatted buffer with rustfmt."))
 
 (defun rust-enable-format-on-save ()
@@ -1441,14 +1338,15 @@ This is written mainly to be used as 
`end-of-defun-function' for Rust."
   :group 'rust-mode
   :syntax-table rust-mode-syntax-table
 
+  ;; Syntax.
+  (setq-local syntax-propertize-function #'rust-syntax-propertize)
+
   ;; Indentation
   (setq-local indent-line-function 'rust-mode-indent-line)
 
   ;; Fonts
-  (add-to-list 'font-lock-extend-region-functions 
'rust-font-lock-extend-region)
   (setq-local font-lock-defaults '(rust-mode-font-lock-keywords
                                    nil nil nil nil
-                                   (font-lock-syntactic-keywords . 
rust-mode-font-lock-syntactic-keywords)
                                    (font-lock-syntactic-face-function . 
rust-mode-syntactic-face-function)
                                    ))
 
@@ -1480,7 +1378,6 @@ This is written mainly to be used as 
`end-of-defun-function' for Rust."
   (setq-local end-of-defun-function 'rust-end-of-defun)
   (setq-local parse-sexp-lookup-properties t)
   (setq-local electric-pair-inhibit-predicate 
'rust-electric-pair-inhibit-predicate-wrap)
-  (add-hook 'after-revert-hook 'rust--after-revert-hook nil t)
   (add-hook 'before-save-hook 'rust--before-save-hook nil t))
 
 ;;;###autoload
@@ -1492,18 +1389,6 @@ This is written mainly to be used as 
`end-of-defun-function' for Rust."
   (require 'rust-mode)
   (rust-mode))
 
-;; Issue #104: When reverting the buffer, make sure all fontification is redone
-;; so that we don't end up missing a non-angle-bracket '<' or '>' character.
-(defun rust--after-revert-hook ()
-  ;; In Emacs 25 and later, the preferred method to force fontification is
-  ;; to use `font-lock-ensure', which doesn't exist in Emacs 24 and earlier.
-  ;; If it's not available, fall back to calling `font-lock-fontify-region'
-  ;; on the whole buffer.
-  (save-excursion
-    (if (fboundp 'font-lock-ensure)
-        (font-lock-ensure)
-      (font-lock-fontify-region (point-min) (point-max)))))
-
 (defun rust--before-save-hook ()
   (when rust-format-on-save (rust-format-buffer)))