bug#3824: 23.1.50; too much effort is put into handling Scheme S-expression comments, causing problems

[Stefan Monnier]

> on imbalanced parentheses.  This is because Emacs goes to excessive
> effort to handle S-expression comments in Scheme Mode, which causes
> more problems than it solves.

Thank you for appreciating my efforts.  The first part of the problem is
known (quoting from the code:)

                   ;; FIXME: this doesn't handle the case where the sexp
                   ;; itself contains a #; comment.

The second part is indeed new: by marking the closing-paren as
a "comment end marker", we prevent the paren from being counted by
forward-sexp, so it can't stop there.

> Emacs should treat `#;' as whitespace, nothing more.

Of course, that's also incorrect with its own set of problems (mostly
indentation), but admittedly, those problems are easier for the user to
understand/predict.

> As far as I can imagine, the only legitimate context for treating `#;'
> as more than whitespace is Font Lock, but since Font Lock works with
> regular expressions, it, too, will do the wrong thing.  (In the above

Font lock can work with arbitrary Elisp code (tho it mostly relies on
regexps).

Can you try the patch below?


        Stefan


=== modified file 'lisp/progmodes/scheme.el'
--- lisp/progmodes/scheme.el    2009-06-06 04:49:20 +0000
+++ lisp/progmodes/scheme.el    2009-07-14 19:50:40 +0000
@@ -99,7 +99,8 @@
     (modify-syntax-entry ?\( "()  " st)
     (modify-syntax-entry ?\) ")(  " st)
     ;; It's used for single-line comments as well as for #;(...) sexp-comments.
-    (modify-syntax-entry ?\; "< 2 " st)
+    ;; It's too difficult to make sexp-comment work right with syntax-tables.
+    (modify-syntax-entry ?\; "<   " st)
     (modify-syntax-entry ?\" "\"   " st)
     (modify-syntax-entry ?' "'   " st)
     (modify-syntax-entry ?` "'   " st)
@@ -170,8 +171,9 @@
          nil t (("+-*/.<>=!?$%_&~^:" . "w") (?#. "w 14"))
          beginning-of-defun
          (font-lock-mark-block-function . mark-defun)
-         (font-lock-syntactic-face-function
-          . scheme-font-lock-syntactic-face-function)
+         (font-lock-syntactic-keywords . scheme-font-lock-syntactic-keywords)
+         ;; (font-lock-syntactic-face-function
+         ;;  . scheme-font-lock-syntactic-face-function)
          (parse-sexp-lookup-properties . t)
          (font-lock-extra-managed-props syntax-table)))
   (set (make-local-variable 'lisp-doc-string-elt-property)
@@ -290,6 +292,13 @@
      "^(declare\\(-\\sw+\\)+\\>\\s-+\\(\\sw+\\)" 2))
   "Imenu generic expression for DSSSL mode.  See `imenu-generic-expression'.")
 
+(defconst scheme-font-lock-syntactic-keywords
+  ;; Treat sexp-comment markers as "whitespace".
+  '(("#\\(;\\)"
+     (1 (if (nth 8 (save-excursion (syntax-ppss (match-beginning 0))))
+            ;; Check parser state to avoid problem with #|comment1|#;comment2
+            nil '(6))))))
+
 (defconst scheme-font-lock-keywords-1
   (eval-when-compile
     (list
@@ -317,8 +326,21 @@
      ))
   "Subdued expressions to highlight in Scheme modes.")
 
+(defun scheme-font-lock-sexp-comment (limit)
+  (when (search-forward "#;" limit t)
+    (let ((beg (match-beginning 0)))
+      (if (nth 8 (save-excursion (syntax-ppss beg)))
+          ;; Not a sexp-comment: keep looking.
+          (scheme-font-lock-sexp-comment limit)
+        (ignore-errors
+          (forward-sexp 1)
+          (set-match-data (list beg (point)))
+          (point))))))
+
 (defconst scheme-font-lock-keywords-2
-  (append scheme-font-lock-keywords-1
+  (append
+   '((scheme-font-lock-sexp-comment (0 font-lock-comment-face)))
+   scheme-font-lock-keywords-1
    (eval-when-compile
      (list
       ;;
@@ -388,6 +410,10 @@
         (when (< pos (- end 2))
           (put-text-property pos (- end 2)
                              'syntax-table scheme-sexp-comment-syntax-table))
+        ;; FIXME: This marks the closing paren as a comment-close
+        ;; instead, which means that forward-sexp won't stop here.
+        ;; I.e. it is fundamentally flawed: we should instead use a zero-width
+        ;; "comment-close" right after the closing paren.
         (put-text-property (- end 1) end 'syntax-table '(12)))))
   ;; Choose the face to use.
   (lisp-font-lock-syntactic-face-function state))