bug#13793: 24.3.50; M-x broken in viper and X

[Stefan Monnier]

> Heya, just checking back about the status of this bug report. I'm
> affected by it and I've been stuck with Emacs 23 since its first arrival
> (or else I can jump back on Emacs 24.3.x ditching viper all together,
> which might even be a good thing for my saneness :-))

I don't understand all of what Viper does with the ESC key, nor do
I know what are the different cases where ESC has to behave in
a specific way, so it's difficult for me to come up with
a trustworthy patch.

Could you try the patch below under X11 and under a tty, ideally even
using emacsclient to create new X11 and tty frames?

Michael, could you also take a look at this patch?


        Stefan


=== modified file 'lisp/emulation/viper-cmd.el'
--- lisp/emulation/viper-cmd.el 2013-06-18 20:24:44 +0000
+++ lisp/emulation/viper-cmd.el 2013-06-22 21:40:32 +0000
@@ -996,93 +996,7 @@
        (suspend-emacs))
     (viper-change-state-to-emacs)))
 
-
-;; Intercept ESC sequences on dumb terminals.
-;; Based on the idea contributed by Marcelino Veiga Tuimil <mveiga@dit.upm.es>
-
-;; Check if last key was ESC and if so try to reread it as a function key.
-;; But only if there are characters to read during a very short time.
-;; Returns the last event, if any.
-(defun viper-envelop-ESC-key ()
-  (let ((event last-input-event)
-       (keyseq [nil])
-       (inhibit-quit t))
-    (if (viper-ESC-event-p event)
-       (progn
-         ;; Some versions of Emacs (eg., 22.50.8 (?)) have a bug, which makes
-          ;; even a single ESC into a fast keyseq. To guard against this, we
-         ;; added a check if there are other events as well.  Keep the next
-         ;; line for the next time the bug reappears, so that will remember to
-         ;; report it.
-         ;;(if (and (viper-fast-keysequence-p) unread-command-events)
-         (if (viper-fast-keysequence-p) ;; for Emacsen without the above bug
-             (progn
-               (let (minor-mode-map-alist emulation-mode-map-alists)
-                 (viper-set-unread-command-events event)
-                 (setq keyseq (read-key-sequence nil 'continue-echo))
-                 ) ; let
-               ;; If keyseq translates into something that still has ESC
-               ;; at the beginning, separate ESC from the rest of the seq.
-               ;; In XEmacs we check for events that are keypress meta-key
-               ;; and convert them into [escape key]
-               ;;
-               ;; This is needed for the following reason:
-               ;; If ESC is the first symbol, we interpret it as if the
-               ;; user typed ESC and then quickly some other symbols.
-               ;; If ESC is not the first one, then the key sequence
-               ;; entered was apparently translated into a function key or
-               ;; something (e.g., one may have
-               ;; (define-key function-key-map "\e[192z" [f11])
-               ;; which would translate the escape-sequence generated by
-               ;; f11 in an xterm window into the symbolic key f11.
-               ;;
-               ;; If `first-key' is not an ESC event, we make it into the
-               ;; last-command-event in order to pretend that this key was
-               ;; pressed.  This is needed to allow arrow keys to be bound to
-               ;; macros.  Otherwise, viper-exec-mapped-kbd-macro will think
-               ;; that the last event was ESC and so it'll execute whatever is
-               ;; bound to ESC. (Viper macros can't be bound to
-               ;; ESC-sequences).
-               (let* ((first-key (elt keyseq 0))
-                      (key-mod (event-modifiers first-key)))
-                 (cond ((and (viper-ESC-event-p first-key)
-                             (not (viper-translate-all-ESC-keysequences)))
-                        ;; put keys following ESC on the unread list
-                        ;; and return ESC as the key-sequence
-                        (viper-set-unread-command-events (viper-subseq keyseq 
1))
-                        (setq last-input-event event
-                              keyseq (if (featurep 'emacs)
-                                         "\e"
-                                       (vector (character-to-event ?\e)))))
-                       ((and (featurep 'xemacs)
-                             (key-press-event-p first-key)
-                             (equal '(meta) key-mod))
-                        (viper-set-unread-command-events
-                         (vconcat (vector
-                                   (character-to-event (event-key first-key)))
-                                  (viper-subseq keyseq 1)))
-                        (setq last-input-event event
-                              keyseq (vector (character-to-event ?\e))))
-                       ((eventp first-key)
-                        (setq last-command-event
-                              (viper-copy-event first-key)))
-                       ))
-               ) ; end progn
-
-           ;; this is escape event with nothing after it
-           ;; put in unread-command-event and then re-read
-           (viper-set-unread-command-events event)
-           (setq keyseq (read-key-sequence nil))
-           ))
-      ;; not an escape event
-      (setq keyseq (vector event)))
-    keyseq))
-
-
-
 ;; Listen to ESC key.
-;; If a sequence of keys starting with ESC is issued with very short delays,
-;; interpret these keys in Emacs mode, so ESC won't be interpreted as a Vi key.
 (defun viper-intercept-ESC-key ()
   "Function that implements ESC key in Viper emulation of Vi."
   (interactive)
@@ -1090,13 +1004,7 @@
   ;; minor-mode map(s) have been temporarily disabled so the ESC
   ;; binding to viper-intercept-ESC-key doesn't hide the binding we're
   ;; looking for (Bug#9146):
-  (let* ((event (viper-envelop-ESC-key))
-        (cmd (cond ((equal event viper-ESC-key)
-                    'viper-intercept-ESC-key)
-                   ((let ((emulation-mode-map-alists nil))
-                      (key-binding event)))
-                   (t
-                    (error "Viper bell")))))
+  (let* ((cmd 'viper-intercept-ESC-key))
 
     ;; call the actual function to execute ESC (if no other symbols followed)
     ;; or the key bound to the ESC sequence (if the sequence was issued

=== modified file 'lisp/emulation/viper-keym.el'
--- lisp/emulation/viper-keym.el        2013-01-01 09:11:05 +0000
+++ lisp/emulation/viper-keym.el        2013-06-22 21:43:10 +0000
@@ -192,7 +192,7 @@
   :type 'string
   :group 'viper)
 
-(defvar viper-ESC-key (kbd "ESC")
+(defconst viper-ESC-key [escape]
   "Key used to ESC.")
 
 

=== modified file 'lisp/emulation/viper.el'
--- lisp/emulation/viper.el     2013-03-12 02:08:21 +0000
+++ lisp/emulation/viper.el     2013-06-22 21:54:55 +0000
@@ -660,7 +660,7 @@
 undone.
 It also can't undo some Viper settings."
   (interactive)
-
+  (viper-setup-ESC-to-escape nil)
   ;; restore non-viper vars
   (setq-default
    next-line-add-newlines
@@ -825,6 +825,58 @@
   (add-hook 'viper-post-command-hooks 'set-viper-state-in-major-mode t))
 
 
+;;; Handling of tty's ESC event
+
+;; On a tty, an ESC event can either be the user hitting the escape key, or
+;; some element of a byte sequence used to encode for example cursor keys.
+;; So we try to recognize those events that correspond to the escape key and
+;; turn them into `escape' events (same as used under GUIs).  The heuristic we
+;; use to distinguish the two cases is based, as usual, on a timeout, and on
+;; the fact that the special ESC=>escape mapping only takes place if the whole
+;; last key-sequence so far is just [?\e], i.e. either we're still in
+;; read-key-sequence, or the last read-key-sequence only read [?\e], which
+;; should ideally never happen because it should have been mapped to [escape].
+
+(defun viper--tty-ESC-filter (map)
+  (if (and (equal (this-single-command-keys) [?\e])
+           (sit-for (/ viper-fast-keyseq-timeout 1000)))
+      [escape] map))
+
+(defun viper--lookup-key (map key)
+  "Kind of like `lookup-key'.
+Two differences:
+- KEY is a single key, not a sequence.
+- the result is the \"raw\" binding, so it can be a `menu-item', rather than 
the
+  binding contained in that menu item."
+  (catch 'found
+    (map-keymap (lambda (k b) (if (equal key k) (throw 'found b))) map)))
+
+(defun viper-catch-tty-ESC ()
+  "Setup key mappings of current terminal to turn a tty's ESC into `escape'."
+  (when (memq (terminal-live-p (frame-terminal)) '(t pc))
+    (let ((esc-binding (viper-uncatch-tty-ESC)))
+      (define-key input-decode-map
+        [?\e] `(menu-item "" ,esc-binding :filter viper--tty-ESC-filter)))))
+
+(defun viper-uncatch-tty-ESC ()
+  "Don't hack ESC into `escape' any more."
+  (let ((b (viper--lookup-key input-decode-map ?\e)))
+    (and (eq 'menu-item (car-safe b))
+         (eq 'viper--tty-ESC-filter (nth 4 b))
+         (define-key input-decode-map [?\e] (setq b (nth 2 b))))
+    b))
+
+(defun viper-setup-ESC-to-escape (enable)
+  (if enable
+      (add-hook 'tty-setup-hook 'viper-catch-tty-ESC)
+    (remove-hook 'tty-setup-hook 'viper-catch-tty-ESC))
+  (let ((seen ()))
+    (dolist (frame (frame-list))
+      (let ((terminal (frame-terminal frame)))
+        (unless (memq terminal seen)
+          (push terminal seen)
+          (with-selected-frame frame
+            (if enable (viper-catch-tty-ESC) (viper-uncatch-tty-ESC))))))))
 
 ;; This sets major mode hooks to make them come up in vi-state.
 (defun viper-set-hooks ()
@@ -837,6 +889,8 @@
   (if (eq (default-value 'major-mode) 'fundamental-mode)
       (setq-default major-mode 'viper-mode))
 
+  (viper-setup-ESC-to-escape t)
+
   (add-hook 'change-major-mode-hook 'viper-major-mode-change-sentinel)
   (add-hook 'find-file-hooks 'set-viper-state-in-major-mode)
 
@@ -847,13 +901,6 @@
   (defvar emerge-startup-hook)
   (add-hook 'emerge-startup-hook 'viper-change-state-to-emacs)
 
-  ;; Zap bad bindings in flyspell-mouse-map, which prevent ESC from working
-  ;; over misspelled words (due to the overlay keymaps)
-  (defvar flyspell-mode-hook)
-  (defvar flyspell-mouse-map)
-  (add-hook 'flyspell-mode-hook
-           (lambda ()
-              (define-key flyspell-mouse-map viper-ESC-key nil)))
   ;; if viper is started from .emacs, it might be impossible to get certain
   ;; info about the display and windows until emacs initialization is complete
   ;; So do it via the window-setup-hook

=== modified file 'lisp/faces.el'
--- lisp/faces.el       2013-05-15 23:55:41 +0000
+++ lisp/faces.el       2013-06-22 20:22:31 +0000
@@ -2126,7 +2126,8 @@
                       type)
        (when (fboundp term-init-func)
          (funcall term-init-func))
-       (set-terminal-parameter frame 'terminal-initted term-init-func)))))
+        (set-terminal-parameter frame 'terminal-initted term-init-func)
+       (run-hooks 'tty-setup-hook)))))
 
 ;; Called from C function init_display to initialize faces of the
 ;; dumped terminal frame on startup.