nnimap: support for non-ASCII folder names (incomplete patch & request for help)

[Orivej Desh]

I'm migrating to Gnus, and I'm in need for non-ASCII folder names.

   I've found them not supported (they just didn't appear in *Gnus
Browse Server*) so I made a copy of nnimap.el and discovered that though
some support for UTF-7 was in place, it was not working as expected
(particularly, double-encoded UTF-7 strings happened).  Investigating
which `utf7-encode' or `utf7-decode' were missing and which superfluous
went difficult, so I made a redesing: everything in nnimap deals with
multibyte, except for two functions most close to the IMAP stream.

   I almost succeeded: the only `utf7-encode' found it's place in
`nnimap-send-command' and `utf7-decode' — in `nnimap-parse-response',
*Gnus Browse Server* displays all of the folders with correct message
count (with a hack from http://www.emacswiki.org/emacs/GnusNiftyTricks).
However I still can't enter non-ASCII folders, and if I subscribe to
them, neither can I enter nor can I see message count.  This has
something to do with UTF-8 unibyte strings which spring at me when I try
to enter or when I subscribe.  I assume this has something to do with
Gnus<+>nnimap interaction, though I don't know what.

   Here goes the patch to be applied explicitly to `nnimap.el'
(generated with `diff -u' against gnus in emacs tree of 2010-11-22),
which is mostly clean, no debugging `message' calls left, except for the
function `nnimap-possibly-change-group' still trying unsuccessfully to
deal with UTF-8 unibyte input (`decode-coding-string' works, but
debugged `nnimap-send-command' reports that it receives unibyte
nonetheless).

--- nnimap.el.old       2010-11-28 19:00:38.953333383 +0300
+++ nnimap.el.new       2010-11-28 20:51:40.906666922 +0300
@@ -687,7 +687,7 @@
          (with-current-buffer (nnimap-buffer)
            (erase-buffer)
            (let ((group-sequence
-                  (nnimap-send-command "SELECT %S" (utf7-encode group t)))
+                  (nnimap-send-command "SELECT %S" group))
                  (flag-sequence
                   (nnimap-send-command "UID FETCH 1:* FLAGS")))
              (setf (nnimap-group nnimap-object) group)
@@ -717,12 +717,12 @@
 (deffoo nnimap-request-create-group (group &optional server args)
   (when (nnimap-possibly-change-group nil server)
     (with-current-buffer (nnimap-buffer)
-      (car (nnimap-command "CREATE %S" (utf7-encode group t))))))
+      (car (nnimap-command "CREATE %S" group)))))
 
 (deffoo nnimap-request-delete-group (group &optional force server)
   (when (nnimap-possibly-change-group nil server)
     (with-current-buffer (nnimap-buffer)
-      (car (nnimap-command "DELETE %S" (utf7-encode group t))))))
+      (car (nnimap-command "DELETE %S" group)))))
 
 (deffoo nnimap-request-rename-group (group new-name &optional server)
   (when (nnimap-possibly-change-group nil server)
@@ -731,8 +731,7 @@
       ;; to examine a mailbox that doesn't exist.  This seems to be
       ;; the only way that allows us to reliably go back to unselected
       ;; state on Courier.
-      (car (nnimap-command "RENAME %S %S"
-                          (utf7-encode group t) (utf7-encode new-name t))))))
+      (car (nnimap-command "RENAME %S %S" group new-name)))))
 
 (deffoo nnimap-request-expunge-group (group &optional server)
   (when (nnimap-possibly-change-group group server)
@@ -776,7 +775,7 @@
                   (with-current-buffer (nnimap-buffer)
                     (nnimap-command "UID COPY %d %S"
                                     article
-                                    (utf7-encode internal-move-group t)))))
+                                    internal-move-group))))
              (when (car result)
                (nnimap-delete-article article)
                (cons internal-move-group
@@ -863,7 +862,7 @@
     (erase-buffer)
     (unless (equal group (nnimap-group nnimap-object))
       (setf (nnimap-group nnimap-object) nil)
-      (nnimap-send-command "EXAMINE %S" (utf7-encode group t)))
+      (nnimap-send-command "EXAMINE %S" group))
     (let ((sequence
           (nnimap-send-command "UID SEARCH HEADER Message-Id %S" message-id))
          article result)
@@ -938,7 +937,7 @@
       (with-current-buffer (nnimap-buffer)
        (erase-buffer)
        (setq sequence (nnimap-send-command
-                       "APPEND %S {%d}" (utf7-encode group t)
+                       "APPEND %S {%d}" group
                        (length message)))
        (unless nnimap-streaming
          (nnimap-wait-for-connection "^[+]"))
@@ -994,7 +993,7 @@
        (with-current-buffer (nnimap-buffer)
          (setf (nnimap-group nnimap-object) nil)
          (dolist (group groups)
-           (push (list (nnimap-send-command "EXAMINE %S" (utf7-encode group t))
+           (push (list (nnimap-send-command "EXAMINE %S" group)
                        group)
                  sequences))
          (nnimap-wait-for-response (caar sequences))
@@ -1015,14 +1014,14 @@
                  (setq highest (1- (string-to-number (car uidnext)))))
                (cond
                 ((null highest)
-                 (insert (format "%S 0 1 y\n" (utf7-decode group t))))
+                 (insert (format "%S 0 1 y\n" group)))
                 ((zerop exists)
                  ;; Empty group.
                  (insert (format "%S %d %d y\n"
-                                 (utf7-decode group t) highest (1+ highest))))
+                                 group highest (1+ highest))))
                 (t
                  ;; Return the widest possible range.
-                 (insert (format "%S %d 1 y\n" (utf7-decode group t)
+                 (insert (format "%S %d 1 y\n" group
                                  (or highest exists)))))))))
        t))))
 
@@ -1057,7 +1056,7 @@
                   modseq)
              (push
               (list (nnimap-send-command "EXAMINE %S (QRESYNC (%s %s))"
-                                         (utf7-encode group t)
+                                         group
                                          uidvalidity modseq)
                     'qresync
                     nil group 'qresync)
@@ -1077,7 +1076,7 @@
                     ;; is read-only or not.
                     "SELECT")))
              (push (list (nnimap-send-command "%s %S" command
-                                              (utf7-encode group t))
+                                              group)
                          (nnimap-send-command "UID FETCH %d:* FLAGS" start)
                          start group command)
                    sequences))))
@@ -1430,9 +1429,11 @@
       t)
      (t
       (with-current-buffer (nnimap-buffer)
+       (when (not (multibyte-string-p group))
+         (setq group (decode-coding-string group 'utf-8-emacs)))
        (if (equal group (nnimap-group nnimap-object))
            t
-         (let ((result (nnimap-command "SELECT %S" (utf7-encode group t))))
+         (let ((result (nnimap-command "SELECT %S" group)))
            (when (car result)
              (setf (nnimap-group nnimap-object) group
                    (nnimap-select-result nnimap-object) result)
@@ -1458,7 +1459,7 @@
    (nnimap-log-command
     (format "%d %s%s\n"
            (incf nnimap-sequence)
-           (apply #'format args)
+           (utf7-encode (apply #'format args) t)
            (if (nnimap-newlinep nnimap-object)
                ""
              "\r"))))
@@ -1532,7 +1533,7 @@
        nil))))
 
 (defun nnimap-parse-response ()
-  (let ((lines (split-string (nnimap-last-response-string) "\r\n" t))
+  (let ((lines (split-string (utf7-decode (nnimap-last-response-string) t) 
"\r\n" t))
        result)
     (dolist (line lines)
       (push (cdr (nnimap-parse-line line)) result))
@@ -1547,7 +1548,6 @@
 (defun nnimap-parse-line (line)
   (let (char result)
     (with-temp-buffer
-      (mm-disable-multibyte)
       (insert line)
       (goto-char (point-min))
       (while (not (eobp))
@@ -1656,7 +1656,7 @@
            (dolist (spec specs)
              (when (and (not (member (car spec) groups))
                         (not (eq (car spec) 'junk)))
-               (nnimap-command "CREATE %S" (utf7-encode (car spec) t))))
+               (nnimap-command "CREATE %S" (car spec))))
            ;; Then copy over all the messages.
            (erase-buffer)
            (dolist (spec specs)
@@ -1667,7 +1667,7 @@
                  (push (list (nnimap-send-command
                               "UID COPY %s %S"
                               (nnimap-article-ranges ranges)
-                              (utf7-encode group t))
+                              group)
                              ranges)
                        sequences))))
            ;; Wait for the last COPY response...