[elpa] externals/auctex 370a4b9 31/57: Resolve preview-latex incompatibility with Japanese TeX (Bug#25322)

[Tassilo Horn]

branch: externals/auctex
commit 370a4b92155aebf3d0860c1211617d14d8e7951c
Author: Ikumi Keita <address@hidden>
Commit: Ikumi Keita <address@hidden>

    Resolve preview-latex incompatibility with Japanese TeX (Bug#25322)
    
    * preview.el.in (preview-error-quote): Work consistently with
    decoded text.  The idea of "encoding entire string beforehand and
    decoding it at the last stage" was used previously, but that failed
    with `japanese-shift-jis' coding system containing bytes which
    happen to coincide with regexp meta characters.
    (preview--decode-^^ab):
    (preview--convert-^^ab): New functions.
    (TeX-inline-preview-internal): Record the process coding system
    for decode assigned already so that `preview-error-quote' can
    decode the given text correctly afterward.  To achive that, change
    the role of `preview-coding-system' together with the changes to
    `preview-error-quote'.
    Give back the command options provided in `(TeX-engine-alist)' to
    latex command when preamble caching is enabled.
    (preview-coding-system): Modify doc string according to the
    changes above to `preview-error-quote' and
    `TeX-inline-preview-internal'.
    * tex-buf.el (TeX-adjust-process-coding-system): New function.
    Adjust the process coding system for asynchronous process launched
    within AUCTeX.
    (TeX-after-start-process-function): Change default value to
    `TeX-adjust-process-coding-system'.
    * tex-jp.el (japanese-TeX-set-process-coding-system): Specify end
    of line format on the coding systems assigned to process.
    (): Don't set `japanese-TeX-set-process-coding-system' to
    `TeX-after-start-process-function'.  It is now called within
    `TeX-adjust-process-coding-system'.
    * prv-xemacs.el (preview-buffer-recoding-alist): Fix docstring.
    (coding-system-change-eol-conversion): Add advice because XEmacs
    21.4 mule-ucs fails to define `utf-8' coding system to respond
    properly to this function.
    (preview-ps-quote-filename): Add advice so that this function is
    not confused by backslashes as path separator in the return value
    of XEmacs function.
    * tests/japanese/preview-latex.el:
    * tests/japanese/preview-error-test.tex:
    * tests/japanese/preview-error-test2.tex:
    * tests/japanese/prv-dif-code.tex: New files for regression test.
---
 preview.el.in                          | 141 +++++++++++----
 prv-xemacs.el                          |  22 ++-
 tests/japanese/preview-error-test.tex  |  42 +++++
 tests/japanese/preview-error-test2.tex |  20 +++
 tests/japanese/preview-latex.el        | 302 +++++++++++++++++++++++++++++++++
 tests/japanese/prv-dif-code.tex        |  42 +++++
 tex-buf.el                             |  54 +++++-
 tex-jp.el                              |  12 +-
 8 files changed, 590 insertions(+), 45 deletions(-)

diff --git a/preview.el.in b/preview.el.in
index e4bcd7f..1b24b5b 100644
--- a/preview.el.in
+++ b/preview.el.in
@@ -295,7 +295,7 @@ If `preview-fast-conversion' is set, this option is not
   :type 'number)
 
 (defvar preview-coding-system nil
-  "Coding system used for LaTeX process.")
+  "Proper coding system to decode output from LaTeX process.")
 (make-variable-buffer-local 'preview-coding-system)
 (defvar preview-parsed-font-size nil
   "Font size as parsed from the log of LaTeX run.")
@@ -2614,37 +2614,98 @@ later while in use."
   "Turn STRING with potential ^^ sequences into a regexp.
 To preserve sanity, additional ^ prefixes are matched literally,
 so the character represented by ^^^ preceding extended characters
-will not get matched, usually."
+will not get matched, usually.
+
+If decoding the process output was suppressed during receiving,
+decode first with RUN-CODING-SYSTEM."
   (let (output case-fold-search)
-    (when (featurep 'mule)
-      (setq string (encode-coding-string string run-coding-system)))
-    (while (string-match 
"\\^\\{2,\\}\\(\\(address@hidden)\\|[8-9a-f][0-9a-f]\\)"
-                        string)
+    ;; Some coding systems (e.g. japanese-shift-jis) use regexp meta
+    ;; characters on encoding.  Such meta characters would be
+    ;; interfered with `regexp-quote' below.  Thus the idea of
+    ;; "encoding entire string beforehand and decoding it at the last
+    ;; stage" does not work for such coding systems.
+    ;; Rather, we work consistently with decoded text.
+    (if (and (featurep 'mule)
+            (not (eq run-coding-system
+                     (preview-buffer-recode-system run-coding-system))))
+       (setq string
+             (decode-coding-string string run-coding-system)))
+
+    ;; Next, bytes with value from 0x80 to 0xFF represented with ^^
+    ;; form are converted to byte sequence, and decoded by the file
+    ;; coding system.
+    (setq string
+         (preview--decode-^^ab string
+                               (if (featurep 'mule)
+                                   buffer-file-coding-system nil)))
+
+    ;; Then, control characters are taken into account.
+    (while (string-match "\\^\\{2,\\}\\(address@hidden)" string)
       (setq output
            (concat output
                    (regexp-quote (substring string
                                             0
                                             (- (match-beginning 1) 2)))
-                   (if (match-beginning 2)
-                       (concat
-                        "\\(?:" (regexp-quote
-                                 (substring string
-                                            (- (match-beginning 1) 2)
-                                            (match-end 0)))
-                        "\\|"
-                        (char-to-string
-                         (logxor (aref string (match-beginning 2)) 64))
-                        "\\)")
-                     (char-to-string
-                      (string-to-number (match-string 1 string) 16))))
+                   (concat
+                    "\\(?:" (regexp-quote
+                             (substring string
+                                        (- (match-beginning 1) 2)
+                                        (match-end 0)))
+                    "\\|"
+                    (char-to-string
+                     (logxor (aref string (match-beginning 1)) 64))
+                    "\\)"))
            string (substring string (match-end 0))))
     (setq output (concat output (regexp-quote string)))
-    (if (featurep 'mule)
-       (decode-coding-string output
-                             (or (and (boundp 
'TeX-japanese-process-output-coding-system)
-                                      
TeX-japanese-process-output-coding-system)
-                                 buffer-file-coding-system))
-      output)))
+    output))
+
+(defun preview--decode-^^ab (string coding-system)
+  "Decode ^^ sequences in STRING with CODING-SYSTEM.
+Sequences of control characters such as ^^I are left untouched.
+
+Return a new string."
+  ;; Since the given string can contain multibyte characters, decoding
+  ;; should be performed seperately on each segment made up entirely
+  ;; with ASCII characters.
+  (let ((result ""))
+    (while (string-match "[\x00-\x7F]+" string)
+      (setq result
+           (concat result
+                   (substring string 0 (match-beginning 0))
+                   (let ((text
+                          (save-match-data
+                            (preview--convert-^^ab
+                             (match-string 0 string)))))
+                     (if (featurep 'mule)
+                         (decode-coding-string text coding-system)
+                       text)))
+           string (substring string (match-end 0))))
+    (setq result (concat result string))
+    result))
+
+(defun preview--convert-^^ab (string)
+  "Convert ^^ sequences in STRING to raw 8bit.
+Sequences of control characters such as ^^I are left untouched.
+
+Return a new string."
+  (let ((result ""))
+    (while (string-match "\\^\\^[8-9a-f][0-9a-f]" string)
+      (setq result
+           (concat result
+                   (substring string 0 (match-beginning 0))
+                   (let ((byte (string-to-number
+                                (substring string
+                                           (+ (match-beginning 0) 2)
+                                           (match-end 0)) 16)))
+                     ;; `char-to-string' is not appropriate in
+                     ;; Emacs >= 23 because it converts #xAB into
+                     ;; "\u00AB" (multibyte string), not "\xAB"
+                     ;; (raw 8bit unibyte string).
+                     (if (fboundp 'byte-to-string)
+                         (byte-to-string byte) (char-to-string byte))))
+           string (substring string (match-end 0))))
+    (setq result (concat result string))
+    result))
 
 (defun preview-parse-messages (open-closure)
   "Turn all preview snippets into overlays.
@@ -3484,7 +3545,13 @@ internal parameters, STR may be a log to insert into the 
current log."
         "Preview-LaTeX"
         (if (consp (cdr dumped-cons))
             (preview-do-replacements
-             command preview-undump-replacements)
+             command
+             (append preview-undump-replacements
+                     ;; Since the command options provided in
+                     ;; (TeX-engine-alist) are dropped, give them
+                     ;; back.
+                     (list (list "\\`\\([^ ]+\\)"
+                           (TeX-command-expand "%(latex)" nil)))))
           command) file)))
     (condition-case err
        (progn
@@ -3497,18 +3564,20 @@ internal parameters, STR may be a log to insert into 
the current log."
          (preview-set-geometry geometry)
          (setq preview-gs-file pr-file)
          (setq TeX-sentinel-function 'preview-TeX-inline-sentinel)
+         ;; Postpone decoding of process output for xemacs 21.4,
+         ;; which is rather bad at preserving incomplete multibyte
+         ;; characters.
          (when (featurep 'mule)
-           (setq preview-coding-system
-                 (or (and (boundp 'TeX-japanese-process-output-coding-system)
-                          TeX-japanese-process-output-coding-system)
-                     (with-current-buffer commandbuff
-                       buffer-file-coding-system)))
-           (when preview-coding-system
-             (setq preview-coding-system
-                   (preview-buffer-recode-system
-                    (coding-system-base preview-coding-system))))
-           (set-process-coding-system
-            process preview-coding-system))
+           ;; Get process coding system set in `TeX-run-command'.
+           (setq preview-coding-system (process-coding-system process))
+           ;; Substitute coding system for decode with `raw-text' if
+           ;; necessary and save the original coding system for
+           ;; decode for later use in `preview-error-quote'.
+           (set-process-coding-system process
+                                      (preview-buffer-recode-system
+                                       (car preview-coding-system))
+                                      (cdr preview-coding-system))
+           (setq preview-coding-system (car preview-coding-system)))
          (TeX-parse-reset)
          (setq TeX-parse-function 'TeX-parse-TeX)
          (if TeX-process-asynchronous
diff --git a/prv-xemacs.el b/prv-xemacs.el
index b4b22f7..32949d1 100644
--- a/prv-xemacs.el
+++ b/prv-xemacs.el
@@ -413,10 +413,9 @@ stream before the buffer characters can be identified.  
XEmacs
 21.4 is rather bad at preserving incomplete multibyte characters
 in that process.  This variable makes it possible to use a
 reconstructable coding system in the run buffer instead.  Specify
-an alist of base coding system names here, which you can get
-using
+an alist of coding system names here, which you can get using
 
-  \(coding-system-name (coding-system-base buffer-file-coding-system))
+  \(coding-system-name buffer-file-coding-system)
 
 in properly detected buffers."
   :group 'preview-latex
@@ -431,6 +430,17 @@ in properly detected buffers."
                 preview-buffer-recoding-alist))
       base))
 
+(if (and (featurep 'mule)
+        (= emacs-major-version 21)
+        (< emacs-minor-version 5))
+    (defadvice coding-system-change-eol-conversion
+       (after fallback activate)
+      "Return CODING-SYSTEM as-is if the result is nil.
+XEmacs 21.4 mule-ucs fails to define utf-8 to respond properly to
+this function."
+      (unless ad-return-value
+       (setq ad-return-value (ad-get-arg 0)))))
+
 (defun preview-mode-setup ()
   "Setup proper buffer hooks and behavior for previews."
   (set (make-local-variable 'desktop-save-buffer)
@@ -734,6 +744,12 @@ of an insertion."
                                (nth 1 image)
                                (nth 2 image)))))
 
+(if (eq system-type 'windows-nt)
+    (defadvice preview-ps-quote-filename (around path-sep-to-slash)
+      "Make path separator to slash so that the function will not be confused."
+      (let ((directory-sep-char ?/))
+       ad-do-it)))
+
 (provide 'prv-xemacs)
 
 ;;; Local variables:
diff --git a/tests/japanese/preview-error-test.tex 
b/tests/japanese/preview-error-test.tex
new file mode 100644
index 0000000..4ae90cd
--- /dev/null
+++ b/tests/japanese/preview-error-test.tex
@@ -0,0 +1,42 @@
+% Please check that all preview images in this buffer come out at the
+% correct position.  I.e., preview images should hide each \section
+% command.
+% When done, type C-M-c or M-x exit-recursive-edit.
+%
+% If the image is at the beginning of the line and "\section{xxx}" is
+% placed far rightward of the image, then the result should be considered
+% as fail.
+% In addition, if the image covers "\section{xxx}" only partially, the
+% result should be considered as fail, too.
+% On the contrary, if the text shown in the image is garbled while the
+% position of the image is correct, then the result should be considered as
+% OK.  That means that your ghostscript is not configured to handle
+% Japanese postscript font names correctly and the functionality of
+% preview-latex is just fine.
+\documentclass{jarticle}
+
+\begin{document}
+
+\section{�\(1)}
+
+\section{�\{a}}
+
+\section{\(�\\|\)}
+
+\section{�A�[�X}
+
+\section{�^}
+
+% The comment "%�\" will be displayed rightward to the image on the next
+% line.  That is a normal outcome and should be considered as OK.
+\section{��} %�\
+
+\end{document}
+
+%%% Local Variables:
+%%% coding: shift_jis
+%%% mode: japanese-latex
+%%% TeX-master: t
+%%% TeX-engine: ptex
+%%% TeX-PDF-mode: nil
+%%% End:
diff --git a/tests/japanese/preview-error-test2.tex 
b/tests/japanese/preview-error-test2.tex
new file mode 100644
index 0000000..3acdf7b
--- /dev/null
+++ b/tests/japanese/preview-error-test2.tex
@@ -0,0 +1,20 @@
+% Please check that the preview image in this buffer comes out at the
+% correct position.  I.e., preview images should hide the equation.
+% When done, type C-M-c or M-x exit-recursive-edit.
+%
+% If the image is at the beginning of the line and does not cover the
+% equation, then the result should be considered as fail.
+\documentclass{jarticle}
+
+\begin{document}
+preview-latex �� \(a^{2}=b^{2}+c^{2}\) �Τ褦�ʿ������ܸ� LaTeX �Ǥ�
+preview ��������
+\end{document}
+
+%%% Local Variables:
+%%% coding: euc-jp
+%%% mode: japanese-latex
+%%% TeX-master: t
+%%% TeX-engine: ptex
+%%% TeX-PDF-mode: nil
+%%% End:
diff --git a/tests/japanese/preview-latex.el b/tests/japanese/preview-latex.el
new file mode 100644
index 0000000..4fac047
--- /dev/null
+++ b/tests/japanese/preview-latex.el
@@ -0,0 +1,302 @@
+;;; preview-latex.el --- tests for preview-latex compatibility
+
+;; Copyright (C) 2017 Free Software Foundation, Inc.
+
+;; This file is part of AUCTeX.
+
+;; AUCTeX is free software; you can redistribute it and/or modify it
+;; under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3, or (at your option)
+;; any later version.
+
+;; AUCTeX is distributed in the hope that it will be useful, but
+;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+;; General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with AUCTeX; see the file COPYING.  If not, write to the Free
+;; Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+;; 02110-1301, USA.
+
+;;; Code:
+
+(require 'ert)
+(let ((japanese-TeX-error-messages nil))
+  (require 'tex-jp))
+(require 'preview)
+
+(defun AUCTeX-set-ert-path (&rest sym-val)
+  "Set first element of SYM-VAL to the next one, and so on.
+
+The value is the path to the test file, make sure it is expanded
+in the right directory even when the ERT test from the command
+line and from another directory."
+  (while sym-val
+    (set (pop sym-val)
+        (expand-file-name (pop sym-val)
+                          (when load-file-name
+                            (file-name-directory load-file-name))))))
+
+(AUCTeX-set-ert-path
+ 'platex-shift-jis
+ "preview-error-test.tex"
+ 'preserve-kanji-option
+ "preview-error-test2.tex"
+ 'different-coding-system
+ "prv-dif-code.tex"
+)
+
+;; Make sure coding system output from tex process to be expected
+;; value.
+(setq japanese-TeX-use-kanji-opt-flag t) ; assume unix or darwin.
+
+(setq TeX-process-asynchronous t)
+(setq TeX-after-start-process-function #'TeX-adjust-process-coding-system)
+
+(ert-deftest japanese-preview-shift-jis ()
+  "Coding system `shift_jis' is harmless to preview-latex or not.
+The second byte in `shift_jis' encoding which coincides with a regexp meta
+character used to cause trouble.  Such patterns are tested."
+  ;; The test is meaningful only in interactive session.  Skip in
+  ;; batch mode.
+  (skip-unless (not noninteractive))
+  (let ((TeX-clean-confirm nil)
+       (preview-auto-cache-preamble nil)
+       (process-environment process-environment)
+       (locale-coding-system 'shift_jis)
+       (TeX-japanese-process-output-coding-system nil)
+       (TeX-japanese-process-input-coding-system nil))
+    ;; Make platex binary to output in `shift_jis' encoding.
+    (setenv "LC_ALL" "ja_JP.SJIS")
+    ;; If your startup script for `TeX-shell' (normally "/bin/sh")
+    ;; overwrites LC_ALL, you cannot trust the result of this test.
+    ;; I.e., the positive result can be reported as negative, and the
+    ;; negative can be as positive.
+    (unwind-protect
+       (save-window-excursion
+         (find-file platex-shift-jis)
+         (delete-other-windows)
+         (preview-document)
+         (message "Please wait for asynchronous process to finish...")
+         (sleep-for 5)
+         ;; Actually, this type of trouble seems to be captured early by
+         ;; ert mechanism as error and not to reach here.
+         (should-not (string-match "error in process sentinel:"
+                                   (current-message)))
+         (message "Please wait for asynchronous process to finish...done")
+         (message "Type %s when checking is done."
+                  (substitute-command-keys "\\[exit-recursive-edit]"))
+         (recursive-edit)
+         (should (yes-or-no-p "\
+Did all images come out at the correct position? ")))
+      ;; Cleanup.
+      (set-buffer (get-file-buffer platex-shift-jis))
+      (let* ((buffer (TeX-process-buffer-name (TeX-master-file nil t)))
+            (process (get-buffer-process buffer)))
+       (if process (delete-process process))
+       (kill-buffer buffer))
+      (preview-clearout-document)
+      (TeX-clean t)
+      (dolist (dir preview-temp-dirs)
+       (if (file-exists-p (directory-file-name dir))
+           (delete-directory dir t)))
+      (kill-buffer))))
+
+(ert-deftest japanese-preview-different-coding-system ()
+  "Different coding systems between file and process are OK or not.
+Japanese TeX by itself converts encoding of Japanese text, so sometimes
+`buffer-file-coding-system' and the coding system of the output from
+the process differ."
+  ;; The test is meaningful only in interactive session.  Skip in
+  ;; batch mode.
+  (skip-unless (not noninteractive))
+  (let ((TeX-clean-confirm nil)
+       (preview-auto-cache-preamble nil)
+       (process-environment process-environment)
+       (locale-coding-system 'shift_jis)
+       (TeX-japanese-process-output-coding-system nil)
+       (TeX-japanese-process-input-coding-system nil))
+    ;; Make platex binary to output in `shift_jis' encoding.
+    (setenv "LC_ALL" "ja_JP.SJIS")
+    ;; If your startup script for `TeX-shell' (normally "/bin/sh")
+    ;; overwrites LC_ALL, you cannot trust the result of this test.
+    ;; I.e., the positive result can be reported as negative, and the
+    ;; negative can be as positive.
+    (unwind-protect
+       (save-window-excursion
+         (find-file different-coding-system)
+         (delete-other-windows)
+         (preview-document)
+         (message "Please wait for asynchronous process to finish...")
+         (sleep-for 5)
+         ;; Actually, this type of trouble seems to be captured early by
+         ;; ert mechanism as error and not to reach here.
+         (should-not (string-match "error in process sentinel:"
+                                   (current-message)))
+         (message "Please wait for asynchronous process to finish...done")
+         (message "Type %s when checking is done."
+                  (substitute-command-keys "\\[exit-recursive-edit]"))
+         (recursive-edit)
+         (should (yes-or-no-p "\
+Did all images come out at the correct position? ")))
+      ;; Cleanup.
+      (set-buffer (get-file-buffer different-coding-system))
+      (let* ((buffer (TeX-process-buffer-name (TeX-master-file nil t)))
+            (process (get-buffer-process buffer)))
+       (if process (delete-process process))
+       (kill-buffer buffer))
+      (preview-clearout-document)
+      (TeX-clean t)
+      (dolist (dir preview-temp-dirs)
+       (if (file-exists-p (directory-file-name dir))
+           (delete-directory dir t)))
+      (kill-buffer))))
+
+(ert-deftest japanese-preview-preserve-kanji-option ()
+  "`TeX-inline-preview-internal' preserves kanji option or not.
+Internal Japanese encoding of `platex' is utf-8 by default in TeXLive of
+unix flavors.  So the document encoded in `euc-jp' is not processed
+correctly without kanji option, which used to be dropped during the
+command substitutions performed within preview-latex when preamble cache
+is enabled."
+  ;; The test is meaningful only in interactive session.  Skip in
+  ;; batch mode.
+  (skip-unless (not noninteractive))
+  (let ((TeX-clean-confirm nil)
+       (preview-auto-cache-preamble t)
+       (TeX-japanese-process-output-coding-system nil)
+       (TeX-japanese-process-input-coding-system nil))
+    (unwind-protect
+       (save-window-excursion
+         (find-file preserve-kanji-option)
+         (delete-other-windows)
+         (preview-document)
+         (message "Please wait for asynchronous process to finish...")
+         (sleep-for 3)
+         (message "Please wait for asynchronous process to finish...done")
+         (message "Type %s when checking is done."
+                  (substitute-command-keys "\\[exit-recursive-edit]"))
+         (recursive-edit)
+         (should (yes-or-no-p "\
+Did the image come out at the correct position? ")))
+      ;; Cleanup.
+      (set-buffer (get-file-buffer preserve-kanji-option))
+      (let* ((buffer (TeX-process-buffer-name (TeX-master-file nil t)))
+            (process (get-buffer-process buffer)))
+       (if process (delete-process process))
+       (kill-buffer buffer))
+      (preview-clearout-document)
+      (TeX-clean t)
+      (dolist (dir preview-temp-dirs)
+       (if (file-exists-p (directory-file-name dir))
+           (delete-directory dir t)))
+      (kill-buffer))))
+
+;; The following tests the individual parts fixed in May 2017 and can be
+;; automated with batch mode.  Note that these tests just check specific
+;; parts of preview-latex and do not gurarantee that final outcome of
+;; the preview images are fine in total even if all these tests pass.
+
+(ert-deftest japanese-preview-error-quote-shift-jis ()
+  "`preview-error-quote' is robust against `shift_jis' or not.
+String encoded in `shift_jis' can have regexp meta characters in it."
+  (let (case-fold-search
+       (buffer-file-coding-system 'shift_jis)
+       (TeX-japanese-process-output-coding-system nil))
+    (dolist (str '("$BI=(B(1)" "$BM=(B{a}" "$BG=(B\|" "{$B$"(B} 
%$BG=(B" "$B%"!<%9(B" "$B7?(B"))
+      (should (string-match (preview-error-quote str 'shift_jis) str)))))
+
+(ert-deftest japanese-preview-decode-^^ab ()
+  "`preview--decode-^^ab' doesn't leave regexp meta characters in results."
+  (let (case-fold-search)
+    ;; "$B$"(B" is encoded as \x82 \xa0 in SJIS.
+    (should (string= (preview--decode-^^ab "^^82^^a0" 'shift_jis) "$B$"(B"))
+    ;; "$BI=(B" is encoded as \x95 '\' in SJIS.
+    (should (string= (preview--decode-^^ab "^^95\\" 'shift_jis) "$BI=(B"))
+    ;; "$B!<(B" is encoded as \x81 '[' in SJIS.
+    (should (string= (preview--decode-^^ab "^^81[^^Ab" 'shift_jis) 
"$B!<(B^^Ab"))
+    ;; "$B7?(B" is encoded as \x8c '^' in SJIS.
+    (should (string= (preview--decode-^^ab "$B7?(B^ab" 'shift_jis) 
"$B7?(B^ab"))))
+
+(ert-deftest japanese-preview-convert-^^ab ()
+  "`preview--convert-^^ab' converts ^^ab to raw 8bits and leaves ^^Ab."
+  (let (case-fold-search)
+    (should (string= (preview--convert-^^ab "^^80") "\x80"))
+    (should (string= (preview--convert-^^ab "^^80^^f0") "\x80\xf0"))
+    (should (string= (preview--convert-^^ab "^^^a0") "^\xa0"))
+    (should (string= (preview--convert-^^ab "^^c0^^Ab") "\xc0^^Ab"))))
+
+(ert-deftest japanese-preview-process-coding-system ()
+  "`TeX-inline-preview-internal' records process coding system or not.
+It used to discard the coding system for decode without recording
+previously set by `japanese-TeX-set-process-coding-system'."
+  (let ((dummyfile (make-temp-file "japanese-TeX-ert"))
+       (file-cs 'japanese-shift-jis-unix)
+       (locale-cs 'japanese-iso-8bit-unix)
+       ;; Make `preview-call-hook' inactive.
+       (preview-image-creators nil)
+       process)
+    (find-file dummyfile)
+    ;; Make `japanese-TeX-set-process-coding-system' to be called in
+    ;; `TeX-adjust-process-coding-system'.
+    (setq japanese-TeX-mode t)
+    (setq buffer-file-coding-system file-cs)
+    (unwind-protect
+       (progn
+         (setq process (TeX-inline-preview-internal
+                        "echo foo" dummyfile '(nil . nil) (current-buffer)
+                        '(nil . nil) dummyfile '(nil nil nil)))
+         ;; coding system assigned by `TeX-run-command' should be saved in
+         ;; `preview-coding-system'.
+         (should (coding-system-equal locale-cs preview-coding-system))
+         ;; actual process coding system should be the one derived from the
+         ;; original coding system via `preview-buffer-recode-system'.
+         (should (coding-system-equal
+                  (car (process-coding-system process))
+                  (preview-buffer-recode-system locale-cs)))))
+    ;; Cleanup.
+    ;; Let process to exit before finishing test.
+    (accept-process-output process)
+    (set-buffer (get-file-buffer dummyfile))
+    (let* ((buffer (TeX-process-buffer-name (TeX-master-file nil t)))
+          (process (get-buffer-process buffer)))
+      (if process (delete-process process))
+      (kill-buffer buffer))
+    (kill-buffer)
+    (delete-file dummyfile)))
+
+(ert-deftest japanese-preview-preserve-kanji-option2 ()
+  "`TeX-inline-preview-internal' preserve kanji option or not."
+  (let ((TeX-clean-confirm nil)
+       ;; Make `preview-call-hook' inactive.
+       (preview-image-creators nil)
+       dummy process)
+    (unwind-protect
+       (save-window-excursion
+         (find-file preserve-kanji-option)
+         (setq dummyfile (TeX-master-file))
+         (delete-other-windows)
+         (setq process (TeX-inline-preview-internal
+                        "platex" dummyfile '(nil . nil) (current-buffer)
+                        '(nil . (t . t)) dummyfile '(nil nil nil)))
+         (let ((cmd (process-command process)))
+           (should (string-match "-kanji" (nth (1- (length cmd)) cmd)))))
+      ;; Cleanup.
+      (accept-process-output process)
+      (set-buffer (get-file-buffer preserve-kanji-option))
+      (let* ((buffer (TeX-process-buffer-name (TeX-master-file nil t)))
+            (process (get-buffer-process buffer)))
+       (if process (delete-process process))
+       (kill-buffer buffer))
+      (TeX-clean t)
+      (dolist (dir preview-temp-dirs)
+       (if (file-exists-p (directory-file-name dir))
+           (delete-directory dir t)))
+      (kill-buffer))))
+
+;;; preview-latex.el ends here
+
+;; Local Variables:
+;; coding: iso-2022-jp
+;; End:
diff --git a/tests/japanese/prv-dif-code.tex b/tests/japanese/prv-dif-code.tex
new file mode 100644
index 0000000..f8d1c82
--- /dev/null
+++ b/tests/japanese/prv-dif-code.tex
@@ -0,0 +1,42 @@
+% Please check that all preview images in this buffer come out at the
+% correct position.  I.e., preview images should hide each \section
+% command.
+% When done, type C-M-c or M-x exit-recursive-edit.
+%
+% If the image is at the beginning of the line and "\section{xxx}" is
+% placed far rightward of the image, then the result should be considered
+% as fail.
+% In addition, if the image covers "\section{xxx}" only partially, the
+% result should be considered as fail, too.
+% On the contrary, if the text shown in the image is garbled while the
+% position of the image is correct, then the result should be considered as
+% OK.  That means that your ghostscript is not configured to handle
+% Japanese postscript font names correctly and the functionality of
+% preview-latex is just fine.
+\documentclass{jarticle}
+
+\begin{document}
+
+\section{$BI=(B(1)}
+
+\section{$BM=(B{a}}
+
+\section{\($BG=(B\|\)}
+
+\section{$B%"!<%9(B}
+
+\section{$B7?(B}
+
+% The comment "%$BG=(B" will be displayed rightward to the image on the next
+% line.  That is a normal outcome and should be considered as OK.
+\section{$B$"(B} %$BG=(B
+
+\end{document}
+
+%%% Local Variables:
+%%% coding: iso-2022-jp
+%%% mode: japanese-latex
+%%% TeX-master: t
+%%% TeX-engine: ptex
+%%% TeX-PDF-mode: nil
+%%% End:
diff --git a/tex-buf.el b/tex-buf.el
index 1d72803..0af6551 100644
--- a/tex-buf.el
+++ b/tex-buf.el
@@ -978,9 +978,57 @@ requires that the corresponding mode defines a sensible
       (with-current-buffer buf
        (revert-buffer nil t t)))))
 
-(defvar TeX-after-start-process-function nil
-  "Hooks to run after starting an asynchronous process.
-Used by Japanese TeX to set the coding system.")
+(defvar TeX-after-start-process-function
+  #'TeX-adjust-process-coding-system
+  "Function to adjust coding system of an asynchronous process.
+Called with one argument PROCESS.")
+
+(defun TeX-adjust-process-coding-system (process)
+  "Adjust coding system of PROCESS to suitable value.
+Usually coding system is the same as the TeX file with eol format
+adjusted to OS default value.  Take care of Japanese TeX, which
+requires special treatment."
+  (when (featurep 'mule)
+    (if (and (boundp 'japanese-TeX-mode)
+            (with-current-buffer TeX-command-buffer
+              japanese-TeX-mode))
+       (japanese-TeX-set-process-coding-system process)
+      (let ((cs (with-current-buffer TeX-command-buffer
+                 buffer-file-coding-system)))
+       ;; The value of `buffer-file-coding-system' is sometimes
+       ;; undecided-{unix,dos,mac}.  That happens when the file
+       ;; contains no multibyte chars and only end of line format is
+       ;; determined.  Emacs lisp reference recommends not to use
+       ;; undecided-* for process coding system, so it might seem
+       ;; reasonable to change undecided-* to some fixed coding
+       ;; system like this:
+       ;; (if (eq 'undecided (coding-sytem-type cs))
+       ;;     (setq cs 'utf-8))
+       ;; However, that can lose when the following conditions are
+       ;; met:
+       ;; (1) The document is divided into multiple files.
+       ;; (2) The command buffer contains no multibyte chars.
+       ;; (3) The other files contain mutlibyte chars and saved in
+       ;;     a coding system other than the coding system chosen
+       ;;     above.
+       ;; So we leave undecided-* unchanged here.  Although
+       ;; undecided-* is not quite safe for the coding system for
+       ;; encoding, i.e., keyboard input to the TeX process, we
+       ;; expect that this does not raise serious problems because it
+       ;; is pretty rare that TeX process needs keyboard input of
+       ;; multibyte chars.
+
+       ;; Eol format of TeX files can differ from OS default. TeX
+       ;; binaries accept all type of eol format in the given files
+       ;; and output messages according to OS default.  So we set eol
+       ;; format to OS default value.
+       (setq cs (coding-system-change-eol-conversion
+                 cs
+                 ;; The eol of macosX is LF, not CR.  So we choose
+                 ;; other than `unix' only for w32 system.
+                 ;; FIXME: what should we do for cygwin?
+                 (if (eq system-type 'windows-nt) 'dos 'unix)))
+       (set-process-coding-system process cs cs)))))
 
 (defcustom TeX-show-compilation nil
   "*If non-nil, show output of TeX compilation in other window."
diff --git a/tex-jp.el b/tex-jp.el
index 5bdae54..09909ba 100644
--- a/tex-jp.el
+++ b/tex-jp.el
@@ -446,14 +446,20 @@ See also a user custom option 
`TeX-japanese-process-input-coding-system'."
                 ;; uptex $B$J$i(B utf-8 $B$K8GDj$9$k!#(B
                 (t
                  'utf-8))))
+
+       ;; $B2~9T%3!<%I$r;XDj!#(B
+       (setq dec (coding-system-change-eol-conversion
+                  dec
+                  (if (eq system-type 'windows-nt) 'dos 'unix))
+             enc (coding-system-change-eol-conversion
+                  enc
+                  (if (eq system-type 'windows-nt) 'dos 'unix)))
+
        ;; Customize $BCM$,$"address@hidden(B
        (set-process-coding-system
         process
         (or TeX-japanese-process-output-coding-system dec)
         (or TeX-japanese-process-input-coding-system enc))))))
-(when (featurep 'mule)
-  (setq TeX-after-start-process-function
-        #'japanese-TeX-set-process-coding-system))
 
 (defun japanese-TeX-coding-ejsu (coding-system)
   "Convert japanese CODING-SYSTEM to mnemonic string.