Re: ispell.el doesn't find all of aspell's dictionaries

[Magnus Henoch]

Juri Linkov <address@hidden> writes:

>>     There is one problem with this code: the dictionary list in the menu
>>     bar is only updated when ispell.el is loaded (probably due to a call 
>>     to an auto-loaded function).  So before any ispell-* command is run,
>>     the menu shows the static list of ispell dictionaries, and if new
>>     dictionaries are added, the menu is not updated.  I'm not sure what
>>     can/should be done about that.
>>
>> Some ideas:
>>
>> 1. Have a menu item, "Update Dictionary List", to check again.
>>
>> 2. If checking is fast, check whenever the user is about to
>> specify a dictionary.
>
> Rechecking the dictionary list is needed also after changing the
> program name in `ispell-program-name' (also don't forget to call
> `ispell-kill-ispell').

Looking closer at it, I see that the dictionary menu is far too long.
It extends beyond the bottom of the screen, so it isn't very useful.
As there is a "Change Dictionary" menu item just above, I propose that
the dictionary list in the menu be removed.  (I've done so in my patch
below)

>> In the mean time, it would be useful for people to start testing your
>> patch, so that by the time your papers have arrived, it will be
>> thoroughly ready to go.
>
> I tested the patch and discovered several problems:
>
>> +   (let ((dictionaries
>> +     (split-string
>> +      (with-temp-buffer
>> +        (call-process ispell-program-name nil t nil "dicts")
>
> `aspell dicts' returns plain dictionary names like
> `en_GB-ize-w_accents', but misses readable aliases like `english'.
> Is it possible for aspell to return all aliases as well?
>
> Aspell aliases will also make many aspell dictionary names compatible
> with ispell dictionary names, so it would be good if users will be able
> to use the same dictionary names for aspell and ispell without the need
> to care about differences.

Good point.  I've added code to find all aliases.  Now it should be
entirely compatible with any way of specifying dictionaries for
Ispell.

>> +     (list dict-name
>> +      "[[:alpha:]]"
>> +      "[^[:alpha:]]"
>
> This code doesn't take into account the real charsets used in
> dictionaries.  Maybe it should extract the `charset' field
> from `dat' files?

Is that important?  Aspell can handle anything we throw at it, as long
as it is UTF-8.

>> + (defun aspell-find-dictonaries ()
>                        ===========
> Typo.  Use a spell checker to check your code ;-)

Thanks ;-)

In this new patch I've also included your changes for user awareness
of which program is being used, and changed aspell-* to
ispell-aspell-*.

*** orig/lisp/textmodes/ispell.el
--- mod/lisp/textmodes/ispell.el
***************
*** 862,870 ****
--- 862,981 ----
                                   )
    "Non-nil means that the OS supports asynchronous processes.")
  
+ ;; Make ispell.el work better with aspell.
+ 
+ (defvar ispell-aspell-have-dictionaries nil
+   "Non-nil if we have queried aspell for dictionaries at least once.")
+ 
+ (defun ispell-aspell-find-dictionaries ()
+   "Find aspell's dictionaries, and record into `ispell-dictionary-alist'."
+   (interactive)
+   (unless ispell-really-aspell
+     (error "This function only works with aspell."))
+   (let ((dictionaries
+        (split-string
+         (with-temp-buffer
+           (call-process ispell-program-name nil t nil "dicts")
+           (buffer-string)))))
+     (setq ispell-dictionary-alist
+         (mapcar #'ispell-aspell-find-dictionary dictionaries))
+     (ispell-aspell-add-aliases)
+     ;; Add a default entry
+     (let* ((english-dict (assoc "en" ispell-dictionary-alist))
+          (default-dict (cons nil (cdr english-dict))))
+       (push default-dict ispell-dictionary-alist))
+     (setq ispell-aspell-have-dictionaries t)))
+ 
+ (defvar ispell-aspell-data-dir nil
+   "Data directory of aspell.")
+ 
+ (defvar ispell-aspell-dict-dir nil
+   "Dictionary directory of aspell.")
+ 
+ (defun ispell-aspell-get-config-value (key)
+   "Return value of aspell configuration option KEY.
+ Assumes that value contains no whitespace."
+   (with-temp-buffer
+     (call-process ispell-program-name nil t nil "config" key)
+     (car (split-string (buffer-string)))))
+ 
+ (defun ispell-aspell-find-data-dir ()
+   "Find aspell's data directory and set `ispell-aspell-data-dir'.
+ Return ispell-aspell-data-dir."
+   (setq ispell-aspell-data-dir (ispell-aspell-get-config-value "data-dir")))
+ 
+ (defun ispell-aspell-find-dict-dir ()
+   "Find aspell's dictionary directory and set `ispell-aspell-dict-dir'.
+ Return ispell-aspell-dict-dir."
+   (setq ispell-aspell-dict-dir (ispell-aspell-get-config-value "dict-dir")))
+ 
+ (defun ispell-aspell-find-dictionary (dict-name)
+   (let* ((lang ;; Strip out region, variant, etc.
+         (and (string-match "^[[:alpha:]]+" dict-name)
+              (match-string 0 dict-name)))
+        (data-file
+         (concat (or ispell-aspell-data-dir (ispell-aspell-find-data-dir))
+                 "/" lang ".dat"))
+        otherchars)
+     ;; This file really should exist; there is no sensible recovery.
+     (with-temp-buffer
+       (insert-file-contents data-file)
+       ;; There is zero or one line with special characters declarations.
+       (when (search-forward-regexp "^special" nil t)
+       (let ((specials (split-string 
+                        (buffer-substring (point)
+                                          (progn (end-of-line) (point))))))
+         ;; The line looks like: special ' -** - -** . -** : -*-
+         ;; -** means that this character
+         ;;    - doesn't appear at word start
+         ;;    * may appear in the middle of a word
+         ;;    * may appear at word end
+         ;; `otherchars' is about the middle case.
+         (while specials
+           (when (eq (aref (cadr specials) 1) ?*)
+             (push (car specials) otherchars))
+           (setq specials (cddr specials))))))
+     (list dict-name
+         "[[:alpha:]]"
+         "[^[:alpha:]]"
+         (regexp-opt otherchars)
+         t                             ; We can't tell, so set this to t
+         (list "-d" dict-name "--encoding=utf-8")
+         nil                           ; aspell doesn't support this
+         ;; Here we specify the encoding to use while communicating with
+         ;; aspell.  This doesn't apply to command line arguments, so
+         ;; just don't pass words to spellcheck as arguments...
+         'utf-8)))
+ 
+ (defun ispell-aspell-add-aliases ()
+   "Find aspell's dictionary aliases and add them to 
`ispell-dictionary-alist'."
+   (let ((aliases (file-expand-wildcards
+                 (concat (or ispell-aspell-dict-dir
+                             (ispell-aspell-find-dict-dir))
+                         "/*.alias"))))
+     (dolist (alias-file aliases)
+       (with-temp-buffer
+       (insert-file-contents alias-file)
+       ;; Look for a line "add FOO.multi", extract FOO
+       (when (search-forward-regexp "^add \\([^.]+\\)\\.multi" nil t)
+         (let* ((aliasname (file-name-sans-extension 
+                            (file-name-nondirectory alias-file)))
+                (already-exists-p (assoc aliasname ispell-dictionary-alist))
+                (realname (match-string 1))
+                (realdict (assoc realname ispell-dictionary-alist)))
+           (when (and realdict (not already-exists-p))
+             (push (cons aliasname (cdr realdict)) 
ispell-dictionary-alist))))))))
+ 
  (defun ispell-valid-dictionary-list ()
    "Returns a list of valid dictionaries.
  The variable `ispell-library-directory' defines the library location."
+   ;; If Ispell is really Aspell, query it for the dictionary list.
+   (when (and (not ispell-aspell-have-dictionaries)
+            (condition-case ()
+                (progn (ispell-check-version) t)
+              (error nil))
+            ispell-really-aspell)
+     (ispell-aspell-find-dictionaries))
    (let ((dicts (append ispell-local-dictionary-alist ispell-dictionary-alist))
        (dict-list (cons "default" nil))
        name load-dict)
***************
*** 875,881 ****
        (if (and
           name
           ;; include all dictionaries if lib directory not known.
!          (or (not ispell-library-directory)
               (file-exists-p (concat ispell-library-directory
                                      "/" name ".hash"))
               (file-exists-p (concat ispell-library-directory "/" name ".has"))
--- 986,994 ----
        (if (and
           name
           ;; include all dictionaries if lib directory not known.
!          ;; For Aspell, we already know which dictionaries exist.
!          (or ispell-really-aspell
!              (not ispell-library-directory)
               (file-exists-p (concat ispell-library-directory
                                      "/" name ".hash"))
               (file-exists-p (concat ispell-library-directory "/" name ".has"))
***************
*** 887,922 ****
          (setq dict-list (cons name dict-list))))
      dict-list))
  
- ;;;###autoload
- (if ispell-menu-map-needed
-     (let ((dicts (if (fboundp 'ispell-valid-dictionary-list)
-                    (ispell-valid-dictionary-list)
-                  ;; This case is used in loaddefs.el
-                  ;; since ispell-valid-dictionary-list isn't defined then.
-                  (mapcar (lambda (x) (or (car x) "default"))
-                          ispell-dictionary-alist)))
-         (dict-map (make-sparse-keymap "Dictionaries")))
-       (setq ispell-menu-map (make-sparse-keymap "Spell"))
-       ;; add the dictionaries to the bottom of the list.
-       (if (not dicts)
-         (define-key ispell-menu-map [default]
-           '("Select Default Dict"
-             "Dictionary for which Ispell was configured"
-             . (lambda () (interactive)
-                 (ispell-change-dictionary "default")))))
-       (fset 'ispell-dict-map dict-map)
-       (define-key ispell-menu-map [dictionaries]
-       `(menu-item "Select Dict" ispell-dict-map))
-       (dolist (name dicts)
-       (define-key dict-map (vector (intern name))
-         (cons (concat "Select " (capitalize name) " Dict")
-               `(lambda () (interactive)
-                  (ispell-change-dictionary ,name)))))))
- 
  ;;; define commands in menu in opposite order you want them to appear.
  ;;;###autoload
  (if ispell-menu-map-needed
      (progn
        (define-key ispell-menu-map [ispell-change-dictionary]
        '(menu-item "Change Dictionary..." ispell-change-dictionary
                    :help "Supply explicit dictionary file name"))
--- 1000,1010 ----
          (setq dict-list (cons name dict-list))))
      dict-list))
  
  ;;; define commands in menu in opposite order you want them to appear.
  ;;;###autoload
  (if ispell-menu-map-needed
      (progn
+       (setq ispell-menu-map (make-sparse-keymap "Spell"))
        (define-key ispell-menu-map [ispell-change-dictionary]
        '(menu-item "Change Dictionary..." ispell-change-dictionary
                    :help "Supply explicit dictionary file name"))
***************
*** 1660,1666 ****
      ;; setup the *Choices* buffer with valid data.
      (save-excursion
        (set-buffer (get-buffer-create ispell-choices-buffer))
!       (setq mode-line-format (concat "--  %b  --  word: " word))
        ;; XEmacs: no need for horizontal scrollbar in choices window
        (with-no-warnings
         (and (fboundp 'set-specifier)
--- 1748,1757 ----
      ;; setup the *Choices* buffer with valid data.
      (save-excursion
        (set-buffer (get-buffer-create ispell-choices-buffer))
!       (setq mode-line-format
!           (concat "--  %b  --  word: " word
!                   "  --  dict: " (or ispell-current-dictionary "default")
!                   "  --  prog: " (file-name-nondirectory 
ispell-program-name)))
        ;; XEmacs: no need for horizontal scrollbar in choices window
        (with-no-warnings
         (and (fboundp 'set-specifier)
***************
*** 1820,1828 ****
                              (erase-buffer)
                              (setq count ?0
                                    skipped 0
!                                   mode-line-format (concat
!                                                     "--  %b  --  word: "
!                                                     new-word)
                                    miss (lookup-words new-word)
                                    choices miss
                                    line ispell-choices-win-default-height)
--- 1911,1920 ----
                              (erase-buffer)
                              (setq count ?0
                                    skipped 0
!                                   mode-line-format
!                                   (concat "--  %b  --  word: " new-word
!                                           "  --  dict: "
!                                           ispell-alternate-dictionary)
                                    miss (lookup-words new-word)
                                    choices miss
                                    line ispell-choices-win-default-height)
***************
*** 2513,2521 ****
        (rstart (make-marker)))
    (unwind-protect
        (save-excursion
!       (message "Spell checking %s using %s dictionary..."
                 (if (and (= reg-start (point-min)) (= reg-end (point-max)))
                     (buffer-name) "region")
                 (or ispell-current-dictionary "default"))
        ;; Returns cursor to original location.
        (save-window-excursion
--- 2605,2614 ----
        (rstart (make-marker)))
    (unwind-protect
        (save-excursion
!       (message "Spell checking %s using %s with %s dictionary..."
                 (if (and (= reg-start (point-min)) (= reg-end (point-max)))
                     (buffer-name) "region")
+                (file-name-nondirectory ispell-program-name)
                 (or ispell-current-dictionary "default"))
        ;; Returns cursor to original location.
        (save-window-excursion
***************
*** 2533,2539 ****
                  (set-marker skip-region-start (- (point) (length key)))
                  (goto-char reg-start)))
            (let (message-log-max)
!             (message "Continuing spelling check using %s dictionary..."
                       (or ispell-current-dictionary "default")))
            (set-marker rstart reg-start)
            (set-marker ispell-region-end reg-end)
--- 2626,2633 ----
                  (set-marker skip-region-start (- (point) (length key)))
                  (goto-char reg-start)))
            (let (message-log-max)
!             (message "Continuing spelling check using %s with %s 
dictionary..."
!                      (file-name-nondirectory ispell-program-name)
                       (or ispell-current-dictionary "default")))
            (set-marker rstart reg-start)
            (set-marker ispell-region-end reg-end)
***************
*** 2610,2616 ****
        (if (not recheckp) (set-marker ispell-region-end nil))
        ;; Only save if successful exit.
        (ispell-pdict-save ispell-silently-savep)
!       (message "Spell-checking done")))))
  
  
  (defun ispell-begin-skip-region-regexp ()
--- 2704,2712 ----
        (if (not recheckp) (set-marker ispell-region-end nil))
        ;; Only save if successful exit.
        (ispell-pdict-save ispell-silently-savep)
!       (message "Spell-checking using %s with %s dictionary done"
!              (file-name-nondirectory ispell-program-name)
!              (or ispell-current-dictionary "default"))))))
  
  
  (defun ispell-begin-skip-region-regexp ()