From bd1c7ca67e7429e07f78d4ff49163fd7a67a6765 Mon Sep 17 00:00:00 2001 From: Paul Eggert Date: Thu, 21 Apr 2016 19:26:34 -0700 Subject: [PATCH] Improve character name escapes * doc/lispref/nonascii.texi (Character Properties): Avoid duplication of Unicode names. Reformat examples to fit in narrow pages. * doc/lispref/objects.texi (General Escape Syntax): Simplify and better-organize explanation of \N{...} escapes. * src/character.h (CHAR_SURROGATE_PAIR_P): Remove; unused. (char_surrogate_p): New inline function. * src/lread.c: Do not include string.h; no longer needed. (invalid_character_name, check_scalar_value): Remove; the ideas behind these functions are now bundled into character_name_to_code. (character_name_to_code): Remove undocumented support for "CJK IDEOGRAPH-XXXX" names, as "U+XXXX" suffices. Reject monstrosities like "\N{U+-0}" and null bytes in \N escapes. Reject floating point in \N escapes instead of returning garbage. Use AUTO_STRING_WITH_LEN to lessen pressure on the garbage collector. * test/src/lread-tests.el (lread-char-number, lread-char-name) (lread-string-char-number, lread-string-char-name): Test runtime behavior, not compile-time, as the test framework is not set up to test compile-time. (lread-char-surrogate-1, lread-char-surrogate-2) (lread-char-surrogate-3, lread-char-surrogate-4) (lread-string-char-number-2, lread-string-char-number-3): New tests. (lread-string-char-number-1): Rename from lread-string-char-number. --- doc/lispref/nonascii.texi | 15 ++++--- doc/lispref/objects.texi | 52 +++++++++++------------ src/character.h | 13 +++--- src/lread.c | 104 +++++++++++++--------------------------------- test/src/lread-tests.el | 32 ++++++++------ 5 files changed, 89 insertions(+), 127 deletions(-) diff --git a/doc/lispref/nonascii.texi b/doc/lispref/nonascii.texi index 66ad9ac..0e4aa86 100644 --- a/doc/lispref/nonascii.texi +++ b/doc/lispref/nonascii.texi @@ -622,18 +622,21 @@ Character Properties @result{} Nd @end group @group -;; U+2084 SUBSCRIPT FOUR -(get-char-code-property ?\u2084 'digit-value) +;; U+2084 +(get-char-code-property address@hidden address@hidden + 'digit-value) @result{} 4 @end group @group -;; U+2155 VULGAR FRACTION ONE FIFTH -(get-char-code-property ?\u2155 'numeric-value) +;; U+2155 +(get-char-code-property address@hidden FRACTION ONE address@hidden + 'numeric-value) @result{} 0.2 @end group @group -;; U+2163 ROMAN NUMERAL FOUR -(get-char-code-property address@hidden NUMERAL address@hidden 'numeric-value) +;; U+2163 +(get-char-code-property address@hidden NUMERAL address@hidden + 'numeric-value) @result{} 4 @end group @group diff --git a/doc/lispref/objects.texi b/doc/lispref/objects.texi index 96b334d..54894b8 100644 --- a/doc/lispref/objects.texi +++ b/doc/lispref/objects.texi @@ -353,25 +353,32 @@ General Escape Syntax control characters, Emacs provides several types of escape syntax that you can use to specify address@hidden text characters. address@hidden address@hidden @cindex @samp{\} in character constant @cindex backslash in character constants @cindex unicode character escape - Firstly, you can specify characters by their Unicode values. address@hidden@var{nnnn}} represents a character with Unicode code point address@hidden@var{nnnn}}, where @var{nnnn} is (by convention) a hexadecimal -number with exactly four digits. The backslash indicates that the -subsequent characters form an escape sequence, and the @samp{u} -specifies a Unicode escape sequence. - - There is a slightly different syntax for specifying Unicode -characters with code points higher than @address@hidden: address@hidden@var{nnnnnn}} represents the character with code point address@hidden@var{nnnnnn}}, where @var{nnnnnn} is a six-digit hexadecimal -number. The Unicode Standard only defines code points up to address@hidden@var{10ffff}}, so if you specify a code point higher than -that, Emacs signals an error. - - Secondly, you can specify characters by their hexadecimal character +You can specify characters by their Unicode names, if any. address@hidden@address@hidden@}} represents the Unicode character named address@hidden Thus, @address@hidden SMALL LETTER A WITH address@hidden is +equivalent to @code{?à} and denotes the Unicode character U+00E0. To +simplify entering multi-line strings, you can replace spaces in the +names by non-empty sequences of whitespace (e.g., newlines). + address@hidden +You can specify characters by their Unicode values. address@hidden@address@hidden@}} represents a character with Unicode code point address@hidden, where @var{X} is a hexadecimal number. Also, address@hidden@var{xxxx}} and @address@hidden represent code +points @var{xxxx} and @var{xxxxxxxx}, respectively, where each @var{x} +is a single hexadecimal digit. For example, @address@hidden@}}, address@hidden and @code{?\U000000E0} are all equivalent to @code{?à} +and to @address@hidden SMALL LETTER A WITH address@hidden The Unicode +Standard defines code points only up to @address@hidden, so if +you specify a code point higher than that, Emacs signals an error. + address@hidden +You can specify characters by their hexadecimal character codes. A hexadecimal escape sequence consists of a backslash, @samp{x}, and the hexadecimal character code. Thus, @samp{?\x41} is the character @kbd{A}, @samp{?\x1} is the character @kbd{C-a}, and @@ -379,23 +386,16 @@ General Escape Syntax You can use any number of hex digits, so you can represent any character code in this way. address@hidden @cindex octal character code - Thirdly, you can specify characters by their character code in +You can specify characters by their character code in octal. An octal escape sequence consists of a backslash followed by up to three octal digits; thus, @samp{?\101} for the character @kbd{A}, @samp{?\001} for the character @kbd{C-a}, and @code{?\002} for the character @kbd{C-b}. Only characters up to octal code 777 can be specified this way. - Fourthly, you can specify characters by their name. A character -name escape sequence consists of a backslash, @address@hidden, the Unicode -character name, and @address@hidden Alternatively, you can also put the -numeric code point value between the braces, using the syntax address@hidden@address@hidden, where @samp{nnnn} denotes between one and eight -hexadecimal digits. Thus, @address@hidden CAPITAL LETTER address@hidden and address@hidden@address@hidden both denote the character @kbd{A}. To simplify -entering multi-line strings, you can replace spaces in the character -names by arbitrary non-empty sequence of whitespace (e.g., newlines). address@hidden enumerate These escape sequences may also be used in strings. @xref{Non-ASCII in Strings}. diff --git a/src/character.h b/src/character.h index bc3e155..586f330 100644 --- a/src/character.h +++ b/src/character.h @@ -612,14 +612,13 @@ sanitize_char_width (EMACS_INT width) : (c) <= 0xE01EF ? (c) - 0xE0100 + 17 \ : 0) -/* If C is a high surrogate, return 1. If C is a low surrogate, - return 2. Otherwise, return 0. */ +/* Return true if C is a surrogate. */ -#define CHAR_SURROGATE_PAIR_P(c) \ - ((c) < 0xD800 ? 0 \ - : (c) <= 0xDBFF ? 1 \ - : (c) <= 0xDFFF ? 2 \ - : 0) +INLINE bool +char_surrogate_p (int c) +{ + return 0xD800 <= c && c <= 0xDFFF; +} /* Data type for Unicode general category. diff --git a/src/lread.c b/src/lread.c index c3b6bd7..a42c1f6 100644 --- a/src/lread.c +++ b/src/lread.c @@ -44,7 +44,6 @@ along with GNU Emacs. If not, see . */ #include "termhooks.h" #include "blockinput.h" #include -#include #ifdef MSDOS #include "msdos.h" @@ -2151,88 +2150,42 @@ grow_read_buffer (void) MAX_MULTIBYTE_LENGTH, -1, 1); } -/* Signal an invalid-read-syntax error indicating that the character - name in an \N{…} literal is invalid. */ -static _Noreturn void -invalid_character_name (Lisp_Object name) -{ - AUTO_STRING (format, "\\N{%s}"); - xsignal1 (Qinvalid_read_syntax, CALLN (Fformat, format, name)); -} - -/* Check that CODE is a valid Unicode scalar value, and return its - value. CODE should be parsed from the character name given by - NAME. NAME is used for error messages. */ +/* Return the scalar value that has the Unicode character name NAME. + Raise 'invalid-read-syntax' if there is no such character. */ static int -check_scalar_value (Lisp_Object code, Lisp_Object name) +character_name_to_code (char const *name, ptrdiff_t name_len) { - if (! NUMBERP (code)) - invalid_character_name (name); - EMACS_INT i = XINT (code); - if (! (0 <= i && i <= MAX_UNICODE_CHAR) - /* Don't allow surrogates. */ - || (0xD800 <= code && code <= 0xDFFF)) - invalid_character_name (name); - return i; -} + Lisp_Object code; -/* If NAME starts with PREFIX, interpret the rest as a hexadecimal - number and return its value. Raise invalid-read-syntax if the - number is not a valid scalar value. Return −1 if NAME doesn’t - start with PREFIX. */ -static int -parse_code_after_prefix (Lisp_Object name, const char *prefix) -{ - ptrdiff_t name_len = SBYTES (name); - ptrdiff_t prefix_len = strlen (prefix); - /* Allow between one and eight hexadecimal digits after the - prefix. */ - if (prefix_len < name_len && name_len <= prefix_len + 8 - && memcmp (SDATA (name), prefix, prefix_len) == 0) + /* Code point as U+XXXX.... */ + if (name[0] == 'U' && name[1] == '+') { - Lisp_Object code = string_to_number (SDATA (name) + prefix_len, 16, false); - if (NUMBERP (code)) - return check_scalar_value (code, name); + /* Pass the leading '+' to string_to_number, so that it + rejects monstrosities such as negative values. */ + code = string_to_number (name + 1, 16, false); + } + else + { + /* Look up the name in the table returned by 'ucs-names'. */ + AUTO_STRING_WITH_LEN (namestr, name, name_len); + Lisp_Object names = call0 (Qucs_names); + code = CDR (Fassoc (namestr, names)); } - return -1; -} -/* Returns the scalar value that has the Unicode character name NAME. - Raises `invalid-read-syntax' if there is no such character. */ -static int -character_name_to_code (Lisp_Object name) -{ - /* Code point as U+N, where N is between 1 and 8 hexadecimal - digits. */ - int code = parse_code_after_prefix (name, "U+"); - if (code >= 0) - return code; - - /* CJK ideographs are not contained in the association list returned - by `ucs-names'. But they follow a predictable naming pattern: a - fixed prefix plus the hexadecimal codepoint value. */ - code = parse_code_after_prefix (name, "CJK IDEOGRAPH-"); - if (code >= 0) + if (! (INTEGERP (code) + && 0 <= XINT (code) && XINT (code) <= MAX_UNICODE_CHAR + && ! char_surrogate_p (XINT (code)))) { - /* Various ranges of CJK characters; see UnicodeData.txt. */ - if ((0x3400 <= code && code <= 0x4DB5) - || (0x4E00 <= code && code <= 0x9FD5) - || (0x20000 <= code && code <= 0x2A6D6) - || (0x2A700 <= code && code <= 0x2B734) - || (0x2B740 <= code && code <= 0x2B81D) - || (0x2B820 <= code && code <= 0x2CEA1)) - return code; - else - invalid_character_name (name); + AUTO_STRING (format, "\\N{%s}"); + AUTO_STRING_WITH_LEN (namestr, name, name_len); + xsignal1 (Qinvalid_read_syntax, CALLN (Fformat, format, namestr)); } - /* Look up the name in the table returned by `ucs-names'. */ - Lisp_Object names = call0 (Qucs_names); - return check_scalar_value (CDR (Fassoc (name, names)), name); + return XINT (code); } /* Bound on the length of a Unicode character name. As of - Unicode 9.0.0 the maximum is 83, so this should be safe. */ + Unicode 9.0.0 the maximum is 83, so this should be safe. */ enum { UNICODE_CHARACTER_NAME_LENGTH_BOUND = 200 }; /* Read a \-escape sequence, assuming we already read the `\'. @@ -2458,14 +2411,14 @@ read_escape (Lisp_Object readcharfun, bool stringp) end_of_file_error (); if (c == '}') break; - if (! c_isascii (c)) + if (! (0 < c && c < 0x80)) { AUTO_STRING (format, - "Non-ASCII character U+%04X in character name"); + "Invalid character U+%04X in character name"); xsignal1 (Qinvalid_read_syntax, CALLN (Fformat, format, make_natnum (c))); } - /* We treat multiple adjacent whitespace characters as a + /* Treat multiple adjacent whitespace characters as a single space character. This makes it easier to use character names in e.g. multi-line strings. */ if (c_isspace (c)) @@ -2483,7 +2436,8 @@ read_escape (Lisp_Object readcharfun, bool stringp) } if (length == 0) invalid_syntax ("Empty character name"); - return character_name_to_code (make_unibyte_string (name, length)); + name[length] = '\0'; + return character_name_to_code (name, length); } default: diff --git a/test/src/lread-tests.el b/test/src/lread-tests.el index ff5d0f6..2ebaf49 100644 --- a/test/src/lread-tests.el +++ b/test/src/lread-tests.el @@ -1,6 +1,6 @@ ;;; lread-tests.el --- tests for lread.c -*- lexical-binding: t; -*- -;; Copyright (C) 2016 Google Inc. +;; Copyright (C) 2016 Free Software Foundation, Inc. ;; Author: Philipp Stephani @@ -26,11 +26,10 @@ ;;; Code: (ert-deftest lread-char-number () - (should (equal ?\N{U+A817} #xA817))) + (should (equal (read "?\\N{U+A817}") #xA817))) (ert-deftest lread-char-name () - (should (equal ?\N{SYLOTI NAGRI LETTER - DHO} + (should (equal (read "?\\N{SYLOTI NAGRI LETTER \n DHO}") #xA817))) (ert-deftest lread-char-invalid-number () @@ -46,16 +45,23 @@ (ert-deftest lread-char-empty-name () (should-error (read "?\\N{}") :type 'invalid-read-syntax)) -(ert-deftest lread-char-cjk-name () - (should (equal ?\N{CJK IDEOGRAPH-2B734} #x2B734))) - -(ert-deftest lread-char-invalid-cjk-name () - (should-error (read "?\\N{CJK IDEOGRAPH-2B735}") :type 'invalid-read-syntax)) - -(ert-deftest lread-string-char-number () - (should (equal "a\N{U+A817}b" "a\uA817b"))) +(ert-deftest lread-char-surrogate-1 () + (should-error (read "?\\N{U+D800}") :type 'invalid-read-syntax)) +(ert-deftest lread-char-surrogate-2 () + (should-error (read "?\\N{U+D801}") :type 'invalid-read-syntax)) +(ert-deftest lread-char-surrogate-3 () + (should-error (read "?\\N{U+Dffe}") :type 'invalid-read-syntax)) +(ert-deftest lread-char-surrogate-4 () + (should-error (read "?\\N{U+DFFF}") :type 'invalid-read-syntax)) + +(ert-deftest lread-string-char-number-1 () + (should (equal (read "a\\N{U+A817}b") "a\uA817bx"))) +(ert-deftest lread-string-char-number-2 () + (should-error (read "?\\N{0.5}") :type 'invalid-read-syntax)) +(ert-deftest lread-string-char-number-3 () + (should-error (read "?\\N{U+-0}") :type 'invalid-read-syntax)) (ert-deftest lread-string-char-name () - (should (equal "a\N{SYLOTI NAGRI LETTER DHO}b" "a\uA817b"))) + (should (equal (read "a\\N{SYLOTI NAGRI LETTER DHO}b") "a\uA817b"))) ;;; lread-tests.el ends here -- 2.5.5