01-fyi-undef-token-number.patch

[Akim Demaille]

Index: ChangeLog
from  Akim Demaille  <address@hidden>
        
        * src/gram.h, src/gram.c (error_token_number): Remove, use
        errtoken->number.
        * src/reader.c (reader): Don't specify the user token number (2)
        for $undefined, as it uselessly prevents using it.
        * src/gram.h (token_number_t): Move to...
        * src/symtab.h: here.
        (state_t.number): Is a token_number_t.
        * src/print.c, src/reader.c: Use undeftoken->number instead of
        hard coded 2.
        (Even though this 2 is not the same as above: the number of the
        undeftoken remains being 2, it is its user token number which
        might not be 2).
        * src/output.c (prepare_tokens): Rename the `maxtok' muscle with
        `user_token_number_max'.
        Output `undef_token_number'.
        * data/bison.simple, data/bison.c++: Use them.
        Be sure to map invalid yylex return values to
        `undef_token_number'.  This saves us from gratuitous SEGV.
        
        * tests/conflicts.at (Solved SR Conflicts)
        (Unresolved SR Conflicts): Adjust.
        * tests/regression.at (Web2c Actions): Adjust.
        
Index: NEWS
--- NEWS Mon, 08 Apr 2002 13:57:45 +0200 akim
+++ NEWS Mon, 08 Apr 2002 23:19:57 +0200 akim
@@ -3,6 +3,14 @@
 
 Changes in version 1.49a:
 
+* Undefined token
+  The undefined token was systematically mapped to 2 which prevented
+  the use of 2 from the user.  This is no longer the case.
+
+* Undefined token
+  If yylex returned a code out of range, yyparse could die.  This is
+  no longer the case.
+
 * Large grammars
   Are now supported (large token numbers, large grammar size (= sum of
   the LHS and RHS lengths).
Index: src/gram.c
--- src/gram.c Sun, 07 Apr 2002 20:28:06 +0200 akim
+++ src/gram.c Mon, 08 Apr 2002 23:37:31 +0200 akim
@@ -48,8 +48,6 @@
 
 int pure_parser = 0;
 
-int error_token_number = 0;
-
 
 /*--------------------------------------.
 | Return the number of symbols in RHS.  |
Index: src/gram.h
--- src/gram.h Sun, 07 Apr 2002 21:22:52 +0200 akim
+++ src/gram.h Mon, 08 Apr 2002 23:37:31 +0200 akim
@@ -154,7 +154,6 @@
 /* TOKEN_TRANSLATION -- a table indexed by a token number as returned
    by the user's yylex routine, it yields the internal token number
    used by the parser and throughout bison.  */
-typedef short token_number_t;
 extern token_number_t *token_translations;
 extern int max_user_token_number;
 
@@ -168,10 +167,6 @@
    and reentrant.  */
 
 extern int pure_parser;
-
-/* ERROR_TOKEN_NUMBER is the token number of the error token.  */
-
-extern int error_token_number;
 
 /* Report the length of the RHS. */
 int rule_rhs_length PARAMS ((rule_t *rule));
Index: src/output.c
--- src/output.c Sun, 07 Apr 2002 21:22:52 +0200 akim
+++ src/output.c Mon, 08 Apr 2002 23:37:31 +0200 akim
@@ -417,7 +417,7 @@
 
       /* Do not use any default reduction if there is a shift for
         error */
-      if (symbol == error_token_number)
+      if (symbol == errtoken->number)
        nodefault = 1;
     }
 
@@ -614,7 +614,7 @@
       if (number == SALIAS)
        continue;
       /* Skip error token.  */
-      if (symbol->number == error_token_number)
+      if (symbol == errtoken)
        continue;
       if (symbol->tag[0] == '\'')
        continue;               /* skip literal character */
@@ -1078,7 +1078,8 @@
   MUSCLE_INSERT_INT ("nsym", nsyms);
   MUSCLE_INSERT_INT ("debug", debug_flag);
   MUSCLE_INSERT_INT ("final", final_state);
-  MUSCLE_INSERT_INT ("maxtok", max_user_token_number);
+  MUSCLE_INSERT_INT ("undef_token_number", undeftoken->number);
+  MUSCLE_INSERT_INT ("user_token_number_max", max_user_token_number);
   MUSCLE_INSERT_INT ("error_verbose", error_verbose);
   MUSCLE_INSERT_STRING ("prefix", spec_name_prefix ? spec_name_prefix : "yy");
 
Index: src/print.c
--- src/print.c Sun, 07 Apr 2002 21:22:52 +0200 akim
+++ src/print.c Mon, 08 Apr 2002 19:42:12 +0200 akim
@@ -386,7 +386,7 @@
   /* TERMINAL (type #) : rule #s terminal is on RHS */
   fprintf (out, "%s\n\n", _("Terminals, with rules where they appear"));
   for (i = 0; i < max_user_token_number + 1; i++)
-    if (token_translations[i] != 2)
+    if (token_translations[i] != undeftoken->number)
       {
        buffer[0] = 0;
        column = strlen (escape (symbols[token_translations[i]]->tag));
Index: src/reader.c
--- src/reader.c Sun, 07 Apr 2002 21:22:52 +0200 akim
+++ src/reader.c Tue, 09 Apr 2002 19:29:17 +0200 akim
@@ -237,7 +237,7 @@
       && this->user_token_number != SALIAS)
     {
       /* A token which translation has already been set? */
-      if (token_translations[this->user_token_number] != 2)
+      if (token_translations[this->user_token_number] != undeftoken->number)
        complain (_("tokens %s and %s both assigned number %d"),
                  symbols[token_translations[this->user_token_number]]->tag,
                  this->tag, this->user_token_number);
@@ -1697,7 +1697,7 @@
      token number for $undefined., which represents all invalid
      inputs.  */
   for (i = 0; i < max_user_token_number + 1; i++)
-    token_translations[i] = 2;
+    token_translations[i] = undeftoken->number;
 
   symbols_do (symbol_translation, NULL);
 }
@@ -1718,8 +1718,6 @@
 
   token_translations_init ();
 
-  error_token_number = errtoken->number;
-
   if (startval->class == unknown_sym)
     fatal (_("the start symbol %s is undefined"), startval->tag);
   else if (startval->class == token_sym)
@@ -1831,7 +1829,6 @@
   undeftoken = getsym ("$undefined.");
   undeftoken->class = token_sym;
   undeftoken->number = ntokens++;
-  undeftoken->user_token_number = 2;
 
   /* Initialize the obstacks. */
   obstack_init (&action_obstack);
Index: src/state.h
--- src/state.h Sun, 07 Apr 2002 21:22:52 +0200 akim
+++ src/state.h Mon, 08 Apr 2002 23:37:29 +0200 akim
@@ -122,7 +122,7 @@
 /* Is the SHIFTS->shifts[Shift] then handling of the error token?. */
 
 #define SHIFT_IS_ERROR(Shifts, Shift) \
-  (SHIFT_SYMBOL (Shifts, Shift) == error_token_number)
+  (SHIFT_SYMBOL (Shifts, Shift) == errtoken->number)
 
 /* When resolving a SR conflicts, if the reduction wins, the shift is
    disabled.  */
Index: src/symtab.h
--- src/symtab.h Sun, 07 Apr 2002 22:07:25 +0200 akim
+++ src/symtab.h Mon, 08 Apr 2002 19:54:42 +0200 akim
@@ -41,6 +41,8 @@
   nterm_sym            /* non-terminal */
 } symbol_class;
 
+/* Internal token numbers. */
+typedef short token_number_t;
 #define SUNDEF  -1              /* For undefined user number. */
 #define SALIAS -9991           /* for symbol generated with an alias */
 
@@ -50,7 +52,7 @@
   char *tag;
   /* Its type. */
   char *type_name;
-  short number;
+  token_number_t number;
   short prec;
   associativity assoc;
   int user_token_number;
Index: data/bison.simple
--- data/bison.simple Mon, 08 Apr 2002 18:44:40 +0200 akim
+++ data/bison.simple Mon, 08 Apr 2002 23:16:50 +0200 akim
@@ -1,4 +1,4 @@
-m4_divert(-1)
+m4_divert(-1)                                                       -*- C -*-
 
 # b4_sint_type(MAX)
 # -----------------
@@ -265,11 +265,14 @@ m4_define([b4_token_defines],
 #define YYNRULES  b4_nrules
 /* YYNRULES -- Number of states. */
 #define YYNSTATES  b4_nstates
-#define YYMAXUTOK  b4_maxtok
 
 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX.  */
+#define YYUNDEFTOK  b4_undef_token_number
+#define YYMAXUTOK   b4_user_token_number_max
+
 typedef b4_uint_type(b4_token_number_max) yy_token_number_type;
-#define YYTRANSLATE(x) ((unsigned)(x) <= b4_maxtok ? yytranslate[[x]] : 
b4_nsym)
+#define YYTRANSLATE(X) \
+  ((unsigned)(X) <= YYMAXUTOK ? yytranslate[[X]] : YYUNDEFTOK)
 
 /* YYTRANSLATE[[YYLEX]] -- Bison symbol number corresponding to YYLEX.  */
 static const yy_token_number_type yytranslate[[]] =
Index: data/bison.c++
--- data/bison.c++ Mon, 08 Apr 2002 18:44:40 +0200 akim
+++ data/bison.c++ Tue, 09 Apr 2002 19:18:03 +0200 akim
@@ -257,7 +257,8 @@ m4_define([b4_copyright],
     static const int errcode_;
     static const int ntokens_;
     static const int initdepth_;
-    static const unsigned maxtok_;
+    static const unsigned user_token_number_max_;
+    static const TokenNumberType undef_token_;
 
     /* State.  */
     int n_;
@@ -722,7 +723,10 @@ yy::b4_name::translate_ (int token)
   {
     b4_translate
   };
-  return (unsigned)(token) <= maxtok_ ? translate_[[token]] : nsym_;
+  if ((unsigned) token <= user_token_number_max_)
+    return translate_[[token]];
+  else
+    return undef_token_;
 }
 
 const int yy::b4_name::eof_ = 0;
@@ -737,7 +741,8 @@ yy::b4_name::translate_ (int token)
 const int yy::b4_name::ntokens_ = b4_ntokens;
 const int yy::b4_name::initdepth_ = b4_initdepth;
 
-const unsigned yy::b4_name::maxtok_ = b4_maxtok;
+const unsigned yy::b4_name::user_token_number_max_ = b4_user_token_number_max;
+const yy::b4_name::TokenNumberType yy::b4_name::undef_token_ = 
b4_undef_token_number;
 
 b4_epilogue
 
Index: tests/conflicts.at
--- tests/conflicts.at Wed, 03 Apr 2002 22:07:02 +0200 akim
+++ tests/conflicts.at Tue, 09 Apr 2002 19:32:51 +0200 akim
@@ -151,8 +151,8 @@ exp: exp OP exp | NUM;
 
 $ (0) 0
 error (256)
-NUM (257) 2
-OP (258) 1
+NUM (258) 2
+OP (259) 1
 
 
 Nonterminals, with rules where they appear
@@ -257,8 +257,8 @@ exp: exp OP exp | NUM;
 
 $ (0) 0
 error (256)
-NUM (257) 2
-OP (258) 1
+NUM (258) 2
+OP (259) 1
 
 
 Nonterminals, with rules where they appear
Index: tests/regression.at
--- tests/regression.at Sun, 07 Apr 2002 21:28:00 +0200 akim
+++ tests/regression.at Tue, 09 Apr 2002 19:34:26 +0200 akim
@@ -336,8 +336,8 @@ input.y:2: fatal error: no input grammar
 ';' (59) 5
 '=' (61) 5
 error (256)
-undef_id_tok (257) 5
-const_id_tok (258) 5
+undef_id_tok (258) 5
+const_id_tok (259) 5
 Nonterminals, with rules where they appear
 $axiom (7)
     on left: 0
@@ -460,8 +460,8 @@ else: "else" statement;
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-       2,     2,     2,     2,     2,     2,     1,     3,     4,     5,
-       6
+       2,     2,     2,     2,     2,     2,     1,     2,     3,     4,
+       5,     6
 };
 static const short yyprhs[] =
 {
@@ -483,7 +483,7 @@ else: "else" statement;
 };
 static const short yytoknum[] =
 {
-       0,   256,     2,   257,   258,   259,   260,    -1
+       0,   256,   257,   258,   259,   260,   261,    -1
 };
 static const short yyr1[] =
 {