05-free-user-token-numbers.patch

[Akim Demaille]

Index: ChangeLog
from  Akim Demaille  <address@hidden>
        
        * src/reader.c (token_translations_init): 256 is now the default
        value for the error token, i.e., it will be assigned another
        number if the user assigned 256 to one of her tokens.
        (reader): Don't force 256 to error.
        * doc/bison.texinfo (Symbols): Adjust.
        * tests/torture.at (AT_DATA_HORIZONTAL_GRAMMAR)
        (AT_DATA_TRIANGULAR_GRAMMAR): Number the tokens as 1, 2, 3
        etc. instead of 10, 20, 30 (which was used to `jump' over error
        (256) and undefined (2)).
        
Index: NEWS
--- NEWS Tue, 09 Apr 2002 19:37:32 +0200 akim
+++ NEWS Thu, 18 Apr 2002 20:31:58 +0200 akim
@@ -11,6 +11,12 @@
   If yylex returned a code out of range, yyparse could die.  This is
   no longer the case.
 
+* Error token
+  According to POSIX, the error token should be numbered as 256.
+  Bison extends this requirement by making it a preference: *if* the
+  user specified that one of her tokens is numbered 256, then error
+  will be mapped onto another number.
+
 * Large grammars
   Are now supported (large token numbers, large grammar size (= sum of
   the LHS and RHS lengths).
@@ -19,11 +25,11 @@
   Bison used to play hacks with the initial rule, which the user does
   not write.  It is now explicit, and visible in the reports and
   graphs as rule 0.
-      
+
 * Useless rules are actually removed.
   Before, Bison reported the useless rules, but, although not used,
   included them in the parsers.
-        
+
 * False `Token not used' report fixed.
   On a grammar such as
 
Index: doc/bison.texinfo
--- doc/bison.texinfo Sun, 07 Apr 2002 20:00:35 +0200 akim
+++ doc/bison.texinfo Thu, 18 Apr 2002 20:46:13 +0200 akim
@@ -2218,11 +2218,9 @@ @node Symbols
 
 The symbol @code{error} is a terminal symbol reserved for error recovery
 (@pxref{Error Recovery}); you shouldn't use it for any other purpose.
-In particular, @code{yylex} should never return this value.
-The default value of the error token is 256, so in the
-unlikely event that you need to use a character token with numeric
-value 256 you must reassign the error token's value with a
address@hidden declaration.
+In particular, @code{yylex} should never return this value.  The default
+value of the error token is 256, unless you explicitly assigned 256 to
+one of your tokens with a @code{%token} declaration.
 
 @node Rules
 @section Syntax of Grammar Rules
Index: src/reader.c
--- src/reader.c Wed, 10 Apr 2002 23:20:06 +0200 akim
+++ src/reader.c Thu, 18 Apr 2002 20:50:25 +0200 akim
@@ -1678,15 +1678,37 @@
 static void
 token_translations_init (void)
 {
-  int last_user_token_number = 256;
+  int num_256_available_p = TRUE;
   int i;
 
-  /* Set the user numbers. */
+  /* Find the highest user token number, and whether 256, the POSIX
+     preferred user token number for the error token, is used.  */
+  max_user_token_number = 0;
+  for (i = 0; i < ntokens; ++i)
+    {
+      symbol_t *this = symbols[i];
+      if (this->user_token_number != SUNDEF)
+       {
+         if (this->user_token_number > max_user_token_number)
+           max_user_token_number = this->user_token_number;
+         if (this->user_token_number == 256)
+           num_256_available_p = FALSE;
+       }
+    }
+
+  /* If 256 is not used, assign it to error, to follow POSIX.  */
+  if (num_256_available_p && errtoken->user_token_number == SUNDEF)
+    errtoken->user_token_number = 256;
+
+  /* Set the missing user numbers. */
+  if (max_user_token_number < 256)
+    max_user_token_number = 256;
+
   for (i = 0; i < ntokens; ++i)
     {
       symbol_t *this = symbols[i];
       if (this->user_token_number == SUNDEF)
-       this->user_token_number = ++last_user_token_number;
+       this->user_token_number = ++max_user_token_number;
       if (this->user_token_number > max_user_token_number)
        max_user_token_number = this->user_token_number;
     }
@@ -1698,7 +1720,6 @@
      inputs.  */
   for (i = 0; i < max_user_token_number + 1; i++)
     token_translations[i] = undeftoken->number;
-
   symbols_do (symbol_translation, NULL);
 }
 
@@ -1824,7 +1845,6 @@
   errtoken = getsym ("error");
   errtoken->class = token_sym;
   errtoken->number = ntokens++;
-  errtoken->user_token_number = 256;   /* Value specified by POSIX.  */
 
   /* Construct a token that represents all undefined literal tokens.
      It is always token number 2.  */
Index: tests/torture.at
--- tests/torture.at Sun, 07 Apr 2002 22:07:25 +0200 akim
+++ tests/torture.at Thu, 18 Apr 2002 20:48:22 +0200 akim
@@ -60,7 +60,7 @@ m4_define([AT_DATA_TRIANGULAR_GRAMMAR],
 
 for my $size (1 .. $max)
   {
-    print "%token \"$size\" ", $size * 10, "\n";
+    print "%token \"$size\" ", $size, "\n";
   };
 
 print <<EOF;
@@ -100,7 +100,7 @@ exp:
       ++outer;
       return END;
     }
-  return inner++ * 10;
+  return inner++;
 }
 
 static void
@@ -170,7 +170,7 @@ m4_define([AT_DATA_HORIZONTAL_GRAMMAR],
 
 for my $size (1 .. $max)
   {
-    print "%token \"$size\" ", $size * 10, "\n";
+    print "%token \"$size\" ", $size, "\n";
   };
 
 print <<EOF;
@@ -192,7 +192,7 @@ m4_define([AT_DATA_HORIZONTAL_GRAMMAR],
   if (counter > $max)
     return 0;
   else
-  return counter++ * 10;
+  return counter++;
 }
 
 static void