26-fyi-many-tokens.patch

[Akim Demaille]

Index: ChangeLog
from  Akim Demaille  <address@hidden>

        * src/muscle_tab.h (MUSCLE_INSERT_LONG_INT): New.
        * src/output.c (output_table_data): Return the longest number.
        (prepare_tokens): Output `token_number_max').
        * data/bison.simple, data/bison.c++ (b4_sint_type, b4_uint_type):
        New.
        Use them to define yy_token_number_type/TokenNumberType.
        Use this type for yytranslate.
        * tests/torture.at (Big triangle): Push the limit from 124 to
        253.
        * tests/regression.at (Web2c Actions): Adjust.

Index: TODO
--- TODO Sun, 07 Apr 2002 11:45:10 +0200 akim
+++ TODO Sun, 07 Apr 2002 12:11:14 +0200 akim
@@ -16,9 +16,8 @@

 * Huge Grammars
 Currently, not only is Bison unable to handle huge grammars because of
-internal limitations, but the test `big triangle' also demonstrates
-that it can produce SEGVing executables!  Push the limit beyond 124,
-and have a core dump.  Be my guest: fix this!
+internal limitations (see test `big triangle').  Push the limit beyond
+253.  Be my guest: fix this!

 * read_pipe.c
 This is not portable to DOS for instance.  Implement a more portable
Index: src/gram.c
--- src/gram.c Sat, 06 Apr 2002 22:33:26 +0200 akim
+++ src/gram.c Sun, 07 Apr 2002 13:03:43 +0200 akim
@@ -38,7 +38,7 @@
 int ntokens = 1;
 int nvars = 0;

-short *token_translations = NULL;
+token_number_t *token_translations = NULL;

 int start_symbol = 0;

Index: src/gram.h
--- src/gram.h Sat, 06 Apr 2002 22:33:26 +0200 akim
+++ src/gram.h Sun, 07 Apr 2002 17:20:34 +0200 akim
@@ -148,11 +148,11 @@
 /* Table of the symbols, indexed by the symbol number. */
 extern symbol_t **symbols;

-/* token translation table: indexed by a token number as returned by
-   the user's yylex routine, it yields the internal token number used
-   by the parser and throughout bison.  */
-
-extern short *token_translations;
+/* TOKEN_TRANSLATION -- a table indexed by a token number as returned
+   by the user's yylex routine, it yields the internal token number
+   used by the parser and throughout bison.  */
+typedef short token_number_t;
+extern token_number_t *token_translations;
 extern int max_user_token_number;

 /* SEMANTIC_PARSER is nonzero if the input file says to use the hairy
Index: src/muscle_tab.h
--- src/muscle_tab.h Mon, 04 Feb 2002 23:18:59 +0100 akim
+++ src/muscle_tab.h Sun, 07 Apr 2002 12:08:10 +0200 akim
@@ -40,6 +40,13 @@
   muscle_insert (Key, obstack_finish (&muscle_obstack));       \
 }

+#define MUSCLE_INSERT_LONG_INT(Key, Value)                     \
+{                                                              \
+  obstack_fgrow1 (&muscle_obstack, "%ld", Value);              \
+  obstack_1grow (&muscle_obstack, 0);                          \
+  muscle_insert (Key, obstack_finish (&muscle_obstack));       \
+}
+
 #define MUSCLE_INSERT_STRING(Key, Value)                       \
 {                                                              \
   obstack_sgrow (&muscle_obstack, Value);                      \
Index: src/output.c
--- src/output.c Sat, 06 Apr 2002 22:14:47 +0200 akim
+++ src/output.c Sun, 07 Apr 2002 12:53:31 +0200 akim
@@ -147,15 +147,19 @@
 }


-/* FIXME. */
+/*----------------------------------------------------------------.
+| Format the FIRST and then TABLE_DATA[BEGIN..END[ into OOUT, and |
+| return the number of bits needed for its longuest value.        |
+`----------------------------------------------------------------*/

-static inline void
+static inline long int
 output_table_data (struct obstack *oout,
                   short *table_data,
                   short first,
                   int begin,
                   int end)
 {
+  long int max = first;
   int i;
   int j = 1;

@@ -171,8 +175,12 @@
       else
        ++j;
       obstack_fgrow1 (oout, "%6d", table_data[i]);
+      if (table_data[i] > max)
+       max = table_data[i];
     }
   obstack_1grow (oout, 0);
+
+  return max;
 }


@@ -184,9 +192,10 @@
 static void
 prepare_tokens (void)
 {
-  output_table_data (&format_obstack, token_translations,
-                    0, 1, max_user_token_number + 1);
+  long int max = output_table_data (&format_obstack, token_translations,
+                                   0, 1, max_user_token_number + 1);
   muscle_insert ("translate", obstack_finish (&format_obstack));
+  MUSCLE_INSERT_LONG_INT ("token_number_max", max);
   XFREE (token_translations);

   {
Index: src/reader.c
--- src/reader.c Sat, 06 Apr 2002 22:33:26 +0200 akim
+++ src/reader.c Sun, 07 Apr 2002 13:05:59 +0200 akim
@@ -1691,7 +1691,7 @@
        max_user_token_number = this->user_token_number;
     }

-  token_translations = XCALLOC (short, max_user_token_number + 1);
+  token_translations = XCALLOC (token_number_t, max_user_token_number + 1);

   /* Initialize all entries for literal tokens to 2, the internal
      token number for $undefined., which represents all invalid
Index: data/bison.simple
--- data/bison.simple Sat, 23 Mar 2002 15:31:26 +0100 akim
+++ data/bison.simple Sun, 07 Apr 2002 12:54:51 +0200 akim
@@ -1,5 +1,25 @@
 m4_divert(-1)

+# b4_sint_type(MAX)
+# -----------------
+# Return the smallest signed int type able to handle the number MAX.
+m4_define([b4_sint_type],
+[m4_if(m4_eval([$1 <= 127]),        [1], [signed char],
+       m4_eval([$1 <= 32767]),      [1], [signed short],
+       m4_eval([$1 <= 2147483647]), [1], [signed int],
+       [m4_fatal([no signed int type for $1])])])
+
+
+# b4_uint_type(MAX)
+# -----------------
+# Return the smallest unsigned int type able to handle the number MAX.
+m4_define([b4_uint_type],
+[m4_if(m4_eval([$1 <= 255]),        [1], [unsigned char],
+       m4_eval([$1 <= 65535]),      [1], [unsigned short],
+       m4_eval([$1 <= 4294967295]), [1], [unsigned int],
+       [m4_fatal([no unsigned int type for $1])])])
+
+
 m4_define_default([b4_input_suffix], [.y])

 m4_define_default([b4_output_parser_suffix],
@@ -231,10 +251,11 @@ m4_define([b4_token_defines],
 #define YYMAXUTOK  b4_maxtok

 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX.  */
+typedef b4_uint_type(b4_token_number_max) yy_token_number_type;
 #define YYTRANSLATE(x) ((unsigned)(x) <= b4_maxtok ? yytranslate[[x]] : 
b4_nsym)

 /* YYTRANSLATE[[YYLEX]] -- Bison symbol number corresponding to YYLEX.  */
-static const char yytranslate[[]] =
+static const yy_token_number_type yytranslate[[]] =
 {
   b4_translate
 };
Index: data/bison.c++
--- data/bison.c++ Sat, 16 Mar 2002 13:28:35 +0100 akim
+++ data/bison.c++ Sun, 07 Apr 2002 12:54:44 +0200 akim
@@ -1,5 +1,25 @@
 m4_divert(-1)

+# b4_sint_type(MAX)
+# -----------------
+# Return the smallest signed int type able to handle the number MAX.
+m4_define([b4_sint_type],
+[m4_if(m4_eval([$1 <= 127]),        [1], [signed char],
+       m4_eval([$1 <= 32767]),      [1], [signed short],
+       m4_eval([$1 <= 2147483647]), [1], [signed int],
+       [m4_fatal([no signed int type for $1])])])
+
+
+# b4_uint_type(MAX)
+# -----------------
+# Return the smallest unsigned int type able to handle the number MAX.
+m4_define([b4_uint_type],
+[m4_if(m4_eval([$1 <= 255]),        [1], [unsigned char],
+       m4_eval([$1 <= 65535]),      [1], [unsigned short],
+       m4_eval([$1 <= 4294967295]), [1], [unsigned int],
+       [m4_fatal([no unsigned int type for $1])])])
+
+
 # b4_token_defines(TOKEN-NAME, TOKEN-NUMBER)
 # ------------------------------------------
 # Output the definition of this token as #define.
@@ -147,6 +167,7 @@ m4_define([b4_copyright],
   template < >
   struct Traits< b4_name >
   {
+    typedef typedef b4_uint_type(b4_token_number_max) TokenNumberType;
     typedef int      StateType;
     typedef yystype  SemanticType;
     typedef b4_ltype LocationType;
@@ -159,9 +180,10 @@ m4_define([b4_copyright],
   {
   public:

-    typedef Traits< b4_name >::StateType    StateType;
-    typedef Traits< b4_name >::SemanticType SemanticType;
-    typedef Traits< b4_name >::LocationType LocationType;
+    typedef Traits< b4_name >::TokenNumberType TokenNumberType;
+    typedef Traits< b4_name >::StateType       StateType;
+    typedef Traits< b4_name >::SemanticType     SemanticType;
+    typedef Traits< b4_name >::LocationType    LocationType;

     typedef Stack< StateType >    StateStack;
     typedef Stack< SemanticType > SemanticStack;
@@ -218,7 +240,7 @@ m4_define([b4_copyright],
 #endif

     /* Even more tables.  */
-    static inline char translate_ (int token);
+    static inline TokenNumberType translate_ (int token);

     /* Constants.  */
     static const int eof_;
@@ -685,16 +707,16 @@ yy::b4_name::rline_[[]] =
 #endif

 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX.  */
-char
+TokenNumberType
 yy::b4_name::translate_ (int token)
 {
   static
-  const char
+  const TokenNumberType
   translate_[[]] =
   {
     b4_translate
   };
-  return ((unsigned)(token) <= maxtok_ ? translate_[[token]] : nsym_);
+  return (unsigned)(token) <= maxtok_ ? translate_[[token]] : nsym_;
 }

 const int yy::b4_name::eof_ = 0;
Index: tests/regression.at
--- tests/regression.at Wed, 03 Apr 2002 22:07:02 +0200 akim
+++ tests/regression.at Sun, 07 Apr 2002 12:56:57 +0200 akim
@@ -433,7 +433,7 @@ else: "else" statement;
 # Check only the tables.  We don't use --no-parser, because it is
 # still to be implemented in the experimental branch of Bison.
 AT_CHECK([[sed -n 's/  *$//;/^static const.*\[\] =/,/^}/p' input.c]], 0,
-[[static const char yytranslate[] =
+[[static const yy_token_number_type yytranslate[] =
 {
        0,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
Index: tests/torture.at
--- tests/torture.at Sun, 07 Apr 2002 11:45:10 +0200 akim
+++ tests/torture.at Sun, 07 Apr 2002 12:11:31 +0200 akim
@@ -127,11 +127,9 @@ exp:
 ## Big triangle.  ##
 ## -------------- ##

-# Arg, the upper limit, currently, is 124.  Afterwards, the
-# executable dumps core...
 AT_SETUP([Big triangle])

-AT_DATA_TRIANGULAR_GRAMMAR([input.y], [124])
+AT_DATA_TRIANGULAR_GRAMMAR([input.y], [253])
 AT_CHECK([bison input.y -v -o input.c])
 AT_CHECK([$CC $CFLAGS $CPPFLAGS input.c -o input], 0, [], [ignore])
 AT_CHECK([./input])