Re: FYI: alias undeclared or declared after use

[Joel E. Denny]

On Fri, 18 Aug 2006, Joel E. Denny wrote:

> I just realized why my change might be wrong.  %token-table reveals the 
> token number associated with any undeclared string literal.  Thus, my 
> change may create a backward incompatibility in real parsers.

I committed the following.

Joel

Index: ChangeLog
===================================================================
RCS file: /sources/bison/bison/ChangeLog,v
retrieving revision 1.1551
diff -p -u -r1.1551 ChangeLog
--- ChangeLog   18 Aug 2006 10:25:48 -0000      1.1551
+++ ChangeLog   18 Aug 2006 21:38:02 -0000
@@ -1,5 +1,27 @@
 2006-08-18  Joel E. Denny  <address@hidden>
 
+       Redo some of the previous commit: add back the ability to use
+       non-aliased/undeclared string literals since it might be useful to
+       those declaring %token-table.
+       * src/reader.c (check_and_convert_grammar): Undo changes in previous
+       commit: don't worry about complaints from symbols_pack.
+       * src/symtab.c (symbol_new, symbol_class_set,
+       symbol_check_alias_consistency): Undo changes in previous commit: count
+       each string literal as a new symbol and token, assign it a symbol
+       number, and don't complain about non-aliased string literals.
+       (symbols_pack): Since symbol_make_alias still does not decrement symbol
+       and token counts but does still set aliased tokens to the same number,
+       symbol_pack_processor now leaves empty slots in the symbols array.
+       Remove those slots.
+       * tests/regression.at (Undeclared string literal): Remove test case
+       added in previous commit since non-aliased string literals are allowed
+       again.
+       (Characters Escapes, Web2c Actions): Undo changes in previous commit:
+       remove unnecessary string literal declarations.
+       * tests/sets.at (Firsts): Likewise.
+
+2006-08-18  Joel E. Denny  <address@hidden>
+
        Don't allow an undeclared string literal, but allow a string literal to
        be used before its declaration.
        * src/reader.c (check_and_convert_grammar): Don't invoke packgram if
@@ -18,7 +40,7 @@
        symbol_pack asserts that every token has been assigned a symbol number
        although undeclared string literals have not.
        * tests/regression.at (String alias declared after use, Undeclared
-       string literal): New test case.
+       string literal): New test cases.
        (Characters Escapes, Web2c Actions): Declare string literals as
        aliases.
        * tests/sets.at (Firsts): Likewise.
Index: src/reader.c
===================================================================
RCS file: /sources/bison/bison/src/reader.c,v
retrieving revision 1.268
diff -p -u -r1.268 reader.c
--- src/reader.c        18 Aug 2006 10:25:49 -0000      1.268
+++ src/reader.c        18 Aug 2006 21:38:03 -0000
@@ -630,8 +630,7 @@ check_and_convert_grammar (void)
   symbols_pack ();
 
   /* Convert the grammar into the format described in gram.h.  */
-  if (!complaint_issued)
-    packgram ();
+  packgram ();
 
   /* The grammar as a symbol_list is no longer needed. */
   LIST_FREE (symbol_list, grammar);
Index: src/symtab.c
===================================================================
RCS file: /sources/bison/bison/src/symtab.c,v
retrieving revision 1.76
diff -p -u -r1.76 symtab.c
--- src/symtab.c        18 Aug 2006 10:25:50 -0000      1.76
+++ src/symtab.c        18 Aug 2006 21:38:03 -0000
@@ -79,8 +79,7 @@ symbol_new (uniqstr tag, location loc)
   if (nsyms == SYMBOL_NUMBER_MAXIMUM)
     fatal (_("too many symbols in input grammar (limit is %d)"),
           SYMBOL_NUMBER_MAXIMUM);
-  if (tag[0] != '"')
-    nsyms++;
+  nsyms++;
   return res;
 }
 
@@ -267,8 +266,7 @@ symbol_class_set (symbol *sym, symbol_cl
 
   if (class == nterm_sym && sym->class != nterm_sym)
     sym->number = nvars++;
-  else if (class == token_sym && sym->number == NUMBER_UNDEFINED
-           && sym->tag[0] != '"')
+  else if (class == token_sym && sym->number == NUMBER_UNDEFINED)
     sym->number = ntokens++;
 
   sym->class = class;
@@ -380,9 +378,6 @@ symbol_check_alias_consistency (symbol *
   symbol *alias = this;
   symbol *orig  = this->alias;
 
-  if (this->tag[0] == '"' && !this->alias)
-    complain_at (this->location, _("%s undeclared"), this->tag);
-
   /* Check only those that _are_ the aliases.  */
   if (!(this->alias && this->user_token_number == USER_NUMBER_ALIAS))
     return;
@@ -720,13 +715,35 @@ symbols_token_translations_init (void)
 void
 symbols_pack (void)
 {
-  symbols = xcalloc (nsyms, sizeof *symbols);
-
   symbols_do (symbol_check_alias_consistency_processor, NULL);
-  if (complaint_issued)
-    return;
+
+  symbols = xcalloc (nsyms, sizeof *symbols);
   symbols_do (symbol_pack_processor, NULL);
 
+  /* Aliases leave empty slots in symbols, so remove them.  */
+  {
+    int writei;
+    int readi;
+    int nsyms_old = nsyms;
+    for (writei = 0, readi = 0; readi < nsyms_old; readi += 1)
+      {
+        if (symbols[readi] == NULL)
+          {
+            nsyms -= 1;
+            ntokens -= 1;
+          }
+        else
+          {
+            symbols[writei] = symbols[readi];
+            symbols[writei]->number = writei;
+            if (symbols[writei]->alias)
+              symbols[writei]->alias->number = writei;
+            writei += 1;
+          }
+      }
+  }
+  symbols = xnrealloc (symbols, nsyms, sizeof *symbols);
+
   symbols_token_translations_init ();
 
   if (startsymbol->class == unknown_sym)
Index: tests/regression.at
===================================================================
RCS file: /sources/bison/bison/tests/regression.at,v
retrieving revision 1.106
diff -p -u -r1.106 regression.at
--- tests/regression.at 18 Aug 2006 10:25:50 -0000      1.106
+++ tests/regression.at 18 Aug 2006 21:38:03 -0000
@@ -489,9 +489,7 @@ AT_DATA_GRAMMAR([input.y],
 void yyerror (const char *s);
 int yylex (void);
 %}
-[%token QUOTES "\""
-%token TICK "'"
-%%
+[%%
 exp:
   '\'' "\'"
 | '\"' "\""
@@ -702,10 +700,6 @@ statement:  struct_stat;
 struct_stat:  /* empty. */ | if else;
 if: "if" "const" "then" statement;
 else: "else" statement;
-%token IF "if";
-%token CONST "const";
-%token THEN "then";
-%token ELSE "else";
 %%
 ]])
 
@@ -1126,7 +1120,7 @@ AT_SETUP([String alias declared after us
 # Bison once incorrectly asserted that the symbol number for either a token or
 # its alias was the highest symbol number so far at the point of the alias
 # declaration.  That was true unless the declaration appeared after their first
-# uses.
+# uses and other tokens appeared in between.
 
 AT_DATA([input.y],
 [[%%
@@ -1137,25 +1131,3 @@ start: 'a' "A" 'b';
 AT_CHECK([bison -t -o input.c input.y])
 
 AT_CLEANUP
-
-
-
-## --------------------------- ##
-## Undeclared string literal.  ##
-## --------------------------- ##
-
-AT_SETUP([Undeclared string literal])
-
-# Bison once allowed a string literal to be used in the grammar without any
-# declaration assigning it as an alias of another token.
-
-AT_DATA([input.y],
-[[%%
-start: "abc";
-]])
-
-AT_CHECK([bison -t -o input.c input.y], [1], [],
-[[input.y:2.8-12: "abc" undeclared
-]])
-
-AT_CLEANUP
Index: tests/sets.at
===================================================================
RCS file: /sources/bison/bison/tests/sets.at,v
retrieving revision 1.22
diff -p -u -r1.22 sets.at
--- tests/sets.at       18 Aug 2006 10:25:50 -0000      1.22
+++ tests/sets.at       18 Aug 2006 21:38:03 -0000
@@ -196,7 +196,6 @@ AT_DATA([input.y],
 [[%nonassoc '<' '>'
 %left '+' '-'
 %right '^' '='
-%token EXP "exp"
 %%
 exp:
    exp '<' exp