Re: FYI: Missing \n

[Akim Demaille]

Here is a protopatch that definitely needs a means to "use" tokens
that are not used by the grammar itself.  If someone wants to take the
ball, she is most welcome.

I left some failures in the test suite because we might want to
address them by declaring the unused tokens are used externally.

I fell on a confusing error message from Bison because of

%token ID_COLON        "identifier:"

which looks like a good idea, but results in:

~/src/bison % cat >/tmp/foo.y
foo: bar baz;
~/src/bison % LC_ALL=C _build/tests/bison /tmp/foo.y            nostromo Err 1
/tmp/foo.y:1.1-3: syntax error, unexpected identifier:

this really looks like an incomplete error message :)  Maybe for this
one we should define

%token ID_COLON        "\"identifier:\""

Index: ChangeLog
from  Akim Demaille  <address@hidden>
        * src/reduce.c (reduce_grammar): Warn about unused terminals.
        Use a more traditional message for unused terminals in the
        report.
        * tests/reduce.at, tests/glr-regression.at, tests/input.at,
        * tests/existing.at, tests/regression.at:
        Adjust expected error messages.

+2007-01-17  Akim Demaille  <address@hidden>
+
        * doc/bison.texinfo (Calc++ Parser): Don't try to alias
        nonterminals.
        Use per-type %printer.
Index: src/reduce.c
===================================================================
RCS file: /cvsroot/bison/bison/src/reduce.c,v
retrieving revision 1.89
diff -u -u -r1.89 reduce.c
--- src/reduce.c 3 Jan 2006 22:25:58 -0000 1.89
+++ src/reduce.c 17 Jan 2007 09:51:43 -0000
@@ -362,7 +362,7 @@
       if (!bitset_test (V, i) && !bitset_test (V1, i))
        {
          if (!b)
-           fprintf (out, "%s\n\n", _("Terminals which are not used"));
+           fprintf (out, "%s\n\n", _("Unused terminals"));
          b = true;
          fprintf (out, "   %s\n", symbols[i]->tag);
        }
@@ -426,6 +426,16 @@
   useless_nonterminals ();
   inaccessable_symbols ();
 
+  /* We don't remove useless terminals, the scanner might want to use
+     them anyway.  But we should warn about them.  */
+  {
+    int i;
+    for (i = 0; i < ntokens; i++)
+      if (!bitset_test (V, i) && !bitset_test (V1, i))
+       warn_at (symbols[i]->location,
+                _("unused terminal: %s"), symbols[i]->tag);
+  }
+
   reduced = (nuseless_nonterminals + nuseless_productions > 0);
   if (!reduced)
     return;
Index: tests/existing.at
===================================================================
RCS file: /cvsroot/bison/bison/tests/existing.at,v
retrieving revision 1.10
diff -u -u -r1.10 existing.at
--- tests/existing.at 22 Jan 2006 08:02:47 -0000 1.10
+++ tests/existing.at 17 Jan 2007 09:51:43 -0000
@@ -1,6 +1,6 @@
 # Exercising Bison on actual grammars.                   -*- Autotest -*-
 
-# Copyright (C) 1989, 1990, 1991, 1992, 2000, 2001, 2002, 2003, 2004, 2005
+# Copyright (C) 1989, 1990, 1991, 1992, 2000, 2001, 2002, 2003, 2004, 2005, 
2007
 # Free Software Foundation, Inc.
 
 # This program is free software; you can redistribute it and/or modify
@@ -353,7 +353,9 @@
 # Pass plenty of options, to exercise plenty of code, even if we
 # don't actually check the output.  But SEGV is watching us, and
 # so might do dmalloc.
-AT_CHECK([[bison --verbose --defines input.y]])
+AT_CHECK([[bison --verbose --defines input.y]], 0, [],
+[[input.y:4.8-12: warning: unused terminal: ERROR
+]])
 
 AT_CLEANUP
 
@@ -956,7 +958,26 @@
 # don't actually check the output.  But SEGV is watching us, and
 # so might do dmalloc.
 AT_CHECK([[bison --verbose --defines input.y]], 0, [],
-[[input.y: conflicts: 78 shift/reduce, 10 reduce/reduce
+[[input.y:8.20-22: warning: unused terminal: HEQ
+input.y:10.9-11: warning: unused terminal: HGE
+input.y:10.23-25: warning: unused terminal: HGT
+input.y:12.22-24: warning: unused terminal: HIN
+input.y:12.51-53: warning: unused terminal: HIS
+input.y:13.16-18: warning: unused terminal: HLE
+input.y:13.26-28: warning: unused terminal: HLT
+input.y:14.15-17: warning: unused terminal: HNE
+input.y:14.39-45: warning: unused terminal: HNOTEXT
+input.y:25.9-20: warning: unused terminal: HASSIGNVALUE
+input.y:25.22-31: warning: unused terminal: HASSIGNREF
+input.y:28.9-12: warning: unused terminal: HEQR
+input.y:28.14-17: warning: unused terminal: HNER
+input.y:29.9-12: warning: unused terminal: HADD
+input.y:29.14-17: warning: unused terminal: HSUB
+input.y:29.19-22: warning: unused terminal: HMUL
+input.y:29.24-27: warning: unused terminal: HDIV
+input.y:29.29-35: warning: unused terminal: HINTDIV
+input.y:29.37-40: warning: unused terminal: HEXP
+input.y: conflicts: 78 shift/reduce, 10 reduce/reduce
 ]])
 
 AT_CHECK([[grep '^State.*conflicts:' input.output]], 0,
Index: tests/glr-regression.at
===================================================================
RCS file: /cvsroot/bison/bison/tests/glr-regression.at,v
retrieving revision 1.43
diff -u -u -r1.43 glr-regression.at
--- tests/glr-regression.at 17 Jan 2007 08:36:07 -0000 1.43
+++ tests/glr-regression.at 17 Jan 2007 09:51:43 -0000
@@ -328,7 +328,8 @@
 ]])
 
 AT_CHECK([[bison -o glr-regr3.c glr-regr3.y]], 0, [],
-[glr-regr3.y: conflicts: 1 shift/reduce, 1 reduce/reduce
+[glr-regr3.y:27.8-15: warning: unused terminal: BAD_CHAR
+glr-regr3.y: conflicts: 1 shift/reduce, 1 reduce/reduce
 ])
 AT_COMPILE([glr-regr3])
 
@@ -1628,7 +1629,8 @@
 ]])
 
 AT_CHECK([[bison -o glr-regr16.c glr-regr16.y]], 0, [],
-[glr-regr16.y: conflicts: 1 reduce/reduce
+[glr-regr16.y:11.38-40: warning: unused terminal: 'b'
+glr-regr16.y: conflicts: 1 reduce/reduce
 ])
 AT_COMPILE([glr-regr16])
 
Index: tests/input.at
===================================================================
RCS file: /cvsroot/bison/bison/tests/input.at,v
retrieving revision 1.75
diff -u -u -r1.75 input.at
--- tests/input.at 17 Jan 2007 08:36:07 -0000 1.75
+++ tests/input.at 17 Jan 2007 09:51:43 -0000
@@ -539,7 +539,9 @@
 }
 ]])
 
-AT_CHECK([bison -d -v -o input.c input.y])
+AT_CHECK([bison -d -v -o input.c input.y], 0, [],
+[[input.y:77.13-94: warning: unused terminal: "fake @<:@@:>@ 
\a\b\f\n\r\t\v\"'?\\@<:@\\ ?\?!?\?'?\?(?\?)?\?-?\?/?\?<?\?=?\?> \001\001"
+]])
 AT_COMPILE([input.o], [-c input.c])
 AT_COMPILE([main.o], [-c main.c])
 AT_COMPILE([input], [input.o main.o])
Index: tests/reduce.at
===================================================================
RCS file: /cvsroot/bison/bison/tests/reduce.at,v
retrieving revision 1.15
diff -u -u -r1.15 reduce.at
--- tests/reduce.at 2 Dec 2006 01:52:16 -0000 1.15
+++ tests/reduce.at 17 Jan 2007 09:51:43 -0000
@@ -1,5 +1,5 @@
 # Exercising Bison Grammar Reduction.                      -*- Autotest -*-
-# Copyright (C) 2001, 2002 Free Software Foundation, Inc.
+# Copyright (C) 2001, 2002, 2007 Free Software Foundation, Inc.
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,11 +19,11 @@
 AT_BANNER([[Grammar Reduction.]])
 
 
-## ------------------- ##
-## Useless Terminals.  ##
-## ------------------- ##
+## ------------------ ##
+## Unused Terminals.  ##
+## ------------------ ##
 
-AT_SETUP([Useless Terminals])
+AT_SETUP([Unused Terminals])
 
 AT_DATA([[input.y]],
 [[%verbose
@@ -44,10 +44,20 @@
 exp: useful;
 ]])
 
-AT_CHECK([[bison input.y]])
+AT_CHECK([[bison input.y]], 0, [],
+[[input.y:4.8-15: warning: unused terminal: useless1
+input.y:5.8-15: warning: unused terminal: useless2
+input.y:6.8-15: warning: unused terminal: useless3
+input.y:7.8-15: warning: unused terminal: useless4
+input.y:8.8-15: warning: unused terminal: useless5
+input.y:9.8-15: warning: unused terminal: useless6
+input.y:10.8-15: warning: unused terminal: useless7
+input.y:11.8-15: warning: unused terminal: useless8
+input.y:12.8-15: warning: unused terminal: useless9
+]])
 
 AT_CHECK([[sed -n '/^Grammar/q;/^$/!p' input.output]], 0,
-[[Terminals which are not used
+[[Unused terminals
    useless1
    useless2
    useless3
@@ -144,7 +154,16 @@
 ]])
 
 AT_CHECK([[bison input.y]], 0, [],
-[[input.y: warning: 9 useless nonterminals and 9 useless rules
+[[input.y:6.11-13: warning: unused terminal: '1'
+input.y:7.11-13: warning: unused terminal: '2'
+input.y:8.11-13: warning: unused terminal: '3'
+input.y:9.11-13: warning: unused terminal: '4'
+input.y:10.11-13: warning: unused terminal: '5'
+input.y:11.11-13: warning: unused terminal: '6'
+input.y:12.11-13: warning: unused terminal: '7'
+input.y:13.11-13: warning: unused terminal: '8'
+input.y:14.11-13: warning: unused terminal: '9'
+input.y: warning: 9 useless nonterminals and 9 useless rules
 input.y:6.1-8: warning: useless nonterminal: useless1
 input.y:7.1-8: warning: useless nonterminal: useless2
 input.y:8.1-8: warning: useless nonterminal: useless3
@@ -176,7 +195,7 @@
    useless7
    useless8
    useless9
-Terminals which are not used
+Unused terminals
    '1'
    '2'
    '3'
@@ -239,7 +258,8 @@
 ]])
 
 AT_CHECK([[bison not-reduced.y]], 0, [],
-[[not-reduced.y: warning: 2 useless nonterminals and 3 useless rules
+[[not-reduced.y:2.8-20: warning: unused terminal: useless_token
+not-reduced.y: warning: 2 useless nonterminals and 3 useless rules
 not-reduced.y:14.1-13: warning: useless nonterminal: not_reachable
 not-reduced.y:11.6-19: warning: useless nonterminal: non_productive
 not-reduced.y:11.6-57: warning: useless rule: exp: non_productive
@@ -251,7 +271,7 @@
 [[Useless nonterminals
    not_reachable
    non_productive
-Terminals which are not used
+Unused terminals
    useless_token
 Useless rules
     2 exp: non_productive
@@ -284,7 +304,9 @@
 %%
 ]])
 
-AT_CHECK([[bison reduced.y]])
+AT_CHECK([[bison reduced.y]], 0, [],
+[[reduced.y:2.8-20: warning: unused terminal: useless_token
+]])
 
 # Comparing the parsers.
 cp reduced.c expout
Index: tests/regression.at
===================================================================
RCS file: /cvsroot/bison/bison/tests/regression.at,v
retrieving revision 1.112
diff -u -u -r1.112 regression.at
--- tests/regression.at 21 Oct 2006 04:52:43 -0000 1.112
+++ tests/regression.at 17 Jan 2007 09:51:43 -0000
@@ -1,6 +1,6 @@
 # Bison Regressions.                               -*- Autotest -*-
 
-# Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006 Free Software
+# Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007 Free Software
 # Foundation, Inc.
 
 # This program is free software; you can redistribute it and/or modify
@@ -193,7 +193,6 @@
 %token NUM
 %token LE "<="
 %token GE "<="
-
 %%
 exp: '(' exp ')' | NUM ;
 %%
@@ -201,8 +200,8 @@
 
 AT_CHECK([bison -v -o input.c input.y], 0, [],
 [[input.y:6.8-14: warning: symbol `"<="' used more than once as a literal 
string
+input.y:6.8-9: warning: unused terminal: GE
 ]])
-
 AT_CLEANUP
 
 
@@ -362,7 +361,7 @@
 %token  <operator>  LE 134  "<="
 %left  OR  "<="
 %%
-exp: ;
+exp: "||" "<=";
 %%
 ]])
 
@@ -447,7 +446,8 @@
 %token 'd' D_TOKEN
 %token SPECIAL "\\\'\?\"\a\b\f\n\r\t\v\001\201\x001\x000081??!"
 %%
-exp: "a" "\\\'\?\"\a\b\f\n\r\t\v\001\201\x001\x000081??!";
+exp: "a" "\\\'\?\"\a\b\f\n\r\t\v\001\201\x001\x000081??!"
+     "b" C_TOKEN 'c' 'd' D_TOKEN;
 %%
 void
 yyerror (char const *s)
@@ -458,7 +458,7 @@
 int
 yylex (void)
 {
-  static int called;
+  static int called = 0;
   if (called++)
     abort ();
   return SPECIAL;
@@ -1099,7 +1099,9 @@
 }
 ]])
 
-AT_CHECK([bison -t -o input.c input.y])
+AT_CHECK([bison -t -o input.c input.y], 0, [],
+[[input.y:26.51-53: warning: unused terminal: 'a'
+]])
 AT_COMPILE([input])
 AT_PARSER_CHECK([./input], 0,
 [[Bison would once convert this action to a midrule because of the subsequent 
braced code.