Re: FYI: lalr1.cc: Move tokens and YYSTYPE into the class

[Paul Eggert]

The patch in your email didn't correspond to what was installed --
perhaps you picked up the wrong file when composing the email?  To
save you some time here is what I have for the patch that was actually
installed.

Also, the ChangeLog entry missed examples/extexi, and the nearby
comment in examples/extexi seems to be dangling now?  I can't quite
figure that out.  Could you please look into that?  Thanks.

Index: ChangeLog
===================================================================
RCS file: /cvsroot/bison/bison/ChangeLog,v
retrieving revision 1.1318
retrieving revision 1.1319
diff -p -u -r1.1318 -r1.1319
--- ChangeLog   29 Sep 2005 19:02:52 -0000      1.1318
+++ ChangeLog   30 Sep 2005 17:57:05 -0000      1.1319
@@ -1,3 +1,21 @@
+2005-09-30  Akim Demaille  <address@hidden>,
+           Alexandre Duret-Lutz <address@hidden>
+
+       Move the token type and YYSTYPE in the parser class.
+       * data/lalr1.cc (stack.hh, location.hh): Include earlier.
+       (parser::token): New, from the moved free definition of tokens.
+       (parser::semantic_value): Now a full definition instead of an
+       indirection to YYSTYPE.
+       (b4_post_prologue): No longer included in the header file, but
+       in the implementation file.
+       * doc/bison.texi (C+ Language Interface): Update.
+       * src/parse-gram.y: Support unary %define.
+       * tests/actions.at: Define global_tokens_and_yystype for backward
+       compatibility until we update the tests.
+       * tests/calc.at: Idem.
+       (first_line, first_column, last_line, last_column): Define for lalr1.cc
+       to simplify the code.
+
 2005-09-29  Paul Eggert  <address@hidden>
 
        Port to SunOS 4.1.4, which lacks strtoul and strerror.
@@ -6,7 +24,7 @@
        * lib/.cvsignore: Add strerror.c, strtol.c, strtoul.c
        * m4/.cvsignore: Add strerror.m4, strtol.m4, strtoul.m4.
 
-2005-09-29  Akim  <address@hidden>
+2005-09-29  Akim Demaille  <address@hidden>
 
        * data/c.m4 (b4_error_verbose_if): New.
        * data/lalr1.cc: Use it.
Index: NEWS
===================================================================
RCS file: /cvsroot/bison/bison/NEWS,v
retrieving revision 1.124
retrieving revision 1.125
diff -p -u -r1.124 -r1.125
--- NEWS        19 Sep 2005 07:08:00 -0000      1.124
+++ NEWS        30 Sep 2005 17:57:05 -0000      1.125
@@ -3,6 +3,17 @@ Bison News
 
 Changes in version 2.1a:
 
+* lalr1.cc: The token and value types are now class members.
+  The tokens where defined as free form enums and cpp macros.  YYSTYPE
+  was defined as a free form union.  Both are now class members:
+  tokens are enumerations of the `yy::parser::token' struct, and the
+  semantic values have the `yy::parser::semantic_type' type.
+
+  If you do not want or can update to this scheme, the directive
+  `%define "global_tokens_and_yystype" "1"' triggers the global
+  definition of tokens and YYSTYPE.
+
+
 Changes in version 2.1, 2005-09-16:
 
 * Bison-generated parsers now support the translation of diagnostics like
Index: data/lalr1.cc
===================================================================
RCS file: /cvsroot/bison/bison/data/lalr1.cc,v
retrieving revision 1.102
retrieving revision 1.103
diff -p -u -r1.102 -r1.103
--- data/lalr1.cc       29 Sep 2005 06:50:57 -0000      1.102
+++ data/lalr1.cc       30 Sep 2005 17:57:05 -0000      1.103
@@ -27,35 +27,27 @@ m4_divert(0)dnl
 m4_if(b4_defines_flag, 0, [],
 address@hidden @output_header_name@
 b4_copyright([C++ Skeleton parser for LALR(1) parsing with Bison],
-             [2002, 2003, 2004, 2005])[
-/* FIXME: This is wrong, we want computed header guards.
-   I don't know why the macros are missing now. :( */
+             [2002, 2003, 2004, 2005])
+dnl FIXME: This is wrong, we want computed header guards.
+dnl FIXME: I don\'t know why the macros are missing now. :(
+[
 #ifndef PARSER_HEADER_H
 # define PARSER_HEADER_H
 
 #include <string>
 #include <iostream>
+#include "stack.hh"
+#include "location.hh"
 
 /* Using locations.  */
 #define YYLSP_NEEDED ]b4_locations_flag[
 
-namespace yy
-{
-  class position;
-  class location;
-}
-
-]b4_token_enums(b4_tokens)[
-
-/* Copy the first part of user declarations.  */
+/* First part of user declarations.  */
 ]b4_pre_prologue[
 
 ]/* Line __line__ of lalr1.cc.  */
 b4_syncline(address@hidden@], address@hidden@])[
 
-#include "stack.hh"
-#include "location.hh"
-
 /* Enabling traces.  */
 #ifndef YYDEBUG
 # define YYDEBUG ]b4_debug[
@@ -74,23 +66,6 @@ b4_syncline(address@hidden@], address@hidden@])[
 # define YYTOKEN_TABLE ]b4_token_table[
 #endif
 
-#if ! defined (YYSTYPE) && ! defined (YYSTYPE_IS_DECLARED)
-]m4_ifdef([b4_stype],
-[b4_syncline([b4_stype_line], [b4_file_name])
-union YYSTYPE b4_stype;
-/* Line __line__ of lalr1.cc.  */
-b4_syncline(address@hidden@], address@hidden@])],
-[typedef int YYSTYPE;])[
-# define yystype YYSTYPE /* obsolescent; will be withdrawn */
-# define YYSTYPE_IS_DECLARED 1
-# define YYSTYPE_IS_TRIVIAL 1
-#endif
-
-/* Copy the second part of user declarations.  */
-]b4_post_prologue[
-
-]/* Line __line__ of lalr1.cc.  */
-b4_syncline(address@hidden@], address@hidden@])[
 /* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N].
    If N is 0, then set CURRENT to the empty location which ends
    the previous symbol: RHS[0] (always defined).  */
@@ -116,12 +91,26 @@ namespace yy
   /// A Bison parser.
   class ]b4_parser_class_name[
   {
+  public:
     /// Symbol semantic values.
+#if ! defined (YYSTYPE)
+]m4_ifdef([b4_stype],
+[b4_syncline([b4_stype_line], [b4_file_name])
+    union semantic_type b4_stype;
+/* Line __line__ of lalr1.cc.  */
+b4_syncline(address@hidden@], address@hidden@])],
+[    typedef int semantic_type;])[
+#else
     typedef YYSTYPE semantic_type;
+#endif
     /// Symbol locations.
     typedef ]b4_location_type[ location_type;
+    /// Tokens.
+    struct token
+    {
+      ]b4_token_enums(b4_tokens)[
+    };
 
-  public:
     /// Build a parser object.
     ]b4_parser_class_name[ (]b4_parse_param_decl[) :
       yydebug_ (false),
@@ -285,6 +274,14 @@ b4_error_verbose_if([, int tok])[);
   };
 }
 
+]m4_ifset([b4_global_tokens_and_yystype],
+[b4_token_defines(b4_tokens)
+
+#ifndef YYSTYPE
+ /* Redirection for backward compatibility. */
+# define YYSTYPE yy::b4_parser_class_name::semantic_type
+#endif
+])[
 #endif /* ! defined PARSER_HEADER_H */]
 ])dnl
 @output @output_parser_name@
@@ -298,6 +295,12 @@ m4_if(b4_defines_flag, 0, [],
 [
 #include @address@hidden)[
 
+/* User implementation prologue.  */
+]b4_post_prologue[
+
+]/* Line __line__ of lalr1.cc.  */
+b4_syncline(address@hidden@], address@hidden@])[
+
 #ifndef YY_
 # if YYENABLE_NLS
 #  if ENABLE_NLS
@@ -788,8 +791,8 @@ yyreturn:
 
 // Generate an error message.
 std::string
-yy::]b4_parser_class_name[::
-yysyntax_error_ (int yystate]b4_error_verbose_if([, int tok])[)
+yy::]b4_parser_class_name[::yysyntax_error_ (int yystate]dnl
+b4_error_verbose_if([, int tok])[)
 {
   std::string res;
 #if YYERROR_VERBOSE
Index: doc/bison.texinfo
===================================================================
RCS file: /cvsroot/bison/bison/doc/bison.texinfo,v
retrieving revision 1.161
retrieving revision 1.162
diff -p -u -r1.161 -r1.162
--- doc/bison.texinfo   20 Sep 2005 23:06:10 -0000      1.161
+++ doc/bison.texinfo   30 Sep 2005 17:57:05 -0000      1.162
@@ -6933,12 +6933,13 @@ for a complete and accurate documentatio
 The @code{%union} directive works as for C, see @ref{Union Decl, ,The
 Collection of Value Types}.  In particular it produces a genuine
 @address@hidden the future techniques to allow complex types
-within pseudo-unions (variants) might be implemented to alleviate
-these issues.}, which have a few specific features in C++.
+within pseudo-unions (similar to Boost variants) might be implemented to
+alleviate these issues.}, which have a few specific features in C++.
 @itemize @minus
 @item
-The name @code{YYSTYPE} also denotes @samp{union YYSTYPE}.  You may
-forward declare it just with @samp{union YYSTYPE;}.
+The type @code{YYSTYPE} is defined but its use is discouraged: rather
+you should refer to the parser's encapsulated type
address@hidden::parser::semantic_type}.
 @item
 Non POD (Plain Old Data) types cannot be used.  C++ forbids any
 instance of classes with constructors in unions: only @emph{pointers}
@@ -7139,7 +7140,8 @@ transforming the simple parsing context 
 
 The declaration of this driver class, @file{calc++-driver.hh}, is as
 follows.  The first part includes the CPP guard and imports the
-required standard library components.
+required standard library components, and the declaration of the parser
+class.
 
 @comment file: calc++-driver.hh
 @example
@@ -7147,26 +7149,9 @@ required standard library components.
 # define CALCXX_DRIVER_HH
 # include <string>
 # include <map>
+# include "calc++-parser.hh"
 @end example
 
address@hidden
-Then come forward declarations.  Because the parser uses the parsing
-driver and reciprocally, simple inclusions of header files will not
-do.  Because the driver's declaration is the one that will be imported
-by the rest of the project, it is saner to forward declare the
-parser's information here.
-
address@hidden file: calc++-driver.hh
address@hidden
-// Forward declarations.
-union YYSTYPE;
-namespace yy
address@hidden
-  class location;
-  class calcxx_parser;
address@hidden
-class calcxx_driver;
address@hidden example
 
 @noindent
 Then comes the declaration of the scanning function.  Flex expects
@@ -7178,7 +7163,9 @@ factor both as follows.
 @example
 // Announce to Flex the prototype we want for lexing function, ...
 # define YY_DECL                                                \
-  int yylex (YYSTYPE* yylval, yy::location* yylloc, calcxx_driver& driver)
+  int yylex (yy::calcxx_parser::semantic_type* yylval,           \
+             yy::calcxx_parser::location_type* yylloc,           \
+             calcxx_driver& driver)
 // ... and declare it for the parser's sake.
 YY_DECL;
 @end example
@@ -7289,18 +7276,29 @@ calcxx_driver::error (const std::string&
 @subsection Calc++ Parser
 
 The parser definition file @file{calc++-parser.yy} starts by asking
-for the C++ skeleton, the creation of the parser header file, and
-specifies the name of the parser class.  It then includes the required
-headers.
+for the C++ LALR(1) skeleton, the creation of the parser header file, and
+specifies the name of the parser class.
 
 @comment file: calc++-parser.yy
 @example
 %skeleton "lalr1.cc"                          /*  -*- C++ -*- */
-%define "parser_class_name" "calcxx_parser"
 %defines
+%define "parser_class_name" "calcxx_parser"
address@hidden example
+
address@hidden
+Then come the declarations/inclusions needed to define the
address@hidden  Because the parser uses the parsing driver and
+reciprocally, both cannot include the header of the other.  Because the
+driver's header needs detailed knowledge about the parser class (in
+particular its inner types), it is the parser's header which will simply
+use a forward declaration of the driver.
+
address@hidden file: calc++-parser.yy
address@hidden
 address@hidden
 # include <string>
-# include "calc++-driver.hh"
+class calcxx_driver;
 address@hidden
 @end example
 
@@ -7357,6 +7355,19 @@ them.
 @end example
 
 @noindent
+The code between @address@hidden and @address@hidden after the introduction of 
the
address@hidden is output in the @file{*.cc} file; it needs detailed
+knowledge about the driver.
+
address@hidden file: calc++-parser.yy
address@hidden
address@hidden
+# include "calc++-driver.hh"
address@hidden
address@hidden example
+
+
address@hidden
 The token numbered as 0 corresponds to end of file; the following line
 allows for nicer error messages referring to ``end of file'' instead
 of ``$end''.  Similarly user friendly named are provided for each
@@ -7365,11 +7376,11 @@ avoid name clashes.
 
 @comment file: calc++-parser.yy
 @example
-%token        TOKEN_EOF      0 "end of file"
-%token        TOKEN_ASSIGN     ":="
-%token <sval> TOKEN_IDENTIFIER "identifier"
-%token <ival> TOKEN_NUMBER     "number"
-%type  <ival> exp              "expression"
+%token        END      0 "end of file"
+%token        ASSIGN     ":="
+%token <sval> IDENTIFIER "identifier"
+%token <ival> NUMBER     "number"
+%type  <ival> exp        "expression"
 @end example
 
 @noindent
@@ -7396,7 +7407,7 @@ unit: assignments exp  @{ driver.result 
 assignments: assignments assignment @address@hidden
            | /* Nothing. */         @address@hidden;
 
-assignment: TOKEN_IDENTIFIER ":=" exp @{ driver.variables[*$1] = $3; @};
+assignment: "identifier" ":=" exp @{ driver.variables[*$1] = $3; @};
 
 %left '+' '-';
 %left '*' '/';
@@ -7404,8 +7415,8 @@ exp: exp '+' exp   @{ $$ = $1 + $3; @}
    | exp '-' exp   @{ $$ = $1 - $3; @}
    | exp '*' exp   @{ $$ = $1 * $3; @}
    | exp '/' exp   @{ $$ = $1 / $3; @}
-   | TOKEN_IDENTIFIER  @{ $$ = driver.variables[*$1]; @}
-   | TOKEN_NUMBER      @{ $$ = $1; @};
+   | "identifier"  @{ $$ = driver.variables[*$1]; @}
+   | "number"      @{ $$ = $1; @};
 %%
 @end example
 
@@ -7485,22 +7496,28 @@ preceding tokens.  Comments would be tre
 @end example
 
 @noindent
-The rules are simple, just note the use of the driver to report
-errors.
+The rules are simple, just note the use of the driver to report errors.
+It is convenient to use a typedef to shorten
address@hidden::calcxx_parser::token::identifier} into
address@hidden::identifier} for isntance.
 
 @comment file: calc++-scanner.ll
 @example
address@hidden
+  typedef yy::calcxx_parser::token token;
address@hidden
+
 [-+*/]     return yytext[0];
-":="       return TOKEN_ASSIGN;
+":="       return token::ASSIGN;
 @address@hidden      @{
   errno = 0;
   long n = strtol (yytext, NULL, 10);
   if (! (INT_MIN <= n && n <= INT_MAX && errno != ERANGE))
     driver.error (*yylloc, "integer is out of range");
   yylval->ival = n;
-  return TOKEN_NUMBER;
+  return token::NUMBER;
 @}
address@hidden@}       yylval->sval = new std::string (yytext); return 
TOKEN_IDENTIFIER;
address@hidden@}       yylval->sval = new std::string (yytext); return 
token::IDENTIFIER;
 .          driver.error (*yylloc, "invalid character");
 %%
 @end example
Index: examples/extexi
===================================================================
RCS file: /cvsroot/bison/bison/examples/extexi,v
retrieving revision 1.3
retrieving revision 1.4
diff -p -u -r1.3 -r1.4
--- examples/extexi     23 Jul 2005 17:06:41 -0000      1.3
+++ examples/extexi     30 Sep 2005 17:57:05 -0000      1.4
@@ -64,8 +64,6 @@ BEGIN {
       # #line report the line number of the *next* line.
       # => + 2.
       # Note that recent Bison support it, but not Flex.
-      if (file ~ /\.[chy]*$/)
-       input = "#line " (FNR + 1) " \"" FILENAME "\"\n";
       next;
     }
 
Index: src/parse-gram.y
===================================================================
RCS file: /cvsroot/bison/bison/src/parse-gram.y,v
retrieving revision 1.57
retrieving revision 1.58
diff -p -u -r1.57 -r1.58
--- src/parse-gram.y    25 Jul 2005 03:38:41 -0000      1.57
+++ src/parse-gram.y    30 Sep 2005 17:57:05 -0000      1.58
@@ -202,6 +202,7 @@ declaration:
   grammar_declaration
 | PROLOGUE                                 { prologue_augment ($1, @1); }
 | "%debug"                                 { debug_flag = true; }
+| "%define" string_content                 { muscle_insert ($2, "1"); }
 | "%define" string_content string_content  { muscle_insert ($2, $3); }
 | "%defines"                               { defines_flag = true; }
 | "%error-verbose"                         { error_verbose = true; }
Index: tests/actions.at
===================================================================
RCS file: /cvsroot/bison/bison/tests/actions.at,v
retrieving revision 1.53
retrieving revision 1.54
diff -p -u -r1.53 -r1.54
--- tests/actions.at    19 Sep 2005 21:08:21 -0000      1.53
+++ tests/actions.at    30 Sep 2005 17:57:05 -0000      1.54
@@ -187,6 +187,7 @@ m4_ifval([$6], [%union
 {
   int ival;
 }])
+AT_LALR1_CC_IF([%define "global_tokens_and_yystype"])
 [
 %{
 ]AT_LALR1_CC_IF([typedef yy::location YYLTYPE;
Index: tests/calc.at
===================================================================
RCS file: /cvsroot/bison/bison/tests/calc.at,v
retrieving revision 1.79
retrieving revision 1.80
diff -p -u -r1.79 -r1.80
--- tests/calc.at       21 Sep 2005 21:05:52 -0000      1.79
+++ tests/calc.at       30 Sep 2005 17:57:05 -0000      1.80
@@ -39,7 +39,9 @@ m4_define([_AT_DATA_CALC_Y],
        [m4_fatal([$0: Invalid arguments: address@hidden)])dnl
 AT_DATA_GRAMMAR([calc.y],
 [[/* Infix notation calculator--calc */
-]$4[
+]$4
+AT_LALR1_CC_IF(
+[%define "global_tokens_and_yystype"])[
 %{
 #include <stdio.h>
 
@@ -68,7 +70,13 @@ static int global_count = 0;
 
 %{
 static int power (int base, int exponent);
-]AT_LALR1_CC_IF([typedef yy::location YYLTYPE;],
+]AT_LALR1_CC_IF(
+[typedef yy::location YYLTYPE;
+#define first_line   begin.line
+#define first_column begin.column
+#define last_line    end.line
+#define last_column  end.column
+],
 [/* yyerror receives the location if:
    - %location & %pure & %glr
    - %location & %pure & %yacc & %parse-param. */
@@ -176,16 +184,11 @@ get_char (]AT_LEX_FORMALS[)
   last_yylloc = AT_LOC;
   if (res == '\n')
     {
-AT_LALR1_CC_IF(
-[     AT_LOC.end.line++;
-      AT_LOC.end.column = 0;],
-[     AT_LOC.last_line++;
-      AT_LOC.last_column = 0;])
+      AT_LOC.last_line++;
+      AT_LOC.last_column = 0;
     }
   else
-AT_LALR1_CC_IF(
-[    AT_LOC.end.column++;],
-[    AT_LOC.last_column++;])
+    AT_LOC.last_column++;
 ])[
   return res;
 }
@@ -244,27 +247,24 @@ yylex (]AT_LEX_FORMALS[)
   if (init)
     {
       init = 0;
-]AT_LALR1_CC_IF([],
-[AT_LOCATION_IF([
+]AT_LOCATION_IF([
       AT_LOC.last_column = 0;
       AT_LOC.last_line = 1;
-])])[
+])[
     }
 
-]AT_LOCATION_IF([AT_LALR1_CC_IF(
-[ AT_LOC.begin = AT_LOC.end;],
-[ AT_LOC.first_column = AT_LOC.last_column;
+]AT_LOCATION_IF([
+ AT_LOC.first_column = AT_LOC.last_column;
   AT_LOC.first_line   = AT_LOC.last_line;
-])])[
+])[
 
   /* Skip white space.  */
   while ((c = get_char (]AT_LEX_ARGS[)) == ' ' || c == '\t')
     {
-]AT_LOCATION_IF([AT_LALR1_CC_IF(
-[     AT_LOC.begin = AT_LOC.end;],
+]AT_LOCATION_IF(
 [     AT_LOC.first_column = AT_LOC.last_column;
       AT_LOC.first_line   = AT_LOC.last_line;
-])])[
+])[
     }
 
   /* process numbers   */
@@ -428,8 +428,8 @@ AT_CHECK([cat stderr], 0, [expout])
 ])
 
 
-# AT_CHECK_CALC([BISON-OPTIONS [, EXPECTED-TO-FAIL]])
-# ------------------------------
+# AT_CHECK_CALC([BISON-OPTIONS, [EXPECTED-TO-FAIL]])
+# --------------------------------------------------
 # Start a testing chunk which compiles `calc' grammar with
 # BISON-OPTIONS, and performs several tests over the parser.
 # However, if EXPECTED-TO-FAIL is nonempty, this test is expected to fail.