Re: push parser

[Joel E. Denny]

On Wed, 20 Dec 2006, Joel E. Denny wrote:

> On Wed, 20 Dec 2006, Joel E. Denny wrote:
> 
> >  2006-12-20  Joel E. Denny  <address@hidden>
> >  
> > +   Enable push parsers to operate in impure mode.  Thus, %push-parser no
> > +   longer implies %pure-parser.  The point of this change is to move
> > +   towards being able to test the push parser code by running the entire
> > +   test suite as if %push-parser had been declared.
> 
> Another advantage of this patch is that it eliminates any debate over how 
> a %pure-parser should be handled when there's a %push-parser.  That is, 
> %pure-parser now has an obvious meaning.

I committed the above and the following.  Some of these changes may need 
to be reconsidered (especially the yyparse implementation in push mode), 
but we can revise later if necessary.  For now, we are at least able to 
run the test suite in push mode using the procedure I described earlier in 
this thread:

a. cp push.c yacc.c

b. In yacc.c, look for this line:

  m4_include(b4_pkgdatadir/[c.m4])

Add this line afterwards:

  m4_define([b4_push_if], [$1])

c. Now look for this line in yacc.c:

   YYDPRINTF ((stderr, "Return for a new token:\n"));

Delete it.

d. Run make maintainer-check.

I'm thinking it might be useful to automate this at some point.

I'll follow soon with a patch that addresses performance.

Index: ChangeLog
===================================================================
RCS file: /sources/bison/bison/ChangeLog,v
retrieving revision 1.1637
diff -p -u -r1.1637 ChangeLog
--- ChangeLog   25 Dec 2006 17:25:39 -0000      1.1637
+++ ChangeLog   25 Dec 2006 17:30:34 -0000
@@ -1,5 +1,18 @@
 2006-12-25  Joel E. Denny  <address@hidden>
 
+       For push mode, add pull wrappers around yypush_parse.
+       * data/push.c: (b4_generate_macro_args, b4_parenthesize): New macros.
+       (yypull_parse): New function wrapping yypush_parse.
+       (yyparse): New #define wrapping yypull_parse.
+       * tests/calc.at (_AT_DATA_CALC_Y): Call yyparse even when %push-parser
+       is declared.
+       * tests/headers.at (export YYLTYPE): Make yylex global.  For push mode,
+       prototype yylex in the module that calls yyparse, and don't prototype
+       yyparse there.  Otherwise, the yyparse expansion won't compile.
+       * tests/input.at (Torturing the Scanner): Likewise.
+
+2006-12-25  Joel E. Denny  <address@hidden>
+
        Enable push parsers to operate in impure mode.  Thus, %push-parser no
        longer implies %pure-parser.  The point of this change is to move
        towards being able to test the push parser code by running the entire
Index: data/push.c
===================================================================
RCS file: /sources/bison/bison/data/push.c,v
retrieving revision 1.24
diff -p -u -r1.24 push.c
--- data/push.c 25 Dec 2006 17:25:39 -0000      1.24
+++ data/push.c 25 Dec 2006 17:30:34 -0000
@@ -63,6 +63,31 @@ b4_locations_if([, [[YYLTYPE *], [&yyllo
 m4_ifdef([b4_lex_param], b4_lex_param)))
 
 
+# b4_generate_macro_args([A], [B], [C], ...)
+# ---------------------------------------------------
+# Generate a comma-delimited list whose size is equal to the number of input
+# arguments and whose form is:
+#
+#   YYARG1, YYARG2, YYARG3, ...
+#
+# No argument should be the empty string except A in the special invocation
+# b4_generate_macro_args(), which generates an empty string.
+m4_define([b4_generate_macro_args],
+[m4_if([$1], [], [], [$#], [1], [[YYARG1]],
+       [b4_generate_macro_args(m4_shift($@)), [YYARG$#]])])
+
+
+# b4_parenthesize([A], [B], [C], ...)
+# ---------------------------------------------------
+# Convert arguments to the form:
+#
+#   (A), (B), (C), ...
+#
+# No argument should be the empty string except A in the special invocation
+# b4_parenthesize(), which generates an empty string.
+m4_define([b4_parenthesize],
+[m4_if([$1], [], [], [$#], [1], [[($1)]],
+       [($1), b4_parenthesize(m4_shift($@))])])
 
 ## ------------ ##
 ## Data Types.  ##
@@ -152,6 +177,7 @@ b4_copyright([Skeleton implementation fo
 m4_if(b4_prefix, [yy], [],
 [/* Substitute the variable and function names.  */
 ]b4_push_if([#define yypush_parse b4_prefix[]push_parse
+#define yypull_parse b4_prefix[]pull_parse
 #define yypstate_new b4_prefix[]pstate_new
 #define yypstate_delete b4_prefix[]pstate_delete
 #define yypstate b4_prefix[]pstate],
@@ -222,6 +248,7 @@ b4_push_if([[#ifndef YYPUSH_DECLS
 struct yypstate;
 typedef struct yypstate yypstate;
 enum { YYPUSH_MORE = 4 };
+#  define yyparse(]b4_generate_macro_args(b4_parse_param))[ yypull_parse (0, 
&yylex]m4_ifset([b4_parse_param], [, 
b4_parenthesize(b4_generate_macro_args(b4_parse_param))])[)
 ]b4_c_function_decl([[yypstate_new]], [[yypstate *]], [[[void]], []])
 b4_c_function_decl([[yypstate_delete]], [[void]],
                    [[[yypstate *yyps]], [[yyps]]])
@@ -230,6 +257,10 @@ b4_c_function_decl([[yypush_parse]], [[i
   [[[int yypushed_char]], [[yypushed_char]]],
   [[[YYSTYPE const *yypushed_val]], [[yypushed_val]]]b4_locations_if([,
   [[[YYLTYPE const *yypushed_loc]], 
[[yypushed_loc]]]])])m4_ifset([b4_parse_param], [,
+  b4_parse_param]))
+b4_c_function_decl([[yypull_parse]], [[int]],
+  [[[yypstate *yyps]], [[yyps]]],
+  [[[int (*yylexp)(]b4_c_ansi_formals(b4_lex_param)[)]], 
[[yylexp]]]m4_ifset([b4_parse_param], [,
   b4_parse_param]))[
 #endif
 ]])
@@ -1065,6 +1096,31 @@ b4_push_if(
     int yynew;
   };
 
+]b4_c_function_def([[yypull_parse]], [[int]],
+  [[[yypstate *yyps]], [[yyps]]],
+  [[[int (*yylexp)(]b4_c_ansi_formals(b4_lex_param)[)]], 
[[yylexp]]]m4_ifset([b4_parse_param], [,
+  b4_parse_param]))[
+{
+  int yystatus;
+  yypstate *yyps_local;
+]b4_pure_if([[  int yychar;
+  YYSTYPE yylval;
+]b4_locations_if([[  YYLTYPE yylloc;
+]])])[
+  if (yyps == 0)
+     yyps_local = yypstate_new ();
+   else
+     yyps_local = yyps;
+   do {
+     yychar = ]b4_c_function_call([yylexp], [int], b4_lex_param)[;
+     yystatus =
+       yypush_parse (yyps_local]b4_pure_if([[, yychar, 
&yylval]b4_locations_if([[, &yylloc]])])m4_ifset([b4_parse_param], [, 
b4_c_args(b4_parse_param)])[);
+  } while (yystatus == YYPUSH_MORE);
+  if (yyps == 0)
+    yypstate_delete (yyps_local);
+  return yystatus;
+}
+
 /* Initialize the parser data structure.  */
 ]b4_c_function_def([[yypstate_new]], [[yypstate *]])[
 {
@@ -1651,6 +1707,7 @@ b4_push_if([[#ifndef YYPUSH_DECLS
 struct ]b4_prefix[pstate;
 typedef struct ]b4_prefix[pstate ]b4_prefix[pstate;
 enum { YYPUSH_MORE = 4 };
+#  define ]b4_prefix[parse(]b4_generate_macro_args(b4_parse_param)) 
b4_prefix[pull_parse (0, &]b4_prefix[lex]m4_ifset([b4_parse_param], [, 
b4_parenthesize(b4_generate_macro_args(b4_parse_param))])[)
 ]b4_c_function_decl([b4_prefix[pstate_new]], [b4_prefix[pstate *]],
                     [[[void]], []])
 b4_c_function_decl([b4_prefix[pstate_delete]], [[void]],
@@ -1660,6 +1717,10 @@ b4_c_function_decl([b4_prefix[push_parse
   [[[int yypushed_char]], [[yypushed_char]]],
   [[[YYSTYPE const *yypushed_val]], [[yypushed_val]]]b4_locations_if([,
   [[[YYLTYPE const *yypushed_loc]], 
[[yypushed_loc]]]])])m4_ifset([b4_parse_param], [,
+  b4_parse_param]))
+b4_c_function_decl([b4_prefix[pull_parse]], [[int]],
+  [[b4_prefix[pstate *yyps]], [[yyps]]],
+  [[[int (*yylexp)(]b4_c_ansi_formals(b4_lex_param)[)]], 
[[yylexp]]]m4_ifset([b4_parse_param], [,
   b4_parse_param]))[
 #endif
 ]])
Index: tests/calc.at
===================================================================
RCS file: /sources/bison/bison/tests/calc.at,v
retrieving revision 1.100
diff -p -u -r1.100 calc.at
--- tests/calc.at       25 Dec 2006 17:25:39 -0000      1.100
+++ tests/calc.at       25 Dec 2006 17:30:35 -0000
@@ -336,17 +336,7 @@ main (int argc, const char **argv)
 
 ]AT_SKEL_CC_IF([], [m4_bmatch([$4], [%debug],
 [  yydebug = 1;])])[
-]AT_PUSH_IF([
-  {
-    yypstate *pstate = yypstate_new ();
-    YYSTYPE my_lval;
-    ]AT_LOCATION_IF([YYLTYPE my_lloc;])[
-    do {
-      status = yypush_parse (pstate, yylex (&my_lval]AT_LOCATION_IF([[, 
&my_lloc]])[), &my_lval]AT_LOCATION_IF([[, &my_lloc]])[);
-    } while (status == YYPUSH_MORE);
-    yypstate_delete (pstate);
-  }],[
-  status = yyparse (]AT_PARAM_IF([[&result, &count]])[);])[
+  status = yyparse (]AT_PARAM_IF([[&result, &count]])[);
   fclose (input);
   if (global_result != result)
     abort ();
Index: tests/headers.at
===================================================================
RCS file: /sources/bison/bison/tests/headers.at,v
retrieving revision 1.15
diff -p -u -r1.15 headers.at
--- tests/headers.at    2 Dec 2006 01:52:16 -0000       1.15
+++ tests/headers.at    25 Dec 2006 17:30:35 -0000
@@ -94,7 +94,9 @@ AT_DATA_GRAMMAR([input.y],
 #include <stdio.h>
 #include <stdlib.h>
 
-static int
+int my_lex (void);
+
+int
 my_lex (void)
 {
   return EOF;
@@ -118,7 +120,11 @@ AT_DATA([caller.c],
 [[#include "input.h"
 YYLTYPE *my_llocp = &my_lloc;
 
+#ifndef YYPUSH_DECLS
 int my_parse (void);
+#else
+int my_lex (void);
+#endif
 
 int
 main (void)
Index: tests/input.at
===================================================================
RCS file: /sources/bison/bison/tests/input.at,v
retrieving revision 1.64
diff -p -u -r1.64 input.at
--- tests/input.at      21 Nov 2006 07:07:35 -0000      1.64
+++ tests/input.at      25 Dec 2006 17:30:35 -0000
@@ -470,7 +470,7 @@ char quote[] = "@:>@@:>@,";
 
 %{
 static void yyerror (const char *s);
-static int yylex (void);
+int yylex (void);
 %}
 
 %type <ival> '@<:@'
@@ -503,7 +503,7 @@ value_as_yystype (value val)
   return res;
 }
 
-static int
+int
 yylex (void)
 {
   static char const input[] = 
"@<:@address@hidden@address@hidden@&address@hidden
@@ -529,7 +529,11 @@ AT_DATA([main.c],
 [[typedef int value;
 #include "input.h"
 
+#ifndef YYPUSH_DECLS
 int yyparse (void);
+#else
+int yylex (void);
+#endif
 
 int
 main (void)