[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
miscellaneous cleanups for Bison scanner and parser
From: |
Paul Eggert |
Subject: |
miscellaneous cleanups for Bison scanner and parser |
Date: |
Thu, 7 Nov 2002 21:28:53 -0800 (PST) |
I noticed that the Bison parser defines a token CHARACTER that is
never used, and started cleaning things up. One thing led to another,
and I ended up simplifying the scanner a bit. It's now simpler and
easier to read (at least for me), and it generates an object module
that is 10% smaller (on Solaris sparc with GCC 3.2). I installed the
following patch.
2002-11-07 Paul Eggert <address@hidden>
* src/parse-gram.y (CHARACTER): Remove unused token.
All uses removed.
* src/scan-gram.l: Remove stack option. We no longer use the
stack, since the stack was never deeper than 1; instead, use the
new auto var c_context to record the stacked value.
Remove nounput option. At an unexpected end of file, we now unput
the minimal input necessary to end cleanly; this simplifies the
code.
Avoid unbounded token sizes where this is easy.
(unexpected_end_of_file): Reword so that beginning of token isn't
needed.
(last-string): Now auto, not static.
(YY_OBS_FREE): Remove unnecessary do while (0) wrapper.
(scanner_last_string_free): Remove; not used.
(percent_percent_count): Move decl to just before use.
(SC_ESCAPED_CHARACTER): Return ID at unexpected end of file,
not the (never otherwised-used) CHARACTER.
Index: src/parse-gram.y
===================================================================
RCS file: /cvsroot/bison/bison/src/parse-gram.y,v
retrieving revision 1.24
diff -p -u -r1.24 parse-gram.y
--- src/parse-gram.y 19 Oct 2002 14:38:06 -0000 1.24
+++ src/parse-gram.y 8 Nov 2002 05:11:58 -0000
@@ -97,7 +97,6 @@ braced_code_t current_braced_code = acti
/* Define the tokens together with their human representation. */
%token GRAM_EOF 0 "end of file"
%token STRING "string"
-%token CHARACTER "character"
%token INT "integer"
%token PERCENT_TOKEN "%token"
@@ -157,7 +156,7 @@ braced_code_t current_braced_code = acti
%token BRACED_CODE "{...}"
-%type <string> CHARACTER TYPE STRING string_content
+%type <string> TYPE STRING string_content
BRACED_CODE PROLOGUE EPILOGUE epilogue.opt action
%type <integer> INT
%type <symbol> ID symbol string_as_id
@@ -285,7 +284,7 @@ precedence_declarator:
;
type.opt:
- /* Nothing. */ { current_type = NULL;}
+ /* Nothing. */ { current_type = NULL; }
| TYPE { current_type = $1; }
;
@@ -390,7 +389,6 @@ rhs:
symbol:
ID { $$ = $1; }
| string_as_id { $$ = $1; }
-| CHARACTER { $$ = symbol_get ($1, @1); }
;
action:
@@ -443,10 +441,6 @@ yyprint (FILE *file,
fputc (' ', file);
switch (type)
{
- case CHARACTER:
- fprintf (file, " = '%s'", value->string);
- break;
-
case ID:
fprintf (file, " = %s", value->symbol->tag);
break;
Index: src/scan-gram.l
===================================================================
RCS file: /cvsroot/bison/bison/src/scan-gram.l,v
retrieving revision 1.35
diff -p -u -r1.35 scan-gram.l
--- src/scan-gram.l 6 Nov 2002 08:08:46 -0000 1.35
+++ src/scan-gram.l 8 Nov 2002 05:11:59 -0000
@@ -19,7 +19,7 @@
02111-1307 USA
*/
-%option debug nodefault noyywrap nounput never-interactive stack
+%option debug nodefault noyywrap never-interactive
%option prefix="gram_" outfile="lex.yy.c"
%{
@@ -140,7 +140,6 @@ extend_location (location_t *loc, char c
used, and which is used by YY_OBS_FREE to free the last string. */
static struct obstack string_obstack;
-char *last_string;
#define YY_OBS_GROW \
obstack_grow (&string_obstack, yytext, yyleng)
@@ -151,19 +150,9 @@ char *last_string;
last_string = obstack_finish (&string_obstack); \
} while (0)
-#define YY_OBS_FREE \
- do { \
- obstack_free (&string_obstack, last_string); \
- } while (0)
-
-void
-scanner_last_string_free (void)
-{
- YY_OBS_FREE;
-}
-
+#define YY_OBS_FREE \
+ obstack_free (&string_obstack, last_string)
-static int percent_percent_count = 0;
/* Within well-formed rules, RULE_LENGTH is the number of values in
the current rule so far, which says where to find `$0' with respect
@@ -179,6 +168,7 @@ static void handle_at (braced_code_t cod
char *cp, location_t location);
static void handle_syncline (char *args, location_t *location);
static int convert_ucn_to_byte (char const *hex_text);
+static void unexpected_end_of_file (location_t, char const *);
%}
%x SC_COMMENT SC_LINE_COMMENT SC_YACC_COMMENT
@@ -202,8 +192,15 @@ splice (\\[ \f\t\v]*\n)*
%%
%{
+ /* Nesting level of the current code in braces. */
int braces_level IF_LINT (= 0);
+ /* Scanner context when scanning C code. */
+ int c_context IF_LINT (= 0);
+
+ /* A string representing the most recently saved token. */
+ char *last_string;
+
/* At each yylex invocation, mark the current position as the
start of the next token. */
YY_STEP;
@@ -263,7 +260,7 @@ splice (\\[ \f\t\v]*\n)*
"," return COMMA;
";" return SEMICOLON;
- [ \f\n\t\v]+ YY_STEP;
+ [ \f\n\t\v] YY_STEP;
{id} {
yylval->symbol = symbol_get (yytext, *yylloc);
@@ -285,20 +282,20 @@ splice (\\[ \f\t\v]*\n)*
}
/* Characters. We don't check there is only one. */
- "'" YY_OBS_GROW; yy_push_state (SC_ESCAPED_CHARACTER);
+ "'" YY_OBS_GROW; BEGIN SC_ESCAPED_CHARACTER;
/* Strings. */
- "\"" YY_OBS_GROW; yy_push_state (SC_ESCAPED_STRING);
+ "\"" YY_OBS_GROW; BEGIN SC_ESCAPED_STRING;
/* Comments. */
"/*" BEGIN SC_YACC_COMMENT;
"//".* YY_STEP;
/* Prologue. */
- "%{" yy_push_state (SC_PROLOGUE);
+ "%{" BEGIN SC_PROLOGUE;
/* Code in between braces. */
- "{" YY_OBS_GROW; braces_level = 0; yy_push_state (SC_BRACED_CODE);
+ "{" YY_OBS_GROW; braces_level = 0; BEGIN SC_BRACED_CODE;
/* A type. */
"<"{tag}">" {
@@ -308,33 +305,20 @@ splice (\\[ \f\t\v]*\n)*
return TYPE;
}
-
- "%%" {
+ "%%" {
+ static int percent_percent_count;
if (++percent_percent_count == 2)
- yy_push_state (SC_EPILOGUE);
+ BEGIN SC_EPILOGUE;
return PERCENT_PERCENT;
}
- . {
+ . {
complain_at (*yylloc, _("invalid character: %s"), quote (yytext));
YY_STEP;
}
}
- /*-------------------------------------------------------------------.
- | Whatever the start condition (but those which correspond to |
- | entities `swallowed' by Bison: SC_YACC_COMMENT, SC_ESCAPED_STRING, |
- | and SC_ESCAPED_CHARACTER), no M4 character must escape as is. |
- `-------------------------------------------------------------------*/
-
-<SC_COMMENT,SC_LINE_COMMENT,SC_STRING,SC_CHARACTER,SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>
-{
- \[ obstack_sgrow (&string_obstack, "@<:@");
- \] obstack_sgrow (&string_obstack, "@:>@");
-}
-
-
/*---------------------------------------------------------------.
| Scanning a Yacc comment. The initial `/ *' is already eaten. |
`---------------------------------------------------------------*/
@@ -346,12 +330,8 @@ splice (\\[ \f\t\v]*\n)*
BEGIN INITIAL;
}
- [^*]+|"*" ;
-
- <<EOF>> {
- complain_at (*yylloc, _("unexpected end of file in a comment"));
- BEGIN INITIAL;
- }
+ .|\n ;
+ <<EOF>> unexpected_end_of_file (*yylloc, "*/");
}
@@ -361,13 +341,8 @@ splice (\\[ \f\t\v]*\n)*
<SC_COMMENT>
{
- "*"{splice}"/" YY_OBS_GROW; yy_pop_state ();
- [^*\[\]]+|"*" YY_OBS_GROW;
-
- <<EOF>> {
- complain_at (*yylloc, _("unexpected end of file in a comment"));
- yy_pop_state ();
- }
+ "*"{splice}"/" YY_OBS_GROW; BEGIN c_context;
+ <<EOF>> unexpected_end_of_file (*yylloc, "*/");
}
@@ -377,9 +352,9 @@ splice (\\[ \f\t\v]*\n)*
<SC_LINE_COMMENT>
{
- "\n" YY_OBS_GROW; yy_pop_state ();
- ([^\n\[\]]|{splice})+ YY_OBS_GROW;
- <<EOF>> yy_pop_state ();
+ "\n" YY_OBS_GROW; BEGIN c_context;
+ {splice} YY_OBS_GROW;
+ <<EOF>> BEGIN c_context;
}
@@ -391,25 +366,16 @@ splice (\\[ \f\t\v]*\n)*
<SC_ESCAPED_STRING>
{
"\"" {
- assert (yy_top_state () == INITIAL);
YY_OBS_GROW;
YY_OBS_FINISH;
yylval->string = last_string;
- yy_pop_state ();
rule_length++;
+ BEGIN INITIAL;
return STRING;
}
- [^\"\\]+ YY_OBS_GROW;
-
- <<EOF>> {
- complain_at (*yylloc, _("unexpected end of file in a string"));
- assert (yy_top_state () == INITIAL);
- YY_OBS_FINISH;
- yylval->string = last_string;
- yy_pop_state ();
- return STRING;
- }
+ .|\n YY_OBS_GROW;
+ <<EOF>> unexpected_end_of_file (*yylloc, "\"");
}
/*---------------------------------------------------------------.
@@ -421,30 +387,19 @@ splice (\\[ \f\t\v]*\n)*
{
"'" {
YY_OBS_GROW;
- assert (yy_top_state () == INITIAL);
- {
- YY_OBS_FINISH;
- yylval->symbol = symbol_get (last_string, *yylloc);
- symbol_class_set (yylval->symbol, token_sym, *yylloc);
- symbol_user_token_number_set (yylval->symbol,
- (unsigned char) last_string[1], *yylloc);
- YY_OBS_FREE;
- yy_pop_state ();
- rule_length++;
- return ID;
- }
- }
-
- [^\'\\]+ YY_OBS_GROW;
-
- <<EOF>> {
- complain_at (*yylloc, _("unexpected end of file in a character literal"));
- assert (yy_top_state () == INITIAL);
YY_OBS_FINISH;
- yylval->string = last_string;
- yy_pop_state ();
- return CHARACTER;
+ yylval->symbol = symbol_get (last_string, *yylloc);
+ symbol_class_set (yylval->symbol, token_sym, *yylloc);
+ symbol_user_token_number_set (yylval->symbol,
+ (unsigned char) last_string[1], *yylloc);
+ YY_OBS_FREE;
+ rule_length++;
+ BEGIN INITIAL;
+ return ID;
}
+
+ .|\n YY_OBS_GROW;
+ <<EOF>> unexpected_end_of_file (*yylloc, "'");
}
@@ -504,8 +459,6 @@ splice (\\[ \f\t\v]*\n)*
quote (yytext));
YY_OBS_GROW;
}
- /* FLex wants this rule, in case of a `\<<EOF>>'. */
- \\ YY_OBS_GROW;
}
@@ -516,23 +469,10 @@ splice (\\[ \f\t\v]*\n)*
<SC_CHARACTER>
{
- "'" {
- YY_OBS_GROW;
- assert (yy_top_state () != INITIAL);
- yy_pop_state ();
- }
-
- [^'\[\]\\]+ YY_OBS_GROW;
+ "'" YY_OBS_GROW; BEGIN c_context;
\\{splice}[^\[\]] YY_OBS_GROW;
{splice} YY_OBS_GROW;
- /* Needed for `\<<EOF>>', `\\<<newline>>[', and `\\<<newline>>]'. */
- \\ YY_OBS_GROW;
-
- <<EOF>> {
- complain_at (*yylloc, _("unexpected end of file in a character literal"));
- assert (yy_top_state () != INITIAL);
- yy_pop_state ();
- }
+ <<EOF>> unexpected_end_of_file (*yylloc, "'");
}
@@ -543,23 +483,10 @@ splice (\\[ \f\t\v]*\n)*
<SC_STRING>
{
- "\"" {
- assert (yy_top_state () != INITIAL);
- YY_OBS_GROW;
- yy_pop_state ();
- }
-
- [^\"\[\]\\]+ YY_OBS_GROW;
+ "\"" YY_OBS_GROW; BEGIN c_context;
\\{splice}[^\[\]] YY_OBS_GROW;
{splice} YY_OBS_GROW;
- /* Needed for `\<<EOF>>', `\\<<newline>>[', and `\\<<newline>>]'. */
- \\ YY_OBS_GROW;
-
- <<EOF>> {
- complain_at (*yylloc, _("unexpected end of file in a string"));
- assert (yy_top_state () != INITIAL);
- yy_pop_state ();
- }
+ <<EOF>> unexpected_end_of_file (*yylloc, "\"");
}
@@ -569,18 +496,10 @@ splice (\\[ \f\t\v]*\n)*
<SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>
{
- /* Characters. We don't check there is only one. */
- "'" YY_OBS_GROW; yy_push_state (SC_CHARACTER);
-
- /* Strings. */
- "\"" YY_OBS_GROW; yy_push_state (SC_STRING);
-
- /* Comments. */
- "/"{splice}"*" YY_OBS_GROW; yy_push_state (SC_COMMENT);
- "/"{splice}"/" YY_OBS_GROW; yy_push_state (SC_LINE_COMMENT);
-
- /* Not comments. */
- "/" YY_OBS_GROW;
+ "'" YY_OBS_GROW; c_context = YY_START; BEGIN SC_CHARACTER;
+ "\"" YY_OBS_GROW; c_context = YY_START; BEGIN SC_STRING;
+ "/"{splice}"*" YY_OBS_GROW; c_context = YY_START; BEGIN SC_COMMENT;
+ "/"{splice}"/" YY_OBS_GROW; c_context = YY_START; BEGIN SC_LINE_COMMENT;
}
@@ -598,31 +517,24 @@ splice (\\[ \f\t\v]*\n)*
braces_level--;
if (braces_level < 0)
{
- yy_pop_state ();
YY_OBS_FINISH;
yylval->string = last_string;
rule_length++;
+ BEGIN INITIAL;
return BRACED_CODE;
}
}
+ /* Tokenize `<<%' correctly (as `<<' `%') rather than incorrrectly
+ (as `<' `<%'). */
+ "<"{splice}"<" YY_OBS_GROW;
+
"$"("<"{tag}">")?(-?[0-9]+|"$") { handle_dollar (current_braced_code,
yytext, *yylloc); }
"@"(-?[0-9]+|"$") { handle_at (current_braced_code,
yytext, *yylloc); }
- /* `"<"{splice}"<"' tokenizes `<<%' correctly (as `<<' `%') rather
- than incorrrectly (as `<' `<%'). */
- [^\"$%\'/<@\[\]\{\}]+|[$%/<@]|"<"{splice}"<" YY_OBS_GROW;
-
- <<EOF>> {
- complain_at (*yylloc, _("unexpected end of file in `{ ... }'"));
- yy_pop_state ();
- YY_OBS_FINISH;
- yylval->string = last_string;
- return BRACED_CODE;
- }
-
+ <<EOF>> unexpected_end_of_file (*yylloc, "}");
}
@@ -633,22 +545,13 @@ splice (\\[ \f\t\v]*\n)*
<SC_PROLOGUE>
{
"%}" {
- yy_pop_state ();
YY_OBS_FINISH;
yylval->string = last_string;
+ BEGIN INITIAL;
return PROLOGUE;
}
- [^%\[\]/\'\"]+ YY_OBS_GROW;
- "%" YY_OBS_GROW;
-
- <<EOF>> {
- complain_at (*yylloc, _("unexpected end of file in `%%{ ... %%}'"));
- yy_pop_state ();
- YY_OBS_FINISH;
- yylval->string = last_string;
- return PROLOGUE;
- }
+ <<EOF>> unexpected_end_of_file (*yylloc, "%}");
}
@@ -659,17 +562,28 @@ splice (\\[ \f\t\v]*\n)*
<SC_EPILOGUE>
{
- [^\[\]]+ YY_OBS_GROW;
-
<<EOF>> {
- yy_pop_state ();
YY_OBS_FINISH;
yylval->string = last_string;
+ BEGIN INITIAL;
return EPILOGUE;
}
}
+ /*----------------------------------------------------------------.
+ | By default, grow the string obstack with the input, escaping M4 |
+ | quoting characters. |
+ `----------------------------------------------------------------*/
+
+<SC_COMMENT,SC_LINE_COMMENT,SC_STRING,SC_CHARACTER,SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>
+{
+ \[ obstack_sgrow (&string_obstack, "@<:@");
+ \] obstack_sgrow (&string_obstack, "@:>@");
+ .|\n YY_OBS_GROW;
+}
+
+
%%
/*------------------------------------------------------------------.
@@ -871,7 +785,7 @@ convert_ucn_to_byte (char const *ucn)
/* A non-ASCII host. Use CODE to index into a table of the C
basic execution character set, which is guaranteed to exist on
all Standard C platforms. This table also includes '$', '@',
- and '`', which not in the basic execution character set but
+ and '`', which are not in the basic execution character set but
which are unibyte characters on all the platforms that we know
about. */
static signed char const table[] =
@@ -919,6 +833,24 @@ handle_syncline (char *args, location_t
location->file = infile;
location->last_line = lineno;
}
+
+
+/*-------------------------------------------------------------.
+| Report an unexpected end of file at LOC. An end of file was |
+| encountered and the expected TOKEN_END was missing. After |
+| reporting the problem, pretend that TOKEN_END was found. |
+`-------------------------------------------------------------*/
+
+static void
+unexpected_end_of_file (location_t loc, char const *token_end)
+{
+ size_t i;
+
+ complain_at (loc, _("missing `%s' at end of file"), token_end);
+ for (i = strlen (token_end); i != 0; i--)
+ unput (token_end[i - 1]);
+}
+
/*-------------------------.
| Initialize the scanner. |
[Prev in Thread] |
Current Thread |
[Next in Thread] |
- miscellaneous cleanups for Bison scanner and parser,
Paul Eggert <=