[PATCH 04/10] api.token.raw: apply to the other skeletons

[Akim Demaille]

* data/skeletons/c++.m4, data/skeletons/glr.c,
* data/skeletons/lalr1.c, data/skeletons/lalr1.java:
Add support for api.token.raw.

* tests/scanner.at: Check them.
---
 data/skeletons/c++.m4     |   6 +-
 data/skeletons/glr.c      |   6 +-
 data/skeletons/lalr1.d    |  21 +++---
 data/skeletons/lalr1.java |  13 +++-
 tests/local.at            |   6 +-
 tests/scanner.at          | 155 ++++++++++++++++++++++++++++++--------
 6 files changed, 155 insertions(+), 52 deletions(-)

diff --git a/data/skeletons/c++.m4 b/data/skeletons/c++.m4
index 2d05699b..1971c7df 100644
--- a/data/skeletons/c++.m4
+++ b/data/skeletons/c++.m4
@@ -532,7 +532,9 @@ m4_define([b4_yytranslate_define],
   ]b4_parser_class[::yytranslate_ (]b4_token_ctor_if([token_type],
                                                           [int])[ t)
   {
-    // YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to
+]b4_api_token_raw_if(
+[[    return static_cast<yy::parser::token_number_type> (t);]],
+[[    // YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to
     // TOKEN-NUM as returned by yylex.
     static
     const token_number_type
@@ -548,7 +550,7 @@ m4_define([b4_yytranslate_define],
     else if (static_cast<unsigned> (t) <= user_token_number_max_)
       return translate_table[t];
     else
-      return undef_token_;
+      return undef_token_;]])[
   }
 ]])
 
diff --git a/data/skeletons/glr.c b/data/skeletons/glr.c
index 55cf3830..d9ab37d5 100644
--- a/data/skeletons/glr.c
+++ b/data/skeletons/glr.c
@@ -346,7 +346,9 @@ static YYLTYPE yyloc_default][]b4_yyloc_default;])[
 
 /* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM
    as returned by yylex, with out-of-bounds checking.  */
-#define YYTRANSLATE(YYX)                                                \
+]b4_api_token_raw_if(dnl
+[[#define YYTRANSLATE(YYX) (YYX)]],
+[[#define YYTRANSLATE(YYX)                         \
   ((unsigned) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
 
 /* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM
@@ -354,7 +356,7 @@ static YYLTYPE yyloc_default][]b4_yyloc_default;])[
 static const ]b4_int_type_for([b4_translate])[ yytranslate[] =
 {
   ]b4_translate[
-};
+};]])[
 
 #if ]b4_api_PREFIX[DEBUG
 /* YYRLINE[YYN] -- source line where rule number YYN was defined.  */
diff --git a/data/skeletons/lalr1.d b/data/skeletons/lalr1.d
index dc155ef4..fd19e852 100644
--- a/data/skeletons/lalr1.d
+++ b/data/skeletons/lalr1.d
@@ -822,18 +822,21 @@ m4_popdef([b4_at_dollar])])dnl
   }
 ]])[
 
-  /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX.  */
-  private static immutable ]b4_int_type_for([b4_translate])[[] 
yytranslate_table_ =
-  @{
-  ]b4_translate[
-  @};
-
   private static ]b4_int_type_for([b4_translate])[ yytranslate_ (int t)
   {
-    if (t >= 0 && t <= yyuser_token_number_max_)
-      return yytranslate_table_[t];
+]b4_api_token_raw_if(
+[[    import std.conv : to;
+    return to!byte (t);]],
+[[    /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX.  */
+    immutable ]b4_int_type_for([b4_translate])[[] translate_table =
+    @{
+  ]b4_translate[
+    @};
+
+    if (0 <= t && t <= yyuser_token_number_max_)
+      return translate_table[t];
     else
-      return yyundef_token_;
+      return yyundef_token_;]])[
   }
 
   private static immutable int yylast_ = ]b4_last[;
diff --git a/data/skeletons/lalr1.java b/data/skeletons/lalr1.java
index d0778bdf..af120404 100644
--- a/data/skeletons/lalr1.java
+++ b/data/skeletons/lalr1.java
@@ -1017,17 +1017,22 @@ b4_dollar_popdef[]dnl
                      b4_rhs_location(yynrhs, yyi + 1)])[);
   }]])[
 
-  /* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM
+  /* YYTRANSLATE_(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM
      as returned by yylex, with out-of-bounds checking.  */
-  ]b4_integral_parser_table_define([translate_table], [b4_translate])[
-
   private static final ]b4_int_type_for([b4_translate])[ yytranslate_ (int t)
-  {
+]b4_api_token_raw_if(dnl
+[[  {
+    return t;
+  }
+]],
+[[  {
     if (0 <= t && t <= yyuser_token_number_max_)
       return yytranslate_table_[t];
     else
       return yyundef_token_;
   }
+  ]b4_integral_parser_table_define([translate_table], [b4_translate])[
+]])[
 
   private static final int yylast_ = ]b4_last[;
   private static final int yynnts_ = ]b4_nterms_number[;
diff --git a/tests/local.at b/tests/local.at
index e70da61e..de396496 100644
--- a/tests/local.at
+++ b/tests/local.at
@@ -684,9 +684,9 @@ public void yyerror (]AT_LOCATION_IF([[YYLocation l, 
]])[string m)
 m4_define([AT_MAIN_DEFINE(d)],
 [[int main ()
 {
-  Lexer l = new Lexer ();
-  Parser p = new Parser (l);
-  p.parse ();
+  auto l = new ]AT_API_prefix[Lexer ();
+  auto p = new ]AT_API_PREFIX[Parser (l);
+  return p.parse ();
 }]])
 
 
diff --git a/tests/scanner.at b/tests/scanner.at
index a3a8b479..543d5cfc 100644
--- a/tests/scanner.at
+++ b/tests/scanner.at
@@ -18,6 +18,115 @@
 AT_BANNER([[Interface with the scanner.]])
 
 
+# -------------- #
+# AT_RAW_YYLEX.  #
+# -------------- #
+
+m4_pushdef([AT_RAW_YYLEX],   [AT_LANG_DISPATCH([$0], $@)])
+
+m4_define([AT_RAW_YYLEX(c)],
+[#include <stdlib.h> /* abort */
+AT_YYLEX_PROTOTYPE[
+{
+  static const char* input = "0-(1+2)*3/9";
+  int c = *input++;
+  switch (c)
+    {
+    case '0':
+    case '1':
+    case '2':
+    case '3':
+    case '4':
+    case '5':
+    case '6':
+    case '7':
+    case '8':
+    case '9':
+      ]AT_VAL[.val = c - '0';
+      return ]AT_CXX_IF([yy::parser::token::])[NUM;
+    case '+': return ]AT_CXX_IF([yy::parser::token::])[PLUS;
+    case '-': return ]AT_CXX_IF([yy::parser::token::])[MINUS;
+    case '*': return ]AT_CXX_IF([yy::parser::token::])[STAR;
+    case '/': return ]AT_CXX_IF([yy::parser::token::])[SLASH;
+    case '(': return ]AT_CXX_IF([yy::parser::token::])[LPAR;
+    case ')': return ]AT_CXX_IF([yy::parser::token::])[RPAR;
+    case 0: return 0;
+    }
+  abort ();
+}
+]])
+
+m4_copy([AT_RAW_YYLEX(c)], [AT_RAW_YYLEX(c++)])
+
+m4_define([AT_RAW_YYLEX(d)],
+[[import std.range.primitives;
+import std.stdio;
+
+auto yyLexer(R)(R range)
+  if (isInputRange!R && is (ElementType!R : dchar))
+{
+  return new YYLexer!R(range);
+}
+
+auto yyLexer ()
+{
+  return yyLexer("0-(1+2)*3/9");
+}
+
+class YYLexer(R) : Lexer
+  if (isInputRange!R && is (ElementType!R : dchar))
+{
+  R input;
+
+  this(R r) {
+    input = r;
+  }
+
+  ]AT_YYERROR_DEFINE[
+
+  YYSemanticType semanticVal_;
+  public final @property YYSemanticType semanticVal ()
+  {
+    return semanticVal_;
+  }
+
+  int yylex ()
+  {
+    import std.uni : isNumber;
+    // Handle EOF.
+    if (input.empty)
+      return YYTokenType.EOF;
+
+    auto c = input.front;
+    input.popFront;
+
+    // Numbers.
+    switch (c)
+    {
+    case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+      semanticVal_.val = c - '0';
+      return YYTokenType.NUM;
+    case '+': return YYTokenType.PLUS;
+    case '-': return YYTokenType.MINUS;
+    case '*': return YYTokenType.STAR;
+    case '/': return YYTokenType.SLASH;
+    case '(': return YYTokenType.LPAR;
+    case ')': return YYTokenType.RPAR;
+    default: assert(0);
+    }
+  }
+}
+]])
+
+m4_pushdef([AT_MAIN_DEFINE(d)],
+[[int main ()
+{
+  auto l = yyLexer ();
+  auto p = new YYParser (l);
+  return !p.parse ();
+}]])
+
+
 ## ------------------- ##
 ## Raw token numbers.  ##
 ## ------------------- ##
@@ -30,12 +139,13 @@ AT_BISON_OPTION_PUSHDEFS([%debug $1])
 AT_DATA_GRAMMAR([[input.y]],
 [[$1
 %debug
+]AT_D_IF([], [[
 %code
 {
 #include <stdio.h>
 ]AT_YYERROR_DECLARE[
 ]AT_YYLEX_DECLARE[
-}
+}]])[
 
 %union {
   int val;
@@ -70,40 +180,18 @@ exp
 
 %%
 ]AT_YYERROR_DEFINE[
+]AT_RAW_YYLEX[
 ]AT_MAIN_DEFINE[
-
-int yylex (void)
-{
-  static const char* input = "0-(1+2)*3/9";
-  int c = *input++;
-  switch (c)
-    {
-    case '0':
-    case '1':
-    case '2':
-    case '3':
-    case '4':
-    case '5':
-    case '6':
-    case '7':
-    case '8':
-    case '9':
-      yylval.val = c - '0';
-      return NUM;
-    case '+': return PLUS;
-    case '-': return MINUS;
-    case '*': return STAR;
-    case '/': return SLASH;
-    case '(': return LPAR;
-    case ')': return RPAR;
-    case 0: return 0;
-    }
-}
 ]])
 
 AT_FULL_COMPILE([input])
 
-AT_CHECK([grep -c yytranslate input.c], [ignore], [AT_TOKEN_RAW_IF([0], [2])[
+# yacc.c, glr.c and glr.cc use 'yytranslate' (and YYTRANSLATE).
+# lalr1.cc uses 'translate_table' (and yytranslate_).
+# lalr1.d uses 'byte[] translate_table =' (and yytranslate_).
+AT_CHECK([[grep -Ec 'yytranslate\[\]|translate_table\[\]|translate_table =' 
input.]AT_LANG_EXT],
+         [ignore],
+         [AT_TOKEN_RAW_IF([0], [1])[
 ]])
 
 AT_PARSER_CHECK([input], 0,
@@ -114,7 +202,10 @@ AT_BISON_OPTION_POPDEFS
 AT_CLEANUP
 ])
 
-AT_TEST([])
-AT_TEST([%define api.token.raw])
+m4_foreach([b4_skel], [[yacc.c], [glr.c], [lalr1.cc], [glr.cc], [lalr1.d]],
+[AT_TEST([%skeleton "]b4_skel["])
+ AT_TEST([%skeleton "]b4_skel[" %define api.token.raw])])
+
 
+m4_popdef([AT_MAIN_DEFINE(d)])
 m4_popdef([AT_TEST])
-- 
2.23.0