[PATCH 11/11] glr2.cc: add support for api.token.constructor

[Akim Demaille]

* data/skeletons/glr2.cc: Add support for api.token.constructor.
* examples/c++/glr/c++-types.yy: Use it.
* examples/c++/glr/c++-types.test: Adjust expectations for error
messages.
---
 TODO                            |   5 --
 data/skeletons/glr2.cc          |  25 +++++--
 examples/c++/glr/c++-types.test |   2 +-
 examples/c++/glr/c++-types.yy   | 120 +++++++++++++++++---------------
 4 files changed, 86 insertions(+), 66 deletions(-)

diff --git a/TODO b/TODO
index 33501acf..0fae0db5 100644
--- a/TODO
+++ b/TODO
@@ -28,11 +28,6 @@ Discourage the use of YYDEBUG in C++ (see thread with Jot).  
Stop supporting
 
 Add value_type as a synonym for semantic_type.
 
-** Asymmetries
-Why are yylval and yylloc treated differently?
-
-    yystack.yyglrShift (create_state_set_index(0), 0, 0, yylval, &yylloc);
-
 ** yyerrok in Java
 And add tests in calc.at, to prepare work for D.
 
diff --git a/data/skeletons/glr2.cc b/data/skeletons/glr2.cc
index bdbdfb06..19c0550c 100644
--- a/data/skeletons/glr2.cc
+++ b/data/skeletons/glr2.cc
@@ -332,6 +332,7 @@ const std::ptrdiff_t strong_index_alias<T>::INVALID_INDEX =
     /// YYSYMBOL.  No bounds checking.
     static std::string symbol_name (symbol_kind_type yysymbol);]])[
 
+]b4_token_constructor_define[
 # if ]b4_api_PREFIX[DEBUG
   public:
     /// \brief Report a symbol value on the debug stream.
@@ -385,6 +386,8 @@ const std::ptrdiff_t strong_index_alias<T>::INVALID_INDEX =
 ]b4_parse_param_vars[
   };
 
+]b4_token_ctor_if([b4_yytranslate_define([$1])[
+]b4_public_types_define([$1])])[
 ]b4_namespace_close[
 
 ]b4_percent_code_get([[provides]])[
@@ -2911,7 +2914,18 @@ yygetToken (]b4_namespace_ref[::]b4_parser_class[& 
yyparser, glr_stack& yystack]
       try
         {
 #endif // YY_EXCEPTIONS
-          yychar = ]b4_lex[;
+          {]b4_token_ctor_if([[
+            typedef ]b4_namespace_ref[::]b4_parser_class[::symbol_type 
symbol_type;
+            typedef ]b4_namespace_ref[::]b4_parser_class[::symbol_kind 
symbol_kind;
+            symbol_type yylookahead = ]b4_lex[;
+            yystack.yytoken = yylookahead.kind ();]b4_variant_if([[
+            ]b4_symbol_variant([yystack.yytoken],
+                               [yystack.yylval], [move], 
[yylookahead.value])], [[
+            yystack.yylval = yylookahead.value;]])[]b4_locations_if([
+            yystack.yylloc = yylookahead.location;
+            yylookahead.kind_ = symbol_kind::S_YYEMPTY;])[]], [[
+            yychar = ]b4_lex[;]])[
+          }
 #if YY_EXCEPTIONS
         }
       catch (const ]b4_namespace_ref[::]b4_parser_class[::syntax_error& yyexc)
@@ -2920,12 +2934,13 @@ yygetToken (]b4_namespace_ref[::]b4_parser_class[& 
yyparser, glr_stack& yystack]
           yystack.yylloc = yyexc.location;])[
           yyparser.error (]b4_locations_if([yystack.yylloc, ])[yyexc.what ());
           // Map errors caught in the scanner to the error token, so that error
-          // handling is started.
-          yychar = 
]b4_namespace_ref[::]b4_parser_class[::token::]b4_symbol(error, id)[;
+          // handling is started.]b4_token_ctor_if([[
+          yystack.yytoken = 
]b4_namespace_ref[::]b4_parser_class[::]b4_symbol(error, kind)[;]], [[
+          yychar = 
]b4_namespace_ref[::]b4_parser_class[::token::]b4_symbol(error, id)[;]])[
         }
-#endif // YY_EXCEPTIONS
+#endif // YY_EXCEPTIONS]b4_token_ctor_if([], [[
       yystack.yytoken
-        = ]b4_namespace_ref[::]b4_parser_class[::yytranslate_ (yychar);
+        = ]b4_namespace_ref[::]b4_parser_class[::yytranslate_ (yychar);]])[
     }
   if (yystack.yytoken == 
]b4_namespace_ref[::]b4_parser_class[::]b4_symbol(eof, kind)[)
     YYCDEBUG << "Now at end of input.\n";
diff --git a/examples/c++/glr/c++-types.test b/examples/c++/glr/c++-types.test
index 6a00cd8c..9a3f7506 100644
--- a/examples/c++/glr/c++-types.test
+++ b/examples/c++/glr/c++-types.test
@@ -47,4 +47,4 @@ run 0 "\
 5.0-13: <OR>(<init-declare>(T, y, +(z, q)), =(<cast>(y, T), +(z, q)))
 7.0-15: <error>
 9.0-5: +(z, q)
-err: 7.5: syntax error, unexpected identifier, expecting '=' or '+' or ')'"
+err: 7.5: syntax error, unexpected identifier, expecting = or + or )"
diff --git a/examples/c++/glr/c++-types.yy b/examples/c++/glr/c++-types.yy
index 594b356e..c4012a00 100644
--- a/examples/c++/glr/c++-types.yy
+++ b/examples/c++/glr/c++-types.yy
@@ -22,6 +22,7 @@
 %glr-parser
 %skeleton "glr2.cc"
 %define parse.assert
+%define api.token.constructor
 %header
 %locations
 %debug
@@ -46,8 +47,8 @@
   static Node
   stmtMerge (const Node& x0, const Node& x1);
 
-  static int
-  yylex (yy::parser::value_type* val, yy::parser::location_type* loc);
+  static yy::parser::symbol_type
+  yylex ();
 }
 
 %expect-rr 1
@@ -56,11 +57,16 @@
 %printer { yyo << $$; } <Node>
 
 %token
-  TYPENAME "typename"
-  ID "identifier"
+  TYPENAME  "typename"
+  ID        "identifier"
+  SEMICOLON ";"
+  EQUAL     "="
+  PLUS      "+"
+  LPAREN    "("
+  RPAREN    ")"
 
-%right '='
-%left '+'
+%right "="
+%left "+"
 
 %%
 
@@ -68,30 +74,31 @@ prog : %empty
      | prog stmt   { std::cout << @2 << ": " << $2 << '\n'; }
      ;
 
-stmt : expr ';'  %merge <stmtMerge>   { $$ = $1; }
+stmt : expr ";"  %merge <stmtMerge>   { $$ = $1; }
      | decl      %merge <stmtMerge>
-     | error ';'                      { $$ = Nterm ("<error>"); }
+     | error ";"                      { $$ = Nterm ("<error>"); }
      ;
 
 expr : ID
-     | TYPENAME '(' expr ')'  { $$ = Nterm ("<cast>", $3, $1); }
-     | expr '+' expr          { $$ = Nterm ("+", $1, $3); }
-     | expr '=' expr          { $$ = Nterm ("=", $1, $3); }
+     | TYPENAME "(" expr ")"  { $$ = Nterm ("<cast>", $3, $1); }
+     | expr "+" expr          { $$ = Nterm ("+", $1, $3); }
+     | expr "=" expr          { $$ = Nterm ("=", $1, $3); }
      ;
 
-decl : TYPENAME declarator ';'
+decl : TYPENAME declarator ";"
                         { $$ = Nterm ("<declare>", $1, $2); }
-     | TYPENAME declarator '=' expr ';'
+     | TYPENAME declarator "=" expr ";"
                         { $$ = Nterm ("<init-declare>", $1, $2, $4); }
      ;
 
 declarator
      : ID
-     | '(' declarator ')' { $$ = $2; }
+     | "(" declarator ")" { $$ = $2; }
      ;
 
 %%
 std::istream* input = nullptr;
+yy::parser::location_type loc;
 
 // An error reporting function.
 void
@@ -100,61 +107,63 @@ yy::parser::error (const location_type& l, const 
std::string& m)
   std::cerr << l << ": " << m << '\n';
 }
 
-static int
-yylex (yy::parser::value_type* lvalp, yy::parser::location_type* llocp)
+static yy::parser::symbol_type
+yylex ()
 {
-  static int lineNum = 1;
-  static int colNum = 0;
-
   while (true)
     {
+      loc.step ();
+      loc += 1;
       assert (!input->eof ());
       switch (int c = input->get ())
         {
         case EOF:
-          return 0;
+          return yy::parser::make_YYEOF (loc);
         case '\t':
-          colNum = (colNum + 7) & ~7;
+          loc.end.column = (loc.end.column + 7) & ~7;
+          loc.step ();
           break;
         case ' ': case '\f':
-          colNum += 1;
+          loc.step ();
           break;
         case '\n':
-          lineNum += 1;
-          colNum = 0;
+          loc.lines (1);
+          loc.end.column = 0;
+          loc.step ();
           break;
+        case '+':
+          return yy::parser::make_PLUS (loc);
+        case '=':
+          return yy::parser::make_EQUAL (loc);
+        case '(':
+          return yy::parser::make_LPAREN (loc);
+        case ')':
+          return yy::parser::make_RPAREN (loc);
+        case ';':
+          return yy::parser::make_SEMICOLON (loc);
         default:
-          {
-            llocp->begin.line = llocp->end.line = lineNum;
-            llocp->begin.column = colNum;
-            int tok;
-            if (isalpha (c))
-              {
-                std::string form;
-                do
-                  {
-                    form += static_cast<char> (c);
-                    colNum += 1;
-                    c = input->get ();
-                  }
-                while (isalnum (c) || c == '_');
-
-                input->unget ();
-                tok
-                  = isupper (static_cast <unsigned char> (form[0]))
-                  ? yy::parser::token::TYPENAME
-                  : yy::parser::token::ID;
-                lvalp->emplace<Node> (Term (form));
-              }
-            else
-              {
-                colNum += 1;
-                tok = c;
-                lvalp = nullptr;
-              }
-            llocp->end.column = colNum;
-            return tok;
-          }
+          if (isalpha (c))
+            {
+              std::string form;
+              do
+                {
+                  form += static_cast<char> (c);
+                  loc += 1;
+                  c = input->get ();
+                }
+              while (isalnum (c) || c == '_');
+              input->unget ();
+              loc -= 1;
+              if (isupper (static_cast <unsigned char> (form[0])))
+                return yy::parser::make_TYPENAME (Term (form), loc);
+              else
+                return yy::parser::make_ID (Term (form), loc);
+            }
+          else
+            {
+              auto msg = "invalid character: " + std::string(1, 
static_cast<char> (c));
+              throw yy::parser::syntax_error (loc, msg);
+            }
         }
     }
 }
@@ -173,6 +182,7 @@ process (yy::parser& parse, const std::string& file)
     input = &std::cin;
   else
     input = new std::ifstream (file.c_str ());
+  loc.initialize (nullptr, 1, 0);
   int status = parse ();
   if (!is_stdin)
     delete input;
-- 
2.30.0