[PATCH 14/16] c: don't emit an error message when the scanner returns YYERRCODE

[Akim Demaille]

* data/skeletons/yacc.c (yyparse): When the scanner returns YYERRCODE,
go directly to error recovery (yyerrlab1).
However, don't keep the error token as lookahead, that token is too
special.
* data/skeletons/lalr1.cc: Likewise.

* examples/c/bistromathic/parse.y (yylex): Use that feature to report
nicely invalid characters.
* examples/c/bistromathic/bistromathic.test: Check that.

* tests/calc.at: Check that case in C only.
The other case seem to be working, but that's an illusion that the
next commit will address (in fact, they can enter endless loops, and
report the error several times anyway).
---
 data/skeletons/lalr1.cc                   |  10 ++
 data/skeletons/yacc.c                     |  11 ++
 examples/c/bistromathic/bistromathic.test | 122 +++++++++++++++++++++-
 examples/c/bistromathic/parse.y           |  21 ++--
 tests/calc.at                             |  46 ++++++--
 5 files changed, 194 insertions(+), 16 deletions(-)

diff --git a/data/skeletons/lalr1.cc b/data/skeletons/lalr1.cc
index 99c64331..cdf42c6c 100644
--- a/data/skeletons/lalr1.cc
+++ b/data/skeletons/lalr1.cc
@@ -981,6 +981,16 @@ b4_dollar_popdef])[]dnl
       }
     YY_SYMBOL_PRINT ("Next token is", yyla);
 
+    if (yyla.kind () == ]symbol_kind::b4_symbol(1, kind)[)
+    {
+      // The scanner already issued an error message, process directly
+      // to error recovery.  But do not keep the error token as
+      // lookahead, it is too special and may lead us to an endless
+      // loop in error recovery. */
+      yyla.type = ]symbol_kind::b4_symbol(2, kind)[;
+      goto yyerrlab1;
+    }
+
     /* If the proper action on seeing token YYLA.TYPE is to reduce or
        to detect an error, take that action.  */
     yyn += yyla.kind ();
diff --git a/data/skeletons/yacc.c b/data/skeletons/yacc.c
index 1ac8e9bc..f7a12dc7 100644
--- a/data/skeletons/yacc.c
+++ b/data/skeletons/yacc.c
@@ -1770,6 +1770,17 @@ yyread_pushed_token:]])[
       yytoken = ]b4_symbol(0, [kind])[;
       YYDPRINTF ((stderr, "Now at end of input.\n"));
     }
+  else if (yychar == ]b4_symbol(1, [id])[)
+    {
+      /* The scanner already issued an error message, process directly
+         to error recovery.  But do not keep the error token as
+         lookahead, it is too special and may lead us to an endless
+         loop in error recovery. */
+      yychar = ]b4_symbol(2, [id])[;
+      yytoken = ]b4_symbol(1, [kind])[;]b4_locations_if([[
+      yyerror_range[1] = yylloc;]])[
+      goto yyerrlab1;
+    }
   else
     {
       yytoken = YYTRANSLATE (yychar);
diff --git a/examples/c/bistromathic/bistromathic.test 
b/examples/c/bistromathic/bistromathic.test
index 629c32c5..9a5b6d11 100755
--- a/examples/c/bistromathic/bistromathic.test
+++ b/examples/c/bistromathic/bistromathic.test
@@ -82,7 +82,11 @@ run 0 '> 1 / 0
 > ''
 err: 1.1-5: error: division by zero'
 
-# Error recovery.
+
+## ---------------- ##
+## Error recovery.  ##
+## ---------------- ##
+
 cat >input <<EOF
 ((1 ++ 2) ** 3)
 (1 ++ 2) + (3 ** 4)
@@ -96,14 +100,124 @@ err: 1.6: syntax error: expected - or ( or number or 
function or variable before
 err: 2.5: syntax error: expected - or ( or number or function or variable 
before +
 err: 2.16: syntax error: expected - or ( or number or function or variable 
before *'
 
-# This is really stupid: we just discarded % and did not enter error recovery.
+# The rule "( error )" should work even if there are no tokens between "(" and 
")".
+cat >input <<EOF
+()
+EOF
+run 0 '> ()
+666
+> ''
+err: 1.2: syntax error: expected - or ( or number or function or variable 
before )'
+
+
 cat >input <<EOF
 100% + 10
 EOF
 run 0 '> 100% + 10
-110
 > ''
-err: 1.4: error: invalid character'
+err: 1.4: syntax error: invalid character: %'
+
+# Traces.  This allows to check the location of the error.  If we
+# forget to map YYERRCODE to YYUNDEF, error recovery enters an endless
+# loop with this input.
+cat >input <<EOF
+(+_)
+EOF
+run 0 '> (+_)
+666
+> ''
+err: Starting parse
+err: Entering state 0
+err: Stack now 0
+err: Reading a token
+err: Next token is token ( (1.1: )
+err: Shifting token ( (1.1: )
+err: Entering state 2
+err: Stack now 0 2
+err: Return for a new token:
+err: Reading a token
+err: Next token is token + (1.2: )
+err: LAC: initial context established for +
+err: LAC: checking lookahead +: Err
+err: LAC: checking lookahead end of file: Err
+err: LAC: checking lookahead +: Err
+err: LAC: checking lookahead -: S1
+err: LAC: checking lookahead *: Err
+err: LAC: checking lookahead /: Err
+err: LAC: checking lookahead ^: Err
+err: LAC: checking lookahead (: S2
+err: LAC: checking lookahead ): Err
+err: LAC: checking lookahead =: Err
+err: LAC: checking lookahead exit: Err
+err: LAC: checking lookahead number: S4
+err: LAC: checking lookahead function: S5
+err: LAC: checking lookahead variable: S6
+err: LAC: checking lookahead NEG: Err
+err: 1.2: syntax error: expected - or ( or number or function or variable 
before +
+err: LAC: initial context discarded due to error recovery
+err: Shifting token error (1.2: )
+err: Entering state 10
+err: Stack now 0 2 10
+err: Next token is token + (1.2: )
+err: LAC: initial context established for +
+err: LAC: checking lookahead +: Err
+err: Error: discarding token + (1.2: )
+err: Error: popping token error (1.2: )
+err: Stack now 0 2
+err: LAC: initial context discarded due to error recovery
+err: Shifting token error (1.2: )
+err: Entering state 10
+err: Stack now 0 2 10
+err: Return for a new token:
+err: 1.3: syntax error: invalid character: _
+err: Reading a token
+err: Error: popping token error (1.2: )
+err: Stack now 0 2
+err: Shifting token error (1.2-3: )
+err: Entering state 10
+err: Stack now 0 2 10
+err: Next token is token invalid token (1.3: )
+err: LAC: initial context established for invalid token
+err: LAC: checking lookahead invalid token: Always Err
+err: Error: discarding token invalid token (1.3: )
+err: Error: popping token error (1.2-3: )
+err: Stack now 0 2
+err: LAC: initial context discarded due to error recovery
+err: Shifting token error (1.2-3: )
+err: Entering state 10
+err: Stack now 0 2 10
+err: Return for a new token:
+err: Reading a token
+err: Next token is token ) (1.4: )
+err: Shifting token ) (1.4: )
+err: Entering state 20
+err: Stack now 0 2 10 20
+err: Reducing stack by rule 15 (line 151):
+err:    $1 = token ( (1.1: )
+err:    $2 = token error (1.2-3: )
+err:    $3 = token ) (1.4: )
+err: -> $$ = nterm exp (1.1-4: 666)
+err: Entering state 7
+err: Stack now 0 7
+err: Return for a new token:
+err: Reading a token
+err: Now at end of input.
+err: LAC: initial context established for end of file
+err: LAC: checking lookahead end of file: R2 G8 S19
+err: Reducing stack by rule 2 (line 126):
+err:    $1 = nterm exp (1.1-4: 666)
+err: -> $$ = nterm input (1.1-4: )
+err: Entering state 8
+err: Stack now 0 8
+err: Now at end of input.
+err: Shifting token end of file (1.5: )
+err: LAC: initial context discarded due to shift
+err: Entering state 19
+err: Stack now 0 8 19
+err: Stack now 0 8 19
+err: Cleanup: popping token end of file (1.5: )
+err: Cleanup: popping nterm input (1.1-4: )' -p
+
 
 
 ## ------------ ##
diff --git a/examples/c/bistromathic/parse.y b/examples/c/bistromathic/parse.y
index 5d422651..aee11270 100644
--- a/examples/c/bistromathic/parse.y
+++ b/examples/c/bistromathic/parse.y
@@ -3,6 +3,7 @@
 %code top {
   #include <ctype.h>  // isdigit
   #include <math.h>   // cos, sin, etc.
+  #include <stdarg.h> // va_start
   #include <stdio.h>  // printf
   #include <stdlib.h> // calloc
   #include <string.h> // strcmp
@@ -46,7 +47,8 @@
 
 %code provides {
   int yylex (const char **line, YYSTYPE *yylval, YYLTYPE *yylloc);
-  void yyerror (YYLTYPE *yylloc, char const *msg);
+  void yyerror (YYLTYPE *loc, char const *format, ...)
+    __attribute__ ((__format__ (__printf__, 2, 3)));
 }
 
 %code {
@@ -249,6 +251,8 @@ yylex (const char **line, YYSTYPE *yylval, YYLTYPE *yylloc)
     case '(': return TOK_LPAREN;
     case ')': return TOK_RPAREN;
 
+    case '!': return TOK_YYUNDEF;
+
     case '\0': return TOK_YYEOF;
 
       // Numbers.
@@ -290,8 +294,8 @@ yylex (const char **line, YYSTYPE *yylval, YYLTYPE *yylloc)
 
       // Stray characters.
     default:
-      yyerror (yylloc, "error: invalid character");
-      return yylex (line, yylval, yylloc);
+      yyerror (yylloc, "syntax error: invalid character: %c", c);
+      return TOK_YYERRCODE;
     }
 }
 
@@ -371,10 +375,15 @@ yyreport_syntax_error (const yypcontext_t *ctx)
 
 
 // Called by yyparse on error.
-void yyerror (YYLTYPE *loc, char const *msg)
+void yyerror (YYLTYPE *loc, char const *format, ...)
 {
   YY_LOCATION_PRINT (stderr, *loc);
-  fprintf (stderr, ": %s\n", msg);
+  fputs (": ", stderr);
+  va_list args;
+  va_start (args, format);
+  vfprintf (stderr, format, args);
+  va_end (args);
+  putc ('\n', stderr);
 }
 
 
@@ -535,7 +544,7 @@ int main (int argc, char const* argv[])
 #endif
 
   // Enable parse traces on option -p.
-  if (argc == 2 && strcmp (argv[1], "-p") == 0)
+  if (1 < argc && strcmp (argv[1], "-p") == 0)
     yydebug = 1;
   init_table ();
   init_readline ();
diff --git a/tests/calc.at b/tests/calc.at
index 992b911c..a3711612 100644
--- a/tests/calc.at
+++ b/tests/calc.at
@@ -201,8 +201,8 @@ read_integer (]AT_YYLEX_FORMALS[)
   /* Skip white spaces.  */
   do
     {
-]AT_LOCATION_IF(
-[     AT_LOC_FIRST_COLUMN = AT_LOC_LAST_COLUMN;
+]AT_LOCATION_IF([
+      AT_LOC_FIRST_COLUMN = AT_LOC_LAST_COLUMN;
       AT_LOC_FIRST_LINE   = AT_LOC_LAST_LINE;
 ])[
     }
@@ -220,6 +220,15 @@ read_integer (]AT_YYLEX_FORMALS[)
   if (c == EOF)
     return ]AT_TOKEN_PREFIX[CALC_EOF;
 
+  /* An explicit error raised by the scanner. */
+  if (c == '#')
+    {]AT_LOCATION_IF([
+      fprintf (stderr, "%d.%d: ",
+               AT_LOC_FIRST_LINE, AT_LOC_FIRST_COLUMN);])[
+      fputs ("syntax error: invalid character: '#'\n", stderr);
+      return ]AT_TOKEN_PREFIX[]AT_API_PREFIX[ERRCODE;
+    }
+
   /* Return single chars. */
   return c;
 }
@@ -304,7 +313,7 @@ class CalcLexer(R) : Lexer
         location.end.column += 1;]])[
       }
 
-    // Handle EOF.
+    // EOF.
     if (input.empty)
       return TokenKind.CALC_EOF;
 
@@ -325,6 +334,14 @@ class CalcLexer(R) : Lexer
     else
       location.end.column += 1;]])[
     input.popFront;
+
+    // An explicit error raised by the scanner. */
+    if (c == '#')
+      {
+        stderr.writeln (]AT_LOCATION_IF([location, ": ", ])["syntax error: 
invalid character: '#'");
+        return TokenKind.YYERRCODE;
+      }
+
     return c;
   }
 }
@@ -370,9 +387,9 @@ m4_define([AT_CALC_YYLEX(java)],
 
   public int yylex () throws IOException {;]AT_LOCATION_IF([[
     start.set (reader.getPosition ());]])[
-    int ttype = st.nextToken ();]AT_LOCATION_IF([[
+    int tkind = st.nextToken ();]AT_LOCATION_IF([[
     end.set (reader.getPosition ());]])[
-    switch (ttype)
+    switch (tkind)
       {
       case StreamTokenizer.TT_EOF:
         return EOF;
@@ -386,8 +403,11 @@ m4_define([AT_CALC_YYLEX(java)],
         return NUM;
       case ' ': case '\t':
         return yylex ();
+      case '#':
+        System.err.println(]AT_LOCATION_IF([[start + ": " + ]])["syntax error: 
invalid character: '#'");
+        return YYERRCODE;
       default:
-        return ttype;
+        return tkind;
       }
   }
 ]AT_LEXPARAM_IF([], [[}]])[
@@ -1004,6 +1024,20 @@ _AT_CHECK_CALC_ERROR([$1], [0], [(* *) + (*) + (*)],
 ]AT_JAVA_IF([1.10-1.11], [1.10])[: syntax error on token ['*'] (expected: 
[number] ['-'] ['('] ['!'])
 ]AT_JAVA_IF([1.16-1.17], [1.16])[: syntax error on token ['*'] (expected: 
[number] ['-'] ['('] ['!'])]])
 
+
+# YYERRCODE.
+# ---------
+# Check that returning YYERRCODE from the scanner properly enters
+# error-recovery without issuing a second error message.
+
+_AT_CHECK_CALC_ERROR([$1], [0], [(#) + (#) = 2222],
+                     [[final: 2222 0 0]],
+                     [102],
+[[1.2: syntax error: invalid character: '#'
+1.8: syntax error: invalid character: '#']])
+
+
+
 AT_BISON_OPTION_POPDEFS
 
 AT_CLEANUP
-- 
2.26.2