Re: Proposed interface changes for "parse.error custom"

[Akim Demaille]

Hi Adrian, hi all,

> Le 12 mars 2020 à 06:50, Akim Demaille <address@hidden> a écrit :
> 
> Yes, there's no question about that: in push parsers, yyparse_context_t 
> _must_ embed yypstate.

I'm installing this.  We should now get rid of yysyntax_error_arguments.

commit 44ac18d13681e05f38626427783bb5aad7d08020
Author: Akim Demaille <address@hidden>
Date:   Sun Mar 8 17:31:28 2020 +0100

    yacc.c: yypstate_expected_tokens
    
    In push parsers, when asking for the list of expected tokens at some
    point, it makes no sense to build a yyparse_context_t: the yypstate
    alone suffices (the only difference being the lookahead).  Instead of
    forcing the user to build a useless shell around yypstate, let's offer
    yypstate_expected_tokens.
    
    See https://lists.gnu.org/r/bison-patches/2020-03/msg00025.html.
    
    * data/skeletons/yacc.c (yypstate): Declare earlier, so that we can
    use it for...
    (yypstate_expected_tokens): this new function, when in push parsers.
    Adjust dependencies.
    * examples/c/bistromathic/parse.y: Simplify: use
    yypstate_expected_tokens.
    Style fixes.
    Reduce scopes (reported by Joel E. Denny).

diff --git a/NEWS b/NEWS
index 7ee66c07..8287f735 100644
--- a/NEWS
+++ b/NEWS
@@ -75,6 +75,16 @@ GNU Bison NEWS
   the translated symbol (i.e., it returns '_("variable")' rather that
   '"variable"').
 
+*** List of expected tokens (yacc.c)
+
+  At any point during parsing (including even before submitting the first
+  token), push parsers may now invoke yypstate_expected_tokens to get the
+  list of possible tokens.  This feature can be used to propose
+  autocompletion (see below the "bistromathic" example).
+
+  It makes little sense to use this feature without enabling LAC (lookahead
+  correction).
+
 *** Modernize display of explanatory statements in diagnostics
 
   Since Bison 2.7, output was indented four spaces for explanatory
@@ -106,9 +116,9 @@ GNU Bison NEWS
 
   A new C example, bistromathic, is a fully featured interactive calculator
   using many Bison features: pure interface, push parser, autocompletion
-  based on the current parser state (using yyexpected_tokens), location
-  tracking, internationalized custom error messages, lookahead-correction,
-  rich debug traces, etc.
+  based on the current parser state (using yypstate_expected_tokens),
+  location tracking, internationalized custom error messages, lookahead
+  correction, rich debug traces, etc.
 
 * Noteworthy changes in release 3.5.3 (2020-03-08) [stable]
 
diff --git a/data/skeletons/yacc.c b/data/skeletons/yacc.c
index 7f92a200..5a2b17ce 100644
--- a/data/skeletons/yacc.c
+++ b/data/skeletons/yacc.c
@@ -856,7 +856,17 @@ int yydebug;
 
 #ifndef YYMAXDEPTH
 # define YYMAXDEPTH ]b4_stack_depth_max[
-#endif]b4_lac_if([[
+#endif]b4_push_if([[
+/* Parser data structure.  */
+struct yypstate
+  {]b4_declare_parser_state_variables[
+    /* Whether this instance has not started parsing yet.  */
+    int yynew;
+  };]b4_pure_if([], [[
+
+/* Whether the only allowed instance of yypstate is allocated.  */
+static char yypstate_allocated = 0;]])])[
+]b4_lac_if([[
 
 /* Given a state stack such that *YYBOTTOM is its bottom, such that
    *YYTOP is either its top or is YYTOP_EMPTY to indicate an empty
@@ -1108,23 +1118,28 @@ yy_lac (yy_state_t *yyesa, yy_state_t **yyes,
 }]])[
 
 ]b4_parse_error_case([simple], [],
-[[typedef struct
-{
-  yy_state_t *yyssp;
-  int yytoken;]b4_locations_if([[
-  YYLTYPE *yylloc;]])[]b4_lac_if([[
+[[/* Context of a parse error.  */
+typedef struct
+{]b4_push_if([[
+  yypstate* yyps;]], [[
+  yy_state_t *yyssp;]b4_lac_if([[
   yy_state_t *yyesa;
   yy_state_t **yyes;
-  YYPTRDIFF_T *yyes_capacity;]])[
+  YYPTRDIFF_T *yyes_capacity;]])])[
+  int yytoken;]b4_locations_if([[
+  YYLTYPE *yylloc;]])[
 } yyparse_context_t;
 
 /* Put in YYARG at most YYARGN of the expected tokens given the
    current YYCTX, and return the number of tokens stored in YYARG.  If
    YYARG is null, return the number of expected tokens (guaranteed to
-   be less than YYNTOKENS).  */
+   be less than YYNTOKENS).  */]b4_push_if([[
+static int
+yypstate_expected_tokens (yypstate *yyps,
+                          int yyarg[], int yyargn)]], [[
 static int
 yyexpected_tokens (const yyparse_context_t *yyctx,
-                   int yyarg[], int yyargn)
+                   int yyarg[], int yyargn)]])[
 {
   /* Actual size of YYARG. */
   int yycount = 0;
@@ -1134,8 +1149,9 @@ yyexpected_tokens (const yyparse_context_t *yyctx,
     if (yyx != YYTERROR && yyx != YYUNDEFTOK)
       {
         {
-          int yy_lac_status = yy_lac (yyctx->yyesa, yyctx->yyes, 
yyctx->yyes_capacity,
-                                      yyctx->yyssp, yyx);
+          int yy_lac_status
+            = yy_lac (]b4_push_if([[yyps->yyesa, &yyps->yyes, 
&yyps->yyes_capacity, yyps->yyssp, yyx]],
+                                  [[yyctx->yyesa, yyctx->yyes, 
yyctx->yyes_capacity, yyctx->yyssp, yyx]])[);
           if (yy_lac_status == 2)
             return -2;
           if (yy_lac_status == 1)
@@ -1148,7 +1164,7 @@ yyexpected_tokens (const yyparse_context_t *yyctx,
         else
           yyarg[yycount++] = yyx;
       }]],
-[[  int yyn = yypact[+*yyctx->yyssp];
+[[  int yyn = yypact@{+*]b4_push_if([yyps], [yyctx])[->yyssp@};
   if (!yypact_value_is_default (yyn))
     {
       /* Start YYX at -YYN if negative to avoid negative indexes in
@@ -1174,6 +1190,15 @@ yyexpected_tokens (const yyparse_context_t *yyctx,
   return yycount;
 }
 
+]b4_push_if([[
+/* Similar to the previous function.  */
+static int
+yyexpected_tokens (const yyparse_context_t *yyctx,
+                   int yyarg[], int yyargn)
+{
+  return yypstate_expected_tokens (yyctx->yyps, yyarg, yyargn);
+}]])[
+
 static int
 yysyntax_error_arguments (const yyparse_context_t *yyctx,
                           int yyarg[], int yyargn)
@@ -1430,14 +1455,7 @@ yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg,
 
 ]b4_pure_if([], [b4_declare_scanner_communication_variables])[
 
-]b4_push_if([[
-struct yypstate
-  {]b4_declare_parser_state_variables[
-    /* Whether this instance has not started parsing yet.  */
-    int yynew;
-  };]b4_pure_if([], [[
-
-static char yypstate_allocated = 0;]])b4_pull_if([[
+]b4_push_if([b4_pull_if([[
 
 int
 yyparse (]m4_ifset([b4_parse_param], [b4_formals(b4_parse_param)], [void])[)
@@ -1873,7 +1891,7 @@ yyerrlab:
          [custom],
 [[      {
         yyparse_context_t yyctx
-          = {yyssp, yytoken]b4_locations_if([[, &yylloc]])[]b4_lac_if([[, 
yyesa, &yyes, &yyes_capacity]])[};]b4_lac_if([[
+          = {]b4_push_if([[yyps]], [[yyssp]b4_lac_if([[, yyesa, &yyes, 
&yyes_capacity]])])[, yytoken]b4_locations_if([[, &yylloc]])[};]b4_lac_if([[
         if (yychar != YYEMPTY)
           YY_LAC_ESTABLISH;]])[
         if (yyreport_syntax_error (&yyctx]m4_ifset([b4_parse_param],
@@ -1883,9 +1901,9 @@ yyerrlab:
          [simple],
 [[      yyerror (]b4_yyerror_args[YY_("syntax error"));]],
 [[      {
-        char const *yymsgp = YY_("syntax error");
         yyparse_context_t yyctx
-          = {yyssp, yytoken]b4_locations_if([[, &yylloc]])[]b4_lac_if([[, 
yyesa, &yyes, &yyes_capacity]])[};
+          = {]b4_push_if([[yyps]], [[yyssp]b4_lac_if([[, yyesa, &yyes, 
&yyes_capacity]])])[, yytoken]b4_locations_if([[, &yylloc]])[};
+        char const *yymsgp = YY_("syntax error");
         int yysyntax_error_status;]b4_lac_if([[
         if (yychar != YYEMPTY)
           YY_LAC_ESTABLISH;]])[
diff --git a/examples/c/bistromathic/parse.y b/examples/c/bistromathic/parse.y
index b91fd094..45bcfd62 100644
--- a/examples/c/bistromathic/parse.y
+++ b/examples/c/bistromathic/parse.y
@@ -332,24 +332,23 @@ int
 expected_tokens (const char *input,
                  int *tokens, int ntokens)
 {
-  YYDPRINTF ((stderr, "expected_tokens(\"%s\")", input));
+  YYDPRINTF ((stderr, "expected_tokens (\"%s\")", input));
 
   // Parse the current state of the line.
-  YYLTYPE lloc;
   yypstate *ps = yypstate_new ();
   int status = 0;
   do {
+    YYLTYPE lloc;
     YYSTYPE lval;
     int token = yylex (&input, &lval, &lloc);
+    // Don't let the parse know when we reach the end of input.
     if (!token)
       break;
     status = yypush_parse (ps, token, &lval, &lloc);
   } while (status == YYPUSH_MORE);
 
   // Then query for the accepted tokens at this point.
-  yyparse_context_t yyctx
-    = {ps->yyssp, YYEMPTY, &lloc, ps->yyesa, &ps->yyes, &ps->yyes_capacity};
-  int res = yyexpected_tokens (&yyctx, tokens, ntokens);
+  int res = yypstate_expected_tokens (ps, tokens, ntokens);
   yypstate_delete (ps);
   return res;
 }
@@ -362,7 +361,7 @@ expected_tokens (const char *input,
 char **
 completion (const char *text, int start, int end)
 {
-  YYDPRINTF ((stderr, "completion(\"%.*s[%.*s]%s\")\n",
+  YYDPRINTF ((stderr, "completion (\"%.*s[%.*s]%s\")\n",
               start, rl_line_buffer,
               end - start, rl_line_buffer + start,
               rl_line_buffer + end));
@@ -414,7 +413,7 @@ completion (const char *text, int start, int end)
 
   if (yydebug)
     {
-      fprintf (stderr, "completion(\"%.*s[%.*s]%s\") = ",
+      fprintf (stderr, "completion (\"%.*s[%.*s]%s\") = ",
                start, rl_line_buffer,
                end - start, rl_line_buffer + start,
                rl_line_buffer + end);
@@ -452,7 +451,7 @@ void init_readline (void)
 int main (int argc, char const* argv[])
 {
   // Enable parse traces on option -p.
-  if (argc == 2 && strcmp(argv[1], "-p") == 0)
+  if (argc == 2 && strcmp (argv[1], "-p") == 0)
     yydebug = 1;
   init_table ();
   init_readline ();