tokens: clean up the translation of special symbols

[Akim Demaille]

commit 5ab00861576908c7a14637ffca20e1c94daa15e5
Author: Akim Demaille <address@hidden>
Date:   Sun Apr 19 15:07:05 2020 +0200

    tokens: clean up the translation of special symbols
    
    * src/output.c (prepare_symbol_names): Don't play tricks with the
    symbols, it's quite too late.
    (has_translations): Move to...
    * src/symtab.c: here.
    (symbols_pack): Use it to enable translation for special symbols.

diff --git a/TODO b/TODO
index 4d2a8acb..aa071a8d 100644
--- a/TODO
+++ b/TODO
@@ -52,6 +52,8 @@ Rename user_token_number for tokens as "code".  It's not a 
"user number",
 it's the token code, and the user can control it, but this code always
 exists.
 
+See also USER_NUMBER_UNDEFINED and NUMBER_UNDEFINED.
+
 Rename endtoken as eoftoken.
 
 Also do it in data/skeletons.
diff --git a/src/output.c b/src/output.c
index 01669f5a..b649737d 100644
--- a/src/output.c
+++ b/src/output.c
@@ -186,16 +186,6 @@ xescape_trigraphs (const char *src)
   return buf;
 }
 
-/* Whether some symbol requires internationalization.  */
-static bool
-has_translations (void)
-{
-  for (int i = 0; i < nsyms; i++)
-    if (symbols[i]->translatable)
-      return true;
-  return false;
-}
-
 /* The tag to show in the generated parsers.  Use "end of file" rather
    than "$end".  But keep "$end" in the reports, it's shorter and more
    consistent.  Support i18n if the user already uses it.  */
@@ -219,11 +209,9 @@ symbol_tag (const symbol *sym)
 static void
 prepare_symbol_names (char const *muscle_name)
 {
-  const bool eof_is_user_defined
-    = !endtoken->alias || STRNEQ (endtoken->alias->tag, "$end");
-
+  // Whether to add a pair of quotes around the name.
   const bool quote = STREQ (muscle_name, "tname");
-  const bool with_translations = !quote && has_translations ();
+  bool has_translations = false;
 
   /* We assume that the table will be output starting at column 2. */
   int col = 2;
@@ -233,11 +221,9 @@ prepare_symbol_names (char const *muscle_name)
   for (int i = 0; i < nsyms; i++)
     {
       const char *tag = symbol_tag (symbols[i]);
-      bool translatable =
-        with_translations
-        && (symbols[i]->translatable
-            || (!eof_is_user_defined && symbols[i]->content == 
endtoken->content)
-            || symbols[i]->content == undeftoken->content);
+      bool translatable = !quote && symbols[i]->translatable;
+      if (translatable)
+        has_translations = true;
 
       char *cp
         = tag[0] == '"' && !quote
@@ -273,8 +259,7 @@ prepare_symbol_names (char const *muscle_name)
   muscle_insert (muscle_name, obstack_finish0 (&format_obstack));
 
   /* Announce whether translation support is needed.  */
-  if (!quote)
-    MUSCLE_INSERT_BOOL ("has_translations", with_translations);
+  MUSCLE_INSERT_BOOL ("has_translations", has_translations);
 }
 
 
diff --git a/src/parse-gram.c b/src/parse-gram.c
index 0100d6ba..86671538 100644
--- a/src/parse-gram.c
+++ b/src/parse-gram.c
@@ -1,4 +1,4 @@
-/* A Bison parser, made by GNU Bison 3.5.4.236-8d75.  */
+/* A Bison parser, made by GNU Bison 3.5.90.  */
 
 /* Bison implementation for Yacc-like parsers in C
 
@@ -48,7 +48,7 @@
 #define YYBISON 1
 
 /* Bison version.  */
-#define YYBISON_VERSION "3.5.4.236-8d75"
+#define YYBISON_VERSION "3.5.90"
 
 /* Skeleton name.  */
 #define YYSKELETON_NAME "yacc.c"
@@ -643,7 +643,7 @@ yysymbol_name (yysymbol_kind_t yysymbol)
 {
   static const char *const yy_sname[] =
   {
-  N_("end of file"), "error", N_("invalid token"), N_("string"),
+  N_("end of file"), N_("error"), N_("invalid token"), N_("string"),
   N_("translatable string"), "%token", "%nterm", "%type", "%destructor",
   "%printer", "%left", "%right", "%nonassoc", "%precedence", "%prec",
   "%dprec", "%merge", "%code", "%default-prec", "%define", "%defines",
@@ -670,7 +670,7 @@ yysymbol_name (yysymbol_kind_t yysymbol)
      internationalizable.  */
   static yytype_int8 yytranslatable[] =
   {
-       0,     0,     0,     1,     1,     0,     0,     0,     0,     0,
+       1,     1,     1,     1,     1,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
        0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
@@ -2619,7 +2619,6 @@ yyerrlab:
   /* Make sure we have latest lookahead translation.  See comments at
      user semantic actions for why this is necessary.  */
   yytoken = yychar == GRAM_EMPTY ? YYSYMBOL_YYEMPTY : YYTRANSLATE (yychar);
-
   /* If not already recovering from an error, report this error.  */
   if (!yyerrstatus)
     {
@@ -2684,6 +2683,7 @@ yyerrorlab:
 yyerrlab1:
   yyerrstatus = 3;      /* Each real token shifted decrements this.  */
 
+  // Pop stack until we find a state that shifts the error token.
   for (;;)
     {
       yyn = yypact[yystate];
diff --git a/src/parse-gram.h b/src/parse-gram.h
index 66feb31c..3ae6f375 100644
--- a/src/parse-gram.h
+++ b/src/parse-gram.h
@@ -1,4 +1,4 @@
-/* A Bison parser, made by GNU Bison 3.5.4.236-8d75.  */
+/* A Bison parser, made by GNU Bison 3.5.90.  */
 
 /* Bison interface for Yacc-like parsers in C
 
diff --git a/src/symtab.c b/src/symtab.c
index d817d7c8..34ae2bc8 100644
--- a/src/symtab.c
+++ b/src/symtab.c
@@ -1095,6 +1095,21 @@ symbols_token_translations_init (void)
 }
 
 
+/* Whether some symbol requires internationalization.  */
+static bool
+has_translations (void)
+{
+  for (const void *entry = hash_get_first (symbol_table);
+       entry;
+       entry = hash_get_next (symbol_table, entry))
+    {
+      const symbol *sym = (const symbol *) entry;
+      if (sym->translatable)
+        return true;
+    }
+  return false;
+}
+
 /*----------------------------------------------------------------.
 | Assign symbol numbers, and write definition of token names into |
 | FDEFINES.  Set up vectors SYMBOL_TABLE, TAGS of symbols.        |
@@ -1137,6 +1152,18 @@ symbols_pack (void)
     complain (&startsymbol_loc, fatal,
               _("the start symbol %s is a token"),
               startsymbol->tag);
+
+  // If some user tokens are internationalized, the internal ones
+  // should be too.
+  if (has_translations ())
+    {
+      const bool eof_is_user_defined
+        = !endtoken->alias || STRNEQ (endtoken->alias->tag, "$end");
+      if (!eof_is_user_defined)
+        endtoken->alias->translatable = true;
+      undeftoken->alias->translatable = true;
+      errtoken->alias->translatable = true;
+    }
 }
 
 /*---------------------------------.