localename: Fix test failure on AIX 7

[Bruno Haible]

Hi,

Assaf Gordon wrote in
<https://lists.gnu.org/archive/html/sed-devel/2018-12/msg00019.html>:

> On AIX 7.1 and 7.2, one gnulib failure (FAIL: test-localename) -
> not a show-stopper.

This patch fixes it.

Jim: To avoid risk, I would NOT include this patch in the upcoming
'sed' and 'grep' releases.


2018-12-18  Bruno Haible  <address@hidden>

        localename: Fix test failure on AIX 7.
        Reported by Assaf Gordon in
        <https://lists.gnu.org/archive/html/sed-devel/2018-12/msg00019.html>.
        * m4/intl-thread-locale.m4 (gt_FUNC_USELOCALE): New macro.
        (gt_INTL_THREAD_LOCALE_NAME): Invoke it. Test gt_cv_func_uselocale_works
        instead of ac_cv_func_uselocale.
        * lib/localename.c: Test HAVE_WORKING_USELOCALE instead of
        HAVE_USELOCALE.
        * lib/localename-table.h: Likewise.
        * lib/localename-table.c: Likewise.
        * tests/test-localename.c: Likewise.
        * doc/posix-functions/uselocale.texi: Mention the AIX problem.

(This is a whitespace-insensitive diff.)
diff --git a/m4/intl-thread-locale.m4 b/m4/intl-thread-locale.m4
index 9ac98c5..661a23f 100644
--- a/m4/intl-thread-locale.m4
+++ b/m4/intl-thread-locale.m4
@@ -1,4 +1,4 @@
-# intl-thread-locale.m4 serial 3
+# intl-thread-locale.m4 serial 4
 dnl Copyright (C) 2015-2018 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -22,7 +22,8 @@ AC_DEFUN([gt_INTL_THREAD_LOCALE_NAME],
   dnl Persuade Solaris <locale.h> to define 'locale_t'.
   AC_REQUIRE([AC_USE_SYSTEM_EXTENSIONS])
 
-  AC_CHECK_FUNCS_ONCE([uselocale])
+  dnl Test whether uselocale() exists and works at all.
+  gt_FUNC_USELOCALE
 
   dnl On OpenBSD >= 6.2, the locale_t type and the uselocale(), newlocale(),
   dnl duplocale(), freelocale() functions exist but are effectively useless,
@@ -37,7 +38,8 @@ AC_DEFUN([gt_INTL_THREAD_LOCALE_NAME],
   dnl they did not think about the programs.
   dnl In this situation, even the HAVE_NAMELESS_LOCALES support does not work.
   dnl So, define HAVE_FAKE_LOCALES and disable all locale_t support.
-  if test $ac_cv_func_uselocale = yes; then
+  case "$gt_cv_func_uselocale_works" in
+    *yes)
       AC_CHECK_HEADERS_ONCE([xlocale.h])
       AC_CACHE_CHECK([for fake locale system (OpenBSD)],
         [gt_cv_locale_fake],
@@ -65,9 +67,9 @@ int main ()
             esac
            ])
         ])
-  else
-    gt_cv_locale_fake=no
-  fi
+      ;;
+    *) gt_cv_locale_fake=no ;;
+  esac
   case "$gt_cv_locale_fake" in
     *yes)
       AC_DEFINE([HAVE_FAKE_LOCALES], [1],
@@ -75,7 +77,8 @@ int main ()
       ;;
   esac
 
-  if test $ac_cv_func_uselocale = yes; then
+  case "$gt_cv_func_uselocale_works" in
+    *yes)
       AC_CACHE_CHECK([for Solaris 11.4 locale system],
         [gt_cv_locale_solaris114],
         [case "$host_os" in
@@ -99,9 +102,9 @@ int main ()
            *) gt_cv_locale_solaris114=no ;;
          esac
         ])
-  else
-    gt_cv_locale_solaris114=no
-  fi
+      ;;
+    *) gt_cv_locale_solaris114=no ;;
+  esac
   if test $gt_cv_locale_solaris114 = yes; then
     AC_DEFINE([HAVE_SOLARIS114_LOCALES], [1],
       [Define if the locale_t type is as on Solaris 11.4.])
@@ -110,9 +113,11 @@ int main ()
   dnl Solaris 12 will maybe provide getlocalename_l.  If it does, it will
   dnl improve the implementation of gl_locale_name_thread(), by removing
   dnl the use of undocumented structures.
-  if test $ac_cv_func_uselocale = yes; then
+  case "$gt_cv_func_uselocale_works" in
+    *yes)
       AC_CHECK_FUNCS([getlocalename_l])
-  fi
+      ;;
+  esac
 
   dnl This code is for future use, in case we some day have to port to a
   dnl platform where the locale_t type does not provide access to the name of
@@ -127,3 +132,52 @@ int main ()
       [Define if the locale_t type does not contain the name of each locale 
category.])
   fi
 ])
+
+dnl Tests whether uselocale() exists and is usable.
+dnl Sets gt_cv_func_uselocale_works. Defines HAVE_WORKING_USELOCALE.
+AC_DEFUN([gt_FUNC_USELOCALE],
+[
+  AC_REQUIRE([AC_CANONICAL_HOST]) dnl for cross-compiles
+
+  dnl Persuade Solaris <locale.h> to define 'locale_t'.
+  AC_REQUIRE([AC_USE_SYSTEM_EXTENSIONS])
+
+  AC_CHECK_FUNCS_ONCE([uselocale])
+
+  dnl On AIX 7.2, the uselocale() function is not documented and leads to
+  dnl crashes in subsequent setlocale() invocations.
+  if test $ac_cv_func_uselocale = yes; then
+    AC_CHECK_HEADERS_ONCE([xlocale.h])
+    AC_CACHE_CHECK([whether uselocale works],
+      [gt_cv_func_uselocale_works],
+      [AC_RUN_IFELSE(
+         [AC_LANG_SOURCE([[
+#include <locale.h>
+#if HAVE_XLOCALE_H
+# include <xlocale.h>
+#endif
+int main ()
+{
+  uselocale (NULL);
+  setlocale (LC_ALL, "en_US.UTF-8");
+  return 0;
+}]])],
+         [gt_cv_func_uselocale_works=yes],
+         [gt_cv_func_uselocale_works=no],
+         [# Guess no on AIX, yes otherwise.
+          case "$host_os" in
+            aix*) gt_cv_func_uselocale_works="guessing no" ;;
+            *)    gt_cv_func_uselocale_works="guessing yes" ;;
+          esac
+         ])
+      ])
+  else
+    gt_cv_func_uselocale_works=no
+  fi
+  case "$gt_cv_func_uselocale_works" in
+    *yes)
+      AC_DEFINE([HAVE_WORKING_USELOCALE], [1],
+        [Define if the uselocale function exists any may safely be called.])
+      ;;
+  esac
+])
diff --git a/lib/localename.c b/lib/localename.c
index 735f427..736c585 100644
--- a/lib/localename.c
+++ b/lib/localename.c
@@ -37,7 +37,7 @@
 
 /* We cannot support uselocale() on platforms where the locale_t type is fake.
    See intl-thread-locale.m4 for details.  */
-#if HAVE_USELOCALE && !HAVE_FAKE_LOCALES
+#if HAVE_WORKING_USELOCALE && !HAVE_FAKE_LOCALES
 # define HAVE_GOOD_USELOCALE 1
 #endif
 
@@ -2629,7 +2629,7 @@ get_lcid (const char *locale_name)
 #endif
 
 
-#if HAVE_GOOD_USELOCALE /* glibc, Mac OS X, FreeBSD >= 9.1, AIX >= 7, Cygwin 
>= 2.6,
+#if HAVE_GOOD_USELOCALE /* glibc, Mac OS X, FreeBSD >= 9.1, Cygwin >= 2.6,
                            Solaris 11 OpenIndiana, or Solaris >= 11.4  */
 
 /* Simple hash set of strings.  We don't want to drag in lots of hash table
diff --git a/lib/localename-table.h b/lib/localename-table.h
index 07e92f6..6b2a807 100644
--- a/lib/localename-table.h
+++ b/lib/localename-table.h
@@ -16,7 +16,7 @@
 
 /* Written by Bruno Haible <address@hidden>, 2018.  */
 
-#if HAVE_USELOCALE && HAVE_NAMELESS_LOCALES
+#if HAVE_WORKING_USELOCALE && HAVE_NAMELESS_LOCALES
 
 # include <stddef.h>
 # include <locale.h>
diff --git a/lib/localename-table.c b/lib/localename-table.c
index 4b0810d..10be7be 100644
--- a/lib/localename-table.c
+++ b/lib/localename-table.c
@@ -18,7 +18,7 @@
 
 #include <config.h>
 
-#if HAVE_USELOCALE && HAVE_NAMELESS_LOCALES
+#if HAVE_WORKING_USELOCALE && HAVE_NAMELESS_LOCALES
 
 /* Specification.  */
 #include "localename-table.h"
diff --git a/tests/test-localename.c b/tests/test-localename.c
index 2b91782..21a5f5d 100644
--- a/tests/test-localename.c
+++ b/tests/test-localename.c
@@ -26,7 +26,7 @@
 
 #include "macros.h"
 
-#if HAVE_NEWLOCALE && HAVE_USELOCALE && !HAVE_FAKE_LOCALES
+#if HAVE_NEWLOCALE && HAVE_WORKING_USELOCALE && !HAVE_FAKE_LOCALES
 # define HAVE_GOOD_USELOCALE 1
 #endif
 
diff --git a/doc/posix-functions/uselocale.texi 
b/doc/posix-functions/uselocale.texi
index 1343214..83d859b 100644
--- a/doc/posix-functions/uselocale.texi
+++ b/doc/posix-functions/uselocale.texi
@@ -16,6 +16,10 @@ Portability problems not fixed by Gnulib:
 This function is missing on many platforms:
 Mac OS X 10.3, FreeBSD 9.0, NetBSD 5.0, OpenBSD 6.1, Minix 3.1.8, AIX 6.1, 
HP-UX 11, IRIX 6.5, OSF/1 5.1, Solaris 11.3, Cygwin 2.5.x, mingw, MSVC 14, 
Interix 3.5, BeOS, Android 4.4.
 @item
+This function is not documented and leads to crashes in subsequent
address@hidden invocations on some platforms:
+AIX 7.2.
address@hidden
 This function is useless because the @code{locale_t} type contains basically
 no information on some platforms:
 OpenBSD 6.3.