bug in stub_charset rollup diff of changes to unicode code.

[John Kearney]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

Configuration Information [Automatically generated, do not change]:
Machine: x86_64
OS: linux-gnu
Compiler: gcc
Compilation CFLAGS:  -DPROGRAM='bash' -DCONF_HOSTTYPE='x86_64'
- -DCONF_OSTYPE='linux-gnu' -DCONF_MACHTYPE='x86_64-pc-linux-gnu'
- -DCONF_VENDOR='pc' -DLOCALEDIR='/usr/s
uname output: Linux DETH00 3.0.0-15-generic #26-Ubuntu SMP Fri Jan 20
17:23:00 UTC 2012 x86_64 x86_64 x86_64 GNU/Linux
Machine Type: x86_64-pc-linux-gnu

Bash Version: 4.2
Patch Level: 10
Release Status: release

Description:
  stub_charset
  if locale == '\0'
    return ASCII
  else if locale=~m/.*\.(.*)(@.*)/
   return $1
  else if locale=UTF-8
   return UTF-8
  else
   return ASCII

should be
  if locale == '\0'
    return ASCII
  else if locale=~m/.*\.(.*)(@.*)/
   return $1
  else
   return locale
 because its output is only being used in iconv, so let it decide if the
locale makes sense.




   I've attached a diff of all my changes to the unicode code.
   Including
   renamed u2cconv to utf32tomb
   move special handling of ascii charcter to start of function and
remove related call wrapper code.
   tried to reationalize the code in utf32tomb so its easier to read and
understand what is happening.
   added utf32toutf16
   use utf32toutf16 in case wchar_t=2 with wctomb

  removed dangerious code that was using iconv_open (charset, "ASCII");
as fallback. pointless anyway as we already assign a ascii value if posible.

  added warning message if encode fails

always terminate mb output string.


haven't started to test these changes yet firstly would like to know
if these changes are acceptable, any observations, I'm still reviewing
it myself for consistency.

Plus can somebody tell me how this was tested originally? I've got
some ideas myself but would like to know what has already been done in
that direction.

Repeat-By:
  .....

Fix:
diff --git a/builtins/printf.def b/builtins/printf.def
index 9eca215..3680419 100644
- --- a/builtins/printf.def
+++ b/builtins/printf.def
@@ -859,15 +859,9 @@ tescape (estart, cp, lenp, sawc)
            *cp = '\\';
            return 0;
          }
- -     if (uvalue <= UCHAR_MAX)
- -       *cp = uvalue;
- -     else
- -       {
- -         temp = u32cconv (uvalue, cp);
- -         cp[temp] = '\0';
- -         if (lenp)
- -           *lenp = temp;
- -       }
+       temp = utf32tomb (uvalue, cp);
+       if (lenp)
+         *lenp = temp;
        break;
 #endif
        
diff --git a/externs.h b/externs.h
index 09244fa..8868b55 100644
- --- a/externs.h
+++ b/externs.h
@@ -460,7 +460,7 @@ extern unsigned int falarm __P((unsigned int,
unsigned int));
 extern unsigned int fsleep __P((unsigned int, unsigned int));

 /* declarations for functions defined in lib/sh/unicode.c */
- -extern int u32cconv __P((unsigned long, char *));
+extern int utf32tomb __P((unsigned long, char *));

 /* declarations for functions defined in lib/sh/winsize.c */
 extern void get_new_window_size __P((int, int *, int *));
diff --git a/lib/sh/strtrans.c b/lib/sh/strtrans.c
index 2265782..495d9c4 100644
- --- a/lib/sh/strtrans.c
+++ b/lib/sh/strtrans.c
@@ -144,16 +144,10 @@ ansicstr (string, len, flags, sawc, rlen)
                  *r++ = '\\';  /* c remains unchanged */
                  break;
                }
- -           else if (v <= UCHAR_MAX)
- -             {
- -               c = v;
- -               break;
- -             }
              else
                {
- -               temp = u32cconv (v, r);
- -               r += temp;
- -               continue;
+                 r += utf32tomb (v, r);
+                 break;
                }
 #endif
            case '\\':
diff --git a/lib/sh/unicode.c b/lib/sh/unicode.c
index d34fa08..9a557a9 100644
- --- a/lib/sh/unicode.c
+++ b/lib/sh/unicode.c
@@ -36,14 +36,6 @@

 #include <xmalloc.h>

- -#ifndef USHORT_MAX
- -#  ifdef USHRT_MAX
- -#    define USHORT_MAX USHRT_MAX
- -#  else
- -#    define USHORT_MAX ((unsigned short) ~(unsigned short)0)
- -#  endif
- -#endif
- -
 #if !defined (STREQ)
 #  define STREQ(a, b) ((a)[0] == (b)[0] && strcmp ((a), (b)) == 0)
 #endif /* !STREQ */
@@ -54,13 +46,14 @@ extern const char *locale_charset __P((void));
 extern char *get_locale_var __P((char *));
 #endif

- -static int u32init = 0;
+const char *charset;
 static int utf8locale = 0;
 #if defined (HAVE_ICONV)
 static iconv_t localconv;
 #endif

 #ifndef HAVE_LOCALE_CHARSET
+static char CType[40]={0};
 static char *
 stub_charset ()
 {
@@ -69,6 +62,7 @@ stub_charset ()
   locale = get_locale_var ("LC_CTYPE");
   if (locale == 0 || *locale == 0)
     return "ASCII";
+  strcpy(CType, locale);
   s = strrchr (locale, '.');
   if (s)
     {
@@ -77,159 +71,230 @@ stub_charset ()
        *t = 0;
       return ++s;
     }
- -  else if (STREQ (locale, "UTF-8"))
- -    return "UTF-8";
   else
- -    return "ASCII";
+    return CType;
 }
 #endif

- -/* u32toascii ? */
 int
- -u32tochar (wc, s)
- -     wchar_t wc;
+utf32_2_utf8 (c, s)
+     unsigned long c;
      char *s;
 {
- -  unsigned long x;
   int l;

- -  x = wc;
- -  l = (x <= UCHAR_MAX) ? 1 : ((x <= USHORT_MAX) ? 2 : 4);
- -
- -  if (x <= UCHAR_MAX)
- -    s[0] = x & 0xFF;
- -  else if (x <= USHORT_MAX)  /* assume unsigned short = 16 bits */
+  if (c <= 0x7F)
     {
- -      s[0] = (x >> 8) & 0xFF;
- -      s[1] = x & 0xFF;
+      s[0] = (char)c;
+      l = 1;
     }
- -  else
+  else if (c <= 0x7FF)
     {
- -      s[0] = (x >> 24) & 0xFF;
- -      s[1] = (x >> 16) & 0xFF;
- -      s[2] = (x >> 8) & 0xFF;
- -      s[3] = x & 0xFF;
+      s[0] = (c >>   6)                | 0xc0; /* 110x xxxx */
+      s[1] = (c                & 0x3f) | 0x80; /* 10xx xxxx */
+      l = 2;
     }
- -  s[l] = '\0';
- -  return l;
- -}
- -
- -int
- -u32toutf8 (wc, s)
- -     wchar_t wc;
- -     char *s;
- -{
- -  int l;
- -
- -  l = (wc < 0x0080) ? 1 : ((wc < 0x0800) ? 2 : 3);
- -
- -  if (wc < 0x0080)
- -    s[0] = (unsigned char)wc;
- -  else if (wc < 0x0800)
+  else if (c <= 0xFFFF)
+    {
+      s[0] =  (c >> 12)         | 0xe0; /* 1110 xxxx */
+      s[1] = ((c >>  6) & 0x3f) | 0x80; /* 10xx xxxx */
+      s[2] =  (c        & 0x3f) | 0x80; /* 10xx xxxx */
+      l = 3;
+    }
+  else if (c <= 0x1FFFFF)
+    {
+      s[0] =  (c >> 18)         | 0xf0; /* 1111 0xxx */
+      s[1] = ((c >> 12) & 0x3f) | 0x80; /* 10xx xxxx */
+      s[2] = ((c >>  6) & 0x3f) | 0x80; /* 10xx xxxx */
+      s[3] = ( c        & 0x3f) | 0x80; /* 10xx xxxx */
+      l = 4;
+    }
+  else if (c <= 0x3FFFFFF)
     {
- -      s[0] = (wc >> 6) | 0xc0;
- -      s[1] = (wc & 0x3f) | 0x80;
+      s[0] =  (c >> 24)         | 0xf8; /* 1111 10xx */
+      s[1] = ((c >> 18) & 0x3f) | 0x80; /* 10xx xxxx */
+      s[2] = ((c >> 12) & 0x3f) | 0x80; /* 10xx xxxx */
+      s[3] = ((c >>  6) & 0x3f) | 0x80; /* 10xx xxxx */
+      s[4] = ( c        & 0x3f) | 0x80; /* 10xx xxxx */
+      l = 5;
+    }
+  else if (c <= 0x7FFFFFFF)
+    {
+      s[0] =  (c >> 30)         | 0xfc; /* 1111 110x */
+      s[1] = ((c >> 24) & 0x3f) | 0x80; /* 10xx xxxx */
+      s[2] = ((c >> 18) & 0x3f) | 0x80; /* 10xx xxxx */
+      s[3] = ((c >> 12) & 0x3f) | 0x80; /* 10xx xxxx */
+      s[4] = ((c >>  6) & 0x3f) | 0x80; /* 10xx xxxx */
+      s[5] = ( c        & 0x3f) | 0x80; /* 10xx xxxx */
+      l = 6;
     }
   else
     {
- -      s[0] = (wc >> 12) | 0xe0;
- -      s[1] = ((wc >> 6) & 0x3f) | 0x80;
- -      s[2] = (wc & 0x3f) | 0x80;
+      /* Error Invalid UTF-8 */
+      l = 0;
     }
   s[l] = '\0';
   return l;
 }
+int
+utf32_2_utf16 (c, s)
+     unsigned long c;
+     uint16_t *s;
+{
+  int l=0;
+  if (c < 0xFFFF)
+  {
+      // Character can be converted directly to 16 bits, just need to
check it's in the valid range
+      if ((c >= 0xD800) && (c <= 0xDFFF))
+      {
+         // Invalid character (this range is reserved)
+         s[0] = 0;
+      }
+      else
+      {
+         // Valid character directly convertible to 16 bits
+         s[0] = (uint16_t)(c&0xFFFF);
+         l=1;
+      }
+  }
+  else if (c > 0x0010FFFF)
+  {
+      // Invalid character (greater than the maximum unicode value)
+      s[0] = 0;
+  }
+  else
+  {
+      // Character will be converted to 2 UTF-16 elements
+      c -= 0x0010000;
+      s[0] = (uint16_t)((c >> 10)     + 0xD800);
+      s[1] = (uint16_t)((c & 0x3FFUL) + 0xDC00);
+      l=2;
+  }
+  s[l] = 0;
+  return l;
+}

 /* convert a single unicode-32 character into a multibyte string and
put the
    result in S, which must be large enough (at least MB_LEN_MAX bytes) */
 int
- -u32cconv (c, s)
+utf32tomb (c, s)
      unsigned long c;
      char *s;
 {
- -  wchar_t wc;
   int n;
+  uint16_t utf16_s[3]
 #if HAVE_ICONV
- -  const char *charset;
+  const char *ncharset;
   char obuf[25], *optr;
   size_t obytesleft;
   const char *iptr;
   size_t sn;
 #endif

- -  wc = c;
+  /*
+   *  Encode Method 1
+   *  UTF 0x00 -> 0x7f = ASCII Just copy
+   */
+  if ( c < 0x7f )
+    {
+      s[0]=(char)c;
+      n=1;
+    }

 #if __STDC_ISO_10646__
- -  if (sizeof (wchar_t) == 4)
+  if ( n == 0 )
     {
- -      n = wctomb (s, wc);
- -      return n;
+     /*
+      *  Encode Method 2
+      *  Use wctomb
+      */
+      if ( sizeof (wchar_t) == 4)
+         n = wctomb (s, (wchar_t)c);
+      else if ( sizeof (wchar_t) == 2)
+       {
+         if( utf32_2_utf16(c, utf16_s) )
+           {
+             n=0;
+             while (*utf16_s != 0)
+                 n += wctomb (s+n, (wchar_t)*utf16_s);
+           }
+       }
     }
 #endif

 #if HAVE_NL_LANGINFO
- -  codeset = nl_langinfo (CODESET);
- -  if (STREQ (codeset, "UTF-8"))
+  if ( n == 0 )
     {
- -      n = u32toutf8 (wc, s);
- -      return n;
+     /*
+      *  Encode Method 3
+      *  Targets UTF-8 cool just encode.
+      */
+      codeset = nl_langinfo (CODESET);
+      if (STREQ (codeset, "UTF-8"))
+         n = utf32_2_utf8 (c, s);
     }
 #endif

 #if HAVE_ICONV
- -  /* this is mostly from coreutils-8.5/lib/unicodeio.c */
- -  if (u32init == 0)
+  if ( n == 0 )
     {
+     /*
+      *  Encode Method 4
+      *  Lets try iconv.
+      */
 #  if HAVE_LOCALE_CHARSET
- -      charset = locale_charset ();   /* XXX - fix later */
+      ncharset = locale_charset ();    /* XXX - fix later */
 #  else
- -      charset = stub_charset ();
+      ncharset = stub_charset ();
 #  endif
- -      if (STREQ (charset, "UTF-8"))
- -     utf8locale = 1;
- -      else
+      /* this is mostly from coreutils-8.5/lib/unicodeio.c */
+      if (STREQ (charset, ncharset))
        {
- -       localconv = iconv_open (charset, "UTF-8");
- -       if (localconv == (iconv_t)-1)
- -         localconv = iconv_open (charset, "ASCII");
+         /* Free Old charset str ? */
+         charset=ncharset;
+         utf8locale = STREQ (charset, "UTF-8");
+         if ( ! utf8locale )
+             localconv = iconv_open (charset, "UTF-8");
        }
- -      u32init = 1;
- -    }
- -
- -  if (utf8locale)
- -    {
- -      n = u32toutf8 (wc, s);
- -      return n;
- -    }

- -  if (localconv == (iconv_t)-1)
- -    {
- -      n = u32tochar (wc, s);
- -      return n;
+       if (utf8locale)
+         {
+           n = utf32_2_utf8 (c, s);
+         }
+       else  if (localconv != (iconv_t)-1)
+         {
+           n = utf32_2_utf8 (c, s);
+
+           optr = obuf;
+           obytesleft = sizeof (obuf);
+           iptr = s;
+           sn = n;
+
+           iconv (localconv, NULL, NULL, NULL, NULL); /* Reset iconv internal
state */
+           n=0;
+           if (iconv (localconv, (ICONV_CONST char **)&iptr, &sn, &optr,
&obytesleft) != (size_t)-1)
+             {
+               *optr = '\0';
+
+               /* number of chars to be copied is optr - obuf if we want to do 
bounds
+                 checking */
+               strcpy (s, obuf);
+               n=(optr - obuf);
+             }
+       }
     }
- -
- -  n = u32toutf8 (wc, s);
- -
- -  optr = obuf;
- -  obytesleft = sizeof (obuf);
- -  iptr = s;
- -  sn = n;
- -
- -  iconv (localconv, NULL, NULL, NULL, NULL);
- -
- -  if (iconv (localconv, (ICONV_CONST char **)&iptr, &sn, &optr,
&obytesleft) == (size_t)-1)
- -    return n;        /* You get utf-8 if iconv fails */
- -
- -  *optr = '\0';
- -
- -  /* number of chars to be copied is optr - obuf if we want to do bounds
- -     checking */
- -  strcpy (s, obuf);
- -  return (optr - obuf);
 #endif

- -  n = u32tochar (wc, s);     /* fallback */
- -  return n;
- -}
+      if ( n == 0 )
+       {
+        /*
+         *  Error Encoding
+         */
+         n=snprintf(s, 23, "<U+%08lx>", c); /* s buffer only 24 characters
long */
+         builtin_warning (_("Encode Error unicode U+%08lx"), c);
+       }
+      s[n]=0;
+      return n;
+  }

 #endif /* HANDLE_MULTIBYTE */

-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.11 (GNU/Linux)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/

iQEcBAEBAgAGBQJPQzdAAAoJEKUDtR0WmS05ktIH/2KxMWa01hEQ1AKBgvtnYilu
XILDo7lHhUHk2w+SJ9TEyBqgF5bHPgLYY5g0xK8DLPH+O+NK0bJFieovTUDpUML4
m6OZ3BHPTJuFuPPYncFrWe9VTziHTAQ4H4rHe5NDHPTNB5Wnd8l1x5mtoOSzgtDM
f1PxmhM+dnIJ7uSsTJGG88FkXOCGkvMsHuQXQqMgJ5rfTYkBJTPh9j3HfDPLr5FP
YWdxEV/IvCaXZpzlugMGwQXcFvzDJM4t3BEKRbRaJFrE3JVEqylPyuglVVTkGl6v
U38afVaw+kjJuzMRL316JGcjrR22/YJPdsYnklXowaWe/YejLlqWjyMyiUYr7eg=
=PfVz
-----END PGP SIGNATURE-----