[Bug-gnulib] addition: utf8-ucs4.h, ucs4-utf8.h, utf16-ucs4.h, ucs4-utf16.h

[Bruno Haible]

Hi,

Here come 4 new modules, for converting between UTF-8 / UTF-16 and
UCS-4. I've needed these conversions sufficiently often in gettext:

    utf8-ucs4.h   4 times
    ucs4-utf8.h   2 times
    utf16-ucs4.h  2 times

The functions convert just one character. It is simple to convert an
entire string by calling the functions in a loop. For speed, the case
of ASCII characters is optimized through an inline function.

Any objections?

Bruno

========================= utf8-ucs4.h =========================
/* Conversion UTF-8 to UCS-4.
   Copyright (C) 2001-2002 Free Software Foundation, Inc.
   Written by Bruno Haible <address@hidden>, 2001.

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2, or (at your option)
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */


#include <stddef.h>

/* Return the length (number of units) of the first character in S, putting
   its 'ucs4_t' representation in *PUC.  */
static int
u8_mbtouc_aux (unsigned int *puc, const unsigned char *s, size_t n)
{
  unsigned char c = *s;

  if (c >= 0xc2)
    {
      if (c < 0xe0)
        {
          if (n >= 2)
            {
              if ((s[1] ^ 0x80) < 0x40)
                {
                  *puc = ((unsigned int) (c & 0x1f) << 6)
                         | (unsigned int) (s[1] ^ 0x80);
                  return 2;
                }
              /* invalid multibyte character */
            }
          else
            {
              /* incomplete multibyte character */
              *puc = 0xfffd;
              return n;
            }
        }
      else if (c < 0xf0)
        {
          if (n >= 3)
            {
              if ((s[1] ^ 0x80) < 0x40 && (s[2] ^ 0x80) < 0x40
                  && (c >= 0xe1 || s[1] >= 0xa0))
                {
                  *puc = ((unsigned int) (c & 0x0f) << 12)
                         | ((unsigned int) (s[1] ^ 0x80) << 6)
                         | (unsigned int) (s[2] ^ 0x80);
                  return 3;
                }
              /* invalid multibyte character */
            }
          else
            {
              /* incomplete multibyte character */
              *puc = 0xfffd;
              return n;
            }
        }
      else if (c < 0xf8)
        {
          if (n >= 4)
            {
              if ((s[1] ^ 0x80) < 0x40 && (s[2] ^ 0x80) < 0x40
                  && (s[3] ^ 0x80) < 0x40
                  && (c >= 0xf1 || s[1] >= 0x90)
#if 1
                  && (c < 0xf4 || (c == 0xf4 && s[1] < 0x90))
#endif
                 )
                {
                  *puc = ((unsigned int) (c & 0x07) << 18)
                         | ((unsigned int) (s[1] ^ 0x80) << 12)
                         | ((unsigned int) (s[2] ^ 0x80) << 6)
                         | (unsigned int) (s[3] ^ 0x80);
                  return 4;
                }
              /* invalid multibyte character */
            }
          else
            {
              /* incomplete multibyte character */
              *puc = 0xfffd;
              return n;
            }
        }
#if 0
      else if (c < 0xfc)
        {
          if (n >= 5)
            {
              if ((s[1] ^ 0x80) < 0x40 && (s[2] ^ 0x80) < 0x40
                  && (s[3] ^ 0x80) < 0x40 && (s[4] ^ 0x80) < 0x40
                  && (c >= 0xf9 || s[1] >= 0x88))
                {
                  *puc = ((unsigned int) (c & 0x03) << 24)
                         | ((unsigned int) (s[1] ^ 0x80) << 18)
                         | ((unsigned int) (s[2] ^ 0x80) << 12)
                         | ((unsigned int) (s[3] ^ 0x80) << 6)
                         | (unsigned int) (s[4] ^ 0x80);
                  return 5;
                }
              /* invalid multibyte character */
            }
          else
            {
              /* incomplete multibyte character */
              *puc = 0xfffd;
              return n;
            }
        }
      else if (c < 0xfe)
        {
          if (n >= 6)
            {
              if ((s[1] ^ 0x80) < 0x40 && (s[2] ^ 0x80) < 0x40
                  && (s[3] ^ 0x80) < 0x40 && (s[4] ^ 0x80) < 0x40
                  && (s[5] ^ 0x80) < 0x40
                  && (c >= 0xfd || s[1] >= 0x84))
                {
                  *puc = ((unsigned int) (c & 0x01) << 30)
                         | ((unsigned int) (s[1] ^ 0x80) << 24)
                         | ((unsigned int) (s[2] ^ 0x80) << 18)
                         | ((unsigned int) (s[3] ^ 0x80) << 12)
                         | ((unsigned int) (s[4] ^ 0x80) << 6)
                         | (unsigned int) (s[5] ^ 0x80);
                  return 6;
                }
              /* invalid multibyte character */
            }
          else
            {
              /* incomplete multibyte character */
              *puc = 0xfffd;
              return n;
            }
        }
#endif
    }
  /* invalid multibyte character */
  *puc = 0xfffd;
  return 1;
}
static inline int
u8_mbtouc (unsigned int *puc, const unsigned char *s, size_t n)
{
  unsigned char c = *s;

  if (c < 0x80)
    {
      *puc = c;
      return 1;
    }
  else
    return u8_mbtouc_aux (puc, s, n);
}
========================= ucs4-utf8.h =========================
/* Conversion UCS-4 to UTF-8.
   Copyright (C) 2002 Free Software Foundation, Inc.
   Written by Bruno Haible <address@hidden>, 2002.

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2, or (at your option)
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */


#include <stddef.h>

/* Return the length (number of units) of the UTF-8 representation of uc,
   after storing it at S.  Return -1 upon failure, -2 if the number of
   available units, N, is too small.  */
static int
u8_uctomb_aux (unsigned char *s, unsigned int uc, int n)
{
  int count;

  if (uc < 0x80)
    count = 1;
  else if (uc < 0x800)
    count = 2;
  else if (uc < 0x10000)
    count = 3;
#if 0
  else if (uc < 0x200000)
    count = 4;
  else if (uc < 0x4000000)
    count = 5;
  else if (uc <= 0x7fffffff)
    count = 6;
#else
  else if (uc < 0x110000)
    count = 4;
#endif
  else
    return -1;

  if (n < count)
    return -2;

  switch (count) /* note: code falls through cases! */
    {
#if 0
    case 6: s[5] = 0x80 | (uc & 0x3f); uc = uc >> 6; uc |= 0x4000000;
    case 5: s[4] = 0x80 | (uc & 0x3f); uc = uc >> 6; uc |= 0x200000;
#endif
    case 4: s[3] = 0x80 | (uc & 0x3f); uc = uc >> 6; uc |= 0x10000;
    case 3: s[2] = 0x80 | (uc & 0x3f); uc = uc >> 6; uc |= 0x800;
    case 2: s[1] = 0x80 | (uc & 0x3f); uc = uc >> 6; uc |= 0xc0;
    case 1: s[0] = uc;
    }
  return count;
}

static inline int
u8_uctomb (unsigned char *s, unsigned int uc, int n)
{
  if (uc < 0x80 && n > 0)
    {
      s[0] = uc;
      return 1;
    }
  else
    return u8_uctomb_aux (s, uc, n);
}
========================= utf16-ucs4.h =========================
/* Conversion UTF-16 to UCS-4.
   Copyright (C) 2001-2002 Free Software Foundation, Inc.
   Written by Bruno Haible <address@hidden>, 2001.

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2, or (at your option)
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */


#include <stddef.h>

/* Return the length (number of units) of the first character in S, putting
   its 'ucs4_t' representation in *PUC.  */
static int
u16_mbtouc_aux (unsigned int *puc, const unsigned short *s, size_t n)
{
  unsigned short c = *s;

  if (c < 0xdc00)
    {
      if (n >= 2)
        {
          if (s[1] >= 0xdc00 && s[1] < 0xe000)
            {
              *puc = 0x10000 + ((c - 0xd800) << 10) + (s[1] - 0xdc00);
              return 2;
            }
          /* invalid multibyte character */
        }
      else
        {
          /* incomplete multibyte character */
          *puc = 0xfffd;
          return n;
        }
    }
  /* invalid multibyte character */
  *puc = 0xfffd;
  return 1;
}
static inline int
u16_mbtouc (unsigned int *puc, const unsigned short *s, size_t n)
{
  unsigned short c = *s;

  if (c < 0xd800 || c >= 0xe000)
    {
      *puc = c;
      return 1;
    }
  else
    return u16_mbtouc_aux (puc, s, n);
}
========================= ucs4-utf16.h =========================
/* Conversion UCS-4 to UTF-16.
   Copyright (C) 2002 Free Software Foundation, Inc.
   Written by Bruno Haible <address@hidden>, 2002.

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2, or (at your option)
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */


#include <stddef.h>

/* Return the length (number of units) of the UTF-16 representation of uc,
   after storing it at S.  Return -1 upon failure, -2 if the number of
   available units, N, is too small.  */
static int
u16_uctomb_aux (unsigned short *s, unsigned int uc, int n)
{
  if (uc >= 0x10000)
    {
      if (uc < 0x110000)
        {
          if (n >= 2)
            {
              s[0] = 0xd800 + ((uc - 0x10000) >> 10);
              s[1] = 0xdc00 + ((uc - 0x10000) & 0x3ff);
              return 2;
            }
        }
      else
        return -1;
    }
  return -2;
}

static inline int
u16_uctomb (unsigned short *s, unsigned int uc, int n)
{
  if (uc < 0x10000 && n > 0)
    {
      s[0] = uc;
      return 1;
    }
  else
    return u16_uctomb_aux (s, uc, n);
}