[PATCH] uniname/uniname: support character alias

[Daiki Ueno]

I need this for gettext to support Python 3.3's change to its "\N{...}"
notation in string literals, which supports aliased character names:
https://docs.python.org/3/reference/lexical_analysis.html#literals

As usual, the patch does not include generated/copied files.
To generate, do the following from the top directory of gnulib:

  $ sed -e '/^$/d' -e '/^#/d' < $UCD/NameAliases.txt > \
    tests/uniname/NameAliases.txt
  $ (cd lib/uniname \
     && clisp gen-uninames.lisp $UCD/UnicodeData.txt uninames.h \
     tests/uniname/NameAliases.txt)

Regards,
--
Daiki Ueno

>From bf9ae34b0c3803785b7d49f05edd6c6f6de2bbb3 Mon Sep 17 00:00:00 2001
From: Daiki Ueno <address@hidden>
Date: Thu, 5 Feb 2015 16:56:41 +0900
Subject: [PATCH] uniname/uniname: support character alias

* lib/uniname/gen-uninames.lisp (main): New argument ALIASFILE.
Register one-way mapping from aliases to codepoints in the
generated tables.
* lib/uniname/uninames.h: Regenerate.
* tests/uniname/NameAliases.txt: New file, taken from UCD 7.0.0.
* modules/uniname/uniname-tests (Files): Add
tests/uniname/NameAliases.txt.
* tests/uniname/test-uninames.c: Mark as static.
(ALIASLEN): Define.
(struct unicode_alias): New struct.
(unicode_aliases): New variable.
(fill_aliases): New function.
(test_alias_lookup): New test function.
(main): Run the 'test_alias_lookup' test if the second argument is
given.
* tests/uniname/test-uninames.sh: Supply NameAliases.txt as the
second argument.
---
 lib/uniname/gen-uninames.lisp  |  49 ++++++++++++++-----
 modules/uniname/uniname-tests  |   1 +
 tests/uniname/test-uninames.c  | 108 ++++++++++++++++++++++++++++++++++++++++-
 tests/uniname/test-uninames.sh |   2 +-
 4 files changed, 145 insertions(+), 15 deletions(-)

diff --git a/lib/uniname/gen-uninames.lisp b/lib/uniname/gen-uninames.lisp
index e7de0a1..060dda1 100755
--- a/lib/uniname/gen-uninames.lisp
+++ b/lib/uniname/gen-uninames.lisp
@@ -25,10 +25,13 @@
   length                        ; number of words
 )
 
-(defun main (inputfile outputfile)
-  (declare (type string inputfile outputfile))
+(defun main (inputfile outputfile aliasfile)
+  (declare (type string inputfile outputfile aliasfile))
   #+UNICODE (setq *default-file-encoding* charset:utf-8)
   (let ((all-chars '())
+        (all-chars-hashed (make-hash-table :test #'equal))
+        (all-aliases '())
+        all-chars-and-aliases
         (all-ranges '())
         (name-index 0)
         range)
@@ -53,6 +56,7 @@
                   (push (make-unicode-char :index name-index
                                            :name name-string)
                         all-chars)
+                  (setf (gethash code all-chars-hashed) (car all-chars))
                   ;; Update the contiguous range, or start a new range.
                   (if (and range (= (1+ (range-end-code range)) code))
                       (setf (range-end-code range) code)
@@ -70,9 +74,28 @@
     (if range
         (push range all-ranges))
     (setq all-ranges (nreverse all-ranges))
+    (when aliasfile
+      ;; Read all characters and names from the alias file.
+      (with-open-file (istream aliasfile :direction :input)
+        (loop
+         (let ((line (read-line istream nil nil)))
+           (unless line (return))
+           (let* ((i1 (position #\; line))
+                  (i2 (position #\; line :start (1+ i1)))
+                  (code-string (subseq line 0 i1))
+                  (code (parse-integer code-string :radix 16))
+                  (name-string (subseq line (1+ i1) i2))
+                  (uc (gethash code all-chars-hashed)))
+             (when uc
+               (push (make-unicode-char :index (unicode-char-index uc)
+                                        :name name-string)
+                     all-aliases)
+             ) ) ) ) ) )
+    (setq all-aliases (nreverse all-aliases)
+          all-chars-and-aliases (append all-chars all-aliases))
     ;; Split into words.
     (let ((words-by-length (make-array 0 :adjustable t)))
-      (dolist (name (list* "HANGUL SYLLABLE" "CJK COMPATIBILITY" (mapcar 
#'unicode-char-name all-chars)))
+      (dolist (name (list* "HANGUL SYLLABLE" "CJK COMPATIBILITY" (mapcar 
#'unicode-char-name all-chars-and-aliases)))
         (let ((i1 0))
           (loop
             (when (>= i1 (length name)) (return))
@@ -195,7 +218,7 @@
                           (gethash word (word-list-hashed (aref 
words-by-length (length word))))
         ) )
         ;; Compute the word-indices for every unicode-char.
-        (dolist (uc all-chars)
+        (dolist (uc all-chars-and-aliases)
           (let ((name (unicode-char-name uc))
                 (indices '()))
             (let ((i1 0))
@@ -215,8 +238,8 @@
             )
         ) )
         ;; Sort the list of unicode-chars by word-indices.
-        (setq all-chars
-              (sort all-chars
+        (setq all-chars-and-aliases
+              (sort all-chars-and-aliases
                     (lambda (vec1 vec2)
                       (let ((len1 (length vec1))
                             (len2 (length vec2)))
@@ -235,10 +258,10 @@
         )     )
         ;; Output the word-indices.
         (format ostream "static const uint16_t unicode_names[~D] = {~%"
-                        (reduce #'+ (mapcar (lambda (uc) (length 
(unicode-char-word-indices uc))) all-chars))
+                        (reduce #'+ (mapcar (lambda (uc) (length 
(unicode-char-word-indices uc))) all-chars-and-aliases))
         )
         (let ((i 0))
-          (dolist (uc all-chars)
+          (dolist (uc all-chars-and-aliases)
             (format ostream " ~{ ~D,~}"
                             (maplist (lambda (r) (+ (* 2 (car r)) (if (cdr r) 
1 0)))
                                      (coerce (unicode-char-word-indices uc) 
'list)
@@ -255,9 +278,9 @@
         (format ostream "static const struct { uint16_t index; uint32_t 
name:24; }~%")
         (format ostream "#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ 
>= 7)~%__attribute__((__packed__))~%#endif~%")
         (format ostream "unicode_name_to_index[~D] = {~%"
-                        (length all-chars)
+                        (length all-chars-and-aliases)
         )
-        (dolist (uc all-chars)
+        (dolist (uc all-chars-and-aliases)
           (format ostream "  { 0x~4,'0X, ~D },"
                           (unicode-char-index uc)
                           (unicode-char-word-indices-index uc)
@@ -285,10 +308,10 @@
         )
         (format ostream "};~%")
         (format ostream "#define UNICODE_CHARNAME_MAX_LENGTH ~D~%"
-                        (reduce #'max (mapcar (lambda (uc) (length 
(unicode-char-name uc))) all-chars))
+                        (reduce #'max (mapcar (lambda (uc) (length 
(unicode-char-name uc))) all-chars-and-aliases))
         )
         (format ostream "#define UNICODE_CHARNAME_MAX_WORDS ~D~%"
-                        (reduce #'max (mapcar (lambda (uc) (length 
(unicode-char-word-indices uc))) all-chars))
+                        (reduce #'max (mapcar (lambda (uc) (length 
(unicode-char-word-indices uc))) all-chars-and-aliases))
         )
         (format ostream "static const struct { uint16_t index; uint32_t gap; 
uint16_t length; } unicode_ranges[~D] = {~%"
                         (length all-ranges))
@@ -302,4 +325,4 @@
       )
 ) ) )
 
-(main (first *args*) (second *args*))
+(main (first *args*) (second *args*) (third *args*))
diff --git a/modules/uniname/uniname-tests b/modules/uniname/uniname-tests
index 305e6a1..512b035 100644
--- a/modules/uniname/uniname-tests
+++ b/modules/uniname/uniname-tests
@@ -2,6 +2,7 @@ Files:
 tests/uniname/test-uninames.sh
 tests/uniname/test-uninames.c
 tests/uniname/UnicodeDataNames.txt
+tests/uniname/NameAliases.txt
 
 Depends-on:
 xalloc
diff --git a/tests/uniname/test-uninames.c b/tests/uniname/test-uninames.c
index f8fb077..eccf2f4 100644
--- a/tests/uniname/test-uninames.c
+++ b/tests/uniname/test-uninames.c
@@ -27,7 +27,19 @@
 
 /* The names according to the UnicodeData.txt file, modified to contain the
    Hangul syllable names, as described in the Unicode 3.0 book.  */
-const char * unicode_names [0x110000];
+static const char * unicode_names [0x110000];
+
+/* Maximum entries in unicode_aliases.  */
+#define ALIASLEN 0x200
+
+/* The aliases according to the NameAliases.txt file.  */
+struct unicode_alias
+{
+  const char *name;
+  unsigned int uc;
+};
+
+static struct unicode_alias unicode_aliases [ALIASLEN];
 
 /* Maximum length of a field in the UnicodeData.txt file.  */
 #define FIELDLEN 120
@@ -113,6 +125,62 @@ fill_names (const char *unicodedata_filename)
     }
 }
 
+/* Stores in unicode_aliases[] the relevant contents of the NameAliases.txt
+   file.  */
+static void
+fill_aliases (const char *namealiases_filename)
+{
+  int i;
+  FILE *stream;
+  char field0[FIELDLEN];
+  char field1[FIELDLEN];
+  int lineno = 0;
+
+  for (i = 0; i < ALIASLEN; i++)
+    unicode_aliases[i].uc = UNINAME_INVALID;
+
+  stream = fopen (namealiases_filename, "r");
+  if (stream == NULL)
+    {
+      fprintf (stderr, "error during fopen of '%s'\n", namealiases_filename);
+      exit (EXIT_FAILURE);
+    }
+
+  for (i = 0; i < ALIASLEN; i++)
+    {
+      int n;
+      int c;
+      unsigned int uc;
+
+      lineno++;
+      n = getfield (stream, field0, ';');
+      n += getfield (stream, field1, ';');
+      if (n == 0)
+        break;
+      if (n != 2)
+        {
+          fprintf (stderr, "short line in '%s':%d\n",
+                   namealiases_filename, lineno);
+          exit (EXIT_FAILURE);
+        }
+      for (; (c = getc (stream)), (c != EOF && c != '\n'); )
+        ;
+      uc = strtoul (field0, NULL, 16);
+      if (uc >= 0x110000)
+        {
+          fprintf (stderr, "index too large\n");
+          exit (EXIT_FAILURE);
+        }
+      unicode_aliases[i].name = xstrdup (field1);
+      unicode_aliases[i].uc = uc;
+    }
+  if (ferror (stream) || fclose (stream))
+    {
+      fprintf (stderr, "error reading from '%s'\n", namealiases_filename);
+      exit (1);
+    }
+}
+
 /* Perform an exhaustive test of the unicode_character_name function.  */
 static int
 test_name_lookup ()
@@ -246,6 +314,38 @@ test_inverse_lookup ()
   return error;
 }
 
+/* Perform a test of the unicode_name_character function for aliases.  */
+static int
+test_alias_lookup ()
+{
+  int error = 0;
+  unsigned int i;
+  char buf[UNINAME_MAX];
+
+  /* Verify all valid character names are recognized.  */
+  for (i = 0; i < ALIASLEN; i++)
+    if (unicode_aliases[i].uc != UNINAME_INVALID
+        /* Skip if the character has no canonical name (e.g. control
+           characters).  */
+        && unicode_character_name (unicode_aliases[i].uc, buf))
+      {
+        unsigned int result = unicode_name_character (unicode_aliases[i].name);
+        if (result != unicode_aliases[i].uc)
+          {
+            if (result == UNINAME_INVALID)
+              fprintf (stderr, "inverse name lookup of \"%s\" failed\n",
+                       unicode_aliases[i]);
+            else
+              fprintf (stderr,
+                       "inverse name lookup of \"%s\" returned 0x%04X\n",
+                       unicode_aliases[i], result);
+            error = 1;
+          }
+      }
+
+  return error;
+}
+
 int
 main (int argc, char *argv[])
 {
@@ -258,5 +358,11 @@ main (int argc, char *argv[])
   error |= test_name_lookup ();
   error |= test_inverse_lookup ();
 
+  if (argc > 2)
+    {
+      fill_aliases (argv[2]);
+      error |= test_alias_lookup ();
+    }
+
   return error;
 }
diff --git a/tests/uniname/test-uninames.sh b/tests/uniname/test-uninames.sh
index f26c275..0e6a018 100755
--- a/tests/uniname/test-uninames.sh
+++ b/tests/uniname/test-uninames.sh
@@ -1,2 +1,2 @@
 #!/bin/sh
-exec ./test-uninames${EXEEXT} "$srcdir/uniname/UnicodeDataNames.txt"
+exec ./test-uninames${EXEEXT} "$srcdir/uniname/UnicodeDataNames.txt" 
"$srcdir/uniname/NameAliases.txt"
-- 
2.1.0