Unicode filename support for Windows

[Masamichi HOSODA]

On Windows, lilypond can't treat unicode filenames.
On linux, it can (UTF-8 unicode filenames).

So, I'm trying to add a unicode filename support for Windows lilypond.
I attach the patch.

It replaces main(), and hooks filename related functions.
This converts between UTF-16 unicode (Windows) and
UTF-8 unicode (lilypond, libguile etc.).

As a result, lilypond can treat unicode filenames of *.ly, *.mid, *.ps.

However, it can't treat *.pdf, yet.
Ghostscript-8.70 that are included in the binary distribution of lilypond
can't treat the unicode filenames.
Ghostscript-9.10 or later can treat the unicode filenames.

>From bb35555f8bd6bcaf41de57e8981ac57a3ac0ef8c Mon Sep 17 00:00:00 2001
From: Masamichi Hosoda <address@hidden>
Date: Sat, 7 Mar 2015 18:42:57 +0900
Subject: [PATCH] Add unicode filename support for Windows

---
 flower/include/mingw-utf8-conv.hh |   7 ++
 flower/include/mingw-utf8-func.hh |  16 ++++
 flower/include/mingw-utf8-hook.hh |   6 ++
 flower/include/mingw-utf8.hh      |  11 +++
 flower/mingw-utf8-conv.cc         |  58 ++++++++++++
 flower/mingw-utf8-func.cc         | 157 +++++++++++++++++++++++++++++++++
 flower/mingw-utf8-hook.cc         | 179 ++++++++++++++++++++++++++++++++++++++
 flower/mingw-utf8-main.cc         |  84 ++++++++++++++++++
 lily/GNUmakefile                  |   1 +
 lily/main.cc                      |   4 +
 10 files changed, 523 insertions(+)
 create mode 100644 flower/include/mingw-utf8-conv.hh
 create mode 100644 flower/include/mingw-utf8-func.hh
 create mode 100644 flower/include/mingw-utf8-hook.hh
 create mode 100644 flower/include/mingw-utf8.hh
 create mode 100644 flower/mingw-utf8-conv.cc
 create mode 100644 flower/mingw-utf8-func.cc
 create mode 100644 flower/mingw-utf8-hook.cc
 create mode 100644 flower/mingw-utf8-main.cc

diff --git a/flower/include/mingw-utf8-conv.hh 
b/flower/include/mingw-utf8-conv.hh
new file mode 100644
index 0000000..1cabe21
--- /dev/null
+++ b/flower/include/mingw-utf8-conv.hh
@@ -0,0 +1,7 @@
+#ifndef __MINGW_UTF8_CONV_H__
+#define __MINGW_UTF8_CONV_H__
+
+std::vector<char> mingw_utf8_16to8 (const wchar_t *wc);
+std::vector<wchar_t> mingw_utf8_8to16 (const char *c);
+
+#endif // __MINGW_UTF8_CONV_H__
diff --git a/flower/include/mingw-utf8-func.hh 
b/flower/include/mingw-utf8-func.hh
new file mode 100644
index 0000000..d4a0a29
--- /dev/null
+++ b/flower/include/mingw-utf8-func.hh
@@ -0,0 +1,16 @@
+#ifndef __MINGW_UTF8_FUNC_H__
+#define __MINGW_UTF8_FUNC_H__
+
+FILE *utf8_fopen(const char *path, const char *mode);
+FILE *utf8_freopen (const char *path, const char *mode, FILE *stream);
+int utf8__stat (const char *path, struct _stat *buff);
+char *utf8_getcwd(char *buff, int size);
+char *utf8_getenv(const char *var);
+int utf8_open (const char *path, int flag, ...);
+int utf8_system (const char *command);
+int utf8__unlink (const char *path);
+gchar *utf8_g_locale_to_from_utf8 (const gchar *string, gssize len,
+                                   gsize *bytes_read, gssize *bytes_written,
+                                   GError **error);
+
+#endif // __MINGW_UTF8_FUNC_H__
diff --git a/flower/include/mingw-utf8-hook.hh 
b/flower/include/mingw-utf8-hook.hh
new file mode 100644
index 0000000..0eea976
--- /dev/null
+++ b/flower/include/mingw-utf8-hook.hh
@@ -0,0 +1,6 @@
+#ifndef __MINGW_UTF8_HOOK_H__
+#define __MINGW_UTF8_HOOK_H__
+
+void mingw_utf8_hook(void);
+
+#endif // __MINGW_UTF8_HOOK_H__
diff --git a/flower/include/mingw-utf8.hh b/flower/include/mingw-utf8.hh
new file mode 100644
index 0000000..feb2fab
--- /dev/null
+++ b/flower/include/mingw-utf8.hh
@@ -0,0 +1,11 @@
+#ifdef __MINGW32__
+
+#ifndef __MINGW_UTF8_H__
+#define __MINGW_UTF8_H__
+
+#define main(...) utf8_main(__VA_ARGS__)
+
+int utf8_main(int argc, char **argv, char **envp);
+
+#endif // __MINGW_UTF8_H__
+#endif // __MINGW32__
diff --git a/flower/mingw-utf8-conv.cc b/flower/mingw-utf8-conv.cc
new file mode 100644
index 0000000..fa04da9
--- /dev/null
+++ b/flower/mingw-utf8-conv.cc
@@ -0,0 +1,58 @@
+/*
+  This file is part of LilyPond, the GNU music typesetter.
+
+  Copyright (C) 2015 Masamichi Hosoda <address@hidden>
+
+  LilyPond is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  LilyPond is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with LilyPond.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifdef __MINGW32__
+
+#include <cwchar>
+#include <vector>
+#include <windows.h>
+
+#include "mingw-utf8-conv.hh"
+
+// Convert from UTF-16 to UTF-8
+std::vector<char> mingw_utf8_16to8 (const wchar_t *wc)
+{
+  int size = WideCharToMultiByte (CP_UTF8, 0, wc, -1, NULL, 0, NULL, NULL);
+  std::vector<char> retval (size);
+  if (size)
+    {
+      WideCharToMultiByte (CP_UTF8, 0, wc, -1,
+                          retval.data(), retval.size(), NULL, NULL);
+    }
+  else
+    retval.push_back ('\0');
+  return retval;
+}
+
+// Convert from UTF-8 to UTF-16
+std::vector<wchar_t> mingw_utf8_8to16 (const char *c)
+{
+  int size = MultiByteToWideChar (CP_UTF8, 0, c, -1, NULL, 0);
+  std::vector<wchar_t> retval (size);
+  if (size)
+    {
+      MultiByteToWideChar (CP_UTF8, 0, c, -1,
+                          retval.data(), retval.size());
+    }
+  else
+    retval.push_back (L'\0');
+  return retval;
+}
+
+#endif // __MINGW32__
diff --git a/flower/mingw-utf8-func.cc b/flower/mingw-utf8-func.cc
new file mode 100644
index 0000000..b19ead4
--- /dev/null
+++ b/flower/mingw-utf8-func.cc
@@ -0,0 +1,157 @@
+/*
+  This file is part of LilyPond, the GNU music typesetter.
+
+  Copyright (C) 2015 Masamichi Hosoda <address@hidden>
+
+  LilyPond is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  LilyPond is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with LilyPond.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifdef __MINGW32__
+
+#include <cwchar>
+#include <cstdio>
+#include <cstdlib>
+#include <cstdarg>
+#include <vector>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <glib.h>
+
+#include "mingw-utf8-func.hh"
+#include "mingw-utf8-conv.hh"
+
+// msvcrt fopen()
+FILE *utf8_fopen (const char *path, const char *mode)
+{
+  //printf("hook!! fopen (\"%s\", \"%s\")\n", path, mode);
+
+  std::vector<wchar_t> wpath (mingw_utf8_8to16 (path));
+  std::vector<wchar_t> wmode (mingw_utf8_8to16 (mode));
+  return _wfopen (wpath.data(), wmode.data());
+}
+
+// msvcrt freopen()
+FILE *utf8_freopen (const char *path, const char *mode, FILE *stream)
+{
+  //printf("hook!! freopen (\"%s\", \"%s\", stream)\n", path, mode);
+
+  std::vector<wchar_t> wpath (mingw_utf8_8to16 (path));
+  std::vector<wchar_t> wmode (mingw_utf8_8to16 (mode));
+  return _wfreopen (wpath.data(), wmode.data(), stream);
+}
+
+// msvcrt _stat() / stat()
+int utf8__stat (const char *path, struct _stat *buff)
+{
+  //printf("hook!! _stat (\"%s\", buff)\n", path);
+
+  std::vector<wchar_t> wpath (mingw_utf8_8to16 (path));
+  return _wstat (wpath.data(), buff);
+}
+
+// msvcrt getcwd() / _getcwd()
+char *utf8_getcwd(char *buff, int size)
+{
+  //printf("hook!! getcwd (0x%p, %d)\n", buff, size);
+
+  wchar_t *wp = _wgetcwd(NULL, 0);
+  std::vector<char> p (mingw_utf8_16to8 (wp));
+  free (wp);
+
+  if (buff == NULL)
+    buff = (char*) malloc (p.size());
+  strncpy(buff, p.data(), size);
+  buff[size]=0;
+  return buff;
+}
+
+// msvcrt getenv()
+char *utf8_getenv(const char *var)
+{
+  //fputs("hook!! getenv (\"", stdout);
+  //fputs(var, stdout);
+  //fputs("\")\n", stdout);
+
+  static std::vector<char> env_ret;
+  std::vector<wchar_t> wvar (mingw_utf8_8to16 (var));
+  wchar_t *e = _wgetenv(wvar.data());
+  if (e == NULL)
+    return NULL;
+  env_ret = mingw_utf8_16to8 (e);
+  return env_ret.data();
+}
+
+// msvcrt open() / _open()
+int utf8_open (const char *path, int flag, ...)
+{
+  //printf("hook!! open (\"%s\", flag, ...)\n", path);
+
+  int mode = 0777;
+  va_list list;
+  va_start (list, flag);
+  if ( flag & O_CREAT )
+    mode = va_arg (list, int);
+  va_end (list);
+
+  std::vector<wchar_t> wpath (mingw_utf8_8to16 (path));
+  return _wopen(wpath.data(), flag, mode);
+}
+
+// msvcrt system()
+int utf8_system (const char *command)
+{
+  //printf("hook!! system (\"%s\")\n", command);
+
+  std::vector<wchar_t> wcommand (mingw_utf8_8to16 (command));
+  return _wsystem(wcommand.data());
+}
+
+// msvcrt _unlink() / unlink()
+int utf8__unlink (const char *path)
+{
+  //printf("hook!! _unlink (\"%s\")\n", path);
+
+  std::vector<wchar_t> wpath (mingw_utf8_8to16 (path));
+  return _wunlink(wpath.data());
+}
+
+// glib g_locale_{to|from}_utf8()
+// from gconvert.c g_locale_{to|from}_utf8() and strdup_len()
+gchar *utf8_g_locale_to_from_utf8 (const gchar *string, gssize len,
+                                   gsize *bytes_read, gssize *bytes_written,
+                                   GError **error)
+{
+  //printf("hook!! g_locale_{to|from}_utf8\n");
+
+  gsize real_len;
+
+  if (len < 0)
+    real_len = strlen (string);
+  else
+    {
+      real_len = 0;
+
+      while (real_len < len && string[real_len])
+       real_len++;
+    }
+
+  if (bytes_read)
+    *bytes_read = real_len;
+  if (bytes_written)
+    *bytes_written = real_len;
+
+  return g_strndup (string, real_len);
+}
+
+#endif // __MINGW32__
diff --git a/flower/mingw-utf8-hook.cc b/flower/mingw-utf8-hook.cc
new file mode 100644
index 0000000..ddf73f1
--- /dev/null
+++ b/flower/mingw-utf8-hook.cc
@@ -0,0 +1,179 @@
+/*
+  This file is part of LilyPond, the GNU music typesetter.
+
+  Copyright (C) 2015 Masamichi Hosoda <address@hidden>
+
+  LilyPond is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  LilyPond is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with LilyPond.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifdef __MINGW32__
+
+#include <cwchar>
+#include <vector>
+#include <windows.h>
+#include <tlhelp32.h>
+#include <dbghelp.h>
+#include <glib.h>
+
+//#include <cstdio>
+
+#include "mingw-utf8-hook.hh"
+#include "mingw-utf8-func.hh"
+
+// Internal use
+namespace
+{
+  void hook_all (PROC func_old, PROC func_new);
+  void hook_one (PROC func_old, PROC func_new,
+                 HMODULE hmod, LPBYTE pbase);
+}
+
+// Address of hook target functions.
+extern "C"
+{
+  extern PROC _imp__fopen;
+  extern PROC _imp__freopen;
+  extern PROC _imp___stat;
+  extern PROC _imp__stat;
+  extern PROC _imp__getcwd;
+  extern PROC _imp___getcwd;
+  extern PROC _imp__getenv;
+  extern PROC _imp__open;
+  extern PROC _imp___open;
+  extern PROC _imp__system;
+  extern PROC _imp___unlink;
+  extern PROC _imp__unlink;
+  extern PROC _imp__g_locale_to_utf8;
+  extern PROC _imp__g_locale_from_utf8;
+}
+
+// Hook filename related functions.
+void mingw_utf8_hook (void)
+{
+  //printf("try to hook fopen\n");
+  hook_all (_imp__fopen, (PROC)utf8_fopen);
+  //printf("try to hook freopen\n");
+  hook_all (_imp__freopen, (PROC)utf8_freopen);
+  //printf("try to hook _stat\n");
+  hook_all (_imp___stat, (PROC)utf8__stat);
+  //printf("try to hook stat\n");
+  hook_all (_imp__stat, (PROC)utf8__stat);
+  //printf("try to hook getcwd\n");
+  hook_all (_imp__getcwd, (PROC)utf8_getcwd);
+  //printf("try to hook _getcwd\n");
+  hook_all (_imp___getcwd, (PROC)utf8_getcwd);
+  //printf("try to hook getenv\n");
+  hook_all (_imp__getenv, (PROC)utf8_getenv);
+  //printf("try to hook open\n");
+  hook_all (_imp__open, (PROC)utf8_open);
+  //printf("try to hook _open\n");
+  hook_all (_imp___open, (PROC)utf8_open);
+  //printf("try to hook system\n");
+  hook_all (_imp__system, (PROC)utf8_system);
+  //printf("try to hook _unlink\n");
+  hook_all (_imp___unlink, (PROC)utf8__unlink);
+  //printf("try to hook unlink\n");
+  hook_all (_imp__unlink, (PROC)utf8__unlink);
+  //printf("try to hook g_locale_to_utf8\n");
+  hook_all (_imp__g_locale_to_utf8, (PROC)utf8_g_locale_to_from_utf8);
+  //printf("try to hook g_locale_from_utf8\n");
+  hook_all (_imp__g_locale_from_utf8, (PROC)utf8_g_locale_to_from_utf8);
+}
+
+namespace
+{
+  // Hook all loaded modules.
+  void hook_all (PROC func_old, PROC func_new)
+  {
+    //printf ("hook_all enter old %p, new %p\n", func_old, func_new);
+    //fflush (stdout);
+
+    // Take a snapshot of modules.
+    HANDLE hmod_snap =
+      CreateToolhelp32Snapshot (TH32CS_SNAPMODULE, GetCurrentProcessId ());
+    if (hmod_snap == INVALID_HANDLE_VALUE)
+      return;
+
+    // Enumerate snapshotted modules.
+    MODULEENTRY32W me;
+    me.dwSize = sizeof (me);
+    BOOL bresult = Module32FirstW (hmod_snap, &me);
+
+    while (bresult)
+      {
+        //printf (" Module %S...\n", me.szModule);
+        //fflush (stdout);
+
+        // Hook one module.
+        hook_one(func_old, func_new, me.hModule, me.modBaseAddr);
+        bresult = Module32NextW (hmod_snap, &me);
+      }
+
+    //printf("hook_all exit\n");
+    //fflush(stdout);
+  }
+
+  // Hook one module.
+  void hook_one (PROC func_old, PROC func_new,
+                 HMODULE hmod, LPBYTE pbase)
+  {
+    ULONG size;
+
+    // Get import descriptor.
+    PIMAGE_IMPORT_DESCRIPTOR pimp_desc = (PIMAGE_IMPORT_DESCRIPTOR)
+      ImageDirectoryEntryToData (hmod, TRUE,
+                                 IMAGE_DIRECTORY_ENTRY_IMPORT, &size);
+    if (pimp_desc == NULL)
+      return;
+
+    // Enumerate import modules.
+    while (pimp_desc->Name)
+      {
+        // Get import address table.
+        PIMAGE_THUNK_DATA pthunk = (PIMAGE_THUNK_DATA) 
+          (pbase + pimp_desc->FirstThunk);
+
+        // Enumerate import address table.
+        while (pthunk->u1.Function)
+          {
+            // Get address of imported function.
+            PROC *p = (PROC*) &pthunk->u1.Function;
+
+            // Compare address of hook target.
+            if (*p == func_old)
+              {
+                //printf ("  Hit!! %p\n", *p);
+                //fflush (stdout);
+
+                DWORD before;
+                // Change memory protection in order to write.
+                VirtualProtect(p, sizeof(p),
+                               PAGE_EXECUTE_READWRITE, &before);
+                // Change hook target to new function.
+                WriteProcessMemory(GetCurrentProcess(),
+                                   p, &func_new, sizeof(func_new), NULL);
+                // Revert memory protection.
+                VirtualProtect(p, sizeof(p), before, &before);
+
+                //printf ("  Changed!!\n");
+                //fflush (stdout);
+              }
+            pthunk++;
+          }
+        pimp_desc++;
+      }
+  }
+}
+
+#endif // __MINGW32__
diff --git a/flower/mingw-utf8-main.cc b/flower/mingw-utf8-main.cc
new file mode 100644
index 0000000..47771a1
--- /dev/null
+++ b/flower/mingw-utf8-main.cc
@@ -0,0 +1,84 @@
+/*
+  This file is part of LilyPond, the GNU music typesetter.
+
+  Copyright (C) 2015 Masamichi Hosoda <address@hidden>
+
+  LilyPond is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  LilyPond is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with LilyPond.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifdef __MINGW32__
+
+#include <cwchar>
+#include <vector>
+
+#include "mingw-utf8.hh"
+#include "mingw-utf8-hook.hh"
+#include "mingw-utf8-conv.hh"
+
+extern "C"
+int __wgetmainargs (int*, wchar_t***, wchar_t***, int, int*);
+
+#undef main
+
+// Replaced main()
+int main (int, char**, char**)
+{
+  // Hook filename related functions.
+  mingw_utf8_hook();
+
+  // Get UTF-16 commandline arguments and environment variables.
+  int argc, si=0;
+  wchar_t **wargv;
+  wchar_t **wenvp;
+  __wgetmainargs (&argc, &wargv, &wenvp, 1, &si);
+
+  // Convert commandline arguments from UTF-16 to UTF-8.
+  std::vector<std::vector<char> > argv_vvc (argc);
+  std::vector<char*> argv_vc (argc);
+
+  for (int i = 0; i < argc; i++)
+    {
+      argv_vvc.at(i) = mingw_utf8_16to8 (wargv[i]);
+      argv_vc.at(i) = argv_vvc.at(i).data();
+    }
+  argv_vc.push_back (NULL);
+
+  // Count environment variables.
+  wchar_t **wenv_tmp=wenvp;
+  int env_count=0;
+  while(*wenv_tmp)
+    {
+      wenv_tmp++;
+      env_count++;
+    }
+
+  // Convert environment variables from UTF-16 to UTF-8.
+  std::vector<std::vector<char> > env_vvc (env_count);
+  std::vector<char*> env_vc (env_count);
+
+  for(int i = 0; i < env_count; i++)
+    {
+      env_vvc.at(i) = mingw_utf8_16to8 (wenvp[i]);
+      env_vc.at(i) = env_vvc.at(i).data();
+    }
+  env_vc.push_back (NULL);
+
+  // Call original main().
+  char **argv = argv_vc.data();
+  char **envp = env_vc.data();
+
+  return utf8_main (argc, argv, envp);
+}
+
+#endif // __MINGW32__
diff --git a/lily/GNUmakefile b/lily/GNUmakefile
index 6d9afd7..25ab6e3 100644
--- a/lily/GNUmakefile
+++ b/lily/GNUmakefile
@@ -25,6 +25,7 @@ CXXFLAGS += -Woverloaded-virtual
 #
 
 ifeq ($(PLATFORM_WINDOWS),yes)
+CONFIG_LDFLAGS += -ldbghelp
 WINDRES_FLAGS += -DLilyPond=0 -DLY=1
 O_FILES += $(outdir)/lilypond.rc.o
 $(outdir)/lilypond: $(outdir)/lilypond.rc.o
diff --git a/lily/main.cc b/lily/main.cc
index 808274c..e552702 100644
--- a/lily/main.cc
+++ b/lily/main.cc
@@ -56,6 +56,10 @@ using namespace std;
 #include "version.hh"
 #include "warn.hh"
 
+#ifdef __MINGW32__
+#include "mingw-utf8.hh"
+#endif
+
 /*
  * Global options that can be overridden through command line.
  */
-- 
2.1.4