Merge pull request #183 from archshift/lowpath

Add support for UTF-16 strings for LowPaths in FS:USER

src/common/string_util.cpp

@@ -9,6 +9,7 @@
 
 #ifdef _WIN32
     #include <Windows.h>
+    #include <codecvt>
 #else
     #include <iconv.h>
 #endif
@@ -411,7 +412,19 @@ std::string UriEncode(const std::string & sSrc)
 
 #ifdef _WIN32
 
-std::string UTF16ToUTF8(const std::wstring& input)
+std::string UTF16ToUTF8(const std::u16string& input)
+{
+    std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> convert;
+    return convert.to_bytes(input);
+}
+
+std::u16string UTF8ToUTF16(const std::string& input)
+{
+    std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> convert;
+    return convert.from_bytes(input);
+}
+
+static std::string UTF16ToUTF8(const std::wstring& input)
 {
     auto const size = WideCharToMultiByte(CP_UTF8, 0, input.data(), input.size(), nullptr, 0, nullptr, nullptr);
 
@@ -424,7 +437,7 @@ std::string UTF16ToUTF8(const std::wstring& input)
     return output;
 }
 
-std::wstring CPToUTF16(u32 code_page, const std::string& input)
+static std::wstring CPToUTF16(u32 code_page, const std::string& input)
 {
     auto const size = MultiByteToWideChar(code_page, 0, input.data(), input.size(), nullptr, 0);
 
@@ -437,7 +450,7 @@ std::wstring CPToUTF16(u32 code_page, const std::string& input)
     return output;
 }
 
-std::wstring UTF8ToUTF16(const std::string& input)
+std::wstring UTF8ToUTF16W(const std::string &input)
 {
     return CPToUTF16(CP_UTF8, input);
 }
@@ -455,19 +468,21 @@ std::string CP1252ToUTF8(const std::string& input)
 #else
 
 template <typename T>
-std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
+static std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
 {
     std::string result;
 
     iconv_t const conv_desc = iconv_open("UTF-8", fromcode);
-    if ((iconv_t)-1 == conv_desc)
+    if ((iconv_t)(-1) == conv_desc)
     {
         ERROR_LOG(COMMON, "Iconv initialization failure [%s]: %s", fromcode, strerror(errno));
+        iconv_close(conv_desc);
+        return {};
     }
-    else
+
-    {
+    const size_t in_bytes = sizeof(T) * input.size();
-        size_t const in_bytes = sizeof(T) * input.size();
+    // Multiply by 4, which is the max number of bytes to encode a codepoint
-        size_t const out_buffer_size = 4 * in_bytes;
+    const size_t out_buffer_size = 4 * in_bytes;
 
     std::string out_buffer;
     out_buffer.resize(out_buffer_size);
@@ -477,17 +492,17 @@ std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
     auto dst_buffer = &out_buffer[0];
     size_t dst_bytes = out_buffer.size();
 
-        while (src_bytes != 0)
+    while (0 != src_bytes)
     {
         size_t const iconv_result = iconv(conv_desc, (char**)(&src_buffer), &src_bytes,
             &dst_buffer, &dst_bytes);
 
-            if ((size_t)-1 == iconv_result)
+        if (static_cast<size_t>(-1) == iconv_result)
         {
             if (EILSEQ == errno || EINVAL == errno)
             {
                 // Try to skip the bad character
-                    if (src_bytes != 0)
+                if (0 != src_bytes)
                 {
                     --src_bytes;
                     ++src_buffer;
@@ -505,11 +520,71 @@ std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
     out_buffer.swap(result);
     
     iconv_close(conv_desc);
-    }
     
     return result;
 }
 
+std::u16string UTF8ToUTF16(const std::string& input)
+{
+    std::u16string result;
+
+    iconv_t const conv_desc = iconv_open("UTF-16", "UTF-8");
+    if ((iconv_t)(-1) == conv_desc)
+    {
+        ERROR_LOG(COMMON, "Iconv initialization failure [UTF-8]: %s", strerror(errno));
+        iconv_close(conv_desc);
+        return {};
+    }
+
+    const size_t in_bytes = sizeof(char) * input.size();
+    // Multiply by 4, which is the max number of bytes to encode a codepoint
+    const size_t out_buffer_size = 4 * sizeof(char16_t) * in_bytes;
+
+    std::u16string out_buffer;
+    out_buffer.resize(out_buffer_size);
+
+    char* src_buffer = const_cast<char*>(&input[0]);
+    size_t src_bytes = in_bytes;
+    char* dst_buffer = (char*)(&out_buffer[0]);
+    size_t dst_bytes = out_buffer.size();
+
+    while (0 != src_bytes)
+    {
+        size_t const iconv_result = iconv(conv_desc, &src_buffer, &src_bytes,
+                                          &dst_buffer, &dst_bytes);
+
+        if (static_cast<size_t>(-1) == iconv_result)
+        {
+            if (EILSEQ == errno || EINVAL == errno)
+            {
+                // Try to skip the bad character
+                if (0 != src_bytes)
+                {
+                    --src_bytes;
+                    ++src_buffer;
+                }
+            }
+            else
+            {
+                ERROR_LOG(COMMON, "iconv failure [UTF-8]: %s", strerror(errno));
+                break;
+            }
+        }
+    }
+
+    out_buffer.resize(out_buffer_size - dst_bytes);
+    out_buffer.swap(result);
+
+    iconv_close(conv_desc);
+    
+    return result;
+}
+
+std::string UTF16ToUTF8(const std::u16string& input)
+{
+    return CodeToUTF8("UTF-16", input);
+}
+
 std::string CP1252ToUTF8(const std::string& input)
 {
     //return CodeToUTF8("CP1252//TRANSLIT", input);
@@ -523,19 +598,6 @@ std::string SHIFTJISToUTF8(const std::string& input)
     return CodeToUTF8("SJIS", input);
 }
 
-std::string UTF16ToUTF8(const std::wstring& input)
-{
-    std::string result =
-    //    CodeToUTF8("UCS-2", input);
-    //    CodeToUTF8("UCS-2LE", input);
-    //    CodeToUTF8("UTF-16", input);
-        CodeToUTF8("UTF-16LE", input);
-
-    // TODO: why is this needed?
-    result.erase(std::remove(result.begin(), result.end(), 0x00), result.end());
-    return result;
-}
-
 #endif
 
 }

src/common/string_util.h

@@ -89,20 +89,22 @@ std::string ReplaceAll(std::string result, const std::string& src, const std::st
 std::string UriDecode(const std::string & sSrc);
 std::string UriEncode(const std::string & sSrc);
 
+std::string UTF16ToUTF8(const std::u16string& input);
+std::u16string UTF8ToUTF16(const std::string& input);
+
 std::string CP1252ToUTF8(const std::string& str);
 std::string SHIFTJISToUTF8(const std::string& str);
-std::string UTF16ToUTF8(const std::wstring& str);
 
 #ifdef _WIN32
 
-std::wstring UTF8ToUTF16(const std::string& str);
+std::wstring UTF8ToUTF16W(const std::string& str);
 
 #ifdef _UNICODE
 inline std::string TStrToUTF8(const std::wstring& str)
 { return UTF16ToUTF8(str); }
 
 inline std::wstring UTF8ToTStr(const std::string& str)
-{ return UTF8ToUTF16(str); }
+{ return UTF8ToUTF16W(str); }
 #else
 inline std::string TStrToUTF8(const std::string& str)
 { return str; }

src/core/file_sys/archive.h

@@ -7,11 +7,13 @@
 #include <memory>
 
 #include "common/common_types.h"
+#include "common/string_util.h"
 #include "common/bit_field.h"
 
 #include "core/file_sys/file.h"
 #include "core/file_sys/directory.h"
 
+#include "core/mem_map.h"
 #include "core/hle/kernel/kernel.h"
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -19,6 +21,15 @@
 
 namespace FileSys {
 
+// Path string type
+enum LowPathType : u32 {
+    Invalid = 0,
+    Empty   = 1,
+    Binary  = 2,
+    Char    = 3,
+    Wchar   = 4
+};
+
 union Mode {
     u32 hex;
     BitField<0, 1, u32> read_flag;
@@ -26,6 +37,94 @@ union Mode {
     BitField<2, 1, u32> create_flag;
 };
 
+class Path {
+public:
+
+    Path():
+        type(Invalid)
+    {
+    }
+
+    Path(LowPathType type, u32 size, u32 pointer):
+        type(type)
+    {
+        switch (type) {
+            case Binary:
+            {
+                u8* data = Memory::GetPointer(pointer);
+                binary = std::vector<u8>(data, data + size);
+                break;
+            }
+            case Char:
+            {
+                const char* data = reinterpret_cast<const char*>(Memory::GetPointer(pointer));
+                string = std::string(data, size - 1); // Data is always null-terminated.
+                break;
+            }
+            case Wchar:
+            {
+                const char16_t* data = reinterpret_cast<const char16_t*>(Memory::GetPointer(pointer));
+                u16str = std::u16string(data, size/2 - 1); // Data is always null-terminated.
+                break;
+            }
+        }
+    }
+
+    LowPathType GetType() const {
+        return type;
+    }
+
+    const std::string AsString() const {
+        switch (GetType()) {
+            case Char:
+                return string;
+            case Wchar:
+                return Common::UTF16ToUTF8(u16str);
+            case Empty:
+                return {};
+            default:
+                ERROR_LOG(KERNEL, "LowPathType cannot be converted to string!");
+                return {};
+        }
+    }
+
+    const std::u16string AsU16Str() const {
+        switch (GetType()) {
+            case Char:
+                return Common::UTF8ToUTF16(string);
+            case Wchar:
+                return u16str;
+            case Empty:
+                return {};
+            default:
+                ERROR_LOG(KERNEL, "LowPathType cannot be converted to u16string!");
+                return {};
+        }
+    }
+
+    const std::vector<u8> AsBinary() const {
+        switch (GetType()) {
+            case Binary:
+                return binary;
+            case Char:
+                return std::vector<u8>(string.begin(), string.end());
+            case Wchar:
+                return std::vector<u8>(u16str.begin(), u16str.end());
+            case Empty:
+                return {};
+            default:
+                ERROR_LOG(KERNEL, "LowPathType cannot be converted to binary!");
+                return {};
+        }
+    }
+
+private:
+    LowPathType type;
+    std::vector<u8> binary;
+    std::string string;
+    std::u16string u16str;
+};
+
 class Archive : NonCopyable {
 public:
     /// Supported archive types

src/core/hle/service/fs_user.cpp

@@ -5,6 +5,7 @@
 #include "common/common.h"
 
 #include "fs_user.h"
+#include "common/string_util.h"
 #include "core/settings.h"
 #include "core/hle/kernel/archive.h"
 
@@ -13,20 +14,6 @@
 
 namespace FS_User {
 
-// Command to access archive file
-enum class LowPathType : u32 {
-    Invalid = 0,
-    Empty   = 1,
-    Binary  = 2,
-    Char    = 3,
-    Wchar   = 4
-};
-
-std::string GetStringFromCmdBuff(const u32 pointer, const u32 size) {
-    auto data = reinterpret_cast<const char*>(Memory::GetPointer(pointer));
-    return std::string(data, size - 1);
-}
-
 // We currently return 0 for success and -1 for failure in cmd_buff[1].  -1 was chosen because it
 // puts all the sections of the http://3dbrew.org/wiki/Error_codes to something non-zero, to make
 // sure we don't mislead the application into thinking something worked.
@@ -44,32 +31,36 @@ void Initialize(Service::Interface* self) {
 void OpenFile(Service::Interface* self) {
     u32* cmd_buff = Service::GetCommandBuffer();
 
-    u32 transaction = cmd_buff[1];
     // TODO(Link Mauve): cmd_buff[2], aka archive handle lower word, isn't used according to
     // 3dmoo's or ctrulib's implementations.  Triple check if it's really the case.
     Handle archive_handle = static_cast<Handle>(cmd_buff[3]);
-    LowPathType type = static_cast<LowPathType>(cmd_buff[4]);
+    auto filename_type    = static_cast<FileSys::LowPathType>(cmd_buff[4]);
-    u32 size = cmd_buff[5];
+    u32 filename_size     = cmd_buff[5];
     FileSys::Mode mode; mode.hex = cmd_buff[6];
     u32 attributes        = cmd_buff[7]; // TODO(Link Mauve): do something with those attributes.
-    u32 pointer = cmd_buff[9];
+    u32 filename_ptr      = cmd_buff[9];
 
-    if (type != LowPathType::Char) {
+    FileSys::Path file_path(filename_type, filename_size, filename_ptr);
-        ERROR_LOG(KERNEL, "file LowPath type other than char is currently unsupported");
+    std::string file_string;
-        cmd_buff[1] = -1;
+    switch (file_path.GetType()) {
+    case FileSys::Char:
+    case FileSys::Wchar:
+        file_string = file_path.AsString();
+        break;
+    default:
+        WARN_LOG(KERNEL, "file LowPath type is currently unsupported; returning archive handle instead");
         return;
     }
 
-    std::string file_name = GetStringFromCmdBuff(pointer, size);
+    DEBUG_LOG(KERNEL, "type=%d size=%d mode=%d attrs=%d data=%s",
+              filename_type, filename_size, mode, attributes, file_string.c_str());
 
-    DEBUG_LOG(KERNEL, "type=%d size=%d mode=%d attrs=%d data=%s", type, size, mode, attributes, file_name.c_str());
+    Handle handle = Kernel::OpenFileFromArchive(archive_handle, file_string, mode);
-
-    Handle handle = Kernel::OpenFileFromArchive(archive_handle, file_name, mode);
     if (handle) {
         cmd_buff[1] = 0;
         cmd_buff[3] = handle;
     } else {
-        ERROR_LOG(KERNEL, "failed to get a handle for file %s", file_name.c_str());
+        ERROR_LOG(KERNEL, "failed to get a handle for file %s", file_string.c_str());
         // TODO(Link Mauve): check for the actual error values, this one was just chosen arbitrarily.
         cmd_buff[1] = -1;
     }
@@ -80,31 +71,25 @@ void OpenFile(Service::Interface* self) {
 void OpenFileDirectly(Service::Interface* self) {
     u32* cmd_buff = Service::GetCommandBuffer();
 
-    u32 transaction = cmd_buff[1];
     auto archive_id       = static_cast<FileSys::Archive::IdCode>(cmd_buff[2]);
-    LowPathType archive_type = static_cast<LowPathType>(cmd_buff[3]);
+    auto archivename_type = static_cast<FileSys::LowPathType>(cmd_buff[3]);
-    u32 archive_size = cmd_buff[4];
+    u32 archivename_size  = cmd_buff[4];
-    LowPathType file_type = static_cast<LowPathType>(cmd_buff[5]);
+    auto filename_type    = static_cast<FileSys::LowPathType>(cmd_buff[5]);
-    u32 size = cmd_buff[6];
+    u32 filename_size     = cmd_buff[6];
     FileSys::Mode mode; mode.hex = cmd_buff[7];
     u32 attributes        = cmd_buff[8]; // TODO(Link Mauve): do something with those attributes.
-    u32 archive_pointer = cmd_buff[10];
+    u32 archivename_ptr   = cmd_buff[10];
-    u32 pointer = cmd_buff[12];
+    u32 filename_ptr      = cmd_buff[12];
 
-    if (archive_type != LowPathType::Empty) {
+    DEBUG_LOG(KERNEL, "archive_type=%d archive_size=%d file_type=%d file_size=%d file_mode=%d file_attrs=%d",
+              archivename_type, archivename_size, filename_type, filename_size, mode, attributes);
+
+    if (archivename_type != FileSys::Empty) {
         ERROR_LOG(KERNEL, "archive LowPath type other than empty is currently unsupported");
         cmd_buff[1] = -1;
         return;
     }
 
-    std::string archive_name = GetStringFromCmdBuff(archive_pointer, archive_size);
-    std::string file_name = GetStringFromCmdBuff(pointer, size);
-
-    DEBUG_LOG(KERNEL, "archive_type=%d archive_size=%d archive_data=%s "
-                      "file_type=%d file_size=%d file_mode=%d file_attrs=%d file_data=%s",
-              archive_type, archive_size, archive_name.c_str(),
-              file_type, size, mode, attributes, file_name.c_str());
-
     // TODO(Link Mauve): check if we should even get a handle for the archive, and don't leak it.
     Handle archive_handle = Kernel::OpenArchive(archive_id);
     if (archive_handle) {
@@ -112,23 +97,30 @@ void OpenFileDirectly(Service::Interface* self) {
         // cmd_buff[2] isn't used according to 3dmoo's implementation.
         cmd_buff[3] = archive_handle;
     } else {
-        ERROR_LOG(KERNEL, "failed to get a handle for archive %s", archive_name.c_str());
+        ERROR_LOG(KERNEL, "failed to get a handle for archive");
         // TODO(Link Mauve): check for the actual error values, this one was just chosen arbitrarily.
         cmd_buff[1] = -1;
         return;
     }
 
-    if (file_type != LowPathType::Char) {
+    FileSys::Path file_path(filename_type, filename_size, filename_ptr);
-        WARN_LOG(KERNEL, "file LowPath type other than char is currently unsupported; returning archive handle instead");
+    std::string file_string;
+    switch (file_path.GetType()) {
+    case FileSys::Char:
+    case FileSys::Wchar:
+        file_string = file_path.AsString();
+        break;
+    default:
+        WARN_LOG(KERNEL, "file LowPath type is currently unsupported; returning archive handle instead");
         return;
     }
 
-    Handle handle = Kernel::OpenFileFromArchive(archive_handle, file_name, mode);
+    Handle handle = Kernel::OpenFileFromArchive(archive_handle, file_string, mode);
     if (handle) {
         cmd_buff[1] = 0;
         cmd_buff[3] = handle;
     } else {
-        ERROR_LOG(KERNEL, "failed to get a handle for file %s", file_name.c_str());
+        ERROR_LOG(KERNEL, "failed to get a handle for file %s", file_string.c_str());
         // TODO(Link Mauve): check for the actual error values, this one was just chosen arbitrarily.
         cmd_buff[1] = -1;
     }
@@ -153,21 +145,25 @@ void CreateDirectory(Service::Interface* self) {
     // TODO: cmd_buff[2], aka archive handle lower word, isn't used according to
     // 3dmoo's or ctrulib's implementations.  Triple check if it's really the case.
     Handle archive_handle = static_cast<Handle>(cmd_buff[3]);
-    LowPathType type = static_cast<LowPathType>(cmd_buff[4]);
+    auto dirname_type = static_cast<FileSys::LowPathType>(cmd_buff[4]);
-    u32 name_size = cmd_buff[5];
+    u32 dirname_size = cmd_buff[5];
-    u32 name_offset = cmd_buff[8];
+    u32 dirname_ptr = cmd_buff[8];
 
-    if (type != LowPathType::Char) {
+    FileSys::Path dir_path(dirname_type, dirname_size, dirname_ptr);
-        ERROR_LOG(KERNEL, "directory LowPath type other than char is currently unsupported");
+    std::string dir_string;
+    switch (dir_path.GetType()) {
+    case FileSys::Char:
+    case FileSys::Wchar:
+        dir_string = dir_path.AsString();
+        break;
+    default:
         cmd_buff[1] = -1;
         return;
     }
 
-    std::string dir_name = GetStringFromCmdBuff(name_offset, name_size);
+    DEBUG_LOG(KERNEL, "type=%d size=%d data=%s", dirname_type, dirname_size, dir_string.c_str());
 
-    DEBUG_LOG(KERNEL, "type=%d size=%d data=%s", type, name_size, dir_name.c_str());
+    cmd_buff[1] = Kernel::CreateDirectoryFromArchive(archive_handle, dir_string);
-
-    cmd_buff[1] = Kernel::CreateDirectoryFromArchive(archive_handle, dir_name);
 
     DEBUG_LOG(KERNEL, "called");
 }
@@ -178,26 +174,30 @@ void OpenDirectory(Service::Interface* self) {
     // TODO(Link Mauve): cmd_buff[2], aka archive handle lower word, isn't used according to
     // 3dmoo's or ctrulib's implementations.  Triple check if it's really the case.
     Handle archive_handle = static_cast<Handle>(cmd_buff[2]);
-    LowPathType type = static_cast<LowPathType>(cmd_buff[3]);
+    auto dirname_type = static_cast<FileSys::LowPathType>(cmd_buff[3]);
-    u32 size = cmd_buff[4];
+    u32 dirname_size = cmd_buff[4];
-    u32 pointer = cmd_buff[6];
+    u32 dirname_ptr = cmd_buff[6];
 
-    if (type != LowPathType::Char) {
+    FileSys::Path dir_path(dirname_type, dirname_size, dirname_ptr);
-        ERROR_LOG(KERNEL, "directory LowPath type other than char is currently unsupported");
+    std::string dir_string;
+    switch (dir_path.GetType()) {
+    case FileSys::Char:
+    case FileSys::Wchar:
+        dir_string = dir_path.AsString();
+        break;
+    default:
         cmd_buff[1] = -1;
         return;
     }
 
-    std::string dir_name = GetStringFromCmdBuff(pointer, size);
+    DEBUG_LOG(KERNEL, "type=%d size=%d data=%s", dirname_type, dirname_size, dir_string.c_str());
 
-    DEBUG_LOG(KERNEL, "type=%d size=%d data=%s", type, size, dir_name.c_str());
+    Handle handle = Kernel::OpenDirectoryFromArchive(archive_handle, dir_string);
-
-    Handle handle = Kernel::OpenDirectoryFromArchive(archive_handle, dir_name);
     if (handle) {
         cmd_buff[1] = 0;
         cmd_buff[3] = handle;
     } else {
-        ERROR_LOG(KERNEL, "failed to get a handle for directory %s", dir_name.c_str());
+        ERROR_LOG(KERNEL, "failed to get a handle for directory %s", dir_string.c_str());
         // TODO(Link Mauve): check for the actual error values, this one was just chosen arbitrarily.
         cmd_buff[1] = -1;
     }
@@ -208,28 +208,26 @@ void OpenDirectory(Service::Interface* self) {
 void OpenArchive(Service::Interface* self) {
     u32* cmd_buff = Service::GetCommandBuffer();
 
-    auto arch_id = static_cast<FileSys::Archive::IdCode>(cmd_buff[1]);
+    auto archive_id = static_cast<FileSys::Archive::IdCode>(cmd_buff[1]);
-    LowPathType type = static_cast<LowPathType>(cmd_buff[2]);
+    auto archivename_type = static_cast<FileSys::LowPathType>(cmd_buff[2]);
-    u32 size = cmd_buff[3];
+    u32 archivename_size = cmd_buff[3];
-    u32 pointer = cmd_buff[5];
+    u32 archivename_ptr = cmd_buff[5];
 
-    if (type != LowPathType::Empty) {
+    DEBUG_LOG(KERNEL, "type=%d size=%d", archivename_type, archivename_size);
+
+    if (archivename_type != FileSys::Empty) {
         ERROR_LOG(KERNEL, "archive LowPath type other than empty is currently unsupported");
         cmd_buff[1] = -1;
         return;
     }
 
-    std::string archive_name = GetStringFromCmdBuff(pointer, size);
+    Handle handle = Kernel::OpenArchive(archive_id);
-
-    DEBUG_LOG(KERNEL, "type=%d size=%d data=%s", type, size, archive_name.c_str());
-
-    Handle handle = Kernel::OpenArchive(arch_id);
     if (handle) {
         cmd_buff[1] = 0;
         // cmd_buff[2] isn't used according to 3dmoo's implementation.
         cmd_buff[3] = handle;
     } else {
-        ERROR_LOG(KERNEL, "failed to get a handle for archive %s", archive_name.c_str());
+        ERROR_LOG(KERNEL, "failed to get a handle for archive");
         // TODO(Link Mauve): check for the actual error values, this one was just chosen arbitrarily.
         cmd_buff[1] = -1;
     }