citra-emu
/
citra
Archived
1
0
Fork 0
Code

Merge pull request #183 from archshift/lowpath 

Add support for UTF-16 strings for LowPaths in FS:USER
This commit is contained in:
bunnei 2014-11-12 22:56:27 -05:00
parent a490490336 1f7c4ab7f6
commit 3cfdabb2e3
4 changed files with 295 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
src/common/string_util.cpp
View File

@ -9,6 +9,7 @@
#ifdef _WIN32
#include <Windows.h>
#include <codecvt>
#else
#include <iconv.h>
#endif
@ -411,7 +412,19 @@ std::string UriEncode(const std::string & sSrc)
#ifdef _WIN32
std::string UTF16ToUTF8(const std::wstring& input)
std::string UTF16ToUTF8(const std::u16string& input)
{
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> convert;
return convert.to_bytes(input);
}
std::u16string UTF8ToUTF16(const std::string& input)
{
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> convert;
return convert.from_bytes(input);
}
static std::string UTF16ToUTF8(const std::wstring& input)
{
auto const size = WideCharToMultiByte(CP_UTF8, 0, input.data(), input.size(), nullptr, 0, nullptr, nullptr);
@ -424,7 +437,7 @@ std::string UTF16ToUTF8(const std::wstring& input)
return output;
}
std::wstring CPToUTF16(u32 code_page, const std::string& input)
static std::wstring CPToUTF16(u32 code_page, const std::string& input)
{
auto const size = MultiByteToWideChar(code_page, 0, input.data(), input.size(), nullptr, 0);
@ -437,7 +450,7 @@ std::wstring CPToUTF16(u32 code_page, const std::string& input)
return output;
}
std::wstring UTF8ToUTF16(const std::string& input)
std::wstring UTF8ToUTF16W(const std::string &input)
{
return CPToUTF16(CP_UTF8, input);
}
@ -455,61 +468,123 @@ std::string CP1252ToUTF8(const std::string& input)
#else
template <typename T>
std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
static std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
{
std::string result;
iconv_t const conv_desc = iconv_open("UTF-8", fromcode);
if ((iconv_t)-1 == conv_desc)
if ((iconv_t)(-1) == conv_desc)
{
ERROR_LOG(COMMON, "Iconv initialization failure [%s]: %s", fromcode, strerror(errno));
iconv_close(conv_desc);
return {};
}
else
const size_t in_bytes = sizeof(T) * input.size();
// Multiply by 4, which is the max number of bytes to encode a codepoint
const size_t out_buffer_size = 4 * in_bytes;
std::string out_buffer;
out_buffer.resize(out_buffer_size);
auto src_buffer = &input[0];
size_t src_bytes = in_bytes;
auto dst_buffer = &out_buffer[0];
size_t dst_bytes = out_buffer.size();
while (0 != src_bytes)
{
size_t const in_bytes = sizeof(T) * input.size();
size_t const out_buffer_size = 4 * in_bytes;
size_t const iconv_result = iconv(conv_desc, (char**)(&src_buffer), &src_bytes,
&dst_buffer, &dst_bytes);
std::string out_buffer;
out_buffer.resize(out_buffer_size);
auto src_buffer = &input[0];
size_t src_bytes = in_bytes;
auto dst_buffer = &out_buffer[0];
size_t dst_bytes = out_buffer.size();
while (src_bytes != 0)
if (static_cast<size_t>(-1) == iconv_result)
{
size_t const iconv_result = iconv(conv_desc, (char**)(&src_buffer), &src_bytes,
&dst_buffer, &dst_bytes);
if ((size_t)-1 == iconv_result)
if (EILSEQ == errno || EINVAL == errno)
{
if (EILSEQ == errno || EINVAL == errno)
// Try to skip the bad character
if (0 != src_bytes)
{
// Try to skip the bad character
if (src_bytes != 0)
{
--src_bytes;
++src_buffer;
}
}
else
{
ERROR_LOG(COMMON, "iconv failure [%s]: %s", fromcode, strerror(errno));
break;
--src_bytes;
++src_buffer;
}
}
else
{
ERROR_LOG(COMMON, "iconv failure [%s]: %s", fromcode, strerror(errno));
break;
}
}
out_buffer.resize(out_buffer_size - dst_bytes);
out_buffer.swap(result);
iconv_close(conv_desc);
}
out_buffer.resize(out_buffer_size - dst_bytes);
out_buffer.swap(result);
iconv_close(conv_desc);
return result;
}
std::u16string UTF8ToUTF16(const std::string& input)
{
std::u16string result;
iconv_t const conv_desc = iconv_open("UTF-16", "UTF-8");
if ((iconv_t)(-1) == conv_desc)
{
ERROR_LOG(COMMON, "Iconv initialization failure [UTF-8]: %s", strerror(errno));
iconv_close(conv_desc);
return {};
}
const size_t in_bytes = sizeof(char) * input.size();
// Multiply by 4, which is the max number of bytes to encode a codepoint
const size_t out_buffer_size = 4 * sizeof(char16_t) * in_bytes;
std::u16string out_buffer;
out_buffer.resize(out_buffer_size);
char* src_buffer = const_cast<char*>(&input[0]);
size_t src_bytes = in_bytes;
char* dst_buffer = (char*)(&out_buffer[0]);
size_t dst_bytes = out_buffer.size();
while (0 != src_bytes)
{
size_t const iconv_result = iconv(conv_desc, &src_buffer, &src_bytes,
&dst_buffer, &dst_bytes);
if (static_cast<size_t>(-1) == iconv_result)
{
if (EILSEQ == errno || EINVAL == errno)
{
// Try to skip the bad character
if (0 != src_bytes)
{
--src_bytes;
++src_buffer;
}
}
else
{
ERROR_LOG(COMMON, "iconv failure [UTF-8]: %s", strerror(errno));
break;
}
}
}
out_buffer.resize(out_buffer_size - dst_bytes);
out_buffer.swap(result);
iconv_close(conv_desc);
return result;
}
std::string UTF16ToUTF8(const std::u16string& input)
{
return CodeToUTF8("UTF-16", input);
}
std::string CP1252ToUTF8(const std::string& input)
{
//return CodeToUTF8("CP1252//TRANSLIT", input);
@ -523,19 +598,6 @@ std::string SHIFTJISToUTF8(const std::string& input)
return CodeToUTF8("SJIS", input);
}
std::string UTF16ToUTF8(const std::wstring& input)
{
std::string result =
// CodeToUTF8("UCS-2", input);
// CodeToUTF8("UCS-2LE", input);
// CodeToUTF8("UTF-16", input);
CodeToUTF8("UTF-16LE", input);
// TODO: why is this needed?
result.erase(std::remove(result.begin(), result.end(), 0x00), result.end());
return result;
}
#endif
}

8
src/common/string_util.h
View File

@ -89,20 +89,22 @@ std::string ReplaceAll(std::string result, const std::string& src, const std::st
std::string UriDecode(const std::string & sSrc);
std::string UriEncode(const std::string & sSrc);
std::string UTF16ToUTF8(const std::u16string& input);
std::u16string UTF8ToUTF16(const std::string& input);
std::string CP1252ToUTF8(const std::string& str);
std::string SHIFTJISToUTF8(const std::string& str);
std::string UTF16ToUTF8(const std::wstring& str);
#ifdef _WIN32
std::wstring UTF8ToUTF16(const std::string& str);
std::wstring UTF8ToUTF16W(const std::string& str);
#ifdef _UNICODE
inline std::string TStrToUTF8(const std::wstring& str)
{ return UTF16ToUTF8(str); }
inline std::wstring UTF8ToTStr(const std::string& str)
{ return UTF8ToUTF16(str); }
{ return UTF8ToUTF16W(str); }
#else
inline std::string TStrToUTF8(const std::string& str)
{ return str; }

99
src/core/file_sys/archive.h
View File

@ -7,11 +7,13 @@
#include <memory>
#include "common/common_types.h"
#include "common/string_util.h"
#include "common/bit_field.h"
#include "core/file_sys/file.h"
#include "core/file_sys/directory.h"
#include "core/mem_map.h"
#include "core/hle/kernel/kernel.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
@ -19,6 +21,15 @@
namespace FileSys {
// Path string type
enum LowPathType : u32 {
Invalid = 0,
Empty = 1,
Binary = 2,
Char = 3,
Wchar = 4
};
union Mode {
u32 hex;
BitField<0, 1, u32> read_flag;
@ -26,6 +37,94 @@ union Mode {
BitField<2, 1, u32> create_flag;
};
class Path {
public:
Path():
type(Invalid)
{
}
Path(LowPathType type, u32 size, u32 pointer):
type(type)
{
switch (type) {
case Binary:
{
u8* data = Memory::GetPointer(pointer);
binary = std::vector<u8>(data, data + size);
break;
}
case Char:
{
const char* data = reinterpret_cast<const char*>(Memory::GetPointer(pointer));
string = std::string(data, size - 1); // Data is always null-terminated.
break;
}
case Wchar:
{
const char16_t* data = reinterpret_cast<const char16_t*>(Memory::GetPointer(pointer));
u16str = std::u16string(data, size/2 - 1); // Data is always null-terminated.
break;
}
}
}
LowPathType GetType() const {
return type;
}
const std::string AsString() const {
switch (GetType()) {
case Char:
return string;
case Wchar:
return Common::UTF16ToUTF8(u16str);
case Empty:
return {};
default:
ERROR_LOG(KERNEL, "LowPathType cannot be converted to string!");
return {};
}
}
const std::u16string AsU16Str() const {
switch (GetType()) {
case Char:
return Common::UTF8ToUTF16(string);
case Wchar:
return u16str;
case Empty:
return {};
default:
ERROR_LOG(KERNEL, "LowPathType cannot be converted to u16string!");
return {};
}
}
const std::vector<u8> AsBinary() const {
switch (GetType()) {
case Binary:
return binary;
case Char:
return std::vector<u8>(string.begin(), string.end());
case Wchar:
return std::vector<u8>(u16str.begin(), u16str.end());
case Empty:
return {};
default:
ERROR_LOG(KERNEL, "LowPathType cannot be converted to binary!");
return {};
}
}
private:
LowPathType type;
std::vector<u8> binary;
std::string string;
std::u16string u16str;
};
class Archive : NonCopyable {
public:
/// Supported archive types

158
src/core/hle/service/fs_user.cpp
View File

@ -5,6 +5,7 @@
#include "common/common.h"
#include "fs_user.h"
#include "common/string_util.h"
#include "core/settings.h"
#include "core/hle/kernel/archive.h"
@ -13,20 +14,6 @@
namespace FS_User {
// Command to access archive file
enum class LowPathType : u32 {
Invalid = 0,
Empty = 1,
Binary = 2,
Char = 3,
Wchar = 4
};
std::string GetStringFromCmdBuff(const u32 pointer, const u32 size) {
auto data = reinterpret_cast<const char*>(Memory::GetPointer(pointer));
return std::string(data, size - 1);
}
// We currently return 0 for success and -1 for failure in cmd_buff[1]. -1 was chosen because it
// puts all the sections of the http://3dbrew.org/wiki/Error_codes to something non-zero, to make
// sure we don't mislead the application into thinking something worked.
@ -44,32 +31,36 @@ void Initialize(Service::Interface* self) {
void OpenFile(Service::Interface* self) {
u32* cmd_buff = Service::GetCommandBuffer();
u32 transaction = cmd_buff[1];
// TODO(Link Mauve): cmd_buff[2], aka archive handle lower word, isn't used according to
// 3dmoo's or ctrulib's implementations. Triple check if it's really the case.
Handle archive_handle = static_cast<Handle>(cmd_buff[3]);
LowPathType type = static_cast<LowPathType>(cmd_buff[4]);
u32 size = cmd_buff[5];
auto filename_type = static_cast<FileSys::LowPathType>(cmd_buff[4]);
u32 filename_size = cmd_buff[5];
FileSys::Mode mode; mode.hex = cmd_buff[6];
u32 attributes = cmd_buff[7]; // TODO(Link Mauve): do something with those attributes.
u32 pointer = cmd_buff[9];
u32 attributes = cmd_buff[7]; // TODO(Link Mauve): do something with those attributes.
u32 filename_ptr = cmd_buff[9];
if (type != LowPathType::Char) {
ERROR_LOG(KERNEL, "file LowPath type other than char is currently unsupported");
cmd_buff[1] = -1;
FileSys::Path file_path(filename_type, filename_size, filename_ptr);
std::string file_string;
switch (file_path.GetType()) {
case FileSys::Char:
case FileSys::Wchar:
file_string = file_path.AsString();
break;
default:
WARN_LOG(KERNEL, "file LowPath type is currently unsupported; returning archive handle instead");
return;
}
std::string file_name = GetStringFromCmdBuff(pointer, size);
DEBUG_LOG(KERNEL, "type=%d size=%d mode=%d attrs=%d data=%s",
filename_type, filename_size, mode, attributes, file_string.c_str());
DEBUG_LOG(KERNEL, "type=%d size=%d mode=%d attrs=%d data=%s", type, size, mode, attributes, file_name.c_str());
Handle handle = Kernel::OpenFileFromArchive(archive_handle, file_name, mode);
Handle handle = Kernel::OpenFileFromArchive(archive_handle, file_string, mode);
if (handle) {
cmd_buff[1] = 0;
cmd_buff[3] = handle;
} else {
ERROR_LOG(KERNEL, "failed to get a handle for file %s", file_name.c_str());
ERROR_LOG(KERNEL, "failed to get a handle for file %s", file_string.c_str());
// TODO(Link Mauve): check for the actual error values, this one was just chosen arbitrarily.
cmd_buff[1] = -1;
}
@ -80,31 +71,25 @@ void OpenFile(Service::Interface* self) {
void OpenFileDirectly(Service::Interface* self) {
u32* cmd_buff = Service::GetCommandBuffer();
u32 transaction = cmd_buff[1];
auto archive_id = static_cast<FileSys::Archive::IdCode>(cmd_buff[2]);
LowPathType archive_type = static_cast<LowPathType>(cmd_buff[3]);
u32 archive_size = cmd_buff[4];
LowPathType file_type = static_cast<LowPathType>(cmd_buff[5]);
u32 size = cmd_buff[6];
auto archive_id = static_cast<FileSys::Archive::IdCode>(cmd_buff[2]);
auto archivename_type = static_cast<FileSys::LowPathType>(cmd_buff[3]);
u32 archivename_size = cmd_buff[4];
auto filename_type = static_cast<FileSys::LowPathType>(cmd_buff[5]);
u32 filename_size = cmd_buff[6];
FileSys::Mode mode; mode.hex = cmd_buff[7];
u32 attributes = cmd_buff[8]; // TODO(Link Mauve): do something with those attributes.
u32 archive_pointer = cmd_buff[10];
u32 pointer = cmd_buff[12];
u32 attributes = cmd_buff[8]; // TODO(Link Mauve): do something with those attributes.
u32 archivename_ptr = cmd_buff[10];
u32 filename_ptr = cmd_buff[12];
if (archive_type != LowPathType::Empty) {
DEBUG_LOG(KERNEL, "archive_type=%d archive_size=%d file_type=%d file_size=%d file_mode=%d file_attrs=%d",
archivename_type, archivename_size, filename_type, filename_size, mode, attributes);
if (archivename_type != FileSys::Empty) {
ERROR_LOG(KERNEL, "archive LowPath type other than empty is currently unsupported");
cmd_buff[1] = -1;
return;
}
std::string archive_name = GetStringFromCmdBuff(archive_pointer, archive_size);
std::string file_name = GetStringFromCmdBuff(pointer, size);
DEBUG_LOG(KERNEL, "archive_type=%d archive_size=%d archive_data=%s "
"file_type=%d file_size=%d file_mode=%d file_attrs=%d file_data=%s",
archive_type, archive_size, archive_name.c_str(),
file_type, size, mode, attributes, file_name.c_str());
// TODO(Link Mauve): check if we should even get a handle for the archive, and don't leak it.
Handle archive_handle = Kernel::OpenArchive(archive_id);
if (archive_handle) {
@ -112,23 +97,30 @@ void OpenFileDirectly(Service::Interface* self) {
// cmd_buff[2] isn't used according to 3dmoo's implementation.
cmd_buff[3] = archive_handle;
} else {
ERROR_LOG(KERNEL, "failed to get a handle for archive %s", archive_name.c_str());
ERROR_LOG(KERNEL, "failed to get a handle for archive");
// TODO(Link Mauve): check for the actual error values, this one was just chosen arbitrarily.
cmd_buff[1] = -1;
return;
}
if (file_type != LowPathType::Char) {
WARN_LOG(KERNEL, "file LowPath type other than char is currently unsupported; returning archive handle instead");
FileSys::Path file_path(filename_type, filename_size, filename_ptr);
std::string file_string;
switch (file_path.GetType()) {
case FileSys::Char:
case FileSys::Wchar:
file_string = file_path.AsString();
break;
default:
WARN_LOG(KERNEL, "file LowPath type is currently unsupported; returning archive handle instead");
return;
}
Handle handle = Kernel::OpenFileFromArchive(archive_handle, file_name, mode);
Handle handle = Kernel::OpenFileFromArchive(archive_handle, file_string, mode);
if (handle) {
cmd_buff[1] = 0;
cmd_buff[3] = handle;
} else {
ERROR_LOG(KERNEL, "failed to get a handle for file %s", file_name.c_str());
ERROR_LOG(KERNEL, "failed to get a handle for file %s", file_string.c_str());
// TODO(Link Mauve): check for the actual error values, this one was just chosen arbitrarily.
cmd_buff[1] = -1;
}
@ -153,21 +145,25 @@ void CreateDirectory(Service::Interface* self) {
// TODO: cmd_buff[2], aka archive handle lower word, isn't used according to
// 3dmoo's or ctrulib's implementations. Triple check if it's really the case.
Handle archive_handle = static_cast<Handle>(cmd_buff[3]);
LowPathType type = static_cast<LowPathType>(cmd_buff[4]);
u32 name_size = cmd_buff[5];
u32 name_offset = cmd_buff[8];
auto dirname_type = static_cast<FileSys::LowPathType>(cmd_buff[4]);
u32 dirname_size = cmd_buff[5];
u32 dirname_ptr = cmd_buff[8];
if (type != LowPathType::Char) {
ERROR_LOG(KERNEL, "directory LowPath type other than char is currently unsupported");
FileSys::Path dir_path(dirname_type, dirname_size, dirname_ptr);
std::string dir_string;
switch (dir_path.GetType()) {
case FileSys::Char:
case FileSys::Wchar:
dir_string = dir_path.AsString();
break;
default:
cmd_buff[1] = -1;
return;
}
std::string dir_name = GetStringFromCmdBuff(name_offset, name_size);
DEBUG_LOG(KERNEL, "type=%d size=%d data=%s", dirname_type, dirname_size, dir_string.c_str());
DEBUG_LOG(KERNEL, "type=%d size=%d data=%s", type, name_size, dir_name.c_str());
cmd_buff[1] = Kernel::CreateDirectoryFromArchive(archive_handle, dir_name);
cmd_buff[1] = Kernel::CreateDirectoryFromArchive(archive_handle, dir_string);
DEBUG_LOG(KERNEL, "called");
}
@ -178,26 +174,30 @@ void OpenDirectory(Service::Interface* self) {
// TODO(Link Mauve): cmd_buff[2], aka archive handle lower word, isn't used according to
// 3dmoo's or ctrulib's implementations. Triple check if it's really the case.
Handle archive_handle = static_cast<Handle>(cmd_buff[2]);
LowPathType type = static_cast<LowPathType>(cmd_buff[3]);
u32 size = cmd_buff[4];
u32 pointer = cmd_buff[6];
auto dirname_type = static_cast<FileSys::LowPathType>(cmd_buff[3]);
u32 dirname_size = cmd_buff[4];
u32 dirname_ptr = cmd_buff[6];
if (type != LowPathType::Char) {
ERROR_LOG(KERNEL, "directory LowPath type other than char is currently unsupported");
FileSys::Path dir_path(dirname_type, dirname_size, dirname_ptr);
std::string dir_string;
switch (dir_path.GetType()) {
case FileSys::Char:
case FileSys::Wchar:
dir_string = dir_path.AsString();
break;
default:
cmd_buff[1] = -1;
return;
}
std::string dir_name = GetStringFromCmdBuff(pointer, size);
DEBUG_LOG(KERNEL, "type=%d size=%d data=%s", dirname_type, dirname_size, dir_string.c_str());
DEBUG_LOG(KERNEL, "type=%d size=%d data=%s", type, size, dir_name.c_str());
Handle handle = Kernel::OpenDirectoryFromArchive(archive_handle, dir_name);
Handle handle = Kernel::OpenDirectoryFromArchive(archive_handle, dir_string);
if (handle) {
cmd_buff[1] = 0;
cmd_buff[3] = handle;
} else {
ERROR_LOG(KERNEL, "failed to get a handle for directory %s", dir_name.c_str());
ERROR_LOG(KERNEL, "failed to get a handle for directory %s", dir_string.c_str());
// TODO(Link Mauve): check for the actual error values, this one was just chosen arbitrarily.
cmd_buff[1] = -1;
}
@ -208,28 +208,26 @@ void OpenDirectory(Service::Interface* self) {
void OpenArchive(Service::Interface* self) {
u32* cmd_buff = Service::GetCommandBuffer();
auto arch_id = static_cast<FileSys::Archive::IdCode>(cmd_buff[1]);
LowPathType type = static_cast<LowPathType>(cmd_buff[2]);
u32 size = cmd_buff[3];
u32 pointer = cmd_buff[5];
auto archive_id = static_cast<FileSys::Archive::IdCode>(cmd_buff[1]);
auto archivename_type = static_cast<FileSys::LowPathType>(cmd_buff[2]);
u32 archivename_size = cmd_buff[3];
u32 archivename_ptr = cmd_buff[5];
if (type != LowPathType::Empty) {
DEBUG_LOG(KERNEL, "type=%d size=%d", archivename_type, archivename_size);
if (archivename_type != FileSys::Empty) {
ERROR_LOG(KERNEL, "archive LowPath type other than empty is currently unsupported");
cmd_buff[1] = -1;
return;
}
std::string archive_name = GetStringFromCmdBuff(pointer, size);
DEBUG_LOG(KERNEL, "type=%d size=%d data=%s", type, size, archive_name.c_str());
Handle handle = Kernel::OpenArchive(arch_id);
Handle handle = Kernel::OpenArchive(archive_id);
if (handle) {
cmd_buff[1] = 0;
// cmd_buff[2] isn't used according to 3dmoo's implementation.
cmd_buff[3] = handle;
} else {
ERROR_LOG(KERNEL, "failed to get a handle for archive %s", archive_name.c_str());
ERROR_LOG(KERNEL, "failed to get a handle for archive");
// TODO(Link Mauve): check for the actual error values, this one was just chosen arbitrarily.
cmd_buff[1] = -1;
}