From e6a003cd2562d5e366b3ad5bed708819a9e5a07c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tur=C3=A1nszki=20J=C3=A1nos?= Date: Tue, 20 May 2025 06:49:03 +0200 Subject: [PATCH] stringconvert replacement --- WickedEngine/wiGraphicsDevice_DX12.cpp | 12 +- WickedEngine/wiHelper.cpp | 306 ++++++++++++++++++++----- WickedEngine/wiHelper.h | 4 +- WickedEngine/wiInput.cpp | 2 +- WickedEngine/wiVersion.cpp | 2 +- 5 files changed, 253 insertions(+), 73 deletions(-) diff --git a/WickedEngine/wiGraphicsDevice_DX12.cpp b/WickedEngine/wiGraphicsDevice_DX12.cpp index e2e056156..4153773ec 100644 --- a/WickedEngine/wiGraphicsDevice_DX12.cpp +++ b/WickedEngine/wiGraphicsDevice_DX12.cpp @@ -5633,11 +5633,11 @@ std::mutex queue_locker; // Note: pCommandListDebugNameA and similar doesn't seem to contain anything while pCommandListDebugNameW does, so convert these by hand: if (pNode->pCommandListDebugNameW != nullptr) { - wi::helper::StringConvert(pNode->pCommandListDebugNameW, commandlistname); + wi::helper::StringConvert(pNode->pCommandListDebugNameW, commandlistname, arraysize(commandlistname)); } if (pNode->pCommandQueueDebugNameW != nullptr) { - wi::helper::StringConvert(pNode->pCommandQueueDebugNameW, commandqueuename); + wi::helper::StringConvert(pNode->pCommandQueueDebugNameW, commandqueuename, arraysize(commandqueuename)); } log += std::string("[DRED] Commandlist = [") + commandlistname + std::string("], CommandQueue = [") + commandqueuename + std::string("], lastCompletedOp = [") + std::to_string(lastCompletedOp) + "], BreadCrumbCount = [" + std::to_string(pNode->BreadcrumbCount) + "]\n"; @@ -5692,7 +5692,7 @@ std::mutex queue_locker; char objectname[1024] = {}; if (pNode->ObjectNameW != nullptr) { - wi::helper::StringConvert(pNode->ObjectNameW, objectname); + wi::helper::StringConvert(pNode->ObjectNameW, objectname, arraysize(objectname)); } log += std::string("\tName: ") + objectname + std::string(" ") + AllocTypeName + std::string("\n"); pNode = pNode->pNext; @@ -5710,7 +5710,7 @@ std::mutex queue_locker; char objectname[1024] = {}; if (pNode->ObjectNameW != nullptr) { - wi::helper::StringConvert(pNode->ObjectNameW, objectname); + wi::helper::StringConvert(pNode->ObjectNameW, objectname, arraysize(objectname)); } log += std::string("\tName: ") + objectname + std::string(" (Type: ") + AllocTypeName + std::string(")\n"); pNode = pNode->pNext; @@ -7670,7 +7670,7 @@ std::mutex queue_locker; if (commandlist.queue == QUEUE_VIDEO_DECODE) return; wchar_t text[128]; - if (wi::helper::StringConvert(name, text) > 0) + if (wi::helper::StringConvert(name, text, arraysize(text)) > 0) { PIXBeginEvent(commandlist.GetGraphicsCommandList(), 0xFF000000, text); } @@ -7688,7 +7688,7 @@ std::mutex queue_locker; if (commandlist.queue == QUEUE_VIDEO_DECODE) return; wchar_t text[128]; - if (wi::helper::StringConvert(name, text) > 0) + if (wi::helper::StringConvert(name, text, arraysize(text)) > 0) { PIXSetMarker(commandlist.GetGraphicsCommandList(), 0xFFFF0000, text); } diff --git a/WickedEngine/wiHelper.cpp b/WickedEngine/wiHelper.cpp index e47bf6f4a..c6d9ec624 100644 --- a/WickedEngine/wiHelper.cpp +++ b/WickedEngine/wiHelper.cpp @@ -1387,84 +1387,264 @@ namespace wi::helper void StringConvert(const std::string& from, std::wstring& to) { -#ifdef _WIN32 - int num = MultiByteToWideChar(CP_UTF8, 0, from.c_str(), -1, NULL, 0); - if (num > 0) + to.clear(); + size_t i = 0; + while (i < from.size()) { - to.resize(size_t(num) - 1); - MultiByteToWideChar(CP_UTF8, 0, from.c_str(), -1, &to[0], num); - } -#else - std::wstring_convert> cv; - to = cv.from_bytes(from); -#endif // _WIN32 - } + uint32_t codepoint = 0; + unsigned char c = from[i]; + if (c < 0x80) + { + codepoint = c; + i += 1; + } + else if ((c & 0xE0) == 0xC0) + { + if (i + 1 >= from.size()) + break; + codepoint = ((c & 0x1F) << 6) | (from[i + 1] & 0x3F); + i += 2; + } + else if ((c & 0xF0) == 0xE0) + { + if (i + 2 >= from.size()) + break; + codepoint = ((c & 0x0F) << 12) | ((from[i + 1] & 0x3F) << 6) | (from[i + 2] & 0x3F); + i += 3; + } + else if ((c & 0xF8) == 0xF0) + { + if (i + 3 >= from.size()) + break; + codepoint = ((c & 0x07) << 18) | ((from[i + 1] & 0x3F) << 12) | ((from[i + 2] & 0x3F) << 6) | (from[i + 3] & 0x3F); + i += 4; + } + else + { + ++i; + continue; + } + + if constexpr (sizeof(wchar_t) >= 4) + { + to += static_cast(codepoint); + } + else + { + if (codepoint <= 0xFFFF) + { + to += static_cast(codepoint); + } + else + { + codepoint -= 0x10000; + to += static_cast((codepoint >> 10) + 0xD800); + to += static_cast((codepoint & 0x3FF) + 0xDC00); + } + } + } + } + void StringConvert(const std::wstring& from, std::string& to) { -#ifdef _WIN32 - int num = WideCharToMultiByte(CP_UTF8, 0, from.c_str(), -1, NULL, 0, NULL, NULL); - if (num > 0) + to.clear(); + for (size_t i = 0; i < from.size(); ++i) { - to.resize(size_t(num) - 1); - WideCharToMultiByte(CP_UTF8, 0, from.c_str(), -1, &to[0], num, NULL, NULL); - } -#else - std::wstring_convert> cv; - to = cv.to_bytes(from); -#endif // _WIN32 - } + uint32_t codepoint = 0; + wchar_t wc = from[i]; + if constexpr (sizeof(wchar_t) >= 4) + { + codepoint = static_cast(wc); + } + else + { + if (wc >= 0xD800 && wc <= 0xDBFF) + { + if (i + 1 < from.size()) + { + wchar_t wc_low = from[i + 1]; + if (wc_low >= 0xDC00 && wc_low <= 0xDFFF) + { + codepoint = ((static_cast(wc - 0xD800) << 10) | (static_cast(wc_low - 0xDC00))) + 0x10000; + ++i; + } + } + } + else + { + codepoint = static_cast(wc); + } + } + + if (codepoint <= 0x7F) + { + to += static_cast(codepoint); + } + else if (codepoint <= 0x7FF) + { + to += static_cast(0xC0 | ((codepoint >> 6) & 0x1F)); + to += static_cast(0x80 | (codepoint & 0x3F)); + } + else if (codepoint <= 0xFFFF) + { + to += static_cast(0xE0 | ((codepoint >> 12) & 0x0F)); + to += static_cast(0x80 | ((codepoint >> 6) & 0x3F)); + to += static_cast(0x80 | (codepoint & 0x3F)); + } + else if (codepoint <= 0x10FFFF) + { + to += static_cast(0xF0 | ((codepoint >> 18) & 0x07)); + to += static_cast(0x80 | ((codepoint >> 12) & 0x3F)); + to += static_cast(0x80 | ((codepoint >> 6) & 0x3F)); + to += static_cast(0x80 | (codepoint & 0x3F)); + } + } + } + int StringConvert(const char* from, wchar_t* to, int dest_size_in_characters) { -#ifdef _WIN32 - int num = MultiByteToWideChar(CP_UTF8, 0, from, -1, NULL, 0); - if (num > 0) - { - if (dest_size_in_characters >= 0) - { - num = std::min(num, dest_size_in_characters); - } - MultiByteToWideChar(CP_UTF8, 0, from, -1, &to[0], num); - } - return num; -#else - std::wstring_convert> cv; - auto result = cv.from_bytes(from).c_str(); - int num = (int)cv.converted(); - if (dest_size_in_characters >= 0) - { - num = std::min(num, dest_size_in_characters); - } - std::memcpy(to, result, num * sizeof(wchar_t)); - return num; -#endif // _WIN32 - } + if (!from || !to || dest_size_in_characters <= 0) + return 0; + const unsigned char* src = reinterpret_cast(from); + int written = 0; + + while (*src && written < dest_size_in_characters - 1) + { + uint32_t codepoint = 0; + unsigned char c = *src; + + if (c < 0x80) + { + codepoint = c; + ++src; + } + else if ((c & 0xE0) == 0xC0) + { + if (!src[1]) + break; + codepoint = ((c & 0x1F) << 6) | (src[1] & 0x3F); + src += 2; + } + else if ((c & 0xF0) == 0xE0) + { + if (!src[1] || !src[2]) + break; + codepoint = ((c & 0x0F) << 12) | ((src[1] & 0x3F) << 6) | (src[2] & 0x3F); + src += 3; + } + else if ((c & 0xF8) == 0xF0) + { + if (!src[1] || !src[2] || !src[3]) + break; + codepoint = ((c & 0x07) << 18) | ((src[1] & 0x3F) << 12) | ((src[2] & 0x3F) << 6) | (src[3] & 0x3F); + src += 4; + } + else + { + ++src; + continue; + } + + if constexpr (sizeof(wchar_t) >= 4) + { + to[written++] = static_cast(codepoint); + } + else + { + if (codepoint <= 0xFFFF) + { + to[written++] = static_cast(codepoint); + } + else + { + if (written + 1 >= dest_size_in_characters - 1) + break; + codepoint -= 0x10000; + to[written++] = static_cast((codepoint >> 10) + 0xD800); + to[written++] = static_cast((codepoint & 0x3FF) + 0xDC00); + } + } + } + + to[written] = 0; + return written; + } + int StringConvert(const wchar_t* from, char* to, int dest_size_in_characters) { -#ifdef _WIN32 - int num = WideCharToMultiByte(CP_UTF8, 0, from, -1, NULL, 0, NULL, NULL); - if (num > 0) + if (!from || !to || dest_size_in_characters <= 0) + return 0; + + int written = 0; + for (int i = 0; from[i] != 0 && written < dest_size_in_characters - 1; ++i) { - if (dest_size_in_characters >= 0) + uint32_t codepoint = 0; + wchar_t wc = from[i]; + + if constexpr (sizeof(wchar_t) >= 4) { - num = std::min(num, dest_size_in_characters); + codepoint = static_cast(wc); + } + else + { + if (wc >= 0xD800 && wc <= 0xDBFF) + { + wchar_t wc_low = from[i + 1]; + if (wc_low >= 0xDC00 && wc_low <= 0xDFFF) + { + codepoint = ((static_cast(wc - 0xD800) << 10) | (static_cast(wc_low - 0xDC00))) + 0x10000; + ++i; + } + else + { + codepoint = wc; + } + } + else + { + codepoint = static_cast(wc); + } + } + + if (codepoint <= 0x7F) + { + if (written + 1 >= dest_size_in_characters) + break; + to[written++] = static_cast(codepoint); + } + else if (codepoint <= 0x7FF) + { + if (written + 2 >= dest_size_in_characters) + break; + to[written++] = static_cast(0xC0 | ((codepoint >> 6) & 0x1F)); + to[written++] = static_cast(0x80 | (codepoint & 0x3F)); + } + else if (codepoint <= 0xFFFF) + { + if (written + 3 >= dest_size_in_characters) + break; + to[written++] = static_cast(0xE0 | ((codepoint >> 12) & 0x0F)); + to[written++] = static_cast(0x80 | ((codepoint >> 6) & 0x3F)); + to[written++] = static_cast(0x80 | (codepoint & 0x3F)); + } + else if (codepoint <= 0x10FFFF) + { + if (written + 4 >= dest_size_in_characters) + break; + to[written++] = static_cast(0xF0 | ((codepoint >> 18) & 0x07)); + to[written++] = static_cast(0x80 | ((codepoint >> 12) & 0x3F)); + to[written++] = static_cast(0x80 | ((codepoint >> 6) & 0x3F)); + to[written++] = static_cast(0x80 | (codepoint & 0x3F)); } - WideCharToMultiByte(CP_UTF8, 0, from, -1, &to[0], num, NULL, NULL); } - return num; -#else - std::wstring_convert> cv; - auto result = cv.to_bytes(from).c_str(); - int num = (size_t)cv.converted(); - if (dest_size_in_characters >= 0) - { - num = std::min(num, dest_size_in_characters); - } - std::memcpy(to, result, num * sizeof(char)); - return num; -#endif // _WIN32 + + if (written < dest_size_in_characters) + to[written] = '\0'; + + return written; } void DebugOut(const std::string& str, DebugLevel level) diff --git a/WickedEngine/wiHelper.h b/WickedEngine/wiHelper.h index f1732a91d..7dc0ca47d 100644 --- a/WickedEngine/wiHelper.h +++ b/WickedEngine/wiHelper.h @@ -145,12 +145,12 @@ namespace wi::helper // Parameter - to - must be pre-allocated! // dest_size_in_characters : number of characters in the pre-allocated string memory // returns result string length - int StringConvert(const char* from, wchar_t* to, int dest_size_in_characters = -1); + int StringConvert(const char* from, wchar_t* to, int dest_size_in_characters); // Parameter - to - must be pre-allocated! // dest_size_in_characters : number of characters in the pre-allocated string memory // returns result string length - int StringConvert(const wchar_t* from, char* to, int dest_size_in_characters = -1); + int StringConvert(const wchar_t* from, char* to, int dest_size_in_characters); // Prints debug info to the console output enum class DebugLevel diff --git a/WickedEngine/wiInput.cpp b/WickedEngine/wiInput.cpp index 37738da7a..8a12351b3 100644 --- a/WickedEngine/wiInput.cpp +++ b/WickedEngine/wiInput.cpp @@ -825,7 +825,7 @@ namespace wi::input { #ifdef PLATFORM_WINDOWS_DESKTOP wchar_t wfilename[1024] = {}; - wi::helper::StringConvert(filename, wfilename); + wi::helper::StringConvert(filename, wfilename, arraysize(wfilename)); cursor_table[cursor] = LoadCursorFromFile(wfilename); #endif // PLATFORM_WINDOWS_DESKTOP diff --git a/WickedEngine/wiVersion.cpp b/WickedEngine/wiVersion.cpp index 1440b67c3..04fd1702f 100644 --- a/WickedEngine/wiVersion.cpp +++ b/WickedEngine/wiVersion.cpp @@ -9,7 +9,7 @@ namespace wi::version // minor features, major updates, breaking compatibility changes const int minor = 71; // minor bug fixes, alterations, refactors, updates - const int revision = 770; + const int revision = 771; const std::string version_string = std::to_string(major) + "." + std::to_string(minor) + "." + std::to_string(revision);