Removing deprecated wstring_convert

eduardodoria · eduardodoria · commit a9e2074925f2 · 2025-12-17T22:07:31.000-03:00
diff --git a/engine/core/subsystem/UISystem.cpp b/engine/core/subsystem/UISystem.cpp
@@ -4,8 +4,6 @@
 
 #include "UISystem.h"
 
-#include <locale>
-#include <codecvt>
 #include <algorithm>
 #include "Scene.h"
 #include "Input.h"
@@ -1501,10 +1499,7 @@ bool UISystem::eventOnCharInput(wchar_t codepoint){
                 TextComponent& text = scene->getComponent<TextComponent>(textedit.text);
                 if (codepoint == '\b'){
                     if (text.text.length() > 0){
-                        std::wstring_convert<std::codecvt_utf8_utf16<wchar_t> > convert;
-                        std::wstring utf16OldText = convert.from_bytes(text.text);
-
-                        text.text = convert.to_bytes(utf16OldText.substr(0, utf16OldText.size() - 1));
+                        StringUtils::eraseLastCodepointUtf8(text.text);
                     }
                 }else{
                     text.text = text.text + StringUtils::toUTF8(codepoint);
@@ -1590,10 +1585,9 @@ bool UISystem::eventOnPointerDown(float x, float y){
                 TextEditComponent& textedit = scene->getComponent<TextEditComponent>(lastUIFromPointer);
                 TextComponent& text = scene->getComponent<TextComponent>(textedit.text);
 
-                std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> convert;
-                std::wstring utf16Text = convert.from_bytes(text.text);
-
-                System::instance().showVirtualKeyboard(utf16Text);
+                bool hadInvalid = false;
+                std::wstring wideText = StringUtils::utf8ToWString(text.text, hadInvalid);
+                System::instance().showVirtualKeyboard(wideText);
             }else{
                 System::instance().hideVirtualKeyboard();
             }
diff --git a/engine/core/util/STBText.cpp b/engine/core/util/STBText.cpp
@@ -7,9 +7,8 @@
 #include <string>
 #include "Log.h"
 #include "io/Data.h"
-#include <codecvt>
-#include <locale>
 #include "DefaultFont.h"
+#include "StringUtils.h"
 
 using namespace Supernova;
 
@@ -155,14 +154,10 @@ TextureData* STBText::load(const std::string& fontpath, unsigned int fontSize){
 
 void STBText::createText(const std::string& text, Buffer* buffer, std::vector<uint16_t>& indices, std::vector<Vector2>& charPositions,
                          unsigned int& width, unsigned int& height, bool fixedWidth, bool fixedHeight, bool multiline, bool invert){
-    
-    std::wstring_convert< std::codecvt_utf8_utf16<wchar_t> > convert;
-    std::wstring utf16String;
 
-    try {
-        utf16String = convert.from_bytes( text );
-    } catch(const std::range_error& e) {
-        utf16String = convert.from_bytes(text.substr(0, convert.converted()));
+    bool hadInvalid = false;
+    std::vector<uint32_t> codepoints = StringUtils::decodeUtf8ToCodepoints(text, hadInvalid);
+    if (hadInvalid) {
         Log::warn("Invalid character");
     }
     
@@ -175,8 +170,8 @@ void STBText::createText(const std::string& text, Buffer* buffer, std::vector<ui
     if (multiline && fixedWidth){
 
         int lastSpace = 0;
-        for (int i = 0; i < utf16String.size(); i++){
-            int intchar = uint_least32_t(utf16String[i]);
+        for (int i = 0; i < (int)codepoints.size(); i++){
+            int intchar = (int)codepoints[(size_t)i];
             if (intchar == 32){ //space
                 lastSpace = i;
             }
@@ -187,13 +182,13 @@ void STBText::createText(const std::string& text, Buffer* buffer, std::vector<ui
                 stbtt_aligned_quad quad;
                 stbtt_GetPackedQuad(charInfo, atlasWidth, atlasHeight, intchar - firstChar, &offsetX, &offsetY, &quad, 1);
                 
-                if (offsetX > width){
+                if (offsetX > (float)width){
                     if (lastSpace > 0){
-                        utf16String[lastSpace] = '\n';
+                        codepoints[(size_t)lastSpace] = '\n';
                         i = lastSpace;
                         lastSpace = 0;
                     }else{
-                        utf16String.insert(i, { '\n' });
+                        codepoints.insert(codepoints.begin() + i, (uint32_t)'\n');
                     }
                     offsetX = 0;
                 }
@@ -209,9 +204,9 @@ void STBText::createText(const std::string& text, Buffer* buffer, std::vector<ui
     int lineCount = 1;
     charPositions.clear();
 
-    for (int i = 0; i < utf16String.size(); i++){
+    for (int i = 0; i < (int)codepoints.size(); i++){
 
-        int intchar = uint_least32_t(utf16String[i]);
+        int intchar = (int)codepoints[(size_t)i];
 
         if (intchar == 10){ //\n
             offsetY += lineHeight;
@@ -277,7 +272,7 @@ void STBText::createText(const std::string& text, Buffer* buffer, std::vector<ui
 
     }
     //Empty text
-    if (utf16String.size() == 0){
+    if (codepoints.size() == 0){
         buffer->addVector3(atrVertice, Vector3(0.0f, 0.0f, 0.0f));
         buffer->addVector3(atrVertice, Vector3(0.0f, 0.0f, 0.0f));
         buffer->addVector3(atrVertice, Vector3(0.0f, 0.0f, 0.0f));
diff --git a/engine/core/util/STBText.h b/engine/core/util/STBText.h
@@ -5,6 +5,8 @@
 #ifndef STBText_h
 #define STBText_h
 
+#include <cstdint>
+#include <string>
 #include <vector>
 #include "math/Vector2.h"
 #include "math/Vector3.h"
diff --git a/engine/core/util/StringUtils.cpp b/engine/core/util/StringUtils.cpp
@@ -4,13 +4,172 @@
 
 #include "StringUtils.h"
 
-#include <codecvt>
-#include <iterator>
-#include <locale>
+#include <cstdint>
+#include <limits>
 
 using namespace Supernova;
 
 std::string StringUtils::toUTF8(wchar_t codepoint){
-    std::wstring_convert<std::codecvt_utf8<wchar_t>,wchar_t> convert;
-    return convert.to_bytes(codepoint);
+    uint32_t cp = static_cast<uint32_t>(codepoint);
+
+    // Replace invalid Unicode scalar values.
+    if (cp > 0x10FFFFu || (cp >= 0xD800u && cp <= 0xDFFFu)) {
+        cp = 0xFFFDu;
+    }
+
+    std::string out;
+    if (cp <= 0x7Fu) {
+        out.push_back(static_cast<char>(cp));
+    } else if (cp <= 0x7FFu) {
+        out.push_back(static_cast<char>(0xC0u | (cp >> 6)));
+        out.push_back(static_cast<char>(0x80u | (cp & 0x3Fu)));
+    } else if (cp <= 0xFFFFu) {
+        out.push_back(static_cast<char>(0xE0u | (cp >> 12)));
+        out.push_back(static_cast<char>(0x80u | ((cp >> 6) & 0x3Fu)));
+        out.push_back(static_cast<char>(0x80u | (cp & 0x3Fu)));
+    } else {
+        out.push_back(static_cast<char>(0xF0u | (cp >> 18)));
+        out.push_back(static_cast<char>(0x80u | ((cp >> 12) & 0x3Fu)));
+        out.push_back(static_cast<char>(0x80u | ((cp >> 6) & 0x3Fu)));
+        out.push_back(static_cast<char>(0x80u | (cp & 0x3Fu)));
+    }
+    return out;
+}
+
+std::vector<uint32_t> StringUtils::decodeUtf8ToCodepoints(const std::string& text, bool& hadInvalid) {
+    hadInvalid = false;
+
+    std::vector<uint32_t> out;
+    out.reserve(text.size());
+
+    const auto* bytes = reinterpret_cast<const unsigned char*>(text.data());
+    size_t i = 0;
+    while (i < text.size()) {
+        uint32_t cp = 0;
+        unsigned char b0 = bytes[i];
+
+        if (b0 <= 0x7F) {
+            cp = b0;
+            i += 1;
+        } else if ((b0 & 0xE0) == 0xC0) {
+            if (i + 1 >= text.size()) {
+                hadInvalid = true;
+                cp = 0xFFFD;
+                i += 1;
+            } else {
+                unsigned char b1 = bytes[i + 1];
+                uint32_t v = ((uint32_t)(b0 & 0x1F) << 6) | (uint32_t)(b1 & 0x3F);
+                if ((b1 & 0xC0) != 0x80 || v < 0x80) {
+                    hadInvalid = true;
+                    cp = 0xFFFD;
+                    i += 1;
+                } else {
+                    cp = v;
+                    i += 2;
+                }
+            }
+        } else if ((b0 & 0xF0) == 0xE0) {
+            if (i + 2 >= text.size()) {
+                hadInvalid = true;
+                cp = 0xFFFD;
+                i += 1;
+            } else {
+                unsigned char b1 = bytes[i + 1];
+                unsigned char b2 = bytes[i + 2];
+                uint32_t v = ((uint32_t)(b0 & 0x0F) << 12) | ((uint32_t)(b1 & 0x3F) << 6) | (uint32_t)(b2 & 0x3F);
+                if ((b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80 || v < 0x800 || (v >= 0xD800 && v <= 0xDFFF)) {
+                    hadInvalid = true;
+                    cp = 0xFFFD;
+                    i += 1;
+                } else {
+                    cp = v;
+                    i += 3;
+                }
+            }
+        } else if ((b0 & 0xF8) == 0xF0) {
+            if (i + 3 >= text.size()) {
+                hadInvalid = true;
+                cp = 0xFFFD;
+                i += 1;
+            } else {
+                unsigned char b1 = bytes[i + 1];
+                unsigned char b2 = bytes[i + 2];
+                unsigned char b3 = bytes[i + 3];
+                uint32_t v = ((uint32_t)(b0 & 0x07) << 18) | ((uint32_t)(b1 & 0x3F) << 12) |
+                             ((uint32_t)(b2 & 0x3F) << 6) | (uint32_t)(b3 & 0x3F);
+                if ((b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80 || (b3 & 0xC0) != 0x80 || v < 0x10000 || v > 0x10FFFF) {
+                    hadInvalid = true;
+                    cp = 0xFFFD;
+                    i += 1;
+                } else {
+                    cp = v;
+                    i += 4;
+                }
+            }
+        } else {
+            hadInvalid = true;
+            cp = 0xFFFD;
+            i += 1;
+        }
+
+        out.push_back(cp);
+    }
+
+    return out;
+}
+
+std::wstring StringUtils::utf8ToWString(const std::string& text, bool& hadInvalid) {
+    std::vector<uint32_t> cps = decodeUtf8ToCodepoints(text, hadInvalid);
+
+    std::wstring out;
+    out.reserve(cps.size());
+
+    if constexpr (sizeof(wchar_t) == 2) {
+        for (uint32_t cp : cps) {
+            if (cp <= 0xFFFFu) {
+                out.push_back(static_cast<wchar_t>(cp));
+            } else {
+                cp -= 0x10000u;
+                wchar_t high = static_cast<wchar_t>(0xD800u + (cp >> 10));
+                wchar_t low = static_cast<wchar_t>(0xDC00u + (cp & 0x3FFu));
+                out.push_back(high);
+                out.push_back(low);
+            }
+        }
+    } else {
+        for (uint32_t cp : cps) {
+            if (cp > static_cast<uint32_t>(std::numeric_limits<wchar_t>::max())) {
+                out.push_back(static_cast<wchar_t>(0xFFFDu));
+                hadInvalid = true;
+            } else {
+                out.push_back(static_cast<wchar_t>(cp));
+            }
+        }
+    }
+
+    return out;
+}
+
+void StringUtils::eraseLastCodepointUtf8(std::string& text) {
+    if (text.empty()) {
+        return;
+    }
+
+    // Remove trailing UTF-8 continuation bytes (10xxxxxx)
+    size_t i = text.size();
+    while (i > 0) {
+        unsigned char c = static_cast<unsigned char>(text[i - 1]);
+        if ((c & 0xC0) != 0x80) {
+            break;
+        }
+        --i;
+    }
+
+    // If we ended up at 0, the string was all continuation bytes; clear it.
+    if (i == 0) {
+        text.clear();
+        return;
+    }
+
+    text.erase(i - 1);
 }
diff --git a/engine/core/util/StringUtils.h b/engine/core/util/StringUtils.h
@@ -6,12 +6,23 @@
 #define StringUtils_h
 
 #include "Export.h"
+#include <cstdint>
 #include <string>
+#include <vector>
 
 namespace Supernova{
     class SUPERNOVA_API StringUtils{
     public:
         static std::string toUTF8(wchar_t codepoint);
+
+        // Decodes UTF-8 into Unicode codepoints. Invalid sequences are replaced with U+FFFD and hadInvalid is set.
+        static std::vector<uint32_t> decodeUtf8ToCodepoints(const std::string& text, bool& hadInvalid);
+
+        // Converts UTF-8 to std::wstring (UTF-16 on Windows; UTF-32 elsewhere). Invalid sequences become U+FFFD.
+        static std::wstring utf8ToWString(const std::string& text, bool& hadInvalid);
+
+        // Removes the last UTF-8 encoded Unicode scalar (safe for multi-byte characters).
+        static void eraseLastCodepointUtf8(std::string& text);
     };
 }
 
