From 71788fb8314f4e4ed5f91f6101b37af1afa18939 Mon Sep 17 00:00:00 2001 From: Demitry235 Date: Mon, 26 Jan 2026 23:28:45 +0300 Subject: [PATCH 1/4] fix cyrillic and non-ASCII support --- Module/Public/ModLoader/Handlers/LocalizationHandler.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Module/Public/ModLoader/Handlers/LocalizationHandler.h b/Module/Public/ModLoader/Handlers/LocalizationHandler.h index 63415825..4976369a 100644 --- a/Module/Public/ModLoader/Handlers/LocalizationHandler.h +++ b/Module/Public/ModLoader/Handlers/LocalizationHandler.h @@ -1,8 +1,9 @@ #pragma once #include - #include +#include +#include namespace Kyber { @@ -20,8 +21,9 @@ class LocalizationHandler : public GenericCustomAssetHandler ModifyHistogram(uint8_t** histogramData); + std::vector ModifyHistogram(uint8_t** histogramData, uint32_t sizeBytes, LocalizationMergeData* data, uint32_t tableOffset, std::map& outCharMap); + std::vector ModifyChunk( - uint8_t* chunkData, uint32_t chunkSize, LocalizationMergeData* data, const std::vector& values); + uint8_t* chunkData, uint32_t chunkSize, LocalizationMergeData* data, const std::map& charMap); }; } // namespace Kyber From dac7326f9fd3025cd0824416216b4dea12d5c963 Mon Sep 17 00:00:00 2001 From: Demitry235 Date: Mon, 26 Jan 2026 23:31:34 +0300 Subject: [PATCH 2/4] fix cyrillic non-ASCII support --- .../Handlers/LocalizationHandler.cpp | 252 +++++++++--------- 1 file changed, 121 insertions(+), 131 deletions(-) diff --git a/Module/Source/ModLoader/Handlers/LocalizationHandler.cpp b/Module/Source/ModLoader/Handlers/LocalizationHandler.cpp index fbb6c96e..b9ca5eae 100644 --- a/Module/Source/ModLoader/Handlers/LocalizationHandler.cpp +++ b/Module/Source/ModLoader/Handlers/LocalizationHandler.cpp @@ -1,12 +1,14 @@ // Copyright Armchair Developers / Sean Kahler. Licensed under GPLv3. #include - #include #include #include #include +#include +#include +#include namespace Kyber { @@ -17,16 +19,7 @@ LocalizationHandler::LocalizationHandler() void LocalizationHandler::Load(const eastl::string& modName, bb::ByteBuffer& buf, LocalizationMergeData* data) { uint32_t magic = buf.getInt(); - - int32_t count = 0; - if (magic != 0xABCD0001) - { - count = (int32_t)magic; - } - else - { - count = buf.getInt(); - } + int32_t count = (magic == 0xABCD0001) ? buf.getInt() : (int32_t)magic; for (int i = 0; i < count; i++) { @@ -40,174 +33,171 @@ bool LocalizationHandler::Modify(CustomAssetHandlerContext& ctx, DataContainer* { UITextDatabase* db = static_cast(container); - uint8_t* histogramData = new uint8_t[db->HistogramChunkSize]; - ModLoader::ReadChunkSync(db->HistogramChunk, histogramData, db->HistogramChunkSize); + // 1. Process Histogram + uint8_t* histogramDataRaw = new uint8_t[db->HistogramChunkSize]; + ModLoader::ReadChunkSync(db->HistogramChunk, histogramDataRaw, db->HistogramChunkSize); + + // Detect character table start (skipping header and padding) + uint32_t tableOffset = 0x100; + for (uint32_t i = 8; i < 0x200; i += 2) { + if (*(uint16_t*)(histogramDataRaw + i) != 0) { + tableOffset = i; + break; + } + } + + // Mapping of Char -> Byte Index + std::map charMap; + std::vector patchedTable = ModifyHistogram(&histogramDataRaw, db->HistogramChunkSize, data, tableOffset, charMap); + + uint8_t* persistentHistogram = new uint8_t[db->HistogramChunkSize]; + memset(persistentHistogram, 0, db->HistogramChunkSize); + memcpy(persistentHistogram, histogramDataRaw, db->HistogramChunkSize); + + // Inject our custom character table + memcpy(persistentHistogram + tableOffset, patchedTable.data(), patchedTable.size() * 2); - std::vector values = ModifyHistogram(&histogramData); + ModLoader::ModifyChunk(db->HistogramChunk, (const void*)persistentHistogram, db->HistogramChunkSize); + delete[] histogramDataRaw; + // 2. Process Binary Strings uint8_t* chunkData = new uint8_t[db->BinaryChunkSize]; ModLoader::ReadChunkSync(db->BinaryChunk, chunkData, db->BinaryChunkSize); - std::vector chunk = ModifyChunk(chunkData, db->BinaryChunkSize, data, values); + std::vector newBinaryVec = ModifyChunk(chunkData, db->BinaryChunkSize, data, charMap); - KYBER_LOG(Info, "[ModLoader] Wrote Localization chunk (Sz: " << chunk.size() << ", " << db->BinaryChunkSize << ")"); - db->BinaryChunkSize = chunk.size(); + uint8_t* persistentBinary = new uint8_t[newBinaryVec.size()]; + memcpy(persistentBinary, newBinaryVec.data(), newBinaryVec.size()); - uint8_t* newData = new uint8_t[chunk.size()]; - memcpy(newData, chunk.data(), chunk.size()); - - ModLoader::ModifyChunk(db->BinaryChunk, newData, chunk.size()); + db->BinaryChunkSize = (uint32_t)newBinaryVec.size(); + ModLoader::ModifyChunk(db->BinaryChunk, (const void*)persistentBinary, (uint32_t)newBinaryVec.size()); delete[] chunkData; return true; } -std::vector LocalizationHandler::ModifyHistogram(uint8_t** histogramData) +std::vector LocalizationHandler::ModifyHistogram(uint8_t** histogramDataPtr, uint32_t sizeBytes, LocalizationMergeData* data, uint32_t tableOffset, std::map& outCharMap) { - std::vector values; - uint32_t unk = RT_READ_BUF(uint32_t, histogramData); - uint32_t size = RT_READ_BUF(uint32_t, histogramData); - uint32_t unk2 = RT_READ_BUF(uint32_t, histogramData); + uint8_t* raw = *histogramDataPtr; + const int numEntries = 2048; + std::vector histogram(numEntries, 0); - for (int i = 0; i < (size / 2); i++) - { - wchar_t data = RT_READ_BUF(wchar_t, histogramData); - values.push_back(data); + // Copy original character map (latin, icons, numbers) + uint32_t bytesToCopy = std::min((uint32_t)(numEntries * 2), sizeBytes - tableOffset); + memcpy(histogram.data(), raw + tableOffset, bytesToCopy); + + // Populate existing characters into our map for strings processing + for (int i = 0; i < 128; i++) { + if (histogram[i] != 0) outCharMap[(wchar_t)histogram[i]] = (uint8_t)i; + } + + // Find all custom non-ASCII characters across all mod strings + std::set neededChars; + for (const auto& entry : data->strings) { + for (wchar_t c : entry.second) { + if (c >= 0x80) neededChars.insert(c); + } + } + + // Remove characters that already exist in the original histogram + for (int i = 0; i < numEntries; i++) { + if (neededChars.count((wchar_t)histogram[i])) { + neededChars.erase((wchar_t)histogram[i]); + } + } + + // In English Frostbite engine, indices 128 and 129 (0x80, 0x81) are often ignored + // or treated as internal control codes. To fix text rendering, we start + // custom characters at index 130 and apply a -2 offset during encoding. + int slotPtr = 130; + for (wchar_t c : neededChars) { + while (slotPtr < 255 && histogram[slotPtr] != 0) slotPtr++; + + if (slotPtr < 255) { + histogram[slotPtr] = (uint16_t)c; + outCharMap[c] = (uint8_t)(slotPtr - 2); // Save index with -2 correction + slotPtr++; + } } - return values; + return histogram; } std::vector LocalizationHandler::ModifyChunk( - uint8_t* chunkData, uint32_t chunkSize, LocalizationMergeData* data, const std::vector& values) + uint8_t* chunkData, uint32_t chunkSize, LocalizationMergeData* data, const std::map& charMap) { bb::ByteBuffer inBuf(chunkData, chunkSize); - uint32_t magic = inBuf.getInt(); - if (magic != 0x00039000) - { - KYBER_LOG(Error, "Failed to merge localization: Invalid Chunk Header"); - return std::vector(); - } - uint32_t size = inBuf.getInt(); int32_t count = inBuf.getInt(); uint32_t dataOffset = inBuf.getInt(); uint32_t stringsOffset = inBuf.getInt(); - std::string tag = inBuf.getNullTerminatedString(); - inBuf.setReadPos(dataOffset + 8); - std::map strings; std::vector ids(count); - std::vector offsets(count); - - for (int i = 0; i < count; i++) - { + + // Read original string database + inBuf.setReadPos(dataOffset + 8); + for (int i = 0; i < count; i++) { ids[i] = inBuf.getInt(); - offsets[i] = inBuf.getInt(); - } - - for (int i = 0; i < count; i++) - { - inBuf.setReadPos(stringsOffset + offsets[i] + 8); - - std::wstring str = inBuf.getNullTerminatedWideStringAsAscii(); - strings[ids[i]] = str; + uint32_t offset = inBuf.getInt(); + size_t savedPos = inBuf.getReadPos(); + inBuf.setReadPos(stringsOffset + offset + 8); + strings[ids[i]] = inBuf.getNullTerminatedWideStringAsAscii(); + inBuf.setReadPos(savedPos); } - std::vector histogramShifts; - for (int i = 0x1FE; i >= 0x80; i--) - { - if (values[i] < 0x80) - { - histogramShifts.push_back((uint8_t)i); - } + // Merge mod strings + for (const auto& entry : data->strings) { + if (std::find(ids.begin(), ids.end(), entry.first) == ids.end()) ids.push_back(entry.first); + strings[entry.first] = entry.second; } + std::sort(ids.begin(), ids.end()); - for (const auto& entry : data->strings) - { - std::wstring sb; - for (char16_t b : entry.second) - { - if (b < 0x80) - { - sb += b; - continue; - } + bb::ByteBuffer stringBuf; + std::vector newOffsets; - auto it = std::find(values.begin(), values.end(), b); - if (it == values.end()) - { - KYBER_LOG(Debug, "Character not supported: " << (uint16_t)b << " from string: " << entry.first); - continue; - } + for (uint32_t id : ids) { + newOffsets.push_back((uint32_t)stringBuf.getWritePos()); + std::wstring& text = strings[id]; - auto index = std::distance(values.begin(), it); - if (index <= 0xFF) - { - sb += (char16_t)((uint8_t)index); + for (wchar_t c : text) { + // Native ASCII support + if (c < 128) { + stringBuf.put((uint8_t)c); continue; } - for (uint8_t shift : histogramShifts) - { - if ((index - (values[shift] << 7)) < 0x80) - { - sb += (char16_t)shift; - sb += (char16_t)((uint8_t)(index - (values[shift] << 7) + 0x80)); - break; - } + // Lookup custom character from our corrected map + if (charMap.count(c)) { + stringBuf.put(charMap.at(c)); + } else { + stringBuf.put((uint8_t)'?'); } } - - if (std::find(ids.begin(), ids.end(), entry.first) == ids.end()) - { - ids.push_back(entry.first); - } - - strings[entry.first] = sb; - } - - std::sort(ids.begin(), ids.end()); - offsets.clear(); - - bb::ByteBuffer stringBuf; - for (int i = 0; i < ids.size(); i++) - { - offsets.push_back(stringBuf.getWritePos()); - - std::wstring& str = strings[ids[i]]; - stringBuf.putNullTerminatedWideString(str.c_str()); + stringBuf.put((uint8_t)0x00); } - bb::ByteBuffer buf; - buf.putInt(0x00039000); // magic - buf.putInt(0xdeadbeef); // size - buf.putInt(ids.size()); - buf.putInt(0x8C); // dataOffset - - buf.putInt(0x8C + (8 * ids.size())); // stringsOffset - buf.putNullTerminatedString(tag.c_str()); + bb::ByteBuffer outBuf; + outBuf.putInt(magic); + outBuf.putInt(0); + outBuf.putInt((int32_t)ids.size()); + outBuf.putInt(0x8C); + outBuf.putInt(0x8C + (8 * (uint32_t)ids.size())); + outBuf.putNullTerminatedString(tag.c_str()); - while (buf.getWritePos() < 0x8C + 8) - { - buf.put((uint8_t)0x00); - } + while (outBuf.getWritePos() < 0x8C + 8) outBuf.put((uint8_t)0x00); - for (int i = 0; i < ids.size(); i++) - { - buf.putInt(ids[i]); - buf.putInt(offsets[i]); + for (int i = 0; i < (int)ids.size(); i++) { + outBuf.putInt(ids[i]); + outBuf.putInt(newOffsets[i]); } + outBuf.put(&stringBuf); - buf.put(&stringBuf); - - size = buf.getWritePos() - 8; - buf.setWritePos(4); - buf.putInt(size); + outBuf.setWritePos(4); + outBuf.putInt((uint32_t)outBuf.getWritePos() - 8); - return buf.getBuf(); + return outBuf.getBuf(); } } // namespace Kyber From 7e3cc4b983beec35e7224c1c9081dba9aef58ca8 Mon Sep 17 00:00:00 2001 From: Demitry235 Date: Tue, 27 Jan 2026 01:36:48 +0300 Subject: [PATCH 3/4] Refactor to follow project standards From 1ca20fc60c0b7e2f5a265dfe1f1b322131e3e9bb Mon Sep 17 00:00:00 2001 From: Demitry235 Date: Tue, 27 Jan 2026 01:38:56 +0300 Subject: [PATCH 4/4] Refactor to follow project standards --- .../Handlers/LocalizationHandler.cpp | 104 +++++++++++------- 1 file changed, 64 insertions(+), 40 deletions(-) diff --git a/Module/Source/ModLoader/Handlers/LocalizationHandler.cpp b/Module/Source/ModLoader/Handlers/LocalizationHandler.cpp index b9ca5eae..3c991c10 100644 --- a/Module/Source/ModLoader/Handlers/LocalizationHandler.cpp +++ b/Module/Source/ModLoader/Handlers/LocalizationHandler.cpp @@ -5,6 +5,8 @@ #include #include +#include + #include #include #include @@ -14,15 +16,15 @@ namespace Kyber { LocalizationHandler::LocalizationHandler() : GenericCustomAssetHandler(CustomAssetHandlerLoadStage_PostLoad) -{} +{ +} void LocalizationHandler::Load(const eastl::string& modName, bb::ByteBuffer& buf, LocalizationMergeData* data) { uint32_t magic = buf.getInt(); - int32_t count = (magic == 0xABCD0001) ? buf.getInt() : (int32_t)magic; + int32_t count = (magic == 0xABCD0001) ? buf.getInt() : static_cast(magic); - for (int i = 0; i < count; i++) - { + for (int32_t i = 0; i < count; i++) { uint32_t hash = buf.getInt(); std::wstring str = buf.getNullTerminatedWideString(); data->strings[hash] = str; @@ -34,13 +36,15 @@ bool LocalizationHandler::Modify(CustomAssetHandlerContext& ctx, DataContainer* UITextDatabase* db = static_cast(container); // 1. Process Histogram - uint8_t* histogramDataRaw = new uint8_t[db->HistogramChunkSize]; + // Using FB_GLOBAL_ARENA for memory allocation + uint8_t* histogramDataRaw = static_cast(FB_GLOBAL_ARENA->alloc(db->HistogramChunkSize)); ModLoader::ReadChunkSync(db->HistogramChunk, histogramDataRaw, db->HistogramChunkSize); // Detect character table start (skipping header and padding) uint32_t tableOffset = 0x100; + for (uint32_t i = 8; i < 0x200; i += 2) { - if (*(uint16_t*)(histogramDataRaw + i) != 0) { + if (*reinterpret_cast(histogramDataRaw + i) != 0) { tableOffset = i; break; } @@ -50,72 +54,79 @@ bool LocalizationHandler::Modify(CustomAssetHandlerContext& ctx, DataContainer* std::map charMap; std::vector patchedTable = ModifyHistogram(&histogramDataRaw, db->HistogramChunkSize, data, tableOffset, charMap); - uint8_t* persistentHistogram = new uint8_t[db->HistogramChunkSize]; + // Allocate persistent memory in Arena + uint8_t* persistentHistogram = static_cast(FB_GLOBAL_ARENA->alloc(db->HistogramChunkSize)); memset(persistentHistogram, 0, db->HistogramChunkSize); memcpy(persistentHistogram, histogramDataRaw, db->HistogramChunkSize); // Inject our custom character table memcpy(persistentHistogram + tableOffset, patchedTable.data(), patchedTable.size() * 2); - ModLoader::ModifyChunk(db->HistogramChunk, (const void*)persistentHistogram, db->HistogramChunkSize); - delete[] histogramDataRaw; + ModLoader::ModifyChunk(db->HistogramChunk, static_cast(persistentHistogram), db->HistogramChunkSize); // 2. Process Binary Strings - uint8_t* chunkData = new uint8_t[db->BinaryChunkSize]; + uint8_t* chunkData = static_cast(FB_GLOBAL_ARENA->alloc(db->BinaryChunkSize)); ModLoader::ReadChunkSync(db->BinaryChunk, chunkData, db->BinaryChunkSize); std::vector newBinaryVec = ModifyChunk(chunkData, db->BinaryChunkSize, data, charMap); - uint8_t* persistentBinary = new uint8_t[newBinaryVec.size()]; + uint8_t* persistentBinary = static_cast(FB_GLOBAL_ARENA->alloc(newBinaryVec.size())); memcpy(persistentBinary, newBinaryVec.data(), newBinaryVec.size()); - db->BinaryChunkSize = (uint32_t)newBinaryVec.size(); - ModLoader::ModifyChunk(db->BinaryChunk, (const void*)persistentBinary, (uint32_t)newBinaryVec.size()); + db->BinaryChunkSize = static_cast(newBinaryVec.size()); + ModLoader::ModifyChunk(db->BinaryChunk, static_cast(persistentBinary), static_cast(newBinaryVec.size())); - delete[] chunkData; return true; } std::vector LocalizationHandler::ModifyHistogram(uint8_t** histogramDataPtr, uint32_t sizeBytes, LocalizationMergeData* data, uint32_t tableOffset, std::map& outCharMap) { uint8_t* raw = *histogramDataPtr; - const int numEntries = 2048; + const int32_t numEntries = 2048; std::vector histogram(numEntries, 0); // Copy original character map (latin, icons, numbers) - uint32_t bytesToCopy = std::min((uint32_t)(numEntries * 2), sizeBytes - tableOffset); + uint32_t bytesToCopy = std::min(static_cast(numEntries * 2), sizeBytes - tableOffset); memcpy(histogram.data(), raw + tableOffset, bytesToCopy); // Populate existing characters into our map for strings processing - for (int i = 0; i < 128; i++) { - if (histogram[i] != 0) outCharMap[(wchar_t)histogram[i]] = (uint8_t)i; + for (int32_t i = 0; i < 128; i++) { + if (histogram[i] != 0) { + outCharMap[static_cast(histogram[i])] = static_cast(i); + } } // Find all custom non-ASCII characters across all mod strings std::set neededChars; + for (const auto& entry : data->strings) { for (wchar_t c : entry.second) { - if (c >= 0x80) neededChars.insert(c); + if (c >= 0x80) { + neededChars.insert(c); + } } } // Remove characters that already exist in the original histogram - for (int i = 0; i < numEntries; i++) { - if (neededChars.count((wchar_t)histogram[i])) { - neededChars.erase((wchar_t)histogram[i]); + for (int32_t i = 0; i < numEntries; i++) { + if (neededChars.count(static_cast(histogram[i]))) { + neededChars.erase(static_cast(histogram[i])); } } // In English Frostbite engine, indices 128 and 129 (0x80, 0x81) are often ignored // or treated as internal control codes. To fix text rendering, we start // custom characters at index 130 and apply a -2 offset during encoding. - int slotPtr = 130; + int32_t slotPtr = 130; + for (wchar_t c : neededChars) { - while (slotPtr < 255 && histogram[slotPtr] != 0) slotPtr++; + while (slotPtr < 255 && histogram[slotPtr] != 0) { + slotPtr++; + } if (slotPtr < 255) { - histogram[slotPtr] = (uint16_t)c; - outCharMap[c] = (uint8_t)(slotPtr - 2); // Save index with -2 correction + histogram[slotPtr] = static_cast(c); + outCharMap[c] = static_cast(slotPtr - 2); slotPtr++; } } @@ -128,6 +139,12 @@ std::vector LocalizationHandler::ModifyChunk( { bb::ByteBuffer inBuf(chunkData, chunkSize); uint32_t magic = inBuf.getInt(); + + if (magic != 0x00039000) { + KYBER_LOG(Error, "Failed to merge localization: Invalid Chunk Header"); + return std::vector(); + } + uint32_t size = inBuf.getInt(); int32_t count = inBuf.getInt(); uint32_t dataOffset = inBuf.getInt(); @@ -137,12 +154,13 @@ std::vector LocalizationHandler::ModifyChunk( std::map strings; std::vector ids(count); - // Read original string database inBuf.setReadPos(dataOffset + 8); - for (int i = 0; i < count; i++) { + + for (int32_t i = 0; i < count; i++) { ids[i] = inBuf.getInt(); uint32_t offset = inBuf.getInt(); size_t savedPos = inBuf.getReadPos(); + inBuf.setReadPos(stringsOffset + offset + 8); strings[ids[i]] = inBuf.getNullTerminatedWideStringAsAscii(); inBuf.setReadPos(savedPos); @@ -150,53 +168,59 @@ std::vector LocalizationHandler::ModifyChunk( // Merge mod strings for (const auto& entry : data->strings) { - if (std::find(ids.begin(), ids.end(), entry.first) == ids.end()) ids.push_back(entry.first); + if (std::find(ids.begin(), ids.end(), entry.first) == ids.end()) { + ids.push_back(entry.first); + } + strings[entry.first] = entry.second; } + std::sort(ids.begin(), ids.end()); bb::ByteBuffer stringBuf; std::vector newOffsets; for (uint32_t id : ids) { - newOffsets.push_back((uint32_t)stringBuf.getWritePos()); + newOffsets.push_back(static_cast(stringBuf.getWritePos())); std::wstring& text = strings[id]; for (wchar_t c : text) { - // Native ASCII support if (c < 128) { - stringBuf.put((uint8_t)c); + stringBuf.put(static_cast(c)); continue; } - // Lookup custom character from our corrected map if (charMap.count(c)) { stringBuf.put(charMap.at(c)); } else { - stringBuf.put((uint8_t)'?'); + stringBuf.put(static_cast('?')); } } - stringBuf.put((uint8_t)0x00); + + stringBuf.put(static_cast(0x00)); } bb::ByteBuffer outBuf; outBuf.putInt(magic); outBuf.putInt(0); - outBuf.putInt((int32_t)ids.size()); + outBuf.putInt(static_cast(ids.size())); outBuf.putInt(0x8C); - outBuf.putInt(0x8C + (8 * (uint32_t)ids.size())); + outBuf.putInt(0x8C + (8 * static_cast(ids.size()))); outBuf.putNullTerminatedString(tag.c_str()); - while (outBuf.getWritePos() < 0x8C + 8) outBuf.put((uint8_t)0x00); + while (outBuf.getWritePos() < 0x8C + 8) { + outBuf.put(static_cast(0x00)); + } - for (int i = 0; i < (int)ids.size(); i++) { + for (int32_t i = 0; i < static_cast(ids.size()); i++) { outBuf.putInt(ids[i]); outBuf.putInt(newOffsets[i]); } + outBuf.put(&stringBuf); outBuf.setWritePos(4); - outBuf.putInt((uint32_t)outBuf.getWritePos() - 8); + outBuf.putInt(static_cast(outBuf.getWritePos()) - 8); return outBuf.getBuf(); }