diff --git a/Module/Public/ModLoader/Handlers/LocalizationHandler.h b/Module/Public/ModLoader/Handlers/LocalizationHandler.h index 63415825..4976369a 100644 --- a/Module/Public/ModLoader/Handlers/LocalizationHandler.h +++ b/Module/Public/ModLoader/Handlers/LocalizationHandler.h @@ -1,8 +1,9 @@ #pragma once #include - #include +#include +#include namespace Kyber { @@ -20,8 +21,9 @@ class LocalizationHandler : public GenericCustomAssetHandler ModifyHistogram(uint8_t** histogramData); + std::vector ModifyHistogram(uint8_t** histogramData, uint32_t sizeBytes, LocalizationMergeData* data, uint32_t tableOffset, std::map& outCharMap); + std::vector ModifyChunk( - uint8_t* chunkData, uint32_t chunkSize, LocalizationMergeData* data, const std::vector& values); + uint8_t* chunkData, uint32_t chunkSize, LocalizationMergeData* data, const std::map& charMap); }; } // namespace Kyber diff --git a/Module/Source/ModLoader/Handlers/LocalizationHandler.cpp b/Module/Source/ModLoader/Handlers/LocalizationHandler.cpp index fbb6c96e..3c991c10 100644 --- a/Module/Source/ModLoader/Handlers/LocalizationHandler.cpp +++ b/Module/Source/ModLoader/Handlers/LocalizationHandler.cpp @@ -1,35 +1,30 @@ // Copyright Armchair Developers / Sean Kahler. Licensed under GPLv3. #include - #include #include #include +#include + #include +#include +#include +#include namespace Kyber { LocalizationHandler::LocalizationHandler() : GenericCustomAssetHandler(CustomAssetHandlerLoadStage_PostLoad) -{} +{ +} void LocalizationHandler::Load(const eastl::string& modName, bb::ByteBuffer& buf, LocalizationMergeData* data) { uint32_t magic = buf.getInt(); + int32_t count = (magic == 0xABCD0001) ? buf.getInt() : static_cast(magic); - int32_t count = 0; - if (magic != 0xABCD0001) - { - count = (int32_t)magic; - } - else - { - count = buf.getInt(); - } - - for (int i = 0; i < count; i++) - { + for (int32_t i = 0; i < count; i++) { uint32_t hash = buf.getInt(); std::wstring str = buf.getNullTerminatedWideString(); data->strings[hash] = str; @@ -40,52 +35,112 @@ bool LocalizationHandler::Modify(CustomAssetHandlerContext& ctx, DataContainer* { UITextDatabase* db = static_cast(container); - uint8_t* histogramData = new uint8_t[db->HistogramChunkSize]; - ModLoader::ReadChunkSync(db->HistogramChunk, histogramData, db->HistogramChunkSize); + // 1. Process Histogram + // Using FB_GLOBAL_ARENA for memory allocation + uint8_t* histogramDataRaw = static_cast(FB_GLOBAL_ARENA->alloc(db->HistogramChunkSize)); + ModLoader::ReadChunkSync(db->HistogramChunk, histogramDataRaw, db->HistogramChunkSize); - std::vector values = ModifyHistogram(&histogramData); + // Detect character table start (skipping header and padding) + uint32_t tableOffset = 0x100; - uint8_t* chunkData = new uint8_t[db->BinaryChunkSize]; - ModLoader::ReadChunkSync(db->BinaryChunk, chunkData, db->BinaryChunkSize); + for (uint32_t i = 8; i < 0x200; i += 2) { + if (*reinterpret_cast(histogramDataRaw + i) != 0) { + tableOffset = i; + break; + } + } + + // Mapping of Char -> Byte Index + std::map charMap; + std::vector patchedTable = ModifyHistogram(&histogramDataRaw, db->HistogramChunkSize, data, tableOffset, charMap); + + // Allocate persistent memory in Arena + uint8_t* persistentHistogram = static_cast(FB_GLOBAL_ARENA->alloc(db->HistogramChunkSize)); + memset(persistentHistogram, 0, db->HistogramChunkSize); + memcpy(persistentHistogram, histogramDataRaw, db->HistogramChunkSize); + + // Inject our custom character table + memcpy(persistentHistogram + tableOffset, patchedTable.data(), patchedTable.size() * 2); + + ModLoader::ModifyChunk(db->HistogramChunk, static_cast(persistentHistogram), db->HistogramChunkSize); - std::vector chunk = ModifyChunk(chunkData, db->BinaryChunkSize, data, values); + // 2. Process Binary Strings + uint8_t* chunkData = static_cast(FB_GLOBAL_ARENA->alloc(db->BinaryChunkSize)); + ModLoader::ReadChunkSync(db->BinaryChunk, chunkData, db->BinaryChunkSize); - KYBER_LOG(Info, "[ModLoader] Wrote Localization chunk (Sz: " << chunk.size() << ", " << db->BinaryChunkSize << ")"); - db->BinaryChunkSize = chunk.size(); + std::vector newBinaryVec = ModifyChunk(chunkData, db->BinaryChunkSize, data, charMap); - uint8_t* newData = new uint8_t[chunk.size()]; - memcpy(newData, chunk.data(), chunk.size()); + uint8_t* persistentBinary = static_cast(FB_GLOBAL_ARENA->alloc(newBinaryVec.size())); + memcpy(persistentBinary, newBinaryVec.data(), newBinaryVec.size()); - ModLoader::ModifyChunk(db->BinaryChunk, newData, chunk.size()); + db->BinaryChunkSize = static_cast(newBinaryVec.size()); + ModLoader::ModifyChunk(db->BinaryChunk, static_cast(persistentBinary), static_cast(newBinaryVec.size())); - delete[] chunkData; return true; } -std::vector LocalizationHandler::ModifyHistogram(uint8_t** histogramData) +std::vector LocalizationHandler::ModifyHistogram(uint8_t** histogramDataPtr, uint32_t sizeBytes, LocalizationMergeData* data, uint32_t tableOffset, std::map& outCharMap) { - std::vector values; - uint32_t unk = RT_READ_BUF(uint32_t, histogramData); - uint32_t size = RT_READ_BUF(uint32_t, histogramData); - uint32_t unk2 = RT_READ_BUF(uint32_t, histogramData); - - for (int i = 0; i < (size / 2); i++) - { - wchar_t data = RT_READ_BUF(wchar_t, histogramData); - values.push_back(data); + uint8_t* raw = *histogramDataPtr; + const int32_t numEntries = 2048; + std::vector histogram(numEntries, 0); + + // Copy original character map (latin, icons, numbers) + uint32_t bytesToCopy = std::min(static_cast(numEntries * 2), sizeBytes - tableOffset); + memcpy(histogram.data(), raw + tableOffset, bytesToCopy); + + // Populate existing characters into our map for strings processing + for (int32_t i = 0; i < 128; i++) { + if (histogram[i] != 0) { + outCharMap[static_cast(histogram[i])] = static_cast(i); + } + } + + // Find all custom non-ASCII characters across all mod strings + std::set neededChars; + + for (const auto& entry : data->strings) { + for (wchar_t c : entry.second) { + if (c >= 0x80) { + neededChars.insert(c); + } + } + } + + // Remove characters that already exist in the original histogram + for (int32_t i = 0; i < numEntries; i++) { + if (neededChars.count(static_cast(histogram[i]))) { + neededChars.erase(static_cast(histogram[i])); + } + } + + // In English Frostbite engine, indices 128 and 129 (0x80, 0x81) are often ignored + // or treated as internal control codes. To fix text rendering, we start + // custom characters at index 130 and apply a -2 offset during encoding. + int32_t slotPtr = 130; + + for (wchar_t c : neededChars) { + while (slotPtr < 255 && histogram[slotPtr] != 0) { + slotPtr++; + } + + if (slotPtr < 255) { + histogram[slotPtr] = static_cast(c); + outCharMap[c] = static_cast(slotPtr - 2); + slotPtr++; + } } - return values; + return histogram; } std::vector LocalizationHandler::ModifyChunk( - uint8_t* chunkData, uint32_t chunkSize, LocalizationMergeData* data, const std::vector& values) + uint8_t* chunkData, uint32_t chunkSize, LocalizationMergeData* data, const std::map& charMap) { bb::ByteBuffer inBuf(chunkData, chunkSize); - uint32_t magic = inBuf.getInt(); - if (magic != 0x00039000) - { + + if (magic != 0x00039000) { KYBER_LOG(Error, "Failed to merge localization: Invalid Chunk Header"); return std::vector(); } @@ -94,120 +149,79 @@ std::vector LocalizationHandler::ModifyChunk( int32_t count = inBuf.getInt(); uint32_t dataOffset = inBuf.getInt(); uint32_t stringsOffset = inBuf.getInt(); - std::string tag = inBuf.getNullTerminatedString(); - inBuf.setReadPos(dataOffset + 8); - std::map strings; std::vector ids(count); - std::vector offsets(count); + + inBuf.setReadPos(dataOffset + 8); - for (int i = 0; i < count; i++) - { + for (int32_t i = 0; i < count; i++) { ids[i] = inBuf.getInt(); - offsets[i] = inBuf.getInt(); - } - - for (int i = 0; i < count; i++) - { - inBuf.setReadPos(stringsOffset + offsets[i] + 8); + uint32_t offset = inBuf.getInt(); + size_t savedPos = inBuf.getReadPos(); - std::wstring str = inBuf.getNullTerminatedWideStringAsAscii(); - strings[ids[i]] = str; + inBuf.setReadPos(stringsOffset + offset + 8); + strings[ids[i]] = inBuf.getNullTerminatedWideStringAsAscii(); + inBuf.setReadPos(savedPos); } - std::vector histogramShifts; - for (int i = 0x1FE; i >= 0x80; i--) - { - if (values[i] < 0x80) - { - histogramShifts.push_back((uint8_t)i); + // Merge mod strings + for (const auto& entry : data->strings) { + if (std::find(ids.begin(), ids.end(), entry.first) == ids.end()) { + ids.push_back(entry.first); } + + strings[entry.first] = entry.second; } - for (const auto& entry : data->strings) - { - std::wstring sb; - for (char16_t b : entry.second) - { - if (b < 0x80) - { - sb += b; - continue; - } + std::sort(ids.begin(), ids.end()); - auto it = std::find(values.begin(), values.end(), b); - if (it == values.end()) - { - KYBER_LOG(Debug, "Character not supported: " << (uint16_t)b << " from string: " << entry.first); - continue; - } + bb::ByteBuffer stringBuf; + std::vector newOffsets; + + for (uint32_t id : ids) { + newOffsets.push_back(static_cast(stringBuf.getWritePos())); + std::wstring& text = strings[id]; - auto index = std::distance(values.begin(), it); - if (index <= 0xFF) - { - sb += (char16_t)((uint8_t)index); + for (wchar_t c : text) { + if (c < 128) { + stringBuf.put(static_cast(c)); continue; } - for (uint8_t shift : histogramShifts) - { - if ((index - (values[shift] << 7)) < 0x80) - { - sb += (char16_t)shift; - sb += (char16_t)((uint8_t)(index - (values[shift] << 7) + 0x80)); - break; - } + if (charMap.count(c)) { + stringBuf.put(charMap.at(c)); + } else { + stringBuf.put(static_cast('?')); } } - if (std::find(ids.begin(), ids.end(), entry.first) == ids.end()) - { - ids.push_back(entry.first); - } - - strings[entry.first] = sb; + stringBuf.put(static_cast(0x00)); } - std::sort(ids.begin(), ids.end()); - offsets.clear(); - - bb::ByteBuffer stringBuf; - for (int i = 0; i < ids.size(); i++) - { - offsets.push_back(stringBuf.getWritePos()); - - std::wstring& str = strings[ids[i]]; - stringBuf.putNullTerminatedWideString(str.c_str()); - } - - bb::ByteBuffer buf; - buf.putInt(0x00039000); // magic - buf.putInt(0xdeadbeef); // size - buf.putInt(ids.size()); - buf.putInt(0x8C); // dataOffset - - buf.putInt(0x8C + (8 * ids.size())); // stringsOffset - buf.putNullTerminatedString(tag.c_str()); + bb::ByteBuffer outBuf; + outBuf.putInt(magic); + outBuf.putInt(0); + outBuf.putInt(static_cast(ids.size())); + outBuf.putInt(0x8C); + outBuf.putInt(0x8C + (8 * static_cast(ids.size()))); + outBuf.putNullTerminatedString(tag.c_str()); - while (buf.getWritePos() < 0x8C + 8) - { - buf.put((uint8_t)0x00); + while (outBuf.getWritePos() < 0x8C + 8) { + outBuf.put(static_cast(0x00)); } - for (int i = 0; i < ids.size(); i++) - { - buf.putInt(ids[i]); - buf.putInt(offsets[i]); + for (int32_t i = 0; i < static_cast(ids.size()); i++) { + outBuf.putInt(ids[i]); + outBuf.putInt(newOffsets[i]); } - buf.put(&stringBuf); + outBuf.put(&stringBuf); - size = buf.getWritePos() - 8; - buf.setWritePos(4); - buf.putInt(size); + outBuf.setWritePos(4); + outBuf.putInt(static_cast(outBuf.getWritePos()) - 8); - return buf.getBuf(); + return outBuf.getBuf(); } } // namespace Kyber