diff --git a/docs/PC.md b/docs/PC.md index 579c8e78f..50a8d758f 100644 --- a/docs/PC.md +++ b/docs/PC.md @@ -45,7 +45,13 @@ port/build/pc/darkcloud --data data --save save ``` The extractor accepts a PAL or NTSC disc image. For NTSC data it supplies the -localized filenames and pack members expected by the shared game code. Original +localized filenames and pack members expected by the shared game code. For the +PAL disc's English data it gives every weapon icon sheet the one in the HUD's +palette, which the language's dungeon-entry packs carry. The game copies the +HUD's weapon icon by palette index from whichever sheet a menu last loaded, and +English's others keep the American palette, so on closing the main menu or the +quick-change ring the icon came out in the wrong colours. The other languages' +fishing and shop sheets get it too. Original disc files remain in the extracted tree for verification against `DATA.HD2`; normalized pack copies live under `data/normalized` and are selected by the port's file index. Extract again after updating the port to refresh them. diff --git a/port/src/tests/data_extract_test.cpp b/port/src/tests/data_extract_test.cpp index 803085ee7..4a47cac0a 100644 --- a/port/src/tests/data_extract_test.cpp +++ b/port/src/tests/data_extract_test.cpp @@ -2,6 +2,7 @@ #include #include +#include #include "data_fixture.hpp" @@ -190,6 +191,84 @@ TEST(DataExtract, NormalizesNtscAssets) { fs::remove_all(dir); } +// An IM2 bank of 16 by 16 eight-bit pictures, each with its own palette and pixel fill. +static Bytes MakeIconBank(const std::vector> &pictures) { + constexpr std::size_t picture = 16 + 48 + 256 + 1024; + Bytes bank(16 + pictures.size() * 48, 0); + std::memcpy(bank.data(), "IM2", 3); + Put32(bank, 4, static_cast(pictures.size())); + for (std::size_t i = 0; i < pictures.size(); i++) { + const auto &[name, colour, pixel] = pictures[i]; + std::memcpy(bank.data() + 16 + i * 48, name.c_str(), name.size()); + Put32(bank, 16 + i * 48 + 32, static_cast(bank.size())); + Bytes tim(picture, 0); + std::memcpy(tim.data(), "TIM2", 4); + Put32(tim, 16, 0x30030); // the wrong total the game's images carry + Put32(tim, 16 + 4, 1024); + Put32(tim, 16 + 8, 256); + tim[16 + 12] = 0x30; + tim[16 + 15] = 0x01; + tim[16 + 19] = 5; + tim[16 + 20] = 16; + tim[16 + 22] = 16; + std::fill(tim.begin() + 64, tim.begin() + 64 + 256, pixel); + for (int c = 0; c < 256; c++) { + Put32(tim, 64 + 256 + c * 4, colour + c); + } + bank.insert(bank.end(), tim.begin(), tim.end()); + } + return bank; +} + +TEST(DataExtract, NormalizesIconSheets) { + fs::path dir = TempDir("normalize_icons"); + Bytes american = MakeIconBank({ + {"quickchara", 0x300, 3}, + {"wepicon", 0x200, 4} + }); + Bytes english = MakeIconBank({ + {"wepicon", 0x100, 5} + }); + Disc disc = MakeDisc({ + {"commenu/a_eng/itempack.img", MakeIconBank({{"itempack", 0x100, 1}}) }, + {"commenu/a_eng/quickchr.pac", MakePack({{"qchr.mes", Pattern(8, 2)}, {"quickchr.img", american}})}, + {"commenu/a_eng/itemlst.img", MakeIconBank({{"wepicon", 0x200, 4}, {"itemicon", 0x200, 7}}) }, + {"commenu/a_eng/dunenter/dunenter2.pak", MakePack({{"dunenter.img", english}}) }, + {"commenu/a_eng/kgetoan2.img", MakeIconBank({{"wepicon", 0x200, 4}}) }, + {"commenu/a_eng/_charatex.img", MakeIconBank({{"wepicon", 0x200, 4}}) }, + {"commenu/a_eng/manual/m10.pac", Pattern(80, 9) }, + {"commenu/a_fre/itempack.img", MakeIconBank({{"itempack", 0x200, 6}}) }, + {"commenu/a_fre/quickchr.pac", MakePack({{"quickchr.img", american}}) }, + {"commenu/a_fre/dunenter/dunenter2.pak", MakePack({{"dunenter.img", english}}) }, + }); + fs::path out = dir / "data"; + dcdata::Extract(dcdata::OpenArchive(WriteStandardIso(dir, disc)), out, nullptr); + + // English: the American sheets take the English one; the rest of each file is as it was. + dcdata::Im2Picture donor = dcdata::FindIm2Picture(english, "wepicon"); + auto holds_donor = [&](const Bytes &bank) { + dcdata::Im2Picture icons = dcdata::FindIm2Picture(bank, "wepicon"); + return icons.size == donor.size && + std::equal(english.begin() + donor.offset, english.begin() + donor.offset + donor.size, bank.begin() + icons.offset); + }; + auto quick = dcdata::ReadPack(ReadBytes(out / "normalized/commenu/a_eng/quickchr.pac")); + ASSERT_EQ(quick.size(), 2); + EXPECT_EQ(quick[0].data, Pattern(8, 2)); + EXPECT_TRUE(holds_donor(quick[1].data)); + dcdata::Im2Picture portraits = dcdata::FindIm2Picture(quick[1].data, "quickchara"); + ASSERT_NE(portraits.size, 0); + EXPECT_TRUE(std::equal(american.begin() + portraits.offset, american.begin() + portraits.offset + portraits.size, + quick[1].data.begin() + portraits.offset)); + EXPECT_TRUE(holds_donor(ReadBytes(out / "normalized/commenu/a_eng/itemlst.img"))); + // The main menu's sheet too, but not a leftover the game never loads, and a .pac without a pack. + EXPECT_TRUE(holds_donor(ReadBytes(out / "normalized/commenu/a_eng/kgetoan2.img"))); + EXPECT_FALSE(fs::exists(out / "normalized/commenu/a_eng/_charatex.img")); + EXPECT_FALSE(fs::exists(out / "normalized/commenu/a_eng/manual/m10.pac")); + // French: its sheet already shares itempack's palette. + EXPECT_FALSE(fs::exists(out / "normalized/commenu/a_fre/quickchr.pac")); + fs::remove_all(dir); +} + TEST(DataExtract, FromDirectory) { fs::path dir = TempDir("extract_dir"); Disc disc = StandardDisc(); diff --git a/tools/dcdata/dcdata.hpp b/tools/dcdata/dcdata.hpp index 780353152..f2e9e582c 100644 --- a/tools/dcdata/dcdata.hpp +++ b/tools/dcdata/dcdata.hpp @@ -506,7 +506,157 @@ inline bool AddPackAlias(std::vector &members, std::string_view name return false; } +// Where a picture named `name` lies in an IM2 bank: the offset of its TIM2 file and its length. +struct Im2Picture { + std::size_t offset = 0; + std::size_t size = 0; +}; + +inline Im2Picture FindIm2Picture(std::span bank, std::string_view name) { + if (bank.size() < 16 || std::memcmp(bank.data(), "IM2", 3) != 0) { + return {}; + } + std::uint32_t count = Le32(bank.data() + 4); + if (16 + std::uint64_t(count) * 48 > bank.size()) { + return {}; + } + for (std::uint32_t i = 0; i < count; i++) { + const unsigned char *entry = bank.data() + 16 + i * 48; + std::size_t length = strnlen(reinterpret_cast(entry), 32); + if (std::string_view(reinterpret_cast(entry), length) != name) { + continue; + } + std::size_t offset = Le32(entry + 32); + if (offset + 32 > bank.size() || std::memcmp(bank.data() + offset, "TIM2", 4) != 0) { + return {}; + } + // The picture header's total size is not reliable in the game's images; its parts are. + const unsigned char *picture = bank.data() + offset + 16; + std::size_t size = 16 + std::size_t(picture[12] | picture[13] << 8) + Le32(picture + 8) + Le32(picture + 4); + if (size > bank.size() - offset) { + return {}; + } + return {offset, size}; + } + return {}; +} + +// The colours of an 8-bit TIM2 picture's 256-entry CLUT as RGB, or none. +inline std::vector Tim2Colours(std::span tim) { + if (tim.size() < 48 || tim[16 + 0x13] != 5 || (tim[16 + 0x0E] | tim[16 + 0x0F] << 8) != 256) { + return {}; + } + std::size_t clut = 16 + (tim[16 + 0x0C] | tim[16 + 0x0D] << 8) + Le32(tim.data() + 16 + 8); + if (clut + 1024 > tim.size()) { + return {}; + } + std::vector colours(256); + for (int i = 0; i < 256; i++) { + colours[i] = Le32(tim.data() + clut + i * 4) & 0xFFFFFF; + } + return colours; +} + +inline int SharedColours(const std::vector &a, const std::vector &b) { + int shared = 0; + for (std::size_t i = 0; i < a.size() && i < b.size(); i++) { + shared += a[i] == b[i]; + } + return shared; +} + +// The dungeon copies weapon icons by palette index from whichever wepicon sheet is loaded into the +// HUD's itempack: a dungeon-entry pack's on a new floor, then a menu's once the main menu or the +// quick-change ring closes or a script reloads the item list. So every wepicon sheet of a language +// has to share itempack's palette, as the American ones do. On the PAL disc only English's +// dungeon-entry sheets do, and the HUD icon comes out in the wrong colours after any of those; the +// other languages' fishing and shop sheets are off as well, though only the town shows them. A +// sheet in another palette takes the dungeon-entry sheet's place: the same icons in itempack's. +inline void NormalizeIconSheets(const fs::path &out, const std::vector &records) { + // A sheet in another palette shares a handful of its colours with itempack's. + constexpr int kSharedPalette = 250; + constexpr std::string_view prefix = "commenu/"; + constexpr std::string_view suffix = "/itempack.img"; + for (const Record &record : records) { + if (!record.path.starts_with(prefix) || !record.path.ends_with(suffix)) { + continue; + } + std::string language = record.path.substr(prefix.size(), record.path.size() - prefix.size() - suffix.size()); + if (language.empty() || language.find('/') != std::string::npos) { + continue; + } + std::vector itempack = ReadFile(out / record.path); + Im2Picture hud = FindIm2Picture(itempack, "itempack"); + if (hud.size == 0) { + continue; + } + std::vector hud_colours = Tim2Colours(std::span(itempack).subspan(hud.offset, hud.size)); + + std::vector donor; + std::string entry_packs = std::format("commenu/{}/dunenter/", language); + for (const Record &other : records) { + if (!donor.empty() || !other.path.starts_with(entry_packs) || !other.path.ends_with(".pak")) { + continue; + } + for (const PackMember &member : ReadPack(ReadFile(out / other.path))) { + Im2Picture icons = FindIm2Picture(member.data, "wepicon"); + if (icons.size != 0 && + SharedColours(Tim2Colours(std::span(member.data).subspan(icons.offset, icons.size)), hud_colours) >= kSharedPalette) { + donor.assign(member.data.begin() + icons.offset, member.data.begin() + icons.offset + icons.size); + break; + } + } + } + if (donor.empty()) { + continue; + } + + // Whether the bank's wepicon was in another palette and the donor took its place. + auto replace = [&](std::vector &bank) { + Im2Picture icons = FindIm2Picture(bank, "wepicon"); + if (icons.size != donor.size() || + SharedColours(Tim2Colours(std::span(bank).subspan(icons.offset, icons.size)), hud_colours) >= kSharedPalette) { + return false; + } + std::copy(donor.begin(), donor.end(), bank.begin() + icons.offset); + return true; + }; + // Every image bank and pack of the language, but the leftovers the game never names (_x, x.old). + std::string folder = std::format("commenu/{}/", language); + for (const Record &file : records) { + std::string_view path = file.path; + std::string_view name = path.substr(path.rfind('/') + 1); + bool bank = path.ends_with(".img"); + if (!path.starts_with(folder) || name.starts_with('_') || + !(bank || path.ends_with(".pak") || path.ends_with(".pac"))) { + continue; + } + std::vector data = ReadFile(out / file.path); + if (bank) { + if (replace(data)) { + WriteFile(out / "normalized" / file.path, data); + } + continue; + } + std::vector members; + try { + members = ReadPack(data); + } catch (const Error &) { + continue; // a .pac that holds no pack + } + bool changed = false; + for (PackMember &member : members) { + changed |= replace(member.data); + } + if (changed) { + WriteFile(out / "normalized" / file.path, WritePack(members)); + } + } + } +} + inline void Normalize(const fs::path &out, const std::vector &records) { + NormalizeIconSheets(out, records); std::unordered_set paths; for (const Record &record : records) { paths.insert(record.path);