#include "gamedata.h" #include "indexparser.h" #include "crc32checksum.h" #include "compression.h" #include "string_utils.h" #include "exlparser.h" #include #include #include #include // TODO: should be enum? // taken from https://xiv.dev/data-files/sqpack#categories std::unordered_map categoryToID = { {"common", 0}, {"bgcommon", 1}, {"bg", 2}, {"cut", 3}, {"chara", 4}, {"shader", 5}, {"ui", 6}, {"sound", 7}, {"vfx", 8}, {"ui_script", 9}, {"exd", 10}, {"game_script", 11}, {"music", 12}, {"sqpack_test", 13}, {"debug", 14}, }; GameData::GameData(const std::string_view dataDirectory) { this->dataDirectory = dataDirectory; extractFile("exd/root.exl", "root.exl"); rootEXL = readEXL("root.exl"); } std::vector GameData::getAllSheetNames() { std::vector names; for(auto row : rootEXL.rows) { names.push_back(row.name); } return names; } uint64_t GameData::calculateHash(const std::string_view path) { std::string data = toLowercase(path.data()); auto lastSeperator = data.find_last_of('/'); const std::string filename = data.substr(lastSeperator + 1, data.length()); const std::string directory = data.substr(0, lastSeperator); uint32_t table[256] = {}; CRC32::generate_table(table); // we actually want JAMCRC, which is just the bitwise not of a regular crc32 hash const uint32_t directoryCrc = ~CRC32::update(table, 0, directory.data(), directory.size()); const uint32_t filenameCrc = ~CRC32::update(table, 0, filename.data(), filename.size()); return static_cast(directoryCrc) << 32 | filenameCrc; } std::tuple GameData::calculateRepositoryCategory(std::string_view path) { std::string repository, category; auto tokens = tokenize(path, "/"); if(stringContains(tokens[1], "ex") && !stringContains(tokens[0], "exd") && !stringContains(tokens[0], "exh")) { repository = tokens[1]; } else { repository = "ffxiv"; } category = tokens[0]; return {repository, category}; } int getExpansionID(std::string_view repositoryName) { if(repositoryName == "ffxiv") return 0; return std::stoi(std::string(repositoryName.substr(2, 2))); } std::string GameData::calculateFilename(const int category, const int expansion, const int chunk, const std::string_view platform, const std::string_view type) { return fmt::sprintf("%02x%02x%02x.%s.%s", category, expansion, chunk, platform, type); } void GameData::extractFile(std::string_view dataFilePath, std::string_view outPath) { const uint64_t hash = calculateHash(dataFilePath); auto [repository, category] = calculateRepositoryCategory(dataFilePath); fmt::print("repository = {}\n", repository); fmt::print("category = {}\n", category); // TODO: handle platforms other than win32 auto indexFilename = calculateFilename(categoryToID[category], getExpansionID(repository), 0, "win32", "index"); fmt::print("calculated index filename: "); // TODO: handle hashes in index2 files (we can read them but it's not setup yet.) auto indexFile = readIndexFile(dataDirectory + "/" + repository + "/" + indexFilename); for(const auto entry : indexFile.entries) { if(entry.hash == hash) { auto dataFilename = calculateFilename(categoryToID[category], getExpansionID(repository), entry.dataFileId, "win32", "dat0"); fmt::print("Opening data file {}...\n", dataFilename); FILE* file = fopen((dataDirectory + "/" + repository + "/" + dataFilename).c_str(), "rb"); if(file == nullptr) { throw std::runtime_error("Failed to open data file: " + dataFilename); } const size_t offset = entry.offset * 0x80; fseek(file, offset, SEEK_SET); enum FileType : int32_t { Empty = 1, Standard = 2, Model = 3, Texture = 4 }; struct FileInfo { uint32_t size; FileType fileType; int32_t fileSize; uint32_t dummy[2]; uint32_t numBlocks; } info; fread(&info, sizeof(FileInfo), 1, file); fmt::print("file size = {}\n", info.fileSize); if(info.fileType != FileType::Standard) { throw std::runtime_error("File type is not handled yet for " + std::string(dataFilePath)); } struct Block { int32_t offset; int16_t dummy; int16_t dummy2; }; std::vector blocks; for(int i = 0; i < info.numBlocks; i++) { Block block; fread(&block, sizeof(Block), 1, file); blocks.push_back(block); } std::vector data; const size_t startingPos = offset + info.size; for(auto block : blocks) { struct BlockHeader { int32_t size; int32_t dummy; int32_t compressedLength; // < 32000 is uncompressed data int32_t decompressedLength; } header; fseek(file, startingPos + block.offset, SEEK_SET); fread(&header, sizeof(BlockHeader), 1, file); std::vector localdata; bool isCompressed = header.compressedLength < 32000; if(isCompressed) { localdata.resize(header.decompressedLength); std::vector compressed_data; compressed_data.resize(header.compressedLength); fread(compressed_data.data(), header.compressedLength, 1, file); zlib::no_header_decompress(reinterpret_cast(compressed_data.data()), compressed_data.size(), reinterpret_cast(localdata.data()), header.decompressedLength); } else { localdata.resize(header.decompressedLength); fread(localdata.data(), header.decompressedLength, 1, file); } data.insert(data.end(), localdata.begin(), localdata.end()); } fclose(file); FILE* newFile = fopen(outPath.data(), "w"); fwrite(data.data(), data.size(), 1, newFile); fclose(newFile); } } fmt::print("Extracted {} to {}\n", dataFilePath, outPath); } std::optional GameData::readExcelSheet(std::string_view name) { fmt::print("Beginning to read excel sheet {}...\n", name); for(auto row : rootEXL.rows) { if(row.name == name) { fmt::print("Found row {} at id {}!\n", name, row.id); // we want it as lowercase (Item -> item) std::string newFilename = name.data(); std::transform(newFilename.begin(), newFilename.end(), newFilename.begin(), [](unsigned char c){ return std::tolower(c); }); std::string exhFilename = "exd/" + newFilename + ".exh"; extractFile(exhFilename, newFilename + ".exh"); fmt::print("Done extracting files, now parsing...\n"); return readEXH(newFilename + ".exh"); } } return {}; }