From 7006cc46625dac73cc64b489174179cbb9e41262 Mon Sep 17 00:00:00 2001 From: Joshua Goins Date: Mon, 11 Apr 2022 10:18:29 -0400 Subject: [PATCH] Add exl parser, make exd category configurable Now you can easily parse EXL files, and this commit includes changes to GameData to easily extract a EXH from just an excel sheet name like "Item" or "Map". However this still doesn't work for language seperated excel sheets yet. --- CMakeLists.txt | 3 ++- include/exdparser.h | 4 +++- include/exlparser.h | 16 ++++++++++++++++ include/gamedata.h | 9 +++++++++ src/exdparser.cpp | 12 ++++++++---- src/exlparser.cpp | 27 +++++++++++++++++++++++++++ src/gamedata.cpp | 44 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 include/exlparser.h create mode 100644 src/exlparser.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4571e06..93b6e30 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,7 +70,8 @@ add_library(libxiv STATIC src/exhparser.cpp src/exdparser.cpp src/installextract.cpp - src/patch.cpp) + src/patch.cpp + src/exlparser.cpp) target_include_directories(libxiv PUBLIC include PRIVATE src) target_link_libraries(libxiv PUBLIC ${LIBRARIES}) target_link_directories(libxiv PUBLIC ${LIB_DIRS}) \ No newline at end of file diff --git a/include/exdparser.h b/include/exdparser.h index deed40a..8b63f71 100644 --- a/include/exdparser.h +++ b/include/exdparser.h @@ -19,4 +19,6 @@ struct EXD { std::vector rows; }; -EXD readEXD(EXH& exh, ExcelDataPagination& page); \ No newline at end of file +std::string getEXDFilename(EXH& exh, std::string_view name, ExcelDataPagination& page); + +EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page); \ No newline at end of file diff --git a/include/exlparser.h b/include/exlparser.h new file mode 100644 index 0000000..3ad3c8e --- /dev/null +++ b/include/exlparser.h @@ -0,0 +1,16 @@ +#pragma once + +#include +#include +#include + +struct EXLRow { + std::string name; + int id; +}; + +struct EXL { + std::vector rows; +}; + +EXL readEXL(std::string_view path); \ No newline at end of file diff --git a/include/gamedata.h b/include/gamedata.h index e23022c..9490747 100644 --- a/include/gamedata.h +++ b/include/gamedata.h @@ -2,6 +2,9 @@ #include #include +#include +#include "exhparser.h" +#include "exlparser.h" /* * This handles reading/extracting the raw data from game data packs, such as dat0, index and index2 files. @@ -23,6 +26,10 @@ public: */ void extractFile(std::string_view dataFilePath, std::string_view outPath); + std::optional readExcelSheet(std::string_view name); + + std::vector getAllSheetNames(); + private: /* * This returns a proper SQEX-style filename for index, index2, and dat files. @@ -41,4 +48,6 @@ private: uint64_t calculateHash(std::string_view path); std::string dataDirectory; + + EXL rootEXL; }; \ No newline at end of file diff --git a/src/exdparser.cpp b/src/exdparser.cpp index 59c1918..7813e31 100644 --- a/src/exdparser.cpp +++ b/src/exdparser.cpp @@ -36,10 +36,14 @@ std::string readData(FILE* file, int offset) { return std::to_string(value); } -EXD readEXD(EXH& exh, ExcelDataPagination& page) { - EXD exd; +std::string getEXDFilename(EXH& exh, std::string_view name, ExcelDataPagination& page) { + auto path = fmt::format("{}_{}.exd", name, page.startId); - auto path = fmt::format("{}_{}.exd", "map", page.startId); + return path; +} + +EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page) { + EXD exd; FILE* file = fopen(path.data(), "rb"); if(file == nullptr) { @@ -90,7 +94,7 @@ EXD readEXD(EXH& exh, ExcelDataPagination& page) { std::string string; int ch; - while ((ch = fgetc(file)) != '\0') { + while ((ch = fgetc(file)) != '\0' && ch != '\377') { string.push_back((char)ch); } diff --git a/src/exlparser.cpp b/src/exlparser.cpp new file mode 100644 index 0000000..97f9f73 --- /dev/null +++ b/src/exlparser.cpp @@ -0,0 +1,27 @@ +#include "exlparser.h" +#include + +EXL readEXL(std::string_view path) { + FILE* file = fopen(path.data(), "rb"); + if(!file) { + throw std::runtime_error("Failed to read exl file from " + std::string(path.data())); + } + + EXL exl; + + char* data = nullptr; + size_t len = 0; + + while ((getline(&data, &len, file)) != -1) { + std::string line = data; + + const size_t comma = line.find_first_of(','); + + std::string name = line.substr(0, comma); + std::string id = line.substr(comma + 1, line.length()); + + exl.rows.push_back({name, std::stoi(id)}); + } + + return exl; +} \ No newline at end of file diff --git a/src/gamedata.cpp b/src/gamedata.cpp index ccc5000..d0c306c 100644 --- a/src/gamedata.cpp +++ b/src/gamedata.cpp @@ -3,6 +3,7 @@ #include "crc32checksum.h" #include "compression.h" #include "string_utils.h" +#include "exlparser.h" #include #include @@ -31,6 +32,20 @@ std::unordered_map categoryToID = { GameData::GameData(const std::string_view dataDirectory) { this->dataDirectory = dataDirectory; + + extractFile("exd/root.exl", "root.exl"); + + rootEXL = readEXL("root.exl"); +} + +std::vector GameData::getAllSheetNames() { + std::vector names; + + for(auto row : rootEXL.rows) { + names.push_back(row.name); + } + + return names; } uint64_t GameData::calculateHash(const std::string_view path) { @@ -86,6 +101,8 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa // TODO: handle platforms other than win32 auto indexFilename = calculateFilename(categoryToID[category], getExpansionID(repository), 0, "win32", "index"); + fmt::print("calculated index filename: "); + // TODO: handle hashes in index2 files (we can read them but it's not setup yet.) auto indexFile = readIndexFile(dataDirectory + "/" + repository + "/" + indexFilename); @@ -93,6 +110,8 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa if(entry.hash == hash) { auto dataFilename = calculateFilename(categoryToID[category], getExpansionID(repository), entry.dataFileId, "win32", "dat0"); + fmt::print("Opening data file {}...\n", dataFilename); + FILE* file = fopen((dataDirectory + "/" + repository + "/" + dataFilename).c_str(), "rb"); if(file == nullptr) { throw std::runtime_error("Failed to open data file: " + dataFilename); @@ -187,3 +206,28 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa fmt::print("Extracted {} to {}\n", dataFilePath, outPath); } + +std::optional GameData::readExcelSheet(std::string_view name) { + fmt::print("Beginning to read excel sheet {}...\n", name); + + for(auto row : rootEXL.rows) { + if(row.name == name) { + fmt::print("Found row {} at id {}!\n", name, row.id); + + // we want it as lowercase (Item -> item) + std::string newFilename = name.data(); + std::transform(newFilename.begin(), newFilename.end(), newFilename.begin(), + [](unsigned char c){ return std::tolower(c); }); + + std::string exhFilename = "exd/" + newFilename + ".exh"; + + extractFile(exhFilename, newFilename + ".exh"); + + fmt::print("Done extracting files, now parsing...\n"); + + return readEXH(newFilename + ".exh"); + } + } + + return {}; +} \ No newline at end of file