Add exl parser, make exd category configurable

Now you can easily parse EXL files, and this commit includes changes
to GameData to easily extract a EXH from just an excel sheet name
like "Item" or "Map". However this still doesn't work for language
seperated excel sheets yet.

CMakeLists.txt

@@ -70,7 +70,8 @@ add_library(libxiv STATIC
         src/exhparser.cpp
         src/exdparser.cpp
         src/installextract.cpp
-        src/patch.cpp)
+        src/patch.cpp
+        src/exlparser.cpp)
 target_include_directories(libxiv PUBLIC include PRIVATE src)
 target_link_libraries(libxiv PUBLIC ${LIBRARIES})
 target_link_directories(libxiv PUBLIC ${LIB_DIRS})

include/exdparser.h

@@ -19,4 +19,6 @@ struct EXD {
     std::vector<Row> rows;
 };
 
-EXD readEXD(EXH& exh, ExcelDataPagination& page);
+std::string getEXDFilename(EXH& exh, std::string_view name, ExcelDataPagination& page);
+
+EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page);

include/exlparser.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <string_view>
+#include <vector>
+#include <string>
+
+struct EXLRow {
+    std::string name;
+    int id;
+};
+
+struct EXL {
+    std::vector<EXLRow> rows;
+};
+
+EXL readEXL(std::string_view path);

include/gamedata.h

@@ -2,6 +2,9 @@
 
 #include <string_view>
 #include <string>
+#include <optional>
+#include "exhparser.h"
+#include "exlparser.h"
 
 /*
  * This handles reading/extracting the raw data from game data packs, such as dat0, index and index2 files.
@@ -23,6 +26,10 @@ public:
      */
     void extractFile(std::string_view dataFilePath, std::string_view outPath);
 
+    std::optional<EXH> readExcelSheet(std::string_view name);
+
+    std::vector<std::string> getAllSheetNames();
+
 private:
     /*
      * This returns a proper SQEX-style filename for index, index2, and dat files.
@@ -41,4 +48,6 @@ private:
     uint64_t calculateHash(std::string_view path);
 
     std::string dataDirectory;
+
+    EXL rootEXL;
 };

src/exdparser.cpp

@@ -36,10 +36,14 @@ std::string readData(FILE* file, int offset) {
     return std::to_string(value);
 }
 
-EXD readEXD(EXH& exh, ExcelDataPagination& page) {
+std::string getEXDFilename(EXH& exh, std::string_view name, ExcelDataPagination& page) {
-    EXD exd;
+    auto path = fmt::format("{}_{}.exd", name, page.startId);
 
-    auto path = fmt::format("{}_{}.exd", "map", page.startId);
+    return path;
+}
+
+EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page) {
+    EXD exd;
 
     FILE* file = fopen(path.data(), "rb");
     if(file == nullptr) {
@@ -90,7 +94,7 @@ EXD readEXD(EXH& exh, ExcelDataPagination& page) {
                     std::string string;
 
                     int ch;
-                    while ((ch = fgetc(file)) != '\0') {
+                    while ((ch = fgetc(file)) != '\0' && ch != '\377') {
                         string.push_back((char)ch);
                     }
 

src/exlparser.cpp

@@ -0,0 +1,27 @@
+#include "exlparser.h"
+#include <stdexcept>
+
+EXL readEXL(std::string_view path) {
+    FILE* file = fopen(path.data(), "rb");
+    if(!file) {
+        throw std::runtime_error("Failed to read exl file from " + std::string(path.data()));
+    }
+
+    EXL exl;
+
+    char* data = nullptr;
+    size_t len = 0;
+
+    while ((getline(&data, &len, file)) != -1) {
+        std::string line = data;
+
+        const size_t comma = line.find_first_of(',');
+
+        std::string name = line.substr(0, comma);
+        std::string id = line.substr(comma + 1, line.length());
+
+        exl.rows.push_back({name, std::stoi(id)});
+    }
+
+    return exl;
+}

src/gamedata.cpp

@@ -3,6 +3,7 @@
 #include "crc32checksum.h"
 #include "compression.h"
 #include "string_utils.h"
+#include "exlparser.h"
 
 #include <string>
 #include <algorithm>
@@ -31,6 +32,20 @@ std::unordered_map<std::string_view, int> categoryToID = {
 
 GameData::GameData(const std::string_view dataDirectory) {
     this->dataDirectory = dataDirectory;
+
+    extractFile("exd/root.exl", "root.exl");
+
+    rootEXL = readEXL("root.exl");
+}
+
+std::vector<std::string> GameData::getAllSheetNames() {
+    std::vector<std::string> names;
+
+    for(auto row : rootEXL.rows) {
+        names.push_back(row.name);
+    }
+
+    return names;
 }
 
 uint64_t GameData::calculateHash(const std::string_view path) {
@@ -86,6 +101,8 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa
     // TODO: handle platforms other than win32
     auto indexFilename = calculateFilename(categoryToID[category], getExpansionID(repository), 0, "win32", "index");
 
+    fmt::print("calculated index filename: ");
+
     // TODO: handle hashes in index2 files (we can read them but it's not setup yet.)
     auto indexFile = readIndexFile(dataDirectory + "/" + repository + "/" + indexFilename);
 
@@ -93,6 +110,8 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa
         if(entry.hash == hash) {
             auto dataFilename = calculateFilename(categoryToID[category], getExpansionID(repository), entry.dataFileId, "win32", "dat0");
 
+            fmt::print("Opening data file {}...\n", dataFilename);
+
             FILE* file = fopen((dataDirectory + "/" + repository + "/" + dataFilename).c_str(), "rb");
             if(file == nullptr) {
                 throw std::runtime_error("Failed to open data file: " + dataFilename);
@@ -187,3 +206,28 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa
 
     fmt::print("Extracted {} to {}\n", dataFilePath, outPath);
 }
+
+std::optional<EXH> GameData::readExcelSheet(std::string_view name) {
+    fmt::print("Beginning to read excel sheet {}...\n", name);
+
+    for(auto row : rootEXL.rows) {
+        if(row.name == name) {
+            fmt::print("Found row {} at id {}!\n", name, row.id);
+
+            // we want it as lowercase (Item -> item)
+            std::string newFilename = name.data();
+            std::transform(newFilename.begin(), newFilename.end(), newFilename.begin(),
+                           [](unsigned char c){ return std::tolower(c); });
+
+            std::string exhFilename = "exd/" + newFilename + ".exh";
+
+            extractFile(exhFilename, newFilename + ".exh");
+
+            fmt::print("Done extracting files, now parsing...\n");
+
+            return readEXH(newFilename + ".exh");
+        }
+    }
+
+    return {};
+}