Add EXD/EXH file parsing

CMakeLists.txt

@@ -5,5 +5,7 @@ add_library(libxiv STATIC
         src/indexparser.cpp
         src/crc32.cpp
         src/gamedata.cpp
-        src/compression.cpp)
+        src/compression.cpp
+        src/exhparser.cpp
+        src/exdparser.cpp)
 target_include_directories(libxiv PUBLIC include PRIVATE src)

include/exdparser.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include <string_view>
+
+struct EXH;
+struct ExcelDataPagination;
+
+void readEXD(EXH& exh, ExcelDataPagination& page);

include/exhparser.h

@@ -0,0 +1,82 @@
+#pragma once
+
+#include <string_view>
+#include <vector>
+
+// taken from https://xiv.dev/game-data/file-formats/excel
+struct ExhHeader {
+    char magic[0x4];
+    uint16_t unknown;
+    uint16_t dataOffset;
+    uint16_t columnCount;
+    uint16_t pageCount;
+    uint16_t languageCount;
+    uint16_t unknown1;
+    uint8_t u2;
+    uint8_t variant;
+    uint16_t u3;
+    uint32_t rowCount;
+    uint32_t u4[2];
+};
+
+enum ExcelColumnDataType : uint16_t {
+    String = 0x0,
+    Bool = 0x1,
+    Int8 = 0x2,
+    UInt8 = 0x3,
+    Int16 = 0x4,
+    UInt16 = 0x5,
+    Int32 = 0x6,
+    UInt32 = 0x7,
+    Float32 = 0x9,
+    Int64 = 0xA,
+    UInt64 = 0xB,
+
+    // 0 is read like data & 1, 1 is like data & 2, 2 = data & 4, etc...
+    PackedBool0 = 0x19,
+    PackedBool1 = 0x1A,
+    PackedBool2 = 0x1B,
+    PackedBool3 = 0x1C,
+    PackedBool4 = 0x1D,
+    PackedBool5 = 0x1E,
+    PackedBool6 = 0x1F,
+    PackedBool7 = 0x20,
+};
+
+struct ExcelColumnDefinition {
+    ExcelColumnDataType type;
+    uint16_t offset;
+};
+
+struct ExcelDataPagination {
+    uint32_t startId;
+    uint32_t rowCount;
+};
+
+enum Language : uint16_t {
+    None,
+    // ja
+    Japanese,
+    // en
+    English,
+    // de
+    German,
+    // fr
+    French,
+    // chs
+    ChineseSimplified,
+    // cht
+    ChineseTraditional,
+    // ko
+    Korean
+};
+
+struct EXH {
+    ExhHeader header;
+
+    std::vector<ExcelColumnDefinition> columnDefinitions;
+    std::vector<ExcelDataPagination> pages;
+    std::vector<Language> language;
+};
+
+EXH readEXH(std::string_view path);

include/utility.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <algorithm>
+
+template <class T>
+void endianSwap(T *objp) {
+    auto memp = reinterpret_cast<unsigned char*>(objp);
+    std::reverse(memp, memp + sizeof(T));
+}

src/exdparser.cpp

@@ -0,0 +1,89 @@
+#include "exdparser.h"
+
+#include <cstdio>
+#include <stdexcept>
+#include <algorithm>
+#include <fmt/format.h>
+
+#include "exhparser.h"
+#include "utility.h"
+
+struct ExcelDataHeader {
+    char magic[0x4];
+    uint16_t version;
+    uint16_t unknown1;
+    uint32_t indexSize;
+    uint32_t unknown2[5];
+};
+
+struct ExcelDataOffset {
+    uint32_t rowId;
+    uint32_t offset;
+};
+
+struct ExcelDataRowHeader {
+    uint32_t dataSize;
+    uint16_t rowCount;
+};
+
+void readEXD(EXH& exh, ExcelDataPagination& page) {
+    auto path = fmt::format("{}_{}.exd", "map", page.startId);
+
+    FILE* file = fopen(path.data(), "rb");
+    if(file == nullptr) {
+        throw std::runtime_error("Failed to open exd file " + std::string(path));
+    }
+
+    ExcelDataHeader header;
+    fread(&header, sizeof(ExcelDataHeader), 1, file);
+
+    endianSwap(&header.indexSize);
+
+    std::vector<ExcelDataOffset> dataOffsets;
+    dataOffsets.resize(header.indexSize / sizeof( ExcelDataOffset ));
+
+    fread(dataOffsets.data(), sizeof(ExcelDataOffset) * dataOffsets.size(), 1, file);
+
+    for(auto& offset : dataOffsets) {
+        endianSwap(&offset.offset);
+        endianSwap(&offset.rowId);
+    }
+
+    for(auto& offset : dataOffsets) {
+        fseek(file, exh.header.dataOffset + offset.offset, SEEK_SET);
+
+        ExcelDataRowHeader rowHeader;
+        fread(&rowHeader, sizeof(ExcelDataRowHeader), 1, file);
+
+        endianSwap(&rowHeader.dataSize);
+        endianSwap(&rowHeader.rowCount);
+
+        const int rowOffset = offset.offset + 6;
+
+        for(auto column : exh.columnDefinitions) {
+            switch(column.type) {
+                case String:
+                {
+                    fseek(file, rowOffset + column.offset, SEEK_SET);
+
+                    uint32_t stringLength;
+                    fread(&stringLength, sizeof(uint32_t), 1, file);
+
+                    fseek(file, rowOffset + exh.header.dataOffset + stringLength, SEEK_SET);
+
+                    std::string string;
+
+                    int ch;
+                    while ((ch = fgetc(file)) != '\0') {
+                        string.push_back((char)ch);
+                    }
+
+                    fmt::print("{}\n", string.data());
+                }
+                break;
+                default:
+                    break;
+            }
+        }
+    }
+}

src/exhparser.cpp

@@ -0,0 +1,47 @@
+#include "exhparser.h"
+#include "utility.h"
+
+#include <cstdio>
+#include <stdexcept>
+#include <vector>
+#include <algorithm>
+
+EXH readEXH(const std::string_view path) {
+    EXH exh;
+
+    FILE* file = fopen(path.data(), "rb");
+    if(file == nullptr) {
+        throw std::runtime_error("Failed to open exh file " + std::string(path));
+    }
+
+    fread(&exh.header, sizeof(ExhHeader), 1, file);
+
+    endianSwap(&exh.header.dataOffset);
+    endianSwap(&exh.header.columnCount);
+    endianSwap(&exh.header.pageCount);
+    endianSwap(&exh.header.languageCount);
+    endianSwap(&exh.header.rowCount);
+
+    exh.columnDefinitions.resize(exh.header.columnCount);
+
+    fread(exh.columnDefinitions.data(), sizeof(ExcelColumnDefinition) * exh.header.columnCount, 1, file);
+
+    exh.pages.resize(exh.header.pageCount);
+
+    fread(exh.pages.data(), sizeof(ExcelDataPagination) * exh.header.pageCount, 1, file);
+
+    exh.language.resize(exh.header.languageCount);
+    fread(exh.language.data(), sizeof(Language) * exh.header.languageCount, 1, file);
+
+    for(auto& columnDef : exh.columnDefinitions) {
+        endianSwap(&columnDef.offset);
+        endianSwap(&columnDef.type);
+    }
+
+    for(auto& page : exh.pages) {
+        endianSwap(&page.rowCount);
+        endianSwap(&page.startId);
+    }
+
+    return exh;
+}

src/gamedata.cpp

@@ -53,7 +53,7 @@ std::tuple<std::string, std::string> GameData::calculateRepositoryCategory(std::
     std::string repository, category;
 
     auto tokens = tokenize(path, "/");
-    if(stringContains(tokens[1], "ex") && !stringContains(tokens[0], "exd")) {
+    if(stringContains(tokens[1], "ex") && !stringContains(tokens[0], "exd") && !stringContains(tokens[0], "exh")) {
         repository = tokens[1];
     } else {
         repository = "ffxiv";
@@ -79,6 +79,9 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa
     const uint64_t hash = calculateHash(dataFilePath);
     auto [repository, category] = calculateRepositoryCategory(dataFilePath);
 
+    fmt::print("repository = {}\n", repository);
+    fmt::print("category = {}\n", category);
+
     // TODO: handle platforms other than win32
     auto indexFilename = calculateFilename(categoryToID[category], getExpansionID(repository), 0, "win32", "index");
 
@@ -114,6 +117,8 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa
 
             fread(&info, sizeof(FileInfo), 1, file);
 
+            fmt::print("file size = {}\n", info.fileSize);
+
             if(info.fileType != FileType::Standard) {
                 throw std::runtime_error("File type is not handled yet for " + std::string(dataFilePath));
             }
@@ -179,5 +184,5 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa
         }
     }
 
-    fmt::print("Extracted {} to {}", dataFilePath, outPath);
+    fmt::print("Extracted {} to {}\n", dataFilePath, outPath);
 }