diff --git a/include/exdparser.h b/include/exdparser.h index c61b8c4..3648ed5 100644 --- a/include/exdparser.h +++ b/include/exdparser.h @@ -4,6 +4,8 @@ #include #include +#include "memorybuffer.h" + struct EXH; struct ExcelDataPagination; @@ -23,4 +25,4 @@ struct EXD { std::string getEXDFilename(EXH& exh, std::string_view name, std::string_view lang, ExcelDataPagination& page); -EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page); \ No newline at end of file +EXD readEXD(EXH& exh, MemorySpan data, ExcelDataPagination& page); \ No newline at end of file diff --git a/include/exhparser.h b/include/exhparser.h index 479f626..48bd578 100644 --- a/include/exhparser.h +++ b/include/exhparser.h @@ -4,6 +4,7 @@ #include #include "language.h" +#include "memorybuffer.h" // taken from https://xiv.dev/game-data/file-formats/excel struct ExhHeader { @@ -63,4 +64,4 @@ struct EXH { std::vector language; }; -EXH readEXH(std::string_view path); \ No newline at end of file +EXH readEXH(MemorySpan data); \ No newline at end of file diff --git a/include/exlparser.h b/include/exlparser.h index 3ad3c8e..43a38eb 100644 --- a/include/exlparser.h +++ b/include/exlparser.h @@ -3,6 +3,7 @@ #include #include #include +#include "memorybuffer.h" struct EXLRow { std::string name; @@ -13,4 +14,4 @@ struct EXL { std::vector rows; }; -EXL readEXL(std::string_view path); \ No newline at end of file +EXL readEXL(MemorySpan data); \ No newline at end of file diff --git a/include/gamedata.h b/include/gamedata.h index 0a48f16..88770fa 100644 --- a/include/gamedata.h +++ b/include/gamedata.h @@ -7,6 +7,7 @@ #include "exlparser.h" #include "indexparser.h" #include "sqpack.h" +#include "memorybuffer.h" /* * This handles reading/extracting the raw data from game data packs, such as dat0, index and index2 files. @@ -26,7 +27,8 @@ public: /* * This extracts the raw file from dataFilePath to outPath; */ - void extractFile(std::string_view dataFilePath, std::string_view outPath); + [[nodiscard]] + std::optional extractFile(std::string_view data_file_path); IndexFile getIndexListing(std::string_view folder); diff --git a/include/mdlparser.h b/include/mdlparser.h index 901bf7d..246a725 100644 --- a/include/mdlparser.h +++ b/include/mdlparser.h @@ -4,6 +4,8 @@ #include #include +#include "memorybuffer.h" + struct Vertex { std::array position; std::array normal; @@ -22,4 +24,4 @@ struct Model { std::vector lods; }; -Model parseMDL(const std::string_view path); \ No newline at end of file +Model parseMDL(MemorySpan data); \ No newline at end of file diff --git a/include/memorybuffer.h b/include/memorybuffer.h new file mode 100644 index 0000000..6d82fd8 --- /dev/null +++ b/include/memorybuffer.h @@ -0,0 +1,141 @@ +#pragma once + +#include +#include +#include + +enum class Seek { + Current, + End, + Set +}; + +struct MemoryBuffer { + MemoryBuffer() {} + MemoryBuffer(const std::vector& new_data) : data(new_data) {} + + void seek(const size_t pos, const Seek seek_type) { + switch(seek_type) { + case Seek::Current: + position += pos; + break; + case Seek::End: + position = size() - pos; + break; + case Seek::Set: + position = pos; + break; + } + } + + template + void write(const T& t) { + size_t end = position + sizeof(T); + if(end > data.size()) + data.resize(end); + + memcpy(data.data() + position, &t, sizeof(T)); + + position = end; + } + + template <> + void write>(const std::vector& t) { + size_t end = position + (sizeof(uint8_t) * t.size()); + if(end > data.size()) + data.resize(end); + + data.insert(data.begin() + position, t.begin(), t.end()); + position = end; + } + + size_t size() const { + return data.size(); + } + + size_t current_position() const { + return position; + } + + std::vector data; + +private: + size_t position = 0; +}; + +struct MemorySpan { + MemorySpan(const MemoryBuffer& new_buffer) : buffer(new_buffer) {} + + std::istream read_as_stream() { + auto char_data = cast_data(); + mem = std::make_unique(char_data, char_data + size()); + return std::istream(mem.get()); + } + + template + void read(T* t) { + *t = *reinterpret_cast(buffer.data.data() + position); + position += sizeof(T); + } + + template + void read(T* t, const size_t size) { + *t = *reinterpret_cast(buffer.data.data() + position); + position += size; + } + + template + void read_structures(std::vector* vector, const size_t count) { + vector->resize(count); + for(size_t i = 0; i < count; i++) + read(&vector->at(i)); + } + + template + void read_array(T* array, const size_t count) { + for(size_t i = 0; i < count; i++) + read((array + i)); + } + + void seek(const size_t pos, const Seek seek_type) { + switch(seek_type) { + case Seek::Current: + position += pos; + break; + case Seek::End: + position = buffer.size() - pos; + break; + case Seek::Set: + position = pos; + break; + } + } + + size_t size() const { + return buffer.data.size(); + } + + size_t current_position() const { + return position; + } + +private: + const MemoryBuffer& buffer; + + struct membuf : std::streambuf { + inline membuf(char* begin, char* end) { + this->setg(begin, begin, end); + } + }; + + template + T* cast_data() { + return (T*)(buffer.data.data()); + } + + std::unique_ptr mem; + + size_t position = 0; +}; + +void write_buffer_to_file(const MemoryBuffer& buffer, std::string_view path); \ No newline at end of file diff --git a/src/exdparser.cpp b/src/exdparser.cpp index db91565..bfce408 100644 --- a/src/exdparser.cpp +++ b/src/exdparser.cpp @@ -27,18 +27,19 @@ struct ExcelDataRowHeader { }; template -T readDataRaw(FILE* file, int offset) { - fseek(file, offset, SEEK_SET); +T readDataRaw(MemorySpan& span, int offset) { + span.seek(offset, Seek::Set); T value; - fread(&value, sizeof value, 1, file); + span.read(&value); endianSwap(&value); + return value; } template -std::string readData(FILE* file, int offset) { - return std::to_string(readDataRaw(file, offset)); +std::string readData(MemorySpan& span, int offset) { + return std::to_string(readDataRaw(span, offset)); } std::string getEXDFilename(EXH& exh, std::string_view name, std::string_view lang, ExcelDataPagination& page) { @@ -49,23 +50,16 @@ std::string getEXDFilename(EXH& exh, std::string_view name, std::string_view lan } } -EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page) { +EXD readEXD(EXH& exh, MemorySpan data, ExcelDataPagination& page) { EXD exd; - FILE* file = fopen(path.data(), "rb"); - if(file == nullptr) { - throw std::runtime_error("Failed to open exd file " + std::string(path)); - } - ExcelDataHeader header; - fread(&header, sizeof(ExcelDataHeader), 1, file); + data.read(&header); endianSwap(&header.indexSize); std::vector dataOffsets; - dataOffsets.resize(header.indexSize / sizeof(ExcelDataOffset)); - - fread(dataOffsets.data(), sizeof(ExcelDataOffset) * dataOffsets.size(), 1, file); + data.read_structures(&dataOffsets, header.indexSize / sizeof(ExcelDataOffset)); for(auto& offset : dataOffsets) { endianSwap(&offset.offset); @@ -75,17 +69,17 @@ EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page) { for(int i = 0; i < exh.header.rowCount; i++) { for(auto& offset : dataOffsets) { if (offset.rowId == i) { - fseek(file, offset.offset, SEEK_SET); + data.seek(offset.offset, Seek::Set); ExcelDataRowHeader rowHeader; - fread(&rowHeader, sizeof(ExcelDataRowHeader), 1, file); + data.read(&rowHeader); endianSwap(&rowHeader.dataSize); endianSwap(&rowHeader.rowCount); const int headerOffset = offset.offset + 6; - const auto readRow = [&exd, &exh, file, rowHeader](int offset) { + const auto readRow = [&exd, &exh, &data, rowHeader](int offset) { Row row; for (auto column: exh.columnDefinitions) { @@ -93,22 +87,22 @@ EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page) { switch (column.type) { case String: { - fseek(file, offset + column.offset, SEEK_SET); + data.seek(offset + column.offset, Seek::Set); uint32_t stringLength = 0; // this is actually offset? - fread(&stringLength, sizeof(uint32_t), 1, file); + data.read(&stringLength); endianSwap(&stringLength); - fseek(file, offset + exh.header.dataOffset + stringLength, SEEK_SET); + data.seek(offset + exh.header.dataOffset + stringLength, Seek::Set); std::string string; uint8_t byte; - fread(&byte, sizeof(uint8_t), 1, file); + data.read(&byte); while(byte != 0) { string.push_back(byte); - fread(&byte, sizeof(uint8_t), 1, file); + data.read(&byte); } c.data = string; @@ -116,46 +110,46 @@ EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page) { } break; case Int8: - c.data = readData(file, offset + column.offset); + c.data = readData(data, offset + column.offset); c.type = "Int"; - c.uint64Data = readDataRaw(file, offset + column.offset); + c.uint64Data = readDataRaw(data, offset + column.offset); break; case UInt8: - c.data = readData(file, offset + column.offset); + c.data = readData(data, offset + column.offset); c.type = "Unsigned Int"; - c.uint64Data = readDataRaw(file, offset + column.offset); + c.uint64Data = readDataRaw(data, offset + column.offset); break; case Int16: - c.data = readData(file, offset + column.offset); + c.data = readData(data, offset + column.offset); c.type = "Int"; - c.uint64Data = readDataRaw(file, offset + column.offset); + c.uint64Data = readDataRaw(data, offset + column.offset); break; case UInt16: - c.data = readData(file, offset + column.offset); + c.data = readData(data, offset + column.offset); c.type = "Unsigned Int"; break; case Int32: - c.data = readData(file, offset + column.offset); + c.data = readData(data, offset + column.offset); c.type = "Int"; - c.uint64Data = readDataRaw(file, offset + column.offset); + c.uint64Data = readDataRaw(data, offset + column.offset); break; case UInt32: - c.data = readData(file, offset + column.offset); + c.data = readData(data, offset + column.offset); c.type = "Unsigned Int"; break; case Float32: - c.data = readData(file, offset + column.offset); + c.data = readData(data, offset + column.offset); c.type = "Float"; break; case Int64: - c.data = readData(file, offset + column.offset); + c.data = readData(data, offset + column.offset); c.type = "Int"; - c.uint64Data = readDataRaw(file, offset + column.offset); + c.uint64Data = readDataRaw(data, offset + column.offset); break; case UInt64: - c.data = readData(file, offset + column.offset); + c.data = readData(data, offset + column.offset); c.type = "Unsigned Int"; - c.uint64Data = readDataRaw(file, offset + column.offset); + c.uint64Data = readDataRaw(data, offset + column.offset); break; case PackedBool0: case PackedBool1: @@ -167,8 +161,8 @@ EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page) { case PackedBool7: { int shift = (int) column.type - (int) PackedBool0; int bit = 1 << shift; - int32_t data = readDataRaw(file, offset + column.offset); - c.data = std::to_string((data & bit) == bit); + int32_t boolData = readDataRaw(data, offset + column.offset); + c.data = std::to_string((boolData & bit) == bit); c.type = "Boolean"; } break; diff --git a/src/exhparser.cpp b/src/exhparser.cpp index bb93826..55c0638 100644 --- a/src/exhparser.cpp +++ b/src/exhparser.cpp @@ -7,17 +7,12 @@ #include #include -EXH readEXH(const std::string_view path) { +EXH readEXH(MemorySpan data) { EXH exh; - FILE* file = fopen(path.data(), "rb"); - if(file == nullptr) { - throw std::runtime_error("Failed to open exh file " + std::string(path)); - } + data.read(&exh.header); - fread(&exh.header, sizeof(ExhHeader), 1, file); - - fseek(file, 0x20, SEEK_SET); + data.seek(0x20, Seek::Set); endianSwap(&exh.header.dataOffset); endianSwap(&exh.header.columnCount); @@ -25,16 +20,9 @@ EXH readEXH(const std::string_view path) { endianSwap(&exh.header.languageCount); endianSwap(&exh.header.rowCount); - exh.columnDefinitions.resize(exh.header.columnCount); - - fread(exh.columnDefinitions.data(), sizeof(ExcelColumnDefinition) * exh.header.columnCount, 1, file); - - exh.pages.resize(exh.header.pageCount); - - fread(exh.pages.data(), sizeof(ExcelDataPagination) * exh.header.pageCount, 1, file); - - exh.language.resize(exh.header.languageCount); - fread(exh.language.data(), sizeof(Language) * exh.header.languageCount, 1, file); + data.read_structures(&exh.columnDefinitions, exh.header.columnCount); + data.read_structures(&exh.pages, exh.header.pageCount); + data.read_structures(&exh.language, exh.header.languageCount); for(auto& columnDef : exh.columnDefinitions) { endianSwap(&columnDef.offset); diff --git a/src/exlparser.cpp b/src/exlparser.cpp index d00013e..7db6ab4 100644 --- a/src/exlparser.cpp +++ b/src/exlparser.cpp @@ -3,18 +3,13 @@ #include #include -EXL readEXL(std::string_view path) { - std::fstream file; - file.open(path.data(), std::iostream::in); - - if(!file.is_open()) { - throw std::runtime_error("Failed to read exl file from " + std::string(path.data())); - } +EXL readEXL(MemorySpan data) { + auto stream = data.read_as_stream(); EXL exl; std::string line; - while (std::getline(file, line)) { + while (std::getline(stream, line)) { const size_t comma = line.find_first_of(','); std::string name = line.substr(0, comma); diff --git a/src/gamedata.cpp b/src/gamedata.cpp index cd351a4..d4b47d1 100644 --- a/src/gamedata.cpp +++ b/src/gamedata.cpp @@ -49,9 +49,8 @@ GameData::GameData(const std::string_view dataDirectory) { repositories.push_back(repository); } - extractFile("exd/root.exl", "root.exl"); - - rootEXL = readEXL("root.exl"); + auto root_exl_data = extractFile("exd/root.exl"); + rootEXL = readEXL(*root_exl_data); } std::vector GameData::getAllSheetNames() { @@ -96,9 +95,9 @@ std::tuple GameData::calculateRepositoryCategory(std::s return {getBaseRepository(), tokens[0]}; } -void GameData::extractFile(std::string_view dataFilePath, std::string_view outPath) { - const uint64_t hash = calculateHash(dataFilePath); - auto [repository, category] = calculateRepositoryCategory(dataFilePath); +std::optional GameData::extractFile(const std::string_view data_file_path) { + const uint64_t hash = calculateHash(data_file_path); + auto [repository, category] = calculateRepositoryCategory(data_file_path); auto [index_filename, index2_filename] = repository.get_index_filenames(categoryToID[category]); auto index_path = fmt::format("{data_directory}/{repository}/{filename}", @@ -160,11 +159,9 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa fclose(file); - FILE* newFile = fopen(outPath.data(), "w"); - fwrite(data.data(), data.size(), 1, newFile); - fclose(newFile); + return {data}; } else if(info.fileType == FileType::Model) { - FILE* newFile = fopen(outPath.data(), "w"); + MemoryBuffer buffer; // reset fseek(file, offset, SEEK_SET); @@ -223,49 +220,54 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa std::vector compressedBlockSizes(totalBlocks); fread(compressedBlockSizes.data(), compressedBlockSizes.size() * sizeof(uint16_t), 1, file); + int currentBlock = 0; - int stackSize = 0; - int runtimeSize = 0; + uint32_t stackSize = 0; + uint32_t runtimeSize = 0; - std::array vertexDataOffsets; - std::array indexDataOffsets; + std::array vertexDataOffsets; + std::array indexDataOffsets; - std::array vertexDataSizes; - std::array indexDataSizes; + std::array vertexDataSizes; + std::array indexDataSizes; // data.append 0x44 - fseek(newFile, 0x44, SEEK_SET); + buffer.seek(0x44, Seek::Set); fseek(file, baseOffset + modelInfo.stackOffset, SEEK_SET); - size_t stackStart = ftell(newFile); + size_t stackStart = buffer.current_position(); for(int i = 0; i < modelInfo.stackBlockNum; i++) { size_t lastPos = ftell(file); + auto data = read_data_block(file, lastPos); - fwrite(data.data(), data.size(), 1, newFile); // i think we write this to file? + buffer.write(data); + fseek(file, lastPos + compressedBlockSizes[currentBlock], SEEK_SET); currentBlock++; } - size_t stackEnd = ftell(newFile); + size_t stackEnd = buffer.current_position(); stackSize = (int)(stackEnd - stackStart); fseek(file, baseOffset + modelInfo.runtimeOffset, SEEK_SET); - size_t runtimeStart = ftell(newFile); + size_t runtimeStart = buffer.current_position(); for(int i = 0; i < modelInfo.runtimeBlockNum; i++) { size_t lastPos = ftell(file); + auto data = read_data_block(file, lastPos); - fwrite(data.data(), data.size(), 1, newFile); + buffer.write(data); + fseek(file, lastPos + compressedBlockSizes[currentBlock], SEEK_SET); currentBlock++; } - size_t runtimeEnd = ftell(newFile); + size_t runtimeEnd = buffer.current_position(); runtimeSize = (int)(runtimeEnd - runtimeStart); // process all 3 lods for(int i = 0; i < 3; i++) { if(modelInfo.vertexBlockBufferBlockNum[i] != 0) { - int currentVertexOffset = ftell(newFile); + int currentVertexOffset = buffer.current_position(); if(i == 0 || currentVertexOffset != vertexDataOffsets[i - 1]) vertexDataOffsets[i] = currentVertexOffset; else @@ -275,8 +277,10 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa for(int j = 0; j < modelInfo.vertexBlockBufferBlockNum[i]; j++) { size_t lastPos = ftell(file); + auto data = read_data_block(file, lastPos); - fwrite(data.data(), data.size(), 1, newFile); // i think we write this to file? + buffer.write(data); + vertexDataSizes[i] += (int)data.size(); fseek(file, lastPos + compressedBlockSizes[currentBlock], SEEK_SET); currentBlock++; @@ -286,7 +290,7 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa // TODO: lol no edge geometry if(modelInfo.indexBufferBlockNum[i] != 0) { - int currentIndexOffset = ftell(newFile); + int currentIndexOffset = buffer.current_position(); if(i == 0 || currentIndexOffset != indexDataOffsets[i - 1]) indexDataOffsets[i] = currentIndexOffset; else @@ -294,8 +298,10 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa for(int j = 0; j < modelInfo.indexBufferBlockNum[i]; j++) { size_t lastPos = ftell(file); + auto data = read_data_block(file, lastPos); - fwrite(data.data(), data.size(), 1, newFile); // i think we write this to file? + buffer.write(data); + indexDataSizes[i] += (int)data.size(); fseek(file, lastPos + compressedBlockSizes[currentBlock], SEEK_SET); currentBlock++; @@ -304,45 +310,43 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa } // now write mdl header - fseek(newFile, 0, SEEK_SET); - fwrite(&modelInfo.version, sizeof(uint32_t), 1, newFile); - fwrite(&stackSize, sizeof(uint32_t), 1, newFile); - fwrite(&runtimeSize, sizeof(uint32_t), 1, newFile); - fwrite(&modelInfo.vertexDeclarationNum, sizeof(unsigned short), 1, newFile); - fwrite(&modelInfo.materialNum, sizeof(unsigned short), 1, newFile); + buffer.seek(0, Seek::Set); + + buffer.write(modelInfo.version); + buffer.write(stackSize); + buffer.write(runtimeSize); + buffer.write(modelInfo.vertexDeclarationNum); + buffer.write(modelInfo.materialNum); for(int i = 0; i < 3; i++) - fwrite(&vertexDataOffsets[i], sizeof(uint32_t), 1, newFile); + buffer.write(vertexDataOffsets[i]); for(int i = 0; i < 3; i++) - fwrite(&indexDataOffsets[i], sizeof(uint32_t), 1, newFile); + buffer.write(indexDataOffsets[i]); for(int i = 0; i < 3; i++) - fwrite(&vertexDataSizes[i], sizeof(uint32_t), 1, newFile); + buffer.write(vertexDataSizes[i]); for(int i = 0; i < 3; i++) - fwrite(&indexDataSizes[i], sizeof(uint32_t), 1, newFile); + buffer.write(indexDataSizes[i]); - fwrite(&modelInfo.numLods, sizeof(uint8_t), 1, file); - fwrite(&modelInfo.indexBufferStreamingEnabled, sizeof(bool), 1, file); - fwrite(&modelInfo.edgeGeometryEnabled, sizeof(bool), 1, file); + buffer.write(modelInfo.numLods); + buffer.write(modelInfo.indexBufferStreamingEnabled); + buffer.write(modelInfo.edgeGeometryEnabled); - uint8_t dummy[] = {0}; - fwrite(dummy, sizeof(uint8_t), 1, file); + uint8_t dummy = 0; + buffer.write(dummy); - fclose(newFile); fclose(file); + + return {buffer}; } else { - throw std::runtime_error("File type is not handled yet for " + std::string(dataFilePath)); + throw std::runtime_error("File type is not handled yet for " + std::string(data_file_path)); } - - fmt::print("Extracted {} to {}!\n", dataFilePath, outPath); - - return; } } - fmt::print("Failed to find file {}.\n", dataFilePath); + fmt::print("Failed to find file {}.\n", data_file_path); } std::optional GameData::readExcelSheet(std::string_view name) { @@ -358,9 +362,8 @@ std::optional GameData::readExcelSheet(std::string_view name) { std::string outPath = newFilename + ".exh"; std::replace(outPath.begin(), outPath.end(), '/', '_'); - extractFile(exhFilename, outPath); - - return readEXH(outPath); + auto exh_data = extractFile(exhFilename); + return readEXH(*exh_data); } } @@ -368,53 +371,45 @@ std::optional GameData::readExcelSheet(std::string_view name) { } void GameData::extractSkeleton() { - std::string path = fmt::format("chara/human/c0201/skeleton/base/b0001/skl_c0201b0001.sklb"); - - extractFile(path, "test.skel"); - - FILE* file = fopen("test.skel", "rb"); - - fseek(file, 0, SEEK_END); - size_t end = ftell(file); - fseek(file, 0, SEEK_SET); + const std::string path = fmt::format("chara/human/c0201/skeleton/base/b0001/skl_c0201b0001.sklb"); + auto skel_data = extractFile(path); + auto skel_span = MemorySpan(*skel_data); int32_t magic; - fread(&magic, sizeof(int32_t), 1, file); - - int32_t format; - fread(&format, sizeof(int32_t), 1, file); - - fseek(file, sizeof(uint16_t), SEEK_CUR); + skel_span.read(&magic); if(magic != 0x736B6C62) fmt::print("INVALID SKLB magic"); - size_t dataOffset = 0; + int32_t format; + skel_span.read(&format); + skel_span.seek(sizeof(uint16_t), Seek::Current); + + int16_t dataOffset = 0; switch(format) { case 0x31323030: - fread(&dataOffset, sizeof(int16_t), 1, file); + skel_span.read(&dataOffset); break; case 0x31333030: case 0x31333031: - fseek(file, sizeof(uint16_t), SEEK_CUR); - fread(&dataOffset, sizeof(int16_t), 1, file); + skel_span.seek(sizeof(uint16_t), Seek::Current); + skel_span.read(&dataOffset); break; default: fmt::print("INVALID SKLB format {}", format); break; } - fseek(file, dataOffset, SEEK_SET); + skel_span.seek(dataOffset, Seek::Set); - std::vector havokData(end - dataOffset); - fread(havokData.data(), havokData.size(), 1, file); + std::vector havokData(skel_span.size() - dataOffset); + skel_span.read_structures(&havokData, havokData.size()); FILE* newFile = fopen("test.sklb.havok", "wb"); fwrite(havokData.data(), havokData.size(), 1, newFile); fclose(newFile); - fclose(file); } IndexFile GameData::getIndexListing(std::string_view folder) { diff --git a/src/mdlparser.cpp b/src/mdlparser.cpp index b07efdf..2e7bafe 100644 --- a/src/mdlparser.cpp +++ b/src/mdlparser.cpp @@ -8,19 +8,7 @@ #include #include -Model parseMDL(const std::string_view path) { - FILE* file = fopen(path.data(), "rb"); - if(file == nullptr) { - throw std::runtime_error("Failed to open exh file " + std::string(path)); - } - - enum class FileType : int32_t { - Empty = 1, - Standard = 2, - Model = 3, - Texture = 4 - }; - +Model parseMDL(MemorySpan data) { struct ModelFileHeader { uint32_t version; uint32_t stackSize; @@ -37,9 +25,7 @@ Model parseMDL(const std::string_view path) { uint8_t padding; } modelFileHeader; - fread(&modelFileHeader, sizeof(ModelFileHeader), 1, file); - - fmt::print("stack size: {}\n", modelFileHeader.stackSize); + data.read(&modelFileHeader); struct VertexElement { uint8_t stream, offset, type, usage, usageIndex; @@ -53,29 +39,28 @@ Model parseMDL(const std::string_view path) { std::vector vertexDecls(modelFileHeader.vertexDeclarationCount); for(int i = 0; i < modelFileHeader.vertexDeclarationCount; i++) { VertexElement element {}; - fread(&element, sizeof(VertexElement), 1, file); + data.read(&element); + do { vertexDecls[i].elements.push_back(element); - fread(&element, sizeof(VertexElement), 1, file); + data.read(&element); } while (element.stream != 255); int toSeek = 17 * 8 - (vertexDecls[i].elements.size() + 1) * 8; - fseek(file, toSeek, SEEK_CUR); + data.seek(toSeek, Seek::Current); } uint16_t stringCount; - fread(&stringCount, sizeof(uint16_t), 1, file); - - fmt::print("string count: {}\n", stringCount); + data.read(&stringCount); // dummy - fseek(file, sizeof(uint16_t), SEEK_CUR); + data.seek(sizeof(uint16_t), Seek::Current); uint32_t stringSize; - fread(&stringSize, sizeof(uint32_t), 1, file); + data.read(&stringSize); - std::vector strings(stringSize); - fread(strings.data(), stringSize, 1, file); + std::vector strings; + data.read_structures(&strings, stringSize); enum ModelFlags1 : uint8_t { @@ -135,28 +120,23 @@ Model parseMDL(const std::string_view path) { uint8_t padding[6]; } modelHeader; - fread(&modelHeader, sizeof(modelHeader), 1, file); - - fmt::print("mesh count: {}\n", modelHeader.meshCount); - fmt::print("attribute count: {}\n", modelHeader.attributeCount); + data.read(&modelHeader); struct ElementId { - unsigned int elementId; - unsigned int parentBoneName; + uint32_t elementId; + uint32_t parentBoneName; std::vector translate; std::vector rotate; }; std::vector elementIds(modelHeader.elementIdCount); for(int i = 0; i < modelHeader.elementIdCount; i++) { - fread(&elementIds[i].elementId, sizeof(uint32_t), 1, file); - fread(&elementIds[i].parentBoneName, sizeof(uint32_t), 1, file); + data.read(&elementIds[i].elementId); + data.read(&elementIds[i].parentBoneName); - elementIds[i].translate.resize(3); // FIXME: these always seem to be 3, convert to static array? then we could probably fread this all in one go! - elementIds[i].rotate.resize(3); - - fread(elementIds[i].translate.data(), sizeof(float) * 3, 1, file); - fread(elementIds[i].rotate.data(), sizeof(float) * 3, 1, file); + // FIXME: these always seem to be 3, convert to static array? then we could probably read this all in one go! + data.read_structures(&elementIds[i].translate, 3); + data.read_structures(&elementIds[i].rotate, 3); } struct Lod { @@ -184,10 +164,10 @@ Model parseMDL(const std::string_view path) { unsigned int indexDataOffset; }; - std::array lods; - fread(lods.data(), sizeof(Lod) * 3, 1, file); + std::vector lods; // TODO: support models that support more than 3 lods + data.read_structures(&lods, 3); struct Mesh { unsigned short vertexCount; @@ -199,7 +179,7 @@ Model parseMDL(const std::string_view path) { unsigned short boneTableIndex; unsigned int startIndex; - std::vector vertexBufferOffset; + std::vector vertexBufferOffset; std::vector vertexBufferStride; uint8_t vertexStreamCount; @@ -207,26 +187,23 @@ Model parseMDL(const std::string_view path) { std::vector meshes(modelHeader.meshCount); for(int i = 0; i < modelHeader.meshCount; i++) { - fread(&meshes[i].vertexCount, sizeof(uint16_t), 1, file); - fread(&meshes[i].padding, sizeof(uint16_t), 1, file); - fread(&meshes[i].indexCount, sizeof(uint32_t), 1, file); - fread(&meshes[i].materialIndex, sizeof(uint16_t), 1, file); - fread(&meshes[i].subMeshIndex, sizeof(uint16_t), 1, file); - fread(&meshes[i].subMeshCount, sizeof(uint16_t), 1, file); - fread(&meshes[i].boneTableIndex, sizeof(uint16_t), 1, file); - fread(&meshes[i].startIndex, sizeof(uint32_t), 1, file); + data.read(&meshes[i].vertexCount); + data.read(&meshes[i].padding); + data.read(&meshes[i].indexCount); + data.read(&meshes[i].materialIndex); + data.read(&meshes[i].subMeshIndex); + data.read(&meshes[i].subMeshCount); + data.read(&meshes[i].boneTableIndex); + data.read(&meshes[i].startIndex); - meshes[i].vertexBufferOffset.resize(3); - fread(meshes[i].vertexBufferOffset.data(), sizeof(uint32_t) * 3, 1, file); + data.read_structures(&meshes[i].vertexBufferOffset, 3); + data.read_structures(&meshes[i].vertexBufferStride, 3); - meshes[i].vertexBufferStride.resize(3); - fread(meshes[i].vertexBufferStride.data(), sizeof(uint8_t) * 3, 1, file); - - fread(&meshes[i].vertexStreamCount, sizeof(uint8_t), 1, file); + data.read(&meshes[i].vertexStreamCount); } - std::vector attributeNameOffsets(modelHeader.attributeCount); - fread(attributeNameOffsets.data(), sizeof(uint32_t) * modelHeader.attributeCount, 1, file); + std::vector attributeNameOffsets; + data.read_structures(&attributeNameOffsets, modelHeader.attributeCount); // TODO: implement terrain shadow meshes @@ -238,69 +215,60 @@ Model parseMDL(const std::string_view path) { unsigned short boneCount; }; - std::vector submeshes(modelHeader.submeshCount); - for(int i = 0; i < modelHeader.submeshCount; i++) { - fread(&submeshes[i], sizeof(Submesh), 1, file); - } + std::vector submeshes; + data.read_structures(&submeshes, modelHeader.submeshCount); // TODO: implement terrain shadow submeshes - std::vector materialNameOffsets(modelHeader.materialCount); - fread(materialNameOffsets.data(), sizeof(uint32_t) * modelHeader.materialCount, 1, file); + std::vector materialNameOffsets; + data.read_structures(&materialNameOffsets, modelHeader.materialCount); - std::vector boneNameOffsets(modelHeader.boneCount); - fread(boneNameOffsets.data(), sizeof(uint32_t) * modelHeader.boneCount, 1, file); + std::vector boneNameOffsets; + data.read_structures(&boneNameOffsets, modelHeader.boneCount); struct BoneTable { - std::vector boneIndex; + std::vector boneIndex; uint8_t boneCount; std::vector padding; }; std::vector boneTables(modelHeader.boneTableCount); for(int i = 0; i < modelHeader.boneTableCount; i++) { - boneTables[i].boneIndex.resize(64); - fread(boneTables[i].boneIndex.data(), 64 * sizeof(uint16_t), 1, file); - fread(&boneTables[i].boneCount, sizeof(uint8_t), 1, file); - boneTables[i].padding.resize(3); - fread(boneTables[i].padding.data(), sizeof(uint8_t) * 3, 1, file); + data.read_structures(&boneTables[i].boneIndex, 64); - fmt::print("bone count: {}\n", boneTables[i].boneCount); + data.read(&boneTables[i].boneCount); + + data.read_structures(&boneTables[i].padding, 3); } // TODO: implement shapes - unsigned int submeshBoneMapSize; - fread(&submeshBoneMapSize, sizeof(uint32_t), 1, file); + uint32_t submeshBoneMapSize; + data.read(&submeshBoneMapSize); - std::vector submeshBoneMap((int)submeshBoneMapSize / 2); - fread(submeshBoneMap.data(), submeshBoneMap.size() * sizeof(uint16_t), 1, file); + std::vector submeshBoneMap; + data.read_structures(&submeshBoneMap, (int)submeshBoneMapSize / 2); uint8_t paddingAmount; - fread(&paddingAmount, sizeof(uint8_t), 1, file); + data.read(&paddingAmount); - fseek(file, paddingAmount, SEEK_CUR); + data.seek(paddingAmount, Seek::Current); struct BoundingBox { std::array min, max; }; BoundingBox boundingBoxes, modelBoundingBoxes, waterBoundingBoxes, verticalFogBoundingBoxes; - fread(&boundingBoxes, sizeof(BoundingBox), 1, file); - fread(&modelBoundingBoxes, sizeof(BoundingBox), 1, file); - fread(&waterBoundingBoxes, sizeof(BoundingBox), 1, file); - fread(&verticalFogBoundingBoxes, sizeof(BoundingBox), 1, file); + data.read(&boundingBoxes); + data.read(&modelBoundingBoxes); + data.read(&waterBoundingBoxes); + data.read(&verticalFogBoundingBoxes); - std::vector boneBoundingBoxes(modelHeader.boneCount); - fread(boneBoundingBoxes.data(), modelHeader.boneCount * sizeof(BoundingBox), 1, file); - - fmt::print("Successfully read mdl file!\n"); - - fmt::print("Now exporting as test.obj...\n"); + std::vector boneBoundingBoxes; + data.read_structures(&boneBoundingBoxes, modelHeader.boneCount); Model model; - // TODO: doesn't work for lod above 0 for(int i = 0; i < modelHeader.lodCount; i++) { ::Lod lod; @@ -333,44 +301,48 @@ Model parseMDL(const std::string_view path) { std::vector vertices(vertexCount); for(int k = 0; k < vertexCount; k++) { - for(auto & orderedElement : decl.elements) { - VertexType type = (VertexType)orderedElement.type; - VertexUsage usage = (VertexUsage)orderedElement.usage; + for(auto& orderedElement : decl.elements) { + auto type = static_cast(orderedElement.type); + auto usage = static_cast(orderedElement.usage); const int stream = orderedElement.stream; - fseek(file, lods[i].vertexDataOffset + meshes[j].vertexBufferOffset[stream] + orderedElement.offset + meshes[i].vertexBufferStride[stream] * k, SEEK_SET); + data.seek(lods[i].vertexDataOffset + meshes[j].vertexBufferOffset[stream] + orderedElement.offset + meshes[i].vertexBufferStride[stream] * k, Seek::Set); std::array floatData = {}; switch(type) { case VertexType::Single3: - fread(floatData.data(), sizeof(float) * 3, 1, file); + data.read_array(floatData.data(), 3); break; case VertexType::Single4: - fread(floatData.data(), sizeof(float) * 4, 1, file); + data.read_array(floatData.data(), 4); break; case VertexType::UInt: - fseek(file, sizeof(uint8_t) * 4, SEEK_CUR); + data.seek(sizeof(uint8_t) * 4, Seek::Current); break; - case VertexType::ByteFloat4: + case VertexType::ByteFloat4: { uint8_t values[4]; - fread(values, sizeof(uint8_t) * 4, 1, file); + data.read_array(values, 4); + floatData[0] = byte_to_float(values[0]); floatData[1] = byte_to_float(values[1]); floatData[2] = byte_to_float(values[2]); floatData[3] = byte_to_float(values[3]); + } break; case VertexType::Half2: { uint16_t values[2]; - fread(values, sizeof(uint16_t) * 2, 1, file); + data.read_array(values, 2); + floatData[0] = half_to_float(values[0]); floatData[1] = half_to_float(values[1]); } break; case VertexType::Half4: { uint16_t values[4]; - fread(values, sizeof(uint16_t) * 4, 1, file); + data.read_array(values, 4); + floatData[0] = half_to_float(values[0]); floatData[1] = half_to_float(values[1]); floatData[2] = half_to_float(values[2]); @@ -390,9 +362,10 @@ Model parseMDL(const std::string_view path) { } } - fseek(file, modelFileHeader.indexOffsets[i] + (meshes[j].startIndex * 2), SEEK_SET); - std::vector indices(meshes[j].indexCount); - fread(indices.data(), meshes[j].indexCount * sizeof(uint16_t), 1, file); + data.seek(modelFileHeader.indexOffsets[i] + (meshes[j].startIndex * 2), Seek::Set); + + std::vector indices; + data.read_structures(&indices, meshes[j].indexCount); part.indices = indices; part.vertices = vertices;