redstrate/libxiv
Archived
1
Fork 0
Code Activity

Big refactoring commit pt. 2

Browse source 
Now instead of hammering your working directory, all operations happen
in-memory using two new classes, MemoryBuffer and MemorySpan. This has
already fixed numerous bugs especially around handling model files.

All operations on files inside dat's now happen on memory buffers, with
an option to write them to a file.

A lot of now useless debug messages are removed too, as that made
some operations needlessly bound by the speed of your console output.
This commit is contained in:
Joshua Goins 2022-04-17 19:08:34 -04:00
parent b11767dc02
commit c78a1ab245
11 changed files with 344 additions and 250 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
include/exdparser.h
View file

@ -4,6 +4,8 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "memorybuffer.h"
struct EXH; struct EXH;
struct ExcelDataPagination; struct ExcelDataPagination;
@ -23,4 +25,4 @@ struct EXD {
std::string getEXDFilename(EXH& exh, std::string_view name, std::string_view lang, ExcelDataPagination& page); std::string getEXDFilename(EXH& exh, std::string_view name, std::string_view lang, ExcelDataPagination& page);
EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page); EXD readEXD(EXH& exh, MemorySpan data, ExcelDataPagination& page);

3
include/exhparser.h
View file

@ -4,6 +4,7 @@
#include <vector> #include <vector>
#include "language.h" #include "language.h"
#include "memorybuffer.h"
// taken from https://xiv.dev/game-data/file-formats/excel // taken from https://xiv.dev/game-data/file-formats/excel
struct ExhHeader { struct ExhHeader {
@ -63,4 +64,4 @@ struct EXH {
std::vector<Language> language; std::vector<Language> language;
}; };
EXH readEXH(std::string_view path); EXH readEXH(MemorySpan data);

3
include/exlparser.h
View file

@ -3,6 +3,7 @@
#include <string_view> #include <string_view>
#include <vector> #include <vector>
#include <string> #include <string>
#include "memorybuffer.h"
struct EXLRow { struct EXLRow {
std::string name; std::string name;
@ -13,4 +14,4 @@ struct EXL {
std::vector<EXLRow> rows; std::vector<EXLRow> rows;
}; };
EXL readEXL(std::string_view path); EXL readEXL(MemorySpan data);

4
include/gamedata.h
View file

@ -7,6 +7,7 @@
#include "exlparser.h" #include "exlparser.h"
#include "indexparser.h" #include "indexparser.h"
#include "sqpack.h" #include "sqpack.h"
#include "memorybuffer.h"
/* /*
* This handles reading/extracting the raw data from game data packs, such as dat0, index and index2 files. * This handles reading/extracting the raw data from game data packs, such as dat0, index and index2 files.
@ -26,7 +27,8 @@ public:
/* /*
* This extracts the raw file from dataFilePath to outPath; * This extracts the raw file from dataFilePath to outPath;
*/ */
void extractFile(std::string_view dataFilePath, std::string_view outPath); [[nodiscard]]
std::optional<MemoryBuffer> extractFile(std::string_view data_file_path);
IndexFile<IndexHashTableEntry> getIndexListing(std::string_view folder); IndexFile<IndexHashTableEntry> getIndexListing(std::string_view folder);

4
include/mdlparser.h
View file

@ -4,6 +4,8 @@
#include <vector> #include <vector>
#include <array> #include <array>
#include "memorybuffer.h"
struct Vertex { struct Vertex {
std::array<float, 3> position; std::array<float, 3> position;
std::array<float, 3> normal; std::array<float, 3> normal;
@ -22,4 +24,4 @@ struct Model {
std::vector<Lod> lods; std::vector<Lod> lods;
}; };
Model parseMDL(const std::string_view path); Model parseMDL(MemorySpan data);

141
include/memorybuffer.h Normal file
View file

@ -0,0 +1,141 @@
#pragma once
#include <string_view>
#include <streambuf>
#include <istream>
enum class Seek {
Current,
End,
Set
};
struct MemoryBuffer {
MemoryBuffer() {}
MemoryBuffer(const std::vector<uint8_t>& new_data) : data(new_data) {}
void seek(const size_t pos, const Seek seek_type) {
switch(seek_type) {
case Seek::Current:
position += pos;
break;
case Seek::End:
position = size() - pos;
break;
case Seek::Set:
position = pos;
break;
}
}
template<typename T>
void write(const T& t) {
size_t end = position + sizeof(T);
if(end > data.size())
data.resize(end);
memcpy(data.data() + position, &t, sizeof(T));
position = end;
}
template <>
void write<std::vector<uint8_t>>(const std::vector<uint8_t>& t) {
size_t end = position + (sizeof(uint8_t) * t.size());
if(end > data.size())
data.resize(end);
data.insert(data.begin() + position, t.begin(), t.end());
position = end;
}
size_t size() const {
return data.size();
}
size_t current_position() const {
return position;
}
std::vector<uint8_t> data;
private:
size_t position = 0;
};
struct MemorySpan {
MemorySpan(const MemoryBuffer& new_buffer) : buffer(new_buffer) {}
std::istream read_as_stream() {
auto char_data = cast_data<char>();
mem = std::make_unique<membuf>(char_data, char_data + size());
return std::istream(mem.get());
}
template<typename T>
void read(T* t) {
*t = *reinterpret_cast<const T*>(buffer.data.data() + position);
position += sizeof(T);
}
template<typename T>
void read(T* t, const size_t size) {
*t = *reinterpret_cast<const T*>(buffer.data.data() + position);
position += size;
}
template<typename T>
void read_structures(std::vector<T>* vector, const size_t count) {
vector->resize(count);
for(size_t i = 0; i < count; i++)
read(&vector->at(i));
}
template<typename T>
void read_array(T* array, const size_t count) {
for(size_t i = 0; i < count; i++)
read((array + i));
}
void seek(const size_t pos, const Seek seek_type) {
switch(seek_type) {
case Seek::Current:
position += pos;
break;
case Seek::End:
position = buffer.size() - pos;
break;
case Seek::Set:
position = pos;
break;
}
}
size_t size() const {
return buffer.data.size();
}
size_t current_position() const {
return position;
}
private:
const MemoryBuffer& buffer;
struct membuf : std::streambuf {
inline membuf(char* begin, char* end) {
this->setg(begin, begin, end);
}
};
template<typename T>
T* cast_data() {
return (T*)(buffer.data.data());
}
std::unique_ptr<membuf> mem;
size_t position = 0;
};
void write_buffer_to_file(const MemoryBuffer& buffer, std::string_view path);

74
src/exdparser.cpp
View file

@ -27,18 +27,19 @@ struct ExcelDataRowHeader {
}; };
template<typename T> template<typename T>
T readDataRaw(FILE* file, int offset) { T readDataRaw(MemorySpan& span, int offset) {
fseek(file, offset, SEEK_SET); span.seek(offset, Seek::Set);
T value; T value;
fread(&value, sizeof value, 1, file); span.read(&value);
endianSwap(&value); endianSwap(&value);
return value; return value;
} }
template<typename T> template<typename T>
std::string readData(FILE* file, int offset) { std::string readData(MemorySpan& span, int offset) {
return std::to_string(readDataRaw<T>(file, offset)); return std::to_string(readDataRaw<T>(span, offset));
} }
std::string getEXDFilename(EXH& exh, std::string_view name, std::string_view lang, ExcelDataPagination& page) { std::string getEXDFilename(EXH& exh, std::string_view name, std::string_view lang, ExcelDataPagination& page) {
@ -49,23 +50,16 @@ std::string getEXDFilename(EXH& exh, std::string_view name, std::string_view lan
} }
} }
EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page) { EXD readEXD(EXH& exh, MemorySpan data, ExcelDataPagination& page) {
EXD exd; EXD exd;
FILE* file = fopen(path.data(), "rb");
if(file == nullptr) {
throw std::runtime_error("Failed to open exd file " + std::string(path));
}
ExcelDataHeader header; ExcelDataHeader header;
fread(&header, sizeof(ExcelDataHeader), 1, file); data.read(&header);
endianSwap(&header.indexSize); endianSwap(&header.indexSize);
std::vector<ExcelDataOffset> dataOffsets; std::vector<ExcelDataOffset> dataOffsets;
dataOffsets.resize(header.indexSize / sizeof(ExcelDataOffset)); data.read_structures(&dataOffsets, header.indexSize / sizeof(ExcelDataOffset));
fread(dataOffsets.data(), sizeof(ExcelDataOffset) * dataOffsets.size(), 1, file);
for(auto& offset : dataOffsets) { for(auto& offset : dataOffsets) {
endianSwap(&offset.offset); endianSwap(&offset.offset);
@ -75,17 +69,17 @@ EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page) {
for(int i = 0; i < exh.header.rowCount; i++) { for(int i = 0; i < exh.header.rowCount; i++) {
for(auto& offset : dataOffsets) { for(auto& offset : dataOffsets) {
if (offset.rowId == i) { if (offset.rowId == i) {
fseek(file, offset.offset, SEEK_SET); data.seek(offset.offset, Seek::Set);
ExcelDataRowHeader rowHeader; ExcelDataRowHeader rowHeader;
fread(&rowHeader, sizeof(ExcelDataRowHeader), 1, file); data.read(&rowHeader);
endianSwap(&rowHeader.dataSize); endianSwap(&rowHeader.dataSize);
endianSwap(&rowHeader.rowCount); endianSwap(&rowHeader.rowCount);
const int headerOffset = offset.offset + 6; const int headerOffset = offset.offset + 6;
const auto readRow = [&exd, &exh, file, rowHeader](int offset) { const auto readRow = [&exd, &exh, &data, rowHeader](int offset) {
Row row; Row row;
for (auto column: exh.columnDefinitions) { for (auto column: exh.columnDefinitions) {
@ -93,22 +87,22 @@ EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page) {
switch (column.type) { switch (column.type) {
case String: { case String: {
fseek(file, offset + column.offset, SEEK_SET); data.seek(offset + column.offset, Seek::Set);
uint32_t stringLength = 0; // this is actually offset? uint32_t stringLength = 0; // this is actually offset?
fread(&stringLength, sizeof(uint32_t), 1, file); data.read(&stringLength);
endianSwap(&stringLength); endianSwap(&stringLength);
fseek(file, offset + exh.header.dataOffset + stringLength, SEEK_SET); data.seek(offset + exh.header.dataOffset + stringLength, Seek::Set);
std::string string; std::string string;
uint8_t byte; uint8_t byte;
fread(&byte, sizeof(uint8_t), 1, file); data.read(&byte);
while(byte != 0) { while(byte != 0) {
string.push_back(byte); string.push_back(byte);
fread(&byte, sizeof(uint8_t), 1, file); data.read(&byte);
} }
c.data = string; c.data = string;
@ -116,46 +110,46 @@ EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page) {
} }
break; break;
case Int8: case Int8:
c.data = readData<int8_t>(file, offset + column.offset); c.data = readData<int8_t>(data, offset + column.offset);
c.type = "Int"; c.type = "Int";
c.uint64Data = readDataRaw<int8_t>(file, offset + column.offset); c.uint64Data = readDataRaw<int8_t>(data, offset + column.offset);
break; break;
case UInt8: case UInt8:
c.data = readData<uint8_t>(file, offset + column.offset); c.data = readData<uint8_t>(data, offset + column.offset);
c.type = "Unsigned Int"; c.type = "Unsigned Int";
c.uint64Data = readDataRaw<uint8_t>(file, offset + column.offset); c.uint64Data = readDataRaw<uint8_t>(data, offset + column.offset);
break; break;
case Int16: case Int16:
c.data = readData<int16_t>(file, offset + column.offset); c.data = readData<int16_t>(data, offset + column.offset);
c.type = "Int"; c.type = "Int";
c.uint64Data = readDataRaw<int16_t>(file, offset + column.offset); c.uint64Data = readDataRaw<int16_t>(data, offset + column.offset);
break; break;
case UInt16: case UInt16:
c.data = readData<uint16_t>(file, offset + column.offset); c.data = readData<uint16_t>(data, offset + column.offset);
c.type = "Unsigned Int"; c.type = "Unsigned Int";
break; break;
case Int32: case Int32:
c.data = readData<int32_t>(file, offset + column.offset); c.data = readData<int32_t>(data, offset + column.offset);
c.type = "Int"; c.type = "Int";
c.uint64Data = readDataRaw<int32_t>(file, offset + column.offset); c.uint64Data = readDataRaw<int32_t>(data, offset + column.offset);
break; break;
case UInt32: case UInt32:
c.data = readData<uint32_t>(file, offset + column.offset); c.data = readData<uint32_t>(data, offset + column.offset);
c.type = "Unsigned Int"; c.type = "Unsigned Int";
break; break;
case Float32: case Float32:
c.data = readData<float>(file, offset + column.offset); c.data = readData<float>(data, offset + column.offset);
c.type = "Float"; c.type = "Float";
break; break;
case Int64: case Int64:
c.data = readData<int64_t>(file, offset + column.offset); c.data = readData<int64_t>(data, offset + column.offset);
c.type = "Int"; c.type = "Int";
c.uint64Data = readDataRaw<int64_t>(file, offset + column.offset); c.uint64Data = readDataRaw<int64_t>(data, offset + column.offset);
break; break;
case UInt64: case UInt64:
c.data = readData<uint64_t>(file, offset + column.offset); c.data = readData<uint64_t>(data, offset + column.offset);
c.type = "Unsigned Int"; c.type = "Unsigned Int";
c.uint64Data = readDataRaw<uint64_t>(file, offset + column.offset); c.uint64Data = readDataRaw<uint64_t>(data, offset + column.offset);
break; break;
case PackedBool0: case PackedBool0:
case PackedBool1: case PackedBool1:
@ -167,8 +161,8 @@ EXD readEXD(EXH& exh, std::string_view path, ExcelDataPagination& page) {
case PackedBool7: { case PackedBool7: {
int shift = (int) column.type - (int) PackedBool0; int shift = (int) column.type - (int) PackedBool0;
int bit = 1 << shift; int bit = 1 << shift;
int32_t data = readDataRaw<int32_t>(file, offset + column.offset); int32_t boolData = readDataRaw<int32_t>(data, offset + column.offset);
c.data = std::to_string((data & bit) == bit); c.data = std::to_string((boolData & bit) == bit);
c.type = "Boolean"; c.type = "Boolean";
} }
break; break;

24
src/exhparser.cpp
View file

@ -7,17 +7,12 @@
#include <algorithm> #include <algorithm>
#include <string> #include <string>
EXH readEXH(const std::string_view path) { EXH readEXH(MemorySpan data) {
EXH exh; EXH exh;
FILE* file = fopen(path.data(), "rb"); data.read(&exh.header);
if(file == nullptr) {
throw std::runtime_error("Failed to open exh file " + std::string(path));
}
fread(&exh.header, sizeof(ExhHeader), 1, file); data.seek(0x20, Seek::Set);
fseek(file, 0x20, SEEK_SET);
endianSwap(&exh.header.dataOffset); endianSwap(&exh.header.dataOffset);
endianSwap(&exh.header.columnCount); endianSwap(&exh.header.columnCount);
@ -25,16 +20,9 @@ EXH readEXH(const std::string_view path) {
endianSwap(&exh.header.languageCount); endianSwap(&exh.header.languageCount);
endianSwap(&exh.header.rowCount); endianSwap(&exh.header.rowCount);
exh.columnDefinitions.resize(exh.header.columnCount); data.read_structures(&exh.columnDefinitions, exh.header.columnCount);
data.read_structures(&exh.pages, exh.header.pageCount);
fread(exh.columnDefinitions.data(), sizeof(ExcelColumnDefinition) * exh.header.columnCount, 1, file); data.read_structures(&exh.language, exh.header.languageCount);
exh.pages.resize(exh.header.pageCount);
fread(exh.pages.data(), sizeof(ExcelDataPagination) * exh.header.pageCount, 1, file);
exh.language.resize(exh.header.languageCount);
fread(exh.language.data(), sizeof(Language) * exh.header.languageCount, 1, file);
for(auto& columnDef : exh.columnDefinitions) { for(auto& columnDef : exh.columnDefinitions) {
endianSwap(&columnDef.offset); endianSwap(&columnDef.offset);

11
src/exlparser.cpp
View file

@ -3,18 +3,13 @@
#include <stdexcept> #include <stdexcept>
#include <fstream> #include <fstream>
EXL readEXL(std::string_view path) { EXL readEXL(MemorySpan data) {
std::fstream file; auto stream = data.read_as_stream();
file.open(path.data(), std::iostream::in);
if(!file.is_open()) {
throw std::runtime_error("Failed to read exl file from " + std::string(path.data()));
}
EXL exl; EXL exl;
std::string line; std::string line;
while (std::getline(file, line)) { while (std::getline(stream, line)) {
const size_t comma = line.find_first_of(','); const size_t comma = line.find_first_of(',');
std::string name = line.substr(0, comma); std::string name = line.substr(0, comma);

145
src/gamedata.cpp
View file

@ -49,9 +49,8 @@ GameData::GameData(const std::string_view dataDirectory) {
repositories.push_back(repository); repositories.push_back(repository);
} }
extractFile("exd/root.exl", "root.exl"); auto root_exl_data = extractFile("exd/root.exl");
rootEXL = readEXL(*root_exl_data);
rootEXL = readEXL("root.exl");
} }
std::vector<std::string> GameData::getAllSheetNames() { std::vector<std::string> GameData::getAllSheetNames() {
@ -96,9 +95,9 @@ std::tuple<Repository, std::string> GameData::calculateRepositoryCategory(std::s
return {getBaseRepository(), tokens[0]}; return {getBaseRepository(), tokens[0]};
} }
void GameData::extractFile(std::string_view dataFilePath, std::string_view outPath) { std::optional<MemoryBuffer> GameData::extractFile(const std::string_view data_file_path) {
const uint64_t hash = calculateHash(dataFilePath); const uint64_t hash = calculateHash(data_file_path);
auto [repository, category] = calculateRepositoryCategory(dataFilePath); auto [repository, category] = calculateRepositoryCategory(data_file_path);
auto [index_filename, index2_filename] = repository.get_index_filenames(categoryToID[category]); auto [index_filename, index2_filename] = repository.get_index_filenames(categoryToID[category]);
auto index_path = fmt::format("{data_directory}/{repository}/{filename}", auto index_path = fmt::format("{data_directory}/{repository}/{filename}",
@ -160,11 +159,9 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa
fclose(file); fclose(file);
FILE* newFile = fopen(outPath.data(), "w"); return {data};
fwrite(data.data(), data.size(), 1, newFile);
fclose(newFile);
} else if(info.fileType == FileType::Model) { } else if(info.fileType == FileType::Model) {
FILE* newFile = fopen(outPath.data(), "w"); MemoryBuffer buffer;
// reset // reset
fseek(file, offset, SEEK_SET); fseek(file, offset, SEEK_SET);
@ -223,49 +220,54 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa
std::vector<uint16_t> compressedBlockSizes(totalBlocks); std::vector<uint16_t> compressedBlockSizes(totalBlocks);
fread(compressedBlockSizes.data(), compressedBlockSizes.size() * sizeof(uint16_t), 1, file); fread(compressedBlockSizes.data(), compressedBlockSizes.size() * sizeof(uint16_t), 1, file);
int currentBlock = 0; int currentBlock = 0;
int stackSize = 0; uint32_t stackSize = 0;
int runtimeSize = 0; uint32_t runtimeSize = 0;
std::array<int, 3> vertexDataOffsets; std::array<uint32_t, 3> vertexDataOffsets;
std::array<int, 3> indexDataOffsets; std::array<uint32_t, 3> indexDataOffsets;
std::array<int, 3> vertexDataSizes; std::array<uint32_t, 3> vertexDataSizes;
std::array<int, 3> indexDataSizes; std::array<uint32_t, 3> indexDataSizes;
// data.append 0x44 // data.append 0x44
fseek(newFile, 0x44, SEEK_SET); buffer.seek(0x44, Seek::Set);
fseek(file, baseOffset + modelInfo.stackOffset, SEEK_SET); fseek(file, baseOffset + modelInfo.stackOffset, SEEK_SET);
size_t stackStart = ftell(newFile); size_t stackStart = buffer.current_position();
for(int i = 0; i < modelInfo.stackBlockNum; i++) { for(int i = 0; i < modelInfo.stackBlockNum; i++) {
size_t lastPos = ftell(file); size_t lastPos = ftell(file);
auto data = read_data_block(file, lastPos); auto data = read_data_block(file, lastPos);
fwrite(data.data(), data.size(), 1, newFile); // i think we write this to file? buffer.write(data);
fseek(file, lastPos + compressedBlockSizes[currentBlock], SEEK_SET); fseek(file, lastPos + compressedBlockSizes[currentBlock], SEEK_SET);
currentBlock++; currentBlock++;
} }
size_t stackEnd = ftell(newFile); size_t stackEnd = buffer.current_position();
stackSize = (int)(stackEnd - stackStart); stackSize = (int)(stackEnd - stackStart);
fseek(file, baseOffset + modelInfo.runtimeOffset, SEEK_SET); fseek(file, baseOffset + modelInfo.runtimeOffset, SEEK_SET);
size_t runtimeStart = ftell(newFile); size_t runtimeStart = buffer.current_position();
for(int i = 0; i < modelInfo.runtimeBlockNum; i++) { for(int i = 0; i < modelInfo.runtimeBlockNum; i++) {
size_t lastPos = ftell(file); size_t lastPos = ftell(file);
auto data = read_data_block(file, lastPos); auto data = read_data_block(file, lastPos);
fwrite(data.data(), data.size(), 1, newFile); buffer.write(data);
fseek(file, lastPos + compressedBlockSizes[currentBlock], SEEK_SET); fseek(file, lastPos + compressedBlockSizes[currentBlock], SEEK_SET);
currentBlock++; currentBlock++;
} }
size_t runtimeEnd = ftell(newFile); size_t runtimeEnd = buffer.current_position();
runtimeSize = (int)(runtimeEnd - runtimeStart); runtimeSize = (int)(runtimeEnd - runtimeStart);
// process all 3 lods // process all 3 lods
for(int i = 0; i < 3; i++) { for(int i = 0; i < 3; i++) {
if(modelInfo.vertexBlockBufferBlockNum[i] != 0) { if(modelInfo.vertexBlockBufferBlockNum[i] != 0) {
int currentVertexOffset = ftell(newFile); int currentVertexOffset = buffer.current_position();
if(i == 0 || currentVertexOffset != vertexDataOffsets[i - 1]) if(i == 0 || currentVertexOffset != vertexDataOffsets[i - 1])
vertexDataOffsets[i] = currentVertexOffset; vertexDataOffsets[i] = currentVertexOffset;
else else
@ -275,8 +277,10 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa
for(int j = 0; j < modelInfo.vertexBlockBufferBlockNum[i]; j++) { for(int j = 0; j < modelInfo.vertexBlockBufferBlockNum[i]; j++) {
size_t lastPos = ftell(file); size_t lastPos = ftell(file);
auto data = read_data_block(file, lastPos); auto data = read_data_block(file, lastPos);
fwrite(data.data(), data.size(), 1, newFile); // i think we write this to file? buffer.write(data);
vertexDataSizes[i] += (int)data.size(); vertexDataSizes[i] += (int)data.size();
fseek(file, lastPos + compressedBlockSizes[currentBlock], SEEK_SET); fseek(file, lastPos + compressedBlockSizes[currentBlock], SEEK_SET);
currentBlock++; currentBlock++;
@ -286,7 +290,7 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa
// TODO: lol no edge geometry // TODO: lol no edge geometry
if(modelInfo.indexBufferBlockNum[i] != 0) { if(modelInfo.indexBufferBlockNum[i] != 0) {
int currentIndexOffset = ftell(newFile); int currentIndexOffset = buffer.current_position();
if(i == 0 || currentIndexOffset != indexDataOffsets[i - 1]) if(i == 0 || currentIndexOffset != indexDataOffsets[i - 1])
indexDataOffsets[i] = currentIndexOffset; indexDataOffsets[i] = currentIndexOffset;
else else
@ -294,8 +298,10 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa
for(int j = 0; j < modelInfo.indexBufferBlockNum[i]; j++) { for(int j = 0; j < modelInfo.indexBufferBlockNum[i]; j++) {
size_t lastPos = ftell(file); size_t lastPos = ftell(file);
auto data = read_data_block(file, lastPos); auto data = read_data_block(file, lastPos);
fwrite(data.data(), data.size(), 1, newFile); // i think we write this to file? buffer.write(data);
indexDataSizes[i] += (int)data.size(); indexDataSizes[i] += (int)data.size();
fseek(file, lastPos + compressedBlockSizes[currentBlock], SEEK_SET); fseek(file, lastPos + compressedBlockSizes[currentBlock], SEEK_SET);
currentBlock++; currentBlock++;
@ -304,45 +310,43 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa
} }
// now write mdl header // now write mdl header
fseek(newFile, 0, SEEK_SET); buffer.seek(0, Seek::Set);
fwrite(&modelInfo.version, sizeof(uint32_t), 1, newFile);
fwrite(&stackSize, sizeof(uint32_t), 1, newFile); buffer.write(modelInfo.version);
fwrite(&runtimeSize, sizeof(uint32_t), 1, newFile); buffer.write(stackSize);
fwrite(&modelInfo.vertexDeclarationNum, sizeof(unsigned short), 1, newFile); buffer.write(runtimeSize);
fwrite(&modelInfo.materialNum, sizeof(unsigned short), 1, newFile); buffer.write(modelInfo.vertexDeclarationNum);
buffer.write(modelInfo.materialNum);
for(int i = 0; i < 3; i++) for(int i = 0; i < 3; i++)
fwrite(&vertexDataOffsets[i], sizeof(uint32_t), 1, newFile); buffer.write(vertexDataOffsets[i]);
for(int i = 0; i < 3; i++) for(int i = 0; i < 3; i++)
fwrite(&indexDataOffsets[i], sizeof(uint32_t), 1, newFile); buffer.write(indexDataOffsets[i]);
for(int i = 0; i < 3; i++) for(int i = 0; i < 3; i++)
fwrite(&vertexDataSizes[i], sizeof(uint32_t), 1, newFile); buffer.write(vertexDataSizes[i]);
for(int i = 0; i < 3; i++) for(int i = 0; i < 3; i++)
fwrite(&indexDataSizes[i], sizeof(uint32_t), 1, newFile); buffer.write(indexDataSizes[i]);
fwrite(&modelInfo.numLods, sizeof(uint8_t), 1, file); buffer.write(modelInfo.numLods);
fwrite(&modelInfo.indexBufferStreamingEnabled, sizeof(bool), 1, file); buffer.write(modelInfo.indexBufferStreamingEnabled);
fwrite(&modelInfo.edgeGeometryEnabled, sizeof(bool), 1, file); buffer.write(modelInfo.edgeGeometryEnabled);
uint8_t dummy[] = {0}; uint8_t dummy = 0;
fwrite(dummy, sizeof(uint8_t), 1, file); buffer.write(dummy);
fclose(newFile);
fclose(file); fclose(file);
return {buffer};
} else { } else {
throw std::runtime_error("File type is not handled yet for " + std::string(dataFilePath)); throw std::runtime_error("File type is not handled yet for " + std::string(data_file_path));
} }
fmt::print("Extracted {} to {}!\n", dataFilePath, outPath);
return;
} }
} }
fmt::print("Failed to find file {}.\n", dataFilePath); fmt::print("Failed to find file {}.\n", data_file_path);
} }
std::optional<EXH> GameData::readExcelSheet(std::string_view name) { std::optional<EXH> GameData::readExcelSheet(std::string_view name) {
@ -358,9 +362,8 @@ std::optional<EXH> GameData::readExcelSheet(std::string_view name) {
std::string outPath = newFilename + ".exh"; std::string outPath = newFilename + ".exh";
std::replace(outPath.begin(), outPath.end(), '/', '_'); std::replace(outPath.begin(), outPath.end(), '/', '_');
extractFile(exhFilename, outPath); auto exh_data = extractFile(exhFilename);
return readEXH(*exh_data);
return readEXH(outPath);
} }
} }
@ -368,53 +371,45 @@ std::optional<EXH> GameData::readExcelSheet(std::string_view name) {
} }
void GameData::extractSkeleton() { void GameData::extractSkeleton() {
std::string path = fmt::format("chara/human/c0201/skeleton/base/b0001/skl_c0201b0001.sklb"); const std::string path = fmt::format("chara/human/c0201/skeleton/base/b0001/skl_c0201b0001.sklb");
auto skel_data = extractFile(path);
extractFile(path, "test.skel"); auto skel_span = MemorySpan(*skel_data);
FILE* file = fopen("test.skel", "rb");
fseek(file, 0, SEEK_END);
size_t end = ftell(file);
fseek(file, 0, SEEK_SET);
int32_t magic; int32_t magic;
fread(&magic, sizeof(int32_t), 1, file); skel_span.read(&magic);
int32_t format;
fread(&format, sizeof(int32_t), 1, file);
fseek(file, sizeof(uint16_t), SEEK_CUR);
if(magic != 0x736B6C62) if(magic != 0x736B6C62)
fmt::print("INVALID SKLB magic"); fmt::print("INVALID SKLB magic");
size_t dataOffset = 0; int32_t format;
skel_span.read(&format);
skel_span.seek(sizeof(uint16_t), Seek::Current);
int16_t dataOffset = 0;
switch(format) { switch(format) {
case 0x31323030: case 0x31323030:
fread(&dataOffset, sizeof(int16_t), 1, file); skel_span.read(&dataOffset);
break; break;
case 0x31333030: case 0x31333030:
case 0x31333031: case 0x31333031:
fseek(file, sizeof(uint16_t), SEEK_CUR); skel_span.seek(sizeof(uint16_t), Seek::Current);
fread(&dataOffset, sizeof(int16_t), 1, file); skel_span.read(&dataOffset);
break; break;
default: default:
fmt::print("INVALID SKLB format {}", format); fmt::print("INVALID SKLB format {}", format);
break; break;
} }
fseek(file, dataOffset, SEEK_SET); skel_span.seek(dataOffset, Seek::Set);
std::vector<uint8_t> havokData(end - dataOffset); std::vector<uint8_t> havokData(skel_span.size() - dataOffset);
fread(havokData.data(), havokData.size(), 1, file); skel_span.read_structures(&havokData, havokData.size());
FILE* newFile = fopen("test.sklb.havok", "wb"); FILE* newFile = fopen("test.sklb.havok", "wb");
fwrite(havokData.data(), havokData.size(), 1, newFile); fwrite(havokData.data(), havokData.size(), 1, newFile);
fclose(newFile); fclose(newFile);
fclose(file);
} }
IndexFile<IndexHashTableEntry> GameData::getIndexListing(std::string_view folder) { IndexFile<IndexHashTableEntry> GameData::getIndexListing(std::string_view folder) {

181
src/mdlparser.cpp
View file

@ -8,19 +8,7 @@
#include <fstream> #include <fstream>
#include <algorithm> #include <algorithm>
Model parseMDL(const std::string_view path) { Model parseMDL(MemorySpan data) {
FILE* file = fopen(path.data(), "rb");
if(file == nullptr) {
throw std::runtime_error("Failed to open exh file " + std::string(path));
}
enum class FileType : int32_t {
Empty = 1,
Standard = 2,
Model = 3,
Texture = 4
};
struct ModelFileHeader { struct ModelFileHeader {
uint32_t version; uint32_t version;
uint32_t stackSize; uint32_t stackSize;
@ -37,9 +25,7 @@ Model parseMDL(const std::string_view path) {
uint8_t padding; uint8_t padding;
} modelFileHeader; } modelFileHeader;
fread(&modelFileHeader, sizeof(ModelFileHeader), 1, file); data.read(&modelFileHeader);
fmt::print("stack size: {}\n", modelFileHeader.stackSize);
struct VertexElement { struct VertexElement {
uint8_t stream, offset, type, usage, usageIndex; uint8_t stream, offset, type, usage, usageIndex;
@ -53,29 +39,28 @@ Model parseMDL(const std::string_view path) {
std::vector<VertexDeclaration> vertexDecls(modelFileHeader.vertexDeclarationCount); std::vector<VertexDeclaration> vertexDecls(modelFileHeader.vertexDeclarationCount);
for(int i = 0; i < modelFileHeader.vertexDeclarationCount; i++) { for(int i = 0; i < modelFileHeader.vertexDeclarationCount; i++) {
VertexElement element {}; VertexElement element {};
fread(&element, sizeof(VertexElement), 1, file); data.read(&element);
do { do {
vertexDecls[i].elements.push_back(element); vertexDecls[i].elements.push_back(element);
fread(&element, sizeof(VertexElement), 1, file); data.read(&element);
} while (element.stream != 255); } while (element.stream != 255);
int toSeek = 17 * 8 - (vertexDecls[i].elements.size() + 1) * 8; int toSeek = 17 * 8 - (vertexDecls[i].elements.size() + 1) * 8;
fseek(file, toSeek, SEEK_CUR); data.seek(toSeek, Seek::Current);
} }
uint16_t stringCount; uint16_t stringCount;
fread(&stringCount, sizeof(uint16_t), 1, file); data.read(&stringCount);
fmt::print("string count: {}\n", stringCount);
// dummy // dummy
fseek(file, sizeof(uint16_t), SEEK_CUR); data.seek(sizeof(uint16_t), Seek::Current);
uint32_t stringSize; uint32_t stringSize;
fread(&stringSize, sizeof(uint32_t), 1, file); data.read(&stringSize);
std::vector<uint8_t> strings(stringSize); std::vector<uint8_t> strings;
fread(strings.data(), stringSize, 1, file); data.read_structures(&strings, stringSize);
enum ModelFlags1 : uint8_t enum ModelFlags1 : uint8_t
{ {
@ -135,28 +120,23 @@ Model parseMDL(const std::string_view path) {
uint8_t padding[6]; uint8_t padding[6];
} modelHeader; } modelHeader;
fread(&modelHeader, sizeof(modelHeader), 1, file); data.read(&modelHeader);
fmt::print("mesh count: {}\n", modelHeader.meshCount);
fmt::print("attribute count: {}\n", modelHeader.attributeCount);
struct ElementId { struct ElementId {
unsigned int elementId; uint32_t elementId;
unsigned int parentBoneName; uint32_t parentBoneName;
std::vector<float> translate; std::vector<float> translate;
std::vector<float> rotate; std::vector<float> rotate;
}; };
std::vector<ElementId> elementIds(modelHeader.elementIdCount); std::vector<ElementId> elementIds(modelHeader.elementIdCount);
for(int i = 0; i < modelHeader.elementIdCount; i++) { for(int i = 0; i < modelHeader.elementIdCount; i++) {
fread(&elementIds[i].elementId, sizeof(uint32_t), 1, file); data.read(&elementIds[i].elementId);
fread(&elementIds[i].parentBoneName, sizeof(uint32_t), 1, file); data.read(&elementIds[i].parentBoneName);
elementIds[i].translate.resize(3); // FIXME: these always seem to be 3, convert to static array? then we could probably fread this all in one go! // FIXME: these always seem to be 3, convert to static array? then we could probably read this all in one go!
elementIds[i].rotate.resize(3); data.read_structures(&elementIds[i].translate, 3);
data.read_structures(&elementIds[i].rotate, 3);
fread(elementIds[i].translate.data(), sizeof(float) * 3, 1, file);
fread(elementIds[i].rotate.data(), sizeof(float) * 3, 1, file);
} }
struct Lod { struct Lod {
@ -184,10 +164,10 @@ Model parseMDL(const std::string_view path) {
unsigned int indexDataOffset; unsigned int indexDataOffset;
}; };
std::array<Lod, 3> lods; std::vector<Lod> lods;
fread(lods.data(), sizeof(Lod) * 3, 1, file);
// TODO: support models that support more than 3 lods // TODO: support models that support more than 3 lods
data.read_structures(&lods, 3);
struct Mesh { struct Mesh {
unsigned short vertexCount; unsigned short vertexCount;
@ -199,7 +179,7 @@ Model parseMDL(const std::string_view path) {
unsigned short boneTableIndex; unsigned short boneTableIndex;
unsigned int startIndex; unsigned int startIndex;
std::vector<unsigned int> vertexBufferOffset; std::vector<uint32_t> vertexBufferOffset;
std::vector<uint8_t> vertexBufferStride; std::vector<uint8_t> vertexBufferStride;
uint8_t vertexStreamCount; uint8_t vertexStreamCount;
@ -207,26 +187,23 @@ Model parseMDL(const std::string_view path) {
std::vector<Mesh> meshes(modelHeader.meshCount); std::vector<Mesh> meshes(modelHeader.meshCount);
for(int i = 0; i < modelHeader.meshCount; i++) { for(int i = 0; i < modelHeader.meshCount; i++) {
fread(&meshes[i].vertexCount, sizeof(uint16_t), 1, file); data.read(&meshes[i].vertexCount);
fread(&meshes[i].padding, sizeof(uint16_t), 1, file); data.read(&meshes[i].padding);
fread(&meshes[i].indexCount, sizeof(uint32_t), 1, file); data.read(&meshes[i].indexCount);
fread(&meshes[i].materialIndex, sizeof(uint16_t), 1, file); data.read(&meshes[i].materialIndex);
fread(&meshes[i].subMeshIndex, sizeof(uint16_t), 1, file); data.read(&meshes[i].subMeshIndex);
fread(&meshes[i].subMeshCount, sizeof(uint16_t), 1, file); data.read(&meshes[i].subMeshCount);
fread(&meshes[i].boneTableIndex, sizeof(uint16_t), 1, file); data.read(&meshes[i].boneTableIndex);
fread(&meshes[i].startIndex, sizeof(uint32_t), 1, file); data.read(&meshes[i].startIndex);
meshes[i].vertexBufferOffset.resize(3); data.read_structures(&meshes[i].vertexBufferOffset, 3);
fread(meshes[i].vertexBufferOffset.data(), sizeof(uint32_t) * 3, 1, file); data.read_structures(&meshes[i].vertexBufferStride, 3);
meshes[i].vertexBufferStride.resize(3); data.read(&meshes[i].vertexStreamCount);
fread(meshes[i].vertexBufferStride.data(), sizeof(uint8_t) * 3, 1, file);
fread(&meshes[i].vertexStreamCount, sizeof(uint8_t), 1, file);
} }
std::vector<uint32_t> attributeNameOffsets(modelHeader.attributeCount); std::vector<uint32_t> attributeNameOffsets;
fread(attributeNameOffsets.data(), sizeof(uint32_t) * modelHeader.attributeCount, 1, file); data.read_structures(&attributeNameOffsets, modelHeader.attributeCount);
// TODO: implement terrain shadow meshes // TODO: implement terrain shadow meshes
@ -238,69 +215,60 @@ Model parseMDL(const std::string_view path) {
unsigned short boneCount; unsigned short boneCount;
}; };
std::vector<Submesh> submeshes(modelHeader.submeshCount); std::vector<Submesh> submeshes;
for(int i = 0; i < modelHeader.submeshCount; i++) { data.read_structures(&submeshes, modelHeader.submeshCount);
fread(&submeshes[i], sizeof(Submesh), 1, file);
}
// TODO: implement terrain shadow submeshes // TODO: implement terrain shadow submeshes
std::vector<uint32_t> materialNameOffsets(modelHeader.materialCount); std::vector<uint32_t> materialNameOffsets;
fread(materialNameOffsets.data(), sizeof(uint32_t) * modelHeader.materialCount, 1, file); data.read_structures(&materialNameOffsets, modelHeader.materialCount);
std::vector<uint32_t> boneNameOffsets(modelHeader.boneCount); std::vector<uint32_t> boneNameOffsets;
fread(boneNameOffsets.data(), sizeof(uint32_t) * modelHeader.boneCount, 1, file); data.read_structures(&boneNameOffsets, modelHeader.boneCount);
struct BoneTable { struct BoneTable {
std::vector<unsigned short> boneIndex; std::vector<uint16_t> boneIndex;
uint8_t boneCount; uint8_t boneCount;
std::vector<uint8_t> padding; std::vector<uint8_t> padding;
}; };
std::vector<BoneTable> boneTables(modelHeader.boneTableCount); std::vector<BoneTable> boneTables(modelHeader.boneTableCount);
for(int i = 0; i < modelHeader.boneTableCount; i++) { for(int i = 0; i < modelHeader.boneTableCount; i++) {
boneTables[i].boneIndex.resize(64); data.read_structures(&boneTables[i].boneIndex, 64);
fread(boneTables[i].boneIndex.data(), 64 * sizeof(uint16_t), 1, file);
fread(&boneTables[i].boneCount, sizeof(uint8_t), 1, file);
boneTables[i].padding.resize(3);
fread(boneTables[i].padding.data(), sizeof(uint8_t) * 3, 1, file);
fmt::print("bone count: {}\n", boneTables[i].boneCount); data.read(&boneTables[i].boneCount);
data.read_structures(&boneTables[i].padding, 3);
} }
// TODO: implement shapes // TODO: implement shapes
unsigned int submeshBoneMapSize; uint32_t submeshBoneMapSize;
fread(&submeshBoneMapSize, sizeof(uint32_t), 1, file); data.read(&submeshBoneMapSize);
std::vector<uint16_t > submeshBoneMap((int)submeshBoneMapSize / 2); std::vector<uint16_t> submeshBoneMap;
fread(submeshBoneMap.data(), submeshBoneMap.size() * sizeof(uint16_t), 1, file); data.read_structures(&submeshBoneMap, (int)submeshBoneMapSize / 2);
uint8_t paddingAmount; uint8_t paddingAmount;
fread(&paddingAmount, sizeof(uint8_t), 1, file); data.read(&paddingAmount);
fseek(file, paddingAmount, SEEK_CUR); data.seek(paddingAmount, Seek::Current);
struct BoundingBox { struct BoundingBox {
std::array<float, 4> min, max; std::array<float, 4> min, max;
}; };
BoundingBox boundingBoxes, modelBoundingBoxes, waterBoundingBoxes, verticalFogBoundingBoxes; BoundingBox boundingBoxes, modelBoundingBoxes, waterBoundingBoxes, verticalFogBoundingBoxes;
fread(&boundingBoxes, sizeof(BoundingBox), 1, file); data.read(&boundingBoxes);
fread(&modelBoundingBoxes, sizeof(BoundingBox), 1, file); data.read(&modelBoundingBoxes);
fread(&waterBoundingBoxes, sizeof(BoundingBox), 1, file); data.read(&waterBoundingBoxes);
fread(&verticalFogBoundingBoxes, sizeof(BoundingBox), 1, file); data.read(&verticalFogBoundingBoxes);
std::vector<BoundingBox> boneBoundingBoxes(modelHeader.boneCount); std::vector<BoundingBox> boneBoundingBoxes;
fread(boneBoundingBoxes.data(), modelHeader.boneCount * sizeof(BoundingBox), 1, file); data.read_structures(&boneBoundingBoxes, modelHeader.boneCount);
fmt::print("Successfully read mdl file!\n");
fmt::print("Now exporting as test.obj...\n");
Model model; Model model;
// TODO: doesn't work for lod above 0
for(int i = 0; i < modelHeader.lodCount; i++) { for(int i = 0; i < modelHeader.lodCount; i++) {
::Lod lod; ::Lod lod;
@ -333,44 +301,48 @@ Model parseMDL(const std::string_view path) {
std::vector<Vertex> vertices(vertexCount); std::vector<Vertex> vertices(vertexCount);
for(int k = 0; k < vertexCount; k++) { for(int k = 0; k < vertexCount; k++) {
for(auto & orderedElement : decl.elements) { for(auto& orderedElement : decl.elements) {
VertexType type = (VertexType)orderedElement.type; auto type = static_cast<VertexType>(orderedElement.type);
VertexUsage usage = (VertexUsage)orderedElement.usage; auto usage = static_cast<VertexUsage>(orderedElement.usage);
const int stream = orderedElement.stream; const int stream = orderedElement.stream;
fseek(file, lods[i].vertexDataOffset + meshes[j].vertexBufferOffset[stream] + orderedElement.offset + meshes[i].vertexBufferStride[stream] * k, SEEK_SET); data.seek(lods[i].vertexDataOffset + meshes[j].vertexBufferOffset[stream] + orderedElement.offset + meshes[i].vertexBufferStride[stream] * k, Seek::Set);
std::array<float, 4> floatData = {}; std::array<float, 4> floatData = {};
switch(type) { switch(type) {
case VertexType::Single3: case VertexType::Single3:
fread(floatData.data(), sizeof(float) * 3, 1, file); data.read_array(floatData.data(), 3);
break; break;
case VertexType::Single4: case VertexType::Single4:
fread(floatData.data(), sizeof(float) * 4, 1, file); data.read_array(floatData.data(), 4);
break; break;
case VertexType::UInt: case VertexType::UInt:
fseek(file, sizeof(uint8_t) * 4, SEEK_CUR); data.seek(sizeof(uint8_t) * 4, Seek::Current);
break; break;
case VertexType::ByteFloat4: case VertexType::ByteFloat4: {
uint8_t values[4]; uint8_t values[4];
fread(values, sizeof(uint8_t) * 4, 1, file); data.read_array(values, 4);
floatData[0] = byte_to_float(values[0]); floatData[0] = byte_to_float(values[0]);
floatData[1] = byte_to_float(values[1]); floatData[1] = byte_to_float(values[1]);
floatData[2] = byte_to_float(values[2]); floatData[2] = byte_to_float(values[2]);
floatData[3] = byte_to_float(values[3]); floatData[3] = byte_to_float(values[3]);
}
break; break;
case VertexType::Half2: { case VertexType::Half2: {
uint16_t values[2]; uint16_t values[2];
fread(values, sizeof(uint16_t) * 2, 1, file); data.read_array(values, 2);
floatData[0] = half_to_float(values[0]); floatData[0] = half_to_float(values[0]);
floatData[1] = half_to_float(values[1]); floatData[1] = half_to_float(values[1]);
} }
break; break;
case VertexType::Half4: { case VertexType::Half4: {
uint16_t values[4]; uint16_t values[4];
fread(values, sizeof(uint16_t) * 4, 1, file); data.read_array(values, 4);
floatData[0] = half_to_float(values[0]); floatData[0] = half_to_float(values[0]);
floatData[1] = half_to_float(values[1]); floatData[1] = half_to_float(values[1]);
floatData[2] = half_to_float(values[2]); floatData[2] = half_to_float(values[2]);
@ -390,9 +362,10 @@ Model parseMDL(const std::string_view path) {
} }
} }
fseek(file, modelFileHeader.indexOffsets[i] + (meshes[j].startIndex * 2), SEEK_SET); data.seek(modelFileHeader.indexOffsets[i] + (meshes[j].startIndex * 2), Seek::Set);
std::vector<uint16_t> indices(meshes[j].indexCount);
fread(indices.data(), meshes[j].indexCount * sizeof(uint16_t), 1, file); std::vector<uint16_t> indices;
data.read_structures(&indices, meshes[j].indexCount);
part.indices = indices; part.indices = indices;
part.vertices = vertices; part.vertices = vertices;