From 2610de0f7fa7a3756f97151d5278e346c9307640 Mon Sep 17 00:00:00 2001 From: Joshua Goins Date: Mon, 11 Apr 2022 20:02:20 -0400 Subject: [PATCH] Add basic MDL parsing This can read basic MDL files (no terrain, shadow, edge geometry, etc.) and export them to OBJ. This will be expanded in future commits to let you export this to an easily readable vertex/index list. This code is mostly based off of xivModdingFramework/Lumina, and just ported to C++. All of it will eventually be refactored as there is duplicate structs EVERYWHERE. --- CMakeLists.txt | 3 +- README.md | 1 + include/mdlparser.h | 21 +++ src/gamedata.cpp | 26 +-- src/mdlparser.cpp | 397 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 436 insertions(+), 12 deletions(-) create mode 100644 include/mdlparser.h create mode 100644 src/mdlparser.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 93b6e30..e373412 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,7 +71,8 @@ add_library(libxiv STATIC src/exdparser.cpp src/installextract.cpp src/patch.cpp - src/exlparser.cpp) + src/exlparser.cpp + src/mdlparser.cpp) target_include_directories(libxiv PUBLIC include PRIVATE src) target_link_libraries(libxiv PUBLIC ${LIBRARIES}) target_link_directories(libxiv PUBLIC ${LIB_DIRS}) \ No newline at end of file diff --git a/README.md b/README.md index fc9ce79..b01b24e 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ A modding framework for FFXIV written in C++. This is used in [Novus](https://gi * [EXL](include/exlparser.h) * [FIIN](include/fiinparser.h) * [INDEX/INDEX2](include/indexparser.h) + * [MDL](include/mdlparser.h) ## Dependencies **Note:** Some of these dependencies will automatically be downloaded from the Internet if not found diff --git a/include/mdlparser.h b/include/mdlparser.h new file mode 100644 index 0000000..67891f2 --- /dev/null +++ b/include/mdlparser.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include +#include + +struct Vertex { + std::array position; + float blendWeights[4]; + std::vector blendIndices; + float normal[3]; + float uv[4]; + float color[4]; + float tangent2[4]; + float tangent1[4]; +}; + +struct Model { +}; + +Model parseMDL(const std::string_view path); \ No newline at end of file diff --git a/src/gamedata.cpp b/src/gamedata.cpp index 0471e9a..1c7c532 100644 --- a/src/gamedata.cpp +++ b/src/gamedata.cpp @@ -88,7 +88,11 @@ int getExpansionID(std::string_view repositoryName) { } std::string GameData::calculateFilename(const int category, const int expansion, const int chunk, const std::string_view platform, const std::string_view type) { - return fmt::sprintf("%02x%02x%02x.%s.%s", category, expansion, chunk, platform, type); + if(type == "index") { + return fmt::sprintf("%02x%02x%02x.%s.%s", category, expansion, chunk, platform, type); + } else if(type == "dat") { + return fmt::sprintf("%02x%02x00.%s.%s%01x", category, expansion, platform, type, chunk); + } } void GameData::extractFile(std::string_view dataFilePath, std::string_view outPath) { @@ -108,7 +112,7 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa for(const auto entry : indexFile.entries) { if(entry.hash == hash) { - auto dataFilename = calculateFilename(categoryToID[category], getExpansionID(repository), entry.dataFileId, "win32", "dat0"); + auto dataFilename = calculateFilename(categoryToID[category], getExpansionID(repository), entry.dataFileId, "win32", "dat"); fmt::print("Opening data file {}...\n", dataFilename); @@ -380,29 +384,29 @@ void GameData::extractFile(std::string_view dataFilePath, std::string_view outPa // now write mdl header fseek(newFile, 0, SEEK_SET); fwrite(&modelInfo.version, sizeof(uint32_t), 1, newFile); - fwrite(&stackSize, sizeof(int), 1, newFile); - fwrite(&runtimeSize, sizeof(int), 1, newFile); - fwrite(&modelInfo.vertexDeclarationNum, sizeof(uint16_t), 1, newFile); - fwrite(&modelInfo.materialNum, sizeof(uint16_t), 1, newFile); + fwrite(&stackSize, sizeof(uint32_t), 1, newFile); + fwrite(&runtimeSize, sizeof(uint32_t), 1, newFile); + fwrite(&modelInfo.vertexDeclarationNum, sizeof(unsigned short), 1, newFile); + fwrite(&modelInfo.materialNum, sizeof(unsigned short), 1, newFile); for(int i = 0; i < 3; i++) - fwrite(&vertexDataOffsets[i], sizeof(int), 1, newFile); + fwrite(&vertexDataOffsets[i], sizeof(uint32_t), 1, newFile); for(int i = 0; i < 3; i++) - fwrite(&indexDataOffsets[i], sizeof(int), 1, newFile); + fwrite(&indexDataOffsets[i], sizeof(uint32_t), 1, newFile); for(int i = 0; i < 3; i++) - fwrite(&vertexDataSizes[i], sizeof(int), 1, newFile); + fwrite(&vertexDataSizes[i], sizeof(uint32_t), 1, newFile); for(int i = 0; i < 3; i++) - fwrite(&indexDataSizes[i], sizeof(int), 1, newFile); + fwrite(&indexDataSizes[i], sizeof(uint32_t), 1, newFile); fwrite(&modelInfo.numLods, sizeof(uint8_t), 1, file); fwrite(&modelInfo.indexBufferStreamingEnabled, sizeof(bool), 1, file); fwrite(&modelInfo.edgeGeometryEnabled, sizeof(bool), 1, file); uint8_t dummy[] = {0}; - fwrite(&dummy, sizeof(dummy), 1, file); + fwrite(dummy, sizeof(uint8_t), 1, file); fmt::print("data size: {}\n", modelInfo.fileSize); diff --git a/src/mdlparser.cpp b/src/mdlparser.cpp new file mode 100644 index 0000000..78b4383 --- /dev/null +++ b/src/mdlparser.cpp @@ -0,0 +1,397 @@ +#include "mdlparser.h" + +#include +#include +#include +#include +#include +#include + +Model parseMDL(const std::string_view path) { + FILE* file = fopen(path.data(), "rb"); + if(file == nullptr) { + throw std::runtime_error("Failed to open exh file " + std::string(path)); + } + + enum FileType : int32_t { + Empty = 1, + Standard = 2, + Model = 3, + Texture = 4 + }; + + struct ModelFileHeader { + uint32_t version; + uint32_t stackSize; + uint32_t runtimeSize; + unsigned short vertexDeclarationCount; + unsigned short materialCount; + uint32_t vertexOffsets[3]; + uint32_t indexOffsets[3]; + uint32_t vertexBufferSize[3]; + uint32_t indexBufferSize[3]; + uint8_t lodCount; + bool indexBufferStreamingEnabled; + bool hasEdgeGeometry; + uint8_t padding; + } modelFileHeader; + + fread(&modelFileHeader, sizeof(ModelFileHeader), 1, file); + + fmt::print("stack size: {}\n", modelFileHeader.stackSize); + + struct VertexElement { + uint8_t stream, offset, type, usage, usageIndex; + uint8_t padding[3]; + }; + + struct VertexDeclaration { + std::vector elements; + }; + + std::vector vertexDecls(modelFileHeader.vertexDeclarationCount); + for(int i = 0; i < modelFileHeader.vertexDeclarationCount; i++) { + VertexElement element {}; + fread(&element, sizeof(VertexElement), 1, file); + do { + vertexDecls[i].elements.push_back(element); + fread(&element, sizeof(VertexElement), 1, file); + } while (element.stream != 255); + + int toSeek = 17 * 8 - (vertexDecls[i].elements.size() + 1) * 8; + fseek(file, toSeek, SEEK_CUR); + } + + uint16_t stringCount; + fread(&stringCount, sizeof(uint16_t), 1, file); + + fmt::print("string count: {}\n", stringCount); + + // dummy + fseek(file, sizeof(uint16_t), SEEK_CUR); + + uint32_t stringSize; + fread(&stringSize, sizeof(uint32_t), 1, file); + + std::vector strings(stringSize); + fread(strings.data(), stringSize, 1, file); + + enum ModelFlags1 : uint8_t + { + DustOcclusionEnabled = 0x80, + SnowOcclusionEnabled = 0x40, + RainOcclusionEnabled = 0x20, + Unknown1 = 0x10, + LightingReflectionEnabled = 0x08, + WavingAnimationDisabled = 0x04, + LightShadowDisabled = 0x02, + ShadowDisabled = 0x01, + }; + + enum ModelFlags2 : uint8_t + { + Unknown2 = 0x80, + BgUvScrollEnabled = 0x40, + EnableForceNonResident = 0x20, + ExtraLodEnabled = 0x10, + ShadowMaskEnabled = 0x08, + ForceLodRangeEnabled = 0x04, + EdgeGeometryEnabled = 0x02, + Unknown3 = 0x01 + }; + + struct ModelHeader { + float radius; + unsigned short meshCount; + unsigned short attributeCount; + unsigned short submeshCount; + unsigned short materialCount; + unsigned short boneCount; + unsigned short boneTableCount; + unsigned short shapeCount; + unsigned short shapeMeshCount; + unsigned short shapeValueCount; + uint8_t lodCount; + + ModelFlags1 flags1; + + unsigned short elementIdCount; + uint8_t terrainShadowMeshCount; + + ModelFlags2 flags2; + + float modelClipOutDistance; + float shadowClipOutDistance; + unsigned short unknown4; + unsigned short terrainShadowSubmeshCount; + + uint8_t unknown5; + + uint8_t bgChangeMaterialIndex; + uint8_t bgCrestChangeMaterialIndex; + uint8_t unknown6; + unsigned short unknown7, unknown8, unknown9; + uint8_t padding[6]; + } modelHeader; + + fread(&modelHeader, sizeof(modelHeader), 1, file); + + fmt::print("mesh count: {}\n", modelHeader.meshCount); + fmt::print("attribute count: {}\n", modelHeader.attributeCount); + + struct ElementId { + unsigned int elementId; + unsigned int parentBoneName; + std::vector translate; + std::vector rotate; + }; + + std::vector elementIds(modelHeader.elementIdCount); + for(int i = 0; i < modelHeader.elementIdCount; i++) { + fread(&elementIds[i].elementId, sizeof(uint32_t), 1, file); + fread(&elementIds[i].parentBoneName, sizeof(uint32_t), 1, file); + + elementIds[i].translate.resize(3); // FIXME: these always seem to be 3, convert to static array? then we could probably fread this all in one go! + elementIds[i].rotate.resize(3); + + fread(elementIds[i].translate.data(), sizeof(float) * 3, 1, file); + fread(elementIds[i].rotate.data(), sizeof(float) * 3, 1, file); + } + + struct Lod { + unsigned short meshIndex; + unsigned short meshCount; + float modelLodRange; + float textureLodRange; + unsigned short waterMeshIndex; + unsigned short waterMeshCount; + unsigned short shadowMeshIndex; + unsigned short shadowMeshCount; + unsigned short terrainShadowMeshIndex; + unsigned short terrainShadowMeshCount; + unsigned short verticalFogMeshIndex; + unsigned short verticalFogMeshCount; + + // unused on win32 according to lumina devs + unsigned int edgeGeometrySize; + unsigned int edgeGeometryDataOffset; + unsigned int polygonCount; + unsigned int unknown1; + unsigned int vertexBufferSize; + unsigned int indexBufferSize; + unsigned int vertexDataOffset; + unsigned int indexDataOffset; + }; + + std::array lods; + fread(lods.data(), sizeof(Lod) * 3, 1, file); + + // TODO: support models that support more than 3 lods + + struct Mesh { + unsigned short vertexCount; + unsigned short padding; + unsigned int indexCount; + unsigned short materialIndex; + unsigned short subMeshIndex; + unsigned short subMeshCount; + unsigned short boneTableIndex; + unsigned int startIndex; + + std::vector vertexBufferOffset; + std::vector vertexBufferStride; + + uint8_t vertexStreamCount; + }; + + std::vector meshes(modelHeader.meshCount); + for(int i = 0; i < modelHeader.meshCount; i++) { + fread(&meshes[i].vertexCount, sizeof(uint16_t), 1, file); + fread(&meshes[i].padding, sizeof(uint16_t), 1, file); + fread(&meshes[i].indexCount, sizeof(uint32_t), 1, file); + fread(&meshes[i].materialIndex, sizeof(uint16_t), 1, file); + fread(&meshes[i].subMeshIndex, sizeof(uint16_t), 1, file); + fread(&meshes[i].subMeshCount, sizeof(uint16_t), 1, file); + fread(&meshes[i].boneTableIndex, sizeof(uint16_t), 1, file); + fread(&meshes[i].startIndex, sizeof(uint32_t), 1, file); + + meshes[i].vertexBufferOffset.resize(3); + fread(meshes[i].vertexBufferOffset.data(), sizeof(uint32_t) * 3, 1, file); + + meshes[i].vertexBufferStride.resize(3); + fread(meshes[i].vertexBufferStride.data(), sizeof(uint8_t) * 3, 1, file); + + fread(&meshes[i].vertexStreamCount, sizeof(uint8_t), 1, file); + } + + std::vector attributeNameOffsets(modelHeader.attributeCount); + fread(attributeNameOffsets.data(), sizeof(uint32_t) * modelHeader.attributeCount, 1, file); + + // TODO: implement terrain shadow meshes + + struct Submesh { + unsigned int indexOffset; + unsigned int indexCount; + unsigned int attributeIndexMask; + unsigned short boneStartIndex; + unsigned short boneCount; + }; + + std::vector submeshes(modelHeader.submeshCount); + for(int i = 0; i < modelHeader.submeshCount; i++) { + fread(&submeshes[i], sizeof(Submesh), 1, file); + } + + // TODO: implement terrain shadow submeshes + + std::vector materialNameOffsets(modelHeader.materialCount); + fread(materialNameOffsets.data(), sizeof(uint32_t) * modelHeader.materialCount, 1, file); + + std::vector boneNameOffsets(modelHeader.boneCount); + fread(boneNameOffsets.data(), sizeof(uint32_t) * modelHeader.boneCount, 1, file); + + struct BoneTable { + std::vector boneIndex; + uint8_t boneCount; + std::vector padding; + }; + + std::vector boneTables(modelHeader.boneTableCount); + for(int i = 0; i < modelHeader.boneTableCount; i++) { + boneTables[i].boneIndex.resize(64); + fread(boneTables[i].boneIndex.data(), 64 * sizeof(uint16_t), 1, file); + fread(&boneTables[i].boneCount, sizeof(uint8_t), 1, file); + boneTables[i].padding.resize(3); + fread(boneTables[i].padding.data(), sizeof(uint8_t) * 3, 1, file); + + fmt::print("bone count: {}\n", boneTables[i].boneCount); + } + + // TODO: implement shapes + + unsigned int submeshBoneMapSize; + fread(&submeshBoneMapSize, sizeof(uint32_t), 1, file); + + std::vector submeshBoneMap((int)submeshBoneMapSize / 2); + fread(submeshBoneMap.data(), submeshBoneMap.size() * sizeof(uint16_t), 1, file); + + uint8_t paddingAmount; + fread(&paddingAmount, sizeof(uint8_t), 1, file); + + fseek(file, paddingAmount, SEEK_CUR); + + struct BoundingBox { + std::array min, max; + }; + + BoundingBox boundingBoxes, modelBoundingBoxes, waterBoundingBoxes, verticalFogBoundingBoxes; + fread(&boundingBoxes, sizeof(BoundingBox), 1, file); + fread(&modelBoundingBoxes, sizeof(BoundingBox), 1, file); + fread(&waterBoundingBoxes, sizeof(BoundingBox), 1, file); + fread(&verticalFogBoundingBoxes, sizeof(BoundingBox), 1, file); + + std::vector boneBoundingBoxes(modelHeader.boneCount); + fread(boneBoundingBoxes.data(), modelHeader.boneCount * sizeof(BoundingBox), 1, file); + + fmt::print("Successfully read mdl file!\n"); + + fmt::print("Now exporting as test.obj...\n"); + + // TODO: doesn't work for lod above 0 + for(int i = 0; i < modelHeader.lodCount; i++) { + for(int j = lods[i].meshIndex; j < (lods[i].meshIndex + lods[i].meshCount); j++) { + std::ofstream out(fmt::format("lod{}_part{}.obj", i, j)); + + const VertexDeclaration decl = vertexDecls[j]; + + std::vector orderedElements = decl.elements; + std::sort(orderedElements.begin(), orderedElements.end(), [](VertexElement a, VertexElement b) { + return a.offset > b.offset; + }); + + enum VertexType : uint8_t { + Single3 = 2, + Single4 = 3, + UInt = 5, + ByteFloat4 = 8, + Half2 = 13, + Half4 = 14 + }; + + enum VertexUsage : uint8_t { + Position = 0, + BlendWeights = 1, + BlendIndices = 2, + Normal = 3, + UV = 4, + Tangent2 = 5, + Tangent1 = 6, + Color = 7, + }; + + int vertexCount = meshes[j].vertexCount; + std::vector vertices(vertexCount); + + for(int k = 0; k < vertexCount; k++) { + for(auto & orderedElement : orderedElements) { + VertexType type = (VertexType)orderedElement.type; + VertexUsage usage = (VertexUsage)orderedElement.usage; + + const int stream = orderedElement.stream; + + fseek(file, lods[i].vertexDataOffset + meshes[j].vertexBufferOffset[stream] + orderedElement.offset + meshes[i].vertexBufferStride[stream] * k, SEEK_SET); + + std::array floatData = {}; + + switch(type) { + case VertexType::Single3: + fread(floatData.data(), sizeof(float) * 3, 1, file); + break; + case VertexType::Single4: + fread(floatData.data(), sizeof(float) * 4, 1, file); + break; + case VertexType::UInt: + fseek(file, sizeof(uint8_t) * 4, SEEK_CUR); + break; + case VertexType::ByteFloat4: + fseek(file, sizeof(uint8_t) * 4, SEEK_CUR); + break; + case VertexType::Half2: + fseek(file, sizeof(uint16_t) * 2, SEEK_CUR); + break; + case VertexType::Half4: + fseek(file, sizeof(uint16_t) * 4, SEEK_CUR); + break; + } + + switch(usage) { + case VertexUsage::Position: + vertices[k].position = floatData; + break; + } + } + + out << "v " << vertices[k].position[0] << " " << vertices[k].position[1] << " " << vertices[k].position[2] << std::endl; + } + + fseek(file, modelFileHeader.indexOffsets[i] + (meshes[j].startIndex * 2), SEEK_SET); + std::vector indices(meshes[j].indexCount); + fread(indices.data(), meshes[j].indexCount * sizeof(uint16_t), 1, file); + + for(int k = 0; k < indices.size(); k += 3) { + unsigned short x = indices[k + 0] + 1; + unsigned short y = indices[k + 1] + 1; + unsigned short z = indices[k + 2] + 1; + + out << "f "; + out << x << "/" << x << "/" << x << " "; + out << y << "/" << y << "/" << y << " "; + out << z << "/" << z << "/" << z << std::endl; + } + + out.close(); + } + } + + return {}; +} \ No newline at end of file