From 0afb688374b8b0fc6e8a157c63b67736e46f142a Mon Sep 17 00:00:00 2001 From: Mordred Date: Thu, 22 Mar 2018 23:01:55 +0100 Subject: [PATCH] Added discovery parser --- CMakeLists.txt | 1 + src/tools/discovery_parser/CMakeLists.txt | 39 ++ src/tools/discovery_parser/README.md | 26 + src/tools/discovery_parser/lgb.h | 363 ++++++++++ src/tools/discovery_parser/main.cpp | 818 ++++++++++++++++++++++ src/tools/discovery_parser/matrix4.h | 100 +++ src/tools/discovery_parser/pcb.h | 92 +++ src/tools/discovery_parser/sgb.h | 213 ++++++ src/tools/discovery_parser/tex.h | 65 ++ src/tools/discovery_parser/tex_decode.h | 250 +++++++ src/tools/discovery_parser/vec3.h | 31 + 11 files changed, 1998 insertions(+) create mode 100644 src/tools/discovery_parser/CMakeLists.txt create mode 100644 src/tools/discovery_parser/README.md create mode 100644 src/tools/discovery_parser/lgb.h create mode 100644 src/tools/discovery_parser/main.cpp create mode 100644 src/tools/discovery_parser/matrix4.h create mode 100644 src/tools/discovery_parser/pcb.h create mode 100644 src/tools/discovery_parser/sgb.h create mode 100644 src/tools/discovery_parser/tex.h create mode 100644 src/tools/discovery_parser/tex_decode.h create mode 100644 src/tools/discovery_parser/vec3.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8fb79881..3ada33ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,5 +51,6 @@ add_subdirectory( "src/tools/exd_common_gen" ) add_subdirectory( "src/tools/exd_struct_gen" ) add_subdirectory( "src/tools/exd_struct_test" ) add_subdirectory( "src/tools/quest_parser" ) +add_subdirectory( "src/tools/discovery_parser" ) #add_subdirectory("src/tools/pcb_reader") #add_subdirectory("src/tools/event_object_parser") diff --git a/src/tools/discovery_parser/CMakeLists.txt b/src/tools/discovery_parser/CMakeLists.txt new file mode 100644 index 00000000..ac6f5744 --- /dev/null +++ b/src/tools/discovery_parser/CMakeLists.txt @@ -0,0 +1,39 @@ +cmake_minimum_required(VERSION 2.6) +cmake_policy(SET CMP0015 NEW) +project(Tool_discovery_parser) + +set(SAPPHIRE_BOOST_VER 1.63.0) +set(SAPPHIRE_BOOST_FOLDER_NAME boost_1_63_0) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../bin/") + + +file(GLOB SERVER_PUBLIC_INCLUDE_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/*" + "${CMAKE_CURRENT_SOURCE_DIR}/BMP-DDS_Converter/*" +) +file(GLOB SERVER_SOURCE_FILES + "${CMAKE_CURRENT_SOURCE_DIR}*.c*" + "${CMAKE_CURRENT_SOURCE_DIR}/BMP-DDS_Converter/*.c*" +) + +#set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../bin/") +add_executable(discovery_parser ${SERVER_PUBLIC_INCLUDE_FILES} ${SERVER_SOURCE_FILES}) + +set_target_properties(discovery_parser PROPERTIES + CXX_STANDARD 14 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS ON + RUNTIME_OUTPUT_DIRECTORY_DEBUG "${CMAKE_CURRENT_SOURCE_DIR}/../bin/" + RUNTIME_OUTPUT_DIRECTORY_RELEASE "${CMAKE_CURRENT_SOURCE_DIR}/../bin/" + RUNTIME_OUTPUT_DIRECTORY_RELWITHDEBINFO "${CMAKE_CURRENT_SOURCE_DIR}/../bin/" + RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL "${CMAKE_CURRENT_SOURCE_DIR}/../bin/" +) + +if (UNIX) + target_link_libraries (discovery_parser common xivdat pthread mysqlclient dl z) +else() + target_link_libraries (discovery_parser common xivdat libmysql zlib1) +endif() + +target_link_libraries(discovery_parser ${Boost_LIBRARIES} ${Boost_LIBRARIES}) + diff --git a/src/tools/discovery_parser/README.md b/src/tools/discovery_parser/README.md new file mode 100644 index 00000000..66fb849f --- /dev/null +++ b/src/tools/discovery_parser/README.md @@ -0,0 +1,26 @@ +collision data exporter for sapphire + +compile with STANDALONE defined to compile without boost and sapphire dependencies + +usage: +- regular + - compile with root sapphire dir cmakelists + - sapphire/src/tools/bin/pcb_reader2 "" +- standalone + - compile main.cpp with STANDALONE defined in build arg + - download ffxivexplorer + - ffxivexplorer > path/to/ffxiv's/game/sqpack/ffxiv/0a0000.dat + - exd/territorytype.exh > `File > Export` and copy `territorytype.exh.csv` from exproted directory to `pcb_reader.exe` directory + - ffxivexplorer > path/to/ffxiv's/game/sqpack/ffxiv/020000.dat + - ctrl click the following: + - `bg/ffxiv/[REGION]/common/collision` + - `bg/ffxiv/[REGION]/[dun|fld|twn|etc..]/common/collision/` + - `bg/ffxiv/[REGION]/[dun|fld|twn|etc..]/collision/` + - `bg/ffxiv/region/shared/[for_bg|for_hou]/` + - `bg/ffxiv/[REGION]/[dun|fld|twn|etc..]/ZONE/level/` + - `bg/ffxiv/[REGION]/[dun|fld|twn|etc..]/ZONE/collision/` + - `bgcommon/world/sys/shared/for_bg/` + and `File > Export Raw` to pcb_reader exe dir (common and shared files are optional but you will be missing a lot of objects if you skip them) + - note: at this time ffxivexplorer is still missing some hashes, though any tool which can export the exds should work fine + - main "" + diff --git a/src/tools/discovery_parser/lgb.h b/src/tools/discovery_parser/lgb.h new file mode 100644 index 00000000..2b6aa003 --- /dev/null +++ b/src/tools/discovery_parser/lgb.h @@ -0,0 +1,363 @@ +#ifndef _LGB_H +#define _LGB_H + +#include +#include +#include +#include +#include +#include +#include + +#include "matrix4.h" +#include "vec3.h" +#include "sgb.h" + +// garbage to skip model loading +extern bool ignoreModels; + +// all credit to +// https://github.com/ufx/SaintCoinach/blob/master/SaintCoinach/Graphics/Lgb/ +// this is simply their work ported to c++ since we dont c# +struct LGB_FILE; +struct LGB_FILE_HEADER; +struct LGB_GROUP; +struct LGB_GROUP_HEADER; + +enum class LgbEntryType : uint32_t +{ + BgParts = 1, + Light = 3, + Vfx = 4, + PositionMarker = 5, + Gimmick = 6, + SharedGroup6 = 6,// secondary variable is set to 2 + Sound = 7, + EventNpc = 8, + BattleNpc = 9, + Aetheryte = 12, + EnvSpace = 13, + Gathering = 14, + SharedGroup15 = 15,// secondary variable is set to 13 + Treasure = 16, + Weapon = 39, + PopRange = 40, + ExitRange = 41, + MapRange = 43, + NaviMeshRange = 44, + EventObject = 45, + EnvLocation = 47, + EventRange = 49, + QuestMarker = 51, + CollisionBox = 57, + DoorRange = 58, + LineVfx = 59, + ClientPath = 65, + ServerPath = 66, + GimmickRange = 67, + TargetMarker = 68, + ChairMarker = 69, + ClickableRange = 70, + PrefetchRange = 71, + FateRange = 72, + SphereCastRange = 75, +}; + +struct LGB_ENTRY_HEADER +{ + LgbEntryType type; + uint32_t unknown; + uint32_t nameOffset; + vec3 translation; + vec3 rotation; + vec3 scale; +}; + +class LGB_ENTRY +{ +public: + char* m_buf; + uint32_t m_offset; + LGB_ENTRY_HEADER header; + + LGB_ENTRY() + { + m_buf = nullptr; + m_offset = 0; + memset( &header, 0, sizeof( header ) ); + }; + LGB_ENTRY( char* buf, uint32_t offset ) + { + m_buf = buf; + m_offset = offset; + header = *reinterpret_cast< LGB_ENTRY_HEADER* >( buf + offset ); + }; + const LgbEntryType getType() const + { + return header.type; + }; + virtual ~LGB_ENTRY() {}; +}; + + +struct LGB_BGPARTS_HEADER : public LGB_ENTRY_HEADER +{ + uint32_t modelFileOffset; + uint32_t collisionFileOffset; + uint32_t unknown4; + uint32_t unknown5; + uint32_t unknown6; + uint32_t unknown7; + uint32_t unknown8; + uint32_t unknown9; +}; + +class LGB_BGPARTS_ENTRY : public LGB_ENTRY +{ +public: + LGB_BGPARTS_HEADER header; + std::string name; + std::string modelFileName; + std::string collisionFileName; + LGB_BGPARTS_ENTRY() {}; + LGB_BGPARTS_ENTRY( char* buf, uint32_t offset ) : LGB_ENTRY( buf, offset ) + { + header = *reinterpret_cast( buf + offset ); + name = std::string( buf + offset + header.nameOffset ); + modelFileName = std::string( buf + offset + header.modelFileOffset ); + collisionFileName = std::string( buf + offset + header.collisionFileOffset ); + }; +}; + +struct LGB_GIMMICK_HEADER : public LGB_ENTRY_HEADER +{ + uint32_t gimmickFileOffset; + char unknownBytes[100]; +}; + +class LGB_GIMMICK_ENTRY : public LGB_ENTRY +{ +public: + LGB_GIMMICK_HEADER header; + std::string name; + std::string gimmickFileName; + + LGB_GIMMICK_ENTRY( char* buf, uint32_t offset ) : LGB_ENTRY( buf, offset ) + { + header = *reinterpret_cast( buf + offset ); + name = std::string( buf + offset + header.nameOffset ); + gimmickFileName = std::string( buf + offset + header.gimmickFileOffset ); + //std::cout << "\t " << gimmickFileName << " unknown: " << header.unknown << "\n"; + }; +}; + +struct LGB_ENPC_HEADER : public LGB_ENTRY_HEADER +{ + uint32_t enpcId; + uint8_t unknown1[0x24]; +}; + +class LGB_ENPC_ENTRY : public LGB_ENTRY +{ +public: + LGB_ENPC_HEADER header; + std::string name; + + LGB_ENPC_ENTRY( char* buf, uint32_t offset ) : LGB_ENTRY( buf, offset ) + { + header = *reinterpret_cast< LGB_ENPC_HEADER* >( buf + offset ); + name = std::string( buf + offset + header.nameOffset ); + //std::cout << "\t ENpc " << header.enpcId << " " << name << "\n"; + }; +}; + +struct LGB_EOBJ_HEADER : public LGB_ENTRY_HEADER +{ + uint32_t eobjId; + uint32_t levelHierachyId; + uint8_t unknown1[0xC]; +}; + +class LGB_EOBJ_ENTRY : public LGB_ENTRY +{ +public: + LGB_EOBJ_HEADER header; + std::string name; + + LGB_EOBJ_ENTRY( char* buf, uint32_t offset ) : LGB_ENTRY( buf, offset ) + { + header = *reinterpret_cast< LGB_EOBJ_HEADER* >( buf + offset ); + //std::cout << "\t " << header.eobjId << " " << name << " unknown: " << header.unknown << "\n"; + name = std::string( buf + offset + header.nameOffset ); + }; +}; + +struct LGB_MAPRANGE_HEADER : public LGB_ENTRY_HEADER +{ + uint32_t type; + uint16_t unknown2; + uint16_t unknown3; + uint8_t unknown4[0x10]; +}; + +struct LGB_MAPRANGE_ENTRY : public LGB_ENTRY +{ +public: + LGB_MAPRANGE_HEADER header; + std::string name; + + LGB_MAPRANGE_ENTRY( char* buf, uint32_t offset ) : LGB_ENTRY( buf, offset ) + { + header = *reinterpret_cast< LGB_MAPRANGE_HEADER* >( buf + offset ); + name = std::string( buf + offset + header.nameOffset ); + }; +}; + +struct LGB_GROUP_HEADER +{ + uint32_t unknown; + int32_t groupNameOffset; + int32_t entriesOffset; + int32_t entryCount; + uint32_t unknown2; + uint32_t unknown3; + uint32_t unknown4; + uint32_t unknown5; + uint32_t unknown6; + uint32_t unknown7; + uint32_t unknown8; + uint32_t unknown9; + uint32_t unknown10; +}; + +struct LGB_GROUP +{ + LGB_FILE* parent; + LGB_GROUP_HEADER header; + std::string name; + std::vector< std::shared_ptr< LGB_ENTRY > > entries; + + LGB_GROUP( char* buf, LGB_FILE* parentStruct, uint32_t offset ) + { + parent = parentStruct; + header = *reinterpret_cast< LGB_GROUP_HEADER* >( buf + offset ); + name = std::string( buf + offset + header.groupNameOffset ); + //entries.resize( header.entryCount ); + //std::cout << name << "\n\t unknown: " << header.unknown << "\n"; + const auto entriesOffset = offset + header.entriesOffset; + for( auto i = 0; i < header.entryCount; ++i ) + { + const auto entryOffset = entriesOffset + *reinterpret_cast< int32_t* >( buf + ( entriesOffset + i * 4 ) ); + + try + { + const auto type = *reinterpret_cast( buf + entryOffset ); + // garbage to skip model loading + if( !ignoreModels && type == LgbEntryType::BgParts ) + { + entries.push_back( std::make_shared< LGB_BGPARTS_ENTRY >( buf, entryOffset ) ); + } + else if( !ignoreModels && type == LgbEntryType::Gimmick ) + { + entries.push_back( std::make_shared< LGB_GIMMICK_ENTRY >( buf, entryOffset ) ); + } + else if( type == LgbEntryType::EventNpc ) + { + entries.push_back( std::make_shared< LGB_ENPC_ENTRY >( buf, entryOffset ) ); + } + else if( type == LgbEntryType::EventObject ) + { + entries.push_back( std::make_shared< LGB_EOBJ_ENTRY >( buf, entryOffset ) ); + } + else if( type == LgbEntryType::MapRange ) + { + entries.push_back( std::make_shared< LGB_MAPRANGE_ENTRY >( buf, entryOffset ) ); + } + /* + else + { + entries[i] = nullptr; + } + */ + + } + catch( std::exception& e ) + { + std::cout << name << " " << e.what() << std::endl; + } + } + }; +}; + +struct LGB_FILE_HEADER +{ + char magic[4]; // LGB 1 + uint32_t fileSize; + uint32_t unknown; + char magic2[4]; // LGP1 + uint32_t unknown2; + uint32_t unknown3; + uint32_t unknown4; + uint32_t unknown5; + int32_t groupCount; +}; + +struct LGB_FILE +{ + LGB_FILE_HEADER header; + std::vector< LGB_GROUP > groups; + std::string name; + + LGB_FILE( char* buf, const std::string& name ) + { + header = *reinterpret_cast< LGB_FILE_HEADER* >( buf ); + if( strncmp( &header.magic[0], "LGB1", 4 ) != 0 || strncmp( &header.magic2[0], "LGP1", 4 ) != 0 ) + throw std::runtime_error( "Invalid LGB file!" ); + + //groups.resize(header.groupCount); + + constexpr auto baseOffset = sizeof( header ); + for( auto i = 0; i < header.groupCount; ++i ) + { + const auto groupOffset = baseOffset + *reinterpret_cast< int32_t* >( buf + ( baseOffset + i * 4 ) ); + const auto group = LGB_GROUP( buf, this, groupOffset ); + groups.push_back( group ); + } + }; +}; + +/* +#if __cplusplus >= 201703L +#include +std::map getLgbFiles( const std::string& dir ) +{ + namespace fs = std::experimental::filesystem; + std::map fileMap; + for( const auto& path : fs::recursive_directory_iterator( dir ) ) + { + if( path.path().extension() == ".lgb" ) + { + const auto& strPath = path.path().string(); + auto f = fopen( strPath.c_str(), "rb" ); + fseek( f, 0, SEEK_END ); + const auto size = ftell( f ); + std::vector bytes( size ); + rewind( f ); + fread( bytes.data(), 1, size, f ); + fclose( f ); + try + { + LGB_FILE lgbFile( bytes.data() ); + fileMap.insert( std::make_pair( strPath, lgbFile ) ); + } + catch( std::exception& e ) + { + std::cout << "Unable to load " << strPath << std::endl; + } + } + } + return fileMap; +} +#endif +*/ +#endif \ No newline at end of file diff --git a/src/tools/discovery_parser/main.cpp b/src/tools/discovery_parser/main.cpp new file mode 100644 index 00000000..73ef43e7 --- /dev/null +++ b/src/tools/discovery_parser/main.cpp @@ -0,0 +1,818 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pcb.h" +#include "lgb.h" +#include "sgb.h" +#include "tex.h" +#include "tex_decode.h" + +//#include "s3tc/s3tc.h" + +#ifndef STANDALONE +#include +#include +#include +#include +#include +#include +#include +#endif + +// garbage to ignore models +bool ignoreModels = false; + +// parsing shit +std::string gamePath( "C:\\SquareEnix\\FINAL FANTASY XIV - A Realm Reborn\\game\\sqpack" ); +std::unordered_map< uint32_t, std::string > eobjNameMap; +std::unordered_map< uint16_t, std::string > zoneNameMap; +std::unordered_map< uint16_t, std::vector< std::pair< uint16_t, std::string > > > zoneInstanceMap; +uint32_t zoneId; + +std::set< std::string > zoneDumpList; + +xiv::dat::GameData* data1 = nullptr; +xiv::exd::ExdData* eData = nullptr; + +void readFileToBuffer( const std::string& path, std::vector< char >& buf ); + + + +// discovery shit +struct vec2 +{ + float x, y; +}; + +struct DiscoveryMap +{ + std::string path; + Image img; + uint16_t mapScale; + int16_t mapOffsetX, mapOffsetY; + int mapId; + constexpr static int discoveryMapRows = 3; + constexpr static int discoveryMapCols = 4; + constexpr static int tileWidth = 128; + constexpr static int tiles = discoveryMapCols * discoveryMapRows; + + uint32_t getColour( uint8_t mapIndex, float x, float y ) + { + auto ogX = x, ogY = y; + int col = mapIndex % (img.width / tileWidth); + int row = mapIndex / (img.width / tileWidth); + x = ( x / 2048.0f ) * (float)tileWidth; + y = ( y / 2048.0f ) * (float)tileWidth; + int tileX = (col * tileWidth) + x; + int tileY = ( row * tileWidth ) + y; + + if (tileX < 0 || tileY < 0) + { + std::cout << "Unable to find tile coord for " << x << " " << y << " mapIndex " << mapIndex << "\n"; + return 0; + } + if( tileY > img.data.size() ) + return 0; + + if( tileX > img.data[0].size() ) + return 0; + + //std::cout << "getColour col " << col << " row " << row << " tileX " << tileX << " tileY " << tileY << " tile index " << std::to_string( mapIndex ) << "\n"; + auto colour = img.data[tileY][tileX]; + + return colour; + } + + vec3 get3dPosFrom2d( float x, float y ) + { + vec3 ret; + float scale2 = mapScale / 100; + ret.x = ( x * scale2 ) + ( img.height * 2 ); //( x / scale2 ) - mapOffsetX; + ret.z = ( y * scale2 ) + ( img.height * 2 ); //( y / scale2 ) - mapOffsetY; + + return ret; + } + + vec2 get2dPosFrom3d( float x, float y ) + { + //int a = (mapPictureBox.Height / 2) + (x / (System.Convert.ToInt32(myMap.sizeFactor) / 100)); + //int b = (mapPictureBox.Height / 2) + (y / (System.Convert.ToInt32(myMap.sizeFactor) / 100)); + + vec2 ret; + float scale2 = mapScale / 100; + ret.x = ( x * scale2 ) + (2048.f /2); + ret.y = ( y * scale2 ) + (2048.f /2); + //ret.x = ( x * scale2 ) + mapOffsetX; + //ret.y = ( y * scale2 ) + mapOffsetY; + + return ret; + } +}; +std::map< uint16_t, DiscoveryMap > discoveryMaps; + + + + +enum class TerritoryTypeExdIndexes : size_t +{ + TerritoryType = 0, + Path = 1 +}; + +using namespace std::chrono_literals; + +struct face +{ + int32_t f1, f2, f3; +}; + +// init +void initExd( const std::string& gamePath ) +{ + data1 = data1 ? data1 : new xiv::dat::GameData( gamePath ); + eData = eData ? eData : new xiv::exd::ExdData( *data1 ); +} + +int parseBlockEntry( char* data, std::vector& entries, int gOff ) +{ + int offset = 0; + bool isgroup = true; + while( isgroup ) + { + PCB_BLOCK_ENTRY block_entry; + memcpy( &block_entry.header, data + offset, sizeof( block_entry.header ) ); + isgroup = block_entry.header.type == 0x30; + + //printf( " BLOCKHEADER_%X: type: %i, group_size: %i\n", gOff + offset, block_entry.header.type, block_entry.header.group_size ); + + if( isgroup ) + { + parseBlockEntry( data + offset + 0x30, entries, gOff + offset ); + offset += block_entry.header.group_size; + } + else + { + /* printf( "\tnum_v16: %i, num_indices: %i, num_vertices: %i\n\n", + block_entry.header.num_v16, block_entry.header.num_indices, block_entry.header.num_vertices );*/ + int doffset = sizeof( block_entry.header ) + offset; + uint16_t block_size = sizeof( block_entry.header ) + + block_entry.header.num_vertices * 3 * 4 + + block_entry.header.num_v16 * 6 + + block_entry.header.num_indices * 6; + + if( block_entry.header.num_vertices != 0 ) + { + block_entry.data.vertices.resize( block_entry.header.num_vertices ); + + int32_t size_vertexbuffer = block_entry.header.num_vertices * 3; + memcpy( &block_entry.data.vertices[0], data + doffset, size_vertexbuffer * 4 ); + doffset += size_vertexbuffer * 4; + } + if( block_entry.header.num_v16 != 0 ) + { + block_entry.data.vertices_i16.resize( block_entry.header.num_v16 ); + int32_t size_unknownbuffer = block_entry.header.num_v16 * 6; + memcpy( &block_entry.data.vertices_i16[0], data + doffset, size_unknownbuffer ); + doffset += block_entry.header.num_v16 * 6; + } + if( block_entry.header.num_indices != 0 ) + { + block_entry.data.indices.resize( block_entry.header.num_indices ); + int32_t size_indexbuffer = block_entry.header.num_indices * 12; + memcpy( &block_entry.data.indices[0], data + doffset, size_indexbuffer ); + doffset += size_indexbuffer; + } + entries.push_back( block_entry ); + } + } + + return 0; +} + +std::string getMapExdEntries( uint32_t mapId ) +{ + static auto& cat = eData->get_category( "Map" ); + static auto& exd = static_cast< xiv::exd::Exd >( cat.get_data_ln( xiv::exd::Language::none ) ); + //static std::unique_ptr< Converter > pConverter = std::make_unique< Converter >(); + + static auto& rows = exd.get_rows(); + for( auto& row : rows ) + { + // fields from SaintCoinach https://github.com/ufx/SaintCoinach/blob/master/SaintCoinach/ex.json#L6358 + auto id = row.first; + if( id != mapId ) + continue; + + auto& fields = row.second; + + /* TYPES !! + case DataType::boolean: 1 + case DataType::int8: 2 + case DataType::uint8: 3 + case DataType::int16: 4 + case DataType::uint16: 5 + case DataType::int32: 6 + case DataType::uint32: 7 + case DataType::float32: 8 + case DataType::uint64: 9 + */ + + auto mapZoneIndex = *boost::get< int8_t >( &fields.at( 2 ) ); + auto hierarchy = *boost::get< uint8_t >( &fields.at( 3 ) ); + auto pathStr = *boost::get< std::string >( &fields.at( 5 ) ); + auto sizeFactor = *boost::get< uint16_t >( &fields.at( 6 ) ); + auto mapOffsetX = *boost::get< int16_t >( &fields.at( 7 ) ); + auto mapOffsetY = *boost::get< int16_t >( &fields.at( 8 ) ); + auto discoveryIdx = *boost::get< int16_t >( &fields.at( 12 ) ); + auto discoveryCompleteBitmask = *boost::get< uint32_t >( &fields.at( 13 ) ); + auto territory = *boost::get< uint16_t >( &fields.at( 14 ) ); + char texStr[255]; + auto teriStr = pathStr.substr( 0, pathStr.find_first_of( '/' ) ); + sprintf( &texStr[0], "ui/map/%s/%s%02Xd.tex", pathStr.c_str(), teriStr.c_str(), mapZoneIndex ); + + + if( discoveryMaps.find( territory ) == discoveryMaps.end() ) + { + auto texFile = data1->getFile( &texStr[0] ); + std::string rawTexFile( teriStr + "0" + std::to_string( mapZoneIndex ) ); + texFile->exportToFile( rawTexFile + "d.tex" ); + auto tex = TEX_FILE( rawTexFile + "d.tex" ); + + int mipMapDivide = 1; + int h = tex.header.uncompressedHeight; + int w = tex.header.uncompressedWidth; + DiscoveryMap discoveryMap; + discoveryMap.img = DecodeTexDXT1( tex, tex.header.mipMaps[0], h / mipMapDivide, w / mipMapDivide, + ( h / mipMapDivide ) / 4, ( w / mipMapDivide ) / 4 + ); + + discoveryMap.img.toFile( rawTexFile + ".img" ); + discoveryMap.mapId = id; + discoveryMap.path = &texStr[0]; + discoveryMap.mapOffsetX = mapOffsetX; + discoveryMap.mapOffsetY = mapOffsetY; + discoveryMap.mapScale = sizeFactor; + + std::cout << "Image Height: " << discoveryMap.img.height << " Width: " << discoveryMap.img.width << "\n"; + + discoveryMaps.emplace( territory, discoveryMap ); + } + return std::string( std::to_string( mapZoneIndex ) + ", " + std::to_string( hierarchy ) + ", " + "\"" + std::string( &texStr[0] ) + "\", " + + std::to_string( discoveryIdx ) + ", " + std::to_string( discoveryCompleteBitmask ) ); + } + return ""; +} + +void dumpLevelExdEntries( uint32_t zoneId, const std::string& name = std::string() ) +{ + static auto& cat = eData->get_category( "Level" ); + static auto& exd = static_cast< xiv::exd::Exd >( cat.get_data_ln( xiv::exd::Language::none ) ); + + std::string fileName( name + "_" + std::to_string( zoneId ) + "_Level" + ".csv" ); + std::ofstream outfile( fileName, std::ios::trunc ); + std::cout << "[Info] Writing level.exd entries to " << fileName << "\n"; + if( outfile.good() ) + { + outfile.close(); + outfile.open( fileName, std::ios::app ); + static std::string levelHeader( "id, objectid, mapid, x, y, z, yaw, radius, type, zone, " ); + static std::string header( levelHeader + "mapZoneIdx, hierarchy, path, size, discoveryIdx, discoveryCompleteBitmask \n" ); + outfile.write( header.c_str(), header.size() ); + + static auto& rows = exd.get_rows(); + for( auto& row : rows ) + { + auto id = row.first; + auto& fields = row.second; + auto x = *boost::get< float >( &fields.at( 0 ) ); + auto y = *boost::get< float >( &fields.at( 1 ) ); + auto z = *boost::get< float >( &fields.at( 2 ) ); + auto yaw = *boost::get< float >( &fields.at( 3 ) ); + auto radius = *boost::get< float >( &fields.at( 4 ) ); + auto type = *boost::get< uint8_t >( &fields.at( 5 ) ); + auto objectid = *boost::get< uint32_t >( &fields.at( 6 ) ); + auto mapid = *boost::get< uint16_t >( &fields.at( 7 ) ); + auto zone = *boost::get< uint16_t >( &fields.at( 9 ) ); + + if( zone == zoneId ) + { + std::string outStr( + std::to_string( id ) + ", " + std::to_string( objectid ) + ", " + std::to_string( mapid ) + ", " + + std::to_string( x ) + ", " + std::to_string( y ) + ", " + std::to_string( z ) + ", " + + std::to_string( yaw ) + ", " + std::to_string( radius ) + ", " + std::to_string( type ) + ", " + std::to_string( zone ) + ", " + + getMapExdEntries( mapid ) + "\n" + ); + outfile.write( outStr.c_str(), outStr.size() ); + } + } + } +} + +std::string zoneNameToPath( const std::string& name ) +{ + std::string path; + bool found = false; + +#ifdef STANDALONE + auto inFile = std::ifstream( "territorytype.exh.csv" ); + if( inFile.good() ) + { + std::string line; + std::regex re( "(\\d+),\"(.*)\",\"(.*)\",.*" ); + while( std::getline( inFile, line ) ) + { + std::smatch match; + if( std::regex_match( line, match, re ) + { + auto tmpId = std::stoul( match[1].str() ); + if( !found && name == match[2].str() ) + { + zoneId = tmpId; + path = match[3].str(); + found = true; + } + zoneNameMap[tmpId] = match[2].str(); + } + } + inFile.close(); + } +#else + + static auto& cat = eData->get_category( "TerritoryType" ); + static auto& exd = static_cast< xiv::exd::Exd >( cat.get_data_ln( xiv::exd::Language::none ) ); + static auto& rows = exd.get_rows(); + for( auto& row : rows ) + { + auto& fields = row.second; + auto teriName = *boost::get< std::string >( &fields.at( static_cast< size_t >( TerritoryTypeExdIndexes::TerritoryType ) ) ); + if( teriName.empty() ) + continue; + auto teriPath = *boost::get< std::string >( &fields.at( static_cast< size_t >( TerritoryTypeExdIndexes::Path ) ) ); + if( !found && boost::iequals( name, teriName ) ) + { + path = teriPath; + found = true; + zoneId = row.first; + } + zoneNameMap[row.first] = teriName; + } +#endif + + if( found ) + { + //path = path.substr( path.find_first_of( "/" ) + 1, path.size() - path.find_first_of( "/" )); + //path = std::string( "ffxiv/" ) + path; + path = std::string( "bg/" ) + path.substr( 0, path.find( "/level/" ) ); + std::cout << "[Info] " << "Found path for " << name << ": " << path << std::endl; + } + else + { + throw std::runtime_error( "Unable to find path for " + name + + ".\n\tPlease double check spelling or open 0a0000.win32.index with FFXIV Explorer and extract territorytype.exh as CSV\n\tand copy territorytype.exh.csv into pcb_reader.exe directory if using standalone" ); + } + + return path; +} + +void loadEobjNames() +{ + auto& cat = eData->get_category( "EObjName" ); + auto& exd = static_cast< xiv::exd::Exd >( cat.get_data_ln( xiv::exd::Language::en ) ); + for( auto& row : exd.get_rows() ) + { + auto id = row.first; + auto& fields = row.second; + auto name = *boost::get< std::string >( &fields.at( 0 ) ); + eobjNameMap[id] = name; + } +} + +void writeEobjEntry( std::ofstream& out, LGB_ENTRY* pObj ) +{ + static std::string mapRangeStr( "\"MapRange\", " ); + static std::ofstream discoverySql( "discovery.sql" , std::ios::app ); + uint32_t id; + uint32_t unknown = 0, unknown2 = 0; + std::string name; + std::string typeStr; + uint32_t eobjlevelHierachyId = 0; + + auto pMapRange = reinterpret_cast< LGB_MAPRANGE_ENTRY* >( pObj ); + id = pMapRange->header.unknown; + unknown = pMapRange->header.unknown2; + unknown2 = pMapRange->header.unknown3; + typeStr = mapRangeStr; + + // discovery shit + vec2 pos; + auto subArea = -1; + auto mapId = -1; + + bool found = false; + auto it = discoveryMaps.find( zoneId ); + if( it != discoveryMaps.end() ) + { + auto map = it->second; + pos = map.get2dPosFrom3d( pObj->header.translation.x, pObj->header.translation.z ); + mapId = map.mapId; + + //std::cout << "3d coords " << pObj->header.translation.x << " " << pObj->header.translation.z << "\n"; + //std::cout << "2d coords " << pos.x << " " << pos.y << "\n"; + for( auto i = 0; i < map.tiles; ++i ) + { + auto colour = map.getColour( i, pos.x, pos.y ); + + auto r = ( colour >> 16 ) & 0xFF; + auto g = ( colour >> 8 ) & 0xFF; + auto b = ( colour >> 0 ) & 0xFF; + + //std::cout << "R " << r << " G " << g << " B " << b << "\n"; + + if( ( found = ( r != 0 || g != 0 || b != 0 ) ) ) + { + if( r != 0 ) + { + // out of bounds + if( i == 0 ) + break; + subArea = i * 3 + 1; + } + else if( g != 0 ) + { + subArea = i * 3 + 2; + } + else if( b != 0 ) + { + subArea = i * 3 + 3; + } + break; + } + } + } + subArea--; + + if( subArea == -2 || mapId == -1 ) + { + if( it != discoveryMaps.end() ) + { + auto map = it->second; + pos = map.get2dPosFrom3d( pObj->header.translation.x, pObj->header.translation.z ); + mapId = map.mapId; + + //std::cout << "3d coords " << pObj->header.translation.x << " " << pObj->header.translation.z << "\n"; + //std::cout << "2d coords " << pos.x << " " << pos.y << "\n"; + for( auto i = 0; i < map.tiles; ++i ) + { + auto colour = map.getColour( i, pos.x, pos.y ); + + auto r = ( colour >> 16 ) & 0xFF; + auto g = ( colour >> 8 ) & 0xFF; + auto b = ( colour >> 0 ) & 0xFF; + + //std::cout << "R " << r << " G " << g << " B " << b << "\n"; + + if( ( found = ( r != 0 || g != 0 || b != 0 ) ) ) + { + if( r != 0 ) + { + // out of bounds + if( i == 0 ) + break; + //subArea = i * 3 + 1; + } + else if( g != 0 ) + { + // subArea = i * 3 + 2; + } + else if( b != 0 ) + { + // subArea = i * 3 + 3; + } + break; + } + } + } + std::cout << "\tUnable to find subarea for maprange " << std::to_string( id ) << "\n"; + return; + } + std::string outStr( "INSERT INTO discoveryinfo VALUES (" + + std::to_string( id ) + ", " + std::to_string( mapId ) + ", " + std::to_string( subArea ) + ");\n" + //std::to_string( pObj->header.translation.x ) + ", " + std::to_string( pObj->header.translation.y ) + ", " + std::to_string( pObj->header.translation.z ) + + //", " + std::to_string( subArea ) + "" + "\n" + ); + discoverySql.write( outStr.c_str(), outStr.size() ); + //out.write( outStr.c_str(), outStr.size() ); +} + +void readFileToBuffer( const std::string& path, std::vector< char >& buf ) +{ + auto inFile = std::ifstream( path, std::ios::binary ); + if( inFile.good() ) + { + inFile.seekg( 0, inFile.end ); + int32_t fileSize = (int32_t)inFile.tellg(); + buf.resize( fileSize ); + inFile.seekg( 0, inFile.beg ); + inFile.read( &buf[0], fileSize ); + inFile.close(); + } + else + { + throw std::runtime_error( "Unable to open " + path ); + } +} + +int main( int argc, char* argv[] ) +{ + auto startTime = std::chrono::system_clock::now(); + auto entryStartTime = std::chrono::system_clock::now(); + + std::vector< std::string > argVec( argv + 1, argv + argc ); + // todo: support expansions + std::string zoneName = "f1f1"; + + bool dumpAll = ignoreModels = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ){ return arg == "--dump-all"; } ) != argVec.end(); + dumpAll = true; + ignoreModels = false; + if( argc > 1 ) + { + zoneName = argv[1]; + if( argc > 2 ) + { + std::string tmpPath( argv[2] ); + if( !tmpPath.empty() ) + gamePath = argv[2]; + } + } + + initExd( gamePath ); + std::ofstream discoverySql( "discovery.sql", std::ios::trunc ); + discoverySql.close(); + + if( dumpAll ) + { + zoneNameToPath( "f1t1" ); + + for( const auto& zone : zoneNameMap ) + zoneDumpList.emplace( zone.second ); + } + else + { + zoneDumpList.emplace( zoneName ); + } + +LABEL_DUMP: + entryStartTime = std::chrono::system_clock::now(); + zoneName = *zoneDumpList.begin(); + try + { + const auto& zonePath = zoneNameToPath( zoneName ); + + std::string listPcbPath( zonePath + "/collision/list.pcb" ); + std::string bgLgbPath( zonePath + "/level/bg.lgb" ); + std::string planmapLgbPath( zonePath + "/level/planmap.lgb" ); + std::string collisionFilePath( zonePath + "/collision/" ); + std::vector< char > section; + std::vector< char > section1; + std::vector< char > section2; + +#ifndef STANDALONE + const xiv::dat::Cat& test = data1->getCategory( "bg" ); + + auto test_file = data1->getFile( bgLgbPath ); + section = test_file->access_data_sections().at( 0 ); + + auto planmap_file = data1->getFile( planmapLgbPath ); + section2 = planmap_file->access_data_sections().at( 0 ); + + auto test_file1 = data1->getFile( listPcbPath ); + section1 = test_file1->access_data_sections().at( 0 ); +#else + { + readFileToBuffer( bgLgbPath, section ); + readFileToBuffer( listPcbPath, section1 ); + } +#endif + + std::vector< std::string > stringList; + + uint32_t offset1 = 0x20; + + loadEobjNames(); + dumpLevelExdEntries( zoneId, zoneName ); + std::string eobjFileName( zoneName + "_eobj.csv" ); + std::ofstream eobjOut( eobjFileName, std::ios::trunc ); + if( !eobjOut.good() ) + throw std::string( "Unable to create " + zoneName + "_eobj.csv for eobj entries. Run as admin or check there isnt already a handle on the file." ).c_str(); + + eobjOut.close(); + eobjOut.open( eobjFileName, std::ios::app ); + + if( !eobjOut.good() ) + throw std::string( "Unable to create " + zoneName + "_eobj.csv for eobj entries. Run as admin or check there isnt already a handle on the file." ).c_str(); + + if( 0 ) + { + for( ; ; ) + { + + uint16_t trId = *(uint16_t*)§ion1[offset1]; + + char someString[200]; + sprintf( someString, "%str%04d.pcb", collisionFilePath.c_str(), trId ); + stringList.push_back( std::string( someString ) ); + //std::cout << someString << "\n"; + offset1 += 0x20; + + if( offset1 >= section1.size() ) + { + break; + } + } + } + LGB_FILE bgLgb( §ion[0], "bg" ); + LGB_FILE planmapLgb( §ion2[0], "planmap" ); + + std::vector< LGB_FILE > lgbList { bgLgb, planmapLgb }; + uint32_t max_index = 0; + + // dont bother if we cant write to a file + FILE* fp_out = nullptr; + //auto fp_out = ignoreModels ? ( FILE* )nullptr : fopen( ( zoneName + ".obj" ).c_str(), "w" ); + if( fp_out ) + { + fprintf( fp_out, "\n" ); + fclose( fp_out ); + } + else if( /*!ignoreModels*/ false ) + { + std::string errorMessage( "Cannot create " + zoneName + ".obj\n" + + " Check no programs have a handle to file and run as admin.\n" ); + std::cout << errorMessage; + throw std::runtime_error( errorMessage.c_str() ); + return 0; + } + + + { + std::map< std::string, PCB_FILE > pcbFiles; + std::map< std::string, SGB_FILE > sgbFiles; + std::map< std::string, uint32_t > objCount; + auto loadPcbFile = [&]( const std::string& fileName ) -> bool + { + if( ignoreModels ) + return false; + try + { + if( fileName.find( '.' ) == std::string::npos ) + return false; + else if( fileName.substr( fileName.find_last_of( '.' ) ) != ".pcb" ) + throw std::runtime_error( "Not a PCB file." ); + + char* dataSection = nullptr; + //std::cout << fileName << " "; +#ifndef STANDALONE + auto file = data1->getFile( fileName ); + auto sections = file->get_data_sections(); + dataSection = §ions.at( 0 )[0]; +#else + std::vector< char > buf; + readFileToBuffer( fileName, buf ); + dataSection = &buf[0]; +#endif + //std::cout << sections.size() << "\n"; + + uint32_t offset = 0; + PCB_FILE pcb_file; + memcpy( &pcb_file.header, &dataSection[0], sizeof( pcb_file.header ) ); + offset += sizeof( pcb_file.header ); + pcb_file.entries.resize( pcb_file.header.num_entries ); + bool isgroup = true; + while( isgroup ) + { + PCB_BLOCK_ENTRY block_entry; + memcpy( &block_entry.header, &dataSection[0] + offset, sizeof( block_entry.header ) ); + isgroup = block_entry.header.type == 0x30; + + //printf( "BLOCKHEADER_%X: type: %i, group_size: %i\n", offset, block_entry.header.type, block_entry.header.group_size ); + // + if( isgroup ) + { + parseBlockEntry( &dataSection[0] + offset + 0x30, pcb_file.entries, offset ); + offset += block_entry.header.group_size; + } + else + { + parseBlockEntry( &dataSection[0] + offset, pcb_file.entries, offset ); + } + } + pcbFiles.insert( std::make_pair( fileName, pcb_file ) ); + return true; + } + catch( std::exception& e ) + { + std::cout << "[Error] " << "Unable to load collision mesh " << fileName << "\n\tError:\n\t" << e.what() << "\n"; + return false; + } + }; + + auto loadSgbFile = [&]( const std::string& fileName ) -> bool + { + SGB_FILE sgbFile; + try + { + char* dataSection = nullptr; + //std::cout << fileName << " "; +#ifndef STANDALONE + auto file = data1->getFile( fileName ); + auto sections = file->get_data_sections(); + dataSection = §ions.at( 0 )[0]; +#else + std::vector< char > buf; + readFileToBuffer( fileName, buf ); + dataSection = &buf[0]; +#endif + sgbFile = SGB_FILE( &dataSection[0] ); + sgbFiles.insert( std::make_pair( fileName, sgbFile ) ); + return true; + } + catch( std::exception& e ) + { + std::cout << "[Error] " << "Unable to load SGB " << fileName << "\n\tError:\n\t" << e.what() << "\n"; + sgbFiles.insert( std::make_pair( fileName, sgbFile ) ); + } + return false; + }; + + { + for( const auto& fileName : stringList ) + { + loadPcbFile( fileName ); + } + } + + std::cout << "[Info] " << ( ignoreModels ? "Dumping MapRange and EObj" : "Writing obj file " ) << "\n"; + uint32_t totalGroups = 0; + uint32_t totalGroupEntries = 0; + + for( const auto& lgb : lgbList ) + { + for( const auto& group : lgb.groups ) + { + //std::cout << "\t" << group.name << " Size " << group.header.entryCount << "\n"; + totalGroups++; + for( const auto& pEntry : group.entries ) + { + auto pGimmick = dynamic_cast< LGB_GIMMICK_ENTRY* >( pEntry.get() ); + auto pBgParts = dynamic_cast< LGB_BGPARTS_ENTRY* >( pEntry.get() ); + + std::string fileName( "" ); + fileName.resize( 256 ); + totalGroupEntries++; + + if( pEntry->getType() == LgbEntryType::MapRange ) + { + writeEobjEntry( eobjOut, pEntry.get() ); + } + } + } + } + std::cout << "[Info] " << "Loaded " << pcbFiles.size() << " PCB Files \n"; + std::cout << "[Info] " << "Total Groups " << totalGroups << " Total entries " << totalGroupEntries << "\n"; + } + std::cout << "[Success] " << "Exported " << zoneName << " in " << + std::chrono::duration_cast< std::chrono::seconds >( std::chrono::system_clock::now() - entryStartTime ).count() << " seconds\n"; + } + catch( std::exception& e ) + { + std::cout << "[Error] " << e.what() << std::endl; + std::cout << "[Error] " << "Unable to extract collision data.\n\tIf using standalone ensure your working directory folder layout is \n\tbg/[ffxiv|ex1|ex2]/teri/type/zone/[level|collision]" << std::endl; + std::cout << std::endl; + std::cout << "[Info] " << "Usage: pcb_reader2 territory \"path/to/game/sqpack/ffxiv\" " << std::endl; + } + std::cout << "\n\n\n"; + LABEL_NEXT_ZONE_ENTRY: + zoneDumpList.erase( zoneName ); + if( !zoneDumpList.empty() ) + goto LABEL_DUMP; + + std::cout << "\n\n\n[Success] Finished all tasks in " << + std::chrono::duration_cast< std::chrono::seconds >( std::chrono::system_clock::now() - startTime ).count() << " seconds\n"; + + getchar(); + + if( eData ) + delete eData; + if( data1 ) + delete data1; + return 0; +} diff --git a/src/tools/discovery_parser/matrix4.h b/src/tools/discovery_parser/matrix4.h new file mode 100644 index 00000000..d02d2c84 --- /dev/null +++ b/src/tools/discovery_parser/matrix4.h @@ -0,0 +1,100 @@ +#ifndef _MATRIX4_H +#define _MATRIX4_H + +#include +#include + +// https://github.com/jpd002/Play--Framework/tree/master/include/math +struct matrix4 +{ + // 4x4 + float grid[16]; + matrix4() + { + memset( &grid[0], 0, sizeof( grid ) ); + } + + float operator()( int row, int col ) const + { + return grid[(row * 4) + col]; + } + + float& operator()( int row, int col ) + { + return grid[(row * 4) + col]; + } + static matrix4 rotateX( float angle ) + { + matrix4 ret = matrix4(); + ret(0, 0) = 1.000000000f; + ret(1, 1) = cos(angle); + ret(1, 2) = -sin(angle); + ret(2, 1) = sin(angle); + ret(2, 2) = cos(angle); + ret(3, 3) = 1.000000000f; + return ret; + } + + static matrix4 rotateY( float angle ) + { + matrix4 ret = matrix4(); + ret(0, 0) = cos(angle); + ret(0, 2) = sin(angle); + ret(1, 1) = 1.000000000f; + ret(2, 0) = -sin(angle); + ret(2, 2) = cos(angle); + ret(3, 3) = 1.000000000f; + return ret; + } + + static matrix4 rotateZ( float angle ) + { + matrix4 ret = matrix4(); + ret(0, 0) = cos(angle); + ret(0, 1) = -sin(angle); + ret(1, 0) = sin(angle); + ret(1, 1) = cos(angle); + ret(2, 2) = 1.000000000f; + ret(3, 3) = 1.000000000f; + return ret; + } + + static matrix4 scale( float x, float y, float z ) + { + matrix4 ret = matrix4(); + ret(0, 0) = x; + ret(1, 1) = y; + ret(2, 2) = z; + ret(3, 3) = 1; + + return ret; + } + + static matrix4 translate( float x, float y, float z ) + { + matrix4 ret = matrix4(); + ret(0, 0) = 1; + ret(1, 1) = 1; + ret(2, 2) = 1; + ret(3, 3) = 1; + + ret(3, 0) = x; + ret(3, 1) = y; + ret(3, 2) = z; + return ret; + } + + matrix4 operator *( const matrix4& rhs ) const + { + matrix4 ret; + for( unsigned int i = 0; i < 4; i++ ) + { + ret( i, 0 ) = (*this)(i, 0) * rhs( 0, 0 ) + (*this)(i, 1) * rhs( 1, 0 ) + (*this)(i, 2) * rhs( 2, 0 ) + (*this)(i, 3) * rhs( 3, 0 ); + ret( i, 1 ) = (*this)(i, 0) * rhs( 0, 1 ) + (*this)(i, 1) * rhs( 1, 1 ) + (*this)(i, 2) * rhs( 2, 1 ) + (*this)(i, 3) * rhs( 3, 1 ); + ret( i, 2 ) = (*this)(i, 0) * rhs( 0, 2 ) + (*this)(i, 1) * rhs( 1, 2 ) + (*this)(i, 2) * rhs( 2, 2 ) + (*this)(i, 3) * rhs( 3, 2 ); + ret( i, 3 ) = (*this)(i, 0) * rhs( 0, 3 ) + (*this)(i, 1) * rhs( 1, 3 ) + (*this)(i, 2) * rhs( 2, 3 ) + (*this)(i, 3) * rhs( 3, 3 ); + } + return ret; + } +}; +#endif diff --git a/src/tools/discovery_parser/pcb.h b/src/tools/discovery_parser/pcb.h new file mode 100644 index 00000000..4b775d84 --- /dev/null +++ b/src/tools/discovery_parser/pcb.h @@ -0,0 +1,92 @@ +#ifndef _PCB_H +#define _PCB_H + +#include +#include + +struct PCB_HEADER +{ + uint32_t unknown_1; + uint32_t unknown_2; + uint32_t num_entries; // count starts at 0 + uint32_t total_indices; + uint64_t padding; +}; + +struct PCB_BLOCK_HEADER +{ + uint32_t type; // 0 for entry, 0x30 for group + uint32_t group_size; // when group size in bytes for the group block + // bounding box + float x; + float y; + float z; + float x1; + float y1; + float z1; + // number of vertices packed into 16 bit + uint16_t num_v16; + // number of indices + uint16_t num_indices; + // number of normal floar vertices + uint32_t num_vertices; +}; + +struct PCB_VERTEXDATA +{ + float x; + float y; + float z; +}; + +struct PCB_INDEXDATA +{ + uint8_t index[3]; + uint8_t unknown[3]; + uint8_t unknown1[6]; +}; + +struct PCB_VERTEXDATAI16 +{ + uint16_t x; + uint16_t y; + uint16_t z; +}; + +struct PCB_BLOCK_DATA +{ + std::vector< PCB_VERTEXDATA > vertices; + std::vector< PCB_VERTEXDATAI16 > vertices_i16; + std::vector< PCB_INDEXDATA > indices; +}; + +struct PCB_BLOCK_ENTRY +{ + PCB_BLOCK_HEADER header; + PCB_BLOCK_DATA data; +}; + +struct PCB_FILE +{ + PCB_HEADER header; + std::vector< PCB_BLOCK_ENTRY > entries; +}; + +struct PCB_LIST_ENTRY +{ + uint32_t id; + float x, y, z, x2, y2, z2, rot; +}; + +struct PCB_LIST_BASE_ENTRY +{ + float x, y, z, x2, y2, z2, rot; +}; + +struct PCB_LIST_FILE +{ + uint32_t count; + PCB_LIST_BASE_ENTRY entry; + std::vector entries; +}; +#endif \ No newline at end of file diff --git a/src/tools/discovery_parser/sgb.h b/src/tools/discovery_parser/sgb.h new file mode 100644 index 00000000..2aa675aa --- /dev/null +++ b/src/tools/discovery_parser/sgb.h @@ -0,0 +1,213 @@ +#ifndef _SGB_H +#define _SGB_H + +#include +#include +#include +#include +#include +#include +#include + +#include "vec3.h" + +// garbage to skip model loading +extern bool ignoreModels; + +// +// ported from https://github.com/ufx/SaintCoinach/blob/master/SaintCoinach/Graphics/Sgb/SgbDataType.cs + +struct SGB_FILE; +struct SGB_HEADER; +struct SGB_MODEL_ENTRY; +struct SGB_MODEL_HEADER; +struct SGB_GROUP; +struct SGB_GROUP_HEADER; + + +enum SgbDataType : uint32_t +{ + Unknown0008 = 0x0008, + Group = 0x0100, +}; + +enum SgbGroupEntryType : uint32_t +{ + Model = 0x01, +}; + +struct SGB_GROUP_HEADER +{ + SgbDataType type; + int32_t nameOffset; + uint32_t unknown08; + uint32_t unknown0C; + + uint32_t unknown10; + uint32_t unknown14; + uint32_t unknown18; + uint32_t unknown1C; + + int32_t entryCount; + uint32_t unknown24; + uint32_t unknown28; + uint32_t unknown2C; + + uint32_t unknown30; + uint32_t unknown34; + uint32_t unknown38; + uint32_t unknown3C; + + uint32_t unknown40; + uint32_t unknown44; +}; + +struct SGB_GROUP_ENTRY +{ +public: + char* m_buf; + uint32_t m_offset; + + SGB_GROUP_ENTRY() + { + m_buf = nullptr; + m_offset = 0; + }; + SGB_GROUP_ENTRY( char* buf, uint32_t offset ) + { + m_buf = buf; + m_offset = offset; + }; + virtual ~SGB_GROUP_ENTRY() {}; +}; + +struct SGB_ENTRY_HEADER +{ + SgbGroupEntryType type; + uint32_t unknown2; + int32_t nameOffset; + vec3 translation; + vec3 rotation; + vec3 scale; +}; + +struct SGB_MODEL_HEADER : public SGB_ENTRY_HEADER +{ + int32_t modelFileOffset; + int32_t collisionFileOffset; +}; + +struct SGB_MODEL_ENTRY : public SGB_GROUP_ENTRY +{ + SGB_MODEL_HEADER header; + SgbGroupEntryType type; + std::string name; + std::string modelFileName; + std::string collisionFileName; + + SGB_MODEL_ENTRY( char* buf, uint32_t offset ) + { + header = *reinterpret_cast< SGB_MODEL_HEADER* >( buf + offset ); + name = std::string( buf + offset + header.nameOffset ); + modelFileName = std::string( buf + offset + header.modelFileOffset ); + collisionFileName = std::string( buf + offset + header.collisionFileOffset ); + } +}; + +struct SGB_GROUP +{ + SGB_GROUP_HEADER header; + std::string name; + SGB_FILE* parent; + std::vector< std::shared_ptr< SGB_GROUP_ENTRY > > entries; + + SGB_GROUP( char* buf, SGB_FILE* file, uint32_t fileSize, uint32_t offset ) + { + parent = file; + header = *reinterpret_cast< SGB_GROUP_HEADER* >( buf + offset ); + name = std::string( buf + offset + header.nameOffset ); + + auto entriesOffset = offset + sizeof( header ); + + for( auto i = 0; i < header.entryCount; ++i ) + { + auto entryOffset = entriesOffset + *reinterpret_cast< uint32_t* >( buf + ( entriesOffset + ( i * 4 ) ) ); + if( entryOffset > fileSize ) + throw std::runtime_error( "SGB_GROUP entry offset was larger than SGB file size!" ); + auto type = *reinterpret_cast< uint32_t* >( buf + entryOffset ); + if( type == SgbGroupEntryType::Model && !ignoreModels ) + { + entries.push_back( std::make_shared< SGB_MODEL_ENTRY >( buf, entryOffset ) ); + } + else + { + // std::cout << "\t\tUnknown SGB entry! Group: " << name << " type: " << type << " index: " << i << " entryOffset: " << entryOffset << "\n"; + } + } + } +}; + +struct SGB_HEADER +{ + char magic[4]; // SGB1 + uint32_t fileSize; + uint32_t unknown1; + char magic2[4]; // SCN1 + + uint32_t unknown10; + int32_t sharedOffset; + uint32_t unknown18; + int32_t offset1C; + + uint32_t unknown20; + uint32_t unknown24; + uint32_t unknown28; + uint32_t unknown2C; + + uint32_t unknown30; + uint32_t unknown34; + uint32_t unknown38; + uint32_t unknown3C; + + uint32_t unknown40; + uint32_t unknown44; + uint32_t unknown48; + uint32_t unknown4C; + + uint32_t unknown50; + uint32_t unknown54; +}; + +struct SGB_FILE +{ + SGB_HEADER header; + std::vector entries; + + SGB_FILE() + { + memset( &header, 0, sizeof( header ) ); + } + SGB_FILE( char* buf ) + { + constexpr int baseOffset = 0x14; + header = *reinterpret_cast< SGB_HEADER* >( buf ); + + if( strncmp( &header.magic[0], "SGB1", 4 ) != 0 || strncmp( &header.magic2[0], "SCN1", 4 ) != 0 ) + throw std::runtime_error( "Unable to load SGB File!" ); + + try + { + auto group = SGB_GROUP( buf, this, header.fileSize, baseOffset + header.sharedOffset ); + entries.push_back( group ); + auto group2 = SGB_GROUP( buf, this, header.fileSize, baseOffset + header.offset1C ); + entries.push_back( group2 ); + } + catch( std::exception& e ) + { + std::cout << e.what() << "\n"; + } + }; +}; + + +#endif // !_SGB_H diff --git a/src/tools/discovery_parser/tex.h b/src/tools/discovery_parser/tex.h new file mode 100644 index 00000000..79a6e310 --- /dev/null +++ b/src/tools/discovery_parser/tex.h @@ -0,0 +1,65 @@ +#ifndef _TEX_H +#define _TEX_H + +#include +#include +#include +#include +#include +#include + +// ported Ioncannon's tex file parsing +// https://bitbucket.org/Ioncannon/ffxiv-explorer/src/9330429c1540cf35f947fe7321739d857f4f31a7/src/com/fragmenterworks/ffxivextract/models/Texture_File.java + +struct TEX_HEADER +{ + uint32_t unknown; // 00 + uint16_t compressionType; // 04 + uint16_t unknown2; // 06 + uint16_t uncompressedWidth; // 08 + uint16_t uncompressedHeight; // 0A + uint16_t unknown5; // 0C + uint16_t numMipMaps; // 0F + uint16_t unknown4[0x0B]; // 11 - 0x1C + std::vector< uint32_t > mipMaps; +}; + +struct TEX_FILE +{ + std::string name; + TEX_HEADER header; + std::vector< char > data; + + TEX_FILE( const std::string& path ) + { + name = path; + std::ifstream in( path, std::ios::binary ); + if ( in.good() ) + { + std::size_t size = 0; + in.seekg( 0, in.end ); + size = in.tellg(); + in.seekg( 0, in.beg ); + data.resize( size ); + in.read( &data[0], size ); + + header = *reinterpret_cast< TEX_HEADER* >( &data[0] ); + + header.mipMaps.clear(); + + for( auto i = 0; i < header.numMipMaps; ++i) + header.mipMaps.push_back( *reinterpret_cast< uint32_t* >( &data[0x1C + ( i * 4 )] ) ); + + std::cout << path << "\n\tcompressionType " << header.compressionType << " uncompressedWidth " << + header.uncompressedWidth << " uncompressedHeight " << header.uncompressedHeight << " numMipMaps " << + header.numMipMaps << " mipMaps " << header.mipMaps.size() << "\n"; + } + else + { + std::string errorStr( "Unable to open " + path ); + throw std::runtime_error( errorStr.c_str() ); + } + } +}; + +#endif \ No newline at end of file diff --git a/src/tools/discovery_parser/tex_decode.h b/src/tools/discovery_parser/tex_decode.h new file mode 100644 index 00000000..f746dcc4 --- /dev/null +++ b/src/tools/discovery_parser/tex_decode.h @@ -0,0 +1,250 @@ +#ifndef _TEX_DECODE_H +#define _TEX_DECODE_H + +#include +#include + +#include "tex.h" + +// all credit to Ioncannon +// copied/pasted from https://bitbucket.org/Ioncannon/ffxiv-explorer/src/9330429c1540cf35f947fe7321739d857f4f31a7/src/com/fragmenterworks/ffxivextract/helpers/ImageDecoding.java + +struct Colour +{ + uint8_t r, g, b, a; + + Colour( int r, int g, int b, int a ) + { + this->r = r; + this->g = g; + this->b = b; + this->a = a; + } + + uint32_t getRGB() + { + return ( a << 24 ) | ( r << 16 ) | ( g << 8 ) | ( b << 0 ); + } +}; + +struct ImageHeader +{ + uint32_t height; + uint32_t width; +}; + +struct Image +{ + uint32_t height, width; + std::vector< std::vector< uint32_t > > data; + + Image() + { + + } + + Image( int height, int width ) + { + this->height = height; + this->width = width; + } + + Image( char* buf ) + { + height = *reinterpret_cast< uint32_t* >( buf + 0 ); + width = *reinterpret_cast< uint32_t* >( buf + 4 ); + data.resize( *reinterpret_cast< uint32_t* >( buf + 8 ) ); + + auto offset = 12; + for( auto y = 0; y < data.size(); ++y ) + { + auto row = *reinterpret_cast< uint32_t* >( buf + offset ); + auto entries = *reinterpret_cast< uint32_t* >( buf + offset + 4 ); + data[y].resize( entries ); + offset += 8; + + for( auto x = 0; x < entries; ++x ) + data[y][x] = *reinterpret_cast< uint32_t* >( buf + offset + ( x * 4 ) ); + offset += entries * 4; + } + } + + void toFile( const std::string& path ) + { + std::ofstream out( path.c_str(), std::ios::trunc ); + out.close(); + out.open( path, std::ios::binary | std::ios::app ); + + out.write( reinterpret_cast< char* >( &height ), 4 ); // 0 + out.write( reinterpret_cast< char* >( &width ), 4 ); // 4 + + auto size = data.size(); + out.write( reinterpret_cast< char* >( &size ), 4 ); // 8 + + for( auto y = 0; y < data.size(); ++y ) + { + out.write( reinterpret_cast< char* >( &y ), 4 ); + auto entries = data[y].size(); + out.write( reinterpret_cast< char* >( &entries ), 4); + for( auto x = 0; x < data[y].size(); ++x ) + { + out.write( reinterpret_cast< char* >( &data[y][x] ), 4 ); + } + } + out.close(); + } + + void setRGB( unsigned int x, unsigned int y, uint32_t colour ) + { + if( data.size() <= y ) + data.resize( y + 1 ); + if( data[y].size() <= x ) + data[y].resize( x + 1 ); + data[y][x] = colour; + } + +}; + + +void DecompressBlockDTX1( int x, const int y, const int width, const int color0, const int color1, const int txl1, const int txl2, Image& img ) +{ + float temp = ((color0 >> 11) * 255.0f) + 16.0f; + const float r0 = (((temp / 32.0f) + temp) / 32.0f); + temp = (((color0 & 0x07E0) >> 5) * 255.0f) + 32.0f; + const float g0 = (((temp / 64.0f) + temp) / 64.0f); + temp = ((color0 & 0x001F) * 255.0f) + 16.0f; + const float b0 = (((temp / 32.0f) + temp) / 32.0f); + temp = ((color1 >> 11) * 255.0f) + 16.0f; + const float r1 = (((temp / 32.0f) + temp) / 32.0f); + temp = (((color1 & 0x07E0) >> 5) * 255.0f) + 32.0f; + const float g1 = (((temp / 64.0f) + temp) / 64.0f); + temp = ((color1 & 0x001F) * 255.0f) + 16.0f; + const float b1 = (((temp / 32.0f) + temp) / 32.0f); + for (int j = 0; j < 4; j++) { + for (int i = 0; i < 4; i++) { + // Color FinalColor; + const int d = (4 * j) + i; + int positionCode; + if ((d * 2) >= 16) { + positionCode = (txl2 >> ((d * 2) % 16)) & 0x03; + } else { + positionCode = (txl1 >> (d * 2)) & 0x03; + } + float fr, fg, fb, fa; + if (color0 > color1) { + switch (positionCode) { + case 0: { + fr = r0; + fg = g0; + fb = b0; + fa = 0; + break; + } + case 1: { + fr = r1; + fg = g1; + fb = b1; + fa = 0; + break; + } + case 2: { + fr = ((2.0f * (float) r0) + (float) r1) / 3.0f; + fg = ((2.0f * (float) g0) + (float) g1) / 3.0f; + fb = ((2.0f * (float) b0) + (float) b1) / 3.0f; + fa = 0; + break; + } + case 3: { + fr = ((float) r0 + (2.0f * (float) r1)) / 3.0f; + fg = ((float) g0 + (2.0f * (float) g1)) / 3.0f; + fb = ((float) b0 + (2.0f * (float) b1)) / 3.0f; + fa = 0; + break; + } + default: { + fr = 0; + fg = 0; + fb = 0; + fa = 0; + } + } + } else { + switch (positionCode) { + case 0: { + fr = r0; + fg = g0; + fb = b0; + fa = 0xff; + break; + } + case 1: { + fr = r1; + fg = g1; + fb = b1; + fa = 0xff; + break; + } + case 2: { + fr = ((float) r0 + (float) r1) / 2.0f; + fg = ((float) g0 + (float) g1) / 2.0f; + fb = ((float) b0 + (float) b1) / 2.0f; + fa = 0xff; + break; + } + case 3: { + fr = 0; + fg = 0; + fb = 0; + fa = 0xff; + break; + } + default: { + fr = 0; + fg = 0; + fb = 0; + fa = 0; + } + } + } + if ((x + i) < width) { + int alpha = 0; + if ((fr == fg) && (fr == fb) && (fr == 0)) { + alpha = 0xff; + } + img.setRGB(x + i, y + j, Colour((int) fr, (int) fg, (int) fb, 255 - alpha).getRGB()); + } + } + } +} + +Image DecodeTexDXT1( const TEX_FILE& tex, uint32_t offset, uint32_t targetHeight, uint32_t targetWidth, + uint32_t compressedHeight, uint32_t compressedWidth ) +{ + if( offset > tex.data.size() ) + throw std::runtime_error( "Unable to decode TEX file " + tex.name + ": offset too large" ); + else if( tex.data.size() < ( ( targetHeight * targetWidth ) / 2 ) ) + throw std::runtime_error( "Unable to decode TEX file " + tex.name + ": data too small" ); + + std::vector< char > ret; + auto data = tex.data.data() + offset; + int pos = 0; + + Image img( targetHeight, targetWidth ); + + for( int y = 0; y < compressedHeight; y++ ) + { + for( int x = 0; x < compressedWidth; x++ ) + { + const int t0 = *reinterpret_cast< const uint16_t* >( data + pos + 0 ) & 0xffff; + const int t1 = *reinterpret_cast< const uint16_t* >( data + pos + 2 ) & 0xffff; + const int t2 = *reinterpret_cast< const uint16_t* >( data + pos + 4 ) & 0xffff; + const int t3 = *reinterpret_cast< const uint16_t* >( data + pos + 6 ) & 0xffff; + + pos += 8; + DecompressBlockDTX1( x * 4, y * 4, targetWidth, t0, t1, t2, t3, img ); + } + } + return img; +} + +#endif \ No newline at end of file diff --git a/src/tools/discovery_parser/vec3.h b/src/tools/discovery_parser/vec3.h new file mode 100644 index 00000000..a8fdfbd1 --- /dev/null +++ b/src/tools/discovery_parser/vec3.h @@ -0,0 +1,31 @@ +#ifndef _VEC3_H +#define _VEC3_H + +#include +#include "matrix4.h" + +struct vec3 +{ + float x, y, z; + vec3() + { + x = 0.0f; + y = 0.0f; + z = 0.0f; + } + vec3(float x, float y, float z) + { + this->x = x; + this->y = y; + this->z = z; + }; +}; +static vec3 operator *(const vec3& lhs, const matrix4& rhs) +{ + vec3 ret; + ret.x = rhs(0, 0) * lhs.x + rhs(0, 1) * lhs.y + rhs(0, 2) * lhs.z; + ret.y = rhs(1, 0) * lhs.x + rhs(1, 1) * lhs.y + rhs(1, 2) * lhs.z; + ret.z = rhs(2, 0) * lhs.x + rhs(2, 1) * lhs.y + rhs(2, 2) * lhs.z; + return ret; +}; +#endif \ No newline at end of file