From ba6ed64982bb8a7619908f0a18028fd9ebfb3092 Mon Sep 17 00:00:00 2001 From: Tahir Akhlaq Date: Sat, 19 Jan 2019 18:46:21 +0000 Subject: [PATCH 1/4] pcb_reader things because lets use two threads and keep one locked for no reason --- src/tools/pcb_reader/lgb.h | 2 +- src/tools/pcb_reader/main.cpp | 256 +++++++++++++++++++++------------- src/tools/pcb_reader/sgb.h | 2 +- 3 files changed, 164 insertions(+), 96 deletions(-) diff --git a/src/tools/pcb_reader/lgb.h b/src/tools/pcb_reader/lgb.h index 902d89c3..7b30682e 100644 --- a/src/tools/pcb_reader/lgb.h +++ b/src/tools/pcb_reader/lgb.h @@ -14,7 +14,7 @@ #include "sgb.h" // garbage to skip model loading -extern bool ignoreModels; +extern bool noObj; // all credit to // https://github.com/ufx/SaintCoinach/blob/master/SaintCoinach/Graphics/Lgb/ diff --git a/src/tools/pcb_reader/main.cpp b/src/tools/pcb_reader/main.cpp index a361951d..e56d1ff1 100644 --- a/src/tools/pcb_reader/main.cpp +++ b/src/tools/pcb_reader/main.cpp @@ -25,8 +25,10 @@ #include #include +#include + // garbage to ignore models -bool ignoreModels = false; +bool noObj = false; std::string gamePath( "C:\\SquareEnix\\FINAL FANTASY XIV - A Realm Reborn\\game\\sqpack" ); std::unordered_map< uint16_t, std::string > zoneNameMap; @@ -37,6 +39,7 @@ std::set< std::string > zoneDumpList; xiv::dat::GameData* data1 = nullptr; xiv::exd::ExdData* eData = nullptr; + enum class TerritoryTypeExdIndexes : size_t { @@ -173,11 +176,15 @@ void readFileToBuffer( const std::string& path, std::vector< char >& buf ) int main( int argc, char* argv[] ) { - auto startTime = std::chrono::system_clock::now(); - auto entryStartTime = std::chrono::system_clock::now(); + auto startTime = std::chrono::high_resolution_clock::now(); + auto entryStartTime = std::chrono::high_resolution_clock::now(); + + std::condition_variable cv; std::vector< std::string > argVec( argv + 1, argv + argc ); std::string zoneName = "r2t2"; + noObj = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ) + { return arg == "--no-obj"; } ) != argVec.end(); bool dumpAllZones = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ) { return arg == "--dump-all"; } ) != argVec.end(); bool generateNavmesh = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ) @@ -210,29 +217,58 @@ int main( int argc, char* argv[] ) std::mutex navmeshMutex; std::queue< std::string > exportedGroups; + // todo: +#ifdef WIN32 std::string exportArg( "RecastDemo.exe --type tileMesh --obj " ); - std::thread navmeshThread( [&navmeshMutex, &exportedGroups, &exportArg, &generateNavmesh]() +#else + std::string exportArg( "./RecastDemo --type tileMesh --obj "); +#endif + + std::thread navmeshThread( [&navmeshMutex, &exportedGroups, &exportArg, &generateNavmesh, &cv]() { while( generateNavmesh ) { - std::this_thread::sleep_for( std::chrono::milliseconds( 1 ) ); - std::lock_guard< std::mutex > lock( navmeshMutex ); - { - if( !exportedGroups.empty() ) + std::string currFile; + + std::unique_lock lk( navmeshMutex ); + while( exportedGroups.empty() ) { - auto& currFile = exportedGroups.front(); - std::cout << "\nGenerating navmesh for " << currFile << std::endl; - system( ( exportArg + currFile ).c_str() ); - std::cout << "\nFinished generating navmesh for " << currFile << std::endl; + cv.wait( lk ); + } + + //if( !exportedGroups.empty() ) + { + currFile = exportedGroups.front(); exportedGroups.pop(); } + + if( !currFile.empty() ) + { + std::error_code e; + if( std::experimental::filesystem::exists( currFile ) && std::experimental::filesystem::file_size( currFile, e ) > 1024 ) + { + std::string generateMessage( "\nGenerating navmesh for " + currFile + "\n" ); + std::cout << generateMessage << std::endl; + + auto start = std::chrono::high_resolution_clock::now(); + + system( ( exportArg + currFile ).c_str() ); + + std::string finishMessage( "\nFinished generating navmesh for " + currFile + " in " + + std::to_string( std::chrono::duration_cast< std::chrono::seconds >( std::chrono::high_resolution_clock::now() - start ).count() ) + " seconds\n" ); + std::cout << finishMessage << std::endl; + } + else + { + std::cout << ( std::string( "Unable to load OBJ file for " ) + currFile + "\n" ) << std::endl; + } } } }); navmeshThread.detach(); LABEL_DUMP: - entryStartTime = std::chrono::system_clock::now(); + entryStartTime = std::chrono::high_resolution_clock::now(); zoneName = *zoneDumpList.begin(); try { @@ -286,13 +322,13 @@ int main( int argc, char* argv[] ) uint32_t max_index = 0; // dont bother if we cant write to a file - auto fp_out = ignoreModels ? ( FILE* ) nullptr : fopen( ( zoneName + ".obj" ).c_str(), "w" ); + auto fp_out = noObj ? ( FILE* ) nullptr : fopen( ( zoneName + ".obj" ).c_str(), "w" ); if( fp_out ) { fprintf( fp_out, "\n" ); fclose( fp_out ); } - else if( !ignoreModels ) + else if( !noObj ) { std::string errorMessage( "Cannot create " + zoneName + ".obj\n" + " Check no programs have a handle to file and run as admin.\n" ); @@ -301,15 +337,16 @@ int main( int argc, char* argv[] ) return 0; } - if( ignoreModels || ( fp_out = fopen( ( zoneName + ".obj" ).c_str(), "ab+" ) ) ) + if( noObj || ( fp_out = fopen( ( zoneName + ".obj" ).c_str(), "ab+" ) ) ) { std::map< std::string, PCB_FILE > pcbFiles; std::map< std::string, SGB_FILE > sgbFiles; std::map< std::string, uint32_t > objCount; auto loadPcbFile = [ & ]( const std::string& fileName )->bool { - if( ignoreModels ) + if( noObj ) return false; + try { if( fileName.find( '.' ) == std::string::npos ) @@ -388,6 +425,9 @@ int main( int argc, char* argv[] ) const vec3* translation = nullptr, const SGB_MODEL_ENTRY* pSgbEntry = nullptr ) { + if( noObj ) + return; + char name2[0x100]; memset( name2, 0, 0x100 ); sprintf( &name2[ 0 ], "%s_%u", &name[ 0 ], objCount[ name ]++ ); @@ -465,10 +505,13 @@ int main( int argc, char* argv[] ) } }; - for( const auto& fileName : stringList ) + if( !noObj ) { - loadPcbFile( fileName ); - writeToFile( pcbFiles[ fileName ], fileName, zoneName ); + for( const auto& fileName : stringList ) + { + loadPcbFile( fileName ); + writeToFile( pcbFiles[ fileName ], fileName, zoneName ); + } } std::cout << "[Info] " << "Writing obj file " << "\n"; @@ -481,99 +524,113 @@ int main( int argc, char* argv[] ) { max_index = 0; std::string outfile_name( zoneName + "_" + group.name + ".obj" ); - - fp_out = fopen( outfile_name.c_str(), "w" ); - if( fp_out ) - { - fprintf( fp_out, "" ); - fclose( fp_out ); - fp_out = fopen( outfile_name.c_str(), "ab+" ); - } - //std::cout << "\t" << group.name << " Size " << group.header.entryCount << "\n"; totalGroups++; - for( const auto& pEntry : group.entries ) + + if( !noObj ) { - auto pGimmick = dynamic_cast< LGB_GIMMICK_ENTRY* >( pEntry.get() ); - auto pBgParts = dynamic_cast< LGB_BGPARTS_ENTRY* >( pEntry.get() ); - - std::string fileName( "" ); - fileName.resize( 256 ); - totalGroupEntries++; - - // write files - auto writeOutput = [ & ]( const std::string& fileName, const vec3* scale, const vec3* rotation, - const vec3* translation, const SGB_MODEL_ENTRY* pModel = nullptr )->bool + fp_out = fopen( outfile_name.c_str(), "w" ); + if( fp_out ) { - { - const auto& it = pcbFiles.find( fileName ); - if( it == pcbFiles.end() ) - { - if( fileName.empty() || !loadPcbFile( fileName ) ) - return false; - //std::cout << "\t\tLoaded PCB File " << pBgParts->collisionFileName << "\n"; - } - } - const auto& it = pcbFiles.find( fileName ); - if( it != pcbFiles.end() ) - { - const auto& pcb_file = it->second; - writeToFile( pcb_file, fileName, group.name, scale, rotation, translation, pModel ); - } - return true; - }; - - if( pBgParts ) - { - fileName = pBgParts->collisionFileName; - writeOutput( fileName, &pBgParts->header.scale, &pBgParts->header.rotation, - &pBgParts->header.translation ); + // blank otherwise recast tries to load them.. + fprintf( fp_out, "" ); + fclose( fp_out ); + fp_out = fopen( outfile_name.c_str(), "ab+" ); } - - // gimmick entry - if( pGimmick ) + + //std::cout << "\t" << group.name << " Size " << group.header.entryCount << "\n"; + for( const auto& pEntry : group.entries ) { + std::string fileName( "" ); + fileName.resize( 256 ); + totalGroupEntries++; + + // write files + auto writeOutput = [ & ]( const std::string& fileName, const vec3* scale, const vec3* rotation, + const vec3* translation, const SGB_MODEL_ENTRY* pModel = nullptr )->bool { - const auto& it = sgbFiles.find( pGimmick->gimmickFileName ); - if( it == sgbFiles.end() ) { - // std::cout << "\tGIMMICK:\n\t\t" << pGimmick->gimmickFileName << "\n"; - loadSgbFile( pGimmick->gimmickFileName ); - } - } - const auto& it = sgbFiles.find( pGimmick->gimmickFileName ); - if( it != sgbFiles.end() ) - { - const auto& sgbFile = it->second; - for( const auto& group : sgbFile.entries ) - { - for( const auto& pEntry : group.entries ) + const auto& it = pcbFiles.find( fileName ); + if( it == pcbFiles.end() ) { - auto pModel = dynamic_cast< SGB_MODEL_ENTRY* >( pEntry.get() ); - fileName = pModel->collisionFileName; - writeOutput( fileName, &pGimmick->header.scale, &pGimmick->header.rotation, - &pGimmick->header.translation, pModel ); + if( fileName.empty() || !loadPcbFile( fileName ) ) + return false; + //std::cout << "\t\tLoaded PCB File " << pBgParts->collisionFileName << "\n"; } } + const auto& it = pcbFiles.find( fileName ); + if( it != pcbFiles.end() ) + { + const auto& pcb_file = it->second; + writeToFile( pcb_file, fileName, group.name, scale, rotation, translation, pModel ); + } + return true; + }; + + switch( pEntry->getType() ) + { + case LgbEntryType::BgParts: + { + auto pBgParts = static_cast< LGB_BGPARTS_ENTRY* >( pEntry.get() ); + fileName = pBgParts->collisionFileName; + writeOutput( fileName, &pBgParts->header.scale, &pBgParts->header.rotation, + &pBgParts->header.translation ); + } + break; + + // gimmick entry + case LgbEntryType::Gimmick: + { + auto pGimmick = static_cast< LGB_GIMMICK_ENTRY* >( pEntry.get() ); + { + const auto& it = sgbFiles.find( pGimmick->gimmickFileName ); + if( it == sgbFiles.end() ) + { + // std::cout << "\tGIMMICK:\n\t\t" << pGimmick->gimmickFileName << "\n"; + loadSgbFile( pGimmick->gimmickFileName ); + } + } + const auto& it = sgbFiles.find( pGimmick->gimmickFileName ); + if( it != sgbFiles.end() ) + { + const auto& sgbFile = it->second; + for( const auto& group : sgbFile.entries ) + { + for( const auto& pEntry : group.entries ) + { + auto pModel = dynamic_cast< SGB_MODEL_ENTRY* >( pEntry.get() ); + fileName = pModel->collisionFileName; + writeOutput( fileName, &pGimmick->header.scale, &pGimmick->header.rotation, + &pGimmick->header.translation, pModel ); + } + } + } + } + + case LgbEntryType::EventObject: + { + writeOutput( fileName, &pEntry->header.scale, &pEntry->header.rotation, &pEntry->header.translation ); + } + break; } } - - if( pEntry->getType() == LgbEntryType::EventObject ) - { - writeOutput( fileName, &pEntry->header.scale, &pEntry->header.rotation, &pEntry->header.translation ); - } } - if( fp_out ) - fclose( fp_out ); - std::lock_guard< std::mutex > lock( navmeshMutex ); - exportedGroups.push( outfile_name ); + if( generateNavmesh ) + { + if( fp_out ) + fclose( fp_out ); + std::unique_lock lock( navmeshMutex ); + exportedGroups.push( outfile_name ); + cv.notify_one(); + } } } std::cout << "[Info] " << "Loaded " << pcbFiles.size() << " PCB Files \n"; std::cout << "[Info] " << "Total Groups " << totalGroups << " Total entries " << totalGroupEntries << "\n"; } + std::cout << "[Success] " << "Exported " << zoneName << " in " << std::chrono::duration_cast< std::chrono::seconds >( - std::chrono::system_clock::now() - entryStartTime ).count() << " seconds\n"; + std::chrono::high_resolution_clock::now() - entryStartTime ).count() << " seconds\n"; } catch( std::exception& e ) { @@ -587,11 +644,22 @@ int main( int argc, char* argv[] ) std::cout << "\n\n\n"; LABEL_NEXT_ZONE_ENTRY: zoneDumpList.erase( zoneName ); + if( !zoneDumpList.empty() ) goto LABEL_DUMP; + while( 1 ) + { + std::lock_guard< std::mutex > lock( navmeshMutex ); + if( exportedGroups.empty() ) + generateNavmesh = false; + + if( navmeshThread.joinable() ) + navmeshThread.join(); + std::this_thread::sleep_for( 1s ); + } std::cout << "\n\n\n[Success] Finished all tasks in " << - std::chrono::duration_cast< std::chrono::seconds >( std::chrono::system_clock::now() - startTime ).count() + std::chrono::duration_cast< std::chrono::seconds >( std::chrono::high_resolution_clock::now() - startTime ).count() << " seconds\n"; getchar(); diff --git a/src/tools/pcb_reader/sgb.h b/src/tools/pcb_reader/sgb.h index c6bbee1e..c6972a84 100644 --- a/src/tools/pcb_reader/sgb.h +++ b/src/tools/pcb_reader/sgb.h @@ -12,7 +12,7 @@ #include "vec3.h" // garbage to skip model loading -extern bool ignoreModels; +extern bool noObj; // // ported from https://github.com/ufx/SaintCoinach/blob/master/SaintCoinach/Graphics/Sgb/SgbDataType.cs From 59725025e7b384266a1e7d61315747021fb8c9d9 Mon Sep 17 00:00:00 2001 From: Tahir Akhlaq Date: Sun, 20 Jan 2019 17:47:39 +0000 Subject: [PATCH 2/4] refactored pcb_reader - now uses a shitty threadpool which never terminates apparently - todo: fix indices --- src/tools/pcb_reader/cache.h | 112 ++++ src/tools/pcb_reader/exporter.h | 68 +++ src/tools/pcb_reader/exportmgr.h | 41 ++ src/tools/pcb_reader/lgb.h | 39 +- src/tools/pcb_reader/main.cpp | 659 ++++++++---------------- src/tools/pcb_reader/navmesh_exporter.h | 44 ++ src/tools/pcb_reader/obj_exporter.h | 98 ++++ src/tools/pcb_reader/pcb.h | 84 +++ src/tools/pcb_reader/threadpool.h | 91 ++++ 9 files changed, 752 insertions(+), 484 deletions(-) create mode 100644 src/tools/pcb_reader/cache.h create mode 100644 src/tools/pcb_reader/exporter.h create mode 100644 src/tools/pcb_reader/exportmgr.h create mode 100644 src/tools/pcb_reader/navmesh_exporter.h create mode 100644 src/tools/pcb_reader/obj_exporter.h create mode 100644 src/tools/pcb_reader/threadpool.h diff --git a/src/tools/pcb_reader/cache.h b/src/tools/pcb_reader/cache.h new file mode 100644 index 00000000..771ef2dc --- /dev/null +++ b/src/tools/pcb_reader/cache.h @@ -0,0 +1,112 @@ +#ifndef CACHE_H +#define CACHE_H + +#include +#include +#include +#include +#include + +#include "pcb.h" +#include "lgb.h" +#include "sgb.h" + +#include +#include +#include + +class Cache : public std::enable_shared_from_this< Cache > +{ +public: + Cache( xiv::dat::GameData* pData ) + { + if( !pData ) + throw std::runtime_error( "Unable to initialise cache without game data" ); + m_pData = pData; + } + ~Cache(){} + + std::shared_ptr< SGB_FILE > getSgbFile( const std::string& filepath ) + { + std::scoped_lock lock( m_mutex ); + + auto it = m_sgbCache.find( filepath ); + if( it != m_sgbCache.end() ) + return it->second; + + auto pFile = loadFile< SGB_FILE >( filepath ); + m_sgbCache[ filepath ] = pFile; + return pFile; + } + + std::shared_ptr< LGB_FILE > getLgbFile( const std::string& filepath ) + { + std::scoped_lock lock( m_mutex ); + + auto it = m_lgbCache.find( filepath ); + if( it != m_lgbCache.end() ) + return it->second; + + auto pFile = loadFile< LGB_FILE >( filepath ); + m_lgbCache[ filepath ] = pFile; + return pFile; + } + + std::shared_ptr< PCB_FILE > getPcbFile( const std::string& filepath ) + { + std::scoped_lock lock( m_mutex ); + + auto it = m_pcbCache.find( filepath ); + if( it != m_pcbCache.end() ) + return it->second; + + auto pFile = loadFile< PCB_FILE >( filepath ); + m_pcbCache[ filepath ] = pFile; + return pFile; + } + + +private: + template< typename T > + std::shared_ptr< T > loadFile( const std::string& filepath ) + { + auto buf = getFileBuffer( filepath ); + if( !buf.empty() ) + { + try + { + return std::make_shared< T >( &buf[0] ); + } + catch( std::exception& e ) + { + std::string err( filepath + " " + e.what() ); + std::cout << err << std::endl; + } + } + return nullptr; + } + + std::vector< char > getFileBuffer( const std::string& filepath ) + { + try + { + //std::cout << fileName << " \n"; + auto pFile = m_pData->getFile( filepath ); + auto& sections = pFile->get_data_sections(); + auto& section = sections.at( 0 ); + return section; + } + catch( std::exception& e ) + { + std::vector< char > empty; + return empty; + } + } + std::mutex m_mutex; + xiv::dat::GameData* m_pData; + std::map< std::string, std::shared_ptr< LGB_FILE > > m_lgbCache; + std::map< std::string, std::shared_ptr< SGB_FILE > > m_sgbCache; + std::map< std::string, std::shared_ptr< PCB_FILE > > m_pcbCache; +}; + +#endif \ No newline at end of file diff --git a/src/tools/pcb_reader/exporter.h b/src/tools/pcb_reader/exporter.h new file mode 100644 index 00000000..c0b8a252 --- /dev/null +++ b/src/tools/pcb_reader/exporter.h @@ -0,0 +1,68 @@ +#ifndef EXPORTER_H +#define EXPORTER_H + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "threadpool.h" + +enum ExportFileType : int +{ + WavefrontObj = 0x01, + Navmesh = 0x02, +}; + +enum ExportSplitType +{ + None, + SplitByGroup, + SingleZone +}; + +struct ExportedMesh +{ + std::vector< float > verts; + std::vector< int > indices; +}; + +struct ExportedModel +{ + std::string name; + std::vector< ExportedMesh > meshes; +}; + +struct ExportedGroup +{ + std::string name; + std::map< std::string, ExportedModel > models; +}; + +struct ExportedZone +{ + std::string name; + std::map< std::string, ExportedGroup > groups; +}; + +class Exporter +{ +public: + Exporter() { } + ~Exporter(){} + + virtual void exportZone( const ExportedZone& zone ) = 0; + virtual void exportGroup( const ExportedGroup& group ) = 0; + +protected: + ExportFileType m_exportFileType; +}; + +#endif \ No newline at end of file diff --git a/src/tools/pcb_reader/exportmgr.h b/src/tools/pcb_reader/exportmgr.h new file mode 100644 index 00000000..7673dc34 --- /dev/null +++ b/src/tools/pcb_reader/exportmgr.h @@ -0,0 +1,41 @@ +#ifndef EXPORTMGR_H +#define EXPORTMGR_H + +#include "exporter.h" +#include "obj_exporter.h" +#include "threadpool.h" + +class ExportMgr +{ +public: + ExportMgr(){} + ~ExportMgr() + { + waitForTasks(); + } + + void exportZone(const ExportedZone& zone, ExportFileType exportFileTypes) + { + if( exportFileTypes & ExportFileType::WavefrontObj ) + { + m_threadpool.queue( [zone](){ ObjExporter::exportZone( zone ); } ); + } + } + + void exportGroup( const std::string& zoneName, const ExportedGroup& group, ExportFileType exportFileTypes ) + { + if( exportFileTypes & ExportFileType::WavefrontObj ) + { + m_threadpool.queue( [zoneName, group](){ ObjExporter::exportGroup( zoneName, group ); } ); + } + } + + void waitForTasks() + { + m_threadpool.complete(); + } +private: + ThreadPool m_threadpool; +}; + +#endif \ No newline at end of file diff --git a/src/tools/pcb_reader/lgb.h b/src/tools/pcb_reader/lgb.h index 7b30682e..bf2a3157 100644 --- a/src/tools/pcb_reader/lgb.h +++ b/src/tools/pcb_reader/lgb.h @@ -318,9 +318,8 @@ struct LGB_FILE { LGB_FILE_HEADER header; std::vector< LGB_GROUP > groups; - std::string name; - - LGB_FILE( char* buf, const std::string& name ) + + LGB_FILE( char* buf ) { header = *reinterpret_cast< LGB_FILE_HEADER* >( buf ); if( strncmp( &header.magic[ 0 ], "LGB1", 4 ) != 0 || strncmp( &header.magic2[ 0 ], "LGP1", 4 ) != 0 ) @@ -338,38 +337,4 @@ struct LGB_FILE }; }; -/* -#if __cplusplus >= 201703L -#include -std::map getLgbFiles( const std::string& dir ) -{ - namespace fs = std::experimental::filesystem; - std::map fileMap; - for( const auto& path : fs::recursive_directory_iterator( dir ) ) - { - if( path.path().extension() == ".lgb" ) - { - const auto& strPath = path.path().string(); - auto f = fopen( strPath.c_str(), "rb" ); - fseek( f, 0, SEEK_END ); - const auto size = ftell( f ); - std::vector bytes( size ); - rewind( f ); - fread( bytes.data(), 1, size, f ); - fclose( f ); - try - { - LGB_FILE lgbFile( bytes.data() ); - fileMap.insert( std::make_pair( strPath, lgbFile ) ); - } - catch( std::exception& e ) - { - std::cout << "Unable to load " << strPath << std::endl; - } - } - } - return fileMap; -} -#endif -*/ #endif \ No newline at end of file diff --git a/src/tools/pcb_reader/main.cpp b/src/tools/pcb_reader/main.cpp index e56d1ff1..dcc498ba 100644 --- a/src/tools/pcb_reader/main.cpp +++ b/src/tools/pcb_reader/main.cpp @@ -14,6 +14,10 @@ #include #include +#include "exporter.h" +#include "exportmgr.h" + +#include "cache.h" #include "pcb.h" #include "lgb.h" #include "sgb.h" @@ -27,6 +31,10 @@ #include +#include +#include + + // garbage to ignore models bool noObj = false; @@ -36,6 +44,8 @@ uint32_t zoneId; std::set< std::string > zoneDumpList; +std::shared_ptr< Cache > pCache; + xiv::dat::GameData* data1 = nullptr; xiv::exd::ExdData* eData = nullptr; @@ -49,72 +59,13 @@ enum class TerritoryTypeExdIndexes : using namespace std::chrono_literals; -struct face -{ - int32_t f1, f2, f3; -}; - void initExd( const std::string& gamePath ) { data1 = data1 ? data1 : new xiv::dat::GameData( gamePath ); eData = eData ? eData : new xiv::exd::ExdData( *data1 ); + pCache = std::make_shared< Cache >( data1 ); } -int parseBlockEntry( char* data, std::vector< PCB_BLOCK_ENTRY >& entries, int gOff ) -{ - int offset = 0; - bool isgroup = true; - while( isgroup ) - { - PCB_BLOCK_ENTRY block_entry; - memcpy( &block_entry.header, data + offset, sizeof( block_entry.header ) ); - isgroup = block_entry.header.type == 0x30; - - //printf( " BLOCKHEADER_%X: type: %i, group_size: %i\n", gOff + offset, block_entry.header.type, block_entry.header.group_size ); - - if( isgroup ) - { - parseBlockEntry( data + offset + 0x30, entries, gOff + offset ); - offset += block_entry.header.group_size; - } - else - { - /* printf( "\tnum_v16: %i, num_indices: %i, num_vertices: %i\n\n", - block_entry.header.num_v16, block_entry.header.num_indices, block_entry.header.num_vertices );*/ - int doffset = sizeof( block_entry.header ) + offset; - uint16_t block_size = sizeof( block_entry.header ) + - block_entry.header.num_vertices * 3 * 4 + - block_entry.header.num_v16 * 6 + - block_entry.header.num_indices * 6; - - if( block_entry.header.num_vertices != 0 ) - { - block_entry.data.vertices.resize( block_entry.header.num_vertices ); - - int32_t size_vertexbuffer = block_entry.header.num_vertices * 3; - memcpy( &block_entry.data.vertices[ 0 ], data + doffset, size_vertexbuffer * 4 ); - doffset += size_vertexbuffer * 4; - } - if( block_entry.header.num_v16 != 0 ) - { - block_entry.data.vertices_i16.resize( block_entry.header.num_v16 ); - int32_t size_unknownbuffer = block_entry.header.num_v16 * 6; - memcpy( &block_entry.data.vertices_i16[ 0 ], data + doffset, size_unknownbuffer ); - doffset += block_entry.header.num_v16 * 6; - } - if( block_entry.header.num_indices != 0 ) - { - block_entry.data.indices.resize( block_entry.header.num_indices ); - int32_t size_indexbuffer = block_entry.header.num_indices * 12; - memcpy( &block_entry.data.indices[ 0 ], data + doffset, size_indexbuffer ); - doffset += size_indexbuffer; - } - entries.push_back( block_entry ); - } - } - - return 0; -} std::string zoneNameToPath( const std::string& name ) { @@ -156,39 +107,27 @@ std::string zoneNameToPath( const std::string& name ) return path; } -void readFileToBuffer( const std::string& path, std::vector< char >& buf ) -{ - auto inFile = std::ifstream( path, std::ios::binary ); - if( inFile.good() ) - { - inFile.seekg( 0, inFile.end ); - int32_t fileSize = ( int32_t ) inFile.tellg(); - buf.resize( fileSize ); - inFile.seekg( 0, inFile.beg ); - inFile.read( &buf[ 0 ], fileSize ); - inFile.close(); - } - else - { - throw std::runtime_error( "Unable to open " + path ); - } -} - int main( int argc, char* argv[] ) { auto startTime = std::chrono::high_resolution_clock::now(); auto entryStartTime = std::chrono::high_resolution_clock::now(); - std::condition_variable cv; - std::vector< std::string > argVec( argv + 1, argv + argc ); std::string zoneName = "r2t2"; + noObj = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ) { return arg == "--no-obj"; } ) != argVec.end(); bool dumpAllZones = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ) { return arg == "--dump-all"; } ) != argVec.end(); bool generateNavmesh = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ) { return arg == "--navmesh"; } ) != argVec.end(); + + int exportFileType = 0; + if( !noObj ) + exportFileType |= ExportFileType::WavefrontObj; + if( generateNavmesh ) + exportFileType |= ExportFileType::Navmesh; + if( argc > 1 ) { zoneName = argv[ 1 ]; @@ -200,8 +139,16 @@ int main( int argc, char* argv[] ) } } - initExd( gamePath ); - + try + { + initExd( gamePath ); + } + catch( std::exception& e ) + { + std::cout << "Unable to initialise EXD! Usage: pcb_reader \"path/to/FINAL FANTASY XIV - A REALM REBORN/game/sqpack\"" << std::endl; + return -1; + } + ExportMgr exportMgr; zoneNameToPath( zoneName ); if( dumpAllZones ) @@ -214,354 +161,198 @@ int main( int argc, char* argv[] ) zoneDumpList.emplace( zoneName ); } - std::mutex navmeshMutex; - std::queue< std::string > exportedGroups; - - // todo: -#ifdef WIN32 - std::string exportArg( "RecastDemo.exe --type tileMesh --obj " ); -#else - std::string exportArg( "./RecastDemo --type tileMesh --obj "); -#endif - - std::thread navmeshThread( [&navmeshMutex, &exportedGroups, &exportArg, &generateNavmesh, &cv]() + for( const auto& zoneName : zoneDumpList ) { - while( generateNavmesh ) + try { - std::string currFile; - - std::unique_lock lk( navmeshMutex ); - while( exportedGroups.empty() ) - { - cv.wait( lk ); - } + ExportedZone exportedZone; + exportedZone.name = zoneName; + + const auto& zonePath = zoneNameToPath( zoneName ); + + std::string listPcbPath( zonePath + "/collision/list.pcb" ); + std::string bgLgbPath( zonePath + "/level/bg.lgb" ); + std::string planmapLgbPath( zonePath + "/level/planmap.lgb" ); + std::string collisionFilePath( zonePath + "/collision/" ); + std::vector< char > section; + std::vector< char > section1; + std::vector< char > section2; + + const xiv::dat::Cat& test = data1->getCategory( "bg" ); + + auto test_file = data1->getFile( bgLgbPath ); + section = test_file->access_data_sections().at( 0 ); + + auto planmap_file = data1->getFile( planmapLgbPath ); + section2 = planmap_file->access_data_sections().at( 0 ); + + auto test_file1 = data1->getFile( listPcbPath ); + section1 = test_file1->access_data_sections().at( 0 ); + + std::vector< std::string > stringList; + + int totalGroups = 0; + int totalEntries = 0; + + uint32_t offset1 = 0x20; - //if( !exportedGroups.empty() ) - { - currFile = exportedGroups.front(); - exportedGroups.pop(); - } - - if( !currFile.empty() ) { - std::error_code e; - if( std::experimental::filesystem::exists( currFile ) && std::experimental::filesystem::file_size( currFile, e ) > 1024 ) + for( ;; ) { - std::string generateMessage( "\nGenerating navmesh for " + currFile + "\n" ); - std::cout << generateMessage << std::endl; + if( offset1 >= section1.size() ) + { + break; + } + uint16_t trId = *( uint16_t* ) §ion1[ offset1 ]; - auto start = std::chrono::high_resolution_clock::now(); + char someString[200]; + sprintf( someString, "%str%04d.pcb", collisionFilePath.c_str(), trId ); + stringList.push_back( std::string( someString ) ); + //std::cout << someString << "\n"; + offset1 += 0x20; - system( ( exportArg + currFile ).c_str() ); - std::string finishMessage( "\nFinished generating navmesh for " + currFile + " in " + - std::to_string( std::chrono::duration_cast< std::chrono::seconds >( std::chrono::high_resolution_clock::now() - start ).count() ) + " seconds\n" ); - std::cout << finishMessage << std::endl; - } - else - { - std::cout << ( std::string( "Unable to load OBJ file for " ) + currFile + "\n" ) << std::endl; } } - } - }); - navmeshThread.detach(); + LGB_FILE bgLgb( §ion[ 0 ] ); + LGB_FILE planmapLgb( §ion2[ 0 ] ); - LABEL_DUMP: - entryStartTime = std::chrono::high_resolution_clock::now(); - zoneName = *zoneDumpList.begin(); - try - { - const auto& zonePath = zoneNameToPath( zoneName ); + std::vector< LGB_FILE > lgbList{ bgLgb, planmapLgb }; + uint32_t max_index = 0; + int totalModels = 0; - std::string listPcbPath( zonePath + "/collision/list.pcb" ); - std::string bgLgbPath( zonePath + "/level/bg.lgb" ); - std::string planmapLgbPath( zonePath + "/level/planmap.lgb" ); - std::string collisionFilePath( zonePath + "/collision/" ); - std::vector< char > section; - std::vector< char > section1; - std::vector< char > section2; - - const xiv::dat::Cat& test = data1->getCategory( "bg" ); - - auto test_file = data1->getFile( bgLgbPath ); - section = test_file->access_data_sections().at( 0 ); - - auto planmap_file = data1->getFile( planmapLgbPath ); - section2 = planmap_file->access_data_sections().at( 0 ); - - auto test_file1 = data1->getFile( listPcbPath ); - section1 = test_file1->access_data_sections().at( 0 ); - - std::vector< std::string > stringList; - - uint32_t offset1 = 0x20; - - { - for( ;; ) { - if( offset1 >= section1.size() ) + + auto buildModelEntry = [ & ]( std::shared_ptr< PCB_FILE > pPcbFile, ExportedGroup& exportedGroup, + const std::string& name, const std::string& groupName, + const vec3* scale = nullptr, + const vec3* rotation = nullptr, + const vec3* translation = nullptr, + const SGB_MODEL_ENTRY* pSgbEntry = nullptr ) { - break; - } - uint16_t trId = *( uint16_t* ) §ion1[ offset1 ]; + + auto& pcb_file = *pPcbFile.get(); - char someString[200]; - sprintf( someString, "%str%04d.pcb", collisionFilePath.c_str(), trId ); - stringList.push_back( std::string( someString ) ); - //std::cout << someString << "\n"; - offset1 += 0x20; + ExportedModel model; + model.name = name + "_" + std::to_string( totalModels++ ); + model.meshes.resize( pcb_file.entries.size() ); - - } - } - LGB_FILE bgLgb( §ion[ 0 ], "bg" ); - LGB_FILE planmapLgb( §ion2[ 0 ], "planmap" ); - - std::vector< LGB_FILE > lgbList{ bgLgb, planmapLgb }; - uint32_t max_index = 0; - - // dont bother if we cant write to a file - auto fp_out = noObj ? ( FILE* ) nullptr : fopen( ( zoneName + ".obj" ).c_str(), "w" ); - if( fp_out ) - { - fprintf( fp_out, "\n" ); - fclose( fp_out ); - } - else if( !noObj ) - { - std::string errorMessage( "Cannot create " + zoneName + ".obj\n" + - " Check no programs have a handle to file and run as admin.\n" ); - std::cout << errorMessage; - throw std::runtime_error( errorMessage.c_str() ); - return 0; - } - - if( noObj || ( fp_out = fopen( ( zoneName + ".obj" ).c_str(), "ab+" ) ) ) - { - std::map< std::string, PCB_FILE > pcbFiles; - std::map< std::string, SGB_FILE > sgbFiles; - std::map< std::string, uint32_t > objCount; - auto loadPcbFile = [ & ]( const std::string& fileName )->bool - { - if( noObj ) - return false; - - try - { - if( fileName.find( '.' ) == std::string::npos ) - return false; - else if( fileName.substr( fileName.find_last_of( '.' ) ) != ".pcb" ) - throw std::runtime_error( "Not a PCB file." ); - - char* dataSection = nullptr; - //std::cout << fileName << " "; - auto file = data1->getFile( fileName ); - auto sections = file->get_data_sections(); - dataSection = §ions.at( 0 )[ 0 ]; - //std::cout << sections.size() << "\n"; - - uint32_t offset = 0; - PCB_FILE pcb_file; - memcpy( &pcb_file.header, &dataSection[ 0 ], sizeof( pcb_file.header ) ); - offset += sizeof( pcb_file.header ); - pcb_file.entries.resize( pcb_file.header.num_entries ); - bool isgroup = true; - while( isgroup ) + uint32_t groupCount = 0; + for( const auto& entry : pcb_file.entries ) { - PCB_BLOCK_ENTRY block_entry; - memcpy( &block_entry.header, &dataSection[ 0 ] + offset, sizeof( block_entry.header ) ); - isgroup = block_entry.header.type == 0x30; + ExportedMesh mesh; - //printf( "BLOCKHEADER_%X: type: %i, group_size: %i\n", offset, block_entry.header.type, block_entry.header.group_size ); - // - if( isgroup ) + int verts = 0; + int indices = 0; + mesh.verts.resize( ( entry.header.num_vertices + entry.header.num_v16 ) * 3 ); + mesh.indices.resize( entry.header.num_indices * 3 ); + + float x_base = abs( float( entry.header.x1 - entry.header.x ) ); + float y_base = abs( float( entry.header.y1 - entry.header.y ) ); + float z_base = abs( float( entry.header.z1 - entry.header.z ) ); + + auto makeTranslation = [ & ]( vec3& v ) { - parseBlockEntry( &dataSection[ 0 ] + offset + 0x30, pcb_file.entries, offset ); - offset += block_entry.header.group_size; - } - else + if( pSgbEntry ) + { + v.x *= pSgbEntry->header.scale.x; + v.y *= pSgbEntry->header.scale.y; + v.z *= pSgbEntry->header.scale.z; + + v = v * matrix4::rotateX( pSgbEntry->header.rotation.x ); + v = v * matrix4::rotateY( pSgbEntry->header.rotation.y ); + v = v * matrix4::rotateZ( pSgbEntry->header.rotation.z ); + + v.x += pSgbEntry->header.translation.x; + v.y += pSgbEntry->header.translation.y; + v.z += pSgbEntry->header.translation.z; + } + + if( scale ) + { + v.x *= scale->x; + v.y *= scale->y; + v.z *= scale->z; + + v = v * matrix4::rotateX( rotation->x ); + v = v * matrix4::rotateY( rotation->y ); + v = v * matrix4::rotateZ( rotation->z ); + + v.x += translation->x; + v.y += translation->y; + v.z += translation->z; + } + + }; + + for( auto& vertex : entry.data.vertices ) { - parseBlockEntry( &dataSection[ 0 ] + offset, pcb_file.entries, offset ); - } - } - pcbFiles.insert( std::make_pair( fileName, pcb_file ) ); - return true; - } - catch( std::exception& e ) - { - std::cout << "[Error] " << "Unable to load collision mesh " << fileName << "\n\tError:\n\t" << e.what() - << "\n"; - return false; - } - }; + vec3 v( vertex.x, vertex.y, vertex.z ); + makeTranslation( v ); - auto loadSgbFile = [ & ]( const std::string& fileName )->bool - { - SGB_FILE sgbFile; - try - { - char* dataSection = nullptr; - //std::cout << fileName << " "; - - auto file = data1->getFile( fileName ); - auto sections = file->get_data_sections(); - dataSection = §ions.at( 0 )[ 0 ]; - - sgbFile = SGB_FILE( &dataSection[ 0 ] ); - sgbFiles.insert( std::make_pair( fileName, sgbFile ) ); - return true; - } - catch( std::exception& e ) - { - std::cout << "[Error] " << "Unable to load SGB " << fileName << "\n\tError:\n\t" << e.what() << "\n"; - sgbFiles.insert( std::make_pair( fileName, sgbFile ) ); - } - return false; - }; - auto writeToFile = [ & ]( const PCB_FILE& pcb_file, const std::string& name, const std::string& groupName, - const vec3* scale = nullptr, - const vec3* rotation = nullptr, - const vec3* translation = nullptr, - const SGB_MODEL_ENTRY* pSgbEntry = nullptr ) - { - if( noObj ) - return; - - char name2[0x100]; - memset( name2, 0, 0x100 ); - sprintf( &name2[ 0 ], "%s_%u", &name[ 0 ], objCount[ name ]++ ); - fprintf( fp_out, "o %s\n", &name2[ 0 ] ); - - uint32_t groupCount = 0; - for( const auto& entry : pcb_file.entries ) - { - float x_base = abs( float( entry.header.x1 - entry.header.x ) ); - float y_base = abs( float( entry.header.y1 - entry.header.y ) ); - float z_base = abs( float( entry.header.z1 - entry.header.z ) ); - - auto makeTranslation = [ & ]( vec3& v ) - { - if( pSgbEntry ) - { - v.x *= pSgbEntry->header.scale.x; - v.y *= pSgbEntry->header.scale.y; - v.z *= pSgbEntry->header.scale.z; - - v = v * matrix4::rotateX( pSgbEntry->header.rotation.x ); - v = v * matrix4::rotateY( pSgbEntry->header.rotation.y ); - v = v * matrix4::rotateZ( pSgbEntry->header.rotation.z ); - - v.x += pSgbEntry->header.translation.x; - v.y += pSgbEntry->header.translation.y; - v.z += pSgbEntry->header.translation.z; - } - if( scale ) - { - v.x *= scale->x; - v.y *= scale->y; - v.z *= scale->z; - - v = v * matrix4::rotateX( rotation->x ); - v = v * matrix4::rotateY( rotation->y ); - v = v * matrix4::rotateZ( rotation->z ); - - v.x += translation->x; - v.y += translation->y; - v.z += translation->z; + mesh.verts[ verts++ ] = v.x; + mesh.verts[ verts++ ] = v.y; + mesh.verts[ verts++ ] = v.z; } - }; + for( const auto& link : entry.data.vertices_i16 ) + { + vec3 v( float( link.x ) / 0xFFFF, float( link.y ) / 0xFFFF, float( link.z ) / 0xFFFF ); - for( auto& vertex : entry.data.vertices ) - { - vec3 v( vertex.x, vertex.y, vertex.z ); - makeTranslation( v ); - fprintf( fp_out, "v %f %f %f\n", v.x, v.y, v.z ); + v.x = v.x * x_base + entry.header.x; + v.y = v.y * y_base + entry.header.y; + v.z = v.z * z_base + entry.header.z; + + makeTranslation( v ); + + mesh.verts[ verts++ ] = v.x; + mesh.verts[ verts++ ] = v.y; + mesh.verts[ verts++ ] = v.z; + } + + //fprintf( fp_out, "g %s_", (name2 + "_" + std::to_string( groupCount++ )).c_str() ); + for( const auto& index : entry.data.indices ) + { + mesh.indices[ indices++ ] = index.index[ 0 ]; + mesh.indices[ indices++ ] = index.index[ 1 ]; + mesh.indices[ indices++ ] = index.index[ 2 ]; + // std::cout << std::to_string( index.unknown[0] )<< " " << std::to_string( index.unknown[1] )<< " " << std::to_string( index.unknown[2]) << std::endl; + } + max_index += entry.data.vertices.size() + entry.data.vertices_i16.size(); + model.meshes.push_back( mesh ); } - - for( const auto& link : entry.data.vertices_i16 ) - { - vec3 v( float( link.x ) / 0xFFFF, float( link.y ) / 0xFFFF, float( link.z ) / 0xFFFF ); - - v.x = v.x * x_base + entry.header.x; - v.y = v.y * y_base + entry.header.y; - v.z = v.z * z_base + entry.header.z; - - makeTranslation( v ); - fprintf( fp_out, "v %f %f %f\n", v.x, v.y, v.z ); - } - - //fprintf( fp_out, "g %s_", (name2 + "_" + std::to_string( groupCount++ )).c_str() ); - for( const auto& index : entry.data.indices ) - { - fprintf( fp_out, "f %i %i %i\n", - index.index[ 0 ] + max_index + 1, - index.index[ 1 ] + max_index + 1, - index.index[ 2 ] + max_index + 1 ); - // std::cout << std::to_string( index.unknown[0] )<< " " << std::to_string( index.unknown[1] )<< " " << std::to_string( index.unknown[2]) << std::endl; - } - max_index += entry.data.vertices.size() + entry.data.vertices_i16.size(); - } - }; - - if( !noObj ) - { + exportedGroup.models[model.name] = model; + }; + ExportedGroup exportedTerrainGroup; + exportedTerrainGroup.name = zoneName; for( const auto& fileName : stringList ) { - loadPcbFile( fileName ); - writeToFile( pcbFiles[ fileName ], fileName, zoneName ); + if( auto pPcbFile = pCache->getPcbFile( fileName ) ) + buildModelEntry( pPcbFile, exportedTerrainGroup, fileName, zoneName ); } - } - - std::cout << "[Info] " << "Writing obj file " << "\n"; - uint32_t totalGroups = 0; - uint32_t totalGroupEntries = 0; - - for( const auto& lgb : lgbList ) - { - for( const auto& group : lgb.groups ) + + for( const auto& lgb : lgbList ) { - max_index = 0; - std::string outfile_name( zoneName + "_" + group.name + ".obj" ); - totalGroups++; - - if( !noObj ) + for( const auto& group : lgb.groups ) { - fp_out = fopen( outfile_name.c_str(), "w" ); - if( fp_out ) - { - // blank otherwise recast tries to load them.. - fprintf( fp_out, "" ); - fclose( fp_out ); - fp_out = fopen( outfile_name.c_str(), "ab+" ); - } - + ExportedGroup exportedGroup; + exportedGroup.name = group.name; + + max_index = 0; + //std::cout << "\t" << group.name << " Size " << group.header.entryCount << "\n"; for( const auto& pEntry : group.entries ) { std::string fileName( "" ); fileName.resize( 256 ); - totalGroupEntries++; // write files - auto writeOutput = [ & ]( const std::string& fileName, const vec3* scale, const vec3* rotation, - const vec3* translation, const SGB_MODEL_ENTRY* pModel = nullptr )->bool + auto writeOutput = [&](const std::string& fileName, const vec3* scale, const vec3* rotation, + const vec3* translation, const SGB_MODEL_ENTRY* pModel = nullptr)->bool { + if( auto pPcbFile = pCache->getPcbFile( fileName ) ) { - const auto& it = pcbFiles.find( fileName ); - if( it == pcbFiles.end() ) - { - if( fileName.empty() || !loadPcbFile( fileName ) ) - return false; - //std::cout << "\t\tLoaded PCB File " << pBgParts->collisionFileName << "\n"; - } - } - const auto& it = pcbFiles.find( fileName ); - if( it != pcbFiles.end() ) - { - const auto& pcb_file = it->second; - writeToFile( pcb_file, fileName, group.name, scale, rotation, translation, pModel ); + buildModelEntry( pPcbFile, exportedGroup, fileName, group.name, scale, rotation, translation, pModel ); } return true; }; @@ -570,37 +361,28 @@ int main( int argc, char* argv[] ) { case LgbEntryType::BgParts: { - auto pBgParts = static_cast< LGB_BGPARTS_ENTRY* >( pEntry.get() ); + auto pBgParts = static_cast(pEntry.get()); fileName = pBgParts->collisionFileName; writeOutput( fileName, &pBgParts->header.scale, &pBgParts->header.rotation, - &pBgParts->header.translation ); + &pBgParts->header.translation ); } break; // gimmick entry case LgbEntryType::Gimmick: { - auto pGimmick = static_cast< LGB_GIMMICK_ENTRY* >( pEntry.get() ); + auto pGimmick = static_cast(pEntry.get()); + if (auto pSgbFile = pCache->getSgbFile(pGimmick->gimmickFileName)) { - const auto& it = sgbFiles.find( pGimmick->gimmickFileName ); - if( it == sgbFiles.end() ) + const auto& sgbFile = *pSgbFile; + for (const auto& group : sgbFile.entries) { - // std::cout << "\tGIMMICK:\n\t\t" << pGimmick->gimmickFileName << "\n"; - loadSgbFile( pGimmick->gimmickFileName ); - } - } - const auto& it = sgbFiles.find( pGimmick->gimmickFileName ); - if( it != sgbFiles.end() ) - { - const auto& sgbFile = it->second; - for( const auto& group : sgbFile.entries ) - { - for( const auto& pEntry : group.entries ) + for (const auto& pEntry : group.entries) { - auto pModel = dynamic_cast< SGB_MODEL_ENTRY* >( pEntry.get() ); + auto pModel = dynamic_cast(pEntry.get()); fileName = pModel->collisionFileName; writeOutput( fileName, &pGimmick->header.scale, &pGimmick->header.rotation, - &pGimmick->header.translation, pModel ); + &pGimmick->header.translation, pModel ); } } } @@ -611,53 +393,36 @@ int main( int argc, char* argv[] ) writeOutput( fileName, &pEntry->header.scale, &pEntry->header.rotation, &pEntry->header.translation ); } break; + default: + break; } } - } - if( generateNavmesh ) - { - if( fp_out ) - fclose( fp_out ); - std::unique_lock lock( navmeshMutex ); - exportedGroups.push( outfile_name ); - cv.notify_one(); + exportMgr.exportGroup( zoneName, exportedGroup, ( ExportFileType )exportFileType ); + //exportedZone.groups.emplace( group.name, exportedGroup ); } } + //exportMgr.exportZone( exportedZone, ( ExportFileType )exportFileType ); + //std::cout << "[Info] " << "Loaded " << pcbFiles.size() << " PCB Files \n"; + std::cout << "[Info] " << "Total Groups " << totalGroups << "\n"; } - std::cout << "[Info] " << "Loaded " << pcbFiles.size() << " PCB Files \n"; - std::cout << "[Info] " << "Total Groups " << totalGroups << " Total entries " << totalGroupEntries << "\n"; + + std::cout << "[Success] " << "Exported " << zoneName << " in " << + std::chrono::duration_cast< std::chrono::seconds >( + std::chrono::high_resolution_clock::now() - entryStartTime ).count() << " seconds\n"; + } + catch( std::exception& e ) + { + std::cout << "[Error] " << e.what() << std::endl; + std::cout << "[Error] " + << "Unable to extract collision data.\n" + << std::endl; + std::cout << std::endl; + std::cout << "[Info] " << "Usage: pcb_reader2 territory \"path/to/game/sqpack/ffxiv\" " << std::endl; } - - std::cout << "[Success] " << "Exported " << zoneName << " in " << - std::chrono::duration_cast< std::chrono::seconds >( - std::chrono::high_resolution_clock::now() - entryStartTime ).count() << " seconds\n"; - } - catch( std::exception& e ) - { - std::cout << "[Error] " << e.what() << std::endl; - std::cout << "[Error] " - << "Unable to extract collision data.\n" - << std::endl; - std::cout << std::endl; - std::cout << "[Info] " << "Usage: pcb_reader2 territory \"path/to/game/sqpack/ffxiv\" " << std::endl; } + exportMgr.waitForTasks(); std::cout << "\n\n\n"; - LABEL_NEXT_ZONE_ENTRY: - zoneDumpList.erase( zoneName ); - - if( !zoneDumpList.empty() ) - goto LABEL_DUMP; - while( 1 ) - { - std::lock_guard< std::mutex > lock( navmeshMutex ); - if( exportedGroups.empty() ) - generateNavmesh = false; - - if( navmeshThread.joinable() ) - navmeshThread.join(); - std::this_thread::sleep_for( 1s ); - } std::cout << "\n\n\n[Success] Finished all tasks in " << std::chrono::duration_cast< std::chrono::seconds >( std::chrono::high_resolution_clock::now() - startTime ).count() << " seconds\n"; diff --git a/src/tools/pcb_reader/navmesh_exporter.h b/src/tools/pcb_reader/navmesh_exporter.h new file mode 100644 index 00000000..d4580d95 --- /dev/null +++ b/src/tools/pcb_reader/navmesh_exporter.h @@ -0,0 +1,44 @@ +#ifndef OBJ_EXPORTER_H +#define OBJ_EXPORTER_H + +#include +#include +#include +#include + +#include + +#include "exporter.h" + +static class ObjExporter : public Exporter +{ +public: + static void exportZone( const ExportedZone& zone ) + { + auto start = std::chrono::high_resolution_clock::now(); + + auto fileName = zone.name + ".obj"; + + auto end = std::chrono::high_resolution_clock::now(); + std::cout << ( "Finished exporting " + fileName + " in " + + std::to_string( std::chrono::duration_cast< std::chrono::seconds >( end - start ).count() ) + " seconds\n" ); + } + + static void exportGroup( const std::string& zoneName, const ExportedGroup& group ) + { + auto start = std::chrono::high_resolution_clock::now(); + + auto fileName = zoneName + "_" + group.name + ".obj"; + + auto end = std::chrono::high_resolution_clock::now(); + + std::cout << ( "Finished exporting " + fileName + " in " + + std::to_string( std::chrono::duration_cast< std::chrono::seconds >( end - start ).count() ) + " seconds\n" ); + } +private: + static void exportGroup( const ExportedGroup& group, std::ofstream& of, int& indicesOffset, int& modelCount ) + { + + } +}; +#endif // !OBJ_EXPORTER_H diff --git a/src/tools/pcb_reader/obj_exporter.h b/src/tools/pcb_reader/obj_exporter.h new file mode 100644 index 00000000..0ff70213 --- /dev/null +++ b/src/tools/pcb_reader/obj_exporter.h @@ -0,0 +1,98 @@ +#ifndef OBJ_EXPORTER_H +#define OBJ_EXPORTER_H + +#include +#include +#include + +#include + +#include "exporter.h" + +class ObjExporter : public Exporter +{ +public: + static void exportZone( const ExportedZone& zone ) + { + auto start = std::chrono::high_resolution_clock::now(); + + auto fileName = zone.name + ".obj"; + std::ofstream of( fileName, std::ios::trunc ); + int indicesOffset = 0; + int meshesCount = 0; + + if( of.good() ) + { + of.close(); + of.open( fileName, std::ios::app ); + for( const auto& group : zone.groups ) + { + exportGroup( group.second, of, indicesOffset, meshesCount ); + } + of.flush(); + of.close(); + } + + auto end = std::chrono::high_resolution_clock::now(); + std::cout << ( "Finished exporting " + fileName + " in " + + std::to_string( std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() ) + "ms \n" ); + } + + static void exportGroup( const std::string& zoneName, const ExportedGroup& group ) + { + auto start = std::chrono::high_resolution_clock::now(); + + auto fileName = zoneName + "_" + group.name + ".obj"; + std::ofstream of( fileName, std::ios::trunc ); + int indicesOffset = 0; + int modelCount = 0; + + if( of.good() ) + { + of.close(); + of.open( fileName, std::ios::app ); + exportGroup( group, of, indicesOffset, modelCount ); + of.flush(); + of.close(); + } + + auto end = std::chrono::high_resolution_clock::now(); + std::cout << ( "Finished exporting " + fileName + " in " + + std::to_string( std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() ) + "ms\n" ); + } +private: + static void exportGroup( const ExportedGroup& group, std::ofstream& of, int& indicesOffset, int& modelCount ) + { + int currModelCount = modelCount; + + //of << ( "o " + group.name + "_" + std::to_string( currModelCount ) + "\n" ); + for( const auto& model : group.models ) + { + of << ( "o " + model.second.name + "_" + std::to_string( currModelCount ) + "_" + std::to_string( modelCount++ ) + "\n" ); + + int meshCount = 0; + for( const auto& mesh : model.second.meshes ) + { + for( int i = 0; i + 2 < mesh.verts.size(); i += 3 ) + { + of << ( + "v " + std::to_string( mesh.verts[ i ] ) + " " + + std::to_string( mesh.verts[ i + 1 ] ) + " " + + std::to_string( mesh.verts[ i + 2 ] ) + "\n" + ); + } + //of << ( "g " + model.second.name + "_" + std::to_string( currModelCount ) + "_" + std::to_string( modelCount ) + "_" + std::to_string( meshCount++ ) + "\n" ); + for( int i = 0; i + 2 < mesh.indices.size(); i += 3 ) + { + of << ( + "f " + std::to_string( mesh.indices[ i ] + indicesOffset + 1 ) + " " + + std::to_string( mesh.indices[i + 1] + indicesOffset + 1 ) + " " + + std::to_string( mesh.indices[i + 2] + indicesOffset + 1 ) + "\n" + ); + } + indicesOffset += mesh.indices.size(); + } + } + } +}; +#endif // !OBJ_EXPORTER_H diff --git a/src/tools/pcb_reader/pcb.h b/src/tools/pcb_reader/pcb.h index 77a50d81..647c56ca 100644 --- a/src/tools/pcb_reader/pcb.h +++ b/src/tools/pcb_reader/pcb.h @@ -70,6 +70,90 @@ struct PCB_FILE { PCB_HEADER header; std::vector< PCB_BLOCK_ENTRY > entries; + + PCB_FILE( char* buf ) + { + uint32_t offset = 0; + memcpy( &header, buf, sizeof( header )); + offset += sizeof( header ); + entries.resize( header.num_entries ); + bool isgroup = true; + while( isgroup ) + { + PCB_BLOCK_ENTRY block_entry; + memcpy( &block_entry.header, buf + offset, sizeof( block_entry.header ) ); + isgroup = block_entry.header.type == 0x30; + + //printf( "BLOCKHEADER_%X: type: %i, group_size: %i\n", offset, block_entry.header.type, block_entry.header.group_size ); + // + if( isgroup ) + { + parseBlockEntry( buf + offset + 0x30, entries, offset); + offset += block_entry.header.group_size; + } + else + { + parseBlockEntry( buf + offset, entries, offset ); + } + } + } + + int parseBlockEntry( char* data, std::vector< PCB_BLOCK_ENTRY >& entries, int gOff ) + { + int offset = 0; + bool isgroup = true; + while( isgroup ) + { + PCB_BLOCK_ENTRY block_entry; + memcpy( &block_entry.header, data + offset, sizeof( block_entry.header ) ); + isgroup = block_entry.header.type == 0x30; + + //printf( " BLOCKHEADER_%X: type: %i, group_size: %i\n", gOff + offset, block_entry.header.type, block_entry.header.group_size ); + + if( isgroup ) + { + parseBlockEntry( data + offset + 0x30, entries, gOff + offset ); + offset += block_entry.header.group_size; + } + else + { + /* printf( "\tnum_v16: %i, num_indices: %i, num_vertices: %i\n\n", + block_entry.header.num_v16, block_entry.header.num_indices, block_entry.header.num_vertices );*/ + int doffset = sizeof( block_entry.header ) + offset; + uint16_t block_size = sizeof( block_entry.header ) + + block_entry.header.num_vertices * 3 * 4 + + block_entry.header.num_v16 * 6 + + block_entry.header.num_indices * 6; + + if( block_entry.header.num_vertices != 0 ) + { + block_entry.data.vertices.resize( block_entry.header.num_vertices ); + + int32_t size_vertexbuffer = block_entry.header.num_vertices * 3; + memcpy( &block_entry.data.vertices[ 0 ], data + doffset, size_vertexbuffer * 4 ); + doffset += size_vertexbuffer * 4; + } + if( block_entry.header.num_v16 != 0 ) + { + block_entry.data.vertices_i16.resize( block_entry.header.num_v16 ); + int32_t size_unknownbuffer = block_entry.header.num_v16 * 6; + memcpy( &block_entry.data.vertices_i16[ 0 ], data + doffset, size_unknownbuffer ); + doffset += block_entry.header.num_v16 * 6; + } + if( block_entry.header.num_indices != 0 ) + { + block_entry.data.indices.resize( block_entry.header.num_indices ); + int32_t size_indexbuffer = block_entry.header.num_indices * 12; + memcpy( &block_entry.data.indices[ 0 ], data + doffset, size_indexbuffer ); + doffset += size_indexbuffer; + } + entries.push_back( block_entry ); + } + } + + return 0; + } + }; struct PCB_LIST_ENTRY diff --git a/src/tools/pcb_reader/threadpool.h b/src/tools/pcb_reader/threadpool.h new file mode 100644 index 00000000..23c542bd --- /dev/null +++ b/src/tools/pcb_reader/threadpool.h @@ -0,0 +1,91 @@ +#ifndef THREADPOOL_H +#define THREADPOOL_H + +#include +#include +#include +#include +#include +#include +#include +#include + +class ThreadPool +{ +public: + ThreadPool( unsigned int numJobs = std::thread::hardware_concurrency() ) + { + for( auto i = 0; i < numJobs; ++i ) + { + m_workers.push_back( std::async( std::launch::async, [this]{ run(); } ) ); + } + } + + ~ThreadPool() + { + complete(); + } + + template< class Func, class Ret = std::result_of_t< Func&() > > + std::future< Ret > queue( Func&& f ) + { + std::packaged_task< Ret() > task( std::forward< Func >( f ) ); + auto ret = task.get_future(); + { + std::unique_lock lock( m_mutex ); + m_pendingJobs.emplace_back( std::move( task ) ); + } + m_cv.notify_one(); + return ret; + } + + void cancel() + { + { + std::unique_lock lock( m_mutex ); + m_pendingJobs.clear(); + } + complete(); + } + + bool complete() + { + std::unique_lock lock( m_mutex ); + for( auto&& worker : m_workers ) + { + m_pendingJobs.push_back( {} ); + } + m_cv.notify_all(); + m_workers.clear(); + return true; + } +private: + void run() + { + while( 1 ) + { + std::packaged_task< void() > func; + { + std::unique_lock< std::mutex > lock( m_mutex ); + if( m_pendingJobs.empty() ) + { + m_cv.wait( lock, [&](){ return !m_pendingJobs.empty(); } ); + } + func = std::move( m_pendingJobs.front() ); + m_pendingJobs.pop_front(); + } + if( !func.valid() ) + { + return; + } + func(); + } + } + + std::mutex m_mutex; + std::condition_variable m_cv; + std::deque< std::packaged_task< void() > > m_pendingJobs; + std::vector< std::future< void > > m_workers; +}; + +#endif \ No newline at end of file From b5ecf60f7a07c68985bd2739267b9fa0f0366104 Mon Sep 17 00:00:00 2001 From: Tahir Akhlaq Date: Sun, 20 Jan 2019 20:14:40 +0000 Subject: [PATCH 3/4] clean up some includes in pcb_reader - fix deadlock --- src/tools/pcb_reader/exportmgr.h | 14 +- src/tools/pcb_reader/lgb.h | 2 +- src/tools/pcb_reader/main.cpp | 355 ++++++++++++------------ src/tools/pcb_reader/navmesh_exporter.h | 21 +- src/tools/pcb_reader/obj_exporter.h | 81 ++++-- src/tools/pcb_reader/sgb.h | 2 +- src/tools/pcb_reader/threadpool.h | 26 +- 7 files changed, 277 insertions(+), 224 deletions(-) diff --git a/src/tools/pcb_reader/exportmgr.h b/src/tools/pcb_reader/exportmgr.h index 7673dc34..656e15f7 100644 --- a/src/tools/pcb_reader/exportmgr.h +++ b/src/tools/pcb_reader/exportmgr.h @@ -2,13 +2,17 @@ #define EXPORTMGR_H #include "exporter.h" +#include "navmesh_exporter.h" #include "obj_exporter.h" #include "threadpool.h" class ExportMgr { public: - ExportMgr(){} + ExportMgr( unsigned int maxJobs = 0 ) + { + m_threadpool.addWorkers( maxJobs ); + } ~ExportMgr() { waitForTasks(); @@ -20,6 +24,10 @@ public: { m_threadpool.queue( [zone](){ ObjExporter::exportZone( zone ); } ); } + if( exportFileTypes & ExportFileType::Navmesh ) + { + m_threadpool.queue( [zone](){ NavmeshExporter::exportZone( zone ); } ); + } } void exportGroup( const std::string& zoneName, const ExportedGroup& group, ExportFileType exportFileTypes ) @@ -28,6 +36,10 @@ public: { m_threadpool.queue( [zoneName, group](){ ObjExporter::exportGroup( zoneName, group ); } ); } + if( exportFileTypes & ExportFileType::Navmesh ) + { + m_threadpool.queue( [zoneName, group](){ NavmeshExporter::exportGroup( zoneName, group ); } ); + } } void waitForTasks() diff --git a/src/tools/pcb_reader/lgb.h b/src/tools/pcb_reader/lgb.h index bf2a3157..21628b68 100644 --- a/src/tools/pcb_reader/lgb.h +++ b/src/tools/pcb_reader/lgb.h @@ -295,7 +295,7 @@ struct LGB_GROUP } catch( std::exception& e ) { - std::cout << name << " " << e.what() << std::endl; + std::cout << ( name + " " + e.what() + "\n" ); } } }; diff --git a/src/tools/pcb_reader/main.cpp b/src/tools/pcb_reader/main.cpp index dcc498ba..53a1bb7c 100644 --- a/src/tools/pcb_reader/main.cpp +++ b/src/tools/pcb_reader/main.cpp @@ -29,12 +29,6 @@ #include #include -#include - -#include -#include - - // garbage to ignore models bool noObj = false; @@ -42,6 +36,7 @@ std::string gamePath( "C:\\SquareEnix\\FINAL FANTASY XIV - A Realm Reborn\\game\ std::unordered_map< uint16_t, std::string > zoneNameMap; uint32_t zoneId; + std::set< std::string > zoneDumpList; std::shared_ptr< Cache > pCache; @@ -96,12 +91,12 @@ std::string zoneNameToPath( const std::string& name ) //path = path.substr( path.find_first_of( "/" ) + 1, path.size() - path.find_first_of( "/" )); //path = std::string( "ffxiv/" ) + path; path = std::string( "bg/" ) + path.substr( 0, path.find( "/level/" ) ); - std::cout << "[Info] " << "Found path for " << name << ": " << path << std::endl; + printf( "[Info] Found path for %s\n", name.c_str() ); } else { throw std::runtime_error( "Unable to find path for " + name + - ".\n\tPlease double check spelling or open 0a0000.win32.index with FFXIV Explorer and extract territorytype.exh as CSV\n\tand copy territorytype.exh.csv into pcb_reader.exe directory if using standalone" ); + ".\n\tPlease double check spelling." ); } return path; @@ -145,7 +140,7 @@ int main( int argc, char* argv[] ) } catch( std::exception& e ) { - std::cout << "Unable to initialise EXD! Usage: pcb_reader \"path/to/FINAL FANTASY XIV - A REALM REBORN/game/sqpack\"" << std::endl; + printf( "Unable to initialise EXD! Usage: pcb_reader \"path/to/FINAL FANTASY XIV - A REALM REBORN/game/sqpack\" [--no-obj, --dump-all, --navmesh]" ); return -1; } ExportMgr exportMgr; @@ -221,211 +216,205 @@ int main( int argc, char* argv[] ) uint32_t max_index = 0; int totalModels = 0; + + + auto buildModelEntry = [ & ]( std::shared_ptr< PCB_FILE > pPcbFile, ExportedGroup& exportedGroup, + const std::string& name, const std::string& groupName, + const vec3* scale = nullptr, + const vec3* rotation = nullptr, + const vec3* translation = nullptr, + const SGB_MODEL_ENTRY* pSgbEntry = nullptr ) { + + auto& pcb_file = *pPcbFile.get(); - auto buildModelEntry = [ & ]( std::shared_ptr< PCB_FILE > pPcbFile, ExportedGroup& exportedGroup, - const std::string& name, const std::string& groupName, - const vec3* scale = nullptr, - const vec3* rotation = nullptr, - const vec3* translation = nullptr, - const SGB_MODEL_ENTRY* pSgbEntry = nullptr ) + ExportedModel model; + model.name = name + "_" + std::to_string( totalModels++ ); + model.meshes.resize( pcb_file.entries.size() ); + + uint32_t meshCount = 0; + for( const auto& entry : pcb_file.entries ) { - - auto& pcb_file = *pPcbFile.get(); + ExportedMesh mesh; - ExportedModel model; - model.name = name + "_" + std::to_string( totalModels++ ); - model.meshes.resize( pcb_file.entries.size() ); + mesh.verts.resize( ( entry.header.num_vertices + entry.header.num_v16 ) * 3 ); + mesh.indices.resize( entry.header.num_indices * 3 ); - uint32_t groupCount = 0; - for( const auto& entry : pcb_file.entries ) + float x_base = abs( float( entry.header.x1 - entry.header.x ) ); + float y_base = abs( float( entry.header.y1 - entry.header.y ) ); + float z_base = abs( float( entry.header.z1 - entry.header.z ) ); + + auto makeTranslation = [ & ]( vec3& v ) { - ExportedMesh mesh; - - int verts = 0; - int indices = 0; - mesh.verts.resize( ( entry.header.num_vertices + entry.header.num_v16 ) * 3 ); - mesh.indices.resize( entry.header.num_indices * 3 ); - - float x_base = abs( float( entry.header.x1 - entry.header.x ) ); - float y_base = abs( float( entry.header.y1 - entry.header.y ) ); - float z_base = abs( float( entry.header.z1 - entry.header.z ) ); - - auto makeTranslation = [ & ]( vec3& v ) + if( pSgbEntry ) { - if( pSgbEntry ) + v.x *= pSgbEntry->header.scale.x; + v.y *= pSgbEntry->header.scale.y; + v.z *= pSgbEntry->header.scale.z; + + v = v * matrix4::rotateX( pSgbEntry->header.rotation.x ); + v = v * matrix4::rotateY( pSgbEntry->header.rotation.y ); + v = v * matrix4::rotateZ( pSgbEntry->header.rotation.z ); + + v.x += pSgbEntry->header.translation.x; + v.y += pSgbEntry->header.translation.y; + v.z += pSgbEntry->header.translation.z; + } + + if( scale ) + { + v.x *= scale->x; + v.y *= scale->y; + v.z *= scale->z; + + v = v * matrix4::rotateX( rotation->x ); + v = v * matrix4::rotateY( rotation->y ); + v = v * matrix4::rotateZ( rotation->z ); + + v.x += translation->x; + v.y += translation->y; + v.z += translation->z; + } + + }; + int verts = 0; + int indices = 0; + + for( auto& vertex : entry.data.vertices ) + { + vec3 v( vertex.x, vertex.y, vertex.z ); + makeTranslation( v ); + + mesh.verts[ verts++ ] = v.x; + mesh.verts[ verts++ ] = v.y; + mesh.verts[ verts++ ] = v.z; + } + + for( const auto& link : entry.data.vertices_i16 ) + { + vec3 v( float( link.x ) / 0xFFFF, float( link.y ) / 0xFFFF, float( link.z ) / 0xFFFF ); + + v.x = v.x * x_base + entry.header.x; + v.y = v.y * y_base + entry.header.y; + v.z = v.z * z_base + entry.header.z; + + makeTranslation( v ); + + mesh.verts[ verts++ ] = v.x; + mesh.verts[ verts++ ] = v.y; + mesh.verts[ verts++ ] = v.z; + } + + for( const auto& index : entry.data.indices ) + { + mesh.indices[ indices++ ] = index.index[ 0 ]; + mesh.indices[ indices++ ] = index.index[ 1 ]; + mesh.indices[ indices++ ] = index.index[ 2 ]; + // std::cout << std::to_string( index.unknown[0] )<< " " << std::to_string( index.unknown[1] )<< " " << std::to_string( index.unknown[2]) << std::endl; + } + max_index += entry.data.vertices.size() + entry.data.vertices_i16.size(); + model.meshes[ meshCount++ ] = mesh; + } + exportedGroup.models[model.name] = model; + }; + ExportedGroup exportedTerrainGroup; + exportedTerrainGroup.name = zoneName + "_terrain"; + for( const auto& fileName : stringList ) + { + if( auto pPcbFile = pCache->getPcbFile( fileName ) ) + buildModelEntry( pPcbFile, exportedTerrainGroup, fileName, zoneName ); + } + exportMgr.exportGroup( zoneName, exportedTerrainGroup, ( ExportFileType )exportFileType ); + + for( const auto& lgb : lgbList ) + { + for( const auto& group : lgb.groups ) + { + ExportedGroup exportedGroup; + exportedGroup.name = group.name; + + max_index = 0; + + //std::cout << "\t" << group.name << " Size " << group.header.entryCount << "\n"; + for( const auto& pEntry : group.entries ) + { + std::string fileName( "" ); + fileName.resize( 256 ); + + // write files + auto pcbTransformModel = [&]( const std::string& fileName, const vec3* scale, const vec3* rotation, + const vec3* translation, const SGB_MODEL_ENTRY* pModel = nullptr )-> bool + { + if( auto pPcbFile = pCache->getPcbFile( fileName ) ) { - v.x *= pSgbEntry->header.scale.x; - v.y *= pSgbEntry->header.scale.y; - v.z *= pSgbEntry->header.scale.z; - - v = v * matrix4::rotateX( pSgbEntry->header.rotation.x ); - v = v * matrix4::rotateY( pSgbEntry->header.rotation.y ); - v = v * matrix4::rotateZ( pSgbEntry->header.rotation.z ); - - v.x += pSgbEntry->header.translation.x; - v.y += pSgbEntry->header.translation.y; - v.z += pSgbEntry->header.translation.z; + buildModelEntry( pPcbFile, exportedGroup, fileName, group.name, scale, rotation, translation, pModel ); } - - if( scale ) - { - v.x *= scale->x; - v.y *= scale->y; - v.z *= scale->z; - - v = v * matrix4::rotateX( rotation->x ); - v = v * matrix4::rotateY( rotation->y ); - v = v * matrix4::rotateZ( rotation->z ); - - v.x += translation->x; - v.y += translation->y; - v.z += translation->z; - } - + return true; }; - for( auto& vertex : entry.data.vertices ) + switch( pEntry->getType() ) { - vec3 v( vertex.x, vertex.y, vertex.z ); - makeTranslation( v ); - - mesh.verts[ verts++ ] = v.x; - mesh.verts[ verts++ ] = v.y; - mesh.verts[ verts++ ] = v.z; - } - - for( const auto& link : entry.data.vertices_i16 ) - { - vec3 v( float( link.x ) / 0xFFFF, float( link.y ) / 0xFFFF, float( link.z ) / 0xFFFF ); - - v.x = v.x * x_base + entry.header.x; - v.y = v.y * y_base + entry.header.y; - v.z = v.z * z_base + entry.header.z; - - makeTranslation( v ); - - mesh.verts[ verts++ ] = v.x; - mesh.verts[ verts++ ] = v.y; - mesh.verts[ verts++ ] = v.z; - } - - //fprintf( fp_out, "g %s_", (name2 + "_" + std::to_string( groupCount++ )).c_str() ); - for( const auto& index : entry.data.indices ) - { - mesh.indices[ indices++ ] = index.index[ 0 ]; - mesh.indices[ indices++ ] = index.index[ 1 ]; - mesh.indices[ indices++ ] = index.index[ 2 ]; - // std::cout << std::to_string( index.unknown[0] )<< " " << std::to_string( index.unknown[1] )<< " " << std::to_string( index.unknown[2]) << std::endl; - } - max_index += entry.data.vertices.size() + entry.data.vertices_i16.size(); - model.meshes.push_back( mesh ); - } - exportedGroup.models[model.name] = model; - }; - ExportedGroup exportedTerrainGroup; - exportedTerrainGroup.name = zoneName; - for( const auto& fileName : stringList ) - { - if( auto pPcbFile = pCache->getPcbFile( fileName ) ) - buildModelEntry( pPcbFile, exportedTerrainGroup, fileName, zoneName ); - } - - for( const auto& lgb : lgbList ) - { - for( const auto& group : lgb.groups ) - { - ExportedGroup exportedGroup; - exportedGroup.name = group.name; - - max_index = 0; - - //std::cout << "\t" << group.name << " Size " << group.header.entryCount << "\n"; - for( const auto& pEntry : group.entries ) - { - std::string fileName( "" ); - fileName.resize( 256 ); - - // write files - auto writeOutput = [&](const std::string& fileName, const vec3* scale, const vec3* rotation, - const vec3* translation, const SGB_MODEL_ENTRY* pModel = nullptr)->bool + case LgbEntryType::BgParts: { - if( auto pPcbFile = pCache->getPcbFile( fileName ) ) - { - buildModelEntry( pPcbFile, exportedGroup, fileName, group.name, scale, rotation, translation, pModel ); - } - return true; - }; + auto pBgParts = static_cast(pEntry.get()); + fileName = pBgParts->collisionFileName; + pcbTransformModel( fileName, &pBgParts->header.scale, &pBgParts->header.rotation, + &pBgParts->header.translation ); + } + break; - switch( pEntry->getType() ) + // gimmick entry + case LgbEntryType::Gimmick: { - case LgbEntryType::BgParts: + auto pGimmick = static_cast( pEntry.get() ); + if( auto pSgbFile = pCache->getSgbFile( pGimmick->gimmickFileName ) ) { - auto pBgParts = static_cast(pEntry.get()); - fileName = pBgParts->collisionFileName; - writeOutput( fileName, &pBgParts->header.scale, &pBgParts->header.rotation, - &pBgParts->header.translation ); - } - break; - - // gimmick entry - case LgbEntryType::Gimmick: - { - auto pGimmick = static_cast(pEntry.get()); - if (auto pSgbFile = pCache->getSgbFile(pGimmick->gimmickFileName)) + const auto& sgbFile = *pSgbFile; + for( const auto& group : sgbFile.entries ) { - const auto& sgbFile = *pSgbFile; - for (const auto& group : sgbFile.entries) + for( const auto& pEntry : group.entries ) { - for (const auto& pEntry : group.entries) - { - auto pModel = dynamic_cast(pEntry.get()); - fileName = pModel->collisionFileName; - writeOutput( fileName, &pGimmick->header.scale, &pGimmick->header.rotation, - &pGimmick->header.translation, pModel ); - } + auto pModel = dynamic_cast< SGB_MODEL_ENTRY* >( pEntry.get() ); + fileName = pModel->collisionFileName; + pcbTransformModel( fileName, &pGimmick->header.scale, &pGimmick->header.rotation, + &pGimmick->header.translation, pModel ); } } } - - case LgbEntryType::EventObject: - { - writeOutput( fileName, &pEntry->header.scale, &pEntry->header.rotation, &pEntry->header.translation ); - } - break; - default: - break; } - } - exportMgr.exportGroup( zoneName, exportedGroup, ( ExportFileType )exportFileType ); - //exportedZone.groups.emplace( group.name, exportedGroup ); - } - } - //exportMgr.exportZone( exportedZone, ( ExportFileType )exportFileType ); - //std::cout << "[Info] " << "Loaded " << pcbFiles.size() << " PCB Files \n"; - std::cout << "[Info] " << "Total Groups " << totalGroups << "\n"; - } - std::cout << "[Success] " << "Exported " << zoneName << " in " << - std::chrono::duration_cast< std::chrono::seconds >( - std::chrono::high_resolution_clock::now() - entryStartTime ).count() << " seconds\n"; + case LgbEntryType::EventObject: + { + pcbTransformModel( fileName, &pEntry->header.scale, &pEntry->header.rotation, &pEntry->header.translation ); + } + break; + default: + break; + } + } + exportMgr.exportGroup( zoneName, exportedGroup, ( ExportFileType )exportFileType ); + //exportedZone.groups.emplace( group.name, exportedGroup ); + } + } + //exportMgr.exportZone( exportedZone, ( ExportFileType )exportFileType ); + + + printf( "Exported %s in %u seconds \n", + zoneName.c_str(), + std::chrono::duration_cast< std::chrono::seconds >( std::chrono::high_resolution_clock::now() - entryStartTime ) ); } catch( std::exception& e ) { - std::cout << "[Error] " << e.what() << std::endl; - std::cout << "[Error] " - << "Unable to extract collision data.\n" - << std::endl; - std::cout << std::endl; - std::cout << "[Info] " << "Usage: pcb_reader2 territory \"path/to/game/sqpack/ffxiv\" " << std::endl; + printf( ( std::string( e.what() ) + "\n" ).c_str() ); + printf( "Unable to extract collision data.\n" ); + printf( "Usage: pcb_reader2 territory \"path/to/game/sqpack/ffxiv\"\n" ); } } exportMgr.waitForTasks(); std::cout << "\n\n\n"; - std::cout << "\n\n\n[Success] Finished all tasks in " << - std::chrono::duration_cast< std::chrono::seconds >( std::chrono::high_resolution_clock::now() - startTime ).count() - << " seconds\n"; + printf( "Finished all tasks in %u seconds\n", + std::chrono::duration_cast< std::chrono::seconds >( std::chrono::high_resolution_clock::now() - startTime ).count() ); getchar(); diff --git a/src/tools/pcb_reader/navmesh_exporter.h b/src/tools/pcb_reader/navmesh_exporter.h index d4580d95..be3fb349 100644 --- a/src/tools/pcb_reader/navmesh_exporter.h +++ b/src/tools/pcb_reader/navmesh_exporter.h @@ -1,5 +1,5 @@ -#ifndef OBJ_EXPORTER_H -#define OBJ_EXPORTER_H +#ifndef NAVMESH_EXPORTER_H +#define NAVMESH_EXPORTER_H #include #include @@ -10,7 +10,12 @@ #include "exporter.h" -static class ObjExporter : public Exporter +#include +#include +#include +#include + +class NavmeshExporter : public Exporter { public: static void exportZone( const ExportedZone& zone ) @@ -20,8 +25,9 @@ public: auto fileName = zone.name + ".obj"; auto end = std::chrono::high_resolution_clock::now(); - std::cout << ( "Finished exporting " + fileName + " in " + - std::to_string( std::chrono::duration_cast< std::chrono::seconds >( end - start ).count() ) + " seconds\n" ); + printf( "[Navmesh] Finished exporting %s in %u ms\n", + fileName, + std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() ); } static void exportGroup( const std::string& zoneName, const ExportedGroup& group ) @@ -32,8 +38,9 @@ public: auto end = std::chrono::high_resolution_clock::now(); - std::cout << ( "Finished exporting " + fileName + " in " + - std::to_string( std::chrono::duration_cast< std::chrono::seconds >( end - start ).count() ) + " seconds\n" ); + printf( "[Navmesh] Finished exporting %s in %u ms\n", + fileName, + std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() ); } private: static void exportGroup( const ExportedGroup& group, std::ofstream& of, int& indicesOffset, int& modelCount ) diff --git a/src/tools/pcb_reader/obj_exporter.h b/src/tools/pcb_reader/obj_exporter.h index 0ff70213..2a39850d 100644 --- a/src/tools/pcb_reader/obj_exporter.h +++ b/src/tools/pcb_reader/obj_exporter.h @@ -1,22 +1,38 @@ #ifndef OBJ_EXPORTER_H #define OBJ_EXPORTER_H +#include #include +#include #include #include -#include #include "exporter.h" + class ObjExporter : public Exporter { public: static void exportZone( const ExportedZone& zone ) { + static std::string currPath = std::experimental::filesystem::current_path().string(); + auto start = std::chrono::high_resolution_clock::now(); - auto fileName = zone.name + ".obj"; + auto dir = currPath + "/" + zone.name + "/"; + auto fileName = dir + "/" + zone.name + ".obj"; + + std::error_code e; + + if( !std::experimental::filesystem::exists( dir, e ) ) + { + if( !std::experimental::filesystem::create_directory( dir, e ) ) + { + printf( "Unable to create directory '%s'", ( dir ).c_str() ); + return; + } + } std::ofstream of( fileName, std::ios::trunc ); int indicesOffset = 0; int meshesCount = 0; @@ -34,15 +50,29 @@ public: } auto end = std::chrono::high_resolution_clock::now(); - std::cout << ( "Finished exporting " + fileName + " in " + - std::to_string( std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() ) + "ms \n" ); + printf( "[Obj] Finished exporting %s in %u ms\n", + fileName, + std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() ); } static void exportGroup( const std::string& zoneName, const ExportedGroup& group ) { + static std::string currPath = std::experimental::filesystem::current_path().string(); + auto start = std::chrono::high_resolution_clock::now(); - auto fileName = zoneName + "_" + group.name + ".obj"; + auto dir = currPath + "/" + zoneName + "/"; + auto fileName = dir + "/" + group.name + ".obj"; + + std::error_code e; + if( !std::experimental::filesystem::exists( dir, e ) ) + { + if( !std::experimental::filesystem::create_directory( dir, e ) ) + { + printf( "Unable to create directory '%s'", ( dir ).c_str() ); + return; + } + } std::ofstream of( fileName, std::ios::trunc ); int indicesOffset = 0; int modelCount = 0; @@ -57,42 +87,47 @@ public: } auto end = std::chrono::high_resolution_clock::now(); - std::cout << ( "Finished exporting " + fileName + " in " + - std::to_string( std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() ) + "ms\n" ); + printf( "[Obj] Finished exporting %s in %u ms\n", + fileName.c_str(), + std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() ); } private: static void exportGroup( const ExportedGroup& group, std::ofstream& of, int& indicesOffset, int& modelCount ) { int currModelCount = modelCount; - //of << ( "o " + group.name + "_" + std::to_string( currModelCount ) + "\n" ); + of << "o " << group.name << '_' << std::to_string( currModelCount ) << '\n'; for( const auto& model : group.models ) { - of << ( "o " + model.second.name + "_" + std::to_string( currModelCount ) + "_" + std::to_string( modelCount++ ) + "\n" ); + modelCount++; + of << "o " << model.second.name << '_' << std::to_string( currModelCount ) << '_' << std::to_string( modelCount ) << '\n'; int meshCount = 0; for( const auto& mesh : model.second.meshes ) { - for( int i = 0; i + 2 < mesh.verts.size(); i += 3 ) + for( int i = 0; i < mesh.verts.size(); i += 3 ) { - of << ( - "v " + std::to_string( mesh.verts[ i ] ) + " " + - std::to_string( mesh.verts[ i + 1 ] ) + " " + - std::to_string( mesh.verts[ i + 2 ] ) + "\n" - ); + of << "v " << + std::to_string( mesh.verts[ i ] ) << ' ' << + std::to_string( mesh.verts[ i + 1 ] ) << ' ' << + std::to_string( mesh.verts[ i + 2 ] ) << '\n'; } - //of << ( "g " + model.second.name + "_" + std::to_string( currModelCount ) + "_" + std::to_string( modelCount ) + "_" + std::to_string( meshCount++ ) + "\n" ); - for( int i = 0; i + 2 < mesh.indices.size(); i += 3 ) + + of << "g " << + model.second.name << '_' << + std::to_string( currModelCount ) << '_' << std::to_string( modelCount ) << '_' << std::to_string( meshCount++ ) << '\n'; + + for( int i = 0; i < mesh.indices.size(); i += 3 ) { - of << ( - "f " + std::to_string( mesh.indices[ i ] + indicesOffset + 1 ) + " " + - std::to_string( mesh.indices[i + 1] + indicesOffset + 1 ) + " " + - std::to_string( mesh.indices[i + 2] + indicesOffset + 1 ) + "\n" - ); + of << "f " << + std::to_string( mesh.indices[ i ] + indicesOffset + 1 ) << ' ' << + std::to_string( mesh.indices[ i + 1 ] + indicesOffset + 1 ) << ' ' + + std::to_string( mesh.indices[ i + 2 ] + indicesOffset + 1 ) << '\n'; } - indicesOffset += mesh.indices.size(); + indicesOffset += mesh.verts.size() / 3; } } + //of.flush(); } }; #endif // !OBJ_EXPORTER_H diff --git a/src/tools/pcb_reader/sgb.h b/src/tools/pcb_reader/sgb.h index c6972a84..6e471be5 100644 --- a/src/tools/pcb_reader/sgb.h +++ b/src/tools/pcb_reader/sgb.h @@ -213,7 +213,7 @@ struct SGB_FILE } catch( std::exception& e ) { - std::cout << e.what() << "\n"; + std::cout << ( std::string( e.what() ) + "\n" ); } }; }; diff --git a/src/tools/pcb_reader/threadpool.h b/src/tools/pcb_reader/threadpool.h index 23c542bd..7ae96891 100644 --- a/src/tools/pcb_reader/threadpool.h +++ b/src/tools/pcb_reader/threadpool.h @@ -13,12 +13,9 @@ class ThreadPool { public: - ThreadPool( unsigned int numJobs = std::thread::hardware_concurrency() ) + ThreadPool() { - for( auto i = 0; i < numJobs; ++i ) - { - m_workers.push_back( std::async( std::launch::async, [this]{ run(); } ) ); - } + } ~ThreadPool() @@ -26,6 +23,17 @@ public: complete(); } + void addWorkers( unsigned int num ) + { + if( num == 0 ) + num = std::thread::hardware_concurrency() - 1; + + for( auto i = 0; i < num; ++i ) + { + m_workers.push_back( std::async( std::launch::async, [this]{ run(); } ) ); + } + } + template< class Func, class Ret = std::result_of_t< Func&() > > std::future< Ret > queue( Func&& f ) { @@ -50,10 +58,12 @@ public: bool complete() { - std::unique_lock lock( m_mutex ); - for( auto&& worker : m_workers ) { - m_pendingJobs.push_back( {} ); + std::scoped_lock lock( m_mutex ); + for( auto&& worker : m_workers ) + { + m_pendingJobs.push_back( {} ); + } } m_cv.notify_all(); m_workers.clear(); From 3bbf8fe2aaa038b3d84278c2672ae426b73548f4 Mon Sep 17 00:00:00 2001 From: Tahir Akhlaq Date: Sun, 20 Jan 2019 21:24:36 +0000 Subject: [PATCH 4/4] hopefully builds on linux this time --- src/tools/pcb_reader/exporter.h | 13 ------------- src/tools/pcb_reader/navmesh_exporter.h | 6 +++--- src/tools/pcb_reader/obj_exporter.h | 4 ++-- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/src/tools/pcb_reader/exporter.h b/src/tools/pcb_reader/exporter.h index c0b8a252..5d738cb9 100644 --- a/src/tools/pcb_reader/exporter.h +++ b/src/tools/pcb_reader/exporter.h @@ -52,17 +52,4 @@ struct ExportedZone std::map< std::string, ExportedGroup > groups; }; -class Exporter -{ -public: - Exporter() { } - ~Exporter(){} - - virtual void exportZone( const ExportedZone& zone ) = 0; - virtual void exportGroup( const ExportedGroup& group ) = 0; - -protected: - ExportFileType m_exportFileType; -}; - #endif \ No newline at end of file diff --git a/src/tools/pcb_reader/navmesh_exporter.h b/src/tools/pcb_reader/navmesh_exporter.h index be3fb349..ca4763d5 100644 --- a/src/tools/pcb_reader/navmesh_exporter.h +++ b/src/tools/pcb_reader/navmesh_exporter.h @@ -15,7 +15,7 @@ #include #include -class NavmeshExporter : public Exporter +class NavmeshExporter { public: static void exportZone( const ExportedZone& zone ) @@ -26,7 +26,7 @@ public: auto end = std::chrono::high_resolution_clock::now(); printf( "[Navmesh] Finished exporting %s in %u ms\n", - fileName, + fileName.c_str(), std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() ); } @@ -39,7 +39,7 @@ public: auto end = std::chrono::high_resolution_clock::now(); printf( "[Navmesh] Finished exporting %s in %u ms\n", - fileName, + fileName.c_str(), std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() ); } private: diff --git a/src/tools/pcb_reader/obj_exporter.h b/src/tools/pcb_reader/obj_exporter.h index 2a39850d..378e14fb 100644 --- a/src/tools/pcb_reader/obj_exporter.h +++ b/src/tools/pcb_reader/obj_exporter.h @@ -11,7 +11,7 @@ #include "exporter.h" -class ObjExporter : public Exporter +class ObjExporter { public: static void exportZone( const ExportedZone& zone ) @@ -51,7 +51,7 @@ public: auto end = std::chrono::high_resolution_clock::now(); printf( "[Obj] Finished exporting %s in %u ms\n", - fileName, + fileName.c_str(), std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() ); }