From cb5f8067d6331f10b1a1ddef95069bf365aa97c3 Mon Sep 17 00:00:00 2001 From: Tahir Date: Mon, 16 Jan 2023 02:15:08 +0000 Subject: [PATCH] fix nav_export for 3.05 - re-add multithreaded exporting and make it eat less ram --- src/tools/nav_export/exportmgr.h | 2 +- src/tools/nav_export/main.cpp | 23 +++++++++++++++-------- src/tools/nav_export/pcb.h | 9 ++++----- src/tools/nav_export/threadpool.h | 11 +++++++++-- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/tools/nav_export/exportmgr.h b/src/tools/nav_export/exportmgr.h index b2f5a67f..ed822d23 100644 --- a/src/tools/nav_export/exportmgr.h +++ b/src/tools/nav_export/exportmgr.h @@ -37,7 +37,7 @@ public: if( exportFileTypes & ExportFileType::Navmesh ) NavmeshExporter::exportZone( zone ); - } ); + }, true ); } void waitForTasks() diff --git a/src/tools/nav_export/main.cpp b/src/tools/nav_export/main.cpp index c42f8c17..fc4d0565 100644 --- a/src/tools/nav_export/main.cpp +++ b/src/tools/nav_export/main.cpp @@ -34,7 +34,7 @@ using namespace Sapphire; // garbage to ignore models bool noObj = false; -std::string gamePath( "C:\\SquareEnix\\FINAL FANTASY XIV - A Realm Reborn\\game\\sqpack" ); +std::string gamePath( "H:\\Games\\ffxiv3.05\\game\\sqpack" ); std::unordered_map< uint16_t, std::string > zoneNameMap; std::map< std::string, std::string > exportedTeriMap; @@ -82,7 +82,7 @@ std::string getEobjSgbPath( uint32_t eobjId ) return exportedSgMap[ eobjSgbPaths[ eobjId ] ]; auto& eobjCat = eData->get_category( "EObj" ); - auto eObjExd = static_cast< xiv::exd::Exd >( eobjCat.get_data_ln( xiv::exd::Language::none ) ); + auto eObjExd = static_cast< xiv::exd::Exd >( eobjCat.get_data_ln( xiv::exd::Language::en ) ); auto& exportedSgCat = eData->get_category( "ExportedSG" ); auto exportedSgExd = static_cast< xiv::exd::Exd >( exportedSgCat.get_data_ln( xiv::exd::Language::none ) ); @@ -300,6 +300,9 @@ void exportSgbModel( const std::string& sgbFilePath, LgbEntry* pGimmick, Exporte int main( int argc, char* argv[] ) { + printf("Usage: nav_export \"path/to/FINAL FANTASY XIV - A REALM REBORN/game/sqpack\" \n" \ + "- default is 4, memory usage may increase to multiple GB as this increases. ~3.5GB RAM using 16 jobs.\n"); + auto startTime = std::chrono::high_resolution_clock::now(); auto entryStartTime = std::chrono::high_resolution_clock::now(); @@ -319,6 +322,9 @@ int main( int argc, char* argv[] ) if( argc > 1 ) gamePath = std::string( argv[ 1 ] ); + if( argc > 2 ) + nJobs = std::atoi( argv[ 2 ] ); + try { initExd( gamePath ); @@ -326,9 +332,10 @@ int main( int argc, char* argv[] ) } catch( std::exception& e ) { - printf( "Unable to initialise EXD!\n Usage: nav_export \"path/to/FINAL FANTASY XIV - A REALM REBORN/game/sqpack\"\n" ); + printf( "Unable to initialise EXD!\n" ); return -1; } + ExportMgr exportMgr( nJobs ); zoneNameToPath( zoneName ); @@ -475,17 +482,17 @@ int main( int argc, char* argv[] ) exportedZone.groups.emplace( group.name, exportedGroup ); } } + + // :( + if( zoneCount % nJobs == 0 ) + pCache->purge(); + exportMgr.exportZone( exportedZone, static_cast< ExportFileType >( exportFileType ) ); exportedZone.groups.clear(); printf( "Built export struct for %s in %lu seconds \n", zoneName.c_str(), std::chrono::duration_cast< std::chrono::seconds >( std::chrono::high_resolution_clock::now() - entryStartTime ).count() ); - //if( zoneCount++ % nJobs == 0 ) - { - exportMgr.restart(); - pCache->purge(); - } } catch( std::exception& e ) { diff --git a/src/tools/nav_export/pcb.h b/src/tools/nav_export/pcb.h index 647c56ca..3b7f1fdb 100644 --- a/src/tools/nav_export/pcb.h +++ b/src/tools/nav_export/pcb.h @@ -43,7 +43,6 @@ struct PCB_INDEXDATA { uint8_t index[3]; uint8_t unknown[3]; - uint8_t unknown1[6]; }; struct PCB_VERTEXDATAI16 @@ -104,7 +103,7 @@ struct PCB_FILE bool isgroup = true; while( isgroup ) { - PCB_BLOCK_ENTRY block_entry; + PCB_BLOCK_ENTRY block_entry{0}; memcpy( &block_entry.header, data + offset, sizeof( block_entry.header ) ); isgroup = block_entry.header.type == 0x30; @@ -117,10 +116,10 @@ struct PCB_FILE } else { - /* printf( "\tnum_v16: %i, num_indices: %i, num_vertices: %i\n\n", + /*printf( "\tnum_v16: %i, num_indices: %i, num_vertices: %i\n\n", block_entry.header.num_v16, block_entry.header.num_indices, block_entry.header.num_vertices );*/ int doffset = sizeof( block_entry.header ) + offset; - uint16_t block_size = sizeof( block_entry.header ) + + uint32_t block_size = sizeof( block_entry.header ) + block_entry.header.num_vertices * 3 * 4 + block_entry.header.num_v16 * 6 + block_entry.header.num_indices * 6; @@ -143,7 +142,7 @@ struct PCB_FILE if( block_entry.header.num_indices != 0 ) { block_entry.data.indices.resize( block_entry.header.num_indices ); - int32_t size_indexbuffer = block_entry.header.num_indices * 12; + int32_t size_indexbuffer = block_entry.header.num_indices * 6; memcpy( &block_entry.data.indices[ 0 ], data + doffset, size_indexbuffer ); doffset += size_indexbuffer; } diff --git a/src/tools/nav_export/threadpool.h b/src/tools/nav_export/threadpool.h index 78657bd3..9da3f63e 100644 --- a/src/tools/nav_export/threadpool.h +++ b/src/tools/nav_export/threadpool.h @@ -41,7 +41,7 @@ public: } template< class Func, class Ret = std::result_of_t< Func&() > > - std::future< Ret > queue( Func&& f ) + std::future< Ret > queue( Func&& f, bool waitForWorker = false ) { std::packaged_task< Ret() > task( std::forward< Func >( f ) ); auto ret = task.get_future(); @@ -50,6 +50,12 @@ public: m_pendingJobs.emplace_back( std::move( task ) ); } m_cv.notify_one(); + + if( waitForWorker ) + { + std::unique_lock lock( m_mutex ); + m_cv2.wait( lock, [&]() { return m_pendingJobs.size() <= m_workers.size(); } ); + } return ret; } @@ -95,12 +101,13 @@ private: return; } func(); + m_cv2.notify_all(); } } bool m_runFlag{ true }; std::mutex m_mutex; - std::condition_variable m_cv; + std::condition_variable m_cv, m_cv2; std::deque< std::packaged_task< void() > > m_pendingJobs; std::vector< std::future< void > > m_workers; };