From ebeb5db3ba78faa2440318ffa72c547a4f54f16c Mon Sep 17 00:00:00 2001 From: Tahir Akhlaq Date: Sat, 26 Jan 2019 15:39:20 +0000 Subject: [PATCH 1/2] pcb_reader: - try free up some memory (recast still leaking a lot though) --- src/tools/pcb_reader/cache.h | 17 +++++++++++++---- src/tools/pcb_reader/exportmgr.h | 17 ++++++++++------- src/tools/pcb_reader/lgb.h | 2 +- src/tools/pcb_reader/main.cpp | 5 ++++- .../pcb_reader/nav/TiledNavmeshGenerator.cpp | 17 +++++++++++------ src/tools/pcb_reader/navmesh_exporter.h | 11 ++++++----- 6 files changed, 45 insertions(+), 24 deletions(-) diff --git a/src/tools/pcb_reader/cache.h b/src/tools/pcb_reader/cache.h index 3821fd07..b9f79226 100644 --- a/src/tools/pcb_reader/cache.h +++ b/src/tools/pcb_reader/cache.h @@ -65,8 +65,20 @@ public: return pFile; } + void purge() + { + std::scoped_lock lock( m_mutex ); + _purge(); + } private: + void _purge() + { + m_lgbCache.clear(); + m_sgbCache.clear(); + m_pcbCache.clear(); + //std::cout << "Purged PCB/SGB/LGB cache \n"; + } template< typename T > std::shared_ptr< T > loadFile( const std::string& filepath ) { @@ -80,10 +92,7 @@ private: m_totalFiles++; if( m_totalFiles % 1000 == 0 ) { - m_lgbCache.clear(); - m_sgbCache.clear(); - m_pcbCache.clear(); - std::cout << "Purged PCB/SGB/LGB cache \n"; + _purge(); m_totalFiles = 1; } diff --git a/src/tools/pcb_reader/exportmgr.h b/src/tools/pcb_reader/exportmgr.h index c448b5d2..5d378e59 100644 --- a/src/tools/pcb_reader/exportmgr.h +++ b/src/tools/pcb_reader/exportmgr.h @@ -32,14 +32,17 @@ public: void exportGroup( const std::string& zoneName, const ExportedGroup& group, ExportFileType exportFileTypes ) { - if( exportFileTypes & ExportFileType::WavefrontObj ) + m_threadpool.queue( [zoneName, group, exportFileTypes]() { - m_threadpool.queue( [zoneName, group](){ ObjExporter::exportGroup( zoneName, group ); } ); - } -// if( exportFileTypes & ExportFileType::Navmesh ) -// { -// m_threadpool.queue( [zoneName, group](){ NavmeshExporter::exportGroup( zoneName, group ); } ); -// } + if( exportFileTypes & ExportFileType::WavefrontObj ) + { + ObjExporter::exportGroup( zoneName, group ); + } + if( exportFileTypes & ExportFileType::Navmesh ) + { + NavmeshExporter::exportGroup( zoneName, group ); + } + }); } void waitForTasks() diff --git a/src/tools/pcb_reader/lgb.h b/src/tools/pcb_reader/lgb.h index d59feb2c..b363dcb3 100644 --- a/src/tools/pcb_reader/lgb.h +++ b/src/tools/pcb_reader/lgb.h @@ -335,7 +335,7 @@ struct LGB_GROUP entries.push_back( std::make_shared< LGB_EOBJ_ENTRY >( buf, entryOffset ) ); break; case LgbEntryType::CollisionBox: - entries.push_back( std::make_shared< LGB_COLLISION_BOX_ENTRY >( buf, entryOffset ) ); + //entries.push_back( std::make_shared< LGB_COLLISION_BOX_ENTRY >( buf, entryOffset ) ); break; default: //std::cout << "\t\tUnknown SGB entry! Group: " << name << " type: " << ( int )type << " index: " << i << " entryOffset: " << entryOffset << "\n"; diff --git a/src/tools/pcb_reader/main.cpp b/src/tools/pcb_reader/main.cpp index 0856e7e5..d4eab40b 100644 --- a/src/tools/pcb_reader/main.cpp +++ b/src/tools/pcb_reader/main.cpp @@ -206,6 +206,7 @@ int main( int argc, char* argv[] ) zoneDumpList.emplace( zoneName ); } + int zoneCount = 0; for( auto zoneName : zoneDumpList ) { try @@ -495,7 +496,8 @@ int main( int argc, char* argv[] ) } } exportMgr.exportZone( exportedZone, static_cast< ExportFileType >( exportFileType ) ); - + if( zoneCount++ % 3 == 0 ) + pCache->purge(); printf( "Exported %s in %lu seconds \n", zoneName.c_str(), @@ -508,6 +510,7 @@ int main( int argc, char* argv[] ) printf( "Usage: pcb_reader2 territory \"path/to/game/sqpack/ffxiv\"\n" ); } } + pCache->purge(); exportMgr.waitForTasks(); std::cout << "\n\n\n"; diff --git a/src/tools/pcb_reader/nav/TiledNavmeshGenerator.cpp b/src/tools/pcb_reader/nav/TiledNavmeshGenerator.cpp index 55f00c19..a120d116 100644 --- a/src/tools/pcb_reader/nav/TiledNavmeshGenerator.cpp +++ b/src/tools/pcb_reader/nav/TiledNavmeshGenerator.cpp @@ -87,14 +87,19 @@ bool TiledNavmeshGenerator::init( const std::string& path ) TiledNavmeshGenerator::~TiledNavmeshGenerator() { - if( m_mesh ) - delete m_mesh; - if( m_chunkyMesh ) - delete m_chunkyMesh; + delete m_mesh; + delete m_chunkyMesh; - if( m_ctx ) - delete m_ctx; + if( m_triareas ) + delete[] m_triareas; + if( m_solid ) + delete m_solid; + delete m_ctx; + rcFreeContourSet( m_cset ); + rcFreeCompactHeightfield(m_chf); + rcFreePolyMesh( m_pmesh ); + rcFreePolyMeshDetail( m_dmesh ); dtFreeNavMesh( m_navMesh ); dtFreeNavMeshQuery( m_navQuery ); } diff --git a/src/tools/pcb_reader/navmesh_exporter.h b/src/tools/pcb_reader/navmesh_exporter.h index 79608607..29e4cecd 100644 --- a/src/tools/pcb_reader/navmesh_exporter.h +++ b/src/tools/pcb_reader/navmesh_exporter.h @@ -27,14 +27,14 @@ public: auto dir = fs::current_path().string() + "/pcb_export/" + zone.name + "/"; auto fileName = dir + zone.name; + auto objName = fileName + ".obj"; std::error_code e; - if( !fs::exists( fileName, e ) ) + if( !fs::exists( objName, e ) ) ObjExporter::exportZone( zone ); - + { TiledNavmeshGenerator gen; - auto objName = fileName + ".obj"; if( !gen.init( objName ) ) { printf( "[Navmesh] failed to init TiledNavmeshGenerator for file '%s'\n", zone.name.c_str() ); @@ -48,6 +48,7 @@ public: } gen.saveNavmesh( zone.name ); + } auto end = std::chrono::high_resolution_clock::now(); printf( "[Navmesh] Finished exporting %s in %lu ms\n", @@ -64,15 +65,15 @@ public: auto dir = fs::current_path().string() + "/pcb_export/" + zoneName + "/"; auto fileName = dir + zoneName + "_" + group.name; + auto objName = fileName + ".obj"; std::error_code e; - if( !fs::exists( fileName, e ) ) + if( !fs::exists( objName, e ) ) ObjExporter::exportGroup( zoneName, group ); TiledNavmeshGenerator gen; - auto objName = fileName + ".obj"; if( !gen.init( objName ) ) { printf( "[Navmesh] failed to init TiledNavmeshGenerator for file '%s'\n", fileName.c_str() ); From 9711cc475dd91da8dd8b80c80adbe8ff976da789 Mon Sep 17 00:00:00 2001 From: Tahir Akhlaq Date: Sat, 26 Jan 2019 16:07:07 +0000 Subject: [PATCH 2/2] pcb_reader: - block main export thread while tasks are pending --- src/tools/pcb_reader/exportmgr.h | 10 ++++++++ src/tools/pcb_reader/main.cpp | 43 ++++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/tools/pcb_reader/exportmgr.h b/src/tools/pcb_reader/exportmgr.h index 5d378e59..f4d7f931 100644 --- a/src/tools/pcb_reader/exportmgr.h +++ b/src/tools/pcb_reader/exportmgr.h @@ -18,6 +18,16 @@ public: waitForTasks(); } + void restart( bool cancel = false, unsigned int maxJobs = 0 ) + { + if( cancel ) + m_threadpool.cancel(); + + m_threadpool.complete(); + + m_threadpool.addWorkers( maxJobs ); + } + void exportZone(const ExportedZone& zone, ExportFileType exportFileTypes) { m_threadpool.queue( [zone, exportFileTypes]() diff --git a/src/tools/pcb_reader/main.cpp b/src/tools/pcb_reader/main.cpp index d4eab40b..8b44878e 100644 --- a/src/tools/pcb_reader/main.cpp +++ b/src/tools/pcb_reader/main.cpp @@ -156,16 +156,36 @@ int main( int argc, char* argv[] ) std::vector< std::string > argVec( argv + 1, argv + argc ); std::string zoneName = "r2t2"; - noObj = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ) + noObj = std::find_if( argVec.begin(), argVec.end(), []( auto arg ) { return arg == "--no-obj"; } ) != argVec.end(); - bool dumpAllZones = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ) + bool dumpAllZones = std::find_if( argVec.begin(), argVec.end(), []( auto arg ) { return arg == "--dump-all"; } ) != argVec.end(); - bool generateNavmesh = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ) + bool generateNavmesh = std::find_if( argVec.begin(), argVec.end(), []( auto arg ) { return arg == "--navmesh"; } ) != argVec.end(); - bool splitByGroup = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ) + bool splitByGroup = std::find_if( argVec.begin(), argVec.end(), []( auto arg ) { return arg == "--split-by-group"; }) != argVec.end(); - bool splitByZone = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ) + bool splitByZone = std::find_if( argVec.begin(), argVec.end(), []( auto arg ) { return arg == "--split-by-zone"; }) != argVec.end(); + int nJobs = std::thread::hardware_concurrency(); + if( auto it = std::find_if( argVec.begin(), argVec.end(), []( auto arg ) + { return arg == "--jobs"; }); it != argVec.end() ) + { + + try + { + auto it2 = ( it + 1 ); + nJobs = std::stoi( *it2 ); + if( nJobs < 0 ) + throw std::runtime_error( "Number of jobs < 0\n" ); + } + catch( std::exception& e ) + { + std::cout << e.what() << "\n"; + std::cout << "Invalid number of jobs " << nJobs << "\n"; + std::cout << "--jobs \n"; + return -1; + } + } int exportFileType = 0; if( !noObj ) exportFileType |= ExportFileType::WavefrontObj; @@ -190,10 +210,10 @@ int main( int argc, char* argv[] ) } catch( std::exception& e ) { - printf( "Unable to initialise EXD!\n Usage: pcb_reader \"path/to/FINAL FANTASY XIV - A REALM REBORN/game/sqpack\" [--no-obj, --dump-all, --navmesh]\n" ); + printf( "Unable to initialise EXD!\n Usage: pcb_reader \"path/to/FINAL FANTASY XIV - A REALM REBORN/game/sqpack\" [--no-obj, --dump-all, --navmesh, --jobs #]\n" ); return -1; } - ExportMgr exportMgr; + ExportMgr exportMgr( nJobs ); zoneNameToPath( zoneName ); if( dumpAllZones ) @@ -496,12 +516,15 @@ int main( int argc, char* argv[] ) } } exportMgr.exportZone( exportedZone, static_cast< ExportFileType >( exportFileType ) ); - if( zoneCount++ % 3 == 0 ) - pCache->purge(); - printf( "Exported %s in %lu seconds \n", + printf( "Built export struct for %s in %lu seconds \n", zoneName.c_str(), std::chrono::duration_cast< std::chrono::seconds >( std::chrono::high_resolution_clock::now() - entryStartTime ).count() ); + if( zoneCount++ % nJobs == 0 ) + { + exportMgr.restart(); + pCache->purge(); + } } catch( std::exception& e ) {