re-add multithreaded exporting and make it eat less ram

src/tools/nav_export/exportmgr.h

@@ -37,7 +37,7 @@ public:
 
       if( exportFileTypes & ExportFileType::Navmesh )
         NavmeshExporter::exportZone( zone );
-    } );
+    }, true );
   }
 
   void waitForTasks()

src/tools/nav_export/main.cpp

@@ -82,7 +82,7 @@ std::string getEobjSgbPath( uint32_t eobjId )
     return exportedSgMap[ eobjSgbPaths[ eobjId ] ];
 
   auto& eobjCat = eData->get_category( "EObj" );
-  auto eObjExd = static_cast< xiv::exd::Exd >( eobjCat.get_data_ln( xiv::exd::Language::none ) );
+  auto eObjExd = static_cast< xiv::exd::Exd >( eobjCat.get_data_ln( xiv::exd::Language::en ) );
 
   auto& exportedSgCat = eData->get_category( "ExportedSG" );
   auto exportedSgExd = static_cast< xiv::exd::Exd >( exportedSgCat.get_data_ln( xiv::exd::Language::none ) );
@@ -300,6 +300,9 @@ void exportSgbModel( const std::string& sgbFilePath, LgbEntry* pGimmick, Exporte
 
 int main( int argc, char* argv[] )
 {
+  printf("Usage: nav_export \"path/to/FINAL FANTASY XIV - A REALM REBORN/game/sqpack\" <jobs>\n" \
+    "- <jobs> default is 4, memory usage may increase to multiple GB as this increases. ~3.5GB RAM using 16 jobs.\n");
+
   auto startTime = std::chrono::high_resolution_clock::now();
   auto entryStartTime = std::chrono::high_resolution_clock::now();
 
@@ -319,6 +322,9 @@ int main( int argc, char* argv[] )
   if( argc > 1 )
     gamePath = std::string( argv[ 1 ] );
 
+  if( argc > 2 )
+    nJobs = std::atoi( argv[ 2 ] );
+
   try
   {
     initExd( gamePath );
@@ -326,9 +332,10 @@ int main( int argc, char* argv[] )
   }
   catch( std::exception& e )
   {
-    printf( "Unable to initialise EXD!\n Usage: nav_export \"path/to/FINAL FANTASY XIV - A REALM REBORN/game/sqpack\"\n" );
+    printf( "Unable to initialise EXD!\n" );
     return -1;
   }
+
   ExportMgr exportMgr( nJobs );
   zoneNameToPath( zoneName );
 
@@ -475,17 +482,17 @@ int main( int argc, char* argv[] )
           exportedZone.groups.emplace( group.name, exportedGroup );
         }
       }
+
+      // :(
+      if( zoneCount % nJobs == 0 )
+        pCache->purge();
+
       exportMgr.exportZone( exportedZone, static_cast< ExportFileType >( exportFileType ) );
       exportedZone.groups.clear();
 
       printf( "Built export struct for %s in %lu seconds \n",
         zoneName.c_str(),
         std::chrono::duration_cast< std::chrono::seconds >( std::chrono::high_resolution_clock::now() - entryStartTime ).count() );
-      //if( zoneCount++ % nJobs == 0 )
-      {
-        exportMgr.restart();
-        pCache->purge();
-      }
     }
     catch( std::exception& e )
     {

src/tools/nav_export/pcb.h

@@ -104,7 +104,7 @@ struct PCB_FILE
     bool isgroup = true;
     while( isgroup )
     {
-      PCB_BLOCK_ENTRY block_entry;
+      PCB_BLOCK_ENTRY block_entry{0};
       memcpy( &block_entry.header, data + offset, sizeof( block_entry.header ) );
       isgroup = block_entry.header.type == 0x30;
   
@@ -117,10 +117,10 @@ struct PCB_FILE
       }
       else
       {
-        /*   printf( "\tnum_v16: %i, num_indices: %i, num_vertices: %i\n\n",
+        /*printf( "\tnum_v16: %i, num_indices: %i, num_vertices: %i\n\n",
                    block_entry.header.num_v16, block_entry.header.num_indices, block_entry.header.num_vertices );*/
         int doffset = sizeof( block_entry.header ) + offset;
-        uint16_t block_size = sizeof( block_entry.header ) +
+        uint32_t block_size = sizeof( block_entry.header ) +
                               block_entry.header.num_vertices * 3 * 4 +
                               block_entry.header.num_v16 * 6 +
                               block_entry.header.num_indices * 6;

src/tools/nav_export/threadpool.h

@@ -41,7 +41,7 @@ public:
   }
 
   template< class Func, class Ret = std::result_of_t< Func&() > >
-  std::future< Ret > queue( Func&& f )
+  std::future< Ret > queue( Func&& f, bool waitForWorker = false )
   {
     std::packaged_task< Ret() > task( std::forward< Func >( f ) );
     auto ret = task.get_future();
@@ -50,6 +50,12 @@ public:
       m_pendingJobs.emplace_back( std::move( task ) );
     }
     m_cv.notify_one();
+
+    if( waitForWorker )
+    {
+      std::unique_lock lock( m_mutex );
+      m_cv2.wait( lock, [&]() { return m_pendingJobs.size() <= m_workers.size(); } );
+    }
     return ret;
   }
 
@@ -95,12 +101,13 @@ private:
         return;
       }
       func();
+      m_cv2.notify_all();
     }
   }
 
   bool m_runFlag{ true };
   std::mutex m_mutex;
-  std::condition_variable m_cv;
+  std::condition_variable m_cv, m_cv2;
   std::deque< std::packaged_task< void() > > m_pendingJobs;
   std::vector< std::future< void > > m_workers;
 };