Merge pull request #2 from takhlaq/develop

pcb reader shit

src/tools/pcb_reader/CMakeLists.txt

@@ -11,9 +11,9 @@ file(GLOB SERVER_SOURCE_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
 add_executable(pcb_reader2 ${SERVER_PUBLIC_INCLUDE_FILES} ${SERVER_SOURCE_FILES})
 
 if (UNIX)
-    target_link_libraries( pcb_reader2 common xivdat pthread mysqlclient dl z stdc++fs Recast Detour )
+    target_link_libraries( pcb_reader2 common xivdat pthread mysqlclient dl z stdc++fs Recast Detour DetourTileCache )
 else()
-    target_link_libraries( pcb_reader2 common xivdat mysql zlib Recast Detour )
+    target_link_libraries( pcb_reader2 common xivdat mysql zlib Recast Detour DetourTileCache )
 endif()
 
 target_include_directories( pcb_reader2

src/tools/pcb_reader/cache.h

@@ -83,7 +83,7 @@ private:
           m_lgbCache.clear();
           m_sgbCache.clear();
           m_pcbCache.clear();
-          std::cout << "Purged PCB/SGB/PCB cache \n";
+          std::cout << "Purged PCB/SGB/LGB cache \n";
           m_totalFiles = 1;
         }
 

src/tools/pcb_reader/lgb.h

@@ -9,6 +9,10 @@
 #include <map>
 #include <string>
 
+#include <iostream>
+#include <sstream>
+#include <iomanip>
+
 #include "matrix4.h"
 #include "vec3.h"
 #include "sgb.h"
@@ -238,6 +242,47 @@ public:
   };
 };
 
+struct LGB_COLLISION_BOX_HEADER :
+  public LGB_ENTRY_HEADER
+{
+  uint8_t unk[100];
+};
+
+struct LGB_COLLISION_BOX_ENTRY :
+  public LGB_ENTRY
+{
+  LGB_COLLISION_BOX_HEADER header;
+  std::string name;
+
+  LGB_COLLISION_BOX_ENTRY( char* buf, uint32_t offset ) :
+    LGB_ENTRY( buf, offset )
+  {
+    header = *reinterpret_cast< LGB_COLLISION_BOX_HEADER* >( buf + offset );
+    header.type = LgbEntryType::CollisionBox;
+    name = std::string( buf + offset + header.nameOffset );
+    std::stringstream ss;
+    ss << "\nName: " << name << "Id: " << header.unknown << "\n";
+    ss << "Pos: " << header.translation.x << " " << header.translation.y << " " << header.translation.z << "\n";
+    ss << "Rot?: " << header.rotation.x << " " << header.rotation.y << " " << header.rotation.z << "\n";
+    ss << "Scale?: " << header.scale.x << " " << header.scale.y << " " << header.scale.z << "\n";
+    ss << "00 01 02 03 04 05 06 07 | 08 09 0A 0B 0C 0D 0E 0F\n";
+    ss << "-------------------------------------------------\n";
+    ss << std::hex;
+    ss << std::setw( 2 );
+    ss << std::setfill( '0' );
+
+    for( auto i = 1; i < sizeof( header.unk ); ++i )
+      if( i % 16 == 0 )
+        ss << std::setw(2) << (int)header.unk[i - 1] << "\n";
+      else if( i % 8 == 0 )
+        ss << std::setw(2) << (int)header.unk[i - 1] << " | ";
+      else
+        ss << std::setw(2) << (int)header.unk[i - 1] << " ";
+    ss << "\n";
+    std::cout << ss.str();
+  }
+};
+
 struct LGB_GROUP_HEADER
 {
   uint32_t unknown;
@@ -289,7 +334,11 @@ struct LGB_GROUP
           case LgbEntryType::EventObject:
             entries.push_back( std::make_shared< LGB_EOBJ_ENTRY >( buf, entryOffset ) );
             break;
+          case LgbEntryType::CollisionBox:
+            entries.push_back( std::make_shared< LGB_COLLISION_BOX_ENTRY >( buf, entryOffset ) );
+            break;
           default:
+            //std::cout << "\t\tUnknown SGB entry! Group: " << name << " type: " << ( int )type << " index: " << i << " entryOffset: " << entryOffset << "\n";
             break;
         }
       }

src/tools/pcb_reader/main.cpp

@@ -34,12 +34,15 @@ bool noObj = false;
 
 std::string gamePath( "/mnt/c/Program Files (x86)/Steam/steamapps/common/FINAL FANTASY XIV Online/game/sqpack" );
 std::unordered_map< uint16_t, std::string > zoneNameMap;
+std::map< std::string, std::string > exportedTeriMap;
+
 uint32_t zoneId;
 
 
 std::set< std::string > zoneDumpList;
 
 std::shared_ptr< Cache > pCache;
+std::map< uint32_t, uint16_t > eobjSgbPaths;
 
 xiv::dat::GameData* data1 = nullptr;
 xiv::exd::ExdData* eData = nullptr;
@@ -61,6 +64,49 @@ void initExd( const std::string& gamePath )
   pCache = std::make_shared< Cache >( data1 );
 }
 
+void replaceAll( std::string& str, const std::string& from, const std::string& to ) {
+  if( from.empty() )
+    return;
+  size_t start_pos = 0;
+  while( ( start_pos = str.find( from, start_pos ) ) != std::string::npos ) {
+    str.replace( start_pos, from.length(), to );
+    start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx'
+  }
+}
+
+std::string getEobjSgbPath( uint32_t eobjId )
+{
+  static std::map< uint16_t, std::string > exportedSgMap;
+
+  if( !exportedSgMap.empty() )
+    return exportedSgMap[ eobjSgbPaths[ eobjId ] ];
+
+  auto& eobjCat = eData->get_category( "EObj" );
+  auto eObjExd = static_cast< xiv::exd::Exd >( eobjCat.get_data_ln( xiv::exd::Language::none ) );
+
+  auto& exportedSgCat = eData->get_category( "ExportedSG" );
+  auto exportedSgExd = static_cast< xiv::exd::Exd >( exportedSgCat.get_data_ln( xiv::exd::Language::none ) );
+
+  for( auto& row : exportedSgExd.get_rows() )
+  {
+    auto id = row.first;
+    auto& fields = row.second;
+
+    auto path = std::get< std::string >( fields.at( 0 ) );
+    exportedSgMap[id] = path;
+  }
+
+  uint16_t exportedSgId{0};
+
+  for( auto& row : eObjExd.get_rows() )
+  {
+    auto id = row.first;
+    auto& fields = row.second;
+
+    eobjSgbPaths[id] = std::get< uint16_t >( fields.at( 11 ) );
+  }
+  return exportedSgMap[exportedSgId];
+}
 
 std::string zoneNameToPath( const std::string& name )
 {
@@ -116,7 +162,10 @@ int main( int argc, char* argv[] )
   { return arg == "--dump-all"; } ) != argVec.end();
   bool generateNavmesh = std::remove_if( argVec.begin(), argVec.end(), []( auto arg )
   { return arg == "--navmesh"; } ) != argVec.end();
-
+  bool splitByGroup = std::remove_if( argVec.begin(), argVec.end(), []( auto arg )
+  { return arg == "--split-by-group"; }) != argVec.end();
+  bool splitByZone = std::remove_if( argVec.begin(), argVec.end(), []( auto arg )
+  { return arg == "--split-by-zone"; }) != argVec.end();
   int exportFileType = 0;
   if( !noObj )
     exportFileType |= ExportFileType::WavefrontObj;
@@ -137,6 +186,7 @@ int main( int argc, char* argv[] )
   try
   {
     initExd( gamePath );
+    getEobjSgbPath( 0 );
   }
   catch( std::exception& e )
   {
@@ -156,14 +206,19 @@ int main( int argc, char* argv[] )
     zoneDumpList.emplace( zoneName );
   }
 
-  for( const auto& zoneName : zoneDumpList )
+  for( auto zoneName : zoneDumpList )
   {
     try
     {
+      const auto& zonePath = zoneNameToPath( zoneName );
+      if( exportedTeriMap.find( zonePath ) != exportedTeriMap.end() )
+        continue;
+
+      zoneName = zonePath.substr( zonePath.find_last_of( '/' ) );
+
       ExportedZone exportedZone;
       exportedZone.name = zoneName;
-
+      exportedTeriMap[ zonePath ] = zoneName;
-      const auto& zonePath = zoneNameToPath( zoneName );
 
       std::string listPcbPath( zonePath + "/collision/list.pcb" );
       std::string bgLgbPath( zonePath + "/level/bg.lgb" );
@@ -324,9 +379,8 @@ int main( int argc, char* argv[] )
         if( auto pPcbFile = pCache->getPcbFile( fileName ) )
           buildModelEntry( pPcbFile, exportedTerrainGroup, fileName, zoneName );
       }
-      exportMgr.exportGroup( zoneName, exportedTerrainGroup, ( ExportFileType )exportFileType );
+      exportedZone.groups.emplace( exportedTerrainGroup.name, exportedTerrainGroup );
-      exportedZone.groups.emplace( zoneName, exportedTerrainGroup );
+      
-
       for( const auto& lgb : lgbList )
       {
         for( const auto& group : lgb.groups )
@@ -352,12 +406,51 @@ int main( int argc, char* argv[] )
               }
               return true;
             };
+            auto exportSgbModel = [&]( const std::string& sgbFilePath, LGB_ENTRY* pGimmick, bool isEobj = false  )
+            {
+              if( auto pSgbFile = pCache->getSgbFile( sgbFilePath ) )
+              {
+                const auto& sgbFile = *pSgbFile;
+                for( const auto& group : sgbFile.entries )
+                {
+                  for( const auto& pSgbEntry : group.entries )
+                  {
+                    auto pModel = dynamic_cast< SGB_MODEL_ENTRY* >( pSgbEntry.get() );
+                    fileName = pModel->collisionFileName;
+                    if( pModel->type == SgbGroupEntryType::Gimmick )
+                    {
+                      if( auto pSubSgbFile = pCache->getSgbFile( pModel->modelFileName ) )
+                      {
+                        for( const auto& subGroup : pSubSgbFile->entries )
+                        {
+                          for( const auto& pSubEntry : subGroup.entries )
+                          {
+                            auto pSubModel = dynamic_cast< SGB_MODEL_ENTRY* >( pSubEntry.get() );
+                            std::string subModelFile = pSubModel->modelFileName;
+                            //"bg/ex1/02_dra_d2/alx/common/bgparts/d2a0_a7_btog2.mdl"
+                            //"bg/ex1/02_dra_d2/alx/common/collision/d2a0_a1_twl01.pcb"
+                           replaceAll( subModelFile, "/bgparts/", "/collision/" );
+                           replaceAll( subModelFile, ".mdl", ".pcb ");
 
+                            if( pSubModel && pSubModel->type == SgbGroupEntryType::Model )
+                              pcbTransformModel( subModelFile, &pGimmick->header.scale, &pGimmick->header.rotation,
+                                &pGimmick->header.translation, pSubModel );
+                          }
+                        }
+                      }
+                    }
+                    pcbTransformModel( fileName, &pGimmick->header.scale, &pGimmick->header.rotation,
+                      &pGimmick->header.translation, pModel );
+
+                  }
+                }
+              }
+            };
             switch( pEntry->getType() )
             {
               case LgbEntryType::BgParts:
               {
-                auto pBgParts = static_cast<LGB_BGPARTS_ENTRY*>(pEntry.get());
+                auto pBgParts = static_cast< LGB_BGPARTS_ENTRY* >( pEntry.get() );
                 fileName = pBgParts->collisionFileName;
                 pcbTransformModel( fileName, &pBgParts->header.scale, &pBgParts->header.rotation,
                   &pBgParts->header.translation );
@@ -367,37 +460,41 @@ int main( int argc, char* argv[] )
               // gimmick entry
               case LgbEntryType::Gimmick:
               {
-                auto pGimmick = static_cast<LGB_GIMMICK_ENTRY*>( pEntry.get() );
+                auto pGimmick = static_cast< LGB_GIMMICK_ENTRY* >( pEntry.get() );
-                if( auto pSgbFile = pCache->getSgbFile( pGimmick->gimmickFileName ) )
+
-                {
+                exportSgbModel( pGimmick->gimmickFileName, pGimmick );
-                  const auto& sgbFile = *pSgbFile;
-                  for( const auto& group : sgbFile.entries )
-                  {
-                    for( const auto& pEntry : group.entries )
-                    {
-                      auto pModel = dynamic_cast< SGB_MODEL_ENTRY* >( pEntry.get() );
-                      fileName = pModel->collisionFileName;
-                      pcbTransformModel( fileName, &pGimmick->header.scale, &pGimmick->header.rotation,
-                        &pGimmick->header.translation, pModel );
-                    }
-                  }
-                }
               }
+              break;
 
               case LgbEntryType::EventObject:
               {
+                auto pEobj = static_cast< LGB_EOBJ_ENTRY* >( pEntry.get() );
                 pcbTransformModel( fileName, &pEntry->header.scale, &pEntry->header.rotation, &pEntry->header.translation );
+
+                auto sgbPath = getEobjSgbPath( pEobj->header.eobjId );
+                if ( !sgbPath.empty() )
+                {
+                  exportSgbModel( sgbPath, pEobj, true );
+
+                  if( auto pGimmick = pCache->getSgbFile( sgbPath ) )
+                  {
+                    for( const auto& offset1cFile : pGimmick->offset1cObjects )
+                      exportSgbModel( offset1cFile, pEobj, true );
+                  }
+                }
+                
               }
               break;
               default:
                 break;
             }
           }
-          exportMgr.exportGroup( zoneName, exportedGroup, ( ExportFileType )exportFileType );
+          if( splitByGroup )
+            exportMgr.exportGroup( zoneName, exportedGroup, ( ExportFileType )exportFileType );
           exportedZone.groups.emplace( group.name, exportedGroup );
         }
       }
-      exportMgr.exportZone( exportedZone, ( ExportFileType )exportFileType );
+      exportMgr.exportZone( exportedZone, ExportFileType::Navmesh );
 
 
       printf( "Exported %s in %lu seconds \n",

src/tools/pcb_reader/navmesh_exporter.h

@@ -18,9 +18,9 @@ namespace fs = std::experimental::filesystem;
 class NavmeshExporter
 {
 public:
-  static void exportZone( const ExportedZone& zone )
+  static void exportZone( const ExportedZone& zone, bool deleteObj = false )
   {
-    auto start = std::chrono::high_resolution_clock::now();
+    static std::string currPath = std::experimental::filesystem::current_path().string();
 
     auto dir = fs::current_path().string() + "/pcb_export/" + zone.name + "/";
     auto fileName = dir + zone.name + ".obj";

src/tools/pcb_reader/obj_exporter.h

@@ -14,7 +14,7 @@
 class ObjExporter
 {
 public:
-  static void exportZone( const ExportedZone& zone )
+  static std::string exportZone( const ExportedZone& zone )
   {
     static std::string currPath = std::experimental::filesystem::current_path().string();
 
@@ -30,7 +30,7 @@ public:
       if( !std::experimental::filesystem::create_directories( dir, e ) )
       {
         printf( "Unable to create directory '%s'", ( dir ).c_str() );
-        return;
+        return "";
       }
     }
     std::ofstream of( fileName, std::ios::trunc );
@@ -50,12 +50,14 @@ public:
     }
 
     auto end = std::chrono::high_resolution_clock::now();
+
     printf( "[Obj] Finished exporting %s in %lu ms\n",
             fileName.substr( fileName.find( "pcb_export" ) - 1 ).c_str(),
             std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() );
+    return fileName;
   }
 
-  static void exportGroup( const std::string& zoneName, const ExportedGroup& group )
+  static std::string exportGroup( const std::string& zoneName, const ExportedGroup& group )
   {
     static std::string currPath = std::experimental::filesystem::current_path().string();
 
@@ -70,7 +72,7 @@ public:
       if( !std::experimental::filesystem::create_directories( dir, e ) )
       {
         printf( "Unable to create directory '%s'", ( dir ).c_str() );
-        return;
+        return "";
       }
     }
     std::ofstream of( fileName, std::ios::trunc );
@@ -90,6 +92,8 @@ public:
     printf( "[Obj] Finished exporting %s in %lu ms\n",
             fileName.substr( fileName.find( "pcb_export" ) - 1 ).c_str(),
             std::chrono::duration_cast< std::chrono::milliseconds >( end - start ).count() );
+    
+    return fileName;
   }
 private:
   static void exportGroup( const ExportedGroup& group, std::ofstream& of, int& indicesOffset, int& modelCount )

src/tools/pcb_reader/sgb.h

@@ -36,6 +36,7 @@ enum SgbGroupEntryType :
   uint32_t
 {
   Model = 0x01,
+  Gimmick = 0x06,
 };
 
 struct SGB_GROUP_HEADER
@@ -64,6 +65,35 @@ struct SGB_GROUP_HEADER
   uint32_t unknown44;
 };
 
+struct SGB_GROUP1C_HEADER
+{
+  SgbDataType type;
+  int32_t nameOffset;
+  uint32_t unknown08;
+  
+  int32_t entryCount;
+  uint32_t unknown14;
+  int32_t modelFileOffset;
+  vec3 unknownFloat3;
+  vec3 unknownFloat3_2;
+  int32_t stateOffset;
+  int32_t modelFileOffset2;
+  uint32_t unknown3;
+  float unknown4;
+  int32_t nameOffset2;
+  vec3 unknownFloat3_3;
+};
+
+struct SGB_GROUP1C_ENTRY
+{
+  uint32_t unk;
+  uint32_t unk2;
+  int32_t nameOffset;
+  uint32_t index;
+  uint32_t unk3;
+  int32_t modelFileOffset;
+};
+
 struct SGB_GROUP_ENTRY
 {
 public:
@@ -113,8 +143,9 @@ struct SGB_MODEL_ENTRY :
   std::string modelFileName;
   std::string collisionFileName;
 
-  SGB_MODEL_ENTRY( char* buf, uint32_t offset )
+  SGB_MODEL_ENTRY( char* buf, uint32_t offset, SgbGroupEntryType type )
   {
+    this->type = type;
     header = *reinterpret_cast< SGB_MODEL_HEADER* >( buf + offset );
     name = std::string( buf + offset + header.nameOffset );
     modelFileName = std::string( buf + offset + header.modelFileOffset );
@@ -129,23 +160,45 @@ struct SGB_GROUP
   SGB_FILE* parent;
   std::vector< std::shared_ptr< SGB_GROUP_ENTRY > > entries;
 
-  SGB_GROUP( char* buf, SGB_FILE* file, uint32_t fileSize, uint32_t offset )
+  SGB_GROUP( char* buf, SGB_FILE* file, std::set< std::string >* offset1cObjects, uint32_t fileSize, uint32_t offset, bool isOffset1C = false )
   {
     parent = file;
+
+
+    if( isOffset1C )
+    {
+      auto header1c = *reinterpret_cast< SGB_GROUP1C_HEADER* >( buf + offset );
+
+      auto entriesOffset = offset + sizeof( header1c );
+
+      auto entryCount = header1c.entryCount;
+      for( auto i = 0; i < entryCount; ++i )
+      {
+        auto entryOffset = entriesOffset + ( i * 24 );
+        auto entry = *reinterpret_cast< SGB_GROUP1C_ENTRY* >( buf + entryOffset );
+
+        std::string entryModelFile( buf + entryOffset + entry.modelFileOffset + 9 );
+        if( entryModelFile.find( ".sgb" ) != std::string::npos )
+        {
+          offset1cObjects->emplace( entryModelFile );
+        }
+      }
+      return;
+    }
+    auto entriesOffset = offset + sizeof( header );
+
     header = *reinterpret_cast< SGB_GROUP_HEADER* >( buf + offset );
     name = std::string( buf + offset + header.nameOffset );
 
-    auto entriesOffset = offset + sizeof( header );
-
     for( auto i = 0; i < header.entryCount; ++i )
     {
       auto entryOffset = entriesOffset + *reinterpret_cast< uint32_t* >( buf + ( entriesOffset + ( i * 4 ) ) );
       if( entryOffset > fileSize )
         throw std::runtime_error( "SGB_GROUP entry offset was larger than SGB file size!" );
       auto type = *reinterpret_cast< uint32_t* >( buf + entryOffset );
-      if( type == SgbGroupEntryType::Model )
+      if( type == SgbGroupEntryType::Model || type == SgbGroupEntryType::Gimmick )
       {
-        entries.push_back( std::make_shared< SGB_MODEL_ENTRY >( buf, entryOffset ) );
+        entries.push_back( std::make_shared< SGB_MODEL_ENTRY >( buf, entryOffset, ( SgbGroupEntryType )type ) );
       }
       else
       {
@@ -190,6 +243,7 @@ struct SGB_FILE
 {
   SGB_HEADER header;
   std::vector< SGB_GROUP > entries;
+  std::set< std::string > offset1cObjects;
 
   SGB_FILE()
   {
@@ -206,9 +260,9 @@ struct SGB_FILE
 
     try
     {
-      auto group = SGB_GROUP( buf, this, header.fileSize, baseOffset + header.sharedOffset );
+      auto group = SGB_GROUP( buf, this, &offset1cObjects, header.fileSize, baseOffset + header.sharedOffset );
       entries.push_back( group );
-      auto group2 = SGB_GROUP( buf, this, header.fileSize, baseOffset + header.offset1C );
+      auto group2 = SGB_GROUP( buf, this, &offset1cObjects, header.fileSize, baseOffset+ header.offset1C, true );
       entries.push_back( group2 );
     }
     catch( std::exception& e )

src/tools/pcb_reader/threadpool.h

@@ -10,6 +10,9 @@
 #include <mutex>
 #include <thread>
 
+// credit to
+// https://riptutorial.com/cplusplus/example/15806/create-a-simple-thread-pool
+
 class ThreadPool
 {
 public:
@@ -55,23 +58,20 @@ public:
     {
       std::unique_lock lock( m_mutex );
       m_pendingJobs.clear();
-      for( auto&& worker : m_workers )
-      {
-        m_pendingJobs.emplace( {} );
-      }
     }
-    m_cv.notify_all();
+    complete();
-    m_workers.clear();
   }
 
   bool complete()
   {
-    m_cv.notify_all();
     {
       std::unique_lock lock( m_mutex );
-      m_runFlag = false;
+      for( auto&& worker : m_workers )
-      m_cv.wait( lock, [&]{ return m_pendingJobs.empty(); } );
+      {
+        m_pendingJobs.push_back( {} );
+      }
     }
+    m_cv.notify_all();
     m_workers.clear();
     return true;
   }
@@ -85,11 +85,6 @@ private:
         std::unique_lock lock( m_mutex );
         if( m_pendingJobs.empty() )
         {
-          if( !m_runFlag )
-          {
-            m_cv.notify_all();
-            return;
-          }
           m_cv.wait( lock, [&](){ return !m_pendingJobs.empty(); } );
         }
         func = std::move( m_pendingJobs.front() );