begin tools cleanup, lgb group dumping

src/common/Exd/StructureDef/lgb.h

@@ -278,7 +278,7 @@ struct LGB_GROUP
       {
         const auto type = *reinterpret_cast<LgbEntryType*>( buf + entryOffset );
         // garbage to skip model loading
-        if( !ignoreModels && type == LgbEntryType::BgParts )
+        if( type == LgbEntryType::BgParts )
         {
           entries.push_back( std::make_shared< LGB_BGPARTS_ENTRY >( buf, entryOffset ) );
         }
@@ -331,9 +331,15 @@ struct LGB_FILE
 {
   LGB_FILE_HEADER header;
   std::vector< LGB_GROUP > groups;
-  std::string name;
+  std::string m_name;
 
-  LGB_FILE( char* buf, const std::string& name )
+  LGB_FILE( char* buf, const std::string& name ) :
+    LGB_FILE( buf )
+  {
+    m_name = name;
+  }
+
+  LGB_FILE( char* buf )
   {
     header = *reinterpret_cast< LGB_FILE_HEADER* >( buf );
     if( strncmp( &header.magic[ 0 ], "LGB1", 4 ) != 0 || strncmp( &header.magic2[ 0 ], "LGP1", 4 ) != 0 )

src/common/Exd/StructureDef/sgb.h

@@ -11,9 +11,6 @@
 
 #include "vec3.h"
 
-// garbage to skip model loading
-extern bool ignoreModels;
-
 //
 // ported from https://github.com/ufx/SaintCoinach/blob/master/SaintCoinach/Graphics/Sgb/SgbDataType.cs
 
@@ -36,6 +33,7 @@ enum SgbGroupEntryType :
   uint32_t
 {
   Model = 0x01,
+  Gimmick = 0x06,
 };
 
 struct SGB_GROUP_HEADER
@@ -64,6 +62,35 @@ struct SGB_GROUP_HEADER
   uint32_t unknown44;
 };
 
+struct SGB_GROUP1C_HEADER
+{
+  SgbDataType type;
+  int32_t nameOffset;
+  uint32_t unknown08;
+
+  int32_t entryCount;
+  uint32_t unknown14;
+  int32_t modelFileOffset;
+  vec3 unknownFloat3;
+  vec3 unknownFloat3_2;
+  int32_t stateOffset;
+  int32_t modelFileOffset2;
+  uint32_t unknown3;
+  float unknown4;
+  int32_t nameOffset2;
+  vec3 unknownFloat3_3;
+};
+
+struct SGB_GROUP1C_ENTRY
+{
+  uint32_t unk;
+  uint32_t unk2;
+  int32_t nameOffset;
+  uint32_t index;
+  uint32_t unk3;
+  int32_t modelFileOffset;
+};
+
 struct SGB_GROUP_ENTRY
 {
 public:
@@ -113,8 +140,9 @@ struct SGB_MODEL_ENTRY :
   std::string modelFileName;
   std::string collisionFileName;
 
-  SGB_MODEL_ENTRY( char* buf, uint32_t offset )
+  SGB_MODEL_ENTRY( char* buf, uint32_t offset, SgbGroupEntryType type )
   {
+    this->type = type;
     header = *reinterpret_cast< SGB_MODEL_HEADER* >( buf + offset );
     name = std::string( buf + offset + header.nameOffset );
     modelFileName = std::string( buf + offset + header.modelFileOffset );
@@ -143,9 +171,9 @@ struct SGB_GROUP
       if( entryOffset > fileSize )
         throw std::runtime_error( "SGB_GROUP entry offset was larger than SGB file size!" );
       auto type = *reinterpret_cast< uint32_t* >( buf + entryOffset );
-      if( type == SgbGroupEntryType::Model && !ignoreModels )
+      if( type == SgbGroupEntryType::Model || type == SgbGroupEntryType::Gimmick )
       {
-        entries.push_back( std::make_shared< SGB_MODEL_ENTRY >( buf, entryOffset ) );
+        entries.push_back( std::make_shared< SGB_MODEL_ENTRY >( buf, entryOffset, ( SgbGroupEntryType )type ) );
       }
       else
       {
@@ -249,5 +277,4 @@ struct SGB_FILE
   };
 };
 
-
+#endif // !_SGB_H
-#endif // !_SGB_H

src/tools/discovery_parser/main.cpp

@@ -46,7 +46,7 @@ std::string zoneName;
 
 std::set< std::string > zoneDumpList;
 
-xiv::dat::GameData* data1 = nullptr;
+xiv::dat::GameData* gameData = nullptr;
 xiv::exd::ExdData* eData = nullptr;
 
 void readFileToBuffer( const std::string& path, std::vector< char >& buf );
@@ -135,8 +135,8 @@ struct face
 // init
 void initExd( const std::string& gamePath )
 {
-  data1 = data1 ? data1 : new xiv::dat::GameData( gamePath );
+  gameData = gameData ? gameData : new xiv::dat::GameData( gamePath );
-  eData = eData ? eData : new xiv::exd::ExdData( *data1 );
+  eData = eData ? eData : new xiv::exd::ExdData( *gameData );
 }
 
 std::string zoneNameToPath( const std::string& name )
@@ -338,13 +338,13 @@ int main( int argc, char* argv[] )
     std::vector< char > section1;
     std::vector< char > section2;
 
-    auto test_file = data1->getFile( bgLgbPath );
+    auto test_file = gameData->getFile( bgLgbPath );
     section = test_file->access_data_sections().at( 0 );
 
-    auto planmap_file = data1->getFile( planmapLgbPath );
+    auto planmap_file = gameData->getFile( planmapLgbPath );
     section2 = planmap_file->access_data_sections().at( 0 );
 
-    auto test_file1 = data1->getFile( listPcbPath );
+    auto test_file1 = gameData->getFile( listPcbPath );
     section1 = test_file1->access_data_sections().at( 0 );
 
     std::vector< std::string > stringList;
@@ -467,7 +467,7 @@ int main( int argc, char* argv[] )
 
   if( eData )
     delete eData;
-  if( data1 )
+  if( gameData )
-    delete data1;
+    delete gameData;
   return 0;
 }

src/tools/event_object_parser/main.cpp

@@ -11,9 +11,9 @@
 #include <variant>
 #include <Util/Util.h>
 
-#include "pcb.h"
+#include <Exd/StructureDef/pcb.h>
-#include "lgb.h"
+#include <Exd/StructureDef/lgb.h>
-#include "sgb.h"
+#include <Exd/StructureDef/sgb.h>
 
 #include <Exd/ExdDataGenerated.h>
 #include <Logging/Logger.h>
@@ -40,7 +40,7 @@ namespace fs = std::experimental::filesystem;
 
 // garbage to ignore models
 bool ignoreModels = false;
-std::string gamePath( "/home/mordred/sqpack" );
+std::string gamePath( "/mnt/c/Program Files (x86)/Steam/steamapps/common/FINAL FANTASY XIV Online/game/sqpack" );
 //std::string gamePath( "C:\\SquareEnix\\FINAL FANTASY XIV - A Realm Reborn\\game\\sqpack" );
 std::unordered_map< uint32_t, std::string > eobjNameMap;
 std::unordered_map< uint16_t, std::vector< std::pair< uint16_t, std::string > > > zoneInstanceMap;
@@ -58,15 +58,15 @@ std::vector< instanceContent > contentList;
 
 std::set< std::string > zoneDumpList;
 
-xiv::dat::GameData* data1 = nullptr;
+xiv::dat::GameData* gameData = nullptr;
 xiv::exd::ExdData* eData = nullptr;
 
 using namespace std::chrono_literals;
 
 void initExd( const std::string& gamePath )
 {
-  data1 = data1 ? data1 : new xiv::dat::GameData( gamePath );
+  gameData = gameData ? gameData : new xiv::dat::GameData( gamePath );
-  eData = eData ? eData : new xiv::exd::ExdData( *data1 );
+  eData = eData ? eData : new xiv::exd::ExdData( *gameData );
 }
 
 int parseBlockEntry( char* data, std::vector< PCB_BLOCK_ENTRY >& entries, int gOff )
@@ -348,19 +348,48 @@ int main( int argc, char* argv[] )
       std::string listPcbPath( zonePath + "/collision/list.pcb" );
       std::string bgLgbPath( zonePath + "/level/bg.lgb" );
       std::string planmapLgbPath( zonePath + "/level/planmap.lgb" );
+      std::string planeventLgbPath( zonePath + "/level/planevent.lgb" );
       std::string collisionFilePath( zonePath + "/collision/" );
+      std::string plannerFilePath( zonePath + "/level/planner.lgb" );
+      std::string lcbFilePath( zonePath + "/level/" + zoneName + ".lcb" );
+      std::string svbFilePath( zonePath + "/level/" + zoneName + ".svb" );
       std::vector< char > section;
       std::vector< char > section1;
       std::vector< char > section2;
+      std::vector< char > section3;
 
-      const xiv::dat::Cat& test = data1->getCategory( "bg" );
+      const xiv::dat::Cat& test = gameData->getCategory( "bg" );
 
-      auto test_file = data1->getFile( bgLgbPath );
+      auto test_file = gameData->getFile( bgLgbPath );
       section = test_file->access_data_sections().at( 0 );
 
-      auto planmap_file = data1->getFile( planmapLgbPath );
+      auto planmap_file = gameData->getFile( planmapLgbPath );
       section2 = planmap_file->access_data_sections().at( 0 );
 
+      auto planeventFile = gameData->getFile( planeventLgbPath );
+      section3 = planeventFile->access_data_sections().at( 0 );
+
+      auto exportFile = [&]( const std::string& path )
+      {
+        try
+        {
+          auto file = gameData->getFile( path );
+
+          if( !file )
+          {
+            return;
+          }
+          auto p = fs::path( path );
+          fs::create_directories( p.parent_path() );
+          file->exportToFile( p );
+        }
+        catch( const std::exception& ex ) {}
+      };
+
+      exportFile( planeventLgbPath );
+      exportFile( plannerFilePath );
+      exportFile( lcbFilePath );
+      exportFile( svbFilePath );
 
       std::vector< std::string > stringList;
 
@@ -370,8 +399,9 @@ int main( int argc, char* argv[] )
 
       LGB_FILE bgLgb( &section[ 0 ], "bg" );
       LGB_FILE planmapLgb( &section2[ 0 ], "planmap" );
+      LGB_FILE planeventLgb( &section3[ 0 ], "planevent" );
 
-      std::vector< LGB_FILE > lgbList{ bgLgb, planmapLgb };
+      std::vector< LGB_FILE > lgbList{ bgLgb, planmapLgb, planeventLgb };
       uint32_t max_index = 0;
 
       //if( ignoreModels )
@@ -385,7 +415,7 @@ int main( int argc, char* argv[] )
           {
             char* dataSection = nullptr;
             //std::cout << fileName << " ";
-            auto file = data1->getFile( fileName );
+            auto file = gameData->getFile( fileName );
             auto sections = file->get_data_sections();
             dataSection = &sections.at( 0 )[ 0 ];
             sgbFile = SGB_FILE( &dataSection[ 0 ] );
@@ -453,7 +483,7 @@ int main( int argc, char* argv[] )
                       char* dataSection = nullptr;
                       //std::cout << fileName << " ";
 
-                      auto file = data1->getFile( pGObjR->gimmickFileName );
+                      auto file = gameData->getFile( pGObjR->gimmickFileName );
                       auto sections = file->get_data_sections();
                       dataSection = &sections.at( 0 )[ 0 ];
                       auto sgbFile = SGB_FILE( &dataSection[ 0 ] );
@@ -487,7 +517,7 @@ int main( int argc, char* argv[] )
                 if( eobjNameMap.find( id ) != eobjNameMap.end() )
                 {
                   name = eobjNameMap[ id ];
-                  std::string remove = ",★_ '()[]-\xae\x1a\x1\x2\x1f\x1\x3.:";
+                  std::string remove = "\",★_ '()[]-\xae\x1a\x1\x2\x1f\x1\x3.:";
                   Sapphire::Util::eraseAllIn( name, remove );
 
                   name[ 0 ] = toupper( name[ 0 ] );
@@ -614,7 +644,7 @@ int main( int argc, char* argv[] )
 
   if( eData )
     delete eData;
-  if( data1 )
+  if( gameData )
-    delete data1;
+    delete gameData;
   return 0;
 }

src/tools/exd_struct_test/main.cpp

@@ -16,18 +16,54 @@
 #include <streambuf>
 #include <regex>
 
+#include <Exd/StructureDef/pcb.h>
+#include <Exd/StructureDef/lgb.h>
+#include <Exd/StructureDef/sgb.h>
+
 #include <Util/CrashHandler.h>
 
+#include <experimental/filesystem>
+
 Sapphire::Common::Util::CrashHandler crashHandler;
 
 Sapphire::Data::ExdDataGenerated g_exdData;
+xiv::dat::GameData* gameData = nullptr;
 
 using namespace Sapphire;
 
+namespace fs = std::experimental::filesystem;
+
 //const std::string datLocation( "/opt/sapphire_3_15_0/bin/sqpack" );
 const std::string datLocation( "/mnt/c/Program Files (x86)/Steam/steamapps/common/FINAL FANTASY XIV Online/game/sqpack" );
 
-int main()
+void exportFile( const std::string& path )
+{
+  try
+  {
+    auto file = gameData->getFile( path );
+
+    if( !file )
+    {
+      return;
+    }
+    auto p = fs::path( path );
+    fs::create_directories( p.parent_path() );
+    file->exportToFile( p );
+  }
+  catch( const std::exception& ex )
+  {
+    Logger::error( "Failed to export file {0}, error: {1}", path, ex.what() );
+  }
+}
+
+struct DupeResult
+{
+  std::string groupName;
+  std::string lgb;
+  uint32_t id;
+};
+
+int main( int argc, char* argv[] )
 {
 
   Logger::init( "struct_test" );
@@ -39,14 +75,97 @@ int main()
     return 0;
   }
 
-  auto gld = g_exdData.get< Sapphire::Data::ClassJob >( 1 );
+  gameData = new xiv::dat::GameData( datLocation );
-  if( gld )
-  {
-    Logger::info( "got {0}", gld->name );
-  }
-  else
-    Logger::warn( "failed to get classjob {}", 1 );
 
+  if( argc > 1 )
+  {
+    while( argc-- > 1 )
+    {
+      Logger::info( "Exporting file: {}", argv[ argc ] );
+      exportFile( argv[ argc ] );
+    }
+
+    return 0;
+  }
+
+  std::vector< std::string > levelNames;
+  std::vector< std::string > paths;
+  
+  std::vector< uint32_t > lgbGroupIds;
+  std::vector< DupeResult > lgbGroupDupes;
+
+  paths.emplace_back( "level/bg.lgb" );
+  paths.emplace_back( "level/planmap.lgb" );
+  paths.emplace_back( "level/planevent.lgb" );
+  paths.emplace_back( "level/planlive.lgb" );
+  paths.emplace_back( "level/sound.lgb" );
+  paths.emplace_back( "level/planner.lgb" );
+
+  auto ids = g_exdData.getTerritoryTypeIdList();
+
+  for( auto id : ids )
+  {
+    auto territoryType = g_exdData.get< Data::TerritoryType >( id );
+    if( !territoryType )
+      continue;
+
+    auto bg = territoryType->bg;
+    bg = bg.substr( 0, bg.find( "/level/" ) );
+
+    if( bg.empty() )
+      continue;
+
+    if( std::find( levelNames.begin(), levelNames.end(), bg ) != levelNames.end() )
+      continue;
+
+    levelNames.push_back( bg );
+
+    Logger::info( "LGB groups for: {}", bg );
+
+    for( const auto& path : paths )
+    {
+      auto filePath = fmt::format( "bg/{}/{}", bg, path );
+
+      try
+      {
+        auto file = gameData->getFile( filePath );
+        auto data = file->access_data_sections().at( 0 );
+
+        LGB_FILE lgb( &data[ 0 ], "wtf" );
+
+        Logger::info(" {} groups: {}", path, lgb.header.groupCount );
+
+        for( const auto& group : lgb.groups )
+        {
+          Logger::info( " - {:<7} {:<25} children: {}", group.header.unknown, group.name, group.header.entryCount );
+
+          if( std::find( lgbGroupIds.begin(), lgbGroupIds.end(), group.header.unknown ) == lgbGroupIds.end() )
+          {
+            lgbGroupIds.emplace_back( group.header.unknown );
+          }
+          else
+          {
+            lgbGroupDupes.push_back( { group.name, filePath, group.header.unknown } );
+          }
+        }
+        Logger::info( "" );
+      }
+      catch( const std::exception& ex )
+      {
+        Logger::warn( "Failed on file {}, error: {}", path, ex.what() );
+      }
+    }
+  }
+
+  if( !lgbGroupDupes.empty() )
+  {
+    Logger::info( "Found duplicate LGB group ids: " );
+
+    for( const auto& result : lgbGroupDupes )
+    {
+      Logger::info( " - file: {:<50} group: {:<30} id: {}", result.lgb, result.groupName, result.id );
+    }
+  }
 
   return 0;
 }

src/tools/nav_export/main.cpp

@@ -42,7 +42,7 @@ std::set< std::string > zoneDumpList;
 std::shared_ptr< Cache > pCache;
 std::map< uint32_t, uint16_t > eobjSgbPaths;
 
-xiv::dat::GameData* data1 = nullptr;
+xiv::dat::GameData* gameData = nullptr;
 xiv::exd::ExdData* eData = nullptr;
 
 
@@ -57,9 +57,9 @@ using namespace std::chrono_literals;
 
 void initExd( const std::string& gamePath )
 {
-  data1 = data1 ? data1 : new xiv::dat::GameData( gamePath );
+  gameData = gameData ? gameData : new xiv::dat::GameData( gamePath );
-  eData = eData ? eData : new xiv::exd::ExdData( *data1 );
+  eData = eData ? eData : new xiv::exd::ExdData( *gameData );
-  pCache = std::make_shared< Cache >( data1 );
+  pCache = std::make_shared< Cache >( gameData );
 }
 
 void replaceAll( std::string& str, const std::string& from, const std::string& to ) {
@@ -360,15 +360,15 @@ int main( int argc, char* argv[] )
       std::vector< char > section1;
       std::vector< char > section2;
 
-      const xiv::dat::Cat& test = data1->getCategory( "bg" );
+      const xiv::dat::Cat& test = gameData->getCategory( "bg" );
 
-      auto test_file = data1->getFile( bgLgbPath );
+      auto test_file = gameData->getFile( bgLgbPath );
       section = test_file->access_data_sections().at( 0 );
 
-      auto planmap_file = data1->getFile( planmapLgbPath );
+      auto planmap_file = gameData->getFile( planmapLgbPath );
       section2 = planmap_file->access_data_sections().at( 0 );
 
-      auto test_file1 = data1->getFile( listPcbPath );
+      auto test_file1 = gameData->getFile( listPcbPath );
       section1 = test_file1->access_data_sections().at( 0 );
 
       std::vector< std::string > stringList;
@@ -456,7 +456,7 @@ int main( int argc, char* argv[] )
 
                   if( auto pGimmick = pCache->getSgbFile( sgbPath ) )
                   {
-                    for( const auto& offset1cFile : pGimmick->offset1cObjects )
+                    for( const auto& offset1cFile : pGimmick->stateEntries )
                       exportSgbModel( offset1cFile, pEobj, exportedGroup, true );
                   }
                 }
@@ -497,7 +497,7 @@ int main( int argc, char* argv[] )
             std::chrono::duration_cast< std::chrono::seconds >( std::chrono::high_resolution_clock::now() - startTime ).count() );
 
   delete eData;
-  delete data1;
+  delete gameData;
   
   return 0;
 }

src/tools/nav_export/sgb.h

@@ -243,7 +243,7 @@ struct SGB_FILE
 {
   SGB_HEADER header;
   std::vector< SGB_GROUP > entries;
-  std::set< std::string > offset1cObjects;
+  std::set< std::string > stateEntries;
 
   SGB_FILE()
   {
@@ -260,9 +260,9 @@ struct SGB_FILE
 
     try
     {
-      auto group = SGB_GROUP( buf, this, &offset1cObjects, header.fileSize, baseOffset + header.sharedOffset );
+      auto group = SGB_GROUP( buf, this, &stateEntries, header.fileSize, baseOffset + header.sharedOffset );
       entries.push_back( group );
-      auto group2 = SGB_GROUP( buf, this, &offset1cObjects, header.fileSize, baseOffset+ header.offset1C, true );
+      auto group2 = SGB_GROUP( buf, this, &stateEntries, header.fileSize, baseOffset+ header.offset1C, true );
       entries.push_back( group2 );
     }
     catch( std::exception& e )

src/tools/pcb_reader/main.cpp

@@ -44,7 +44,7 @@ std::set< std::string > zoneDumpList;
 std::shared_ptr< Cache > pCache;
 std::map< uint32_t, uint16_t > eobjSgbPaths;
 
-xiv::dat::GameData* data1 = nullptr;
+xiv::dat::GameData* gameData = nullptr;
 xiv::exd::ExdData* eData = nullptr;
 
 
@@ -59,9 +59,9 @@ using namespace std::chrono_literals;
 
 void initExd( const std::string& gamePath )
 {
-  data1 = data1 ? data1 : new xiv::dat::GameData( gamePath );
+  gameData = gameData ? gameData : new xiv::dat::GameData( gamePath );
-  eData = eData ? eData : new xiv::exd::ExdData( *data1 );
+  eData = eData ? eData : new xiv::exd::ExdData( *gameData );
-  pCache = std::make_shared< Cache >( data1 );
+  pCache = std::make_shared< Cache >( gameData );
 }
 
 void replaceAll( std::string& str, const std::string& from, const std::string& to ) {
@@ -249,15 +249,15 @@ int main( int argc, char* argv[] )
       std::vector< char > section1;
       std::vector< char > section2;
 
-      const xiv::dat::Cat& test = data1->getCategory( "bg" );
+      const xiv::dat::Cat& test = gameData->getCategory( "bg" );
 
-      auto test_file = data1->getFile( bgLgbPath );
+      auto test_file = gameData->getFile( bgLgbPath );
       section = test_file->access_data_sections().at( 0 );
 
-      auto planmap_file = data1->getFile( planmapLgbPath );
+      auto planmap_file = gameData->getFile( planmapLgbPath );
       section2 = planmap_file->access_data_sections().at( 0 );
 
-      auto test_file1 = data1->getFile( listPcbPath );
+      auto test_file1 = gameData->getFile( listPcbPath );
       section1 = test_file1->access_data_sections().at( 0 );
 
       std::vector< std::string > stringList;
@@ -499,7 +499,7 @@ int main( int argc, char* argv[] )
 
                   if( auto pGimmick = pCache->getSgbFile( sgbPath ) )
                   {
-                    for( const auto& offset1cFile : pGimmick->offset1cObjects )
+                    for( const auto& offset1cFile : pGimmick->stateEntries )
                       exportSgbModel( offset1cFile, pEobj, true );
                   }
                 }
@@ -541,7 +541,7 @@ int main( int argc, char* argv[] )
             std::chrono::duration_cast< std::chrono::seconds >( std::chrono::high_resolution_clock::now() - startTime ).count() );
 
   delete eData;
-  delete data1;
+  delete gameData;
   
   return 0;
 }

src/tools/pcb_reader/sgb.h

@@ -243,7 +243,7 @@ struct SGB_FILE
 {
   SGB_HEADER header;
   std::vector< SGB_GROUP > entries;
-  std::set< std::string > offset1cObjects;
+  std::set< std::string > stateEntries;
 
   SGB_FILE()
   {
@@ -260,9 +260,9 @@ struct SGB_FILE
 
     try
     {
-      auto group = SGB_GROUP( buf, this, &offset1cObjects, header.fileSize, baseOffset + header.sharedOffset );
+      auto group = SGB_GROUP( buf, this, &stateEntries, header.fileSize, baseOffset + header.sharedOffset );
       entries.push_back( group );
-      auto group2 = SGB_GROUP( buf, this, &offset1cObjects, header.fileSize, baseOffset+ header.offset1C, true );
+      auto group2 = SGB_GROUP( buf, this, &stateEntries, header.fileSize, baseOffset+ header.offset1C, true );
       entries.push_back( group2 );
     }
     catch( std::exception& e )