Initial commit of Sapphire for FFXIV 3.0 "light rewrite"

.appveyor.yml

@@ -1,5 +1,5 @@
 os:
-  - Visual Studio 2019
+  - Visual Studio 2017
 
 configuration:
   - Debug
@@ -20,8 +20,8 @@ before_build:
   - git submodule update --init
   - mkdir build
   - cd build
-  - cmake .. -G "Visual Studio 16 2019" -A x64
+  - cmake .. -G "Visual Studio 15 2017 Win64" 
-  - cmake --build . --target ALL_BUILD --config RelWithDebInfo
+  - cmake --build . --target ALL_BUILD --config Release
 
 build_script:
   - cd bin  

.github/CONTRIBUTING.md

@@ -1,63 +0,0 @@
-# Contributing
-
-Thanks for contributing to Sapphire!
-
-First, we'd like to mention that a lot of discussion regarding the project happens in our Discord server.
-We value research and discussion as to how we should tackle our issues as well as improving what is already in.
-Of course we also value testing - many things tend to break due to patches, or mistakes/edge cases.
-
-Regardless of how you plan on contributing, your thoughts are appreciated and you're welcome to join our Discord (link in README.md).
-
-## Research
-
-Care in implementating features should be taken. It tends to be end up weird, and replicating the expected behavior 
-is always preferred. Avoid assumptions and guesswork whenever possible.
-
-As much research possible should be done before writing it out - on game data, testing with retail, 
-and even common practices in server development (emulators or not).
-
-## Pull Requests
-
-When making a PR, please make sure that it follows our style guidelines and good practices.
-
-### Coding style
-
-Indentations are Allman-style based, 2-space, no tabs.
-Space between arguments in function calls, as well as for types.
-
-Example (shortened from ActionHandler.cpp):
-
-```cpp
-switch( commandId )
-{
-  case 0x01:  // Toggle sheathe
-  {
-    if( param11 == 1 )
-       pPlayer->setStance( Entity::Chara::Stance::Active );
-    else
-    {
-      pPlayer->setStance( Entity::Chara::Stance::Passive );
-      pPlayer->setAutoattack( false );
-    }
-
-    pPlayer->sendToInRangeSet( ActorControlPacket142( pPlayer->getId(), 0, param11, 1 ) );
-
-    break;
-  }
-  case 0x03: // Change target
-  {
-    uint64_t targetId = inPacket.getValAt< uint64_t >( 0x24 );
-    pPlayer->changeTarget( targetId );
-    break;
-  }
-  default:
-  {
-    break;
-  }
-}
-```
-
-### Feature implementation
-
-Please make sure edge cases have been tested, behavior is aligned with retail and (if applicable) your queries make sense.
-Any changes to the SQL base should be noted (and reflected in the update.sql file in rootDirectory/sql). 

.github/ISSUE_TEMPLATE.md

@@ -1,9 +0,0 @@
-**Describe the problem:** *Please describe the problem you're encountering accurately, including steps to reproduce/recreate the issue.*
-
-**Screenshots or videos needed to identify the problem:** *Please attach screenshots or videos showing the problem to this issue, if applicable.*
-
-**Sapphire branch and commit:** *You can get this by checking the server startup message in your console window, running `` git describe --all`` in a console window or looking at your git client.*
-
-**Logs:** *You can get these from the /bin/logs directory in your sapphire clone, please attach them to this issue.*
-
-**Setup:** *Please note down which operating system you are using and any other information about your setup which could be of use to us, like compiler/Visual Studio version and FFXIV version.*

.gitignore

@@ -129,7 +129,7 @@ src/common/Version\.cpp
 .mtime_cache
 
 # generated script loader files
-**/ScriptLoader.cpp
+src/scripts/*/ScriptLoader.cpp
 
 # cotire generated files/folders
 cotire/

.travis.yml

@@ -14,10 +14,9 @@ matrix:
           sources:
           - ubuntu-toolchain-r-test
           packages:
-          - g++-8
+          - g++-7
       env:
-      - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
+      - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7"
-      - CXX=g++-8
 
 # Setup cache
 cache:

CMakeLists.txt

@@ -13,16 +13,13 @@ add_custom_target( copy_runtime_files ALL
                      COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/config ${CMAKE_BINARY_DIR}/bin/config
                      COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/sql ${CMAKE_BINARY_DIR}/bin/sql
                      COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/web ${CMAKE_BINARY_DIR}/bin/web
-                     COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/sql_import.sh ${CMAKE_BINARY_DIR}/bin/sql_import.sh
+                     COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/sql_import.sh ${CMAKE_BINARY_DIR}/bin/sql_import.sh )
-                     COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/bin/data/actions
-                     COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/deps/ffxiv-actions/actions ${CMAKE_BINARY_DIR}/bin/data/actions )
 
 ######################################
 # Dependencies and compiler settings #
 ######################################
 include( "cmake/paths.cmake" )
 include( "cmake/compiler.cmake" )
-include( "cmake/cotire.cmake" )
 
 ##############################
 #             Git            #
@@ -44,6 +41,7 @@ find_package( MySQL )
 add_subdirectory( "deps/zlib" )
 add_subdirectory( "deps/MySQL" )
 add_subdirectory( "deps/datReader" )
+add_subdirectory( "deps/datReaderPs3" )
 add_subdirectory( "deps/mysqlConnector" )
 add_subdirectory( "deps/recastnavigation" )
 

CMakeSettings.json

@@ -1,32 +1,21 @@
 {
   // See https://go.microsoft.com//fwlink//?linkid=834763 for more information about this file.
-  "environments": [
-    {
-      "BuildDir": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\build"
-    }
-  ],
   "configurations": [
     {
       "name": "x64-Debug",
-      "generator": "Visual Studio 16 2019 Win64",
+      "generator": "Visual Studio 15 2017 Win64",
       "configurationType": "Debug",
-      "buildRoot": "${env.BuildDir}\\${name}",
+      "buildRoot": "H:\\Sapphire\\Sapphire-2.3-Build",
       "cmakeCommandArgs": "-DCMAKE_BUILD_TYPE=\"Debug\"",
-      "buildCommandArgs": "-m -v:minimal",
+      "buildCommandArgs": "-m -v:minimal"
-      "inheritEnvironments": [
-        "msvc_x64"
-      ]
     },
     {
       "name": "x64-Release",
-      "generator": "Visual Studio 16 2019 Win64",
+      "generator": "Visual Studio 15 2017 Win64",
       "configurationType": "Release",
-      "buildRoot": "${env.BuildDir}\\${name}",
+      "buildRoot": "H:\\Sapphire\\Sapphire-2.3-Build",
-      "cmakeCommandArgs": "-DCMAKE_BUILD_TYPE=\"RelWithDebInfo\"",
+      "cmakeCommandArgs": "",
-      "buildCommandArgs": "-m -v:minimal",
+      "buildCommandArgs": "-m -v:minimal"
-      "inheritEnvironments": [
-        "msvc_x64"
-      ]
     }
   ]
 }

README.md

@@ -18,7 +18,7 @@ Sapphire requires the following software:
 
 | *Name* | *Windows* | *Linux* |
 | ------ | --------- | ------- |
-| CMake 3.0.2+ and C++17 capable compiler | [Visual Studio 2019](https://www.visualstudio.com/) | `gcc 8` and `g++ 8` or newer, or equivalent `clang` version. |
+| CMake 3.0.2+ and C++17 capable compiler | [Visual Studio 2017](https://www.visualstudio.com/) | `gcc 7` and `g++ 7` or newer |
 | MySQL Server 5.7 | [Official Site](https://dev.mysql.com/downloads/mysql/) | MySQL server from your distribution's package manager |
 
 Please check the [wiki](https://github.com/SapphireMordred/Sapphire/wiki) for detailed installation/build instructions for your OS.

cmake/FindMySQL.cmake

@@ -85,8 +85,6 @@ find_path(MYSQL_INCLUDE_DIR
     "${PROGRAM_FILES_64}/MySQL/MySQL Server 5.7/include"
     "${PROGRAM_FILES_64}/MySQL/include"
     "${PROGRAM_FILES_64}/MariaDB 10.3/include/mysql"
-    "${PROGRAM_FILES_64}/MariaDB 10.4/include/mysql"
-    "${PROGRAM_FILES_64}/MariaDB 10.5/include/mysql"
     "C:/MySQL/include"
     "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MySQL AB\\MySQL Server 5.7;Location]/include"
     "$ENV{ProgramFiles}/MySQL/MySQL Server 5.7/include"
@@ -126,8 +124,6 @@ if( WIN32 )
       "${PROGRAM_FILES_64}/MySQL/MySQL Server 5.7/lib/opt"
       "${PROGRAM_FILES_64}/MySQL/lib"
       "${PROGRAM_FILES_64}/MariaDB 10.3/lib"
-      "${PROGRAM_FILES_64}/MariaDB 10.4/lib"
-      "${PROGRAM_FILES_64}/MariaDB 10.5/lib"
       "C:/MySQL/lib/debug"
       "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MySQL AB\\MySQL Server 5.7;Location]/lib"
       "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MySQL AB\\MySQL Server 5.7;Location]/lib/opt"
@@ -181,8 +177,6 @@ if( WIN32 )
         "${PROGRAM_FILES_64}/MySQL/MySQL Server 5.7/bin/opt"
         "${PROGRAM_FILES_64}/MySQL/bin"
         "${PROGRAM_FILES_64}/MariaDB 10.3/bin"
-        "${PROGRAM_FILES_64}/MariaDB 10.4/bin"
-        "${PROGRAM_FILES_64}/MariaDB 10.5/bin"
         "C:/MySQL/bin/debug"
         "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MySQL AB\\MySQL Server 5.7;Location]/bin"
         "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MySQL AB\\MySQL Server 5.7;Location]/bin/opt"

cmake/paths.cmake

@@ -13,4 +13,3 @@ endif()
 # Create log folder
 file( MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin/log )
 file( MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin/navi )
-file( MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin/data )

config/global.ini.default

@@ -11,7 +11,7 @@ AsyncThreads = 2
 ServerSecret = default
 DataPath = C:\\SquareEnix\\FINAL FANTASY XIV - A Realm Reborn\\game\\sqpack
 WorldID = 67
-DefaultGMRank = 255
+DefaultGMRank = 90
 LogLevel = 1
 LogFilter = 0
 

deps/datReader/Dat.cpp

@@ -70,9 +70,9 @@ namespace xiv::utils::bparse
     xiv::utils::bparse::reorder( i_struct.size );
     xiv::utils::bparse::reorder( i_struct.entry_type );
     xiv::utils::bparse::reorder( i_struct.total_uncompressed_size );
-    for( int32_t i = 0; i < 0x2; ++i )
+    for( unsigned int & i : i_struct.unknown )
     {
-      xiv::utils::bparse::reorder( i_struct.unknown[ i ] );
+      xiv::utils::bparse::reorder( i );
     }
   }
 
@@ -81,9 +81,9 @@ namespace xiv::utils::bparse
   {
     xiv::utils::bparse::reorder( i_struct.offset );
     xiv::utils::bparse::reorder( i_struct.size );
-    for( int32_t i = 0; i < 0x4; ++i )
+    for( unsigned int & i : i_struct.unknown )
     {
-      xiv::utils::bparse::reorder( i_struct.unknown[ i ] );
+      xiv::utils::bparse::reorder( i );
     }
     xiv::utils::bparse::reorder( i_struct.block_hash );
   }
@@ -109,29 +109,29 @@ namespace xiv::utils::bparse
   inline void reorder< xiv::dat::DatMdlFileBlockInfos >( xiv::dat::DatMdlFileBlockInfos& i_struct )
   {
     xiv::utils::bparse::reorder( i_struct.unknown1 );
-    for( auto i = 0; i < ::model_section_count; ++i )
+    for( unsigned int& uncompressed_size : i_struct.uncompressed_sizes )
     {
-      xiv::utils::bparse::reorder( i_struct.uncompressed_sizes[ i ] );
+      xiv::utils::bparse::reorder( uncompressed_size );
     }
-    for( auto i = 0; i < ::model_section_count; ++i )
+    for( unsigned int& compressed_size : i_struct.compressed_sizes )
     {
-      xiv::utils::bparse::reorder( i_struct.compressed_sizes[ i ] );
+      xiv::utils::bparse::reorder( compressed_size );
     }
-    for( auto i = 0; i < ::model_section_count; ++i )
+    for( unsigned int& offset : i_struct.offsets )
     {
-      xiv::utils::bparse::reorder( i_struct.offsets[ i ] );
+      xiv::utils::bparse::reorder( offset );
     }
-    for( auto i = 0; i < ::model_section_count; ++i )
+    for( unsigned short& block_id : i_struct.block_ids )
     {
-      xiv::utils::bparse::reorder( i_struct.block_ids[ i ] );
+      xiv::utils::bparse::reorder( block_id );
     }
-    for( auto i = 0; i < ::model_section_count; ++i )
+    for( unsigned short& block_count : i_struct.block_counts )
     {
-      xiv::utils::bparse::reorder( i_struct.block_counts[ i ] );
+      xiv::utils::bparse::reorder( block_count );
     }
-    for( auto i = 0; i < 0x2; ++i )
+    for( unsigned int &i : i_struct.unknown2 )
     {
-      xiv::utils::bparse::reorder( i_struct.unknown2[ i ] );
+      xiv::utils::bparse::reorder( i );
     }
   }
 
@@ -160,9 +160,7 @@ namespace xiv::dat
     isBlockValid( block_record.offset, block_record.size, block_record.block_hash );
   }
 
-  Dat::~Dat()
+  Dat::~Dat() = default;
-  {
-  }
 
   std::unique_ptr< File > Dat::getFile( uint32_t i_offset )
   {
@@ -184,7 +182,7 @@ namespace xiv::dat
         {
           outputFile->_type = FileType::standard;
 
-          uint32_t number_of_blocks = extract< uint32_t >( m_handle, "number_of_blocks" );
+          auto number_of_blocks = extract< uint32_t >( m_handle, "number_of_blocks" );
 
           // Just extract offset infos for the blocks to extract
           std::vector< DatStdFileBlockInfos > std_file_block_infos;
@@ -207,7 +205,7 @@ namespace xiv::dat
         {
           outputFile->_type = FileType::model;
 
-          DatMdlFileBlockInfos mdlBlockInfo = extract< DatMdlFileBlockInfos >( m_handle );
+          auto mdlBlockInfo = extract< DatMdlFileBlockInfos >( m_handle );
 
           // Getting the block number and read their sizes
           const uint32_t block_count = mdlBlockInfo.block_ids[ ::model_section_count - 1 ] +
@@ -239,7 +237,7 @@ namespace xiv::dat
           outputFile->_type = FileType::texture;
 
           // Extracts mipmap entries and the block sizes
-          uint32_t sectionCount = extract< uint32_t >( m_handle, "sections_count" );
+          auto sectionCount = extract< uint32_t >( m_handle, "sections_count" );
 
           std::vector< DatTexFileBlockInfos > texBlockInfo;
           extract< DatTexFileBlockInfos >( m_handle, sectionCount, texBlockInfo );
@@ -289,10 +287,10 @@ namespace xiv::dat
   {
     m_handle.seekg( i_offset );
 
-    DatBlockHeader block_header = extract< DatBlockHeader >( m_handle );
+    auto block_header = extract< DatBlockHeader >( m_handle );
 
     // Resizing the vector to write directly into it
-    const uint32_t data_size = o_data.size();
+    const auto data_size = o_data.size();
     o_data.resize( data_size + block_header.uncompressed_size );
 
     // 32000 in compressed_size means it is not compressed so take uncompressed_size
@@ -307,10 +305,10 @@ namespace xiv::dat
       std::vector< char > temp_buffer( block_header.compressed_size );
       m_handle.read( temp_buffer.data(), block_header.compressed_size );
 
-      utils::zlib::no_header_decompress( reinterpret_cast<uint8_t*>(temp_buffer.data()),
+      utils::zlib::no_header_decompress( reinterpret_cast< uint8_t* >( temp_buffer.data() ),
                                          temp_buffer.size(),
-                                         reinterpret_cast<uint8_t*>(o_data.data() + data_size),
+                                         reinterpret_cast< uint8_t* >( o_data.data() + data_size ),
-                                         block_header.uncompressed_size );
+                                         static_cast< size_t >( block_header.uncompressed_size ) );
     }
   }
 

deps/datReader/Dat.h

@@ -1,6 +1,4 @@
-#ifndef XIV_DAT_DAT_H
+#pragma once
-#define XIV_DAT_DAT_H
-
 #include "SqPack.h"
 
 #include <mutex>
@@ -39,4 +37,3 @@ namespace xiv::dat
 
 }
 
-#endif // XIV_DAT_DAT_H

deps/datReader/DatCategories/bg/lgb.h

@@ -34,10 +34,10 @@ public:
     memset( &header, 0, sizeof( header ) );
   };
 
-  LgbEntry( char* buf, uint32_t offset )
+  LgbEntry( char* buf, size_t offset )
   {
     m_buf = buf;
-    m_offset = offset;
+    m_offset = static_cast< uint32_t >( offset );
     header = *reinterpret_cast< InstanceObject* >( buf + offset );
   };
 
@@ -60,11 +60,9 @@ public:
   std::string modelFileName;
   std::string collisionFileName;
 
-  LGB_BGPARTS_ENTRY()
+  LGB_BGPARTS_ENTRY() = default;
-  {
-  };
 
-  LGB_BGPARTS_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  LGB_BGPARTS_ENTRY( char* buf, size_t offset ) : LgbEntry( buf, offset )
   {
     data = *reinterpret_cast< BgPartsData* >( buf + offset );
     name = std::string( buf + offset + header.nameOffset );
@@ -80,7 +78,7 @@ public:
   std::string name;
   std::string gimmickFileName;
 
-  LGB_GIMMICK_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  LGB_GIMMICK_ENTRY( char* buf, size_t offset ) : LgbEntry( buf, offset )
   {
     data = *reinterpret_cast< GimmickData* >( buf + offset );
     name = std::string( buf + offset + header.nameOffset );
@@ -94,7 +92,7 @@ public:
   ENpcData data;
   std::string name;
 
-  LGB_ENPC_ENTRY( char* buf, uint32_t offset ) :
+  LGB_ENPC_ENTRY( char* buf, size_t offset ) :
     LgbEntry( buf, offset )
   {
     data = *reinterpret_cast< ENpcData* >( buf + offset );
@@ -108,7 +106,7 @@ public:
   EObjData data;
   std::string name;
 
-  LGB_EOBJ_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  LGB_EOBJ_ENTRY( char* buf, size_t offset ) : LgbEntry( buf, offset )
   {
     data = *reinterpret_cast< EObjData* >( buf + offset );
     name = std::string( buf + offset + header.nameOffset );
@@ -121,7 +119,7 @@ public:
   MapRangeData data;
   std::string name;
 
-  LGB_MAP_RANGE_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  LGB_MAP_RANGE_ENTRY( char* buf, size_t offset ) : LgbEntry( buf, offset )
   {
     data = *reinterpret_cast< MapRangeData* >( buf + offset );
     name = std::string( buf + offset + header.nameOffset );
@@ -134,7 +132,7 @@ public:
   ExitRangeData data;
   std::string name;
 
-  LGB_EXIT_RANGE_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  LGB_EXIT_RANGE_ENTRY( char* buf, size_t offset ) : LgbEntry( buf, offset )
   {
     data = *reinterpret_cast< ExitRangeData* >( buf + offset );
     name = std::string( buf + offset + header.nameOffset );
@@ -146,7 +144,7 @@ struct LGB_POP_RANGE_ENTRY : public LgbEntry
 public:
   PopRangeData data;
 
-  LGB_POP_RANGE_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  LGB_POP_RANGE_ENTRY( char* buf, size_t offset ) : LgbEntry( buf, offset )
   {
     data = *reinterpret_cast< PopRangeData* >( buf + offset );
   };
@@ -176,7 +174,7 @@ struct LGB_GROUP
   std::string name;
   std::vector< std::shared_ptr< LgbEntry > > entries;
 
-  LGB_GROUP( char* buf, LGB_FILE* parentStruct, uint32_t offset )
+  LGB_GROUP( char* buf, LGB_FILE* parentStruct, size_t offset )
   {
     parent = parentStruct;
     header = *reinterpret_cast< LGB_GROUP_HEADER* >( buf + offset );
@@ -257,7 +255,7 @@ struct LGB_FILE
       throw std::runtime_error( "Invalid LGB file!" );
 
     constexpr auto baseOffset = sizeof( header );
-    for( auto i = 0; i < header.groupCount; ++i )
+    for( size_t i = 0; i < header.groupCount; ++i )
     {
       const auto groupOffset = baseOffset + *reinterpret_cast< int32_t* >( buf + ( baseOffset + i * 4 ) );
       const auto group = LGB_GROUP( buf, this, groupOffset );

deps/datReader/DatCategories/bg/sgb.h

@@ -136,7 +136,7 @@ struct SGB_MODEL_ENTRY : public SGB_GROUP_ENTRY
   std::string modelFileName;
   std::string collisionFileName;
 
-  SGB_MODEL_ENTRY( char* buf, uint32_t offset, SgbGroupEntryType type )
+  SGB_MODEL_ENTRY( char* buf, size_t offset, SgbGroupEntryType type )
   {
     this->type = type;
     header = *reinterpret_cast< SGB_MODEL_HEADER* >( buf + offset );
@@ -258,7 +258,7 @@ struct SGB_FILE
       if( stateCount > 0 )
       {
         stateCount = stateCount;
-        for( int i = 0; i < stateCount; ++i )
+        for( size_t i = 0; i < stateCount; ++i )
         {
           auto state = SGB_STATE_ENTRY( buf + baseOffset + header.statesOffset + 8 + i * sizeof( SGB_STATE_HEADER ) );
           stateEntries.push_back( state );

deps/datReader/Exd.cpp

@@ -1,6 +1,8 @@
 #include "Exd.h"
 
 #include "bparse.h"
+#include "stream.h"
+#include <fstream>
 #include "Exh.h"
 
 using xiv::utils::bparse::extract;
@@ -51,24 +53,32 @@ template<>
 
 namespace xiv::exd
 {
-  Exd::Exd( std::shared_ptr< Exh > exh, const std::vector< std::shared_ptr< dat::File > >& files )
+  Exd::Exd( std::shared_ptr< Exh > i_exh, const std::vector< std::shared_ptr< dat::File>>& i_files )
   {
-    _exh = exh;
+    _exh = i_exh;
+    _files = i_files;
 
 
     // Iterates over all the files
-    for( auto& file : files )
+    const uint32_t member_count = static_cast< uint32_t >( _exh->get_members().size() );
+    for( auto& file_ptr : _files )
     {
-      std::vector< char > dataCpy = file->get_data_sections().front();
+      // Get a stream
+      std::vector< char > dataCpy = file_ptr->get_data_sections().front();
+      std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
 
-      // Extract the header
+      // Extract the header and skip to the record indices
-      auto exdHeader = extract< ExdHeader >( dataCpy, 0 );
+      auto exd_header = extract< ExdHeader >( iss );
+      iss.seekg( 0x20 );
 
-      const uint32_t recordCount = exdHeader.index_size / sizeof( ExdRecordIndex );
+      // Preallocate and extract the record_indices
-      for( uint32_t i = 0; i < recordCount; ++i )
+      const uint32_t record_count = exd_header.index_size / sizeof( ExdRecordIndex );
+      std::vector< ExdRecordIndex > record_indices;
+      record_indices.reserve( record_count );
+      for( uint32_t i = 0; i < record_count; ++i )
       {
-        auto recordIndex = extract< ExdRecordIndex >( dataCpy, 32 + ( i * sizeof( ExdRecordIndex ) ) );
+        auto recordIndex = extract< ExdRecordIndex >( iss );
-        _idCache[ recordIndex.id ] = ExdCacheEntry{ file, recordIndex.offset + 6, extract< uint8_t >( dataCpy, recordIndex.offset + 5 ) };
+        _idCache[ recordIndex.id ] = ExdCacheEntry{ file_ptr, recordIndex.offset };
       }
     }
   }
@@ -81,24 +91,40 @@ namespace xiv::exd
   {
 
     auto cacheEntryIt = _idCache.find( id );
-    if( cacheEntryIt == _idCache.end() || subRow >= cacheEntryIt->second.subRows )
+    if( cacheEntryIt == _idCache.end() )
-      throw std::runtime_error( "Id + SubId combination not found: " + std::to_string( id ) + "." + std::to_string( subRow ) );
+      throw std::runtime_error( "Id not found: " + std::to_string( id ) );
 
-    auto dataCpy = cacheEntryIt->second.file->get_data_sections().front();
+    // Iterates over all the files
+    const uint32_t member_count = static_cast< uint32_t >( _exh->get_members().size() );
+    auto& file_ptr = cacheEntryIt->second.file;
 
-    std::vector< Field > fields;
+    std::vector< char > dataCpy = file_ptr->get_data_sections().front();
-    fields.reserve( _exh->get_members().size() );
+    std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
 
-    uint32_t baseOffset = cacheEntryIt->second.offset + ( subRow * _exh->get_header().data_offset + 2 * ( subRow + 1 ) );
+    // Get the vector fields for the given record and preallocate it
+    auto fields = _data[ id ];
+    fields.reserve( member_count );
+    iss.seekg( cacheEntryIt->second.offset + 6 );
 
-    for( auto& memberEntry : _exh->get_exh_members() )
+    uint8_t subRows = *reinterpret_cast< uint8_t* >( &dataCpy[ cacheEntryIt->second.offset + 5 ] );
+
+    if( subRow >= subRows )
+      throw std::runtime_error( "Out of bounds sub-row!" );
+
+    int offset = cacheEntryIt->second.offset + 6 + ( subRow * _exh->get_header().data_offset + 2 * ( subRow + 1 ) );
+
+    for( auto& member_entry : _exh->get_exh_members() )
     {
+      // Seek to the position of the member to extract.
+      // 6 is because we have uint32_t/uint16_t at the start of each record
+      iss.seekg( offset + member_entry.offset );
+
       // Switch depending on the type to extract
-      switch( memberEntry.type )
+      switch( member_entry.type )
       {
         case DataType::string:
           // Extract the offset to the actual string
-          // Then extract the actual string from that offset
+          // Seek to it then extract the actual string
         {
           throw std::runtime_error( "String not implemented for variant 2!" );
           //auto string_offset = extract<uint32_t>( iss, "string_offset", false );
@@ -108,46 +134,50 @@ namespace xiv::exd
           break;
 
         case DataType::boolean:
-          fields.emplace_back( extract< bool >( dataCpy, baseOffset + memberEntry.offset ) );
+          fields.emplace_back( extract< bool >( iss, "bool" ) );
           break;
 
         case DataType::int8:
-          fields.emplace_back( extract< int8_t >( dataCpy, baseOffset + memberEntry.offset ) );
+          fields.emplace_back( extract< int8_t >( iss, "int8_t" ) );
           break;
 
         case DataType::uint8:
-          fields.emplace_back( extract< uint8_t >( dataCpy, baseOffset + memberEntry.offset ) );
+          fields.emplace_back( extract< uint8_t >( iss, "uint8_t" ) );
           break;
 
         case DataType::int16:
-          fields.emplace_back( extract< int16_t >( dataCpy, baseOffset + memberEntry.offset, false ) );
+          fields.emplace_back( extract< int16_t >( iss, "int16_t", false ) );
           break;
 
         case DataType::uint16:
-          fields.emplace_back( extract< uint16_t >( dataCpy, baseOffset + memberEntry.offset, false ) );
+          fields.emplace_back( extract< uint16_t >( iss, "uint16_t", false ) );
           break;
 
         case DataType::int32:
-          fields.emplace_back( extract< int32_t >( dataCpy, baseOffset + memberEntry.offset, false ) );
+          fields.emplace_back( extract< int32_t >( iss, "int32_t", false ) );
           break;
 
         case DataType::uint32:
-          fields.emplace_back( extract< uint32_t >( dataCpy, baseOffset + memberEntry.offset, false ) );
+          fields.emplace_back( extract< uint32_t >( iss, "uint32_t", false ) );
           break;
 
         case DataType::float32:
-          fields.emplace_back( extract< float >( dataCpy, baseOffset + memberEntry.offset, false ) );
+          fields.emplace_back( extract< float >( iss, "float", false ) );
           break;
 
         case DataType::uint64:
-          fields.emplace_back( extract< uint64_t >( dataCpy, baseOffset + memberEntry.offset, false ) );
+          fields.emplace_back( extract< uint64_t >( iss, "uint64_t", false ) );
           break;
 
         default:
-          auto type = static_cast< uint16_t >( memberEntry.type );
+          auto type = static_cast< uint16_t >( member_entry.type );
           if( type < 0x19 || type > 0x20 )
             throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
-          fields.emplace_back( ( extract< uint8_t >( dataCpy, baseOffset + memberEntry.offset ) & ( 1 << ( type - 0x19 ) ) ) != 0 );
+          uint64_t val = extract< uint64_t >( iss, "bool" );
+          int32_t shift = type - 0x19;
+          int32_t i = 1 << shift;
+          val &= i;
+          fields.emplace_back( ( val & i ) == i );
           break;
       }
     }
@@ -163,68 +193,84 @@ namespace xiv::exd
     if( cacheEntryIt == _idCache.end() )
       throw std::runtime_error( "Id not found: " + std::to_string( id ) );
 
-    auto dataCpy = cacheEntryIt->second.file->get_data_sections().front();
+    // Iterates over all the files
+    const uint32_t member_count =  static_cast< uint32_t >( _exh->get_members().size() );
+    auto& file_ptr = cacheEntryIt->second.file;
 
-    std::vector< Field > fields;
+    std::vector< char > dataCpy = file_ptr->get_data_sections().front();
-    fields.reserve( _exh->get_members().size() );
+    std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
 
-    auto stringBaseOffset = cacheEntryIt->second.offset + _exh->get_header().data_offset;
+    // Get the vector fields for the given record and preallocate it
+    auto fields = _data[ id ];
+    fields.reserve( member_count );
+    iss.seekg( cacheEntryIt->second.offset + 6 );
 
-    for( auto& memberEntry : _exh->get_exh_members() )
+    uint8_t subRows = *reinterpret_cast< uint8_t* >( &dataCpy[ cacheEntryIt->second.offset + 5 ] );
+
+    for( auto& member_entry : _exh->get_exh_members() )
     {
+      // Seek to the position of the member to extract.
+      // 6 is because we have uint32_t/uint16_t at the start of each record
+      iss.seekg( cacheEntryIt->second.offset + 6 + member_entry.offset );
+
       // Switch depending on the type to extract
-      switch( memberEntry.type )
+      switch( member_entry.type )
       {
         case DataType::string:
           // Extract the offset to the actual string
-          // Then extract the actual string from that offset
+          // Seek to it then extract the actual string
         {
-          auto stringOffset = extract< uint32_t >( dataCpy, cacheEntryIt->second.offset + memberEntry.offset, false );
+          auto string_offset = extract< uint32_t >( iss, "string_offset", false );
-          fields.emplace_back( utils::bparse::extract_cstring( dataCpy, stringBaseOffset + stringOffset ) );
+          iss.seekg( cacheEntryIt->second.offset + 6 + _exh->get_header().data_offset + string_offset );
+          fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
         }
           break;
 
         case DataType::boolean:
-          fields.emplace_back( extract< bool >( dataCpy, cacheEntryIt->second.offset + memberEntry.offset ) );
+          fields.emplace_back( extract< bool >( iss, "bool" ) );
           break;
 
         case DataType::int8:
-          fields.emplace_back( extract< int8_t >( dataCpy, cacheEntryIt->second.offset + memberEntry.offset ) );
+          fields.emplace_back( extract< int8_t >( iss, "int8_t" ) );
           break;
 
         case DataType::uint8:
-          fields.emplace_back( extract< uint8_t >( dataCpy, cacheEntryIt->second.offset + memberEntry.offset ) );
+          fields.emplace_back( extract< uint8_t >( iss, "uint8_t" ) );
           break;
 
         case DataType::int16:
-          fields.emplace_back( extract< int16_t >( dataCpy, cacheEntryIt->second.offset + memberEntry.offset, false ) );
+          fields.emplace_back( extract< int16_t >( iss, "int16_t", false ) );
           break;
 
         case DataType::uint16:
-          fields.emplace_back( extract< uint16_t >( dataCpy, cacheEntryIt->second.offset + memberEntry.offset, false ) );
+          fields.emplace_back( extract< uint16_t >( iss, "uint16_t", false ) );
           break;
 
         case DataType::int32:
-          fields.emplace_back( extract< int32_t >( dataCpy, cacheEntryIt->second.offset + memberEntry.offset, false ) );
+          fields.emplace_back( extract< int32_t >( iss, "int32_t", false ) );
           break;
 
         case DataType::uint32:
-          fields.emplace_back( extract< uint32_t >( dataCpy, cacheEntryIt->second.offset + memberEntry.offset, false ) );
+          fields.emplace_back( extract< uint32_t >( iss, "uint32_t", false ) );
           break;
 
         case DataType::float32:
-          fields.emplace_back( extract< float >( dataCpy, cacheEntryIt->second.offset + memberEntry.offset, false ) );
+          fields.emplace_back( extract< float >( iss, "float", false ) );
           break;
 
         case DataType::uint64:
-          fields.emplace_back( extract< uint64_t >( dataCpy, cacheEntryIt->second.offset + memberEntry.offset, false ) );
+          fields.emplace_back( extract< uint64_t >( iss, "uint64_t", false ) );
           break;
 
         default:
-          auto type = static_cast< uint16_t >( memberEntry.type );
+          auto type = static_cast< uint16_t >( member_entry.type );
           if( type < 0x19 || type > 0x20 )
             throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
-          fields.emplace_back( ( extract< uint8_t >( dataCpy, cacheEntryIt->second.offset + memberEntry.offset ) & ( 1 << ( type - 0x19 ) ) ) != 0 );
+          uint64_t val = extract< uint64_t >( iss, "bool" );
+          int32_t shift = type - 0x19;
+          int32_t i = 1 << shift;
+          val &= i;
+          fields.emplace_back( ( val & i ) == i );
           break;
       }
     }
@@ -233,99 +279,106 @@ namespace xiv::exd
   }
 
   // Get all rows
-  const std::map< ExdRow, std::vector< Field >, exdRowSort > Exd::get_rows()
+  const std::map< uint32_t, std::vector< Field>>& Exd::get_rows()
   {
-    std::map< ExdRow, std::vector< Field >, exdRowSort > data;
+    // Iterates over all the files
-
+    const uint32_t member_count =  static_cast< uint32_t >( _exh->get_members().size() );
-    // Iterates over all the cached ids
+    for( auto& file_ptr : _files )
-    const uint32_t memberCount = _exh->get_members().size();
-    for( auto& cacheEntry : _idCache )
     {
-      std::vector< char > dataCpy = cacheEntry.second.file->get_data_sections().front();
+      // Get a stream
+      std::vector< char > dataCpy = file_ptr->get_data_sections().front();
+      std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
 
-      auto baseOffset = cacheEntry.second.offset;
+      // Extract the header and skip to the record indices
-      auto stringBaseOffset = baseOffset + _exh->get_header().data_offset;
+      auto exd_header = extract< ExdHeader >( iss );
+      iss.seekg( 0x20 );
 
-      for( int32_t i = 0; i < cacheEntry.second.subRows; i++ )
+      // Preallocate and extract the record_indices
+      const uint32_t record_count = exd_header.index_size / sizeof( ExdRecordIndex );
+      std::vector< ExdRecordIndex > record_indices;
+      record_indices.reserve( record_count );
+      for( uint32_t i = 0; i < record_count; ++i )
+      {
+        record_indices.emplace_back( extract< ExdRecordIndex >( iss ) );
+      }
+
+      for( auto& record_index : record_indices )
       {
         // Get the vector fields for the given record and preallocate it
-        ExdRow row = { cacheEntry.first, i };
+        auto& fields = _data[ record_index.id ];
-        auto& fields = data[ row ];
+        fields.reserve( member_count );
-        fields.reserve( memberCount );
 
-        if( _exh->get_header().variant == 2 )
+        for( auto& member_entry : _exh->get_exh_members() )
-          baseOffset = cacheEntry.second.offset + ( i * _exh->get_header().data_offset + 2 * ( i + 1 ) );
-
-        for( auto& memberEntry : _exh->get_exh_members() )
           //for( auto& member_entry : _exh->get_members() )
         {
+          // Seek to the position of the member to extract.
+          // 6 is because we have uint32_t/uint16_t at the start of each record
+          iss.seekg( record_index.offset + 6 + member_entry.offset );
+
           // Switch depending on the type to extract
-          switch( memberEntry.type )
+          switch( member_entry.type )
           {
             case DataType::string:
               // Extract the offset to the actual string
-              // Then extract the actual string from that offset
+              // Seek to it then extract the actual string
             {
-              if( _exh->get_header().variant == 1 )
+              auto string_offset = extract< uint32_t >( iss, "string_offset", false );
-              {
+              iss.seekg( record_index.offset + 6 + _exh->get_header().data_offset + string_offset );
-                auto stringOffset = extract< uint32_t >( dataCpy, baseOffset + memberEntry.offset, false );
+              fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
-                fields.emplace_back( utils::bparse::extract_cstring( dataCpy, stringBaseOffset + stringOffset ) );
-              }
-              else if( _exh->get_header().variant == 2 )
-              {
-                throw std::runtime_error( "String not implemented for variant 2!" );
-              }
             }
               break;
 
             case DataType::boolean:
-              fields.emplace_back( extract< bool >( dataCpy, baseOffset + memberEntry.offset ) );
+              fields.emplace_back( extract< bool >( iss, "bool" ) );
               break;
 
             case DataType::int8:
-              fields.emplace_back( extract< int8_t >( dataCpy, baseOffset + memberEntry.offset ) );
+              fields.emplace_back( extract< int8_t >( iss, "int8_t" ) );
               break;
 
             case DataType::uint8:
-              fields.emplace_back( extract< uint8_t >( dataCpy, baseOffset + memberEntry.offset ) );
+              fields.emplace_back( extract< uint8_t >( iss, "uint8_t" ) );
               break;
 
             case DataType::int16:
-              fields.emplace_back( extract< int16_t >( dataCpy, baseOffset + memberEntry.offset, false ) );
+              fields.emplace_back( extract< int16_t >( iss, "int16_t", false ) );
               break;
 
             case DataType::uint16:
-              fields.emplace_back( extract< uint16_t >( dataCpy, baseOffset + memberEntry.offset, false ) );
+              fields.emplace_back( extract< uint16_t >( iss, "uint16_t", false ) );
               break;
 
             case DataType::int32:
-              fields.emplace_back( extract< int32_t >( dataCpy, baseOffset + memberEntry.offset, false ) );
+              fields.emplace_back( extract< int32_t >( iss, "int32_t", false ) );
               break;
 
             case DataType::uint32:
-              fields.emplace_back( extract< uint32_t >( dataCpy, baseOffset + memberEntry.offset, false ) );
+              fields.emplace_back( extract< uint32_t >( iss, "uint32_t", false ) );
               break;
 
             case DataType::float32:
-              fields.emplace_back( extract< float >( dataCpy, baseOffset + memberEntry.offset, false ) );
+              fields.emplace_back( extract< float >( iss, "float", false ) );
               break;
 
             case DataType::uint64:
-              fields.emplace_back( extract< uint64_t >( dataCpy, baseOffset + memberEntry.offset, false ) );
+              fields.emplace_back( extract< uint64_t >( iss, "uint64_t", false ) );
               break;
 
             default:
-              auto type = static_cast< uint16_t >( memberEntry.type );
+              auto type = static_cast< uint16_t >( member_entry.type );
               if( type < 0x19 || type > 0x20 )
                 throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
-              fields.emplace_back( ( extract< uint8_t >( dataCpy, baseOffset + memberEntry.offset ) & ( 1 << ( type - 0x19 ) ) ) != 0 );
+              uint64_t val = extract< uint64_t >( iss, "bool" );
+              int32_t shift = type - 0x19;
+              int32_t i = 1 << shift;
+              val &= i;
+              fields.emplace_back( ( val & i ) == i );
               break;
           }
         }
       }
     }
-    return data;
+    return _data;
-
   }
 
 }

deps/datReader/Exd.h

@@ -1,15 +1,17 @@
-#ifndef XIV_EXD_EXD_H
+#pragma once
-#define XIV_EXD_EXD_H
 
 #include <memory>
 #include <map>
-#include <unordered_map>
-#include <set>
 
 #include <variant>
 
 #include "File.h"
-
+#include "Exd/Common.h"
+#include "Exd/Structs.h"
+#include "stream.h"
+#include <fstream>
+#include "Exh.h"
+#include "bparse.h"
 namespace xiv::exd
 {
 
@@ -32,54 +34,165 @@ namespace xiv::exd
   {
     std::shared_ptr< dat::File > file;
     uint32_t offset;
-    uint8_t subRows;
-  };
-
-  struct ExdRow
-  {
-    uint32_t rowId;
-    uint8_t subRowId;
-  };
-
-  struct exdRowSort
-  {
-    constexpr bool operator()( const ExdRow& _Left, const ExdRow& _Right ) const
-    {
-      if( _Left.rowId == _Right.rowId )
-        return _Left.subRowId < _Right.subRowId;
-
-      return _Left.rowId < _Right.rowId;
-    }
   };
 
   // Data for a given language
   class Exd
   {
   public:
-    // exh: the header
+    // i_exh: the header
-    // files: the multiple exd files
+    // i_files: the multiple exd files
     Exd()
     {
     }
 
-    Exd( std::shared_ptr< Exh > exh, const std::vector< std::shared_ptr< dat::File > >& files );
+    Exd( std::shared_ptr< Exh > i_exh, const std::vector< std::shared_ptr< dat::File>>& i_files );
 
     ~Exd();
 
     // Get a row by its id
     const std::vector< Field > get_row( uint32_t id );
 
+    template< typename T >
+    std::shared_ptr< Component::Excel::ExcelStruct< T > > get_row( uint32_t id )
+    {
+      using namespace xiv::utils;
+      auto cacheEntryIt = _idCache.find( id );
+      if( cacheEntryIt == _idCache.end() )
+        throw std::out_of_range( "Id not found: " + std::to_string( id ) );
+
+      if( sizeof( T ) != _exh->get_header().data_offset )
+      {
+        throw std::runtime_error(
+          "the struct size (" + std::to_string( sizeof( T ) ) + ") doesn't match the size in the header (" +
+          std::to_string( _exh->get_header().data_offset ) + ")!" );
+      }
+
+      // Iterates over all the files
+      const uint32_t member_count = static_cast< uint32_t >( _exh->get_members().size() );
+      auto& file_ptr = cacheEntryIt->second.file;
+
+      std::vector< char > dataCpy = file_ptr->get_data_sections().front();
+      std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
+
+      auto pSheet = std::make_shared< Component::Excel::ExcelStruct< T > >();
+
+      // Get the vector fields for the given record and preallocate it
+      auto fields = _data[ id ];
+      fields.reserve( member_count );
+      iss.seekg( cacheEntryIt->second.offset + 6 );
+
+      iss.read( reinterpret_cast<char*>( &pSheet.get()->_data ), sizeof( T ) );
+
+      int stringCount = 0;
+      for( auto& member_entry : _exh->get_exh_members() )
+      {
+
+        // Seek to the position of the member to extract.
+        // 6 is because we have uint32_t/uint16_t at the start of each record
+        iss.seekg( cacheEntryIt->second.offset + 6 + member_entry.offset );
+
+        // Switch depending on the type to extract
+        switch( member_entry.type )
+        {
+          case DataType::string:
+            // Extract the offset to the actual string
+            // Seek to it then extract the actual string
+            {
+              auto string_offset = bparse::extract< uint32_t >( iss, "string_offset", false );
+              iss.seekg( cacheEntryIt->second.offset + 6 + _exh->get_header().data_offset + string_offset );
+              std::string value = utils::bparse::extract_cstring( iss, "string" );
+              auto it = pSheet->_strings.insert( pSheet->_strings.end(), value );
+              *reinterpret_cast< uint32_t* >( pSheet->ptr() + member_entry.offset ) =
+                static_cast< uint32_t >( std::distance( pSheet->_strings.begin(), it ) );
+            }
+            break;
+
+          case DataType::boolean:
+            bparse::extract< bool >( iss, "bool" );
+            break;
+
+          case DataType::int8:
+            bparse::extract< int8_t >( iss, "int8_t" );
+            break;
+
+          case DataType::uint8:
+            bparse::extract< uint8_t >( iss, "uint8_t" );
+            break;
+
+
+          case DataType::int16:
+            {
+              int16_t value = bparse::extract< int16_t >( iss, "int16_t", false );
+              *reinterpret_cast< int16_t* >( pSheet->ptr() + member_entry.offset ) = value;
+            }
+            break;
+
+          case DataType::uint16:
+            {
+              uint16_t value = bparse::extract< uint16_t >( iss, "uint16_t", false );
+              *reinterpret_cast< uint16_t* >( pSheet->ptr() + member_entry.offset ) = value;
+            }
+            break;
+
+          case DataType::int32:
+            {
+              int32_t value = bparse::extract< int32_t >( iss, "int32_t", false );
+              *reinterpret_cast< int32_t* >( pSheet->ptr() + member_entry.offset ) = value;
+            }
+            break;
+
+          case DataType::uint32:
+            {
+              uint32_t value = bparse::extract< uint32_t >( iss, "uint32_t", false );
+              *reinterpret_cast< uint32_t* >( pSheet->ptr() + member_entry.offset ) = value;
+            }
+            break;
+
+          case DataType::float32:
+            {
+              float value = bparse::extract< float >( iss, "float", false );
+              *reinterpret_cast< float* >( pSheet->ptr() + member_entry.offset ) = value;
+            }
+            break;
+
+          case DataType::uint64:
+            {
+              uint64_t value = bparse::extract< uint64_t >( iss, "uint64_t", false );
+              *reinterpret_cast< uint64_t* >( pSheet->ptr() + member_entry.offset ) = value;
+            }
+            break;
+
+          default:
+            auto type = static_cast< uint16_t >( member_entry.type );
+            if( type < 0x19 || type > 0x20 )
+              throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
+            uint64_t val = bparse::extract< uint64_t >( iss, "bool" );
+            int32_t shift = type - 0x19;
+            int32_t i = 1 << shift;
+            val &= i;
+            fields.emplace_back( ( val & i ) == i );
+            break;
+        }
+      }
+
+      return pSheet;
+
+    }
+
     // Get a row by its id and sub-row
     const std::vector< Field > get_row( uint32_t id, uint32_t subRow );
 
     // Get all rows
-    const std::map< ExdRow, std::vector< Field >, exdRowSort > get_rows();
+    const std::map< uint32_t, std::vector< Field>>& get_rows();
 
   protected:
+    // Data indexed by the ID of the row, the vector is field with the same order as exh.members
+    std::map< uint32_t, std::vector< Field>> _data;
+    std::vector< std::shared_ptr< dat::File>> _files;
     std::shared_ptr< Exh > _exh;
     std::map< uint32_t, ExdCacheEntry > _idCache;
   };
 
 }
 
-#endif // XIV_EXD_EXD_H

deps/datReader/Exd/Common.h

@@ -0,0 +1,103 @@
+#pragma once
+namespace Component::Excel
+{
+
+  struct ExcelDataRowHeader
+  {
+    uint32_t dataSize;
+    uint16_t rowCount;
+  };
+
+  enum Language : int32_t
+  {
+    LANGUAGE_ALL = 0x0,
+    LANGUAGE_JP = 0x1,
+    LANGUAGE_EN = 0x2,
+    LANGUAGE_DE = 0x3,
+    LANGUAGE_FR = 0x4,
+    LANGUAGE_CH = 0x5,
+    LANGUAGE_MAX = 0x6,
+  };
+
+  struct StringOffset
+  {
+    uint32_t m_offset;
+  };
+
+  typedef int32_t LinkList[12];
+
+/*  struct LinkList::Holder
+  {
+    Common::Component::Excel::LinkList m_begin;
+    Common::Component::Excel::LinkList m_end;
+    uint32_t m_size;
+  };*/
+
+  struct ExcelEntryKey
+  {
+    uint32_t m_mainkey;
+    uint16_t m_hash;
+    uint16_t m_subkey_info[3];
+  };
+
+  typedef int32_t StringPOD[4];
+
+  union ExcelCell
+  {
+    bool b;
+    int8_t s8;
+    uint8_t u8;
+    int16_t s16;
+    uint16_t u16;
+    int32_t s32;
+    uint32_t u32;
+    int64_t s64;
+    uint64_t u64;
+    float f;
+    StringPOD str_pod;
+    StringPOD str_old;
+    StringOffset str_new;
+    StringPOD str;
+    StringOffset str_ofs;
+    int8_t *bin;
+    uint8_t boolean0;
+    uint8_t boolean1;
+    uint8_t boolean2;
+    uint8_t boolean3;
+    uint8_t boolean4;
+    uint8_t boolean5;
+    uint8_t boolean6;
+    uint8_t boolean7;
+  };
+
+  using ExdCell = ExcelCell;
+
+  enum CELL_TYPE : uint32_t
+  {
+    TYPE_NONE = 0xFFFFFFFF,
+    TYPE_STR = 0x0,
+    TYPE_BOOL = 0x1,
+    TYPE_S8 = 0x2,
+    TYPE_U8 = 0x3,
+    TYPE_S16 = 0x4,
+    TYPE_U16 = 0x5,
+    TYPE_S32 = 0x6,
+    TYPE_U32 = 0x7,
+    TYPE_F16 = 0x8,
+    TYPE_FLOAT = 0x9,
+    TYPE_S64 = 0xA,
+    TYPE_U64 = 0xB,
+    TYPE_BIN = 0xC,
+    TYPE_BOOLEAN0 = 0x19,
+    TYPE_BOOLEAN1 = 0x1A,
+    TYPE_BOOLEAN2 = 0x1B,
+    TYPE_BOOLEAN3 = 0x1C,
+    TYPE_BOOLEAN4 = 0x1D,
+    TYPE_BOOLEAN5 = 0x1E,
+    TYPE_BOOLEAN6 = 0x1F,
+    TYPE_BOOLEAN7 = 0x20,
+    TYPE_MAX_4 = 0x21,
+  };
+
+
+}

deps/datReader/ExdCat.cpp

@@ -78,4 +78,15 @@ namespace xiv::exd
     return *( ln_it->second );
   }
 
+  const Exd& Cat::get_data( Language language ) const
+  {
+    auto ln_it = _data.find( language );
+    if( ln_it == _data.end() )
+    {
+      return get_data_ln( Language::none );
+    }
+
+    return *( ln_it->second );
+  }
+
 }

deps/datReader/ExdCat.h

@@ -1,5 +1,4 @@
-#ifndef XIV_EXD_CAT_H
+#pragma once
-#define XIV_EXD_CAT_H
 
 #include <memory>
 #include <map>
@@ -50,6 +49,8 @@ namespace xiv
 
          // Returns data for a specific language
          const Exd& get_data_ln( Language i_language = Language::none ) const;
+
+         const Exd& get_data( Language language = Language::none ) const;
       protected:
          const std::string _name;
 
@@ -62,5 +63,3 @@ namespace xiv
 
    }
 }
-
-#endif // XIV_EXD_CAT_H

deps/datReader/ExdData.h

@@ -1,12 +1,12 @@
-#ifndef XIV_EXD_EXDDATA_H
+#pragma once
-#define XIV_EXD_EXDDATA_H
 
 #include <unordered_map>
+#include <string>
 #include <memory>
+#include <vector>
 #include <mutex>
 
 #include <filesystem>
-#include <vector>
 
 namespace xiv
 {
@@ -54,5 +54,3 @@ namespace xiv
 
    }
 }
-
-#endif // XIV_EXD_EXDDATA_H

deps/datReader/Exh.h

@@ -1,5 +1,4 @@
-#ifndef XIV_EXD_EXH_H
+#pragma once
-#define XIV_EXD_EXH_H
 
 #include <map>
 
@@ -7,8 +6,7 @@
 
 namespace xiv::exd
 {
-  enum class DataType :
+  enum class DataType : uint16_t
-    uint16_t
   {
     string = 0,
     boolean = 1,
@@ -98,8 +96,7 @@ namespace xiv
   namespace exd
   {
 
-    enum Language :
+    enum Language : uint16_t;
-      uint16_t;
 
     // Header file for exd data
     class Exh
@@ -132,4 +129,3 @@ namespace xiv
   }
 }
 
-#endif // XIV_EXD_EXH_H

deps/datReader/File.h

@@ -1,5 +1,4 @@
-#ifndef XIV_DAT_FILE_H
+#pragma once
-#define XIV_DAT_FILE_H
 
 #include <vector>
 
@@ -45,4 +44,3 @@ namespace xiv::dat
   };
 }
 
-#endif // XIV_DAT_FILE_H

deps/datReader/GameData.cpp

@@ -5,6 +5,7 @@
 #include <algorithm>
 
 #include <map>
+#include <utility>
 #include <zlib/zlib.h>
 
 #include "bparse.h"
@@ -49,8 +50,8 @@ std::unordered_map< uint32_t, std::string > categoryIdToNameMap =
 
 namespace xiv::dat
 {
-  GameData::GameData( const std::filesystem::path& path ) try :
+  GameData::GameData( std::filesystem::path path ) try :
-    m_path( path )
+    m_path( std::move( path ) )
   {
     int maxExLevel = 0;
 
@@ -273,8 +274,8 @@ namespace xiv::dat
     std::string filenamePart = pathLower.substr( lastSlashPos + 1 );
 
     // Get the crc32 values from zlib, to compensate the final XOR 0xFFFFFFFF that isnot done in the exe we just reXOR
-    dirHash = crc32( 0, reinterpret_cast<const uint8_t*>( dirPart.data() ), dirPart.size() ) ^ 0xFFFFFFFF;
+    dirHash = crc32( 0, reinterpret_cast< const uint8_t* >( dirPart.data() ), static_cast< uInt >( dirPart.size() ) ) ^ 0xFFFFFFFF;
-    filenameHash = crc32( 0, reinterpret_cast<const uint8_t*>( filenamePart.data() ), filenamePart.size() ) ^ 0xFFFFFFFF;
+    filenameHash = crc32( 0, reinterpret_cast< const uint8_t* >( filenamePart.data() ), static_cast< uInt >( filenamePart.size() ) ) ^ 0xFFFFFFFF;
   }
 
   void GameData::createCategory( uint32_t catNum )

deps/datReader/GameData.h

@@ -1,12 +1,13 @@
-#ifndef XIV_DAT_GAMEDATA_H
+#pragma once
-#define XIV_DAT_GAMEDATA_H
 
-#include <memory>
 #include <unordered_map>
+#include <string>
+#include <memory>
+#include <vector>
+#include <stdint.h>
 #include <mutex>
 
 #include <filesystem>
-#include <vector>
 
 namespace xiv::dat
 {
@@ -21,7 +22,7 @@ namespace xiv::dat
   {
   public:
     // This should be the path in which the .index/.datX files are located
-    GameData( const std::filesystem::path& path );
+    GameData( std::filesystem::path path );
 
     ~GameData();
 
@@ -90,4 +91,3 @@ namespace xiv::dat
 
 }
 
-#endif // XIV_DAT_GAMEDATA_H

deps/datReader/bparse.cpp

@@ -6,8 +6,3 @@ std::string xiv::utils::bparse::extract_cstring( std::istream& i_stream, const s
    std::getline( i_stream, temp_str, '\0' );
    return temp_str;
 }
-
-std::string xiv::utils::bparse::extract_cstring( std::vector< char >& data, uint32_t pos )
-{
-  return &data[ pos ];
-}

deps/datReader/bparse.h

@@ -1,6 +1,4 @@
-#ifndef XIV_UTILS_BPARSE_H
+#pragma once
-#define XIV_UTILS_BPARSE_H
-
 #include <type_traits>
 #include <iomanip>
 #include <sstream>
@@ -31,7 +29,7 @@ namespace xiv::utils::bparse
   template< typename StructType >
   void read( std::istream& i_stream, StructType& i_struct )
   {
-    static_assert( std::is_pod< StructType >::value, "StructType must be a POD to be able to use read." );
+    static_assert( std::is_trivially_copyable< StructType >::value, "StructType must be a POD to be able to use read." );
     i_stream.read( reinterpret_cast<char*>( &i_struct ), sizeof( StructType ) );
   }
 
@@ -93,27 +91,7 @@ namespace xiv::utils::bparse
     }
   }
 
-  template< typename StructType >
-  StructType extract( std::vector< char >& data, uint32_t pos, bool isLe = true )
-  {
-    StructType tempStruct = *reinterpret_cast< StructType* >( &data[ pos ] );
-
-    if( std::is_class< StructType >::value )
-    {
-      reorder( tempStruct );
-    }
-    else if( !isLe )
-    {
-      tempStruct = byteswap( tempStruct );
-    }
-    return tempStruct;
-  }
-
   // For cstrings
   std::string extract_cstring( std::istream& i_stream, const std::string& i_name );
 
-  std::string extract_cstring( std::vector< char >& data, uint32_t pos );
-
 }
-
-#endif // XIV_UTILS_BPARSE_H

deps/datReader/conv.h

@@ -1,5 +1,4 @@
-#ifndef XIV_UTILS_CONV_H
+#pragma once
-#define XIV_UTILS_CONV_H
 
 #include <cstdint>
 #include <vector>
@@ -12,4 +11,3 @@ namespace xiv::utils::conv
   float ubyte2float( const uint8_t i_value );
 }
 
-#endif // XIV_UTILS_CONV_H

deps/datReader/crc32.cpp

@@ -101,7 +101,7 @@ namespace xiv::utils::crc32
   void generate_hashes_1( std::string& i_format, const uint32_t i_first_index, std::vector< uint32_t >& o_hashes )
   {
     char* str = const_cast<char*>(i_format.data());
-    const uint32_t str_size = i_format.size();
+    const auto str_size = static_cast< uInt >( i_format.size() );
 
     o_hashes.resize( 10000 );
 
@@ -130,7 +130,7 @@ namespace xiv::utils::crc32
                           std::vector< uint32_t >& o_hashes )
   {
     char* str = const_cast<char*>(i_format.data());
-    const uint32_t str_size = i_format.size();
+    const auto str_size = static_cast< uInt >( i_format.size() );
 
     o_hashes.resize( 100000000 );
 

deps/datReader/crc32.h

@@ -1,5 +1,4 @@
-#ifndef XIV_UTILS_CRC32_H
+#pragma once
-#define XIV_UTILS_CRC32_H
 
 #include <cstdint>
 #include <vector>
@@ -22,4 +21,3 @@ namespace xiv::utils::crc32
 
 }
 
-#endif // XIV_UTILS_CRC32_H

deps/datReader/zlib.cpp

@@ -10,11 +10,11 @@ namespace xiv::utils::zlib
   void compress( const std::vector< char >& in, std::vector< char >& out )
   {
     // Fetching upper bound for out size
-    auto out_size = compressBound( in.size() );
+    auto out_size = compressBound( static_cast< uLong >( in.size() ) );
     out.resize( out_size );
 
-    auto ret = compress2( reinterpret_cast<uint8_t*>(out.data()), &out_size,
+    auto ret = compress2( reinterpret_cast< uint8_t* >( out.data() ), &out_size,
-                          reinterpret_cast<const uint8_t*>(in.data()), in.size(), Z_BEST_COMPRESSION );
+                          reinterpret_cast< const uint8_t* >( in.data() ), static_cast< uLong >( in.size() ), Z_BEST_COMPRESSION );
 
     if( ret != Z_OK )
     {
@@ -24,13 +24,13 @@ namespace xiv::utils::zlib
     out.resize( out_size );
   }
 
-  void no_header_decompress( uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t out_size )
+  void no_header_decompress( uint8_t* in, size_t in_size, uint8_t* out, size_t out_size )
   {
     z_stream strm;
     strm.zalloc = Z_NULL;
     strm.zfree = Z_NULL;
     strm.opaque = Z_NULL;
-    strm.avail_in = in_size;
+    strm.avail_in = static_cast< uInt >( in_size );
     strm.next_in = Z_NULL;
 
     // Init with -15 because we do not have header in this compressed data
@@ -42,7 +42,7 @@ namespace xiv::utils::zlib
 
     // Set pointers to the right addresses
     strm.next_in = in;
-    strm.avail_out = out_size;
+    strm.avail_out = static_cast< uInt >( out_size );
     strm.next_out = out;
 
     // Effectively decompress data

deps/datReader/zlib.h

@@ -9,7 +9,7 @@ namespace xiv::utils::zlib
 
   void compress( const std::vector< char >& in, std::vector< char >& out );
 
-  void no_header_decompress( uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t out_size );
+  void no_header_decompress( uint8_t* in, size_t in_size, uint8_t* out, size_t out_size );
 
 }
 

deps/datReaderPs3/CMakeLists.txt

@@ -0,0 +1,20 @@
+cmake_minimum_required(VERSION 3.0.2)
+project(Sapphire)
+
+include_directories( "../" )
+
+file( GLOB UTILS_PUBLIC_INCLUDE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*" )
+file( GLOB UTILS_SOURCE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/src/*" )
+
+set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} )
+
+add_library( xivdatps3 ${UTILS_PUBLIC_INCLUDE_FILES} ${UTILS_SOURCE_FILES} )
+
+if (UNIX)
+    target_link_libraries( xivdatps3 PUBLIC dl )
+    target_link_libraries( xivdatps3 PUBLIC z )
+else()
+    target_link_libraries( xivdatps3 PUBLIC zlib )
+endif()
+target_include_directories( xivdatps3 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} )
+#cotire( xivdat )

deps/datReaderPs3/Dat.cpp

@@ -0,0 +1,334 @@
+#include "Dat.h"
+
+#include "zlib.h"
+
+#include "File.h"
+
+namespace {
+  const uint32_t model_section_count = 0xB;
+}
+
+namespace xivps3::dat
+{
+  struct DatFileHeader
+  {
+    uint32_t size;
+    FileType entry_type;
+    uint32_t total_uncompressed_size;
+    uint32_t unknown[0x2];
+  };
+
+  struct DatBlockRecord
+  {
+    uint32_t offset;
+    uint32_t size;
+    uint32_t unknown[0x4];
+    SqPackBlockHash block_hash;
+  };
+
+  struct DatBlockHeader
+  {
+    uint32_t size;
+    uint32_t unknown1;
+    uint32_t compressed_size;
+    uint32_t uncompressed_size;
+  };
+
+  struct DatStdFileBlockInfos
+  {
+    uint32_t offset;
+    uint16_t size;
+    uint16_t uncompressed_size;
+  };
+
+  struct DatMdlFileBlockInfos
+  {
+    uint32_t unknown1;
+    uint32_t uncompressed_sizes[::model_section_count];
+    uint32_t compressed_sizes[::model_section_count];
+    uint32_t offsets[::model_section_count];
+    uint16_t block_ids[::model_section_count];
+    uint16_t block_counts[::model_section_count];
+    uint32_t unknown2[0x2];
+  };
+
+  struct DatTexFileBlockInfos
+  {
+    uint32_t offset;
+    uint32_t size;
+    uint32_t uncompressed_size;
+    uint32_t block_id;
+    uint32_t block_count;
+  };
+}
+
+namespace xivps3::utils::bparse
+{
+  template<>
+  inline void reorder< xivps3::dat::DatFileHeader >( xivps3::dat::DatFileHeader& i_struct )
+  {
+    xivps3::utils::bparse::reorder( i_struct.size );
+    xivps3::utils::bparse::reorder( i_struct.entry_type );
+    xivps3::utils::bparse::reorder( i_struct.total_uncompressed_size );
+    i_struct.size = xivps3::utils::bparse::byteswap( i_struct.size );
+    i_struct.entry_type = xivps3::utils::bparse::byteswap( i_struct.entry_type );
+    i_struct.total_uncompressed_size = xivps3::utils::bparse::byteswap( i_struct.total_uncompressed_size );
+    for( int32_t i = 0; i < 0x2; ++i )
+    {
+      xivps3::utils::bparse::reorder( i_struct.unknown[ i ] );
+    }
+  }
+
+  template<>
+  inline void reorder< xivps3::dat::DatBlockRecord >( xivps3::dat::DatBlockRecord& i_struct )
+  {
+    i_struct.offset = xivps3::utils::bparse::byteswap( i_struct.offset );
+    i_struct.size = xivps3::utils::bparse::byteswap( i_struct.size );
+    for( int32_t i = 0; i < 0x4; ++i )
+    {
+      xivps3::utils::bparse::reorder( i_struct.unknown[ i ] );
+    }
+    xivps3::utils::bparse::reorder( i_struct.block_hash );
+  }
+
+  template<>
+  inline void reorder< xivps3::dat::DatBlockHeader >( xivps3::dat::DatBlockHeader& i_struct )
+  {
+    xivps3::utils::bparse::reorder( i_struct.size );
+    xivps3::utils::bparse::reorder( i_struct.unknown1 );
+    xivps3::utils::bparse::reorder( i_struct.compressed_size );
+    xivps3::utils::bparse::reorder( i_struct.uncompressed_size );
+
+    i_struct.size = xivps3::utils::bparse::byteswap( i_struct.size );
+    i_struct.unknown1 = xivps3::utils::bparse::byteswap( i_struct.unknown1 );
+    i_struct.compressed_size = xivps3::utils::bparse::byteswap( i_struct.compressed_size );
+    i_struct.uncompressed_size = xivps3::utils::bparse::byteswap( i_struct.uncompressed_size );
+  }
+
+  template<>
+  inline void reorder< xivps3::dat::DatStdFileBlockInfos >( xivps3::dat::DatStdFileBlockInfos& i_struct )
+  {
+    xivps3::utils::bparse::reorder( i_struct.offset );
+    xivps3::utils::bparse::reorder( i_struct.size );
+    xivps3::utils::bparse::reorder( i_struct.uncompressed_size );
+
+    i_struct.offset = xivps3::utils::bparse::byteswap( i_struct.offset );
+    i_struct.size = xivps3::utils::bparse::byteswap( i_struct.size );
+    i_struct.uncompressed_size = xivps3::utils::bparse::byteswap( i_struct.uncompressed_size );
+  }
+
+  template<>
+  inline void reorder< xivps3::dat::DatMdlFileBlockInfos >( xivps3::dat::DatMdlFileBlockInfos& i_struct )
+  {
+    xivps3::utils::bparse::reorder( i_struct.unknown1 );
+    for( auto i = 0; i < ::model_section_count; ++i )
+    {
+      xivps3::utils::bparse::reorder( i_struct.uncompressed_sizes[ i ] );
+    }
+    for( auto i = 0; i < ::model_section_count; ++i )
+    {
+      xivps3::utils::bparse::reorder( i_struct.compressed_sizes[ i ] );
+    }
+    for( auto i = 0; i < ::model_section_count; ++i )
+    {
+      xivps3::utils::bparse::reorder( i_struct.offsets[ i ] );
+    }
+    for( auto i = 0; i < ::model_section_count; ++i )
+    {
+      xivps3::utils::bparse::reorder( i_struct.block_ids[ i ] );
+    }
+    for( auto i = 0; i < ::model_section_count; ++i )
+    {
+      xivps3::utils::bparse::reorder( i_struct.block_counts[ i ] );
+    }
+    for( auto i = 0; i < 0x2; ++i )
+    {
+      xivps3::utils::bparse::reorder( i_struct.unknown2[ i ] );
+    }
+  }
+
+  template<>
+  inline void reorder< xivps3::dat::DatTexFileBlockInfos >( xivps3::dat::DatTexFileBlockInfos& i_struct )
+  {
+    xivps3::utils::bparse::reorder( i_struct.offset );
+    xivps3::utils::bparse::reorder( i_struct.size );
+    xivps3::utils::bparse::reorder( i_struct.uncompressed_size );
+    xivps3::utils::bparse::reorder( i_struct.block_id );
+    xivps3::utils::bparse::reorder( i_struct.block_count );
+  }
+}
+
+using xivps3::utils::bparse::extract;
+
+namespace xivps3::dat
+{
+
+  Dat::Dat( const std::filesystem::path& i_path, uint32_t i_nb ) :
+    SqPack( i_path ),
+    m_num( i_nb )
+  {
+    auto block_record = extract< DatBlockRecord >( m_handle );
+    block_record.offset *= 0x80;
+    isBlockValid( block_record.offset, block_record.size, block_record.block_hash );
+  }
+
+  Dat::~Dat()
+  {
+  }
+
+  std::unique_ptr< File > Dat::getFile( uint32_t i_offset )
+  {
+    std::unique_ptr< File > outputFile( new File() );
+    {
+      // Lock in this scope
+      std::lock_guard< std::mutex > lock( m_fileMutex );
+
+      // Seek to the start of the header of the file record and extract it
+      m_handle.seekg( i_offset );
+      auto file_header = extract< DatFileHeader >( m_handle );
+
+      switch( file_header.entry_type )
+      {
+        case FileType::empty:
+          throw std::runtime_error( "File is empty" );
+
+        case FileType::standard:
+        {
+          outputFile->_type = FileType::standard;
+
+          uint32_t number_of_blocks = extract< uint32_t >( m_handle, "number_of_blocks", false );
+
+          // Just extract offset infos for the blocks to extract
+          std::vector< DatStdFileBlockInfos > std_file_block_infos;
+          extract< DatStdFileBlockInfos >( m_handle, number_of_blocks, std_file_block_infos );
+
+          // Pre allocate data vector for the whole file
+          outputFile->_data_sections.resize( 1 );
+          auto& data_section = outputFile->_data_sections.front();
+
+          data_section.reserve( file_header.total_uncompressed_size );
+          // Extract each block
+          for( auto& file_block_info : std_file_block_infos )
+          {
+            extractBlock( i_offset + file_header.size + file_block_info.offset, data_section );
+          }
+        }
+          break;
+
+        case FileType::model:
+        {
+          outputFile->_type = FileType::model;
+
+          DatMdlFileBlockInfos mdlBlockInfo = extract< DatMdlFileBlockInfos >( m_handle );
+
+          // Getting the block number and read their sizes
+          const uint32_t block_count = mdlBlockInfo.block_ids[ ::model_section_count - 1 ] +
+                                       mdlBlockInfo.block_counts[ ::model_section_count - 1 ];
+          std::vector< uint16_t > block_sizes;
+          extract< uint16_t >( m_handle, "block_size", block_count, block_sizes );
+
+          // Preallocate sufficient space
+          outputFile->_data_sections.resize( ::model_section_count );
+
+          for( uint32_t i = 0; i < ::model_section_count; ++i )
+          {
+            // Preallocating for section
+            auto& data_section = outputFile->_data_sections[ i ];
+            data_section.reserve( mdlBlockInfo.uncompressed_sizes[ i ] );
+
+            uint32_t current_offset = i_offset + file_header.size + mdlBlockInfo.offsets[ i ];
+            for( uint32_t j = 0; j < mdlBlockInfo.block_counts[ i ]; ++j )
+            {
+              extractBlock( current_offset, data_section );
+              current_offset += block_sizes[ mdlBlockInfo.block_ids[ i ] + j ];
+            }
+          }
+        }
+          break;
+
+        case FileType::texture:
+        {
+          outputFile->_type = FileType::texture;
+
+          // Extracts mipmap entries and the block sizes
+          uint32_t sectionCount = extract< uint32_t >( m_handle, "sections_count" );
+
+          std::vector< DatTexFileBlockInfos > texBlockInfo;
+          extract< DatTexFileBlockInfos >( m_handle, sectionCount, texBlockInfo );
+
+          // Extracting block sizes
+          uint32_t block_count = texBlockInfo.back().block_id + texBlockInfo.back().block_count;
+          std::vector< uint16_t > block_sizes;
+          extract< uint16_t >( m_handle, "block_size", block_count, block_sizes );
+
+          outputFile->_data_sections.resize( sectionCount + 1 );
+
+          // Extracting header in section 0
+          const uint32_t header_size = texBlockInfo.front().offset;
+          auto& header_section = outputFile->_data_sections[ 0 ];
+          header_section.resize( header_size );
+
+          m_handle.seekg( i_offset + file_header.size );
+          m_handle.read( header_section.data(), header_size );
+
+          // Extracting other sections
+          for( uint32_t i = 0; i < sectionCount; ++i )
+          {
+            auto& data_section = outputFile->_data_sections[ i + 1 ];
+            auto& section_infos = texBlockInfo[ i ];
+            data_section.reserve( section_infos.uncompressed_size );
+
+            uint32_t current_offset = i_offset + file_header.size + section_infos.offset;
+            for( uint32_t j = 0; j < section_infos.block_count; ++j )
+            {
+              extractBlock( current_offset, data_section );
+              current_offset += block_sizes[ section_infos.block_id + j ];
+            }
+          }
+        }
+          break;
+
+        default:
+          throw std::runtime_error(
+            "Invalid entry_type: " + std::to_string( static_cast<uint32_t>(file_header.entry_type) ) );
+      }
+    }
+
+    return outputFile;
+  }
+
+  void Dat::extractBlock( uint32_t i_offset, std::vector< char >& o_data )
+  {
+    m_handle.seekg( i_offset );
+
+    DatBlockHeader block_header = extract< DatBlockHeader >( m_handle );
+
+    // Resizing the vector to write directly into it
+    const uint32_t data_size = o_data.size();
+    o_data.resize( data_size + block_header.uncompressed_size );
+
+    // 32000 in compressed_size means it is not compressed so take uncompressed_size
+    if( block_header.compressed_size == 32000 )
+    {
+      m_handle.read( o_data.data() + data_size, block_header.uncompressed_size );
+    }
+    else
+    {
+      // If it is compressed use zlib
+      // Read the data to be decompressed
+      std::vector< char > temp_buffer( block_header.compressed_size );
+      m_handle.read( temp_buffer.data(), block_header.compressed_size );
+
+      utils::zlib::no_header_decompress( reinterpret_cast<uint8_t*>(temp_buffer.data()),
+                                         temp_buffer.size(),
+                                         reinterpret_cast<uint8_t*>(o_data.data() + data_size),
+                                         block_header.uncompressed_size );
+    }
+  }
+
+  uint32_t Dat::getNum() const
+  {
+    return m_num;
+  }
+
+}

deps/datReaderPs3/Dat.h

@@ -0,0 +1,42 @@
+#ifndef XIV_DAT_DAT_H
+#define XIV_DAT_DAT_H
+
+#include "SqPack.h"
+
+#include <mutex>
+
+#include <filesystem>
+
+namespace xivps3::dat
+{
+
+  class File;
+
+  class Dat : public SqPack
+  {
+  public:
+     // Full path to the dat file
+     Dat( const std::filesystem::path& i_path, uint32_t i_nb );
+     virtual ~Dat();
+
+     // Retrieves a file given the offset in the dat file
+     std::unique_ptr<File> getFile( uint32_t i_offset );
+
+     // Appends to the vector the data of this block, it is assumed to be preallocated
+     // Is it also assumed that the m_fileMutex is currently locked by this thread before the call
+     void extractBlock( uint32_t i_offset, std::vector<char>& o_data );
+
+     // Returns the dat number
+     uint32_t getNum() const;
+
+  protected:
+     // File reading mutex to have only one thread reading the file at a time
+     std::mutex m_fileMutex;
+
+     // Dat nb
+     uint32_t m_num;
+  };
+
+}
+
+#endif // XIV_DAT_DAT_H

deps/datReaderPs3/DatCat.cpp

@@ -0,0 +1,86 @@
+#include "DatCat.h"
+
+#include "Index.h"
+#include "Dat.h"
+#include "File.h"
+#include "GameData.h"
+
+namespace xivps3
+{
+namespace dat
+{
+
+Cat::Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::string& name ) :
+   m_name( name ),
+   m_catNum( catNum ),
+   m_chunk( -1 )
+{
+   // From the category number, compute back the real filename for.index .datXs
+   std::stringstream ss;
+   ss << std::setw( 2 ) << std::setfill( '0' ) << std::hex << catNum;
+   std::string prefix = ss.str() + "0000.ps3.d";
+
+   // Creates the index: XX0000.win32.index
+   m_index = std::unique_ptr<Index>( new Index( basePath / "ffxiv" / ( prefix + ".index" ) ) );
+
+   // For all dat files linked to this index, create it: XX0000.win32.datX
+   for( uint32_t i = 0; i < getIndex().getDatCount(); ++i )
+   {
+      m_dats.emplace_back( std::make_unique< Dat >(basePath / "ffxiv" / ( prefix + ".dat" + std::to_string( i ) ), i ) );
+   }
+}
+
+Cat::Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::string& name, uint32_t exNum, uint32_t chunk ) :
+   m_name( name ),
+   m_catNum( catNum ),
+   m_chunk( chunk )
+{
+   // Creates the index: XX0000.win32.index
+   m_index = std::make_unique< Index >( basePath / GameData::buildDatStr( "ex" + std::to_string( exNum ), catNum, exNum, chunk, "ps3.d", "index" ) );
+
+   // For all dat files linked to this index, create it: XX0000.win32.datX
+   for( uint32_t i = 0; i < getIndex().getDatCount(); ++i )
+   {
+      m_dats.emplace_back( std::make_unique< Dat >( basePath / GameData::buildDatStr( "ex" + std::to_string( exNum ), catNum, exNum, chunk, "ps3.d", "dat" + std::to_string( i ) ), i ) );
+   }
+}
+
+Cat::~Cat()
+{
+
+}
+
+const Index& Cat::getIndex() const
+{
+   return *m_index;
+}
+
+std::unique_ptr<File> Cat::getFile(uint32_t dir_hash, uint32_t filename_hash) const
+{
+    // Fetch the correct hash_table_entry for these hashes, from that request the file from the right dat file
+    auto& hash_table_entry = getIndex().getHashTableEntry(dir_hash, filename_hash);
+    return m_dats[hash_table_entry.datNum]->getFile(hash_table_entry.datOffset);
+}
+
+bool Cat::doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const
+{
+   return getIndex().doesFileExist( dir_hash, filename_hash );
+}
+
+bool Cat::doesDirExist( uint32_t dir_hash ) const
+{
+   return getIndex().doesDirExist( dir_hash );
+}
+
+const std::string& Cat::getName() const
+{
+   return m_name;
+}
+
+uint32_t Cat::getCatNum() const
+{
+   return m_catNum;
+}
+
+}
+}

deps/datReaderPs3/DatCat.h

@@ -0,0 +1,68 @@
+#ifndef XIV_DAT_CAT_H
+#define XIV_DAT_CAT_H
+
+#include <memory>
+#include <vector>
+#include <filesystem>
+
+namespace xivps3::dat
+{
+
+  class Index;
+
+  class Dat;
+
+  class File;
+
+  // A category represents an .index and its associated .datX
+  class Cat
+  {
+  public:
+    // basePath: Path to the folder containingthe datfiles
+    // catNum: The number of the category
+    // name: The name of the category, empty if not known
+    Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::string& name );
+
+    // basePath: Path to the folder containingthe datfiles
+    // catNum: The number of the category
+    // name: The name of the category, empty if not known
+    // exNum: The number of the expansion to load from
+    // chunk: The chunk to load from
+    Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::string& name, uint32_t exNum,
+         uint32_t chunk );
+
+    ~Cat();
+
+    // Returns .index of the category
+    const Index& getIndex() const;
+
+    // Retrieve a file from the category given its hashes
+    std::unique_ptr< File > getFile( uint32_t dir_hash, uint32_t filename_hash ) const;
+
+
+    bool doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const;
+
+    bool doesDirExist( uint32_t dir_hash ) const;
+
+
+    // Returns thename of the category
+    const std::string& getName() const;
+
+    // Returns the number of the category
+    uint32_t getCatNum() const;
+
+  protected:
+    const std::string m_name;
+    const uint32_t m_catNum;
+    const uint32_t m_chunk;
+
+    // The .index
+    std::unique_ptr< Index > m_index;
+
+    // The .datXs such as dat nb X => m_dats[X]
+    std::vector< std::unique_ptr< Dat>> m_dats;
+  };
+    
+}
+
+#endif // XIV_DAT_CAT_H

deps/datReaderPs3/DatCategories/bg/LgbTypes.h

@@ -0,0 +1,283 @@
+#ifndef SAPPHIRE_LGBTYPES_H
+#define SAPPHIRE_LGBTYPES_H
+
+#include "vec3.h"
+
+enum class LgbEntryType : uint32_t
+{
+  BgParts = 1,
+  Attribute = 2,
+  Light = 3,
+  Vfx = 4,
+  PositionMarker = 5,
+  Gimmick = 6,
+  SharedGroup6 = 6,// secondary variable is set to 2
+  Sound = 7,
+  EventNpc = 8,
+  BattleNpc = 9,
+  RoutePath = 10,
+  Character = 11,
+  Aetheryte = 12,
+  EnvSpace = 13,
+  Gathering = 14,
+  SharedGroup15 = 15,// secondary variable is set to 13
+  Treasure = 16,
+  Clip = 0x11,
+  ClipCtrlPoint = 0x12,
+  ClipCamera = 0x13,
+  ClipLight = 0x14,
+  ClipReserve00 = 0x15,
+  ClipReserve01 = 0x16,
+  ClipReserve02 = 0x17,
+  ClipReserve03 = 0x18,
+  ClipReserve04 = 0x19,
+  ClipReserve05 = 0x1A,
+  ClipReserve06 = 0x1B,
+  ClipReserve07 = 0x1C,
+  ClipReserve08 = 0x1D,
+  ClipReserve09 = 0x1E,
+  ClipReserve10 = 0x1F,
+  ClipReserve11 = 0x20,
+  ClipReserve12 = 0x21,
+  ClipReserve13 = 0x22,
+  ClipReserve14 = 0x23,
+  CutAssetOnlySelectable = 0x24,
+  Player = 37,
+  Monster = 38,
+  Weapon = 39,
+  PopRange = 40,
+  ExitRange = 41,
+  LVB = 42,
+  MapRange = 43,
+  NaviMeshRange = 44,
+  EventObject = 45,
+  DemiHuman = 46,
+  EnvLocation = 47,
+  ControlPoint = 48,
+  EventRange = 49,
+  RestBonusRange = 50,
+  QuestMarker = 51,
+  TimeLine = 52,
+  ObjectBehaviorSet = 53,
+  Movie = 54,
+  ScenarioEXD = 55,
+  ScenarioText = 56,
+  CollisionBox = 57,
+  DoorRange = 58,
+  LineVfx = 59,
+  SoundEnvSet = 60,
+  CutActionTimeline = 61,
+  CharaScene = 62,
+  CutAction = 63,
+  EquipPreset = 64,
+  ClientPath = 65,
+  ServerPath = 66,
+  GimmickRange = 67,
+  TargetMarker = 68,
+  ChairMarker = 69,
+  ClickableRange = 70,
+  PrefetchRange = 71,
+  FateRange = 72,
+  PartyMember = 73,
+  KeepRange = 74,
+  SphereCastRange = 75,
+  IndoorObject = 76,
+  OutdoorObject = 77,
+  EditGroup = 78,
+  StableChocobo = 79
+};
+
+enum PopType : uint32_t
+{
+  PopTypePC = 0x1,
+  PopTypeNPC = 0x2,
+  PopTypeBNPC = 0x2,
+  PopTypeContent = 0x3,
+};
+
+struct Transformation
+{
+  vec3 translation;
+  vec3 rotation;
+  vec3 scale;
+};
+
+struct RelativePositions_
+{
+  int32_t Pos;
+  int32_t PosCount;
+};
+
+struct InstanceObject
+{
+  LgbEntryType type;
+  uint32_t instanceId;
+  uint32_t nameOffset;
+  Transformation transform;
+};
+
+struct GameInstanceObject : public InstanceObject
+{
+  uint32_t BaseId;
+};
+
+struct NPCInstanceObject : public GameInstanceObject
+{
+  uint32_t PopWeather;
+  uint8_t PopTimeStart;
+  uint8_t PopTimeEnd;
+  uint8_t Padding00[2];
+  uint32_t MoveAI;
+  uint8_t WanderingRange;
+  uint8_t Route;
+  uint16_t EventGroup;
+  uint32_t Reserved1;
+  uint32_t Reserved2;
+};
+
+
+struct BNpcBaseData
+{
+  uint16_t TerritoryRange;
+  uint8_t Sense[2];
+  uint8_t SenseRange[2];
+};
+
+
+struct BNPCInstanceObject : public NPCInstanceObject
+{
+  uint32_t NameId;
+  uint32_t DropItem;
+  float SenseRangeRate;
+  uint16_t Level;
+  uint8_t ActiveType;
+  uint8_t PopInterval;
+  uint8_t PopRate;
+  uint8_t PopEvent;
+  uint8_t LinkGroup;
+  uint8_t LinkFamily;
+  uint8_t LinkRange;
+  uint8_t LinkCountLimit;
+  int8_t NonpopInitZone;
+  int8_t InvalidRepop;
+  int8_t LinkParent;
+  int8_t LinkOverride;
+  int8_t LinkReply;
+  int8_t Nonpop;
+  RelativePositions_ RelativePositions;
+  float HorizontalPopRange;
+  float VerticalPopRange;
+  int32_t BNpcBaseData;
+  uint8_t RepopId;
+  uint8_t BNPCRankId;
+  uint16_t TerritoryRange;
+  uint32_t BoundInstanceID;
+  uint32_t FateLayoutLabelId;
+  uint32_t NormalAI;
+  uint32_t ServerPathId;
+  uint32_t EquipmentID;
+  uint32_t CustomizeID;
+};
+
+struct BgPartsData : public InstanceObject
+{
+  uint32_t modelFileOffset;
+  uint32_t collisionFileOffset;
+  uint32_t unknown4;
+  uint32_t unknown5;
+  uint32_t unknown6;
+  uint32_t unknown7;
+  uint32_t unknown8;
+  uint32_t unknown9;
+};
+
+
+
+struct PopRangeData : public InstanceObject
+{
+  PopType popType;
+  RelativePositions_ relativePositions;
+  float innerRadiusRatio;
+  uint8_t index;
+  uint8_t padding00[3];
+  uint32_t reserved;
+};
+
+struct GimmickData : public InstanceObject
+{
+  uint32_t gimmickFileOffset;
+  char unknownBytes[100];
+};
+
+struct ENpcData : public InstanceObject
+{
+  uint32_t enpcId;
+  uint8_t unknown1[0x24];
+};
+
+struct EObjData : public InstanceObject
+{
+  uint32_t eobjId;
+  uint32_t levelHierachyId;
+  uint8_t unknown1[0xC];
+};
+
+enum TriggerBoxShape : uint32_t
+{
+  TriggerBoxShapeBox = 0x1,
+  TriggerBoxShapeSphere = 0x2,
+  TriggerBoxShapeCylinder = 0x3,
+  TriggerBoxShapeBoard = 0x4,
+  TriggerBoxShapeMesh = 0x5,
+  TriggerBoxShapeBoardBothSides = 0x6,
+};
+
+struct TriggerBoxInstanceObject
+{
+  TriggerBoxShape triggerBoxShape;
+  int16_t priority;
+  int8_t enabled;
+  uint8_t padding;
+  uint32_t reserved;
+};
+
+struct ExitRangeData : public InstanceObject
+{
+  TriggerBoxInstanceObject triggerBoxType;
+  uint32_t exitType;
+  uint16_t zoneId;
+  uint16_t destTerritoryType;
+  int index;
+  uint32_t destInstanceObjectId;
+  uint32_t returnInstanceObjectId;
+  float direction;
+  uint32_t reserved;
+};
+
+struct MapRangeData : public InstanceObject
+{
+  TriggerBoxInstanceObject triggerBoxType;
+  uint32_t mapId;
+  uint32_t placeNameBlock;
+  uint32_t placeNameSpot;
+  uint32_t bGM;
+  uint32_t weather;
+  uint32_t reserved;
+  uint32_t reserved2;
+  uint16_t reserved3;
+  uint8_t housingBlockId;
+  int8_t restBonusEffective;
+  uint8_t discoveryIndex;
+  int8_t mapEnabled;
+  int8_t placeNameEnabled;
+  int8_t discoveryEnabled;
+  int8_t bGMEnabled;
+  int8_t weatherEnabled;
+  int8_t restBonusEnabled;
+  int8_t bGMPlayZoneInOnly;
+  int8_t liftEnabled;
+  int8_t housingEnabled;
+  uint16_t padding;
+};
+
+#endif //SAPPHIRE_LGBTYPES_H

deps/datReaderPs3/DatCategories/bg/lgb.h

@@ -0,0 +1,467 @@
+#ifndef _LGB_H
+#define _LGB_H
+
+#include <cstring>
+#include <memory>
+#include <cstdint>
+#include <iostream>
+#include <vector>
+#include <map>
+#include <string>
+
+#include "matrix4.h"
+#include "vec3.h"
+#include "sgb.h"
+#include "LgbTypes.h"
+#include "../../bparse.h"
+
+
+// based on https://github.com/ufx/SaintCoinach/blob/master/SaintCoinach/Graphics/Lgb/
+struct LGB_FILE;
+struct LGB_FILE_HEADER;
+struct LGB_GROUP;
+struct LGB_GROUP_HEADER;
+
+/* 253494 */
+struct LayerSetReferenced
+{
+  uint32_t LayerSetID;
+};
+
+/* 253496 */
+enum LayerSetReferencedType : int32_t
+{
+  All = 0x0,
+  Include = 0x1,
+  Exclude = 0x2,
+  Undetermined = 0x3,
+};
+
+/* 253495 */
+struct LayerSetReferencedList
+{
+  LayerSetReferencedType ReferencedType;
+  int32_t LayerSets;
+  int32_t LayerSet_Count;
+};
+
+
+class LgbEntry
+{
+public:
+  char* m_buf;
+  uint32_t m_offset;
+  InstanceObject header;
+
+  LgbEntry()
+  {
+    m_buf = nullptr;
+    m_offset = 0;
+    memset( &header, 0, sizeof( header ) );
+  };
+
+  LgbEntry( char* buf, uint32_t offset )
+  {
+    m_buf = buf;
+    m_offset = offset;
+    header = *reinterpret_cast< InstanceObject* >( buf + offset );
+    header.instanceId = xivps3::utils::bparse::byteswap( header.instanceId );
+    header.nameOffset = xivps3::utils::bparse::byteswap( header.nameOffset );
+    header.type = xivps3::utils::bparse::byteswap( header.type );
+
+    header.transform.translation.x = xivps3::utils::bparse::byteswap( header.transform.translation.x );
+    header.transform.translation.y = xivps3::utils::bparse::byteswap( header.transform.translation.y );
+    header.transform.translation.z = xivps3::utils::bparse::byteswap( header.transform.translation.z );
+
+    header.transform.rotation.x = xivps3::utils::bparse::byteswap( header.transform.rotation.x );
+    header.transform.rotation.y = xivps3::utils::bparse::byteswap( header.transform.rotation.y );
+    header.transform.rotation.z = xivps3::utils::bparse::byteswap( header.transform.rotation.z );
+  };
+
+  const LgbEntryType getType() const
+  {
+    return header.type;
+  };
+
+  virtual ~LgbEntry()
+  {
+  };
+};
+
+
+class LGB_BGPARTS_ENTRY : public LgbEntry
+{
+public:
+  BgPartsData data;
+  std::string name;
+  std::string modelFileName;
+  std::string collisionFileName;
+
+  LGB_BGPARTS_ENTRY()
+  {
+  };
+
+  LGB_BGPARTS_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  {
+    data = *reinterpret_cast< BgPartsData* >( buf + offset );
+    name = std::string( buf + offset + header.nameOffset );
+    modelFileName = std::string( buf + offset + data.modelFileOffset );
+    collisionFileName = std::string( buf + offset + data.collisionFileOffset );
+  };
+};
+
+class LGB_GIMMICK_ENTRY : public LgbEntry
+{
+public:
+  GimmickData data;
+  std::string name;
+  std::string gimmickFileName;
+
+  LGB_GIMMICK_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  {
+    data = *reinterpret_cast< GimmickData* >( buf + offset );
+    name = std::string( buf + offset + header.nameOffset );
+    gimmickFileName = std::string( buf + offset + data.gimmickFileOffset );
+  };
+};
+
+class LGB_ENPC_ENTRY : public LgbEntry
+{
+public:
+  ENpcData data;
+  std::string name;
+
+  LGB_ENPC_ENTRY( char* buf, uint32_t offset ) :
+    LgbEntry( buf, offset )
+  {
+    data = *reinterpret_cast< ENpcData* >( buf + offset );
+    name = std::string( buf + offset + header.nameOffset );
+  };
+};
+
+class LGB_BNPC_ENTRY : public LgbEntry
+{
+public:
+  BNPCInstanceObject data;
+  BNpcBaseData baseData;
+  std::string name;
+
+  LGB_BNPC_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  {
+    data = *reinterpret_cast< BNPCInstanceObject* >( buf + offset );
+    data.BaseId = xivps3::utils::bparse::byteswap( data.BaseId );
+    data.BNpcBaseData = xivps3::utils::bparse::byteswap( data.BNpcBaseData );
+    data.BoundInstanceID = xivps3::utils::bparse::byteswap( data.BoundInstanceID );
+    data.CustomizeID = xivps3::utils::bparse::byteswap( data.CustomizeID );
+    data.DropItem = xivps3::utils::bparse::byteswap( data.DropItem );
+    data.EquipmentID = xivps3::utils::bparse::byteswap( data.EquipmentID );
+    data.EventGroup = xivps3::utils::bparse::byteswap( data.EventGroup );
+    data.FateLayoutLabelId = xivps3::utils::bparse::byteswap( data.FateLayoutLabelId );
+    data.HorizontalPopRange = xivps3::utils::bparse::byteswap( data.HorizontalPopRange );
+
+    data.type = xivps3::utils::bparse::byteswap( data.type );
+    data.instanceId = xivps3::utils::bparse::byteswap( data.instanceId );
+    data.nameOffset = xivps3::utils::bparse::byteswap( data.nameOffset );
+
+    data.transform.translation.x = xivps3::utils::bparse::byteswap( data.transform.translation.x );
+    data.transform.translation.y = xivps3::utils::bparse::byteswap( data.transform.translation.y );
+    data.transform.translation.z = xivps3::utils::bparse::byteswap( data.transform.translation.z );
+
+    data.transform.rotation.x = xivps3::utils::bparse::byteswap( data.transform.rotation.x );
+    data.transform.rotation.y = xivps3::utils::bparse::byteswap( data.transform.rotation.y );
+    data.transform.rotation.z = xivps3::utils::bparse::byteswap( data.transform.rotation.z );
+
+    data.transform.scale.x = xivps3::utils::bparse::byteswap( data.transform.scale.x );
+    data.transform.scale.y = xivps3::utils::bparse::byteswap( data.transform.scale.y );
+    data.transform.scale.z = xivps3::utils::bparse::byteswap( data.transform.scale.z );
+
+    data.PopWeather = xivps3::utils::bparse::byteswap( data.PopWeather );
+    data.MoveAI = xivps3::utils::bparse::byteswap( data.MoveAI );
+    data.Level = xivps3::utils::bparse::byteswap( data.Level );
+    data.NormalAI = xivps3::utils::bparse::byteswap( data.NormalAI );
+    data.SenseRangeRate = xivps3::utils::bparse::byteswap( data.SenseRangeRate );
+    data.ServerPathId = xivps3::utils::bparse::byteswap( data.ServerPathId );
+    data.TerritoryRange = xivps3::utils::bparse::byteswap( data.TerritoryRange );
+    data.VerticalPopRange = xivps3::utils::bparse::byteswap( data.VerticalPopRange );
+    data.NameId = xivps3::utils::bparse::byteswap( data.NameId );
+
+    name = std::string( buf + offset + header.nameOffset );
+
+    baseData = *reinterpret_cast< BNpcBaseData* >( buf + offset + data.BNpcBaseData );
+
+    baseData.TerritoryRange = xivps3::utils::bparse::byteswap( baseData.TerritoryRange );
+
+    std::cout << data.BNpcBaseData << "\n";
+
+  };
+};
+
+class LGB_EOBJ_ENTRY : public LgbEntry
+{
+public:
+  EObjData data;
+  std::string name;
+
+  LGB_EOBJ_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  {
+    data = *reinterpret_cast< EObjData* >( buf + offset );
+    name = std::string( buf + offset + header.nameOffset );
+  };
+};
+
+struct LGB_MAP_RANGE_ENTRY : public LgbEntry
+{
+public:
+  MapRangeData data;
+  std::string name;
+
+  LGB_MAP_RANGE_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  {
+    data = *reinterpret_cast< MapRangeData* >( buf + offset );
+    name = std::string( buf + offset + header.nameOffset );
+  };
+};
+
+struct LGB_EXIT_RANGE_ENTRY : public LgbEntry
+{
+public:
+  ExitRangeData data;
+  std::string name;
+
+  LGB_EXIT_RANGE_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  {
+    data = *reinterpret_cast< ExitRangeData* >( buf + offset );
+    name = std::string( buf + offset + header.nameOffset );
+  };
+};
+
+struct LGB_POP_RANGE_ENTRY : public LgbEntry
+{
+public:
+  PopRangeData data;
+
+  LGB_POP_RANGE_ENTRY( char* buf, uint32_t offset ) : LgbEntry( buf, offset )
+  {
+    data = *reinterpret_cast< PopRangeData* >( buf + offset );
+  };
+};
+
+struct LGB_GROUP_HEADER
+{
+  uint32_t id;
+  int32_t groupNameOffset;
+  int32_t entriesOffset;
+  int32_t entryCount;
+  int8_t ToolModeVisible;
+  int8_t ToolModeReadOnly;
+  int8_t IsBushLayer;
+  int8_t PS3Visible;
+  int32_t LayerSetRef;
+  uint16_t FestivalID;
+  uint16_t FestivalPhaseID;
+  int8_t IsTemporary;
+  int8_t IsHousing;
+  uint16_t VersionMask;
+  uint32_t Reserved;
+  int32_t OBSetReferencedList;
+  int32_t OBSetReferencedList_Count;
+  int32_t OBSetEnableReferencedList;
+  int32_t OBSetEnableReferencedList_Count;
+};
+
+struct LGB_GROUP
+{
+  LGB_FILE* parent;
+  LGB_GROUP_HEADER header;
+  LayerSetReferencedList layerSetReferencedList;
+
+  std::string name;
+  std::vector< std::shared_ptr< LgbEntry > > entries;
+  std::vector< LayerSetReferenced > refs;
+
+  LGB_GROUP( char* buf, LGB_FILE* parentStruct, uint32_t offset )
+  {
+    parent = parentStruct;
+    header = *reinterpret_cast< LGB_GROUP_HEADER* >( buf + offset );
+    header.entriesOffset = xivps3::utils::bparse::byteswap( header.entriesOffset );
+    header.entryCount = xivps3::utils::bparse::byteswap( header.entryCount );
+    header.groupNameOffset = xivps3::utils::bparse::byteswap( header.groupNameOffset );
+    header.LayerSetRef = xivps3::utils::bparse::byteswap( header.LayerSetRef );
+    header.id = xivps3::utils::bparse::byteswap( header.id );
+    name = std::string( buf + offset + header.groupNameOffset );
+
+    layerSetReferencedList = *reinterpret_cast< LayerSetReferencedList* >( buf + offset + header.LayerSetRef );
+    layerSetReferencedList.LayerSet_Count = xivps3::utils::bparse::byteswap( layerSetReferencedList.LayerSet_Count );
+    layerSetReferencedList.LayerSets = xivps3::utils::bparse::byteswap( layerSetReferencedList.LayerSets );
+    layerSetReferencedList.ReferencedType = xivps3::utils::bparse::byteswap( layerSetReferencedList.ReferencedType );
+
+    if( layerSetReferencedList.LayerSet_Count > 0 )
+    {
+      refs.resize( layerSetReferencedList.LayerSet_Count );
+      memcpy( (char*)&refs[0], buf + offset + header.LayerSetRef + layerSetReferencedList.LayerSets, layerSetReferencedList.LayerSet_Count * sizeof( LayerSetReferenced ) );
+
+      for( auto& ref : refs )
+      {
+        ref = xivps3::utils::bparse::byteswap( ref );
+      }
+    }
+    const auto entriesOffset = offset + header.entriesOffset;
+    for( auto i = 0; i < header.entryCount; ++i )
+    {
+
+      const auto entryOffset = entriesOffset + xivps3::utils::bparse::byteswap( *reinterpret_cast< int32_t* >( buf + ( entriesOffset + i * 4 ) ) );
+
+      try
+      {
+        const auto type =  xivps3::utils::bparse::byteswap( *reinterpret_cast< LgbEntryType* >( buf + entryOffset ) );
+        if( type == LgbEntryType::BgParts )
+        {
+        //  entries.push_back( std::make_shared< LGB_BGPARTS_ENTRY >( buf, entryOffset ) );
+        }
+        else if( type == LgbEntryType::Gimmick )
+        {
+        //  entries.push_back( std::make_shared< LGB_GIMMICK_ENTRY >( buf, entryOffset ) );
+        }
+        else if( type == LgbEntryType::EventNpc )
+        {
+        //  entries.push_back( std::make_shared< LGB_ENPC_ENTRY >( buf, entryOffset ) );
+        }
+        else if( type == LgbEntryType::EventObject )
+        {
+        //  entries.push_back( std::make_shared< LGB_EOBJ_ENTRY >( buf, entryOffset ) );
+        }
+        else if( type == LgbEntryType::ExitRange )
+        {
+        //  entries.push_back( std::make_shared< LGB_EXIT_RANGE_ENTRY >( buf, entryOffset ) );
+        }
+        else if( type == LgbEntryType::MapRange )
+        {
+        //  entries.push_back( std::make_shared< LGB_MAP_RANGE_ENTRY >( buf, entryOffset ) );
+        }
+        else if( type == LgbEntryType::BattleNpc )
+        {
+          entries.push_back( std::make_shared< LGB_BNPC_ENTRY >( buf, entryOffset ) );
+        }
+        else
+        {
+       //   entries.push_back( std::make_shared< LgbEntry >( buf, entryOffset ) );
+        }
+      }
+      catch( std::exception& e )
+      {
+        std::cout << name << " " << e.what() << std::endl;
+      }
+    }
+  };
+};
+
+struct LGB_FILE_HEADER
+{
+  char magic[4]; // LGB 1
+  uint32_t fileSize;
+  uint32_t unknown;
+  char magic2[4]; // LGP1
+  uint32_t dataOffset;
+  uint32_t unknown3;
+  uint32_t unknown4;
+  uint32_t unknown5;
+  int32_t groupCount;
+};
+
+struct LGB_FILE
+{
+  LGB_FILE_HEADER header;
+  std::vector< LGB_GROUP > groups;
+  std::string m_name;
+
+  LGB_FILE( char* buf, const std::string& name ) : LGB_FILE( buf )
+  {
+    m_name = name;
+  }
+
+  LGB_FILE( char* buf )
+  {
+    header = *reinterpret_cast< LGB_FILE_HEADER* >( buf );
+
+    header.fileSize = xivps3::utils::bparse::byteswap( header.fileSize );
+
+
+ /*   header.unknown = xivps3::utils::bparse::byteswap( header.unknown );
+    header.unknown2 = xivps3::utils::bparse::byteswap( header.unknown2 );
+    header.unknown3 = xivps3::utils::bparse::byteswap( header.unknown3 );
+    header.unknown4 = xivps3::utils::bparse::byteswap( header.unknown4 );
+    header.unknown5 = xivps3::utils::bparse::byteswap( header.unknown5 );*/
+
+
+    auto baseOffset = sizeof( header );
+
+    header.dataOffset = xivps3::utils::bparse::byteswap( header.dataOffset );
+
+    if( strncmp( &header.magic[ 0 ], "LGB1", 4 ) != 0 )
+      throw std::runtime_error( "Invalid LGB file!" );
+
+   if( strncmp( &header.magic2[ 0 ], "LGP1", 4 ) != 0  )
+    {
+      throw std::runtime_error( "Invalid LGB file, LGP section not found!" );
+      /*     if( strncmp( &header.magic2[ 0 ] + 0x14 , "LGP1", 4 ) == 0 )
+          {
+            memcpy( &header.magic2[ 0 ], buf + 0x20, 24 );
+            baseOffset = 0x44;
+
+            header.dataOffset = *reinterpret_cast< int32_t* >( buf + 0x24 );
+            header.dataOffset = xivps3::utils::bparse::byteswap( header.dataOffset );
+            header.groupCount = *reinterpret_cast< int32_t* >( buf + 0x20 + header.dataOffset );
+          }
+          else
+            throw std::runtime_error( "Invalid LGB file, LGP section not found!" );
+
+    */
+    }
+
+    header.groupCount = xivps3::utils::bparse::byteswap( header.groupCount );
+
+    for( auto i = 0; i < header.groupCount; ++i )
+    {
+      auto groupOffset = *reinterpret_cast< int32_t* >( buf + ( baseOffset + i * 4 ) );
+      groupOffset = baseOffset + xivps3::utils::bparse::byteswap( groupOffset );
+      const auto group = LGB_GROUP( buf, this, groupOffset );
+      groups.push_back( group );
+    }
+  };
+};
+
+/*
+#if __cplusplus >= 201703L
+#include <experimental/filesystem>
+std::map<std::string, LGB_FILE> getLgbFiles( const std::string& dir )
+{
+   namespace fs = std::experimental::filesystem;
+   std::map<std::string, LGB_FILE> fileMap;
+   for( const auto& path : fs::recursive_directory_iterator( dir ) )
+   {
+      if( path.path().extension() == ".lgb" )
+      {
+         const auto& strPath = path.path().string();
+         auto f = fopen( strPath.c_str(), "rb" );
+         fseek( f, 0, SEEK_END );
+         const auto size = ftell( f );
+         std::vector<char> bytes( size );
+         rewind( f );
+         fread( bytes.data(), 1, size, f );
+         fclose( f );
+         try
+         {
+            LGB_FILE lgbFile( bytes.data() );
+            fileMap.insert( std::make_pair( strPath, lgbFile ) );
+         }
+         catch( std::exception& e )
+         {
+            std::cout << "Unable to load " << strPath << std::endl;
+         }
+      }
+   }
+   return fileMap;
+}
+#endif
+*/
+#endif

deps/datReaderPs3/DatCategories/bg/lvb.h

@@ -0,0 +1,296 @@
+#ifndef _LVB_H
+#define _LVB_H
+
+#include <cstring>
+#include <memory>
+#include <cstdint>
+#include <iostream>
+#include <vector>
+#include <map>
+#include <string>
+
+#include "matrix4.h"
+#include "vec3.h"
+#include "sgb.h"
+#include "LgbTypes.h"
+#include "../../bparse.h"
+
+struct FileHeader
+{
+  char FileID[4];
+  int32_t FileSize;
+  int32_t TotalChunkCount;
+  char ChunkID[4];
+  int32_t ChunkSize;
+};
+static_assert( sizeof( FileHeader ) == 20 );
+
+struct LayoutLayerSet
+{
+  uint32_t m_layerSetId;
+  uint32_t m_territoryTypeId;
+  char m_layerSetName[32];
+};
+static_assert( sizeof( LayoutLayerSet ) == 40 );
+
+struct LayerSet
+{
+  int32_t NavimeshAssetPath;
+  uint32_t LayerSetID;
+  int32_t LayerReferences;
+  int32_t LayerReference_Count;
+  uint32_t TerritoryTypeID;
+  int32_t Name;
+};
+static_assert( sizeof( LayerSet ) == 24 );
+
+struct SceneChunk
+{
+  int32_t LayerGroups;
+  int32_t LayerGroup_Count;
+  int32_t Settings;
+  int32_t LayerSetFold;
+  int32_t SGTimelineFold;
+  int32_t LGBAssetPaths;
+  int32_t LGBAssetPathCount;
+  int32_t _SGDoorSettings;
+  int32_t SGSetting;
+  int32_t _SGRotationSettings;
+  int32_t _SGRandomTimelineSettings;
+  int32_t HousingSetting;
+  int32_t _SGClockSettings;
+  int32_t Reserved1[3];
+};
+static_assert( sizeof( SceneChunk ) == 64 );
+
+struct SceneSettings
+{
+  int8_t IsPartialOutput;
+  int8_t ContainsLayerSetRef;
+  int8_t IsDungeon;
+  int8_t ExistsGrassData;
+  int32_t TerrainAssetPath;
+  int32_t EnvSetAttrReferences;
+  int32_t EnvSetAttrReference_Count;
+  int32_t SunriseAngle;
+  int32_t SkyVisibilityPath;
+  float CameraFarClipDistance;
+  float MainLightOrbitCurve;
+  float MainLightOrbitClamp;
+  float ShadowFarDistance;
+  float ShadowDistanceFade;
+  float BGSkyVisibility;
+  int32_t BGMaterialColor;
+  int32_t LightClipAABBPath;
+  int8_t TerrainOcclusionRainEnabled;
+  int8_t TerrainOcclusionDustEnabled;
+  int8_t ConstantTimeModeEnabled;
+  uint8_t Padding00[1];
+  float ConstantTime;
+  int32_t LevelWeatherTable;
+  int32_t Reserved1[5];
+};
+static_assert( sizeof( SceneSettings ) == 88 );
+
+struct LayerSetFolder
+{
+  int32_t LayerSets;
+  int32_t LayerSetCount;
+};
+static_assert( sizeof( LayerSetFolder ) == 8 );
+
+struct SGTimelineFolder
+{
+  int32_t SGTimelines;
+  int32_t SGTimelineCount;
+};
+static_assert( sizeof( SGTimelineFolder ) ==  8 );
+
+enum eShowHideAnimationType : int32_t
+{
+  Invalid = 0x0,
+  None_1 = 0x1,
+  Auto = 0x2,
+  Timeline_0 = 0x3,
+  AutoWithAnimationTime = 0x4,
+};
+
+/* 20883 */
+struct SGSettings
+{
+  uint8_t NamePlateInstanceID;
+  uint8_t TimelineShowingID;
+  uint8_t TimelineHidingID;
+  uint8_t TimelineShownID;
+  uint8_t TimelineHiddenID;
+  uint8_t GeneralPurposeTimelineIDs[16];
+  int8_t TimelineShowingIDEnabled;
+  int8_t TimelineHidingIDEnabled;
+  uint8_t Padding00[1];
+  eShowHideAnimationType ShowHideAnimationType;
+  uint16_t ShowAnimationTime;
+  uint16_t HideAnimationTime;
+  int32_t SGActionFolder;
+};
+static_assert( sizeof( SGSettings ) == 36 );
+
+struct LVB_FILE
+{
+  FileHeader header;
+  SceneChunk sceneChunk;
+  SceneSettings sceneSettings;
+
+  LayerSetFolder layerSetFolder;
+  SGTimelineFolder sgTimelineFolder;
+  SGSettings sgSettings;
+
+  std::vector< LayerSet > layers;
+  std::vector< LayoutLayerSet > layoutLayers;
+  std::vector< std::string > layerNames;
+
+  //std::vector< LGB_GROUP > groups;
+  std::string m_name;
+
+  LVB_FILE( char* buf, const std::string& name ) : LVB_FILE( buf )
+  {
+    m_name = name;
+  }
+
+  LVB_FILE( char* buf )
+  {
+    header = *reinterpret_cast< FileHeader* >( buf );
+    convertToLEBytes( header );
+
+    if( strncmp( &header.FileID[ 0 ], "LVB1", 4 ) != 0 )
+      throw std::runtime_error( "Invalid LVB file!" );
+
+    auto baseOffset = sizeof( header );
+    auto dataOffset = baseOffset;
+
+    sceneChunk = *reinterpret_cast< SceneChunk* >( buf + baseOffset );
+    convertToLEBytes( sceneChunk );
+
+    baseOffset += sizeof( sceneChunk );
+
+    sceneSettings = *reinterpret_cast< SceneSettings* >( buf + baseOffset );
+    convertToLEBytes( sceneSettings );
+
+    layerSetFolder = *reinterpret_cast< LayerSetFolder* >( buf + dataOffset + sceneChunk.LayerSetFold );
+    convertToLEBytes( layerSetFolder );
+
+    sgTimelineFolder = *reinterpret_cast< SGTimelineFolder* >( buf + dataOffset + sceneChunk.SGTimelineFold );
+    convertToLEBytes( sgTimelineFolder );
+
+    sgSettings = *reinterpret_cast< SGSettings* >( buf + dataOffset + sceneChunk.SGSetting );
+    convertToLEBytes( sgSettings );
+
+    layers.resize( layerSetFolder.LayerSetCount );
+    memcpy( (char*)&layers[0], buf + dataOffset + sceneChunk.LayerSetFold + layerSetFolder.LayerSets, sizeof( LayerSet ) * layerSetFolder.LayerSetCount );
+
+    int i = 0;
+    for( auto& entry : layers )
+    {
+      convertToLEBytes( entry );
+
+      auto entryOffset = buf + dataOffset + sceneChunk.LayerSetFold + layerSetFolder.LayerSets + ( sizeof( LayerSet ) * i++ );
+
+      layerNames.emplace_back( entryOffset + entry.Name );
+
+//      for( int j = 0; j < entry.LayerReference_Count; ++j )
+//      {
+//        LayoutLayerSet layoutLayer{};
+//
+//        memcpy(
+//          ( char* ) &layoutLayer,
+//          entryOffset + entry.LayerReferences + ( sizeof( LayoutLayerSet ) * j ),
+//          sizeof( LayoutLayerSet )
+//        );
+//
+//        convertToLEBytes( layoutLayer );
+//
+//        layoutLayers.emplace_back( layoutLayer );
+//      }
+    }
+
+  }
+
+  void convertToLEBytes( LayoutLayerSet& lls ) const
+  {
+    lls.m_layerSetId = xivps3::utils::bparse::byteswap( lls.m_layerSetId );
+    lls.m_territoryTypeId = xivps3::utils::bparse::byteswap( lls.m_territoryTypeId );
+  }
+
+  void convertToLEBytes( FileHeader& header ) const
+  {
+    header.FileSize = xivps3::utils::bparse::byteswap( header.FileSize );
+    header.TotalChunkCount = xivps3::utils::bparse::byteswap( header.TotalChunkCount );
+    header.ChunkSize = xivps3::utils::bparse::byteswap( header.ChunkSize );
+  }
+
+  void convertToLEBytes( LayerSet& entry ) const
+  {
+    entry.LayerReference_Count = xivps3::utils::bparse::byteswap( entry.LayerReference_Count );
+    entry.LayerReferences = xivps3::utils::bparse::byteswap( entry.LayerReferences );
+    entry.LayerSetID = xivps3::utils::bparse::byteswap( entry.LayerSetID );
+    entry.Name = xivps3::utils::bparse::byteswap( entry.Name );
+    entry.NavimeshAssetPath = xivps3::utils::bparse::byteswap( entry.NavimeshAssetPath );
+    entry.TerritoryTypeID = xivps3::utils::bparse::byteswap( entry.TerritoryTypeID );
+  }
+
+  void convertToLEBytes( SGSettings& sgSettings ) const
+  {
+    sgSettings.HideAnimationTime = xivps3::utils::bparse::byteswap( sgSettings.HideAnimationTime );
+    sgSettings.SGActionFolder = xivps3::utils::bparse::byteswap( sgSettings.SGActionFolder );
+    sgSettings.ShowAnimationTime = xivps3::utils::bparse::byteswap( sgSettings.ShowAnimationTime );
+    sgSettings.ShowHideAnimationType = xivps3::utils::bparse::byteswap( sgSettings.ShowHideAnimationType );
+  }
+
+  void convertToLEBytes( SGTimelineFolder& sgTimelineFolder ) const
+  {
+    sgTimelineFolder.SGTimelineCount = xivps3::utils::bparse::byteswap( sgTimelineFolder.SGTimelineCount );
+    sgTimelineFolder.SGTimelines = xivps3::utils::bparse::byteswap( sgTimelineFolder.SGTimelines );
+  }
+
+  void convertToLEBytes( LayerSetFolder& layerSetFolder ) const
+  {
+    layerSetFolder.LayerSetCount = xivps3::utils::bparse::byteswap( layerSetFolder.LayerSetCount );
+    layerSetFolder.LayerSets = xivps3::utils::bparse::byteswap( layerSetFolder.LayerSets );
+  }
+
+  void convertToLEBytes( SceneSettings& sceneSettings ) const
+  {
+    sceneSettings.BGMaterialColor = xivps3::utils::bparse::byteswap( sceneSettings.BGMaterialColor );
+    sceneSettings.BGSkyVisibility = xivps3::utils::bparse::byteswap( sceneSettings.BGSkyVisibility );
+    sceneSettings.CameraFarClipDistance = xivps3::utils::bparse::byteswap( sceneSettings.CameraFarClipDistance );
+    sceneSettings.ConstantTime = xivps3::utils::bparse::byteswap( sceneSettings.ConstantTime );
+    sceneSettings.EnvSetAttrReference_Count = xivps3::utils::bparse::byteswap( sceneSettings.EnvSetAttrReference_Count );
+    sceneSettings.EnvSetAttrReferences = xivps3::utils::bparse::byteswap( sceneSettings.EnvSetAttrReferences );
+    sceneSettings.LevelWeatherTable = xivps3::utils::bparse::byteswap( sceneSettings.LevelWeatherTable );
+    sceneSettings.LightClipAABBPath = xivps3::utils::bparse::byteswap( sceneSettings.LightClipAABBPath );
+    sceneSettings.MainLightOrbitClamp = xivps3::utils::bparse::byteswap( sceneSettings.MainLightOrbitClamp );
+    sceneSettings.ShadowDistanceFade = xivps3::utils::bparse::byteswap( sceneSettings.ShadowDistanceFade );
+    sceneSettings.ShadowFarDistance = xivps3::utils::bparse::byteswap( sceneSettings.ShadowFarDistance );
+    sceneSettings.SkyVisibilityPath = xivps3::utils::bparse::byteswap( sceneSettings.SkyVisibilityPath );
+    sceneSettings.SunriseAngle = xivps3::utils::bparse::byteswap( sceneSettings.SunriseAngle );
+    sceneSettings.TerrainAssetPath = xivps3::utils::bparse::byteswap( sceneSettings.TerrainAssetPath );
+  }
+
+  void convertToLEBytes( SceneChunk& sceneChunk ) const
+  {
+    sceneChunk._SGClockSettings = xivps3::utils::bparse::byteswap( sceneChunk._SGClockSettings );
+    sceneChunk._SGDoorSettings = xivps3::utils::bparse::byteswap( sceneChunk._SGDoorSettings );
+    sceneChunk._SGRandomTimelineSettings = xivps3::utils::bparse::byteswap( sceneChunk._SGRandomTimelineSettings );
+    sceneChunk._SGRotationSettings = xivps3::utils::bparse::byteswap( sceneChunk._SGRotationSettings );
+    sceneChunk.HousingSetting = xivps3::utils::bparse::byteswap( sceneChunk.HousingSetting );
+    sceneChunk.LayerGroup_Count = xivps3::utils::bparse::byteswap( sceneChunk.LayerGroup_Count );
+    sceneChunk.LayerGroups = xivps3::utils::bparse::byteswap( sceneChunk.LayerGroups );
+    sceneChunk.LayerSetFold = xivps3::utils::bparse::byteswap( sceneChunk.LayerSetFold );
+    sceneChunk.LGBAssetPathCount = xivps3::utils::bparse::byteswap( sceneChunk.LGBAssetPathCount );
+    sceneChunk.LGBAssetPaths = xivps3::utils::bparse::byteswap( sceneChunk.LGBAssetPaths );
+    sceneChunk.Settings = xivps3::utils::bparse::byteswap( sceneChunk.Settings );
+    sceneChunk.SGSetting = xivps3::utils::bparse::byteswap( sceneChunk.SGSetting );
+    sceneChunk.SGTimelineFold = xivps3::utils::bparse::byteswap( sceneChunk.SGTimelineFold );
+  };
+};
+
+#endif

deps/datReaderPs3/DatCategories/bg/matrix4.h

@@ -0,0 +1,111 @@
+#ifndef _MATRIX4_H
+#define _MATRIX4_H
+
+#include <cstdint>
+#include <cmath>
+
+// https://github.com/jpd002/Play--Framework/tree/master/include/math
+struct matrix4
+{
+  // 4x4
+  float grid[16];
+
+  matrix4()
+  {
+    memset( &grid[ 0 ], 0, sizeof( grid ) );
+  }
+
+  float operator()( int row, int col ) const
+  {
+    return grid[ ( row * 4 ) + col ];
+  }
+
+  float& operator()( int row, int col )
+  {
+    return grid[ ( row * 4 ) + col ];
+  }
+
+  static matrix4 rotateX( float angle )
+  {
+    matrix4 ret = matrix4();
+    ret( 0, 0 ) = 1.000000000f;
+    ret( 1, 1 ) = cos( angle );
+    ret( 1, 2 ) = -sin( angle );
+    ret( 2, 1 ) = sin( angle );
+    ret( 2, 2 ) = cos( angle );
+    ret( 3, 3 ) = 1.000000000f;
+    return ret;
+  }
+
+  static matrix4 rotateY( float angle )
+  {
+    matrix4 ret = matrix4();
+    ret( 0, 0 ) = cos( angle );
+    ret( 0, 2 ) = sin( angle );
+    ret( 1, 1 ) = 1.000000000f;
+    ret( 2, 0 ) = -sin( angle );
+    ret( 2, 2 ) = cos( angle );
+    ret( 3, 3 ) = 1.000000000f;
+    return ret;
+  }
+
+  static matrix4 rotateZ( float angle )
+  {
+    matrix4 ret = matrix4();
+    ret( 0, 0 ) = cos( angle );
+    ret( 0, 1 ) = -sin( angle );
+    ret( 1, 0 ) = sin( angle );
+    ret( 1, 1 ) = cos( angle );
+    ret( 2, 2 ) = 1.000000000f;
+    ret( 3, 3 ) = 1.000000000f;
+    return ret;
+  }
+
+  static matrix4 scale( float x, float y, float z )
+  {
+    matrix4 ret = matrix4();
+    ret( 0, 0 ) = x;
+    ret( 1, 1 ) = y;
+    ret( 2, 2 ) = z;
+    ret( 3, 3 ) = 1;
+
+    return ret;
+  }
+
+  static matrix4 translate( float x, float y, float z )
+  {
+    matrix4 ret = matrix4();
+    ret( 0, 0 ) = 1;
+    ret( 1, 1 ) = 1;
+    ret( 2, 2 ) = 1;
+    ret( 3, 3 ) = 1;
+
+    ret( 3, 0 ) = x;
+    ret( 3, 1 ) = y;
+    ret( 3, 2 ) = z;
+    return ret;
+  }
+
+  matrix4 operator*( const matrix4& rhs ) const
+  {
+    matrix4 ret;
+    for( unsigned int i = 0; i < 4; i++ )
+    {
+      ret( i, 0 ) =
+        ( *this )( i, 0 ) * rhs( 0, 0 ) + ( *this )( i, 1 ) * rhs( 1, 0 ) + ( *this )( i, 2 ) * rhs( 2, 0 ) +
+        ( *this )( i, 3 ) * rhs( 3, 0 );
+      ret( i, 1 ) =
+        ( *this )( i, 0 ) * rhs( 0, 1 ) + ( *this )( i, 1 ) * rhs( 1, 1 ) + ( *this )( i, 2 ) * rhs( 2, 1 ) +
+        ( *this )( i, 3 ) * rhs( 3, 1 );
+      ret( i, 2 ) =
+        ( *this )( i, 0 ) * rhs( 0, 2 ) + ( *this )( i, 1 ) * rhs( 1, 2 ) + ( *this )( i, 2 ) * rhs( 2, 2 ) +
+        ( *this )( i, 3 ) * rhs( 3, 2 );
+      ret( i, 3 ) =
+        ( *this )( i, 0 ) * rhs( 0, 3 ) + ( *this )( i, 1 ) * rhs( 1, 3 ) + ( *this )( i, 2 ) * rhs( 2, 3 ) +
+        ( *this )( i, 3 ) * rhs( 3, 3 );
+    }
+    return ret;
+  }
+};
+
+#endif

deps/datReaderPs3/DatCategories/bg/pcb.h

@@ -0,0 +1,92 @@
+#ifndef _PCB_H
+#define _PCB_H
+
+#include <stdint.h>
+#include <vector>
+
+struct PCB_HEADER
+{
+  uint32_t unknown_1;
+  uint32_t unknown_2;
+  uint32_t num_entries; // count starts at 0
+  uint32_t total_indices;
+  uint64_t padding;
+};
+
+struct PCB_BLOCK_HEADER
+{
+  uint32_t type; // 0 for entry, 0x30 for group
+  uint32_t group_size; // when group size in bytes for the group block
+  // bounding box
+  float x;
+  float y;
+  float z;
+  float x1;
+  float y1;
+  float z1;
+  // number of vertices packed into 16 bit
+  uint16_t num_v16;
+  // number of indices
+  uint16_t num_indices;
+  // number of normal floar vertices
+  uint32_t num_vertices;
+};
+
+struct PCB_VERTEXDATA
+{
+  float x;
+  float y;
+  float z;
+};
+
+struct PCB_INDEXDATA
+{
+  uint8_t index[3];
+  uint8_t unknown[3];
+  uint8_t unknown1[6];
+};
+
+struct PCB_VERTEXDATAI16
+{
+  uint16_t x;
+  uint16_t y;
+  uint16_t z;
+};
+
+struct PCB_BLOCK_DATA
+{
+  std::vector< PCB_VERTEXDATA > vertices;
+  std::vector< PCB_VERTEXDATAI16 > vertices_i16;
+  std::vector< PCB_INDEXDATA > indices;
+};
+
+struct PCB_BLOCK_ENTRY
+{
+  PCB_BLOCK_HEADER header;
+  PCB_BLOCK_DATA data;
+};
+
+struct PCB_FILE
+{
+  PCB_HEADER header;
+  std::vector< PCB_BLOCK_ENTRY > entries;
+};
+
+struct PCB_LIST_ENTRY
+{
+  uint32_t id;
+  float x, y, z, x2, y2, z2, rot;
+};
+
+struct PCB_LIST_BASE_ENTRY
+{
+  float x, y, z, x2, y2, z2, rot;
+};
+
+struct PCB_LIST_FILE
+{
+  uint32_t count;
+  PCB_LIST_BASE_ENTRY entry;
+  std::vector< PCB_LIST_ENTRY > entries;
+};
+#endif

deps/datReaderPs3/DatCategories/bg/sgb.h

@@ -0,0 +1,276 @@
+#ifndef _SGB_H
+#define _SGB_H
+
+#include <cstring>
+#include <memory>
+#include <cstdint>
+#include <iostream>
+#include <vector>
+#include <map>
+#include <string>
+
+#include "vec3.h"
+
+//
+// ported from https://github.com/ufx/SaintCoinach/blob/master/SaintCoinach/Graphics/Sgb/SgbDataType.cs
+
+struct SGB_FILE;
+struct SGB_HEADER;
+struct SGB_MODEL_ENTRY;
+struct SGB_MODEL_HEADER;
+struct SGB_GROUP;
+struct SGB_GROUP_HEADER;
+
+
+enum SgbDataType : uint32_t
+{
+  Unknown0008 = 0x0008,
+  Group = 0x0100,
+};
+
+enum SgbGroupEntryType : uint32_t
+{
+  Model = 0x01,
+  Gimmick = 0x06,
+};
+
+struct SGB_GROUP_HEADER
+{
+  SgbDataType type;
+  int32_t nameOffset;
+  uint32_t unknown08;
+  uint32_t unknown0C;
+
+  uint32_t unknown10;
+  uint32_t unknown14;
+  uint32_t unknown18;
+  uint32_t unknown1C;
+
+  int32_t entryCount;
+  uint32_t unknown24;
+  uint32_t unknown28;
+  uint32_t unknown2C;
+
+  uint32_t unknown30;
+  uint32_t unknown34;
+  uint32_t unknown38;
+  uint32_t unknown3C;
+
+  uint32_t unknown40;
+  uint32_t unknown44;
+};
+
+struct SGB_GROUP1C_HEADER
+{
+  SgbDataType type;
+  int32_t nameOffset;
+  uint32_t unknown08;
+
+  int32_t entryCount;
+  uint32_t unknown14;
+  int32_t modelFileOffset;
+  vec3 unknownFloat3;
+  vec3 unknownFloat3_2;
+  int32_t stateOffset;
+  int32_t modelFileOffset2;
+  uint32_t unknown3;
+  float unknown4;
+  int32_t nameOffset2;
+  vec3 unknownFloat3_3;
+};
+
+struct SGB_GROUP1C_ENTRY
+{
+  uint32_t unk;
+  uint32_t unk2;
+  int32_t nameOffset;
+  uint32_t index;
+  uint32_t unk3;
+  int32_t modelFileOffset;
+};
+
+struct SGB_GROUP_ENTRY
+{
+public:
+  char* m_buf;
+  uint32_t m_offset;
+
+  SGB_GROUP_ENTRY()
+  {
+    m_buf = nullptr;
+    m_offset = 0;
+  };
+
+  SGB_GROUP_ENTRY( char* buf, uint32_t offset )
+  {
+    m_buf = buf;
+    m_offset = offset;
+  };
+
+  virtual ~SGB_GROUP_ENTRY()
+  {
+  };
+};
+
+struct SGB_ENTRY_HEADER
+{
+  SgbGroupEntryType type;
+  uint32_t unknown2;
+  int32_t nameOffset;
+  vec3 translation;
+  vec3 rotation;
+  vec3 scale;
+};
+
+struct SGB_MODEL_HEADER : public SGB_ENTRY_HEADER
+{
+  int32_t modelFileOffset;
+  int32_t collisionFileOffset;
+};
+
+struct SGB_MODEL_ENTRY : public SGB_GROUP_ENTRY
+{
+  SGB_MODEL_HEADER header;
+  SgbGroupEntryType type;
+  std::string name;
+  std::string modelFileName;
+  std::string collisionFileName;
+
+  SGB_MODEL_ENTRY( char* buf, uint32_t offset, SgbGroupEntryType type )
+  {
+    this->type = type;
+    header = *reinterpret_cast< SGB_MODEL_HEADER* >( buf + offset );
+    name = std::string( buf + offset + header.nameOffset );
+    modelFileName = std::string( buf + offset + header.modelFileOffset );
+    collisionFileName = std::string( buf + offset + header.collisionFileOffset );
+  }
+};
+
+struct SGB_GROUP
+{
+  SGB_GROUP_HEADER header;
+  std::string name;
+  SGB_FILE* parent;
+  std::vector< std::shared_ptr< SGB_GROUP_ENTRY > > entries;
+
+  SGB_GROUP( char* buf, SGB_FILE* file, uint32_t fileSize, uint32_t offset )
+  {
+    parent = file;
+    header = *reinterpret_cast< SGB_GROUP_HEADER* >( buf + offset );
+    name = std::string( buf + offset + header.nameOffset );
+
+    auto entriesOffset = offset + sizeof( header );
+
+    for( auto i = 0; i < header.entryCount; ++i )
+    {
+      auto entryOffset = entriesOffset + *reinterpret_cast< uint32_t* >( buf + ( entriesOffset + ( i * 4 ) ) );
+      if( entryOffset > fileSize )
+        throw std::runtime_error( "SGB_GROUP entry offset was larger than SGB file size!" );
+      auto type = *reinterpret_cast< uint32_t* >( buf + entryOffset );
+      if( type == SgbGroupEntryType::Model || type == SgbGroupEntryType::Gimmick )
+      {
+        entries.push_back( std::make_shared< SGB_MODEL_ENTRY >( buf, entryOffset, ( SgbGroupEntryType )type ) );
+      }
+      else
+      {
+        // std::cout << "\t\tUnknown SGB entry! Group: " << name << " type: " << type << " index: " << i << " entryOffset: " << entryOffset << "\n";
+      }
+    }
+  }
+};
+
+struct SGB_HEADER
+{
+  char magic[4];     // SGB1
+  uint32_t fileSize;
+  uint32_t unknown1;
+  char magic2[4];    // SCN1
+
+  uint32_t unknown10;
+  int32_t sharedOffset;
+  uint32_t unknown18;
+  int32_t offset1C;
+
+  uint32_t unknown20;
+  uint32_t statesOffset;
+  uint32_t unknown28;
+  uint32_t unknown2C;
+
+  uint32_t unknown30;
+  uint32_t unknown34;
+  uint32_t unknown38;
+  uint32_t unknown3C;
+
+  uint32_t unknown40;
+  uint32_t unknown44;
+  uint32_t unknown48;
+  uint32_t unknown4C;
+
+  uint32_t unknown50;
+  uint32_t unknown54;
+};
+
+struct SGB_STATE_HEADER
+{
+  uint32_t id;
+  uint32_t nameOffset;
+  char unknown[0x24];
+};
+
+struct SGB_STATE_ENTRY
+{
+  SGB_STATE_HEADER header;
+  std::string name;
+
+  SGB_STATE_ENTRY( char* buf )
+  {
+    header = *reinterpret_cast< SGB_STATE_HEADER* >( buf );
+    name = std::string( buf + header.nameOffset );
+  }
+};
+
+struct SGB_FILE
+{
+  SGB_HEADER header;
+  std::vector< SGB_GROUP > entries;
+  std::vector< SGB_STATE_ENTRY > stateEntries;
+
+  SGB_FILE()
+  {
+    memset( &header, 0, sizeof( header ) );
+  }
+
+  SGB_FILE( char* buf )
+  {
+    constexpr int baseOffset = 0x14;
+    header = *reinterpret_cast< SGB_HEADER* >( buf );
+
+    if( strncmp( &header.magic[ 0 ], "SGB1", 4 ) != 0 || strncmp( &header.magic2[ 0 ], "SCN1", 4 ) != 0 )
+      throw std::runtime_error( "Unable to load SGB File!" );
+
+    try
+    {
+      auto group = SGB_GROUP( buf, this, header.fileSize, baseOffset + header.sharedOffset );
+      entries.push_back( group );
+      auto group2 = SGB_GROUP( buf, this, header.fileSize, baseOffset + header.offset1C );
+      entries.push_back( group2 );
+      uint32_t stateCount = *reinterpret_cast< uint32_t* >( buf + baseOffset + header.statesOffset + 4 );
+      if( stateCount > 0 )
+      {
+        stateCount = stateCount;
+        for( int i = 0; i < stateCount; ++i )
+        {
+          auto state = SGB_STATE_ENTRY( buf + baseOffset + header.statesOffset + 8 + i * sizeof( SGB_STATE_HEADER ) );
+          stateEntries.push_back( state );
+          std::cout << state.name << "\n";
+        }
+      }
+    }
+    catch( std::exception& e )
+    {
+      std::cout << e.what() << "\n";
+    }
+  };
+};
+
+#endif // !_SGB_H

deps/datReaderPs3/DatCategories/bg/vec3.h

@@ -0,0 +1,34 @@
+#ifndef _VEC3_H
+#define _VEC3_H
+
+#include <cstdint>
+#include "matrix4.h"
+
+struct vec3
+{
+  float x, y, z;
+
+  vec3()
+  {
+    x = 0.0f;
+    y = 0.0f;
+    z = 0.0f;
+  }
+
+  vec3( float x, float y, float z )
+  {
+    this->x = x;
+    this->y = y;
+    this->z = z;
+  };
+};
+
+static vec3 operator*( const vec3& lhs, const matrix4& rhs )
+{
+  vec3 ret;
+  ret.x = rhs( 0, 0 ) * lhs.x + rhs( 0, 1 ) * lhs.y + rhs( 0, 2 ) * lhs.z;
+  ret.y = rhs( 1, 0 ) * lhs.x + rhs( 1, 1 ) * lhs.y + rhs( 1, 2 ) * lhs.z;
+  ret.z = rhs( 2, 0 ) * lhs.x + rhs( 2, 1 ) * lhs.y + rhs( 2, 2 ) * lhs.z;
+  return ret;
+};
+#endif

deps/datReaderPs3/Exd.cpp

@@ -0,0 +1,385 @@
+#include "Exd.h"
+
+#include "bparse.h"
+#include "stream.h"
+#include <fstream>
+#include "Exh.h"
+
+using xivps3::utils::bparse::extract;
+
+
+namespace xivps3::exd
+{
+  struct ExdHeader
+  {
+    char magic[0x4];
+    uint16_t unknown;
+    uint16_t unknown2;
+    uint32_t index_size;
+  };
+
+  struct ExdRecordIndex
+  {
+    uint32_t id;
+    uint32_t offset;
+  };
+}
+
+namespace xivps3::utils::bparse {
+template<>
+  inline void reorder< xivps3::exd::ExdHeader >( xivps3::exd::ExdHeader& i_struct )
+  {
+    for( int32_t i = 0; i < 0x4; ++i )
+    {
+      xivps3::utils::bparse::reorder( i_struct.magic[ i ] );
+    }
+    i_struct.unknown = xivps3::utils::bparse::byteswap( i_struct.unknown );
+    xivps3::utils::bparse::reorder( i_struct.unknown );
+    i_struct.unknown2 = xivps3::utils::bparse::byteswap( i_struct.unknown2 );
+    xivps3::utils::bparse::reorder( i_struct.unknown2 );
+    i_struct.index_size = xivps3::utils::bparse::byteswap( i_struct.index_size );
+    xivps3::utils::bparse::reorder( i_struct.index_size );
+  }
+
+  template<>
+  inline void reorder< xivps3::exd::ExdRecordIndex >( xivps3::exd::ExdRecordIndex& i_struct )
+  {
+    i_struct.id = xivps3::utils::bparse::byteswap( i_struct.id );
+    xivps3::utils::bparse::reorder( i_struct.id );
+    i_struct.offset = xivps3::utils::bparse::byteswap( i_struct.offset );
+    xivps3::utils::bparse::reorder( i_struct.offset );
+  }
+};
+
+namespace xivps3::exd
+{
+  Exd::Exd( std::shared_ptr< Exh > i_exh, const std::vector< std::shared_ptr< dat::File>>& i_files )
+  {
+    _exh = i_exh;
+    _files = i_files;
+
+
+    // Iterates over all the files
+    const uint32_t member_count = static_cast< uint32_t >( _exh->get_members().size() );
+    for( auto& file_ptr : _files )
+    {
+      // Get a stream
+      std::vector< char > dataCpy = file_ptr->get_data_sections().front();
+      std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
+
+      // Extract the header and skip to the record indices
+      auto exd_header = extract< ExdHeader >( iss );
+      iss.seekg( 0x20 );
+
+      // Preallocate and extract the record_indices
+      const uint32_t record_count = exd_header.index_size / sizeof( ExdRecordIndex );
+      std::vector< ExdRecordIndex > record_indices;
+      record_indices.reserve( record_count );
+      for( uint32_t i = 0; i < record_count; ++i )
+      {
+        auto recordIndex = extract< ExdRecordIndex >( iss );
+        _idCache[ recordIndex.id ] = ExdCacheEntry{ file_ptr, recordIndex.offset };
+      }
+    }
+  }
+
+  Exd::~Exd()
+  {
+  }
+
+  const std::vector< Field > Exd::get_row( uint32_t id, uint32_t subRow )
+  {
+
+    auto cacheEntryIt = _idCache.find( id );
+    if( cacheEntryIt == _idCache.end() )
+      throw std::runtime_error( "Id not found: " + std::to_string( id ) );
+
+    // Iterates over all the files
+    const uint32_t member_count = static_cast< uint32_t >( _exh->get_members().size() );
+    auto& file_ptr = cacheEntryIt->second.file;
+
+    std::vector< char > dataCpy = file_ptr->get_data_sections().front();
+    std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
+
+    // Get the vector fields for the given record and preallocate it
+    auto fields = _data[ id ];
+    fields.reserve( member_count );
+    iss.seekg( cacheEntryIt->second.offset + 6 );
+
+    uint8_t subRows = *reinterpret_cast< uint8_t* >( &dataCpy[ cacheEntryIt->second.offset + 5 ] );
+
+    if( subRow >= subRows )
+      throw std::runtime_error( "Out of bounds sub-row!" );
+
+    int offset = cacheEntryIt->second.offset + 6 + ( subRow * _exh->get_header().data_offset + 2 * ( subRow + 1 ) );
+
+    for( auto& member_entry : _exh->get_exh_members() )
+    {
+      // Seek to the position of the member to extract.
+      // 6 is because we have uint32_t/uint16_t at the start of each record
+      iss.seekg( offset + member_entry.offset );
+
+      // Switch depending on the type to extract
+      switch( member_entry.type )
+      {
+        case DataType::string:
+          // Extract the offset to the actual string
+          // Seek to it then extract the actual string
+        {
+          throw std::runtime_error( "String not implemented for variant 2!" );
+          //auto string_offset = extract<uint32_t>( iss, "string_offset", false );
+          //iss.seekg( cacheEntryIt->second.offset + 6 + _exh->get_header().data_offset + string_offset );
+          //fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
+        }
+          break;
+
+        case DataType::boolean:
+          fields.emplace_back( extract< bool >( iss, "bool" ) );
+          break;
+
+        case DataType::int8:
+          fields.emplace_back( extract< int8_t >( iss, "int8_t" ) );
+          break;
+
+        case DataType::uint8:
+          fields.emplace_back( extract< uint8_t >( iss, "uint8_t" ) );
+          break;
+
+        case DataType::int16:
+          fields.emplace_back( extract< int16_t >( iss, "int16_t", false ) );
+          break;
+
+        case DataType::uint16:
+          fields.emplace_back( extract< uint16_t >( iss, "uint16_t", false ) );
+          break;
+
+        case DataType::int32:
+          fields.emplace_back( extract< int32_t >( iss, "int32_t", false ) );
+          break;
+
+        case DataType::uint32:
+          fields.emplace_back( extract< uint32_t >( iss, "uint32_t", false ) );
+          break;
+
+        case DataType::float32:
+          fields.emplace_back( extract< float >( iss, "float", false ) );
+          break;
+
+        case DataType::uint64:
+          fields.emplace_back( extract< uint64_t >( iss, "uint64_t", false ) );
+          break;
+
+        default:
+          auto type = static_cast< uint16_t >( member_entry.type );
+          if( type < 0x19 || type > 0x20 )
+            throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
+          uint64_t val = extract< uint64_t >( iss, "bool" );
+          int32_t shift = type - 0x19;
+          int32_t i = 1 << shift;
+          val &= i;
+          fields.emplace_back( ( val & i ) == i );
+          break;
+      }
+    }
+    return fields;
+
+  }
+
+
+  const std::vector< Field > Exd::get_row( uint32_t id )
+  {
+
+    auto cacheEntryIt = _idCache.find( id );
+    if( cacheEntryIt == _idCache.end() )
+      throw std::runtime_error( "Id not found: " + std::to_string( id ) );
+
+    // Iterates over all the files
+    const uint32_t member_count =  static_cast< uint32_t >( _exh->get_members().size() );
+    auto& file_ptr = cacheEntryIt->second.file;
+
+    std::vector< char > dataCpy = file_ptr->get_data_sections().front();
+    std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
+
+    // Get the vector fields for the given record and preallocate it
+    auto fields = _data[ id ];
+    fields.reserve( member_count );
+    iss.seekg( cacheEntryIt->second.offset + 6 );
+
+    uint8_t subRows = *reinterpret_cast< uint8_t* >( &dataCpy[ cacheEntryIt->second.offset + 5 ] );
+
+    for( auto& member_entry : _exh->get_exh_members() )
+    {
+      // Seek to the position of the member to extract.
+      // 6 is because we have uint32_t/uint16_t at the start of each record
+      iss.seekg( cacheEntryIt->second.offset + 6 + member_entry.offset );
+
+      // Switch depending on the type to extract
+      switch( member_entry.type )
+      {
+        case DataType::string:
+          // Extract the offset to the actual string
+          // Seek to it then extract the actual string
+        {
+          auto string_offset = extract< uint32_t >( iss, "string_offset", false );
+          iss.seekg( cacheEntryIt->second.offset + 6 + _exh->get_header().data_offset + string_offset );
+          fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
+        }
+          break;
+
+        case DataType::boolean:
+          fields.emplace_back( extract< bool >( iss, "bool" ) );
+          break;
+
+        case DataType::int8:
+          fields.emplace_back( extract< int8_t >( iss, "int8_t" ) );
+          break;
+
+        case DataType::uint8:
+          fields.emplace_back( extract< uint8_t >( iss, "uint8_t" ) );
+          break;
+
+        case DataType::int16:
+          fields.emplace_back( extract< int16_t >( iss, "int16_t", false ) );
+          break;
+
+        case DataType::uint16:
+          fields.emplace_back( extract< uint16_t >( iss, "uint16_t", false ) );
+          break;
+
+        case DataType::int32:
+          fields.emplace_back( extract< int32_t >( iss, "int32_t", false ) );
+          break;
+
+        case DataType::uint32:
+          fields.emplace_back( extract< uint32_t >( iss, "uint32_t", false ) );
+          break;
+
+        case DataType::float32:
+          fields.emplace_back( extract< float >( iss, "float", false ) );
+          break;
+
+        case DataType::uint64:
+          fields.emplace_back( extract< uint64_t >( iss, "uint64_t", false ) );
+          break;
+
+        default:
+          auto type = static_cast< uint16_t >( member_entry.type );
+          if( type < 0x19 || type > 0x20 )
+            throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
+          uint64_t val = extract< uint64_t >( iss, "bool" );
+          int32_t shift = type - 0x19;
+          int32_t i = 1 << shift;
+          val &= i;
+          fields.emplace_back( ( val & i ) == i );
+          break;
+      }
+    }
+    return fields;
+
+  }
+
+  // Get all rows
+  const std::map< uint32_t, std::vector< Field>>& Exd::get_rows()
+  {
+    // Iterates over all the files
+    const uint32_t member_count =  static_cast< uint32_t >( _exh->get_members().size() );
+    for( auto& file_ptr : _files )
+    {
+      // Get a stream
+      std::vector< char > dataCpy = file_ptr->get_data_sections().front();
+      std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
+
+      // Extract the header and skip to the record indices
+      auto exd_header = extract< ExdHeader >( iss );
+      iss.seekg( 0x20 );
+
+      // Preallocate and extract the record_indices
+      const uint32_t record_count = exd_header.index_size / sizeof( ExdRecordIndex );
+      std::vector< ExdRecordIndex > record_indices;
+      record_indices.reserve( record_count );
+      for( uint32_t i = 0; i < record_count; ++i )
+      {
+        record_indices.emplace_back( extract< ExdRecordIndex >( iss ) );
+      }
+
+      for( auto& record_index : record_indices )
+      {
+        // Get the vector fields for the given record and preallocate it
+        auto& fields = _data[ record_index.id ];
+        fields.reserve( member_count );
+
+        for( auto& member_entry : _exh->get_exh_members() )
+          //for( auto& member_entry : _exh->get_members() )
+        {
+          // Seek to the position of the member to extract.
+          // 6 is because we have uint32_t/uint16_t at the start of each record
+          iss.seekg( record_index.offset + 6 + member_entry.offset );
+
+          // Switch depending on the type to extract
+          switch( member_entry.type )
+          {
+            case DataType::string:
+              // Extract the offset to the actual string
+              // Seek to it then extract the actual string
+            {
+              auto string_offset = extract< uint32_t >( iss, "string_offset", false );
+              iss.seekg( record_index.offset + 6 + _exh->get_header().data_offset + string_offset );
+              fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
+            }
+              break;
+
+            case DataType::boolean:
+              fields.emplace_back( extract< bool >( iss, "bool" ) );
+              break;
+
+            case DataType::int8:
+              fields.emplace_back( extract< int8_t >( iss, "int8_t" ) );
+              break;
+
+            case DataType::uint8:
+              fields.emplace_back( extract< uint8_t >( iss, "uint8_t" ) );
+              break;
+
+            case DataType::int16:
+              fields.emplace_back( extract< int16_t >( iss, "int16_t", false ) );
+              break;
+
+            case DataType::uint16:
+              fields.emplace_back( extract< uint16_t >( iss, "uint16_t", false ) );
+              break;
+
+            case DataType::int32:
+              fields.emplace_back( extract< int32_t >( iss, "int32_t", false ) );
+              break;
+
+            case DataType::uint32:
+              fields.emplace_back( extract< uint32_t >( iss, "uint32_t", false ) );
+              break;
+
+            case DataType::float32:
+              fields.emplace_back( extract< float >( iss, "float", false ) );
+              break;
+
+            case DataType::uint64:
+              fields.emplace_back( extract< uint64_t >( iss, "uint64_t", false ) );
+              break;
+
+            default:
+              auto type = static_cast< uint16_t >( member_entry.type );
+              if( type < 0x19 || type > 0x20 )
+                throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
+              uint64_t val = extract< uint64_t >( iss, "bool" );
+              int32_t shift = type - 0x19;
+              int32_t i = 1 << shift;
+              val &= i;
+              fields.emplace_back( ( val & i ) == i );
+              break;
+          }
+        }
+      }
+    }
+    return _data;
+  }
+
+}
+

deps/datReaderPs3/Exd.h

@@ -0,0 +1,200 @@
+#ifndef XIV_EXD_EXD_H
+#define XIV_EXD_EXD_H
+
+#include <memory>
+#include <map>
+
+#include <variant>
+
+#include "File.h"
+#include "Exd/Common.h"
+#include "Exd/Structs.h"
+#include "stream.h"
+#include <fstream>
+#include "Exh.h"
+#include "bparse.h"
+namespace xivps3::exd
+{
+
+  class Exh;
+
+  // Field type containing all the possible types in the data files
+  using Field = std::variant<
+    std::string,
+    bool,
+    int8_t,
+    uint8_t,
+    int16_t,
+    uint16_t,
+    int32_t,
+    uint32_t,
+    float,
+    uint64_t >;
+
+  struct ExdCacheEntry
+  {
+    std::shared_ptr< dat::File > file;
+    uint32_t offset;
+  };
+
+  // Data for a given language
+  class Exd
+  {
+  public:
+    // i_exh: the header
+    // i_files: the multiple exd files
+    Exd()
+    {
+    }
+
+    Exd( std::shared_ptr< Exh > i_exh, const std::vector< std::shared_ptr< dat::File>>& i_files );
+
+    ~Exd();
+
+    // Get a row by its id
+    const std::vector< Field > get_row( uint32_t id );
+
+    template< typename T >
+    std::shared_ptr< Component::Excel::Ps3::ExcelStruct< T > > get_row( uint32_t id )
+    {
+      using namespace xivps3::utils;
+      auto cacheEntryIt = _idCache.find( id );
+      if( cacheEntryIt == _idCache.end() )
+        throw std::out_of_range( "Id not found: " + std::to_string( id ) );
+
+      if( sizeof( T ) != _exh->get_header().data_offset )
+      {
+        throw std::runtime_error(
+          "the struct size (" + std::to_string( sizeof( T ) ) + ") doesn't match the size in the header (" +
+          std::to_string( _exh->get_header().data_offset ) + ")!" );
+      }
+
+      // Iterates over all the files
+      const uint32_t member_count = static_cast< uint32_t >( _exh->get_members().size() );
+      auto& file_ptr = cacheEntryIt->second.file;
+
+      std::vector< char > dataCpy = file_ptr->get_data_sections().front();
+      std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
+
+      auto pSheet = std::make_shared< Component::Excel::Ps3::ExcelStruct< T > >();
+
+      // Get the vector fields for the given record and preallocate it
+      auto fields = _data[ id ];
+      fields.reserve( member_count );
+      iss.seekg( cacheEntryIt->second.offset + 6 );
+
+      iss.read( reinterpret_cast<char*>( &pSheet.get()->_data ), sizeof( T ) );
+
+      int stringCount = 0;
+      for( auto& member_entry : _exh->get_exh_members() )
+      {
+
+        // Seek to the position of the member to extract.
+        // 6 is because we have uint32_t/uint16_t at the start of each record
+        iss.seekg( cacheEntryIt->second.offset + 6 + member_entry.offset );
+
+        // Switch depending on the type to extract
+        switch( member_entry.type )
+        {
+          case DataType::string:
+            // Extract the offset to the actual string
+            // Seek to it then extract the actual string
+            {
+              auto string_offset = bparse::extract< uint32_t >( iss, "string_offset", false );
+              iss.seekg( cacheEntryIt->second.offset + 6 + _exh->get_header().data_offset + string_offset );
+              std::string value = utils::bparse::extract_cstring( iss, "string" );
+              auto it = pSheet->_strings.insert( pSheet->_strings.end(), value );
+              *reinterpret_cast< uint32_t* >( pSheet->ptr() + member_entry.offset ) =
+                static_cast< uint32_t >( std::distance( pSheet->_strings.begin(), it ) );
+            }
+            break;
+
+          case DataType::boolean:
+            bparse::extract< bool >( iss, "bool" );
+            break;
+
+          case DataType::int8:
+            bparse::extract< int8_t >( iss, "int8_t" );
+            break;
+
+          case DataType::uint8:
+            bparse::extract< uint8_t >( iss, "uint8_t" );
+            break;
+
+
+          case DataType::int16:
+            {
+              int16_t value = bparse::extract< int16_t >( iss, "int16_t", false );
+              *reinterpret_cast< int16_t* >( pSheet->ptr() + member_entry.offset ) = value;
+            }
+            break;
+
+          case DataType::uint16:
+            {
+              uint16_t value = bparse::extract< uint16_t >( iss, "uint16_t", false );
+              *reinterpret_cast< uint16_t* >( pSheet->ptr() + member_entry.offset ) = value;
+            }
+            break;
+
+          case DataType::int32:
+            {
+              int32_t value = bparse::extract< int32_t >( iss, "int32_t", false );
+              *reinterpret_cast< int32_t* >( pSheet->ptr() + member_entry.offset ) = value;
+            }
+            break;
+
+          case DataType::uint32:
+            {
+              uint32_t value = bparse::extract< uint32_t >( iss, "uint32_t", false );
+              *reinterpret_cast< uint32_t* >( pSheet->ptr() + member_entry.offset ) = value;
+            }
+            break;
+
+          case DataType::float32:
+            {
+              float value = bparse::extract< float >( iss, "float", false );
+              *reinterpret_cast< float* >( pSheet->ptr() + member_entry.offset ) = value;
+            }
+            break;
+
+          case DataType::uint64:
+            {
+              uint64_t value = bparse::extract< uint64_t >( iss, "uint64_t", false );
+              *reinterpret_cast< uint64_t* >( pSheet->ptr() + member_entry.offset ) = value;
+            }
+            break;
+
+          default:
+            auto type = static_cast< uint16_t >( member_entry.type );
+            if( type < 0x19 || type > 0x20 )
+              throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
+            uint64_t val = bparse::extract< uint64_t >( iss, "bool" );
+            int32_t shift = type - 0x19;
+            int32_t i = 1 << shift;
+            val &= i;
+            fields.emplace_back( ( val & i ) == i );
+            break;
+        }
+      }
+
+      return pSheet;
+
+    }
+
+    // Get a row by its id and sub-row
+    const std::vector< Field > get_row( uint32_t id, uint32_t subRow );
+
+    // Get all rows
+    const std::map< uint32_t, std::vector< Field>>& get_rows();
+
+  protected:
+    // Data indexed by the ID of the row, the vector is field with the same order as exh.members
+    std::map< uint32_t, std::vector< Field>> _data;
+    std::vector< std::shared_ptr< dat::File>> _files;
+    std::shared_ptr< Exh > _exh;
+    std::map< uint32_t, ExdCacheEntry > _idCache;
+  };
+
+}
+
+#endif // XIV_EXD_EXD_H

deps/datReaderPs3/Exd/Common.h

@@ -0,0 +1,109 @@
+#ifndef SAPPHIRE_EXD_COMMON_H
+#define SAPPHIRE_EXD_COMMON_H
+
+namespace Component::Excel::Ps3
+{
+
+  struct ExcelDataRowHeader
+  {
+    uint32_t dataSize;
+    uint16_t rowCount;
+  };
+
+  enum Language : int32_t
+  {
+    LANGUAGE_ALL = 0x0,
+    LANGUAGE_JP = 0x1,
+    LANGUAGE_EN = 0x2,
+    LANGUAGE_DE = 0x3,
+    LANGUAGE_FR = 0x4,
+    LANGUAGE_CH = 0x5,
+    LANGUAGE_MAX = 0x6,
+  };
+
+  struct StringOffset
+  {
+    uint32_t m_offset;
+  };
+
+  typedef int32_t LinkList[12];
+
+/*  struct LinkList::Holder
+  {
+    Common::Component::Excel::LinkList m_begin;
+    Common::Component::Excel::LinkList m_end;
+    uint32_t m_size;
+  };*/
+
+  struct ExcelEntryKey
+  {
+    uint32_t m_mainkey;
+    uint16_t m_hash;
+    uint16_t m_subkey_info[3];
+  };
+
+  typedef int32_t StringPOD[4];
+
+  union ExcelCell
+  {
+    bool b;
+    int8_t s8;
+    uint8_t u8;
+    int16_t s16;
+    uint16_t u16;
+    int32_t s32;
+    uint32_t u32;
+    int64_t s64;
+    uint64_t u64;
+    float f;
+    StringPOD str_pod;
+    StringPOD str_old;
+    StringOffset str_new;
+    StringPOD str;
+    StringOffset str_ofs;
+    int8_t *bin;
+    uint8_t boolean0;
+    uint8_t boolean1;
+    uint8_t boolean2;
+    uint8_t boolean3;
+    uint8_t boolean4;
+    uint8_t boolean5;
+    uint8_t boolean6;
+    uint8_t boolean7;
+  };
+
+  using ExdCell = ExcelCell;
+
+  enum CELL_TYPE : uint32_t
+  {
+    TYPE_NONE = 0xFFFFFFFF,
+    TYPE_STR = 0x0,
+    TYPE_BOOL = 0x1,
+    TYPE_S8 = 0x2,
+    TYPE_U8 = 0x3,
+    TYPE_S16 = 0x4,
+    TYPE_U16 = 0x5,
+    TYPE_S32 = 0x6,
+    TYPE_U32 = 0x7,
+    TYPE_F16 = 0x8,
+    TYPE_FLOAT = 0x9,
+    TYPE_S64 = 0xA,
+    TYPE_U64 = 0xB,
+    TYPE_BIN = 0xC,
+    TYPE_BOOLEAN0 = 0x19,
+    TYPE_BOOLEAN1 = 0x1A,
+    TYPE_BOOLEAN2 = 0x1B,
+    TYPE_BOOLEAN3 = 0x1C,
+    TYPE_BOOLEAN4 = 0x1D,
+    TYPE_BOOLEAN5 = 0x1E,
+    TYPE_BOOLEAN6 = 0x1F,
+    TYPE_BOOLEAN7 = 0x20,
+    TYPE_MAX_4 = 0x21,
+  };
+
+
+}
+
+
+
+#endif //SAPPHIRE_EXD_COMMON_H

deps/datReaderPs3/ExdCat.cpp

@@ -0,0 +1,92 @@
+#include "ExdCat.h"
+
+#include <fstream>
+
+#include "GameData.h"
+
+#include "Exh.h"
+#include "Exd.h"
+
+namespace
+{
+  // Suffix of the filenames given a language
+  std::map<xivps3::exd::Language, std::string> language_map =
+  {
+    {xivps3::exd::Language::none, ""},
+    {xivps3::exd::Language::ja, "_ja"},
+    {xivps3::exd::Language::en, "_en"},
+    {xivps3::exd::Language::de, "_de"},
+    {xivps3::exd::Language::fr, "_fr"},
+    {xivps3::exd::Language::chs, "_chs"}
+  };
+}
+
+namespace xivps3::exd
+{
+  Cat::Cat( dat::GameData& i_game_data, const std::string& i_name ) :
+    _name( i_name )
+  {
+    //XIV_INFO(xiv_exd_logger, "Initializing Cat with name: " << i_name);
+    // creates the header .exh
+    {
+      auto header_file = i_game_data.getFile( "exd/" + i_name + ".exh" );
+      _header = std::shared_ptr< Exh >( new Exh( *header_file ) );
+
+    }
+
+    for( auto language: _header->get_languages() )
+    {
+      // chs not yet in data files
+      if( language == Language::en || language == Language::none )
+      {
+        // Get all the files for a given category/language, in case of multiple range of IDs in separate files (like Quest)
+        std::vector< std::shared_ptr< dat::File>> files;
+        for( auto& exd_def: _header->get_exd_defs() )
+        {
+          files.emplace_back( i_game_data.getFile(
+            "exd/" + i_name + "_" + std::to_string( exd_def.start_id ) + language_map.at( language ) + ".exd" ) );
+        }
+        // Instantiate the data for this language
+        _data[ language ] = std::make_unique< Exd >( _header, files );
+      }
+    }
+  }
+
+  Cat::~Cat()
+  {
+
+  }
+
+  const std::string& Cat::get_name() const
+  {
+    return _name;
+  }
+
+  const Exh& Cat::get_header() const
+  {
+    return *_header;
+  }
+
+  const Exd& Cat::get_data_ln( Language i_language ) const
+  {
+    auto ln_it = _data.find( i_language );
+    if( ln_it == _data.end() )
+    {
+      throw std::runtime_error( "No data for language: " + std::to_string( uint16_t( i_language ) ) );
+    }
+
+    return *( ln_it->second );
+  }
+
+  const Exd& Cat::get_data( Language language ) const
+  {
+    auto ln_it = _data.find( language );
+    if( ln_it == _data.end() )
+    {
+      return get_data_ln( Language::none );
+    }
+
+    return *( ln_it->second );
+  }
+
+}

deps/datReaderPs3/ExdCat.h

@@ -0,0 +1,68 @@
+#ifndef XIV_EXD_CAT_H
+#define XIV_EXD_CAT_H
+
+#include <memory>
+#include <map>
+
+#include <filesystem>
+
+#include "bparse.h"
+#include "Exd.h"
+
+
+namespace xivps3
+{
+   namespace dat
+   {
+      class GameData;
+   }
+   namespace exd
+   {
+
+      // Language in the exd files - note: chs/chinese is present in the languages array but not in the data files
+      enum Language : uint16_t
+      {
+         none = 0,
+         ja = 1,
+         en = 2,
+         de = 3,
+         fr = 4,
+         chs = 5,
+      };
+
+      class Exh;
+      class Exd;
+
+      // A category repesent a several data sheets in the dats all under the same category
+      class Cat
+      {
+      public:
+         // i_name: name of the category
+         // i_game_data: used to fetch the files needed
+         Cat( dat::GameData& i_game_data, const std::string& i_name );
+         ~Cat();
+
+         // Returns the name of the category
+         const std::string& get_name() const;
+
+         // Returns the header
+         const Exh& get_header() const;
+
+         // Returns data for a specific language
+         const Exd& get_data_ln( Language i_language = Language::none ) const;
+
+         const Exd& get_data( Language language = Language::none ) const;
+      protected:
+         const std::string _name;
+
+         // The header file of the category *.exh
+         std::shared_ptr<Exh> _header;
+         // The data files of the category, indexed by language *.exd
+         // Note that if we have multiple files for different range of IDs, they are merged here
+         std::map<Language, std::unique_ptr<Exd>> _data;
+      };
+
+   }
+}
+
+#endif // XIV_EXD_CAT_H

deps/datReaderPs3/ExdData.cpp

@@ -0,0 +1,96 @@
+#include "ExdData.h"
+
+#include "stream.h"
+
+#include "GameData.h"
+#include "File.h"
+
+#include "ExdCat.h"
+
+namespace xivps3::exd {
+
+ExdData::ExdData( dat::GameData& i_game_data ) try :
+  _game_data( i_game_data )
+{
+  //XIV_INFO(xiv_exd_logger, "Initializing ExdData");
+
+  // Fetch the root.exl and get a stream from it
+  auto root_exl = i_game_data.getFile( "exd/root.exl" );
+  std::vector< char > dataCpy = root_exl->get_data_sections().front();
+  xivps3::utils::stream::vectorwrapbuf< char > databuf( dataCpy );
+  std::istream stream( &databuf );
+
+  // Iterates over the lines while skipping the first one
+  std::string line;
+  std::getline( stream, line ); // extract first line EXLT,2
+  std::getline( stream, line );
+
+  // Until the EOF
+  while( !line.empty() )
+  {
+    // Format is cat_name,XX
+    // XX being an internal identifier
+    // Get only the cat_name
+    auto sep = line.find( ',' );
+    auto category = line.substr( 0, sep );
+
+    // Add to the list of category name
+    // creates the empty category in the cats map
+    // instantiate the creation mutex for this category
+    _cat_names.push_back( category );
+    _cats[ category ] = std::unique_ptr< Cat >();
+    _cat_creation_mutexes[ category ] = std::make_unique< std::mutex >();
+
+    std::getline( stream, line );
+  }
+}
+catch( std::exception& e )
+{
+  // In case of failure here, client is supposed to catch the exception because it is not recoverable on our side
+  throw std::runtime_error( "ExdData initialization failed: " + std::string( e.what() ) );
+}
+
+ExdData::~ExdData()
+{
+
+}
+
+const std::vector< std::string >& ExdData::get_cat_names() const
+{
+  return _cat_names;
+}
+
+const Cat& ExdData::get_category( const std::string& i_cat_name )
+{
+  // Get the category from its name
+  auto cat_it = _cats.find( i_cat_name );
+  if( cat_it == _cats.end() )
+  {
+    throw std::runtime_error( "Category not found: " + i_cat_name );
+  }
+
+  if( cat_it->second )
+  {
+    // If valid return it
+    return *( cat_it->second );
+  }
+  else
+  {
+    // If not, create it and return it
+    create_category( i_cat_name );
+    return *( _cats[ i_cat_name ] );
+  }
+}
+
+void ExdData::create_category( const std::string& i_cat_name )
+{
+  // Lock mutex in this scope
+  std::lock_guard< std::mutex > lock( *( _cat_creation_mutexes[ i_cat_name ] ) );
+  // Maybe after unlocking it has already been created, so check (most likely if it blocked)
+  if( !_cats[ i_cat_name ] )
+  {
+    _cats[ i_cat_name ] = std::make_unique< Cat >( _game_data, i_cat_name );
+  }
+}
+
+}

deps/datReaderPs3/ExdData.h

@@ -0,0 +1,59 @@
+#ifndef XIV_EXD_EXDDATA_H
+#define XIV_EXD_EXDDATA_H
+
+#include <unordered_map>
+#include <string>
+#include <memory>
+#include <vector>
+#include <mutex>
+
+#include <filesystem>
+
+namespace xivps3
+{
+   namespace dat
+   {
+      class GameData;
+   }
+   namespace exd
+   {
+
+      class Cat;
+
+      // Interface for retrieval of exd data - Main entry point
+      // the game_data object should outlive the exd_data object
+      class ExdData
+      {
+      public:
+          // Need an initialized dat::GameData to retrieve the files from the dat
+          ExdData(dat::GameData& i_game_data);
+          ~ExdData();
+
+          // Get the list of thenames of the categories
+          const std::vector<std::string>& get_cat_names() const;
+
+          // Get a category by its name
+          const Cat& get_category(const std::string& i_cat_name);
+
+          // Export in csv in base flder i_ouput_path
+          void export_as_csvs(const std::filesystem::path& i_output_path);
+
+      protected:
+          // Lazy instantiation of category
+          void create_category(const std::string& i_cat_name);
+
+          // Reference to the game_data object
+          dat::GameData& _game_data;
+
+          // Categories, indexed by their name
+          std::unordered_map<std::string, std::unique_ptr<Cat>> _cats;
+          // List of category names = m_cats.keys()
+          std::vector<std::string> _cat_names;
+          // Mutexes used to avoid race condition when lazy instantiating a category
+          std::unordered_map<std::string, std::unique_ptr<std::mutex>> _cat_creation_mutexes;
+      };
+
+   }
+}
+
+#endif // XIV_EXD_EXDDATA_H

deps/datReaderPs3/Exh.cpp

@@ -0,0 +1,75 @@
+#include "Exh.h"
+
+#include "stream.h"
+
+#include "File.h"
+#include <sstream>
+
+using xivps3::utils::bparse::extract;
+
+namespace xivps3::exd
+{
+
+  Exh::Exh( const dat::File& i_file )
+  {
+    // Get a stream from the file
+    std::vector< char > dataCpy = i_file.get_data_sections().front();
+    std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
+
+    // Extract header and skip to member definitions
+    _header = extract< ExhHeader >( iss );
+    iss.seekg( 0x20 );
+
+    // Extract all the members and feed the _members map
+    for( auto i = 0; i < _header.field_count; ++i )
+    {
+      auto member = extract< ExhMember >( iss );
+      _members[ member.offset ] = member;
+      _exh_defs.push_back( member );
+    }
+
+    // Extract all the exd_defs
+    _exd_defs.reserve( _header.exd_count );
+    for( auto i = 0; i < _header.exd_count; ++i )
+    {
+      _exd_defs.emplace_back( extract< ExhExdDef >( iss ) );
+    }
+
+    // Extract all the languages
+    _languages.reserve( _header.language_count );
+    for( auto i = 0; i < _header.language_count; ++i )
+    {
+      _languages.emplace_back( Language( extract< uint16_t >( iss, "language" ) ) );
+    }
+  }
+
+  Exh::~Exh()
+  {
+  }
+
+  const ExhHeader& Exh::get_header() const
+  {
+    return _header;
+  }
+
+  const std::vector< ExhExdDef >& Exh::get_exd_defs() const
+  {
+    return _exd_defs;
+  }
+
+  const std::vector< Language >& Exh::get_languages() const
+  {
+    return _languages;
+  }
+
+  const std::map< uint32_t, ExhMember >& Exh::get_members() const
+  {
+    return _members;
+  }
+
+  const std::vector< ExhMember >& Exh::get_exh_members() const
+  {
+    return _exh_defs;
+  }
+
+}

deps/datReaderPs3/Exh.h

@@ -0,0 +1,133 @@
+#ifndef XIV_EXD_EXH_H
+#define XIV_EXD_EXH_H
+
+#include <map>
+
+#include "bparse.h"
+
+namespace xivps3::exd
+{
+  enum class DataType : uint16_t
+  {
+    string = 0,
+    boolean = 1,
+    int8 = 2,
+    uint8 = 3,
+    int16 = 4,
+    uint16 = 5,
+    int32 = 6,
+    uint32 = 7,
+    float32 = 9,
+    uint64 = 11,
+  };
+
+  struct ExhHeader
+  {
+    char magic[0x4];
+    uint16_t unknown;
+    uint16_t data_offset;
+    uint16_t field_count;
+    uint16_t exd_count;
+    uint16_t language_count;
+    uint16_t unknown1;
+    uint8_t u2;
+    uint8_t variant;
+  };
+
+  struct ExhMember
+  {
+    DataType type;
+    uint16_t offset;
+  };
+
+  struct ExhExdDef
+  {
+    uint32_t start_id;
+    uint32_t count_id;
+  };
+};
+
+namespace xivps3::utils::bparse {
+template<>
+inline void reorder< xivps3::exd::ExhHeader >( xivps3::exd::ExhHeader& i_struct )
+{
+  for( int32_t i = 0; i < 0x4; ++i )
+  {
+    xivps3::utils::bparse::reorder( i_struct.magic[ i ] );
+  }
+  i_struct.unknown = xivps3::utils::bparse::byteswap( i_struct.unknown );
+  xivps3::utils::bparse::reorder( i_struct.unknown );
+  i_struct.data_offset = xivps3::utils::bparse::byteswap( i_struct.data_offset );
+  xivps3::utils::bparse::reorder( i_struct.data_offset );
+  i_struct.field_count = xivps3::utils::bparse::byteswap( i_struct.field_count );
+  xivps3::utils::bparse::reorder( i_struct.field_count );
+  i_struct.exd_count = xivps3::utils::bparse::byteswap( i_struct.exd_count );
+  xivps3::utils::bparse::reorder( i_struct.exd_count );
+  i_struct.language_count = xivps3::utils::bparse::byteswap( i_struct.language_count );
+  xivps3::utils::bparse::reorder( i_struct.language_count );
+}
+
+template<>
+inline void reorder< xivps3::exd::ExhMember >( xivps3::exd::ExhMember& i_struct )
+{
+  i_struct.type = xivps3::utils::bparse::byteswap( i_struct.type );
+  xivps3::utils::bparse::reorder( i_struct.type );
+  i_struct.offset = xivps3::utils::bparse::byteswap( i_struct.offset );
+  xivps3::utils::bparse::reorder( i_struct.offset );
+}
+
+template<>
+inline void reorder< xivps3::exd::ExhExdDef >( xivps3::exd::ExhExdDef& i_struct )
+{
+  i_struct.start_id = xivps3::utils::bparse::byteswap( i_struct.start_id );
+  xivps3::utils::bparse::reorder( i_struct.start_id );
+  i_struct.count_id = xivps3::utils::bparse::byteswap( i_struct.count_id );
+  xivps3::utils::bparse::reorder( i_struct.count_id );
+}
+};
+
+namespace xivps3
+{
+
+  namespace dat
+  {
+    class File;
+  }
+
+  namespace exd
+  {
+
+    enum Language : uint16_t;
+
+    // Header file for exd data
+    class Exh
+    {
+    public:
+      // The header file
+      Exh( const dat::File& i_file );
+
+      ~Exh();
+
+      const ExhHeader& get_header() const;
+
+      const std::vector< ExhExdDef >& get_exd_defs() const;
+
+      const std::vector< Language >& get_languages() const;
+
+      const std::map< uint32_t, ExhMember >& get_members() const;
+
+      const std::vector< ExhMember >& get_exh_members() const;
+
+    protected:
+      ExhHeader _header;
+      // Members of the datastruct ordered(indexed) by offset
+      std::map< uint32_t, ExhMember > _members;
+      std::vector< ExhMember > _exh_defs;
+      std::vector< ExhExdDef > _exd_defs;
+      std::vector< Language > _languages;
+    };
+
+  }
+}
+
+#endif // XIV_EXD_EXH_H

deps/datReaderPs3/File.cpp

@@ -0,0 +1,42 @@
+#include "File.h"
+
+#include <fstream>
+
+namespace xivps3::dat
+{
+
+  File::File() :
+    _type( FileType::empty )
+  {
+  }
+
+  File::~File()
+  {
+  }
+
+  FileType File::get_type() const
+  {
+    return _type;
+  }
+
+  const std::vector< std::vector< char>>& File::get_data_sections() const
+  {
+    return _data_sections;
+  }
+
+  std::vector< std::vector< char>>& File::access_data_sections()
+  {
+    return _data_sections;
+  }
+
+  void File::exportToFile( const std::filesystem::path& i_path ) const
+  {
+    std::ofstream ofs( i_path.string(), std::ios_base::binary | std::ios_base::out );
+    for( auto& data_section : _data_sections )
+    {
+      ofs.write( data_section.data(), data_section.size() );
+    }
+    ofs.close();
+  }
+
+}

deps/datReaderPs3/File.h

@@ -0,0 +1,48 @@
+#ifndef XIV_DAT_FILE_H
+#define XIV_DAT_FILE_H
+
+#include <vector>
+
+#include <filesystem>
+#include <stdint.h>
+#include "bparse.h"
+
+namespace xivps3::dat
+{
+  enum class FileType :
+    uint32_t
+  {
+    empty = 1,
+    standard = 2,
+    model = 3,
+    texture = 4,
+  };
+
+  class Dat;
+
+  // Basic file from the dats
+  class File
+  {
+    friend class Dat;
+
+  public:
+    File();
+
+    ~File();
+
+    FileType get_type() const;
+
+    // Getters functions for the data in the file
+    const std::vector< std::vector< char>>& get_data_sections() const;
+
+    std::vector< std::vector< char>>& access_data_sections();
+
+    void exportToFile( const std::filesystem::path& i_path ) const;
+
+  protected:
+    FileType _type;
+    std::vector< std::vector< char>> _data_sections;
+  };
+}
+
+#endif // XIV_DAT_FILE_H

deps/datReaderPs3/GameData.cpp

@@ -0,0 +1,325 @@
+#include "GameData.h"
+
+#include <string>
+#include <sstream>
+#include <algorithm>
+
+#include <map>
+#include <zlib/zlib.h>
+
+#include "bparse.h"
+#include "DatCat.h"
+#include "File.h"
+
+namespace {
+// Relation between category number and category name
+// These names are taken straight from the exe, it helps resolve dispatching when getting files by path
+
+std::unordered_map< std::string, uint32_t > categoryNameToIdMap =
+  { { "common",      0x00 },
+    { "bgcommon",    0x01 },
+    { "bg",          0x02 },
+    { "cut",         0x03 },
+    { "chara",       0x04 },
+    { "shader",      0x05 },
+    { "ui",          0x06 },
+    { "sound",       0x07 },
+    { "vfx",         0x08 },
+    { "ui_script",   0x09 },
+    { "exd",         0x0A },
+    { "game_script", 0x0B },
+    { "music",       0x0C }
+  };
+
+std::unordered_map< uint32_t, std::string > categoryIdToNameMap =
+  { { 0x00, "common" },
+    { 0x01, "bgcommon" },
+    { 0x02, "bg" },
+    { 0x03, "cut" },
+    { 0x04, "chara" },
+    { 0x05, "shader" },
+    { 0x06, "ui" },
+    { 0x07, "sound" },
+    { 0x08, "vfx" },
+    { 0x09, "ui_script" },
+    { 0x0A, "exd" },
+    { 0x0B, "game_script" },
+    { 0x0C, "music" } };
+}
+
+namespace xivps3::dat
+{
+  GameData::GameData( const std::filesystem::path& path ) try :
+    m_path( path )
+  {
+    int maxExLevel = 0;
+
+    // msvc has retarded stdlib implementation
+  #ifdef _WIN32
+    static constexpr auto sep = "\\";
+  #else
+    static constexpr auto sep = std::filesystem::path::preferred_separator;
+  #endif
+
+    // Determine which expansions are available
+    while( std::filesystem::exists( std::filesystem::path(
+      m_path.string() + sep + "ex" + std::to_string( maxExLevel + 1 ) + sep + "ex" + std::to_string( maxExLevel + 1 ) +
+      ".ver" ) ) )
+    {
+      maxExLevel++;
+    }
+
+
+    // Iterate over the files in path
+    for( auto it = std::filesystem::directory_iterator( m_path.string() + "//ffxiv" );
+         it != std::filesystem::directory_iterator(); ++it )
+    {
+      // Get the filename of the current element
+      auto filename = it->path().filename().string();
+
+      // If it contains ".ps3.d.index" this is most likely a hit for a category
+      if( filename.find( ".ps3.d.index" ) != std::string::npos && filename.find( ".ps3.d.index2" ) == std::string::npos )
+      {
+        // Format of indexes is XX0000.win32.index, so fetch the hex number for category number
+        std::istringstream iss( filename.substr( 0, 2 ) );
+        uint32_t cat_nb;
+        iss >> std::hex >> cat_nb;
+
+
+        // Add to the list of category number
+        // creates the empty category in the cats map
+        // instantiate the creation mutex for this category
+        m_catNums.push_back( cat_nb );
+        m_cats[ cat_nb ] = std::unique_ptr< Cat >();
+        m_catCreationMutexes[ cat_nb ] = std::make_unique< std::mutex >();
+
+        // Check for expansion
+        for( int exNum = 1; exNum <= maxExLevel; exNum++ )
+        {
+          const std::string path =
+            m_path.string() + sep + buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, 0, "ps3.d", "index" );
+
+          if( std::filesystem::exists( std::filesystem::path( path ) ) )
+          {
+
+            int chunkCount = 0;
+
+            for( int chunkTest = 0; chunkTest < 256; chunkTest++ )
+            {
+              if( std::filesystem::exists( m_path.string() + sep +
+                                           buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, chunkTest, "ps3.d",
+                                                        "index" ) ) )
+              {
+                m_exCats[ cat_nb ].exNumToChunkMap[ exNum ].chunkToCatMap[ chunkTest ] = std::unique_ptr< Cat >();
+                chunkCount++;
+              }
+            }
+
+          }
+        }
+      }
+    }
+
+  }
+  catch( std::exception& e )
+  {
+    // In case of failure here, client is supposed to catch the exception because it is not recoverable on our side
+    throw std::runtime_error( "GameData initialization failed: " + std::string( e.what() ) );
+  }
+
+  GameData::~GameData()
+  {
+
+  }
+
+  const std::string GameData::buildDatStr( const std::string folder, const int cat, const int exNum, const int chunk,
+                                           const std::string platform, const std::string type )
+  {
+    char dat[1024];
+    sprintf( dat, "%s/%02x%02x%02x.%s.%s", folder.c_str(), cat, exNum, chunk, platform.c_str(), type.c_str() );
+    return std::string( dat );
+  }
+
+  const std::vector< uint32_t >& GameData::getCatNumbers() const
+  {
+    return m_catNums;
+  }
+
+  std::unique_ptr< File > GameData::getFile( const std::string& path )
+  {
+    // Get the hashes, the category from the path then call the getFile of the category
+    uint32_t dirHash;
+    uint32_t filenameHash;
+    getHashes( path, dirHash, filenameHash );
+
+    return getCategoryFromPath( path ).getFile( dirHash, filenameHash );
+  }
+
+  bool GameData::doesFileExist( const std::string& path )
+  {
+    uint32_t dirHash;
+    uint32_t filenameHash;
+    getHashes( path, dirHash, filenameHash );
+
+    return getCategoryFromPath( path ).doesFileExist( dirHash, filenameHash );
+  }
+
+  bool GameData::doesDirExist( const std::string& i_path )
+  {
+    uint32_t dirHash;
+    uint32_t filenameHash;
+    getHashes( i_path, dirHash, filenameHash );
+
+    return getCategoryFromPath( i_path ).doesDirExist( dirHash );
+  }
+
+  const Cat& GameData::getCategory( uint32_t catNum )
+  {
+    // Check that the category number exists
+    auto catIt = m_cats.find( catNum );
+    if( catIt == m_cats.end() )
+    {
+      throw std::runtime_error( "Category not found: " + std::to_string( catNum ) );
+    }
+
+    // If it exists and already instantiated return it
+    if( catIt->second )
+    {
+      return *( catIt->second );
+    }
+    else
+    {
+      // Else create it and return it
+      createCategory( catNum );
+      return *( m_cats[ catNum ] );
+    }
+  }
+
+  const Cat& GameData::getCategory( const std::string& catName )
+  {
+    // Find the category number from the name
+    auto categoryNameToIdMapIt = ::categoryNameToIdMap.find( catName );
+    if( categoryNameToIdMapIt == ::categoryNameToIdMap.end() )
+    {
+      throw std::runtime_error( "Category not found: " + catName );
+    }
+
+    // From the category number return the category
+    return getCategory( categoryNameToIdMapIt->second );
+  }
+
+  const Cat& GameData::getExCategory( const std::string& catName, uint32_t exNum, const std::string& path )
+  {
+    // Find the category number from the name
+    auto categoryMapIt = ::categoryNameToIdMap.find( catName );
+    if( categoryMapIt == ::categoryNameToIdMap.end() )
+    {
+      throw std::runtime_error( "Category not found: " + catName );
+    }
+
+    uint32_t dirHash;
+    uint32_t filenameHash;
+    getHashes( path, dirHash, filenameHash );
+
+    for( auto const& chunk : m_exCats[ categoryMapIt->second ].exNumToChunkMap[ exNum ].chunkToCatMap )
+    {
+      if( !chunk.second )
+        createExCategory( categoryMapIt->second );
+
+      if( chunk.second->doesFileExist( dirHash, filenameHash ) )
+      {
+        return *( chunk.second );
+      }
+    }
+
+    throw std::runtime_error( "Chunk not found for path: " + path );
+  }
+
+  const Cat& GameData::getCategoryFromPath( const std::string& path )
+  {
+    // Find the first / in the string, paths are in the format CAT_NAME/..../.../../....
+    auto firstSlashPos = path.find( '/' );
+    if( firstSlashPos == std::string::npos )
+    {
+      throw std::runtime_error( "Path does not have a / char: " + path );
+    }
+
+    if( path.substr( firstSlashPos + 1, 2 ) == "ex" )
+    {
+      return getExCategory( path.substr( 0, firstSlashPos ), std::stoi( path.substr( firstSlashPos + 3, 1 ) ), path );
+    }
+    else
+    {
+      // From the sub string found beforethe first / get the category
+      return getCategory( path.substr( 0, firstSlashPos ) );
+    }
+  }
+
+  void GameData::getHashes( const std::string& path, uint32_t& dirHash, uint32_t& filenameHash ) const
+  {
+    // Convert the path to lowercase before getting the hashes
+    std::string pathLower;
+    pathLower.resize( path.size() );
+    std::transform( path.begin(), path.end(), pathLower.begin(), ::tolower );
+
+    // Find last / to separate dir from filename
+    auto lastSlashPos = pathLower.rfind( '/' );
+    if( lastSlashPos == std::string::npos )
+    {
+      throw std::runtime_error( "Path does not have a / char: " + path );
+    }
+
+    std::string dirPart = pathLower.substr( 0, lastSlashPos );
+    std::string filenamePart = pathLower.substr( lastSlashPos + 1 );
+
+    // Get the crc32 values from zlib, to compensate the final XOR 0xFFFFFFFF that isnot done in the exe we just reXOR
+    dirHash = crc32( 0, reinterpret_cast<const uint8_t*>( dirPart.data() ), dirPart.size() ) ^ 0xFFFFFFFF;
+    filenameHash = crc32( 0, reinterpret_cast<const uint8_t*>( filenamePart.data() ), filenamePart.size() ) ^ 0xFFFFFFFF;
+  }
+
+  void GameData::createCategory( uint32_t catNum )
+  {
+    // Lock mutex in this scope
+    std::lock_guard< std::mutex > lock( *( m_catCreationMutexes[ catNum ] ) );
+    // Maybe after unlocking it has already been created, so check (most likely if it blocked)
+    if( !m_cats[ catNum ] )
+    {
+      // Get the category name if we have it
+      std::string catName;
+      auto categoryMapIt = ::categoryIdToNameMap.find( catNum );
+      if( categoryMapIt != ::categoryIdToNameMap.end() )
+      {
+        catName = categoryMapIt->second;
+      }
+
+      // Actually creates the category
+      m_cats[ catNum ] = std::make_unique< Cat >( m_path, catNum, catName );
+    }
+  }
+
+  void GameData::createExCategory( uint32_t catNum )
+  {
+    // Maybe after unlocking it has already been created, so check (most likely if it blocked)
+    if( !m_exCats[ catNum ].exNumToChunkMap[ 1 ].chunkToCatMap[ 0 ] )
+    {
+      // Get the category name if we have it
+      std::string catName;
+      auto categoryMapIt = ::categoryIdToNameMap.find( catNum );
+      if( categoryMapIt != ::categoryIdToNameMap.end() )
+      {
+        catName = categoryMapIt->second;
+      }
+
+      for( auto const& ex : m_exCats[ catNum ].exNumToChunkMap )
+      {
+        for( auto const& chunk : m_exCats[ catNum ].exNumToChunkMap[ ex.first ].chunkToCatMap )
+        {
+          // Actually creates the category
+          m_exCats[ catNum ].exNumToChunkMap[ ex.first ].chunkToCatMap[ chunk.first ] = std::unique_ptr< Cat >(
+            new Cat( m_path, catNum, catName, ex.first, chunk.first ) );
+        }
+      }
+    }
+  }
+
+}

deps/datReaderPs3/GameData.h

@@ -0,0 +1,95 @@
+#ifndef XIV_DAT_GAMEDATA_H
+#define XIV_DAT_GAMEDATA_H
+
+#include <unordered_map>
+#include <string>
+#include <memory>
+#include <vector>
+#include <stdint.h>
+#include <mutex>
+
+#include <filesystem>
+
+namespace xivps3::dat
+{
+
+  class Cat;
+
+  class File;
+
+  // Interface to all the datfiles - Main entry point
+  // All the paths to files/dirs inside the dats are case-insensitive
+  class GameData
+  {
+  public:
+    // This should be the path in which the .index/.datX files are located
+    GameData( const std::filesystem::path& path );
+
+    ~GameData();
+
+    static const std::string
+    buildDatStr( const std::string folder, const int cat, const int exNum, const int chunk, const std::string platform,
+                 const std::string type );
+
+    // Returns all the scanned category number available in the path
+    const std::vector< uint32_t >& getCatNumbers() const;
+
+    // Return a specific category by its number (see getCatNumbers() for loops)
+    const Cat& getCategory( uint32_t catNum );
+
+    // Return a specific category by it's name (e.g.: "exd"/"game_script"/ etc...)
+    const Cat& getCategory( const std::string& catName );
+
+    const Cat& getExCategory( const std::string& catName, uint32_t exNum, const std::string& path );
+
+    // Retrieve a file from the dats given its filename
+    std::unique_ptr< File > getFile( const std::string& path );
+
+    // Checks that a file exists
+    bool doesFileExist( const std::string& path );
+
+    // Checks that a dir exists, there must be a trailing / in the path
+    // Note that it won't work for dirs that don't contain any file
+    // e.g.:  - "ui/icon/" will return False
+    //        - "ui/icon/000000/" will return True
+    bool doesDirExist( const std::string& path );
+
+  protected:
+    // Return a specific category given a path (calls const Cat& getCategory(const std::string& catName))
+    const Cat& getCategoryFromPath( const std::string& path );
+
+    // From a full path, returns the dirHash and the filenameHash
+    void getHashes( const std::string& path, uint32_t& dirHash, uint32_t& filenameHash ) const;
+
+    // Lazy instantiation of category
+    void createCategory( uint32_t catNum );
+
+    void createExCategory( uint32_t catNum );
+
+    // Path given to constructor, pointing to the folder with the .index/.datX files
+    const std::filesystem::path m_path;
+
+    // Stored categories, indexed by their number, categories are instantiated and parsed individually when they are needed
+    std::unordered_map< uint32_t, std::unique_ptr< Cat>> m_cats;
+
+    // List of all the categories numbers, is equal to m_cats.keys()
+    std::vector< uint32_t > m_catNums;
+
+    // Map of all EX categories and their chunks, "CatNum - (ExNum - (ChunkNum - Cat))"
+    // Map of all EX categories and their chunks, "CatNum - (ExNum - (ChunkNum - Cat))"
+    using ChunkToCatMap = struct
+    {
+      std::unordered_map< uint32_t, std::unique_ptr< Cat > > chunkToCatMap;
+    };
+    using ExNumToChunkMap = struct
+    {
+      std::unordered_map< uint32_t, ChunkToCatMap > exNumToChunkMap;
+    };
+    using CatNumToExNumMap = std::unordered_map< uint32_t, ExNumToChunkMap >;
+    CatNumToExNumMap m_exCats;
+    std::unordered_map< uint32_t, std::unique_ptr< std::mutex>> m_catCreationMutexes;
+  };
+
+}
+
+#endif // XIV_DAT_GAMEDATA_H

deps/datReaderPs3/Index.cpp

@@ -0,0 +1,156 @@
+#include "Index.h"
+
+#include "bparse.h"
+
+namespace xivps3::dat
+{
+  struct IndexBlockRecord
+  {
+    uint32_t offset;
+    uint32_t size;
+    SqPackBlockHash blockHash;
+  };
+
+  struct IndexHashTableEntry
+  {
+    uint32_t dirHash;
+    uint32_t filenameHash;
+    uint32_t datOffset;
+    uint32_t padding;
+  };
+}
+
+namespace xivps3::utils::bparse
+{
+  template<>
+  inline void reorder< xivps3::dat::IndexBlockRecord >( xivps3::dat::IndexBlockRecord& i_struct )
+  {
+    i_struct.offset = xivps3::utils::bparse::byteswap( i_struct.offset );
+    i_struct.size = xivps3::utils::bparse::byteswap( i_struct.size );
+
+    xivps3::utils::bparse::reorder( i_struct.blockHash );
+  }
+
+  template<>
+  inline void reorder< xivps3::dat::IndexHashTableEntry >( xivps3::dat::IndexHashTableEntry& i_struct )
+  {
+    i_struct.filenameHash = xivps3::utils::bparse::byteswap( i_struct.filenameHash );
+    i_struct.dirHash = xivps3::utils::bparse::byteswap( i_struct.dirHash );
+    i_struct.datOffset = xivps3::utils::bparse::byteswap( i_struct.datOffset );
+    i_struct.padding = xivps3::utils::bparse::byteswap( i_struct.padding );
+  }
+}
+
+using xivps3::utils::bparse::extract;
+
+namespace xivps3::dat
+{
+
+  Index::Index( const std::filesystem::path& path ) :
+    SqPack( path )
+  {
+    if( !m_handle )
+      throw new std::runtime_error( "Failed to load Index at " + path.string() );
+
+    // Hash Table record
+    auto hashTableBlockRecord = extract< IndexBlockRecord >( m_handle );
+    isIndexBlockValid( hashTableBlockRecord );
+
+    // Save the posin the stream to go back to it later on
+    auto pos = m_handle.tellg();
+
+    // Seek to the pos of the hash table in the file
+    m_handle.seekg( hashTableBlockRecord.offset );
+
+    // Preallocate and extract the index_hash_table_entries
+    std::vector< IndexHashTableEntry > indexHashTableEntries;
+    extract< IndexHashTableEntry >( m_handle, hashTableBlockRecord.size / sizeof( IndexHashTableEntry ),
+                                    indexHashTableEntries );
+
+    // Feed the correct entry in the HashTable for each index_hash_table_entry
+    for( auto& indexHashTableEntry : indexHashTableEntries )
+    {
+      auto& hashTableEntry = m_hashTable[ indexHashTableEntry.dirHash ][ indexHashTableEntry.filenameHash ];
+      // The dat number is found in the offset, last four bits
+      //hashTableEntry.datNum = ( indexHashTableEntry.datOffset & 0xF ) / 0x2;
+      hashTableEntry.datNum = 0;
+      // The offset in the dat file, needs to strip the dat number indicator
+      //hashTableEntry.datOffset = ( indexHashTableEntry.datOffset - ( indexHashTableEntry.datOffset & 0x000F ) ) * 0x08;
+      hashTableEntry.datOffset = indexHashTableEntry.datOffset * 128;
+    }
+
+    // Come back to where we were before reading the HashTable
+    m_handle.seekg( pos );
+
+    // Dat Count
+    m_datCount = extract< uint32_t >( m_handle, "dat_count", false );
+
+    // Free List
+    isIndexBlockValid( extract< IndexBlockRecord >( m_handle ) );
+
+    // Dir Hash Table
+    isIndexBlockValid( extract< IndexBlockRecord >( m_handle ) );
+  }
+
+  Index::~Index()
+  {
+  }
+
+  uint32_t Index::getDatCount() const
+  {
+    return m_datCount;
+  }
+
+  const Index::HashTable& Index::getHashTable() const
+  {
+    return m_hashTable;
+  }
+
+  bool Index::doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const
+  {
+    auto dir_it = getHashTable().find( dir_hash );
+    if( dir_it != getHashTable().end() )
+    {
+      return ( dir_it->second.find( filename_hash ) != dir_it->second.end() );
+    }
+    return false;
+  }
+
+  bool Index::doesDirExist( uint32_t dir_hash ) const
+  {
+    return ( getHashTable().find( dir_hash ) != getHashTable().end() );
+  }
+
+  const Index::DirHashTable& Index::getDirHashTable( uint32_t dir_hash ) const
+  {
+    auto dir_it = getHashTable().find( dir_hash );
+    if( dir_it == getHashTable().end() )
+    {
+      throw std::runtime_error( "dirHash not found" );
+    }
+    else
+    {
+      return dir_it->second;
+    }
+  }
+
+  const Index::HashTableEntry& Index::getHashTableEntry( uint32_t dir_hash, uint32_t filename_hash ) const
+  {
+    auto& dirHashTable = getDirHashTable( dir_hash );
+    auto file_it = dirHashTable.find( filename_hash );
+    if( file_it == dirHashTable.end() )
+    {
+      throw std::runtime_error( "filenameHash not found" );
+    }
+    else
+    {
+      return file_it->second;
+    }
+  }
+
+  void Index::isIndexBlockValid( const IndexBlockRecord& i_index_block_record )
+  {
+    isBlockValid( i_index_block_record.offset, i_index_block_record.size, i_index_block_record.blockHash );
+  }
+
+}

deps/datReaderPs3/Index.h

@@ -0,0 +1,63 @@
+#ifndef XIV_DAT_INDEX_H
+#define XIV_DAT_INDEX_H
+
+#include "SqPack.h"
+
+#include <unordered_map>
+
+#include <filesystem>
+
+namespace xivps3::dat
+{
+
+  struct IndexBlockRecord;
+
+  class Index :
+    public SqPack
+  {
+  public:
+    // Full path to the index file
+    Index( const std::filesystem::path& i_path );
+
+    virtual ~Index();
+
+    // An entry in the hash table, representing a file in a given dat
+    struct HashTableEntry
+    {
+      uint32_t datNum;
+      uint32_t dirHash;
+      uint32_t filenameHash;
+      uint32_t datOffset;
+    };
+
+    // HashTable has dir hashes -> filename hashes -> HashTableEntry
+    using DirHashTable = std::unordered_map< uint32_t, HashTableEntry >;
+    using HashTable = std::unordered_map< uint32_t, DirHashTable >;
+
+    // Get the number of dat files the index is linked to
+    uint32_t getDatCount() const;
+
+    bool doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const;
+
+    bool doesDirExist( uint32_t dir_hash ) const;
+
+    // Returns the whole HashTable
+    const HashTable& getHashTable() const;
+
+    // Returns the hash table for a specific dir
+    const DirHashTable& getDirHashTable( uint32_t dir_hash ) const;
+
+    // Returns the HashTableEntry for a given file given its hashes
+    const HashTableEntry& getHashTableEntry( uint32_t dir_hash, uint32_t filename_hash ) const;
+
+  protected:
+    // Checks that the block is valid with regards to its hash
+    void isIndexBlockValid( const IndexBlockRecord& i_index_block_record );
+
+    uint32_t m_datCount;
+    HashTable m_hashTable;
+  };
+    
+}
+
+#endif // XIV_DAT_INDEX_H

deps/datReaderPs3/SqPack.cpp

@@ -0,0 +1,78 @@
+#include "SqPack.h"
+
+namespace xivps3::dat {
+  enum PlatformId :
+    uint8_t
+  {
+    Win32,
+    PS3,
+    PS4
+  };
+
+  struct SqPackHeader
+  {
+    char magic[0x8];
+    PlatformId platformId;
+    uint8_t padding0[3];
+    uint32_t size;
+    uint32_t version;
+    uint32_t type;
+  };
+
+  struct SqPackIndexHeader
+  {
+    uint32_t size;
+    uint32_t type;
+  };
+}
+
+namespace xivps3::utils:: bparse
+{
+  template<>
+  inline void reorder< xivps3::dat::SqPackHeader >( xivps3::dat::SqPackHeader& i_struct )
+  {
+    for( int32_t i = 0; i < 0x8; ++i )
+    {
+      xivps3::utils::bparse::reorder( i_struct.magic[ i ] );
+    }
+    xivps3::utils::bparse::reorder( i_struct.platformId );
+    xivps3::utils::bparse::reorder( i_struct.size );
+    xivps3::utils::bparse::reorder( i_struct.version );
+    xivps3::utils::bparse::reorder( i_struct.type );
+  }
+
+  template<>
+  inline void reorder< xivps3::dat::SqPackIndexHeader >( xivps3::dat::SqPackIndexHeader& i_struct )
+  {
+    xivps3::utils::bparse::reorder( i_struct.size );
+    xivps3::utils::bparse::reorder( i_struct.type );
+  }
+}
+
+using xivps3::utils::bparse::extract;
+
+namespace xivps3::dat
+{
+
+  // Open the file
+  SqPack::SqPack( const std::filesystem::path& path ) :
+    m_handle( path.string(), std::ios_base::in | std::ios_base::binary )
+  {
+    // Extract the header
+    extract< SqPackHeader >( m_handle );
+
+    // Skip until the IndexHeader the extract it
+    m_handle.seekg( 0x400 );
+    extract< SqPackIndexHeader >( m_handle );
+  }
+
+  SqPack::~SqPack()
+  {
+  }
+
+  void SqPack::isBlockValid( uint32_t i_offset, uint32_t i_size, const SqPackBlockHash& i_block_hash )
+  {
+    // TODO
+  }
+
+}

deps/datReaderPs3/SqPack.h

@@ -0,0 +1,62 @@
+#ifndef XIV_DAT_SQPACK_H
+#define XIV_DAT_SQPACK_H
+
+#include <fstream>
+
+#include <filesystem>
+
+#include "bparse.h"
+
+
+namespace xivps3::dat
+{
+  typedef uint64_t HashType64;
+  struct SqPackBlockHash
+  {
+    HashType64 hash;
+    uint8_t reserved[0x0B];
+    uint32_t padding[0xB];
+  };
+}
+
+namespace xivps3::utils::bparse
+{
+  template<>
+  inline void reorder< xivps3::dat::SqPackBlockHash >( xivps3::dat::SqPackBlockHash& i_struct )
+  {
+    i_struct.hash = xivps3::utils::bparse::byteswap( i_struct.hash );
+    for( auto i = 0; i < 0x14; ++i )
+    {
+    //  xivps3::utils::bparse::reorder( i_struct.hash[ i ] );
+
+    }
+    for( auto i = 0; i < 0xB; ++i )
+    {
+      xivps3::utils::bparse::reorder( i_struct.padding[ i ] );
+    }
+  }
+};
+
+namespace xivps3::dat
+{
+
+  class SqPack
+  {
+
+  public:
+    // Full path to the sqpack file
+    SqPack( const std::filesystem::path& i_path );
+
+    virtual ~SqPack();
+
+  protected:
+    // Checks that a given block is valid iven its hash
+    void isBlockValid( uint32_t i_offset, uint32_t i_size, const SqPackBlockHash& i_block_hash );
+
+    // File handle
+    std::ifstream m_handle;
+  };
+
+}
+
+#endif // XIV_DAT_SQPACK_H

deps/datReaderPs3/bparse.cpp

@@ -0,0 +1,8 @@
+#include "bparse.h"
+
+std::string xivps3::utils::bparse::extract_cstring( std::istream& i_stream, const std::string& i_name )
+{
+   std::string temp_str;
+   std::getline( i_stream, temp_str, '\0' );
+   return temp_str;
+}

deps/datReaderPs3/bparse.h

@@ -0,0 +1,101 @@
+#ifndef XIV_UTILS_BPARSE_H
+#define XIV_UTILS_BPARSE_H
+
+#include <type_traits>
+#include <iomanip>
+#include <sstream>
+#include <vector>
+
+namespace xivps3::utils::bparse
+{
+
+  // Internal macro for byteswapping
+  template< int N >
+  void byteswap_impl( char (& bytes)[N] )
+  {
+    for( auto p = std::begin( bytes ), end = std::end( bytes ) - 1; p < end; ++p, --end )
+    {
+      std::swap( *p, *end );
+    }
+  }
+
+  // byteswapping any type (no pointers to array)
+  template< typename T >
+  T byteswap( T value )
+  {
+    byteswap_impl( *reinterpret_cast<char ( * )[sizeof( T )]>(&value) );
+    return value;
+  }
+
+  // Read a struct from a stream
+  template< typename StructType >
+  void read( std::istream& i_stream, StructType& i_struct )
+  {
+    static_assert( std::is_pod< StructType >::value, "StructType must be a POD to be able to use read." );
+    i_stream.read( reinterpret_cast<char*>( &i_struct ), sizeof( StructType ) );
+  }
+
+  // By default a type does not need reordering
+  template< typename StructType >
+  void reorder( StructType& i_struct )
+  {
+  }
+
+  // "Overload" for passed struct as arg
+  template< typename StructType >
+  void extract( std::istream& i_stream, StructType& o_struct )
+  {
+    read( i_stream, o_struct );
+    reorder( o_struct );
+  }
+
+  // This should not copy because of RVO
+  // Extract a struct from a stream and log it
+  template< typename StructType >
+  StructType extract( std::istream& i_stream )
+  {
+    StructType temp_struct;
+    extract< StructType >( i_stream, temp_struct );
+    return temp_struct;
+  }
+
+  template< typename StructType >
+  void extract( std::istream& i_stream, uint32_t i_size, std::vector< StructType >& o_structs )
+  {
+    o_structs.reserve( i_size );
+    for( uint32_t i = 0; i < i_size; ++i )
+    {
+      o_structs.emplace_back( extract< StructType >( i_stream ) );
+    }
+  }
+
+  // For simple (integral) types just provide name and endianness directly
+  template< typename StructType >
+  StructType extract( std::istream& i_stream, const std::string& i_name, bool i_is_le = true )
+  {
+    StructType temp_struct;
+    read( i_stream, temp_struct );
+    if( !i_is_le )
+    {
+      temp_struct = byteswap( temp_struct );
+    }
+    return temp_struct;
+  }
+
+  template< typename StructType >
+  void extract( std::istream& i_stream, const std::string& i_name, uint32_t i_size, std::vector< StructType >& o_structs,
+                bool i_is_le = true )
+  {
+    o_structs.reserve( i_size );
+    for( uint32_t i = 0; i < i_size; ++i )
+    {
+      o_structs.emplace_back( extract< StructType >( i_stream, i_name ) );
+    }
+  }
+
+  // For cstrings
+  std::string extract_cstring( std::istream& i_stream, const std::string& i_name );
+
+}
+
+#endif // XIV_UTILS_BPARSE_H

deps/datReaderPs3/conv.cpp

@@ -0,0 +1,33 @@
+#include "conv.h"
+
+namespace xivps3::utils::conv
+{
+
+  float half2float( const uint16_t i_value )
+  {
+    uint32_t t1;
+    uint32_t t2;
+    uint32_t t3;
+
+    t1 = i_value & 0x7fff;                  // Non-sign bits
+    t2 = i_value & 0x8000;                  // Sign bit
+    t3 = i_value & 0x7c00;                  // Exponent
+    t1 <<= 13;                              // Align mantissa on MSB
+    t2 <<= 16;                              // Shift sign bit into position
+
+    t1 += 0x38000000;                       // Adjust bias
+
+    t1 = ( t3 == 0 ? 0 : t1 );              // Denormals-as-zero
+
+    t1 |= t2;                               // Re-insert sign bit
+
+    return *reinterpret_cast< float* >( &t1 );
+  }
+
+  float ubyte2float( const uint8_t i_value )
+  {
+    return i_value / 255.0f;
+  }
+
+}
+

deps/datReaderPs3/conv.h

@@ -0,0 +1,15 @@
+#ifndef XIV_UTILS_CONV_H
+#define XIV_UTILS_CONV_H
+
+#include <cstdint>
+#include <vector>
+#include <ostream>
+
+namespace xivps3::utils::conv
+{
+  float half2float( const uint16_t i_value );
+
+  float ubyte2float( const uint8_t i_value );
+}
+
+#endif // XIV_UTILS_CONV_H

deps/datReaderPs3/crc32.cpp

@@ -0,0 +1,175 @@
+#include "crc32.h"
+
+#include <mutex>
+#include <vector>
+
+#include <zlib/zlib.h>
+
+namespace internal
+{
+    // Mutex to prevent two threads from concurrently trying to build the crc tables atthe same time
+    std::mutex crc_creation_mutex;
+
+    typedef std::vector<uint32_t> CrcTable;
+    
+    // Our crc/rev_crc tables
+    CrcTable crc_table;
+    CrcTable rev_crc_table;
+
+    bool crc_tables_created = false;
+
+    void build_crc_tables()
+    {
+        std::lock_guard<std::mutex> lock(crc_creation_mutex);
+        if (!crc_tables_created)
+        {
+            crc_table.resize(0x100);
+            rev_crc_table.resize(0x100);
+            for (auto i = 0; i < 0x100; ++i)
+            {
+                uint32_t crc = i;
+                for (auto j = 0; j < 8; ++j)
+                {
+                    if (crc & 1)
+                    {
+                        crc = 0xEDB88320 ^ (crc >> 1);
+                    }
+                    else
+                    {
+                        crc = crc >> 1;
+                    }
+                }
+                crc_table[i] = crc;
+                rev_crc_table[crc >> 24] = i + ((crc & 0xFFFFFF) << 8);
+            }
+        }
+        crc_tables_created = true;
+    }
+
+    const CrcTable& get_crc_table()
+    {
+        if (!crc_tables_created)
+        {
+            build_crc_tables();
+        }
+        return crc_table;
+    }
+
+    const CrcTable& get_rev_crc_table()
+    {
+        if (!crc_tables_created)
+        {
+            build_crc_tables();
+        }
+        return rev_crc_table;
+    }
+}
+
+namespace xivps3::utils::crc32
+{
+
+  uint32_t compute( const std::string& i_input, uint32_t init_crc )
+  {
+    // Classical crc stuff
+    auto& crc_table = internal::get_crc_table();
+    auto crc = init_crc;
+    for( std::size_t i = 0; i < i_input.size(); ++i )
+    {
+      crc = crc_table[ ( crc ^ i_input[ i ] ) & 0xFF ] ^ ( crc >> 8 );
+    }
+    return crc;
+  }
+
+  uint32_t rev_compute( const std::string& i_input, uint32_t init_crc )
+  {
+    auto& rev_crc_table = internal::get_rev_crc_table();
+    auto crc = init_crc;
+    const auto input_size = i_input.size();
+    // Reverse crc
+    for( auto i = input_size; i > 0; --i )
+    {
+      crc = rev_crc_table[ crc >> 24 ] ^ ( ( crc << 8 ) & 0xFFFFFF00 ) ^ i_input[ input_size - i - 1 ];
+    }
+    // Compute the 4 bytes needed for this init_crc
+    for( auto i = 0; i < 4; ++i )
+    {
+      crc = rev_crc_table[ crc >> 24 ] ^ ( ( crc << 8 ) & 0xFFFFFF00 );
+    }
+    return crc;
+  }
+
+  void generate_hashes_1( std::string& i_format, const uint32_t i_first_index, std::vector< uint32_t >& o_hashes )
+  {
+    char* str = const_cast<char*>(i_format.data());
+    const uint32_t str_size = i_format.size();
+
+    o_hashes.resize( 10000 );
+
+    uint32_t i = 0;
+    for( char a = '0'; a <= '9'; ++a )
+    {
+      str[ i_first_index ] = a;
+      for( char b = '0'; b <= '9'; ++b )
+      {
+        str[ i_first_index + 1 ] = b;
+        for( char c = '0'; c <= '9'; ++c )
+        {
+          str[ i_first_index + 2 ] = c;
+          for( char d = '0'; d <= '9'; ++d )
+          {
+            str[ i_first_index + 3 ] = d;
+            o_hashes[ i ] = ::crc32( 0, reinterpret_cast<uint8_t*>(&( str[ 0 ] )), str_size ) ^ 0xFFFFFFFF;
+            ++i;
+          }
+        }
+      }
+    }
+  }
+
+  void generate_hashes_2( std::string& i_format, const uint32_t i_first_index, const uint32_t i_second_index,
+                          std::vector< uint32_t >& o_hashes )
+  {
+    char* str = const_cast<char*>(i_format.data());
+    const uint32_t str_size = i_format.size();
+
+    o_hashes.resize( 100000000 );
+
+    uint32_t i = 0;
+    for( char a = '0'; a <= '9'; ++a )
+    {
+      str[ i_first_index ] = a;
+      for( char b = '0'; b <= '9'; ++b )
+      {
+        str[ i_first_index + 1 ] = b;
+        for( char c = '0'; c <= '9'; ++c )
+        {
+          str[ i_first_index + 2 ] = c;
+          for( char d = '0'; d <= '9'; ++d )
+          {
+            str[ i_first_index + 3 ] = d;
+            for( char e = '0'; e <= '9'; ++e )
+            {
+              str[ i_second_index ] = e;
+              for( char f = '0'; f <= '9'; ++f )
+              {
+                str[ i_second_index + 1 ] = f;
+                for( char g = '0'; g <= '9'; ++g )
+                {
+                  str[ i_second_index + 2 ] = g;
+                  for( char h = '0'; h <= '9'; ++h )
+                  {
+                    str[ i_second_index + 3 ] = h;
+                    o_hashes[ i ] = ::crc32( 0, reinterpret_cast<uint8_t*>(&( str[ 0 ] )), str_size ) ^ 0xFFFFFFFF;
+                    ++i;
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+
+}

deps/datReaderPs3/crc32.h

@@ -0,0 +1,25 @@
+#ifndef XIV_UTILS_CRC32_H
+#define XIV_UTILS_CRC32_H
+
+#include <cstdint>
+#include <vector>
+#include <string>
+
+namespace xivps3::utils::crc32
+{
+
+  // Normal crc32 computation from a given intial crc value, use zlib.crc32 instead, the final XOR 0xFFFFFFFF is not done
+  uint32_t compute( const std::string& i_input, uint32_t init_crc = 0xFFFFFFFF );
+
+  // Computes the 4 missing bytes XXXX such as init_crc = crc32(prefix_string)
+  // and string_to_find = prefix_string + XXXX + i_input
+  uint32_t rev_compute( const std::string& i_input, uint32_t init_crc = 0 );
+
+  void generate_hashes_1( std::string& i_format, const uint32_t i_first_index, std::vector< uint32_t >& o_hashes );
+
+  void generate_hashes_2( std::string& i_format, const uint32_t i_first_index, const uint32_t i_second_index,
+                          std::vector< uint32_t >& o_hashes );
+
+}
+
+#endif // XIV_UTILS_CRC32_H

deps/datReaderPs3/stream.cpp

@@ -0,0 +1,10 @@
+#include "stream.h"
+
+#include <iostream>
+#include <sstream>
+#include <streambuf>
+
+namespace xivps3::utils::stream
+{
+
+}

deps/datReaderPs3/stream.h

@@ -0,0 +1,21 @@
+#ifndef XIV_UTILS_STREAM_H
+#define XIV_UTILS_STREAM_H
+
+#include <memory>
+#include <iostream>
+#include <vector>
+
+namespace xivps3::utils::stream
+{
+  template< typename CharT, typename TraitsT = std::char_traits< CharT > >
+  class vectorwrapbuf :
+    public std::basic_streambuf< CharT, TraitsT >
+  {
+  public:
+    vectorwrapbuf( std::vector< CharT >& vec )
+    {
+      this->setg( vec.data(), vec.data(), vec.data() + vec.size() );
+    }
+  };
+}
+#endif // XIV_UTILS_STREAM_H

deps/datReaderPs3/zlib.cpp

@@ -0,0 +1,59 @@
+#include "zlib.h"
+#include <string>
+#include <stdexcept>
+#include <zlib/zlib.h>
+#include <vector>
+
+namespace xivps3::utils::zlib
+{
+
+  void compress( const std::vector< char >& in, std::vector< char >& out )
+  {
+    // Fetching upper bound for out size
+    auto out_size = compressBound( in.size() );
+    out.resize( out_size );
+
+    auto ret = compress2( reinterpret_cast<uint8_t*>(out.data()), &out_size,
+                          reinterpret_cast<const uint8_t*>(in.data()), in.size(), Z_BEST_COMPRESSION );
+
+    if( ret != Z_OK )
+    {
+      throw std::runtime_error( "Error at zlib uncompress: " + std::to_string( ret ) );
+    }
+
+    out.resize( out_size );
+  }
+
+  void no_header_decompress( uint8_t* in, size_t in_size, uint8_t* out, size_t out_size )
+  {
+    z_stream strm;
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+    strm.avail_in = in_size;
+    strm.next_in = Z_NULL;
+
+    // Init with -15 because we do not have header in this compressed data
+    auto ret = inflateInit2( &strm, -15 );
+    if( ret != Z_OK )
+    {
+      throw std::runtime_error( "Error at zlib init: " + std::to_string( ret ) );
+    }
+
+    // Set pointers to the right addresses
+    strm.next_in = in;
+    strm.avail_out = out_size;
+    strm.next_out = out;
+
+    // Effectively decompress data
+    ret = inflate( &strm, Z_NO_FLUSH );
+    if( ret != Z_STREAM_END )
+    {
+      throw std::runtime_error( "Error at zlib inflate: " + std::to_string( ret ) );
+    }
+
+    // Clean up
+    inflateEnd( &strm );
+  }
+
+}

deps/datReaderPs3/zlib.h

@@ -0,0 +1,16 @@
+#ifndef XIV_UTILS_ZLIB_H
+#define XIV_UTILS_ZLIB_H
+
+#include <cstdint>
+#include <vector>
+
+namespace xivps3::utils::zlib
+{
+
+  void compress( const std::vector< char >& in, std::vector< char >& out );
+
+  void no_header_decompress( uint8_t* in, size_t in_size, uint8_t* out, size_t out_size );
+
+}
+
+#endif // XIV_UTILS_ZLIB_H

deps/ffxiv-actions

@@ -1 +0,0 @@
-Subproject commit dde9b5bbfc7c0197de0b0b49b982a0ee9fe761ab

deps/mysqlConnector/Connection.cpp

@@ -3,11 +3,9 @@
 #include "Statement.h"
 #include "PreparedStatement.h"
 #include <mysql.h>
+#include <stdexcept>
 
-#ifdef _MSC_VER
+#include <utility>
-  // fixes compile error when compiling with vs2019
-  #include <stdexcept>
-#endif
 #include <vector>
 
 Mysql::Connection::Connection( std::shared_ptr< MySqlBase > pBase,
@@ -15,7 +13,7 @@ Mysql::Connection::Connection( std::shared_ptr< MySqlBase > pBase,
                                const std::string& userName,
                                const std::string& password,
                                uint16_t port ) :
-    m_pBase( pBase ),
+    m_pBase( std::move( pBase ) ),
     m_bConnected( false )
 {
    m_pRawCon = mysql_init( nullptr );
@@ -32,12 +30,12 @@ Mysql::Connection::Connection( std::shared_ptr< MySqlBase > pBase,
                                const std::string& password,
                                const optionMap& options,
                                uint16_t port ) :
-    m_pBase( pBase )
+    m_pBase( std::move( pBase ) )
 {
    m_pRawCon = mysql_init( nullptr );
    // Different mysql versions support different options, for now whatever was unsupporter here was commented out
    // but left there.
-   for( auto entry : options )
+   for( const auto& entry : options )
    {
       switch( entry.first )
       {
@@ -113,9 +111,7 @@ Mysql::Connection::Connection( std::shared_ptr< MySqlBase > pBase,
 }
 
 
-Mysql::Connection::~Connection()
+Mysql::Connection::~Connection() = default;
-{
-}
 
 void Mysql::Connection::setOption( enum mysqlOption option, const void *arg )
 {
@@ -159,7 +155,7 @@ std::shared_ptr< Mysql::MySqlBase > Mysql::Connection::getMySqlBase() const
 
 void Mysql::Connection::setAutoCommit( bool autoCommit )
 {
-   auto b = static_cast< my_bool >( autoCommit == true ? 1 : 0 );
+   auto b = static_cast< my_bool >( autoCommit ? 1 : 0 );
    if( mysql_autocommit( m_pRawCon, b ) != 0 )
       throw std::runtime_error( "Connection::setAutoCommit failed!" );
 }
@@ -168,7 +164,7 @@ bool Mysql::Connection::getAutoCommit()
 {
    // TODO: should be replaced with wrapped sql query function once available
    std::string query("SELECT @@autocommit");
-   auto res = mysql_real_query( m_pRawCon, query.c_str(), query.length() );
+   auto res = mysql_real_query( m_pRawCon, query.c_str(), static_cast< unsigned long >( query.length() ) );
 
    if( res != 0 )
     throw std::runtime_error( "Query failed!" );
@@ -202,7 +198,7 @@ void Mysql::Connection::rollbackTransaction()
 std::string Mysql::Connection::escapeString( const std::string &inData )
 {
    std::unique_ptr< char[] > buffer( new char[inData.length() * 2 + 1] );
-   if( !buffer.get() )
+   if( !buffer )
       return "";
    unsigned long return_len = mysql_real_escape_string( m_pRawCon, buffer.get(),
                                                         inData.c_str(), static_cast< unsigned long > ( inData.length() ) );
@@ -240,7 +236,7 @@ std::shared_ptr< Mysql::PreparedStatement > Mysql::Connection::prepareStatement(
    if( !stmt )
       throw std::runtime_error( "Could not init prepared statement: " + getError() );
 
-   if( mysql_stmt_prepare( stmt, sql.c_str(), sql.size() ) )
+   if( mysql_stmt_prepare( stmt, sql.c_str(), static_cast< unsigned long >( sql.size() ) ) )
       throw std::runtime_error( "Could not prepare statement: " + getError() );
 
    return std::make_shared< PreparedStatement >( stmt, shared_from_this() );

deps/mysqlConnector/Connection.h

@@ -1,10 +1,9 @@
 #ifndef SAPPHIRE_CONNECTION_H
 #define SAPPHIRE_CONNECTION_H
 
+#include <string>
 #include <memory>
 #include <map>
-#include <string>
-
 #include "MysqlCommon.h"
 
 typedef struct st_mysql MYSQL;
@@ -19,18 +18,11 @@ namespace Mysql
    class Connection : public std::enable_shared_from_this< Connection >
    {
    public:
-      Connection( std::shared_ptr< MySqlBase > pBase,
+      Connection( std::shared_ptr< MySqlBase > pBase, const std::string& hostName, const std::string& userName,
-                  const std::string& hostName,
+                  const std::string& password, uint16_t port = 3306 );
-                  const std::string& userName,
-                  const std::string& password,
-                  uint16_t port = 3306);
 
-      Connection( std::shared_ptr< MySqlBase > pBase,
+      Connection( std::shared_ptr< MySqlBase > pBase, const std::string& hostName, const std::string& userName,
-                  const std::string& hostName,
+                  const std::string& password, const optionMap& options, uint16_t port = 3306 );
-                  const std::string& userName,
-                  const std::string& password,
-                  const optionMap& options,
-                  uint16_t port = 3306 );
 
       virtual ~Connection();
 
@@ -79,7 +71,7 @@ namespace Mysql
    private:
       std::shared_ptr< MySqlBase > m_pBase;
       MYSQL* m_pRawCon;
-      bool m_bConnected;
+      bool m_bConnected{};
 
       Connection( const Connection& );
       void operator=( Connection& );

deps/mysqlConnector/PreparedStatement.cpp

@@ -78,7 +78,7 @@ struct LongDataSender
       while( sent < str->length() )
       {
          chunkSize = ( sent + MAX_SEND_LONGDATA_CHUNK > str->length()
-                     ? str->length() - sent
+                     ? static_cast< uint32_t >( str->length() ) - sent
                      : MAX_SEND_LONGDATA_CHUNK );
 
          if( mysql_stmt_send_long_data( m_pStmt, position, str->c_str() + sent, chunkSize ) )

deps/mysqlConnector/Statement.cpp

@@ -17,7 +17,7 @@ Mysql::Statement::Statement( std::shared_ptr< Mysql::Connection > conn ) :
 
 void Mysql::Statement::doQuery( const std::string &q )
 {
-   mysql_real_query( m_pConnection->getRawCon(), q.c_str(), q.length() );
+   mysql_real_query( m_pConnection->getRawCon(), q.c_str(), static_cast< unsigned long >( q.length() ) );
 
    if( errNo() )
       throw std::runtime_error( m_pConnection->getError() );

deps/mysqlConnector/mysql_util.h

@@ -27,8 +27,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include <iostream>
 #include <sstream>
+#include <mysql.h>
 
-typedef struct st_mysql_field  MYSQL_FIELD;
+//using  MYSQL_FIELD = st_mysql_field;
 #ifndef UL64
 #ifdef _WIN32
 #define UL64(x) x##ui64

deps/watchdog/Watchdog.h

@@ -31,8 +31,15 @@
 #include <mutex>
 
 #include <functional>
+
+ // fucking filesystem
+#if _MSC_VER >= 1925
 #include <filesystem>
 namespace ci { namespace fs = std::filesystem; }
+#else
+#include <experimental/filesystem>
+namespace ci { namespace fs = std::experimental::filesystem; }
+#endif
 
 //! Exception for when Watchdog can't locate a file or parse the wildcard
 class WatchedFileSystemExc : public std::exception {
@@ -319,7 +326,7 @@ protected:
       std::string                                                               mFilter;
       std::function<void(const ci::fs::path&)>                                  mCallback;
       std::function<void(const std::vector<ci::fs::path>&)>                     mListCallback;
-      std::map< std::string, std::filesystem::file_time_type >    mModificationTimes;
+      std::map< std::string, ci::fs::file_time_type >    mModificationTimes;
    };
 
    std::mutex                      mMutex;

sql/migrations/20200428074112_AddBattleNpcTable.sql

@@ -0,0 +1,50 @@
+-- Migration generated at 2020/04/28 07:41:12
+-- 20200428074112_AddBattleNpcTable.sql
+
+CREATE TABLE IF NOT EXISTS `battlenpc` (
+  `TerritoryType` int(11) NOT NULL,
+  `TerritoryName` varchar(22) NOT NULL,
+  `name` varchar(12) NOT NULL,
+  `instanceId` int(11) NOT NULL,
+  `x` decimal(12,6) NOT NULL,
+  `y` decimal(11,6) NOT NULL,
+  `z` decimal(11,6) NOT NULL,
+  `BaseId` int(11) NOT NULL,
+  `PopWeather` int(11) NOT NULL,
+  `PopTimeStart` int(11) NOT NULL,
+  `PopTimeEnd` int(11) NOT NULL,
+  `MoveAI` int(11) NOT NULL,
+  `WanderingRange` int(11) NOT NULL,
+  `Route` int(11) NOT NULL,
+  `EventGroup` int(11) NOT NULL,
+  `NameId` int(11) NOT NULL,
+  `DropItem` int(11) NOT NULL,
+  `SenseRangeRate` decimal(9,6) NOT NULL,
+  `Level` int(11) NOT NULL,
+  `ActiveType` int(11) NOT NULL,
+  `PopInterval` int(11) NOT NULL,
+  `PopRate` int(11) NOT NULL,
+  `PopEvent` int(11) NOT NULL,
+  `LinkGroup` int(11) NOT NULL,
+  `LinkFamily` int(11) NOT NULL,
+  `LinkRange` int(11) NOT NULL,
+  `LinkCountLimit` int(11) NOT NULL,
+  `NonpopInitZone` int(11) NOT NULL,
+  `InvalidRepop` int(11) NOT NULL,
+  `LinkParent` int(11) NOT NULL,
+  `LinkOverride` int(11) NOT NULL,
+  `LinkReply` int(11) NOT NULL,
+  `HorizontalPopRange` decimal(9,6) NOT NULL,
+  `VerticalPopRange` decimal(9,6) NOT NULL,
+  `BNpcBaseData` int(11) NOT NULL,
+  `RepopId` int(11) NOT NULL,
+  `BNPCRankId` int(11) NOT NULL,
+  `TerritoryRange` int(11) NOT NULL,
+  `BoundInstanceID` int(11) NOT NULL,
+  `FateLayoutLabelId` int(11) NOT NULL,
+  `NormalAI` int(11) NOT NULL,
+  `ServerPathId` int(11) NOT NULL,
+  `EquipmentID` int(11) NOT NULL,
+  `CustomizeID` int(11) NOT NULL,
+  PRIMARY KEY (`TerritoryType`,`instanceId`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;

sql/migrations/20210910074112_AddFriendlist.sql

@@ -0,0 +1,9 @@
+CREATE TABLE IF NOT EXISTS CharaInfoFriendlist (
+	`CharacterId` int(20) NOT NULL,
+	`CharacterIdList` blob,
+	`InviteDataList` blob,
+	`IS_DELETE` int(3) DEFAULT 0,
+	`IS_NOT_ACTIVE_FLG` int(3) DEFAULT 0,
+	`UPDATE_DATE` datetime,
+	PRIMARY KEY (`CharacterId`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;

sql/migrations/20210911163405_AddLinkshell.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `infolinkshell`
+	CHANGE COLUMN `MasterCharacterId` `MasterCharacterId` BIGINT NULL DEFAULT NULL AFTER `LinkshellId`;

sql/migrations/20210916081902_RenameContentIdToActorId.sql

@@ -0,0 +1,49 @@
+-- Migration generated at 2021/09/16 08:19:02
+-- 20210916081902_RenameContentIdToActorId.sql
+
+ALTER TABLE `accounts`
+	CHANGE COLUMN `account_id` `account_id` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `characlass`
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `charaglobalitem`
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `charainfo`
+	CHANGE COLUMN `AccountId` `AccountId` BIGINT UNSIGNED NOT NULL FIRST,
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL AFTER `AccountId`,
+	CHANGE COLUMN `ContentId` `EntityId` INT UNSIGNED NULL DEFAULT '0' AFTER `CharacterId`;
+
+ALTER TABLE `charainfoblacklist`
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `charainfolinkshell`
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `charainfosearch`
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `charaitemcrystal`
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `charaitemcurrency`
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `charaitemgearset`
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `charaiteminventory`
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `charamonsternote`
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `charaquest`
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `charastatus`
+	CHANGE COLUMN `CharacterId` `CharacterId` BIGINT UNSIGNED NOT NULL FIRST;
+
+ALTER TABLE `infolinkshell`
+	CHANGE COLUMN `MasterCharacterId` `MasterCharacterId` BIGINT UNSIGNED NULL DEFAULT NULL AFTER `LinkshellId`;

sql/migrations/20211003182901_UpdateCharaInfoUpdateDate.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `charainfo`
+	CHANGE COLUMN `UPDATE_DATE` `UPDATE_DATE` DATETIME NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP AFTER `CFPenaltyUntil`;

sql/migrations/20211005112001_DropSpawnTables.sql

@@ -0,0 +1,3 @@
+DROP TABLE IF EXISTS bnpctemplate;
+DROP TABLE IF EXISTS spawnpoint;
+DROP TABLE IF EXISTS spawngroup;

sql/schema/schema.sql

@@ -9,36 +9,6 @@ CREATE TABLE `accounts` (
    UNIQUE KEY `accountname` (`account_name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
-CREATE TABLE `bnpctemplate` (
-  `Id` int(11) NOT NULL AUTO_INCREMENT,
-  `Name` varchar(32) NOT NULL,
-  `bNPCBaseId` int(10) DEFAULT NULL,
-  `bNPCNameId` int(10) NOT NULL,
-  `mainWeaponModel` bigint(20) DEFAULT NULL,
-  `secWeaponModel` bigint(20) DEFAULT NULL,
-  `aggressionMode` int(3) DEFAULT NULL,
-  `enemyType` int(3) DEFAULT NULL,
-  `pose` int(3) DEFAULT NULL,
-  `modelChara` int(5) DEFAULT NULL,
-  `displayFlags` int(10) DEFAULT NULL,
-  `Look` binary(26) DEFAULT NULL,
-  `Models` binary(40) DEFAULT NULL,
-   PRIMARY KEY (`Id`),
-   KEY `templatename` (`name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
-CREATE TABLE `spawnpoint` (
-  `id` int(11) NOT NULL  AUTO_INCREMENT,
-  `spawnGroupId` int(11) NOT NULL,
-  `x` float NOT NULL,
-  `y` float NOT NULL,
-  `z` float NOT NULL,
-  `r` float NOT NULL,
-  `gimmickId` int(11) DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  KEY `spawngroupidx` (`spawnGroupId`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
-
 CREATE TABLE `charainfo` (
   `AccountId` int(11) NOT NULL,
   `CharacterId` int(20) NOT NULL,
@@ -540,15 +510,6 @@ CREATE TABLE `houseiteminventory` (
 	INDEX `landIdent` (`LandIdent`)
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
 
-CREATE TABLE `spawngroup` (
-  `id` int(10) NOT NULL AUTO_INCREMENT,
-  `territoryTypeId` int(5) NOT NULL,
-  `bNpcTemplateId` int(10) NOT NULL,
-  `level` int(3) NOT NULL,
-  `maxHp` int(10) NOT NULL,
-  PRIMARY KEY(`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
-
 CREATE TABLE `uniqueiddata` (
   `NextId` int(20) NOT NULL AUTO_INCREMENT,
   `IdName` varchar(16) DEFAULT 'NOT SET',

src/api/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required( VERSION 2.6 )
+cmake_minimum_required(VERSION 3.0)
 cmake_policy( SET CMP0015 NEW )
 project( Sapphire )
 

src/api/PlayerMinimal.cpp

@@ -1,13 +1,13 @@
 #include "PlayerMinimal.h"
 
 #include <Common.h>
-#include <Exd/ExdDataGenerated.h>
+#include <Exd/ExdData.h>
 
 #include <Database/DatabaseDef.h>
 
 #include <nlohmann/json.hpp>
 
-extern Sapphire::Data::ExdDataGenerated g_exdDataGen;
+extern Sapphire::Data::ExdData g_exdData;
 
 namespace Sapphire::Api {
 
@@ -15,7 +15,7 @@ using namespace Common;
 
 // player constructor
 PlayerMinimal::PlayerMinimal() :
-  m_id( 0 )
+  m_characterId( 0 )
 {
 
 
@@ -23,18 +23,19 @@ PlayerMinimal::PlayerMinimal() :
 
 // load player from the db
 // TODO change void CPlayer::load to bool, we want to know if something went wrong
-void PlayerMinimal::load( uint32_t charId )
+void PlayerMinimal::load( uint64_t charId )
 {
 
   auto stmt = g_charaDb.getPreparedStatement( Db::ZoneDbStatements::CHARA_SEL_MINIMAL );
 
-  stmt->setUInt( 1, charId );
+  stmt->setUInt64( 1, charId );
   auto res = g_charaDb.query( stmt );
 
   if( !res->next() )
     return;
 
-  m_id = charId;
+  m_characterId = charId;
+  m_id = res->getUInt64( "EntityId" );
 
   memset( m_name, 0, 32 );
 
@@ -58,14 +59,13 @@ void PlayerMinimal::load( uint32_t charId )
   setBirthDay( res->getUInt8( "BirthDay" ), res->getUInt8( "BirthMonth" ) );
   m_guardianDeity = res->getUInt8( "GuardianDeity" );
   m_class = res->getUInt8( "Class" );
-  m_contentId = res->getUInt64( "ContentId" );
   m_territoryTypeId = res->getUInt16( "TerritoryType" );
 
   res.reset();
 
   // SELECT ClassIdx, Exp, Lvl
   auto stmtClass = g_charaDb.getPreparedStatement( Db::ZoneDbStatements::CHARA_CLASS_SEL );
-  stmtClass->setInt( 1, m_id );
+  stmtClass->setUInt64( 1, m_characterId );
 
   auto resClass = g_charaDb.query( stmtClass );
 
@@ -122,7 +122,7 @@ std::string PlayerMinimal::getInfoJson()
   c.push_back( std::to_string( getZoneId() ) );
 
   // ContentFinderCondition
-  c.push_back( "0" );
+  //c.push_back( "0" );
 
   // look map
   auto lookArray = nlohmann::json();
@@ -167,7 +167,7 @@ std::string PlayerMinimal::getInfoJson()
   // LoginStatus
   c.push_back( "0" );
   // IsOutTerritory
-  c.push_back( "0" );
+  //c.push_back( "0" );
 
 
   payload["classname"] = "ClientSelectData";
@@ -178,7 +178,7 @@ std::string PlayerMinimal::getInfoJson()
 
 uint8_t PlayerMinimal::getClassLevel()
 {
-  uint8_t classJobIndex = g_exdDataGen.get< Sapphire::Data::ClassJob >( static_cast< uint8_t >( m_class ) )->expArrayIndex;
+  uint8_t classJobIndex = g_exdData.getRow< Component::Excel::ClassJob >( m_class )->data().WorkIndex;
   return static_cast< uint8_t >( m_classMap[ classJobIndex ] );
 }
 
@@ -257,16 +257,17 @@ void PlayerMinimal::saveAsNew()
       break;
   }
 
-  //        "(AccountId, CharacterId, ContentId, Name, Hp, Mp, "
+  //        "(AccountId, CharacterId, EntityId, Name, Hp, Mp, "
   //        "Customize, Voice, IsNewGame, TerritoryType, PosX, PosY, PosZ, PosR, ModelEquip, "
   //        "IsNewAdventurer, GuardianDeity, Birthday, BirthMonth, Class, Status, FirstClass, "
   //        "HomePoint, StartTown, Discovery, HowTo, QuestCompleteFlags, Unlocks, QuestTracking, "
   //        "Aetheryte, GMRank, UPDATE_DATE )
 
   auto stmt = g_charaDb.getPreparedStatement( Db::ZoneDbStatements::CHARA_INS );
-  stmt->setInt( 1, m_accountId );
+  stmt->set( 1, m_accountId );
-  stmt->setInt( 2, m_id );
+  stmt->set( 2, m_characterId );
-  stmt->setInt64( 3, m_contentId );
+  stmt->set( 3, m_id );
+
   stmt->setString( 4, std::string( m_name ) );
   stmt->setInt( 5, 100 );
   stmt->setInt( 6, 100 );
@@ -301,14 +302,14 @@ void PlayerMinimal::saveAsNew()
 
   // CharacterId, ClassIdx, Exp, Lvl
   auto stmtClass = g_charaDb.getPreparedStatement( Db::ZoneDbStatements::CHARA_CLASS_INS );
-  stmtClass->setInt( 1, m_id );
+  stmtClass->setUInt64( 1, m_characterId );
-  stmtClass->setInt( 2, g_exdDataGen.get< Sapphire::Data::ClassJob >( m_class )->expArrayIndex );
+  stmtClass->setInt( 2, g_exdData.getRow< Component::Excel::ClassJob >( m_class )->data().WorkIndex );
   stmtClass->setInt( 3, 0 );
   stmtClass->setInt( 4, 1 );
   g_charaDb.directExecute( stmtClass );
 
   auto stmtSearchInfo = g_charaDb.getPreparedStatement( Db::ZoneDbStatements::CHARA_SEARCHINFO_INS );
-  stmtSearchInfo->setInt( 1, m_id );
+  stmtSearchInfo->setUInt64( 1, m_characterId );
   g_charaDb.directExecute( stmtSearchInfo );
 
   /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -338,21 +339,21 @@ void PlayerMinimal::saveAsNew()
   createInvDbContainer( InventoryType::Crystal );
 
   auto stmtMonsterNote = g_charaDb.getPreparedStatement( Db::ZoneDbStatements::CHARA_MONSTERNOTE_INS );
-  stmtMonsterNote->setInt( 1, m_id );
+  stmtMonsterNote->setUInt64( 1, m_characterId );
   for( uint8_t i = 1; i <= 12; ++i )
     stmtMonsterNote->setBinary( i + 1, monsterNote );
   g_charaDb.directExecute( stmtMonsterNote );
 
   ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
   /// SETUP EQUIPMENT / STARTING GEAR
-  auto classJobInfo = g_exdDataGen.get< Sapphire::Data::ClassJob >( m_class );
+  auto classJobInfo = g_exdData.getRow< Component::Excel::ClassJob >( m_class );
-  uint32_t weaponId = classJobInfo->itemStartingWeapon;
+  uint32_t weaponId = classJobInfo->data().InitWeapon[0];
   uint64_t uniqueId = getNextUId64();
 
   uint8_t race = customize[ CharaLook::Race ];
   uint8_t gender = customize[ CharaLook::Gender ];
 
-  auto raceInfo = g_exdDataGen.get< Sapphire::Data::Race >( race );
+  auto raceInfo = g_exdData.getRow< Component::Excel::Race >( race );
 
   uint32_t body;
   uint32_t hands;
@@ -363,20 +364,10 @@ void PlayerMinimal::saveAsNew()
   uint64_t legsUid = getNextUId64();
   uint64_t feetUid = getNextUId64();
 
-  if( gender == 0 )
+  body = raceInfo->data().Body[ gender ];
-  {
+  hands = raceInfo->data().Hand[ gender ];
-    body = raceInfo->rSEMBody;
+  legs = raceInfo->data().Leg[ gender ];
-    hands = raceInfo->rSEMHands;
+  feet = raceInfo->data().Foot[ gender ];
-    legs = raceInfo->rSEMLegs;
-    feet = raceInfo->rSEMFeet;
-  }
-  else
-  {
-    body = raceInfo->rSEFBody;
-    hands = raceInfo->rSEFHands;
-    legs = raceInfo->rSEFLegs;
-    feet = raceInfo->rSEFFeet;
-  }
 
   insertDbGlobalItem( weaponId, uniqueId );
   insertDbGlobalItem( body, bodyUid );
@@ -384,18 +375,6 @@ void PlayerMinimal::saveAsNew()
   insertDbGlobalItem( legs, legsUid );
   insertDbGlobalItem( feet, feetUid );
 
-  // Universal accessories
-
-  uint64_t neckUid = getNextUId64();
-  uint64_t earUid = getNextUId64();
-  uint64_t wristUid = getNextUId64();
-  uint64_t ringUid = getNextUId64();
-
-
-  insertDbGlobalItem( 15130, neckUid );
-  insertDbGlobalItem( 15131, earUid );
-  insertDbGlobalItem( 15132, wristUid );
-  insertDbGlobalItem( 15133, ringUid );
 
   g_charaDb.execute( "INSERT INTO charaitemgearset (storageId, CharacterId, "
                      "container_" + std::to_string( GearSetSlot::MainHand ) + ", "
@@ -409,23 +388,23 @@ void PlayerMinimal::saveAsNew()
                      "container_" + std::to_string( GearSetSlot::Ring1 ) + ", UPDATE_DATE ) "
                      "VALUES ( " +
                      std::to_string( InventoryType::GearSet0 ) + ", " +
-                     std::to_string( m_id ) + ", " +
+                     std::to_string( m_characterId ) + ", " +
                      std::to_string( uniqueId ) + ", " +
                      std::to_string( bodyUid ) + ", " +
                      std::to_string( handsUid ) + ", " +
                      std::to_string( legsUid ) + ", " +
                      std::to_string( feetUid ) + ", " +
-                     std::to_string( neckUid ) + ", " +
+                     std::to_string( 0 ) + ", " +
-                     std::to_string( earUid ) + ", " +
+                     std::to_string( 0 ) + ", " +
-                     std::to_string( wristUid ) + ", " +
+                     std::to_string( 0 ) + ", " +
-                     std::to_string( ringUid ) + ", NOW());" );
+                     std::to_string( 0 ) + ", NOW());" );
 
 }
 
 void PlayerMinimal::insertDbGlobalItem( uint32_t itemId, uint64_t uniqueId ) const
 {
   auto stmtItemGlobal = g_charaDb.getPreparedStatement( Db::CHARA_ITEMGLOBAL_INS );
-  stmtItemGlobal->setInt( 1, m_id );
+  stmtItemGlobal->setUInt64( 1, m_characterId );
   stmtItemGlobal->setInt64( 2, uniqueId );
   stmtItemGlobal->setInt( 3, itemId );
   stmtItemGlobal->setInt( 4, 1 ); // stack of 1
@@ -435,7 +414,7 @@ void PlayerMinimal::insertDbGlobalItem( uint32_t itemId, uint64_t uniqueId ) con
 void PlayerMinimal::createInvDbContainer( uint16_t slot ) const
 {
   auto stmtCreateInv = g_charaDb.getPreparedStatement( Db::CHARA_ITEMINV_INS );
-  stmtCreateInv->setInt( 1, m_id );
+  stmtCreateInv->setUInt64( 1, m_characterId );
   stmtCreateInv->setInt( 2, slot );
   g_charaDb.directExecute( stmtCreateInv );
 }

src/api/PlayerMinimal.h

@@ -1,10 +1,9 @@
-#ifndef _PLAYERMINIMAL_H
+#pragma once
-#define _PLAYERMINIMAL_H
 
 #include <map>
-#include <cstdint>
+#include <stdint.h>
 #include <string>
-#include <cstring>
+#include <string.h> // c string functions
 
 namespace Sapphire::Api
 {
@@ -20,7 +19,7 @@ namespace Sapphire::Api
     void write();
 
     // load player from db, by id
-    void load( uint32_t charId );
+    void load( uint64_t charId );
 
     void saveAsNew();
 
@@ -28,7 +27,6 @@ namespace Sapphire::Api
 
     uint8_t getClassLevel();
 
-    // return the id of the actor
     uint32_t getId() const
     {
       return m_id;
@@ -39,14 +37,14 @@ namespace Sapphire::Api
       m_id = id;
     }
 
-    void setContentId( uint64_t id )
+    void setCharacterId( uint64_t id )
     {
-      m_contentId = id;
+      m_characterId = id;
     }
 
-    uint64_t getContentId() const
+    uint64_t getCharacterId() const
     {
-      return m_contentId;
+      return m_characterId;
     }
 
 
@@ -168,7 +166,7 @@ namespace Sapphire::Api
   private:
     uint32_t m_accountId;
     uint32_t m_id;
-    uint64_t m_contentId;
+    uint64_t m_characterId;
 
     uint8_t m_guardianDeity;
     uint8_t m_birthMonth;
@@ -199,4 +197,3 @@ namespace Sapphire::Api
   };
 
 }
-#endif

src/api/SapphireApi.cpp

@@ -28,27 +28,29 @@ bool SapphireApi::login( const std::string& username, const std::string& pass, s
 
   // session id string generation
   srand( ( uint32_t ) time( NULL ) + 42 );
-  uint8_t sid[58];
 
-  for( int32_t i = 0; i < 56; i += 4 )
+  std::string sessionId;
+  for( int32_t i = 0; i < 64 / 4; ++i )
   {
     short number = 0x1111 + rand() % 0xFFFF;
-    sprintf( ( char* ) sid + i, "%04hx", number );
+    char part[5];
+    sprintf( part, "%04hx", number );
+
+    if( i == 15 )
+    {
+      part[2] = 0;
+      part[3] = 0;
+    }
+    sessionId += std::string( part );
   }
 
   // create session for the new sessionid and store to sessionlist
   auto pSession = std::make_shared< Session >();
   pSession->setAccountId( accountId );
-  pSession->setSessionId( sid );
+  pSession->setSessionId( sessionId.c_str() );
 
-  std::stringstream ss;
+  m_sessionMap[ sessionId ] = pSession;
-
+  sId = sessionId;
-  for( size_t i = 0; i < 56; i++ )
-  {
-    ss << std::hex << sid[ i ];
-  }
-  m_sessionMap[ ss.str() ] = pSession;
-  sId = ss.str();
 
   return true;
 
@@ -60,7 +62,7 @@ bool SapphireApi::insertSession( const uint32_t accountId, std::string& sId )
   // create session for the new sessionid and store to sessionlist
   auto pSession = std::make_shared< Session >();
   pSession->setAccountId( accountId );
-  pSession->setSessionId( ( uint8_t* ) sId.c_str() );
+  pSession->setSessionId( sId.c_str() );
 
   m_sessionMap[ sId ] = pSession;
 
@@ -105,8 +107,8 @@ int SapphireApi::createCharacter( const uint32_t accountId, const std::string& n
   Api::PlayerMinimal newPlayer;
 
   newPlayer.setAccountId( accountId );
-  newPlayer.setId( getNextCharId() );
+  newPlayer.setId( getNextEntityId() );
-  newPlayer.setContentId( getNextContentId() );
+  newPlayer.setCharacterId( getNextCharaId() );
   newPlayer.setName( name.c_str() );
 
   auto json = nlohmann::json::parse( infoJson );
@@ -186,21 +188,19 @@ void SapphireApi::deleteCharacter( std::string name, const uint32_t accountId )
     }
   }
 
-  int32_t id = deletePlayer.getId();
+  int32_t id = deletePlayer.getCharacterId();
 
-  g_charaDb.execute( "DELETE FROM charainfo WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
+  g_charaDb.execute( "DELETE FROM charainfo WHERE CharacterId = " + std::to_string( id ) + ";" );
-  g_charaDb.execute( "DELETE FROM characlass WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
+  g_charaDb.execute( "DELETE FROM characlass WHERE CharacterId = " + std::to_string( id ) + ";" );
-  g_charaDb.execute( "DELETE FROM charaglobalitem WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
+  g_charaDb.execute( "DELETE FROM charaglobalitem WHERE CharacterId = " + std::to_string( id ) + ";" );
-  g_charaDb.execute( "DELETE FROM charainfoblacklist WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
+  g_charaDb.execute( "DELETE FROM charainfoblacklist WHERE CharacterId = " + std::to_string( id ) + ";" );
-  g_charaDb.execute( "DELETE FROM charainfolinkshell WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
+  g_charaDb.execute( "DELETE FROM charainfofriendlist WHERE CharacterId = " + std::to_string( id ) + ";" );
-  g_charaDb.execute( "DELETE FROM charainfosearch WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
+  g_charaDb.execute( "DELETE FROM charainfolinkshell WHERE CharacterId = " + std::to_string( id ) + ";" );
-  g_charaDb.execute( "DELETE FROM charaitemcrystal WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
+  g_charaDb.execute( "DELETE FROM charainfosearch WHERE CharacterId = " + std::to_string( id ) + ";" );
-  g_charaDb.execute( "DELETE FROM charaitemcurrency WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
+  g_charaDb.execute( "DELETE FROM charaitemcrystal WHERE CharacterId = " + std::to_string( id ) + ";" );
-  g_charaDb.execute( "DELETE FROM charaiteminventory WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
+  g_charaDb.execute( "DELETE FROM charaiteminventory WHERE CharacterId = " + std::to_string( id ) + ";" );
-  g_charaDb.execute( "DELETE FROM charaitemgearset WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
+  g_charaDb.execute( "DELETE FROM charaitemgearset WHERE CharacterId = " + std::to_string( id ) + ";" );
-  g_charaDb.execute( "DELETE FROM charamonsternote WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
+  g_charaDb.execute( "DELETE FROM charaquest WHERE CharacterId = " + std::to_string( id ) + ";" );
-  g_charaDb.execute( "DELETE FROM charaquest WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
-  g_charaDb.execute( "DELETE FROM charastatus WHERE CharacterId LIKE '" + std::to_string( id ) + "';" );
 }
 
 std::vector< PlayerMinimal > SapphireApi::getCharList( uint32_t accountId )
@@ -209,13 +209,13 @@ std::vector< PlayerMinimal > SapphireApi::getCharList( uint32_t accountId )
   std::vector< Api::PlayerMinimal > charList;
 
   auto pQR = g_charaDb.query(
-    "SELECT CharacterId, ContentId FROM charainfo WHERE AccountId = " + std::to_string( accountId ) + ";" );
+    "SELECT CharacterId FROM charainfo WHERE AccountId = " + std::to_string( accountId ) + ";" );
 
   while( pQR->next() )
   {
     Api::PlayerMinimal player;
 
-    uint32_t charId = pQR->getUInt( 1 );
+    auto charId = pQR->getUInt64( 1 );
 
     player.load( charId );
 
@@ -238,11 +238,11 @@ bool SapphireApi::checkNameTaken( std::string name )
     return true;
 }
 
-uint32_t SapphireApi::getNextCharId()
+uint32_t SapphireApi::getNextEntityId()
 {
   uint32_t charId = 0;
 
-  auto pQR = g_charaDb.query( "SELECT MAX(CharacterId) FROM charainfo" );
+  auto pQR = g_charaDb.query( "SELECT MAX(EntityId) FROM charainfo" );
 
   if( !pQR->next() )
     return 0x00200001;
@@ -254,11 +254,11 @@ uint32_t SapphireApi::getNextCharId()
   return charId;
 }
 
-uint64_t SapphireApi::getNextContentId()
+uint64_t SapphireApi::getNextCharaId()
 {
   uint64_t contentId = 0;
 
-  auto pQR = g_charaDb.query( "SELECT MAX(ContentId) FROM charainfo" );
+  auto pQR = g_charaDb.query( "SELECT MAX(CharacterId) FROM charainfo" );
 
   if( !pQR->next() )
     return 0x0040000001000001;

src/api/SapphireApi.h

@@ -1,5 +1,4 @@
-#ifndef _SAPPHIREAPI_H_
+#pragma once
-#define _SAPPHIREAPI_H_
 
 #include <string>
 #include <vector>
@@ -34,9 +33,9 @@ namespace Sapphire::Api
 
     bool checkNameTaken( std::string name );
 
-    uint32_t getNextCharId();
+    uint32_t getNextEntityId();
 
-    uint64_t getNextContentId();
+    uint64_t getNextCharaId();
 
     int32_t checkSession( const std::string& sId );
 
@@ -46,5 +45,3 @@ namespace Sapphire::Api
 
   };
 }
-
-#endif

src/api/Session.cpp

@@ -17,9 +17,9 @@ uint32_t Session::getIp() const
   return m_ip;
 }
 
-void Session::setSessionId( uint8_t* sessionId )
+void Session::setSessionId( const char* sessionId )
 {
-  memcpy( m_sessionId, sessionId, 56 );
+  memcpy( m_sessionId, sessionId, sizeof( m_sessionId ) );
 }
 
 void Session::setIp( uint32_t ip )

src/api/Session.h

@@ -1,5 +1,4 @@
-#ifndef _SESSION_H_
+#pragma once
-#define _SESSION_H_
 
 #include <stdint.h>
 #include <string>
@@ -14,7 +13,7 @@ namespace Sapphire::Api
   private:
     uint32_t m_ip;
     uint32_t m_accountId;
-    uint8_t m_sessionId[56];
+    char m_sessionId[ 64 ];
 
   public:
     std::string newCharName;
@@ -25,7 +24,7 @@ namespace Sapphire::Api
 
     uint32_t getIp() const;
 
-    void setSessionId( uint8_t* sessionId );
+    void setSessionId( const char* sessionId );
 
     void setIp( uint32_t ip );
 
@@ -36,5 +35,3 @@ namespace Sapphire::Api
   };
 
 }
-
-#endif

src/api/client_http.hpp

@@ -1,5 +1,4 @@
-#ifndef CLIENT_HTTP_HPP
+#pragma once
-#define	CLIENT_HTTP_HPP
 
 #include <asio.hpp>
 
@@ -398,4 +397,3 @@ namespace SimpleWeb
     };
 }
 
-#endif	/* CLIENT_HTTP_HPP */

src/api/main.cpp

@@ -10,7 +10,7 @@
 #include <Network/Hive.h>
 #include <Network/Acceptor.h>
 
-#include <Exd/ExdDataGenerated.h>
+#include <Exd/ExdData.h>
 #include <Crypt/base64.h>
 
 #include <Database/DbLoader.h>
@@ -22,23 +22,31 @@
 //Added for the default_resource example
 #include <fstream>
 #include <string>
-#include <filesystem>
+
 #include <vector>
 #include <algorithm>
 
-#include <Logging/Logger.h>
-
 #include "SapphireApi.h"
 
 #include <Util/CrashHandler.h>
 
-[[maybe_unused]] Sapphire::Common::Util::CrashHandler crashHandler;
+
+// fucking filesystem
+#if _MSC_VER >= 1925
+#include <filesystem>
+namespace fs = std::filesystem;
+#else
+#include <experimental/filesystem>
+namespace fs = std::experimental::filesystem;
+#endif
+
+
+Sapphire::Common::Util::CrashHandler crashHandler;
 
 Sapphire::Db::DbWorkerPool< Sapphire::Db::ZoneDbConnection > g_charaDb;
-Sapphire::Data::ExdDataGenerated g_exdDataGen;
+Sapphire::Data::ExdData g_exdData;
 Sapphire::Api::SapphireApi g_sapphireAPI;
 
-namespace fs = std::filesystem;
 
 using namespace std;
 using namespace Sapphire;
@@ -121,7 +129,7 @@ bool loadSettings( int32_t argc, char* argv[] )
 
   Logger::info( "Setting up generated EXD data" );
   auto dataPath = m_config.global.general.dataPath;
-  if( !g_exdDataGen.init( dataPath ) )
+  if( !g_exdData.init( dataPath ) )
   {
     Logger::fatal( "Error setting up generated EXD data. Make sure that DataPath is set correctly in global.ini" );
     Logger::fatal( "DataPath: {0}", dataPath );
@@ -213,10 +221,12 @@ std::string buildHttpResponse( uint16_t rCode, const std::string& content = "",
 void getZoneName( shared_ptr< HttpServer::Response > response, shared_ptr< HttpServer::Request > request )
 {
   string number = request->path_match[ 1 ];
-  auto info = g_exdDataGen.get< Sapphire::Data::TerritoryType >( atoi( number.c_str() ) );
+  auto info = g_exdData.getRow< Component::Excel::TerritoryType >( atoi( number.c_str() ) );
   std::string responseStr = "Not found!";
   if( info )
-    responseStr = info->name + ", " + info->bg;
+  {
+    responseStr = info->getString( info->data().Name ) + ", " + info->getString( info->data().LVB );
+  }
   *response << buildHttpResponse( 200, responseStr );
 }
 
@@ -461,7 +471,7 @@ void checkSession( shared_ptr< HttpServer::Response > response, shared_ptr< Http
       {
         std::string json_string = nlohmann::json( {
           { "result", result }
-        } ).dump()
+        } ).dump(1)
         ;
         *response << buildHttpResponse( 200, json_string, JSON );
       }
@@ -498,7 +508,7 @@ void getNextCharId( shared_ptr< HttpServer::Response > response, shared_ptr< Htt
     }
     else
     {
-      std::string json_string = "{\"result\":\"" + std::to_string( g_sapphireAPI.getNextCharId() ) + "\"}";
+      std::string json_string = "{\"result\":\"" + std::to_string( g_sapphireAPI.getNextEntityId() ) + "\"}";
       *response << buildHttpResponse( 200, json_string, JSON );
     }
   }
@@ -529,7 +539,7 @@ void getNextContentId( shared_ptr< HttpServer::Response > response, shared_ptr<
     }
     else
     {
-      std::string json_string = "{\"result\":\"" + std::to_string( g_sapphireAPI.getNextContentId() ) + "\"}";
+      std::string json_string = "{\"result\":\"" + std::to_string( g_sapphireAPI.getNextCharaId() ) + "\"}";
       *response << buildHttpResponse( 200, json_string, JSON );
     }
   }
@@ -572,15 +582,15 @@ void getCharacterList( shared_ptr< HttpServer::Response > response, shared_ptr<
         {
           json["charArray"].push_back( {
             { "name", std::string( entry.getName() ) },
-            { "charId", std::to_string( entry.getId() ) },
+            { "entityId", std::to_string( entry.getId() ) },
-            { "contentId", std::to_string( entry.getContentId() ) },
+            { "contentId", std::to_string( entry.getCharacterId() ) },
             { "infoJson", std::string( entry.getInfoJson() ) }
           } );
         }
         
         json["result"] = "success";
         
-        *response << buildHttpResponse( 200, json.dump(), JSON );
+        *response << buildHttpResponse( 200, json.dump(1), JSON );
       }
     }
     else
@@ -670,9 +680,8 @@ void defaultGet( shared_ptr< HttpServer::Response > response, shared_ptr< HttpSe
   print_request_info( request );
   try
   {
-    auto web_root_path = fs::current_path() / "web";
+    auto web_root_path = fs::canonical( "web" );
-    auto path = web_root_path / request->path;
+    auto path = fs::canonical( "web" + request->path );
-
     //Check if path is within web_root_path
     if( distance( web_root_path.begin(), web_root_path.end() ) > distance( path.begin(), path.end() ) ||
         !std::equal( web_root_path.begin(), web_root_path.end(), path.begin() ) )
@@ -718,19 +727,19 @@ int main( int argc, char* argv[] )
 
   Logger::setLogLevel( m_config.global.general.logLevel );
 
-  server.resource[ "^ZoneName/([0-9]+)$" ][ "GET" ] = &getZoneName;
+  server.resource[ "^/ZoneName/([0-9]+)$" ][ "GET" ] = &getZoneName;
-  server.resource[ "^sapphire-api/lobby/createAccount" ][ "POST" ] = &createAccount;
+  server.resource[ "^/sapphire-api/lobby/createAccount" ][ "POST" ] = &createAccount;
-  server.resource[ "^sapphire-api/lobby/login" ][ "POST" ] = &login;
+  server.resource[ "^/sapphire-api/lobby/login" ][ "POST" ] = &login;
-  server.resource[ "^sapphire-api/lobby/deleteCharacter" ][ "POST" ] = &deleteCharacter;
+  server.resource[ "^/sapphire-api/lobby/deleteCharacter" ][ "POST" ] = &deleteCharacter;
-  server.resource[ "^sapphire-api/lobby/createCharacter" ][ "POST" ] = &createCharacter;
+  server.resource[ "^/sapphire-api/lobby/createCharacter" ][ "POST" ] = &createCharacter;
-  server.resource[ "^sapphire-api/lobby/insertSession" ][ "POST" ] = &insertSession;
+  server.resource[ "^/sapphire-api/lobby/insertSession" ][ "POST" ] = &insertSession;
-  server.resource[ "^sapphire-api/lobby/checkNameTaken" ][ "POST" ] = &checkNameTaken;
+  server.resource[ "^/sapphire-api/lobby/checkNameTaken" ][ "POST" ] = &checkNameTaken;
-  server.resource[ "^sapphire-api/lobby/checkSession" ][ "POST" ] = &checkSession;
+  server.resource[ "^/sapphire-api/lobby/checkSession" ][ "POST" ] = &checkSession;
-  server.resource[ "^sapphire-api/lobby/getNextCharId" ][ "POST" ] = &getNextCharId;
+  server.resource[ "^/sapphire-api/lobby/getNextEntityId" ][ "POST" ] = &getNextCharId;
-  server.resource[ "^sapphire-api/lobby/getNextContentId" ][ "POST" ] = &getNextContentId;
+  server.resource[ "^/sapphire-api/lobby/getNextCharaId" ][ "POST" ] = &getNextContentId;
-  server.resource[ "^sapphire-api/lobby/getCharacterList" ][ "POST" ] = &getCharacterList;
+  server.resource[ "^/sapphire-api/lobby/getCharacterList" ][ "POST" ] = &getCharacterList;
-  server.resource[ "^(frontier-api/ffxivsupport/view/get_init)(.*)" ][ "GET" ] = &get_init;
+  server.resource[ "^(/frontier-api/ffxivsupport/view/get_init)(.*)" ][ "GET" ] = &get_init;
-  server.resource[ "^(frontier-api/ffxivsupport/information/get_headline_all)(.*)" ][ "GET" ] = &get_headline_all;
+  server.resource[ "^(/frontier-api/ffxivsupport/information/get_headline_all)(.*)" ][ "GET" ] = &get_headline_all;
 
   server.default_resource[ "GET" ] = &defaultGet;