mirrors/sapphire
1
Fork 0
mirror of https://github.com/SapphireServer/Sapphire.git synced 2025-05-12 21:47:45 +00:00
Code Activity

loads of datreader cleanup/consistency fixes

Browse source
This commit is contained in:
NotAdam 2020-02-10 14:05:04 +11:00
parent b7ddb054f4
commit 5f18c43dd6
27 changed files with 1727 additions and 1734 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

509
deps/datReader/Dat.cpp vendored
View file

@ -4,308 +4,319 @@
#include "File.h" #include "File.h"
namespace namespace {
{ const uint32_t model_section_count = 0xB;
const uint32_t model_section_count = 0xB;
} }
namespace xiv namespace xiv::dat
{ {
namespace dat struct DatFileHeader
{ {
struct DatFileHeader uint32_t size;
{ FileType entry_type;
uint32_t size; uint32_t total_uncompressed_size;
FileType entry_type; uint32_t unknown[0x2];
uint32_t total_uncompressed_size; };
uint32_t unknown[0x2];
};
struct DatBlockRecord struct DatBlockRecord
{ {
uint32_t offset; uint32_t offset;
uint32_t size; uint32_t size;
uint32_t unknown[0x4]; uint32_t unknown[0x4];
SqPackBlockHash block_hash; SqPackBlockHash block_hash;
}; };
struct DatBlockHeader struct DatBlockHeader
{ {
uint32_t size; uint32_t size;
uint32_t unknown1; uint32_t unknown1;
uint32_t compressed_size; uint32_t compressed_size;
uint32_t uncompressed_size; uint32_t uncompressed_size;
}; };
struct DatStdFileBlockInfos struct DatStdFileBlockInfos
{ {
uint32_t offset; uint32_t offset;
uint16_t size; uint16_t size;
uint16_t uncompressed_size; uint16_t uncompressed_size;
}; };
struct DatMdlFileBlockInfos struct DatMdlFileBlockInfos
{ {
uint32_t unknown1; uint32_t unknown1;
uint32_t uncompressed_sizes[::model_section_count]; uint32_t uncompressed_sizes[::model_section_count];
uint32_t compressed_sizes[::model_section_count]; uint32_t compressed_sizes[::model_section_count];
uint32_t offsets[::model_section_count]; uint32_t offsets[::model_section_count];
uint16_t block_ids[::model_section_count]; uint16_t block_ids[::model_section_count];
uint16_t block_counts[::model_section_count]; uint16_t block_counts[::model_section_count];
uint32_t unknown2[0x2]; uint32_t unknown2[0x2];
}; };
struct DatTexFileBlockInfos struct DatTexFileBlockInfos
{ {
uint32_t offset; uint32_t offset;
uint32_t size; uint32_t size;
uint32_t uncompressed_size; uint32_t uncompressed_size;
uint32_t block_id; uint32_t block_id;
uint32_t block_count; uint32_t block_count;
}; };
}
} }
namespace xiv namespace xiv::utils::bparse
{ {
namespace utils template<>
{ inline void reorder< xiv::dat::DatFileHeader >( xiv::dat::DatFileHeader& i_struct )
namespace bparse {
{ xiv::utils::bparse::reorder( i_struct.size );
template <> xiv::utils::bparse::reorder( i_struct.entry_type );
inline void reorder<xiv::dat::DatFileHeader>(xiv::dat::DatFileHeader& i_struct) xiv::utils::bparse::reorder( i_struct.total_uncompressed_size );
{ for( int32_t i = 0; i < 0x2; ++i )
xiv::utils::bparse::reorder(i_struct.size); {
xiv::utils::bparse::reorder(i_struct.entry_type); xiv::utils::bparse::reorder( i_struct.unknown[ i ] );
xiv::utils::bparse::reorder(i_struct.total_uncompressed_size); }
for (int32_t i = 0; i < 0x2; ++i) { xiv::utils::bparse::reorder(i_struct.unknown[i]); } }
}
template <> template<>
inline void reorder<xiv::dat::DatBlockRecord>(xiv::dat::DatBlockRecord& i_struct) inline void reorder< xiv::dat::DatBlockRecord >( xiv::dat::DatBlockRecord& i_struct )
{ {
xiv::utils::bparse::reorder(i_struct.offset); xiv::utils::bparse::reorder( i_struct.offset );
xiv::utils::bparse::reorder(i_struct.size); xiv::utils::bparse::reorder( i_struct.size );
for (int32_t i = 0; i < 0x4; ++i) { xiv::utils::bparse::reorder(i_struct.unknown[i]); } for( int32_t i = 0; i < 0x4; ++i )
xiv::utils::bparse::reorder(i_struct.block_hash); {
} xiv::utils::bparse::reorder( i_struct.unknown[ i ] );
}
xiv::utils::bparse::reorder( i_struct.block_hash );
}
template <> template<>
inline void reorder<xiv::dat::DatBlockHeader>(xiv::dat::DatBlockHeader& i_struct) inline void reorder< xiv::dat::DatBlockHeader >( xiv::dat::DatBlockHeader& i_struct )
{ {
xiv::utils::bparse::reorder(i_struct.size); xiv::utils::bparse::reorder( i_struct.size );
xiv::utils::bparse::reorder(i_struct.unknown1); xiv::utils::bparse::reorder( i_struct.unknown1 );
xiv::utils::bparse::reorder(i_struct.compressed_size); xiv::utils::bparse::reorder( i_struct.compressed_size );
xiv::utils::bparse::reorder(i_struct.uncompressed_size); xiv::utils::bparse::reorder( i_struct.uncompressed_size );
} }
template <> template<>
inline void reorder<xiv::dat::DatStdFileBlockInfos>(xiv::dat::DatStdFileBlockInfos& i_struct) inline void reorder< xiv::dat::DatStdFileBlockInfos >( xiv::dat::DatStdFileBlockInfos& i_struct )
{ {
xiv::utils::bparse::reorder(i_struct.offset); xiv::utils::bparse::reorder( i_struct.offset );
xiv::utils::bparse::reorder(i_struct.size); xiv::utils::bparse::reorder( i_struct.size );
xiv::utils::bparse::reorder(i_struct.uncompressed_size); xiv::utils::bparse::reorder( i_struct.uncompressed_size );
} }
template <> template<>
inline void reorder<xiv::dat::DatMdlFileBlockInfos>(xiv::dat::DatMdlFileBlockInfos& i_struct) inline void reorder< xiv::dat::DatMdlFileBlockInfos >( xiv::dat::DatMdlFileBlockInfos& i_struct )
{ {
xiv::utils::bparse::reorder(i_struct.unknown1); xiv::utils::bparse::reorder( i_struct.unknown1 );
for (auto i = 0; i < ::model_section_count; ++i) { xiv::utils::bparse::reorder(i_struct.uncompressed_sizes[i]); } for( auto i = 0; i < ::model_section_count; ++i )
for (auto i = 0; i < ::model_section_count; ++i) { xiv::utils::bparse::reorder(i_struct.compressed_sizes[i]); } {
for (auto i = 0; i < ::model_section_count; ++i) { xiv::utils::bparse::reorder(i_struct.offsets[i]); } xiv::utils::bparse::reorder( i_struct.uncompressed_sizes[ i ] );
for (auto i = 0; i < ::model_section_count; ++i) { xiv::utils::bparse::reorder(i_struct.block_ids[i]); } }
for (auto i = 0; i < ::model_section_count; ++i) { xiv::utils::bparse::reorder(i_struct.block_counts[i]); } for( auto i = 0; i < ::model_section_count; ++i )
for (auto i = 0; i < 0x2; ++i) { xiv::utils::bparse::reorder(i_struct.unknown2[i]); } {
} xiv::utils::bparse::reorder( i_struct.compressed_sizes[ i ] );
}
for( auto i = 0; i < ::model_section_count; ++i )
{
xiv::utils::bparse::reorder( i_struct.offsets[ i ] );
}
for( auto i = 0; i < ::model_section_count; ++i )
{
xiv::utils::bparse::reorder( i_struct.block_ids[ i ] );
}
for( auto i = 0; i < ::model_section_count; ++i )
{
xiv::utils::bparse::reorder( i_struct.block_counts[ i ] );
}
for( auto i = 0; i < 0x2; ++i )
{
xiv::utils::bparse::reorder( i_struct.unknown2[ i ] );
}
}
template <> template<>
inline void reorder<xiv::dat::DatTexFileBlockInfos>(xiv::dat::DatTexFileBlockInfos& i_struct) inline void reorder< xiv::dat::DatTexFileBlockInfos >( xiv::dat::DatTexFileBlockInfos& i_struct )
{ {
xiv::utils::bparse::reorder(i_struct.offset); xiv::utils::bparse::reorder( i_struct.offset );
xiv::utils::bparse::reorder(i_struct.size); xiv::utils::bparse::reorder( i_struct.size );
xiv::utils::bparse::reorder(i_struct.uncompressed_size); xiv::utils::bparse::reorder( i_struct.uncompressed_size );
xiv::utils::bparse::reorder(i_struct.block_id); xiv::utils::bparse::reorder( i_struct.block_id );
xiv::utils::bparse::reorder(i_struct.block_count); xiv::utils::bparse::reorder( i_struct.block_count );
} }
} }
}
};
using xiv::utils::bparse::extract; using xiv::utils::bparse::extract;
namespace xiv namespace xiv::dat
{
namespace dat
{ {
Dat::Dat( const std::filesystem::path& i_path, uint32_t i_nb ) : Dat::Dat( const std::filesystem::path& i_path, uint32_t i_nb ) :
SqPack( i_path ), SqPack( i_path ),
m_num( i_nb ) m_num( i_nb )
{ {
auto block_record = extract<DatBlockRecord>(m_handle); auto block_record = extract< DatBlockRecord >( m_handle );
block_record.offset *= 0x80; block_record.offset *= 0x80;
isBlockValid(block_record.offset, block_record.size, block_record.block_hash); isBlockValid( block_record.offset, block_record.size, block_record.block_hash );
} }
Dat::~Dat() Dat::~Dat()
{ {
} }
std::unique_ptr<File> Dat::getFile( uint32_t i_offset ) std::unique_ptr< File > Dat::getFile( uint32_t i_offset )
{ {
std::unique_ptr<File> outputFile(new File()); std::unique_ptr< File > outputFile( new File() );
{ {
// Lock in this scope // Lock in this scope
std::lock_guard<std::mutex> lock(m_fileMutex); std::lock_guard< std::mutex > lock( m_fileMutex );
// Seek to the start of the header of the file record and extract it // Seek to the start of the header of the file record and extract it
m_handle.seekg(i_offset); m_handle.seekg( i_offset );
auto file_header = extract<DatFileHeader>(m_handle); auto file_header = extract< DatFileHeader >( m_handle );
switch (file_header.entry_type) switch( file_header.entry_type )
{ {
case FileType::empty: case FileType::empty:
throw std::runtime_error("File is empty"); throw std::runtime_error( "File is empty" );
case FileType::standard: case FileType::standard:
{ {
outputFile->_type = FileType::standard; outputFile->_type = FileType::standard;
uint32_t number_of_blocks = extract<uint32_t>(m_handle, "number_of_blocks"); uint32_t number_of_blocks = extract< uint32_t >( m_handle, "number_of_blocks" );
// Just extract offset infos for the blocks to extract // Just extract offset infos for the blocks to extract
std::vector<DatStdFileBlockInfos> std_file_block_infos; std::vector< DatStdFileBlockInfos > std_file_block_infos;
extract<DatStdFileBlockInfos>( m_handle, number_of_blocks, std_file_block_infos ); extract< DatStdFileBlockInfos >( m_handle, number_of_blocks, std_file_block_infos );
// Pre allocate data vector for the whole file // Pre allocate data vector for the whole file
outputFile->_data_sections.resize(1); outputFile->_data_sections.resize( 1 );
auto& data_section = outputFile->_data_sections.front(); auto& data_section = outputFile->_data_sections.front();
data_section.reserve(file_header.total_uncompressed_size); data_section.reserve( file_header.total_uncompressed_size );
// Extract each block // Extract each block
for (auto& file_block_info : std_file_block_infos) for( auto& file_block_info : std_file_block_infos )
{
extractBlock( i_offset + file_header.size + file_block_info.offset, data_section );
}
}
break;
case FileType::model:
{
outputFile->_type = FileType::model;
DatMdlFileBlockInfos mdlBlockInfo = extract< DatMdlFileBlockInfos >( m_handle );
// Getting the block number and read their sizes
const uint32_t block_count = mdlBlockInfo.block_ids[ ::model_section_count - 1 ] +
mdlBlockInfo.block_counts[ ::model_section_count - 1 ];
std::vector< uint16_t > block_sizes;
extract< uint16_t >( m_handle, "block_size", block_count, block_sizes );
// Preallocate sufficient space
outputFile->_data_sections.resize( ::model_section_count );
for( uint32_t i = 0; i < ::model_section_count; ++i )
{
// Preallocating for section
auto& data_section = outputFile->_data_sections[ i ];
data_section.reserve( mdlBlockInfo.uncompressed_sizes[ i ] );
uint32_t current_offset = i_offset + file_header.size + mdlBlockInfo.offsets[ i ];
for( uint32_t j = 0; j < mdlBlockInfo.block_counts[ i ]; ++j )
{ {
extractBlock(i_offset + file_header.size + file_block_info.offset, data_section); extractBlock( current_offset, data_section );
current_offset += block_sizes[ mdlBlockInfo.block_ids[ i ] + j ];
} }
} }
break; }
break;
case FileType::model: case FileType::texture:
{ {
outputFile->_type = FileType::model; outputFile->_type = FileType::texture;
DatMdlFileBlockInfos mdlBlockInfo = extract<DatMdlFileBlockInfos>(m_handle); // Extracts mipmap entries and the block sizes
uint32_t sectionCount = extract< uint32_t >( m_handle, "sections_count" );
// Getting the block number and read their sizes std::vector< DatTexFileBlockInfos > texBlockInfo;
const uint32_t block_count = mdlBlockInfo.block_ids[::model_section_count - 1] + extract< DatTexFileBlockInfos >( m_handle, sectionCount, texBlockInfo );
mdlBlockInfo.block_counts[::model_section_count - 1];
std::vector<uint16_t> block_sizes;
extract<uint16_t>(m_handle, "block_size", block_count, block_sizes);
// Preallocate sufficient space // Extracting block sizes
outputFile->_data_sections.resize(::model_section_count); uint32_t block_count = texBlockInfo.back().block_id + texBlockInfo.back().block_count;
std::vector< uint16_t > block_sizes;
extract< uint16_t >( m_handle, "block_size", block_count, block_sizes );
for (uint32_t i = 0; i < ::model_section_count; ++i) outputFile->_data_sections.resize( sectionCount + 1 );
// Extracting header in section 0
const uint32_t header_size = texBlockInfo.front().offset;
auto& header_section = outputFile->_data_sections[ 0 ];
header_section.resize( header_size );
m_handle.seekg( i_offset + file_header.size );
m_handle.read( header_section.data(), header_size );
// Extracting other sections
for( uint32_t i = 0; i < sectionCount; ++i )
{
auto& data_section = outputFile->_data_sections[ i + 1 ];
auto& section_infos = texBlockInfo[ i ];
data_section.reserve( section_infos.uncompressed_size );
uint32_t current_offset = i_offset + file_header.size + section_infos.offset;
for( uint32_t j = 0; j < section_infos.block_count; ++j )
{ {
// Preallocating for section extractBlock( current_offset, data_section );
auto& data_section = outputFile->_data_sections[i]; current_offset += block_sizes[ section_infos.block_id + j ];
data_section.reserve(mdlBlockInfo.uncompressed_sizes[i]);
uint32_t current_offset = i_offset + file_header.size + mdlBlockInfo.offsets[i];
for (uint32_t j = 0; j < mdlBlockInfo.block_counts[i]; ++j)
{
extractBlock(current_offset, data_section);
current_offset += block_sizes[mdlBlockInfo.block_ids[i] + j];
}
} }
} }
break; }
break;
case FileType::texture: default:
{ throw std::runtime_error(
outputFile->_type = FileType::texture; "Invalid entry_type: " + std::to_string( static_cast<uint32_t>(file_header.entry_type) ) );
// Extracts mipmap entries and the block sizes
uint32_t sectionCount = extract<uint32_t>(m_handle, "sections_count");
std::vector<DatTexFileBlockInfos> texBlockInfo;
extract<DatTexFileBlockInfos>(m_handle, sectionCount, texBlockInfo);
// Extracting block sizes
uint32_t block_count = texBlockInfo.back().block_id + texBlockInfo.back().block_count;
std::vector<uint16_t> block_sizes;
extract<uint16_t>(m_handle, "block_size", block_count, block_sizes);
outputFile->_data_sections.resize(sectionCount + 1);
// Extracting header in section 0
const uint32_t header_size = texBlockInfo.front().offset;
auto& header_section = outputFile->_data_sections[0];
header_section.resize(header_size);
m_handle.seekg(i_offset + file_header.size);
m_handle.read(header_section.data(), header_size);
// Extracting other sections
for (uint32_t i = 0; i < sectionCount; ++i)
{
auto& data_section = outputFile->_data_sections[i + 1];
auto& section_infos = texBlockInfo[i];
data_section.reserve(section_infos.uncompressed_size);
uint32_t current_offset = i_offset + file_header.size + section_infos.offset;
for (uint32_t j = 0; j < section_infos.block_count; ++j)
{
extractBlock(current_offset, data_section);
current_offset += block_sizes[section_infos.block_id + j];
}
}
}
break;
default:
throw std::runtime_error("Invalid entry_type: " + std::to_string(static_cast<uint32_t>(file_header.entry_type)));
} }
} }
return outputFile; return outputFile;
} }
void Dat::extractBlock( uint32_t i_offset, std::vector<char>& o_data ) void Dat::extractBlock( uint32_t i_offset, std::vector< char >& o_data )
{ {
m_handle.seekg(i_offset); m_handle.seekg( i_offset );
DatBlockHeader block_header = extract<DatBlockHeader>(m_handle); DatBlockHeader block_header = extract< DatBlockHeader >( m_handle );
// Resizing the vector to write directly into it // Resizing the vector to write directly into it
const uint32_t data_size = o_data.size(); const uint32_t data_size = o_data.size();
o_data.resize(data_size + block_header.uncompressed_size); o_data.resize( data_size + block_header.uncompressed_size );
// 32000 in compressed_size means it is not compressed so take uncompressed_size // 32000 in compressed_size means it is not compressed so take uncompressed_size
if (block_header.compressed_size == 32000) if( block_header.compressed_size == 32000 )
{ {
m_handle.read(o_data.data() + data_size, block_header.uncompressed_size); m_handle.read( o_data.data() + data_size, block_header.uncompressed_size );
} }
else else
{ {
// If it is compressed use zlib // If it is compressed use zlib
// Read the data to be decompressed // Read the data to be decompressed
std::vector<char> temp_buffer(block_header.compressed_size); std::vector< char > temp_buffer( block_header.compressed_size );
m_handle.read(temp_buffer.data(), block_header.compressed_size); m_handle.read( temp_buffer.data(), block_header.compressed_size );
utils::zlib::no_header_decompress(reinterpret_cast<uint8_t*>(temp_buffer.data()), utils::zlib::no_header_decompress( reinterpret_cast<uint8_t*>(temp_buffer.data()),
temp_buffer.size(), temp_buffer.size(),
reinterpret_cast<uint8_t*>(o_data.data() + data_size), reinterpret_cast<uint8_t*>(o_data.data() + data_size),
block_header.uncompressed_size); block_header.uncompressed_size );
} }
} }
uint32_t Dat::getNum() const uint32_t Dat::getNum() const
{ {
return m_num; return m_num;
} }
} }
}

45
deps/datReader/Dat.h vendored
View file

@ -7,39 +7,36 @@
#include <filesystem> #include <filesystem>
namespace xiv namespace xiv::dat
{
namespace dat
{ {
class File; class File;
class Dat : public SqPack class Dat : public SqPack
{ {
public: public:
// Full path to the dat file // Full path to the dat file
Dat( const std::filesystem::path& i_path, uint32_t i_nb ); Dat( const std::filesystem::path& i_path, uint32_t i_nb );
virtual ~Dat(); virtual ~Dat();
// Retrieves a file given the offset in the dat file // Retrieves a file given the offset in the dat file
std::unique_ptr<File> getFile( uint32_t i_offset ); std::unique_ptr<File> getFile( uint32_t i_offset );
// Appends to the vector the data of this block, it is assumed to be preallocated // Appends to the vector the data of this block, it is assumed to be preallocated
// Is it also assumed that the m_fileMutex is currently locked by this thread before the call // Is it also assumed that the m_fileMutex is currently locked by this thread before the call
void extractBlock( uint32_t i_offset, std::vector<char>& o_data ); void extractBlock( uint32_t i_offset, std::vector<char>& o_data );
// Returns the dat number // Returns the dat number
uint32_t getNum() const; uint32_t getNum() const;
protected: protected:
// File reading mutex to have only one thread reading the file at a time // File reading mutex to have only one thread reading the file at a time
std::mutex m_fileMutex; std::mutex m_fileMutex;
// Dat nb // Dat nb
uint32_t m_num; uint32_t m_num;
}; };
}
} }
#endif // XIV_DAT_DAT_H #endif // XIV_DAT_DAT_H

6
deps/datReader/DatCat.cpp vendored
View file

@ -26,7 +26,7 @@ Cat::Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::str
// For all dat files linked to this index, create it: XX0000.win32.datX // For all dat files linked to this index, create it: XX0000.win32.datX
for( uint32_t i = 0; i < getIndex().getDatCount(); ++i ) for( uint32_t i = 0; i < getIndex().getDatCount(); ++i )
{ {
m_dats.emplace_back( std::unique_ptr<Dat>( new Dat( basePath / "ffxiv" / ( prefix + ".dat" + std::to_string( i ) ), i ) ) ); m_dats.emplace_back( std::make_unique< Dat >(basePath / "ffxiv" / ( prefix + ".dat" + std::to_string( i ) ), i ) );
} }
} }
@ -36,12 +36,12 @@ Cat::Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::str
m_chunk( chunk ) m_chunk( chunk )
{ {
// Creates the index: XX0000.win32.index // Creates the index: XX0000.win32.index
m_index = std::unique_ptr<Index>( new Index( basePath / GameData::buildDatStr( "ex" + std::to_string( exNum ), catNum, exNum, chunk, "win32", "index" ) ) ); m_index = std::make_unique< Index >( basePath / GameData::buildDatStr( "ex" + std::to_string( exNum ), catNum, exNum, chunk, "win32", "index" ) );
// For all dat files linked to this index, create it: XX0000.win32.datX // For all dat files linked to this index, create it: XX0000.win32.datX
for( uint32_t i = 0; i < getIndex().getDatCount(); ++i ) for( uint32_t i = 0; i < getIndex().getDatCount(); ++i )
{ {
m_dats.emplace_back( std::unique_ptr<Dat>( new Dat( basePath / GameData::buildDatStr( "ex" + std::to_string( exNum ), catNum, exNum, chunk, "win32", "dat" + std::to_string( i ) ), i ) ) ); m_dats.emplace_back( std::make_unique< Dat >( basePath / GameData::buildDatStr( "ex" + std::to_string( exNum ), catNum, exNum, chunk, "win32", "dat" + std::to_string( i ) ), i ) );
} }
} }

86
deps/datReader/DatCat.h vendored
View file

@ -5,60 +5,64 @@
#include <vector> #include <vector>
#include <filesystem> #include <filesystem>
namespace xiv { namespace xiv::dat
namespace dat {
class Index;
class Dat;
class File;
// A category represents an .index and its associated .datX
class Cat
{ {
public:
// basePath: Path to the folder containingthe datfiles
// catNum: The number of the category
// name: The name of the category, empty if not known
Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::string& name );
// basePath: Path to the folder containingthe datfiles class Index;
// catNum: The number of the category
// name: The name of the category, empty if not known
// exNum: The number of the expansion to load from
// chunk: The chunk to load from
Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::string& name, uint32_t exNum, uint32_t chunk );
~Cat();
// Returns .index of the category class Dat;
const Index& getIndex() const;
// Retrieve a file from the category given its hashes class File;
std::unique_ptr<File> getFile( uint32_t dir_hash, uint32_t filename_hash ) const;
// A category represents an .index and its associated .datX
class Cat
{
public:
// basePath: Path to the folder containingthe datfiles
// catNum: The number of the category
// name: The name of the category, empty if not known
Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::string& name );
// basePath: Path to the folder containingthe datfiles
// catNum: The number of the category
// name: The name of the category, empty if not known
// exNum: The number of the expansion to load from
// chunk: The chunk to load from
Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::string& name, uint32_t exNum,
uint32_t chunk );
~Cat();
// Returns .index of the category
const Index& getIndex() const;
// Retrieve a file from the category given its hashes
std::unique_ptr< File > getFile( uint32_t dir_hash, uint32_t filename_hash ) const;
bool doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const; bool doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const;
bool doesDirExist( uint32_t dir_hash ) const;
bool doesDirExist( uint32_t dir_hash ) const;
// Returns thename of the category // Returns thename of the category
const std::string& getName() const; const std::string& getName() const;
// Returns the number of the category // Returns the number of the category
uint32_t getCatNum() const; uint32_t getCatNum() const;
protected: protected:
const std::string m_name; const std::string m_name;
const uint32_t m_catNum; const uint32_t m_catNum;
const uint32_t m_chunk; const uint32_t m_chunk;
// The .index // The .index
std::unique_ptr<Index> m_index; std::unique_ptr< Index > m_index;
// The .datXs such as dat nb X => m_dats[X] // The .datXs such as dat nb X => m_dats[X]
std::vector<std::unique_ptr<Dat>> m_dats; std::vector< std::unique_ptr< Dat>> m_dats;
}; };
}
} }
#endif // XIV_DAT_CAT_H #endif // XIV_DAT_CAT_H

655
deps/datReader/Exd.cpp vendored
View file

@ -8,373 +8,378 @@
using xiv::utils::bparse::extract; using xiv::utils::bparse::extract;
namespace xiv namespace xiv::exd
{ {
namespace exd struct ExdHeader
{ {
char magic[0x4];
uint16_t unknown;
uint16_t unknown2;
uint32_t index_size;
};
struct ExdHeader struct ExdRecordIndex
{ {
char magic[0x4]; uint32_t id;
uint16_t unknown; uint32_t offset;
uint16_t unknown2; };
uint32_t index_size;
};
struct ExdRecordIndex
{
uint32_t id;
uint32_t offset;
};
}
} }
namespace xiv namespace xiv::utils::bparse {
{ template<>
namespace utils inline void reorder< xiv::exd::ExdHeader >( xiv::exd::ExdHeader& i_struct )
{ {
namespace bparse for( int32_t i = 0; i < 0x4; ++i )
{ {
template <> inline void reorder<xiv::exd::ExdHeader>( xiv::exd::ExdHeader& i_struct ) { for( int32_t i = 0; i < 0x4; ++i ) { xiv::utils::bparse::reorder( i_struct.magic[i] ); } i_struct.unknown = xiv::utils::bparse::byteswap( i_struct.unknown ); xiv::utils::bparse::reorder( i_struct.unknown ); i_struct.unknown2 = xiv::utils::bparse::byteswap( i_struct.unknown2 ); xiv::utils::bparse::reorder( i_struct.unknown2 ); i_struct.index_size = xiv::utils::bparse::byteswap( i_struct.index_size ); xiv::utils::bparse::reorder( i_struct.index_size ); } xiv::utils::bparse::reorder( i_struct.magic[ i ] );
template <> inline void reorder<xiv::exd::ExdRecordIndex>( xiv::exd::ExdRecordIndex& i_struct ) { i_struct.id = xiv::utils::bparse::byteswap( i_struct.id ); xiv::utils::bparse::reorder( i_struct.id ); i_struct.offset = xiv::utils::bparse::byteswap( i_struct.offset ); xiv::utils::bparse::reorder( i_struct.offset ); } }
} i_struct.unknown = xiv::utils::bparse::byteswap( i_struct.unknown );
} xiv::utils::bparse::reorder( i_struct.unknown );
i_struct.unknown2 = xiv::utils::bparse::byteswap( i_struct.unknown2 );
xiv::utils::bparse::reorder( i_struct.unknown2 );
i_struct.index_size = xiv::utils::bparse::byteswap( i_struct.index_size );
xiv::utils::bparse::reorder( i_struct.index_size );
}
template<>
inline void reorder< xiv::exd::ExdRecordIndex >( xiv::exd::ExdRecordIndex& i_struct )
{
i_struct.id = xiv::utils::bparse::byteswap( i_struct.id );
xiv::utils::bparse::reorder( i_struct.id );
i_struct.offset = xiv::utils::bparse::byteswap( i_struct.offset );
xiv::utils::bparse::reorder( i_struct.offset );
}
}; };
namespace xiv namespace xiv::exd
{ {
namespace exd Exd::Exd( std::shared_ptr< Exh > i_exh, const std::vector< std::shared_ptr< dat::File>>& i_files )
{ {
_exh = i_exh;
_files = i_files;
Exd::Exd( std::shared_ptr<Exh> i_exh, const std::vector<std::shared_ptr<dat::File>>& i_files )
// Iterates over all the files
const uint32_t member_count = _exh->get_members().size();
for( auto& file_ptr : _files )
{
// Get a stream
std::vector< char > dataCpy = file_ptr->get_data_sections().front();
std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
// Extract the header and skip to the record indices
auto exd_header = extract< ExdHeader >( iss );
iss.seekg( 0x20 );
// Preallocate and extract the record_indices
const uint32_t record_count = exd_header.index_size / sizeof( ExdRecordIndex );
std::vector< ExdRecordIndex > record_indices;
record_indices.reserve( record_count );
for( uint32_t i = 0; i < record_count; ++i )
{ {
_exh = i_exh; auto recordIndex = extract< ExdRecordIndex >( iss );
_files = i_files; _idCache[ recordIndex.id ] = ExdCacheEntry{ file_ptr, recordIndex.offset };
}
}
}
Exd::~Exd()
{
}
const std::vector< Field > Exd::get_row( uint32_t id, uint32_t subRow )
{
auto cacheEntryIt = _idCache.find( id );
if( cacheEntryIt == _idCache.end() )
throw std::runtime_error( "Id not found: " + std::to_string( id ) );
// Iterates over all the files
const uint32_t member_count = _exh->get_members().size();
auto& file_ptr = cacheEntryIt->second.file;
std::vector< char > dataCpy = file_ptr->get_data_sections().front();
std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
// Get the vector fields for the given record and preallocate it
auto fields = _data[ id ];
fields.reserve( member_count );
iss.seekg( cacheEntryIt->second.offset + 6 );
uint8_t subRows = *reinterpret_cast< uint8_t* >( &dataCpy[ cacheEntryIt->second.offset + 5 ] );
if( subRow >= subRows )
throw std::runtime_error( "Out of bounds sub-row!" );
int offset = cacheEntryIt->second.offset + 6 + ( subRow * _exh->get_header().data_offset + 2 * ( subRow + 1 ) );
for( auto& member_entry : _exh->get_exh_members() )
{
// Seek to the position of the member to extract.
// 6 is because we have uint32_t/uint16_t at the start of each record
iss.seekg( offset + member_entry.offset );
// Switch depending on the type to extract
switch( member_entry.type )
{
case DataType::string:
// Extract the offset to the actual string
// Seek to it then extract the actual string
{
throw std::runtime_error( "String not implemented for variant 2!" );
//auto string_offset = extract<uint32_t>( iss, "string_offset", false );
//iss.seekg( cacheEntryIt->second.offset + 6 + _exh->get_header().data_offset + string_offset );
//fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
}
break;
case DataType::boolean:
fields.emplace_back( extract< bool >( iss, "bool" ) );
break;
case DataType::int8:
fields.emplace_back( extract< int8_t >( iss, "int8_t" ) );
break;
case DataType::uint8:
fields.emplace_back( extract< uint8_t >( iss, "uint8_t" ) );
break;
case DataType::int16:
fields.emplace_back( extract< int16_t >( iss, "int16_t", false ) );
break;
case DataType::uint16:
fields.emplace_back( extract< uint16_t >( iss, "uint16_t", false ) );
break;
case DataType::int32:
fields.emplace_back( extract< int32_t >( iss, "int32_t", false ) );
break;
case DataType::uint32:
fields.emplace_back( extract< uint32_t >( iss, "uint32_t", false ) );
break;
case DataType::float32:
fields.emplace_back( extract< float >( iss, "float", false ) );
break;
case DataType::uint64:
fields.emplace_back( extract< uint64_t >( iss, "uint64_t", false ) );
break;
default:
auto type = static_cast< uint16_t >( member_entry.type );
if( type < 0x19 || type > 0x20 )
throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
uint64_t val = extract< uint64_t >( iss, "bool" );
int32_t shift = type - 0x19;
int32_t i = 1 << shift;
val &= i;
fields.emplace_back( ( val & i ) == i );
break;
}
}
return fields;
}
// Iterates over all the files const std::vector< Field > Exd::get_row( uint32_t id )
const uint32_t member_count = _exh->get_members().size(); {
for ( auto &file_ptr : _files )
{
// Get a stream
std::vector< char > dataCpy = file_ptr->get_data_sections().front();
std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
// Extract the header and skip to the record indices auto cacheEntryIt = _idCache.find( id );
auto exd_header = extract< ExdHeader >( iss ); if( cacheEntryIt == _idCache.end() )
iss.seekg( 0x20 ); throw std::runtime_error( "Id not found: " + std::to_string( id ) );
// Preallocate and extract the record_indices // Iterates over all the files
const uint32_t record_count = exd_header.index_size / sizeof( ExdRecordIndex ); const uint32_t member_count = _exh->get_members().size();
std::vector< ExdRecordIndex > record_indices; auto& file_ptr = cacheEntryIt->second.file;
record_indices.reserve( record_count );
for ( uint32_t i = 0; i < record_count; ++i ) std::vector< char > dataCpy = file_ptr->get_data_sections().front();
{ std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
auto recordIndex = extract< ExdRecordIndex >( iss );
_idCache[recordIndex.id] = ExdCacheEntry{file_ptr, recordIndex.offset}; // Get the vector fields for the given record and preallocate it
} auto fields = _data[ id ];
} fields.reserve( member_count );
iss.seekg( cacheEntryIt->second.offset + 6 );
uint8_t subRows = *reinterpret_cast< uint8_t* >( &dataCpy[ cacheEntryIt->second.offset + 5 ] );
for( auto& member_entry : _exh->get_exh_members() )
{
// Seek to the position of the member to extract.
// 6 is because we have uint32_t/uint16_t at the start of each record
iss.seekg( cacheEntryIt->second.offset + 6 + member_entry.offset );
// Switch depending on the type to extract
switch( member_entry.type )
{
case DataType::string:
// Extract the offset to the actual string
// Seek to it then extract the actual string
{
auto string_offset = extract< uint32_t >( iss, "string_offset", false );
iss.seekg( cacheEntryIt->second.offset + 6 + _exh->get_header().data_offset + string_offset );
fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
}
break;
case DataType::boolean:
fields.emplace_back( extract< bool >( iss, "bool" ) );
break;
case DataType::int8:
fields.emplace_back( extract< int8_t >( iss, "int8_t" ) );
break;
case DataType::uint8:
fields.emplace_back( extract< uint8_t >( iss, "uint8_t" ) );
break;
case DataType::int16:
fields.emplace_back( extract< int16_t >( iss, "int16_t", false ) );
break;
case DataType::uint16:
fields.emplace_back( extract< uint16_t >( iss, "uint16_t", false ) );
break;
case DataType::int32:
fields.emplace_back( extract< int32_t >( iss, "int32_t", false ) );
break;
case DataType::uint32:
fields.emplace_back( extract< uint32_t >( iss, "uint32_t", false ) );
break;
case DataType::float32:
fields.emplace_back( extract< float >( iss, "float", false ) );
break;
case DataType::uint64:
fields.emplace_back( extract< uint64_t >( iss, "uint64_t", false ) );
break;
default:
auto type = static_cast< uint16_t >( member_entry.type );
if( type < 0x19 || type > 0x20 )
throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
uint64_t val = extract< uint64_t >( iss, "bool" );
int32_t shift = type - 0x19;
int32_t i = 1 << shift;
val &= i;
fields.emplace_back( ( val & i ) == i );
break;
}
}
return fields;
}
// Get all rows
const std::map< uint32_t, std::vector< Field>>& Exd::get_rows()
{
// Iterates over all the files
const uint32_t member_count = _exh->get_members().size();
for( auto& file_ptr : _files )
{
// Get a stream
std::vector< char > dataCpy = file_ptr->get_data_sections().front();
std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
// Extract the header and skip to the record indices
auto exd_header = extract< ExdHeader >( iss );
iss.seekg( 0x20 );
// Preallocate and extract the record_indices
const uint32_t record_count = exd_header.index_size / sizeof( ExdRecordIndex );
std::vector< ExdRecordIndex > record_indices;
record_indices.reserve( record_count );
for( uint32_t i = 0; i < record_count; ++i )
{
record_indices.emplace_back( extract< ExdRecordIndex >( iss ) );
} }
Exd::~Exd() for( auto& record_index : record_indices )
{ {
} // Get the vector fields for the given record and preallocate it
auto& fields = _data[ record_index.id ];
fields.reserve( member_count );
const std::vector<Field> Exd::get_row( uint32_t id, uint32_t subRow ) for( auto& member_entry : _exh->get_exh_members() )
{ //for( auto& member_entry : _exh->get_members() )
{
// Seek to the position of the member to extract.
// 6 is because we have uint32_t/uint16_t at the start of each record
iss.seekg( record_index.offset + 6 + member_entry.offset );
auto cacheEntryIt = _idCache.find( id ); // Switch depending on the type to extract
if( cacheEntryIt == _idCache.end() ) switch( member_entry.type )
throw std::runtime_error( "Id not found: " + std::to_string( id ) ); {
// Iterates over all the files
const uint32_t member_count = _exh->get_members().size();
auto& file_ptr = cacheEntryIt->second.file;
std::vector< char > dataCpy = file_ptr->get_data_sections().front();
std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
// Get the vector fields for the given record and preallocate it
auto fields = _data[id];
fields.reserve( member_count );
iss.seekg( cacheEntryIt->second.offset + 6 );
uint8_t subRows = *reinterpret_cast< uint8_t* >( &dataCpy[ cacheEntryIt->second.offset + 5 ] );
if( subRow >= subRows )
throw std::runtime_error( "Out of bounds sub-row!" );
int offset = cacheEntryIt->second.offset + 6 + ( subRow * _exh->get_header().data_offset + 2 * ( subRow + 1 ) );
for( auto& member_entry : _exh->get_exh_members() )
{
// Seek to the position of the member to extract.
// 6 is because we have uint32_t/uint16_t at the start of each record
iss.seekg( offset + member_entry.offset );
// Switch depending on the type to extract
switch( member_entry.type )
{
case DataType::string: case DataType::string:
// Extract the offset to the actual string // Extract the offset to the actual string
// Seek to it then extract the actual string // Seek to it then extract the actual string
{ {
throw std::runtime_error( "String not implemented for variant 2!" ); auto string_offset = extract< uint32_t >( iss, "string_offset", false );
//auto string_offset = extract<uint32_t>( iss, "string_offset", false ); iss.seekg( record_index.offset + 6 + _exh->get_header().data_offset + string_offset );
//iss.seekg( cacheEntryIt->second.offset + 6 + _exh->get_header().data_offset + string_offset ); fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
//fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
} }
break; break;
case DataType::boolean: case DataType::boolean:
fields.emplace_back( extract<bool>( iss, "bool" ) ); fields.emplace_back( extract< bool >( iss, "bool" ) );
break; break;
case DataType::int8: case DataType::int8:
fields.emplace_back( extract<int8_t>( iss, "int8_t" ) ); fields.emplace_back( extract< int8_t >( iss, "int8_t" ) );
break; break;
case DataType::uint8: case DataType::uint8:
fields.emplace_back( extract<uint8_t>( iss, "uint8_t" ) ); fields.emplace_back( extract< uint8_t >( iss, "uint8_t" ) );
break; break;
case DataType::int16: case DataType::int16:
fields.emplace_back( extract<int16_t>( iss, "int16_t", false ) ); fields.emplace_back( extract< int16_t >( iss, "int16_t", false ) );
break; break;
case DataType::uint16: case DataType::uint16:
fields.emplace_back( extract<uint16_t>( iss, "uint16_t", false ) ); fields.emplace_back( extract< uint16_t >( iss, "uint16_t", false ) );
break; break;
case DataType::int32: case DataType::int32:
fields.emplace_back( extract<int32_t>( iss, "int32_t", false ) ); fields.emplace_back( extract< int32_t >( iss, "int32_t", false ) );
break; break;
case DataType::uint32: case DataType::uint32:
fields.emplace_back( extract<uint32_t>( iss, "uint32_t", false ) ); fields.emplace_back( extract< uint32_t >( iss, "uint32_t", false ) );
break; break;
case DataType::float32: case DataType::float32:
fields.emplace_back( extract<float>( iss, "float", false ) ); fields.emplace_back( extract< float >( iss, "float", false ) );
break; break;
case DataType::uint64: case DataType::uint64:
fields.emplace_back( extract<uint64_t>( iss, "uint64_t", false ) ); fields.emplace_back( extract< uint64_t >( iss, "uint64_t", false ) );
break; break;
default: default:
auto type = static_cast< uint16_t >( member_entry.type ); auto type = static_cast< uint16_t >( member_entry.type );
if( type < 0x19 || type > 0x20 ) if( type < 0x19 || type > 0x20 )
throw std::runtime_error("Unknown DataType: " + std::to_string( type )); throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
uint64_t val = extract< uint64_t >( iss, "bool" ); uint64_t val = extract< uint64_t >( iss, "bool" );
int32_t shift = type - 0x19; int32_t shift = type - 0x19;
int32_t i = 1 << shift; int32_t i = 1 << shift;
val &= i; val &= i;
fields.emplace_back( ( val & i ) == i ); fields.emplace_back( ( val & i ) == i );
break; break;
} }
} }
return fields;
} }
}
return _data;
}
const std::vector<Field> Exd::get_row( uint32_t id )
{
auto cacheEntryIt = _idCache.find( id );
if( cacheEntryIt == _idCache.end() )
throw std::runtime_error( "Id not found: " + std::to_string( id ) );
// Iterates over all the files
const uint32_t member_count = _exh->get_members().size();
auto& file_ptr = cacheEntryIt->second.file;
std::vector< char > dataCpy = file_ptr->get_data_sections().front();
std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
// Get the vector fields for the given record and preallocate it
auto fields = _data[id];
fields.reserve( member_count );
iss.seekg( cacheEntryIt->second.offset + 6 );
uint8_t subRows = *reinterpret_cast< uint8_t* >( &dataCpy[ cacheEntryIt->second.offset + 5 ] );
for( auto& member_entry : _exh->get_exh_members() )
{
// Seek to the position of the member to extract.
// 6 is because we have uint32_t/uint16_t at the start of each record
iss.seekg( cacheEntryIt->second.offset + 6 + member_entry.offset );
// Switch depending on the type to extract
switch( member_entry.type )
{
case DataType::string:
// Extract the offset to the actual string
// Seek to it then extract the actual string
{
auto string_offset = extract<uint32_t>( iss, "string_offset", false );
iss.seekg( cacheEntryIt->second.offset + 6 + _exh->get_header().data_offset + string_offset );
fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
}
break;
case DataType::boolean:
fields.emplace_back( extract<bool>( iss, "bool" ) );
break;
case DataType::int8:
fields.emplace_back( extract<int8_t>( iss, "int8_t" ) );
break;
case DataType::uint8:
fields.emplace_back( extract<uint8_t>( iss, "uint8_t" ) );
break;
case DataType::int16:
fields.emplace_back( extract<int16_t>( iss, "int16_t", false ) );
break;
case DataType::uint16:
fields.emplace_back( extract<uint16_t>( iss, "uint16_t", false ) );
break;
case DataType::int32:
fields.emplace_back( extract<int32_t>( iss, "int32_t", false ) );
break;
case DataType::uint32:
fields.emplace_back( extract<uint32_t>( iss, "uint32_t", false ) );
break;
case DataType::float32:
fields.emplace_back( extract<float>( iss, "float", false ) );
break;
case DataType::uint64:
fields.emplace_back( extract<uint64_t>( iss, "uint64_t", false ) );
break;
default:
auto type = static_cast< uint16_t >( member_entry.type );
if( type < 0x19 || type > 0x20 )
throw std::runtime_error("Unknown DataType: " + std::to_string( type ));
uint64_t val = extract< uint64_t >( iss, "bool" );
int32_t shift = type - 0x19;
int32_t i = 1 << shift;
val &= i;
fields.emplace_back( ( val & i ) == i );
break;
}
}
return fields;
}
// Get all rows
const std::map<uint32_t, std::vector<Field>>& Exd::get_rows()
{
// Iterates over all the files
const uint32_t member_count = _exh->get_members().size();
for( auto& file_ptr : _files )
{
// Get a stream
std::vector< char > dataCpy = file_ptr->get_data_sections().front();
std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
// Extract the header and skip to the record indices
auto exd_header = extract<ExdHeader>( iss );
iss.seekg( 0x20 );
// Preallocate and extract the record_indices
const uint32_t record_count = exd_header.index_size / sizeof( ExdRecordIndex );
std::vector<ExdRecordIndex> record_indices;
record_indices.reserve( record_count );
for( uint32_t i = 0; i < record_count; ++i )
{
record_indices.emplace_back( extract<ExdRecordIndex>( iss ) );
}
for( auto& record_index : record_indices )
{
// Get the vector fields for the given record and preallocate it
auto& fields = _data[record_index.id];
fields.reserve( member_count );
for( auto& member_entry : _exh->get_exh_members() )
//for( auto& member_entry : _exh->get_members() )
{
// Seek to the position of the member to extract.
// 6 is because we have uint32_t/uint16_t at the start of each record
iss.seekg( record_index.offset + 6 + member_entry.offset );
// Switch depending on the type to extract
switch( member_entry.type )
{
case DataType::string:
// Extract the offset to the actual string
// Seek to it then extract the actual string
{
auto string_offset = extract<uint32_t>( iss, "string_offset", false );
iss.seekg( record_index.offset + 6 + _exh->get_header().data_offset + string_offset );
fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
}
break;
case DataType::boolean:
fields.emplace_back( extract<bool>( iss, "bool" ) );
break;
case DataType::int8:
fields.emplace_back( extract<int8_t>( iss, "int8_t" ) );
break;
case DataType::uint8:
fields.emplace_back( extract<uint8_t>( iss, "uint8_t" ) );
break;
case DataType::int16:
fields.emplace_back( extract<int16_t>( iss, "int16_t", false ) );
break;
case DataType::uint16:
fields.emplace_back( extract<uint16_t>( iss, "uint16_t", false ) );
break;
case DataType::int32:
fields.emplace_back( extract<int32_t>( iss, "int32_t", false ) );
break;
case DataType::uint32:
fields.emplace_back( extract<uint32_t>( iss, "uint32_t", false ) );
break;
case DataType::float32:
fields.emplace_back( extract<float>( iss, "float", false ) );
break;
case DataType::uint64:
fields.emplace_back( extract<uint64_t>( iss, "uint64_t", false ) );
break;
default:
auto type = static_cast< uint16_t >( member_entry.type );
if( type < 0x19 || type > 0x20 )
throw std::runtime_error("Unknown DataType: " + std::to_string( type ));
uint64_t val = extract< uint64_t >( iss, "bool" );
int32_t shift = type - 0x19;
int32_t i = 1 << shift;
val &= i;
fields.emplace_back( ( val & i ) == i );
break;
}
}
}
}
return _data;
}
}
} }

54
deps/datReader/Exd.h vendored
View file

@ -8,15 +8,13 @@
#include "File.h" #include "File.h"
namespace xiv namespace xiv::exd
{
namespace exd
{ {
class Exh; class Exh;
// Field type containing all the possible types in the data files // Field type containing all the possible types in the data files
using Field = std::variant< using Field = std::variant<
std::string, std::string,
bool, bool,
int8_t, int8_t,
@ -28,39 +26,43 @@ using Field = std::variant<
float, float,
uint64_t >; uint64_t >;
struct ExdCacheEntry struct ExdCacheEntry
{ {
std::shared_ptr<dat::File> file; std::shared_ptr< dat::File > file;
uint32_t offset; uint32_t offset;
}; };
// Data for a given language // Data for a given language
class Exd class Exd
{ {
public: public:
// i_exh: the header // i_exh: the header
// i_files: the multiple exd files // i_files: the multiple exd files
Exd() {} Exd()
Exd(std::shared_ptr<Exh> i_exh, const std::vector<std::shared_ptr<dat::File>>& i_files); {
}
Exd( std::shared_ptr< Exh > i_exh, const std::vector< std::shared_ptr< dat::File>>& i_files );
~Exd(); ~Exd();
// Get a row by its id // Get a row by its id
const std::vector<Field> get_row(uint32_t id); const std::vector< Field > get_row( uint32_t id );
// Get a row by its id and sub-row // Get a row by its id and sub-row
const std::vector<Field> get_row(uint32_t id, uint32_t subRow); const std::vector< Field > get_row( uint32_t id, uint32_t subRow );
// Get all rows // Get all rows
const std::map<uint32_t, std::vector<Field>>& get_rows(); const std::map< uint32_t, std::vector< Field>>& get_rows();
protected: protected:
// Data indexed by the ID of the row, the vector is field with the same order as exh.members // Data indexed by the ID of the row, the vector is field with the same order as exh.members
std::map<uint32_t, std::vector<Field>> _data; std::map< uint32_t, std::vector< Field>> _data;
std::vector<std::shared_ptr<dat::File>> _files; std::vector< std::shared_ptr< dat::File>> _files;
std::shared_ptr<Exh> _exh; std::shared_ptr< Exh > _exh;
std::map< uint32_t, ExdCacheEntry > _idCache; std::map< uint32_t, ExdCacheEntry > _idCache;
}; };
}
} }
#endif // XIV_EXD_EXD_H #endif // XIV_EXD_EXD_H

113
deps/datReader/ExdCat.cpp vendored
View file

@ -9,74 +9,73 @@
namespace namespace
{ {
// Suffix of the filenames given a language // Suffix of the filenames given a language
std::map<xiv::exd::Language, std::string> language_map = std::map<xiv::exd::Language, std::string> language_map =
{{xiv::exd::Language::none, ""}, {
{xiv::exd::Language::ja, "_ja"}, {xiv::exd::Language::none, ""},
{xiv::exd::Language::en, "_en"}, {xiv::exd::Language::ja, "_ja"},
{xiv::exd::Language::de, "_de"}, {xiv::exd::Language::en, "_en"},
{xiv::exd::Language::fr, "_fr"}, {xiv::exd::Language::de, "_de"},
{xiv::exd::Language::chs, "_chs"}}; {xiv::exd::Language::fr, "_fr"},
{xiv::exd::Language::chs, "_chs"}
};
} }
namespace xiv namespace xiv::exd
{ {
namespace exd Cat::Cat( dat::GameData& i_game_data, const std::string& i_name ) :
{ _name( i_name )
{
//XIV_INFO(xiv_exd_logger, "Initializing Cat with name: " << i_name);
// creates the header .exh
{
auto header_file = i_game_data.getFile( "exd/" + i_name + ".exh" );
_header = std::shared_ptr< Exh >( new Exh( *header_file ) );
Cat::Cat(dat::GameData& i_game_data, const std::string& i_name) : }
_name(i_name)
for( auto language: _header->get_languages() )
{
// chs not yet in data files
if( language == Language::en || language == Language::none )
{ {
//XIV_INFO(xiv_exd_logger, "Initializing Cat with name: " << i_name); // Get all the files for a given category/language, in case of multiple range of IDs in separate files (like Quest)
// creates the header .exh std::vector< std::shared_ptr< dat::File>> files;
{ for( auto& exd_def: _header->get_exd_defs() )
auto header_file = i_game_data.getFile("exd/" + i_name + ".exh"); {
_header = std::shared_ptr< Exh >( new Exh( *header_file ) ); files.emplace_back( i_game_data.getFile(
"exd/" + i_name + "_" + std::to_string( exd_def.start_id ) + language_map.at( language ) + ".exd" ) );
} }
// Instantiate the data for this language
for(auto language: _header->get_languages()) _data[ language ] = std::make_unique< Exd >( _header, files );
{
// chs not yet in data files
if (language == Language::en || language == Language::none)
{
// Get all the files for a given category/language, in case of multiple range of IDs in separate files (like Quest)
std::vector<std::shared_ptr<dat::File>> files;
for(auto& exd_def: _header->get_exd_defs())
{
files.emplace_back( i_game_data.getFile("exd/" + i_name + "_" + std::to_string(exd_def.start_id) + language_map.at(language) + ".exd") );
}
// Instantiate the data for this language
_data[language] = std::unique_ptr<Exd>(new Exd(_header, files));
}
}
} }
}
}
Cat::~Cat() Cat::~Cat()
{ {
} }
const std::string& Cat::get_name() const const std::string& Cat::get_name() const
{ {
return _name; return _name;
} }
const Exh& Cat::get_header() const const Exh& Cat::get_header() const
{ {
return *_header; return *_header;
} }
const Exd& Cat::get_data_ln(Language i_language) const const Exd& Cat::get_data_ln( Language i_language ) const
{ {
auto ln_it = _data.find(i_language); auto ln_it = _data.find( i_language );
if (ln_it == _data.end()) if( ln_it == _data.end() )
{ {
throw std::runtime_error("No data for language: " + std::to_string(uint16_t(i_language))); throw std::runtime_error( "No data for language: " + std::to_string( uint16_t( i_language ) ) );
} }
return *(ln_it->second); return *( ln_it->second );
} }
}
} }

126
deps/datReader/ExdData.cpp vendored
View file

@ -7,50 +7,47 @@
#include "ExdCat.h" #include "ExdCat.h"
namespace xiv namespace xiv::exd {
{
namespace exd ExdData::ExdData( dat::GameData& i_game_data ) try :
_game_data( i_game_data )
{ {
//XIV_INFO(xiv_exd_logger, "Initializing ExdData");
ExdData::ExdData(dat::GameData& i_game_data) try : // Fetch the root.exl and get a stream from it
_game_data(i_game_data) auto root_exl = i_game_data.getFile( "exd/root.exl" );
{ std::vector< char > dataCpy = root_exl->get_data_sections().front();
//XIV_INFO(xiv_exd_logger, "Initializing ExdData"); xiv::utils::stream::vectorwrapbuf< char > databuf( dataCpy );
std::istream stream( &databuf );
// Fetch the root.exl and get a stream from it // Iterates over the lines while skipping the first one
auto root_exl = i_game_data.getFile("exd/root.exl"); std::string line;
std::vector< char > dataCpy = root_exl->get_data_sections().front(); std::getline( stream, line ); // extract first line EXLT,2
xiv::utils::stream::vectorwrapbuf<char> databuf(dataCpy); std::getline( stream, line );
std::istream stream(&databuf);
// Iterates over the lines while skipping the first one // Until the EOF
std::string line; while( !line.empty() )
std::getline(stream, line); // extract first line EXLT,2 {
std::getline(stream, line); // Format is cat_name,XX
// XX being an internal identifier
// Get only the cat_name
auto sep = line.find( ',' );
auto category = line.substr( 0, sep );
// Until the EOF // Add to the list of category name
while (!line.empty()) // creates the empty category in the cats map
{ // instantiate the creation mutex for this category
// Format is cat_name,XX _cat_names.push_back( category );
// XX being an internal identifier _cats[ category ] = std::unique_ptr< Cat >();
// Get only the cat_name _cat_creation_mutexes[ category ] = std::make_unique< std::mutex >();
auto sep = line.find(',');
auto category = line.substr(0, sep);
// Add to the list of category name std::getline( stream, line );
// creates the empty category in the cats map }
// instantiate the creation mutex for this category
_cat_names.push_back(category);
_cats[category] = std::unique_ptr<Cat>();
_cat_creation_mutexes[category] = std::unique_ptr<std::mutex>(new std::mutex());
std::getline(stream, line);
}
} }
catch(std::exception& e) catch( std::exception& e )
{ {
// In case of failure here, client is supposed to catch the exception because it is not recoverable on our side // In case of failure here, client is supposed to catch the exception because it is not recoverable on our side
throw std::runtime_error( "ExdData initialization failed: " + std::string( e.what() ) ); throw std::runtime_error( "ExdData initialization failed: " + std::string( e.what() ) );
} }
ExdData::~ExdData() ExdData::~ExdData()
@ -58,43 +55,42 @@ ExdData::~ExdData()
} }
const std::vector<std::string>& ExdData::get_cat_names() const const std::vector< std::string >& ExdData::get_cat_names() const
{ {
return _cat_names; return _cat_names;
} }
const Cat& ExdData::get_category(const std::string& i_cat_name) const Cat& ExdData::get_category( const std::string& i_cat_name )
{ {
// Get the category from its name // Get the category from its name
auto cat_it = _cats.find(i_cat_name); auto cat_it = _cats.find( i_cat_name );
if (cat_it == _cats.end()) if( cat_it == _cats.end() )
{ {
throw std::runtime_error("Category not found: " + i_cat_name); throw std::runtime_error( "Category not found: " + i_cat_name );
} }
if (cat_it->second) if( cat_it->second )
{ {
// If valid return it // If valid return it
return *(cat_it->second); return *( cat_it->second );
} }
else else
{ {
// If not, create it and return it // If not, create it and return it
create_category(i_cat_name); create_category( i_cat_name );
return *(_cats[i_cat_name]); return *( _cats[ i_cat_name ] );
} }
} }
void ExdData::create_category(const std::string& i_cat_name) void ExdData::create_category( const std::string& i_cat_name )
{ {
// Lock mutex in this scope // Lock mutex in this scope
std::lock_guard<std::mutex> lock(*(_cat_creation_mutexes[i_cat_name])); std::lock_guard< std::mutex > lock( *( _cat_creation_mutexes[ i_cat_name ] ) );
// Maybe after unlocking it has already been created, so check (most likely if it blocked) // Maybe after unlocking it has already been created, so check (most likely if it blocked)
if (!_cats[i_cat_name]) if( !_cats[ i_cat_name ] )
{ {
_cats[i_cat_name] = std::unique_ptr<Cat>(new Cat(_game_data, i_cat_name)); _cats[ i_cat_name ] = std::make_unique< Cat >( _game_data, i_cat_name );
} }
} }
} }
}

74
deps/datReader/Exh.cpp vendored
View file

@ -7,71 +7,69 @@
using xiv::utils::bparse::extract; using xiv::utils::bparse::extract;
namespace xiv namespace xiv::exd
{
namespace exd
{
Exh::Exh(const dat::File& i_file)
{ {
Exh::Exh( const dat::File& i_file )
{
// Get a stream from the file // Get a stream from the file
std::vector< char > dataCpy = i_file.get_data_sections().front(); std::vector< char > dataCpy = i_file.get_data_sections().front();
std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) ); std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
// Extract header and skip to member definitions // Extract header and skip to member definitions
_header = extract<ExhHeader>(iss); _header = extract< ExhHeader >( iss );
iss.seekg(0x20); iss.seekg( 0x20 );
// Extract all the members and feed the _members map // Extract all the members and feed the _members map
for (auto i = 0; i < _header.field_count; ++i) for( auto i = 0; i < _header.field_count; ++i )
{ {
auto member = extract<ExhMember>(iss); auto member = extract< ExhMember >( iss );
_members[member.offset] = member; _members[ member.offset ] = member;
_exh_defs.push_back( member ); _exh_defs.push_back( member );
} }
// Extract all the exd_defs // Extract all the exd_defs
_exd_defs.reserve(_header.exd_count); _exd_defs.reserve( _header.exd_count );
for (auto i = 0; i < _header.exd_count; ++i) for( auto i = 0; i < _header.exd_count; ++i )
{ {
_exd_defs.emplace_back(extract<ExhExdDef>(iss)); _exd_defs.emplace_back( extract< ExhExdDef >( iss ) );
} }
// Extract all the languages // Extract all the languages
_languages.reserve(_header.language_count); _languages.reserve( _header.language_count );
for (auto i = 0; i < _header.language_count; ++i) for( auto i = 0; i < _header.language_count; ++i )
{ {
_languages.emplace_back(Language(extract<uint16_t>(iss, "language"))); _languages.emplace_back( Language( extract< uint16_t >( iss, "language" ) ) );
} }
} }
Exh::~Exh() Exh::~Exh()
{ {
} }
const ExhHeader& Exh::get_header() const const ExhHeader& Exh::get_header() const
{ {
return _header; return _header;
} }
const std::vector<ExhExdDef>& Exh::get_exd_defs() const const std::vector< ExhExdDef >& Exh::get_exd_defs() const
{ {
return _exd_defs; return _exd_defs;
} }
const std::vector<Language>& Exh::get_languages() const const std::vector< Language >& Exh::get_languages() const
{ {
return _languages; return _languages;
} }
const std::map<uint32_t, ExhMember>& Exh::get_members() const const std::map< uint32_t, ExhMember >& Exh::get_members() const
{ {
return _members; return _members;
} }
const std::vector<ExhMember>& Exh::get_exh_members() const const std::vector< ExhMember >& Exh::get_exh_members() const
{ {
return _exh_defs; return _exh_defs;
} }
} }
}

190
deps/datReader/Exh.h vendored
View file

@ -5,99 +5,131 @@
#include "bparse.h" #include "bparse.h"
namespace xiv namespace xiv::exd
{ {
namespace exd enum class DataType :
{ uint16_t
enum class DataType : uint16_t {
{ string = 0,
string = 0, boolean = 1,
boolean = 1, int8 = 2,
int8 = 2, uint8 = 3,
uint8 = 3, int16 = 4,
int16 = 4, uint16 = 5,
uint16 = 5, int32 = 6,
int32 = 6, uint32 = 7,
uint32 = 7, float32 = 9,
float32 = 9, uint64 = 11,
uint64 = 11, };
};
struct ExhHeader struct ExhHeader
{ {
char magic[0x4]; char magic[0x4];
uint16_t unknown; uint16_t unknown;
uint16_t data_offset; uint16_t data_offset;
uint16_t field_count; uint16_t field_count;
uint16_t exd_count; uint16_t exd_count;
uint16_t language_count; uint16_t language_count;
uint16_t unknown1; uint16_t unknown1;
uint8_t u2; uint8_t u2;
uint8_t variant; uint8_t variant;
}; };
struct ExhMember struct ExhMember
{ {
DataType type; DataType type;
uint16_t offset; uint16_t offset;
}; };
struct ExhExdDef struct ExhExdDef
{ {
uint32_t start_id; uint32_t start_id;
uint32_t count_id; uint32_t count_id;
}; };
} };
namespace xiv::utils::bparse {
template<>
inline void reorder< xiv::exd::ExhHeader >( xiv::exd::ExhHeader& i_struct )
{
for( int32_t i = 0; i < 0x4; ++i )
{
xiv::utils::bparse::reorder( i_struct.magic[ i ] );
}
i_struct.unknown = xiv::utils::bparse::byteswap( i_struct.unknown );
xiv::utils::bparse::reorder( i_struct.unknown );
i_struct.data_offset = xiv::utils::bparse::byteswap( i_struct.data_offset );
xiv::utils::bparse::reorder( i_struct.data_offset );
i_struct.field_count = xiv::utils::bparse::byteswap( i_struct.field_count );
xiv::utils::bparse::reorder( i_struct.field_count );
i_struct.exd_count = xiv::utils::bparse::byteswap( i_struct.exd_count );
xiv::utils::bparse::reorder( i_struct.exd_count );
i_struct.language_count = xiv::utils::bparse::byteswap( i_struct.language_count );
xiv::utils::bparse::reorder( i_struct.language_count );
}
template<>
inline void reorder< xiv::exd::ExhMember >( xiv::exd::ExhMember& i_struct )
{
i_struct.type = xiv::utils::bparse::byteswap( i_struct.type );
xiv::utils::bparse::reorder( i_struct.type );
i_struct.offset = xiv::utils::bparse::byteswap( i_struct.offset );
xiv::utils::bparse::reorder( i_struct.offset );
}
template<>
inline void reorder< xiv::exd::ExhExdDef >( xiv::exd::ExhExdDef& i_struct )
{
i_struct.start_id = xiv::utils::bparse::byteswap( i_struct.start_id );
xiv::utils::bparse::reorder( i_struct.start_id );
i_struct.count_id = xiv::utils::bparse::byteswap( i_struct.count_id );
xiv::utils::bparse::reorder( i_struct.count_id );
}
}; };
namespace xiv namespace xiv
{ {
namespace utils
{
namespace bparse
{
template <> inline void reorder<xiv::exd::ExhHeader>( xiv::exd::ExhHeader& i_struct ) { for( int32_t i = 0; i < 0x4; ++i ) { xiv::utils::bparse::reorder( i_struct.magic[i] ); } i_struct.unknown = xiv::utils::bparse::byteswap( i_struct.unknown ); xiv::utils::bparse::reorder( i_struct.unknown ); i_struct.data_offset = xiv::utils::bparse::byteswap( i_struct.data_offset ); xiv::utils::bparse::reorder( i_struct.data_offset ); i_struct.field_count = xiv::utils::bparse::byteswap( i_struct.field_count ); xiv::utils::bparse::reorder( i_struct.field_count ); i_struct.exd_count = xiv::utils::bparse::byteswap( i_struct.exd_count ); xiv::utils::bparse::reorder( i_struct.exd_count ); i_struct.language_count = xiv::utils::bparse::byteswap( i_struct.language_count ); xiv::utils::bparse::reorder( i_struct.language_count ); }
template <> inline void reorder<xiv::exd::ExhMember>( xiv::exd::ExhMember& i_struct ) { i_struct.type = xiv::utils::bparse::byteswap( i_struct.type ); xiv::utils::bparse::reorder( i_struct.type ); i_struct.offset = xiv::utils::bparse::byteswap( i_struct.offset ); xiv::utils::bparse::reorder( i_struct.offset ); }
template <> inline void reorder<xiv::exd::ExhExdDef>( xiv::exd::ExhExdDef& i_struct ) { i_struct.start_id = xiv::utils::bparse::byteswap( i_struct.start_id ); xiv::utils::bparse::reorder( i_struct.start_id ); i_struct.count_id = xiv::utils::bparse::byteswap( i_struct.count_id ); xiv::utils::bparse::reorder( i_struct.count_id ); }
}
}
};
namespace xiv namespace dat
{ {
namespace dat class File;
{ }
class File;
}
namespace exd
{
enum Language : uint16_t; namespace exd
{
// Header file for exd data enum Language :
class Exh uint16_t;
{
public:
// The header file
Exh( const dat::File& i_file );
~Exh();
const ExhHeader& get_header() const; // Header file for exd data
const std::vector<ExhExdDef>& get_exd_defs() const; class Exh
const std::vector<Language>& get_languages() const; {
const std::map<uint32_t, ExhMember>& get_members() const; public:
const std::vector<ExhMember>& get_exh_members() const; // The header file
Exh( const dat::File& i_file );
protected: ~Exh();
ExhHeader _header;
// Members of the datastruct ordered(indexed) by offset
std::map<uint32_t, ExhMember> _members;
std::vector<ExhMember> _exh_defs;
std::vector<ExhExdDef> _exd_defs;
std::vector<Language> _languages;
};
} const ExhHeader& get_header() const;
const std::vector< ExhExdDef >& get_exd_defs() const;
const std::vector< Language >& get_languages() const;
const std::map< uint32_t, ExhMember >& get_members() const;
const std::vector< ExhMember >& get_exh_members() const;
protected:
ExhHeader _header;
// Members of the datastruct ordered(indexed) by offset
std::map< uint32_t, ExhMember > _members;
std::vector< ExhMember > _exh_defs;
std::vector< ExhExdDef > _exd_defs;
std::vector< Language > _languages;
};
}
} }
#endif // XIV_EXD_EXH_H #endif // XIV_EXD_EXH_H

59
deps/datReader/File.cpp vendored
View file

@ -2,44 +2,41 @@
#include <fstream> #include <fstream>
namespace xiv namespace xiv::dat
{
namespace dat
{ {
File::File() : File::File() :
_type(FileType::empty) _type( FileType::empty )
{ {
} }
File::~File() File::~File()
{ {
} }
FileType File::get_type() const FileType File::get_type() const
{ {
return _type; return _type;
} }
const std::vector<std::vector<char>>& File::get_data_sections() const const std::vector< std::vector< char>>& File::get_data_sections() const
{ {
return _data_sections; return _data_sections;
} }
std::vector<std::vector<char>>& File::access_data_sections() std::vector< std::vector< char>>& File::access_data_sections()
{ {
return _data_sections; return _data_sections;
} }
void File::exportToFile(const std::filesystem::path& i_path) const void File::exportToFile( const std::filesystem::path& i_path ) const
{ {
std::ofstream ofs( i_path.string(), std::ios_base::binary | std::ios_base::out ); std::ofstream ofs( i_path.string(), std::ios_base::binary | std::ios_base::out );
for( auto& data_section : _data_sections ) for( auto& data_section : _data_sections )
{ {
ofs.write( data_section.data(), data_section.size() ); ofs.write( data_section.data(), data_section.size() );
} }
ofs.close(); ofs.close();
} }
} }
}

62
deps/datReader/File.h vendored
View file

@ -7,50 +7,42 @@
#include <stdint.h> #include <stdint.h>
#include "bparse.h" #include "bparse.h"
namespace xiv::dat
namespace xiv
{ {
namespace dat enum class FileType :
{ uint32_t
enum class FileType : uint32_t {
{ empty = 1,
empty = 1, standard = 2,
standard = 2, model = 3,
model = 3, texture = 4,
texture = 4, };
};
}
};
namespace xiv class Dat;
{
namespace dat
{
class Dat; // Basic file from the dats
class File
{
friend class Dat;
// Basic file from the dats public:
class File File();
{
friend class Dat;
public:
File();
~File();
FileType get_type() const; ~File();
// Getters functions for the data in the file FileType get_type() const;
const std::vector<std::vector<char>>& get_data_sections() const;
std::vector<std::vector<char>>& access_data_sections();
void exportToFile( const std::filesystem::path& i_path ) const; // Getters functions for the data in the file
const std::vector< std::vector< char>>& get_data_sections() const;
protected: std::vector< std::vector< char>>& access_data_sections();
FileType _type;
std::vector<std::vector<char>> _data_sections;
};
} void exportToFile( const std::filesystem::path& i_path ) const;
protected:
FileType _type;
std::vector< std::vector< char>> _data_sections;
};
} }
#endif // XIV_DAT_FILE_H #endif // XIV_DAT_FILE_H

451
deps/datReader/GameData.cpp vendored
View file

@ -11,312 +11,315 @@
#include "DatCat.h" #include "DatCat.h"
#include "File.h" #include "File.h"
namespace namespace {
{ // Relation between category number and category name
// Relation between category number and category name // These names are taken straight from the exe, it helps resolve dispatching when getting files by path
// These names are taken straight from the exe, it helps resolve dispatching when getting files by path
std::unordered_map< std::string, uint32_t > categoryNameToIdMap = std::unordered_map< std::string, uint32_t > categoryNameToIdMap =
{{"common", 0x00}, { { "common", 0x00 },
{"bgcommon", 0x01}, { "bgcommon", 0x01 },
{"bg", 0x02}, { "bg", 0x02 },
{"cut", 0x03}, { "cut", 0x03 },
{"chara", 0x04}, { "chara", 0x04 },
{"shader", 0x05}, { "shader", 0x05 },
{"ui", 0x06}, { "ui", 0x06 },
{"sound", 0x07}, { "sound", 0x07 },
{"vfx", 0x08}, { "vfx", 0x08 },
{"ui_script", 0x09}, { "ui_script", 0x09 },
{"exd", 0x0A}, { "exd", 0x0A },
{"game_script", 0x0B}, { "game_script", 0x0B },
{"music", 0x0C} { "music", 0x0C }
}; };
std::unordered_map< uint32_t, std::string > categoryIdToNameMap = std::unordered_map< uint32_t, std::string > categoryIdToNameMap =
{{0x00, "common"}, { { 0x00, "common" },
{0x01, "bgcommon"}, { 0x01, "bgcommon" },
{0x02, "bg"}, { 0x02, "bg" },
{0x03, "cut"}, { 0x03, "cut" },
{0x04, "chara"}, { 0x04, "chara" },
{0x05, "shader"}, { 0x05, "shader" },
{0x06, "ui"}, { 0x06, "ui" },
{0x07, "sound"}, { 0x07, "sound" },
{0x08, "vfx"}, { 0x08, "vfx" },
{0x09, "ui_script"}, { 0x09, "ui_script" },
{0x0A, "exd"}, { 0x0A, "exd" },
{0x0B, "game_script"}, { 0x0B, "game_script" },
{0x0C, "music"}}; { 0x0C, "music" } };
} }
namespace xiv namespace xiv::dat
{
namespace dat
{ {
GameData::GameData( const std::filesystem::path& path ) try :
m_path( path )
{
int maxExLevel = 0;
GameData::GameData(const std::filesystem::path& path) try : // msvc has retarded stdlib implementation
m_path(path) #ifdef _WIN32
{ static constexpr auto sep = "\\";
int maxExLevel = 0; #else
static constexpr auto sep = std::filesystem::path::preferred_separator;
#endif
// msvc has retarded stdlib implementation // Determine which expansions are available
#ifdef _WIN32 while( std::filesystem::exists( std::filesystem::path(
static constexpr auto sep = "\\"; m_path.string() + sep + "ex" + std::to_string( maxExLevel + 1 ) + sep + "ex" + std::to_string( maxExLevel + 1 ) +
#else ".ver" ) ) )
static constexpr auto sep = std::filesystem::path::preferred_separator; {
#endif
// Determine which expansions are available
while( std::filesystem::exists( std::filesystem::path( m_path.string() + sep + "ex" + std::to_string( maxExLevel + 1 ) + sep + "ex" + std::to_string( maxExLevel + 1 ) + ".ver" ) ) )
{
maxExLevel++; maxExLevel++;
} }
// Iterate over the files in path // Iterate over the files in path
for( auto it = std::filesystem::directory_iterator( m_path.string() + "//ffxiv" ); it != std::filesystem::directory_iterator(); ++it ) for( auto it = std::filesystem::directory_iterator( m_path.string() + "//ffxiv" );
{ it != std::filesystem::directory_iterator(); ++it )
{
// Get the filename of the current element // Get the filename of the current element
auto filename = it->path().filename().string(); auto filename = it->path().filename().string();
// If it contains ".win32.index" this is most likely a hit for a category // If it contains ".win32.index" this is most likely a hit for a category
if( filename.find( ".win32.index" ) != std::string::npos && filename.find( ".win32.index2" ) == std::string::npos ) if( filename.find( ".win32.index" ) != std::string::npos && filename.find( ".win32.index2" ) == std::string::npos )
{ {
// Format of indexes is XX0000.win32.index, so fetch the hex number for category number // Format of indexes is XX0000.win32.index, so fetch the hex number for category number
std::istringstream iss( filename.substr( 0, 2 ) ); std::istringstream iss( filename.substr( 0, 2 ) );
uint32_t cat_nb; uint32_t cat_nb;
iss >> std::hex >> cat_nb; iss >> std::hex >> cat_nb;
// Add to the list of category number // Add to the list of category number
// creates the empty category in the cats map // creates the empty category in the cats map
// instantiate the creation mutex for this category // instantiate the creation mutex for this category
m_catNums.push_back( cat_nb ); m_catNums.push_back( cat_nb );
m_cats[cat_nb] = std::unique_ptr<Cat>(); m_cats[ cat_nb ] = std::unique_ptr< Cat >();
m_catCreationMutexes[cat_nb] = std::unique_ptr<std::mutex>( new std::mutex() ); m_catCreationMutexes[ cat_nb ] = std::make_unique< std::mutex >();
// Check for expansion // Check for expansion
for( int exNum = 1; exNum <= maxExLevel; exNum++ ) for( int exNum = 1; exNum <= maxExLevel; exNum++ )
{ {
const std::string path = m_path.string() + sep + buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, 0, "win32", "index" ); const std::string path =
m_path.string() + sep + buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, 0, "win32", "index" );
if( std::filesystem::exists( std::filesystem::path( path ) ) ) if( std::filesystem::exists( std::filesystem::path( path ) ) )
{
int chunkCount = 0;
for( int chunkTest = 0; chunkTest < 256; chunkTest++ )
{ {
if( std::filesystem::exists( m_path.string() + sep +
int chunkCount = 0; buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, chunkTest, "win32",
"index" ) ) )
for(int chunkTest = 0; chunkTest < 256; chunkTest++ ) {
{ m_exCats[ cat_nb ].exNumToChunkMap[ exNum ].chunkToCatMap[ chunkTest ] = std::unique_ptr< Cat >();
if( std::filesystem::exists( m_path.string() + sep + buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, chunkTest, "win32", "index" ) ) ) chunkCount++;
{ }
m_exCats[cat_nb].exNumToChunkMap[exNum].chunkToCatMap[chunkTest] = std::unique_ptr<Cat>();
chunkCount++;
}
}
} }
}
}
}
} }
} }
} }
catch( std::exception& e ) catch( std::exception& e )
{ {
// In case of failure here, client is supposed to catch the exception because it is not recoverable on our side // In case of failure here, client is supposed to catch the exception because it is not recoverable on our side
throw std::runtime_error( "GameData initialization failed: " + std::string( e.what() ) ); throw std::runtime_error( "GameData initialization failed: " + std::string( e.what() ) );
} }
GameData::~GameData() GameData::~GameData()
{ {
} }
const std::string GameData::buildDatStr( const std::string folder, const int cat, const int exNum, const int chunk, const std::string platform, const std::string type ) const std::string GameData::buildDatStr( const std::string folder, const int cat, const int exNum, const int chunk,
{ const std::string platform, const std::string type )
char dat[1024]; {
sprintf( dat, "%s/%02x%02x%02x.%s.%s", folder.c_str(), cat, exNum, chunk, platform.c_str(), type.c_str() ); char dat[1024];
return std::string( dat ); sprintf( dat, "%s/%02x%02x%02x.%s.%s", folder.c_str(), cat, exNum, chunk, platform.c_str(), type.c_str() );
} return std::string( dat );
}
const std::vector<uint32_t>& GameData::getCatNumbers() const const std::vector< uint32_t >& GameData::getCatNumbers() const
{ {
return m_catNums; return m_catNums;
} }
std::unique_ptr<File> GameData::getFile(const std::string& path) std::unique_ptr< File > GameData::getFile( const std::string& path )
{ {
// Get the hashes, the category from the path then call the getFile of the category // Get the hashes, the category from the path then call the getFile of the category
uint32_t dirHash; uint32_t dirHash;
uint32_t filenameHash; uint32_t filenameHash;
getHashes( path, dirHash, filenameHash ); getHashes( path, dirHash, filenameHash );
return getCategoryFromPath( path ).getFile( dirHash, filenameHash ); return getCategoryFromPath( path ).getFile( dirHash, filenameHash );
} }
bool GameData::doesFileExist(const std::string& path) bool GameData::doesFileExist( const std::string& path )
{ {
uint32_t dirHash; uint32_t dirHash;
uint32_t filenameHash; uint32_t filenameHash;
getHashes( path, dirHash, filenameHash ); getHashes( path, dirHash, filenameHash );
return getCategoryFromPath( path ).doesFileExist( dirHash, filenameHash ); return getCategoryFromPath( path ).doesFileExist( dirHash, filenameHash );
} }
bool GameData::doesDirExist(const std::string& i_path) bool GameData::doesDirExist( const std::string& i_path )
{ {
uint32_t dirHash; uint32_t dirHash;
uint32_t filenameHash; uint32_t filenameHash;
getHashes( i_path, dirHash, filenameHash ); getHashes( i_path, dirHash, filenameHash );
return getCategoryFromPath( i_path ).doesDirExist( dirHash ); return getCategoryFromPath( i_path ).doesDirExist( dirHash );
} }
const Cat& GameData::getCategory(uint32_t catNum) const Cat& GameData::getCategory( uint32_t catNum )
{ {
// Check that the category number exists // Check that the category number exists
auto catIt = m_cats.find( catNum ); auto catIt = m_cats.find( catNum );
if( catIt == m_cats.end() ) if( catIt == m_cats.end() )
{ {
throw std::runtime_error( "Category not found: " + std::to_string( catNum ) ); throw std::runtime_error( "Category not found: " + std::to_string( catNum ) );
} }
// If it exists and already instantiated return it // If it exists and already instantiated return it
if( catIt->second ) if( catIt->second )
{ {
return *( catIt->second ); return *( catIt->second );
} }
else else
{ {
// Else create it and return it // Else create it and return it
createCategory( catNum ); createCategory( catNum );
return *( m_cats[catNum] ); return *( m_cats[ catNum ] );
} }
} }
const Cat& GameData::getCategory(const std::string& catName) const Cat& GameData::getCategory( const std::string& catName )
{ {
// Find the category number from the name // Find the category number from the name
auto categoryNameToIdMapIt = ::categoryNameToIdMap.find( catName ); auto categoryNameToIdMapIt = ::categoryNameToIdMap.find( catName );
if( categoryNameToIdMapIt == ::categoryNameToIdMap.end() ) if( categoryNameToIdMapIt == ::categoryNameToIdMap.end() )
{ {
throw std::runtime_error( "Category not found: " + catName ); throw std::runtime_error( "Category not found: " + catName );
} }
// From the category number return the category // From the category number return the category
return getCategory( categoryNameToIdMapIt->second ); return getCategory( categoryNameToIdMapIt->second );
} }
const Cat& GameData::getExCategory( const std::string& catName, uint32_t exNum, const std::string& path ) const Cat& GameData::getExCategory( const std::string& catName, uint32_t exNum, const std::string& path )
{ {
// Find the category number from the name // Find the category number from the name
auto categoryMapIt = ::categoryNameToIdMap.find( catName ); auto categoryMapIt = ::categoryNameToIdMap.find( catName );
if( categoryMapIt == ::categoryNameToIdMap.end() ) if( categoryMapIt == ::categoryNameToIdMap.end() )
{ {
throw std::runtime_error( "Category not found: " + catName ); throw std::runtime_error( "Category not found: " + catName );
} }
uint32_t dirHash; uint32_t dirHash;
uint32_t filenameHash; uint32_t filenameHash;
getHashes( path, dirHash, filenameHash ); getHashes( path, dirHash, filenameHash );
for( auto const& chunk : m_exCats[categoryMapIt->second].exNumToChunkMap[exNum].chunkToCatMap ) for( auto const& chunk : m_exCats[ categoryMapIt->second ].exNumToChunkMap[ exNum ].chunkToCatMap )
{ {
if( !chunk.second ) if( !chunk.second )
createExCategory( categoryMapIt->second ); createExCategory( categoryMapIt->second );
if( chunk.second->doesFileExist( dirHash, filenameHash ) ) if( chunk.second->doesFileExist( dirHash, filenameHash ) )
{ {
return *( chunk.second ); return *( chunk.second );
} }
} }
throw std::runtime_error( "Chunk not found for path: " + path ); throw std::runtime_error( "Chunk not found for path: " + path );
} }
const Cat& GameData::getCategoryFromPath(const std::string& path) const Cat& GameData::getCategoryFromPath( const std::string& path )
{ {
// Find the first / in the string, paths are in the format CAT_NAME/..../.../../.... // Find the first / in the string, paths are in the format CAT_NAME/..../.../../....
auto firstSlashPos = path.find( '/' ); auto firstSlashPos = path.find( '/' );
if( firstSlashPos == std::string::npos ) if( firstSlashPos == std::string::npos )
{ {
throw std::runtime_error( "Path does not have a / char: " + path ); throw std::runtime_error( "Path does not have a / char: " + path );
} }
if( path.substr( firstSlashPos + 1, 2) == "ex" ) if( path.substr( firstSlashPos + 1, 2 ) == "ex" )
{ {
return getExCategory( path.substr( 0, firstSlashPos ), std::stoi( path.substr( firstSlashPos + 3, 1 ) ), path ); return getExCategory( path.substr( 0, firstSlashPos ), std::stoi( path.substr( firstSlashPos + 3, 1 ) ), path );
} }
else else
{ {
// From the sub string found beforethe first / get the category // From the sub string found beforethe first / get the category
return getCategory( path.substr( 0, firstSlashPos ) ); return getCategory( path.substr( 0, firstSlashPos ) );
} }
} }
void GameData::getHashes(const std::string& path, uint32_t& dirHash, uint32_t& filenameHash) const void GameData::getHashes( const std::string& path, uint32_t& dirHash, uint32_t& filenameHash ) const
{ {
// Convert the path to lowercase before getting the hashes // Convert the path to lowercase before getting the hashes
std::string pathLower; std::string pathLower;
pathLower.resize( path.size() ); pathLower.resize( path.size() );
std::transform( path.begin(), path.end(), pathLower.begin(), ::tolower ); std::transform( path.begin(), path.end(), pathLower.begin(), ::tolower );
// Find last / to separate dir from filename // Find last / to separate dir from filename
auto lastSlashPos = pathLower.rfind( '/' ); auto lastSlashPos = pathLower.rfind( '/' );
if( lastSlashPos == std::string::npos ) if( lastSlashPos == std::string::npos )
{ {
throw std::runtime_error( "Path does not have a / char: " + path ); throw std::runtime_error( "Path does not have a / char: " + path );
} }
std::string dirPart = pathLower.substr( 0, lastSlashPos ); std::string dirPart = pathLower.substr( 0, lastSlashPos );
std::string filenamePart = pathLower.substr( lastSlashPos + 1 ); std::string filenamePart = pathLower.substr( lastSlashPos + 1 );
// Get the crc32 values from zlib, to compensate the final XOR 0xFFFFFFFF that isnot done in the exe we just reXOR // Get the crc32 values from zlib, to compensate the final XOR 0xFFFFFFFF that isnot done in the exe we just reXOR
dirHash = crc32( 0, reinterpret_cast<const uint8_t*>( dirPart.data() ), dirPart.size() ) ^ 0xFFFFFFFF; dirHash = crc32( 0, reinterpret_cast<const uint8_t*>( dirPart.data() ), dirPart.size() ) ^ 0xFFFFFFFF;
filenameHash = crc32( 0, reinterpret_cast<const uint8_t*>( filenamePart.data() ), filenamePart.size() ) ^ 0xFFFFFFFF; filenameHash = crc32( 0, reinterpret_cast<const uint8_t*>( filenamePart.data() ), filenamePart.size() ) ^ 0xFFFFFFFF;
} }
void GameData::createCategory(uint32_t catNum) void GameData::createCategory( uint32_t catNum )
{ {
// Lock mutex in this scope // Lock mutex in this scope
std::lock_guard<std::mutex> lock( *( m_catCreationMutexes[catNum] ) ); std::lock_guard< std::mutex > lock( *( m_catCreationMutexes[ catNum ] ) );
// Maybe after unlocking it has already been created, so check (most likely if it blocked) // Maybe after unlocking it has already been created, so check (most likely if it blocked)
if( !m_cats[catNum] ) if( !m_cats[ catNum ] )
{ {
// Get the category name if we have it // Get the category name if we have it
std::string catName; std::string catName;
auto categoryMapIt = ::categoryIdToNameMap.find( catNum ); auto categoryMapIt = ::categoryIdToNameMap.find( catNum );
if( categoryMapIt != ::categoryIdToNameMap.end() ) if( categoryMapIt != ::categoryIdToNameMap.end() )
{ {
catName = categoryMapIt->second; catName = categoryMapIt->second;
} }
// Actually creates the category // Actually creates the category
m_cats[catNum] = std::unique_ptr<Cat>( new Cat( m_path, catNum, catName ) ); m_cats[ catNum ] = std::make_unique< Cat >( m_path, catNum, catName );
} }
} }
void GameData::createExCategory( uint32_t catNum ) void GameData::createExCategory( uint32_t catNum )
{ {
// Maybe after unlocking it has already been created, so check (most likely if it blocked) // Maybe after unlocking it has already been created, so check (most likely if it blocked)
if( !m_exCats[catNum].exNumToChunkMap[1].chunkToCatMap[0] ) if( !m_exCats[ catNum ].exNumToChunkMap[ 1 ].chunkToCatMap[ 0 ] )
{ {
// Get the category name if we have it // Get the category name if we have it
std::string catName; std::string catName;
auto categoryMapIt = ::categoryIdToNameMap.find( catNum ); auto categoryMapIt = ::categoryIdToNameMap.find( catNum );
if( categoryMapIt != ::categoryIdToNameMap.end() ) if( categoryMapIt != ::categoryIdToNameMap.end() )
{ {
catName = categoryMapIt->second; catName = categoryMapIt->second;
} }
for( auto const& ex : m_exCats[catNum].exNumToChunkMap ) for( auto const& ex : m_exCats[ catNum ].exNumToChunkMap )
{ {
for( auto const& chunk : m_exCats[catNum].exNumToChunkMap[ex.first].chunkToCatMap ) for( auto const& chunk : m_exCats[ catNum ].exNumToChunkMap[ ex.first ].chunkToCatMap )
{ {
// Actually creates the category // Actually creates the category
m_exCats[catNum].exNumToChunkMap[ex.first].chunkToCatMap[chunk.first] = std::unique_ptr<Cat>( new Cat( m_path, catNum, catName, ex.first, chunk.first ) ); m_exCats[ catNum ].exNumToChunkMap[ ex.first ].chunkToCatMap[ chunk.first ] = std::unique_ptr< Cat >(
} new Cat( m_path, catNum, catName, ex.first, chunk.first ) );
}
} }
} }
} }
} }
}

114
deps/datReader/GameData.h vendored
View file

@ -7,78 +7,86 @@
#include <filesystem> #include <filesystem>
namespace xiv namespace xiv::dat
{
namespace dat
{ {
class Cat; class Cat;
class File;
// Interface to all the datfiles - Main entry point class File;
// All the paths to files/dirs inside the dats are case-insensitive
class GameData
{
public:
// This should be the path in which the .index/.datX files are located
GameData( const std::filesystem::path& path );
~GameData();
static const std::string buildDatStr( const std::string folder, const int cat, const int exNum, const int chunk, const std::string platform, const std::string type ); // Interface to all the datfiles - Main entry point
// All the paths to files/dirs inside the dats are case-insensitive
class GameData
{
public:
// This should be the path in which the .index/.datX files are located
GameData( const std::filesystem::path& path );
// Returns all the scanned category number available in the path ~GameData();
const std::vector<uint32_t>& getCatNumbers() const;
// Return a specific category by its number (see getCatNumbers() for loops) static const std::string
const Cat& getCategory( uint32_t catNum ); buildDatStr( const std::string folder, const int cat, const int exNum, const int chunk, const std::string platform,
// Return a specific category by it's name (e.g.: "exd"/"game_script"/ etc...) const std::string type );
const Cat& getCategory( const std::string& catName );
const Cat& getExCategory( const std::string& catName, uint32_t exNum, const std::string& path ); // Returns all the scanned category number available in the path
const std::vector< uint32_t >& getCatNumbers() const;
// Retrieve a file from the dats given its filename // Return a specific category by its number (see getCatNumbers() for loops)
std::unique_ptr<File> getFile( const std::string& path ); const Cat& getCategory( uint32_t catNum );
// Checks that a file exists // Return a specific category by it's name (e.g.: "exd"/"game_script"/ etc...)
bool doesFileExist( const std::string& path ); const Cat& getCategory( const std::string& catName );
// Checks that a dir exists, there must be a trailing / in the path const Cat& getExCategory( const std::string& catName, uint32_t exNum, const std::string& path );
// Note that it won't work for dirs that don't contain any file
// e.g.: - "ui/icon/" will return False
// - "ui/icon/000000/" will return True
bool doesDirExist( const std::string& path );
protected: // Retrieve a file from the dats given its filename
// Return a specific category given a path (calls const Cat& getCategory(const std::string& catName)) std::unique_ptr< File > getFile( const std::string& path );
const Cat& getCategoryFromPath( const std::string& path );
// From a full path, returns the dirHash and the filenameHash // Checks that a file exists
void getHashes( const std::string& path, uint32_t& dirHash, uint32_t& filenameHash ) const; bool doesFileExist( const std::string& path );
// Lazy instantiation of category // Checks that a dir exists, there must be a trailing / in the path
void createCategory( uint32_t catNum ); // Note that it won't work for dirs that don't contain any file
// e.g.: - "ui/icon/" will return False
// - "ui/icon/000000/" will return True
bool doesDirExist( const std::string& path );
void createExCategory( uint32_t catNum ); protected:
// Return a specific category given a path (calls const Cat& getCategory(const std::string& catName))
const Cat& getCategoryFromPath( const std::string& path );
// Path given to constructor, pointing to the folder with the .index/.datX files // From a full path, returns the dirHash and the filenameHash
const std::filesystem::path m_path; void getHashes( const std::string& path, uint32_t& dirHash, uint32_t& filenameHash ) const;
// Stored categories, indexed by their number, categories are instantiated and parsed individually when they are needed // Lazy instantiation of category
std::unordered_map<uint32_t, std::unique_ptr<Cat>> m_cats; void createCategory( uint32_t catNum );
// List of all the categories numbers, is equal to m_cats.keys() void createExCategory( uint32_t catNum );
std::vector<uint32_t> m_catNums;
// Map of all EX categories and their chunks, "CatNum - (ExNum - (ChunkNum - Cat))" // Path given to constructor, pointing to the folder with the .index/.datX files
// Map of all EX categories and their chunks, "CatNum - (ExNum - (ChunkNum - Cat))" const std::filesystem::path m_path;
using ChunkToCatMap = struct { std::unordered_map< uint32_t, std::unique_ptr< Cat > > chunkToCatMap; };
using ExNumToChunkMap = struct { std::unordered_map< uint32_t, ChunkToCatMap > exNumToChunkMap; }; // Stored categories, indexed by their number, categories are instantiated and parsed individually when they are needed
using CatNumToExNumMap = std::unordered_map< uint32_t, ExNumToChunkMap >; std::unordered_map< uint32_t, std::unique_ptr< Cat>> m_cats;
CatNumToExNumMap m_exCats;
std::unordered_map<uint32_t, std::unique_ptr<std::mutex>> m_catCreationMutexes; // List of all the categories numbers, is equal to m_cats.keys()
}; std::vector< uint32_t > m_catNums;
// Map of all EX categories and their chunks, "CatNum - (ExNum - (ChunkNum - Cat))"
// Map of all EX categories and their chunks, "CatNum - (ExNum - (ChunkNum - Cat))"
using ChunkToCatMap = struct
{
std::unordered_map< uint32_t, std::unique_ptr< Cat > > chunkToCatMap;
};
using ExNumToChunkMap = struct
{
std::unordered_map< uint32_t, ChunkToCatMap > exNumToChunkMap;
};
using CatNumToExNumMap = std::unordered_map< uint32_t, ExNumToChunkMap >;
CatNumToExNumMap m_exCats;
std::unordered_map< uint32_t, std::unique_ptr< std::mutex>> m_catCreationMutexes;
};
}
} }
#endif // XIV_DAT_GAMEDATA_H #endif // XIV_DAT_GAMEDATA_H

228
deps/datReader/Index.cpp vendored
View file

@ -2,166 +2,154 @@
#include "bparse.h" #include "bparse.h"
namespace xiv namespace xiv::dat
{ {
namespace dat struct IndexBlockRecord
{ {
struct IndexBlockRecord uint32_t offset;
{ uint32_t size;
uint32_t offset; SqPackBlockHash blockHash;
uint32_t size; };
SqPackBlockHash blockHash;
};
struct IndexHashTableEntry struct IndexHashTableEntry
{ {
uint32_t filenameHash; uint32_t filenameHash;
uint32_t dirHash; uint32_t dirHash;
uint32_t datOffset; uint32_t datOffset;
uint32_t padding; uint32_t padding;
}; };
}
} }
namespace xiv namespace xiv::utils::bparse
{ {
namespace utils template<>
{ inline void reorder< xiv::dat::IndexBlockRecord >( xiv::dat::IndexBlockRecord& i_struct )
namespace bparse {
{ xiv::utils::bparse::reorder( i_struct.offset );
template <> xiv::utils::bparse::reorder( i_struct.size );
inline void reorder<xiv::dat::IndexBlockRecord>(xiv::dat::IndexBlockRecord& i_struct) xiv::utils::bparse::reorder( i_struct.blockHash );
{ }
xiv::utils::bparse::reorder(i_struct.offset);
xiv::utils::bparse::reorder(i_struct.size);
xiv::utils::bparse::reorder(i_struct.blockHash);
}
template <> template<>
inline void reorder<xiv::dat::IndexHashTableEntry>(xiv::dat::IndexHashTableEntry& i_struct) inline void reorder< xiv::dat::IndexHashTableEntry >( xiv::dat::IndexHashTableEntry& i_struct )
{ {
xiv::utils::bparse::reorder(i_struct.filenameHash); xiv::utils::bparse::reorder( i_struct.filenameHash );
xiv::utils::bparse::reorder(i_struct.dirHash); xiv::utils::bparse::reorder( i_struct.dirHash );
xiv::utils::bparse::reorder(i_struct.datOffset); xiv::utils::bparse::reorder( i_struct.datOffset );
xiv::utils::bparse::reorder(i_struct.padding); xiv::utils::bparse::reorder( i_struct.padding );
} }
} }
}
};
using xiv::utils::bparse::extract; using xiv::utils::bparse::extract;
namespace xiv namespace xiv::dat
{
namespace dat
{ {
Index::Index(const std::filesystem::path& path) : Index::Index( const std::filesystem::path& path ) :
SqPack( path ) SqPack( path )
{ {
if( !m_handle ) if( !m_handle )
throw new std::runtime_error( "Failed to load Index at " + path.string() ); throw new std::runtime_error( "Failed to load Index at " + path.string() );
// Hash Table record // Hash Table record
auto hashTableBlockRecord = extract<IndexBlockRecord>( m_handle ); auto hashTableBlockRecord = extract< IndexBlockRecord >( m_handle );
isIndexBlockValid( hashTableBlockRecord ); isIndexBlockValid( hashTableBlockRecord );
// Save the posin the stream to go back to it later on // Save the posin the stream to go back to it later on
auto pos = m_handle.tellg(); auto pos = m_handle.tellg();
// Seek to the pos of the hash table in the file // Seek to the pos of the hash table in the file
m_handle.seekg( hashTableBlockRecord.offset ); m_handle.seekg( hashTableBlockRecord.offset );
// Preallocate and extract the index_hash_table_entries // Preallocate and extract the index_hash_table_entries
std::vector<IndexHashTableEntry> indexHashTableEntries; std::vector< IndexHashTableEntry > indexHashTableEntries;
extract<IndexHashTableEntry>( m_handle, hashTableBlockRecord.size / sizeof( IndexHashTableEntry ), extract< IndexHashTableEntry >( m_handle, hashTableBlockRecord.size / sizeof( IndexHashTableEntry ),
indexHashTableEntries ); indexHashTableEntries );
// Feed the correct entry in the HashTable for each index_hash_table_entry // Feed the correct entry in the HashTable for each index_hash_table_entry
for( auto& indexHashTableEntry : indexHashTableEntries ) for( auto& indexHashTableEntry : indexHashTableEntries )
{ {
auto& hashTableEntry = m_hashTable[indexHashTableEntry.dirHash][indexHashTableEntry.filenameHash]; auto& hashTableEntry = m_hashTable[ indexHashTableEntry.dirHash ][ indexHashTableEntry.filenameHash ];
// The dat number is found in the offset, last four bits // The dat number is found in the offset, last four bits
hashTableEntry.datNum = ( indexHashTableEntry.datOffset & 0xF ) / 0x2; hashTableEntry.datNum = ( indexHashTableEntry.datOffset & 0xF ) / 0x2;
// The offset in the dat file, needs to strip the dat number indicator // The offset in the dat file, needs to strip the dat number indicator
hashTableEntry.datOffset = ( indexHashTableEntry.datOffset - ( indexHashTableEntry.datOffset & 0x000F ) ) * 0x08; hashTableEntry.datOffset = ( indexHashTableEntry.datOffset - ( indexHashTableEntry.datOffset & 0x000F ) ) * 0x08;
hashTableEntry.dirHash = indexHashTableEntry.dirHash; hashTableEntry.dirHash = indexHashTableEntry.dirHash;
hashTableEntry.filenameHash = indexHashTableEntry.filenameHash; hashTableEntry.filenameHash = indexHashTableEntry.filenameHash;
} }
// Come back to where we were before reading the HashTable // Come back to where we were before reading the HashTable
m_handle.seekg( pos ); m_handle.seekg( pos );
// Dat Count // Dat Count
m_datCount = extract<uint32_t>( m_handle, "dat_count" ); m_datCount = extract< uint32_t >( m_handle, "dat_count" );
// Free List // Free List
isIndexBlockValid( extract<IndexBlockRecord>( m_handle ) ); isIndexBlockValid( extract< IndexBlockRecord >( m_handle ) );
// Dir Hash Table // Dir Hash Table
isIndexBlockValid( extract<IndexBlockRecord>( m_handle ) ); isIndexBlockValid( extract< IndexBlockRecord >( m_handle ) );
} }
Index::~Index() Index::~Index()
{ {
} }
uint32_t Index::getDatCount() const uint32_t Index::getDatCount() const
{ {
return m_datCount; return m_datCount;
} }
const Index::HashTable& Index::getHashTable() const const Index::HashTable& Index::getHashTable() const
{ {
return m_hashTable; return m_hashTable;
} }
bool Index::doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const bool Index::doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const
{ {
auto dir_it = getHashTable().find( dir_hash ); auto dir_it = getHashTable().find( dir_hash );
if( dir_it != getHashTable().end() ) if( dir_it != getHashTable().end() )
{ {
return ( dir_it->second.find( filename_hash ) != dir_it->second.end() ); return ( dir_it->second.find( filename_hash ) != dir_it->second.end() );
} }
return false; return false;
} }
bool Index::doesDirExist( uint32_t dir_hash ) const bool Index::doesDirExist( uint32_t dir_hash ) const
{ {
return ( getHashTable().find( dir_hash ) != getHashTable().end() ); return ( getHashTable().find( dir_hash ) != getHashTable().end() );
} }
const Index::DirHashTable& Index::getDirHashTable( uint32_t dir_hash ) const const Index::DirHashTable& Index::getDirHashTable( uint32_t dir_hash ) const
{ {
auto dir_it = getHashTable().find( dir_hash ); auto dir_it = getHashTable().find( dir_hash );
if( dir_it == getHashTable().end() ) if( dir_it == getHashTable().end() )
{ {
throw std::runtime_error( "dirHash not found" ); throw std::runtime_error( "dirHash not found" );
} }
else else
{ {
return dir_it->second; return dir_it->second;
} }
} }
const Index::HashTableEntry& Index::getHashTableEntry( uint32_t dir_hash, uint32_t filename_hash ) const const Index::HashTableEntry& Index::getHashTableEntry( uint32_t dir_hash, uint32_t filename_hash ) const
{ {
auto& dirHashTable = getDirHashTable( dir_hash ); auto& dirHashTable = getDirHashTable( dir_hash );
auto file_it = dirHashTable.find( filename_hash ); auto file_it = dirHashTable.find( filename_hash );
if( file_it == dirHashTable.end() ) if( file_it == dirHashTable.end() )
{ {
throw std::runtime_error( "filenameHash not found" ); throw std::runtime_error( "filenameHash not found" );
} }
else else
{ {
return file_it->second; return file_it->second;
} }
} }
void Index::isIndexBlockValid( const IndexBlockRecord& i_index_block_record ) void Index::isIndexBlockValid( const IndexBlockRecord& i_index_block_record )
{ {
isBlockValid( i_index_block_record.offset, i_index_block_record.size, i_index_block_record.blockHash ); isBlockValid( i_index_block_record.offset, i_index_block_record.size, i_index_block_record.blockHash );
} }
} }
}

72
deps/datReader/Index.h vendored
View file

@ -7,53 +7,57 @@
#include <filesystem> #include <filesystem>
namespace xiv { namespace xiv::dat
namespace dat {
struct IndexBlockRecord;
class Index : public SqPack
{ {
public:
// Full path to the index file
Index( const std::filesystem::path& i_path );
virtual ~Index();
// An entry in the hash table, representing a file in a given dat struct IndexBlockRecord;
struct HashTableEntry
{ class Index :
public SqPack
{
public:
// Full path to the index file
Index( const std::filesystem::path& i_path );
virtual ~Index();
// An entry in the hash table, representing a file in a given dat
struct HashTableEntry
{
uint32_t datNum; uint32_t datNum;
uint32_t dirHash; uint32_t dirHash;
uint32_t filenameHash; uint32_t filenameHash;
uint32_t datOffset; uint32_t datOffset;
}; };
// HashTable has dir hashes -> filename hashes -> HashTableEntry // HashTable has dir hashes -> filename hashes -> HashTableEntry
using DirHashTable = std::unordered_map< uint32_t, HashTableEntry >; using DirHashTable = std::unordered_map< uint32_t, HashTableEntry >;
using HashTable = std::unordered_map< uint32_t, DirHashTable >; using HashTable = std::unordered_map< uint32_t, DirHashTable >;
// Get the number of dat files the index is linked to // Get the number of dat files the index is linked to
uint32_t getDatCount() const; uint32_t getDatCount() const;
bool doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const; bool doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const;
bool doesDirExist( uint32_t dir_hash ) const;
// Returns the whole HashTable bool doesDirExist( uint32_t dir_hash ) const;
const HashTable& getHashTable() const;
// Returns the hash table for a specific dir
const DirHashTable& getDirHashTable( uint32_t dir_hash ) const;
// Returns the HashTableEntry for a given file given its hashes
const HashTableEntry& getHashTableEntry( uint32_t dir_hash, uint32_t filename_hash ) const;
protected: // Returns the whole HashTable
// Checks that the block is valid with regards to its hash const HashTable& getHashTable() const;
void isIndexBlockValid( const IndexBlockRecord& i_index_block_record );
uint32_t m_datCount; // Returns the hash table for a specific dir
HashTable m_hashTable; const DirHashTable& getDirHashTable( uint32_t dir_hash ) const;
};
// Returns the HashTableEntry for a given file given its hashes
const HashTableEntry& getHashTableEntry( uint32_t dir_hash, uint32_t filename_hash ) const;
protected:
// Checks that the block is valid with regards to its hash
void isIndexBlockValid( const IndexBlockRecord& i_index_block_record );
uint32_t m_datCount;
HashTable m_hashTable;
};
}
} }
#endif // XIV_DAT_INDEX_H #endif // XIV_DAT_INDEX_H

12
deps/datReader/SqPack.cpp vendored
View file

@ -54,12 +54,12 @@ using xiv::utils::bparse::extract;
namespace xiv::dat namespace xiv::dat
{ {
// Open the file // Open the file
SqPack::SqPack( const std::filesystem::path& path ) : SqPack::SqPack( const std::filesystem::path& path ) :
m_handle( path.string(), std::ios_base::in | std::ios_base::binary ) m_handle( path.string(), std::ios_base::in | std::ios_base::binary )
{ {
// Extract the header // Extract the header
extract<SqPackHeader>( m_handle ); extract< SqPackHeader >( m_handle );
// Skip until the IndexHeader the extract it // Skip until the IndexHeader the extract it
m_handle.seekg( 0x400 ); m_handle.seekg( 0x400 );

78
deps/datReader/SqPack.h vendored
View file

@ -8,59 +8,51 @@
#include "bparse.h" #include "bparse.h"
namespace xiv namespace xiv::dat
{ {
namespace dat struct SqPackBlockHash
{ {
uint8_t hash[0x14];
struct SqPackBlockHash uint32_t padding[0xB];
{ };
uint8_t hash[0x14];
uint32_t padding[0xB];
};
}
} }
namespace xiv {
namespace utils { namespace xiv::utils::bparse
namespace bparse { {
template <> inline void reorder<xiv::dat::SqPackBlockHash>( xiv::dat::SqPackBlockHash& i_struct ) template<>
{ inline void reorder< xiv::dat::SqPackBlockHash >( xiv::dat::SqPackBlockHash& i_struct )
for( auto i = 0; i < 0x14; ++i ) {
{ for( auto i = 0; i < 0x14; ++i )
xiv::utils::bparse::reorder( i_struct.hash[i] ); {
} xiv::utils::bparse::reorder( i_struct.hash[ i ] );
for( auto i = 0; i < 0xB; ++i ) }
{ for( auto i = 0; i < 0xB; ++i )
xiv::utils::bparse::reorder( i_struct.padding[i] ); {
} xiv::utils::bparse::reorder( i_struct.padding[ i ] );
} }
} }
}
}; };
namespace xiv namespace xiv::dat
{
namespace dat
{ {
class SqPack class SqPack
{ {
public: public:
// Full path to the sqpack file // Full path to the sqpack file
SqPack( const std::filesystem::path& i_path ); SqPack( const std::filesystem::path& i_path );
virtual ~SqPack();
protected: virtual ~SqPack();
// Checks that a given block is valid iven its hash
void isBlockValid( uint32_t i_offset, uint32_t i_size, const SqPackBlockHash& i_block_hash );
// File handle protected:
std::ifstream m_handle; // Checks that a given block is valid iven its hash
}; void isBlockValid( uint32_t i_offset, uint32_t i_size, const SqPackBlockHash& i_block_hash );
// File handle
std::ifstream m_handle;
};
}
} }
#endif // XIV_DAT_SQPACK_H #endif // XIV_DAT_SQPACK_H

152
deps/datReader/bparse.h vendored
View file

@ -6,98 +6,96 @@
#include <sstream> #include <sstream>
#include <vector> #include <vector>
namespace xiv namespace xiv::utils::bparse
{
namespace utils
{
namespace bparse
{ {
// Internal macro for byteswapping // Internal macro for byteswapping
template <int N> template< int N >
void byteswap_impl(char (&bytes)[N]) void byteswap_impl( char (& bytes)[N] )
{ {
for( auto p = std::begin( bytes ), end = std::end( bytes ) - 1; p < end; ++p, --end ) for( auto p = std::begin( bytes ), end = std::end( bytes ) - 1; p < end; ++p, --end )
{ {
std::swap( *p, *end ); std::swap( *p, *end );
} }
} }
// byteswapping any type (no pointers to array) // byteswapping any type (no pointers to array)
template <typename T> template< typename T >
T byteswap(T value) T byteswap( T value )
{ {
byteswap_impl(*reinterpret_cast<char (*)[sizeof(T)]>(&value)); byteswap_impl( *reinterpret_cast<char ( * )[sizeof( T )]>(&value) );
return value; return value;
} }
// Read a struct from a stream // Read a struct from a stream
template <typename StructType> template< typename StructType >
void read(std::istream& i_stream, StructType& i_struct) void read( std::istream& i_stream, StructType& i_struct )
{ {
static_assert( std::is_pod<StructType>::value, "StructType must be a POD to be able to use read." ); static_assert( std::is_pod< StructType >::value, "StructType must be a POD to be able to use read." );
i_stream.read( reinterpret_cast<char*>( &i_struct ), sizeof( StructType ) ); i_stream.read( reinterpret_cast<char*>( &i_struct ), sizeof( StructType ) );
} }
// By default a type does not need reordering // By default a type does not need reordering
template <typename StructType> void reorder(StructType& i_struct) {} template< typename StructType >
void reorder( StructType& i_struct )
{
}
// "Overload" for passed struct as arg // "Overload" for passed struct as arg
template <typename StructType> template< typename StructType >
void extract(std::istream& i_stream, StructType& o_struct) void extract( std::istream& i_stream, StructType& o_struct )
{ {
read( i_stream, o_struct ); read( i_stream, o_struct );
reorder( o_struct ); reorder( o_struct );
} }
// This should not copy because of RVO // This should not copy because of RVO
// Extract a struct from a stream and log it // Extract a struct from a stream and log it
template <typename StructType> template< typename StructType >
StructType extract( std::istream& i_stream ) StructType extract( std::istream& i_stream )
{ {
StructType temp_struct; StructType temp_struct;
extract<StructType>( i_stream, temp_struct ); extract< StructType >( i_stream, temp_struct );
return temp_struct; return temp_struct;
} }
template <typename StructType> template< typename StructType >
void extract(std::istream& i_stream, uint32_t i_size, std::vector<StructType>& o_structs ) void extract( std::istream& i_stream, uint32_t i_size, std::vector< StructType >& o_structs )
{ {
o_structs.reserve( i_size ); o_structs.reserve( i_size );
for( uint32_t i = 0; i < i_size; ++i ) for( uint32_t i = 0; i < i_size; ++i )
{ {
o_structs.emplace_back( extract<StructType>( i_stream ) ); o_structs.emplace_back( extract< StructType >( i_stream ) );
} }
} }
// For simple (integral) types just provide name and endianness directly // For simple (integral) types just provide name and endianness directly
template <typename StructType> template< typename StructType >
StructType extract(std::istream& i_stream, const std::string& i_name, bool i_is_le = true) StructType extract( std::istream& i_stream, const std::string& i_name, bool i_is_le = true )
{ {
StructType temp_struct; StructType temp_struct;
read( i_stream, temp_struct ); read( i_stream, temp_struct );
if( !i_is_le ) if( !i_is_le )
{ {
temp_struct = byteswap( temp_struct ); temp_struct = byteswap( temp_struct );
} }
return temp_struct; return temp_struct;
} }
template <typename StructType> template< typename StructType >
void extract(std::istream& i_stream, const std::string& i_name, uint32_t i_size, std::vector<StructType>& o_structs, bool i_is_le = true) void extract( std::istream& i_stream, const std::string& i_name, uint32_t i_size, std::vector< StructType >& o_structs,
{ bool i_is_le = true )
o_structs.reserve( i_size ); {
for( uint32_t i = 0; i < i_size; ++i ) o_structs.reserve( i_size );
{ for( uint32_t i = 0; i < i_size; ++i )
o_structs.emplace_back( extract<StructType>( i_stream, i_name ) ); {
} o_structs.emplace_back( extract< StructType >( i_stream, i_name ) );
} }
}
// For cstrings // For cstrings
std::string extract_cstring( std::istream& i_stream, const std::string& i_name ); std::string extract_cstring( std::istream& i_stream, const std::string& i_name );
}
}
} }
#endif // XIV_UTILS_BPARSE_H #endif // XIV_UTILS_BPARSE_H

8
deps/datReader/conv.cpp vendored
View file

@ -1,8 +1,8 @@
#include "conv.h" #include "conv.h"
namespace xiv { namespace xiv::utils::conv
namespace utils { {
namespace conv {
float half2float( const uint16_t i_value ) float half2float( const uint16_t i_value )
{ {
uint32_t t1; uint32_t t1;
@ -30,6 +30,4 @@ namespace conv {
} }
} }
}
}

8
deps/datReader/conv.h vendored
View file

@ -5,13 +5,11 @@
#include <vector> #include <vector>
#include <ostream> #include <ostream>
namespace xiv { namespace xiv::utils::conv
namespace utils { {
namespace conv {
float half2float( const uint16_t i_value ); float half2float( const uint16_t i_value );
float ubyte2float( const uint8_t i_value ); float ubyte2float( const uint8_t i_value );
} }
}
}
#endif // XIV_UTILS_CONV_H #endif // XIV_UTILS_CONV_H

169
deps/datReader/crc32.cpp vendored
View file

@ -65,116 +65,111 @@ namespace internal
} }
} }
namespace xiv namespace xiv::utils::crc32
{
namespace utils
{
namespace crc32
{ {
uint32_t compute(const std::string& i_input, uint32_t init_crc) uint32_t compute( const std::string& i_input, uint32_t init_crc )
{ {
// Classical crc stuff // Classical crc stuff
auto& crc_table = internal::get_crc_table(); auto& crc_table = internal::get_crc_table();
auto crc = init_crc; auto crc = init_crc;
for(std::size_t i = 0; i < i_input.size(); ++i) for( std::size_t i = 0; i < i_input.size(); ++i )
{ {
crc = crc_table[(crc ^ i_input[i]) & 0xFF] ^ (crc >> 8); crc = crc_table[ ( crc ^ i_input[ i ] ) & 0xFF ] ^ ( crc >> 8 );
} }
return crc; return crc;
} }
uint32_t rev_compute(const std::string& i_input, uint32_t init_crc) uint32_t rev_compute( const std::string& i_input, uint32_t init_crc )
{ {
auto& rev_crc_table = internal::get_rev_crc_table(); auto& rev_crc_table = internal::get_rev_crc_table();
auto crc = init_crc; auto crc = init_crc;
const auto input_size = i_input.size(); const auto input_size = i_input.size();
// Reverse crc // Reverse crc
for(auto i = input_size; i > 0; --i) for( auto i = input_size; i > 0; --i )
{ {
crc = rev_crc_table[crc >> 24] ^ ((crc << 8) & 0xFFFFFF00) ^ i_input[input_size - i - 1]; crc = rev_crc_table[ crc >> 24 ] ^ ( ( crc << 8 ) & 0xFFFFFF00 ) ^ i_input[ input_size - i - 1 ];
} }
// Compute the 4 bytes needed for this init_crc // Compute the 4 bytes needed for this init_crc
for (auto i = 0; i < 4; ++i) for( auto i = 0; i < 4; ++i )
{ {
crc = rev_crc_table[crc >> 24] ^ ((crc << 8) & 0xFFFFFF00); crc = rev_crc_table[ crc >> 24 ] ^ ( ( crc << 8 ) & 0xFFFFFF00 );
} }
return crc; return crc;
} }
void generate_hashes_1(std::string& i_format, const uint32_t i_first_index, std::vector<uint32_t>& o_hashes) void generate_hashes_1( std::string& i_format, const uint32_t i_first_index, std::vector< uint32_t >& o_hashes )
{ {
char* str = const_cast<char*>(i_format.data()); char* str = const_cast<char*>(i_format.data());
const uint32_t str_size = i_format.size(); const uint32_t str_size = i_format.size();
o_hashes.resize(10000); o_hashes.resize( 10000 );
uint32_t i = 0; uint32_t i = 0;
for (char a = '0'; a <= '9'; ++a) for( char a = '0'; a <= '9'; ++a )
{ {
str[i_first_index] = a; str[ i_first_index ] = a;
for (char b = '0'; b <= '9'; ++b) for( char b = '0'; b <= '9'; ++b )
{ {
str[i_first_index + 1] = b; str[ i_first_index + 1 ] = b;
for (char c = '0'; c <= '9'; ++c) for( char c = '0'; c <= '9'; ++c )
{ {
str[i_first_index + 2] = c; str[ i_first_index + 2 ] = c;
for (char d = '0'; d <= '9'; ++d) for( char d = '0'; d <= '9'; ++d )
{ {
str[i_first_index + 3] = d; str[ i_first_index + 3 ] = d;
o_hashes[i] = ::crc32(0, reinterpret_cast<uint8_t*>(&(str[0])), str_size) ^ 0xFFFFFFFF; o_hashes[ i ] = ::crc32( 0, reinterpret_cast<uint8_t*>(&( str[ 0 ] )), str_size ) ^ 0xFFFFFFFF;
++i; ++i;
} }
} }
} }
} }
} }
void generate_hashes_2(std::string& i_format, const uint32_t i_first_index, const uint32_t i_second_index, std::vector<uint32_t>& o_hashes) void generate_hashes_2( std::string& i_format, const uint32_t i_first_index, const uint32_t i_second_index,
{ std::vector< uint32_t >& o_hashes )
char* str = const_cast<char*>(i_format.data()); {
const uint32_t str_size = i_format.size(); char* str = const_cast<char*>(i_format.data());
const uint32_t str_size = i_format.size();
o_hashes.resize(100000000); o_hashes.resize( 100000000 );
uint32_t i = 0; uint32_t i = 0;
for (char a = '0'; a <= '9'; ++a) for( char a = '0'; a <= '9'; ++a )
{ {
str[i_first_index] = a; str[ i_first_index ] = a;
for (char b = '0'; b <= '9'; ++b) for( char b = '0'; b <= '9'; ++b )
{ {
str[i_first_index + 1] = b; str[ i_first_index + 1 ] = b;
for (char c = '0'; c <= '9'; ++c) for( char c = '0'; c <= '9'; ++c )
{ {
str[i_first_index + 2] = c; str[ i_first_index + 2 ] = c;
for (char d = '0'; d <= '9'; ++d) for( char d = '0'; d <= '9'; ++d )
{ {
str[i_first_index + 3] = d; str[ i_first_index + 3 ] = d;
for (char e = '0'; e <= '9'; ++e) for( char e = '0'; e <= '9'; ++e )
{ {
str[i_second_index] = e; str[ i_second_index ] = e;
for (char f = '0'; f <= '9'; ++f) for( char f = '0'; f <= '9'; ++f )
{ {
str[i_second_index + 1] = f; str[ i_second_index + 1 ] = f;
for (char g = '0'; g <= '9'; ++g) for( char g = '0'; g <= '9'; ++g )
{ {
str[i_second_index + 2] = g; str[ i_second_index + 2 ] = g;
for (char h = '0'; h <= '9'; ++h) for( char h = '0'; h <= '9'; ++h )
{ {
str[i_second_index + 3] = h; str[ i_second_index + 3 ] = h;
o_hashes[i] = ::crc32(0, reinterpret_cast<uint8_t*>(&(str[0])), str_size) ^ 0xFFFFFFFF; o_hashes[ i ] = ::crc32( 0, reinterpret_cast<uint8_t*>(&( str[ 0 ] )), str_size ) ^ 0xFFFFFFFF;
++i; ++i;
} }
} }
} }
} }
} }
} }
} }
} }
} }
} }
}
}

11
deps/datReader/crc32.h vendored
View file

@ -5,9 +5,8 @@
#include <vector> #include <vector>
#include <string> #include <string>
namespace xiv { namespace xiv::utils::crc32
namespace utils { {
namespace crc32 {
// Normal crc32 computation from a given intial crc value, use zlib.crc32 instead, the final XOR 0xFFFFFFFF is not done // Normal crc32 computation from a given intial crc value, use zlib.crc32 instead, the final XOR 0xFFFFFFFF is not done
uint32_t compute( const std::string& i_input, uint32_t init_crc = 0xFFFFFFFF ); uint32_t compute( const std::string& i_input, uint32_t init_crc = 0xFFFFFFFF );
@ -17,10 +16,10 @@ namespace crc32 {
uint32_t rev_compute( const std::string& i_input, uint32_t init_crc = 0 ); uint32_t rev_compute( const std::string& i_input, uint32_t init_crc = 0 );
void generate_hashes_1( std::string& i_format, const uint32_t i_first_index, std::vector< uint32_t >& o_hashes ); void generate_hashes_1( std::string& i_format, const uint32_t i_first_index, std::vector< uint32_t >& o_hashes );
void generate_hashes_2( std::string& i_format, const uint32_t i_first_index, const uint32_t i_second_index, std::vector< uint32_t >& o_hashes );
} void generate_hashes_2( std::string& i_format, const uint32_t i_first_index, const uint32_t i_second_index,
} std::vector< uint32_t >& o_hashes );
} }
#endif // XIV_UTILS_CRC32_H #endif // XIV_UTILS_CRC32_H

8
deps/datReader/stream.cpp vendored
View file

@ -4,13 +4,7 @@
#include <sstream> #include <sstream>
#include <streambuf> #include <streambuf>
namespace xiv namespace xiv::utils::stream
{
namespace utils
{
namespace stream
{ {
} }
}
}

24
deps/datReader/stream.h vendored
View file

@ -5,23 +5,17 @@
#include <iostream> #include <iostream>
#include <vector> #include <vector>
namespace xiv namespace xiv::utils::stream
{ {
namespace utils template< typename CharT, typename TraitsT = std::char_traits< CharT > >
{ class vectorwrapbuf :
namespace stream public std::basic_streambuf< CharT, TraitsT >
{ {
template<typename CharT, typename TraitsT = std::char_traits<CharT> > public:
class vectorwrapbuf : public std::basic_streambuf<CharT, TraitsT> vectorwrapbuf( std::vector< CharT >& vec )
{
public:
vectorwrapbuf(std::vector<CharT> &vec)
{ {
this->setg(vec.data(), vec.data(), vec.data() + vec.size()); this->setg( vec.data(), vec.data(), vec.data() + vec.size() );
} }
}; };
} }
}
}
#endif // XIV_UTILS_STREAM_H #endif // XIV_UTILS_STREAM_H

48
deps/datReader/zlib.cpp vendored
View file

@ -4,32 +4,28 @@
#include <zlib/zlib.h> #include <zlib/zlib.h>
#include <vector> #include <vector>
namespace xiv namespace xiv::utils::zlib
{
namespace utils
{
namespace zlib
{ {
void compress(const std::vector<char>& in, std::vector<char>& out) void compress( const std::vector< char >& in, std::vector< char >& out )
{ {
// Fetching upper bound for out size // Fetching upper bound for out size
auto out_size = compressBound(in.size()); auto out_size = compressBound( in.size() );
out.resize(out_size); out.resize( out_size );
auto ret = compress2(reinterpret_cast<uint8_t*>(out.data()), &out_size, auto ret = compress2( reinterpret_cast<uint8_t*>(out.data()), &out_size,
reinterpret_cast<const uint8_t*>(in.data()), in.size(), Z_BEST_COMPRESSION); reinterpret_cast<const uint8_t*>(in.data()), in.size(), Z_BEST_COMPRESSION );
if (ret != Z_OK) if( ret != Z_OK )
{ {
throw std::runtime_error("Error at zlib uncompress: " + std::to_string(ret)); throw std::runtime_error( "Error at zlib uncompress: " + std::to_string( ret ) );
} }
out.resize(out_size); out.resize( out_size );
} }
void no_header_decompress(uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t out_size) void no_header_decompress( uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t out_size )
{ {
z_stream strm; z_stream strm;
strm.zalloc = Z_NULL; strm.zalloc = Z_NULL;
strm.zfree = Z_NULL; strm.zfree = Z_NULL;
@ -38,10 +34,10 @@ void no_header_decompress(uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t
strm.next_in = Z_NULL; strm.next_in = Z_NULL;
// Init with -15 because we do not have header in this compressed data // Init with -15 because we do not have header in this compressed data
auto ret = inflateInit2(&strm, -15); auto ret = inflateInit2( &strm, -15 );
if (ret != Z_OK) if( ret != Z_OK )
{ {
throw std::runtime_error("Error at zlib init: " + std::to_string(ret)); throw std::runtime_error( "Error at zlib init: " + std::to_string( ret ) );
} }
// Set pointers to the right addresses // Set pointers to the right addresses
@ -50,16 +46,14 @@ void no_header_decompress(uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t
strm.next_out = out; strm.next_out = out;
// Effectively decompress data // Effectively decompress data
ret = inflate(&strm, Z_NO_FLUSH); ret = inflate( &strm, Z_NO_FLUSH );
if (ret != Z_STREAM_END) if( ret != Z_STREAM_END )
{ {
throw std::runtime_error("Error at zlib inflate: " + std::to_string(ret)); throw std::runtime_error( "Error at zlib inflate: " + std::to_string( ret ) );
} }
// Clean up // Clean up
inflateEnd(&strm); inflateEnd( &strm );
} }
} }
}
}

13
deps/datReader/zlib.h vendored
View file

@ -4,18 +4,13 @@
#include <cstdint> #include <cstdint>
#include <vector> #include <vector>
namespace xiv namespace xiv::utils::zlib
{
namespace utils
{
namespace zlib
{ {
void compress(const std::vector<char>& in, std::vector<char>& out); void compress( const std::vector< char >& in, std::vector< char >& out );
void no_header_decompress(uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t out_size);
void no_header_decompress( uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t out_size );
}
}
} }
#endif // XIV_UTILS_ZLIB_H #endif // XIV_UTILS_ZLIB_H