mirrors/sapphire
1
Fork 0
mirror of https://github.com/SapphireServer/Sapphire.git synced 2025-05-12 13:37:46 +00:00
Code Activity

loads of datreader cleanup/consistency fixes

Browse source
This commit is contained in:
NotAdam 2020-02-10 14:05:04 +11:00
parent b7ddb054f4
commit 5f18c43dd6
27 changed files with 1727 additions and 1734 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

265
deps/datReader/Dat.cpp vendored
View file

@ -4,14 +4,11 @@
#include "File.h"
namespace
{
namespace {
const uint32_t model_section_count = 0xB;
}
namespace xiv
{
namespace dat
namespace xiv::dat
{
struct DatFileHeader
{
@ -63,131 +60,145 @@ namespace dat
uint32_t block_id;
uint32_t block_count;
};
}
}
namespace xiv
namespace xiv::utils::bparse
{
namespace utils
{
namespace bparse
{
template <>
inline void reorder<xiv::dat::DatFileHeader>(xiv::dat::DatFileHeader& i_struct)
template<>
inline void reorder< xiv::dat::DatFileHeader >( xiv::dat::DatFileHeader& i_struct )
{
xiv::utils::bparse::reorder(i_struct.size);
xiv::utils::bparse::reorder(i_struct.entry_type);
xiv::utils::bparse::reorder(i_struct.total_uncompressed_size);
for (int32_t i = 0; i < 0x2; ++i) { xiv::utils::bparse::reorder(i_struct.unknown[i]); }
xiv::utils::bparse::reorder( i_struct.size );
xiv::utils::bparse::reorder( i_struct.entry_type );
xiv::utils::bparse::reorder( i_struct.total_uncompressed_size );
for( int32_t i = 0; i < 0x2; ++i )
{
xiv::utils::bparse::reorder( i_struct.unknown[ i ] );
}
}
template <>
inline void reorder<xiv::dat::DatBlockRecord>(xiv::dat::DatBlockRecord& i_struct)
template<>
inline void reorder< xiv::dat::DatBlockRecord >( xiv::dat::DatBlockRecord& i_struct )
{
xiv::utils::bparse::reorder(i_struct.offset);
xiv::utils::bparse::reorder(i_struct.size);
for (int32_t i = 0; i < 0x4; ++i) { xiv::utils::bparse::reorder(i_struct.unknown[i]); }
xiv::utils::bparse::reorder(i_struct.block_hash);
xiv::utils::bparse::reorder( i_struct.offset );
xiv::utils::bparse::reorder( i_struct.size );
for( int32_t i = 0; i < 0x4; ++i )
{
xiv::utils::bparse::reorder( i_struct.unknown[ i ] );
}
xiv::utils::bparse::reorder( i_struct.block_hash );
}
template <>
inline void reorder<xiv::dat::DatBlockHeader>(xiv::dat::DatBlockHeader& i_struct)
template<>
inline void reorder< xiv::dat::DatBlockHeader >( xiv::dat::DatBlockHeader& i_struct )
{
xiv::utils::bparse::reorder(i_struct.size);
xiv::utils::bparse::reorder(i_struct.unknown1);
xiv::utils::bparse::reorder(i_struct.compressed_size);
xiv::utils::bparse::reorder(i_struct.uncompressed_size);
xiv::utils::bparse::reorder( i_struct.size );
xiv::utils::bparse::reorder( i_struct.unknown1 );
xiv::utils::bparse::reorder( i_struct.compressed_size );
xiv::utils::bparse::reorder( i_struct.uncompressed_size );
}
template <>
inline void reorder<xiv::dat::DatStdFileBlockInfos>(xiv::dat::DatStdFileBlockInfos& i_struct)
template<>
inline void reorder< xiv::dat::DatStdFileBlockInfos >( xiv::dat::DatStdFileBlockInfos& i_struct )
{
xiv::utils::bparse::reorder(i_struct.offset);
xiv::utils::bparse::reorder(i_struct.size);
xiv::utils::bparse::reorder(i_struct.uncompressed_size);
xiv::utils::bparse::reorder( i_struct.offset );
xiv::utils::bparse::reorder( i_struct.size );
xiv::utils::bparse::reorder( i_struct.uncompressed_size );
}
template <>
inline void reorder<xiv::dat::DatMdlFileBlockInfos>(xiv::dat::DatMdlFileBlockInfos& i_struct)
template<>
inline void reorder< xiv::dat::DatMdlFileBlockInfos >( xiv::dat::DatMdlFileBlockInfos& i_struct )
{
xiv::utils::bparse::reorder(i_struct.unknown1);
for (auto i = 0; i < ::model_section_count; ++i) { xiv::utils::bparse::reorder(i_struct.uncompressed_sizes[i]); }
for (auto i = 0; i < ::model_section_count; ++i) { xiv::utils::bparse::reorder(i_struct.compressed_sizes[i]); }
for (auto i = 0; i < ::model_section_count; ++i) { xiv::utils::bparse::reorder(i_struct.offsets[i]); }
for (auto i = 0; i < ::model_section_count; ++i) { xiv::utils::bparse::reorder(i_struct.block_ids[i]); }
for (auto i = 0; i < ::model_section_count; ++i) { xiv::utils::bparse::reorder(i_struct.block_counts[i]); }
for (auto i = 0; i < 0x2; ++i) { xiv::utils::bparse::reorder(i_struct.unknown2[i]); }
xiv::utils::bparse::reorder( i_struct.unknown1 );
for( auto i = 0; i < ::model_section_count; ++i )
{
xiv::utils::bparse::reorder( i_struct.uncompressed_sizes[ i ] );
}
for( auto i = 0; i < ::model_section_count; ++i )
{
xiv::utils::bparse::reorder( i_struct.compressed_sizes[ i ] );
}
for( auto i = 0; i < ::model_section_count; ++i )
{
xiv::utils::bparse::reorder( i_struct.offsets[ i ] );
}
for( auto i = 0; i < ::model_section_count; ++i )
{
xiv::utils::bparse::reorder( i_struct.block_ids[ i ] );
}
for( auto i = 0; i < ::model_section_count; ++i )
{
xiv::utils::bparse::reorder( i_struct.block_counts[ i ] );
}
for( auto i = 0; i < 0x2; ++i )
{
xiv::utils::bparse::reorder( i_struct.unknown2[ i ] );
}
}
template <>
inline void reorder<xiv::dat::DatTexFileBlockInfos>(xiv::dat::DatTexFileBlockInfos& i_struct)
template<>
inline void reorder< xiv::dat::DatTexFileBlockInfos >( xiv::dat::DatTexFileBlockInfos& i_struct )
{
xiv::utils::bparse::reorder(i_struct.offset);
xiv::utils::bparse::reorder(i_struct.size);
xiv::utils::bparse::reorder(i_struct.uncompressed_size);
xiv::utils::bparse::reorder(i_struct.block_id);
xiv::utils::bparse::reorder(i_struct.block_count);
xiv::utils::bparse::reorder( i_struct.offset );
xiv::utils::bparse::reorder( i_struct.size );
xiv::utils::bparse::reorder( i_struct.uncompressed_size );
xiv::utils::bparse::reorder( i_struct.block_id );
xiv::utils::bparse::reorder( i_struct.block_count );
}
}
}
};
using xiv::utils::bparse::extract;
namespace xiv
{
namespace dat
namespace xiv::dat
{
Dat::Dat( const std::filesystem::path& i_path, uint32_t i_nb ) :
Dat::Dat( const std::filesystem::path& i_path, uint32_t i_nb ) :
SqPack( i_path ),
m_num( i_nb )
{
auto block_record = extract<DatBlockRecord>(m_handle);
{
auto block_record = extract< DatBlockRecord >( m_handle );
block_record.offset *= 0x80;
isBlockValid(block_record.offset, block_record.size, block_record.block_hash);
}
isBlockValid( block_record.offset, block_record.size, block_record.block_hash );
}
Dat::~Dat()
{
}
Dat::~Dat()
{
}
std::unique_ptr<File> Dat::getFile( uint32_t i_offset )
{
std::unique_ptr<File> outputFile(new File());
std::unique_ptr< File > Dat::getFile( uint32_t i_offset )
{
std::unique_ptr< File > outputFile( new File() );
{
// Lock in this scope
std::lock_guard<std::mutex> lock(m_fileMutex);
std::lock_guard< std::mutex > lock( m_fileMutex );
// Seek to the start of the header of the file record and extract it
m_handle.seekg(i_offset);
auto file_header = extract<DatFileHeader>(m_handle);
m_handle.seekg( i_offset );
auto file_header = extract< DatFileHeader >( m_handle );
switch (file_header.entry_type)
switch( file_header.entry_type )
{
case FileType::empty:
throw std::runtime_error("File is empty");
throw std::runtime_error( "File is empty" );
case FileType::standard:
{
outputFile->_type = FileType::standard;
uint32_t number_of_blocks = extract<uint32_t>(m_handle, "number_of_blocks");
uint32_t number_of_blocks = extract< uint32_t >( m_handle, "number_of_blocks" );
// Just extract offset infos for the blocks to extract
std::vector<DatStdFileBlockInfos> std_file_block_infos;
extract<DatStdFileBlockInfos>( m_handle, number_of_blocks, std_file_block_infos );
std::vector< DatStdFileBlockInfos > std_file_block_infos;
extract< DatStdFileBlockInfos >( m_handle, number_of_blocks, std_file_block_infos );
// Pre allocate data vector for the whole file
outputFile->_data_sections.resize(1);
outputFile->_data_sections.resize( 1 );
auto& data_section = outputFile->_data_sections.front();
data_section.reserve(file_header.total_uncompressed_size);
data_section.reserve( file_header.total_uncompressed_size );
// Extract each block
for (auto& file_block_info : std_file_block_infos)
for( auto& file_block_info : std_file_block_infos )
{
extractBlock(i_offset + file_header.size + file_block_info.offset, data_section);
extractBlock( i_offset + file_header.size + file_block_info.offset, data_section );
}
}
break;
@ -196,28 +207,28 @@ std::unique_ptr<File> Dat::getFile( uint32_t i_offset )
{
outputFile->_type = FileType::model;
DatMdlFileBlockInfos mdlBlockInfo = extract<DatMdlFileBlockInfos>(m_handle);
DatMdlFileBlockInfos mdlBlockInfo = extract< DatMdlFileBlockInfos >( m_handle );
// Getting the block number and read their sizes
const uint32_t block_count = mdlBlockInfo.block_ids[::model_section_count - 1] +
mdlBlockInfo.block_counts[::model_section_count - 1];
std::vector<uint16_t> block_sizes;
extract<uint16_t>(m_handle, "block_size", block_count, block_sizes);
const uint32_t block_count = mdlBlockInfo.block_ids[ ::model_section_count - 1 ] +
mdlBlockInfo.block_counts[ ::model_section_count - 1 ];
std::vector< uint16_t > block_sizes;
extract< uint16_t >( m_handle, "block_size", block_count, block_sizes );
// Preallocate sufficient space
outputFile->_data_sections.resize(::model_section_count);
outputFile->_data_sections.resize( ::model_section_count );
for (uint32_t i = 0; i < ::model_section_count; ++i)
for( uint32_t i = 0; i < ::model_section_count; ++i )
{
// Preallocating for section
auto& data_section = outputFile->_data_sections[i];
data_section.reserve(mdlBlockInfo.uncompressed_sizes[i]);
auto& data_section = outputFile->_data_sections[ i ];
data_section.reserve( mdlBlockInfo.uncompressed_sizes[ i ] );
uint32_t current_offset = i_offset + file_header.size + mdlBlockInfo.offsets[i];
for (uint32_t j = 0; j < mdlBlockInfo.block_counts[i]; ++j)
uint32_t current_offset = i_offset + file_header.size + mdlBlockInfo.offsets[ i ];
for( uint32_t j = 0; j < mdlBlockInfo.block_counts[ i ]; ++j )
{
extractBlock(current_offset, data_section);
current_offset += block_sizes[mdlBlockInfo.block_ids[i] + j];
extractBlock( current_offset, data_section );
current_offset += block_sizes[ mdlBlockInfo.block_ids[ i ] + j ];
}
}
}
@ -228,84 +239,84 @@ std::unique_ptr<File> Dat::getFile( uint32_t i_offset )
outputFile->_type = FileType::texture;
// Extracts mipmap entries and the block sizes
uint32_t sectionCount = extract<uint32_t>(m_handle, "sections_count");
uint32_t sectionCount = extract< uint32_t >( m_handle, "sections_count" );
std::vector<DatTexFileBlockInfos> texBlockInfo;
extract<DatTexFileBlockInfos>(m_handle, sectionCount, texBlockInfo);
std::vector< DatTexFileBlockInfos > texBlockInfo;
extract< DatTexFileBlockInfos >( m_handle, sectionCount, texBlockInfo );
// Extracting block sizes
uint32_t block_count = texBlockInfo.back().block_id + texBlockInfo.back().block_count;
std::vector<uint16_t> block_sizes;
extract<uint16_t>(m_handle, "block_size", block_count, block_sizes);
std::vector< uint16_t > block_sizes;
extract< uint16_t >( m_handle, "block_size", block_count, block_sizes );
outputFile->_data_sections.resize(sectionCount + 1);
outputFile->_data_sections.resize( sectionCount + 1 );
// Extracting header in section 0
const uint32_t header_size = texBlockInfo.front().offset;
auto& header_section = outputFile->_data_sections[0];
header_section.resize(header_size);
auto& header_section = outputFile->_data_sections[ 0 ];
header_section.resize( header_size );
m_handle.seekg(i_offset + file_header.size);
m_handle.read(header_section.data(), header_size);
m_handle.seekg( i_offset + file_header.size );
m_handle.read( header_section.data(), header_size );
// Extracting other sections
for (uint32_t i = 0; i < sectionCount; ++i)
for( uint32_t i = 0; i < sectionCount; ++i )
{
auto& data_section = outputFile->_data_sections[i + 1];
auto& section_infos = texBlockInfo[i];
data_section.reserve(section_infos.uncompressed_size);
auto& data_section = outputFile->_data_sections[ i + 1 ];
auto& section_infos = texBlockInfo[ i ];
data_section.reserve( section_infos.uncompressed_size );
uint32_t current_offset = i_offset + file_header.size + section_infos.offset;
for (uint32_t j = 0; j < section_infos.block_count; ++j)
for( uint32_t j = 0; j < section_infos.block_count; ++j )
{
extractBlock(current_offset, data_section);
current_offset += block_sizes[section_infos.block_id + j];
extractBlock( current_offset, data_section );
current_offset += block_sizes[ section_infos.block_id + j ];
}
}
}
break;
default:
throw std::runtime_error("Invalid entry_type: " + std::to_string(static_cast<uint32_t>(file_header.entry_type)));
throw std::runtime_error(
"Invalid entry_type: " + std::to_string( static_cast<uint32_t>(file_header.entry_type) ) );
}
}
return outputFile;
}
}
void Dat::extractBlock( uint32_t i_offset, std::vector<char>& o_data )
{
m_handle.seekg(i_offset);
void Dat::extractBlock( uint32_t i_offset, std::vector< char >& o_data )
{
m_handle.seekg( i_offset );
DatBlockHeader block_header = extract<DatBlockHeader>(m_handle);
DatBlockHeader block_header = extract< DatBlockHeader >( m_handle );
// Resizing the vector to write directly into it
const uint32_t data_size = o_data.size();
o_data.resize(data_size + block_header.uncompressed_size);
o_data.resize( data_size + block_header.uncompressed_size );
// 32000 in compressed_size means it is not compressed so take uncompressed_size
if (block_header.compressed_size == 32000)
if( block_header.compressed_size == 32000 )
{
m_handle.read(o_data.data() + data_size, block_header.uncompressed_size);
m_handle.read( o_data.data() + data_size, block_header.uncompressed_size );
}
else
{
// If it is compressed use zlib
// Read the data to be decompressed
std::vector<char> temp_buffer(block_header.compressed_size);
m_handle.read(temp_buffer.data(), block_header.compressed_size);
std::vector< char > temp_buffer( block_header.compressed_size );
m_handle.read( temp_buffer.data(), block_header.compressed_size );
utils::zlib::no_header_decompress(reinterpret_cast<uint8_t*>(temp_buffer.data()),
utils::zlib::no_header_decompress( reinterpret_cast<uint8_t*>(temp_buffer.data()),
temp_buffer.size(),
reinterpret_cast<uint8_t*>(o_data.data() + data_size),
block_header.uncompressed_size);
block_header.uncompressed_size );
}
}
}
uint32_t Dat::getNum() const
{
uint32_t Dat::getNum() const
{
return m_num;
}
}
}
}

17
deps/datReader/Dat.h vendored
View file

@ -7,16 +7,14 @@
#include <filesystem>
namespace xiv
{
namespace dat
namespace xiv::dat
{
class File;
class File;
class Dat : public SqPack
{
public:
class Dat : public SqPack
{
public:
// Full path to the dat file
Dat( const std::filesystem::path& i_path, uint32_t i_nb );
virtual ~Dat();
@ -31,15 +29,14 @@ public:
// Returns the dat number
uint32_t getNum() const;
protected:
protected:
// File reading mutex to have only one thread reading the file at a time
std::mutex m_fileMutex;
// Dat nb
uint32_t m_num;
};
};
}
}
#endif // XIV_DAT_DAT_H

6
deps/datReader/DatCat.cpp vendored
View file

@ -26,7 +26,7 @@ Cat::Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::str
// For all dat files linked to this index, create it: XX0000.win32.datX
for( uint32_t i = 0; i < getIndex().getDatCount(); ++i )
{
m_dats.emplace_back( std::unique_ptr<Dat>( new Dat( basePath / "ffxiv" / ( prefix + ".dat" + std::to_string( i ) ), i ) ) );
m_dats.emplace_back( std::make_unique< Dat >(basePath / "ffxiv" / ( prefix + ".dat" + std::to_string( i ) ), i ) );
}
}
@ -36,12 +36,12 @@ Cat::Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::str
m_chunk( chunk )
{
// Creates the index: XX0000.win32.index
m_index = std::unique_ptr<Index>( new Index( basePath / GameData::buildDatStr( "ex" + std::to_string( exNum ), catNum, exNum, chunk, "win32", "index" ) ) );
m_index = std::make_unique< Index >( basePath / GameData::buildDatStr( "ex" + std::to_string( exNum ), catNum, exNum, chunk, "win32", "index" ) );
// For all dat files linked to this index, create it: XX0000.win32.datX
for( uint32_t i = 0; i < getIndex().getDatCount(); ++i )
{
m_dats.emplace_back( std::unique_ptr<Dat>( new Dat( basePath / GameData::buildDatStr( "ex" + std::to_string( exNum ), catNum, exNum, chunk, "win32", "dat" + std::to_string( i ) ), i ) ) );
m_dats.emplace_back( std::make_unique< Dat >( basePath / GameData::buildDatStr( "ex" + std::to_string( exNum ), catNum, exNum, chunk, "win32", "dat" + std::to_string( i ) ), i ) );
}
}

38
deps/datReader/DatCat.h vendored
View file

@ -5,17 +5,19 @@
#include <vector>
#include <filesystem>
namespace xiv {
namespace dat {
class Index;
class Dat;
class File;
// A category represents an .index and its associated .datX
class Cat
namespace xiv::dat
{
public:
class Index;
class Dat;
class File;
// A category represents an .index and its associated .datX
class Cat
{
public:
// basePath: Path to the folder containingthe datfiles
// catNum: The number of the category
// name: The name of the category, empty if not known
@ -26,17 +28,20 @@ public:
// name: The name of the category, empty if not known
// exNum: The number of the expansion to load from
// chunk: The chunk to load from
Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::string& name, uint32_t exNum, uint32_t chunk );
Cat( const std::filesystem::path& basePath, uint32_t catNum, const std::string& name, uint32_t exNum,
uint32_t chunk );
~Cat();
// Returns .index of the category
const Index& getIndex() const;
// Retrieve a file from the category given its hashes
std::unique_ptr<File> getFile( uint32_t dir_hash, uint32_t filename_hash ) const;
std::unique_ptr< File > getFile( uint32_t dir_hash, uint32_t filename_hash ) const;
bool doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const;
bool doesDirExist( uint32_t dir_hash ) const;
@ -46,19 +51,18 @@ public:
// Returns the number of the category
uint32_t getCatNum() const;
protected:
protected:
const std::string m_name;
const uint32_t m_catNum;
const uint32_t m_chunk;
// The .index
std::unique_ptr<Index> m_index;
std::unique_ptr< Index > m_index;
// The .datXs such as dat nb X => m_dats[X]
std::vector<std::unique_ptr<Dat>> m_dats;
};
std::vector< std::unique_ptr< Dat>> m_dats;
};
}
}
#endif // XIV_DAT_CAT_H

129
deps/datReader/Exd.cpp vendored
View file

@ -8,11 +8,8 @@
using xiv::utils::bparse::extract;
namespace xiv
namespace xiv::exd
{
namespace exd
{
struct ExdHeader
{
char magic[0x4];
@ -26,28 +23,37 @@ namespace xiv
uint32_t id;
uint32_t offset;
};
}
}
namespace xiv
{
namespace utils
namespace xiv::utils::bparse {
template<>
inline void reorder< xiv::exd::ExdHeader >( xiv::exd::ExdHeader& i_struct )
{
namespace bparse
for( int32_t i = 0; i < 0x4; ++i )
{
template <> inline void reorder<xiv::exd::ExdHeader>( xiv::exd::ExdHeader& i_struct ) { for( int32_t i = 0; i < 0x4; ++i ) { xiv::utils::bparse::reorder( i_struct.magic[i] ); } i_struct.unknown = xiv::utils::bparse::byteswap( i_struct.unknown ); xiv::utils::bparse::reorder( i_struct.unknown ); i_struct.unknown2 = xiv::utils::bparse::byteswap( i_struct.unknown2 ); xiv::utils::bparse::reorder( i_struct.unknown2 ); i_struct.index_size = xiv::utils::bparse::byteswap( i_struct.index_size ); xiv::utils::bparse::reorder( i_struct.index_size ); }
template <> inline void reorder<xiv::exd::ExdRecordIndex>( xiv::exd::ExdRecordIndex& i_struct ) { i_struct.id = xiv::utils::bparse::byteswap( i_struct.id ); xiv::utils::bparse::reorder( i_struct.id ); i_struct.offset = xiv::utils::bparse::byteswap( i_struct.offset ); xiv::utils::bparse::reorder( i_struct.offset ); }
xiv::utils::bparse::reorder( i_struct.magic[ i ] );
}
i_struct.unknown = xiv::utils::bparse::byteswap( i_struct.unknown );
xiv::utils::bparse::reorder( i_struct.unknown );
i_struct.unknown2 = xiv::utils::bparse::byteswap( i_struct.unknown2 );
xiv::utils::bparse::reorder( i_struct.unknown2 );
i_struct.index_size = xiv::utils::bparse::byteswap( i_struct.index_size );
xiv::utils::bparse::reorder( i_struct.index_size );
}
template<>
inline void reorder< xiv::exd::ExdRecordIndex >( xiv::exd::ExdRecordIndex& i_struct )
{
i_struct.id = xiv::utils::bparse::byteswap( i_struct.id );
xiv::utils::bparse::reorder( i_struct.id );
i_struct.offset = xiv::utils::bparse::byteswap( i_struct.offset );
xiv::utils::bparse::reorder( i_struct.offset );
}
};
namespace xiv
namespace xiv::exd
{
namespace exd
{
Exd::Exd( std::shared_ptr<Exh> i_exh, const std::vector<std::shared_ptr<dat::File>>& i_files )
Exd::Exd( std::shared_ptr< Exh > i_exh, const std::vector< std::shared_ptr< dat::File>>& i_files )
{
_exh = i_exh;
_files = i_files;
@ -55,7 +61,7 @@ namespace xiv
// Iterates over all the files
const uint32_t member_count = _exh->get_members().size();
for ( auto &file_ptr : _files )
for( auto& file_ptr : _files )
{
// Get a stream
std::vector< char > dataCpy = file_ptr->get_data_sections().front();
@ -69,10 +75,10 @@ namespace xiv
const uint32_t record_count = exd_header.index_size / sizeof( ExdRecordIndex );
std::vector< ExdRecordIndex > record_indices;
record_indices.reserve( record_count );
for ( uint32_t i = 0; i < record_count; ++i )
for( uint32_t i = 0; i < record_count; ++i )
{
auto recordIndex = extract< ExdRecordIndex >( iss );
_idCache[recordIndex.id] = ExdCacheEntry{file_ptr, recordIndex.offset};
_idCache[ recordIndex.id ] = ExdCacheEntry{ file_ptr, recordIndex.offset };
}
}
}
@ -81,7 +87,7 @@ namespace xiv
{
}
const std::vector<Field> Exd::get_row( uint32_t id, uint32_t subRow )
const std::vector< Field > Exd::get_row( uint32_t id, uint32_t subRow )
{
auto cacheEntryIt = _idCache.find( id );
@ -96,7 +102,7 @@ namespace xiv
std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
// Get the vector fields for the given record and preallocate it
auto fields = _data[id];
auto fields = _data[ id ];
fields.reserve( member_count );
iss.seekg( cacheEntryIt->second.offset + 6 );
@ -128,45 +134,45 @@ namespace xiv
break;
case DataType::boolean:
fields.emplace_back( extract<bool>( iss, "bool" ) );
fields.emplace_back( extract< bool >( iss, "bool" ) );
break;
case DataType::int8:
fields.emplace_back( extract<int8_t>( iss, "int8_t" ) );
fields.emplace_back( extract< int8_t >( iss, "int8_t" ) );
break;
case DataType::uint8:
fields.emplace_back( extract<uint8_t>( iss, "uint8_t" ) );
fields.emplace_back( extract< uint8_t >( iss, "uint8_t" ) );
break;
case DataType::int16:
fields.emplace_back( extract<int16_t>( iss, "int16_t", false ) );
fields.emplace_back( extract< int16_t >( iss, "int16_t", false ) );
break;
case DataType::uint16:
fields.emplace_back( extract<uint16_t>( iss, "uint16_t", false ) );
fields.emplace_back( extract< uint16_t >( iss, "uint16_t", false ) );
break;
case DataType::int32:
fields.emplace_back( extract<int32_t>( iss, "int32_t", false ) );
fields.emplace_back( extract< int32_t >( iss, "int32_t", false ) );
break;
case DataType::uint32:
fields.emplace_back( extract<uint32_t>( iss, "uint32_t", false ) );
fields.emplace_back( extract< uint32_t >( iss, "uint32_t", false ) );
break;
case DataType::float32:
fields.emplace_back( extract<float>( iss, "float", false ) );
fields.emplace_back( extract< float >( iss, "float", false ) );
break;
case DataType::uint64:
fields.emplace_back( extract<uint64_t>( iss, "uint64_t", false ) );
fields.emplace_back( extract< uint64_t >( iss, "uint64_t", false ) );
break;
default:
auto type = static_cast< uint16_t >( member_entry.type );
if( type < 0x19 || type > 0x20 )
throw std::runtime_error("Unknown DataType: " + std::to_string( type ));
throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
uint64_t val = extract< uint64_t >( iss, "bool" );
int32_t shift = type - 0x19;
int32_t i = 1 << shift;
@ -180,7 +186,7 @@ namespace xiv
}
const std::vector<Field> Exd::get_row( uint32_t id )
const std::vector< Field > Exd::get_row( uint32_t id )
{
auto cacheEntryIt = _idCache.find( id );
@ -195,7 +201,7 @@ namespace xiv
std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
// Get the vector fields for the given record and preallocate it
auto fields = _data[id];
auto fields = _data[ id ];
fields.reserve( member_count );
iss.seekg( cacheEntryIt->second.offset + 6 );
@ -214,52 +220,52 @@ namespace xiv
// Extract the offset to the actual string
// Seek to it then extract the actual string
{
auto string_offset = extract<uint32_t>( iss, "string_offset", false );
auto string_offset = extract< uint32_t >( iss, "string_offset", false );
iss.seekg( cacheEntryIt->second.offset + 6 + _exh->get_header().data_offset + string_offset );
fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
}
break;
case DataType::boolean:
fields.emplace_back( extract<bool>( iss, "bool" ) );
fields.emplace_back( extract< bool >( iss, "bool" ) );
break;
case DataType::int8:
fields.emplace_back( extract<int8_t>( iss, "int8_t" ) );
fields.emplace_back( extract< int8_t >( iss, "int8_t" ) );
break;
case DataType::uint8:
fields.emplace_back( extract<uint8_t>( iss, "uint8_t" ) );
fields.emplace_back( extract< uint8_t >( iss, "uint8_t" ) );
break;
case DataType::int16:
fields.emplace_back( extract<int16_t>( iss, "int16_t", false ) );
fields.emplace_back( extract< int16_t >( iss, "int16_t", false ) );
break;
case DataType::uint16:
fields.emplace_back( extract<uint16_t>( iss, "uint16_t", false ) );
fields.emplace_back( extract< uint16_t >( iss, "uint16_t", false ) );
break;
case DataType::int32:
fields.emplace_back( extract<int32_t>( iss, "int32_t", false ) );
fields.emplace_back( extract< int32_t >( iss, "int32_t", false ) );
break;
case DataType::uint32:
fields.emplace_back( extract<uint32_t>( iss, "uint32_t", false ) );
fields.emplace_back( extract< uint32_t >( iss, "uint32_t", false ) );
break;
case DataType::float32:
fields.emplace_back( extract<float>( iss, "float", false ) );
fields.emplace_back( extract< float >( iss, "float", false ) );
break;
case DataType::uint64:
fields.emplace_back( extract<uint64_t>( iss, "uint64_t", false ) );
fields.emplace_back( extract< uint64_t >( iss, "uint64_t", false ) );
break;
default:
auto type = static_cast< uint16_t >( member_entry.type );
if( type < 0x19 || type > 0x20 )
throw std::runtime_error("Unknown DataType: " + std::to_string( type ));
throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
uint64_t val = extract< uint64_t >( iss, "bool" );
int32_t shift = type - 0x19;
int32_t i = 1 << shift;
@ -273,7 +279,7 @@ namespace xiv
}
// Get all rows
const std::map<uint32_t, std::vector<Field>>& Exd::get_rows()
const std::map< uint32_t, std::vector< Field>>& Exd::get_rows()
{
// Iterates over all the files
const uint32_t member_count = _exh->get_members().size();
@ -284,22 +290,22 @@ namespace xiv
std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
// Extract the header and skip to the record indices
auto exd_header = extract<ExdHeader>( iss );
auto exd_header = extract< ExdHeader >( iss );
iss.seekg( 0x20 );
// Preallocate and extract the record_indices
const uint32_t record_count = exd_header.index_size / sizeof( ExdRecordIndex );
std::vector<ExdRecordIndex> record_indices;
std::vector< ExdRecordIndex > record_indices;
record_indices.reserve( record_count );
for( uint32_t i = 0; i < record_count; ++i )
{
record_indices.emplace_back( extract<ExdRecordIndex>( iss ) );
record_indices.emplace_back( extract< ExdRecordIndex >( iss ) );
}
for( auto& record_index : record_indices )
{
// Get the vector fields for the given record and preallocate it
auto& fields = _data[record_index.id];
auto& fields = _data[ record_index.id ];
fields.reserve( member_count );
for( auto& member_entry : _exh->get_exh_members() )
@ -316,52 +322,52 @@ namespace xiv
// Extract the offset to the actual string
// Seek to it then extract the actual string
{
auto string_offset = extract<uint32_t>( iss, "string_offset", false );
auto string_offset = extract< uint32_t >( iss, "string_offset", false );
iss.seekg( record_index.offset + 6 + _exh->get_header().data_offset + string_offset );
fields.emplace_back( utils::bparse::extract_cstring( iss, "string" ) );
}
break;
case DataType::boolean:
fields.emplace_back( extract<bool>( iss, "bool" ) );
fields.emplace_back( extract< bool >( iss, "bool" ) );
break;
case DataType::int8:
fields.emplace_back( extract<int8_t>( iss, "int8_t" ) );
fields.emplace_back( extract< int8_t >( iss, "int8_t" ) );
break;
case DataType::uint8:
fields.emplace_back( extract<uint8_t>( iss, "uint8_t" ) );
fields.emplace_back( extract< uint8_t >( iss, "uint8_t" ) );
break;
case DataType::int16:
fields.emplace_back( extract<int16_t>( iss, "int16_t", false ) );
fields.emplace_back( extract< int16_t >( iss, "int16_t", false ) );
break;
case DataType::uint16:
fields.emplace_back( extract<uint16_t>( iss, "uint16_t", false ) );
fields.emplace_back( extract< uint16_t >( iss, "uint16_t", false ) );
break;
case DataType::int32:
fields.emplace_back( extract<int32_t>( iss, "int32_t", false ) );
fields.emplace_back( extract< int32_t >( iss, "int32_t", false ) );
break;
case DataType::uint32:
fields.emplace_back( extract<uint32_t>( iss, "uint32_t", false ) );
fields.emplace_back( extract< uint32_t >( iss, "uint32_t", false ) );
break;
case DataType::float32:
fields.emplace_back( extract<float>( iss, "float", false ) );
fields.emplace_back( extract< float >( iss, "float", false ) );
break;
case DataType::uint64:
fields.emplace_back( extract<uint64_t>( iss, "uint64_t", false ) );
fields.emplace_back( extract< uint64_t >( iss, "uint64_t", false ) );
break;
default:
auto type = static_cast< uint16_t >( member_entry.type );
if( type < 0x19 || type > 0x20 )
throw std::runtime_error("Unknown DataType: " + std::to_string( type ));
throw std::runtime_error( "Unknown DataType: " + std::to_string( type ) );
uint64_t val = extract< uint64_t >( iss, "bool" );
int32_t shift = type - 0x19;
int32_t i = 1 << shift;
@ -375,6 +381,5 @@ namespace xiv
return _data;
}
}
}

52
deps/datReader/Exd.h vendored
View file

@ -8,15 +8,13 @@
#include "File.h"
namespace xiv
{
namespace exd
namespace xiv::exd
{
class Exh;
class Exh;
// Field type containing all the possible types in the data files
using Field = std::variant<
// Field type containing all the possible types in the data files
using Field = std::variant<
std::string,
bool,
int8_t,
@ -28,39 +26,43 @@ using Field = std::variant<
float,
uint64_t >;
struct ExdCacheEntry
{
std::shared_ptr<dat::File> file;
struct ExdCacheEntry
{
std::shared_ptr< dat::File > file;
uint32_t offset;
};
};
// Data for a given language
class Exd
{
public:
// Data for a given language
class Exd
{
public:
// i_exh: the header
// i_files: the multiple exd files
Exd() {}
Exd(std::shared_ptr<Exh> i_exh, const std::vector<std::shared_ptr<dat::File>>& i_files);
Exd()
{
}
Exd( std::shared_ptr< Exh > i_exh, const std::vector< std::shared_ptr< dat::File>>& i_files );
~Exd();
// Get a row by its id
const std::vector<Field> get_row(uint32_t id);
const std::vector< Field > get_row( uint32_t id );
// Get a row by its id and sub-row
const std::vector<Field> get_row(uint32_t id, uint32_t subRow);
const std::vector< Field > get_row( uint32_t id, uint32_t subRow );
// Get all rows
const std::map<uint32_t, std::vector<Field>>& get_rows();
const std::map< uint32_t, std::vector< Field>>& get_rows();
protected:
protected:
// Data indexed by the ID of the row, the vector is field with the same order as exh.members
std::map<uint32_t, std::vector<Field>> _data;
std::vector<std::shared_ptr<dat::File>> _files;
std::shared_ptr<Exh> _exh;
std::map< uint32_t, std::vector< Field>> _data;
std::vector< std::shared_ptr< dat::File>> _files;
std::shared_ptr< Exh > _exh;
std::map< uint32_t, ExdCacheEntry > _idCache;
};
};
}
}
#endif // XIV_EXD_EXD_H

41
deps/datReader/ExdCat.cpp vendored
View file

@ -11,43 +11,43 @@ namespace
{
// Suffix of the filenames given a language
std::map<xiv::exd::Language, std::string> language_map =
{{xiv::exd::Language::none, ""},
{
{xiv::exd::Language::none, ""},
{xiv::exd::Language::ja, "_ja"},
{xiv::exd::Language::en, "_en"},
{xiv::exd::Language::de, "_de"},
{xiv::exd::Language::fr, "_fr"},
{xiv::exd::Language::chs, "_chs"}};
{xiv::exd::Language::chs, "_chs"}
};
}
namespace xiv
namespace xiv::exd
{
namespace exd
{
Cat::Cat(dat::GameData& i_game_data, const std::string& i_name) :
_name(i_name)
Cat::Cat( dat::GameData& i_game_data, const std::string& i_name ) :
_name( i_name )
{
//XIV_INFO(xiv_exd_logger, "Initializing Cat with name: " << i_name);
// creates the header .exh
{
auto header_file = i_game_data.getFile("exd/" + i_name + ".exh");
auto header_file = i_game_data.getFile( "exd/" + i_name + ".exh" );
_header = std::shared_ptr< Exh >( new Exh( *header_file ) );
}
for(auto language: _header->get_languages())
for( auto language: _header->get_languages() )
{
// chs not yet in data files
if (language == Language::en || language == Language::none)
if( language == Language::en || language == Language::none )
{
// Get all the files for a given category/language, in case of multiple range of IDs in separate files (like Quest)
std::vector<std::shared_ptr<dat::File>> files;
for(auto& exd_def: _header->get_exd_defs())
std::vector< std::shared_ptr< dat::File>> files;
for( auto& exd_def: _header->get_exd_defs() )
{
files.emplace_back( i_game_data.getFile("exd/" + i_name + "_" + std::to_string(exd_def.start_id) + language_map.at(language) + ".exd") );
files.emplace_back( i_game_data.getFile(
"exd/" + i_name + "_" + std::to_string( exd_def.start_id ) + language_map.at( language ) + ".exd" ) );
}
// Instantiate the data for this language
_data[language] = std::unique_ptr<Exd>(new Exd(_header, files));
_data[ language ] = std::make_unique< Exd >( _header, files );
}
}
}
@ -67,16 +67,15 @@ namespace xiv
return *_header;
}
const Exd& Cat::get_data_ln(Language i_language) const
const Exd& Cat::get_data_ln( Language i_language ) const
{
auto ln_it = _data.find(i_language);
if (ln_it == _data.end())
auto ln_it = _data.find( i_language );
if( ln_it == _data.end() )
{
throw std::runtime_error("No data for language: " + std::to_string(uint16_t(i_language)));
throw std::runtime_error( "No data for language: " + std::to_string( uint16_t( i_language ) ) );
}
return *(ln_it->second);
return *( ln_it->second );
}
}
}

62
deps/datReader/ExdData.cpp vendored
View file

@ -7,47 +7,44 @@
#include "ExdCat.h"
namespace xiv
{
namespace exd
{
namespace xiv::exd {
ExdData::ExdData(dat::GameData& i_game_data) try :
_game_data(i_game_data)
ExdData::ExdData( dat::GameData& i_game_data ) try :
_game_data( i_game_data )
{
//XIV_INFO(xiv_exd_logger, "Initializing ExdData");
// Fetch the root.exl and get a stream from it
auto root_exl = i_game_data.getFile("exd/root.exl");
auto root_exl = i_game_data.getFile( "exd/root.exl" );
std::vector< char > dataCpy = root_exl->get_data_sections().front();
xiv::utils::stream::vectorwrapbuf<char> databuf(dataCpy);
std::istream stream(&databuf);
xiv::utils::stream::vectorwrapbuf< char > databuf( dataCpy );
std::istream stream( &databuf );
// Iterates over the lines while skipping the first one
std::string line;
std::getline(stream, line); // extract first line EXLT,2
std::getline(stream, line);
std::getline( stream, line ); // extract first line EXLT,2
std::getline( stream, line );
// Until the EOF
while (!line.empty())
while( !line.empty() )
{
// Format is cat_name,XX
// XX being an internal identifier
// Get only the cat_name
auto sep = line.find(',');
auto category = line.substr(0, sep);
auto sep = line.find( ',' );
auto category = line.substr( 0, sep );
// Add to the list of category name
// creates the empty category in the cats map
// instantiate the creation mutex for this category
_cat_names.push_back(category);
_cats[category] = std::unique_ptr<Cat>();
_cat_creation_mutexes[category] = std::unique_ptr<std::mutex>(new std::mutex());
_cat_names.push_back( category );
_cats[ category ] = std::unique_ptr< Cat >();
_cat_creation_mutexes[ category ] = std::make_unique< std::mutex >();
std::getline(stream, line);
std::getline( stream, line );
}
}
catch(std::exception& e)
catch( std::exception& e )
{
// In case of failure here, client is supposed to catch the exception because it is not recoverable on our side
throw std::runtime_error( "ExdData initialization failed: " + std::string( e.what() ) );
@ -58,43 +55,42 @@ ExdData::~ExdData()
}
const std::vector<std::string>& ExdData::get_cat_names() const
const std::vector< std::string >& ExdData::get_cat_names() const
{
return _cat_names;
}
const Cat& ExdData::get_category(const std::string& i_cat_name)
const Cat& ExdData::get_category( const std::string& i_cat_name )
{
// Get the category from its name
auto cat_it = _cats.find(i_cat_name);
if (cat_it == _cats.end())
auto cat_it = _cats.find( i_cat_name );
if( cat_it == _cats.end() )
{
throw std::runtime_error("Category not found: " + i_cat_name);
throw std::runtime_error( "Category not found: " + i_cat_name );
}
if (cat_it->second)
if( cat_it->second )
{
// If valid return it
return *(cat_it->second);
return *( cat_it->second );
}
else
{
// If not, create it and return it
create_category(i_cat_name);
return *(_cats[i_cat_name]);
create_category( i_cat_name );
return *( _cats[ i_cat_name ] );
}
}
void ExdData::create_category(const std::string& i_cat_name)
void ExdData::create_category( const std::string& i_cat_name )
{
// Lock mutex in this scope
std::lock_guard<std::mutex> lock(*(_cat_creation_mutexes[i_cat_name]));
std::lock_guard< std::mutex > lock( *( _cat_creation_mutexes[ i_cat_name ] ) );
// Maybe after unlocking it has already been created, so check (most likely if it blocked)
if (!_cats[i_cat_name])
if( !_cats[ i_cat_name ] )
{
_cats[i_cat_name] = std::unique_ptr<Cat>(new Cat(_game_data, i_cat_name));
_cats[ i_cat_name ] = std::make_unique< Cat >( _game_data, i_cat_name );
}
}
}
}

70
deps/datReader/Exh.cpp vendored
View file

@ -7,71 +7,69 @@
using xiv::utils::bparse::extract;
namespace xiv
{
namespace exd
{
Exh::Exh(const dat::File& i_file)
namespace xiv::exd
{
Exh::Exh( const dat::File& i_file )
{
// Get a stream from the file
std::vector< char > dataCpy = i_file.get_data_sections().front();
std::istringstream iss( std::string( dataCpy.begin(), dataCpy.end() ) );
// Extract header and skip to member definitions
_header = extract<ExhHeader>(iss);
iss.seekg(0x20);
_header = extract< ExhHeader >( iss );
iss.seekg( 0x20 );
// Extract all the members and feed the _members map
for (auto i = 0; i < _header.field_count; ++i)
for( auto i = 0; i < _header.field_count; ++i )
{
auto member = extract<ExhMember>(iss);
_members[member.offset] = member;
auto member = extract< ExhMember >( iss );
_members[ member.offset ] = member;
_exh_defs.push_back( member );
}
// Extract all the exd_defs
_exd_defs.reserve(_header.exd_count);
for (auto i = 0; i < _header.exd_count; ++i)
_exd_defs.reserve( _header.exd_count );
for( auto i = 0; i < _header.exd_count; ++i )
{
_exd_defs.emplace_back(extract<ExhExdDef>(iss));
_exd_defs.emplace_back( extract< ExhExdDef >( iss ) );
}
// Extract all the languages
_languages.reserve(_header.language_count);
for (auto i = 0; i < _header.language_count; ++i)
_languages.reserve( _header.language_count );
for( auto i = 0; i < _header.language_count; ++i )
{
_languages.emplace_back(Language(extract<uint16_t>(iss, "language")));
_languages.emplace_back( Language( extract< uint16_t >( iss, "language" ) ) );
}
}
}
Exh::~Exh()
{
}
Exh::~Exh()
{
}
const ExhHeader& Exh::get_header() const
{
const ExhHeader& Exh::get_header() const
{
return _header;
}
}
const std::vector<ExhExdDef>& Exh::get_exd_defs() const
{
const std::vector< ExhExdDef >& Exh::get_exd_defs() const
{
return _exd_defs;
}
}
const std::vector<Language>& Exh::get_languages() const
{
const std::vector< Language >& Exh::get_languages() const
{
return _languages;
}
}
const std::map<uint32_t, ExhMember>& Exh::get_members() const
{
const std::map< uint32_t, ExhMember >& Exh::get_members() const
{
return _members;
}
}
const std::vector<ExhMember>& Exh::get_exh_members() const
{
const std::vector< ExhMember >& Exh::get_exh_members() const
{
return _exh_defs;
}
}
}
}

82
deps/datReader/Exh.h vendored
View file

@ -5,11 +5,10 @@
#include "bparse.h"
namespace xiv
namespace xiv::exd
{
namespace exd
{
enum class DataType : uint16_t
enum class DataType :
uint16_t
{
string = 0,
boolean = 1,
@ -47,32 +46,60 @@ namespace xiv
uint32_t start_id;
uint32_t count_id;
};
};
namespace xiv::utils::bparse {
template<>
inline void reorder< xiv::exd::ExhHeader >( xiv::exd::ExhHeader& i_struct )
{
for( int32_t i = 0; i < 0x4; ++i )
{
xiv::utils::bparse::reorder( i_struct.magic[ i ] );
}
i_struct.unknown = xiv::utils::bparse::byteswap( i_struct.unknown );
xiv::utils::bparse::reorder( i_struct.unknown );
i_struct.data_offset = xiv::utils::bparse::byteswap( i_struct.data_offset );
xiv::utils::bparse::reorder( i_struct.data_offset );
i_struct.field_count = xiv::utils::bparse::byteswap( i_struct.field_count );
xiv::utils::bparse::reorder( i_struct.field_count );
i_struct.exd_count = xiv::utils::bparse::byteswap( i_struct.exd_count );
xiv::utils::bparse::reorder( i_struct.exd_count );
i_struct.language_count = xiv::utils::bparse::byteswap( i_struct.language_count );
xiv::utils::bparse::reorder( i_struct.language_count );
}
template<>
inline void reorder< xiv::exd::ExhMember >( xiv::exd::ExhMember& i_struct )
{
i_struct.type = xiv::utils::bparse::byteswap( i_struct.type );
xiv::utils::bparse::reorder( i_struct.type );
i_struct.offset = xiv::utils::bparse::byteswap( i_struct.offset );
xiv::utils::bparse::reorder( i_struct.offset );
}
template<>
inline void reorder< xiv::exd::ExhExdDef >( xiv::exd::ExhExdDef& i_struct )
{
i_struct.start_id = xiv::utils::bparse::byteswap( i_struct.start_id );
xiv::utils::bparse::reorder( i_struct.start_id );
i_struct.count_id = xiv::utils::bparse::byteswap( i_struct.count_id );
xiv::utils::bparse::reorder( i_struct.count_id );
}
};
namespace xiv
{
namespace utils
{
namespace bparse
{
template <> inline void reorder<xiv::exd::ExhHeader>( xiv::exd::ExhHeader& i_struct ) { for( int32_t i = 0; i < 0x4; ++i ) { xiv::utils::bparse::reorder( i_struct.magic[i] ); } i_struct.unknown = xiv::utils::bparse::byteswap( i_struct.unknown ); xiv::utils::bparse::reorder( i_struct.unknown ); i_struct.data_offset = xiv::utils::bparse::byteswap( i_struct.data_offset ); xiv::utils::bparse::reorder( i_struct.data_offset ); i_struct.field_count = xiv::utils::bparse::byteswap( i_struct.field_count ); xiv::utils::bparse::reorder( i_struct.field_count ); i_struct.exd_count = xiv::utils::bparse::byteswap( i_struct.exd_count ); xiv::utils::bparse::reorder( i_struct.exd_count ); i_struct.language_count = xiv::utils::bparse::byteswap( i_struct.language_count ); xiv::utils::bparse::reorder( i_struct.language_count ); }
template <> inline void reorder<xiv::exd::ExhMember>( xiv::exd::ExhMember& i_struct ) { i_struct.type = xiv::utils::bparse::byteswap( i_struct.type ); xiv::utils::bparse::reorder( i_struct.type ); i_struct.offset = xiv::utils::bparse::byteswap( i_struct.offset ); xiv::utils::bparse::reorder( i_struct.offset ); }
template <> inline void reorder<xiv::exd::ExhExdDef>( xiv::exd::ExhExdDef& i_struct ) { i_struct.start_id = xiv::utils::bparse::byteswap( i_struct.start_id ); xiv::utils::bparse::reorder( i_struct.start_id ); i_struct.count_id = xiv::utils::bparse::byteswap( i_struct.count_id ); xiv::utils::bparse::reorder( i_struct.count_id ); }
}
}
};
namespace xiv
{
namespace dat
{
class File;
}
namespace exd
{
enum Language : uint16_t;
enum Language :
uint16_t;
// Header file for exd data
class Exh
@ -80,21 +107,26 @@ namespace xiv
public:
// The header file
Exh( const dat::File& i_file );
~Exh();
const ExhHeader& get_header() const;
const std::vector<ExhExdDef>& get_exd_defs() const;
const std::vector<Language>& get_languages() const;
const std::map<uint32_t, ExhMember>& get_members() const;
const std::vector<ExhMember>& get_exh_members() const;
const std::vector< ExhExdDef >& get_exd_defs() const;
const std::vector< Language >& get_languages() const;
const std::map< uint32_t, ExhMember >& get_members() const;
const std::vector< ExhMember >& get_exh_members() const;
protected:
ExhHeader _header;
// Members of the datastruct ordered(indexed) by offset
std::map<uint32_t, ExhMember> _members;
std::vector<ExhMember> _exh_defs;
std::vector<ExhExdDef> _exd_defs;
std::vector<Language> _languages;
std::map< uint32_t, ExhMember > _members;
std::vector< ExhMember > _exh_defs;
std::vector< ExhExdDef > _exd_defs;
std::vector< Language > _languages;
};
}

43
deps/datReader/File.cpp vendored
View file

@ -2,44 +2,41 @@
#include <fstream>
namespace xiv
{
namespace dat
namespace xiv::dat
{
File::File() :
_type(FileType::empty)
{
}
File::File() :
_type( FileType::empty )
{
}
File::~File()
{
}
File::~File()
{
}
FileType File::get_type() const
{
FileType File::get_type() const
{
return _type;
}
}
const std::vector<std::vector<char>>& File::get_data_sections() const
{
const std::vector< std::vector< char>>& File::get_data_sections() const
{
return _data_sections;
}
}
std::vector<std::vector<char>>& File::access_data_sections()
{
std::vector< std::vector< char>>& File::access_data_sections()
{
return _data_sections;
}
}
void File::exportToFile(const std::filesystem::path& i_path) const
{
void File::exportToFile( const std::filesystem::path& i_path ) const
{
std::ofstream ofs( i_path.string(), std::ios_base::binary | std::ios_base::out );
for( auto& data_section : _data_sections )
{
ofs.write( data_section.data(), data_section.size() );
}
ofs.close();
}
}
}
}

26
deps/datReader/File.h vendored
View file

@ -7,25 +7,16 @@
#include <stdint.h>
#include "bparse.h"
namespace xiv
namespace xiv::dat
{
namespace dat
{
enum class FileType : uint32_t
enum class FileType :
uint32_t
{
empty = 1,
standard = 2,
model = 3,
texture = 4,
};
}
};
namespace xiv
{
namespace dat
{
class Dat;
@ -33,24 +24,25 @@ namespace xiv
class File
{
friend class Dat;
public:
File();
~File();
FileType get_type() const;
// Getters functions for the data in the file
const std::vector<std::vector<char>>& get_data_sections() const;
std::vector<std::vector<char>>& access_data_sections();
const std::vector< std::vector< char>>& get_data_sections() const;
std::vector< std::vector< char>>& access_data_sections();
void exportToFile( const std::filesystem::path& i_path ) const;
protected:
FileType _type;
std::vector<std::vector<char>> _data_sections;
std::vector< std::vector< char>> _data_sections;
};
}
}
#endif // XIV_DAT_FILE_H

211
deps/datReader/GameData.cpp vendored
View file

@ -11,69 +11,68 @@
#include "DatCat.h"
#include "File.h"
namespace
{
// Relation between category number and category name
// These names are taken straight from the exe, it helps resolve dispatching when getting files by path
namespace {
// Relation between category number and category name
// These names are taken straight from the exe, it helps resolve dispatching when getting files by path
std::unordered_map< std::string, uint32_t > categoryNameToIdMap =
{{"common", 0x00},
{"bgcommon", 0x01},
{"bg", 0x02},
{"cut", 0x03},
{"chara", 0x04},
{"shader", 0x05},
{"ui", 0x06},
{"sound", 0x07},
{"vfx", 0x08},
{"ui_script", 0x09},
{"exd", 0x0A},
{"game_script", 0x0B},
{"music", 0x0C}
std::unordered_map< std::string, uint32_t > categoryNameToIdMap =
{ { "common", 0x00 },
{ "bgcommon", 0x01 },
{ "bg", 0x02 },
{ "cut", 0x03 },
{ "chara", 0x04 },
{ "shader", 0x05 },
{ "ui", 0x06 },
{ "sound", 0x07 },
{ "vfx", 0x08 },
{ "ui_script", 0x09 },
{ "exd", 0x0A },
{ "game_script", 0x0B },
{ "music", 0x0C }
};
std::unordered_map< uint32_t, std::string > categoryIdToNameMap =
{{0x00, "common"},
{0x01, "bgcommon"},
{0x02, "bg"},
{0x03, "cut"},
{0x04, "chara"},
{0x05, "shader"},
{0x06, "ui"},
{0x07, "sound"},
{0x08, "vfx"},
{0x09, "ui_script"},
{0x0A, "exd"},
{0x0B, "game_script"},
{0x0C, "music"}};
std::unordered_map< uint32_t, std::string > categoryIdToNameMap =
{ { 0x00, "common" },
{ 0x01, "bgcommon" },
{ 0x02, "bg" },
{ 0x03, "cut" },
{ 0x04, "chara" },
{ 0x05, "shader" },
{ 0x06, "ui" },
{ 0x07, "sound" },
{ 0x08, "vfx" },
{ 0x09, "ui_script" },
{ 0x0A, "exd" },
{ 0x0B, "game_script" },
{ 0x0C, "music" } };
}
namespace xiv
{
namespace dat
{
GameData::GameData(const std::filesystem::path& path) try :
m_path(path)
namespace xiv::dat
{
GameData::GameData( const std::filesystem::path& path ) try :
m_path( path )
{
int maxExLevel = 0;
// msvc has retarded stdlib implementation
#ifdef _WIN32
#ifdef _WIN32
static constexpr auto sep = "\\";
#else
#else
static constexpr auto sep = std::filesystem::path::preferred_separator;
#endif
#endif
// Determine which expansions are available
while( std::filesystem::exists( std::filesystem::path( m_path.string() + sep + "ex" + std::to_string( maxExLevel + 1 ) + sep + "ex" + std::to_string( maxExLevel + 1 ) + ".ver" ) ) )
while( std::filesystem::exists( std::filesystem::path(
m_path.string() + sep + "ex" + std::to_string( maxExLevel + 1 ) + sep + "ex" + std::to_string( maxExLevel + 1 ) +
".ver" ) ) )
{
maxExLevel++;
}
// Iterate over the files in path
for( auto it = std::filesystem::directory_iterator( m_path.string() + "//ffxiv" ); it != std::filesystem::directory_iterator(); ++it )
for( auto it = std::filesystem::directory_iterator( m_path.string() + "//ffxiv" );
it != std::filesystem::directory_iterator(); ++it )
{
// Get the filename of the current element
auto filename = it->path().filename().string();
@ -91,24 +90,27 @@ GameData::GameData(const std::filesystem::path& path) try :
// creates the empty category in the cats map
// instantiate the creation mutex for this category
m_catNums.push_back( cat_nb );
m_cats[cat_nb] = std::unique_ptr<Cat>();
m_catCreationMutexes[cat_nb] = std::unique_ptr<std::mutex>( new std::mutex() );
m_cats[ cat_nb ] = std::unique_ptr< Cat >();
m_catCreationMutexes[ cat_nb ] = std::make_unique< std::mutex >();
// Check for expansion
for( int exNum = 1; exNum <= maxExLevel; exNum++ )
{
const std::string path = m_path.string() + sep + buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, 0, "win32", "index" );
const std::string path =
m_path.string() + sep + buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, 0, "win32", "index" );
if( std::filesystem::exists( std::filesystem::path( path ) ) )
{
int chunkCount = 0;
for(int chunkTest = 0; chunkTest < 256; chunkTest++ )
for( int chunkTest = 0; chunkTest < 256; chunkTest++ )
{
if( std::filesystem::exists( m_path.string() + sep + buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, chunkTest, "win32", "index" ) ) )
if( std::filesystem::exists( m_path.string() + sep +
buildDatStr( "ex" + std::to_string( exNum ), cat_nb, exNum, chunkTest, "win32",
"index" ) ) )
{
m_exCats[cat_nb].exNumToChunkMap[exNum].chunkToCatMap[chunkTest] = std::unique_ptr<Cat>();
m_exCats[ cat_nb ].exNumToChunkMap[ exNum ].chunkToCatMap[ chunkTest ] = std::unique_ptr< Cat >();
chunkCount++;
}
}
@ -118,60 +120,61 @@ GameData::GameData(const std::filesystem::path& path) try :
}
}
}
catch( std::exception& e )
{
}
catch( std::exception& e )
{
// In case of failure here, client is supposed to catch the exception because it is not recoverable on our side
throw std::runtime_error( "GameData initialization failed: " + std::string( e.what() ) );
}
}
GameData::~GameData()
{
GameData::~GameData()
{
}
}
const std::string GameData::buildDatStr( const std::string folder, const int cat, const int exNum, const int chunk, const std::string platform, const std::string type )
{
const std::string GameData::buildDatStr( const std::string folder, const int cat, const int exNum, const int chunk,
const std::string platform, const std::string type )
{
char dat[1024];
sprintf( dat, "%s/%02x%02x%02x.%s.%s", folder.c_str(), cat, exNum, chunk, platform.c_str(), type.c_str() );
return std::string( dat );
}
}
const std::vector<uint32_t>& GameData::getCatNumbers() const
{
const std::vector< uint32_t >& GameData::getCatNumbers() const
{
return m_catNums;
}
}
std::unique_ptr<File> GameData::getFile(const std::string& path)
{
std::unique_ptr< File > GameData::getFile( const std::string& path )
{
// Get the hashes, the category from the path then call the getFile of the category
uint32_t dirHash;
uint32_t filenameHash;
getHashes( path, dirHash, filenameHash );
return getCategoryFromPath( path ).getFile( dirHash, filenameHash );
}
}
bool GameData::doesFileExist(const std::string& path)
{
bool GameData::doesFileExist( const std::string& path )
{
uint32_t dirHash;
uint32_t filenameHash;
getHashes( path, dirHash, filenameHash );
return getCategoryFromPath( path ).doesFileExist( dirHash, filenameHash );
}
}
bool GameData::doesDirExist(const std::string& i_path)
{
bool GameData::doesDirExist( const std::string& i_path )
{
uint32_t dirHash;
uint32_t filenameHash;
getHashes( i_path, dirHash, filenameHash );
return getCategoryFromPath( i_path ).doesDirExist( dirHash );
}
}
const Cat& GameData::getCategory(uint32_t catNum)
{
const Cat& GameData::getCategory( uint32_t catNum )
{
// Check that the category number exists
auto catIt = m_cats.find( catNum );
if( catIt == m_cats.end() )
@ -188,12 +191,12 @@ const Cat& GameData::getCategory(uint32_t catNum)
{
// Else create it and return it
createCategory( catNum );
return *( m_cats[catNum] );
return *( m_cats[ catNum ] );
}
}
}
const Cat& GameData::getCategory(const std::string& catName)
{
const Cat& GameData::getCategory( const std::string& catName )
{
// Find the category number from the name
auto categoryNameToIdMapIt = ::categoryNameToIdMap.find( catName );
if( categoryNameToIdMapIt == ::categoryNameToIdMap.end() )
@ -203,10 +206,10 @@ const Cat& GameData::getCategory(const std::string& catName)
// From the category number return the category
return getCategory( categoryNameToIdMapIt->second );
}
}
const Cat& GameData::getExCategory( const std::string& catName, uint32_t exNum, const std::string& path )
{
const Cat& GameData::getExCategory( const std::string& catName, uint32_t exNum, const std::string& path )
{
// Find the category number from the name
auto categoryMapIt = ::categoryNameToIdMap.find( catName );
if( categoryMapIt == ::categoryNameToIdMap.end() )
@ -218,7 +221,7 @@ const Cat& GameData::getExCategory( const std::string& catName, uint32_t exNum,
uint32_t filenameHash;
getHashes( path, dirHash, filenameHash );
for( auto const& chunk : m_exCats[categoryMapIt->second].exNumToChunkMap[exNum].chunkToCatMap )
for( auto const& chunk : m_exCats[ categoryMapIt->second ].exNumToChunkMap[ exNum ].chunkToCatMap )
{
if( !chunk.second )
createExCategory( categoryMapIt->second );
@ -230,10 +233,10 @@ const Cat& GameData::getExCategory( const std::string& catName, uint32_t exNum,
}
throw std::runtime_error( "Chunk not found for path: " + path );
}
}
const Cat& GameData::getCategoryFromPath(const std::string& path)
{
const Cat& GameData::getCategoryFromPath( const std::string& path )
{
// Find the first / in the string, paths are in the format CAT_NAME/..../.../../....
auto firstSlashPos = path.find( '/' );
if( firstSlashPos == std::string::npos )
@ -241,7 +244,7 @@ const Cat& GameData::getCategoryFromPath(const std::string& path)
throw std::runtime_error( "Path does not have a / char: " + path );
}
if( path.substr( firstSlashPos + 1, 2) == "ex" )
if( path.substr( firstSlashPos + 1, 2 ) == "ex" )
{
return getExCategory( path.substr( 0, firstSlashPos ), std::stoi( path.substr( firstSlashPos + 3, 1 ) ), path );
}
@ -250,10 +253,10 @@ const Cat& GameData::getCategoryFromPath(const std::string& path)
// From the sub string found beforethe first / get the category
return getCategory( path.substr( 0, firstSlashPos ) );
}
}
}
void GameData::getHashes(const std::string& path, uint32_t& dirHash, uint32_t& filenameHash) const
{
void GameData::getHashes( const std::string& path, uint32_t& dirHash, uint32_t& filenameHash ) const
{
// Convert the path to lowercase before getting the hashes
std::string pathLower;
pathLower.resize( path.size() );
@ -272,14 +275,14 @@ void GameData::getHashes(const std::string& path, uint32_t& dirHash, uint32_t& f
// Get the crc32 values from zlib, to compensate the final XOR 0xFFFFFFFF that isnot done in the exe we just reXOR
dirHash = crc32( 0, reinterpret_cast<const uint8_t*>( dirPart.data() ), dirPart.size() ) ^ 0xFFFFFFFF;
filenameHash = crc32( 0, reinterpret_cast<const uint8_t*>( filenamePart.data() ), filenamePart.size() ) ^ 0xFFFFFFFF;
}
}
void GameData::createCategory(uint32_t catNum)
{
void GameData::createCategory( uint32_t catNum )
{
// Lock mutex in this scope
std::lock_guard<std::mutex> lock( *( m_catCreationMutexes[catNum] ) );
std::lock_guard< std::mutex > lock( *( m_catCreationMutexes[ catNum ] ) );
// Maybe after unlocking it has already been created, so check (most likely if it blocked)
if( !m_cats[catNum] )
if( !m_cats[ catNum ] )
{
// Get the category name if we have it
std::string catName;
@ -290,14 +293,14 @@ void GameData::createCategory(uint32_t catNum)
}
// Actually creates the category
m_cats[catNum] = std::unique_ptr<Cat>( new Cat( m_path, catNum, catName ) );
m_cats[ catNum ] = std::make_unique< Cat >( m_path, catNum, catName );
}
}
}
void GameData::createExCategory( uint32_t catNum )
{
void GameData::createExCategory( uint32_t catNum )
{
// Maybe after unlocking it has already been created, so check (most likely if it blocked)
if( !m_exCats[catNum].exNumToChunkMap[1].chunkToCatMap[0] )
if( !m_exCats[ catNum ].exNumToChunkMap[ 1 ].chunkToCatMap[ 0 ] )
{
// Get the category name if we have it
std::string catName;
@ -307,16 +310,16 @@ void GameData::createExCategory( uint32_t catNum )
catName = categoryMapIt->second;
}
for( auto const& ex : m_exCats[catNum].exNumToChunkMap )
for( auto const& ex : m_exCats[ catNum ].exNumToChunkMap )
{
for( auto const& chunk : m_exCats[catNum].exNumToChunkMap[ex.first].chunkToCatMap )
for( auto const& chunk : m_exCats[ catNum ].exNumToChunkMap[ ex.first ].chunkToCatMap )
{
// Actually creates the category
m_exCats[catNum].exNumToChunkMap[ex.first].chunkToCatMap[chunk.first] = std::unique_ptr<Cat>( new Cat( m_path, catNum, catName, ex.first, chunk.first ) );
m_exCats[ catNum ].exNumToChunkMap[ ex.first ].chunkToCatMap[ chunk.first ] = std::unique_ptr< Cat >(
new Cat( m_path, catNum, catName, ex.first, chunk.first ) );
}
}
}
}
}
}
}

50
deps/datReader/GameData.h vendored
View file

@ -7,37 +7,40 @@
#include <filesystem>
namespace xiv
{
namespace dat
namespace xiv::dat
{
class Cat;
class File;
class Cat;
// Interface to all the datfiles - Main entry point
// All the paths to files/dirs inside the dats are case-insensitive
class GameData
{
public:
class File;
// Interface to all the datfiles - Main entry point
// All the paths to files/dirs inside the dats are case-insensitive
class GameData
{
public:
// This should be the path in which the .index/.datX files are located
GameData( const std::filesystem::path& path );
~GameData();
static const std::string buildDatStr( const std::string folder, const int cat, const int exNum, const int chunk, const std::string platform, const std::string type );
static const std::string
buildDatStr( const std::string folder, const int cat, const int exNum, const int chunk, const std::string platform,
const std::string type );
// Returns all the scanned category number available in the path
const std::vector<uint32_t>& getCatNumbers() const;
const std::vector< uint32_t >& getCatNumbers() const;
// Return a specific category by its number (see getCatNumbers() for loops)
const Cat& getCategory( uint32_t catNum );
// Return a specific category by it's name (e.g.: "exd"/"game_script"/ etc...)
const Cat& getCategory( const std::string& catName );
const Cat& getExCategory( const std::string& catName, uint32_t exNum, const std::string& path );
// Retrieve a file from the dats given its filename
std::unique_ptr<File> getFile( const std::string& path );
std::unique_ptr< File > getFile( const std::string& path );
// Checks that a file exists
bool doesFileExist( const std::string& path );
@ -48,7 +51,7 @@ public:
// - "ui/icon/000000/" will return True
bool doesDirExist( const std::string& path );
protected:
protected:
// Return a specific category given a path (calls const Cat& getCategory(const std::string& catName))
const Cat& getCategoryFromPath( const std::string& path );
@ -64,21 +67,26 @@ protected:
const std::filesystem::path m_path;
// Stored categories, indexed by their number, categories are instantiated and parsed individually when they are needed
std::unordered_map<uint32_t, std::unique_ptr<Cat>> m_cats;
std::unordered_map< uint32_t, std::unique_ptr< Cat>> m_cats;
// List of all the categories numbers, is equal to m_cats.keys()
std::vector<uint32_t> m_catNums;
std::vector< uint32_t > m_catNums;
// Map of all EX categories and their chunks, "CatNum - (ExNum - (ChunkNum - Cat))"
// Map of all EX categories and their chunks, "CatNum - (ExNum - (ChunkNum - Cat))"
using ChunkToCatMap = struct { std::unordered_map< uint32_t, std::unique_ptr< Cat > > chunkToCatMap; };
using ExNumToChunkMap = struct { std::unordered_map< uint32_t, ChunkToCatMap > exNumToChunkMap; };
using ChunkToCatMap = struct
{
std::unordered_map< uint32_t, std::unique_ptr< Cat > > chunkToCatMap;
};
using ExNumToChunkMap = struct
{
std::unordered_map< uint32_t, ChunkToCatMap > exNumToChunkMap;
};
using CatNumToExNumMap = std::unordered_map< uint32_t, ExNumToChunkMap >;
CatNumToExNumMap m_exCats;
std::unordered_map<uint32_t, std::unique_ptr<std::mutex>> m_catCreationMutexes;
};
std::unordered_map< uint32_t, std::unique_ptr< std::mutex>> m_catCreationMutexes;
};
}
}
#endif // XIV_DAT_GAMEDATA_H

108
deps/datReader/Index.cpp vendored
View file

@ -2,9 +2,7 @@
#include "bparse.h"
namespace xiv
{
namespace dat
namespace xiv::dat
{
struct IndexBlockRecord
{
@ -21,49 +19,40 @@ namespace dat
uint32_t padding;
};
}
}
namespace xiv
namespace xiv::utils::bparse
{
namespace utils
{
namespace bparse
{
template <>
inline void reorder<xiv::dat::IndexBlockRecord>(xiv::dat::IndexBlockRecord& i_struct)
template<>
inline void reorder< xiv::dat::IndexBlockRecord >( xiv::dat::IndexBlockRecord& i_struct )
{
xiv::utils::bparse::reorder(i_struct.offset);
xiv::utils::bparse::reorder(i_struct.size);
xiv::utils::bparse::reorder(i_struct.blockHash);
xiv::utils::bparse::reorder( i_struct.offset );
xiv::utils::bparse::reorder( i_struct.size );
xiv::utils::bparse::reorder( i_struct.blockHash );
}
template <>
inline void reorder<xiv::dat::IndexHashTableEntry>(xiv::dat::IndexHashTableEntry& i_struct)
template<>
inline void reorder< xiv::dat::IndexHashTableEntry >( xiv::dat::IndexHashTableEntry& i_struct )
{
xiv::utils::bparse::reorder(i_struct.filenameHash);
xiv::utils::bparse::reorder(i_struct.dirHash);
xiv::utils::bparse::reorder(i_struct.datOffset);
xiv::utils::bparse::reorder(i_struct.padding);
xiv::utils::bparse::reorder( i_struct.filenameHash );
xiv::utils::bparse::reorder( i_struct.dirHash );
xiv::utils::bparse::reorder( i_struct.datOffset );
xiv::utils::bparse::reorder( i_struct.padding );
}
}
}
};
using xiv::utils::bparse::extract;
namespace xiv
{
namespace dat
namespace xiv::dat
{
Index::Index(const std::filesystem::path& path) :
Index::Index( const std::filesystem::path& path ) :
SqPack( path )
{
{
if( !m_handle )
throw new std::runtime_error( "Failed to load Index at " + path.string() );
// Hash Table record
auto hashTableBlockRecord = extract<IndexBlockRecord>( m_handle );
auto hashTableBlockRecord = extract< IndexBlockRecord >( m_handle );
isIndexBlockValid( hashTableBlockRecord );
// Save the posin the stream to go back to it later on
@ -73,14 +62,14 @@ Index::Index(const std::filesystem::path& path) :
m_handle.seekg( hashTableBlockRecord.offset );
// Preallocate and extract the index_hash_table_entries
std::vector<IndexHashTableEntry> indexHashTableEntries;
extract<IndexHashTableEntry>( m_handle, hashTableBlockRecord.size / sizeof( IndexHashTableEntry ),
std::vector< IndexHashTableEntry > indexHashTableEntries;
extract< IndexHashTableEntry >( m_handle, hashTableBlockRecord.size / sizeof( IndexHashTableEntry ),
indexHashTableEntries );
// Feed the correct entry in the HashTable for each index_hash_table_entry
for( auto& indexHashTableEntry : indexHashTableEntries )
{
auto& hashTableEntry = m_hashTable[indexHashTableEntry.dirHash][indexHashTableEntry.filenameHash];
auto& hashTableEntry = m_hashTable[ indexHashTableEntry.dirHash ][ indexHashTableEntry.filenameHash ];
// The dat number is found in the offset, last four bits
hashTableEntry.datNum = ( indexHashTableEntry.datOffset & 0xF ) / 0x2;
// The offset in the dat file, needs to strip the dat number indicator
@ -93,46 +82,46 @@ Index::Index(const std::filesystem::path& path) :
m_handle.seekg( pos );
// Dat Count
m_datCount = extract<uint32_t>( m_handle, "dat_count" );
m_datCount = extract< uint32_t >( m_handle, "dat_count" );
// Free List
isIndexBlockValid( extract<IndexBlockRecord>( m_handle ) );
isIndexBlockValid( extract< IndexBlockRecord >( m_handle ) );
// Dir Hash Table
isIndexBlockValid( extract<IndexBlockRecord>( m_handle ) );
}
isIndexBlockValid( extract< IndexBlockRecord >( m_handle ) );
}
Index::~Index()
{
}
Index::~Index()
{
}
uint32_t Index::getDatCount() const
{
uint32_t Index::getDatCount() const
{
return m_datCount;
}
}
const Index::HashTable& Index::getHashTable() const
{
const Index::HashTable& Index::getHashTable() const
{
return m_hashTable;
}
}
bool Index::doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const
{
bool Index::doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const
{
auto dir_it = getHashTable().find( dir_hash );
if( dir_it != getHashTable().end() )
{
return ( dir_it->second.find( filename_hash ) != dir_it->second.end() );
}
return false;
}
}
bool Index::doesDirExist( uint32_t dir_hash ) const
{
bool Index::doesDirExist( uint32_t dir_hash ) const
{
return ( getHashTable().find( dir_hash ) != getHashTable().end() );
}
}
const Index::DirHashTable& Index::getDirHashTable( uint32_t dir_hash ) const
{
const Index::DirHashTable& Index::getDirHashTable( uint32_t dir_hash ) const
{
auto dir_it = getHashTable().find( dir_hash );
if( dir_it == getHashTable().end() )
{
@ -142,10 +131,10 @@ const Index::DirHashTable& Index::getDirHashTable( uint32_t dir_hash ) const
{
return dir_it->second;
}
}
}
const Index::HashTableEntry& Index::getHashTableEntry( uint32_t dir_hash, uint32_t filename_hash ) const
{
const Index::HashTableEntry& Index::getHashTableEntry( uint32_t dir_hash, uint32_t filename_hash ) const
{
auto& dirHashTable = getDirHashTable( dir_hash );
auto file_it = dirHashTable.find( filename_hash );
if( file_it == dirHashTable.end() )
@ -156,12 +145,11 @@ const Index::HashTableEntry& Index::getHashTableEntry( uint32_t dir_hash, uint32
{
return file_it->second;
}
}
}
void Index::isIndexBlockValid( const IndexBlockRecord& i_index_block_record )
{
void Index::isIndexBlockValid( const IndexBlockRecord& i_index_block_record )
{
isBlockValid( i_index_block_record.offset, i_index_block_record.size, i_index_block_record.blockHash );
}
}
}
}

24
deps/datReader/Index.h vendored
View file

@ -7,16 +7,18 @@
#include <filesystem>
namespace xiv {
namespace dat {
struct IndexBlockRecord;
class Index : public SqPack
namespace xiv::dat
{
public:
struct IndexBlockRecord;
class Index :
public SqPack
{
public:
// Full path to the index file
Index( const std::filesystem::path& i_path );
virtual ~Index();
// An entry in the hash table, representing a file in a given dat
@ -36,24 +38,26 @@ public:
uint32_t getDatCount() const;
bool doesFileExist( uint32_t dir_hash, uint32_t filename_hash ) const;
bool doesDirExist( uint32_t dir_hash ) const;
// Returns the whole HashTable
const HashTable& getHashTable() const;
// Returns the hash table for a specific dir
const DirHashTable& getDirHashTable( uint32_t dir_hash ) const;
// Returns the HashTableEntry for a given file given its hashes
const HashTableEntry& getHashTableEntry( uint32_t dir_hash, uint32_t filename_hash ) const;
protected:
protected:
// Checks that the block is valid with regards to its hash
void isIndexBlockValid( const IndexBlockRecord& i_index_block_record );
uint32_t m_datCount;
HashTable m_hashTable;
};
};
}
}
#endif // XIV_DAT_INDEX_H

2
deps/datReader/SqPack.cpp vendored
View file

@ -59,7 +59,7 @@ namespace xiv::dat
m_handle( path.string(), std::ios_base::in | std::ios_base::binary )
{
// Extract the header
extract<SqPackHeader>( m_handle );
extract< SqPackHeader >( m_handle );
// Skip until the IndexHeader the extract it
m_handle.seekg( 0x400 );

36
deps/datReader/SqPack.h vendored
View file

@ -8,51 +8,44 @@
#include "bparse.h"
namespace xiv
namespace xiv::dat
{
namespace dat
{
struct SqPackBlockHash
{
uint8_t hash[0x14];
uint32_t padding[0xB];
};
}
}
namespace xiv {
namespace utils {
namespace bparse {
template <> inline void reorder<xiv::dat::SqPackBlockHash>( xiv::dat::SqPackBlockHash& i_struct )
namespace xiv::utils::bparse
{
template<>
inline void reorder< xiv::dat::SqPackBlockHash >( xiv::dat::SqPackBlockHash& i_struct )
{
for( auto i = 0; i < 0x14; ++i )
{
xiv::utils::bparse::reorder( i_struct.hash[i] );
xiv::utils::bparse::reorder( i_struct.hash[ i ] );
}
for( auto i = 0; i < 0xB; ++i )
{
xiv::utils::bparse::reorder( i_struct.padding[i] );
}
}
xiv::utils::bparse::reorder( i_struct.padding[ i ] );
}
}
};
namespace xiv
{
namespace dat
namespace xiv::dat
{
class SqPack
{
class SqPack
{
public:
public:
// Full path to the sqpack file
SqPack( const std::filesystem::path& i_path );
virtual ~SqPack();
protected:
protected:
// Checks that a given block is valid iven its hash
void isBlockValid( uint32_t i_offset, uint32_t i_size, const SqPackBlockHash& i_block_hash );
@ -61,6 +54,5 @@ protected:
};
}
}
#endif // XIV_DAT_SQPACK_H

108
deps/datReader/bparse.h vendored
View file

@ -6,74 +6,73 @@
#include <sstream>
#include <vector>
namespace xiv
{
namespace utils
{
namespace bparse
namespace xiv::utils::bparse
{
// Internal macro for byteswapping
template <int N>
void byteswap_impl(char (&bytes)[N])
{
// Internal macro for byteswapping
template< int N >
void byteswap_impl( char (& bytes)[N] )
{
for( auto p = std::begin( bytes ), end = std::end( bytes ) - 1; p < end; ++p, --end )
{
std::swap( *p, *end );
}
}
}
// byteswapping any type (no pointers to array)
template <typename T>
T byteswap(T value)
{
byteswap_impl(*reinterpret_cast<char (*)[sizeof(T)]>(&value));
// byteswapping any type (no pointers to array)
template< typename T >
T byteswap( T value )
{
byteswap_impl( *reinterpret_cast<char ( * )[sizeof( T )]>(&value) );
return value;
}
}
// Read a struct from a stream
template <typename StructType>
void read(std::istream& i_stream, StructType& i_struct)
{
static_assert( std::is_pod<StructType>::value, "StructType must be a POD to be able to use read." );
// Read a struct from a stream
template< typename StructType >
void read( std::istream& i_stream, StructType& i_struct )
{
static_assert( std::is_pod< StructType >::value, "StructType must be a POD to be able to use read." );
i_stream.read( reinterpret_cast<char*>( &i_struct ), sizeof( StructType ) );
}
}
// By default a type does not need reordering
template <typename StructType> void reorder(StructType& i_struct) {}
// By default a type does not need reordering
template< typename StructType >
void reorder( StructType& i_struct )
{
}
// "Overload" for passed struct as arg
template <typename StructType>
void extract(std::istream& i_stream, StructType& o_struct)
{
// "Overload" for passed struct as arg
template< typename StructType >
void extract( std::istream& i_stream, StructType& o_struct )
{
read( i_stream, o_struct );
reorder( o_struct );
}
}
// This should not copy because of RVO
// Extract a struct from a stream and log it
template <typename StructType>
StructType extract( std::istream& i_stream )
{
// This should not copy because of RVO
// Extract a struct from a stream and log it
template< typename StructType >
StructType extract( std::istream& i_stream )
{
StructType temp_struct;
extract<StructType>( i_stream, temp_struct );
extract< StructType >( i_stream, temp_struct );
return temp_struct;
}
}
template <typename StructType>
void extract(std::istream& i_stream, uint32_t i_size, std::vector<StructType>& o_structs )
{
template< typename StructType >
void extract( std::istream& i_stream, uint32_t i_size, std::vector< StructType >& o_structs )
{
o_structs.reserve( i_size );
for( uint32_t i = 0; i < i_size; ++i )
{
o_structs.emplace_back( extract<StructType>( i_stream ) );
o_structs.emplace_back( extract< StructType >( i_stream ) );
}
}
}
// For simple (integral) types just provide name and endianness directly
template <typename StructType>
StructType extract(std::istream& i_stream, const std::string& i_name, bool i_is_le = true)
{
// For simple (integral) types just provide name and endianness directly
template< typename StructType >
StructType extract( std::istream& i_stream, const std::string& i_name, bool i_is_le = true )
{
StructType temp_struct;
read( i_stream, temp_struct );
if( !i_is_le )
@ -81,23 +80,22 @@ StructType extract(std::istream& i_stream, const std::string& i_name, bool i_is_
temp_struct = byteswap( temp_struct );
}
return temp_struct;
}
}
template <typename StructType>
void extract(std::istream& i_stream, const std::string& i_name, uint32_t i_size, std::vector<StructType>& o_structs, bool i_is_le = true)
{
template< typename StructType >
void extract( std::istream& i_stream, const std::string& i_name, uint32_t i_size, std::vector< StructType >& o_structs,
bool i_is_le = true )
{
o_structs.reserve( i_size );
for( uint32_t i = 0; i < i_size; ++i )
{
o_structs.emplace_back( extract<StructType>( i_stream, i_name ) );
o_structs.emplace_back( extract< StructType >( i_stream, i_name ) );
}
}
}
// For cstrings
std::string extract_cstring( std::istream& i_stream, const std::string& i_name );
// For cstrings
std::string extract_cstring( std::istream& i_stream, const std::string& i_name );
}
}
}
#endif // XIV_UTILS_BPARSE_H

8
deps/datReader/conv.cpp vendored
View file

@ -1,8 +1,8 @@
#include "conv.h"
namespace xiv {
namespace utils {
namespace conv {
namespace xiv::utils::conv
{
float half2float( const uint16_t i_value )
{
uint32_t t1;
@ -30,6 +30,4 @@ namespace conv {
}
}
}
}

8
deps/datReader/conv.h vendored
View file

@ -5,13 +5,11 @@
#include <vector>
#include <ostream>
namespace xiv {
namespace utils {
namespace conv {
namespace xiv::utils::conv
{
float half2float( const uint16_t i_value );
float ubyte2float( const uint8_t i_value );
}
}
}
#endif // XIV_UTILS_CONV_H

101
deps/datReader/crc32.cpp vendored
View file

@ -65,104 +65,101 @@ namespace internal
}
}
namespace xiv
{
namespace utils
{
namespace crc32
namespace xiv::utils::crc32
{
uint32_t compute(const std::string& i_input, uint32_t init_crc)
{
uint32_t compute( const std::string& i_input, uint32_t init_crc )
{
// Classical crc stuff
auto& crc_table = internal::get_crc_table();
auto crc = init_crc;
for(std::size_t i = 0; i < i_input.size(); ++i)
for( std::size_t i = 0; i < i_input.size(); ++i )
{
crc = crc_table[(crc ^ i_input[i]) & 0xFF] ^ (crc >> 8);
crc = crc_table[ ( crc ^ i_input[ i ] ) & 0xFF ] ^ ( crc >> 8 );
}
return crc;
}
}
uint32_t rev_compute(const std::string& i_input, uint32_t init_crc)
{
uint32_t rev_compute( const std::string& i_input, uint32_t init_crc )
{
auto& rev_crc_table = internal::get_rev_crc_table();
auto crc = init_crc;
const auto input_size = i_input.size();
// Reverse crc
for(auto i = input_size; i > 0; --i)
for( auto i = input_size; i > 0; --i )
{
crc = rev_crc_table[crc >> 24] ^ ((crc << 8) & 0xFFFFFF00) ^ i_input[input_size - i - 1];
crc = rev_crc_table[ crc >> 24 ] ^ ( ( crc << 8 ) & 0xFFFFFF00 ) ^ i_input[ input_size - i - 1 ];
}
// Compute the 4 bytes needed for this init_crc
for (auto i = 0; i < 4; ++i)
for( auto i = 0; i < 4; ++i )
{
crc = rev_crc_table[crc >> 24] ^ ((crc << 8) & 0xFFFFFF00);
crc = rev_crc_table[ crc >> 24 ] ^ ( ( crc << 8 ) & 0xFFFFFF00 );
}
return crc;
}
}
void generate_hashes_1(std::string& i_format, const uint32_t i_first_index, std::vector<uint32_t>& o_hashes)
{
void generate_hashes_1( std::string& i_format, const uint32_t i_first_index, std::vector< uint32_t >& o_hashes )
{
char* str = const_cast<char*>(i_format.data());
const uint32_t str_size = i_format.size();
o_hashes.resize(10000);
o_hashes.resize( 10000 );
uint32_t i = 0;
for (char a = '0'; a <= '9'; ++a)
for( char a = '0'; a <= '9'; ++a )
{
str[i_first_index] = a;
for (char b = '0'; b <= '9'; ++b)
str[ i_first_index ] = a;
for( char b = '0'; b <= '9'; ++b )
{
str[i_first_index + 1] = b;
for (char c = '0'; c <= '9'; ++c)
str[ i_first_index + 1 ] = b;
for( char c = '0'; c <= '9'; ++c )
{
str[i_first_index + 2] = c;
for (char d = '0'; d <= '9'; ++d)
str[ i_first_index + 2 ] = c;
for( char d = '0'; d <= '9'; ++d )
{
str[i_first_index + 3] = d;
o_hashes[i] = ::crc32(0, reinterpret_cast<uint8_t*>(&(str[0])), str_size) ^ 0xFFFFFFFF;
str[ i_first_index + 3 ] = d;
o_hashes[ i ] = ::crc32( 0, reinterpret_cast<uint8_t*>(&( str[ 0 ] )), str_size ) ^ 0xFFFFFFFF;
++i;
}
}
}
}
}
}
void generate_hashes_2(std::string& i_format, const uint32_t i_first_index, const uint32_t i_second_index, std::vector<uint32_t>& o_hashes)
{
void generate_hashes_2( std::string& i_format, const uint32_t i_first_index, const uint32_t i_second_index,
std::vector< uint32_t >& o_hashes )
{
char* str = const_cast<char*>(i_format.data());
const uint32_t str_size = i_format.size();
o_hashes.resize(100000000);
o_hashes.resize( 100000000 );
uint32_t i = 0;
for (char a = '0'; a <= '9'; ++a)
for( char a = '0'; a <= '9'; ++a )
{
str[i_first_index] = a;
for (char b = '0'; b <= '9'; ++b)
str[ i_first_index ] = a;
for( char b = '0'; b <= '9'; ++b )
{
str[i_first_index + 1] = b;
for (char c = '0'; c <= '9'; ++c)
str[ i_first_index + 1 ] = b;
for( char c = '0'; c <= '9'; ++c )
{
str[i_first_index + 2] = c;
for (char d = '0'; d <= '9'; ++d)
str[ i_first_index + 2 ] = c;
for( char d = '0'; d <= '9'; ++d )
{
str[i_first_index + 3] = d;
for (char e = '0'; e <= '9'; ++e)
str[ i_first_index + 3 ] = d;
for( char e = '0'; e <= '9'; ++e )
{
str[i_second_index] = e;
for (char f = '0'; f <= '9'; ++f)
str[ i_second_index ] = e;
for( char f = '0'; f <= '9'; ++f )
{
str[i_second_index + 1] = f;
for (char g = '0'; g <= '9'; ++g)
str[ i_second_index + 1 ] = f;
for( char g = '0'; g <= '9'; ++g )
{
str[i_second_index + 2] = g;
for (char h = '0'; h <= '9'; ++h)
str[ i_second_index + 2 ] = g;
for( char h = '0'; h <= '9'; ++h )
{
str[i_second_index + 3] = h;
o_hashes[i] = ::crc32(0, reinterpret_cast<uint8_t*>(&(str[0])), str_size) ^ 0xFFFFFFFF;
str[ i_second_index + 3 ] = h;
o_hashes[ i ] = ::crc32( 0, reinterpret_cast<uint8_t*>(&( str[ 0 ] )), str_size ) ^ 0xFFFFFFFF;
++i;
}
}
@ -172,9 +169,7 @@ void generate_hashes_2(std::string& i_format, const uint32_t i_first_index, cons
}
}
}
}
}
}
}
}

11
deps/datReader/crc32.h vendored
View file

@ -5,9 +5,8 @@
#include <vector>
#include <string>
namespace xiv {
namespace utils {
namespace crc32 {
namespace xiv::utils::crc32
{
// Normal crc32 computation from a given intial crc value, use zlib.crc32 instead, the final XOR 0xFFFFFFFF is not done
uint32_t compute( const std::string& i_input, uint32_t init_crc = 0xFFFFFFFF );
@ -17,10 +16,10 @@ namespace crc32 {
uint32_t rev_compute( const std::string& i_input, uint32_t init_crc = 0 );
void generate_hashes_1( std::string& i_format, const uint32_t i_first_index, std::vector< uint32_t >& o_hashes );
void generate_hashes_2( std::string& i_format, const uint32_t i_first_index, const uint32_t i_second_index, std::vector< uint32_t >& o_hashes );
}
}
void generate_hashes_2( std::string& i_format, const uint32_t i_first_index, const uint32_t i_second_index,
std::vector< uint32_t >& o_hashes );
}
#endif // XIV_UTILS_CRC32_H

8
deps/datReader/stream.cpp vendored
View file

@ -4,13 +4,7 @@
#include <sstream>
#include <streambuf>
namespace xiv
{
namespace utils
{
namespace stream
namespace xiv::utils::stream
{
}
}
}

24
deps/datReader/stream.h vendored
View file

@ -5,23 +5,17 @@
#include <iostream>
#include <vector>
namespace xiv
namespace xiv::utils::stream
{
namespace utils
{
namespace stream
{
template<typename CharT, typename TraitsT = std::char_traits<CharT> >
class vectorwrapbuf : public std::basic_streambuf<CharT, TraitsT>
{
public:
vectorwrapbuf(std::vector<CharT> &vec)
template< typename CharT, typename TraitsT = std::char_traits< CharT > >
class vectorwrapbuf :
public std::basic_streambuf< CharT, TraitsT >
{
this->setg(vec.data(), vec.data(), vec.data() + vec.size());
public:
vectorwrapbuf( std::vector< CharT >& vec )
{
this->setg( vec.data(), vec.data(), vec.data() + vec.size() );
}
};
};
}
}
}
#endif // XIV_UTILS_STREAM_H

54
deps/datReader/zlib.cpp vendored
View file

@ -4,32 +4,28 @@
#include <zlib/zlib.h>
#include <vector>
namespace xiv
{
namespace utils
{
namespace zlib
namespace xiv::utils::zlib
{
void compress(const std::vector<char>& in, std::vector<char>& out)
{
// Fetching upper bound for out size
auto out_size = compressBound(in.size());
out.resize(out_size);
auto ret = compress2(reinterpret_cast<uint8_t*>(out.data()), &out_size,
reinterpret_cast<const uint8_t*>(in.data()), in.size(), Z_BEST_COMPRESSION);
if (ret != Z_OK)
void compress( const std::vector< char >& in, std::vector< char >& out )
{
throw std::runtime_error("Error at zlib uncompress: " + std::to_string(ret));
// Fetching upper bound for out size
auto out_size = compressBound( in.size() );
out.resize( out_size );
auto ret = compress2( reinterpret_cast<uint8_t*>(out.data()), &out_size,
reinterpret_cast<const uint8_t*>(in.data()), in.size(), Z_BEST_COMPRESSION );
if( ret != Z_OK )
{
throw std::runtime_error( "Error at zlib uncompress: " + std::to_string( ret ) );
}
out.resize(out_size);
}
out.resize( out_size );
}
void no_header_decompress(uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t out_size)
{
void no_header_decompress( uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t out_size )
{
z_stream strm;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
@ -38,10 +34,10 @@ void no_header_decompress(uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t
strm.next_in = Z_NULL;
// Init with -15 because we do not have header in this compressed data
auto ret = inflateInit2(&strm, -15);
if (ret != Z_OK)
auto ret = inflateInit2( &strm, -15 );
if( ret != Z_OK )
{
throw std::runtime_error("Error at zlib init: " + std::to_string(ret));
throw std::runtime_error( "Error at zlib init: " + std::to_string( ret ) );
}
// Set pointers to the right addresses
@ -50,16 +46,14 @@ void no_header_decompress(uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t
strm.next_out = out;
// Effectively decompress data
ret = inflate(&strm, Z_NO_FLUSH);
if (ret != Z_STREAM_END)
ret = inflate( &strm, Z_NO_FLUSH );
if( ret != Z_STREAM_END )
{
throw std::runtime_error("Error at zlib inflate: " + std::to_string(ret));
throw std::runtime_error( "Error at zlib inflate: " + std::to_string( ret ) );
}
// Clean up
inflateEnd(&strm);
}
inflateEnd( &strm );
}
}
}
}

13
deps/datReader/zlib.h vendored
View file

@ -4,18 +4,13 @@
#include <cstdint>
#include <vector>
namespace xiv
{
namespace utils
{
namespace zlib
namespace xiv::utils::zlib
{
void compress(const std::vector<char>& in, std::vector<char>& out);
void no_header_decompress(uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t out_size);
void compress( const std::vector< char >& in, std::vector< char >& out );
void no_header_decompress( uint8_t* in, uint32_t in_size, uint8_t* out, uint32_t out_size );
}
}
}
#endif // XIV_UTILS_ZLIB_H