mirrors/sapphire
1
Fork 0
mirror of https://github.com/SapphireServer/Sapphire.git synced 2025-04-23 21:27:45 +00:00
Code Activity

Merge pull request #370 from takhlaq/discovery_layers

Browse source 
halal discovery
This commit is contained in:
Mordred 2018-08-05 12:05:42 +02:00 • committed by GitHub
commit 132a650bf3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 1695 additions and 2492 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3795
sql/discoveryinfo.sql
View file

File diff suppressed because it is too large Load diff

16
src/tools/discovery_parser/lgb.h
View file

@ -195,9 +195,20 @@ public:
struct LGB_MAPRANGE_HEADER : public LGB_ENTRY_HEADER
{
uint32_t type;
uint16_t unknown2;
uint8_t unknown2;
uint8_t unknown2_1;
uint16_t unknown3;
uint8_t unknown4[0x10];
uint32_t unknown5;
uint32_t mapId;
uint32_t offsetX, offsetY;
uint32_t unkInts[4];
uint16_t unkShort;
uint8_t unkFlag;
uint8_t unkFlag2;
uint8_t discoveryIndex;
uint8_t unkFlag3;
uint8_t unkFlag4;
uint8_t unknown4[0x09];
};
struct LGB_MAPRANGE_ENTRY : public LGB_ENTRY
@ -210,6 +221,7 @@ public:
{
header = *reinterpret_cast< LGB_MAPRANGE_HEADER* >( buf + offset );
name = std::string( buf + offset + header.nameOffset );
};
};

356
src/tools/discovery_parser/main.cpp
View file

@ -16,9 +16,6 @@
#include "tex.h"
#include "tex_decode.h"
//#include "s3tc/s3tc.h"
#ifndef STANDALONE
#include <GameData.h>
#include <File.h>
#include <DatCat.h>
@ -26,15 +23,21 @@
#include <ExdCat.h>
#include <Exd.h>
#include <boost/algorithm/string.hpp>
#endif
// garbage to ignore models
bool ignoreModels = false;
struct ZoneInfo
{
uint16_t id;
std::string name;
std::string path;
};
// parsing shit
std::string gamePath( "C:\\Program Files (x86)\\SquareEnix\\FINAL FANTASY XIV - A Realm Reborn\\game\\sqpack" );
std::unordered_map< uint32_t, std::string > eobjNameMap;
std::unordered_map< uint16_t, std::string > zoneNameMap;
std::unordered_map< uint16_t, ZoneInfo > zoneInfoMap;
std::unordered_map< uint16_t, std::vector< std::pair< uint16_t, std::string > > > zoneInstanceMap;
uint32_t zoneId;
@ -53,7 +56,7 @@ struct vec2
float x, y;
};
struct DiscoveryMap
struct DiscoveryMap : std::enable_shared_from_this< DiscoveryMap >
{
std::string path;
Image img;
@ -65,7 +68,7 @@ struct DiscoveryMap
constexpr static int tileWidth = 128;
constexpr static int tiles = discoveryMapCols * discoveryMapRows;
uint32_t getColour( uint8_t mapIndex, float x, float y )
uint32_t getColour( uint8_t mapIndex, float x, float y, float scale )
{
auto ogX = x, ogY = y;
int col = ( mapIndex % ( int )( ( float )img.width / ( float )tileWidth ) );
@ -75,18 +78,13 @@ struct DiscoveryMap
int tileX = ( col * ( float )tileWidth ) + x;
int tileY = ( row * ( float )tileWidth ) + y;
if( tileX < 0 || tileY < 0 )
if( tileX < 0 || tileY < 0 || tileY > img.data.size() - 1 || tileX > img.data[0].size() - 1 )
{
std::cout << "Unable to find tile coord for " << x << " " << y << " mapIndex " << std::to_string( mapIndex ) << "\n";
return 0;
}
if( tileY > img.data.size() - 1 )
return 0;
if( tileX > img.data[0].size() - 1)
return 0;
//std::cout << "getColour col " << col << " row " << row << " tileX " << tileX << " tileY " << tileY << " tile index " << std::to_string( mapIndex ) << "\n";
//std::cout << "getColour col " << col << " row " << row << " tileX " << tileX << " tileY " << tileY << " tile index " << std::to_string( unkFlag2 ) << "\n";
auto colour = img.data[tileY][tileX];
return colour;
@ -105,16 +103,14 @@ struct DiscoveryMap
vec2 get2dPosFrom3d( float x, float y, float scale )
{
vec2 ret;
float scale2 = ( ( float )mapScale / 100.f );
float scale2 = ( float )( ( float )mapScale / 100.f );
ret.x = ( ( x * scale2 ) + 1024.f );
ret.y = ( ( y * scale2 ) + 1024.f );
//ret.x = ( x * scale2 ) + mapOffsetX;
//ret.y = ( y * scale2 ) + mapOffsetY;
return ret;
}
};
std::map< uint16_t, DiscoveryMap > discoveryMaps;
std::map< uint16_t, std::map< uint16_t, std::map< uint16_t, std::shared_ptr< DiscoveryMap > > > > discoveryMaps;
@ -140,7 +136,7 @@ void initExd( const std::string& gamePath )
}
std::string getMapExdEntries( uint32_t mapId )
void getMapExdEntries( uint32_t zoneId )
{
static auto& cat = eData->get_category( "Map" );
static auto exd = static_cast< xiv::exd::Exd >( cat.get_data_ln( xiv::exd::Language::none ) );
@ -151,8 +147,6 @@ std::string getMapExdEntries( uint32_t mapId )
{
// fields from SaintCoinach https://github.com/ufx/SaintCoinach/blob/master/SaintCoinach/ex.json#L6358
auto id = row.first;
if( id != mapId )
continue;
auto& fields = row.second;
@ -167,7 +161,9 @@ std::string getMapExdEntries( uint32_t mapId )
case DataType::float32: 8
case DataType::uint64: 9
*/
auto territory = *boost::get< uint16_t >( &fields.at( 14 ) );
if( territory != zoneId )
continue;
auto mapZoneIndex = *boost::get< int8_t >( &fields.at( 2 ) );
auto hierarchy = *boost::get< uint8_t >( &fields.at( 3 ) );
auto pathStr = *boost::get< std::string >( &fields.at( 5 ) );
@ -176,94 +172,63 @@ std::string getMapExdEntries( uint32_t mapId )
auto mapOffsetY = *boost::get< int16_t >( &fields.at( 8 ) );
auto discoveryIdx = *boost::get< int16_t >( &fields.at( 12 ) );
auto discoveryCompleteBitmask = *boost::get< uint32_t >( &fields.at( 13 ) );
auto territory = *boost::get< uint16_t >( &fields.at( 14 ) );
char texStr[255];
auto teriStr = pathStr.substr( 0, pathStr.find_first_of( '/' ) );
sprintf( &texStr[0], "ui/map/%s/%s%02Xd.tex", pathStr.c_str(), teriStr.c_str(), mapZoneIndex );
char discoveryFileName[255];
sprintf( &discoveryFileName[0], "%s%02u", teriStr.c_str(), mapZoneIndex );
sprintf( &texStr[0], "ui/map/%s/%sd.tex", pathStr.c_str(), &discoveryFileName[0] );
if( discoveryMaps.find( territory ) == discoveryMaps.end() )
if( discoveryMaps[territory].find( mapZoneIndex ) == discoveryMaps[territory].end() ||
discoveryMaps[territory][mapZoneIndex].find( hierarchy ) == discoveryMaps[territory][mapZoneIndex].end() )
{
std::string fileName( discoveryFileName );
try
{
auto texFile = data1->getFile( &texStr[0] );
std::string rawTexFile( teriStr + "0" + std::to_string( mapZoneIndex ) );
texFile->exportToFile( rawTexFile + "d.tex" );
auto tex = TEX_FILE( rawTexFile + "d.tex" );
std::string rawTexFile( texStr );
auto pDiscoveryMap = std::make_shared< DiscoveryMap >();
auto& discoveryMap = *pDiscoveryMap;
std::ifstream discoveryFile( fileName + ".img", std::ios::binary );
if( !discoveryFile.good() )
{
auto texFile = data1->getFile( rawTexFile );
texFile->exportToFile( fileName + ".tex" );
auto tex = TEX_FILE( fileName + ".tex" );
int mipMapDivide = 1;
int h = tex.header.uncompressedHeight;
int w = tex.header.uncompressedWidth;
DiscoveryMap discoveryMap;
discoveryMap.img = DecodeTexDXT1( tex, tex.header.mipMaps[0], h / mipMapDivide, w / mipMapDivide,
( h / mipMapDivide ) / 4, ( w / mipMapDivide ) / 4
);
int mipMapDivide = 1;
int h = tex.header.uncompressedHeight;
int w = tex.header.uncompressedWidth;
discoveryMap.img = DecodeTexDXT1( tex, tex.header.mipMaps[0], h / mipMapDivide, w / mipMapDivide,
( h / mipMapDivide ) / 4, ( w / mipMapDivide ) / 4
);
discoveryMap.img.toFile( fileName + ".img" );
}
else
{
std::stringstream ss;
ss << discoveryFile.rdbuf();
discoveryMap.img = std::move( Image( &ss.str()[0] ) );
discoveryFile.close();
}
discoveryMap.img.toFile( rawTexFile + ".img" );
discoveryMap.mapId = id;
discoveryMap.path = &texStr[0];
discoveryMap.path = rawTexFile;
discoveryMap.mapOffsetX = mapOffsetX;
discoveryMap.mapOffsetY = mapOffsetY;
discoveryMap.mapScale = sizeFactor;
std::cout << "Image Height: " << discoveryMap.img.height << " Width: " << discoveryMap.img.width << "\n";
discoveryMaps.emplace( territory, discoveryMap );
discoveryMaps[territory][mapZoneIndex][hierarchy] = pDiscoveryMap;
}
catch( std::exception& e )
{
std::cout << "[Error] " << std::string( texStr ) << " " << e.what() << "\n";
}
}
return std::string( std::to_string( mapZoneIndex ) + ", " + std::to_string( hierarchy ) + ", " + "\"" + std::string( &texStr[0] ) + "\", " +
std::to_string( discoveryIdx ) + ", " + std::to_string( discoveryCompleteBitmask ) );
}
return "";
}
void dumpLevelExdEntries( uint32_t zoneId, const std::string& name = std::string() )
{
static auto& cat = eData->get_category( "Level" );
static auto exd = static_cast< xiv::exd::Exd >( cat.get_data_ln( xiv::exd::Language::none ) );
std::string fileName( name + "_" + std::to_string( zoneId ) + "_Level" + ".csv" );
std::ofstream outfile( fileName, std::ios::trunc );
std::cout << "[Info] Writing level.exd entries to " << fileName << "\n";
if( outfile.good() )
{
outfile.close();
outfile.open( fileName, std::ios::app );
static std::string levelHeader( "id, objectid, mapid, x, y, z, yaw, radius, type, zone, " );
static std::string header( levelHeader + "mapZoneIdx, hierarchy, path, size, discoveryIdx, discoveryCompleteBitmask \n" );
outfile.write( header.c_str(), header.size() );
static auto& rows = exd.get_rows();
for( auto& row : rows )
{
auto id = row.first;
auto& fields = row.second;
auto x = *boost::get< float >( &fields.at( 0 ) );
auto y = *boost::get< float >( &fields.at( 1 ) );
auto z = *boost::get< float >( &fields.at( 2 ) );
auto yaw = *boost::get< float >( &fields.at( 3 ) );
auto radius = *boost::get< float >( &fields.at( 4 ) );
auto type = *boost::get< uint8_t >( &fields.at( 5 ) );
auto objectid = *boost::get< uint32_t >( &fields.at( 6 ) );
auto mapid = *boost::get< uint16_t >( &fields.at( 7 ) );
auto zone = *boost::get< uint16_t >( &fields.at( 9 ) );
if( zone == zoneId )
{
std::string outStr(
std::to_string( id ) + ", " + std::to_string( objectid ) + ", " + std::to_string( mapid ) + ", " +
std::to_string( x ) + ", " + std::to_string( y ) + ", " + std::to_string( z ) + ", " +
std::to_string( yaw ) + ", " + std::to_string( radius ) + ", " + std::to_string( type ) + ", " + std::to_string( zone ) + ", " +
getMapExdEntries( mapid ) + "\n"
);
outfile.write( outStr.c_str(), outStr.size() );
}
}
}
return;
}
std::string zoneNameToPath( const std::string& name )
@ -271,50 +236,45 @@ std::string zoneNameToPath( const std::string& name )
std::string path;
bool found = false;
#ifdef STANDALONE
auto inFile = std::ifstream( "territorytype.exh.csv" );
if( inFile.good() )
{
std::string line;
std::regex re( "(\\d+),\"(.*)\",\"(.*)\",.*" );
while( std::getline( inFile, line ) )
{
std::smatch match;
if( std::regex_match( line, match, re )
{
auto tmpId = std::stoul( match[1].str() );
if( !found && name == match[2].str() )
{
zoneId = tmpId;
path = match[3].str();
found = true;
}
zoneNameMap[tmpId] = match[2].str();
}
}
inFile.close();
}
#else
static auto& cat = eData->get_category( "TerritoryType" );
static auto exd = static_cast< xiv::exd::Exd >( cat.get_data_ln( xiv::exd::Language::none ) );
static auto& rows = exd.get_rows();
for( auto& row : rows )
if( zoneInfoMap.size() == 0 )
{
auto& fields = row.second;
auto teriName = *boost::get< std::string >( &fields.at( static_cast< size_t >( TerritoryTypeExdIndexes::TerritoryType ) ) );
if( teriName.empty() )
continue;
auto teriPath = *boost::get< std::string >( &fields.at( static_cast< size_t >( TerritoryTypeExdIndexes::Path ) ) );
if( !found && boost::iequals( name, teriName ) )
for( auto& row : rows )
{
path = teriPath;
found = true;
zoneId = row.first;
auto& fields = row.second;
auto teriName = *boost::get< std::string >( &fields.at( static_cast< size_t >( TerritoryTypeExdIndexes::TerritoryType ) ) );
if( teriName.empty() )
continue;
auto teriPath = *boost::get< std::string >( &fields.at( static_cast< size_t >( TerritoryTypeExdIndexes::Path ) ) );
ZoneInfo info;
info.id = row.first;
info.path = teriPath;
info.name = teriName;
zoneInfoMap[row.first] = info;
if( !found && boost::iequals( name, teriName ) )
{
found = true;
path = teriPath;
zoneId = info.id;
}
}
}
else
{
for( const auto& entry : zoneInfoMap )
{
if( found = boost::iequals( name, entry.second.name ) )
{
path = entry.second.path;
zoneId = entry.second.id;
break;
}
}
zoneNameMap[row.first] = teriName;
}
#endif
if( found )
{
@ -350,78 +310,109 @@ void writeEobjEntry( std::ofstream& out, LGB_ENTRY* pObj )
static std::string mapRangeStr( "\"MapRange\", " );
static std::ofstream discoverySql( "discovery.sql" , std::ios::app );
uint32_t id;
uint32_t unknown = 0, unknown2 = 0;
uint32_t unknown2 = 0, unknown2_1 = 0, unknown3 = 0;
std::string name;
std::string typeStr;
uint32_t eobjlevelHierachyId = 0;
static std::map< uint32_t, std::map< uint32_t, uint32_t > > exportedMapRange;
auto pMapRange = reinterpret_cast< LGB_MAPRANGE_ENTRY* >( pObj );
id = pMapRange->header.unknown;
unknown = pMapRange->header.unknown2;
unknown2 = pMapRange->header.unknown3;
unknown2 = pMapRange->header.unknown2;
unknown2_1 = pMapRange->header.unknown2_1;
unknown3 = pMapRange->header.unknown3;
typeStr = mapRangeStr;
// discovery shit
vec2 pos;
auto subArea = -255;
vec2 pos{0};
auto subArea = 0;
auto mapId = -1;
auto discoveryIndex = pMapRange->header.discoveryIndex;
vec3 translation = pObj->header.translation;
vec3 distanceVec = pObj->header.scale;
bool found = false;
float scale = 100.f; // pMapRange->header.unknown2
auto it = discoveryMaps.find( zoneId );
if( it != discoveryMaps.end() )
float scale = 100.f; //pMapRange->header.unknown2
if( pMapRange->header.mapId == 0 )
{
auto& map = it->second;
pos = map.get2dPosFrom3d( translation.x, translation.z, scale );
mapId = map.mapId;
//std::cout << "3d coords " << pObj->header.translation.x << " " << pObj->header.translation.z << "\n";
//std::cout << "2d coords " << pos.x << " " << pos.y << "\n";
for( int i = 0; i < map.tiles; ++i )
auto it = discoveryMaps.find( zoneId );
if( it != discoveryMaps.end() )
{
auto colour = map.getColour( i, pos.x, pos.y );
auto r = ( colour >> 16 ) & 0xFF;
auto g = ( colour >> 8 ) & 0xFF;
auto b = ( colour >> 0 ) & 0xFF;
//std::cout << "R " << r << " G " << g << " B " << b << "\n";
for( const auto& mapHierarchy : it->second )
{
if( b > 0x40 )
{
subArea = i * 3 + 3;
if( subArea > 0 )
break;
}
else if( g > 0x40 )
for( const auto& levelHierarchy : mapHierarchy.second )
{
subArea = i * 3 + 2;
break;
}
else if( r > 0x40 )
{
// out of bounds
if( i == 0 )
if( subArea > 0 )
break;
subArea = i * 3 + 1;
break;
auto& map = *levelHierarchy.second;
pos = map.get2dPosFrom3d( translation.x, translation.z, scale );
mapId = map.mapId;
for( int i = 0; i < map.tiles; i++ )
{
auto colour = map.getColour( i, pos.x, pos.y, scale );
auto a = ( colour >> 24 ) & 0xFF;
auto r = ( colour >> 16 ) & 0xFF;
auto g = ( colour >> 8 ) & 0xFF;
auto b = ( colour >> 0 ) & 0xFF;
if( a > 0 && ( r + b + g ) > 0 )
{
if( r > 0 )
{
// out of bounds
if( i == 0 )
continue;
subArea = ( i * 3 ) + 1;
}
else if( g > 0 )
{
subArea = ( i * 3 ) + 2;
}
else if( b > 0 )
{
subArea = ( i * 3 ) + 3;
}
break;
}
}
}
}
}
subArea--;
}
subArea--;
if( subArea < -254 )
else
{
std::cout << "\tUnable to find subarea for maprange " << std::to_string( id ) << "\n";
mapId = pMapRange->header.mapId;
}
subArea = pMapRange->header.discoveryIndex + 1;
//if( discoveryIndex == subArea )
{
//std::cout << std::to_string( id ) << " discoveryIndex " << std::to_string( discoveryIndex ) << " subArea " << subArea << "\n";
}
if( discoveryIndex == 0 )
{
//std::cout << "\tUnable to find subarea for maprange " << std::to_string( id ) << " mapCoord " << pos.x << " " << pos.y <<
// "\tzoneCoord " << translation.x << " " << translation.y << " " << translation.z << " " << "\n";
return;
}
else if( mapId == -1 )
{
//std::cout << "\tUnable to find subarea for maprange " << std::to_string(id) << " " << "\n";
return;
}
if( exportedMapRange[id][mapId] == discoveryIndex )
return;
exportedMapRange[id][mapId] = discoveryIndex;
std::string outStr( "INSERT INTO discoveryinfo VALUES (" +
std::to_string( id ) + ", " + std::to_string( mapId ) + ", " + std::to_string( subArea ) + ");\n"
std::to_string( id ) + ", " + std::to_string( mapId ) + ", " + std::to_string( discoveryIndex ) + ");\n"
//std::to_string( pObj->header.translation.x ) + ", " + std::to_string( pObj->header.translation.y ) + ", " + std::to_string( pObj->header.translation.z ) +
//", " + std::to_string( subArea ) + "" + "\n"
);
@ -447,13 +438,14 @@ void readFileToBuffer( const std::string& path, std::vector< char >& buf )
}
}
bool isEx = false;
int main( int argc, char* argv[] )
{
auto startTime = std::chrono::system_clock::now();
auto entryStartTime = std::chrono::system_clock::now();
std::vector< std::string > argVec( argv + 1, argv + argc );
// todo: support expansions
std::string zoneName = "s1d1";
bool dumpAll = ignoreModels = std::remove_if( argVec.begin(), argVec.end(), []( auto arg ){ return arg == "--dump-all"; } ) != argVec.end();
@ -476,10 +468,10 @@ int main( int argc, char* argv[] )
if( dumpAll )
{
zoneNameToPath( "w1f3" );
zoneNameToPath( "r1f1" );
for( const auto& zone : zoneNameMap )
zoneDumpList.emplace( zone.second );
for( const auto& zone : zoneInfoMap )
zoneDumpList.emplace( zone.second.name );
}
else
{
@ -491,19 +483,18 @@ LABEL_DUMP:
zoneName = *zoneDumpList.begin();
try
{
const auto& zonePath = zoneNameToPath( zoneName );
const auto zonePath = zoneNameToPath( zoneName );
std::string listPcbPath( zonePath + "/collision/list.pcb" );
std::string bgLgbPath( zonePath + "/level/bg.lgb" );
std::string planmapLgbPath( zonePath + "/level/planmap.lgb" );
std::string collisionFilePath( zonePath + "/collision/" );
isEx = bgLgbPath.find( "ex1" ) != -1 || bgLgbPath.find( "ex2" ) != -1;
std::vector< char > section;
std::vector< char > section1;
std::vector< char > section2;
#ifndef STANDALONE
const xiv::dat::Cat& test = data1->getCategory( "bg" );
auto test_file = data1->getFile( bgLgbPath );
section = test_file->access_data_sections().at( 0 );
@ -512,19 +503,14 @@ LABEL_DUMP:
auto test_file1 = data1->getFile( listPcbPath );
section1 = test_file1->access_data_sections().at( 0 );
#else
{
readFileToBuffer( bgLgbPath, section );
readFileToBuffer( listPcbPath, section1 );
}
#endif
std::vector< std::string > stringList;
uint32_t offset1 = 0x20;
loadEobjNames();
dumpLevelExdEntries( zoneId, zoneName );
//loadEobjNames();
getMapExdEntries( zoneId );
std::string eobjFileName( zoneName + "_eobj.csv" );
std::ofstream eobjOut( eobjFileName, std::ios::trunc );
if( !eobjOut.good() )

20
src/tools/discovery_parser/vec3.h
View file

@ -28,4 +28,24 @@ static vec3 operator *(const vec3& lhs, const matrix4& rhs)
ret.z = rhs(2, 0) * lhs.x + rhs(2, 1) * lhs.y + rhs(2, 2) * lhs.z;
return ret;
};
static vec3 operator *(const vec3& lhs, float scalar)
{
return {lhs.x * scalar, lhs.y * scalar, lhs.z * scalar};
}
static vec3 operator +(const vec3& lhs, const vec3& rhs)
{
return {lhs.x + rhs.x, lhs.y + rhs.y, lhs.z + rhs.z};
}
static vec3 operator -(const vec3& lhs, const vec3& rhs)
{
return {lhs.x - rhs.x, lhs.y - rhs.y, lhs.z - rhs.z};
}
static vec3 operator /(const vec3& lhs, const vec3& rhs)
{
return {lhs.x / rhs.x, lhs.y / rhs.y, lhs.z / rhs.z};
}
static vec3 operator /(const vec3& lhs, float scalar)
{
return {lhs.x / scalar, lhs.y / scalar, lhs.z / scalar};
}
#endif