From 4fa77965c12abf2f972b5f2cdd936745d8fe0536 Mon Sep 17 00:00:00 2001 From: Joshua Goins Date: Sun, 14 Apr 2024 11:24:18 -0400 Subject: [PATCH] Support index2 files Needed for extracting data from benchmarks, which only ship with this index file type. --- src/gamedata.rs | 91 ++++++++++++++++++++++++++++++++-------------- src/index.rs | 92 +++++++++++++++++++++++++++++++++++++++++++++-- src/repository.rs | 22 +++++++----- src/sqpack.rs | 24 ------------- 4 files changed, 167 insertions(+), 62 deletions(-) diff --git a/src/gamedata.rs b/src/gamedata.rs index c04c28d..f75514b 100755 --- a/src/gamedata.rs +++ b/src/gamedata.rs @@ -13,11 +13,10 @@ use crate::dat::DatFile; use crate::exd::EXD; use crate::exh::EXH; use crate::exl::EXL; -use crate::index::IndexFile; +use crate::index::{Index2File, IndexFile, IndexHashBitfield}; use crate::ByteBuffer; use crate::patch::{apply_patch, PatchError}; use crate::repository::{Category, Repository, string_to_category}; -use crate::sqpack::calculate_hash; /// Framework for operating on game data. pub struct GameData { @@ -27,7 +26,8 @@ pub struct GameData { /// Repositories in the game directory. pub repositories: Vec, - index_files: HashMap + index_files: HashMap, + index2_files: HashMap } fn is_valid(path: &str) -> bool { @@ -78,7 +78,8 @@ impl GameData { true => Some(Self { game_directory: String::from(directory), repositories: vec![], - index_files: HashMap::new() + index_files: HashMap::new(), + index2_files: HashMap::new() }), false => { warn!("Game data is not valid!"); @@ -157,15 +158,19 @@ impl GameData { /// } /// ``` pub fn exists(&mut self, path: &str) -> bool { - let hash = calculate_hash(path); - let index_path = self.get_index_filename(path); + let index_path = self.get_index_filenames(path); - self.cache_index_file(&index_path); - if let Some(index_file) = self.get_index_file(&index_path) { - index_file.entries.iter().any(|s| s.hash == hash) - } else { - false + self.cache_index_file((&index_path.0, &index_path.1)); + + if let Some(index_file) = self.get_index_file(&index_path.0) { + return index_file.exists(path); } + + if let Some(index2_file) = self.get_index2_file(&index_path.1) { + return index2_file.exists(path); + } + + false } /// Extracts the file located at `path`. This is returned as an in-memory buffer, and will usually @@ -185,18 +190,12 @@ impl GameData { pub fn extract(&mut self, path: &str) -> Option { debug!(file=path, "Extracting file"); - let hash = calculate_hash(path); - let index_path = self.get_index_filename(path); - - self.cache_index_file(&index_path); - let index_file = self.get_index_file(&index_path)?; - - let slice = index_file.entries.iter().find(|s| s.hash == hash); + let slice = self.find_entry(path); match slice { Some(entry) => { - let mut dat_file = self.get_dat_file(path, entry.bitfield.data_file_id().into())?; + let mut dat_file = self.get_dat_file(path, entry.data_file_id().into())?; - dat_file.read_from_offset(entry.bitfield.offset()) + dat_file.read_from_offset(entry.offset()) } None => None, } @@ -220,7 +219,7 @@ impl GameData { Some((&self.repositories[0], string_to_category(tokens[0])?)) } - fn get_index_filename(&self, path: &str) -> String { + fn get_index_filenames(&self, path: &str) -> (String, String) { let (repository, category) = self.parse_repository_category(path).unwrap(); let index_path: PathBuf = [ @@ -232,7 +231,16 @@ impl GameData { .iter() .collect(); - index_path.into_os_string().into_string().unwrap() + let index2_path: PathBuf = [ + &self.game_directory, + "sqpack", + &repository.name, + &repository.index2_filename(category), + ] + .iter() + .collect(); + + (index_path.into_os_string().into_string().unwrap(), index2_path.into_os_string().into_string().unwrap()) } /// Read an excel sheet by name (e.g. "Achievement") @@ -384,10 +392,16 @@ impl GameData { Ok(()) } - fn cache_index_file(&mut self, filename: &str) { - if !self.index_files.contains_key(filename) { - if let Some(index_file) = IndexFile::from_existing(filename) { - self.index_files.insert(filename.to_string(), index_file); + fn cache_index_file(&mut self, filenames: (&str, &str)) { + if !self.index_files.contains_key(filenames.0) { + if let Some(index_file) = IndexFile::from_existing(filenames.0) { + self.index_files.insert(filenames.0.to_string(), index_file); + } + } + + if !self.index_files.contains_key(filenames.1) { + if let Some(index_file) = Index2File::from_existing(filenames.1) { + self.index2_files.insert(filenames.1.to_string(), index_file); } } } @@ -395,6 +409,31 @@ impl GameData { fn get_index_file(&self, filename: &str) -> Option<&IndexFile> { self.index_files.get(filename) } + + fn get_index2_file(&self, filename: &str) -> Option<&Index2File> { + println!("Trying {}", filename); + self.index2_files.get(filename) + } + + fn find_entry(&mut self, path: &str) -> Option { + let index_path = self.get_index_filenames(path); + + self.cache_index_file((&index_path.0, &index_path.1)); + + if let Some(index_file) = self.get_index_file(&index_path.0) { + if let Some(entry) = index_file.find_entry(path) { + return Some(entry.bitfield); + } + } + + if let Some(index2_file) = self.get_index2_file(&index_path.1) { + if let Some(entry) = index2_file.find_entry(path) { + return Some(entry.bitfield); + } + } + + None + } } #[cfg(test)] diff --git a/src/index.rs b/src/index.rs index c3c4ea3..2ab1663 100755 --- a/src/index.rs +++ b/src/index.rs @@ -8,6 +8,7 @@ use std::io::SeekFrom; use binrw::BinRead; use binrw::binrw; use modular_bitfield::prelude::*; +use crate::crc::Jamcrc; #[binrw] #[brw(repr = u8)] @@ -40,6 +41,7 @@ pub struct SqPackIndexHeader { #[bitfield] #[binrw] #[br(map = Self::from_bytes)] +#[derive(Clone, Copy, Debug)] pub struct IndexHashBitfield { pub size: B1, pub data_file_id: B3, @@ -53,6 +55,16 @@ pub struct IndexHashTableEntry { pub(crate) bitfield: IndexHashBitfield, } +// The only difference between index and index2 is how the path hash is stored. +// The folder name and the filename are split in index1 (hence why it's 64-bits and not 32-bit) +// But in index2, its both the file and folder name in one single CRC hash. +#[binrw] +#[derive(Debug)] +pub struct Index2HashTableEntry { + pub hash: u32, + pub(crate) bitfield: IndexHashBitfield, +} + #[derive(Debug)] pub struct IndexEntry { pub hash: u64, @@ -69,15 +81,89 @@ pub struct IndexFile { index_header: SqPackIndexHeader, #[br(seek_before = SeekFrom::Start(index_header.index_data_offset.into()))] - #[br(count = index_header.index_data_size / 16)] + // +4 because of padding + #[br(count = index_header.index_data_size / core::mem::size_of::() as u32 + 4)] pub entries: Vec, } +#[binrw] +#[br(little)] +pub struct Index2File { + sqpack_header: SqPackHeader, + + #[br(seek_before = SeekFrom::Start(sqpack_header.size.into()))] + index_header: SqPackIndexHeader, + + #[br(seek_before = SeekFrom::Start(index_header.index_data_offset.into()))] + #[br(count = index_header.index_data_size / core::mem::size_of::() as u32)] + pub entries: Vec, +} + +const CRC: Jamcrc = Jamcrc::new(); + impl IndexFile { /// Creates a new reference to an existing index file. - pub fn from_existing(path: &str) -> Option { + pub fn from_existing(path: &str) -> Option { let mut index_file = std::fs::File::open(path).ok()?; - IndexFile::read(&mut index_file).ok() + Self::read(&mut index_file).ok() + } + + /// Calculates a partial hash for a given path + pub fn calculate_partial_hash(path: &str) -> u32 { + let lowercase = path.to_lowercase(); + + CRC.checksum(lowercase.as_bytes()) + } + + /// Calculates a hash for `index` files from a game path. + pub fn calculate_hash(path: &str) -> u64 { + let lowercase = path.to_lowercase(); + + let pos = lowercase.rfind('/').unwrap(); + + let (directory, filename) = lowercase.split_at(pos); + + let directory_crc = CRC.checksum(directory.as_bytes()); + let filename_crc = CRC.checksum(filename[1..filename.len()].as_bytes()); + + (directory_crc as u64) << 32 | (filename_crc as u64) + } + + // TODO: turn into traits? + pub fn exists(&self, path: &str) -> bool { + let hash = IndexFile::calculate_hash(path); + self.entries.iter().any(|s| s.hash == hash) + } + + pub fn find_entry(&self, path: &str) -> Option<&IndexHashTableEntry> { + let hash = IndexFile::calculate_hash(path); + self.entries.iter().find(|s| s.hash == hash) } } + +impl Index2File { + /// Creates a new reference to an existing index2 file. + pub fn from_existing(path: &str) -> Option { + let mut index_file = std::fs::File::open(path).ok()?; + + Some(Self::read(&mut index_file).unwrap()) + } + + /// Calculates a hash for `index2` files from a game path. + pub fn calculate_hash(path: &str) -> u32 { + let lowercase = path.to_lowercase(); + + CRC.checksum(lowercase.as_bytes()) + } + + pub fn exists(&self, path: &str) -> bool { + let hash = Index2File::calculate_hash(path); + self.entries.iter().any(|s| s.hash == hash) + } + + pub fn find_entry(&self, path: &str) -> Option<&Index2HashTableEntry> { + let hash = Index2File::calculate_hash(path); + self.entries.iter().find(|s| s.hash == hash) + } +} \ No newline at end of file diff --git a/src/repository.rs b/src/repository.rs index 44b8724..ca4d290 100755 --- a/src/repository.rs +++ b/src/repository.rs @@ -177,15 +177,11 @@ impl Repository { d.push("ffxivgame.ver"); let version = read_version(d.as_path()); - if version.is_some() { - Some(Repository { - name: "ffxiv".parse().unwrap(), - repo_type: Base, - version, - }) - } else { - None - } + Some(Repository { + name: "ffxiv".parse().unwrap(), + repo_type: Base, + version, + }) } fn expansion(&self) -> i32 { @@ -205,6 +201,14 @@ impl Repository { "win32" ) } + + /// Calculate an index2 filename for a specific category, like _"0a0000.win32.index"_. + pub fn index2_filename(&self, category: Category) -> String { + format!( + "{}2", + self.index_filename(category) + ) + } /// Calculate a dat filename given a category and a data file id, returns something like _"0a0000.win32.dat0"_. pub fn dat_filename(&self, category: Category, data_file_id: u32) -> String { diff --git a/src/sqpack.rs b/src/sqpack.rs index c26880a..647d1cf 100755 --- a/src/sqpack.rs +++ b/src/sqpack.rs @@ -6,32 +6,8 @@ use std::io::{Read, Seek, SeekFrom}; use binrw::BinRead; use crate::compression::no_header_decompress; -use crate::crc::Jamcrc; use crate::dat::{BlockHeader, CompressionMode}; -const CRC: Jamcrc = Jamcrc::new(); - -/// Calculates a partial hash for a given path -pub fn calculate_partial_hash(path: &str) -> u32 { - let lowercase = path.to_lowercase(); - - CRC.checksum(lowercase.as_bytes()) -} - -/// Calculates a hash for `index` files from a game path. -pub fn calculate_hash(path: &str) -> u64 { - let lowercase = path.to_lowercase(); - - let pos = lowercase.rfind('/').unwrap(); - - let (directory, filename) = lowercase.split_at(pos); - - let directory_crc = CRC.checksum(directory.as_bytes()); - let filename_crc = CRC.checksum(filename[1..filename.len()].as_bytes()); - - (directory_crc as u64) << 32 | (filename_crc as u64) -} - pub fn read_data_block(mut buf: T, starting_position: u64) -> Option> { buf.seek(SeekFrom::Start(starting_position)).ok()?;