From 81fff744f2636926d8bac4c65fedeaa3d4e60699 Mon Sep 17 00:00:00 2001 From: Joshua Goins Date: Tue, 8 Jul 2025 21:37:03 -0400 Subject: [PATCH] Overhaul file resource handling This is a big API change, and it's probably not in it's ideal shape right now. Physis makes a big assumption that the pool of game data that you're reading from is in the compressed, SqPack format like in retail. However, as the projects that use Physis expand - so does the scope of data people want to punt into it. For example, we have Icarus that makes it easy to read Excel data but it can only do so through SqPack which makes it less useful in scenarios where you want to read unpacked Excel modded files or perhaps Excel data from the network. So I created a Resource trait that GameData now implements (and it's now called SqPackResource.) In a similar vain, Physis comes with an UnpackedResource and a ResourceResolver to chain multiple Resources together. --- examples/extractor.rs | 4 +- src/lib.rs | 6 +- src/resource/mod.rs | 48 ++++++++ src/resource/resolver.rs | 54 +++++++++ src/{gamedata.rs => resource/sqpack.rs} | 151 ++++++++++-------------- src/resource/unpacked.rs | 62 ++++++++++ 6 files changed, 232 insertions(+), 93 deletions(-) create mode 100644 src/resource/mod.rs create mode 100644 src/resource/resolver.rs rename src/{gamedata.rs => resource/sqpack.rs} (85%) mode change 100755 => 100644 create mode 100644 src/resource/unpacked.rs diff --git a/examples/extractor.rs b/examples/extractor.rs index 9464614..e0dc553 100644 --- a/examples/extractor.rs +++ b/examples/extractor.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later use physis::common::Platform; -use physis::gamedata::GameData; +use physis::resource::SqPackResource; use std::env; use std::fs::File; use std::io::Write; @@ -21,7 +21,7 @@ fn main() { let destination_path = &args[3]; // Create a GameData struct, this manages the repositories. It allows us to easily extract files. - let mut game_data = GameData::from_existing(Platform::Win32, game_dir); + let mut game_data = SqPackResource::from_existing(Platform::Win32, game_dir); // Extract said file: let Some(game_file) = game_data.extract(file_path) else { diff --git a/src/lib.rs b/src/lib.rs index 8eaa132..36cfe6b 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,9 +11,6 @@ pub type ByteSpan<'a> = &'a [u8]; /// Represents a continuous block of memory which is owned. pub type ByteBuffer = Vec; -/// Reading and writing game data repositories, such as "ffxiv" and "ex1", and so on. -pub mod gamedata; - /// Parsing game repositories, such as "ffxiv", "ex1" and their version information. pub mod repository; @@ -149,6 +146,9 @@ pub mod lvb; /// Reading SVB files pub mod svb; +/// File resource handling. +pub mod resource; + mod bcn; mod error; diff --git a/src/resource/mod.rs b/src/resource/mod.rs new file mode 100644 index 0000000..e2ba470 --- /dev/null +++ b/src/resource/mod.rs @@ -0,0 +1,48 @@ +mod resolver; +pub use resolver::ResourceResolver; + +mod sqpack; +pub use sqpack::SqPackResource; + +mod unpacked; +pub use unpacked::UnpackedResource; + +use crate::ByteBuffer; + +/// Represents a source of files for reading. +/// This abstracts away some of the nitty-gritty of where files come from. These could be coming from a compressed archive like SqPack, unpacked files on disk, or even a network. +pub trait Resource { + /// Reads the file located at `path`. This is returned as an in-memory buffer, and will usually + /// have to be further parsed. + /// + /// # Example + /// + /// ```should_panic + /// # use physis::resource::{Resource, SqPackResource}; + /// # use std::io::Write; + /// # use physis::common::Platform; + /// let mut game = SqPackResource::from_existing(Platform::Win32, "SquareEnix/Final Fantasy XIV - A Realm Reborn/game"); + /// let data = game.read("exd/root.exl").unwrap(); + /// + /// let mut file = std::fs::File::create("root.exl").unwrap(); + /// file.write(data.as_slice()).unwrap(); + /// ``` + fn read(&mut self, path: &str) -> Option; + + /// Checks if a file exists + /// While you could abuse `read` to do this, in some Resources they can optimize this since it doesn't read data. + /// + /// # Example + /// + /// ``` + /// # use physis::common::Platform; + /// # use physis::resource::{Resource, SqPackResource}; + /// let mut game = SqPackResource::from_existing(Platform::Win32, "SquareEnix/Final Fantasy XIV - A Realm Reborn/game"); + /// if game.exists("exd/cid.exl") { + /// println!("Cid really does exist!"); + /// } else { + /// println!("Oh noes!"); + /// } + /// ``` + fn exists(&mut self, path: &str) -> bool; +} diff --git a/src/resource/resolver.rs b/src/resource/resolver.rs new file mode 100644 index 0000000..eb4d3a1 --- /dev/null +++ b/src/resource/resolver.rs @@ -0,0 +1,54 @@ +use crate::ByteBuffer; + +use super::Resource; + +/// Allows chaining multiple FileSources together. +/// +/// # Example +/// +/// ``` +/// # use physis::resource::{ResourceResolver, SqPackResource, UnpackedResource}; +/// # use physis::common::Platform; +/// let sqpack_source = SqPackResource::from_existing(Platform::Win32, "SquareEnix/Final Fantasy XIV - A Realm Reborn/game"); +/// let file_source = UnpackedResource::from_existing("unpacked/"); +/// let mut resolver = ResourceResolver::new(); +/// resolver.add_source(Box::new(file_source)); // first has most priority +/// resolver.add_source(Box::new(sqpack_source)); // this is the fallback +/// ``` +pub struct ResourceResolver { + resolvers: Vec>, +} + +impl ResourceResolver { + pub fn new() -> Self { + Self { + resolvers: Vec::new(), + } + } + + pub fn add_source(&mut self, source: Box) { + self.resolvers.push(source); + } +} + +impl Resource for ResourceResolver { + fn read(&mut self, path: &str) -> Option { + for resolver in &mut self.resolvers { + if let Some(bytes) = resolver.read(path) { + return Some(bytes); + } + } + + return None; + } + + fn exists(&mut self, path: &str) -> bool { + for resolver in &mut self.resolvers { + if resolver.exists(path) { + return true; + } + } + + return false; + } +} diff --git a/src/gamedata.rs b/src/resource/sqpack.rs old mode 100755 new mode 100644 similarity index 85% rename from src/gamedata.rs rename to src/resource/sqpack.rs index a2b6f8f..2e5dfa5 --- a/src/gamedata.rs +++ b/src/resource/sqpack.rs @@ -1,40 +1,21 @@ -// SPDX-FileCopyrightText: 2023 Joshua Goins -// SPDX-License-Identifier: GPL-3.0-or-later +use std::{ + collections::HashMap, + fs::{self, DirEntry, ReadDir}, + path::PathBuf, +}; -use std::collections::HashMap; -use std::fs; -use std::fs::{DirEntry, ReadDir}; -use std::path::PathBuf; +use crate::{ + ByteBuffer, + common::{Language, Platform, read_version}, + exd::EXD, + exh::EXH, + exl::EXL, + patch::{PatchError, ZiPatch}, + repository::{Category, Repository, string_to_category}, + sqpack::{IndexEntry, SqPackData, SqPackIndex}, +}; -use crate::ByteBuffer; -use crate::common::{Language, Platform, read_version}; -use crate::exd::EXD; -use crate::exh::EXH; -use crate::exl::EXL; -use crate::patch::{PatchError, ZiPatch}; -use crate::repository::{Category, Repository, string_to_category}; -use crate::sqpack::{IndexEntry, SqPackData, SqPackIndex}; - -/// Framework for operating on game data. -pub struct GameData { - /// The game directory to operate on. - pub game_directory: String, - - /// Repositories in the game directory. - pub repositories: Vec, - - index_files: HashMap, -} - -fn is_valid(path: &str) -> bool { - let d = PathBuf::from(path); - - if fs::metadata(d.as_path()).is_err() { - return false; - } - - true -} +use super::Resource; /// Possible actions to repair game files #[derive(Debug)] @@ -52,20 +33,19 @@ pub enum RepairError<'a> { FailedRepair(&'a Repository), } -impl GameData { - /// Read game data from an existing game installation. - /// - /// This will return a GameData even if the game directory is technically - /// invalid, but it won't have any repositories. - /// - /// # Example - /// - /// ``` - /// # use physis::common::Platform; - /// use physis::gamedata::GameData; - /// GameData::from_existing(Platform::Win32, "$FFXIV/game"); - /// ``` - pub fn from_existing(platform: Platform, directory: &str) -> GameData { +/// Used to read files from the retail game, in their SqPack-compressed format. +pub struct SqPackResource { + /// The game directory to operate on. + pub game_directory: String, + + /// Repositories in the game directory. + pub repositories: Vec, + + index_files: HashMap, +} + +impl SqPackResource { + pub fn from_existing(platform: Platform, directory: &str) -> Self { match is_valid(directory) { true => { let mut data = Self { @@ -138,43 +118,6 @@ impl GameData { SqPackData::from_existing(dat_path.to_str()?) } - /// Checks if a file located at `path` exists. - /// - /// # Example - /// - /// ``` - /// # use physis::common::Platform; - /// use physis::gamedata::GameData; - /// # let mut game = GameData::from_existing(Platform::Win32, "SquareEnix/Final Fantasy XIV - A Realm Reborn/game"); - /// if game.exists("exd/cid.exl") { - /// println!("Cid really does exist!"); - /// } else { - /// println!("Oh noes!"); - /// } - /// ``` - pub fn exists(&mut self, path: &str) -> bool { - let Some(_) = self.get_index_filenames(path) else { - return false; - }; - - self.find_entry(path).is_some() - } - - /// Extracts the file located at `path`. This is returned as an in-memory buffer, and will usually - /// have to be further parsed. - /// - /// # Example - /// - /// ```should_panic - /// # use physis::gamedata::GameData; - /// # use std::io::Write; - /// use physis::common::Platform; - /// # let mut game = GameData::from_existing(Platform::Win32, "SquareEnix/Final Fantasy XIV - A Realm Reborn/game"); - /// let data = game.extract("exd/root.exl").unwrap(); - /// - /// let mut file = std::fs::File::create("root.exl").unwrap(); - /// file.write(data.as_slice()).unwrap(); - /// ``` pub fn extract(&mut self, path: &str) -> Option { let slice = self.find_entry(path); match slice { @@ -423,19 +366,51 @@ impl GameData { } } +impl Resource for SqPackResource { + fn read(&mut self, path: &str) -> Option { + let slice = self.find_entry(path); + match slice { + Some((entry, chunk)) => { + let mut dat_file = self.get_dat_file(path, chunk, entry.data_file_id.into())?; + + dat_file.read_from_offset(entry.offset) + } + None => None, + } + } + + fn exists(&mut self, path: &str) -> bool { + let Some(_) = self.get_index_filenames(path) else { + return false; + }; + + self.find_entry(path).is_some() + } +} + +fn is_valid(path: &str) -> bool { + let d = PathBuf::from(path); + + if fs::metadata(d.as_path()).is_err() { + return false; + } + + true +} + #[cfg(test)] mod tests { use crate::repository::Category::*; use super::*; - fn common_setup_data() -> GameData { + fn common_setup_data() -> SqPackResource { let mut d = PathBuf::from(env!("CARGO_MANIFEST_DIR")); d.push("resources/tests"); d.push("valid_sqpack"); d.push("game"); - GameData::from_existing(Platform::Win32, d.to_str().unwrap()) + SqPackResource::from_existing(Platform::Win32, d.to_str().unwrap()) } #[test] diff --git a/src/resource/unpacked.rs b/src/resource/unpacked.rs new file mode 100644 index 0000000..fc0a366 --- /dev/null +++ b/src/resource/unpacked.rs @@ -0,0 +1,62 @@ +use std::path::PathBuf; + +use crate::ByteBuffer; + +use super::Resource; + +/// Used to read unpacked files from a directory. +pub struct UnpackedResource { + base_directory: String, +} + +impl UnpackedResource { + pub fn from_existing(base_directory: &str) -> Self { + Self { + base_directory: base_directory.to_string(), + } + } +} + +impl Resource for UnpackedResource { + fn read(&mut self, path: &str) -> Option { + let mut new_path = PathBuf::from(&self.base_directory); + new_path.push(path); + + std::fs::read(new_path).ok() + } + + fn exists(&mut self, path: &str) -> bool { + let mut new_path = PathBuf::from(&self.base_directory); + new_path.push(path); + + std::fs::exists(new_path).unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn common_setup_data() -> UnpackedResource { + let mut d = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + d.push("resources/tests"); + + UnpackedResource::from_existing(d.to_str().unwrap()) + } + + #[test] + fn read_files() { + let mut data = common_setup_data(); + + assert!(data.read("empty_planlive.lgb").is_some()); + assert!(data.read("non_existent.lgb").is_none()); + } + + #[test] + fn exist_files() { + let mut data = common_setup_data(); + + assert_eq!(data.exists("empty_planlive.lgb"), true); + assert_eq!(data.exists("non_existent.lgb"), false); + } +}