redstrate/physis
1
Fork 0
mirror of https://github.com/redstrate/Physis.git synced 2025-04-19 17:36:50 +00:00
Code Activity

Shrink the dependency and feature complexity, auto-cleanup and more

Browse source 
We had a few odd dependencies that caused nothing but pain in dependent projects
like libphysis. One of these was libunshield (a C library) that our game_install
feature used, but to be honest this was the wrong library to put this code. It
was really only ever used by Astra, and should live there instead - there's no
reason to have it shared between applications (and it's small enough to be
copied if *you* need it.) Also that also killed the system-deps dependency which
had a significant impact on our build time.

Another dependency was replaced: libz-sys. This is replaced by the pure Rust
libz-rs (through libz-rs-sys) which should simplify deploying physis without
having to worry about manually linking libz or other nonsense. Some leftover
copied code from flate2 can also be removed.

I also removed the visual_data feature as Astra ended up using it anyway, and
the distinction doesn't make much sense now. It was previously to gate some
dependencies needed for visual data extraction, but the bitflags and half crates
are small. I can look into splitting the crate up into more features if needed
later.

A dependency that was erroneously included in the refactoring was quote, which
has been removed. Also ran cargo fmt, clippy too.
This commit is contained in:
Joshua Goins 2025-03-11 16:25:59 -04:00
parent 5378d72ccf
commit d58a216462
22 changed files with 57 additions and 195 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
Cargo.toml
View file

@ -23,12 +23,6 @@ name = "retail_benchmark"
harness = false
required-features = ["retail_game_testing"]
[build-dependencies]
system-deps = "7"
[package.metadata.system-deps]
libunshield = { version = "1.4", feature = "game_install" }
[dev-dependencies]
hmac-sha512 = "1"
@ -39,19 +33,11 @@ brunch = { version = "0.5.3", default-features = false }
crc = "3"
[features]
default = ["visual_data"]
# enables game installation support using unshield (only supported on Linux and macOS)
game_install = []
# enables support for extracting visual data, such as models, textures, materials, etc.
# this enables a bunch of dependencies!
# tip: can be safely turned off for launchers and other tools that simply need to extract the bare minimum of data
visual_data = ["dep:half", "dep:bitflags"]
default = []
# testing only features
retail_game_testing = []
patch_testing = ["game_install"]
patch_testing = []
[dependencies]
# amazing binary parsing/writing library
@ -60,14 +46,11 @@ binrw = { version = "0.14", features = ["std"], default-features = false }
tracing = { version = "0.1", features = ["std"], default-features = false }
# used for zlib compression in sqpack files
libz-sys = { version = "1.1" }
libz-rs-sys = { version = "0.4", features = ["std", "rust-allocator"], default-features = false }
# needed for half-float support which FFXIV uses in its model data
half = { version = "2", optional = true }
half = { version = "2", features = ["std"], default-features = false }
# needed for c-style bitflags used in some formats (such as tex files)
# cannot upgrade to 2.0.0, breaking changes that aren't recoverable: https://github.com/bitflags/bitflags/issues/314
bitflags = { version = "1.3", optional = true }
# used in bcn decoding
quote = "1.0"
bitflags = { version = "1.3", default-features = false }

3
README.md
View file

@ -45,7 +45,6 @@ Physis also supports doing some other useful things other than reading and writi
* Extract game data from SqPack files, and list file hashes from index/index2.
* Apply game patches. Indexed ZiPatch is not yet supported, though.
* Blockfish ciphers for encrypting and decrypting [SqexArg](https://docs.xiv.zone/concept/sqexarg/).
* Extract retail installer contents, useful on Linux for avoiding having to run the InstallShield installer.
* Construct paths to equipment, items, faces, and other useful models.
* Extract strings from executables.
@ -82,8 +81,6 @@ C/C++ projects (or anything that can interface with C libraries) can use [libphy
Physis only has a few dependencies, and very little if nothing is turned on by default. You need to set up [Rust](https://www.rust-lang.org/learn/get-started) and then run `cargo build`.
If you want to build the `game_install` feature, you also need to install [unshield](https://github.com/twogood/unshield).
## Development
If you're interested to read how these formats work in more detail, see [xiv.dev](https://xiv.dev/) and

8
build.rs
View file

@ -1,8 +0,0 @@
// SPDX-FileCopyrightText: 2023 Joshua Goins <josh@redstrate.com>
// SPDX-License-Identifier: GPL-3.0-or-later
fn main() {
// Windows doesn't ship pkgconfig files, typically. At least in our build system.
#[cfg(all(not(target_os = "windows"), not(target_family = "wasm")))]
system_deps::Config::new().probe().unwrap();
}

2
examples/extractor.rs
View file

@ -39,7 +39,7 @@ fn main() {
};
/// Since GameData::extract returns a byte buffer, it's trivial to write that to a file on disk.
if !file.write_all(&game_file).is_ok() {
if file.write_all(&game_file).is_err() {
println!("Failed to write to file {}.", destination_path);
return;
};

2
src/bcn/bc3.rs
View file

@ -28,7 +28,7 @@ pub fn decode_bc3_alpha(data: &[u8], outbuf: &mut [u32], channel: usize) {
let channel_shift = channel * 8;
let channel_mask = 0xFFFFFFFF ^ (0xFF << channel_shift);
outbuf.iter_mut().for_each(|p| {
*p = (*p & channel_mask) | (a[d & 7] as u32) << channel_shift;
*p = (*p & channel_mask) | ((a[d & 7] as u32) << channel_shift);
d >>= 3;
});
}

6
src/bcn/color.rs
View file

@ -46,9 +46,9 @@ pub const fn color(r: u8, g: u8, b: u8, a: u8) -> u32 {
#[inline]
pub const fn rgb565_le(d: u16) -> (u8, u8, u8) {
(
(d >> 8 & 0xf8) as u8 | (d >> 13) as u8,
(d >> 3 & 0xfc) as u8 | (d >> 9 & 3) as u8,
(d << 3) as u8 | (d >> 2 & 7) as u8,
((d >> 8) & 0xf8) as u8 | (d >> 13) as u8,
((d >> 3) & 0xfc) as u8 | ((d >> 9) & 3) as u8,
(d << 3) as u8 | ((d >> 2) & 7) as u8,
)
}

10
src/bcn/macros.rs
View file

@ -5,13 +5,19 @@
macro_rules! block_decoder {
($name: ident, $block_width: expr, $block_height: expr, $raw_block_size: expr, $block_decode_func: expr) => {
//#[doc = "Decodes a " $name " encoded texture into an image"]
pub fn $name(data: &[u8], width: usize, height: usize, image: &mut [u32]) -> Result<(), &'static str> {
pub fn $name(
data: &[u8],
width: usize,
height: usize,
image: &mut [u32],
) -> Result<(), &'static str> {
const BLOCK_WIDTH: usize = $block_width;
const BLOCK_HEIGHT: usize = $block_height;
const BLOCK_SIZE: usize = BLOCK_WIDTH * BLOCK_HEIGHT;
let num_blocks_x: usize = (width + BLOCK_WIDTH - 1) / BLOCK_WIDTH;
let num_blocks_y: usize = (height + BLOCK_WIDTH - 1) / BLOCK_HEIGHT;
let mut buffer: [u32; BLOCK_SIZE] = [crate::bcn::color::color(0,0,0,255); BLOCK_SIZE];
let mut buffer: [u32; BLOCK_SIZE] =
[crate::bcn::color::color(0, 0, 0, 255); BLOCK_SIZE];
if data.len() < num_blocks_x * num_blocks_y * $raw_block_size {
return Err("Not enough data to decode image!");

8
src/chardat.rs
View file

@ -312,7 +312,7 @@ mod tests {
assert_eq!(chardat.customize.subrace, Subrace::Midlander);
assert_eq!(chardat.customize.face, 5);
assert_eq!(chardat.customize.hair, 1);
assert_eq!(chardat.customize.enable_highlights, false);
assert!(!chardat.customize.enable_highlights);
assert_eq!(chardat.customize.skin_tone, 2);
assert_eq!(chardat.customize.right_eye_color, 37);
assert_eq!(chardat.customize.hair_tone, 53);
@ -347,7 +347,7 @@ mod tests {
assert_eq!(chardat.customize.subrace, Subrace::Xaela);
assert_eq!(chardat.customize.face, 3);
assert_eq!(chardat.customize.hair, 5);
assert_eq!(chardat.customize.enable_highlights, false);
assert!(!chardat.customize.enable_highlights);
assert_eq!(chardat.customize.skin_tone, 160);
assert_eq!(chardat.customize.right_eye_color, 91);
assert_eq!(chardat.customize.hair_tone, 159);
@ -382,7 +382,7 @@ mod tests {
assert_eq!(chardat.customize.subrace, Subrace::Plainsfolk);
assert_eq!(chardat.customize.face, 1);
assert_eq!(chardat.customize.hair, 8);
assert_eq!(chardat.customize.enable_highlights, false);
assert!(!chardat.customize.enable_highlights);
assert_eq!(chardat.customize.skin_tone, 25);
assert_eq!(chardat.customize.right_eye_color, 11);
assert_eq!(chardat.customize.hair_tone, 45);
@ -417,7 +417,7 @@ mod tests {
assert_eq!(chardat.customize.subrace, Subrace::Rava);
assert_eq!(chardat.customize.face, 1);
assert_eq!(chardat.customize.hair, 8);
assert_eq!(chardat.customize.enable_highlights, false);
assert!(!chardat.customize.enable_highlights);
assert_eq!(chardat.customize.skin_tone, 12);
assert_eq!(chardat.customize.right_eye_color, 43);
assert_eq!(chardat.customize.hair_tone, 53);

68
src/compression.rs
View file

@ -3,71 +3,9 @@
use std::ptr::null_mut;
use libz_sys::*;
// This module's functions are licensed under MIT from https://github.com/rust-lang/flate2-rs
mod flate2_zallocation {
use std::alloc::{self, Layout};
use std::ffi::c_void;
use std::ptr::null_mut;
const ALIGN: usize = std::mem::align_of::<usize>();
fn align_up(size: usize, align: usize) -> usize {
(size + align - 1) & !(align - 1)
}
pub extern "C" fn zalloc(_ptr: *mut c_void, items: u32, item_size: u32) -> *mut c_void {
// We need to multiply `items` and `item_size` to get the actual desired
// allocation size. Since `zfree` doesn't receive a size argument we
// also need to allocate space for a `usize` as a header so we can store
// how large the allocation is to deallocate later.
let size = match items
.checked_mul(item_size)
.and_then(|i| usize::try_from(i).ok())
.map(|size| align_up(size, ALIGN))
.and_then(|i| i.checked_add(std::mem::size_of::<usize>()))
{
Some(i) => i,
None => return null_mut(),
};
// Make sure the `size` isn't too big to fail `Layout`'s restrictions
let layout = match Layout::from_size_align(size, ALIGN) {
Ok(layout) => layout,
Err(_) => return null_mut(),
};
unsafe {
// Allocate the data, and if successful store the size we allocated
// at the beginning and then return an offset pointer.
let ptr = alloc::alloc(layout) as *mut usize;
if ptr.is_null() {
return ptr as *mut c_void;
}
*ptr = size;
ptr.add(1) as *mut c_void
}
}
pub extern "C" fn zfree(_ptr: *mut c_void, address: *mut c_void) {
unsafe {
// Move our address being freed back one pointer, read the size we
// stored in `zalloc`, and then free it using the standard Rust
// allocator.
let ptr = (address as *mut usize).offset(-1);
let size = *ptr;
let layout = Layout::from_size_align_unchecked(size, ALIGN);
alloc::dealloc(ptr as *mut u8, layout)
}
}
}
use libz_rs_sys::*;
pub fn no_header_decompress(in_data: &mut [u8], out_data: &mut [u8]) -> bool {
// thanks for not somehow detecting this?
#[allow(unused_imports)]
use crate::compression::flate2_zallocation::{zalloc, zfree};
unsafe {
let mut strm = z_stream {
next_in: null_mut(),
@ -78,8 +16,8 @@ pub fn no_header_decompress(in_data: &mut [u8], out_data: &mut [u8]) -> bool {
total_out: 0,
msg: null_mut(),
state: null_mut(),
zalloc,
zfree,
zalloc: None, // the default alloc is fine
zfree: None, // the default free is fine
opaque: null_mut(),
data_type: 0,
adler: 0,

6
src/crc.rs
View file

@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: 2023 Joshua Goins <josh@redstrate.com>
// SPDX-License-Identifier: GPL-3.0-or-later
use libz_sys::z_off_t;
use libz_rs_sys::z_off_t;
use std::ops::{Add, AddAssign, BitXor, BitXorAssign};
/// CRC used for filepath hashes in index file
@ -45,11 +45,11 @@ impl Jamcrc {
}
fn crc32(crc: u32, s: &[u8]) -> u32 {
unsafe { libz_sys::crc32(crc.into(), s.as_ptr(), s.len() as u32) as u32 }
unsafe { libz_rs_sys::crc32(crc.into(), s.as_ptr(), s.len() as u32) as u32 }
}
fn crc32_combine(crc1: u32, crc2: u32, len2: usize) -> u32 {
unsafe { libz_sys::crc32_combine(crc1.into(), crc2.into(), len2 as z_off_t) as u32 }
libz_rs_sys::crc32_combine(crc1.into(), crc2.into(), len2 as z_off_t) as u32
}
/// CRC used for shader keys

14
src/dat.rs
View file

@ -10,7 +10,6 @@ use binrw::BinWrite;
use binrw::{BinReaderExt, binrw};
use crate::common_file_operations::read_bool_from;
#[cfg(feature = "visual_data")]
use crate::model::ModelFileHeader;
use crate::sqpack::read_data_block;
@ -214,17 +213,7 @@ impl DatFile {
match file_info.file_type {
FileType::Empty => None,
FileType::Standard => self.read_standard_file(offset, &file_info),
FileType::Model => {
#[cfg(feature = "visual_data")]
{
self.read_model_file(offset, &file_info)
}
#[cfg(not(feature = "visual_data"))]
{
panic!("Tried to extract a model without the visual_data feature enabled!")
}
}
FileType::Model => self.read_model_file(offset, &file_info),
FileType::Texture => self.read_texture_file(offset, &file_info),
}
}
@ -257,7 +246,6 @@ impl DatFile {
}
/// Reads a model file block.
#[cfg(feature = "visual_data")]
fn read_model_file(&mut self, offset: u64, file_info: &FileInfo) -> Option<ByteBuffer> {
let model_file_info = file_info.model_info.as_ref()?;

4
src/dic.rs
View file

@ -154,11 +154,11 @@ impl Dictionary {
return 0;
}
return if let Some(new_val) = lookup_table.get(&(higher as u16)) {
if let Some(new_val) = lookup_table.get(&(higher as u16)) {
(((*new_val as u32) << 8) + lower) as i32
} else {
0
};
}
}
fn dump_dict_node(&self, vec: &mut Vec<String>, entry_id: i32, prev: String) {

8
src/index.rs
View file

@ -192,7 +192,7 @@ impl IndexFile {
pub fn calculate_hash(&self, path: &str) -> Hash {
let lowercase = path.to_lowercase();
return match &self.index_header.index_type {
match &self.index_header.index_type {
IndexType::Index1 => {
if let Some(pos) = lowercase.rfind('/') {
let (directory, filename) = lowercase.split_at(pos);
@ -210,7 +210,7 @@ impl IndexFile {
}
}
IndexType::Index2 => Hash::FullPath(CRC.checksum(lowercase.as_bytes())),
};
}
}
pub fn exists(&self, path: &str) -> bool {
@ -223,7 +223,7 @@ impl IndexFile {
if let Some(entry) = self.entries.iter().find(|s| s.hash == hash) {
let full_hash = match hash {
Hash::SplitPath { name, path } => (path as u64) << 32 | (name as u64),
Hash::SplitPath { name, path } => ((path as u64) << 32) | (name as u64),
Hash::FullPath(hash) => hash as u64,
};
return Some(IndexEntry {
@ -272,7 +272,7 @@ mod tests {
path: 5164904,
};
assert_eq!(file_entry.hash, expected_hash);
assert_eq!(file_entry.data.is_synonym, false);
assert!(!file_entry.data.is_synonym);
assert_eq!(file_entry.data.data_file_id, 0);
assert_eq!(file_entry.data.offset, 57674496);

16
src/installer.rs
View file

@ -32,22 +32,6 @@ const BOOT_COMPONENT_FILES: [&str; 18] = [
const GAME_COMPONENT_FILES: [&str; 1] = ["ffxivgame.ver"];
#[repr(C)]
struct Unshield {
_private: [u8; 0],
}
unsafe extern "C" {
fn unshield_open(filename: *const c_char) -> *mut Unshield;
fn unshield_close(unshield: *mut Unshield);
fn unshield_set_log_level(level: i32);
fn unshield_file_count(unshield: *mut Unshield) -> i32;
fn unshield_file_name(unshield: *mut Unshield, index: i32) -> *const c_char;
fn unshield_file_save(unshield: *mut Unshield, index: i32, filename: *const c_char) -> bool;
}
pub enum InstallError {
IOFailure,
FFIFailure,

15
src/lib.rs
View file

@ -30,7 +30,6 @@ mod compression;
mod dat;
/// Reading model (MDL) files.
#[cfg(feature = "visual_data")]
pub mod model;
/// All of the races in Eorzea in a nice enum package.
@ -53,10 +52,6 @@ pub mod blowfish;
mod blowfish_constants;
/// Initializing a new retail game install from the official retail installer. No execution required!
#[cfg(feature = "game_install")]
pub mod installer;
/// Reading Excel header files (EXH).
pub mod exh;
@ -64,7 +59,6 @@ pub mod exh;
pub mod exd;
/// Reading Havok XML sidecar files.
#[cfg(feature = "visual_data")]
pub mod skeleton;
/// Reading file info files (FIIN).
@ -74,15 +68,12 @@ pub mod fiin;
pub mod log;
/// Reading textures (TEX).
#[cfg(feature = "visual_data")]
pub mod tex;
/// Reading material files (MTRL)
#[cfg(feature = "visual_data")]
pub mod mtrl;
/// Reading shader packages (SHPK)
#[cfg(feature = "visual_data")]
pub mod shpk;
/// Reading character parameter files (CMP)
@ -97,26 +88,20 @@ pub mod gearsets;
/// Reading and writing the plaintext config files (CFG) used by the game to store most of it's configuration.
pub mod cfg;
#[cfg(feature = "visual_data")]
mod havok;
/// Reading bone deform matrices.
#[cfg(feature = "visual_data")]
pub mod pbd;
mod crc;
mod sha1;
#[cfg(feature = "visual_data")]
mod model_file_operations;
#[cfg(feature = "visual_data")]
pub mod model_vertex_declarations;
#[cfg(feature = "visual_data")]
pub mod lgb;
#[cfg(feature = "visual_data")]
pub mod tera;
/// Reading data from executables

4
src/model.rs
View file

@ -1352,8 +1352,8 @@ mod tests {
assert_eq!(mdl.file_header.vertex_declaration_count, 6);
assert_eq!(mdl.file_header.material_count, 2);
assert_eq!(mdl.file_header.lod_count, 3);
assert_eq!(mdl.file_header.index_buffer_streaming_enabled, false);
assert_eq!(mdl.file_header.has_edge_geometry, false);
assert!(!mdl.file_header.index_buffer_streaming_enabled);
assert!(!mdl.file_header.has_edge_geometry);
// model header
assert_eq!(mdl.model_data.header.radius, 1.5340779);

8
src/patch.rs
View file

@ -335,7 +335,7 @@ struct SqpkChunk {
operation: SqpkOperation,
}
const WIPE_BUFFER: [u8; 1 << 16] = [0; 1 << 16];
static WIPE_BUFFER: [u8; 1 << 16] = [0; 1 << 16];
fn wipe(mut file: &File, length: usize) -> Result<(), PatchError> {
let mut length: usize = length;
@ -804,8 +804,6 @@ impl ZiPatch {
mod tests {
use std::fs::{read, write};
use std::path::PathBuf;
use std::thread::sleep;
use std::time::Duration;
use super::*;
@ -830,7 +828,7 @@ mod tests {
let data_dir = prepare_data_dir();
write(data_dir.clone() + "/test.patch", &read(d).unwrap()).unwrap();
write(data_dir.clone() + "/test.patch", read(d).unwrap()).unwrap();
// Feeding it invalid data should not panic
let Err(PatchError::ParseError) =
@ -852,7 +850,7 @@ mod tests {
resources_dir.push("resources/tests");
// Let's create a patch that re-creates the resources dir into our data directory
let patch = ZiPatch::create(&*data_dir, resources_dir.to_str().unwrap()).unwrap();
let patch = ZiPatch::create(&data_dir, resources_dir.to_str().unwrap()).unwrap();
write(data_dir.clone() + "/test.patch", &patch).unwrap();

6
src/sqdb.rs
View file

@ -3,13 +3,13 @@
use std::io::Cursor;
use crate::ByteSpan;
use crate::common_file_operations::read_string;
use crate::common_file_operations::write_string;
use crate::sqpack::SqPackHeader;
use crate::ByteSpan;
use binrw::helpers::until_eof;
use binrw::BinRead;
use binrw::binrw;
use binrw::helpers::until_eof;
#[binrw]
#[derive(Debug)]
@ -48,7 +48,7 @@ pub struct SQDB {
header: SQDBHeader,
#[br(parse_with = until_eof)]
entries: Vec<SQDBEntry>
entries: Vec<SQDBEntry>,
}
impl SQDB {

2
src/sqpack.rs
View file

@ -3,7 +3,7 @@
use std::io::{Read, Seek, SeekFrom, Write};
use binrw::{binrw, BinRead, BinWrite};
use binrw::{BinRead, BinWrite, binrw};
use crate::common::{Platform, Region};
use crate::compression::no_header_decompress;

2
src/tex.rs
View file

@ -5,10 +5,10 @@
use std::io::{Cursor, Read, Seek, SeekFrom};
use crate::ByteSpan;
use crate::bcn::decode_bc1;
use crate::bcn::decode_bc3;
use crate::bcn::decode_bc5;
use crate::ByteSpan;
use binrw::BinRead;
use binrw::binrw;
use bitflags::bitflags;

11
tests/patching_test.rs
View file

@ -1,17 +1,8 @@
// SPDX-FileCopyrightText: 2023 Joshua Goins <josh@redstrate.com>
// SPDX-License-Identifier: GPL-3.0-or-later
use hmac_sha512::Hash;
use std::env;
use std::fs::{read, read_dir};
use std::io::Write;
use std::process::Command;
use std::fs::read_dir;
use physis::common::Platform;
use physis::fiin::FileInfo;
use physis::index;
use physis::patch::ZiPatch;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
#[cfg(feature = "patch_testing")]