From 2e0403beb1df43774026381dbe80e14a22648a15 Mon Sep 17 00:00:00 2001 From: Joshua Goins Date: Tue, 25 Feb 2025 19:30:16 -0500 Subject: [PATCH] Add a much better, faster and more ergonomic way to parse structs This adds a macro that helps deserialize structs when their properties & types are known ahead of time. It's sole job is gluing different binrw macros together so I don't have to. The nice thing about this is that you have more direct access to the underlying data. --- Cargo.lock | 30 +++- Cargo.toml | 1 + paramacro/Cargo.toml | 11 ++ paramacro/src/lib.rs | 79 ++++++++++ src/build_data.rs | 32 ++-- src/lib.rs | 3 +- src/name_property.rs | 2 +- src/primary_asset_type.rs | 1 + src/structs.rs | 324 ++++++++++++++++++++++++++------------ 9 files changed, 361 insertions(+), 122 deletions(-) create mode 100644 paramacro/Cargo.toml create mode 100644 paramacro/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 5aecaee..04a5da3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,7 +34,7 @@ dependencies = [ "either", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -78,15 +78,15 @@ dependencies = [ [[package]] name = "either" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" [[package]] name = "flate2" -version = "1.0.35" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", "libz-ng-sys", @@ -99,6 +99,7 @@ version = "0.1.0" dependencies = [ "binrw", "flate2", + "paramacro", ] [[package]] @@ -126,6 +127,14 @@ dependencies = [ "adler2", ] +[[package]] +name = "paramacro" +version = "0.1.0" +dependencies = [ + "quote", + "syn 2.0.98", +] + [[package]] name = "proc-macro2" version = "1.0.93" @@ -161,6 +170,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "unicode-ident" version = "1.0.17" diff --git a/Cargo.toml b/Cargo.toml index 2a01ff3..cbe337a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,3 +6,4 @@ edition = "2024" [dependencies] flate2 = { version = "1.0", features = ["zlib-ng"], default-features = false } binrw = { version = "0.14", features = ["std"], default-features = false } +paramacro = { path = "paramacro" } diff --git a/paramacro/Cargo.toml b/paramacro/Cargo.toml new file mode 100644 index 0000000..3a9e9f3 --- /dev/null +++ b/paramacro/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "paramacro" +version = "0.1.0" +edition = "2024" + +[dependencies] +syn = { version = "2.0", features = ["full"] } +quote = "1.0" + +[lib] +proc-macro = true diff --git a/paramacro/src/lib.rs b/paramacro/src/lib.rs new file mode 100644 index 0000000..6e93a50 --- /dev/null +++ b/paramacro/src/lib.rs @@ -0,0 +1,79 @@ +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::{quote, ToTokens}; +use syn::{parse_macro_input, Expr, Fields, Item, Lit}; + +// TODO: clean up this mess. + +/// Helps read and write serialized structs. These are structs that have named fields with known types. +/// Note: This adds a hidden "None" at the end of the struct as well. +#[proc_macro_attribute] +pub fn serialized_struct(_metadata: TokenStream, input: TokenStream) + -> TokenStream { + let mut input = syn::parse_macro_input!(input as syn::ItemStruct); + + for mut field in &mut input.fields { + let our_custom = &field.attrs[0]; + let our_custom_name = our_custom.meta.require_name_value().unwrap(); + let our_custom_name = match &our_custom_name.value { + Expr::Lit(_0) => { + match &_0.lit { + Lit::Str(_0) => { + Some(_0.value()) + } + _ => None + } + } + _ => None + }.unwrap(); + field.attrs.clear(); + let field_tokens = field.to_token_stream(); + let new_field_token_stream = quote! { + #[br(parse_with = crate::structs::read_struct_field, args(#our_custom_name))] + #field_tokens + }; + let buffer = ::syn::parse::Parser::parse2( + syn::Field::parse_named, + new_field_token_stream, + ).unwrap(); + *field = buffer; + } + + // Add "None" field + let none_field_stream = quote! { + #[br(temp)] + #[bw(ignore)] + none_field: crate::structs::PrimaryAssetNameProperty + }; + let buffer = ::syn::parse::Parser::parse2( + syn::Field::parse_named, + none_field_stream, + ).unwrap(); + match &mut input.fields { + Fields::Named(_0) => { + _0.named.push(buffer) + } + _ => {} + } + + let output = quote! { + #[binrw] + #input + }; + + output.into_token_stream().into() +} + +/// Denotes a field with a known name. +#[proc_macro_attribute] +pub fn serialized_field(attr: TokenStream, input: TokenStream) -> TokenStream { + let input2 = parse_macro_input!(input as Item); + + let output = quote! { + #[binrw] + #input2 + }; + + output.into_token_stream().into() +} diff --git a/src/build_data.rs b/src/build_data.rs index e860695..6a57fe9 100644 --- a/src/build_data.rs +++ b/src/build_data.rs @@ -1,24 +1,28 @@ -use crate::structs::StructField; -use binrw::binrw; +use crate::common::{read_string_with_length, write_string_with_length}; +use crate::str_property::StrProperty; +use crate::structs::{ + DAAssembleIdDataStruct, DACustomizeAssetIdDataStruct, DATriggerDataStruct, DATuningDataStruct, +}; +use binrw::{binrw}; +use std::fmt::Debug; -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct DABuildDataStruct { - #[br(args { name: "Name", r#type: "StrProperty" })] - pub name: StructField, + #[paramacro::serialized_field = "Name"] + pub name: StrProperty, - #[br(args { name: "Assemble", r#type: "StructProperty" })] - pub assemble: StructField, + #[paramacro::serialized_field = "Assemble"] + pub assemble: DAAssembleIdDataStruct, - #[br(args { name: "Trigger", r#type: "StructProperty" })] - pub trigger: StructField, + #[paramacro::serialized_field = "Trigger"] + pub trigger: DATriggerDataStruct, - #[br(args { name: "Customize", r#type: "StructProperty" })] - pub customize: StructField, + #[paramacro::serialized_field = "Customize"] + pub customize: DACustomizeAssetIdDataStruct, - #[br(args { name: "Tuning", r#type: "StructProperty" })] - #[brw(pad_after = 9)] - pub tuning: StructField, + #[paramacro::serialized_field = "Tuning"] + pub tuning: DATuningDataStruct, } #[cfg(test)] diff --git a/src/lib.rs b/src/lib.rs index 5cd7f2f..f4d705a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ use crate::common::{read_string_with_length, write_string_with_length}; use crate::float_property::FloatProperty; use crate::int_property::IntProperty; use crate::map_property::MapProperty; +use crate::name_property::NameProperty; use crate::set_property::SetProperty; use crate::str_property::StrProperty; use crate::struct_property::StructProperty; @@ -37,7 +38,7 @@ use flate2::bufread::ZlibDecoder; #[br(import { magic: &str, name: &str })] pub enum Property { #[br(pre_assert("NameProperty" == magic))] - Name(StrProperty), + Name(NameProperty), #[br(pre_assert("StructProperty" == magic))] Struct(StructProperty), #[br(pre_assert("FloatProperty" == magic))] diff --git a/src/name_property.rs b/src/name_property.rs index a2a43cc..9a21a31 100644 --- a/src/name_property.rs +++ b/src/name_property.rs @@ -4,7 +4,7 @@ use binrw::binrw; #[binrw] #[derive(Debug)] pub struct NameProperty { - #[brw(pad_after = 6)] + #[brw(pad_after = 5)] // Only add + 1 for the null terminator if the string *isn't* empty. #[bw(calc = value.len() as u32 + 4 + if value.is_empty() { 0 } else { 1})] pub size_in_bytes: u32, diff --git a/src/primary_asset_type.rs b/src/primary_asset_type.rs index 8bf924b..2d0e595 100644 --- a/src/primary_asset_type.rs +++ b/src/primary_asset_type.rs @@ -1,6 +1,7 @@ use crate::structs::StructField; use binrw::binrw; +// TODO: im pretty sure everything about this + PrimaryAssetId is wrong. i think primary_asset_name needs to belong in PrimaryAssetId #[binrw] #[derive(Debug)] pub struct PrimaryAssetTypeStruct { diff --git a/src/structs.rs b/src/structs.rs index d0c51f7..d2007c6 100644 --- a/src/structs.rs +++ b/src/structs.rs @@ -1,7 +1,19 @@ use crate::Property; +use crate::array_property::ArrayProperty; +use crate::bool_property::BoolProperty; use crate::build_data::DABuildDataStruct; use crate::common::{read_string_with_length, write_string_with_length}; -use binrw::binrw; +use crate::float_property::FloatProperty; +use crate::guid::Guid; +use crate::int_property::IntProperty; +use crate::linear_color::LinearColorStruct; +use crate::map_property::MapProperty; +use crate::name_property::NameProperty; +use crate::primary_asset_id::PrimaryAssetIdStruct; +use crate::primary_asset_type::PrimaryAssetTypeStruct; +use crate::str_property::StrProperty; +use binrw::{BinRead, BinResult, binrw}; +use std::fmt::Debug; #[binrw] #[derive(Debug)] @@ -9,26 +21,27 @@ pub struct DateTimeStruct { pub unk: [u8; 8], } -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct DALoadOptionStruct { - #[br(args { name: "LoadTypes", r#type: "IntProperty" })] - #[br(pad_after = 9)] // "none" - pub load_types: StructField, + #[paramacro::serialized_field = "LoadTypes"] + pub load_types: IntProperty, } -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct SaveSlotInfoStruct { - #[br(args { name: "Name", r#type: "StrProperty" })] - pub name: StructField, - #[br(args { name: "Timestamp", r#type: "StructProperty" })] - pub timestamp: StructField, - #[br(args { name: "Level", r#type: "NameProperty" })] - pub level: StructField, - #[br(args { name: "Players", r#type: "ArrayProperty" })] - #[br(pad_after = 9)] // "none" - pub players: StructField, + #[paramacro::serialized_field = "Name"] + pub name: StrProperty, + + #[paramacro::serialized_field = "Timestamp"] + pub timestamp: DateTimeStruct, + + #[paramacro::serialized_field = "Level"] + pub level: NameProperty, + + #[paramacro::serialized_field = "Players"] + pub players: ArrayProperty, } #[binrw] @@ -75,74 +88,129 @@ pub struct StructField { pub key: Option>, } -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct DAModuleItemDataStruct { - #[br(pad_after = 9)] // none - pub module_level: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "ModuleLevel"] + pub module_level: IntProperty, } -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct DAAssembleIdDataStruct { - pub hanger: PrimaryAssetNameProperty, - pub headset: PrimaryAssetNameProperty, - pub mobility: PrimaryAssetNameProperty, - pub thruster: PrimaryAssetNameProperty, - pub utility: PrimaryAssetNameProperty, - pub primary_front_weapon: PrimaryAssetNameProperty, - pub secondary_front_weapon: PrimaryAssetNameProperty, - pub left_rear_weapon: PrimaryAssetNameProperty, - pub right_rear_weapon: PrimaryAssetNameProperty, - // has none at the end - #[br(pad_after = 9)] - pub coloring_data: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "Hanger"] + pub hanger: Guid, + + #[paramacro::serialized_field = "Headset"] + pub headset: Guid, + + #[paramacro::serialized_field = "Mobility"] + pub mobility: Guid, + + #[paramacro::serialized_field = "Thruster"] + pub thruster: Guid, + + #[paramacro::serialized_field = "Utility"] + pub utility: Guid, + + #[paramacro::serialized_field = "PrimaryFrontWeapon"] + pub primary_front_weapon: Guid, + + #[paramacro::serialized_field = "SecondaryFrontWeapon"] + pub secondary_front_weapon: Guid, + + #[paramacro::serialized_field = "LeftRearWeapon"] + pub left_rear_weapon: Guid, + + #[paramacro::serialized_field = "RightRearWeapon"] + pub right_rear_weapon: Guid, + + #[paramacro::serialized_field = "ColoringData"] + pub coloring_data: DAMachineColoringDataStruct, } -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct DAMachineColoringDataStruct { - pub hanger: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "Hanger"] + pub hanger: DAModuleColorStruct, - pub headset: PrimaryAssetNameProperty, - pub mobility: PrimaryAssetNameProperty, - pub thruster: PrimaryAssetNameProperty, - pub utility: PrimaryAssetNameProperty, - pub primary_front_weapon: PrimaryAssetNameProperty, - pub secondary_front_weapon: PrimaryAssetNameProperty, - pub left_rear_weapon: PrimaryAssetNameProperty, - // has none at the end - #[br(pad_after = 9)] - pub right_rear_weapon: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "Headset"] + pub headset: DAModuleColorStruct, + + #[paramacro::serialized_field = "Mobility"] + pub mobility: DAModuleColorStruct, + + #[paramacro::serialized_field = "Thruster"] + pub thruster: DAModuleColorStruct, + + #[paramacro::serialized_field = "Utility"] + pub utility: DAModuleColorStruct, + + #[paramacro::serialized_field = "PrimaryFrontWeapon"] + pub primary_front_weapon: DAModuleColorStruct, + + #[paramacro::serialized_field = "SecondaryFrontWeapon"] + pub secondary_front_weapon: DAModuleColorStruct, + + #[paramacro::serialized_field = "LeftRearWeapon"] + pub left_rear_weapon: DAModuleColorStruct, + + #[paramacro::serialized_field = "RightRearWeapon"] + pub right_rear_weapon: DAModuleColorStruct, } -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct DAHumanoidColoringDataStruct { - pub skin: PrimaryAssetNameProperty, - pub hair_base: PrimaryAssetNameProperty, - pub hair_gradation: PrimaryAssetNameProperty, - pub hair_highlight: PrimaryAssetNameProperty, - pub head_option: PrimaryAssetNameProperty, - pub eye_l: PrimaryAssetNameProperty, - pub eye_r: PrimaryAssetNameProperty, - pub body_main: PrimaryAssetNameProperty, - pub body_sub1: PrimaryAssetNameProperty, - pub body_sub2: PrimaryAssetNameProperty, - // has none at the end - #[br(pad_after = 9)] - pub body_sub3: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "Skin"] + pub skin: LinearColorStruct, + + #[paramacro::serialized_field = "HairBase"] + pub hair_base: LinearColorStruct, + + #[paramacro::serialized_field = "HairGradation"] + pub hair_gradation: LinearColorStruct, + + #[paramacro::serialized_field = "HairHighlight"] + pub hair_highlight: LinearColorStruct, + + #[paramacro::serialized_field = "HeadOption"] + pub head_option: LinearColorStruct, + + #[paramacro::serialized_field = "EyeL"] + pub eye_l: LinearColorStruct, + + #[paramacro::serialized_field = "EyeR"] + pub eye_r: LinearColorStruct, + + #[paramacro::serialized_field = "BodyMain"] + pub body_main: LinearColorStruct, + + #[paramacro::serialized_field = "BodySub1"] + pub body_sub1: LinearColorStruct, + + #[paramacro::serialized_field = "BodySub2"] + pub body_sub2: LinearColorStruct, + + #[paramacro::serialized_field = "BodySub3"] + pub body_sub3: LinearColorStruct, } -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct DAModuleColorStruct { - pub main: PrimaryAssetNameProperty, - pub sub: PrimaryAssetNameProperty, - pub inner: PrimaryAssetNameProperty, - // has none at the end - #[br(pad_after = 9)] - pub glow: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "Main"] + pub main: LinearColorStruct, + + #[paramacro::serialized_field = "Sub"] + pub sub: LinearColorStruct, + + #[paramacro::serialized_field = "Inner"] + pub inner: LinearColorStruct, + + #[paramacro::serialized_field = "Glow"] + pub glow: LinearColorStruct, } #[binrw] @@ -151,50 +219,61 @@ pub struct DATriggerDataStruct { pub unk: [u8; 319], // trigger weapon config in game } -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct DACustomizeAssetIdDataStruct { - pub body: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "Body"] + pub body: PrimaryAssetIdStruct, - pub face: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "Face"] + pub face: PrimaryAssetIdStruct, - pub front_hair: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "FrontHair"] + pub front_hair: PrimaryAssetIdStruct, - pub back_hair: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "BackHair"] + pub back_hair: PrimaryAssetIdStruct, - pub coloring_data: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "ColoringData"] + pub coloring_data: DAHumanoidColoringDataStruct, - pub figure_data: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "FigureData"] + pub figure_data: DAHumanoidFigureData, - pub inverse_face_mesh: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "bInverseFaceMesh"] + pub inverse_face_mesh: BoolProperty, - pub inverse_front_hair_mesh: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "bInverseFrontHairMesh"] + pub inverse_front_hair_mesh: BoolProperty, - // has none at the end - #[brw(pad_after = 9)] - pub inverse_back_hair_mesh: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "bInverseBackHairMesh"] + pub inverse_back_hair_mesh: BoolProperty, } -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct DAHumanoidFigureData { - pub bust_up: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "BustUp"] + pub bust_up: FloatProperty, - pub fat_up: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "FatUp"] + pub fat_up: FloatProperty, - pub arm_up: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "ArmUp"] + pub arm_up: FloatProperty, - pub leg_up: PrimaryAssetNameProperty, - // has none at the end - #[brw(pad_after = 9)] - pub waist_up: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "LegUp"] + pub leg_up: FloatProperty, + + #[paramacro::serialized_field = "WaistUp"] + pub waist_up: FloatProperty, } -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct DATuningDataStruct { - #[brw(pad_after = 9)] - pub granted_tuning_point_list: PrimaryAssetNameProperty, + #[paramacro::serialized_field = "GrantedTuningPointList"] + pub granted_tuning_point_list: MapProperty, } #[binrw] @@ -203,26 +282,27 @@ pub struct SavedBuildData { pub build_data: DABuildDataStruct, } -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct DATuningPointData { - #[br(args { name: "TuningPoint", r#type: "IntProperty" })] - tuning_point: StructField, - #[br(args { name: "MaxTuningPoint", r#type: "IntProperty" })] - #[brw(pad_after = 9)] // none - max_tuning_point: StructField, + #[paramacro::serialized_field = "TuningPoint"] + tuning_point: IntProperty, + + #[paramacro::serialized_field = "MaxTuningPoint"] + max_tuning_point: IntProperty, } -#[binrw] +#[paramacro::serialized_struct] #[derive(Debug)] pub struct TransformStruct { - #[br(args { name: "Rotation", r#type: "StructProperty" })] - rotation: StructField, - #[br(args { name: "Translation", r#type: "StructProperty" })] - translation: StructField, - #[br(args { name: "Scale3D", r#type: "StructProperty" })] - #[brw(pad_after = 9)] // none - scale: StructField, + #[paramacro::serialized_field = "Rotation"] + rotation: QuatStruct, + + #[paramacro::serialized_field = "Translation"] + translation: VectorStruct, + + #[paramacro::serialized_field = "Scale3D"] + scale: VectorStruct, } #[binrw] @@ -242,3 +322,45 @@ pub struct VectorStruct { pub y: f32, pub z: f32, } + +#[binrw] +#[derive(Debug)] +pub struct StructFieldPrelude { + #[br(parse_with = read_string_with_length)] + #[bw(write_with = write_string_with_length)] + pub property_name: String, + + #[br(parse_with = read_string_with_length)] + #[bw(write_with = write_string_with_length)] + pub type_name: String, +} + +#[binrw] +#[derive(Debug)] +pub struct StructPrelude { + pub unk: u32, + #[brw(pad_before = 4)] + #[br(parse_with = read_string_with_length)] + #[bw(write_with = write_string_with_length)] + #[br(pad_after = 17)] + pub struct_name: String, +} + +#[binrw::parser(reader, endian)] +pub(crate) fn read_struct_field = ()> + Debug>( + name: &str, +) -> BinResult { + let prelude = StructFieldPrelude::read_le(reader)?; + if prelude.property_name != name { + panic!( + "Property name doesn't match! Is supposed to be {} but is actually {}", + name, prelude.property_name + ); + } + println!("{:#?}", prelude); + if prelude.type_name == "StructProperty" { + println!("{:#?}", StructPrelude::read_le(reader)?); + } + let val = T::read_options(reader, endian, ())?; + Ok(val) +}