From 26c4041b68be116308792851e438404699234517 Mon Sep 17 00:00:00 2001 From: Joshua Goins Date: Mon, 28 Apr 2025 16:42:32 -0400 Subject: [PATCH] Refactor EXD parsing, begin adding write support --- src/common.rs | 2 +- src/exd.rs | 307 ++++++++++++++++++++++++++++++++++++------------ src/exh.rs | 4 + src/gamedata.rs | 2 +- 4 files changed, 236 insertions(+), 79 deletions(-) diff --git a/src/common.rs b/src/common.rs index 7b99fd0..799e19d 100755 --- a/src/common.rs +++ b/src/common.rs @@ -9,7 +9,7 @@ use binrw::binrw; #[binrw] #[brw(repr(u8))] #[repr(u8)] -#[derive(Copy, Clone)] +#[derive(Debug, Copy, Clone)] /// The language the game data is written for. Some of these languages are supported in the Global region. pub enum Language { /// Used for data that is language-agnostic, such as item data. diff --git a/src/exd.rs b/src/exd.rs index 456afeb..d23cb48 100644 --- a/src/exd.rs +++ b/src/exd.rs @@ -1,13 +1,12 @@ // SPDX-FileCopyrightText: 2023 Joshua Goins // SPDX-License-Identifier: GPL-3.0-or-later -use std::io::{Cursor, Seek, SeekFrom}; +use std::io::{BufWriter, Cursor, Read, Seek, SeekFrom, Write}; -use binrw::binrw; -use binrw::helpers::until_eof; +use binrw::{binrw, BinResult, BinWrite}; use binrw::{BinRead, Endian}; -use crate::ByteSpan; +use crate::{ByteBuffer, ByteSpan}; use crate::common::Language; use crate::exh::{ColumnDataType, EXH, ExcelColumnDefinition, ExcelDataPagination}; @@ -15,16 +14,21 @@ use crate::exh::{ColumnDataType, EXH, ExcelColumnDefinition, ExcelDataPagination #[brw(magic = b"EXDF")] #[brw(big)] #[allow(dead_code)] +#[derive(Debug)] struct EXDHeader { version: u16, - #[br(pad_before = 2)] - #[br(pad_after = 20)] + #[brw(pad_before = 2)] // empty + /// Size of the ExcelDataOffset array index_size: u32, + #[brw(pad_after = 16)] // empty + /// Total size of the string data? + data_section_size: u32, } #[binrw] #[brw(big)] +#[derive(Debug)] struct ExcelDataOffset { row_id: u32, pub offset: u32, @@ -38,17 +42,147 @@ struct ExcelDataRowHeader { row_count: u16, } +#[binrw::parser(reader)] +fn parse_rows(exh: &EXH, data_offsets: &Vec) -> BinResult> { + let mut rows = Vec::new(); + + for offset in data_offsets { + reader.seek(SeekFrom::Start(offset.offset.into()))?; + + let row_header = ExcelDataRowHeader::read(reader)?; + + let data_offset = reader.stream_position().unwrap() as u32; + + let mut read_row = |row_offset: u32| -> Option { + let mut subrow = ExcelSingleRow { + columns: Vec::with_capacity(exh.column_definitions.len()), + }; + + for column in &exh.column_definitions { + reader + .seek(SeekFrom::Start((row_offset + column.offset as u32).into())) + .ok()?; + + subrow + .columns + .push(EXD::read_column(reader, exh, column).unwrap()); + } + + Some(subrow) + }; + + let new_row = if row_header.row_count > 1 { + let mut rows = Vec::new(); + for i in 0..row_header.row_count { + let subrow_offset = + data_offset + (i * exh.header.data_offset + 2 * (i + 1)) as u32; + + rows.push(read_row(subrow_offset).unwrap()); + } + ExcelRowKind::SubRows(rows) + } else { + ExcelRowKind::SingleRow(read_row(data_offset).unwrap()) + }; + rows.push(ExcelRow { + row_id: offset.row_id, + kind: new_row + }); + } + + Ok(rows) +} + +#[binrw::writer(writer)] +fn write_rows( + rows: &Vec, + exh: &EXH, +) -> BinResult<()> { + // seek past the data offsets, which we will write later + let data_offsets_pos = writer.stream_position().unwrap(); + writer.seek(SeekFrom::Current((core::mem::size_of::() * rows.len()) as i64)).unwrap(); + + let mut data_offsets = Vec::new(); + + for row in rows { + data_offsets.push(ExcelDataOffset { + row_id: row.row_id, + offset: writer.stream_position().unwrap() as u32 + }); + + let row_header = ExcelDataRowHeader { + data_size: 0, + row_count: 0 + }; + row_header.write(writer).unwrap(); + + // write column data + { + let mut write_row = |row: &ExcelSingleRow| { + for (i, column) in row.columns.iter().enumerate() { + EXD::write_column(writer, &column, &exh.column_definitions[i]); + } + }; + + match &row.kind { + ExcelRowKind::SingleRow(excel_single_row) => write_row(excel_single_row), + ExcelRowKind::SubRows(excel_single_rows) => { + for row in excel_single_rows { + write_row(row); + } + }, + } + } + + // write strings at the end of column data + { + let mut write_row_strings = |row: &ExcelSingleRow| { + for column in &row.columns { + match column { + ColumnData::String(val) => { + let bytes = val.as_bytes(); + bytes.write(writer).unwrap(); + }, + _ => {} + } + } + }; + + match &row.kind { + ExcelRowKind::SingleRow(excel_single_row) => write_row_strings(excel_single_row), + ExcelRowKind::SubRows(excel_single_rows) => { + for row in excel_single_rows { + write_row_strings(row); + } + }, + } + } + + // There's an empty byte between each row... for some reason + 0u8.write_le(writer).unwrap(); + } + + // now write the data offsets + writer.seek(SeekFrom::Start(data_offsets_pos)).unwrap(); + data_offsets.write(writer).unwrap(); + + Ok(()) +} + #[binrw] #[brw(big)] #[allow(dead_code)] +#[derive(Debug)] +#[brw(import(exh: &EXH))] pub struct EXD { header: EXDHeader, #[br(count = header.index_size / core::mem::size_of::() as u32)] + #[bw(ignore)] data_offsets: Vec, - #[br(seek_before = SeekFrom::Start(0), parse_with = until_eof)] - data: Vec, + #[br(parse_with = parse_rows, args(&exh, &data_offsets))] + #[bw(write_with = write_rows, args(&exh))] + rows: Vec, } #[derive(Debug)] @@ -66,71 +200,35 @@ pub enum ColumnData { UInt64(u64), } +#[derive(Debug)] +pub struct ExcelSingleRow { + pub columns: Vec, +} + +#[derive(Debug)] +pub enum ExcelRowKind { + SingleRow(ExcelSingleRow), + SubRows(Vec) +} + #[derive(Debug)] pub struct ExcelRow { - pub data: Vec, + pub row_id: u32, + pub kind: ExcelRowKind, } impl EXD { - pub fn from_existing(buffer: ByteSpan) -> Option { - EXD::read(&mut Cursor::new(&buffer)).ok() + pub fn from_existing(exh: &EXH, buffer: ByteSpan) -> Option { + EXD::read_args(&mut Cursor::new(&buffer), (exh,)).ok() } - pub fn read_row(&self, exh: &EXH, id: u32) -> Option> { - let mut cursor = Cursor::new(&self.data); - - for offset in &self.data_offsets { - if offset.row_id == id { - cursor.seek(SeekFrom::Start(offset.offset.into())).ok()?; - - let row_header = ExcelDataRowHeader::read(&mut cursor).ok()?; - - let header_offset = offset.offset + 6; // std::mem::size_of::() as u32; - - let mut read_row = |row_offset: u32| -> Option { - let mut subrow = ExcelRow { - data: Vec::with_capacity(exh.column_definitions.len()), - }; - - for column in &exh.column_definitions { - cursor - .seek(SeekFrom::Start((row_offset + column.offset as u32).into())) - .ok()?; - - subrow - .data - .push(Self::read_column(&mut cursor, exh, row_offset, column).unwrap()); - } - - Some(subrow) - }; - - return if row_header.row_count > 1 { - let mut rows = Vec::new(); - for i in 0..row_header.row_count { - let subrow_offset = - header_offset + (i * exh.header.data_offset + 2 * (i + 1)) as u32; - - rows.push(read_row(subrow_offset).unwrap()); - } - Some(rows) - } else { - Some(vec![read_row(header_offset).unwrap()]) - }; - } - } - - None - } - - fn read_data_raw = ()>>(cursor: &mut Cursor<&Vec>) -> Option { + fn read_data_raw = ()>>(cursor: &mut T) -> Option { Z::read_options(cursor, Endian::Big, ()).ok() } - fn read_column( - cursor: &mut Cursor<&Vec>, + fn read_column( + cursor: &mut T, exh: &EXH, - row_offset: u32, column: &ExcelColumnDefinition, ) -> Option { let mut read_packed_bool = |shift: i32| -> bool { @@ -144,9 +242,12 @@ impl EXD { ColumnDataType::String => { let string_offset: u32 = Self::read_data_raw(cursor).unwrap(); + let old_pos = cursor.stream_position().unwrap(); + + // -4 to take into account reading string_offset cursor - .seek(SeekFrom::Start( - (row_offset + exh.header.data_offset as u32 + string_offset).into(), + .seek(SeekFrom::Current( + (exh.header.data_offset as u32 + string_offset - 4).into(), )) .ok()?; @@ -158,6 +259,8 @@ impl EXD { byte = Self::read_data_raw(cursor).unwrap(); } + cursor.seek(SeekFrom::Start(old_pos)).unwrap(); + Some(ColumnData::String(string)) } ColumnDataType::Bool => { @@ -169,20 +272,12 @@ impl EXD { ColumnDataType::Int8 => Some(ColumnData::Int8(Self::read_data_raw(cursor).unwrap())), ColumnDataType::UInt8 => Some(ColumnData::UInt8(Self::read_data_raw(cursor).unwrap())), ColumnDataType::Int16 => Some(ColumnData::Int16(Self::read_data_raw(cursor).unwrap())), - ColumnDataType::UInt16 => { - Some(ColumnData::UInt16(Self::read_data_raw(cursor).unwrap())) - } + ColumnDataType::UInt16 => Some(ColumnData::UInt16(Self::read_data_raw(cursor).unwrap())), ColumnDataType::Int32 => Some(ColumnData::Int32(Self::read_data_raw(cursor).unwrap())), - ColumnDataType::UInt32 => { - Some(ColumnData::UInt32(Self::read_data_raw(cursor).unwrap())) - } - ColumnDataType::Float32 => { - Some(ColumnData::Float32(Self::read_data_raw(cursor).unwrap())) - } + ColumnDataType::UInt32 => Some(ColumnData::UInt32(Self::read_data_raw(cursor).unwrap())), + ColumnDataType::Float32 => Some(ColumnData::Float32(Self::read_data_raw(cursor).unwrap())), ColumnDataType::Int64 => Some(ColumnData::Int64(Self::read_data_raw(cursor).unwrap())), - ColumnDataType::UInt64 => { - Some(ColumnData::UInt64(Self::read_data_raw(cursor).unwrap())) - } + ColumnDataType::UInt64 => Some(ColumnData::UInt64(Self::read_data_raw(cursor).unwrap())), ColumnDataType::PackedBool0 => Some(ColumnData::Bool(read_packed_bool(0))), ColumnDataType::PackedBool1 => Some(ColumnData::Bool(read_packed_bool(1))), ColumnDataType::PackedBool2 => Some(ColumnData::Bool(read_packed_bool(2))), @@ -194,6 +289,51 @@ impl EXD { } } + fn write_data_raw = ()>>(cursor: &mut T, value: &Z) { + value.write_options(cursor, Endian::Big, ()).unwrap() + } + + fn write_column( + cursor: &mut T, + column: &ColumnData, + column_definition: &ExcelColumnDefinition, + ) { + let write_packed_bool = |cursor: &mut T, shift: i32, boolean: &bool| { + let val = 0i32; // TODO + Self::write_data_raw(cursor, &val); + }; + + match column { + ColumnData::String(_) => { + let string_offset = 0u32; // TODO, but 0 is fine for single string column data + Self::write_data_raw(cursor, &string_offset); + }, + ColumnData::Bool(val) => { + match column_definition.data_type { + ColumnDataType::Bool => todo!(), + ColumnDataType::PackedBool0 => write_packed_bool(cursor, 0, val), + ColumnDataType::PackedBool1 => write_packed_bool(cursor, 1, val), + ColumnDataType::PackedBool2 => write_packed_bool(cursor, 2, val), + ColumnDataType::PackedBool3 => write_packed_bool(cursor, 3, val), + ColumnDataType::PackedBool4 => write_packed_bool(cursor, 4, val), + ColumnDataType::PackedBool5 => write_packed_bool(cursor, 5, val), + ColumnDataType::PackedBool6 => write_packed_bool(cursor, 6, val), + ColumnDataType::PackedBool7 => write_packed_bool(cursor, 7, val), + _ => panic!("This makes no sense!") + } + }, + ColumnData::Int8(val) => Self::write_data_raw(cursor, val), + ColumnData::UInt8(val) => Self::write_data_raw(cursor, val), + ColumnData::Int16(val) => Self::write_data_raw(cursor, val), + ColumnData::UInt16(val) => Self::write_data_raw(cursor, val), + ColumnData::Int32(val) => Self::write_data_raw(cursor, val), + ColumnData::UInt32(val) => Self::write_data_raw(cursor, val), + ColumnData::Float32(val) => Self::write_data_raw(cursor, val), + ColumnData::Int64(val) => Self::write_data_raw(cursor, val), + ColumnData::UInt64(val) => Self::write_data_raw(cursor, val), + } + } + pub fn calculate_filename( name: &str, language: Language, @@ -210,6 +350,19 @@ impl EXD { } } } + + pub fn write_to_buffer(&self, exh: &EXH) -> Option { + let mut buffer = ByteBuffer::new(); + + { + let cursor = Cursor::new(&mut buffer); + let mut writer = BufWriter::new(cursor); + + self.write_args(&mut writer, (exh,)).unwrap(); + } + + Some(buffer) + } } #[cfg(test)] @@ -241,6 +394,6 @@ mod tests { }; // Feeding it invalid data should not panic - EXD::from_existing(&read(d).unwrap()); + EXD::from_existing(&exh, &read(d).unwrap()); } } diff --git a/src/exh.rs b/src/exh.rs index 1b3f4f1..5a340f2 100644 --- a/src/exh.rs +++ b/src/exh.rs @@ -15,6 +15,7 @@ use crate::common::Language; #[brw(magic = b"EXHF")] #[brw(big)] #[allow(dead_code)] +#[derive(Debug)] pub struct EXHHeader { pub(crate) version: u16, @@ -57,6 +58,7 @@ pub enum ColumnDataType { #[binrw] #[brw(big)] +#[derive(Debug)] pub struct ExcelColumnDefinition { pub data_type: ColumnDataType, pub offset: u16, @@ -65,6 +67,7 @@ pub struct ExcelColumnDefinition { #[binrw] #[brw(big)] #[allow(dead_code)] +#[derive(Debug)] pub struct ExcelDataPagination { pub start_id: u32, pub row_count: u32, @@ -73,6 +76,7 @@ pub struct ExcelDataPagination { #[binrw] #[brw(big)] #[allow(dead_code)] +#[derive(Debug)] pub struct EXH { pub header: EXHHeader, diff --git a/src/gamedata.rs b/src/gamedata.rs index 0976a21..001febd 100755 --- a/src/gamedata.rs +++ b/src/gamedata.rs @@ -291,7 +291,7 @@ impl GameData { let exd_file = self.extract(&exd_path)?; - EXD::from_existing(&exd_file) + EXD::from_existing(&exh, &exd_file) } /// Applies the patch to game data and returns any errors it encounters. This function will not update the version in the GameData struct.